diff --git a/contrib/dbt_factory/.gitignore b/contrib/dbt_factory/.gitignore new file mode 100644 index 00000000..1ae87882 --- /dev/null +++ b/contrib/dbt_factory/.gitignore @@ -0,0 +1,16 @@ +.venv/ +__pycache__/ +*.pyc +.databricks/ +logs/ +dbt_packages/ +dbt_profiles/.user.yml +uv.lock + +# dbt build artifacts, but keep the committed manifest that resources/__init__.py reads at +# deploy time. Regenerate it with `make manifest`. +target/* +!target/manifest.json + +# Serverless base-environment file, regenerated on every `bundle deploy` by resources/__init__.py. +dbt_serverless_env.yaml diff --git a/contrib/dbt_factory/Makefile b/contrib/dbt_factory/Makefile new file mode 100644 index 00000000..66343d8e --- /dev/null +++ b/contrib/dbt_factory/Makefile @@ -0,0 +1,37 @@ +.PHONY: setup deps manifest validate deploy run test test-update-expected-tasks test-e2e + +# Install dependencies into the .venv used by the bundle (databricks.yml -> python.venv_path). +setup: + uv sync --dev + +# Install dbt package dependencies declared in packages.yml / dependencies.yml (if any). +deps: + uv run dbt deps + +# Regenerate the dbt manifest that resources/__init__.py reads at deploy time. +# `dbt parse` does not connect to a warehouse; it only reads the project files. +manifest: deps + uv run dbt parse --profiles-dir dbt_profiles + +validate: + databricks bundle validate + +# Regenerate the manifest and deploy the generated job to the dev target. +deploy: manifest + databricks bundle deploy --target dev + +run: + databricks bundle run dbt_factory_job + +test: + uv run pytest tests + +# Refresh the saved "expected tasks" file the tests compare against, after you intentionally +# change the generated output. +test-update-expected-tasks: + uv run python tests/update_expected_tasks.py + +# End-to-end test: generate a project from the template, deploy the factory-generated job to your +# workspace, run it, verify the output, and tear it all down. Needs DBX_E2E_* env (see tests/e2e/run_e2e.py). +test-e2e: + uv run python tests/e2e/run_e2e.py diff --git a/contrib/dbt_factory/NOTICE b/contrib/dbt_factory/NOTICE new file mode 100644 index 00000000..ebc023a5 --- /dev/null +++ b/contrib/dbt_factory/NOTICE @@ -0,0 +1,34 @@ +This project includes code adapted from the "databricks-dbt-factory" library. + + Source: https://github.com/mwojtyczka/databricks-dbt-factory + Full history: https://github.com/mwojtyczka/databricks-dbt-factory/commits/main + Adapted from: commit e767a9d865581226e4f144fb17b7a822df1ea1f4 (v0.2.1) + Location: src/databricks_dbt_factory/ + +The code under src/databricks_dbt_factory/ originates from that repository (trimmed to the +serverless + notebook-task subset used here, then reformatted to this repository's style) and is +provided under the MIT +license reproduced below, which this NOTICE preserves for attribution. All other files are +part of the bundle-examples repository and are covered by that repository's license. + +-------------------------------------------------------------------------------- + +MIT License + +Copyright (c) 2024-present mwojtyczka + +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the "Software"), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE +FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/contrib/dbt_factory/README.md b/contrib/dbt_factory/README.md new file mode 100644 index 00000000..9a5eac7f --- /dev/null +++ b/contrib/dbt_factory/README.md @@ -0,0 +1,247 @@ +# dbt_factory + +This example runs a [dbt](https://docs.getdbt.com/) project on Databricks as a +**Databricks Workflow with one task per dbt object** (model, seed, snapshot, test) instead of +running the whole project as a single opaque task. + +It does this by combining two pieces: + +* **[databricks-dbt-factory](https://github.com/mwojtyczka/databricks-dbt-factory)** — a small + library that reads a dbt `manifest.json` and expands it into Databricks job tasks, wiring up + the dependencies between them. Its source is included under `src/databricks_dbt_factory/` + (see [`NOTICE`](NOTICE) for attribution and license). +* **[PyDABs](https://docs.databricks.com/dev-tools/bundles/python)** — the Databricks Asset + Bundle Python resources hook. At `databricks bundle deploy` time the Databricks CLI calls + `load_resources` in [`resources/__init__.py`](resources/__init__.py), which runs the factory + against the manifest and returns the generated job. + +The result: **no per-model job YAML is checked in**. The task graph is generated on the fly from +the dbt manifest each time you deploy. + +## Why one task per dbt object? + +By default dbt's integration with Databricks Workflows treats the whole project as a single +task — a black box. Expanding it into one task per object gives: + +* **Faster execution** — independent models run in parallel, and the notebook task type runs dbt + from a pre-built serverless base environment, avoiding a dependency install on every task. +* **Visibility & simplified troubleshooting** — pinpoint and fix issues at the model level right + in the Databricks Workflows UI. +* **Enhanced logging & notifications** — per-task logs and precise, model-level error alerts. +* **Improved retriability** — retry only the failed model tasks without rerunning the whole project. +* **Seamless testing** — dbt data tests run as their own tasks right after each model finishes, + for faster validation and feedback. + +This example uses **serverless compute** and the **notebook task type** (each task triggers dbt +through a small runner notebook using the `dbtRunner` Python API) for the fastest task start +times. See the [databricks-dbt-factory README](https://github.com/mwojtyczka/databricks-dbt-factory#benefits) +for more. + +## How it works + +The [`dbt-factory` template](../templates/dbt-factory) scaffolds a self-contained project. +From then on, each `databricks bundle deploy` regenerates the Workflow from your current dbt +manifest — add or remove a model and the task graph follows on the next deploy, with no per-model +YAML to maintain. + +```mermaid +flowchart TD + subgraph setup["One-time setup"] + T["dbt-factory bundle template"] -->|databricks bundle init| B["Scaffolded project:
dbt project + PyDABs hook + factory code"] + X["Existing dbt project
(optional)"] -.->|move models/seeds/... into src/| B + end + subgraph deploy["Every deploy"] + C["make manifest
(dbt parse)"] --> D["target/manifest.json"] + D --> E["databricks bundle deploy"] + E --> F["PyDABs load_resources reads the
manifest and generates the job"] + end + subgraph runtime["At run time — serverless"] + G["Databricks Workflow:
one task per model / seed / snapshot / test"] --> H["Each task triggers dbt
via the runner notebook"] + H --> I[("SQL warehouse")] + end + B --> C + F --> G + + classDef optional stroke:#999,stroke-dasharray:5 4,color:#888; + class X optional; +``` + +## Project structure + +``` +dbt_factory/ +├── databricks.yml # Bundle definition; wires up the PyDABs `load_resources` hook +├── dbt_project.yml # dbt project (models under src/models, etc.) +├── dbt_profiles/profiles.yml # dbt profiles for the deployed job (dev / prod targets) +├── profile_template.yml # prompts for `dbt init` (local development) +├── resources/__init__.py # PyDABs glue: manifest -> generated job (the only integration code) +├── src/ +│ ├── models/ # your dbt models (example: orders_raw, orders_daily) +│ └── databricks_dbt_factory/ # vendored factory library (trimmed; see NOTICE) +├── target/manifest.json # committed dbt manifest, read at deploy time (regenerate with `make manifest`) +├── tests/ # tests for the vendored factory + the PyDABs integration +├── pyproject.toml # dependencies (installed into .venv via `uv sync`) +└── Makefile # convenience targets: setup, manifest, validate, deploy, run, test, test-e2e +``` + +## Setup + +1. Install the [Databricks CLI](https://docs.databricks.com/dev-tools/cli/databricks-cli.html) + and the [uv](https://docs.astral.sh/uv/) package manager. + +2. Authenticate to your Databricks workspace: + ``` + $ databricks configure + ``` + +3. Install dependencies into the `.venv` the bundle uses: + ``` + $ make setup # == uv sync --dev + ``` + +4. Edit `dbt_profiles/profiles.yml` and set your SQL warehouse `http_path`, `catalog`, and + `schema`. Set the workspace host in `databricks.yml` (and the prod `root_path` / permissions). + +## The dbt manifest + +`resources/__init__.py` reads `target/manifest.json` at deploy time to build the task graph. A +manifest is committed so the bundle deploys out of the box. **After you change your models, +regenerate it:** + +``` +$ make manifest # == uv run dbt deps && uv run dbt parse +``` + +`dbt parse` only reads your project files; it does not connect to a warehouse. The manifest +location is configurable — point at a different file via the `DBT_MANIFEST_PATH` environment +variable or by editing `MANIFEST_PATH` in `resources/__init__.py`. + +> **Faster task startup (automatic).** `make manifest` also writes `target/partial_parse.msgpack` +> next to the manifest. The bundle syncs it (`sync.include` in `databricks.yml`) and each notebook +> task injects it to **skip dbt's parse phase** — a large win on big projects, where parsing (not +> the SQL) dominates each task's time. No `git add -f` needed: because the runtime dbt is pinned to +> your local version (see "dbt version and the serverless environment" below), the version-specific +> msgpack always loads instead of being silently ignored. Deploy with `make deploy` (or run +> `make manifest` first) so the shipped msgpack always matches your current models. + +## Deploy and run + +``` +$ make deploy # regenerates the manifest + parse cache, then deploys +$ make run # == databricks bundle run dbt_factory_job +``` + +`make deploy` regenerates `target/manifest.json` and `target/partial_parse.msgpack` (via `dbt +parse`) before deploying, so the task graph and the synced parse cache always match your current +models. You can also call the CLI directly — just run `make manifest` first: + +``` +$ databricks bundle deploy --target dev +$ databricks bundle run dbt_factory_job +``` + +Open the run URL the CLI prints to watch the generated per-model task graph execute. Deploying +in `dev` mode prefixes resources with `[dev your_name]` and pauses the daily schedule; deploy to +`prod` with `--target prod`. + +## Configuring the generated job + +A few knobs are exposed as constants at the top of `resources/__init__.py`: + +* `BUNDLE_TESTS` — when `True`, single-model tests are bundled into one `dbt test` task per + resource (fewer task startups; faster for test-heavy projects). Default `False` (one task per + test node, for maximum per-test visibility). +* `ENVIRONMENT_KEY` — the serverless environment key (default `Default`). +* `EXTRA_DBT_COMMAND_OPTIONS` — extra options appended to every generated dbt command. + +The dbt target, warehouse, catalog, and schema are configured in `dbt_profiles/profiles.yml` +and selected per bundle target via `--target ${bundle.target}`. + +### dbt version and the serverless environment + +You don't set the runtime dbt version by hand. At deploy time `resources/__init__.py` pins the +serverless environment to the **exact `dbt-databricks` version installed in the bundle's `.venv`** — +the same version you use locally to generate the manifest and develop with. `pyproject.toml` is the +single source of truth: change the version there, re-run `make setup`, and the next deploy uses it. +This guarantees the version running in Databricks matches the one you tested with. + +The version is shipped as a small `dbt_serverless_env.yaml` [base environment](https://docs.databricks.com/aws/en/compute/serverless/dependencies) +that the bundle generates and syncs on every deploy (git-ignored), so Databricks pre-builds the +environment once instead of installing dbt on every task. + +## Migrating an existing dbt project + +Bring your own dbt project by **generating a fresh project from the template and moving your dbt +files into it.** You don't touch dependencies, the vendored factory, or any paths — the generated +project already ships all of that. + +1. Generate a new project (or copy this `dbt_factory` example): + + ``` + $ databricks bundle init https://github.com/databricks/bundle-examples --template-dir contrib/templates/dbt-factory + ``` + +2. Remove the starter models and copy your dbt sources into the matching `src/` subdirectories: + + ``` + $ rm -r src/models/example + # Copy whichever of these your project has (skip the ones you don't use): + $ cp -R /path/to/your/dbt/models/* src/models/ + $ cp -R /path/to/your/dbt/seeds/* src/seeds/ + $ cp -R /path/to/your/dbt/snapshots/* src/snapshots/ + $ cp -R /path/to/your/dbt/macros/* src/macros/ + $ cp -R /path/to/your/dbt/tests/* src/tests/ + ``` + + The generated `dbt_project.yml` already points `model-paths`, `seed-paths`, etc. at these + `src/` folders, so your files are picked up as-is. Merge any model/seed configuration from your + own `dbt_project.yml` into the generated one (keep the generated `name`/`profile`), and remove + the leftover `models: dbt_factory: example:` block that referenced the deleted starter models — + otherwise `dbt parse` warns that those config paths don't apply to any resource. If you use dbt + packages, copy your `packages.yml` to the project root too: `make manifest` installs them + (`dbt deps`) and the bundle syncs the resulting `dbt_packages/` to the workspace, so the job + never installs packages at runtime. + +3. Point `dbt_profiles/profiles.yml` at your warehouse (`http_path`, `catalog`, `schema`). Leave + the `host`/`token` lines as they are — the runner notebook sets those at runtime. + +4. Generate the manifest and deploy: + + ``` + $ make setup + $ make manifest # dbt parse -> target/manifest.json + $ databricks bundle deploy --target dev + ``` + +That's the whole migration: no dependency wrangling and no path edits, because your project keeps +the generated layout (dbt project at the bundle root, factory under `src/`). If you'd rather keep +your project's existing directory structure instead of `src/`, edit the `*-paths` in +`dbt_project.yml` to point at your folders — nothing else changes. + +## Tests + +``` +$ make test # == uv run pytest tests +``` + +This runs the factory's unit tests plus an offline test that exercises the PyDABs integration +against the committed manifest; no workspace is required. One test compares the generated tasks +with a saved snapshot (`tests/test_data/expected_tasks.json`), so unintended changes to the +generated job fail the suite. After an intentional change to the generated output, refresh the +snapshot with `make test-update-expected-tasks`. + +There is also a live end-to-end test that generates a project from the template, deploys it to +your workspace, runs the generated job, verifies the output tables, and tears everything down +again: + +``` +$ make test-e2e +``` + +See [`tests/e2e/README.md`](tests/e2e/README.md) for the required environment variables. + +## Local development with dbt + +You can still develop the dbt project locally with the dbt CLI. Initialize your own profile with +`dbt init` (see `profile_template.yml`), then use `dbt run`, `dbt test`, etc. as usual. See the +[`dbt_sql`](../../dbt_sql) example for a more detailed local-dbt walkthrough. diff --git a/contrib/dbt_factory/databricks.yml b/contrib/dbt_factory/databricks.yml new file mode 100644 index 00000000..3df9de0e --- /dev/null +++ b/contrib/dbt_factory/databricks.yml @@ -0,0 +1,53 @@ +# This is a Databricks Asset Bundle definition for dbt_factory. +# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. +# +# The job for this bundle is NOT defined in YAML. Instead it is generated at deploy time +# from the dbt manifest by resources/__init__.py (see the `python.resources` hook below), +# producing one Databricks task per dbt object (model / seed / snapshot / test). +bundle: + name: dbt_factory + uuid: 19ecc815-cff0-449c-91c1-e68239d49ccb + +# PyDABs: the Databricks CLI calls `load_resources` during `bundle deploy` to build resources +# defined in Python. See resources/__init__.py. +python: + venv_path: .venv + resources: + - "resources:load_resources" + +# These files are generated (by resources/__init__.py / `make manifest`) and git-ignored; include +# them explicitly so `bundle deploy` still syncs them to the workspace: +# - dbt_serverless_env.yaml the base_environment the generated job points at. +# - target/partial_parse.msgpack dbt's parse cache; each task injects it to skip parsing (see +# README). Regenerated by `make manifest`; usable at runtime +# because the runtime dbt is pinned to your local version. +# - dbt_packages/** dbt packages installed by `make manifest` (dbt deps), so tasks +# never install packages at runtime. +sync: + include: + - dbt_serverless_env.yaml + - target/partial_parse.msgpack + - dbt_packages/** + +# Deployment targets. The dbt target is selected via `--target ${bundle.target}`, so these +# names match the dbt outputs in dbt_profiles/profiles.yml. +targets: + dev: + # The default target uses 'mode: development' to create a development copy. + # - Deployed resources get prefixed with '[dev my_user_name]' + # - Any job schedules and triggers are paused by default. + # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html. + mode: development + default: true + workspace: + host: https://company.databricks.com + + prod: + mode: production + workspace: + host: https://company.databricks.com + # We explicitly deploy to /Workspace/Users/user@company.com to make sure we only have a single copy. + root_path: /Workspace/Users/user@company.com/.bundle/${bundle.name}/${bundle.target} + permissions: + - user_name: user@company.com + level: CAN_MANAGE diff --git a/contrib/dbt_factory/dbt_profiles/profiles.yml b/contrib/dbt_factory/dbt_profiles/profiles.yml new file mode 100644 index 00000000..46e604f1 --- /dev/null +++ b/contrib/dbt_factory/dbt_profiles/profiles.yml @@ -0,0 +1,37 @@ + +# This file defines dbt profiles for deployed dbt jobs. +# The generated Databricks job selects the target via `--target ${bundle.target}`, +# so the dbt target names below (dev / prod) match the bundle targets in databricks.yml. +dbt_factory: + target: dev # default target + outputs: + + # Doing local development with the dbt CLI? + # Then you should create your own profile in your .dbt/profiles.yml using 'dbt init' + # (See README.md) + + # The default target when deployed with the Databricks CLI + dev: + type: databricks + method: http + catalog: catalog + schema: default + + http_path: /sql/1.0/warehouses/abcdef1234567890 + + # The workspace host / token are provided by the runner notebook at runtime + # (see src/databricks_dbt_factory/notebook/run_dbt_command.py). + host: "{{ env_var('DBT_HOST', '') }}" + token: "{{ env_var('DBT_ACCESS_TOKEN', '') }}" + + # The production target when deployed with the Databricks CLI + prod: + type: databricks + method: http + catalog: catalog + schema: default + + http_path: /sql/1.0/warehouses/abcdef1234567890 + + host: "{{ env_var('DBT_HOST', '') }}" + token: "{{ env_var('DBT_ACCESS_TOKEN', '') }}" diff --git a/contrib/dbt_factory/dbt_project.yml b/contrib/dbt_factory/dbt_project.yml new file mode 100644 index 00000000..63de848b --- /dev/null +++ b/contrib/dbt_factory/dbt_project.yml @@ -0,0 +1,28 @@ +name: 'dbt_factory' +version: '1.0.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. +profile: 'dbt_factory' + +# These configurations specify where dbt should look for different types of files. +# Everything dbt-related lives under src/ so the project can also hold non-dbt resources +# (such as the vendored databricks_dbt_factory library under src/databricks_dbt_factory). +model-paths: ["src/models"] +analysis-paths: ["src/analyses"] +test-paths: ["src/tests"] +seed-paths: ["src/seeds"] +macro-paths: ["src/macros"] +snapshot-paths: ["src/snapshots"] + +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_packages" + +# In this example config, we tell dbt to build all models in the example/ +# directory as views by default. These settings can be overridden in the +# individual model files using the `{{ config(...) }}` macro. +models: + dbt_factory: + example: + +materialized: view diff --git a/contrib/dbt_factory/profile_template.yml b/contrib/dbt_factory/profile_template.yml new file mode 100644 index 00000000..f26ec0b3 --- /dev/null +++ b/contrib/dbt_factory/profile_template.yml @@ -0,0 +1,23 @@ +# This file defines prompts with defaults for dbt initialization. +# It is used when the `dbt init` command is invoked. +# +fixed: + type: databricks +prompts: + host: + default: company.databricks.com + token: + hint: 'personal access token to use, dapiXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' + hide_input: true + http_path: + hint: 'HTTP path of SQL warehouse to use' + default: /sql/1.0/warehouses/abcdef1234567890 + catalog: + hint: 'initial catalog' + default: catalog + schema: + hint: 'personal schema where dbt will build objects during development, example: user_name' + threads: + hint: 'threads to use during development, 1 or more' + type: 'int' + default: 4 diff --git a/contrib/dbt_factory/pyproject.toml b/contrib/dbt_factory/pyproject.toml new file mode 100644 index 00000000..39989930 --- /dev/null +++ b/contrib/dbt_factory/pyproject.toml @@ -0,0 +1,32 @@ +[project] +name = "dbt_factory" +version = "0.0.1" +requires-python = ">=3.10,<3.13" +# Used at `bundle deploy` time by resources/__init__.py (to write the serverless +# base-environment file) and by the vendored databricks_dbt_factory core. +dependencies = [ + "PyYAML~=6.0.1", +] + +[dependency-groups] +dev = [ + # Required by the PyDABs hook (resources/__init__.py) at `bundle deploy` time. + "databricks-bundles==0.295.0", + # Single source of truth for the dbt-databricks version: used locally to generate the + # manifest (`make manifest`), and resources/__init__.py pins the serverless runtime to this + # exact installed version at deploy time. Pin/lock it for fully reproducible deploys. + "dbt-databricks>=1.8.0,<2.0.0", + "pytest", + "ruff", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +# Only the vendored library is packaged; the dbt project (src/models, ...) is not a package. +[tool.hatch.build.targets.wheel] +packages = ["src/databricks_dbt_factory"] + +[tool.ruff] +line-length = 120 diff --git a/contrib/dbt_factory/resources/__init__.py b/contrib/dbt_factory/resources/__init__.py new file mode 100644 index 00000000..06fa3b74 --- /dev/null +++ b/contrib/dbt_factory/resources/__init__.py @@ -0,0 +1,184 @@ +""" +PyDABs integration for databricks-dbt-factory. + +At ``databricks bundle deploy`` time the Databricks CLI calls :func:`load_resources`, which +reads the dbt manifest and expands it into a Databricks job with one task per dbt object +(model / seed / snapshot / test) using the vendored ``databricks_dbt_factory`` core. No +per-model YAML is generated or checked in — the job graph is built on the fly from the manifest. + +This module is the only integration glue. The vendored core under ``src/databricks_dbt_factory`` +has been trimmed to the single path this example uses — serverless compute with notebook tasks — +and returns a ``Resources`` object for PyDABs. +""" + +import os +from importlib.metadata import PackageNotFoundError, version + +import yaml +from databricks.bundles.core import Bundle, Resources +from databricks.bundles.jobs import Job + +from databricks_dbt_factory.DbtFactory import DbtFactory +from databricks_dbt_factory.DbtTask import DbtTaskOptions +from databricks_dbt_factory.TaskFactory import ( + DbtDependencyResolver, + ModelTaskFactory, + SnapshotTaskFactory, + SeedTaskFactory, + TestTaskFactory, +) +from databricks_dbt_factory.Utils import read_dbt_manifest + +# --- Configuration ------------------------------------------------------------------------- + +# Path to the dbt manifest, read at deploy time. Regenerate with `make manifest` (dbt parse). +# Override the location without editing this file via the DBT_MANIFEST_PATH env var. +MANIFEST_PATH = os.environ.get("DBT_MANIFEST_PATH", "target/manifest.json") + +# Name of the generated Databricks job. +JOB_NAME = "dbt_factory_job" + +# Key of the serverless environment (defined on the job below). +ENVIRONMENT_KEY = "Default" + +# Bundle single-model tests per resource into one `dbt test` task (fewer task startups, +# faster end-to-end runtime for projects with many tests). See the databricks-dbt-factory +# README "DBT Tests handling" section for the trade-offs. +BUNDLE_TESTS = False + +# Extra options appended to every generated dbt command (e.g. "--vars '{...}'"). Empty by default. +EXTRA_DBT_COMMAND_OPTIONS = "" + +# Serverless base-environment file, generated at deploy time from the pinned +# dbt-databricks version and synced with the bundle. The generated job points its +# serverless environment at this file (see build_job) so Databricks pre-builds the +# environment once instead of installing dbt on every task. `${workspace.file_path}` +# is substituted by the CLI at deploy time to the bundle's synced files root. +SERVERLESS_ENV_FILE = "dbt_serverless_env.yaml" +SERVERLESS_ENV_PATH = "${workspace.file_path}/" + SERVERLESS_ENV_FILE + +# The runner notebook shipped with the core, referenced in place (path relative to the bundle +# root). `PROJECT_DIRECTORY` is the path from the notebook's own directory back up to the dbt +# project root (== bundle root), where the runner changes directory to before running dbt. It is +# derived from the notebook path so the two stay consistent if the layout ever changes. +RUNNER_NOTEBOOK_PATH = "src/databricks_dbt_factory/notebook/run_dbt_command.py" +PROJECT_DIRECTORY = os.path.relpath(".", os.path.dirname(RUNNER_NOTEBOOK_PATH)) +PROFILES_DIRECTORY = "dbt_profiles" + + +def _build_tasks(target: str) -> list[dict]: + """Reads the dbt manifest and returns the list of Databricks task dicts (one per dbt node).""" + resolver = DbtDependencyResolver() + task_options = DbtTaskOptions( + environment_key=ENVIRONMENT_KEY, + notebook_path=RUNNER_NOTEBOOK_PATH, + project_directory=PROJECT_DIRECTORY, + profiles_directory=PROFILES_DIRECTORY, + ) + dbt_options = f"--target {target} {EXTRA_DBT_COMMAND_OPTIONS}".strip() + + task_factories = { + "model": ModelTaskFactory(resolver, task_options, dbt_options), + "snapshot": SnapshotTaskFactory(resolver, task_options, dbt_options), + "seed": SeedTaskFactory(resolver, task_options, dbt_options), + "test": TestTaskFactory(resolver, task_options, dbt_options), + } + + factory = DbtFactory(task_factories, bundle_tests=BUNDLE_TESTS) + manifest = read_dbt_manifest(MANIFEST_PATH) + return factory.create_tasks(manifest) + + +def _dbt_databricks_dependency() -> str: + """Pin the runtime to the exact dbt-databricks installed in this bundle's venv — the same + version used to generate the manifest (`make manifest`) and to develop locally. pyproject.toml + is therefore the single source of truth for the dbt version, and the version that runs in + Databricks is guaranteed to match the one you tested with (no separate range that can drift). + Local or dev builds (e.g. `1.9.0+custom`) are rejected: pip cannot resolve them from PyPI + when Databricks builds the serverless environment, which would fail every task at runtime. + """ + try: + installed = version("dbt-databricks") + except PackageNotFoundError as exc: + raise RuntimeError( + "dbt-databricks is not installed in the bundle venv — run `make setup` before deploying." + ) from exc + + from packaging.version import InvalidVersion, Version + + try: + parsed = Version(installed) + except InvalidVersion: + parsed = None + if parsed is None or parsed.local or parsed.is_devrelease: + raise RuntimeError( + f"The installed dbt-databricks version ({installed}) is not a plain PyPI release, so the " + "serverless environment cannot install it. Set a released version in pyproject.toml and " + "re-run `make setup` before deploying." + ) + return f"dbt-databricks=={installed}" + + +def _serverless_environment_spec() -> dict: + """Contents of the base-environment file: the environment version plus the pinned + dbt-databricks dependency.""" + return {"environment_version": "4", "dependencies": [_dbt_databricks_dependency()]} + + +def _write_serverless_environment_file() -> None: + """Write the base-environment file to the bundle root so `bundle deploy` syncs it to the + workspace, where the generated job references it via base_environment. The write is skipped + when the file already has the desired content, so commands like `bundle validate` stay free + of side effects once the file exists.""" + content = yaml.safe_dump(_serverless_environment_spec(), sort_keys=False) + try: + with open(SERVERLESS_ENV_FILE, "r", encoding="utf-8") as f: + if f.read() == content: + return + except OSError: + pass + try: + with open(SERVERLESS_ENV_FILE, "w", encoding="utf-8") as f: + f.write(content) + except OSError as exc: + raise RuntimeError( + f"Cannot write {SERVERLESS_ENV_FILE}, which the generated job needs as its serverless " + f"base environment: {exc}" + ) from exc + + +def build_job(target: str) -> Job: + """Builds the Databricks job that runs the dbt project on serverless via notebook tasks.""" + return Job.from_dict( + { + "name": JOB_NAME, + "queue": {"enabled": True}, + "trigger": { + # Run this job every day, exactly one day from the last run; + # see https://docs.databricks.com/api/workspace/jobs/create#trigger + "periodic": {"interval": 1, "unit": "DAYS"}, + }, + "tasks": _build_tasks(target), + "environments": [ + { + "environment_key": ENVIRONMENT_KEY, + # Pre-built base env (synced with the bundle), not inline deps — + # the two are mutually exclusive; the file pins dbt-databricks. + "spec": {"base_environment": SERVERLESS_ENV_PATH}, + } + ], + } + ) + + +def load_resources(bundle: Bundle) -> Resources: + """ + Called by the Databricks CLI during `bundle deploy` to load resources defined in Python. + + Reads the dbt manifest, writes the serverless base-environment file (synced with the bundle), + and registers the generated job. After deployment this function is not used. + """ + _write_serverless_environment_file() + resources = Resources() + resources.add_job(JOB_NAME, build_job(bundle.target)) + return resources diff --git a/contrib/dbt_factory/src/analyses/.gitkeep b/contrib/dbt_factory/src/analyses/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/dbt_factory/src/databricks_dbt_factory/DbtFactory.py b/contrib/dbt_factory/src/databricks_dbt_factory/DbtFactory.py new file mode 100644 index 00000000..bb6bfa70 --- /dev/null +++ b/contrib/dbt_factory/src/databricks_dbt_factory/DbtFactory.py @@ -0,0 +1,306 @@ +from dataclasses import replace + +from databricks_dbt_factory.TaskFactory import TaskFactory +from databricks_dbt_factory.DbtTask import DbtTask +from databricks_dbt_factory.Utils import build_task_key_maps + + +class DbtFactory: + """Generates Databricks job task definitions from a dbt manifest.""" + + def __init__( + self, + task_factories: dict[str, TaskFactory], + bundle_tests: bool = False, + ): + """ + Initializes the dbt factory. + + Args: + task_factories (dict[str, TaskFactory]): Maps dbt resource types (`model`, `seed`, + `snapshot`, `test`) to their respective `TaskFactory` instances. Omitting `test` + disables test-task generation entirely. + bundle_tests (bool): When True, emit one `_test` task per tested resource + and rewire downstream models/seeds/snapshots to depend on the upstream's + `_test` task so failing tests halt the DAG. When False, emit one task + per dbt test node. + """ + self.task_factories = task_factories + self.bundle_tests = bundle_tests + + def create_tasks(self, dbt_manifest: dict) -> list[dict]: + """ + Generates the Databricks task dictionaries from a dbt manifest. + + Args: + dbt_manifest (dict): Parsed dbt manifest content. + + Returns: + list[dict]: Task dictionaries ready to be injected into the `tasks` list of a + Databricks job spec. + """ + tasks = self._create_tasks(dbt_manifest) + return [task.to_dict() for task in tasks] + + _GATEABLE_TYPES = frozenset({"model", "seed", "snapshot"}) + _DBT_TEST_TARGET_PREFIXES = ("model.", "seed.", "snapshot.", "source.") + + def _create_tasks(self, dbt_manifest: dict) -> list[DbtTask]: + """ + Builds `DbtTask` instances from the manifest, applying the bundling and gating policies. + + Args: + dbt_manifest (dict): Parsed dbt manifest content. + + Returns: + list[DbtTask]: `DbtTask` instances (not yet rendered to dicts). + """ + dbt_nodes = dbt_manifest.get("nodes", {}) + dbt_sources = dbt_manifest.get("sources", {}) + + bundle = "test" in self.task_factories and self.bundle_tests + single_model_tested: set[str] = set() + standalone_tests: list[tuple[str, dict]] = [] + if bundle: + single_model_tested, standalone_tests = self._classify_tests(dbt_nodes, dbt_sources) + standalone_test_ids = {full_name for full_name, _ in standalone_tests} + task_ids = [ + full_name + for full_name, info in dbt_nodes.items() + if info["resource_type"] in self.task_factories + and (not bundle or info["resource_type"] != "test" or full_name in standalone_test_ids) + ] + task_keys, bundled_test_keys = build_task_key_maps(task_ids, sorted(single_model_tested)) + + tests_by_resource: dict[str, list[tuple[str, frozenset[str]]]] = {} + ancestors: dict[str, set[str]] = {} + if not bundle and "test" in self.task_factories: + tests_by_resource = self._index_tests_by_resource(dbt_nodes, dbt_sources, task_keys) + ancestors = self._compute_ancestors(dbt_nodes, dbt_sources) + test_key_by_resource = {task_keys[fn]: bundled_test_keys[fn] for fn in single_model_tested if fn in task_keys} + + tasks = self._build_resource_tasks( + dbt_nodes, bundle, task_keys, test_key_by_resource, tests_by_resource, ancestors + ) + + if bundle: + tasks.extend( + self._build_bundled_test_tasks( + dbt_nodes, dbt_sources, single_model_tested, task_keys, bundled_test_keys + ) + ) + tasks.extend(self._build_standalone_test_tasks(standalone_tests, task_keys)) + + return tasks + + def _compute_ancestors(self, dbt_nodes: dict, dbt_sources: dict) -> dict[str, set[str]]: + """ + Maps each testable resource's full name to the set of resources it transitively depends + on (not including itself). Used in per-test mode to decide whether a test can safely + gate a downstream node: a test `T` with refs `R` is only safe to add to node `N`'s + deps if `R ⊆ ancestors(N)` — i.e. `N` already waits for all of `T`'s endpoints, + transitively. Otherwise adding `T` would create a cycle (since `T` depends on each + ref, and some ref might depend on `N`). + """ + ancestors: dict[str, set[str]] = {} + + def visit(full_name: str) -> set[str]: + cached = ancestors.get(full_name) + if cached is not None: + return cached + result: set[str] = set() + info = dbt_nodes.get(full_name) or dbt_sources.get(full_name) + if info is not None: + for dep in info.get("depends_on", {}).get("nodes", []): + if dep in dbt_nodes or dep in dbt_sources: + result.add(dep) + result.update(visit(dep)) + ancestors[full_name] = result + return result + + for full_name in list(dbt_nodes.keys()) + list(dbt_sources.keys()): + visit(full_name) + return ancestors + + def _index_tests_by_resource( + self, dbt_nodes: dict, dbt_sources: dict, task_keys: dict[str, str] + ) -> dict[str, list[tuple[str, frozenset[str]]]]: + """ + Maps each testable resource's full name to a list of (test_task_key, test_refs) pairs + for tests whose `severity` is `error` (the default). Warn-severity tests still run but + are NOT indexed here, so they do not appear in any downstream model's `depends_on` — + their job is to surface findings, not halt the DAG. This matches `dbt build` semantics: + dbt itself exits 0 on warn-severity failures, so even if we did gate on them the + Databricks task would succeed and downstream would run; keeping warn tests out of the + dep graph just avoids the extra DAG clutter. + + The refs set is carried alongside each test so `_extend_deps_with_upstream_tests` can + avoid cycles: a test with refs that aren't all ancestors of a candidate node would + create a cycle if added as that node's dep. + """ + index: dict[str, list[tuple[str, frozenset[str]]]] = {} + for node_full_name, node_info in dbt_nodes.items(): + if node_info["resource_type"] != "test": + continue + if self._test_severity(node_info) != "error": + continue + test_task_key = task_keys[node_full_name] + refs: set[str] = set() + for dep in node_info.get("depends_on", {}).get("nodes", []): + if dep.startswith(self._DBT_TEST_TARGET_PREFIXES) and (dep in dbt_nodes or dep in dbt_sources): + refs.add(dep) + frozen_refs = frozenset(refs) + for resource_full in refs: + index.setdefault(resource_full, []).append((test_task_key, frozen_refs)) + return index + + @staticmethod + def _test_severity(test_node_info: dict) -> str: + """Reads the test's severity from the manifest, defaulting to `error` when unset.""" + config = test_node_info.get("config") or {} + severity = config.get("severity") + if isinstance(severity, str): + return severity.lower() + return "error" + + @staticmethod + def _extend_deps_with_upstream_tests( + node_full_name: str, + existing_deps: list[str] | None, + tests_by_resource: dict[str, list[tuple[str, frozenset[str]]]], + ancestors_by_node: dict[str, set[str]], + ) -> list[str]: + """ + Appends task keys of tests that safely gate this node — i.e. tests whose refs are all + ancestors of the current node. This prevents both direct and transitive cycles: a test + `T` with refs `R` is added to node `N`'s deps only if `N` transitively depends on every + resource in `R`. If any ref of `T` is downstream of (or equal to) `N`, adding `T` would + cycle because `T` already depends on that ref, and the ref depends on `N`. + """ + extended: list[str] = list(existing_deps or []) + seen = set(extended) + node_ancestors = ancestors_by_node.get(node_full_name, set()) + for ancestor in node_ancestors: + for test_key, test_refs in tests_by_resource.get(ancestor, []): + if test_key in seen: + continue + if test_refs <= node_ancestors: + extended.append(test_key) + seen.add(test_key) + return extended + + def _classify_tests(self, dbt_nodes: dict, dbt_sources: dict) -> tuple[set[str], list[tuple[str, dict]]]: + """ + Classifies test nodes for bundled mode so that no test is silently dropped. + + - Tests with exactly 1 testable dep: will be covered by their resource's bundled + `tests_` task under `--indirect-selection cautious`. + - Tests with >1 testable deps (cross-model, e.g. `relationships`): emitted as their own + tasks with multi-resource deps — `cautious` filters them out of bundles. + - Tests with 0 testable deps (singular/custom tests that don't `ref()` or `source()` + any resource): also emitted as their own tasks, since no bundle would pick them up. + + Returns: + (single_model_tested, standalone_tests): + - `single_model_tested`: full names of resources with at least one single-model + test — these become `tests_` bundled tasks. + - `standalone_tests`: list of `(test_full_name, test_node_info)` for tests + that must run as individual tasks (cross-model or zero-dep). + """ + single_model_tested: set[str] = set() + standalone_tests: list[tuple[str, dict]] = [] + for node_full_name, node_info in dbt_nodes.items(): + if node_info["resource_type"] != "test": + continue + testable_deps: list[str] = [] + for dep in node_info.get("depends_on", {}).get("nodes", []): + if dep.startswith(self._DBT_TEST_TARGET_PREFIXES) and (dep in dbt_nodes or dep in dbt_sources): + testable_deps.append(dep) + if len(testable_deps) == 1: + single_model_tested.add(testable_deps[0]) + else: + standalone_tests.append((node_full_name, node_info)) + return single_model_tested, standalone_tests + + def _build_resource_tasks( + self, + dbt_nodes: dict, + bundle: bool, + task_keys: dict[str, str], + test_key_by_resource: dict[str, str], + tests_by_resource: dict[str, list[tuple[str, frozenset[str]]]], + ancestors_by_node: dict[str, set[str]], + ) -> list[DbtTask]: + """Builds tasks for every non-test resource (plus per-test tasks when not bundling).""" + tasks: list[DbtTask] = [] + for node_full_name, node_info in dbt_nodes.items(): + resource_type = node_info["resource_type"] + if resource_type not in self.task_factories: + continue + if bundle and resource_type == "test": + continue + + task_key = task_keys[node_full_name] + factory = self.task_factories[resource_type] + task = factory.create_task(node_info["name"], node_info, task_key, task_keys) + + if resource_type in self._GATEABLE_TYPES: + if bundle: + task = replace(task, depends_on=self._rewire_deps(task.depends_on, test_key_by_resource)) + elif tests_by_resource: + task = replace( + task, + depends_on=self._extend_deps_with_upstream_tests( + node_full_name, task.depends_on, tests_by_resource, ancestors_by_node + ), + ) + + tasks.append(task) + return tasks + + @staticmethod + def _rewire_deps(deps: list[str] | None, test_key_by_resource: dict[str, str]) -> list[str]: + """Rewrites a dependency on a tested resource to that resource's gating `_test` task.""" + return [test_key_by_resource.get(dep_key, dep_key) for dep_key in (deps or [])] + + def _build_bundled_test_tasks( + self, + dbt_nodes: dict, + dbt_sources: dict, + nodes_with_tests: set[str], + task_keys: dict[str, str], + bundled_test_keys: dict[str, str], + ) -> list[DbtTask]: + """Emits one `_test` task per tested resource using `TestTaskFactory.create_bundled_task`.""" + test_factory = self.task_factories["test"] + tasks: list[DbtTask] = [] + for full_name in sorted(nodes_with_tests): + is_source = full_name.startswith("source.") + info = dbt_sources[full_name] if is_source else dbt_nodes[full_name] + bare_name = info["name"] + qualified = f"{info['package_name']}.{bare_name}" + select = f"source:{info['package_name']}.{info['source_name']}.{bare_name}" if is_source else qualified + tasks.append( + test_factory.create_bundled_task( + task_key=bundled_test_keys[full_name], + select=select, + depends_on=[] if is_source else [task_keys[full_name]], + ) + ) + return tasks + + def _build_standalone_test_tasks( + self, + standalone_tests: list[tuple[str, dict]], + task_keys: dict[str, str], + ) -> list[DbtTask]: + """ + Emits one task per standalone test — cross-model tests (e.g. `relationships`) gated on + every referenced resource, plus any zero-dep singular tests that bundles can't cover. + """ + test_factory = self.task_factories["test"] + tasks: list[DbtTask] = [] + for test_full_name, test_info in sorted(standalone_tests, key=lambda item: item[0]): + test_task_key = task_keys[test_full_name] + tasks.append(test_factory.create_task(test_info["name"], test_info, test_task_key, task_keys)) + return tasks diff --git a/contrib/dbt_factory/src/databricks_dbt_factory/DbtTask.py b/contrib/dbt_factory/src/databricks_dbt_factory/DbtTask.py new file mode 100644 index 00000000..a109c85f --- /dev/null +++ b/contrib/dbt_factory/src/databricks_dbt_factory/DbtTask.py @@ -0,0 +1,48 @@ +import json +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class DbtTaskOptions: + """Options shared by every generated task. All tasks are serverless notebook tasks.""" + + environment_key: str = "Default" + """Key of the serverless environment the task runs on. Must match an environment defined on the job.""" + + notebook_path: str | None = None + """Workspace path to the dbt runner notebook that each task executes.""" + + project_directory: str | None = None + """Path to the dbt project directory, resolved relative to the runner notebook (or absolute).""" + + profiles_directory: str | None = None + """Optional path to the dbt profiles directory (passed to dbt as `--profiles-dir`).""" + + +@dataclass(frozen=True) +class DbtTask: + """A single generated Databricks notebook task that runs one batch of dbt commands.""" + + task_key: str + commands: list[str] + options: DbtTaskOptions + depends_on: list[str] | None = None + + def to_dict(self) -> dict[str, Any]: + """Renders the task as a serverless `notebook_task` dict for the Databricks job spec.""" + base_parameters: dict[str, str] = {"dbt_commands": json.dumps(self.commands)} + if self.options.project_directory: + base_parameters["project_directory"] = self.options.project_directory + if self.options.profiles_directory: + base_parameters["profiles_directory"] = self.options.profiles_directory + + return { + "task_key": self.task_key, + "depends_on": [{"task_key": dep} for dep in (self.depends_on or [])], + "environment_key": self.options.environment_key, + "notebook_task": { + "notebook_path": self.options.notebook_path, + "base_parameters": base_parameters, + }, + } diff --git a/contrib/dbt_factory/src/databricks_dbt_factory/TaskFactory.py b/contrib/dbt_factory/src/databricks_dbt_factory/TaskFactory.py new file mode 100644 index 00000000..b786a155 --- /dev/null +++ b/contrib/dbt_factory/src/databricks_dbt_factory/TaskFactory.py @@ -0,0 +1,123 @@ +from abc import ABC, abstractmethod +from enum import Enum + +from databricks_dbt_factory.DbtTask import DbtTask, DbtTaskOptions + + +class DbtNodeTypes(Enum): + """dbt node types that become Databricks tasks.""" + + MODEL = "model" + TEST = "test" + SEED = "seed" + SNAPSHOT = "snapshot" + + +class DbtDependencyResolver: + @staticmethod + def resolve(node_info: dict, valid_deps_types: list[str], task_keys: dict[str, str]) -> list[str]: + """ + Resolves a dbt node's upstream dependencies to Databricks task keys, keeping only the + dependency types relevant for the node being built. + + Args: + node_info (dict): The dbt manifest entry for the node. + valid_deps_types (list[str]): dbt node types that should become task dependencies. + task_keys (dict[str, str]): Task key per dbt node, from `build_task_key_maps`. + + Returns: + list[str]: Resolved upstream task keys. + """ + deps = node_info.get("depends_on", {}).get("nodes", []) + resolved_deps = [] + for node_full_name in deps: + if any(node_full_name.startswith(dbt_type + ".") for dbt_type in valid_deps_types): + resolved_deps.append(task_keys[node_full_name]) + return resolved_deps + + +class TaskFactory(ABC): + """Abstract base class for building a task (one dbt command batch) from a dbt node.""" + + def __init__(self, resolver: DbtDependencyResolver, task_options: DbtTaskOptions, dbt_options: str = ""): + """ + Args: + resolver (DbtDependencyResolver): Resolves upstream dependencies to task keys. + task_options (DbtTaskOptions): Shared task options (serverless notebook task). + dbt_options (str, optional): Extra options appended to every dbt command. Defaults to "". + """ + self.resolver = resolver + self.task_options = task_options + self.dbt_options = dbt_options + + @abstractmethod + def create_task(self, dbt_node_name: str, dbt_node_info: dict, task_key: str, task_keys: dict[str, str]) -> DbtTask: + """Builds a `DbtTask` for a single dbt node. `task_keys` maps every dbt node to its task key.""" + + def _command(self, verb: str, select: str) -> str: + """Assembles a `dbt --select [extra options]` command string.""" + return f"dbt {verb} --select {select}" + ( + f" {self.dbt_options}" if self.dbt_options else "" + ) + + +class ModelTaskFactory(TaskFactory): + """Factory for model tasks (`dbt run`).""" + + def create_task( + self, + dbt_node_name: str, + dbt_node_info: dict, + task_key: str, + task_keys: dict[str, str], + ) -> DbtTask: + valid_dbt_deps_types = [ + DbtNodeTypes.MODEL.value, + DbtNodeTypes.SEED.value, + DbtNodeTypes.SNAPSHOT.value, + DbtNodeTypes.TEST.value, + ] + depends_on = self.resolver.resolve( + dbt_node_info, valid_dbt_deps_types, task_keys + ) + return DbtTask( + task_key, + [self._command("run", dbt_node_name)], + self.task_options, + depends_on, + ) + + +class SnapshotTaskFactory(TaskFactory): + """Factory for snapshot tasks (`dbt snapshot`).""" + + def create_task( + self, + dbt_node_name: str, + dbt_node_info: dict, + task_key: str, + task_keys: dict[str, str], + ) -> DbtTask: + depends_on = self.resolver.resolve( + dbt_node_info, [DbtNodeTypes.MODEL.value], task_keys + ) + return DbtTask( + task_key, + [self._command("snapshot", dbt_node_name)], + self.task_options, + depends_on, + ) + + +class SeedTaskFactory(TaskFactory): + """Factory for seed tasks (`dbt seed`). Seeds have no dependencies.""" + + def create_task( + self, + dbt_node_name: str, + dbt_node_info: dict, + task_key: str, + task_keys: dict[str, str], + ) -> DbtTask: + depends_on = self.resolver.resolve(dbt_node_info, [], task_keys) + return DbtTask( + task_key, + [self._command("seed", dbt_node_name)], + self.task_options, + depends_on, + ) + + +class TestTaskFactory(TaskFactory): + """Factory for test tasks (`dbt test`).""" + + def create_task( + self, + dbt_node_name: str, + dbt_node_info: dict, + task_key: str, + task_keys: dict[str, str], + ) -> DbtTask: + valid_dbt_deps_types = [ + DbtNodeTypes.MODEL.value, + DbtNodeTypes.SEED.value, + DbtNodeTypes.SNAPSHOT.value, + ] + depends_on = self.resolver.resolve( + dbt_node_info, valid_dbt_deps_types, task_keys + ) + return DbtTask( + task_key, + [self._command("test", dbt_node_name)], + self.task_options, + depends_on, + ) + + def create_bundled_task( + self, task_key: str, select: str, depends_on: list[str] + ) -> DbtTask: + """ + Creates a single task that runs the single-model tests for a resource via + `dbt test --select --indirect-selection cautious`. The cautious selector + includes only tests whose referenced resources are entirely within this bundle; + cross-model tests (e.g. `relationships`) are excluded and handled as standalone tasks. + + Args: + task_key (str): Key for the bundled task. + select (str): Pre-computed dbt `--select` argument (qualified model name, or + `source:..` for sources). + depends_on (list[str]): Upstream task keys this bundled task should gate on. + + Returns: + DbtTask: The bundled test task. + """ + command = f"dbt test --select {select} --indirect-selection cautious" + ( + f" {self.dbt_options}" if self.dbt_options else "" + ) + return DbtTask(task_key, [command], self.task_options, depends_on) diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/Utils.py b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/Utils.py new file mode 100644 index 00000000..b9b8f40d --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/Utils.py @@ -0,0 +1,181 @@ +import json +from collections.abc import Iterable + +# Databricks caps task keys at 100 characters (letters, numbers, underscores, hyphens). +MAX_TASK_KEY_LENGTH = 100 + +# dbt resource type -> the dbt verb the task runs, used as the task-key suffix. +_RUN_SUFFIX = {"model": "run", "seed": "seed", "snapshot": "snapshot"} + + +def _resource_name(unique_id: str) -> str: + """ + The dbt resource name — everything after ``..`` — with dots turned into + underscores. Keeps a versioned model's version, e.g. ``model.shop.dim.v2`` -> ``dim_v2``. + """ + return "_".join(unique_id.split(".")[2:]) + + +def generate_task_key(unique_id: str) -> str: + """ + Builds a readable Databricks task key from a dbt node ``unique_id``. + + The key is based on the dbt resource *name* (not the fully-qualified id), so the package + prefix and dbt's test-name hash never appear, with a verb suffix per resource type: + + * ``model.shop.orders`` -> ``orders_run`` + * ``seed.shop.countries`` -> ``countries_seed`` + * ``snapshot.shop.orders_snap`` -> ``orders_snap_snapshot`` + * ``test.shop.unique_orders_id.9a1`` -> ``unique_orders_id_test`` (hash dropped) + * ``source.shop.raw.customers`` -> ``raw_customers_test`` + + Distinct nodes can map to the same plain key (the same custom test name on two models, a + model name reused across packages), so job generation resolves keys through + :func:`build_task_key_maps`, which keeps these plain keys and disambiguates only actual + collisions. Over-long test keys are truncated and disambiguated with dbt's hash to stay + within the task-key length limit. + """ + parts = unique_id.split(".") + resource_type = parts[0] + + if resource_type in _RUN_SUFFIX: + return f"{_resource_name(unique_id)}_{_RUN_SUFFIX[resource_type]}" + + if resource_type == "source": + # A source only ever surfaces as a test task: source.... + return f"{parts[2]}_{parts[3]}_test" + + if resource_type == "test": + # test..[.] -> _test (drop dbt's uniqueness hash). + test_hash = parts[3] if len(parts) > 3 else "" + return _bounded_test_key(parts[2], test_hash) + + # Unknown type: fall back to the sanitized id (still unique). + return unique_id.replace(".", "_") + + +def bundled_test_key(unique_id: str) -> str: + """ + Key for the single ``dbt test`` task that gates a tested resource in bundled mode: + ``model.shop.orders`` -> ``orders_test``; ``source.shop.raw.customers`` -> ``raw_customers_test``. + """ + parts = unique_id.split(".") + if parts[0] == "source": + return f"{parts[2]}_{parts[3]}_test" + return f"{_resource_name(unique_id)}_test" + + +def build_task_key_maps( + task_ids: Iterable[str], bundled_test_ids: Iterable[str] = () +) -> tuple[dict[str, str], dict[str, str]]: + """ + Assigns every task-producing dbt node a unique Databricks task key. + + ``task_ids`` are nodes that become their own task (keys from :func:`generate_task_key`); + ``bundled_test_ids`` are tested resources that additionally get a bundled test task in + bundled mode (keys from :func:`bundled_test_key`). Plain keys are kept untouched unless + several nodes claim the same key; each claimant of a contested key then gets a + disambiguated key with dbt's test hash (or the package name, when there is no hash) folded + in, falling back to the sanitized ``unique_id``. The returned keys are therefore always + unique, so a valid dbt project cannot fail deployment with a duplicate task key. + + Returns ``(task_keys, bundled_test_keys)``, both keyed by ``unique_id``. + """ + claims: dict[str, list[tuple[str, bool]]] = {} + for uid in task_ids: + claims.setdefault(generate_task_key(uid), []).append((uid, False)) + for uid in bundled_test_ids: + claims.setdefault(bundled_test_key(uid), []).append((uid, True)) + + task_keys: dict[str, str] = {} + bundled_test_keys: dict[str, str] = {} + taken = {key for key, claimants in claims.items() if len(claimants) == 1} + for key, claimants in claims.items(): + if len(claimants) == 1: + uid, is_bundled = claimants[0] + (bundled_test_keys if is_bundled else task_keys)[uid] = key + continue + for uid, is_bundled in claimants: + unique = ( + _disambiguated_bundled_test_key(uid) + if is_bundled + else _disambiguated_task_key(uid) + ) + if unique in taken: + unique = uid.replace(".", "_") + ("_test" if is_bundled else "") + base, counter = unique, 2 + while unique in taken: + unique = f"{base}_{counter}" + counter += 1 + taken.add(unique) + (bundled_test_keys if is_bundled else task_keys)[uid] = unique + return task_keys, bundled_test_keys + + +def _disambiguated_task_key(unique_id: str) -> str: + """ + Key for a node whose plain key collides with another node's: dbt's test hash (or the + package name, when there is no hash) is folded in to keep the key unique yet readable, + e.g. ``test.shop.dup_check.9a1`` -> ``dup_check_9a1_test``, ``model.pkg.orders`` -> + ``pkg_orders_run``. + """ + parts = unique_id.split(".") + resource_type = parts[0] + if resource_type in _RUN_SUFFIX: + return f"{parts[1]}_{_resource_name(unique_id)}_{_RUN_SUFFIX[resource_type]}" + if resource_type == "source": + return f"{parts[1]}_{parts[2]}_{parts[3]}_test" + if resource_type == "test": + if len(parts) > 3: + return _hashed_test_key(parts[2], parts[3]) + return f"{parts[1]}_{parts[2]}_test" + return unique_id.replace(".", "_") + + +def _disambiguated_bundled_test_key(unique_id: str) -> str: + """Package-prefixed variant of :func:`bundled_test_key`, for contested bundled test keys.""" + parts = unique_id.split(".") + if parts[0] == "source": + return f"{parts[1]}_{parts[2]}_{parts[3]}_test" + return f"{parts[1]}_{_resource_name(unique_id)}_test" + + +def _bounded_test_key(test_name: str, test_hash: str) -> str: + """``_test``, truncated and hash-disambiguated if it exceeds the key length limit.""" + key = f"{test_name}_test" + if len(key) <= MAX_TASK_KEY_LENGTH: + return key + return _hashed_test_key(test_name, test_hash) + + +def _hashed_test_key(test_name: str, test_hash: str) -> str: + """``__test``, truncating the name so the hash survives the length limit.""" + tail = (f"_{test_hash}" if test_hash else "") + "_test" + if len(test_name) + len(tail) <= MAX_TASK_KEY_LENGTH: + return test_name + tail + return test_name[: MAX_TASK_KEY_LENGTH - len(tail)] + tail + + +def read_dbt_manifest(path: str) -> dict: + """ + Reads a dbt manifest JSON file and returns its parsed content. + + Args: + path (str): Path to the manifest file. + + Returns: + dict: Parsed manifest content. + + Raises: + FileNotFoundError: If the file does not exist. + ValueError: If the file is not valid JSON. + """ + try: + with open(path, "r", encoding="utf-8") as file: + return json.load(file) + except FileNotFoundError as e: + raise FileNotFoundError(f"Manifest file not found: {path}. Details: {e}") from e + except json.JSONDecodeError as e: + raise ValueError( + f"Error parsing JSON from manifest file: {path}. Details: {e}" + ) from e diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/__about__.py b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/__about__.py new file mode 100644 index 00000000..3ced3581 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/__about__.py @@ -0,0 +1 @@ +__version__ = "0.2.1" diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/__init__.py b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/notebook/run_dbt_command.py b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/notebook/run_dbt_command.py new file mode 100644 index 00000000..157a7a4f --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/notebook/run_dbt_command.py @@ -0,0 +1,93 @@ +# Databricks notebook source + +import json +import os +import shlex +import shutil +import tempfile + +from dbt.cli.main import dbtRunner + +# COMMAND ---------- + +dbutils.widgets.text("dbt_commands", "") +dbutils.widgets.text("project_directory", "") +dbutils.widgets.text("profiles_directory", "") + +dbt_commands = dbutils.widgets.get("dbt_commands") +project_directory = dbutils.widgets.get("project_directory") +profiles_directory = dbutils.widgets.get("profiles_directory") + +if not dbt_commands: + raise ValueError("dbt_commands parameter is required") + +# COMMAND ---------- + +ctx = dbutils.notebook.entry_point.getDbutils().notebook().getContext() +os.environ["DBT_ACCESS_TOKEN"] = ctx.apiToken().get() +# dbt's host must be a bare hostname; apiUrl() includes the https:// scheme. +os.environ["DBT_HOST"] = ( + ctx.apiUrl().get().removeprefix("https://").removeprefix("http://") +) + +if project_directory: + notebook_dir = os.path.dirname("/Workspace" + ctx.notebookPath().get()) + target_dir = ( + project_directory + if os.path.isabs(project_directory) + else os.path.normpath(os.path.join(notebook_dir, project_directory)) + ) + os.chdir(target_dir) + +local_dir = tempfile.mkdtemp(prefix="dbt_local_") +os.environ["DBT_TARGET_PATH"] = local_dir +os.environ["DBT_LOG_PATH"] = local_dir + +manifest = None +prebuilt_manifest_path = os.path.join("target", "partial_parse.msgpack") +if os.path.exists(prebuilt_manifest_path): + try: + from dbt.contracts.graph.manifest import Manifest + + with open(prebuilt_manifest_path, "rb") as f: + manifest = Manifest.from_msgpack(f.read()) + manifest.build_flat_graph() + print( + f"[dbt-factory] injecting pre-built manifest from {prebuilt_manifest_path} (skipping dbt parse)" + ) + except Exception as e: + print( + f"[dbt-factory] manifest injection unavailable, falling back to dbt parse: {e}" + ) + manifest = None + +try: + runner = dbtRunner(manifest=manifest) + + for command_str in json.loads(dbt_commands): + command_str = command_str.strip() + if not command_str: + continue + + if command_str.startswith("dbt "): + command_str = command_str[4:] + + args = shlex.split(command_str) + + if profiles_directory: + args.extend(["--profiles-dir", profiles_directory]) + + print(f"Running: dbt {' '.join(args)}") + + result = runner.invoke(args) + + if not result.success: + detail = result.exception or result.result or "(no further details)" + raise RuntimeError(f"dbt command failed: dbt {' '.join(args)}\n{detail}") +finally: + os.environ.pop("DBT_ACCESS_TOKEN", None) + os.environ.pop("DBT_HOST", None) + os.environ.pop("DBT_TARGET_PATH", None) + os.environ.pop("DBT_LOG_PATH", None) + if local_dir: + shutil.rmtree(local_dir, ignore_errors=True) diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/macros/.gitkeep b/contrib/templates/dbt-factory/template/{{.project_name}}/src/macros/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/orders_daily.sql b/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/orders_daily.sql new file mode 100644 index 00000000..e32736ce --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/orders_daily.sql @@ -0,0 +1,17 @@ + +-- This model file defines a materialized view called 'orders_daily' +-- +-- Read more about materialized at https://docs.getdbt.com/reference/resource-configs/databricks-configs#materialized-views-and-streaming-tables +-- Current limitation: a "full refresh" is needed in case the definition below is changed; see https://github.com/databricks/dbt-databricks/issues/561. +{{ config(materialized = 'materialized_view') }} + +select order_date, count(*) AS number_of_orders + +from {{ ref('orders_raw') }} + +-- During development, only process a smaller range of data +{% if target.name != 'prod' %} +where order_date >= '2019-08-01' and order_date < '2019-09-01' +{% endif %} + +group by order_date diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/orders_raw.sql b/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/orders_raw.sql new file mode 100644 index 00000000..8faf8f38 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/orders_raw.sql @@ -0,0 +1,16 @@ +-- This model file defines a streaming table called 'orders_raw' +-- +-- The streaming table below ingests all JSON files in /databricks-datasets/retail-org/sales_orders/ +-- Read more about streaming tables at https://docs.getdbt.com/reference/resource-configs/databricks-configs#materialized-views-and-streaming-tables +-- Current limitation: a "full refresh" is needed in case the definition below is changed; see https://github.com/databricks/dbt-databricks/issues/561. +{{ config(materialized = 'streaming_table') }} + +select + customer_name, + date(timestamp(from_unixtime(try_cast(order_datetime as bigint)))) as order_date, + order_number +from stream read_files( + "/databricks-datasets/retail-org/sales_orders/", + format => "json", + header => true +) diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/schema.yml b/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/schema.yml new file mode 100644 index 00000000..c17c810e --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/schema.yml @@ -0,0 +1,22 @@ + +version: 2 + +models: + - name: orders_raw + description: "Raw ingested orders" + columns: + - name: customer_name + description: "The name of a customer" + # customer_name repeats across orders, so only a not_null test applies here. + data_tests: + - not_null + + - name: orders_daily + description: "Number of orders by day" + columns: + - name: order_date + description: "The date on which orders took place" + # orders_daily groups by order_date, so it is unique and not null. + data_tests: + - unique + - not_null diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/seeds/.gitkeep b/contrib/templates/dbt-factory/template/{{.project_name}}/src/seeds/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/snapshots/.gitkeep b/contrib/templates/dbt-factory/template/{{.project_name}}/src/snapshots/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/tests/.gitkeep b/contrib/templates/dbt-factory/template/{{.project_name}}/src/tests/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/__init__.py b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/conftest.py b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/conftest.py new file mode 100644 index 00000000..71964e7d --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/conftest.py @@ -0,0 +1,39 @@ +import pytest + +from databricks_dbt_factory.DbtFactory import DbtFactory +from databricks_dbt_factory.DbtTask import DbtTaskOptions +from databricks_dbt_factory.TaskFactory import ( + ModelTaskFactory, + SnapshotTaskFactory, + SeedTaskFactory, + TestTaskFactory, + DbtDependencyResolver, +) + + +def create_dbt_factory(bundle_tests: bool = False) -> DbtFactory: + resolver = DbtDependencyResolver() + task_options = DbtTaskOptions( + environment_key="Default", + notebook_path="./notebooks/dbt_runner.py", + ) + dbt_options = "--target dev" + + task_factories = { + "model": ModelTaskFactory(resolver, task_options, dbt_options), + "snapshot": SnapshotTaskFactory(resolver, task_options, dbt_options), + "seed": SeedTaskFactory(resolver, task_options, dbt_options), + "test": TestTaskFactory(resolver, task_options, dbt_options), + } + + return DbtFactory(task_factories, bundle_tests=bundle_tests) + + +@pytest.fixture +def dbt_factory() -> DbtFactory: + return create_dbt_factory() + + +@pytest.fixture +def dbt_factory_bundled() -> DbtFactory: + return create_dbt_factory(bundle_tests=True) diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/expected_tasks.json b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/expected_tasks.json new file mode 100644 index 00000000..ae6f4f88 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/expected_tasks.json @@ -0,0 +1,552 @@ +[ + { + "task_key": "diamonds_list_colors_run", + "depends_on": [ + { + "task_key": "diamonds_four_cs_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select diamonds_list_colors --target dev\"]" + } + } + }, + { + "task_key": "diamonds_prices_run", + "depends_on": [], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select diamonds_prices --target dev\"]" + } + } + }, + { + "task_key": "diamonds_four_cs_run", + "depends_on": [], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select diamonds_four_cs --target dev\"]" + } + } + }, + { + "task_key": "zzz_game_details_run", + "depends_on": [], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select zzz_game_details --target dev\"]" + } + } + }, + { + "task_key": "second_dbt_model_run", + "depends_on": [ + { + "task_key": "first_dbt_model_run" + }, + { + "task_key": "unique_first_dbt_model_id_test" + }, + { + "task_key": "not_null_first_dbt_model_id_test" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select second_dbt_model --target dev\"]" + } + } + }, + { + "task_key": "first_dbt_model_run", + "depends_on": [], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select first_dbt_model --target dev\"]" + } + } + }, + { + "task_key": "zzz_game_details_check_scores_test", + "depends_on": [ + { + "task_key": "zzz_game_details_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select zzz_game_details_check_scores --target dev\"]" + } + } + }, + { + "task_key": "zzz_game_details_check_dates_test", + "depends_on": [ + { + "task_key": "zzz_game_details_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select zzz_game_details_check_dates --target dev\"]" + } + } + }, + { + "task_key": "zzz_win_loss_records_check_records_test", + "depends_on": [ + { + "task_key": "zzz_win_loss_records_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select zzz_win_loss_records_check_records --target dev\"]" + } + } + }, + { + "task_key": "diamonds_list_colors_test_test", + "depends_on": [ + { + "task_key": "diamonds_list_colors_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select diamonds_list_colors_test --target dev\"]" + } + } + }, + { + "task_key": "country_codes_seed", + "depends_on": [], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt seed --select country_codes --target dev\"]" + } + } + }, + { + "task_key": "unique_zzz_game_details_game_id_test", + "depends_on": [ + { + "task_key": "zzz_game_details_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select unique_zzz_game_details_game_id --target dev\"]" + } + } + }, + { + "task_key": "not_null_zzz_game_details_game_id_test", + "depends_on": [ + { + "task_key": "zzz_game_details_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select not_null_zzz_game_details_game_id --target dev\"]" + } + } + }, + { + "task_key": "not_null_zzz_game_details_home_test", + "depends_on": [ + { + "task_key": "zzz_game_details_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select not_null_zzz_game_details_home --target dev\"]" + } + } + }, + { + "task_key": "accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_test", + "depends_on": [ + { + "task_key": "zzz_game_details_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle --target dev\"]" + } + } + }, + { + "task_key": "not_null_zzz_game_details_visitor_test", + "depends_on": [ + { + "task_key": "zzz_game_details_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select not_null_zzz_game_details_visitor --target dev\"]" + } + } + }, + { + "task_key": "accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_test", + "depends_on": [ + { + "task_key": "zzz_game_details_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle --target dev\"]" + } + } + }, + { + "task_key": "not_null_zzz_game_details_home_score_test", + "depends_on": [ + { + "task_key": "zzz_game_details_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select not_null_zzz_game_details_home_score --target dev\"]" + } + } + }, + { + "task_key": "not_null_zzz_game_details_visitor_score_test", + "depends_on": [ + { + "task_key": "zzz_game_details_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select not_null_zzz_game_details_visitor_score --target dev\"]" + } + } + }, + { + "task_key": "not_null_zzz_game_details_winner_test", + "depends_on": [ + { + "task_key": "zzz_game_details_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select not_null_zzz_game_details_winner --target dev\"]" + } + } + }, + { + "task_key": "accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_test", + "depends_on": [ + { + "task_key": "zzz_game_details_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle --target dev\"]" + } + } + }, + { + "task_key": "not_null_zzz_game_details_date_test", + "depends_on": [ + { + "task_key": "zzz_game_details_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select not_null_zzz_game_details_date --target dev\"]" + } + } + }, + { + "task_key": "unique_first_dbt_model_id_test", + "depends_on": [ + { + "task_key": "first_dbt_model_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select unique_first_dbt_model_id --target dev\"]" + } + } + }, + { + "task_key": "not_null_first_dbt_model_id_test", + "depends_on": [ + { + "task_key": "first_dbt_model_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select not_null_first_dbt_model_id --target dev\"]" + } + } + }, + { + "task_key": "unique_second_dbt_model_id_test", + "depends_on": [ + { + "task_key": "second_dbt_model_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select unique_second_dbt_model_id --target dev\"]" + } + } + }, + { + "task_key": "not_null_second_dbt_model_id_test", + "depends_on": [ + { + "task_key": "second_dbt_model_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select not_null_second_dbt_model_id --target dev\"]" + } + } + }, + { + "task_key": "not_null_second_dbt_model_amount_usd_test", + "depends_on": [ + { + "task_key": "second_dbt_model_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select not_null_second_dbt_model_amount_usd --target dev\"]" + } + } + }, + { + "task_key": "changed_country_run", + "depends_on": [ + { + "task_key": "country_snapshot_snapshot" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select changed_country --target dev\"]" + } + } + }, + { + "task_key": "country_run", + "depends_on": [ + { + "task_key": "country_codes_seed" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select country --target dev\"]" + } + } + }, + { + "task_key": "country_snapshot_snapshot", + "depends_on": [ + { + "task_key": "country_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt snapshot --select country_snapshot --target dev\"]" + } + } + }, + { + "task_key": "zzz_win_loss_records_run", + "depends_on": [ + { + "task_key": "zzz_game_details_run" + }, + { + "task_key": "zzz_game_details_check_scores_test" + }, + { + "task_key": "zzz_game_details_check_dates_test" + }, + { + "task_key": "unique_zzz_game_details_game_id_test" + }, + { + "task_key": "not_null_zzz_game_details_game_id_test" + }, + { + "task_key": "not_null_zzz_game_details_home_test" + }, + { + "task_key": "accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_test" + }, + { + "task_key": "not_null_zzz_game_details_visitor_test" + }, + { + "task_key": "accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_test" + }, + { + "task_key": "not_null_zzz_game_details_home_score_test" + }, + { + "task_key": "not_null_zzz_game_details_visitor_score_test" + }, + { + "task_key": "not_null_zzz_game_details_winner_test" + }, + { + "task_key": "accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_test" + }, + { + "task_key": "not_null_zzz_game_details_date_test" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt run --select zzz_win_loss_records --target dev\"]" + } + } + }, + { + "task_key": "unique_zzz_win_loss_records_team_test", + "depends_on": [ + { + "task_key": "zzz_win_loss_records_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select unique_zzz_win_loss_records_team --target dev\"]" + } + } + }, + { + "task_key": "not_null_zzz_win_loss_records_team_test", + "depends_on": [ + { + "task_key": "zzz_win_loss_records_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select not_null_zzz_win_loss_records_team --target dev\"]" + } + } + }, + { + "task_key": "not_null_zzz_win_loss_records_wins_test", + "depends_on": [ + { + "task_key": "zzz_win_loss_records_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select not_null_zzz_win_loss_records_wins --target dev\"]" + } + } + }, + { + "task_key": "not_null_zzz_win_loss_records_losses_test", + "depends_on": [ + { + "task_key": "zzz_win_loss_records_run" + } + ], + "environment_key": "Default", + "notebook_task": { + "notebook_path": "./notebooks/dbt_runner.py", + "base_parameters": { + "dbt_commands": "[\"dbt test --select not_null_zzz_win_loss_records_losses --target dev\"]" + } + } + } +] diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/manifest.json b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/manifest.json new file mode 100644 index 00000000..44282d07 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/manifest.json @@ -0,0 +1 @@ +{"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v12.json", "dbt_version": "1.8.7", "generated_at": "2024-12-06T13:55:35.349350Z", "invocation_id": "a2d9eb4c-6787-4ede-b5ce-44e61f3769b2", "env": {}, "project_name": "dbt_demo", "project_id": "f005e70093f1ddd3daa6316f5ec6f69e", "user_id": "b802ff80-7d9c-4428-8a39-5f56cf164bf8", "send_anonymous_usage_stats": true, "adapter_type": "databricks"}, "nodes": {"model.dbt_demo.diamonds_list_colors": {"database": "dbt_demo", "schema": "dbt_dev", "name": "diamonds_list_colors", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/diamonds_list_colors.sql", "original_file_path": "src/models/sql_model1/diamonds_list_colors.sql", "unique_id": "model.dbt_demo.diamonds_list_colors", "fqn": ["dbt_demo", "sql_model1", "diamonds_list_colors"], "alias": "diamonds_list_colors", "checksum": {"name": "sha256", "checksum": "fb8e076fbc40ae30c2441fbc660922d1d3428cf1557cc8f400dbd69caa78c266"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/diamonds_list_colors.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "created_at": 1733486794.194898, "relation_name": "`dbt_demo`.`dbt_dev`.`diamonds_list_colors`", "raw_code": "-- create a view\nselect distinct color\nfrom {{ ref('diamonds_four_cs') }}\nsort by color asc", "language": "sql", "refs": [{"name": "diamonds_four_cs", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.diamonds_four_cs"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/diamonds_list_colors.sql", "compiled": true, "compiled_code": "-- create a view\nselect distinct color\nfrom `dbt_demo`.`dbt_dev`.`diamonds_four_cs`\nsort by color asc", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.diamonds_prices": {"database": "dbt_demo", "schema": "dbt_dev", "name": "diamonds_prices", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/diamonds_prices.sql", "original_file_path": "src/models/sql_model1/diamonds_prices.sql", "unique_id": "model.dbt_demo.diamonds_prices", "fqn": ["dbt_demo", "sql_model1", "diamonds_prices"], "alias": "diamonds_prices", "checksum": {"name": "sha256", "checksum": "809b1318095c2871ab3e9d98af3cc11233e1fbb791e19682fbb38070074e53d2"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/diamonds_prices.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "created_at": 1733486794.207654, "relation_name": "`dbt_demo`.`dbt_dev`.`diamonds_prices`", "raw_code": "-- create a view\nselect\n account_id,\n workspace_id,\n usage_date,\n sum(usage_quantity) as quantity,\n sum({{ cents_to_dollars('usage_quantity') }}) as amount_usd\nfrom system.billing.usage\ngroup by account_id, workspace_id, usage_date", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt_demo.cents_to_dollars"], "nodes": []}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/diamonds_prices.sql", "compiled": true, "compiled_code": "-- create a view\nselect\n account_id,\n workspace_id,\n usage_date,\n sum(usage_quantity) as quantity,\n sum(\n (usage_quantity / 100)::numeric(16, 2)\n) as amount_usd\nfrom system.billing.usage\ngroup by account_id, workspace_id, usage_date", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.diamonds_four_cs": {"database": "dbt_demo", "schema": "dbt_dev", "name": "diamonds_four_cs", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/diamonds_four_cs.sql", "original_file_path": "src/models/sql_model1/diamonds_four_cs.sql", "unique_id": "model.dbt_demo.diamonds_four_cs", "fqn": ["dbt_demo", "sql_model1", "diamonds_four_cs"], "alias": "diamonds_four_cs", "checksum": {"name": "sha256", "checksum": "85a0d2493625371fdfbd2d8261dcc22335301c88e08791638ad760e6cb7dd614"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "file_format": "delta"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/diamonds_four_cs.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "materialized": "table", "file_format": "delta"}, "created_at": 1733486794.264364, "relation_name": "`dbt_demo`.`dbt_dev`.`diamonds_four_cs`", "raw_code": "-- create a table\n-- https://docs.getdbt.com/reference/resource-configs/databricks-configs\n{{ config(\n materialized='table',\n file_format='delta'\n) }}\nselect\n carat,\n cut,\n color,\n clarity\n-- get table specification from schema.yml\nfrom {{ source('raw', 'diamonds') }} -- equivalent of: from hive_metastore.default.diamonds", "language": "sql", "refs": [], "sources": [["raw", "diamonds"]], "metrics": [], "depends_on": {"macros": [], "nodes": ["source.dbt_demo.raw.diamonds"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/diamonds_four_cs.sql", "compiled": true, "compiled_code": "-- create a table\n-- https://docs.getdbt.com/reference/resource-configs/databricks-configs\n\nselect\n carat,\n cut,\n color,\n clarity\n-- get table specification from schema.yml\nfrom `hive_metastore`.`default`.`diamonds` -- equivalent of: from hive_metastore.default.diamonds", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.zzz_game_details": {"database": "dbt_demo", "schema": "dbt_dev", "name": "zzz_game_details", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/zzz_game_details.sql", "original_file_path": "src/models/sql_model1/zzz_game_details.sql", "unique_id": "model.dbt_demo.zzz_game_details", "fqn": ["dbt_demo", "sql_model1", "zzz_game_details"], "alias": "zzz_game_details", "checksum": {"name": "sha256", "checksum": "07a0ddce4e171f5838f27124b3574a77cbab23fac66bba7a1b8f87f6bfbd653f"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "file_format": "delta"}, "tags": [], "description": "", "columns": {"game_id": {"name": "game_id", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "home": {"name": "home", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "visitor": {"name": "visitor", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "home_score": {"name": "home_score", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "visitor_score": {"name": "visitor_score", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "winner": {"name": "winner", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "date": {"name": "date", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_demo://src/models/sql_model1/schemas.yml", "build_path": "target/run/dbt_demo/src/models/sql_model1/zzz_game_details.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "materialized": "table", "file_format": "delta"}, "created_at": 1733486794.43343, "relation_name": "`dbt_demo`.`dbt_dev`.`zzz_game_details`", "raw_code": "-- Create a table that provides full details for each game, including\n-- the game ID, the home and visiting teams' city names and scores,\n-- the game winner's city name, and the game date.\n{{ config(\n materialized='table',\n file_format='delta'\n) }}\n-- Step 4 of 4: Replace the visitor team IDs with their city names.\nselect\n game_id,\n home,\n t.team_city as visitor,\n home_score,\n visitor_score,\n -- Step 3 of 4: Display the city name for each game's winner.\n game_date as date,\n case\n when\n home_score > visitor_score\n then\n home\n when\n visitor_score > home_score\n then\n t.team_city\n end as winner\nfrom (\n -- Step 2 of 4: Replace the home team IDs with their actual city names.\n select\n game_id,\n t.team_city as home,\n home_score,\n visitor_team_id,\n visitor_score,\n game_date\n from (\n -- Step 1 of 4: Combine data from various tables (for example, game and team IDs, scores, dates).\n select\n g.game_id,\n go.home_team_id,\n gs.home_team_score as home_score,\n go.visitor_team_id,\n gs.visitor_team_score as visitor_score,\n g.game_date\n from\n marcin_demo.dbt.{{ target.name }}_zzz_games as g,\n marcin_demo.dbt.zzz_game_opponents as go,\n marcin_demo.dbt.zzz_game_scores as gs\n where\n g.game_id = go.game_id\n and g.game_id = gs.game_id\n ) as all_ids,\n marcin_demo.dbt.zzz_teams as t\n where\n all_ids.home_team_id = t.team_id\n) as visitor_ids,\n marcin_demo.dbt.zzz_teams as t\nwhere\n visitor_ids.visitor_team_id = t.team_id\norder by game_date desc", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/zzz_game_details.sql", "compiled": true, "compiled_code": "-- Create a table that provides full details for each game, including\n-- the game ID, the home and visiting teams' city names and scores,\n-- the game winner's city name, and the game date.\n\n-- Step 4 of 4: Replace the visitor team IDs with their city names.\nselect\n game_id,\n home,\n t.team_city as visitor,\n home_score,\n visitor_score,\n -- Step 3 of 4: Display the city name for each game's winner.\n game_date as date,\n case\n when\n home_score > visitor_score\n then\n home\n when\n visitor_score > home_score\n then\n t.team_city\n end as winner\nfrom (\n -- Step 2 of 4: Replace the home team IDs with their actual city names.\n select\n game_id,\n t.team_city as home,\n home_score,\n visitor_team_id,\n visitor_score,\n game_date\n from (\n -- Step 1 of 4: Combine data from various tables (for example, game and team IDs, scores, dates).\n select\n g.game_id,\n go.home_team_id,\n gs.home_team_score as home_score,\n go.visitor_team_id,\n gs.visitor_team_score as visitor_score,\n g.game_date\n from\n marcin_demo.dbt.dev_zzz_games as g,\n marcin_demo.dbt.zzz_game_opponents as go,\n marcin_demo.dbt.zzz_game_scores as gs\n where\n g.game_id = go.game_id\n and g.game_id = gs.game_id\n ) as all_ids,\n marcin_demo.dbt.zzz_teams as t\n where\n all_ids.home_team_id = t.team_id\n) as visitor_ids,\n marcin_demo.dbt.zzz_teams as t\nwhere\n visitor_ids.visitor_team_id = t.team_id\norder by game_date desc", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.second_dbt_model": {"database": "dbt_demo", "schema": "dbt_dev", "name": "second_dbt_model", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model2/second_dbt_model.sql", "original_file_path": "src/models/sql_model2/second_dbt_model.sql", "unique_id": "model.dbt_demo.second_dbt_model", "fqn": ["dbt_demo", "sql_model2", "second_dbt_model"], "alias": "second_dbt_model", "checksum": {"name": "sha256", "checksum": "226c0e1d47e6f9bee7430c7acffadc0d239fa185558a2de703c5f0ab0b01d218"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": true, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "A starter dbt model", "columns": {"id": {"name": "id", "description": "The primary key for this table", "meta": {}, "data_type": "int", "constraints": [], "quote": null, "tags": []}, "amount_usd": {"name": "amount_usd", "description": "The primary key for this table", "meta": {}, "data_type": "decimal", "constraints": [], "quote": null, "tags": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_demo://src/models/sql_model2/schemas.yml", "build_path": "target/run/dbt_demo/src/models/sql_model2/second_dbt_model.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "contract": {"enforced": true}}, "created_at": 1733486794.536133, "relation_name": "`dbt_demo`.`dbt_dev`.`second_dbt_model`", "raw_code": "-- Use the `ref` function to select from other models\n\nselect\n *, {{ cents_to_dollars('id') }} as amount_usd\nfrom {{ ref('first_dbt_model') }}\nwhere id = 1", "language": "sql", "refs": [{"name": "first_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt_demo.cents_to_dollars"], "nodes": ["model.dbt_demo.first_dbt_model"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model2/second_dbt_model.sql", "compiled": true, "compiled_code": "-- Use the `ref` function to select from other models\n\nselect\n *, \n (id / 100)::numeric(16, 2)\n as amount_usd\nfrom `dbt_demo`.`dbt_dev`.`first_dbt_model`\nwhere id = 1", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": true, "alias_types": true, "checksum": "d4b533efdb37067787094e25d1860ddcc473cb5cf92ebe2c358011ff2866658f"}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.first_dbt_model": {"database": "dbt_demo", "schema": "dbt_dev", "name": "first_dbt_model", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model2/first_dbt_model.sql", "original_file_path": "src/models/sql_model2/first_dbt_model.sql", "unique_id": "model.dbt_demo.first_dbt_model", "fqn": ["dbt_demo", "sql_model2", "first_dbt_model"], "alias": "first_dbt_model", "checksum": {"name": "sha256", "checksum": "91d9a8b1e5c899463e52c50492b7b3c7df99fa6635c868f9165ca83e5f3b13f7"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": true, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "A starter dbt model", "columns": {"id": {"name": "id", "description": "The primary key for this table", "meta": {}, "data_type": "int", "constraints": [], "quote": null, "tags": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_demo://src/models/sql_model2/schemas.yml", "build_path": "target/run/dbt_demo/src/models/sql_model2/first_dbt_model.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "contract": {"enforced": true}, "materialized": "table"}, "created_at": 1733486794.534828, "relation_name": "`dbt_demo`.`dbt_dev`.`first_dbt_model`", "raw_code": "/*\n Welcome to your first dbt model!\n Did you know that you can also configure models directly within SQL files?\n This will override configurations stated in dbt_project.yml\n\n Try changing \"table\" to \"view\" below\n*/\n\n{{ config(materialized='table') }}\n\nwith source_data as (\n\n select 1 as id\n union all\n select 2 as id\n\n)\n\nselect *\nfrom source_data\n\n/*\n Uncomment the line below to remove records with null `id` values\n*/\n\n-- where id is not null", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model2/first_dbt_model.sql", "compiled": true, "compiled_code": "/*\n Welcome to your first dbt model!\n Did you know that you can also configure models directly within SQL files?\n This will override configurations stated in dbt_project.yml\n\n Try changing \"table\" to \"view\" below\n*/\n\n\n\nwith source_data as (\n\n select 1 as id\n union all\n select 2 as id\n\n)\n\nselect *\nfrom source_data\n\n/*\n Uncomment the line below to remove records with null `id` values\n*/\n\n-- where id is not null", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": true, "alias_types": true, "checksum": "a31b0e38eb3a9f0c0ed2f45bd615732c39e48b7f698f50a36748a687fd6d7289"}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "test.dbt_demo.zzz_game_details_check_scores": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "zzz_game_details_check_scores", "resource_type": "test", "package_name": "dbt_demo", "path": "data_tests/zzz_game_details_check_scores.sql", "original_file_path": "tests/data_tests/zzz_game_details_check_scores.sql", "unique_id": "test.dbt_demo.zzz_game_details_check_scores", "fqn": ["dbt_demo", "data_tests", "zzz_game_details_check_scores"], "alias": "zzz_game_details_check_scores", "checksum": {"name": "sha256", "checksum": "49cca9739f2738a4609833d267ade1a137756523f0659ec02f0bd31adb3c5789"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.31489, "relation_name": null, "raw_code": "-- This sport allows no negative scores or tie games.\n-- For this test to pass, this query must return no results.\n\nselect home_score, visitor_score\nfrom {{ ref('zzz_game_details') }}\nwhere home_score < 0\nor visitor_score < 0\nor home_score = visitor_score", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "test.dbt_demo.zzz_game_details_check_dates": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "zzz_game_details_check_dates", "resource_type": "test", "package_name": "dbt_demo", "path": "data_tests/zzz_game_details_check_dates.sql", "original_file_path": "tests/data_tests/zzz_game_details_check_dates.sql", "unique_id": "test.dbt_demo.zzz_game_details_check_dates", "fqn": ["dbt_demo", "data_tests", "zzz_game_details_check_dates"], "alias": "zzz_game_details_check_dates", "checksum": {"name": "sha256", "checksum": "ac2ae3e40e5b734327e8a60a8c944e535c472314feee78d9320da1f7ed515918"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.3213592, "relation_name": null, "raw_code": "-- This season's games happened between 2020-12-12 and 2021-02-06.\n-- For this test to pass, this query must return no results.\n\nselect date\nfrom {{ ref('zzz_game_details') }}\nwhere date < '2020-12-12'\nor date > '2021-02-06'", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "test.dbt_demo.zzz_win_loss_records_check_records": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "zzz_win_loss_records_check_records", "resource_type": "test", "package_name": "dbt_demo", "path": "data_tests/zzz_win_loss_records_check_records.sql", "original_file_path": "tests/data_tests/zzz_win_loss_records_check_records.sql", "unique_id": "test.dbt_demo.zzz_win_loss_records_check_records", "fqn": ["dbt_demo", "data_tests", "zzz_win_loss_records_check_records"], "alias": "zzz_win_loss_records_check_records", "checksum": {"name": "sha256", "checksum": "8840acce153171ad015abfe10626d1220cc9533e98c859a2b483f0d25d2e4903"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.323633, "relation_name": null, "raw_code": "-- Each team participated in 4 games this season.\n-- For this test to pass, this query must return no results.\n\nselect wins, losses\nfrom {{ ref('zzz_win_loss_records') }}\nwhere wins < 0 or wins > 4\nor losses < 0 or losses > 4\nor (wins + losses) > 4", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "test.dbt_demo.diamonds_list_colors_test": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "diamonds_list_colors_test", "resource_type": "test", "package_name": "dbt_demo", "path": "data_tests/diamonds_list_colors_test.sql", "original_file_path": "tests/data_tests/diamonds_list_colors_test.sql", "unique_id": "test.dbt_demo.diamonds_list_colors_test", "fqn": ["dbt_demo", "data_tests", "diamonds_list_colors_test"], "alias": "diamonds_list_colors_test", "checksum": {"name": "sha256", "checksum": "4a4d3ee2fa6aad9a7fc4b248111aa6432b67374e217793fbf29d92c2504f13be"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.3264809, "relation_name": null, "raw_code": "SELECT color, COUNT(*)\nFROM {{ ref('diamonds_list_colors') }}\nGROUP BY color\nHAVING COUNT(*) > 1", "language": "sql", "refs": [{"name": "diamonds_list_colors", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.diamonds_list_colors"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "seed.dbt_demo.country_codes": {"database": "dbt_demo", "schema": "dbt_dev", "name": "country_codes", "resource_type": "seed", "package_name": "dbt_demo", "path": "country_codes.csv", "original_file_path": "src/seeds/country_codes.csv", "unique_id": "seed.dbt_demo.country_codes", "fqn": ["dbt_demo", "country_codes"], "alias": "country_codes", "checksum": {"name": "sha256", "checksum": "b54c38e4337b053b07cdc37fa5364e8f159b4e37b77d36281a8d2d11a2c09b8c"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "seed", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "delimiter": ",", "quote_columns": null}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.388313, "relation_name": "`dbt_demo`.`dbt_dev`.`country_codes`", "raw_code": "", "root_path": "/Users/marcin.wojtyczka/Documents/repos/dbt_demo", "depends_on": {"macros": []}}, "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "unique_zzz_game_details_game_id", "resource_type": "test", "package_name": "dbt_demo", "path": "unique_zzz_game_details_game_id.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218", "fqn": ["dbt_demo", "models", "sql_model1", "unique_zzz_game_details_game_id"], "alias": "unique_zzz_game_details_game_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.462498, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_unique"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "game_id", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "unique", "kwargs": {"column_name": "game_id", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_game_id", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_game_id.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_game_id"], "alias": "not_null_zzz_game_details_game_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.4634, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "game_id", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "game_id", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_home", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_home.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_home"], "alias": "not_null_zzz_game_details_home", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.464213, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "home", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "home", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle", "resource_type": "test", "package_name": "dbt_demo", "path": "accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40", "fqn": ["dbt_demo", "models", "sql_model1", "accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle"], "alias": "accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": "accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3", "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"alias": "accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3"}, "created_at": 1733486794.4649322, "relation_name": null, "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3\") }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_accepted_values", "macro.dbt.get_where_subquery"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "home", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "accepted_values", "kwargs": {"values": ["Amsterdam", "San Francisco", "Seattle"], "column_name": "home", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_visitor", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_visitor.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_visitor"], "alias": "not_null_zzz_game_details_visitor", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.474986, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "visitor", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "visitor", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle", "resource_type": "test", "package_name": "dbt_demo", "path": "accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d", "fqn": ["dbt_demo", "models", "sql_model1", "accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle"], "alias": "accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": "accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272", "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"alias": "accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272"}, "created_at": 1733486794.4757178, "relation_name": null, "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272\") }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_accepted_values", "macro.dbt.get_where_subquery"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "visitor", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "accepted_values", "kwargs": {"values": ["Amsterdam", "San Francisco", "Seattle"], "column_name": "visitor", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_home_score", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_home_score.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_home_score"], "alias": "not_null_zzz_game_details_home_score", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.4776351, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "home_score", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "home_score", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_visitor_score", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_visitor_score.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_visitor_score"], "alias": "not_null_zzz_game_details_visitor_score", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.4783318, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "visitor_score", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "visitor_score", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_winner", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_winner.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_winner"], "alias": "not_null_zzz_game_details_winner", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.479018, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "winner", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "winner", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle", "resource_type": "test", "package_name": "dbt_demo", "path": "accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c", "fqn": ["dbt_demo", "models", "sql_model1", "accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle"], "alias": "accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": "accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca", "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"alias": "accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca"}, "created_at": 1733486794.4797802, "relation_name": null, "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca\") }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_accepted_values", "macro.dbt.get_where_subquery"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "winner", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "accepted_values", "kwargs": {"values": ["Amsterdam", "San Francisco", "Seattle"], "column_name": "winner", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_date", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_date.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_date"], "alias": "not_null_zzz_game_details_date", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.4815848, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "date", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "date", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.unique_first_dbt_model_id.9dda05743e": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "unique_first_dbt_model_id", "resource_type": "test", "package_name": "dbt_demo", "path": "unique_first_dbt_model_id.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.unique_first_dbt_model_id.9dda05743e", "fqn": ["dbt_demo", "models", "sql_model2", "unique_first_dbt_model_id"], "alias": "unique_first_dbt_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.536581, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "first_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_unique"], "nodes": ["model.dbt_demo.first_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "id", "file_key_name": "models.first_dbt_model", "attached_node": "model.dbt_demo.first_dbt_model", "test_metadata": {"name": "unique", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('first_dbt_model')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_first_dbt_model_id", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_first_dbt_model_id.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061", "fqn": ["dbt_demo", "models", "sql_model2", "not_null_first_dbt_model_id"], "alias": "not_null_first_dbt_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.537291, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "first_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.first_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "id", "file_key_name": "models.first_dbt_model", "attached_node": "model.dbt_demo.first_dbt_model", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('first_dbt_model')) }}"}, "namespace": null}}, "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "unique_second_dbt_model_id", "resource_type": "test", "package_name": "dbt_demo", "path": "unique_second_dbt_model_id.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4", "fqn": ["dbt_demo", "models", "sql_model2", "unique_second_dbt_model_id"], "alias": "unique_second_dbt_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.537977, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "second_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_unique"], "nodes": ["model.dbt_demo.second_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "id", "file_key_name": "models.second_dbt_model", "attached_node": "model.dbt_demo.second_dbt_model", "test_metadata": {"name": "unique", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('second_dbt_model')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_second_dbt_model_id", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_second_dbt_model_id.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2", "fqn": ["dbt_demo", "models", "sql_model2", "not_null_second_dbt_model_id"], "alias": "not_null_second_dbt_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.538719, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "second_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.second_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "id", "file_key_name": "models.second_dbt_model", "attached_node": "model.dbt_demo.second_dbt_model", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('second_dbt_model')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_second_dbt_model_amount_usd", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_second_dbt_model_amount_usd.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f", "fqn": ["dbt_demo", "models", "sql_model2", "not_null_second_dbt_model_amount_usd"], "alias": "not_null_second_dbt_model_amount_usd", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.539395, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "second_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.second_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "amount_usd", "file_key_name": "models.second_dbt_model", "attached_node": "model.dbt_demo.second_dbt_model", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "amount_usd", "model": "{{ get_where_subquery(ref('second_dbt_model')) }}"}, "namespace": null}}, "model.dbt_demo.changed_country": {"database": "dbt_demo", "schema": "dbt_dev", "name": "changed_country", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/changed_country.sql", "original_file_path": "src/models/sql_model1/changed_country.sql", "unique_id": "model.dbt_demo.changed_country", "fqn": ["dbt_demo", "sql_model1", "changed_country"], "alias": "changed_country", "checksum": {"name": "sha256", "checksum": "e1f9599d0e633856772d321bfc75b89ead550df1ba8fe3dc40a8102c9cc2eb9d"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "file_format": "delta"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/changed_country.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "materialized": "table", "file_format": "delta"}, "created_at": 1733490496.062186, "relation_name": "`dbt_demo`.`dbt_dev`.`changed_country`", "raw_code": "{{ config(\n materialized='table',\n file_format='delta'\n) }}\nselect * from {{ ref('country_snapshot') }}", "language": "sql", "refs": [{"name": "country_snapshot", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["snapshot.dbt_demo.country_snapshot"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/changed_country.sql", "compiled": true, "compiled_code": "\nselect * from `dbt_demo`.`dbt_dev`.`country_snapshot`", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.country": {"database": "dbt_demo", "schema": "dbt_dev", "name": "country", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/country.sql", "original_file_path": "src/models/sql_model1/country.sql", "unique_id": "model.dbt_demo.country", "fqn": ["dbt_demo", "sql_model1", "country"], "alias": "country", "checksum": {"name": "sha256", "checksum": "be347ee69b8c27dec884b8bd692e1aa0ffdac4fb2578c410d048dc9ed16b0b08"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "file_format": "delta"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/country.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "materialized": "table", "file_format": "delta"}, "created_at": 1733492304.094096, "relation_name": "`dbt_demo`.`dbt_dev`.`country`", "raw_code": "-- create a table\n-- https://docs.getdbt.com/reference/resource-configs/databricks-configs\n{{ config(\n materialized='table',\n file_format='delta'\n) }}\nselect\n country_code,\n country_name\nfrom {{ ref('country_codes') }} -- use seed data to populate the table", "language": "sql", "refs": [{"name": "country_codes", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["seed.dbt_demo.country_codes"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/country.sql", "compiled": true, "compiled_code": "-- create a table\n-- https://docs.getdbt.com/reference/resource-configs/databricks-configs\n\nselect\n country_code,\n country_name\nfrom `dbt_demo`.`dbt_dev`.`country_codes` -- use seed data to populate the table", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "snapshot.dbt_demo.country_snapshot": {"database": "dbt_demo", "schema": "dbt_dev", "name": "country_snapshot", "resource_type": "snapshot", "package_name": "dbt_demo", "path": "country_snapshot.sql", "original_file_path": "src/snapshots/country_snapshot.sql", "unique_id": "snapshot.dbt_demo.country_snapshot", "fqn": ["dbt_demo", "country_snapshot", "country_snapshot"], "alias": "country_snapshot", "checksum": {"name": "sha256", "checksum": "100b8c3c320732e98704fe520db0c0bcbf9f1fccf565fccd5807b87cd0d01984"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "snapshot", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "country_code", "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "strategy": "check", "target_schema": "dbt_dev", "target_database": "dbt_demo", "updated_at": null, "check_cols": ["country_code"]}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"target_database": "dbt_demo", "target_schema": "dbt_dev", "unique_key": "country_code", "strategy": "check", "check_cols": ["country_code"]}, "created_at": 1733493205.419168, "relation_name": "`dbt_demo`.`dbt_dev`.`country_snapshot`", "raw_code": "\n\n{% set unique_key = 'country_code' %}\n{% set strategy = 'check' %}\n{% set check_cols = ['country_code'] %}\n{% set target_database = 'dbt_demo' %}\n{% set target_schema = 'dbt_dev' %}\n\n{{\n config(\n target_database=target_database,\n target_schema=target_schema,\n unique_key=unique_key,\n strategy=strategy,\n check_cols=check_cols\n )\n}}\n\n-- SQL query for the snapshot\nSELECT *\nFROM {{ ref('country') }}\n\n", "language": "sql", "refs": [{"name": "country", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.country"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "model.dbt_demo.zzz_win_loss_records": {"database": "dbt_demo", "schema": "dbt_dev", "name": "zzz_win_loss_records", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/zzz_win_loss_records.sql", "original_file_path": "src/models/sql_model1/zzz_win_loss_records.sql", "unique_id": "model.dbt_demo.zzz_win_loss_records", "fqn": ["dbt_demo", "sql_model1", "zzz_win_loss_records"], "alias": "zzz_win_loss_records", "checksum": {"name": "sha256", "checksum": "2af4334f4afcfc71dbb39dd2bb88e64cf6e49fb20ec401ef0abf345aa827eb69"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "", "columns": {"team": {"name": "team", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "wins": {"name": "wins", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "losses": {"name": "losses", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_demo://src/models/sql_model1/schemas.yml", "build_path": "target/run/dbt_demo/src/models/sql_model1/zzz_win_loss_records.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "created_at": 1733493328.3391528, "relation_name": "`dbt_demo`.`dbt_dev`.`zzz_win_loss_records`", "raw_code": "-- Step 2 of 2: Calculate the number of wins and losses for each team.\nselect\n winner as team,\n count(winner) as wins,\n -- Each team played in 4 games.\n (4 - count(winner)) as losses\nfrom (\n -- Step 1 of 2: Determine the winner and loser for each game.\n select\n game_id,\n winner,\n case\n when\n home = winner\n then\n visitor\n else\n home\n end as loser\n from {{ ref('zzz_game_details') }}\n)\ngroup by winner\norder by wins desc", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/zzz_win_loss_records.sql", "compiled": true, "compiled_code": "-- Step 2 of 2: Calculate the number of wins and losses for each team.\nselect\n winner as team,\n count(winner) as wins,\n -- Each team played in 4 games.\n (4 - count(winner)) as losses\nfrom (\n -- Step 1 of 2: Determine the winner and loser for each game.\n select\n game_id,\n winner,\n case\n when\n home = winner\n then\n visitor\n else\n home\n end as loser\n from `dbt_demo`.`dbt_dev`.`zzz_game_details`\n)\ngroup by winner\norder by wins desc", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "unique_zzz_win_loss_records_team", "resource_type": "test", "package_name": "dbt_demo", "path": "unique_zzz_win_loss_records_team.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105", "fqn": ["dbt_demo", "models", "sql_model1", "unique_zzz_win_loss_records_team"], "alias": "unique_zzz_win_loss_records_team", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733493328.3774579, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_unique"], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "team", "file_key_name": "models.zzz_win_loss_records", "attached_node": "model.dbt_demo.zzz_win_loss_records", "test_metadata": {"name": "unique", "kwargs": {"column_name": "team", "model": "{{ get_where_subquery(ref('zzz_win_loss_records')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_win_loss_records_team", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_win_loss_records_team.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_win_loss_records_team"], "alias": "not_null_zzz_win_loss_records_team", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733493328.378356, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "team", "file_key_name": "models.zzz_win_loss_records", "attached_node": "model.dbt_demo.zzz_win_loss_records", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "team", "model": "{{ get_where_subquery(ref('zzz_win_loss_records')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_win_loss_records_wins", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_win_loss_records_wins.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_win_loss_records_wins"], "alias": "not_null_zzz_win_loss_records_wins", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733493328.3790798, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "wins", "file_key_name": "models.zzz_win_loss_records", "attached_node": "model.dbt_demo.zzz_win_loss_records", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "wins", "model": "{{ get_where_subquery(ref('zzz_win_loss_records')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_win_loss_records_losses", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_win_loss_records_losses.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_win_loss_records_losses"], "alias": "not_null_zzz_win_loss_records_losses", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733493328.379801, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "losses", "file_key_name": "models.zzz_win_loss_records", "attached_node": "model.dbt_demo.zzz_win_loss_records", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "losses", "model": "{{ get_where_subquery(ref('zzz_win_loss_records')) }}"}, "namespace": null}}}, "sources": {"source.dbt_demo.raw.diamonds": {"database": "hive_metastore", "schema": "default", "name": "diamonds", "resource_type": "source", "package_name": "dbt_demo", "path": "src/models/sql_model1/sources.yml", "original_file_path": "src/models/sql_model1/sources.yml", "unique_id": "source.dbt_demo.raw.diamonds", "fqn": ["dbt_demo", "sql_model1", "raw", "diamonds"], "source_name": "raw", "source_description": "", "loader": "", "identifier": "diamonds", "quoting": {"database": null, "schema": null, "identifier": null, "column": null}, "loaded_at_field": null, "freshness": {"warn_after": {"count": null, "period": null}, "error_after": {"count": null, "period": null}, "filter": null}, "external": null, "description": "", "columns": {}, "meta": {}, "source_meta": {}, "tags": [], "config": {"enabled": true}, "patch_path": null, "unrendered_config": {}, "relation_name": "`hive_metastore`.`default`.`diamonds`", "created_at": 1733486794.541405}}, "macros": {"macro.dbt_demo.cents_to_dollars": {"name": "cents_to_dollars", "resource_type": "macro", "package_name": "dbt_demo", "path": "src/macros/centrs_to_dollars.sql", "original_file_path": "src/macros/centrs_to_dollars.sql", "unique_id": "macro.dbt_demo.cents_to_dollars", "macro_sql": "{% macro cents_to_dollars(column_name, scale=2) %}\n ({{ column_name }} / 100)::numeric(16, {{ scale }})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.526016, "supported_languages": null}, "macro.dbt_databricks.materialization_materialized_view_databricks": {"name": "materialization_materialized_view_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialization_materialized_view_databricks", "macro_sql": "{% materialization materialized_view, adapter = 'databricks' %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.MaterializedView) %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation) %}\n\n {% if build_sql == '' %}\n {{ materialized_view_execute_no_op(target_relation) }}\n {% else %}\n {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.run_hooks", "macro.dbt_databricks.materialized_view_get_build_sql", "macro.dbt_databricks.materialized_view_execute_no_op", "macro.dbt_databricks.materialized_view_execute_build_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5290601, "supported_languages": ["sql"]}, "macro.dbt_databricks.materialized_view_get_build_sql": {"name": "materialized_view_get_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialized_view_get_build_sql", "macro_sql": "{% macro materialized_view_get_build_sql(existing_relation, target_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_materialized_view %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %}\n\n {% if configuration_changes is none %}\n {% set build_sql = refresh_materialized_view(target_relation) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, None, None) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = \"\" %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt_databricks.get_replace_sql", "macro.dbt.get_materialized_view_configuration_changes", "macro.dbt.refresh_materialized_view", "macro.dbt_databricks.get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.530187, "supported_languages": null}, "macro.dbt_databricks.materialized_view_execute_no_op": {"name": "materialized_view_execute_no_op", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialized_view_execute_no_op", "macro_sql": "{% macro materialized_view_execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5303931, "supported_languages": null}, "macro.dbt_databricks.materialized_view_execute_build_sql": {"name": "materialized_view_execute_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialized_view_execute_build_sql", "macro_sql": "{% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n\n {%- if build_sql is string %}\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n {%- else %}\n {%- for sql in build_sql %}\n {% call statement(name=\"main\") %}\n {{ sql }}\n {% endcall %}\n {% endfor %}\n {% endif %}\n\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model, for_relation=False) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.531114, "supported_languages": null}, "macro.dbt_databricks.materialization_view_databricks": {"name": "materialization_view_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/view.sql", "original_file_path": "macros/materializations/view.sql", "unique_id": "macro.dbt_databricks.materialization_view_databricks", "macro_sql": "{% materialization view, adapter='databricks' -%}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n {% set tags = config.get('databricks_tags') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and not old_relation.is_view -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=True) %}\n\n {%- do apply_tags(target_relation, tags) -%}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.apply_tags"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.532544, "supported_languages": ["sql"]}, "macro.dbt_databricks.materialization_table_databricks": {"name": "materialization_table_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_databricks.materialization_table_databricks", "macro_sql": "{% materialization table, adapter = 'databricks', supported_languages=['sql', 'python'] %}\n {{ log(\"MATERIALIZING TABLE\") }}\n {%- set language = model['language'] -%}\n {%- set identifier = model['alias'] -%}\n {%- set grant_config = config.get('grants') -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- set tags = config.get('databricks_tags') -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier, needs_information=True) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier,\n schema=schema,\n database=database,\n type='table') -%}\n\n {{ run_hooks(pre_hooks) }}\n\n -- setup: if the target relation already exists, drop it\n -- in case if the existing and future table is delta, we want to do a\n -- create or replace table instead of dropping, so we don't have the table unavailable\n {% if old_relation and (not (old_relation.is_delta and config.get('file_format', default='delta') == 'delta')) or (old_relation.is_materialized_view or old_relation.is_streaming_table) -%}\n {{ adapter.drop_relation(old_relation) }}\n {%- endif %}\n\n -- build model\n\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n {% if language==\"python\" %}\n {% do apply_tblproperties(target_relation, tblproperties) %}\n {% endif %}\n {%- do apply_tags(target_relation, tags) -%}\n\n {% do persist_docs(target_relation, model, for_relation=language=='python') %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {% do optimize(target_relation) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]})}}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.apply_tblproperties", "macro.dbt_databricks.apply_tags", "macro.dbt.persist_docs", "macro.dbt_databricks.persist_constraints", "macro.dbt_databricks.optimize"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5349078, "supported_languages": ["sql", "python"]}, "macro.dbt_databricks.databricks_build_snapshot_staging_table": {"name": "databricks_build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_databricks.databricks_build_snapshot_staging_table", "macro_sql": "{% macro databricks_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set tmp_identifier = target_relation.identifier ~ '__dbt_tmp' %}\n\n {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier,\n schema=target_relation.schema,\n database=target_relation.database,\n type='view') -%}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {# needs to be a non-temp view so that its columns can be ascertained via `describe` #}\n {% call statement('build_snapshot_staging_relation') %}\n create or replace view {{ tmp_relation }}\n as\n {{ select }}\n {% endcall %}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_staging_table", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5406778, "supported_languages": null}, "macro.dbt_databricks.materialization_snapshot_databricks": {"name": "materialization_snapshot_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_databricks.materialization_snapshot_databricks", "macro_sql": "{% materialization snapshot, adapter='databricks' %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n {%- set file_format = config.get('file_format', 'delta') -%}\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = databricks__get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table',\n needs_information=True) -%}\n\n {%- if file_format not in ['delta', 'hudi'] -%}\n {% set invalid_format_msg -%}\n Invalid file format: {{ file_format }}\n Snapshot functionality requires file_format be set to 'delta' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n\n {%- if target_relation_exists -%}\n {%- if not target_relation.is_delta and not target_relation.is_hudi -%}\n {% set invalid_format_msg -%}\n The existing table {{ model.schema }}.{{ target_table }} is in another format than 'delta' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n {% endif %}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% do persist_docs(target_relation, model, for_relation=False) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% if target_relation.database is none %}\n {% set staging_table = spark_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% else %}\n {% set staging_table = databricks_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% endif %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% call statement_with_staging_table('main', staging_table) %}\n {{ final_sql }}\n {% endcall %}\n\n {% do persist_docs(target_relation, model, for_relation=True) %}\n\n {% endif %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.statement", "macro.dbt.persist_docs", "macro.dbt_spark.spark_build_snapshot_staging_table", "macro.dbt_databricks.databricks_build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt_databricks.statement_with_staging_table", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.544019, "supported_languages": ["sql"]}, "macro.dbt_databricks.materialization_streaming_table_databricks": {"name": "materialization_streaming_table_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.materialization_streaming_table_databricks", "macro_sql": "{% materialization streaming_table, adapter='databricks' %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.StreamingTable) %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n{% set build_sql = streaming_table_get_build_sql(existing_relation, target_relation) %}\n\n {% if build_sql == '' %}\n {{ streaming_table_execute_no_op(target_relation) }}\n {% else %}\n {{ streaming_table_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.run_hooks", "macro.dbt_databricks.streaming_table_get_build_sql", "macro.dbt_databricks.streaming_table_execute_no_op", "macro.dbt_databricks.streaming_table_execute_build_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.546883, "supported_languages": ["sql"]}, "macro.dbt_databricks.streaming_table_get_build_sql": {"name": "streaming_table_get_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.streaming_table_get_build_sql", "macro_sql": "{% macro streaming_table_get_build_sql(existing_relation, target_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_streaming_table_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_streaming_table %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_streaming_table_configuration_changes(existing_relation, config) %}\n {% if configuration_changes is none %}\n {% set build_sql = refresh_streaming_table(target_relation, sql) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_streaming_table_as_sql(target_relation, configuration_changes, sql, existing_relation, None, None) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = \"\" %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt_databricks.get_create_streaming_table_as_sql", "macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.get_streaming_table_configuration_changes", "macro.dbt_databricks.refresh_streaming_table", "macro.dbt_databricks.get_alter_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5479271, "supported_languages": null}, "macro.dbt_databricks.streaming_table_execute_no_op": {"name": "streaming_table_execute_no_op", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.streaming_table_execute_no_op", "macro_sql": "{% macro streaming_table_execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.548121, "supported_languages": null}, "macro.dbt_databricks.streaming_table_execute_build_sql": {"name": "streaming_table_execute_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.streaming_table_execute_build_sql", "macro_sql": "{% macro streaming_table_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n\n {%- if build_sql is string %}\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n {%- else %}\n {%- for sql in build_sql %}\n {% call statement(name=\"main\") %}\n {{ sql }}\n {% endcall %}\n {% endfor %}\n {% endif %}\n\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model, for_relation=False) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.548807, "supported_languages": null}, "macro.dbt_databricks.databricks__can_clone_table": {"name": "databricks__can_clone_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_databricks.databricks__can_clone_table", "macro_sql": "{% macro databricks__can_clone_table() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.550354, "supported_languages": null}, "macro.dbt_databricks.databricks__create_or_replace_clone": {"name": "databricks__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_databricks.databricks__create_or_replace_clone", "macro_sql": "{% macro databricks__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace\n table {{ this_relation }}\n shallow clone {{ defer_relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.550466, "supported_languages": null}, "macro.dbt_databricks.materialization_clone_databricks": {"name": "materialization_clone_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_databricks.materialization_clone_databricks", "macro_sql": "\n\n{%- materialization clone, adapter='databricks' -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n {%- set file_format = config.get('file_format', validator=validation.any[basestring]) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% call statement('main') %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n {% endif %}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.552143, "supported_languages": ["sql"]}, "macro.dbt_databricks.dbt_databricks_validate_get_file_format": {"name": "dbt_databricks_validate_get_file_format", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_databricks.dbt_databricks_validate_get_file_format", "macro_sql": "{% macro dbt_databricks_validate_get_file_format(raw_file_format) %}\n {#-- Validate the file format #}\n\n {% set accepted_formats = ['text', 'csv', 'json', 'jdbc', 'parquet', 'orc', 'hive', 'delta', 'libsvm', 'hudi'] %}\n\n {% set invalid_file_format_msg -%}\n Invalid file format provided: {{ raw_file_format }}\n Expected one of: {{ accepted_formats | join(', ') }}\n {%- endset %}\n\n {% if raw_file_format not in accepted_formats %}\n {% do exceptions.raise_compiler_error(invalid_file_format_msg) %}\n {% endif %}\n\n {% do return(raw_file_format) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.553088, "supported_languages": null}, "macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy": {"name": "dbt_databricks_validate_get_incremental_strategy", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy", "macro_sql": "{% macro dbt_databricks_validate_get_incremental_strategy(raw_strategy, file_format) %}\n {#-- Validate the incremental strategy #}\n\n {% set invalid_strategy_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n Expected one of: 'merge', 'replace_where', 'append', 'insert_overwrite'\n {%- endset %}\n\n {% set invalid_delta_only_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You can only choose this strategy when file_format is set to 'delta'\n {%- endset %}\n\n {% set invalid_insert_overwrite_endpoint_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You cannot use this strategy when connecting via warehouse\n Use the 'merge' or 'replace_where' strategy instead\n {%- endset %}\n\n {% if raw_strategy not in ['append', 'merge', 'insert_overwrite', 'replace_where'] %}\n {% do exceptions.raise_compiler_error(invalid_strategy_msg) %}\n {%-else %}\n {% if raw_strategy == 'merge' and file_format not in ['delta', 'hudi'] %}\n {% do exceptions.raise_compiler_error(invalid_delta_only_msg) %}\n {% endif %}\n {% if raw_strategy == 'replace_where' and file_format not in ['delta'] %}\n {% do exceptions.raise_compiler_error(invalid_delta_only_msg) %}\n {% endif %}\n {% endif %}\n\n {% do return(raw_strategy) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.55381, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_default_sql": {"name": "databricks__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_default_sql", "macro_sql": "{% macro databricks__get_incremental_default_sql(arg_dict) %}\n {{ return(get_incremental_merge_sql(arg_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.55716, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_append_sql": {"name": "databricks__get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_append_sql", "macro_sql": "{% macro databricks__get_incremental_append_sql(arg_dict) %}\n {% do return(get_insert_into_sql(arg_dict[\"temp_relation\"], arg_dict[\"target_relation\"])) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_insert_into_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5573308, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_replace_where_sql": {"name": "databricks__get_incremental_replace_where_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_replace_where_sql", "macro_sql": "{% macro databricks__get_incremental_replace_where_sql(arg_dict) %}\n {% do return(get_replace_where_sql(arg_dict)) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_replace_where_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.557461, "supported_languages": null}, "macro.dbt_databricks.get_incremental_replace_where_sql": {"name": "get_incremental_replace_where_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_incremental_replace_where_sql", "macro_sql": "{% macro get_incremental_replace_where_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_replace_where_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_replace_where_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.557615, "supported_languages": null}, "macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql": {"name": "databricks__get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql", "macro_sql": "{% macro databricks__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) %}\n {{ return(get_insert_overwrite_sql(source, target)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_insert_overwrite_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5577788, "supported_languages": null}, "macro.dbt_databricks.get_insert_overwrite_sql": {"name": "get_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_insert_overwrite_sql", "macro_sql": "{% macro get_insert_overwrite_sql(source_relation, target_relation) %}\n\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n insert overwrite table {{ target_relation }}\n {{ partition_cols(label=\"partition\") }}\n select {{dest_cols_csv}} from {{ source_relation }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.partition_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.558079, "supported_languages": null}, "macro.dbt_databricks.get_replace_where_sql": {"name": "get_replace_where_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_replace_where_sql", "macro_sql": "{% macro get_replace_where_sql(args_dict) -%}\n {%- set predicates = args_dict['incremental_predicates'] -%}\n {%- set target_relation = args_dict['target_relation'] -%}\n {%- set temp_relation = args_dict['temp_relation'] -%}\n\n insert into {{ target_relation }}\n {% if predicates %}\n {% if predicates is sequence and predicates is not string %}\n replace where {{ predicates | join(' and ') }}\n {% else %}\n replace where {{ predicates }}\n {% endif %}\n {% endif %}\n table {{ temp_relation }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5584981, "supported_languages": null}, "macro.dbt_databricks.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(source_relation, target_relation) %}\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) | map(attribute=\"quoted\") | list -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) | map(attribute=\"quoted\") | list -%}\n {{ insert_into_sql_impl(target_relation, dest_columns, source_relation, source_columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.insert_into_sql_impl"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.558819, "supported_languages": null}, "macro.dbt_databricks.insert_into_sql_impl": {"name": "insert_into_sql_impl", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.insert_into_sql_impl", "macro_sql": "{% macro insert_into_sql_impl(target_relation, dest_columns, source_relation, source_columns) %}\n {%- set common_columns = [] -%}\n {%- for dest_col in dest_columns -%}\n {%- if dest_col in source_columns -%}\n {%- do common_columns.append(dest_col) -%}\n {%- else -%}\n {%- do common_columns.append('DEFAULT') -%}\n {%- endif -%}\n {%- endfor -%}\n {%- set dest_cols_csv = dest_columns | join(', ') -%}\n {%- set source_cols_csv = common_columns | join(', ') -%}\ninsert into table {{ target_relation }} ({{ dest_cols_csv }})\nselect {{source_cols_csv}} from {{ source_relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5592902, "supported_languages": null}, "macro.dbt_databricks.databricks__get_merge_sql": {"name": "databricks__get_merge_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_merge_sql", "macro_sql": "{% macro databricks__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %}\n {# need dest_columns for merge_exclude_columns, default to use \"*\" #}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target) -%}\n {%- set source_columns = (adapter.get_columns_in_relation(source) | map(attribute='quoted') | list)-%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} <=> DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} <=> DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n when matched then update set {{ get_merge_update_set(update_columns, on_schema_change, source_columns) }}\n when not matched then insert {{ get_merge_insert(on_schema_change, source_columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.incremental_validate_on_schema_change", "macro.dbt.get_merge_update_columns", "macro.dbt_databricks.get_merge_update_set", "macro.dbt_databricks.get_merge_insert"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5605829, "supported_languages": null}, "macro.dbt_databricks.get_merge_update_set": {"name": "get_merge_update_set", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_merge_update_set", "macro_sql": "{% macro get_merge_update_set(update_columns, on_schema_change, source_columns) %}\n {%- if update_columns -%}\n {%- for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}{%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n {%- elif on_schema_change == 'ignore' -%}\n *\n {%- else -%}\n {%- for column in source_columns -%}\n {{ column }} = DBT_INTERNAL_SOURCE.{{ column }}{%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.560978, "supported_languages": null}, "macro.dbt_databricks.get_merge_insert": {"name": "get_merge_insert", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_merge_insert", "macro_sql": "{% macro get_merge_insert(on_schema_change, source_columns) %}\n {%- if on_schema_change == 'ignore' -%}\n *\n {%- else -%}\n ({{ source_columns | join(\", \") }}) VALUES (\n {%- for column in source_columns -%}\n DBT_INTERNAL_SOURCE.{{ column }}{%- if not loop.last %}, {% endif -%}\n {%- endfor %})\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.561265, "supported_languages": null}, "macro.dbt_databricks.materialization_incremental_databricks": {"name": "materialization_incremental_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/incremental.sql", "original_file_path": "macros/materializations/incremental/incremental.sql", "unique_id": "macro.dbt_databricks.materialization_incremental_databricks", "macro_sql": "{% materialization incremental, adapter='databricks', supported_languages=['sql', 'python'] -%}\n {#-- Validate early so we don't run SQL if the file_format + strategy combo is invalid --#}\n {%- set raw_file_format = config.get('file_format', default='delta') -%}\n {%- set raw_strategy = config.get('incremental_strategy') or 'merge' -%}\n {%- set grant_config = config.get('grants') -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- set tags = config.get('databricks_tags') -%}\n {%- set unique_tmp_table_suffix = config.get('unique_tmp_table_suffix', False) | as_bool -%}\n\n {%- set file_format = dbt_databricks_validate_get_file_format(raw_file_format) -%}\n {%- set incremental_strategy = dbt_databricks_validate_get_incremental_strategy(raw_strategy, file_format) -%}\n\n {#-- Set vars --#}\n\n {%- set incremental_predicates = config.get('predicates', default=none) or config.get('incremental_predicates', default=none) -%}\n {%- set unique_key = config.get('unique_key', none) -%}\n {%- set partition_by = config.get('partition_by', none) -%}\n {%- set language = model['language'] -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set existing_relation = adapter.get_relation(database=this.database, schema=this.schema, identifier=this.identifier, needs_information=True) -%}\n {%- if unique_tmp_table_suffix == True and raw_strategy == 'replace_where' and raw_file_format == 'delta' -%}\n {%- set temp_relation_suffix = adapter.generate_unique_temporary_table_suffix() -%}\n {%- else -%}\n {%- set temp_relation_suffix = '__dbt_tmp' -%}\n {%- endif -%}\n\n {#-- Set Overwrite Mode to STATIC for initial replace --#}\n {%- if incremental_strategy == 'insert_overwrite' and should_full_refresh() -%}\n {%- call statement() -%}\n set spark.sql.sources.partitionOverwriteMode = STATIC\n {%- endcall -%}\n {%- endif -%}\n\n {#-- Run pre-hooks --#}\n {{ run_hooks(pre_hooks) }}\n\n {#-- Incremental run logic --#}\n {%- if existing_relation is none -%}\n {#-- Relation must be created --#}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {% do apply_tags(target_relation, tags) %}\n {%- if language == 'python' -%}\n {%- do apply_tblproperties(target_relation, tblproperties) %}\n {%- endif -%}\n\n {% do persist_docs(target_relation, model, for_relation=language=='python') %}\n {%- elif existing_relation.is_view or existing_relation.is_materialized_view or existing_relation.is_streaming_table or should_full_refresh() -%}\n {#-- Relation must be dropped & recreated --#}\n {% set is_delta = (file_format == 'delta' and existing_relation.is_delta) %}\n {% if not is_delta %} {#-- If Delta, we will `create or replace` below, so no need to drop --#}\n {% do adapter.drop_relation(existing_relation) %}\n {% endif %}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% if not existing_relation.is_view %}\n {% do persist_constraints(target_relation, model) %}\n {% endif %}\n {% do apply_tags(target_relation, tags) %}\n {% do persist_docs(target_relation, model, for_relation=language=='python') %}\n {%- else -%}\n {#-- Set Overwrite Mode to DYNAMIC for subsequent incremental operations --#}\n {%- if incremental_strategy == 'insert_overwrite' and partition_by -%}\n {%- call statement() -%}\n set spark.sql.sources.partitionOverwriteMode = DYNAMIC\n {%- endcall -%}\n {%- endif -%}\n {#-- Relation must be merged --#}\n {%- set _existing_config = adapter.get_relation_config(existing_relation) -%}\n {%- set model_config = adapter.get_config_from_model(config.model) -%}\n {%- set _configuration_changes = model_config.get_changeset(_existing_config) -%}\n {%- set temp_relation = databricks__make_temp_relation(target_relation, suffix=temp_relation_suffix, as_table=language != 'sql') -%}\n {%- call statement('create_temp_relation', language=language) -%}\n {{ create_table_as(True, temp_relation, compiled_code, language) }}\n {%- endcall -%}\n {%- do process_schema_changes(on_schema_change, temp_relation, existing_relation) -%}\n {%- set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) -%}\n {%- set strategy_arg_dict = ({\n 'target_relation': target_relation,\n 'temp_relation': temp_relation,\n 'unique_key': unique_key,\n 'dest_columns': none,\n 'incremental_predicates': incremental_predicates}) -%}\n {%- set build_sql = strategy_sql_macro_func(strategy_arg_dict) -%}\n {%- if language == 'sql' -%}\n {%- call statement('main') -%}\n {{ build_sql }}\n {%- endcall -%}\n {%- elif language == 'python' -%}\n {%- call statement_with_staging_table('main', temp_relation) -%}\n {{ build_sql }}\n {%- endcall -%}\n {#--\n This is yucky.\n See note in dbt-spark/dbt/include/spark/macros/adapters.sql\n re: python models and temporary views.\n\n Also, why does not either drop_relation or adapter.drop_relation work here?!\n --#}\n {%- endif -%}\n {% do apply_liquid_clustered_cols(target_relation) %}\n {% if _configuration_changes is not none %}\n {% set tags = _configuration_changes.changes.get(\"tags\", None) %}\n {% set tblproperties = _configuration_changes.changes.get(\"tblproperties\", None) %}\n {% if tags is not none %}\n {% do apply_tags(target_relation, tags.set_tags, tags.unset_tags) %}\n {%- endif -%}\n {% if tblproperties is not none %}\n {% do apply_tblproperties(target_relation, tblproperties.tblproperties) %}\n {%- endif -%}\n {%- endif -%}\n {% do persist_docs(target_relation, model, for_relation=True) %}\n {%- endif -%}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n {% do optimize(target_relation) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt_databricks.dbt_databricks_validate_get_file_format", "macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.run_hooks", "macro.dbt.create_table_as", "macro.dbt_databricks.persist_constraints", "macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_tblproperties", "macro.dbt.persist_docs", "macro.dbt_databricks.databricks__make_temp_relation", "macro.dbt.process_schema_changes", "macro.dbt_databricks.statement_with_staging_table", "macro.dbt_databricks.apply_liquid_clustered_cols", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.optimize"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.566802, "supported_languages": ["sql", "python"]}, "macro.dbt_databricks.databricks__get_binding_char": {"name": "databricks__get_binding_char", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__get_binding_char", "macro_sql": "{% macro databricks__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5697222, "supported_languages": null}, "macro.dbt_databricks.databricks__load_csv_rows": {"name": "databricks__load_csv_rows", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__load_csv_rows", "macro_sql": "{% macro databricks__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n {% set column_override = model['config'].get('column_types', {}) %}\n {% set must_cast = model['config'].get(\"file_format\", \"delta\") == \"parquet\" %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert {% if loop.index0 == 0 -%} overwrite {% else -%} into {% endif -%} {{ this.render() }} values\n {% for row in chunk -%}\n ({%- for col_name in agate_table.column_names -%}\n {%- if must_cast -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n cast({{ get_binding_char() }} as {{type}})\n {%- else -%}\n {{ get_binding_char() }}\n {%- endif -%}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True, close_cursor=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.571152, "supported_languages": null}, "macro.dbt_databricks.databricks__reset_csv_table": {"name": "databricks__reset_csv_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__reset_csv_table", "macro_sql": "{% macro databricks__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% if old_relation %}\n {% if old_relation.is_delta and config.get('file_format', default='delta') == 'delta' %}\n {% set sql = create_or_replace_csv_table(model, agate_table, True) %}\n {% else %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% endif %}\n {% else %}\n {% set sql = create_csv_table(model, agate_table) %}\n {% endif %}\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.create_or_replace_csv_table", "macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.57173, "supported_languages": null}, "macro.dbt_databricks.create_or_replace_csv_table": {"name": "create_or_replace_csv_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.create_or_replace_csv_table", "macro_sql": "{% macro create_or_replace_csv_table(model, agate_table, replace=False) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {%- set column_comment = config.persist_column_docs() and model.columns %}\n {%- set identifier = model['alias'] -%}\n {%- set relation = api.Relation.create(database=database, schema=schema, identifier=identifier, type='table') -%}\n {%- set replace_clause = \"\" -%}\n {%- if replace -%}\n {%- set replace_clause = \"or replace\" -%}\n {%- endif -%}\n\n {% set sql %}\n create {{replace_clause}} table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {%- set column_comment_clause = \"\" -%}\n {%- if column_comment and col_name in model.columns.keys() -%} \n {%- set comment = model.columns[col_name]['description'] | replace(\"'\", \"\\\\'\") -%}\n {%- if comment and comment != \"\" -%}\n {%- set column_comment_clause = \"comment '\" ~ comment ~ \"'\" -%}\n {%- endif -%}\n {%- endif -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {{ column_comment_clause }}{%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {{ file_format_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause(relation) }}\n {{ comment_clause() }}\n {{ tblproperties_clause() }}\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.file_format_clause", "macro.dbt_spark.partition_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_databricks.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt_databricks.tblproperties_clause", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.57331, "supported_languages": null}, "macro.dbt_databricks.databricks__create_csv_table": {"name": "databricks__create_csv_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__create_csv_table", "macro_sql": "{% macro databricks__create_csv_table(model, agate_table) %}\n {{ return(create_or_replace_csv_table(model, agate_table)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.create_or_replace_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.573471, "supported_languages": null}, "macro.dbt_databricks.materialization_seed_databricks": {"name": "materialization_seed_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/seeds.sql", "original_file_path": "macros/materializations/seeds/seeds.sql", "unique_id": "macro.dbt_databricks.materialization_seed_databricks", "macro_sql": "{% materialization seed, adapter='databricks' %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier, needs_information=True) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and (old_relation.is_view or old_relation.is_materialized_view)) -%}\n {%- set exists_as_streaming_table = (old_relation is not none and old_relation.is_streaming_table) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparision later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view or a materialized view\".format(old_relation)) }}\n {% elif exists_as_streaming_table %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a streaming table\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n -- No need to persist docs, already handled in seed create\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.get_csv_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.576501, "supported_languages": ["sql"]}, "macro.dbt_databricks.databricks__generate_database_name": {"name": "databricks__generate_database_name", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt_databricks.databricks__generate_database_name", "macro_sql": "{% macro databricks__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n {{ return(default_database) }}\n {%- else -%}\n {{ return(custom_database_name) }}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.576849, "supported_languages": null}, "macro.dbt_databricks.databricks__get_drop_sql": {"name": "databricks__get_drop_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt_databricks.databricks__get_drop_sql", "macro_sql": "{% macro databricks__get_drop_sql(relation) -%}\n {%- if relation.is_materialized_view -%}\n {{ drop_materialized_view(relation) }}\n {%- elif relation.is_streaming_table-%}\n {{ drop_streaming_table(relation) }}\n {%- elif relation.is_view -%}\n {{ drop_view(relation) }}\n {%- else -%}\n {{ drop_table(relation) }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.drop_materialized_view", "macro.dbt_databricks.drop_streaming_table", "macro.dbt.drop_view", "macro.dbt.drop_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.577309, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_relation": {"name": "databricks__drop_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_relation", "macro_sql": "{% macro databricks__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n {{ get_drop_sql(relation) }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.577478, "supported_languages": null}, "macro.dbt_databricks.get_replace_sql": {"name": "get_replace_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt_databricks.get_replace_sql", "macro_sql": "{% macro get_replace_sql(existing_relation, target_relation, sql) %}\n {{- log('Applying REPLACE to: ' ~ existing_relation) -}}\n {% do return(adapter.dispatch('get_replace_sql', 'dbt')(existing_relation, target_relation, sql)) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_replace_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.57809, "supported_languages": null}, "macro.dbt_databricks.databricks__get_replace_sql": {"name": "databricks__get_replace_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt_databricks.databricks__get_replace_sql", "macro_sql": "{% macro databricks__get_replace_sql(existing_relation, target_relation, sql) %}\n {# /* use a create or replace statement if possible */ #}\n\n {% set is_replaceable = existing_relation.type == target_relation_type and existing_relation.can_be_replaced %}\n\n {% if is_replaceable and existing_relation.is_view %}\n {{ get_replace_view_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_table %}\n {{ get_replace_table_sql(target_relation, sql) }}\n\n {# /* a create or replace statement is not possible, so try to stage and/or backup to be safe */ #}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #}\n {%- elif target_relation.can_be_renamed and existing_relation.can_be_renamed -%}\n {{ return([\n get_create_intermediate_sql(target_relation, sql),\n get_create_backup_sql(existing_relation),\n get_rename_intermediate_sql(target_relation),\n get_drop_backup_sql(existing_relation)\n ]) }}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one without using a backup */ #}\n {%- elif target_relation.can_be_renamed -%}\n {{ return([\n get_create_intermediate_sql(target_relation, sql),\n get_drop_sql(existing_relation),\n get_rename_intermediate_sql(target_relation),\n ]) }}\n\n {# /* create target_relation in place by first backing up the existing relation */ #}\n {%- elif existing_relation.can_be_renamed -%}\n {{ return([\n get_create_backup_sql(existing_relation),\n get_create_sql(target_relation, sql),\n get_drop_backup_sql(existing_relation)\n ]) }}\n\n {# /* no renaming is allowed, so just drop and create */ #}\n {%- else -%}\n {{ return([\n get_drop_sql(existing_relation),\n get_create_sql(target_relation, sql)\n ]) }}\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_replace_view_sql", "macro.dbt.get_replace_table_sql", "macro.dbt.get_create_intermediate_sql", "macro.dbt.get_create_backup_sql", "macro.dbt.get_rename_intermediate_sql", "macro.dbt.get_drop_backup_sql", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.579058, "supported_languages": null}, "macro.dbt_databricks.fetch_tags": {"name": "fetch_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.fetch_tags", "macro_sql": "{% macro fetch_tags(relation) -%}\n {% if relation.is_hive_metastore() %}\n {{ exceptions.raise_compiler_error(\"Tags are only supported for Unity Catalog\") }}\n {%- endif %}\n {% call statement('list_tags', fetch_result=True) -%}\n {{ fetch_tags_sql(relation) }}\n {% endcall %}\n {% do return(load_result('list_tags').table) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.fetch_tags_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5802681, "supported_languages": null}, "macro.dbt_databricks.fetch_tags_sql": {"name": "fetch_tags_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.fetch_tags_sql", "macro_sql": "{% macro fetch_tags_sql(relation) -%}\n SELECT tag_name, tag_value\n FROM `system`.`information_schema`.`table_tags`\n WHERE catalog_name = '{{ relation.database|lower }}' \n AND schema_name = '{{ relation.schema|lower }}'\n AND table_name = '{{ relation.identifier|lower }}'\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5804422, "supported_languages": null}, "macro.dbt_databricks.apply_tags": {"name": "apply_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.apply_tags", "macro_sql": "{% macro apply_tags(relation, set_tags, unset_tags=[]) -%}\n {%- if (set_tags or unset_tags) and relation.is_hive_metastore() -%}\n {{ exceptions.raise_compiler_error(\"Tags are only supported for Unity Catalog\") }}\n {%- endif -%}\n {%- if set_tags %}\n {%- call statement('set_tags') -%}\n {{ alter_set_tags(relation, set_tags) }}\n {%- endcall -%}\n {%- endif %}\n {%- if unset_tags %}\n {%- call statement('unset_tags') -%}\n {{ alter_unset_tags(relation, unset_tags) }}\n {%- endcall -%}\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.alter_set_tags", "macro.dbt_databricks.alter_unset_tags"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.580887, "supported_languages": null}, "macro.dbt_databricks.alter_set_tags": {"name": "alter_set_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.alter_set_tags", "macro_sql": "{% macro alter_set_tags(relation, tags) -%}\n ALTER {{ relation.type }} {{ relation }} SET TAGS (\n {% for tag in tags -%}\n '{{ tag }}' = '{{ tags[tag] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n )\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.581147, "supported_languages": null}, "macro.dbt_databricks.alter_unset_tags": {"name": "alter_unset_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.alter_unset_tags", "macro_sql": "{% macro alter_unset_tags(relation, tags) -%}\n ALTER {{ relation.type }} {{ relation }} UNSET TAGS (\n {% for tag in tags -%}\n '{{ tag }}' {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n )\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.58137, "supported_languages": null}, "macro.dbt_databricks.location_clause": {"name": "location_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/location.sql", "original_file_path": "macros/relations/location.sql", "unique_id": "macro.dbt_databricks.location_clause", "macro_sql": "{% macro location_clause(relation) %}\n {%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%}\n {%- set file_format = config.get('file_format', default='delta') -%}\n {%- set identifier = model['alias'] -%}\n {%- if location_root is not none %}\n location '{{ location_root }}/{{ identifier }}'\n {%- elif (not relation.is_hive_metastore()) and file_format != 'delta' -%}\n {{ exceptions.raise_compiler_error(\n 'Incompatible configuration: `location_root` must be set when using a non-delta file format with Unity Catalog'\n ) }}\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5819929, "supported_languages": null}, "macro.dbt_databricks.optimize": {"name": "optimize", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/optimize.sql", "original_file_path": "macros/relations/optimize.sql", "unique_id": "macro.dbt_databricks.optimize", "macro_sql": "{% macro optimize(relation) %}\n {{ return(adapter.dispatch('optimize', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__optimize"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.582578, "supported_languages": null}, "macro.dbt_databricks.databricks__optimize": {"name": "databricks__optimize", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/optimize.sql", "original_file_path": "macros/relations/optimize.sql", "unique_id": "macro.dbt_databricks.databricks__optimize", "macro_sql": "\n\n{%- macro databricks__optimize(relation) -%}\n {%- if var('DATABRICKS_SKIP_OPTIMIZE', 'false')|lower != 'true' and\n var('databricks_skip_optimize', 'false')|lower != 'true' and\n config.get('file_format', 'delta') == 'delta' -%}\n {%- if (config.get('zorder', False) or config.get('liquid_clustered_by', False)) -%}\n {%- call statement('run_optimize_stmt') -%}\n {{ get_optimize_sql(relation) }}\n {%- endcall -%}\n {%- endif -%}\n {%- endif -%}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.get_optimize_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.583037, "supported_languages": null}, "macro.dbt_databricks.get_optimize_sql": {"name": "get_optimize_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/optimize.sql", "original_file_path": "macros/relations/optimize.sql", "unique_id": "macro.dbt_databricks.get_optimize_sql", "macro_sql": "{%- macro get_optimize_sql(relation) %}\n optimize {{ relation }}\n {%- if config.get('zorder', False) and config.get('file_format', 'delta') == 'delta' %}\n {%- if config.get('liquid_clustered_by', False) %}\n {{ exceptions.warn(\"Both zorder and liquid_clustered_by are set but they are incompatible. zorder will be ignored.\") }}\n {%- else %}\n {%- set zorder = config.get('zorder', none) %}\n {# TODO: predicates here? WHERE ... #}\n {%- if zorder is sequence and zorder is not string %}\n zorder by (\n {%- for col in zorder %}\n {{ col }}{% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- else %}\n zorder by ({{zorder}})\n {%- endif %}\n {%- endif %}\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.583699, "supported_languages": null}, "macro.dbt_databricks.file_format_clause": {"name": "file_format_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/file_format.sql", "original_file_path": "macros/relations/file_format.sql", "unique_id": "macro.dbt_databricks.file_format_clause", "macro_sql": "{% macro file_format_clause() %}\n {%- set file_format = config.get('file_format', default='delta') -%}\n using {{ file_format }}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.583901, "supported_languages": null}, "macro.dbt_databricks.liquid_clustered_cols": {"name": "liquid_clustered_cols", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/liquid_clustering.sql", "original_file_path": "macros/relations/liquid_clustering.sql", "unique_id": "macro.dbt_databricks.liquid_clustered_cols", "macro_sql": "{% macro liquid_clustered_cols() -%}\n {%- set cols = config.get('liquid_clustered_by', validator=validation.any[list, basestring]) -%}\n {%- if cols is not none %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n CLUSTER BY ({{ cols | join(', ') }})\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5845, "supported_languages": null}, "macro.dbt_databricks.apply_liquid_clustered_cols": {"name": "apply_liquid_clustered_cols", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/liquid_clustering.sql", "original_file_path": "macros/relations/liquid_clustering.sql", "unique_id": "macro.dbt_databricks.apply_liquid_clustered_cols", "macro_sql": "{% macro apply_liquid_clustered_cols(target_relation) -%}\n {%- set cols = config.get('liquid_clustered_by', validator=validation.any[list, basestring]) -%}\n {%- if cols is not none %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n {%- call statement('set_cluster_by_columns') -%}\n ALTER {{ target_relation.type }} {{ target_relation }} CLUSTER BY ({{ cols | join(', ') }})\n {%- endcall -%}\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.584919, "supported_languages": null}, "macro.dbt_databricks.tblproperties_clause": {"name": "tblproperties_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tblproperties.sql", "original_file_path": "macros/relations/tblproperties.sql", "unique_id": "macro.dbt_databricks.tblproperties_clause", "macro_sql": "{% macro tblproperties_clause() -%}\n {{ return(adapter.dispatch('tblproperties_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.58541, "supported_languages": null}, "macro.dbt_databricks.databricks__tblproperties_clause": {"name": "databricks__tblproperties_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tblproperties.sql", "original_file_path": "macros/relations/tblproperties.sql", "unique_id": "macro.dbt_databricks.databricks__tblproperties_clause", "macro_sql": "{% macro databricks__tblproperties_clause() -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- if tblproperties is not none %}\n tblproperties (\n {%- for prop in tblproperties -%}\n '{{ prop }}' = '{{ tblproperties[prop] }}' {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5857081, "supported_languages": null}, "macro.dbt_databricks.apply_tblproperties": {"name": "apply_tblproperties", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tblproperties.sql", "original_file_path": "macros/relations/tblproperties.sql", "unique_id": "macro.dbt_databricks.apply_tblproperties", "macro_sql": "{% macro apply_tblproperties(relation, tblproperties) -%}\n {% if tblproperties %}\n {%- call statement('apply_tblproperties') -%}\n ALTER {{ relation.type }} {{ relation }} SET TBLPROPERTIES (\n {% for tblproperty in tblproperties -%}\n '{{ tblproperty }}' = '{{ tblproperties[tblproperty] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n )\n {%- endcall -%}\n {% endif %}\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.58604, "supported_languages": null}, "macro.dbt_databricks.persist_constraints": {"name": "persist_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.persist_constraints", "macro_sql": "{% macro persist_constraints(relation, model) %}\n {{ return(adapter.dispatch('persist_constraints', 'dbt')(relation, model)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.599794, "supported_languages": null}, "macro.dbt_databricks.databricks__persist_constraints": {"name": "databricks__persist_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks__persist_constraints", "macro_sql": "{% macro databricks__persist_constraints(relation, model) %}\n {%- set contract_config = config.get('contract') -%}\n {% set has_model_contract = contract_config and contract_config.enforced %}\n {% set has_databricks_constraints = config.get('persist_constraints', False) %}\n\n {% if (has_model_contract or has_databricks_constraints) %}\n {% if config.get('file_format', 'delta') != 'delta' %}\n {# Constraints are only supported for delta tables #}\n {{ exceptions.warn(\"Constraints not supported for file format: \" ~ config.get('file_format')) }}\n {% elif relation.is_view %}\n {# Constraints are not supported for views. This point in the code should not have been reached. #}\n {{ exceptions.raise_compiler_error(\"Constraints not supported for views.\") }}\n {% elif is_incremental() %}\n {# Constraints are not applied for incremental updates. This point in the code should not have been reached #}\n {{ exceptions.raise_compiler_error(\"Constraints are not applied for incremental updates. Full refresh is required to update constraints.\") }}\n {% else %}\n {% do alter_column_set_constraints(relation, model) %}\n {% do alter_table_add_constraints(relation, model) %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.is_incremental", "macro.dbt_databricks.alter_column_set_constraints", "macro.dbt_databricks.alter_table_add_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.600513, "supported_languages": null}, "macro.dbt_databricks.alter_table_add_constraints": {"name": "alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.alter_table_add_constraints", "macro_sql": "{% macro alter_table_add_constraints(relation, constraints) %}\n {{ return(adapter.dispatch('alter_table_add_constraints', 'dbt')(relation, constraints)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_table_add_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.600693, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_table_add_constraints": {"name": "databricks__alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks__alter_table_add_constraints", "macro_sql": "{% macro databricks__alter_table_add_constraints(relation, model) %}\n {% set constraints = get_model_constraints(model) %}\n {% set statements = get_constraints_sql(relation, constraints, model) %}\n {% for stmt in statements %}\n {% call statement() %}\n {{ stmt }}\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_model_constraints", "macro.dbt_databricks.get_constraints_sql", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6010072, "supported_languages": null}, "macro.dbt_databricks.get_model_constraints": {"name": "get_model_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_model_constraints", "macro_sql": "{% macro get_model_constraints(model) %}\n {% set constraints = model.get('constraints', []) %}\n {% if config.get('persist_constraints', False) and model.get('meta', {}).get('constraints') is sequence %}\n {# Databricks constraints implementation. Constraints are in the meta property. #}\n {% set db_constraints = model.get('meta', {}).get('constraints', []) %}\n {% set constraints = databricks_constraints_to_dbt(db_constraints) %}\n {% endif %}\n {{ return(constraints) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks_constraints_to_dbt"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.601491, "supported_languages": null}, "macro.dbt_databricks.get_column_constraints": {"name": "get_column_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_column_constraints", "macro_sql": "{% macro get_column_constraints(column) %}\n {% set constraints = column.get('constraints', []) %}\n {% if config.get('persist_constraints', False) and column.get('meta', {}).get('constraint') %}\n {# Databricks constraints implementation. Constraint is in the meta property. #}\n {% set db_constraints = [column.get('meta', {}).get('constraint')] %}\n {% set constraints = databricks_constraints_to_dbt(db_constraints, column) %}\n {% endif %}\n {{ return(constraints) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks_constraints_to_dbt"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6019762, "supported_languages": null}, "macro.dbt_databricks.alter_column_set_constraints": {"name": "alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.alter_column_set_constraints", "macro_sql": "{% macro alter_column_set_constraints(relation, column_dict) %}\n {{ return(adapter.dispatch('alter_column_set_constraints', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_column_set_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.602155, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_column_set_constraints": {"name": "databricks__alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks__alter_column_set_constraints", "macro_sql": "{% macro databricks__alter_column_set_constraints(relation, model) %}\n {% set column_dict = model.columns %}\n {% for column_name in column_dict %}\n {% set column = column_dict[column_name] %}\n {% set constraints = get_column_constraints(column) %}\n {% set statements = get_constraints_sql(relation, constraints, model, column) %}\n {% for stmt in statements %}\n {% call statement() %}\n {{ stmt }}\n {% endcall %}\n {% endfor %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_column_constraints", "macro.dbt_databricks.get_constraints_sql", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.602596, "supported_languages": null}, "macro.dbt_databricks.get_constraints_sql": {"name": "get_constraints_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_constraints_sql", "macro_sql": "{% macro get_constraints_sql(relation, constraints, model, column={}) %}\n {% set statements = [] %}\n -- Hack so that not null constraints will be applied before primary key constraints\n {% for constraint in constraints|sort(attribute='type') %}\n {% if constraint %}\n {% set constraint_statements = get_constraint_sql(relation, constraint, model, column) %}\n {% for statement in constraint_statements %}\n {% if statement %}\n {% do statements.append(statement) %}\n {% endif %}\n {% endfor %}\n {% endif %}\n {% endfor %}\n\n {{ return(statements) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_constraint_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.603168, "supported_languages": null}, "macro.dbt_databricks.get_constraint_sql": {"name": "get_constraint_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_constraint_sql", "macro_sql": "{% macro get_constraint_sql(relation, constraint, model, column={}) %}\n {% set statements = [] %}\n {% set type = constraint.get(\"type\", \"\") %}\n\n {% if type == 'check' %}\n {% set expression = constraint.get(\"expression\", \"\") %}\n {% if not expression %}\n {{ exceptions.raise_compiler_error('Invalid check constraint expression') }}\n {% endif %}\n\n {% set name = constraint.get(\"name\") %}\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5 (relation.identifier ~ \";\" ~ column.get(\"name\", \"\") ~ \";\" ~ expression ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %}\n {% endif %}\n {% set stmt = \"alter table \" ~ relation ~ \" add constraint \" ~ name ~ \" check (\" ~ expression ~ \");\" %}\n {% do statements.append(stmt) %}\n {% elif type == 'not_null' %}\n {% set column_names = constraint.get(\"columns\", []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if column %}\n {% set quoted_name = adapter.quote(column['name']) if column['quote'] else column['name'] %}\n {% set stmt = \"alter table \" ~ relation ~ \" change column \" ~ quoted_name ~ \" set not null \" ~ (constraint.expression or \"\") ~ \";\" %}\n {% do statements.append(stmt) %}\n {% else %}\n {{ exceptions.warn('not_null constraint on invalid column: ' ~ column_name) }}\n {% endif %}\n {% endfor %}\n {% elif type == 'primary_key' %}\n {% if constraint.get('warn_unenforced') %}\n {{ exceptions.warn(\"unenforced constraint type: \" ~ type)}}\n {% endif %}\n {% set column_names = constraint.get(\"columns\", []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% set quoted_names = [] %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if not column %}\n {{ exceptions.warn('Invalid primary key column: ' ~ column_name) }}\n {% else %}\n {% set quoted_name = adapter.quote(column['name']) if column['quote'] else column['name'] %}\n {% do quoted_names.append(quoted_name) %}\n {% endif %}\n {% endfor %}\n\n {% set joined_names = quoted_names|join(\", \") %}\n\n {% set name = constraint.get(\"name\") %}\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5(\"primary_key;\" ~ relation.identifier ~ \";\" ~ column_names ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %}\n {% endif %}\n {% set stmt = \"alter table \" ~ relation ~ \" add constraint \" ~ name ~ \" primary key(\" ~ joined_names ~ \");\" %}\n {% do statements.append(stmt) %}\n {% elif type == 'foreign_key' %}\n\n {% if constraint.get('warn_unenforced') %}\n {{ exceptions.warn(\"unenforced constraint type: \" ~ constraint.type)}}\n {% endif %}\n\n {% set name = constraint.get(\"name\") %}\n \n {% if constraint.get('expression') %}\n\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5(\"foreign_key;\" ~ relation.identifier ~ \";\" ~ constraint.get('expression') ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %} \n {% endif %}\n\n {% set stmt = \"alter table \" ~ relation ~ \" add constraint \" ~ name ~ \" foreign key\" ~ constraint.get('expression') %}\n {% else %}\n {% set column_names = constraint.get(\"columns\", []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% set quoted_names = [] %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if not column %}\n {{ exceptions.warn('Invalid foreign key column: ' ~ column_name) }}\n {% else %}\n {% set quoted_name = adapter.quote(column['name']) if column['quote'] else column['name'] %}\n {% do quoted_names.append(quoted_name) %}\n {% endif %}\n {% endfor %}\n\n {% set joined_names = quoted_names|join(\", \") %}\n\n {% set parent = constraint.get(\"to\") %}\n {% if not parent %}\n {{ exceptions.raise_compiler_error('No parent table defined for foreign key: ' ~ expression) }}\n {% endif %}\n {% if not \".\" in parent %}\n {% set parent = relation.schema ~ \".\" ~ parent%}\n {% endif %}\n\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5(\"foreign_key;\" ~ relation.identifier ~ \";\" ~ column_names ~ \";\" ~ parent ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %} \n {% endif %}\n\n {% set stmt = \"alter table \" ~ relation ~ \" add constraint \" ~ name ~ \" foreign key(\" ~ joined_names ~ \") references \" ~ parent %}\n {% set parent_columns = constraint.get(\"to_columns\") %}\n {% if parent_columns %}\n {% set stmt = stmt ~ \"(\" ~ parent_columns|join(\", \") ~ \")\"%}\n {% endif %}\n {% endif %}\n {% set stmt = stmt ~ \";\" %}\n {% do statements.append(stmt) %}\n {% elif constraint.get('warn_unsupported') %}\n {{ exceptions.warn(\"unsupported constraint type: \" ~ constraint.type)}}\n {% endif %}\n\n {{ return(statements) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6086242, "supported_languages": null}, "macro.dbt_databricks.databricks_constraints_to_dbt": {"name": "databricks_constraints_to_dbt", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks_constraints_to_dbt", "macro_sql": "{% macro databricks_constraints_to_dbt(constraints, column) %}\n {# convert constraints defined using the original databricks format #}\n {% set dbt_constraints = [] %}\n {% for constraint in constraints %}\n {% if constraint.get and constraint.get(\"type\") %}\n {# already in model contract format #}\n {% do dbt_constraints.append(constraint) %}\n {% else %}\n {% if column %}\n {% if constraint == \"not_null\" %}\n {% do dbt_constraints.append({\"type\": \"not_null\", \"columns\": [column.get(\"name\")]}) %}\n {% else %}\n {{ exceptions.raise_compiler_error('Invalid constraint for column ' ~ column.get(\"name\", \"\") ~ '. Only `not_null` is supported.') }}\n {% endif %}\n {% else %}\n {% set name = constraint['name'] %}\n {% if not name %}\n {{ exceptions.raise_compiler_error('Invalid check constraint name') }}\n {% endif %}\n {% set condition = constraint['condition'] %}\n {% if not condition %}\n {{ exceptions.raise_compiler_error('Invalid check constraint condition') }}\n {% endif %}\n {% do dbt_constraints.append({\"name\": name, \"type\": \"check\", \"expression\": condition}) %}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(dbt_constraints) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.609767, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_sql": {"name": "databricks__get_create_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_sql", "macro_sql": "{%- macro databricks__get_create_sql(relation, sql) -%}\n {%- if relation.is_view -%}\n {{ get_create_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_table -%}\n {{ get_create_table_as_sql(False, relation, sql) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_create_materialized_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_streaming_table -%}\n {{ get_create_streaming_table_as_sql(relation, sql) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_create_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.get_create_view_as_sql", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt_databricks.get_create_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.610386, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_materialized_view": {"name": "databricks__drop_materialized_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_materialized_view", "macro_sql": "{% macro databricks__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.610508, "supported_languages": null}, "macro.dbt_databricks.databricks__refresh_materialized_view": {"name": "databricks__refresh_materialized_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt_databricks.databricks__refresh_materialized_view", "macro_sql": "{% macro databricks__refresh_materialized_view(relation) -%}\n refresh materialized view {{ relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.61062, "supported_languages": null}, "macro.dbt_databricks.get_alter_materialized_view_as_sql": {"name": "get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_materialized_view_as_sql", "macro_sql": "{% macro get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {%- do return(adapter.dispatch('get_alter_materialized_view_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n )) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.611409, "supported_languages": null}, "macro.dbt_databricks.databricks__get_materialized_view_configuration_changes": {"name": "databricks__get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_materialized_view_configuration_changes", "macro_sql": "\n\n\n{%- macro databricks__get_materialized_view_configuration_changes(existing_relation, new_config) -%}\n {%- set _existing_materialized_view = adapter.get_relation_config(existing_relation) -%}\n {%- set materialized_view = adapter.get_config_from_model(config.model) -%}\n {%- set _configuration_changes = materialized_view.get_changeset(_existing_materialized_view) -%}\n {% do return(_configuration_changes) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6116831, "supported_languages": null}, "macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql": {"name": "databricks__get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql", "macro_sql": "{% macro databricks__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n -- apply a full refresh immediately if needed\n {% if configuration_changes.requires_full_refresh %}\n {% do return(get_replace_sql(existing_relation, relation, sql)) %}\n\n -- otherwise apply individual changes as needed\n {% else %}\n {% do return(get_alter_mv_internal(relation, configuration_changes)) %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.get_alter_mv_internal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.611993, "supported_languages": null}, "macro.dbt_databricks.get_alter_mv_internal": {"name": "get_alter_mv_internal", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_mv_internal", "macro_sql": "{% macro get_alter_mv_internal(relation, configuration_changes) %}\n {%- set refresh = configuration_changes.changes[\"refresh\"] -%}\n -- Currently only schedule can be altered\n ALTER MATERIALIZED VIEW {{ relation }}\n {{ get_alter_sql_refresh_schedule(refresh.cron, refresh.time_zone_value, refresh.is_altered) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_alter_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6122189, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_materialized_view_as_sql": {"name": "databricks__get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_materialized_view_as_sql", "macro_sql": "{% macro databricks__get_create_materialized_view_as_sql(relation, sql) -%}\n {%- set materialized_view = adapter.get_config_from_model(config.model) -%}\n {%- set partition_by = materialized_view.config[\"partition_by\"].partition_by -%}\n {%- set tblproperties = materialized_view.config[\"tblproperties\"].tblproperties -%}\n {%- set comment = materialized_view.config[\"comment\"].comment -%}\n {%- set refresh = materialized_view.config[\"refresh\"] -%}\n create materialized view {{ relation }}\n {{ get_create_sql_partition_by(partition_by) }}\n {{ get_create_sql_comment(comment) }}\n {{ get_create_sql_tblproperties(tblproperties) }}\n {{ get_create_sql_refresh_schedule(refresh.cron, refresh.time_zone_value) }}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_create_sql_partition_by", "macro.dbt_databricks.get_create_sql_comment", "macro.dbt_databricks.get_create_sql_tblproperties", "macro.dbt_databricks.get_create_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.612938, "supported_languages": null}, "macro.dbt_databricks.drop_streaming_table": {"name": "drop_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/drop.sql", "original_file_path": "macros/relations/streaming_table/drop.sql", "unique_id": "macro.dbt_databricks.drop_streaming_table", "macro_sql": "{% macro drop_streaming_table(relation) -%}\n {{ return(adapter.dispatch('drop_streaming_table', 'dbt')(relation)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.default__drop_streaming_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.613157, "supported_languages": null}, "macro.dbt_databricks.default__drop_streaming_table": {"name": "default__drop_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/drop.sql", "original_file_path": "macros/relations/streaming_table/drop.sql", "unique_id": "macro.dbt_databricks.default__drop_streaming_table", "macro_sql": "{% macro default__drop_streaming_table(relation) -%}\n drop table if exists {{ relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6132412, "supported_languages": null}, "macro.dbt_databricks.refresh_streaming_table": {"name": "refresh_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/refresh.sql", "original_file_path": "macros/relations/streaming_table/refresh.sql", "unique_id": "macro.dbt_databricks.refresh_streaming_table", "macro_sql": "{% macro refresh_streaming_table(relation, sql) -%}\n {{ adapter.dispatch('refresh_streaming_table', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__refresh_streaming_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.61346, "supported_languages": null}, "macro.dbt_databricks.databricks__refresh_streaming_table": {"name": "databricks__refresh_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/refresh.sql", "original_file_path": "macros/relations/streaming_table/refresh.sql", "unique_id": "macro.dbt_databricks.databricks__refresh_streaming_table", "macro_sql": "{% macro databricks__refresh_streaming_table(relation, sql) -%}\n create or refresh streaming table {{ relation }}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6135712, "supported_languages": null}, "macro.dbt_databricks.get_alter_streaming_table_as_sql": {"name": "get_alter_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_streaming_table_as_sql", "macro_sql": "{% macro get_alter_streaming_table_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {%- do return(adapter.dispatch('get_alter_streaming_table_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n )) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_alter_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.615469, "supported_languages": null}, "macro.dbt_databricks.get_streaming_table_configuration_changes": {"name": "get_streaming_table_configuration_changes", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_streaming_table_configuration_changes", "macro_sql": "{% macro get_streaming_table_configuration_changes(existing_relation, new_config) -%}\n {{- log('Determining configuration changes on: ' ~ existing_relation) -}}\n {%- do return(adapter.dispatch('get_streaming_table_configuration_changes', 'dbt')(existing_relation, new_config)) -%}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_streaming_table_configuration_changes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6158211, "supported_languages": null}, "macro.dbt_databricks.databricks__get_streaming_table_configuration_changes": {"name": "databricks__get_streaming_table_configuration_changes", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_streaming_table_configuration_changes", "macro_sql": "\n\n{%- macro databricks__get_streaming_table_configuration_changes(existing_relation, new_config) -%}\n {%- set _existing_streaming_table = adapter.get_relation_config(existing_relation) -%}\n {%- set streaming_table = adapter.get_config_from_model(config.model) -%}\n {%- set _configuration_changes = streaming_table.get_changeset(_existing_streaming_table) -%}\n {% do return(_configuration_changes) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6161752, "supported_languages": null}, "macro.dbt_databricks.databricks__get_alter_streaming_table_as_sql": {"name": "databricks__get_alter_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_alter_streaming_table_as_sql", "macro_sql": "{% macro databricks__get_alter_streaming_table_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n -- apply a full refresh immediately if needed\n {% if configuration_changes.requires_full_refresh %}\n {% do return(get_replace_sql(existing_relation, relation, sql)) %}\n\n -- otherwise apply individual changes as needed\n {% else %}\n {%- set alter_statement = get_alter_st_internal(relation, configuration_changes) -%}\n {%- set create_statement = get_create_st_internal(relation, configuration_changes, sql) -%}\n {%- set return_statements = [] -%}\n {%- if create_statement -%}\n {{ return_statements.append(create_statement) }}\n {%- endif -%}\n {%- if alter_statement -%}\n {{ return_statements.append(alter_statement) }}\n {%- endif -%}\n {% do return(return_statements) %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.get_alter_st_internal", "macro.dbt_databricks.get_create_st_internal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.616995, "supported_languages": null}, "macro.dbt_databricks.get_create_st_internal": {"name": "get_create_st_internal", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_create_st_internal", "macro_sql": "{% macro get_create_st_internal(relation, configuration_changes, sql) %}\n {%- set partition_by = configuration_changes.changes[\"partition_by\"].partition_by -%}\n {%- set tblproperties = configuration_changes.changes[\"tblproperties\"].tblproperties -%}\n {%- set comment = configuration_changes.changes[\"comment\"].comment -%}\n CREATE OR REFRESH STREAMING TABLE {{ relation }}\n {% if partition_by -%}\n {{ get_create_sql_partition_by(partition_by) }}\n {%- endif %}\n {% if comment -%}\n {{ get_create_sql_comment(comment) }}\n {%- endif %}\n {% if tblproperties -%}\n {{ get_create_sql_tblproperties(tblproperties) }}\n {%- endif %}\n AS {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_create_sql_partition_by", "macro.dbt_databricks.get_create_sql_comment", "macro.dbt_databricks.get_create_sql_tblproperties"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6175182, "supported_languages": null}, "macro.dbt_databricks.get_alter_st_internal": {"name": "get_alter_st_internal", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_st_internal", "macro_sql": "{% macro get_alter_st_internal(relation, configuration_changes) %}\n {%- set refresh = configuration_changes.changes[\"refresh\"] -%}\n {%- if refresh and refresh.cron -%}\n ALTER STREAMING TABLE {{ relation }}\n {{ get_alter_sql_refresh_schedule(refresh.cron, refresh.time_zone_value, False) -}}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_alter_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6177878, "supported_languages": null}, "macro.dbt_databricks.get_create_streaming_table_as_sql": {"name": "get_create_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/create.sql", "original_file_path": "macros/relations/streaming_table/create.sql", "unique_id": "macro.dbt_databricks.get_create_streaming_table_as_sql", "macro_sql": "{% macro get_create_streaming_table_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_streaming_table_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.618218, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_streaming_table_as_sql": {"name": "databricks__get_create_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/create.sql", "original_file_path": "macros/relations/streaming_table/create.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_streaming_table_as_sql", "macro_sql": "{% macro databricks__get_create_streaming_table_as_sql(relation, sql) -%}\n {%- set streaming_table = adapter.get_config_from_model(config.model) -%}\n {%- set partition_by = streaming_table.config[\"partition_by\"].partition_by -%}\n {%- set tblproperties = streaming_table.config[\"tblproperties\"].tblproperties -%}\n {%- set comment = streaming_table.config[\"comment\"].comment -%}\n {%- set refresh = streaming_table.config[\"refresh\"] -%}\n\n CREATE STREAMING TABLE {{ relation }}\n {{ get_create_sql_partition_by(partition_by) }}\n {{ get_create_sql_comment(comment) }}\n {{ get_create_sql_tblproperties(tblproperties) }}\n {{ get_create_sql_refresh_schedule(refresh.cron, refresh.time_zone_value) }}\n AS {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_create_sql_partition_by", "macro.dbt_databricks.get_create_sql_comment", "macro.dbt_databricks.get_create_sql_tblproperties", "macro.dbt_databricks.get_create_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.618762, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_comment": {"name": "get_create_sql_comment", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/comment.sql", "original_file_path": "macros/relations/components/comment.sql", "unique_id": "macro.dbt_databricks.get_create_sql_comment", "macro_sql": "{%- macro get_create_sql_comment(comment) -%}\n{% if comment is string -%}\n COMMENT '{{ comment }}'\n{%- endif -%}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6189408, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_refresh_schedule": {"name": "get_create_sql_refresh_schedule", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/refresh_schedule.sql", "original_file_path": "macros/relations/components/refresh_schedule.sql", "unique_id": "macro.dbt_databricks.get_create_sql_refresh_schedule", "macro_sql": "{% macro get_create_sql_refresh_schedule(cron, time_zone_value) %}\n {%- if cron -%}\n SCHEDULE CRON '{{ cron }}'{%- if time_zone_value %} AT TIME ZONE '{{ time_zone_value }}'{%- endif -%}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.619374, "supported_languages": null}, "macro.dbt_databricks.get_alter_sql_refresh_schedule": {"name": "get_alter_sql_refresh_schedule", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/refresh_schedule.sql", "original_file_path": "macros/relations/components/refresh_schedule.sql", "unique_id": "macro.dbt_databricks.get_alter_sql_refresh_schedule", "macro_sql": "{% macro get_alter_sql_refresh_schedule(cron, time_zone_value, is_altered) %}\n {%- if cron -%}\n {%- if is_altered -%}\n ALTER SCHEDULE CRON '{{ cron }}'{%- if time_zone_value %} AT TIME ZONE '{{ time_zone_value }}'{%- endif -%}\n {%- else -%}\n ADD SCHEDULE CRON '{{ cron }}'{%- if time_zone_value %} AT TIME ZONE '{{ time_zone_value }}'{%- endif -%}\n {%- endif -%}\n {%- else -%}\n DROP SCHEDULE\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.619678, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_partition_by": {"name": "get_create_sql_partition_by", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/partitioning.sql", "original_file_path": "macros/relations/components/partitioning.sql", "unique_id": "macro.dbt_databricks.get_create_sql_partition_by", "macro_sql": "{% macro get_create_sql_partition_by(partition_by) -%}\n{%- if partition_by -%}\n PARTITIONED BY ({%- for col in partition_by -%}{{ col }}{% if not loop.last %}, {% endif %}{%- endfor %})\n{%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.619945, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_tblproperties": {"name": "get_create_sql_tblproperties", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/tblproperties.sql", "original_file_path": "macros/relations/components/tblproperties.sql", "unique_id": "macro.dbt_databricks.get_create_sql_tblproperties", "macro_sql": "{% macro get_create_sql_tblproperties(tblproperties) %}\n {%- if tblproperties and tblproperties|length>0 -%}\n TBLPROPERTIES (\n {%- for prop in tblproperties -%}\n '{{ prop }}' = '{{ tblproperties[prop] }}'{%- if not loop.last -%}, {% endif -%}\n {% endfor -%}\n )\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.62031, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_table": {"name": "databricks__drop_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_table", "macro_sql": "{% macro databricks__drop_table(relation) -%}\n drop table if exists {{ relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.620424, "supported_languages": null}, "macro.dbt_databricks.databricks__create_table_as": {"name": "databricks__create_table_as", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt_databricks.databricks__create_table_as", "macro_sql": "{% macro databricks__create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {%- if language == 'sql' -%}\n {%- if temporary -%}\n {{ create_temporary_view(relation, compiled_code) }}\n {%- else -%}\n {%- set file_format = config.get('file_format', default='delta') -%}\n {% if file_format == 'delta' %}\n create or replace table {{ relation }}\n {% else %}\n create table {{ relation }}\n {% endif %}\n {%- set contract_config = config.get('contract') -%}\n {% if contract_config and contract_config.enforced %}\n {{ get_assert_columns_equivalent(compiled_code) }}\n {%- set compiled_code = get_select_subquery(compiled_code) %}\n {% endif %}\n {{ file_format_clause() }}\n {{ options_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ liquid_clustered_cols() }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause(relation) }}\n {{ comment_clause() }}\n {{ tblproperties_clause() }}\n as\n {{ compiled_code }}\n {%- endif -%}\n {%- elif language == 'python' -%}\n {#--\n N.B. Python models _can_ write to temp views HOWEVER they use a different session\n and have already expired by the time they need to be used (I.E. in merges for incremental models)\n\n TODO: Deep dive into spark sessions to see if we can reuse a single session for an entire\n dbt invocation.\n --#}\n {{ databricks__py_write_table(compiled_code=compiled_code, target_relation=relation) }}\n {%- endif -%}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.create_temporary_view", "macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_select_subquery", "macro.dbt_databricks.file_format_clause", "macro.dbt_spark.options_clause", "macro.dbt_spark.partition_cols", "macro.dbt_databricks.liquid_clustered_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_databricks.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt_databricks.tblproperties_clause", "macro.dbt_databricks.databricks__py_write_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6221719, "supported_languages": null}, "macro.dbt_databricks.databricks__options_clause": {"name": "databricks__options_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt_databricks.databricks__options_clause", "macro_sql": "{% macro databricks__options_clause() -%}\n {%- set options = config.get('options') -%}\n {%- if config.get('file_format', default='delta') == 'hudi' -%}\n {%- set unique_key = config.get('unique_key') -%}\n {%- if unique_key is not none and options is none -%}\n {%- set options = {'primaryKey': config.get('unique_key')} -%}\n {%- elif unique_key is not none and options is not none and 'primaryKey' not in options -%}\n {%- set _ = options.update({'primaryKey': config.get('unique_key')}) -%}\n {%- elif options is not none and 'primaryKey' in options and options['primaryKey'] != unique_key -%}\n {{ exceptions.raise_compiler_error(\"unique_key and options('primaryKey') should be the same column(s).\") }}\n {%- endif %}\n {%- endif %}\n\n {%- if options is not none %}\n options (\n {%- for option in options -%}\n {{ option }} \"{{ options[option] }}\" {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.623204, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_view": {"name": "databricks__drop_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_view", "macro_sql": "{% macro databricks__drop_view(relation) -%}\n drop view if exists {{ relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6233542, "supported_languages": null}, "macro.dbt_databricks.databricks__create_view_as": {"name": "databricks__create_view_as", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt_databricks.databricks__create_view_as", "macro_sql": "{% macro databricks__create_view_as(relation, sql) -%}\n create or replace view {{ relation }}\n {% if config.persist_column_docs() -%}\n {% set model_columns = model.columns %}\n {% set query_columns = get_columns_in_query(sql) %}\n {% if query_columns %}\n (\n {{ get_persist_docs_column_list(model_columns, query_columns) }}\n )\n {% endif %}\n {% endif %}\n {{ comment_clause() }}\n {%- set contract_config = config.get('contract') -%}\n {% if contract_config and contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n {{ tblproperties_clause() }}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query", "macro.dbt_databricks.get_persist_docs_column_list", "macro.dbt_spark.comment_clause", "macro.dbt.get_assert_columns_equivalent", "macro.dbt_databricks.tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6240902, "supported_languages": null}, "macro.dbt_databricks.statement_with_staging_table": {"name": "statement_with_staging_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt_databricks.statement_with_staging_table", "macro_sql": "{% macro statement_with_staging_table(name=None, staging_table=None, fetch_result=False, auto_begin=True) -%}\n {%- if execute: -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- set res, table = adapter.execute(sql, auto_begin=auto_begin, fetch=fetch_result, staging_table=staging_table) -%}\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6248162, "supported_languages": null}, "macro.dbt_databricks.databricks__dateadd": {"name": "databricks__dateadd", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt_databricks.databricks__dateadd", "macro_sql": "{% macro databricks__dateadd(datepart, interval, from_date_or_timestamp) %}\n {%- if adapter.compare_dbr_version(10, 4) >= 0 -%}\n timestampadd({{datepart}}, {{interval}}, {{from_date_or_timestamp}})\n {%- else -%}\n {{ spark__dateadd(datepart, interval, from_date_or_timestamp) }}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6251829, "supported_languages": null}, "macro.dbt_databricks.databricks__datediff": {"name": "databricks__datediff", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt_databricks.databricks__datediff", "macro_sql": "{% macro databricks__datediff(first_date, second_date, datepart) %}\n {%- if adapter.compare_dbr_version(10, 4) >= 0 -%}\n timestampdiff({{datepart}}, {{date_trunc(datepart, first_date)}}, {{date_trunc(datepart, second_date)}})\n {%- else -%}\n {{ spark__datediff(first_date, second_date, datepart) }}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.date_trunc", "macro.dbt_spark.spark__datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6255841, "supported_languages": null}, "macro.dbt_databricks.databricks_copy_into": {"name": "databricks_copy_into", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/copy_into.sql", "original_file_path": "macros/adapters/copy_into.sql", "unique_id": "macro.dbt_databricks.databricks_copy_into", "macro_sql": "{% macro databricks_copy_into(\n target_table,\n source,\n file_format,\n expression_list=none,\n source_credential=none,\n source_encryption=none,\n validate=none,\n files=none,\n pattern=none,\n format_options=none,\n copy_options=none) -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=target.database,\n schema=target.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- set source_clause -%}\n {%- if expression_list -%}\n ( select {{ expression_list }} from '{{ source }}' )\n {%- else -%}\n '{{ source }}'\n {%- endif -%}\n {%- if source_credential or source_encryption %}\n WITH (\n {%- if source_credential %}\n credential (\n {%- for name in source_credential -%}\n '{{ name }}' = '{{ source_credential[name] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {%- if source_encryption %}\n encryption (\n {%- for name in source_encryption -%}\n '{{ name }}' = '{{ source_encryption[name] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n )\n {%- endif -%}\n {%- endset -%}\n\n {% set query %}\n copy into {{ target_relation }}\n from {{ source_clause }}\n fileformat = {{ file_format }}\n {% if validate -%} validate {{ validate }} {%- endif %}\n {% if files and pattern %}\n {{ exceptions.raise_compiler_error(\"You can only specify one of 'files' or 'pattern'\") }}\n {% endif %}\n {% if files -%}\n files = (\n {%- for file in files -%}\n '{{ file }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% if pattern -%}\n pattern = '{{ pattern }}'\n {%- endif %}\n {% if format_options -%}\n format_options (\n {%- for key in format_options -%}\n '{{ key }}' = '{{ format_options[key] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% if copy_options -%}\n copy_options (\n {%- for key in copy_options -%}\n '{{ key }}' = '{{ copy_options[key] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% endset %}\n\n {% do log(\"Running COPY INTO\" ~ adapter.redact_credentials(query), info=True) %}\n {% do run_query(query) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.62939, "supported_languages": null}, "macro.dbt_databricks.databricks__make_temp_relation": {"name": "databricks__make_temp_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt_databricks.databricks__make_temp_relation", "macro_sql": "{% macro databricks__make_temp_relation(base_relation, suffix='__dbt_tmp', as_table=False) %}\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {%- if as_table -%}\n {% set tmp_relation = api.Relation.create(\n identifier=tmp_identifier,\n schema=base_relation.schema,\n database=base_relation.database,\n type='table') %}\n {%- else -%}\n {% set tmp_relation = api.Relation.create(identifier=tmp_identifier, type='view') %}\n {%- endif -%}\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.630194, "supported_languages": null}, "macro.dbt_databricks.databricks__get_or_create_relation": {"name": "databricks__get_or_create_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt_databricks.databricks__get_or_create_relation", "macro_sql": "{% macro databricks__get_or_create_relation(database, schema, identifier, type, needs_information=False) %}\n {%- set target_relation = adapter.get_relation(\n database=database,\n schema=schema,\n identifier=identifier,\n needs_information=needs_information) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6306949, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog": {"name": "databricks__get_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog", "macro_sql": "{% macro databricks__get_catalog(information_schema, schemas) -%}\n\n {% set query %}\n with tables as (\n {{ databricks__get_catalog_tables_sql(information_schema) }}\n {{ databricks__get_catalog_schemas_where_clause_sql(information_schema.database, schemas) }}\n ),\n columns as (\n {{ databricks__get_catalog_columns_sql(information_schema) }}\n {{ databricks__get_catalog_schemas_where_clause_sql(information_schema.database, schemas) }}\n )\n {{ databricks__get_catalog_results_sql() }}\n {%- endset -%}\n\n {{ return(run_query(query)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog_tables_sql", "macro.dbt_databricks.databricks__get_catalog_schemas_where_clause_sql", "macro.dbt_databricks.databricks__get_catalog_columns_sql", "macro.dbt_databricks.databricks__get_catalog_results_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.633368, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_relations": {"name": "databricks__get_catalog_relations", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_relations", "macro_sql": "{% macro databricks__get_catalog_relations(information_schema, relations) -%}\n\n {% set query %}\n with tables as (\n {{ databricks__get_catalog_tables_sql(information_schema) }}\n {{ databricks__get_catalog_relations_where_clause_sql(relations) }}\n ),\n columns as (\n {{ databricks__get_catalog_columns_sql(information_schema) }}\n {{ databricks__get_catalog_relations_where_clause_sql(relations) }}\n )\n {{ databricks__get_catalog_results_sql() }}\n {%- endset -%}\n\n {{ return(run_query(query)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog_tables_sql", "macro.dbt_databricks.databricks__get_catalog_relations_where_clause_sql", "macro.dbt_databricks.databricks__get_catalog_columns_sql", "macro.dbt_databricks.databricks__get_catalog_results_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.63369, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_tables_sql": {"name": "databricks__get_catalog_tables_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_tables_sql", "macro_sql": "{% macro databricks__get_catalog_tables_sql(information_schema) -%}\n select\n table_catalog as table_database,\n table_schema,\n table_name,\n lower(table_type) as table_type,\n comment as table_comment,\n table_owner,\n 'Last Modified' as `stats:last_modified:label`,\n last_altered as `stats:last_modified:value`,\n 'The timestamp for last update/change' as `stats:last_modified:description`,\n (last_altered is not null and table_type not ilike '%VIEW%') as `stats:last_modified:include`\n from `system`.`information_schema`.`tables`\n where table_catalog = '{{ information_schema.database|lower }}'\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6338282, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_columns_sql": {"name": "databricks__get_catalog_columns_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_columns_sql", "macro_sql": "{% macro databricks__get_catalog_columns_sql(information_schema) -%}\n select\n table_catalog as table_database,\n table_schema,\n table_name,\n column_name,\n ordinal_position as column_index,\n lower(data_type) as column_type,\n comment as column_comment\n from `system`.`information_schema`.`columns`\n where table_catalog = '{{ information_schema.database|lower }}'\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.633953, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_results_sql": {"name": "databricks__get_catalog_results_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_results_sql", "macro_sql": "{% macro databricks__get_catalog_results_sql() -%}\n select *\n from tables\n join columns using (table_database, table_schema, table_name)\n order by column_index\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.634028, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_schemas_where_clause_sql": {"name": "databricks__get_catalog_schemas_where_clause_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_schemas_where_clause_sql", "macro_sql": "{% macro databricks__get_catalog_schemas_where_clause_sql(catalog, schemas) -%}\n where table_catalog = '{{ catalog|lower }}' and ({%- for relation in schemas -%}\n table_schema = '{{ relation[1]|lower }}'{%- if not loop.last %} or {% endif -%}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.63427, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_relations_where_clause_sql": {"name": "databricks__get_catalog_relations_where_clause_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_relations_where_clause_sql", "macro_sql": "{% macro databricks__get_catalog_relations_where_clause_sql(catalog, relations) -%}\n where table_catalog = '{{ catalog|lower }}' and (\n {%- for relation in relations -%}\n {% if relation.schema and relation.identifier %}\n (\n table_schema = '{{ relation.schema|lower }}'\n and table_name = '{{ relation.identifier|lower }}'\n )\n {% elif relation.schema %}\n (\n table_schema = '{{ relation.schema|lower }}'\n )\n {% else %}\n {% do exceptions.raise_compiler_error(\n '`get_catalog_relations` requires a list of relations, each with a schema'\n ) %}\n {% endif %}\n\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6348171, "supported_languages": null}, "macro.dbt_databricks.current_catalog": {"name": "current_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.current_catalog", "macro_sql": "{% macro current_catalog() -%}\n {{ return(adapter.dispatch('current_catalog', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__current_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6351402, "supported_languages": null}, "macro.dbt_databricks.databricks__current_catalog": {"name": "databricks__current_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.databricks__current_catalog", "macro_sql": "{% macro databricks__current_catalog() -%}\n {% call statement('current_catalog', fetch_result=True) %}\n select current_catalog()\n {% endcall %}\n {% do return(load_result('current_catalog').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.635343, "supported_languages": null}, "macro.dbt_databricks.use_catalog": {"name": "use_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.use_catalog", "macro_sql": "{% macro use_catalog(catalog) -%}\n {{ return(adapter.dispatch('use_catalog', 'dbt')(catalog)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__use_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6355011, "supported_languages": null}, "macro.dbt_databricks.databricks__use_catalog": {"name": "databricks__use_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.databricks__use_catalog", "macro_sql": "{% macro databricks__use_catalog(catalog) -%}\n {% call statement() %}\n use catalog {{ adapter.quote(catalog)|lower }}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.635657, "supported_languages": null}, "macro.dbt_databricks.databricks__py_write_table": {"name": "databricks__py_write_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/python.sql", "original_file_path": "macros/adapters/python.sql", "unique_id": "macro.dbt_databricks.databricks__py_write_table", "macro_sql": "{% macro databricks__py_write_table(compiled_code, target_relation) %}\n{{ compiled_code }}\n# --- Autogenerated dbt materialization code. --- #\ndbt = dbtObj(spark.table)\ndf = model(dbt, spark)\n\nimport pyspark\n\n{{ py_try_import('pyspark.sql.connect.dataframe', 'newer_pyspark_available') }}\n{{ py_try_import('pandas', 'pandas_available') }}\n{{ py_try_import('pyspark.pandas', 'pyspark_pandas_api_available') }}\n{{ py_try_import('databricks.koalas', 'koalas_available') }}\n\n# preferentially convert pandas DataFrames to pandas-on-Spark or Koalas DataFrames first\n# since they know how to convert pandas DataFrames better than `spark.createDataFrame(df)`\n# and converting from pandas-on-Spark to Spark DataFrame has no overhead\n\nif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n if pyspark_pandas_api_available:\n df = pyspark.pandas.frame.DataFrame(df)\n elif koalas_available:\n df = databricks.koalas.frame.DataFrame(df)\n\n# convert to pyspark.sql.dataframe.DataFrame\nif isinstance(df, pyspark.sql.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif newer_pyspark_available and isinstance(df, pyspark.sql.connect.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif pyspark_pandas_api_available and isinstance(df, pyspark.pandas.frame.DataFrame):\n df = df.to_spark()\nelif koalas_available and isinstance(df, databricks.koalas.frame.DataFrame):\n df = df.to_spark()\nelif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = spark.createDataFrame(df)\nelse:\n msg = f\"{type(df)} is not a supported type for dbt Python materialization\"\n raise Exception(msg)\n\nwriter = (\n df.write\n .mode(\"overwrite\")\n .option(\"overwriteSchema\", \"true\")\n{{ py_get_writer_options()|indent(8, True) }}\n)\n\nwriter.saveAsTable(\"{{ target_relation }}\")\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.py_try_import", "macro.dbt_databricks.py_get_writer_options"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6376588, "supported_languages": null}, "macro.dbt_databricks.py_get_writer_options": {"name": "py_get_writer_options", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/python.sql", "original_file_path": "macros/adapters/python.sql", "unique_id": "macro.dbt_databricks.py_get_writer_options", "macro_sql": "\n\n{%- macro py_get_writer_options() -%}\n{%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%}\n{%- set file_format = config.get('file_format', validator=validation.any[basestring])|default('delta', true) -%}\n{%- set partition_by = config.get('partition_by', validator=validation.any[list, basestring]) -%}\n{%- set liquid_clustered_by = config.get('liquid_clustered_by', validator=validation.any[list, basestring]) -%}\n{%- set clustered_by = config.get('clustered_by', validator=validation.any[list, basestring]) -%}\n{%- set buckets = config.get('buckets', validator=validation.any[int]) -%}\n.format(\"{{ file_format }}\")\n{%- if location_root is not none %}\n{%- set identifier = model['alias'] %}\n{%- if is_incremental() %}\n{%- set identifier = identifier + '__dbt_tmp' %}\n{%- endif %}\n.option(\"path\", \"{{ location_root }}/{{ identifier }}\")\n{%- endif -%}\n{%- if partition_by is not none -%}\n {%- if partition_by is string -%}\n {%- set partition_by = [partition_by] -%}\n {%- endif %}\n.partitionBy({{ partition_by }})\n{%- endif -%}\n{%- if liquid_clustered_by and not is_incremental() -%}\n {%- if liquid_clustered_by is string -%}\n {%- set liquid_clustered_by = [liquid_clustered_by] -%}\n {%- endif %}\n.clusterBy({{ liquid_clustered_by }})\n{%- endif -%}\n{%- if (clustered_by is not none) and (buckets is not none) -%}\n {%- if clustered_by is string -%}\n {%- set clustered_by = [clustered_by] -%}\n {%- endif %}\n.bucketBy({{ buckets }}, {{ clustered_by }})\n{%- endif -%}\n{% endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.is_incremental"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6390772, "supported_languages": null}, "macro.dbt_databricks.py_try_import": {"name": "py_try_import", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/python.sql", "original_file_path": "macros/adapters/python.sql", "unique_id": "macro.dbt_databricks.py_try_import", "macro_sql": "{% macro py_try_import(library, var_name) -%}\n# make sure {{ library }} exists before using it\ntry:\n import {{ library }}\n {{ var_name }} = True\nexcept ImportError:\n {{ var_name }} = False\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.639234, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_column_comment": {"name": "databricks__alter_column_comment", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.databricks__alter_column_comment", "macro_sql": "{% macro databricks__alter_column_comment(relation, column_dict) %}\n {% if config.get('file_format', default='delta') in ['delta', 'hudi'] %}\n {% for column_name in column_dict %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = comment | replace('\\'', '\\\\\\'') %}\n {% set comment_query %}\n alter table {{ relation }} change column\n {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }}\n comment '{{ escaped_comment }}';\n {% endset %}\n {% do run_query(comment_query) %}\n {% endfor %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.641151, "supported_languages": null}, "macro.dbt_databricks.alter_table_comment": {"name": "alter_table_comment", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.alter_table_comment", "macro_sql": "{% macro alter_table_comment(relation, model) %}\n {% set comment_query %}\n comment on table {{ relation|lower }} is '{{ model.description | replace(\"'\", \"\\\\'\") }}'\n {% endset %}\n {% do run_query(comment_query) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.641404, "supported_languages": null}, "macro.dbt_databricks.get_columns_comments": {"name": "get_columns_comments", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.get_columns_comments", "macro_sql": "{% macro get_columns_comments(relation) -%}\n {% call statement('get_columns_comments', fetch_result=True) -%}\n describe table {{ relation|lower }}\n {% endcall %}\n\n {% do return(load_result('get_columns_comments').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6416378, "supported_languages": null}, "macro.dbt_databricks.databricks__persist_docs": {"name": "databricks__persist_docs", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.databricks__persist_docs", "macro_sql": "{% macro databricks__persist_docs(relation, model, for_relation, for_columns) -%}\n {%- if for_relation and config.persist_relation_docs() and model.description %}\n {% do alter_table_comment(relation, model) %}\n {% endif %}\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {%- set existing_columns = adapter.get_columns_in_relation(relation) -%}\n {%- set columns_to_persist_docs = adapter.get_persist_doc_columns(existing_columns, model.columns) -%}\n {% do alter_column_comment(relation, columns_to_persist_docs) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.alter_table_comment", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.642109, "supported_languages": null}, "macro.dbt_databricks.get_column_comment_sql": {"name": "get_column_comment_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.get_column_comment_sql", "macro_sql": "{% macro get_column_comment_sql(column_name, column_dict) -%}\n {% if column_name in column_dict and column_dict[column_name][\"description\"] -%}\n {% set escaped_description = column_dict[column_name][\"description\"] | replace(\"'\", \"\\\\'\") %}\n {% set column_comment_clause = \"comment '\" ~ escaped_description ~ \"'\" %}\n {%- endif -%}\n {{ adapter.quote(column_name) }} {{ column_comment_clause }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.642476, "supported_languages": null}, "macro.dbt_databricks.get_persist_docs_column_list": {"name": "get_persist_docs_column_list", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.get_persist_docs_column_list", "macro_sql": "{% macro get_persist_docs_column_list(model_columns, query_columns) %}\n {% for column_name in query_columns %}\n {{ get_column_comment_sql(column_name, model_columns) }}\n {{- \", \" if not loop.last else \"\" }}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_column_comment_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.642708, "supported_languages": null}, "macro.dbt_databricks.databricks__list_relations_without_caching": {"name": "databricks__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__list_relations_without_caching", "macro_sql": "{% macro databricks__list_relations_without_caching(schema_relation) %}\n {{ return(adapter.get_relations_without_caching(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6461172, "supported_languages": null}, "macro.dbt_databricks.show_table_extended": {"name": "show_table_extended", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_table_extended", "macro_sql": "{% macro show_table_extended(schema_relation) %}\n {{ return(adapter.dispatch('show_table_extended', 'dbt')(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__show_table_extended"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.646267, "supported_languages": null}, "macro.dbt_databricks.databricks__show_table_extended": {"name": "databricks__show_table_extended", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__show_table_extended", "macro_sql": "{% macro databricks__show_table_extended(schema_relation) %}\n {% call statement('show_table_extended', fetch_result=True) -%}\n show table extended in {{ schema_relation.without_identifier()|lower }} like '{{ schema_relation.identifier|lower }}'\n {% endcall %}\n\n {% do return(load_result('show_table_extended').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.646537, "supported_languages": null}, "macro.dbt_databricks.show_tables": {"name": "show_tables", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_tables", "macro_sql": "{% macro show_tables(relation) %}\n {{ return(adapter.dispatch('show_tables', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__show_tables"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.646763, "supported_languages": null}, "macro.dbt_databricks.databricks__show_tables": {"name": "databricks__show_tables", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__show_tables", "macro_sql": "{% macro databricks__show_tables(relation) %}\n {% call statement('show_tables', fetch_result=True) -%}\n show tables in {{ relation|lower }}\n {% endcall %}\n\n {% do return(load_result('show_tables').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.646989, "supported_languages": null}, "macro.dbt_databricks.show_views": {"name": "show_views", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_views", "macro_sql": "{% macro show_views(relation) %}\n {{ return(adapter.dispatch('show_views', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__show_views"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.647151, "supported_languages": null}, "macro.dbt_databricks.databricks__show_views": {"name": "databricks__show_views", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__show_views", "macro_sql": "{% macro databricks__show_views(relation) %}\n {% call statement('show_views', fetch_result=True) -%}\n show views in {{ relation|lower }}\n {% endcall %}\n\n {% do return(load_result('show_views').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.647396, "supported_languages": null}, "macro.dbt_databricks.databricks__get_relation_last_modified": {"name": "databricks__get_relation_last_modified", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__get_relation_last_modified", "macro_sql": "{% macro databricks__get_relation_last_modified(information_schema, relations) -%}\n\n {%- call statement('last_modified', fetch_result=True) -%}\n {% if information_schema.is_hive_metastore() %}\n {%- for relation in relations -%}\n select '{{ relation.schema }}' as schema,\n '{{ relation.identifier }}' as identifier,\n max(timestamp) as last_modified,\n {{ current_timestamp() }} as snapshotted_at\n from (describe history {{ relation.schema|lower }}.{{ relation.identifier|lower }})\n {% if not loop.last %}\n union all\n {% endif %}\n {%- endfor -%}\n {% else %}\n select table_schema as schema,\n table_name as identifier,\n last_altered as last_modified,\n {{ current_timestamp() }} as snapshotted_at\n from `system`.`information_schema`.`tables`\n where table_catalog = '{{ information_schema.database|lower }}' and\n (\n {%- for relation in relations -%}\n (table_schema = '{{ relation.schema|lower }}' and\n table_name = '{{ relation.identifier|lower }}'){%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n {% endif %}\n {%- endcall -%}\n\n {{ return(load_result('last_modified')) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6481018, "supported_languages": null}, "macro.dbt_databricks.get_view_description": {"name": "get_view_description", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.get_view_description", "macro_sql": "{% macro get_view_description(relation) %}\n {% call statement('get_view_description', fetch_result=True) -%}\n select *\n from `system`.`information_schema`.`views`\n where table_catalog = '{{ relation.database|lower }}'\n and table_schema = '{{ relation.schema|lower }}'\n and table_name = '{{ relation.identifier|lower }}'\n {%- endcall -%}\n\n {% do return(load_result('get_view_description').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.648402, "supported_languages": null}, "macro.dbt_databricks.get_uc_tables": {"name": "get_uc_tables", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.get_uc_tables", "macro_sql": "{% macro get_uc_tables(relation) %}\n {% call statement('get_uc_tables', fetch_result=True) -%}\n select\n table_name,\n if(table_type in ('EXTERNAL', 'MANAGED', 'MANAGED_SHALLOW_CLONE', 'EXTERNAL_SHALLOW_CLONE'), 'table', lower(table_type)) as table_type,\n lower(data_source_format) as file_format,\n table_owner\n from `system`.`information_schema`.`tables`\n where table_catalog = '{{ relation.database|lower }}'\n and table_schema = '{{ relation.schema|lower }}'\n {% if relation.identifier %}\n and table_name = '{{ relation.identifier|lower }}'\n {% endif %}\n {% endcall %}\n\n {% do return(load_result('get_uc_tables').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.648753, "supported_languages": null}, "macro.dbt_spark.tblproperties_clause": {"name": "tblproperties_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.tblproperties_clause", "macro_sql": "{% macro tblproperties_clause() %}\n {{ return(adapter.dispatch('tblproperties_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.663456, "supported_languages": null}, "macro.dbt_spark.spark__tblproperties_clause": {"name": "spark__tblproperties_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__tblproperties_clause", "macro_sql": "{% macro spark__tblproperties_clause() -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- if tblproperties is not none %}\n tblproperties (\n {%- for prop in tblproperties -%}\n '{{ prop }}' = '{{ tblproperties[prop] }}' {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.663767, "supported_languages": null}, "macro.dbt_spark.file_format_clause": {"name": "file_format_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.file_format_clause", "macro_sql": "{% macro file_format_clause() %}\n {{ return(adapter.dispatch('file_format_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__file_format_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.663905, "supported_languages": null}, "macro.dbt_spark.spark__file_format_clause": {"name": "spark__file_format_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__file_format_clause", "macro_sql": "{% macro spark__file_format_clause() %}\n {%- set file_format = config.get('file_format', validator=validation.any[basestring]) -%}\n {%- if file_format is not none %}\n using {{ file_format }}\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6641312, "supported_languages": null}, "macro.dbt_spark.location_clause": {"name": "location_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.location_clause", "macro_sql": "{% macro location_clause() %}\n {{ return(adapter.dispatch('location_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__location_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.664276, "supported_languages": null}, "macro.dbt_spark.spark__location_clause": {"name": "spark__location_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__location_clause", "macro_sql": "{% macro spark__location_clause() %}\n {%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%}\n {%- set identifier = model['alias'] -%}\n {%- if location_root is not none %}\n location '{{ location_root }}/{{ identifier }}'\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.664563, "supported_languages": null}, "macro.dbt_spark.options_clause": {"name": "options_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.options_clause", "macro_sql": "{% macro options_clause() -%}\n {{ return(adapter.dispatch('options_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__options_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6646988, "supported_languages": null}, "macro.dbt_spark.spark__options_clause": {"name": "spark__options_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__options_clause", "macro_sql": "{% macro spark__options_clause() -%}\n {%- set options = config.get('options') -%}\n {%- if config.get('file_format') == 'hudi' -%}\n {%- set unique_key = config.get('unique_key') -%}\n {%- if unique_key is not none and options is none -%}\n {%- set options = {'primaryKey': config.get('unique_key')} -%}\n {%- elif unique_key is not none and options is not none and 'primaryKey' not in options -%}\n {%- set _ = options.update({'primaryKey': config.get('unique_key')}) -%}\n {%- elif options is not none and 'primaryKey' in options and options['primaryKey'] != unique_key -%}\n {{ exceptions.raise_compiler_error(\"unique_key and options('primaryKey') should be the same column(s).\") }}\n {%- endif %}\n {%- endif %}\n\n {%- if options is not none %}\n options (\n {%- for option in options -%}\n {{ option }} \"{{ options[option] }}\" {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6656451, "supported_languages": null}, "macro.dbt_spark.comment_clause": {"name": "comment_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.comment_clause", "macro_sql": "{% macro comment_clause() %}\n {{ return(adapter.dispatch('comment_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__comment_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.665798, "supported_languages": null}, "macro.dbt_spark.spark__comment_clause": {"name": "spark__comment_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__comment_clause", "macro_sql": "{% macro spark__comment_clause() %}\n {%- set raw_persist_docs = config.get('persist_docs', {}) -%}\n\n {%- if raw_persist_docs is mapping -%}\n {%- set raw_relation = raw_persist_docs.get('relation', false) -%}\n {%- if raw_relation -%}\n comment '{{ model.description | replace(\"'\", \"\\\\'\") }}'\n {% endif %}\n {%- elif raw_persist_docs -%}\n {{ exceptions.raise_compiler_error(\"Invalid value provided for 'persist_docs'. Expected dict but got value: \" ~ raw_persist_docs) }}\n {% endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.666194, "supported_languages": null}, "macro.dbt_spark.partition_cols": {"name": "partition_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.partition_cols", "macro_sql": "{% macro partition_cols(label, required=false) %}\n {{ return(adapter.dispatch('partition_cols', 'dbt')(label, required)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__partition_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.666384, "supported_languages": null}, "macro.dbt_spark.spark__partition_cols": {"name": "spark__partition_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__partition_cols", "macro_sql": "{% macro spark__partition_cols(label, required=false) %}\n {%- set cols = config.get('partition_by', validator=validation.any[list, basestring]) -%}\n {%- if cols is not none %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n {{ label }} (\n {%- for item in cols -%}\n {{ item }}\n {%- if not loop.last -%},{%- endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.666821, "supported_languages": null}, "macro.dbt_spark.clustered_cols": {"name": "clustered_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.clustered_cols", "macro_sql": "{% macro clustered_cols(label, required=false) %}\n {{ return(adapter.dispatch('clustered_cols', 'dbt')(label, required)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__clustered_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.667007, "supported_languages": null}, "macro.dbt_spark.spark__clustered_cols": {"name": "spark__clustered_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__clustered_cols", "macro_sql": "{% macro spark__clustered_cols(label, required=false) %}\n {%- set cols = config.get('clustered_by', validator=validation.any[list, basestring]) -%}\n {%- set buckets = config.get('buckets', validator=validation.any[int]) -%}\n {%- if (cols is not none) and (buckets is not none) %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n {{ label }} (\n {%- for item in cols -%}\n {{ item }}\n {%- if not loop.last -%},{%- endif -%}\n {%- endfor -%}\n ) into {{ buckets }} buckets\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6675909, "supported_languages": null}, "macro.dbt_spark.fetch_tbl_properties": {"name": "fetch_tbl_properties", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.fetch_tbl_properties", "macro_sql": "{% macro fetch_tbl_properties(relation) -%}\n {% call statement('list_properties', fetch_result=True) -%}\n SHOW TBLPROPERTIES {{ relation }}\n {% endcall %}\n {% do return(load_result('list_properties').table) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.667808, "supported_languages": null}, "macro.dbt_spark.create_temporary_view": {"name": "create_temporary_view", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.create_temporary_view", "macro_sql": "{% macro create_temporary_view(relation, compiled_code) -%}\n {{ return(adapter.dispatch('create_temporary_view', 'dbt')(relation, compiled_code)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__create_temporary_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.667983, "supported_languages": null}, "macro.dbt_spark.spark__create_temporary_view": {"name": "spark__create_temporary_view", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_temporary_view", "macro_sql": "{% macro spark__create_temporary_view(relation, compiled_code) -%}\n create or replace temporary view {{ relation }} as\n {{ compiled_code }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.668092, "supported_languages": null}, "macro.dbt_spark.spark__create_table_as": {"name": "spark__create_table_as", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_table_as", "macro_sql": "{%- macro spark__create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {%- if language == 'sql' -%}\n {%- if temporary -%}\n {{ create_temporary_view(relation, compiled_code) }}\n {%- else -%}\n {% if config.get('file_format', validator=validation.any[basestring]) in ['delta', 'iceberg'] %}\n create or replace table {{ relation }}\n {% else %}\n create table {{ relation }}\n {% endif %}\n {%- set contract_config = config.get('contract') -%}\n {%- if contract_config.enforced -%}\n {{ get_assert_columns_equivalent(compiled_code) }}\n {%- set compiled_code = get_select_subquery(compiled_code) %}\n {% endif %}\n {{ file_format_clause() }}\n {{ options_clause() }}\n {{ tblproperties_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause() }}\n {{ comment_clause() }}\n\n as\n {{ compiled_code }}\n {%- endif -%}\n {%- elif language == 'python' -%}\n {#--\n N.B. Python models _can_ write to temp views HOWEVER they use a different session\n and have already expired by the time they need to be used (I.E. in merges for incremental models)\n\n TODO: Deep dive into spark sessions to see if we can reuse a single session for an entire\n dbt invocation.\n --#}\n {{ py_write_table(compiled_code=compiled_code, target_relation=relation) }}\n {%- endif -%}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_spark.create_temporary_view", "macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_select_subquery", "macro.dbt_spark.file_format_clause", "macro.dbt_spark.options_clause", "macro.dbt_spark.tblproperties_clause", "macro.dbt_spark.partition_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_spark.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt_spark.py_write_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.668963, "supported_languages": null}, "macro.dbt_spark.persist_constraints": {"name": "persist_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.persist_constraints", "macro_sql": "{% macro persist_constraints(relation, model) %}\n {{ return(adapter.dispatch('persist_constraints', 'dbt')(relation, model)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.669142, "supported_languages": null}, "macro.dbt_spark.spark__persist_constraints": {"name": "spark__persist_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__persist_constraints", "macro_sql": "{% macro spark__persist_constraints(relation, model) %}\n {%- set contract_config = config.get('contract') -%}\n {% if contract_config.enforced and config.get('file_format', 'delta') == 'delta' %}\n {% do alter_table_add_constraints(relation, model.constraints) %}\n {% do alter_column_set_constraints(relation, model.columns) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.alter_table_add_constraints", "macro.dbt_spark.alter_column_set_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.669481, "supported_languages": null}, "macro.dbt_spark.alter_table_add_constraints": {"name": "alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.alter_table_add_constraints", "macro_sql": "{% macro alter_table_add_constraints(relation, constraints) %}\n {{ return(adapter.dispatch('alter_table_add_constraints', 'dbt')(relation, constraints)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_table_add_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.669651, "supported_languages": null}, "macro.dbt_spark.spark__alter_table_add_constraints": {"name": "spark__alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_table_add_constraints", "macro_sql": "{% macro spark__alter_table_add_constraints(relation, constraints) %}\n {% for constraint in constraints %}\n {% if constraint.type == 'check' and not is_incremental() %}\n {%- set constraint_hash = local_md5(column_name ~ \";\" ~ constraint.expression ~ \";\" ~ loop.index) -%}\n {% call statement() %}\n alter table {{ relation }} add constraint {{ constraint.name if constraint.name else constraint_hash }} check ({{ constraint.expression }});\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.is_incremental", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.670092, "supported_languages": null}, "macro.dbt_spark.alter_column_set_constraints": {"name": "alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.alter_column_set_constraints", "macro_sql": "{% macro alter_column_set_constraints(relation, column_dict) %}\n {{ return(adapter.dispatch('alter_column_set_constraints', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_column_set_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.67027, "supported_languages": null}, "macro.dbt_spark.spark__alter_column_set_constraints": {"name": "spark__alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_column_set_constraints", "macro_sql": "{% macro spark__alter_column_set_constraints(relation, column_dict) %}\n {% for column_name in column_dict %}\n {% set constraints = column_dict[column_name]['constraints'] %}\n {% for constraint in constraints %}\n {% if constraint.type != 'not_null' %}\n {{ exceptions.warn('Invalid constraint for column ' ~ column_name ~ '. Only `not_null` is supported.') }}\n {% else %}\n {% set quoted_name = adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name %}\n {% call statement() %}\n alter table {{ relation }} change column {{ quoted_name }} set not null {{ constraint.expression or \"\" }};\n {% endcall %}\n {% endif %}\n {% endfor %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.670861, "supported_languages": null}, "macro.dbt_spark.get_column_comment_sql": {"name": "get_column_comment_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.get_column_comment_sql", "macro_sql": "{% macro get_column_comment_sql(column_name, column_dict) -%}\n {% if column_name in column_dict and column_dict[column_name][\"description\"] -%}\n {% set escaped_description = column_dict[column_name][\"description\"] | replace(\"'\", \"\\\\'\") %}\n {% set column_comment_clause = \"comment '\" ~ escaped_description ~ \"'\" %}\n {%- endif -%}\n {{ adapter.quote(column_name) }} {{ column_comment_clause }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.671293, "supported_languages": null}, "macro.dbt_spark.get_persist_docs_column_list": {"name": "get_persist_docs_column_list", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.get_persist_docs_column_list", "macro_sql": "{% macro get_persist_docs_column_list(model_columns, query_columns) %}\n {% for column_name in query_columns %}\n {{ get_column_comment_sql(column_name, model_columns) }}\n {{- \", \" if not loop.last else \"\" }}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.get_column_comment_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.671527, "supported_languages": null}, "macro.dbt_spark.spark__create_view_as": {"name": "spark__create_view_as", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_view_as", "macro_sql": "{% macro spark__create_view_as(relation, sql) -%}\n create or replace view {{ relation }}\n {% if config.persist_column_docs() -%}\n {% set model_columns = model.columns %}\n {% set query_columns = get_columns_in_query(sql) %}\n (\n {{ get_persist_docs_column_list(model_columns, query_columns) }}\n )\n {% endif %}\n {{ comment_clause() }}\n {%- set contract_config = config.get('contract') -%}\n {%- if contract_config.enforced -%}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query", "macro.dbt_spark.get_persist_docs_column_list", "macro.dbt_spark.comment_clause", "macro.dbt.get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6719759, "supported_languages": null}, "macro.dbt_spark.spark__create_schema": {"name": "spark__create_schema", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_schema", "macro_sql": "{% macro spark__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{relation}}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6721191, "supported_languages": null}, "macro.dbt_spark.spark__drop_schema": {"name": "spark__drop_schema", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__drop_schema", "macro_sql": "{% macro spark__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation }} cascade\n {%- endcall -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.672259, "supported_languages": null}, "macro.dbt_spark.get_columns_in_relation_raw": {"name": "get_columns_in_relation_raw", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.get_columns_in_relation_raw", "macro_sql": "{% macro get_columns_in_relation_raw(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation_raw', 'dbt')(relation)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__get_columns_in_relation_raw"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.672405, "supported_languages": null}, "macro.dbt_spark.spark__get_columns_in_relation_raw": {"name": "spark__get_columns_in_relation_raw", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__get_columns_in_relation_raw", "macro_sql": "{% macro spark__get_columns_in_relation_raw(relation) -%}\n {% call statement('get_columns_in_relation_raw', fetch_result=True) %}\n describe extended {{ relation }}\n {% endcall %}\n {% do return(load_result('get_columns_in_relation_raw').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.672622, "supported_languages": null}, "macro.dbt_spark.spark__get_columns_in_relation": {"name": "spark__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__get_columns_in_relation", "macro_sql": "{% macro spark__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n describe extended {{ relation.include(schema=(schema is not none)) }}\n {% endcall %}\n {% do return(load_result('get_columns_in_relation').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.672888, "supported_languages": null}, "macro.dbt_spark.spark__list_relations_without_caching": {"name": "spark__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__list_relations_without_caching", "macro_sql": "{% macro spark__list_relations_without_caching(relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n show table extended in {{ relation.schema }} like '*'\n {% endcall %}\n\n {% do return(load_result('list_relations_without_caching').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.673117, "supported_languages": null}, "macro.dbt_spark.list_relations_show_tables_without_caching": {"name": "list_relations_show_tables_without_caching", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.list_relations_show_tables_without_caching", "macro_sql": "{% macro list_relations_show_tables_without_caching(schema_relation) %}\n {#-- Spark with iceberg tables don't work with show table extended for #}\n {#-- V2 iceberg tables #}\n {#-- https://issues.apache.org/jira/browse/SPARK-33393 #}\n {% call statement('list_relations_without_caching_show_tables', fetch_result=True) -%}\n show tables in {{ schema_relation.schema }} like '*'\n {% endcall %}\n\n {% do return(load_result('list_relations_without_caching_show_tables').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6733599, "supported_languages": null}, "macro.dbt_spark.describe_table_extended_without_caching": {"name": "describe_table_extended_without_caching", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.describe_table_extended_without_caching", "macro_sql": "{% macro describe_table_extended_without_caching(table_name) %}\n {#-- Spark with iceberg tables don't work with show table extended for #}\n {#-- V2 iceberg tables #}\n {#-- https://issues.apache.org/jira/browse/SPARK-33393 #}\n {% call statement('describe_table_extended_without_caching', fetch_result=True) -%}\n describe extended {{ table_name }}\n {% endcall %}\n {% do return(load_result('describe_table_extended_without_caching').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6736, "supported_languages": null}, "macro.dbt_spark.spark__list_schemas": {"name": "spark__list_schemas", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__list_schemas", "macro_sql": "{% macro spark__list_schemas(database) -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n show databases\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.673813, "supported_languages": null}, "macro.dbt_spark.spark__rename_relation": {"name": "spark__rename_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__rename_relation", "macro_sql": "{% macro spark__rename_relation(from_relation, to_relation) -%}\n {% call statement('rename_relation') -%}\n {% if not from_relation.type %}\n {% do exceptions.raise_database_error(\"Cannot rename a relation with a blank type: \" ~ from_relation.identifier) %}\n {% elif from_relation.type in ('table') %}\n alter table {{ from_relation }} rename to {{ to_relation }}\n {% elif from_relation.type == 'view' %}\n alter view {{ from_relation }} rename to {{ to_relation }}\n {% else %}\n {% do exceptions.raise_database_error(\"Unknown type '\" ~ from_relation.type ~ \"' for relation: \" ~ from_relation.identifier) %}\n {% endif %}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.674293, "supported_languages": null}, "macro.dbt_spark.spark__drop_relation": {"name": "spark__drop_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__drop_relation", "macro_sql": "{% macro spark__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n drop {{ relation.type }} if exists {{ relation }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.674477, "supported_languages": null}, "macro.dbt_spark.spark__generate_database_name": {"name": "spark__generate_database_name", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__generate_database_name", "macro_sql": "{% macro spark__generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(None) %}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.674613, "supported_languages": null}, "macro.dbt_spark.spark__persist_docs": {"name": "spark__persist_docs", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__persist_docs", "macro_sql": "{% macro spark__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do alter_column_comment(relation, model.columns) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.674838, "supported_languages": null}, "macro.dbt_spark.spark__alter_column_comment": {"name": "spark__alter_column_comment", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_column_comment", "macro_sql": "{% macro spark__alter_column_comment(relation, column_dict) %}\n {% if config.get('file_format', validator=validation.any[basestring]) in ['delta', 'hudi', 'iceberg'] %}\n {% for column_name in column_dict %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = comment | replace('\\'', '\\\\\\'') %}\n {% set comment_query %}\n {% if relation.is_iceberg %}\n alter table {{ relation }} alter column\n {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }}\n comment '{{ escaped_comment }}';\n {% else %}\n alter table {{ relation }} change column\n {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }}\n comment '{{ escaped_comment }}';\n {% endif %}\n {% endset %}\n {% do run_query(comment_query) %}\n {% endfor %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.675686, "supported_languages": null}, "macro.dbt_spark.spark__make_temp_relation": {"name": "spark__make_temp_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__make_temp_relation", "macro_sql": "{% macro spark__make_temp_relation(base_relation, suffix) %}\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {% set tmp_relation = base_relation.incorporate(path = {\n \"identifier\": tmp_identifier\n }) -%}\n\n {%- set tmp_relation = tmp_relation.include(database=false, schema=false) -%}\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6760252, "supported_languages": null}, "macro.dbt_spark.spark__alter_column_type": {"name": "spark__alter_column_type", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_column_type", "macro_sql": "{% macro spark__alter_column_type(relation, column_name, new_column_type) -%}\n {% call statement('alter_column_type') %}\n alter table {{ relation }} alter column {{ column_name }} type {{ new_column_type }};\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.67621, "supported_languages": null}, "macro.dbt_spark.spark__alter_relation_add_remove_columns": {"name": "spark__alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_relation_add_remove_columns", "macro_sql": "{% macro spark__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if remove_columns %}\n {% if relation.is_delta %}\n {% set platform_name = 'Delta Lake' %}\n {% elif relation.is_iceberg %}\n {% set platform_name = 'Iceberg' %}\n {% else %}\n {% set platform_name = 'Apache Spark' %}\n {% endif %}\n {{ exceptions.raise_compiler_error(platform_name + ' does not support dropping columns from tables') }}\n {% endif %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation }}\n\n {% if add_columns %} add columns {% endif %}\n {% for column in add_columns %}\n {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.676926, "supported_languages": null}, "macro.dbt_spark.spark__copy_grants": {"name": "spark__copy_grants", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__copy_grants", "macro_sql": "{% macro spark__copy_grants() %}\n\n {% if config.materialized == 'view' %}\n {#-- Spark views don't copy grants when they're replaced --#}\n {{ return(False) }}\n\n {% else %}\n {#-- This depends on how we're replacing the table, which depends on its file format\n -- Just play it safe by assuming that grants have been copied over, and need to be checked / possibly revoked\n -- We can make this more efficient in the future\n #}\n {{ return(True) }}\n\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.677521, "supported_languages": null}, "macro.dbt_spark.spark__get_grant_sql": {"name": "spark__get_grant_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__get_grant_sql", "macro_sql": "\n\n\n{%- macro spark__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation }} to {{ adapter.quote(grantees[0]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6777, "supported_languages": null}, "macro.dbt_spark.spark__get_revoke_sql": {"name": "spark__get_revoke_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__get_revoke_sql", "macro_sql": "\n\n\n{%- macro spark__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation }} from {{ adapter.quote(grantees[0]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.677873, "supported_languages": null}, "macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement": {"name": "spark__support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement", "macro_sql": "\n\n\n{%- macro spark__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(False) }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.677966, "supported_languages": null}, "macro.dbt_spark.spark__call_dcl_statements": {"name": "spark__call_dcl_statements", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__call_dcl_statements", "macro_sql": "{% macro spark__call_dcl_statements(dcl_statement_list) %}\n {% for dcl_statement in dcl_statement_list %}\n {% call statement('grant_or_revoke') %}\n {{ dcl_statement }}\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.678161, "supported_languages": null}, "macro.dbt_spark.spark__get_binding_char": {"name": "spark__get_binding_char", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__get_binding_char", "macro_sql": "{% macro spark__get_binding_char() %}\n {{ return('?' if target.method == 'odbc' else '%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.679692, "supported_languages": null}, "macro.dbt_spark.spark__reset_csv_table": {"name": "spark__reset_csv_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__reset_csv_table", "macro_sql": "{% macro spark__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% if old_relation %}\n {{ adapter.drop_relation(old_relation) }}\n {% endif %}\n {% set sql = create_csv_table(model, agate_table) %}\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6799822, "supported_languages": null}, "macro.dbt_spark.spark__load_csv_rows": {"name": "spark__load_csv_rows", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__load_csv_rows", "macro_sql": "{% macro spark__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n {% set column_override = model['config'].get('column_types', {}) %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} values\n {% for row in chunk -%}\n ({%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n cast({{ get_binding_char() }} as {{type}})\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.681087, "supported_languages": null}, "macro.dbt_spark.spark__create_csv_table": {"name": "spark__create_csv_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__create_csv_table", "macro_sql": "{% macro spark__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {{ file_format_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause() }}\n {{ comment_clause() }}\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.file_format_clause", "macro.dbt_spark.partition_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_spark.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6820521, "supported_languages": null}, "macro.dbt_spark.materialization_view_spark": {"name": "materialization_view_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/view.sql", "original_file_path": "macros/materializations/view.sql", "unique_id": "macro.dbt_spark.materialization_view_spark", "macro_sql": "{% materialization view, adapter='spark' -%}\n {{ return(create_or_replace_view()) }}\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.create_or_replace_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.682219, "supported_languages": ["sql"]}, "macro.dbt_spark.materialization_table_spark": {"name": "materialization_table_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_spark.materialization_table_spark", "macro_sql": "{% materialization table, adapter = 'spark', supported_languages=['sql', 'python'] %}\n {%- set language = model['language'] -%}\n {%- set identifier = model['alias'] -%}\n {%- set grant_config = config.get('grants') -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier,\n schema=schema,\n database=database,\n type='table') -%}\n\n {{ run_hooks(pre_hooks) }}\n\n -- setup: if the target relation already exists, drop it\n -- in case if the existing and future table is delta or iceberg, we want to do a\n -- create or replace table instead of dropping, so we don't have the table unavailable\n {% if old_relation is not none %}\n {% set is_delta = (old_relation.is_delta and config.get('file_format', validator=validation.any[basestring]) == 'delta') %}\n {% set is_iceberg = (old_relation.is_iceberg and config.get('file_format', validator=validation.any[basestring]) == 'iceberg') %}\n {% set old_relation_type = old_relation.type %}\n {% else %}\n {% set is_delta = false %}\n {% set is_iceberg = false %}\n {% set old_relation_type = target_relation.type %}\n {% endif %}\n\n {% if not is_delta and not is_iceberg %}\n {% set existing_relation = target_relation %}\n {{ adapter.drop_relation(existing_relation.incorporate(type=old_relation_type)) }}\n {% endif %}\n\n -- build model\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]})}}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt_spark.persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6857522, "supported_languages": ["sql", "python"]}, "macro.dbt_spark.py_write_table": {"name": "py_write_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_spark.py_write_table", "macro_sql": "{% macro py_write_table(compiled_code, target_relation) %}\n{{ compiled_code }}\n# --- Autogenerated dbt materialization code. --- #\ndbt = dbtObj(spark.table)\ndf = model(dbt, spark)\n\n# make sure pyspark exists in the namepace, for 7.3.x-scala2.12 it does not exist\nimport pyspark\n# make sure pandas exists before using it\ntry:\n import pandas\n pandas_available = True\nexcept ImportError:\n pandas_available = False\n\n# make sure pyspark.pandas exists before using it\ntry:\n import pyspark.pandas\n pyspark_pandas_api_available = True\nexcept ImportError:\n pyspark_pandas_api_available = False\n\n# make sure databricks.koalas exists before using it\ntry:\n import databricks.koalas\n koalas_available = True\nexcept ImportError:\n koalas_available = False\n\n# preferentially convert pandas DataFrames to pandas-on-Spark or Koalas DataFrames first\n# since they know how to convert pandas DataFrames better than `spark.createDataFrame(df)`\n# and converting from pandas-on-Spark to Spark DataFrame has no overhead\nif pyspark_pandas_api_available and pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = pyspark.pandas.frame.DataFrame(df)\nelif koalas_available and pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = databricks.koalas.frame.DataFrame(df)\n\n# convert to pyspark.sql.dataframe.DataFrame\nif isinstance(df, pyspark.sql.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif pyspark_pandas_api_available and isinstance(df, pyspark.pandas.frame.DataFrame):\n df = df.to_spark()\nelif koalas_available and isinstance(df, databricks.koalas.frame.DataFrame):\n df = df.to_spark()\nelif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = spark.createDataFrame(df)\nelse:\n msg = f\"{type(df)} is not a supported type for dbt Python materialization\"\n raise Exception(msg)\n\ndf.write.mode(\"overwrite\").format(\"{{ config.get('file_format', 'delta') }}\").option(\"overwriteSchema\", \"true\").saveAsTable(\"{{ target_relation }}\")\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.686034, "supported_languages": null}, "macro.dbt_spark.py_script_comment": {"name": "py_script_comment", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_spark.py_script_comment", "macro_sql": "{%macro py_script_comment()%}\n# how to execute python model in notebook\n# dbt = dbtObj(spark.table)\n# df = model(dbt, spark)\n{%endmacro%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.686104, "supported_languages": null}, "macro.dbt_spark.spark__snapshot_hash_arguments": {"name": "spark__snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__snapshot_hash_arguments", "macro_sql": "{% macro spark__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as string ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.692856, "supported_languages": null}, "macro.dbt_spark.spark__snapshot_string_as_time": {"name": "spark__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__snapshot_string_as_time", "macro_sql": "{% macro spark__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"to_timestamp('\" ~ timestamp ~ \"')\" -%}\n {{ return(result) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.693005, "supported_languages": null}, "macro.dbt_spark.spark__snapshot_merge_sql": {"name": "spark__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__snapshot_merge_sql", "macro_sql": "{% macro spark__snapshot_merge_sql(target, source, insert_cols) -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n {% if target.is_iceberg %}\n {# create view only supports a name (no catalog, or schema) #}\n using {{ source.identifier }} as DBT_INTERNAL_SOURCE\n {% else %}\n using {{ source }} as DBT_INTERNAL_SOURCE\n {% endif %}\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert *\n ;\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.693233, "supported_languages": null}, "macro.dbt_spark.spark_build_snapshot_staging_table": {"name": "spark_build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark_build_snapshot_staging_table", "macro_sql": "{% macro spark_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set tmp_identifier = target_relation.identifier ~ '__dbt_tmp' %}\n\n {% if target_relation.is_iceberg %}\n {# iceberg catalog does not support create view, but regular spark does. We removed the catalog and schema #}\n {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier,\n schema=none,\n database=none,\n type='view') -%}\n {% else %}\n {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier,\n schema=target_relation.schema,\n database=none,\n type='view') -%}\n {% endif %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {# needs to be a non-temp view so that its columns can be ascertained via `describe` #}\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_view_as(tmp_relation, select) }}\n {% endcall %}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.693888, "supported_languages": null}, "macro.dbt_spark.spark__post_snapshot": {"name": "spark__post_snapshot", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__post_snapshot", "macro_sql": "{% macro spark__post_snapshot(staging_relation) %}\n {% do adapter.drop_relation(staging_relation) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.694009, "supported_languages": null}, "macro.dbt_spark.spark__create_columns": {"name": "spark__create_columns", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__create_columns", "macro_sql": "{% macro spark__create_columns(relation, columns) %}\n {% if columns|length > 0 %}\n {% call statement() %}\n alter table {{ relation }} add columns (\n {% for column in columns %}\n `{{ column.name }}` {{ column.data_type }} {{- ',' if not loop.last -}}\n {% endfor %}\n );\n {% endcall %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6943328, "supported_languages": null}, "macro.dbt_spark.materialization_snapshot_spark": {"name": "materialization_snapshot_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.materialization_snapshot_spark", "macro_sql": "{% materialization snapshot, adapter='spark' %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n {%- set file_format = config.get('file_format', 'parquet') -%}\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=none,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if file_format not in ['delta', 'iceberg', 'hudi'] -%}\n {% set invalid_format_msg -%}\n Invalid file format: {{ file_format }}\n Snapshot functionality requires file_format be set to 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n\n {%- if target_relation_exists -%}\n {%- if not target_relation.is_delta and not target_relation.is_iceberg and not target_relation.is_hudi -%}\n {% set invalid_format_msg -%}\n The existing table {{ model.schema }}.{{ target_table }} is in another format than 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n {% endif %}\n\n {% if not adapter.check_schema_exists(model.database, model.schema) %}\n {% do create_schema(model.schema) %}\n {% endif %}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = spark_build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.create_schema", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt_spark.spark_build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6975749, "supported_languages": ["sql"]}, "macro.dbt_spark.spark__can_clone_table": {"name": "spark__can_clone_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_spark.spark__can_clone_table", "macro_sql": "{% macro spark__can_clone_table() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6994078, "supported_languages": null}, "macro.dbt_spark.spark__create_or_replace_clone": {"name": "spark__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_spark.spark__create_or_replace_clone", "macro_sql": "{% macro spark__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace table {{ this_relation }} shallow clone {{ defer_relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.699519, "supported_languages": null}, "macro.dbt_spark.materialization_clone_spark": {"name": "materialization_clone_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_spark.materialization_clone_spark", "macro_sql": "\n\n{%- materialization clone, adapter='spark' -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n {%- set file_format = config.get('file_format', validator=validation.any[basestring]) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if file_format != 'delta' -%}\n {% set invalid_format_msg -%}\n Invalid file format: {{ file_format }}\n shallow clone requires file_format be set to 'delta'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {%- elif other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% call statement('main') %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n {% endif %}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.70128, "supported_languages": ["sql"]}, "macro.dbt_spark.spark__get_merge_update_columns": {"name": "spark__get_merge_update_columns", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/column_helpers.sql", "original_file_path": "macros/materializations/incremental/column_helpers.sql", "unique_id": "macro.dbt_spark.spark__get_merge_update_columns", "macro_sql": "{% macro spark__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = None -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.702069, "supported_languages": null}, "macro.dbt_spark.dbt_spark_validate_get_file_format": {"name": "dbt_spark_validate_get_file_format", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_spark.dbt_spark_validate_get_file_format", "macro_sql": "{% macro dbt_spark_validate_get_file_format(raw_file_format) %}\n {#-- Validate the file format #}\n\n {% set accepted_formats = ['text', 'csv', 'json', 'jdbc', 'parquet', 'orc', 'hive', 'delta', 'iceberg', 'libsvm', 'hudi'] %}\n\n {% set invalid_file_format_msg -%}\n Invalid file format provided: {{ raw_file_format }}\n Expected one of: {{ accepted_formats | join(', ') }}\n {%- endset %}\n\n {% if raw_file_format not in accepted_formats %}\n {% do exceptions.raise_compiler_error(invalid_file_format_msg) %}\n {% endif %}\n\n {% do return(raw_file_format) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.703007, "supported_languages": null}, "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy": {"name": "dbt_spark_validate_get_incremental_strategy", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy", "macro_sql": "{% macro dbt_spark_validate_get_incremental_strategy(raw_strategy, file_format) %}\n {#-- Validate the incremental strategy #}\n\n {% set invalid_strategy_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n Expected one of: 'append', 'merge', 'insert_overwrite'\n {%- endset %}\n\n {% set invalid_merge_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You can only choose this strategy when file_format is set to 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n\n {% set invalid_insert_overwrite_endpoint_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You cannot use this strategy when connecting via endpoint\n Use the 'append' or 'merge' strategy instead\n {%- endset %}\n\n {% if raw_strategy not in ['append', 'merge', 'insert_overwrite'] %}\n {% do exceptions.raise_compiler_error(invalid_strategy_msg) %}\n {%-else %}\n {% if raw_strategy == 'merge' and file_format not in ['delta', 'iceberg', 'hudi'] %}\n {% do exceptions.raise_compiler_error(invalid_merge_msg) %}\n {% endif %}\n {% if raw_strategy == 'insert_overwrite' and target.endpoint %}\n {% do exceptions.raise_compiler_error(invalid_insert_overwrite_endpoint_msg) %}\n {% endif %}\n {% endif %}\n\n {% do return(raw_strategy) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.703701, "supported_languages": null}, "macro.dbt_spark.get_insert_overwrite_sql": {"name": "get_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.get_insert_overwrite_sql", "macro_sql": "{% macro get_insert_overwrite_sql(source_relation, target_relation, existing_relation) %}\n\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n {% if existing_relation.is_iceberg %}\n {# removed table from statement for iceberg #}\n insert overwrite {{ target_relation }}\n {# removed partition_cols for iceberg as well #}\n {% else %}\n insert overwrite table {{ target_relation }}\n {{ partition_cols(label=\"partition\") }}\n {% endif %}\n select {{dest_cols_csv}} from {{ source_relation }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.partition_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.705375, "supported_languages": null}, "macro.dbt_spark.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(source_relation, target_relation) %}\n\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n insert into table {{ target_relation }}\n select {{dest_cols_csv}} from {{ source_relation }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.705639, "supported_languages": null}, "macro.dbt_spark.spark__get_merge_sql": {"name": "spark__get_merge_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.spark__get_merge_sql", "macro_sql": "{% macro spark__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %}\n {# need dest_columns for merge_exclude_columns, default to use \"*\" #}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n\n when matched then update set\n {% if update_columns -%}{%- for column_name in update_columns %}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {%- else %} * {% endif %}\n\n when not matched then insert *\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.706816, "supported_languages": null}, "macro.dbt_spark.dbt_spark_get_incremental_sql": {"name": "dbt_spark_get_incremental_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.dbt_spark_get_incremental_sql", "macro_sql": "{% macro dbt_spark_get_incremental_sql(strategy, source, target, existing, unique_key, incremental_predicates) %}\n {%- if strategy == 'append' -%}\n {#-- insert new records into existing table, without updating or overwriting #}\n {{ get_insert_into_sql(source, target) }}\n {%- elif strategy == 'insert_overwrite' -%}\n {#-- insert statements don't like CTEs, so support them via a temp view #}\n {{ get_insert_overwrite_sql(source, target, existing) }}\n {%- elif strategy == 'merge' -%}\n {#-- merge all columns for datasources which implement MERGE INTO (e.g. databricks, iceberg) - schema changes are handled for us #}\n {{ get_merge_sql(target, source, unique_key, dest_columns=none, incremental_predicates=incremental_predicates) }}\n {%- else -%}\n {% set no_sql_for_strategy_msg -%}\n No known SQL for the incremental strategy provided: {{ strategy }}\n {%- endset %}\n {%- do exceptions.raise_compiler_error(no_sql_for_strategy_msg) -%}\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.get_insert_into_sql", "macro.dbt_spark.get_insert_overwrite_sql", "macro.dbt.get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.707412, "supported_languages": null}, "macro.dbt_spark.materialization_incremental_spark": {"name": "materialization_incremental_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/incremental.sql", "original_file_path": "macros/materializations/incremental/incremental.sql", "unique_id": "macro.dbt_spark.materialization_incremental_spark", "macro_sql": "{% materialization incremental, adapter='spark', supported_languages=['sql', 'python'] -%}\n {#-- Validate early so we don't run SQL if the file_format + strategy combo is invalid --#}\n {%- set raw_file_format = config.get('file_format', default='parquet') -%}\n {%- set raw_strategy = config.get('incremental_strategy') or 'append' -%}\n {%- set grant_config = config.get('grants') -%}\n\n {%- set file_format = dbt_spark_validate_get_file_format(raw_file_format) -%}\n {%- set strategy = dbt_spark_validate_get_incremental_strategy(raw_strategy, file_format) -%}\n\n {#-- Set vars --#}\n\n {%- set unique_key = config.get('unique_key', none) -%}\n {%- set partition_by = config.get('partition_by', none) -%}\n {%- set language = model['language'] -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n {%- set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) -%}\n {%- set target_relation = this -%}\n {%- set existing_relation = load_relation(this) -%}\n {%- set tmp_relation = make_temp_relation(this) -%}\n\n {#-- for SQL model we will create temp view that doesn't have database and schema --#}\n {%- if language == 'sql'-%}\n {%- set tmp_relation = tmp_relation.include(database=false, schema=false) -%}\n {%- endif -%}\n\n {#-- Set Overwrite Mode --#}\n {%- if strategy == 'insert_overwrite' and partition_by -%}\n {%- call statement() -%}\n set spark.sql.sources.partitionOverwriteMode = DYNAMIC\n {%- endcall -%}\n {%- endif -%}\n\n {#-- Run pre-hooks --#}\n {{ run_hooks(pre_hooks) }}\n\n {#-- Incremental run logic --#}\n {%- if existing_relation is none -%}\n {#-- Relation must be created --#}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {%- elif existing_relation.is_view or should_full_refresh() -%}\n {#-- Relation must be dropped & recreated --#}\n {% set is_delta = (file_format == 'delta' and existing_relation.is_delta) %}\n {% if not is_delta %} {#-- If Delta, we will `create or replace` below, so no need to drop --#}\n {% do adapter.drop_relation(existing_relation) %}\n {% endif %}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {%- else -%}\n {#-- Relation must be merged --#}\n {%- call statement('create_tmp_relation', language=language) -%}\n {{ create_table_as(True, tmp_relation, compiled_code, language) }}\n {%- endcall -%}\n {%- do process_schema_changes(on_schema_change, tmp_relation, existing_relation) -%}\n {%- call statement('main') -%}\n {{ dbt_spark_get_incremental_sql(strategy, tmp_relation, target_relation, existing_relation, unique_key, incremental_predicates) }}\n {%- endcall -%}\n {%- if language == 'python' -%}\n {#--\n This is yucky.\n See note in dbt-spark/dbt/include/spark/macros/adapters.sql\n re: python models and temporary views.\n\n Also, why do neither drop_relation or adapter.drop_relation work here?!\n --#}\n {% call statement('drop_relation') -%}\n drop table if exists {{ tmp_relation }}\n {%- endcall %}\n {%- endif -%}\n {%- endif -%}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt_spark.dbt_spark_validate_get_file_format", "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.load_relation", "macro.dbt.make_temp_relation", "macro.dbt.statement", "macro.dbt.run_hooks", "macro.dbt.create_table_as", "macro.dbt_spark.persist_constraints", "macro.dbt.should_full_refresh", "macro.dbt.process_schema_changes", "macro.dbt_spark.dbt_spark_get_incremental_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.710429, "supported_languages": ["sql", "python"]}, "macro.dbt_spark.spark__concat": {"name": "spark__concat", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt_spark.spark__concat", "macro_sql": "{% macro spark__concat(fields) -%}\n concat({{ fields|join(', ') }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.710591, "supported_languages": null}, "macro.dbt_spark.assert_not_null": {"name": "assert_not_null", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/assert_not_null.sql", "original_file_path": "macros/utils/assert_not_null.sql", "unique_id": "macro.dbt_spark.assert_not_null", "macro_sql": "{% macro assert_not_null(function, arg) -%}\n {{ return(adapter.dispatch('assert_not_null', 'dbt')(function, arg)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__assert_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.710856, "supported_languages": null}, "macro.dbt_spark.spark__assert_not_null": {"name": "spark__assert_not_null", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/assert_not_null.sql", "original_file_path": "macros/utils/assert_not_null.sql", "unique_id": "macro.dbt_spark.spark__assert_not_null", "macro_sql": "{% macro spark__assert_not_null(function, arg) %}\n\n coalesce({{function}}({{arg}}), nvl2({{function}}({{arg}}), assert_true({{function}}({{arg}}) is not null), null))\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.711019, "supported_languages": null}, "macro.dbt_spark.spark__dateadd": {"name": "spark__dateadd", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt_spark.spark__dateadd", "macro_sql": "{% macro spark__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n {%- set clock_component -%}\n {# make sure the dates + timestamps are real, otherwise raise an error asap #}\n to_unix_timestamp({{ assert_not_null('to_timestamp', from_date_or_timestamp) }})\n - to_unix_timestamp({{ assert_not_null('date', from_date_or_timestamp) }})\n {%- endset -%}\n\n {%- if datepart in ['day', 'week'] -%}\n\n {%- set multiplier = 7 if datepart == 'week' else 1 -%}\n\n to_timestamp(\n to_unix_timestamp(\n date_add(\n {{ assert_not_null('date', from_date_or_timestamp) }},\n cast({{interval}} * {{multiplier}} as int)\n )\n ) + {{clock_component}}\n )\n\n {%- elif datepart in ['month', 'quarter', 'year'] -%}\n\n {%- set multiplier -%}\n {%- if datepart == 'month' -%} 1\n {%- elif datepart == 'quarter' -%} 3\n {%- elif datepart == 'year' -%} 12\n {%- endif -%}\n {%- endset -%}\n\n to_timestamp(\n to_unix_timestamp(\n add_months(\n {{ assert_not_null('date', from_date_or_timestamp) }},\n cast({{interval}} * {{multiplier}} as int)\n )\n ) + {{clock_component}}\n )\n\n {%- elif datepart in ('hour', 'minute', 'second', 'millisecond', 'microsecond') -%}\n\n {%- set multiplier -%}\n {%- if datepart == 'hour' -%} 3600\n {%- elif datepart == 'minute' -%} 60\n {%- elif datepart == 'second' -%} 1\n {%- elif datepart == 'millisecond' -%} (1/1000000)\n {%- elif datepart == 'microsecond' -%} (1/1000000)\n {%- endif -%}\n {%- endset -%}\n\n to_timestamp(\n {{ assert_not_null('to_unix_timestamp', from_date_or_timestamp) }}\n + cast({{interval}} * {{multiplier}} as int)\n )\n\n {%- else -%}\n\n {{ exceptions.raise_compiler_error(\"macro dateadd not implemented for datepart ~ '\" ~ datepart ~ \"' ~ on Spark\") }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.assert_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.713269, "supported_languages": null}, "macro.dbt_spark.spark__current_timestamp": {"name": "spark__current_timestamp", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/timestamps.sql", "original_file_path": "macros/utils/timestamps.sql", "unique_id": "macro.dbt_spark.spark__current_timestamp", "macro_sql": "{% macro spark__current_timestamp() -%}\n current_timestamp()\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.713365, "supported_languages": null}, "macro.dbt_spark.spark__escape_single_quotes": {"name": "spark__escape_single_quotes", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt_spark.spark__escape_single_quotes", "macro_sql": "{% macro spark__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"\\\\'\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.713526, "supported_languages": null}, "macro.dbt_spark.spark__listagg": {"name": "spark__listagg", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt_spark.spark__listagg", "macro_sql": "{% macro spark__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if order_by_clause %}\n {{ exceptions.warn(\"order_by_clause is not supported for listagg on Spark/Databricks\") }}\n {% endif %}\n\n {% set collect_list %} collect_list({{ measure }}) {% endset %}\n\n {% set limited %} slice({{ collect_list }}, 1, {{ limit_num }}) {% endset %}\n\n {% set collected = limited if limit_num else collect_list %}\n\n {% set final %} array_join({{ collected }}, {{ delimiter_text }}) {% endset %}\n\n {% do return(final) %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.714139, "supported_languages": null}, "macro.dbt_spark.spark__datediff": {"name": "spark__datediff", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt_spark.spark__datediff", "macro_sql": "{% macro spark__datediff(first_date, second_date, datepart) %}\n\n {%- if datepart in ['day', 'week', 'month', 'quarter', 'year'] -%}\n\n {# make sure the dates are real, otherwise raise an error asap #}\n {% set first_date = assert_not_null('date', first_date) %}\n {% set second_date = assert_not_null('date', second_date) %}\n\n {%- endif -%}\n\n {%- if datepart == 'day' -%}\n\n datediff({{second_date}}, {{first_date}})\n\n {%- elif datepart == 'week' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(datediff({{second_date}}, {{first_date}})/7)\n else ceil(datediff({{second_date}}, {{first_date}})/7)\n end\n\n -- did we cross a week boundary (Sunday)?\n + case\n when {{first_date}} < {{second_date}} and dayofweek({{second_date}}) < dayofweek({{first_date}}) then 1\n when {{first_date}} > {{second_date}} and dayofweek({{second_date}}) > dayofweek({{first_date}}) then -1\n else 0 end\n\n {%- elif datepart == 'month' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(months_between(date({{second_date}}), date({{first_date}})))\n else ceil(months_between(date({{second_date}}), date({{first_date}})))\n end\n\n -- did we cross a month boundary?\n + case\n when {{first_date}} < {{second_date}} and dayofmonth({{second_date}}) < dayofmonth({{first_date}}) then 1\n when {{first_date}} > {{second_date}} and dayofmonth({{second_date}}) > dayofmonth({{first_date}}) then -1\n else 0 end\n\n {%- elif datepart == 'quarter' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(months_between(date({{second_date}}), date({{first_date}}))/3)\n else ceil(months_between(date({{second_date}}), date({{first_date}}))/3)\n end\n\n -- did we cross a quarter boundary?\n + case\n when {{first_date}} < {{second_date}} and (\n (dayofyear({{second_date}}) - (quarter({{second_date}}) * 365/4))\n < (dayofyear({{first_date}}) - (quarter({{first_date}}) * 365/4))\n ) then 1\n when {{first_date}} > {{second_date}} and (\n (dayofyear({{second_date}}) - (quarter({{second_date}}) * 365/4))\n > (dayofyear({{first_date}}) - (quarter({{first_date}}) * 365/4))\n ) then -1\n else 0 end\n\n {%- elif datepart == 'year' -%}\n\n year({{second_date}}) - year({{first_date}})\n\n {%- elif datepart in ('hour', 'minute', 'second', 'millisecond', 'microsecond') -%}\n\n {%- set divisor -%}\n {%- if datepart == 'hour' -%} 3600\n {%- elif datepart == 'minute' -%} 60\n {%- elif datepart == 'second' -%} 1\n {%- elif datepart == 'millisecond' -%} (1/1000)\n {%- elif datepart == 'microsecond' -%} (1/1000000)\n {%- endif -%}\n {%- endset -%}\n\n case when {{first_date}} < {{second_date}}\n then ceil((\n {# make sure the timestamps are real, otherwise raise an error asap #}\n {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', second_date)) }}\n - {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', first_date)) }}\n ) / {{divisor}})\n else floor((\n {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', second_date)) }}\n - {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', first_date)) }}\n ) / {{divisor}})\n end\n\n {% if datepart == 'millisecond' %}\n + cast(date_format({{second_date}}, 'SSS') as int)\n - cast(date_format({{first_date}}, 'SSS') as int)\n {% endif %}\n\n {% if datepart == 'microsecond' %}\n {% set capture_str = '[0-9]{4}-[0-9]{2}-[0-9]{2}.[0-9]{2}:[0-9]{2}:[0-9]{2}.([0-9]{6})' %}\n -- Spark doesn't really support microseconds, so this is a massive hack!\n -- It will only work if the timestamp-string is of the format\n -- 'yyyy-MM-dd-HH mm.ss.SSSSSS'\n + cast(regexp_extract({{second_date}}, '{{capture_str}}', 1) as int)\n - cast(regexp_extract({{first_date}}, '{{capture_str}}', 1) as int)\n {% endif %}\n\n {%- else -%}\n\n {{ exceptions.raise_compiler_error(\"macro datediff not implemented for datepart ~ '\" ~ datepart ~ \"' ~ on Spark\") }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.assert_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7200809, "supported_languages": null}, "macro.dbt_spark.spark__safe_cast": {"name": "spark__safe_cast", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt_spark.spark__safe_cast", "macro_sql": "{% macro spark__safe_cast(field, type) %}\n{%- set field_clean = field.strip('\"').strip(\"'\") if (cast_from_string_unsupported_for(type) and field is string) else field -%}\ncast({{field_clean}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.cast_from_string_unsupported_for"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7204401, "supported_languages": null}, "macro.dbt_spark.cast_from_string_unsupported_for": {"name": "cast_from_string_unsupported_for", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt_spark.cast_from_string_unsupported_for", "macro_sql": "{% macro cast_from_string_unsupported_for(type) %}\n {{ return(type.lower().startswith('struct') or type.lower().startswith('array') or type.lower().startswith('map')) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.720673, "supported_languages": null}, "macro.dbt_spark.spark__any_value": {"name": "spark__any_value", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt_spark.spark__any_value", "macro_sql": "{% macro spark__any_value(expression) -%}\n {#-- return any value (non-deterministic) --#}\n first({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.720806, "supported_languages": null}, "macro.dbt_spark.spark__array_concat": {"name": "spark__array_concat", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt_spark.spark__array_concat", "macro_sql": "{% macro spark__array_concat(array_1, array_2) -%}\n concat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.720957, "supported_languages": null}, "macro.dbt_spark.spark__bool_or": {"name": "spark__bool_or", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt_spark.spark__bool_or", "macro_sql": "{% macro spark__bool_or(expression) -%}\n\n max({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.721088, "supported_languages": null}, "macro.dbt_spark.spark__split_part": {"name": "spark__split_part", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt_spark.spark__split_part", "macro_sql": "{% macro spark__split_part(string_text, delimiter_text, part_number) %}\n\n {% set delimiter_expr %}\n\n -- escape if starts with a special character\n case when regexp_extract({{ delimiter_text }}, '([^A-Za-z0-9])(.*)', 1) != '_'\n then concat('\\\\', {{ delimiter_text }})\n else {{ delimiter_text }} end\n\n {% endset %}\n\n {% if part_number >= 0 %}\n\n {% set split_part_expr %}\n\n split(\n {{ string_text }},\n {{ delimiter_expr }}\n )[({{ part_number - 1 if part_number > 0 else part_number }})]\n\n {% endset %}\n\n {% else %}\n\n {% set split_part_expr %}\n\n split(\n {{ string_text }},\n {{ delimiter_expr }}\n )[(\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 1 + {{ part_number }}\n )]\n\n {% endset %}\n\n {% endif %}\n\n {{ return(split_part_expr) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.722142, "supported_languages": null}, "macro.dbt_spark.spark__array_construct": {"name": "spark__array_construct", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt_spark.spark__array_construct", "macro_sql": "{% macro spark__array_construct(inputs, data_type) -%}\n array( {{ inputs|join(' , ') }} )\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.722311, "supported_languages": null}, "macro.dbt_spark.spark__array_append": {"name": "spark__array_append", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt_spark.spark__array_append", "macro_sql": "{% macro spark__array_append(array, new_element) -%}\n {{ array_concat(array, array_construct([new_element])) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.array_concat", "macro.dbt.array_construct"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.722492, "supported_languages": null}, "macro.dbt.run_hooks": {"name": "run_hooks", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.run_hooks", "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.72342, "supported_languages": null}, "macro.dbt.make_hook_config": {"name": "make_hook_config", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.make_hook_config", "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.723584, "supported_languages": null}, "macro.dbt.before_begin": {"name": "before_begin", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.before_begin", "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.723713, "supported_languages": null}, "macro.dbt.in_transaction": {"name": "in_transaction", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.in_transaction", "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.723829, "supported_languages": null}, "macro.dbt.after_commit": {"name": "after_commit", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.after_commit", "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.723947, "supported_languages": null}, "macro.dbt.set_sql_header": {"name": "set_sql_header", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.set_sql_header", "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.724263, "supported_languages": null}, "macro.dbt.should_full_refresh": {"name": "should_full_refresh", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_full_refresh", "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.72451, "supported_languages": null}, "macro.dbt.should_store_failures": {"name": "should_store_failures", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_store_failures", "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.724753, "supported_languages": null}, "macro.dbt.snapshot_merge_sql": {"name": "snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.snapshot_merge_sql", "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__snapshot_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7252579, "supported_languages": null}, "macro.dbt.default__snapshot_merge_sql": {"name": "default__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.default__snapshot_merge_sql", "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n {%- set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() -%}\n\n merge into {{ target.render() }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }}\n\n when matched\n and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.725742, "supported_languages": null}, "macro.dbt.strategy_dispatch": {"name": "strategy_dispatch", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.strategy_dispatch", "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7284422, "supported_languages": null}, "macro.dbt.snapshot_hash_arguments": {"name": "snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_hash_arguments", "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7285912, "supported_languages": null}, "macro.dbt.default__snapshot_hash_arguments": {"name": "default__snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_hash_arguments", "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.728775, "supported_languages": null}, "macro.dbt.snapshot_timestamp_strategy": {"name": "snapshot_timestamp_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_timestamp_strategy", "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}\n {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}\n {% set primary_key = config.get('unique_key') %}\n {% set updated_at = config.get('updated_at') %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes') or false %}\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.{{ columns.dbt_valid_from }} < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7294762, "supported_languages": null}, "macro.dbt.snapshot_string_as_time": {"name": "snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_string_as_time", "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__snapshot_string_as_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7296178, "supported_languages": null}, "macro.dbt.default__snapshot_string_as_time": {"name": "default__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_string_as_time", "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.729759, "supported_languages": null}, "macro.dbt.snapshot_check_all_get_existing_columns": {"name": "snapshot_check_all_get_existing_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_all_get_existing_columns", "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%}\n {%- if not target_exists -%}\n {#-- no table yet -> return whatever the query does --#}\n {{ return((false, query_columns)) }}\n {%- endif -%}\n\n {#-- handle any schema changes --#}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) -%}\n\n {% if check_cols_config == 'all' %}\n {%- set query_columns = get_columns_in_query(node['compiled_code']) -%}\n\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {#-- query for proper casing/quoting, to support comparison below --#}\n {%- set select_check_cols_from_target -%}\n {#-- N.B. The whitespace below is necessary to avoid edge case issue with comments --#}\n {#-- See: https://github.com/dbt-labs/dbt-core/issues/6781 --#}\n select {{ check_cols_config | join(', ') }} from (\n {{ node['compiled_code'] }}\n ) subq\n {%- endset -%}\n {% set query_columns = get_columns_in_query(select_check_cols_from_target) %}\n\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%}\n {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(adapter.quote(col)) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return((ns.column_added, intersection)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7308888, "supported_languages": null}, "macro.dbt.snapshot_check_strategy": {"name": "snapshot_check_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_strategy", "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}\n {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}\n {% set check_cols_config = config.get('check_cols') %}\n {% set primary_key = config.get('unique_key') %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes') or false %}\n {% set updated_at = config.get('updated_at') or snapshot_get_time() %}\n\n {% set column_added = false %}\n\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.snapshot_check_all_get_existing_columns", "macro.dbt.get_true_sql", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7321742, "supported_languages": null}, "macro.dbt.create_columns": {"name": "create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.create_columns", "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__create_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.73771, "supported_languages": null}, "macro.dbt.default__create_columns": {"name": "default__create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__create_columns", "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation.render() }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.737952, "supported_languages": null}, "macro.dbt.post_snapshot": {"name": "post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.post_snapshot", "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.738092, "supported_languages": null}, "macro.dbt.default__post_snapshot": {"name": "default__post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__post_snapshot", "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.738164, "supported_languages": null}, "macro.dbt.get_true_sql": {"name": "get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_true_sql", "macro_sql": "{% macro get_true_sql() %}\n {{ adapter.dispatch('get_true_sql', 'dbt')() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_true_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.738283, "supported_languages": null}, "macro.dbt.default__get_true_sql": {"name": "default__get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__get_true_sql", "macro_sql": "{% macro default__get_true_sql() %}\n {{ return('TRUE') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7383811, "supported_languages": null}, "macro.dbt.snapshot_staging_table": {"name": "snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.snapshot_staging_table", "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__snapshot_staging_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7385468, "supported_languages": null}, "macro.dbt.get_snapshot_table_column_names": {"name": "get_snapshot_table_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_snapshot_table_column_names", "macro_sql": "{% macro get_snapshot_table_column_names() %}\n {{ return({'dbt_valid_to': 'dbt_valid_to', 'dbt_valid_from': 'dbt_valid_from', 'dbt_scd_id': 'dbt_scd_id', 'dbt_updated_at': 'dbt_updated_at'}) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7387378, "supported_languages": null}, "macro.dbt.default__snapshot_staging_table": {"name": "default__snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__snapshot_staging_table", "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where {{ columns.dbt_valid_to }} is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }},\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_to }}\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.{{ columns.dbt_scd_id }}\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }},\n {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }},\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_to }},\n snapshotted_data.{{ columns.dbt_scd_id }}\n\n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.739788, "supported_languages": null}, "macro.dbt.build_snapshot_table": {"name": "build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_table", "macro_sql": "{% macro build_snapshot_table(strategy, sql) -%}\n {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__build_snapshot_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7399502, "supported_languages": null}, "macro.dbt.default__build_snapshot_table": {"name": "default__build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__build_snapshot_table", "macro_sql": "{% macro default__build_snapshot_table(strategy, sql) %}\n {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}\n\n select *,\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }}\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7403069, "supported_languages": null}, "macro.dbt.build_snapshot_staging_table": {"name": "build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_staging_table", "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set temp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, temp_relation, select) }}\n {% endcall %}\n\n {% do return(temp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_temp_relation", "macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.740653, "supported_languages": null}, "macro.dbt.get_updated_at_column_data_type": {"name": "get_updated_at_column_data_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_updated_at_column_data_type", "macro_sql": "{% macro get_updated_at_column_data_type(snapshot_sql) %}\n {% set snapshot_sql_column_schema = get_column_schema_from_query(snapshot_sql) %}\n {% set dbt_updated_at_data_type = null %}\n {% set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {% set ns.dbt_updated_at_data_type = null -%}\n {% for column in snapshot_sql_column_schema %}\n {% if ((column.column == 'dbt_updated_at') or (column.column == 'DBT_UPDATED_AT')) %}\n {% set ns.dbt_updated_at_data_type = column.dtype %}\n {% endif %}\n {% endfor %}\n {{ return(ns.dbt_updated_at_data_type or none) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_column_schema_from_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.741133, "supported_languages": null}, "macro.dbt.check_time_data_types": {"name": "check_time_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.check_time_data_types", "macro_sql": "{% macro check_time_data_types(sql) %}\n {% set dbt_updated_at_data_type = get_updated_at_column_data_type(sql) %}\n {% set snapshot_get_time_data_type = get_snapshot_get_time_data_type() %}\n {% if snapshot_get_time_data_type is not none and dbt_updated_at_data_type is not none and snapshot_get_time_data_type != dbt_updated_at_data_type %}\n {% if exceptions.warn_snapshot_timestamp_data_types %}\n {{ exceptions.warn_snapshot_timestamp_data_types(snapshot_get_time_data_type, dbt_updated_at_data_type) }}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_updated_at_column_data_type", "macro.dbt.get_snapshot_get_time_data_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.741546, "supported_languages": null}, "macro.dbt.materialization_snapshot_default": {"name": "materialization_snapshot_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot.sql", "original_file_path": "macros/materializations/snapshots/snapshot.sql", "unique_id": "macro.dbt.materialization_snapshot_default", "macro_sql": "{% materialization snapshot, default %}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n -- grab current tables grants config for comparision later on\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {# The model['config'] parameter below is no longer used, but passing anyway for compatibility #}\n {# It was a dictionary of config, instead of the config object from the context #}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", model['config'], target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set build_or_select_sql = build_sql %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {{ adapter.valid_snapshot_target(target_relation, columns) }}\n\n {% set build_or_select_sql = snapshot_staging_table(strategy, sql, target_relation) %}\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n\n {{ check_time_data_types(build_or_select_sql) }}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_staging_table", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.check_time_data_types", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.746002, "supported_languages": ["sql"]}, "macro.dbt.materialization_test_default": {"name": "materialization_test_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/test.sql", "original_file_path": "macros/materializations/tests/test.sql", "unique_id": "macro.dbt.materialization_test_default", "macro_sql": "{%- materialization test, default -%}\n\n {% set relations = [] %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% set store_failures_as = config.get('store_failures_as') %}\n -- if `--store-failures` is invoked via command line and `store_failures_as` is not set,\n -- config.get('store_failures_as', 'table') returns None, not 'table'\n {% if store_failures_as == none %}{% set store_failures_as = 'table' %}{% endif %}\n {% if store_failures_as not in ['table', 'view'] %}\n {{ exceptions.raise_compiler_error(\n \"'\" ~ store_failures_as ~ \"' is not a valid value for `store_failures_as`. \"\n \"Accepted values are: ['ephemeral', 'table', 'view']\"\n ) }}\n {% endif %}\n\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type=store_failures_as) -%} %}\n\n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n\n {% call statement(auto_begin=True) %}\n {{ get_create_sql(target_relation, sql) }}\n {% endcall %}\n\n {% do relations.append(target_relation) %}\n\n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n\n {{ adapter.commit() }}\n\n {% else %}\n\n {% set main_sql = sql %}\n\n {% endif %}\n\n {% set limit = config.get('limit') %}\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}\n\n {%- endcall %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.should_store_failures", "macro.dbt.statement", "macro.dbt.get_create_sql", "macro.dbt.get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7482162, "supported_languages": ["sql"]}, "macro.dbt.get_test_sql": {"name": "get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.get_test_sql", "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.749175, "supported_languages": null}, "macro.dbt.default__get_test_sql": {"name": "default__get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.default__get_test_sql", "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7494242, "supported_languages": null}, "macro.dbt.get_unit_test_sql": {"name": "get_unit_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.get_unit_test_sql", "macro_sql": "{% macro get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%}\n {{ adapter.dispatch('get_unit_test_sql', 'dbt')(main_sql, expected_fixture_sql, expected_column_names) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_unit_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.749598, "supported_languages": null}, "macro.dbt.default__get_unit_test_sql": {"name": "default__get_unit_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.default__get_unit_test_sql", "macro_sql": "{% macro default__get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%}\n-- Build actual result given inputs\nwith dbt_internal_unit_test_actual as (\n select\n {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%},{% endif %}{%- endfor -%}, {{ dbt.string_literal(\"actual\") }} as {{ adapter.quote(\"actual_or_expected\") }}\n from (\n {{ main_sql }}\n ) _dbt_internal_unit_test_actual\n),\n-- Build expected result\ndbt_internal_unit_test_expected as (\n select\n {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%}, {% endif %}{%- endfor -%}, {{ dbt.string_literal(\"expected\") }} as {{ adapter.quote(\"actual_or_expected\") }}\n from (\n {{ expected_fixture_sql }}\n ) _dbt_internal_unit_test_expected\n)\n-- Union actual and expected results\nselect * from dbt_internal_unit_test_actual\nunion all\nselect * from dbt_internal_unit_test_expected\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.string_literal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7500942, "supported_languages": null}, "macro.dbt.get_where_subquery": {"name": "get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.get_where_subquery", "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_where_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7504158, "supported_languages": null}, "macro.dbt.default__get_where_subquery": {"name": "default__get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.default__get_where_subquery", "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7507288, "supported_languages": null}, "macro.dbt.materialization_unit_default": {"name": "materialization_unit_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/unit.sql", "original_file_path": "macros/materializations/tests/unit.sql", "unique_id": "macro.dbt.materialization_unit_default", "macro_sql": "{%- materialization unit, default -%}\n\n {% set relations = [] %}\n\n {% set expected_rows = config.get('expected_rows') %}\n {% set expected_sql = config.get('expected_sql') %}\n {% set tested_expected_column_names = expected_rows[0].keys() if (expected_rows | length ) > 0 else get_columns_in_query(sql) %} %}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {% do run_query(get_create_table_as_sql(True, temp_relation, get_empty_subquery_sql(sql))) %}\n {%- set columns_in_relation = adapter.get_columns_in_relation(temp_relation) -%}\n {%- set column_name_to_data_types = {} -%}\n {%- for column in columns_in_relation -%}\n {%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}\n {%- endfor -%}\n\n {% if not expected_sql %}\n {% set expected_sql = get_expected_sql(expected_rows, column_name_to_data_types) %}\n {% endif %}\n {% set unit_test_sql = get_unit_test_sql(sql, expected_sql, tested_expected_column_names) %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ unit_test_sql }}\n\n {%- endcall %}\n\n {% do adapter.drop_relation(temp_relation) %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query", "macro.dbt.make_temp_relation", "macro.dbt.run_query", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_empty_subquery_sql", "macro.dbt.get_expected_sql", "macro.dbt.get_unit_test_sql", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7521741, "supported_languages": ["sql"]}, "macro.dbt.materialization_materialized_view_default": {"name": "materialization_materialized_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialization_materialized_view_default", "macro_sql": "{% materialization materialized_view, default %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.MaterializedView) %}\n {% set intermediate_relation = make_intermediate_relation(target_relation) %}\n {% set backup_relation_type = target_relation.MaterializedView if existing_relation is none else existing_relation.type %}\n {% set backup_relation = make_backup_relation(target_relation, backup_relation_type) %}\n\n {{ materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) }}\n\n {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% if build_sql == '' %}\n {{ materialized_view_execute_no_op(target_relation) }}\n {% else %}\n {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.materialized_view_setup", "macro.dbt.materialized_view_get_build_sql", "macro.dbt.materialized_view_execute_no_op", "macro.dbt.materialized_view_execute_build_sql", "macro.dbt.materialized_view_teardown"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.756201, "supported_languages": ["sql"]}, "macro.dbt.materialized_view_setup": {"name": "materialized_view_setup", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_setup", "macro_sql": "{% macro materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) %}\n\n -- backup_relation and intermediate_relation should not already exist in the database\n -- it's possible these exist because of a previous run that exited unexpectedly\n {% set preexisting_backup_relation = load_cached_relation(backup_relation) %}\n {% set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.756525, "supported_languages": null}, "macro.dbt.materialized_view_teardown": {"name": "materialized_view_teardown", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_teardown", "macro_sql": "{% macro materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) %}\n\n -- drop the temp relations if they exist to leave the database clean for the next run\n {{ drop_relation_if_exists(backup_relation) }}\n {{ drop_relation_if_exists(intermediate_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.756736, "supported_languages": null}, "macro.dbt.materialized_view_get_build_sql": {"name": "materialized_view_get_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_get_build_sql", "macro_sql": "{% macro materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_materialized_view %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %}\n\n {% if configuration_changes is none %}\n {% set build_sql = refresh_materialized_view(target_relation) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = '' %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation.render() ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation.render() ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt.get_replace_sql", "macro.dbt.get_materialized_view_configuration_changes", "macro.dbt.refresh_materialized_view", "macro.dbt.get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.757851, "supported_languages": null}, "macro.dbt.materialized_view_execute_no_op": {"name": "materialized_view_execute_no_op", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_execute_no_op", "macro_sql": "{% macro materialized_view_execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.75805, "supported_languages": null}, "macro.dbt.materialized_view_execute_build_sql": {"name": "materialized_view_execute_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_execute_build_sql", "macro_sql": "{% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7585862, "supported_languages": null}, "macro.dbt.materialization_view_default": {"name": "materialization_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view.sql", "original_file_path": "macros/materializations/models/view.sql", "unique_id": "macro.dbt.materialization_view_default", "macro_sql": "{%- materialization view, default -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='view') -%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"existing_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the existing_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the existing_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.run_hooks", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.760961, "supported_languages": ["sql"]}, "macro.dbt.materialization_table_default": {"name": "materialization_table_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table.sql", "original_file_path": "macros/materializations/models/table.sql", "unique_id": "macro.dbt.materialization_table_default", "macro_sql": "{% materialization table, default %}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') %}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_table_as_sql(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do create_indexes(target_relation) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.get_create_table_as_sql", "macro.dbt.create_indexes", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.763325, "supported_languages": ["sql"]}, "macro.dbt.get_quoted_csv": {"name": "get_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_quoted_csv", "macro_sql": "{% macro get_quoted_csv(column_names) %}\n\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7646668, "supported_languages": null}, "macro.dbt.diff_columns": {"name": "diff_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_columns", "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n\n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.765106, "supported_languages": null}, "macro.dbt.diff_column_data_types": {"name": "diff_column_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_column_data_types", "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.data_type != tc.data_type and not sc.can_expand_to(other_column=tc) %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.76564, "supported_languages": null}, "macro.dbt.get_merge_update_columns": {"name": "get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_merge_update_columns", "macro_sql": "{% macro get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {{ return(adapter.dispatch('get_merge_update_columns', 'dbt')(merge_update_columns, merge_exclude_columns, dest_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.765835, "supported_languages": null}, "macro.dbt.default__get_merge_update_columns": {"name": "default__get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.default__get_merge_update_columns", "macro_sql": "{% macro default__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = dest_columns | map(attribute=\"quoted\") | list -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7664142, "supported_languages": null}, "macro.dbt.get_merge_sql": {"name": "get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_merge_sql", "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n -- back compat for old kwarg name\n {% set incremental_predicates = kwargs.get('predicates', incremental_predicates) %}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.771425, "supported_languages": null}, "macro.dbt.default__get_merge_sql": {"name": "default__get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_merge_sql", "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{\"(\" ~ predicates | join(\") and (\") ~ \")\"}}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv", "macro.dbt.get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.772757, "supported_languages": null}, "macro.dbt.get_delete_insert_merge_sql": {"name": "get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_delete_insert_merge_sql", "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.77298, "supported_languages": null}, "macro.dbt.default__get_delete_insert_merge_sql": {"name": "default__get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_delete_insert_merge_sql", "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not string %}\n delete from {{target }}\n using {{ source }}\n where (\n {% for key in unique_key %}\n {{ source }}.{{ key }} = {{ target }}.{{ key }}\n {{ \"and \" if not loop.last}}\n {% endfor %}\n {% if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {% endif %}\n );\n {% else %}\n delete from {{ target }}\n where (\n {{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n )\n {%- if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {%- endif -%};\n\n {% endif %}\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.773788, "supported_languages": null}, "macro.dbt.get_insert_overwrite_merge_sql": {"name": "get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_insert_overwrite_merge_sql", "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.774013, "supported_languages": null}, "macro.dbt.default__get_insert_overwrite_merge_sql": {"name": "default__get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql", "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {#-- The only time include_sql_header is True: --#}\n {#-- BigQuery + insert_overwrite strategy + \"static\" partitions config --#}\n {#-- We should consider including the sql header at the materialization level instead --#}\n\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.774538, "supported_languages": null}, "macro.dbt.is_incremental": {"name": "is_incremental", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/is_incremental.sql", "original_file_path": "macros/materializations/models/incremental/is_incremental.sql", "unique_id": "macro.dbt.is_incremental", "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.775049, "supported_languages": null}, "macro.dbt.get_incremental_append_sql": {"name": "get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_append_sql", "macro_sql": "{% macro get_incremental_append_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_append_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.775977, "supported_languages": null}, "macro.dbt.default__get_incremental_append_sql": {"name": "default__get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_append_sql", "macro_sql": "{% macro default__get_incremental_append_sql(arg_dict) %}\n\n {% do return(get_insert_into_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_into_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.776169, "supported_languages": null}, "macro.dbt.get_incremental_delete_insert_sql": {"name": "get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_delete_insert_sql", "macro_sql": "{% macro get_incremental_delete_insert_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_delete_insert_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_delete_insert_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.776327, "supported_languages": null}, "macro.dbt.default__get_incremental_delete_insert_sql": {"name": "default__get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_delete_insert_sql", "macro_sql": "{% macro default__get_incremental_delete_insert_sql(arg_dict) %}\n\n {% do return(get_delete_insert_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7765691, "supported_languages": null}, "macro.dbt.get_incremental_merge_sql": {"name": "get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_merge_sql", "macro_sql": "{% macro get_incremental_merge_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.776725, "supported_languages": null}, "macro.dbt.default__get_incremental_merge_sql": {"name": "default__get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_merge_sql", "macro_sql": "{% macro default__get_incremental_merge_sql(arg_dict) %}\n\n {% do return(get_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7769668, "supported_languages": null}, "macro.dbt.get_incremental_insert_overwrite_sql": {"name": "get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_insert_overwrite_sql", "macro_sql": "{% macro get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_insert_overwrite_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_insert_overwrite_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7771251, "supported_languages": null}, "macro.dbt.default__get_incremental_insert_overwrite_sql": {"name": "default__get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_insert_overwrite_sql", "macro_sql": "{% macro default__get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {% do return(get_insert_overwrite_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.777345, "supported_languages": null}, "macro.dbt.get_incremental_default_sql": {"name": "get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_default_sql", "macro_sql": "{% macro get_incremental_default_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_default_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_default_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.777499, "supported_languages": null}, "macro.dbt.default__get_incremental_default_sql": {"name": "default__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_default_sql", "macro_sql": "{% macro default__get_incremental_default_sql(arg_dict) %}\n\n {% do return(get_incremental_append_sql(arg_dict)) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.777623, "supported_languages": null}, "macro.dbt.get_incremental_microbatch_sql": {"name": "get_incremental_microbatch_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_microbatch_sql", "macro_sql": "{% macro get_incremental_microbatch_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_microbatch_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_microbatch_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.777774, "supported_languages": null}, "macro.dbt.default__get_incremental_microbatch_sql": {"name": "default__get_incremental_microbatch_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_microbatch_sql", "macro_sql": "{% macro default__get_incremental_microbatch_sql(arg_dict) %}\n\n {{ exceptions.raise_not_implemented('microbatch materialization strategy not implemented for adapter ' + adapter.type()) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7779121, "supported_languages": null}, "macro.dbt.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(target_relation, temp_relation, dest_columns) %}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ temp_relation }}\n )\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7781398, "supported_languages": null}, "macro.dbt.materialization_incremental_default": {"name": "materialization_incremental_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/incremental.sql", "original_file_path": "macros/materializations/models/incremental/incremental.sql", "unique_id": "macro.dbt.materialization_incremental_default", "macro_sql": "{% materialization incremental, default -%}\n\n -- relations\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n\n -- configs\n {%- set unique_key = config.get('unique_key') -%}\n {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n\n -- the temp_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {% if existing_relation is none %}\n {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}\n {% elif full_refresh_mode %}\n {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}\n {% set need_swap = true %}\n {% else %}\n {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}\n {% set contract_config = config.get('contract') %}\n {% if not contract_config or not contract_config.enforced %}\n {% do adapter.expand_target_column_types(\n from_relation=temp_relation,\n to_relation=target_relation) %}\n {% endif %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n\n {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}\n {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}\n {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}\n {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}\n {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}\n {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% do to_drop.append(backup_relation) %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_temp_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.should_full_refresh", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.get_create_table_as_sql", "macro.dbt.run_query", "macro.dbt.process_schema_changes", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.782465, "supported_languages": ["sql"]}, "macro.dbt.incremental_validate_on_schema_change": {"name": "incremental_validate_on_schema_change", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.incremental_validate_on_schema_change", "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n\n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n\n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n\n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7869802, "supported_languages": null}, "macro.dbt.check_for_schema_changes": {"name": "check_for_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.check_for_schema_changes", "macro_sql": "{% macro check_for_schema_changes(source_relation, target_relation) %}\n\n {% set schema_changed = False %}\n\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}\n {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}\n {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}\n\n {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}\n\n {% if source_not_in_target != [] %}\n {% set schema_changed = True %}\n {% elif target_not_in_source != [] or new_target_types != [] %}\n {% set schema_changed = True %}\n {% elif new_target_types != [] %}\n {% set schema_changed = True %}\n {% endif %}\n\n {% set changes_dict = {\n 'schema_changed': schema_changed,\n 'source_not_in_target': source_not_in_target,\n 'target_not_in_source': target_not_in_source,\n 'source_columns': source_columns,\n 'target_columns': target_columns,\n 'new_target_types': new_target_types\n } %}\n\n {% set msg %}\n In {{ target_relation }}:\n Schema changed: {{ schema_changed }}\n Source columns not in target: {{ source_not_in_target }}\n Target columns not in source: {{ target_not_in_source }}\n New column types: {{ new_target_types }}\n {% endset %}\n\n {% do log(msg) %}\n\n {{ return(changes_dict) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.diff_columns", "macro.dbt.diff_column_data_types"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.787961, "supported_languages": null}, "macro.dbt.sync_column_schemas": {"name": "sync_column_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.sync_column_schemas", "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n\n {% elif on_schema_change == 'sync_all_columns' %}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n\n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n\n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n\n {% do log(schema_change_message) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.alter_relation_add_remove_columns", "macro.dbt.alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.788921, "supported_languages": null}, "macro.dbt.process_schema_changes": {"name": "process_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.process_schema_changes", "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n\n {% if on_schema_change == 'ignore' %}\n\n {{ return({}) }}\n\n {% else %}\n\n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n\n {% if schema_changes_dict['schema_changed'] %}\n\n {% if on_schema_change == 'fail' %}\n\n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways:\n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n\n Additional troubleshooting context:\n Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}\n Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}\n New column types: {{ schema_changes_dict['new_target_types'] }}\n {% endset %}\n\n {% do exceptions.raise_compiler_error(fail_msg) %}\n\n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n\n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {% endif %}\n\n {% endif %}\n\n {{ return(schema_changes_dict['source_columns']) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.check_for_schema_changes", "macro.dbt.sync_column_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.789614, "supported_languages": null}, "macro.dbt.can_clone_table": {"name": "can_clone_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/can_clone_table.sql", "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", "unique_id": "macro.dbt.can_clone_table", "macro_sql": "{% macro can_clone_table() %}\n {{ return(adapter.dispatch('can_clone_table', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__can_clone_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.789815, "supported_languages": null}, "macro.dbt.default__can_clone_table": {"name": "default__can_clone_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/can_clone_table.sql", "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", "unique_id": "macro.dbt.default__can_clone_table", "macro_sql": "{% macro default__can_clone_table() %}\n {{ return(False) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7899132, "supported_languages": null}, "macro.dbt.create_or_replace_clone": {"name": "create_or_replace_clone", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/create_or_replace_clone.sql", "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", "unique_id": "macro.dbt.create_or_replace_clone", "macro_sql": "{% macro create_or_replace_clone(this_relation, defer_relation) %}\n {{ return(adapter.dispatch('create_or_replace_clone', 'dbt')(this_relation, defer_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_or_replace_clone"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.790163, "supported_languages": null}, "macro.dbt.default__create_or_replace_clone": {"name": "default__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/create_or_replace_clone.sql", "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", "unique_id": "macro.dbt.default__create_or_replace_clone", "macro_sql": "{% macro default__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace table {{ this_relation.render() }} clone {{ defer_relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.790299, "supported_languages": null}, "macro.dbt.materialization_clone_default": {"name": "materialization_clone_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/clone.sql", "original_file_path": "macros/materializations/models/clone/clone.sql", "unique_id": "macro.dbt.materialization_clone_default", "macro_sql": "{%- materialization clone, default -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation.render() ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% call statement('main') %}\n {% if target_relation and defer_relation and target_relation == defer_relation %}\n {{ log(\"Target relation and defer relation are the same, skipping clone for relation: \" ~ target_relation.render()) }}\n {% else %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endif %}\n\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n\n {%- endif -%}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.793341, "supported_languages": ["sql"]}, "macro.dbt.materialization_seed_default": {"name": "materialization_seed_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/seed.sql", "original_file_path": "macros/materializations/seeds/seed.sql", "unique_id": "macro.dbt.materialization_seed_default", "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparison later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation.render())) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.get_csv_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.796114, "supported_languages": ["sql"]}, "macro.dbt.create_csv_table": {"name": "create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.create_csv_table", "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.80035, "supported_languages": null}, "macro.dbt.default__create_csv_table": {"name": "default__create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__create_csv_table", "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.80112, "supported_languages": null}, "macro.dbt.reset_csv_table": {"name": "reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.reset_csv_table", "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__reset_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8013299, "supported_languages": null}, "macro.dbt.default__reset_csv_table": {"name": "default__reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__reset_csv_table", "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation.render() %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.801737, "supported_languages": null}, "macro.dbt.get_csv_sql": {"name": "get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_csv_sql", "macro_sql": "{% macro get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ adapter.dispatch('get_csv_sql', 'dbt')(create_or_truncate_sql, insert_sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_csv_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.801903, "supported_languages": null}, "macro.dbt.default__get_csv_sql": {"name": "default__get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_csv_sql", "macro_sql": "{% macro default__get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ create_or_truncate_sql }};\n -- dbt seed --\n {{ insert_sql }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.80202, "supported_languages": null}, "macro.dbt.get_binding_char": {"name": "get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_binding_char", "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8021429, "supported_languages": null}, "macro.dbt.default__get_binding_char": {"name": "default__get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_binding_char", "macro_sql": "{% macro default__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.802243, "supported_languages": null}, "macro.dbt.get_batch_size": {"name": "get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_batch_size", "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_batch_size"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8023849, "supported_languages": null}, "macro.dbt.default__get_batch_size": {"name": "default__get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_batch_size", "macro_sql": "{% macro default__get_batch_size() %}\n {{ return(10000) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.802489, "supported_languages": null}, "macro.dbt.get_seed_column_quoted_csv": {"name": "get_seed_column_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_seed_column_quoted_csv", "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8028958, "supported_languages": null}, "macro.dbt.load_csv_rows": {"name": "load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.load_csv_rows", "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__load_csv_rows"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.803055, "supported_languages": null}, "macro.dbt.default__load_csv_rows": {"name": "default__load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__load_csv_rows", "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_seed_column_quoted_csv", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8041549, "supported_languages": null}, "macro.dbt.generate_alias_name": {"name": "generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.generate_alias_name", "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_alias_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.804568, "supported_languages": null}, "macro.dbt.default__generate_alias_name": {"name": "default__generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.default__generate_alias_name", "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name -%}\n\n {{ custom_alias_name | trim }}\n\n {%- elif node.version -%}\n\n {{ return(node.name ~ \"_v\" ~ (node.version | replace(\".\", \"_\"))) }}\n\n {%- else -%}\n\n {{ node.name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.804885, "supported_languages": null}, "macro.dbt.generate_schema_name": {"name": "generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name", "macro_sql": "{% macro generate_schema_name(custom_schema_name=none, node=none) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8053389, "supported_languages": null}, "macro.dbt.default__generate_schema_name": {"name": "default__generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.default__generate_schema_name", "macro_sql": "{% macro default__generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8055522, "supported_languages": null}, "macro.dbt.generate_schema_name_for_env": {"name": "generate_schema_name_for_env", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name_for_env", "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8057861, "supported_languages": null}, "macro.dbt.generate_database_name": {"name": "generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.generate_database_name", "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__generate_database_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.806116, "supported_languages": null}, "macro.dbt.default__generate_database_name": {"name": "default__generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.default__generate_database_name", "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.806323, "supported_languages": null}, "macro.dbt.get_drop_sql": {"name": "get_drop_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.get_drop_sql", "macro_sql": "{%- macro get_drop_sql(relation) -%}\n {{- log('Applying DROP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.806918, "supported_languages": null}, "macro.dbt.default__get_drop_sql": {"name": "default__get_drop_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.default__get_drop_sql", "macro_sql": "{%- macro default__get_drop_sql(relation) -%}\n\n {%- if relation.is_view -%}\n {{ drop_view(relation) }}\n\n {%- elif relation.is_table -%}\n {{ drop_table(relation) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ drop_materialized_view(relation) }}\n\n {%- else -%}\n drop {{ relation.type }} if exists {{ relation.render() }} cascade\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.drop_view", "macro.dbt.drop_table", "macro.dbt.drop_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.807232, "supported_languages": null}, "macro.dbt.drop_relation": {"name": "drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8073862, "supported_languages": null}, "macro.dbt.default__drop_relation": {"name": "default__drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.default__drop_relation", "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n {{ get_drop_sql(relation) }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.807553, "supported_languages": null}, "macro.dbt.drop_relation_if_exists": {"name": "drop_relation_if_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.drop_relation_if_exists", "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.807711, "supported_languages": null}, "macro.dbt.get_replace_sql": {"name": "get_replace_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt.get_replace_sql", "macro_sql": "{% macro get_replace_sql(existing_relation, target_relation, sql) %}\n {{- log('Applying REPLACE to: ' ~ existing_relation) -}}\n {{- adapter.dispatch('get_replace_sql', 'dbt')(existing_relation, target_relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_replace_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.808418, "supported_languages": null}, "macro.dbt.default__get_replace_sql": {"name": "default__get_replace_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt.default__get_replace_sql", "macro_sql": "{% macro default__get_replace_sql(existing_relation, target_relation, sql) %}\n\n {# /* use a create or replace statement if possible */ #}\n\n {% set is_replaceable = existing_relation.type == target_relation_type and existing_relation.can_be_replaced %}\n\n {% if is_replaceable and existing_relation.is_view %}\n {{ get_replace_view_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_table %}\n {{ get_replace_table_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_materialized_view %}\n {{ get_replace_materialized_view_sql(target_relation, sql) }}\n\n {# /* a create or replace statement is not possible, so try to stage and/or backup to be safe */ #}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #}\n {%- elif target_relation.can_be_renamed and existing_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one without using a backup */ #}\n {%- elif target_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_drop_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }}\n\n {# /* create target_relation in place by first backing up the existing relation */ #}\n {%- elif existing_relation.can_be_renamed -%}\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* no renaming is allowed, so just drop and create */ #}\n {%- else -%}\n {{ get_drop_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_replace_view_sql", "macro.dbt.get_replace_table_sql", "macro.dbt.get_replace_materialized_view_sql", "macro.dbt.get_create_intermediate_sql", "macro.dbt.get_create_backup_sql", "macro.dbt.get_rename_intermediate_sql", "macro.dbt.get_drop_backup_sql", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.80947, "supported_languages": null}, "macro.dbt.get_create_intermediate_sql": {"name": "get_create_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_intermediate.sql", "original_file_path": "macros/relations/create_intermediate.sql", "unique_id": "macro.dbt.get_create_intermediate_sql", "macro_sql": "{%- macro get_create_intermediate_sql(relation, sql) -%}\n {{- log('Applying CREATE INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_intermediate_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_create_intermediate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.809802, "supported_languages": null}, "macro.dbt.default__get_create_intermediate_sql": {"name": "default__get_create_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_intermediate.sql", "original_file_path": "macros/relations/create_intermediate.sql", "unique_id": "macro.dbt.default__get_create_intermediate_sql", "macro_sql": "{%- macro default__get_create_intermediate_sql(relation, sql) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n -- drop any pre-existing intermediate\n {{ get_drop_sql(intermediate_relation) }};\n\n {{ get_create_sql(intermediate_relation, sql) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_intermediate_relation", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.810016, "supported_languages": null}, "macro.dbt.drop_schema_named": {"name": "drop_schema_named", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/schema.sql", "original_file_path": "macros/relations/schema.sql", "unique_id": "macro.dbt.drop_schema_named", "macro_sql": "{% macro drop_schema_named(schema_name) %}\n {{ return(adapter.dispatch('drop_schema_named', 'dbt') (schema_name)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__drop_schema_named"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.81025, "supported_languages": null}, "macro.dbt.default__drop_schema_named": {"name": "default__drop_schema_named", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/schema.sql", "original_file_path": "macros/relations/schema.sql", "unique_id": "macro.dbt.default__drop_schema_named", "macro_sql": "{% macro default__drop_schema_named(schema_name) %}\n {% set schema_relation = api.Relation.create(schema=schema_name) %}\n {{ adapter.drop_schema(schema_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.810434, "supported_languages": null}, "macro.dbt.get_drop_backup_sql": {"name": "get_drop_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop_backup.sql", "original_file_path": "macros/relations/drop_backup.sql", "unique_id": "macro.dbt.get_drop_backup_sql", "macro_sql": "{%- macro get_drop_backup_sql(relation) -%}\n {{- log('Applying DROP BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_drop_backup_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.81071, "supported_languages": null}, "macro.dbt.default__get_drop_backup_sql": {"name": "default__get_drop_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop_backup.sql", "original_file_path": "macros/relations/drop_backup.sql", "unique_id": "macro.dbt.default__get_drop_backup_sql", "macro_sql": "{%- macro default__get_drop_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n {{ get_drop_sql(backup_relation) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_backup_relation", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.810894, "supported_languages": null}, "macro.dbt.get_rename_sql": {"name": "get_rename_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.get_rename_sql", "macro_sql": "{%- macro get_rename_sql(relation, new_name) -%}\n {{- log('Applying RENAME to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_sql', 'dbt')(relation, new_name) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.811506, "supported_languages": null}, "macro.dbt.default__get_rename_sql": {"name": "default__get_rename_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.default__get_rename_sql", "macro_sql": "{%- macro default__get_rename_sql(relation, new_name) -%}\n\n {%- if relation.is_view -%}\n {{ get_rename_view_sql(relation, new_name) }}\n\n {%- elif relation.is_table -%}\n {{ get_rename_table_sql(relation, new_name) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_rename_materialized_view_sql(relation, new_name) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_rename_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.get_rename_view_sql", "macro.dbt.get_rename_table_sql", "macro.dbt.get_rename_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.811878, "supported_languages": null}, "macro.dbt.rename_relation": {"name": "rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.rename_relation", "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__rename_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.812062, "supported_languages": null}, "macro.dbt.default__rename_relation": {"name": "default__rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.default__rename_relation", "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation.render() }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.812324, "supported_languages": null}, "macro.dbt.get_create_backup_sql": {"name": "get_create_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_backup.sql", "original_file_path": "macros/relations/create_backup.sql", "unique_id": "macro.dbt.get_create_backup_sql", "macro_sql": "{%- macro get_create_backup_sql(relation) -%}\n {{- log('Applying CREATE BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_create_backup_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.812639, "supported_languages": null}, "macro.dbt.default__get_create_backup_sql": {"name": "default__get_create_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_backup.sql", "original_file_path": "macros/relations/create_backup.sql", "unique_id": "macro.dbt.default__get_create_backup_sql", "macro_sql": "{%- macro default__get_create_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n -- drop any pre-existing backup\n {{ get_drop_sql(backup_relation) }};\n\n {{ get_rename_sql(relation, backup_relation.identifier) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_backup_relation", "macro.dbt.get_drop_sql", "macro.dbt.get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.812882, "supported_languages": null}, "macro.dbt.get_create_sql": {"name": "get_create_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt.get_create_sql", "macro_sql": "{%- macro get_create_sql(relation, sql) -%}\n {{- log('Applying CREATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8133051, "supported_languages": null}, "macro.dbt.default__get_create_sql": {"name": "default__get_create_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt.default__get_create_sql", "macro_sql": "{%- macro default__get_create_sql(relation, sql) -%}\n\n {%- if relation.is_view -%}\n {{ get_create_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_table -%}\n {{ get_create_table_as_sql(False, relation, sql) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_create_materialized_view_as_sql(relation, sql) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_create_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.get_create_view_as_sql", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_create_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.813689, "supported_languages": null}, "macro.dbt.get_rename_intermediate_sql": {"name": "get_rename_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename_intermediate.sql", "original_file_path": "macros/relations/rename_intermediate.sql", "unique_id": "macro.dbt.get_rename_intermediate_sql", "macro_sql": "{%- macro get_rename_intermediate_sql(relation) -%}\n {{- log('Applying RENAME INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_intermediate_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_rename_intermediate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.813972, "supported_languages": null}, "macro.dbt.default__get_rename_intermediate_sql": {"name": "default__get_rename_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename_intermediate.sql", "original_file_path": "macros/relations/rename_intermediate.sql", "unique_id": "macro.dbt.default__get_rename_intermediate_sql", "macro_sql": "{%- macro default__get_rename_intermediate_sql(relation) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n {{ get_rename_sql(intermediate_relation, relation.identifier) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_intermediate_relation", "macro.dbt.get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8141482, "supported_languages": null}, "macro.dbt.drop_materialized_view": {"name": "drop_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt.drop_materialized_view", "macro_sql": "{% macro drop_materialized_view(relation) -%}\n {{- adapter.dispatch('drop_materialized_view', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8143559, "supported_languages": null}, "macro.dbt.default__drop_materialized_view": {"name": "default__drop_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt.default__drop_materialized_view", "macro_sql": "{% macro default__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.814456, "supported_languages": null}, "macro.dbt.get_replace_materialized_view_sql": {"name": "get_replace_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/replace.sql", "original_file_path": "macros/relations/materialized_view/replace.sql", "unique_id": "macro.dbt.get_replace_materialized_view_sql", "macro_sql": "{% macro get_replace_materialized_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_materialized_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_replace_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8146849, "supported_languages": null}, "macro.dbt.default__get_replace_materialized_view_sql": {"name": "default__get_replace_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/replace.sql", "original_file_path": "macros/relations/materialized_view/replace.sql", "unique_id": "macro.dbt.default__get_replace_materialized_view_sql", "macro_sql": "{% macro default__get_replace_materialized_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.814875, "supported_languages": null}, "macro.dbt.refresh_materialized_view": {"name": "refresh_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt.refresh_materialized_view", "macro_sql": "{% macro refresh_materialized_view(relation) %}\n {{- log('Applying REFRESH to: ' ~ relation) -}}\n {{- adapter.dispatch('refresh_materialized_view', 'dbt')(relation) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__refresh_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.815148, "supported_languages": null}, "macro.dbt.default__refresh_materialized_view": {"name": "default__refresh_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt.default__refresh_materialized_view", "macro_sql": "{% macro default__refresh_materialized_view(relation) %}\n {{ exceptions.raise_compiler_error(\"`refresh_materialized_view` has not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.815263, "supported_languages": null}, "macro.dbt.get_rename_materialized_view_sql": {"name": "get_rename_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/rename.sql", "original_file_path": "macros/relations/materialized_view/rename.sql", "unique_id": "macro.dbt.get_rename_materialized_view_sql", "macro_sql": "{% macro get_rename_materialized_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_materialized_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_rename_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8154979, "supported_languages": null}, "macro.dbt.default__get_rename_materialized_view_sql": {"name": "default__get_rename_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/rename.sql", "original_file_path": "macros/relations/materialized_view/rename.sql", "unique_id": "macro.dbt.default__get_rename_materialized_view_sql", "macro_sql": "{% macro default__get_rename_materialized_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8156219, "supported_languages": null}, "macro.dbt.get_alter_materialized_view_as_sql": {"name": "get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.get_alter_materialized_view_as_sql", "macro_sql": "{% macro get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {{- adapter.dispatch('get_alter_materialized_view_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n ) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8161569, "supported_languages": null}, "macro.dbt.default__get_alter_materialized_view_as_sql": {"name": "default__get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.default__get_alter_materialized_view_as_sql", "macro_sql": "{% macro default__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.816313, "supported_languages": null}, "macro.dbt.get_materialized_view_configuration_changes": {"name": "get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.get_materialized_view_configuration_changes", "macro_sql": "{% macro get_materialized_view_configuration_changes(existing_relation, new_config) %}\n /* {#\n It's recommended that configuration changes be formatted as follows:\n {\"\": [{\"action\": \"\", \"context\": ...}]}\n\n For example:\n {\n \"indexes\": [\n {\"action\": \"drop\", \"context\": \"index_abc\"},\n {\"action\": \"create\", \"context\": {\"columns\": [\"column_1\", \"column_2\"], \"type\": \"hash\", \"unique\": True}},\n ],\n }\n\n Either way, `get_materialized_view_configuration_changes` needs to align with `get_alter_materialized_view_as_sql`.\n #} */\n {{- log('Determining configuration changes on: ' ~ existing_relation) -}}\n {%- do return(adapter.dispatch('get_materialized_view_configuration_changes', 'dbt')(existing_relation, new_config)) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_materialized_view_configuration_changes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8165581, "supported_languages": null}, "macro.dbt.default__get_materialized_view_configuration_changes": {"name": "default__get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.default__get_materialized_view_configuration_changes", "macro_sql": "{% macro default__get_materialized_view_configuration_changes(existing_relation, new_config) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.816685, "supported_languages": null}, "macro.dbt.get_create_materialized_view_as_sql": {"name": "get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt.get_create_materialized_view_as_sql", "macro_sql": "{% macro get_create_materialized_view_as_sql(relation, sql) -%}\n {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.816923, "supported_languages": null}, "macro.dbt.default__get_create_materialized_view_as_sql": {"name": "default__get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt.default__get_create_materialized_view_as_sql", "macro_sql": "{% macro default__get_create_materialized_view_as_sql(relation, sql) -%}\n {{ exceptions.raise_compiler_error(\n \"`get_create_materialized_view_as_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.817044, "supported_languages": null}, "macro.dbt.get_table_columns_and_constraints": {"name": "get_table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_table_columns_and_constraints", "macro_sql": "{%- macro get_table_columns_and_constraints() -%}\n {{ adapter.dispatch('get_table_columns_and_constraints', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_table_columns_and_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.817939, "supported_languages": null}, "macro.dbt.default__get_table_columns_and_constraints": {"name": "default__get_table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_table_columns_and_constraints", "macro_sql": "{% macro default__get_table_columns_and_constraints() -%}\n {{ return(table_columns_and_constraints()) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.table_columns_and_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.818042, "supported_languages": null}, "macro.dbt.table_columns_and_constraints": {"name": "table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.table_columns_and_constraints", "macro_sql": "{% macro table_columns_and_constraints() %}\n {# loop through user_provided_columns to create DDL with data types and constraints #}\n {%- set raw_column_constraints = adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%}\n {%- set raw_model_constraints = adapter.render_raw_model_constraints(raw_constraints=model['constraints']) -%}\n (\n {% for c in raw_column_constraints -%}\n {{ c }}{{ \",\" if not loop.last or raw_model_constraints }}\n {% endfor %}\n {% for c in raw_model_constraints -%}\n {{ c }}{{ \",\" if not loop.last }}\n {% endfor -%}\n )\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8184981, "supported_languages": null}, "macro.dbt.get_assert_columns_equivalent": {"name": "get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_assert_columns_equivalent", "macro_sql": "\n\n{%- macro get_assert_columns_equivalent(sql) -%}\n {{ adapter.dispatch('get_assert_columns_equivalent', 'dbt')(sql) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.818639, "supported_languages": null}, "macro.dbt.default__get_assert_columns_equivalent": {"name": "default__get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_assert_columns_equivalent", "macro_sql": "{% macro default__get_assert_columns_equivalent(sql) -%}\n {{ return(assert_columns_equivalent(sql)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.818753, "supported_languages": null}, "macro.dbt.assert_columns_equivalent": {"name": "assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.assert_columns_equivalent", "macro_sql": "{% macro assert_columns_equivalent(sql) %}\n\n {#-- First ensure the user has defined 'columns' in yaml specification --#}\n {%- set user_defined_columns = model['columns'] -%}\n {%- if not user_defined_columns -%}\n {{ exceptions.raise_contract_error([], []) }}\n {%- endif -%}\n\n {#-- Obtain the column schema provided by sql file. #}\n {%- set sql_file_provided_columns = get_column_schema_from_query(sql, config.get('sql_header', none)) -%}\n {#--Obtain the column schema provided by the schema file by generating an 'empty schema' query from the model's columns. #}\n {%- set schema_file_provided_columns = get_column_schema_from_query(get_empty_schema_sql(user_defined_columns)) -%}\n\n {#-- create dictionaries with name and formatted data type and strings for exception #}\n {%- set sql_columns = format_columns(sql_file_provided_columns) -%}\n {%- set yaml_columns = format_columns(schema_file_provided_columns) -%}\n\n {%- if sql_columns|length != yaml_columns|length -%}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n\n {%- for sql_col in sql_columns -%}\n {%- set yaml_col = [] -%}\n {%- for this_col in yaml_columns -%}\n {%- if this_col['name'] == sql_col['name'] -%}\n {%- do yaml_col.append(this_col) -%}\n {%- break -%}\n {%- endif -%}\n {%- endfor -%}\n {%- if not yaml_col -%}\n {#-- Column with name not found in yaml #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- if sql_col['formatted'] != yaml_col[0]['formatted'] -%}\n {#-- Column data types don't match #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- endfor -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_column_schema_from_query", "macro.dbt.get_empty_schema_sql", "macro.dbt.format_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.819914, "supported_languages": null}, "macro.dbt.format_columns": {"name": "format_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.format_columns", "macro_sql": "{% macro format_columns(columns) %}\n {% set formatted_columns = [] %}\n {% for column in columns %}\n {%- set formatted_column = adapter.dispatch('format_column', 'dbt')(column) -%}\n {%- do formatted_columns.append(formatted_column) -%}\n {% endfor %}\n {{ return(formatted_columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__format_column"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.820245, "supported_languages": null}, "macro.dbt.default__format_column": {"name": "default__format_column", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__format_column", "macro_sql": "{% macro default__format_column(column) -%}\n {% set data_type = column.dtype %}\n {% set formatted = column.column.lower() ~ \" \" ~ data_type %}\n {{ return({'name': column.name, 'data_type': data_type, 'formatted': formatted}) }}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8205361, "supported_languages": null}, "macro.dbt.drop_table": {"name": "drop_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt.drop_table", "macro_sql": "{% macro drop_table(relation) -%}\n {{- adapter.dispatch('drop_table', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.820756, "supported_languages": null}, "macro.dbt.default__drop_table": {"name": "default__drop_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt.default__drop_table", "macro_sql": "{% macro default__drop_table(relation) -%}\n drop table if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82086, "supported_languages": null}, "macro.dbt.get_replace_table_sql": {"name": "get_replace_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt.get_replace_table_sql", "macro_sql": "{% macro get_replace_table_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_table_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_replace_table_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8210938, "supported_languages": null}, "macro.dbt.default__get_replace_table_sql": {"name": "default__get_replace_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt.default__get_replace_table_sql", "macro_sql": "{% macro default__get_replace_table_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8212202, "supported_languages": null}, "macro.dbt.get_rename_table_sql": {"name": "get_rename_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt.get_rename_table_sql", "macro_sql": "{% macro get_rename_table_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_table_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_rename_table_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82146, "supported_languages": null}, "macro.dbt.default__get_rename_table_sql": {"name": "default__get_rename_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt.default__get_rename_table_sql", "macro_sql": "{% macro default__get_rename_table_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82159, "supported_languages": null}, "macro.dbt.get_create_table_as_sql": {"name": "get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.get_create_table_as_sql", "macro_sql": "{% macro get_create_table_as_sql(temporary, relation, sql) -%}\n {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.822352, "supported_languages": null}, "macro.dbt.default__get_create_table_as_sql": {"name": "default__get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_create_table_as_sql", "macro_sql": "{% macro default__get_create_table_as_sql(temporary, relation, sql) -%}\n {{ return(create_table_as(temporary, relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8225112, "supported_languages": null}, "macro.dbt.create_table_as": {"name": "create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.create_table_as", "macro_sql": "{% macro create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {# backward compatibility for create_table_as that does not support language #}\n {% if language == \"sql\" %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code)}}\n {% else %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code, language) }}\n {% endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8228748, "supported_languages": null}, "macro.dbt.default__create_table_as": {"name": "default__create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__create_table_as", "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced and (not temporary) %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_table_columns_and_constraints() }}\n {%- set sql = get_select_subquery(sql) %}\n {% endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_table_columns_and_constraints", "macro.dbt.get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.823435, "supported_languages": null}, "macro.dbt.default__get_column_names": {"name": "default__get_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_column_names", "macro_sql": "{% macro default__get_column_names() %}\n {#- loop through user_provided_columns to get column names -#}\n {%- set user_provided_columns = model['columns'] -%}\n {%- for i in user_provided_columns %}\n {%- set col = user_provided_columns[i] -%}\n {%- set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] -%}\n {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8238149, "supported_languages": null}, "macro.dbt.get_select_subquery": {"name": "get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.get_select_subquery", "macro_sql": "{% macro get_select_subquery(sql) %}\n {{ return(adapter.dispatch('get_select_subquery', 'dbt')(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.823968, "supported_languages": null}, "macro.dbt.default__get_select_subquery": {"name": "default__get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_select_subquery", "macro_sql": "{% macro default__get_select_subquery(sql) %}\n select {{ adapter.dispatch('get_column_names', 'dbt')() }}\n from (\n {{ sql }}\n ) as model_subq\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.824113, "supported_languages": null}, "macro.dbt.drop_view": {"name": "drop_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt.drop_view", "macro_sql": "{% macro drop_view(relation) -%}\n {{- adapter.dispatch('drop_view', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.824321, "supported_languages": null}, "macro.dbt.default__drop_view": {"name": "default__drop_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt.default__drop_view", "macro_sql": "{% macro default__drop_view(relation) -%}\n drop view if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.824422, "supported_languages": null}, "macro.dbt.get_replace_view_sql": {"name": "get_replace_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.get_replace_view_sql", "macro_sql": "{% macro get_replace_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_replace_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.825161, "supported_languages": null}, "macro.dbt.default__get_replace_view_sql": {"name": "default__get_replace_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.default__get_replace_view_sql", "macro_sql": "{% macro default__get_replace_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82529, "supported_languages": null}, "macro.dbt.create_or_replace_view": {"name": "create_or_replace_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.create_or_replace_view", "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82631, "supported_languages": null}, "macro.dbt.handle_existing_table": {"name": "handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.handle_existing_table", "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__handle_existing_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8264818, "supported_languages": null}, "macro.dbt.default__handle_existing_table": {"name": "default__handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.default__handle_existing_table", "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation.render() ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8266811, "supported_languages": null}, "macro.dbt.get_rename_view_sql": {"name": "get_rename_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt.get_rename_view_sql", "macro_sql": "{% macro get_rename_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_rename_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.826918, "supported_languages": null}, "macro.dbt.default__get_rename_view_sql": {"name": "default__get_rename_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt.default__get_rename_view_sql", "macro_sql": "{% macro default__get_rename_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8270478, "supported_languages": null}, "macro.dbt.get_create_view_as_sql": {"name": "get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.get_create_view_as_sql", "macro_sql": "{% macro get_create_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8274088, "supported_languages": null}, "macro.dbt.default__get_create_view_as_sql": {"name": "default__get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.default__get_create_view_as_sql", "macro_sql": "{% macro default__get_create_view_as_sql(relation, sql) -%}\n {{ return(create_view_as(relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8275461, "supported_languages": null}, "macro.dbt.create_view_as": {"name": "create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.create_view_as", "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.827704, "supported_languages": null}, "macro.dbt.default__create_view_as": {"name": "default__create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.default__create_view_as", "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation.render() }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.828066, "supported_languages": null}, "macro.dbt.default__test_relationships": {"name": "default__test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/relationships.sql", "original_file_path": "macros/generic_test_sql/relationships.sql", "unique_id": "macro.dbt.default__test_relationships", "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.828371, "supported_languages": null}, "macro.dbt.default__test_not_null": {"name": "default__test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/not_null.sql", "original_file_path": "macros/generic_test_sql/not_null.sql", "unique_id": "macro.dbt.default__test_not_null", "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\n{% set column_list = '*' if should_store_failures() else column_name %}\n\nselect {{ column_list }}\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_store_failures"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.828622, "supported_languages": null}, "macro.dbt.default__test_unique": {"name": "default__test_unique", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/unique.sql", "original_file_path": "macros/generic_test_sql/unique.sql", "unique_id": "macro.dbt.default__test_unique", "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.828834, "supported_languages": null}, "macro.dbt.default__test_accepted_values": {"name": "default__test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/accepted_values.sql", "original_file_path": "macros/generic_test_sql/accepted_values.sql", "unique_id": "macro.dbt.default__test_accepted_values", "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.829317, "supported_languages": null}, "macro.dbt.statement": {"name": "statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.statement", "macro_sql": "\n{%- macro statement(name=None, fetch_result=False, auto_begin=True, language='sql') -%}\n {%- if execute: -%}\n {%- set compiled_code = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime {} for node \"{}\"'.format(language, model['unique_id'])) }}\n {{ write(compiled_code) }}\n {%- endif -%}\n {%- if language == 'sql'-%}\n {%- set res, table = adapter.execute(compiled_code, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- elif language == 'python' -%}\n {%- set res = submit_python_job(model, compiled_code) -%}\n {#-- TODO: What should table be for python models? --#}\n {%- set table = None -%}\n {%- else -%}\n {% do exceptions.raise_compiler_error(\"statement macro didn't get supported language\") %}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.83059, "supported_languages": null}, "macro.dbt.noop_statement": {"name": "noop_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.noop_statement", "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8313391, "supported_languages": null}, "macro.dbt.run_query": {"name": "run_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.run_query", "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8315961, "supported_languages": null}, "macro.dbt.convert_datetime": {"name": "convert_datetime", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.convert_datetime", "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.833154, "supported_languages": null}, "macro.dbt.dates_in_range": {"name": "dates_in_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.dates_in_range", "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.convert_datetime"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.834102, "supported_languages": null}, "macro.dbt.partition_range": {"name": "partition_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.partition_range", "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.dates_in_range"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8347142, "supported_languages": null}, "macro.dbt.py_current_timestring": {"name": "py_current_timestring", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.py_current_timestring", "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8349109, "supported_languages": null}, "macro.dbt.except": {"name": "except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.except", "macro_sql": "{% macro except() %}\n {{ return(adapter.dispatch('except', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__except"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.835105, "supported_languages": null}, "macro.dbt.default__except": {"name": "default__except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.default__except", "macro_sql": "{% macro default__except() %}\n\n except\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.835169, "supported_languages": null}, "macro.dbt.get_intervals_between": {"name": "get_intervals_between", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.get_intervals_between", "macro_sql": "{% macro get_intervals_between(start_date, end_date, datepart) -%}\n {{ return(adapter.dispatch('get_intervals_between', 'dbt')(start_date, end_date, datepart)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_intervals_between"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.835808, "supported_languages": null}, "macro.dbt.default__get_intervals_between": {"name": "default__get_intervals_between", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.default__get_intervals_between", "macro_sql": "{% macro default__get_intervals_between(start_date, end_date, datepart) -%}\n {%- call statement('get_intervals_between', fetch_result=True) %}\n\n select {{ dbt.datediff(start_date, end_date, datepart) }}\n\n {%- endcall -%}\n\n {%- set value_list = load_result('get_intervals_between') -%}\n\n {%- if value_list and value_list['data'] -%}\n {%- set values = value_list['data'] | map(attribute=0) | list %}\n {{ return(values[0]) }}\n {%- else -%}\n {{ return(1) }}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8363059, "supported_languages": null}, "macro.dbt.date_spine": {"name": "date_spine", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.date_spine", "macro_sql": "{% macro date_spine(datepart, start_date, end_date) %}\n {{ return(adapter.dispatch('date_spine', 'dbt')(datepart, start_date, end_date)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date_spine"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.836503, "supported_languages": null}, "macro.dbt.default__date_spine": {"name": "default__date_spine", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.default__date_spine", "macro_sql": "{% macro default__date_spine(datepart, start_date, end_date) %}\n\n\n {# call as follows:\n\n date_spine(\n \"day\",\n \"to_date('01/01/2016', 'mm/dd/yyyy')\",\n \"dbt.dateadd(week, 1, current_date)\"\n ) #}\n\n\n with rawdata as (\n\n {{dbt.generate_series(\n dbt.get_intervals_between(start_date, end_date, datepart)\n )}}\n\n ),\n\n all_periods as (\n\n select (\n {{\n dbt.dateadd(\n datepart,\n \"row_number() over (order by 1) - 1\",\n start_date\n )\n }}\n ) as date_{{datepart}}\n from rawdata\n\n ),\n\n filtered as (\n\n select *\n from all_periods\n where date_{{datepart}} <= {{ end_date }}\n\n )\n\n select * from filtered\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.generate_series", "macro.dbt.get_intervals_between", "macro.dbt.dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.836822, "supported_languages": null}, "macro.dbt.date": {"name": "date", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date.sql", "original_file_path": "macros/utils/date.sql", "unique_id": "macro.dbt.date", "macro_sql": "{% macro date(year, month, day) %}\n {{ return(adapter.dispatch('date', 'dbt') (year, month, day)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837112, "supported_languages": null}, "macro.dbt.default__date": {"name": "default__date", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date.sql", "original_file_path": "macros/utils/date.sql", "unique_id": "macro.dbt.default__date", "macro_sql": "{% macro default__date(year, month, day) -%}\n {%- set dt = modules.datetime.date(year, month, day) -%}\n {%- set iso_8601_formatted_date = dt.strftime('%Y-%m-%d') -%}\n to_date('{{ iso_8601_formatted_date }}', 'YYYY-MM-DD')\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837353, "supported_languages": null}, "macro.dbt.replace": {"name": "replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.replace", "macro_sql": "{% macro replace(field, old_chars, new_chars) -%}\n {{ return(adapter.dispatch('replace', 'dbt') (field, old_chars, new_chars)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__replace"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837631, "supported_languages": null}, "macro.dbt.default__replace": {"name": "default__replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.default__replace", "macro_sql": "{% macro default__replace(field, old_chars, new_chars) %}\n\n replace(\n {{ field }},\n {{ old_chars }},\n {{ new_chars }}\n )\n\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837771, "supported_languages": null}, "macro.dbt.concat": {"name": "concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.concat", "macro_sql": "{% macro concat(fields) -%}\n {{ return(adapter.dispatch('concat', 'dbt')(fields)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837983, "supported_languages": null}, "macro.dbt.default__concat": {"name": "default__concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.default__concat", "macro_sql": "{% macro default__concat(fields) -%}\n {{ fields|join(' || ') }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.838089, "supported_languages": null}, "macro.dbt.get_powers_of_two": {"name": "get_powers_of_two", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.get_powers_of_two", "macro_sql": "{% macro get_powers_of_two(upper_bound) %}\n {{ return(adapter.dispatch('get_powers_of_two', 'dbt')(upper_bound)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_powers_of_two"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.838809, "supported_languages": null}, "macro.dbt.default__get_powers_of_two": {"name": "default__get_powers_of_two", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.default__get_powers_of_two", "macro_sql": "{% macro default__get_powers_of_two(upper_bound) %}\n\n {% if upper_bound <= 0 %}\n {{ exceptions.raise_compiler_error(\"upper bound must be positive\") }}\n {% endif %}\n\n {% for _ in range(1, 100) %}\n {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %}\n {% endfor %}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.839253, "supported_languages": null}, "macro.dbt.generate_series": {"name": "generate_series", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.generate_series", "macro_sql": "{% macro generate_series(upper_bound) %}\n {{ return(adapter.dispatch('generate_series', 'dbt')(upper_bound)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_series"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.839426, "supported_languages": null}, "macro.dbt.default__generate_series": {"name": "default__generate_series", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.default__generate_series", "macro_sql": "{% macro default__generate_series(upper_bound) %}\n\n {% set n = dbt.get_powers_of_two(upper_bound) %}\n\n with p as (\n select 0 as generated_number union all select 1\n ), unioned as (\n\n select\n\n {% for i in range(n) %}\n p{{i}}.generated_number * power(2, {{i}})\n {% if not loop.last %} + {% endif %}\n {% endfor %}\n + 1\n as generated_number\n\n from\n\n {% for i in range(n) %}\n p as p{{i}}\n {% if not loop.last %} cross join {% endif %}\n {% endfor %}\n\n )\n\n select *\n from unioned\n where generated_number <= {{upper_bound}}\n order by generated_number\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_powers_of_two"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.839862, "supported_languages": null}, "macro.dbt.length": {"name": "length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.length", "macro_sql": "{% macro length(expression) -%}\n {{ return(adapter.dispatch('length', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__length"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.840083, "supported_languages": null}, "macro.dbt.default__length": {"name": "default__length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.default__length", "macro_sql": "{% macro default__length(expression) %}\n\n length(\n {{ expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8401742, "supported_languages": null}, "macro.dbt.dateadd": {"name": "dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.dateadd", "macro_sql": "{% macro dateadd(datepart, interval, from_date_or_timestamp) %}\n {{ return(adapter.dispatch('dateadd', 'dbt')(datepart, interval, from_date_or_timestamp)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8404648, "supported_languages": null}, "macro.dbt.default__dateadd": {"name": "default__dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.default__dateadd", "macro_sql": "{% macro default__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n dateadd(\n {{ datepart }},\n {{ interval }},\n {{ from_date_or_timestamp }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.840607, "supported_languages": null}, "macro.dbt.intersect": {"name": "intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.intersect", "macro_sql": "{% macro intersect() %}\n {{ return(adapter.dispatch('intersect', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__intersect"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.840808, "supported_languages": null}, "macro.dbt.default__intersect": {"name": "default__intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.default__intersect", "macro_sql": "{% macro default__intersect() %}\n\n intersect\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.840875, "supported_languages": null}, "macro.dbt.escape_single_quotes": {"name": "escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.escape_single_quotes", "macro_sql": "{% macro escape_single_quotes(expression) %}\n {{ return(adapter.dispatch('escape_single_quotes', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__escape_single_quotes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.841106, "supported_languages": null}, "macro.dbt.default__escape_single_quotes": {"name": "default__escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.default__escape_single_quotes", "macro_sql": "{% macro default__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"''\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.841234, "supported_languages": null}, "macro.dbt.right": {"name": "right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.right", "macro_sql": "{% macro right(string_text, length_expression) -%}\n {{ return(adapter.dispatch('right', 'dbt') (string_text, length_expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__right"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.84149, "supported_languages": null}, "macro.dbt.default__right": {"name": "default__right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.default__right", "macro_sql": "{% macro default__right(string_text, length_expression) %}\n\n right(\n {{ string_text }},\n {{ length_expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8416152, "supported_languages": null}, "macro.dbt.listagg": {"name": "listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.listagg", "macro_sql": "{% macro listagg(measure, delimiter_text=\"','\", order_by_clause=none, limit_num=none) -%}\n {{ return(adapter.dispatch('listagg', 'dbt') (measure, delimiter_text, order_by_clause, limit_num)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__listagg"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.842151, "supported_languages": null}, "macro.dbt.default__listagg": {"name": "default__listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.default__listagg", "macro_sql": "{% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n array_slice(\n array_agg(\n {{ measure }}\n ){% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n ,0\n ,{{ limit_num }}\n ),\n {{ delimiter_text }}\n )\n {%- else %}\n listagg(\n {{ measure }},\n {{ delimiter_text }}\n )\n {% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.842484, "supported_languages": null}, "macro.dbt.datediff": {"name": "datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.datediff", "macro_sql": "{% macro datediff(first_date, second_date, datepart) %}\n {{ return(adapter.dispatch('datediff', 'dbt')(first_date, second_date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.842758, "supported_languages": null}, "macro.dbt.default__datediff": {"name": "default__datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.default__datediff", "macro_sql": "{% macro default__datediff(first_date, second_date, datepart) -%}\n\n datediff(\n {{ datepart }},\n {{ first_date }},\n {{ second_date }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.842896, "supported_languages": null}, "macro.dbt.safe_cast": {"name": "safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.safe_cast", "macro_sql": "{% macro safe_cast(field, type) %}\n {{ return(adapter.dispatch('safe_cast', 'dbt') (field, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__safe_cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.843133, "supported_languages": null}, "macro.dbt.default__safe_cast": {"name": "default__safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.default__safe_cast", "macro_sql": "{% macro default__safe_cast(field, type) %}\n {# most databases don't support this function yet\n so we just need to use cast #}\n cast({{field}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.843246, "supported_languages": null}, "macro.dbt.hash": {"name": "hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.hash", "macro_sql": "{% macro hash(field) -%}\n {{ return(adapter.dispatch('hash', 'dbt') (field)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__hash"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.843461, "supported_languages": null}, "macro.dbt.default__hash": {"name": "default__hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.default__hash", "macro_sql": "{% macro default__hash(field) -%}\n md5(cast({{ field }} as {{ api.Column.translate_type('string') }}))\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.843594, "supported_languages": null}, "macro.dbt.cast_bool_to_text": {"name": "cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.cast_bool_to_text", "macro_sql": "{% macro cast_bool_to_text(field) %}\n {{ adapter.dispatch('cast_bool_to_text', 'dbt') (field) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__cast_bool_to_text"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8438652, "supported_languages": null}, "macro.dbt.default__cast_bool_to_text": {"name": "default__cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.default__cast_bool_to_text", "macro_sql": "{% macro default__cast_bool_to_text(field) %}\n cast({{ field }} as {{ api.Column.translate_type('string') }})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8439958, "supported_languages": null}, "macro.dbt.cast": {"name": "cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast.sql", "original_file_path": "macros/utils/cast.sql", "unique_id": "macro.dbt.cast", "macro_sql": "{% macro cast(field, type) %}\n {{ return(adapter.dispatch('cast', 'dbt') (field, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.844224, "supported_languages": null}, "macro.dbt.default__cast": {"name": "default__cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast.sql", "original_file_path": "macros/utils/cast.sql", "unique_id": "macro.dbt.default__cast", "macro_sql": "{% macro default__cast(field, type) %}\n cast({{field}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.844336, "supported_languages": null}, "macro.dbt.any_value": {"name": "any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.any_value", "macro_sql": "{% macro any_value(expression) -%}\n {{ return(adapter.dispatch('any_value', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__any_value"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8445451, "supported_languages": null}, "macro.dbt.default__any_value": {"name": "default__any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.default__any_value", "macro_sql": "{% macro default__any_value(expression) -%}\n\n any_value({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.844633, "supported_languages": null}, "macro.dbt.position": {"name": "position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.position", "macro_sql": "{% macro position(substring_text, string_text) -%}\n {{ return(adapter.dispatch('position', 'dbt') (substring_text, string_text)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__position"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8448782, "supported_languages": null}, "macro.dbt.default__position": {"name": "default__position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.default__position", "macro_sql": "{% macro default__position(substring_text, string_text) %}\n\n position(\n {{ substring_text }} in {{ string_text }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.844993, "supported_languages": null}, "macro.dbt.string_literal": {"name": "string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.string_literal", "macro_sql": "{%- macro string_literal(value) -%}\n {{ return(adapter.dispatch('string_literal', 'dbt') (value)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__string_literal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.845202, "supported_languages": null}, "macro.dbt.default__string_literal": {"name": "default__string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.default__string_literal", "macro_sql": "{% macro default__string_literal(value) -%}\n '{{ value }}'\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.845294, "supported_languages": null}, "macro.dbt.type_string": {"name": "type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_string", "macro_sql": "\n\n{%- macro type_string() -%}\n {{ return(adapter.dispatch('type_string', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_string"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846126, "supported_languages": null}, "macro.dbt.default__type_string": {"name": "default__type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_string", "macro_sql": "{% macro default__type_string() %}\n {{ return(api.Column.translate_type(\"string\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846256, "supported_languages": null}, "macro.dbt.type_timestamp": {"name": "type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_timestamp", "macro_sql": "\n\n{%- macro type_timestamp() -%}\n {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846394, "supported_languages": null}, "macro.dbt.default__type_timestamp": {"name": "default__type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_timestamp", "macro_sql": "{% macro default__type_timestamp() %}\n {{ return(api.Column.translate_type(\"timestamp\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8465211, "supported_languages": null}, "macro.dbt.type_float": {"name": "type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_float", "macro_sql": "\n\n{%- macro type_float() -%}\n {{ return(adapter.dispatch('type_float', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_float"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846658, "supported_languages": null}, "macro.dbt.default__type_float": {"name": "default__type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_float", "macro_sql": "{% macro default__type_float() %}\n {{ return(api.Column.translate_type(\"float\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8467891, "supported_languages": null}, "macro.dbt.type_numeric": {"name": "type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_numeric", "macro_sql": "\n\n{%- macro type_numeric() -%}\n {{ return(adapter.dispatch('type_numeric', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_numeric"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846924, "supported_languages": null}, "macro.dbt.default__type_numeric": {"name": "default__type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_numeric", "macro_sql": "{% macro default__type_numeric() %}\n {{ return(api.Column.numeric_type(\"numeric\", 28, 6)) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8470762, "supported_languages": null}, "macro.dbt.type_bigint": {"name": "type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_bigint", "macro_sql": "\n\n{%- macro type_bigint() -%}\n {{ return(adapter.dispatch('type_bigint', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_bigint"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.847214, "supported_languages": null}, "macro.dbt.default__type_bigint": {"name": "default__type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_bigint", "macro_sql": "{% macro default__type_bigint() %}\n {{ return(api.Column.translate_type(\"bigint\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.847407, "supported_languages": null}, "macro.dbt.type_int": {"name": "type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_int", "macro_sql": "\n\n{%- macro type_int() -%}\n {{ return(adapter.dispatch('type_int', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_int"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8475451, "supported_languages": null}, "macro.dbt.default__type_int": {"name": "default__type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_int", "macro_sql": "{%- macro default__type_int() -%}\n {{ return(api.Column.translate_type(\"integer\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8476732, "supported_languages": null}, "macro.dbt.type_boolean": {"name": "type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_boolean", "macro_sql": "\n\n{%- macro type_boolean() -%}\n {{ return(adapter.dispatch('type_boolean', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_boolean"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8478148, "supported_languages": null}, "macro.dbt.default__type_boolean": {"name": "default__type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_boolean", "macro_sql": "{%- macro default__type_boolean() -%}\n {{ return(api.Column.translate_type(\"boolean\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8479412, "supported_languages": null}, "macro.dbt.array_concat": {"name": "array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.array_concat", "macro_sql": "{% macro array_concat(array_1, array_2) -%}\n {{ return(adapter.dispatch('array_concat', 'dbt')(array_1, array_2)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__array_concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.848181, "supported_languages": null}, "macro.dbt.default__array_concat": {"name": "default__array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.default__array_concat", "macro_sql": "{% macro default__array_concat(array_1, array_2) -%}\n array_cat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.848293, "supported_languages": null}, "macro.dbt.bool_or": {"name": "bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.bool_or", "macro_sql": "{% macro bool_or(expression) -%}\n {{ return(adapter.dispatch('bool_or', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__bool_or"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8485062, "supported_languages": null}, "macro.dbt.default__bool_or": {"name": "default__bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.default__bool_or", "macro_sql": "{% macro default__bool_or(expression) -%}\n\n bool_or({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8485959, "supported_languages": null}, "macro.dbt.last_day": {"name": "last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.last_day", "macro_sql": "{% macro last_day(date, datepart) %}\n {{ return(adapter.dispatch('last_day', 'dbt') (date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.848897, "supported_languages": null}, "macro.dbt.default_last_day": {"name": "default_last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default_last_day", "macro_sql": "\n\n{%- macro default_last_day(date, datepart) -%}\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd(datepart, '1', dbt.date_trunc(datepart, date))\n )}}\n as date)\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.dateadd", "macro.dbt.date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.84913, "supported_languages": null}, "macro.dbt.default__last_day": {"name": "default__last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default__last_day", "macro_sql": "{% macro default__last_day(date, datepart) -%}\n {{dbt.default_last_day(date, datepart)}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default_last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.849261, "supported_languages": null}, "macro.dbt.split_part": {"name": "split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.split_part", "macro_sql": "{% macro split_part(string_text, delimiter_text, part_number) %}\n {{ return(adapter.dispatch('split_part', 'dbt') (string_text, delimiter_text, part_number)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__split_part"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8497112, "supported_languages": null}, "macro.dbt.default__split_part": {"name": "default__split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.default__split_part", "macro_sql": "{% macro default__split_part(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8498502, "supported_languages": null}, "macro.dbt._split_part_negative": {"name": "_split_part_negative", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt._split_part_negative", "macro_sql": "{% macro _split_part_negative(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 2 + {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8500328, "supported_languages": null}, "macro.dbt.date_trunc": {"name": "date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.date_trunc", "macro_sql": "{% macro date_trunc(datepart, date) -%}\n {{ return(adapter.dispatch('date_trunc', 'dbt') (datepart, date)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.850267, "supported_languages": null}, "macro.dbt.default__date_trunc": {"name": "default__date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.default__date_trunc", "macro_sql": "{% macro default__date_trunc(datepart, date) -%}\n date_trunc('{{datepart}}', {{date}})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.85038, "supported_languages": null}, "macro.dbt.array_construct": {"name": "array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.array_construct", "macro_sql": "{% macro array_construct(inputs=[], data_type=api.Column.translate_type('integer')) -%}\n {{ return(adapter.dispatch('array_construct', 'dbt')(inputs, data_type)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__array_construct"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.850707, "supported_languages": null}, "macro.dbt.default__array_construct": {"name": "default__array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.default__array_construct", "macro_sql": "{% macro default__array_construct(inputs, data_type) -%}\n {% if inputs|length > 0 %}\n array[ {{ inputs|join(' , ') }} ]\n {% else %}\n array[]::{{data_type}}[]\n {% endif %}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.850914, "supported_languages": null}, "macro.dbt.array_append": {"name": "array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.array_append", "macro_sql": "{% macro array_append(array, new_element) -%}\n {{ return(adapter.dispatch('array_append', 'dbt')(array, new_element)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__array_append"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8511622, "supported_languages": null}, "macro.dbt.default__array_append": {"name": "default__array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.default__array_append", "macro_sql": "{% macro default__array_append(array, new_element) -%}\n array_append({{ array }}, {{ new_element }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.851361, "supported_languages": null}, "macro.dbt.create_schema": {"name": "create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.create_schema", "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__create_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.851688, "supported_languages": null}, "macro.dbt.default__create_schema": {"name": "default__create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__create_schema", "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.851847, "supported_languages": null}, "macro.dbt.drop_schema": {"name": "drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.drop_schema", "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__drop_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.851985, "supported_languages": null}, "macro.dbt.default__drop_schema": {"name": "default__drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__drop_schema", "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.852136, "supported_languages": null}, "macro.dbt.current_timestamp": {"name": "current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp", "macro_sql": "{%- macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.852708, "supported_languages": null}, "macro.dbt.default__current_timestamp": {"name": "default__current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp", "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter ' + adapter.type()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.852832, "supported_languages": null}, "macro.dbt.snapshot_get_time": {"name": "snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.snapshot_get_time", "macro_sql": "\n\n{%- macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.852951, "supported_languages": null}, "macro.dbt.default__snapshot_get_time": {"name": "default__snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__snapshot_get_time", "macro_sql": "{% macro default__snapshot_get_time() %}\n {{ current_timestamp() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853037, "supported_languages": null}, "macro.dbt.get_snapshot_get_time_data_type": {"name": "get_snapshot_get_time_data_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.get_snapshot_get_time_data_type", "macro_sql": "{% macro get_snapshot_get_time_data_type() %}\n {% set snapshot_time = adapter.dispatch('snapshot_get_time', 'dbt')() %}\n {% set time_data_type_sql = 'select ' ~ snapshot_time ~ ' as dbt_snapshot_time' %}\n {% set snapshot_time_column_schema = get_column_schema_from_query(time_data_type_sql) %}\n {% set time_data_type = snapshot_time_column_schema[0].dtype %}\n {{ return(time_data_type or none) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.default__snapshot_get_time", "macro.dbt.get_column_schema_from_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8533702, "supported_languages": null}, "macro.dbt.current_timestamp_backcompat": {"name": "current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_backcompat", "macro_sql": "{% macro current_timestamp_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853508, "supported_languages": null}, "macro.dbt.default__current_timestamp_backcompat": {"name": "default__current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_backcompat", "macro_sql": "{% macro default__current_timestamp_backcompat() %}\n current_timestamp::timestamp\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853577, "supported_languages": null}, "macro.dbt.current_timestamp_in_utc_backcompat": {"name": "current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_in_utc_backcompat", "macro_sql": "{% macro current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_in_utc_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__current_timestamp_in_utc_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853718, "supported_languages": null}, "macro.dbt.default__current_timestamp_in_utc_backcompat": {"name": "default__current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_in_utc_backcompat", "macro_sql": "{% macro default__current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp_backcompat", "macro.dbt.default__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853857, "supported_languages": null}, "macro.dbt.get_create_index_sql": {"name": "get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_create_index_sql", "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.854583, "supported_languages": null}, "macro.dbt.default__get_create_index_sql": {"name": "default__get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_create_index_sql", "macro_sql": "{% macro default__get_create_index_sql(relation, index_dict) -%}\n {% do return(None) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.854707, "supported_languages": null}, "macro.dbt.create_indexes": {"name": "create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.create_indexes", "macro_sql": "{% macro create_indexes(relation) -%}\n {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8548422, "supported_languages": null}, "macro.dbt.default__create_indexes": {"name": "default__create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__create_indexes", "macro_sql": "{% macro default__create_indexes(relation) -%}\n {%- set _indexes = config.get('indexes', default=[]) -%}\n\n {% for _index_dict in _indexes %}\n {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}\n {% if create_index_sql %}\n {% do run_query(create_index_sql) %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_create_index_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8552508, "supported_languages": null}, "macro.dbt.get_drop_index_sql": {"name": "get_drop_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_drop_index_sql", "macro_sql": "{% macro get_drop_index_sql(relation, index_name) -%}\n {{ adapter.dispatch('get_drop_index_sql', 'dbt')(relation, index_name) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_drop_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.855412, "supported_languages": null}, "macro.dbt.default__get_drop_index_sql": {"name": "default__get_drop_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_drop_index_sql", "macro_sql": "{% macro default__get_drop_index_sql(relation, index_name) -%}\n {{ exceptions.raise_compiler_error(\"`get_drop_index_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.855531, "supported_languages": null}, "macro.dbt.get_show_indexes_sql": {"name": "get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_show_indexes_sql", "macro_sql": "{% macro get_show_indexes_sql(relation) -%}\n {{ adapter.dispatch('get_show_indexes_sql', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_show_indexes_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8556669, "supported_languages": null}, "macro.dbt.default__get_show_indexes_sql": {"name": "default__get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_show_indexes_sql", "macro_sql": "{% macro default__get_show_indexes_sql(relation) -%}\n {{ exceptions.raise_compiler_error(\"`get_show_indexes_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.855777, "supported_languages": null}, "macro.dbt.make_intermediate_relation": {"name": "make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_intermediate_relation", "macro_sql": "{% macro make_intermediate_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_intermediate_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_intermediate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.857485, "supported_languages": null}, "macro.dbt.default__make_intermediate_relation": {"name": "default__make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_intermediate_relation", "macro_sql": "{% macro default__make_intermediate_relation(base_relation, suffix) %}\n {{ return(default__make_temp_relation(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.857631, "supported_languages": null}, "macro.dbt.make_temp_relation": {"name": "make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_temp_relation", "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.857822, "supported_languages": null}, "macro.dbt.default__make_temp_relation": {"name": "default__make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_temp_relation", "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {%- set temp_identifier = base_relation.identifier ~ suffix -%}\n {%- set temp_relation = base_relation.incorporate(\n path={\"identifier\": temp_identifier}) -%}\n\n {{ return(temp_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.858082, "supported_languages": null}, "macro.dbt.make_backup_relation": {"name": "make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_backup_relation", "macro_sql": "{% macro make_backup_relation(base_relation, backup_relation_type, suffix='__dbt_backup') %}\n {{ return(adapter.dispatch('make_backup_relation', 'dbt')(base_relation, backup_relation_type, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_backup_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.858318, "supported_languages": null}, "macro.dbt.default__make_backup_relation": {"name": "default__make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_backup_relation", "macro_sql": "{% macro default__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {%- set backup_identifier = base_relation.identifier ~ suffix -%}\n {%- set backup_relation = base_relation.incorporate(\n path={\"identifier\": backup_identifier},\n type=backup_relation_type\n ) -%}\n {{ return(backup_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8586, "supported_languages": null}, "macro.dbt.truncate_relation": {"name": "truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.truncate_relation", "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__truncate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.858756, "supported_languages": null}, "macro.dbt.default__truncate_relation": {"name": "default__truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__truncate_relation", "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation.render() }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.858906, "supported_languages": null}, "macro.dbt.get_or_create_relation": {"name": "get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.get_or_create_relation", "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) -%}\n {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_or_create_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.859114, "supported_languages": null}, "macro.dbt.default__get_or_create_relation": {"name": "default__get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__get_or_create_relation", "macro_sql": "{% macro default__get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.859583, "supported_languages": null}, "macro.dbt.load_cached_relation": {"name": "load_cached_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_cached_relation", "macro_sql": "{% macro load_cached_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.859786, "supported_languages": null}, "macro.dbt.load_relation": {"name": "load_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {{ return(load_cached_relation(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.859914, "supported_languages": null}, "macro.dbt.collect_freshness": {"name": "collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.collect_freshness", "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__collect_freshness"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.860286, "supported_languages": null}, "macro.dbt.default__collect_freshness": {"name": "default__collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.default__collect_freshness", "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86064, "supported_languages": null}, "macro.dbt.validate_sql": {"name": "validate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/validate_sql.sql", "original_file_path": "macros/adapters/validate_sql.sql", "unique_id": "macro.dbt.validate_sql", "macro_sql": "{% macro validate_sql(sql) -%}\n {{ return(adapter.dispatch('validate_sql', 'dbt')(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__validate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8608792, "supported_languages": null}, "macro.dbt.default__validate_sql": {"name": "default__validate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/validate_sql.sql", "original_file_path": "macros/adapters/validate_sql.sql", "unique_id": "macro.dbt.default__validate_sql", "macro_sql": "{% macro default__validate_sql(sql) -%}\n {% call statement('validate_sql') -%}\n explain {{ sql }}\n {% endcall %}\n {{ return(load_result('validate_sql')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8611462, "supported_languages": null}, "macro.dbt.copy_grants": {"name": "copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.copy_grants", "macro_sql": "{% macro copy_grants() %}\n {{ return(adapter.dispatch('copy_grants', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.862612, "supported_languages": null}, "macro.dbt.default__copy_grants": {"name": "default__copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__copy_grants", "macro_sql": "{% macro default__copy_grants() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.862713, "supported_languages": null}, "macro.dbt.support_multiple_grantees_per_dcl_statement": {"name": "support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.support_multiple_grantees_per_dcl_statement", "macro_sql": "{% macro support_multiple_grantees_per_dcl_statement() %}\n {{ return(adapter.dispatch('support_multiple_grantees_per_dcl_statement', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8628511, "supported_languages": null}, "macro.dbt.default__support_multiple_grantees_per_dcl_statement": {"name": "default__support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__support_multiple_grantees_per_dcl_statement", "macro_sql": "\n\n{%- macro default__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(True) }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.862943, "supported_languages": null}, "macro.dbt.should_revoke": {"name": "should_revoke", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.should_revoke", "macro_sql": "{% macro should_revoke(existing_relation, full_refresh_mode=True) %}\n\n {% if not existing_relation %}\n {#-- The table doesn't already exist, so no grants to copy over --#}\n {{ return(False) }}\n {% elif full_refresh_mode %}\n {#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#}\n {{ return(copy_grants()) }}\n {% else %}\n {#-- The table is being merged/upserted/inserted -- grants will be carried over --#}\n {{ return(True) }}\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863224, "supported_languages": null}, "macro.dbt.get_show_grant_sql": {"name": "get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_show_grant_sql", "macro_sql": "{% macro get_show_grant_sql(relation) %}\n {{ return(adapter.dispatch(\"get_show_grant_sql\", \"dbt\")(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_show_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863383, "supported_languages": null}, "macro.dbt.default__get_show_grant_sql": {"name": "default__get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_show_grant_sql", "macro_sql": "{% macro default__get_show_grant_sql(relation) %}\n show grants on {{ relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863481, "supported_languages": null}, "macro.dbt.get_grant_sql": {"name": "get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_grant_sql", "macro_sql": "{% macro get_grant_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863672, "supported_languages": null}, "macro.dbt.default__get_grant_sql": {"name": "default__get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_grant_sql", "macro_sql": "\n\n{%- macro default__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation.render() }} to {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863852, "supported_languages": null}, "macro.dbt.get_revoke_sql": {"name": "get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_revoke_sql", "macro_sql": "{% macro get_revoke_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_revoke_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_revoke_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86405, "supported_languages": null}, "macro.dbt.default__get_revoke_sql": {"name": "default__get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_revoke_sql", "macro_sql": "\n\n{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation.render() }} from {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.864309, "supported_languages": null}, "macro.dbt.get_dcl_statement_list": {"name": "get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_dcl_statement_list", "macro_sql": "{% macro get_dcl_statement_list(relation, grant_config, get_dcl_macro) %}\n {{ return(adapter.dispatch('get_dcl_statement_list', 'dbt')(relation, grant_config, get_dcl_macro)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_dcl_statement_list"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8645408, "supported_languages": null}, "macro.dbt.default__get_dcl_statement_list": {"name": "default__get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_dcl_statement_list", "macro_sql": "\n\n{%- macro default__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}\n {#\n -- Unpack grant_config into specific privileges and the set of users who need them granted/revoked.\n -- Depending on whether this database supports multiple grantees per statement, pass in the list of\n -- all grantees per privilege, or (if not) template one statement per privilege-grantee pair.\n -- `get_dcl_macro` will be either `get_grant_sql` or `get_revoke_sql`\n #}\n {%- set dcl_statements = [] -%}\n {%- for privilege, grantees in grant_config.items() %}\n {%- if support_multiple_grantees_per_dcl_statement() and grantees -%}\n {%- set dcl = get_dcl_macro(relation, privilege, grantees) -%}\n {%- do dcl_statements.append(dcl) -%}\n {%- else -%}\n {%- for grantee in grantees -%}\n {% set dcl = get_dcl_macro(relation, privilege, [grantee]) %}\n {%- do dcl_statements.append(dcl) -%}\n {% endfor -%}\n {%- endif -%}\n {%- endfor -%}\n {{ return(dcl_statements) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.865141, "supported_languages": null}, "macro.dbt.call_dcl_statements": {"name": "call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.call_dcl_statements", "macro_sql": "{% macro call_dcl_statements(dcl_statement_list) %}\n {{ return(adapter.dispatch(\"call_dcl_statements\", \"dbt\")(dcl_statement_list)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.865317, "supported_languages": null}, "macro.dbt.default__call_dcl_statements": {"name": "default__call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__call_dcl_statements", "macro_sql": "{% macro default__call_dcl_statements(dcl_statement_list) %}\n {#\n -- By default, supply all grant + revoke statements in a single semicolon-separated block,\n -- so that they're all processed together.\n\n -- Some databases do not support this. Those adapters will need to override this macro\n -- to run each statement individually.\n #}\n {% call statement('grants') %}\n {% for dcl_statement in dcl_statement_list %}\n {{ dcl_statement }};\n {% endfor %}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.865537, "supported_languages": null}, "macro.dbt.apply_grants": {"name": "apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.apply_grants", "macro_sql": "{% macro apply_grants(relation, grant_config, should_revoke) %}\n {{ return(adapter.dispatch(\"apply_grants\", \"dbt\")(relation, grant_config, should_revoke)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.865737, "supported_languages": null}, "macro.dbt.default__apply_grants": {"name": "default__apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__apply_grants", "macro_sql": "{% macro default__apply_grants(relation, grant_config, should_revoke=True) %}\n {#-- If grant_config is {} or None, this is a no-op --#}\n {% if grant_config %}\n {% if should_revoke %}\n {#-- We think previous grants may have carried over --#}\n {#-- Show current grants and calculate diffs --#}\n {% set current_grants_table = run_query(get_show_grant_sql(relation)) %}\n {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}\n {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}\n {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}\n {% if not (needs_granting or needs_revoking) %}\n {{ log('On ' ~ relation.render() ~': All grants are in place, no revocation or granting needed.')}}\n {% endif %}\n {% else %}\n {#-- We don't think there's any chance of previous grants having carried over. --#}\n {#-- Jump straight to granting what the user has configured. --#}\n {% set needs_revoking = {} %}\n {% set needs_granting = grant_config %}\n {% endif %}\n {% if needs_granting or needs_revoking %}\n {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %}\n {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %}\n {% set dcl_statement_list = revoke_statement_list + grant_statement_list %}\n {% if dcl_statement_list %}\n {{ call_dcl_statements(dcl_statement_list) }}\n {% endif %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.get_show_grant_sql", "macro.dbt.get_dcl_statement_list", "macro.dbt.call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8668082, "supported_languages": null}, "macro.dbt.get_show_sql": {"name": "get_show_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.get_show_sql", "macro_sql": "{% macro get_show_sql(compiled_code, sql_header, limit) -%}\n {%- if sql_header is not none -%}\n {{ sql_header }}\n {%- endif %}\n {{ get_limit_subquery_sql(compiled_code, limit) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_limit_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8671901, "supported_languages": null}, "macro.dbt.get_limit_subquery_sql": {"name": "get_limit_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.get_limit_subquery_sql", "macro_sql": "\n{%- macro get_limit_subquery_sql(sql, limit) -%}\n {{ adapter.dispatch('get_limit_sql', 'dbt')(sql, limit) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_limit_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8673549, "supported_languages": null}, "macro.dbt.default__get_limit_sql": {"name": "default__get_limit_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.default__get_limit_sql", "macro_sql": "{% macro default__get_limit_sql(sql, limit) %}\n {{ compiled_code }}\n {% if limit is not none %}\n limit {{ limit }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86752, "supported_languages": null}, "macro.dbt.alter_column_comment": {"name": "alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_column_comment", "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86812, "supported_languages": null}, "macro.dbt.default__alter_column_comment": {"name": "default__alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_column_comment", "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.868274, "supported_languages": null}, "macro.dbt.alter_relation_comment": {"name": "alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_relation_comment", "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__alter_relation_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.868457, "supported_languages": null}, "macro.dbt.default__alter_relation_comment": {"name": "default__alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_relation_comment", "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8686101, "supported_languages": null}, "macro.dbt.persist_docs": {"name": "persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.persist_docs", "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86885, "supported_languages": null}, "macro.dbt.default__persist_docs": {"name": "default__persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__persist_docs", "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.alter_relation_comment", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8692548, "supported_languages": null}, "macro.dbt.get_catalog_relations": {"name": "get_catalog_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog_relations", "macro_sql": "{% macro get_catalog_relations(information_schema, relations) -%}\n {{ return(adapter.dispatch('get_catalog_relations', 'dbt')(information_schema, relations)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog_relations"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.872187, "supported_languages": null}, "macro.dbt.default__get_catalog_relations": {"name": "default__get_catalog_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog_relations", "macro_sql": "{% macro default__get_catalog_relations(information_schema, relations) -%}\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog_relations not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.872406, "supported_languages": null}, "macro.dbt.get_catalog": {"name": "get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog", "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.872573, "supported_languages": null}, "macro.dbt.default__get_catalog": {"name": "default__get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog", "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8727858, "supported_languages": null}, "macro.dbt.information_schema_name": {"name": "information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.information_schema_name", "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__information_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.872941, "supported_languages": null}, "macro.dbt.default__information_schema_name": {"name": "default__information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__information_schema_name", "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8730712, "supported_languages": null}, "macro.dbt.list_schemas": {"name": "list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_schemas", "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__list_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8732271, "supported_languages": null}, "macro.dbt.default__list_schemas": {"name": "default__list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_schemas", "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.information_schema_name", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.87344, "supported_languages": null}, "macro.dbt.check_schema_exists": {"name": "check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.check_schema_exists", "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__check_schema_exists"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8736901, "supported_languages": null}, "macro.dbt.default__check_schema_exists": {"name": "default__check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__check_schema_exists", "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.replace", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8739462, "supported_languages": null}, "macro.dbt.list_relations_without_caching": {"name": "list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_relations_without_caching", "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__list_relations_without_caching"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.874113, "supported_languages": null}, "macro.dbt.default__list_relations_without_caching": {"name": "default__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_relations_without_caching", "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.874256, "supported_languages": null}, "macro.dbt.get_catalog_for_single_relation": {"name": "get_catalog_for_single_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog_for_single_relation", "macro_sql": "{% macro get_catalog_for_single_relation(relation) %}\n {{ return(adapter.dispatch('get_catalog_for_single_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_catalog_for_single_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.874414, "supported_languages": null}, "macro.dbt.default__get_catalog_for_single_relation": {"name": "default__get_catalog_for_single_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog_for_single_relation", "macro_sql": "{% macro default__get_catalog_for_single_relation(relation) %}\n {{ exceptions.raise_not_implemented(\n 'get_catalog_for_single_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8745549, "supported_languages": null}, "macro.dbt.get_relations": {"name": "get_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_relations", "macro_sql": "{% macro get_relations() %}\n {{ return(adapter.dispatch('get_relations', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_relations"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.874696, "supported_languages": null}, "macro.dbt.default__get_relations": {"name": "default__get_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_relations", "macro_sql": "{% macro default__get_relations() %}\n {{ exceptions.raise_not_implemented(\n 'get_relations macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8748312, "supported_languages": null}, "macro.dbt.get_relation_last_modified": {"name": "get_relation_last_modified", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_relation_last_modified", "macro_sql": "{% macro get_relation_last_modified(information_schema, relations) %}\n {{ return(adapter.dispatch('get_relation_last_modified', 'dbt')(information_schema, relations)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_relation_last_modified"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.87501, "supported_languages": null}, "macro.dbt.default__get_relation_last_modified": {"name": "default__get_relation_last_modified", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_relation_last_modified", "macro_sql": "{% macro default__get_relation_last_modified(information_schema, relations) %}\n {{ exceptions.raise_not_implemented(\n 'get_relation_last_modified macro not implemented for adapter ' + adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8751612, "supported_languages": null}, "macro.dbt.get_columns_in_relation": {"name": "get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_relation", "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.877135, "supported_languages": null}, "macro.dbt.default__get_columns_in_relation": {"name": "default__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_relation", "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.877271, "supported_languages": null}, "macro.dbt.sql_convert_columns_in_relation": {"name": "sql_convert_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.sql_convert_columns_in_relation", "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.877537, "supported_languages": null}, "macro.dbt.get_empty_subquery_sql": {"name": "get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_subquery_sql", "macro_sql": "{% macro get_empty_subquery_sql(select_sql, select_sql_header=none) -%}\n {{ return(adapter.dispatch('get_empty_subquery_sql', 'dbt')(select_sql, select_sql_header)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8777258, "supported_languages": null}, "macro.dbt.default__get_empty_subquery_sql": {"name": "default__get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_subquery_sql", "macro_sql": "{% macro default__get_empty_subquery_sql(select_sql, select_sql_header=none) %}\n {%- if select_sql_header is not none -%}\n {{ select_sql_header }}\n {%- endif -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.877902, "supported_languages": null}, "macro.dbt.get_empty_schema_sql": {"name": "get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_schema_sql", "macro_sql": "{% macro get_empty_schema_sql(columns) -%}\n {{ return(adapter.dispatch('get_empty_schema_sql', 'dbt')(columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_empty_schema_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8780658, "supported_languages": null}, "macro.dbt.default__get_empty_schema_sql": {"name": "default__get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_schema_sql", "macro_sql": "{% macro default__get_empty_schema_sql(columns) %}\n {%- set col_err = [] -%}\n {%- set col_naked_numeric = [] -%}\n select\n {% for i in columns %}\n {%- set col = columns[i] -%}\n {%- if col['data_type'] is not defined -%}\n {%- do col_err.append(col['name']) -%}\n {#-- If this column's type is just 'numeric' then it is missing precision/scale, raise a warning --#}\n {%- elif col['data_type'].strip().lower() in ('numeric', 'decimal', 'number') -%}\n {%- do col_naked_numeric.append(col['name']) -%}\n {%- endif -%}\n {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %}\n {{ cast('null', col['data_type']) }} as {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n {%- if (col_err | length) > 0 -%}\n {{ exceptions.column_type_missing(column_names=col_err) }}\n {%- elif (col_naked_numeric | length) > 0 -%}\n {{ exceptions.warn(\"Detected columns with numeric type and unspecified precision/scale, this can lead to unintended rounding: \" ~ col_naked_numeric ~ \"`\") }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.879136, "supported_languages": null}, "macro.dbt.get_column_schema_from_query": {"name": "get_column_schema_from_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_column_schema_from_query", "macro_sql": "{% macro get_column_schema_from_query(select_sql, select_sql_header=none) -%}\n {% set columns = [] %}\n {# -- Using an 'empty subquery' here to get the same schema as the given select_sql statement, without necessitating a data scan.#}\n {% set sql = get_empty_subquery_sql(select_sql, select_sql_header) %}\n {% set column_schema = adapter.get_column_schema_from_query(sql) %}\n {{ return(column_schema) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.879442, "supported_languages": null}, "macro.dbt.get_columns_in_query": {"name": "get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_query", "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.879605, "supported_languages": null}, "macro.dbt.default__get_columns_in_query": {"name": "default__get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_query", "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n {{ get_empty_subquery_sql(select_sql) }}\n {% endcall %}\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.879909, "supported_languages": null}, "macro.dbt.alter_column_type": {"name": "alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_column_type", "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.880113, "supported_languages": null}, "macro.dbt.default__alter_column_type": {"name": "default__alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_column_type", "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation.render() }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation.render() }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation.render() }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation.render() }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.880679, "supported_languages": null}, "macro.dbt.alter_relation_add_remove_columns": {"name": "alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_relation_add_remove_columns", "macro_sql": "{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}\n {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__alter_relation_add_remove_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8809001, "supported_languages": null}, "macro.dbt.default__alter_relation_add_remove_columns": {"name": "default__alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_relation_add_remove_columns", "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation.render() }}\n\n {% for column in add_columns %}\n add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n\n {% for column in remove_columns %}\n drop column {{ column.name }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.88154, "supported_languages": null}, "macro.dbt.get_fixture_sql": {"name": "get_fixture_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.get_fixture_sql", "macro_sql": "{% macro get_fixture_sql(rows, column_name_to_data_types) %}\n-- Fixture for {{ model.name }}\n{% set default_row = {} %}\n\n{%- if not column_name_to_data_types -%}\n{#-- Use defer_relation IFF it is available in the manifest and 'this' is missing from the database --#}\n{%- set this_or_defer_relation = defer_relation if (defer_relation and not load_relation(this)) else this -%}\n{%- set columns_in_relation = adapter.get_columns_in_relation(this_or_defer_relation) -%}\n\n{%- set column_name_to_data_types = {} -%}\n{%- for column in columns_in_relation -%}\n{#-- This needs to be a case-insensitive comparison --#}\n{%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}\n{%- endfor -%}\n{%- endif -%}\n\n{%- if not column_name_to_data_types -%}\n {{ exceptions.raise_compiler_error(\"Not able to get columns for unit test '\" ~ model.name ~ \"' from relation \" ~ this ~ \" because the relation doesn't exist\") }}\n{%- endif -%}\n\n{%- for column_name, column_type in column_name_to_data_types.items() -%}\n {%- do default_row.update({column_name: (safe_cast(\"null\", column_type) | trim )}) -%}\n{%- endfor -%}\n\n{{ validate_fixture_rows(rows, row_number) }}\n\n{%- for row in rows -%}\n{%- set formatted_row = format_row(row, column_name_to_data_types) -%}\n{%- set default_row_copy = default_row.copy() -%}\n{%- do default_row_copy.update(formatted_row) -%}\nselect\n{%- for column_name, column_value in default_row_copy.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%}, {%- endif %}\n{%- endfor %}\n{%- if not loop.last %}\nunion all\n{% endif %}\n{%- endfor -%}\n\n{%- if (rows | length) == 0 -%}\n select\n {%- for column_name, column_value in default_row.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%},{%- endif %}\n {%- endfor %}\n limit 0\n{%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_relation", "macro.dbt.safe_cast", "macro.dbt.validate_fixture_rows", "macro.dbt.format_row"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.884523, "supported_languages": null}, "macro.dbt.get_expected_sql": {"name": "get_expected_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.get_expected_sql", "macro_sql": "{% macro get_expected_sql(rows, column_name_to_data_types) %}\n\n{%- if (rows | length) == 0 -%}\n select * from dbt_internal_unit_test_actual\n limit 0\n{%- else -%}\n{%- for row in rows -%}\n{%- set formatted_row = format_row(row, column_name_to_data_types) -%}\nselect\n{%- for column_name, column_value in formatted_row.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%}, {%- endif %}\n{%- endfor %}\n{%- if not loop.last %}\nunion all\n{% endif %}\n{%- endfor -%}\n{%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.format_row"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8849862, "supported_languages": null}, "macro.dbt.format_row": {"name": "format_row", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.format_row", "macro_sql": "\n\n{%- macro format_row(row, column_name_to_data_types) -%}\n {#-- generate case-insensitive formatted row --#}\n {% set formatted_row = {} %}\n {%- for column_name, column_value in row.items() -%}\n {% set column_name = column_name|lower %}\n\n {%- if column_name not in column_name_to_data_types %}\n {#-- if user-provided row contains column name that relation does not contain, raise an error --#}\n {% set fixture_name = \"expected output\" if model.resource_type == 'unit_test' else (\"'\" ~ model.name ~ \"'\") %}\n {{ exceptions.raise_compiler_error(\n \"Invalid column name: '\" ~ column_name ~ \"' in unit test fixture for \" ~ fixture_name ~ \".\"\n \"\\nAccepted columns for \" ~ fixture_name ~ \" are: \" ~ (column_name_to_data_types.keys()|list)\n ) }}\n {%- endif -%}\n\n {%- set column_type = column_name_to_data_types[column_name] %}\n\n {#-- sanitize column_value: wrap yaml strings in quotes, apply cast --#}\n {%- set column_value_clean = column_value -%}\n {%- if column_value is string -%}\n {%- set column_value_clean = dbt.string_literal(dbt.escape_single_quotes(column_value)) -%}\n {%- elif column_value is none -%}\n {%- set column_value_clean = 'null' -%}\n {%- endif -%}\n\n {%- set row_update = {column_name: safe_cast(column_value_clean, column_type) } -%}\n {%- do formatted_row.update(row_update) -%}\n {%- endfor -%}\n {{ return(formatted_row) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.string_literal", "macro.dbt.escape_single_quotes", "macro.dbt.safe_cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8859332, "supported_languages": null}, "macro.dbt.validate_fixture_rows": {"name": "validate_fixture_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.validate_fixture_rows", "macro_sql": "{%- macro validate_fixture_rows(rows, row_number) -%}\n {{ return(adapter.dispatch('validate_fixture_rows', 'dbt')(rows, row_number)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__validate_fixture_rows"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8861172, "supported_languages": null}, "macro.dbt.default__validate_fixture_rows": {"name": "default__validate_fixture_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.default__validate_fixture_rows", "macro_sql": "{%- macro default__validate_fixture_rows(rows, row_number) -%}\n {# This is an abstract method for adapter overrides as needed #}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.886199, "supported_languages": null}, "macro.dbt.resolve_model_name": {"name": "resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.resolve_model_name", "macro_sql": "{% macro resolve_model_name(input_model_name) %}\n {{ return(adapter.dispatch('resolve_model_name', 'dbt')(input_model_name)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.887635, "supported_languages": null}, "macro.dbt.default__resolve_model_name": {"name": "default__resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.default__resolve_model_name", "macro_sql": "\n\n{%- macro default__resolve_model_name(input_model_name) -%}\n {{ input_model_name | string | replace('\"', '\\\"') }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.887774, "supported_languages": null}, "macro.dbt.build_ref_function": {"name": "build_ref_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_ref_function", "macro_sql": "{% macro build_ref_function(model) %}\n\n {%- set ref_dict = {} -%}\n {%- for _ref in model.refs -%}\n {% set _ref_args = [_ref.get('package'), _ref['name']] if _ref.get('package') else [_ref['name'],] %}\n {%- set resolved = ref(*_ref_args, v=_ref.get('version')) -%}\n {%- if _ref.get('version') -%}\n {% do _ref_args.extend([\"v\" ~ _ref['version']]) %}\n {%- endif -%}\n {%- do ref_dict.update({_ref_args | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef ref(*args, **kwargs):\n refs = {{ ref_dict | tojson }}\n key = '.'.join(args)\n version = kwargs.get(\"v\") or kwargs.get(\"version\")\n if version:\n key += f\".v{version}\"\n dbt_load_df_function = kwargs.get(\"dbt_load_df_function\")\n return dbt_load_df_function(refs[key])\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.888478, "supported_languages": null}, "macro.dbt.build_source_function": {"name": "build_source_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_source_function", "macro_sql": "{% macro build_source_function(model) %}\n\n {%- set source_dict = {} -%}\n {%- for _source in model.sources -%}\n {%- set resolved = source(*_source) -%}\n {%- do source_dict.update({_source | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef source(*args, dbt_load_df_function):\n sources = {{ source_dict | tojson }}\n key = '.'.join(args)\n return dbt_load_df_function(sources[key])\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8888261, "supported_languages": null}, "macro.dbt.build_config_dict": {"name": "build_config_dict", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_config_dict", "macro_sql": "{% macro build_config_dict(model) %}\n {%- set config_dict = {} -%}\n {% set config_dbt_used = zip(model.config.config_keys_used, model.config.config_keys_defaults) | list %}\n {%- for key, default in config_dbt_used -%}\n {# weird type testing with enum, would be much easier to write this logic in Python! #}\n {%- if key == \"language\" -%}\n {%- set value = \"python\" -%}\n {%- endif -%}\n {%- set value = model.config.get(key, default) -%}\n {%- do config_dict.update({key: value}) -%}\n {%- endfor -%}\nconfig_dict = {{ config_dict }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8893209, "supported_languages": null}, "macro.dbt.py_script_postfix": {"name": "py_script_postfix", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_postfix", "macro_sql": "{% macro py_script_postfix(model) %}\n# This part is user provided model code\n# you will need to copy the next section to run the code\n# COMMAND ----------\n# this part is dbt logic for get ref work, do not modify\n\n{{ build_ref_function(model ) }}\n{{ build_source_function(model ) }}\n{{ build_config_dict(model) }}\n\nclass config:\n def __init__(self, *args, **kwargs):\n pass\n\n @staticmethod\n def get(key, default=None):\n return config_dict.get(key, default)\n\nclass this:\n \"\"\"dbt.this() or dbt.this.identifier\"\"\"\n database = \"{{ this.database }}\"\n schema = \"{{ this.schema }}\"\n identifier = \"{{ this.identifier }}\"\n {% set this_relation_name = resolve_model_name(this) %}\n def __repr__(self):\n return '{{ this_relation_name }}'\n\n\nclass dbtObj:\n def __init__(self, load_df_function) -> None:\n self.source = lambda *args: source(*args, dbt_load_df_function=load_df_function)\n self.ref = lambda *args, **kwargs: ref(*args, **kwargs, dbt_load_df_function=load_df_function)\n self.config = config\n self.this = this()\n self.is_incremental = {{ is_incremental() }}\n\n# COMMAND ----------\n{{py_script_comment()}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.build_ref_function", "macro.dbt.build_source_function", "macro.dbt.build_config_dict", "macro.dbt.resolve_model_name", "macro.dbt.is_incremental", "macro.dbt.py_script_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8898182, "supported_languages": null}, "macro.dbt.py_script_comment": {"name": "py_script_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_comment", "macro_sql": "{%macro py_script_comment()%}\n{%endmacro%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.889886, "supported_languages": null}, "macro.dbt.test_unique": {"name": "test_unique", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_unique", "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_unique"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.890347, "supported_languages": null}, "macro.dbt.test_not_null": {"name": "test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_not_null", "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8905559, "supported_languages": null}, "macro.dbt.test_accepted_values": {"name": "test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_accepted_values", "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_accepted_values"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.890809, "supported_languages": null}, "macro.dbt.test_relationships": {"name": "test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_relationships", "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_relationships"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.891062, "supported_languages": null}}, "docs": {"doc.dbt.__overview__": {"name": "__overview__", "resource_type": "doc", "package_name": "dbt", "path": "overview.md", "original_file_path": "docs/overview.md", "unique_id": "doc.dbt.__overview__", "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/introduction)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion"}}, "exposures": {}, "metrics": {}, "groups": {}, "selectors": {}, "disabled": {}, "parent_map": {"model.dbt_demo.diamonds_list_colors": ["model.dbt_demo.diamonds_four_cs"], "model.dbt_demo.diamonds_prices": [], "model.dbt_demo.diamonds_four_cs": ["source.dbt_demo.raw.diamonds"], "model.dbt_demo.zzz_game_details": [], "model.dbt_demo.second_dbt_model": ["model.dbt_demo.first_dbt_model"], "model.dbt_demo.first_dbt_model": [], "test.dbt_demo.zzz_game_details_check_scores": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.zzz_game_details_check_dates": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.zzz_win_loss_records_check_records": ["model.dbt_demo.zzz_win_loss_records"], "test.dbt_demo.diamonds_list_colors_test": ["model.dbt_demo.diamonds_list_colors"], "seed.dbt_demo.country_codes": [], "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.unique_first_dbt_model_id.9dda05743e": ["model.dbt_demo.first_dbt_model"], "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061": ["model.dbt_demo.first_dbt_model"], "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4": ["model.dbt_demo.second_dbt_model"], "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2": ["model.dbt_demo.second_dbt_model"], "test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f": ["model.dbt_demo.second_dbt_model"], "model.dbt_demo.changed_country": ["snapshot.dbt_demo.country_snapshot"], "model.dbt_demo.country": ["seed.dbt_demo.country_codes"], "snapshot.dbt_demo.country_snapshot": ["model.dbt_demo.country"], "model.dbt_demo.zzz_win_loss_records": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105": ["model.dbt_demo.zzz_win_loss_records"], "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86": ["model.dbt_demo.zzz_win_loss_records"], "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe": ["model.dbt_demo.zzz_win_loss_records"], "test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601": ["model.dbt_demo.zzz_win_loss_records"], "source.dbt_demo.raw.diamonds": [], "unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors": ["model.dbt_demo.diamonds_list_colors"]}, "child_map": {"model.dbt_demo.diamonds_list_colors": ["test.dbt_demo.diamonds_list_colors_test", "unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors"], "model.dbt_demo.diamonds_prices": [], "model.dbt_demo.diamonds_four_cs": ["model.dbt_demo.diamonds_list_colors"], "model.dbt_demo.zzz_game_details": ["model.dbt_demo.zzz_win_loss_records", "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40", "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d", "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c", "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68", "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905", "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5", "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39", "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e", "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f", "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be", "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218", "test.dbt_demo.zzz_game_details_check_dates", "test.dbt_demo.zzz_game_details_check_scores"], "model.dbt_demo.second_dbt_model": ["test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f", "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2", "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4"], "model.dbt_demo.first_dbt_model": ["model.dbt_demo.second_dbt_model", "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061", "test.dbt_demo.unique_first_dbt_model_id.9dda05743e"], "test.dbt_demo.zzz_game_details_check_scores": [], "test.dbt_demo.zzz_game_details_check_dates": [], "test.dbt_demo.zzz_win_loss_records_check_records": [], "test.dbt_demo.diamonds_list_colors_test": [], "seed.dbt_demo.country_codes": ["model.dbt_demo.country"], "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218": [], "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905": [], "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5": [], "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40": [], "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e": [], "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d": [], "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39": [], "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f": [], "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be": [], "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c": [], "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68": [], "test.dbt_demo.unique_first_dbt_model_id.9dda05743e": [], "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061": [], "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4": [], "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2": [], "test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f": [], "model.dbt_demo.changed_country": [], "model.dbt_demo.country": ["snapshot.dbt_demo.country_snapshot"], "snapshot.dbt_demo.country_snapshot": ["model.dbt_demo.changed_country"], "model.dbt_demo.zzz_win_loss_records": ["test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601", "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86", "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe", "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105", "test.dbt_demo.zzz_win_loss_records_check_records"], "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105": [], "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86": [], "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe": [], "test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601": [], "source.dbt_demo.raw.diamonds": ["model.dbt_demo.diamonds_four_cs"], "unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors": []}, "group_map": {}, "saved_queries": {}, "semantic_models": {}, "unit_tests": {"unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors": {"model": "diamonds_list_colors", "given": [{"input": "ref('diamonds_four_cs')", "rows": [{"carat": 0.23, "cut": "Ideal", "color": "E", "clarity": "SI2"}, {"carat": 0.3, "cut": "Premium", "color": "E", "clarity": "SI2"}, {"carat": 0.3, "cut": "Premium", "color": "K", "clarity": "SI2"}], "format": "dict", "fixture": null}], "expect": {"rows": [{"color": "E"}, {"color": "K"}], "format": "dict", "fixture": null}, "name": "test_get_distinct_colors", "resource_type": "unit_test", "package_name": "dbt_demo", "path": "sql_model1/test_get_distinct_colors.yml", "original_file_path": "src/models/sql_model1/test_get_distinct_colors.yml", "unique_id": "unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors", "fqn": ["dbt_demo", "models", "sql_model1", "diamonds_list_colors", "test_get_distinct_colors"], "description": "Check a distinct list of colors is returned.", "overrides": null, "depends_on": {"macros": [], "nodes": ["model.dbt_demo.diamonds_list_colors"]}, "config": {"tags": [], "meta": {}}, "checksum": "483c13eb849bf29c60173f6b4583def3c4a3b8c20e674a28c0db2c3e81834bb1", "schema": "dbt_dev", "created_at": 1733486794.4994261, "versions": null, "version": null}}} \ No newline at end of file diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_dbt_factory.py b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_dbt_factory.py new file mode 100644 index 00000000..51cbf79c --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_dbt_factory.py @@ -0,0 +1,482 @@ +import json +from pathlib import Path + +from databricks_dbt_factory.Utils import read_dbt_manifest + +BASE_PATH = str(Path(__file__).resolve().parent) + + +def _model( + package: str, name: str, depends_on: list[str] | None = None +) -> tuple[str, dict]: + full_name = f"model.{package}.{name}" + return full_name, { + "resource_type": "model", + "name": name, + "package_name": package, + "depends_on": {"nodes": depends_on or []}, + } + + +def _test( + package: str, + name: str, + depends_on: list[str], + severity: str = "error", + test_hash: str = "", +) -> tuple[str, dict]: + full_name = f"test.{package}.{name}" + (f".{test_hash}" if test_hash else "") + return full_name, { + "resource_type": "test", + "name": name, + "package_name": package, + "depends_on": {"nodes": depends_on}, + "config": {"severity": severity}, + } + + +def _seed(package: str, name: str) -> tuple[str, dict]: + full_name = f"seed.{package}.{name}" + return full_name, { + "resource_type": "seed", + "name": name, + "package_name": package, + "depends_on": {"nodes": []}, + } + + +def _snapshot( + package: str, name: str, depends_on: list[str] | None = None +) -> tuple[str, dict]: + full_name = f"snapshot.{package}.{name}" + return full_name, { + "resource_type": "snapshot", + "name": name, + "package_name": package, + "depends_on": {"nodes": depends_on or []}, + } + + +def _source(package: str, source_name: str, table: str) -> tuple[str, dict]: + full_name = f"source.{package}.{source_name}.{table}" + return full_name, { + "resource_type": "source", + "name": table, + "source_name": source_name, + "package_name": package, + } + + +def _commands(task: dict) -> list[str]: + """Extracts the dbt commands from a rendered (serverless notebook) task dict.""" + return json.loads(task["notebook_task"]["base_parameters"]["dbt_commands"]) + + +def test_bundled_model_gates_on_tests_of_all_tested_upstreams(dbt_factory_bundled): + nodes = dict( + [ + _model("pkg", "customers"), + _model("pkg", "products"), + _model( + "pkg", + "orders", + depends_on=["model.pkg.customers", "model.pkg.products"], + ), + _test("pkg", "unique_customers_id", ["model.pkg.customers"]), + _test("pkg", "unique_products_id", ["model.pkg.products"]), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "customers_test" in by_key + assert "products_test" in by_key + assert _commands(by_key["customers_test"]) == [ + "dbt test --select pkg.customers --indirect-selection cautious --target dev" + ] + assert by_key["customers_test"]["depends_on"] == [{"task_key": "customers_run"}] + + # orders depends on both upstreams' bundled test tasks (rewired from their run tasks) + assert {dep["task_key"] for dep in by_key["orders_run"]["depends_on"]} == { + "customers_test", + "products_test", + } + + +def test_tests_on_seed_produce_task_and_gate_downstream(dbt_factory_bundled): + nodes = dict( + [ + _seed("pkg", "countries"), + _model("pkg", "enriched", depends_on=["seed.pkg.countries"]), + _test("pkg", "unique_countries_code", ["seed.pkg.countries"]), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "countries_test" in by_key + assert _commands(by_key["countries_test"]) == [ + "dbt test --select pkg.countries --indirect-selection cautious --target dev" + ] + assert by_key["countries_test"]["depends_on"] == [{"task_key": "countries_seed"}] + assert by_key["enriched_run"]["depends_on"] == [{"task_key": "countries_test"}] + + +def test_tests_on_snapshot_produce_task_and_gate_downstream(dbt_factory_bundled): + nodes = dict( + [ + _snapshot("pkg", "orders_snap"), + _model("pkg", "orders_history", depends_on=["snapshot.pkg.orders_snap"]), + _test("pkg", "not_null_orders_snap_id", ["snapshot.pkg.orders_snap"]), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "orders_snap_test" in by_key + assert _commands(by_key["orders_snap_test"]) == [ + "dbt test --select pkg.orders_snap --indirect-selection cautious --target dev" + ] + assert by_key["orders_snap_test"]["depends_on"] == [ + {"task_key": "orders_snap_snapshot"} + ] + assert by_key["orders_history_run"]["depends_on"] == [ + {"task_key": "orders_snap_test"} + ] + + +def test_tests_on_source_produce_standalone_task(dbt_factory_bundled): + nodes = dict( + [ + _test("pkg", "unique_raw_customers_id", ["source.pkg.raw.customers"]), + ] + ) + sources = dict([_source("pkg", "raw", "customers")]) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes, "sources": sources}) + by_key = {t["task_key"]: t for t in tasks} + + assert "raw_customers_test" in by_key + assert _commands(by_key["raw_customers_test"]) == [ + "dbt test --select source:pkg.raw.customers --indirect-selection cautious --target dev" + ] + assert by_key["raw_customers_test"]["depends_on"] == [] + + +def test_flat_mode_emits_one_task_per_test_node_and_gates_downstream(dbt_factory): + # Per-test mode mirrors `dbt build`: downstream models wait on upstream tests, so a + # failing `severity: error` test skips downstream via Databricks task failure. + nodes = dict( + [ + _model("pkg", "customers"), + _model("pkg", "orders", depends_on=["model.pkg.customers"]), + _test("pkg", "unique_customers_id", ["model.pkg.customers"]), + _test("pkg", "not_null_customers_id", ["model.pkg.customers"]), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "unique_customers_id_test" in by_key + assert "not_null_customers_id_test" in by_key + assert "customers_test" not in by_key # no bundling in per-test mode + + assert _commands(by_key["unique_customers_id_test"]) == [ + "dbt test --select unique_customers_id --target dev" + ] + assert by_key["unique_customers_id_test"]["depends_on"] == [ + {"task_key": "customers_run"} + ] + # orders depends on customers AND every test attached to customers + assert {dep["task_key"] for dep in by_key["orders_run"]["depends_on"]} == { + "customers_run", + "unique_customers_id_test", + "not_null_customers_id_test", + } + + +def test_flat_mode_cross_model_test_does_not_create_cycle(dbt_factory): + # Relationship test references BOTH `orders` and `customers`. Without care, extending + # `orders`'s deps with "tests of upstream (customers)" would pull in the relationship test, + # which itself depends on `orders` — a direct cycle. + nodes = dict( + [ + _model("pkg", "customers"), + _model("pkg", "orders", depends_on=["model.pkg.customers"]), + _model("pkg", "payments", depends_on=["model.pkg.orders"]), + _test("pkg", "unique_customers_id", ["model.pkg.customers"]), + _test( + "pkg", + "relationships_orders_customer_id__ref_customers", + ["model.pkg.orders", "model.pkg.customers"], + ), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + # orders depends on customers + unique_customers_id, but NOT on the relationship test + # (that test references orders itself — including it would cycle) + assert {dep["task_key"] for dep in by_key["orders_run"]["depends_on"]} == { + "customers_run", + "unique_customers_id_test", + } + + # payments (downstream of orders) picks up the relationship test — safe, payments + # transitively depends on both orders and customers (the test's refs) + payments_deps = {dep["task_key"] for dep in by_key["payments_run"]["depends_on"]} + assert "orders_run" in payments_deps + assert "relationships_orders_customer_id__ref_customers_test" in payments_deps + + +def test_flat_mode_transitive_cross_model_test_does_not_create_cycle(dbt_factory): + # Transitive cycle case: test T refs {A, C} where C is downstream of B which is downstream + # of A. Extending B's deps with "tests of upstream (A)" must NOT add T, because T depends + # on C and C depends on B → B → T → C → B cycle. Only nodes downstream of both A and C + # (i.e. downstream of C) should get T. + nodes = dict( + [ + _model("pkg", "a"), + _model("pkg", "b", depends_on=["model.pkg.a"]), + _model("pkg", "c", depends_on=["model.pkg.b"]), + _model("pkg", "d", depends_on=["model.pkg.c"]), + _test("pkg", "relationship_a_c", ["model.pkg.a", "model.pkg.c"]), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + # B's ancestors = {A}. Test T refs = {A, C}. C ∉ ancestors(B) → skip T. + assert by_key["b_run"]["depends_on"] == [{"task_key": "a_run"}] + # C's ancestors = {A, B}. C IS in T.refs → skip T (direct self-reference). + assert by_key["c_run"]["depends_on"] == [{"task_key": "b_run"}] + # D's ancestors = {A, B, C}. T.refs = {A, C} ⊆ ancestors(D) → add T. + d_deps = {dep["task_key"] for dep in by_key["d_run"]["depends_on"]} + assert d_deps == {"c_run", "relationship_a_c_test"} + + +def test_flat_mode_warn_severity_tests_do_not_gate_downstream(dbt_factory): + # Only error-severity tests gate downstream (matches `dbt build`: dbt exits 0 on warn + # so a gate wouldn't block anyway; we just keep the DAG cleaner). + nodes = dict( + [ + _model("pkg", "customers"), + _model("pkg", "orders", depends_on=["model.pkg.customers"]), + _test( + "pkg", "unique_customers_id", ["model.pkg.customers"], severity="warn" + ), + _test( + "pkg", + "not_null_customers_id", + ["model.pkg.customers"], + severity="error", + ), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + # Both test tasks still exist (warn tests still run — they just don't gate anything) + assert "unique_customers_id_test" in by_key + assert "not_null_customers_id_test" in by_key + + # orders gates on customers + the error-severity test, but NOT the warn-severity one + assert {dep["task_key"] for dep in by_key["orders_run"]["depends_on"]} == { + "customers_run", + "not_null_customers_id_test", + } + + +def test_flat_mode_test_on_seed_gates_on_seed(dbt_factory): + nodes = dict( + [ + _seed("pkg", "countries"), + _test("pkg", "unique_countries_code", ["seed.pkg.countries"]), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert by_key["unique_countries_code_test"]["depends_on"] == [ + {"task_key": "countries_seed"} + ] + + +def test_bundled_task_factory_assembles_commands(dbt_factory_bundled): + test_factory = dbt_factory_bundled.task_factories["test"] + task = test_factory.create_bundled_task( + task_key="customers_test", + select="pkg.customers", + depends_on=["customers_run"], + ) + assert task.task_key == "customers_test" + assert task.commands == [ + "dbt test --select pkg.customers --indirect-selection cautious --target dev" + ] + assert task.depends_on == ["customers_run"] + + +def test_cross_model_test_in_bundled_mode_is_emitted_as_standalone_task( + dbt_factory_bundled, +): + # The relationship test spans two models, so it must NOT be collapsed into either model's + # bundled test task (dbt would hit a TABLE_OR_VIEW_NOT_FOUND on the un-built endpoint). + # It should emit its own task with deps on both referenced models. + nodes = dict( + [ + _model("pkg", "team_cities"), + _model("pkg", "game_details", depends_on=["model.pkg.team_cities"]), + _test("pkg", "not_null_team_cities_name", ["model.pkg.team_cities"]), + _test( + "pkg", + "relationships_game_details_winner__team_city__ref_team_cities_", + ["model.pkg.game_details", "model.pkg.team_cities"], + ), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + # Single-model test → bundled with cautious selection (relationship test is excluded by dbt) + assert "team_cities_test" in by_key + assert _commands(by_key["team_cities_test"]) == [ + "dbt test --select pkg.team_cities --indirect-selection cautious --target dev" + ] + + # Cross-model test → its own task, gated on BOTH referenced models + cross_test_key = ( + "relationships_game_details_winner__team_city__ref_team_cities__test" + ) + assert cross_test_key in by_key + assert _commands(by_key[cross_test_key]) == [ + "dbt test --select relationships_game_details_winner__team_city__ref_team_cities_ --target dev" + ] + assert {dep["task_key"] for dep in by_key[cross_test_key]["depends_on"]} == { + "team_cities_run", + "game_details_run", + } + + # `game_details` has no single-model tests, so no bundled `game_details_test` exists + assert "game_details_test" not in by_key + + +def test_single_package_bundled_test_uses_qualified_select(dbt_factory_bundled): + nodes = dict( + [ + _model("pkg_a", "customers"), + _model("pkg_a", "orders", depends_on=["model.pkg_a.customers"]), + _test("pkg_a", "unique_customers_id", ["model.pkg_a.customers"]), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "customers_test" in by_key + assert _commands(by_key["customers_test"]) == [ + "dbt test --select pkg_a.customers --indirect-selection cautious --target dev" + ] + assert by_key["orders_run"]["depends_on"] == [{"task_key": "customers_test"}] + + +def test_flat_mode_same_named_tests_get_unique_keys_and_both_gate_downstream( + dbt_factory, +): + # dbt allows the same custom test name on two different models, telling the nodes apart + # only via the unique_id hash. Both tests must keep distinct task keys AND both must gate + # the downstream model (a post-hoc rename could not fix the dep list, hence the key map). + nodes = dict( + [ + _model("pkg", "customers"), + _model("pkg", "orders", depends_on=["model.pkg.customers"]), + _test("pkg", "dup_check", ["model.pkg.customers"], test_hash="6ea6b2ac82"), + _test("pkg", "dup_check", ["model.pkg.customers"], test_hash="a9ab3a6e12"), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "dup_check_6ea6b2ac82_test" in by_key + assert "dup_check_a9ab3a6e12_test" in by_key + assert {dep["task_key"] for dep in by_key["orders_run"]["depends_on"]} == { + "customers_run", + "dup_check_6ea6b2ac82_test", + "dup_check_a9ab3a6e12_test", + } + + +def test_flat_mode_cross_package_models_get_package_prefixed_keys(dbt_factory): + # Two packages may ship a model with the same name (materialized to different schemas). + nodes = dict( + [ + _model("shop", "stg_orders"), + _model("subpkg", "stg_orders"), + _model( + "shop", + "mart", + depends_on=["model.shop.stg_orders", "model.subpkg.stg_orders"], + ), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "shop_stg_orders_run" in by_key + assert "subpkg_stg_orders_run" in by_key + assert {dep["task_key"] for dep in by_key["mart_run"]["depends_on"]} == { + "shop_stg_orders_run", + "subpkg_stg_orders_run", + } + + +def test_bundled_mode_singular_test_named_like_tested_model_keeps_keys_unique( + dbt_factory_bundled, +): + # The bundled test task for model `orders` claims `orders_test`, and so does a singular + # test file named `orders`. Both stay unique (and deploy cannot fail on a duplicate key). + nodes = dict( + [ + _model("pkg", "orders"), + _test( + "pkg", "unique_orders_id", ["model.pkg.orders"], test_hash="9a1b2c3d4e" + ), + _test("pkg", "orders", []), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + keys = [t["task_key"] for t in tasks] + by_key = {t["task_key"]: t for t in tasks} + + assert len(keys) == len(set(keys)) + assert _commands(by_key["pkg_orders_test"]) == [ + "dbt test --select orders --target dev" + ] + assert _commands(by_key["model_pkg_orders_test"]) == [ + "dbt test --select pkg.orders --indirect-selection cautious --target dev" + ] + + +def test_generated_tasks_match_expected(dbt_factory): + """Safety check: the full set of tasks generated for the sample manifest matches the saved + copy in test_data/expected_tasks.json. If you change the generated output on purpose, refresh + that file with `make test-update-expected-tasks`. + """ + manifest = read_dbt_manifest(BASE_PATH + "/test_data/manifest.json") + tasks = dbt_factory.create_tasks(manifest) + + expected = json.loads( + Path(BASE_PATH + "/test_data/expected_tasks.json").read_text() + ) + assert tasks == expected diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_dbt_task.py b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_dbt_task.py new file mode 100644 index 00000000..f0a993ce --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_dbt_task.py @@ -0,0 +1,47 @@ +import json + +from databricks_dbt_factory.DbtTask import DbtTask, DbtTaskOptions + + +def test_renders_serverless_notebook_task(): + options = DbtTaskOptions( + environment_key="Default", + notebook_path="./notebooks/dbt_runner.py", + project_directory="/project", + profiles_directory="/profiles", + ) + task = DbtTask( + task_key="my_model", + commands=["dbt run --select my_model --target dev"], + options=options, + depends_on=["upstream_model"], + ) + + result = task.to_dict() + + assert result["task_key"] == "my_model" + assert result["environment_key"] == "Default" # serverless + assert "job_cluster_key" not in result + assert result["depends_on"] == [{"task_key": "upstream_model"}] + + notebook_task = result["notebook_task"] + assert notebook_task["notebook_path"] == "./notebooks/dbt_runner.py" + assert notebook_task["base_parameters"]["project_directory"] == "/project" + assert notebook_task["base_parameters"]["profiles_directory"] == "/profiles" + assert json.loads(notebook_task["base_parameters"]["dbt_commands"]) == [ + "dbt run --select my_model --target dev" + ] + + +def test_defaults_and_optional_directories_are_omitted(): + options = DbtTaskOptions(notebook_path="./runner.py") + task = DbtTask(task_key="m", commands=["dbt run --select m"], options=options) + + result = task.to_dict() + + assert result["environment_key"] == "Default" # default serverless environment + assert result["depends_on"] == [] + base_parameters = result["notebook_task"]["base_parameters"] + assert "project_directory" not in base_parameters + assert "profiles_directory" not in base_parameters + assert json.loads(base_parameters["dbt_commands"]) == ["dbt run --select m"] diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_load_resources.py b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_load_resources.py new file mode 100644 index 00000000..37a32d50 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_load_resources.py @@ -0,0 +1,121 @@ +""" +Offline test for the PyDABs integration in resources/__init__.py. + +Exercises the whole glue path against the committed dbt manifest (no Databricks workspace +required): reading the manifest, wiring the factory, generating notebook tasks, and registering +the job on a Resources object (which runs Job.from_dict, validating the task dict schema). +""" + +import json +from importlib.metadata import version +from pathlib import Path + +import pytest +import yaml +from databricks.bundles.core import Bundle + +import resources +from databricks_dbt_factory.Utils import generate_task_key + +PROJECT_ROOT = Path(__file__).resolve().parent.parent + +# The manifest is generated by `make manifest` (dbt parse). In a freshly initialized project it +# does not exist yet, so skip these tests rather than erroring until the user has run it. +pytestmark = pytest.mark.skipif( + not (PROJECT_ROOT / resources.MANIFEST_PATH).exists(), + reason="dbt manifest not found; run `make manifest` (dbt parse) first", +) + + +@pytest.fixture(autouse=True) +def _chdir_to_project_root(monkeypatch): + # resources.MANIFEST_PATH is relative to the bundle root, as it is at deploy time. + monkeypatch.chdir(PROJECT_ROOT) + + +def test_build_tasks_covers_every_buildable_node(): + # Every model/seed/snapshot node gets its own run task regardless of the BUNDLE_TESTS mode + # (tests may be collapsed into per-resource `_test` tasks when bundling is on). + tasks = resources._build_tasks("dev") + task_keys = {task["task_key"] for task in tasks} + + manifest = json.loads((PROJECT_ROOT / resources.MANIFEST_PATH).read_text()) + buildable = [ + uid + for uid, node in manifest.get("nodes", {}).items() + if node.get("resource_type") in {"model", "seed", "snapshot"} + ] + expected_keys = {generate_task_key(uid) for uid in buildable} + + assert buildable, "committed manifest has no model/seed/snapshot nodes" + assert expected_keys <= task_keys + + +def test_generated_tasks_are_serverless_notebook_tasks(): + tasks = resources._build_tasks("dev") + + for task in tasks: + assert "notebook_task" in task, task + assert task["environment_key"] == resources.ENVIRONMENT_KEY + assert "job_cluster_key" not in task # serverless only + notebook_task = task["notebook_task"] + assert notebook_task["notebook_path"] == resources.RUNNER_NOTEBOOK_PATH + # The dbt target is injected into every command. + assert "--target dev" in notebook_task["base_parameters"]["dbt_commands"] + + +def test_environment_pins_installed_dbt_databricks(): + # Single source of truth: the serverless env pins dbt-databricks to the exact + # version installed in the bundle venv (the one used to generate the manifest). + spec = resources._serverless_environment_spec() + assert spec["dependencies"] == [f"dbt-databricks=={version('dbt-databricks')}"] + + # The generated job points its serverless environment at the synced env file. + job = resources.build_job("dev") + assert job.environments[0].spec.base_environment == resources.SERVERLESS_ENV_PATH + + +def test_load_resources_registers_the_job(): + result = resources.load_resources(Bundle(target="dev")) + try: + assert resources.JOB_NAME in result.jobs + job = result.jobs[resources.JOB_NAME] + assert job.environments[0].environment_key == resources.ENVIRONMENT_KEY + + # load_resources writes the base-environment file to the bundle root for + # the bundle to sync; it pins dbt-databricks to the installed version. + env_file = PROJECT_ROOT / resources.SERVERLESS_ENV_FILE + assert env_file.exists() + content = yaml.safe_load(env_file.read_text()) + pin = f"dbt-databricks=={version('dbt-databricks')}" + assert content["dependencies"] == [pin] + finally: + (PROJECT_ROOT / resources.SERVERLESS_ENV_FILE).unlink(missing_ok=True) + + +def test_dependency_pin_rejects_non_pypi_versions(monkeypatch): + # A local or dev build cannot be pip-installed from PyPI when Databricks builds the + # serverless environment, so the deploy must fail early with a clear message. + for installed in ("1.9.0+custom", "1.13.0.dev0", "not-a-version"): + monkeypatch.setattr(resources, "version", lambda name, v=installed: v) + with pytest.raises(RuntimeError, match="PyPI"): + resources._dbt_databricks_dependency() + + +def test_environment_file_write_is_idempotent(): + env_file = PROJECT_ROOT / resources.SERVERLESS_ENV_FILE + try: + resources._write_serverless_environment_file() + env_file.chmod(0o444) + # Content already matches: no write is attempted, so a read-only file is fine. + resources._write_serverless_environment_file() + env_file.chmod(0o644) + + env_file.write_text("stale: true\n") + resources._write_serverless_environment_file() + pin = f"dbt-databricks=={version('dbt-databricks')}" + assert yaml.safe_load(env_file.read_text())["dependencies"] == [pin] + finally: + if env_file.exists(): + env_file.chmod(0o644) + env_file.unlink() diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_utils.py b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_utils.py new file mode 100644 index 00000000..a433c226 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_utils.py @@ -0,0 +1,113 @@ +from databricks_dbt_factory.Utils import ( + MAX_TASK_KEY_LENGTH, + build_task_key_maps, + bundled_test_key, + generate_task_key, +) + + +def test_resource_keys_use_name_and_verb_suffix(): + assert generate_task_key("model.shop.orders") == "orders_run" + assert generate_task_key("seed.shop.countries") == "countries_seed" + assert generate_task_key("snapshot.shop.orders_snap") == "orders_snap_snapshot" + + +def test_versioned_model_keeps_version(): + assert generate_task_key("model.shop.dim_customers.v2") == "dim_customers_v2_run" + + +def test_test_key_drops_package_and_hash(): + assert ( + generate_task_key("test.shop.unique_orders_id.9a1b2c3d4e") + == "unique_orders_id_test" + ) + # singular test (no trailing hash segment) + assert ( + generate_task_key("test.shop.assert_positive_amount") + == "assert_positive_amount_test" + ) + + +def test_source_key(): + assert generate_task_key("source.shop.raw.customers") == "raw_customers_test" + + +def test_bundled_test_key(): + assert bundled_test_key("model.shop.orders") == "orders_test" + assert bundled_test_key("seed.shop.countries") == "countries_test" + assert bundled_test_key("source.shop.raw.customers") == "raw_customers_test" + + +def test_long_test_key_is_truncated_and_hash_disambiguated(): + long_name = "accepted_values_" + "x" * 200 + key = generate_task_key(f"test.shop.{long_name}.9a1b2c3d4e") + assert len(key) <= MAX_TASK_KEY_LENGTH + assert key.endswith( + "_9a1b2c3d4e_test" + ) # hash re-added so truncated names stay unique + + +def test_key_maps_keep_plain_keys_when_unique(): + task_keys, bundled = build_task_key_maps( + ["model.shop.orders", "test.shop.unique_orders_id.9a1b2c3d4e"] + ) + assert task_keys == { + "model.shop.orders": "orders_run", + "test.shop.unique_orders_id.9a1b2c3d4e": "unique_orders_id_test", + } + assert bundled == {} + + +def test_key_maps_disambiguate_same_named_tests_with_dbt_hash(): + # The same custom test name on two models: dbt keeps them apart only via the unique_id hash. + task_keys, _ = build_task_key_maps( + ["test.shop.dup_check.6ea6b2ac82", "test.shop.dup_check.a9ab3a6e12"] + ) + assert task_keys["test.shop.dup_check.6ea6b2ac82"] == "dup_check_6ea6b2ac82_test" + assert task_keys["test.shop.dup_check.a9ab3a6e12"] == "dup_check_a9ab3a6e12_test" + + +def test_key_maps_disambiguate_singular_vs_custom_named_test(): + # A singular test file and a custom-named generic test can share a name; the singular test + # has no hash, so it gets the package folded in instead. + task_keys, _ = build_task_key_maps( + ["test.shop.raw_customers", "test.shop.raw_customers.e58cc24de2"] + ) + assert task_keys["test.shop.raw_customers"] == "shop_raw_customers_test" + assert ( + task_keys["test.shop.raw_customers.e58cc24de2"] + == "raw_customers_e58cc24de2_test" + ) + + +def test_key_maps_disambiguate_cross_package_models_with_package_name(): + task_keys, _ = build_task_key_maps( + ["model.shop.a", "model.subpkg.a", "model.shop.b"] + ) + assert task_keys["model.shop.a"] == "shop_a_run" + assert task_keys["model.subpkg.a"] == "subpkg_a_run" + assert task_keys["model.shop.b"] == "b_run" + + +def test_key_maps_disambiguate_bundled_test_key_against_task_keys(): + # Bundled mode: model `orders` claims the bundled key `orders_test`, and so does a singular + # test named `orders`. Every returned key must still be unique. + task_keys, bundled = build_task_key_maps( + ["model.shop.orders", "test.shop.orders"], + bundled_test_ids=["model.shop.orders"], + ) + all_keys = list(task_keys.values()) + list(bundled.values()) + assert len(all_keys) == len(set(all_keys)) + assert task_keys["model.shop.orders"] == "orders_run" + assert task_keys["test.shop.orders"] == "shop_orders_test" + assert bundled["model.shop.orders"] == "model_shop_orders_test" + + +def test_key_maps_disambiguated_long_test_keys_keep_hash_within_limit(): + long_name = "accepted_values_" + "y" * 200 + ids = [f"test.shop.{long_name}.6ea6b2ac82", f"test.shop.{long_name}.a9ab3a6e12"] + task_keys, _ = build_task_key_maps(ids) + assert len(set(task_keys.values())) == 2 + for uid, key in task_keys.items(): + assert len(key) <= MAX_TASK_KEY_LENGTH + assert key.endswith(f"_{uid.split('.')[3]}_test") diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/update_expected_tasks.py b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/update_expected_tasks.py new file mode 100644 index 00000000..e0ea4e69 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/update_expected_tasks.py @@ -0,0 +1,30 @@ +"""Refresh tests/test_data/expected_tasks.json — the saved copy of the job tasks the factory +generates for the sample manifest. The test `test_generated_tasks_match_expected` compares the +current output against this file, so run this script (via `make test-update-expected-tasks`) whenever +you intentionally change the generated output. + +It builds the factory the exact same way the tests do (conftest.create_dbt_factory), so the saved +copy can't drift from what the tests expect. +""" + +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from conftest import create_dbt_factory # noqa: E402 +from databricks_dbt_factory.Utils import read_dbt_manifest # noqa: E402 + +TEST_DATA = Path(__file__).resolve().parent / "test_data" + + +def main() -> None: + manifest = read_dbt_manifest(str(TEST_DATA / "manifest.json")) + tasks = create_dbt_factory().create_tasks(manifest) + (TEST_DATA / "expected_tasks.json").write_text(json.dumps(tasks, indent=2) + "\n") + print(f"Wrote {len(tasks)} tasks to {TEST_DATA / 'expected_tasks.json'}") + + +if __name__ == "__main__": + main()