diff --git a/.circleci/Dockerfile b/.circleci/Dockerfile deleted file mode 100644 index 0db762fc..00000000 --- a/.circleci/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ - -FROM cimg/python:3.7.10 AS base -USER root - -# Setup dependencies for pyodbc -RUN \ - apt-get update && \ - apt-get install -y unixodbc-dev unixodbc g++ apt-transport-https && \ - gpg --keyserver hkp://keys.gnupg.net:80 --recv-keys 5072E1F5 - -RUN \ - export ACCEPT_EULA='Y' && \ - # Install pyodbc db drivers for MSSQL - curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \ - curl https://packages.microsoft.com/config/debian/9/prod.list > /etc/apt/sources.list.d/mssql-release.list && \ - apt-get update && \ - apt-get install -y msodbcsql17 odbc-postgresql mssql-tools - -# add sqlcmd to the path -ENV PATH="$PATH:/opt/mssql-tools/bin" - -# Update odbcinst.ini to make sure full path to driver is listed -RUN \ - sed 's/Driver=psql/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psql/' /etc/odbcinst.ini > /tmp/temp.ini && \ - mv -f /tmp/temp.ini /etc/odbcinst.ini - -RUN \ - # Cleanup build dependencies - apt-get remove -y curl apt-transport-https debconf-utils rsync build-essential gnupg2 && \ - apt-get autoremove -y && apt-get autoclean -y \ No newline at end of file diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 4ff11346..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,146 +0,0 @@ -version: 2.1 - -orbs: - python: circleci/python@1.3 - azure-cli: circleci/azure-cli@1.1.0 - -jobs: - unit: - docker: &msodbc_py - - image: &docker_image dbtmsft/msodbc_py:0.5 - steps: - - checkout - - python/install-packages: - pkg-manager: pip - pip-dependency-file: dev_requirements.txt - - run: tox -- -v test/unit - integration-sqlserver: &sqlserver - docker: &msodbc_py_&_sqlserver - - image: *docker_image - - image: mcr.microsoft.com/mssql/server:2019-latest - environment: - ACCEPT_EULA: 'yes' - MSSQL_SA_PASSWORD: 5atyaNadella - MSSQL_IP_ADDRESS: 0.0.0.0 - steps: - - checkout - - python/install-packages: - pkg-manager: pip - pip-dependency-file: dev_requirements.txt - - run: - name: wait for SQL Server container to set up - command: sleep 30 - - run: - name: test connection via SQL CMD - command: sqlcmd -S 'localhost,1433' -U sa -P 5atyaNadella -Q 'create database blog' - - run: - name: Test adapter on SQL Server against dbt-adapter-tests - command: tox -- -v test/integration/sqlserver.dbtspec - connection-sqlserver: - <<: *sqlserver - steps: - - checkout - - run: &install-dbt-sqlserver - name: "install dbt-sqlserver" - command: | - pip install . - - run: - name: wait for SQL Server container to set up - command: sleep 30 - - run: &prep=connect - name: prep for connecting - command: | - mkdir -p ~/.dbt - cp test/integration/sample.profiles.yml ~/.dbt/profiles.yml - - run: - name: cnxn -- SQL Server - local sql cred - command: | - cd test/integration - dbt compile --target sqlserver_local_userpass - - run: - name: cnxn -- SQL Server - local sql cred encrypt - command: | - cd test/integration - dbt compile --target sqlserver_local_encrypt - - integration-azuresql: - docker: *msodbc_py - steps: - - checkout - - python/install-packages: - pkg-manager: pip - pip-dependency-file: dev_requirements.txt - - run: - name: Test adapter on Azure SQL against dbt-adapter-tests - command: tox -- -v test/integration/azuresql.dbtspec - connection-azuresql: - docker: *msodbc_py - steps: - - checkout - - run: *install-dbt-sqlserver - - azure-cli/install - - run: *prep=connect - - run: - name: wake up server - command: | - cd test/integration - dbt debug --target azuresql_sqlcred - - run: - name: cnxn -- Azure SQL - SQL CRED user+pass - command: | - cd test/integration - dbt compile --target azuresql_sqlcred - - azure-cli/login-with-user: - azure-username: DBT_AZURE_USERNAME - azure-password: DBT_AZURE_PASSWORD - alternate-tenant: true - azure-tenant: DBT_AZURE_TENANT - - run: - name: cnxn -- Azure SQL - AZ CLI User - command: | - cd test/integration - dbt compile --target azuresql_azcli - - azure-cli/login-with-service-principal: - azure-sp: DBT_AZURE_SP_NAME - azure-sp-password: DBT_AZURE_SP_SECRET - azure-sp-tenant: DBT_AZURE_TENANT - - run: - name: cnxn -- Azure SQL - AZ CLI ServicePrincipal - command: | - cd test/integration - dbt compile --target azuresql_azcli - - run: - name: cnxn -- Azure SQL - AZ CLI auto - command: | - cd test/integration - dbt compile --target azuresql_azauto - - run: - name: az logout - command: az logout - - run: - name: cnxn -- Azure SQL - AZ SP auto - command: | - export AZURE_CLIENT_ID="$DBT_AZURE_SP_NAME" - export AZURE_CLIENT_SECRET="$DBT_AZURE_SP_SECRET" - export AZURE_TENANT_ID="$DBT_AZURE_TENANT" - cd test/integration - dbt compile --target azuresql_azauto - # - run: - # name: cnxn -- Azure SQL - AZ SP env - # command: | - # cd test/integration - # dbt compile --target azuresql_azenv - -workflows: - main: - jobs: - - unit - - connection-azuresql: &profile - context: - - DBT_SYNAPSE_PROFILE - - connection-sqlserver: *profile - - integration-sqlserver: *profile - - integration-azuresql: - <<: *profile - requires: - - connection-azuresql diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..50980f2d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,15 @@ +--- +version: 2 +updates: + - package-ecosystem: pip + directory: "/" + schedule: + interval: daily + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: daily + - package-ecosystem: docker + directory: "/" + schedule: + interval: daily diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml new file mode 100644 index 00000000..eb9c5660 --- /dev/null +++ b/.github/workflows/integration-tests-azure.yml @@ -0,0 +1,73 @@ +--- +name: Integration tests on Azure +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - v* + - azure-testing + pull_request_target: + types: [labeled] + +jobs: + integration-tests-azure: + name: Integration tests on Azure + if: contains(github.event.pull_request.labels.*.name, 'safe to test') || github.ref_name == 'master' || github.ref_name == 'azure-testing' + strategy: + max-parallel: 1 + matrix: + python_version: ["3.7", "3.8", "3.9", "3.10"] + profile: ["ci_azure_cli", "ci_azure_auto", "ci_azure_environment", "ci_azure_basic"] + msodbc_version: ["17", "18"] + runs-on: ubuntu-latest + container: + image: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-msodbc${{ matrix.msodbc_version }} + steps: + - name: AZ CLI login + run: az login --service-principal --username="${AZURE_CLIENT_ID}" --password="${AZURE_CLIENT_SECRET}" --tenant="${AZURE_TENANT_ID}" + env: + AZURE_CLIENT_ID: ${{ secrets.DBT_AZURE_SP_NAME }} + AZURE_CLIENT_SECRET: ${{ secrets.DBT_AZURE_SP_SECRET }} + AZURE_TENANT_ID: ${{ secrets.DBT_AZURE_TENANT }} + + - uses: actions/checkout@v3 + + - name: Install dependencies + run: pip install -r dev_requirements.txt + + - name: Wake up server + env: + DBT_AZURESQL_SERVER: ${{ secrets.DBT_AZURESQL_SERVER }} + DBT_AZURESQL_DB: ${{ secrets.DBT_AZURESQL_DB }} + DBT_AZURESQL_UID: ${{ secrets.DBT_AZURESQL_UID }} + DBT_AZURESQL_PWD: ${{ secrets.DBT_AZURESQL_PWD }} + MSODBC_VERSION: ${{ matrix.msodbc_version }} + run: python devops/scripts/wakeup_azure.py + + - name: Configure test users + run: sqlcmd -b -I -i devops/scripts/create_sql_users.sql + env: + DBT_TEST_USER_1: DBT_TEST_USER_1 + DBT_TEST_USER_2: DBT_TEST_USER_2 + DBT_TEST_USER_3: DBT_TEST_USER_3 + SQLCMDUSER: ${{ secrets.DBT_AZURESQL_UID }} + SQLCMDPASSWORD: ${{ secrets.DBT_AZURESQL_PWD }} + SQLCMDSERVER: ${{ secrets.DBT_AZURESQL_SERVER }} + SQLCMDDBNAME: ${{ secrets.DBT_AZURESQL_DB }} + + - name: Run functional tests + env: + DBT_AZURESQL_SERVER: ${{ secrets.DBT_AZURESQL_SERVER }} + DBT_AZURESQL_DB: ${{ secrets.DBT_AZURESQL_DB }} + DBT_AZURESQL_UID: ${{ secrets.DBT_AZURESQL_UID }} + DBT_AZURESQL_PWD: ${{ secrets.DBT_AZURESQL_PWD }} + DBT_AZURE_USERNAME: ${{ secrets.DBT_AZURE_USERNAME }} + DBT_AZURE_PASSWORD: ${{ secrets.DBT_AZURE_PASSWORD }} + AZURE_CLIENT_ID: ${{ secrets.DBT_AZURE_SP_NAME }} + AZURE_CLIENT_SECRET: ${{ secrets.DBT_AZURE_SP_SECRET }} + AZURE_TENANT_ID: ${{ secrets.DBT_AZURE_TENANT }} + DBT_TEST_USER_1: DBT_TEST_USER_1 + DBT_TEST_USER_2: DBT_TEST_USER_2 + DBT_TEST_USER_3: DBT_TEST_USER_3 + SQLSERVER_TEST_DRIVER: 'ODBC Driver ${{ matrix.msodbc_version }} for SQL Server' + run: pytest tests/functional --profile "${{ matrix.profile }}" diff --git a/.github/workflows/integration-tests-sqlserver.yml b/.github/workflows/integration-tests-sqlserver.yml new file mode 100644 index 00000000..3ffac3de --- /dev/null +++ b/.github/workflows/integration-tests-sqlserver.yml @@ -0,0 +1,45 @@ +--- +name: Integration tests on SQL Server +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - v* + pull_request: + branches: + - master + - v* + +jobs: + integration-tests-sql-server: + name: Integration tests on SQL Server + strategy: + matrix: + python_version: ["3.7", "3.8", "3.9", "3.10"] + msodbc_version: ["17", "18"] + sqlserver_version: ["2017", "2019", "2022"] + runs-on: ubuntu-latest + container: + image: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-msodbc${{ matrix.msodbc_version }} + services: + sqlserver: + image: ghcr.io/${{ github.repository }}:server-${{ matrix.sqlserver_version }} + env: + ACCEPT_EULA: 'Y' + SA_PASSWORD: 5atyaNadella + DBT_TEST_USER_1: DBT_TEST_USER_1 + DBT_TEST_USER_2: DBT_TEST_USER_2 + DBT_TEST_USER_3: DBT_TEST_USER_3 + steps: + - uses: actions/checkout@v3 + + - name: Install dependencies + run: pip install -r dev_requirements.txt + + - name: Run functional tests + run: pytest tests/functional --profile "ci_sql_server" + env: + DBT_TEST_USER_1: DBT_TEST_USER_1 + DBT_TEST_USER_2: DBT_TEST_USER_2 + DBT_TEST_USER_3: DBT_TEST_USER_3 + SQLSERVER_TEST_DRIVER: 'ODBC Driver ${{ matrix.msodbc_version }} for SQL Server' diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml new file mode 100644 index 00000000..a7ba58ea --- /dev/null +++ b/.github/workflows/publish-docker.yml @@ -0,0 +1,70 @@ +--- +name: Publish Docker images for CI/CD +on: # yamllint disable-line rule:truthy + push: + paths: + - 'devops/**' + - '.github/workflows/publish-docker.yml' + branches: + - 'master' + +jobs: + publish-docker-client: + strategy: + matrix: + python_version: ["3.7", "3.8", "3.9", "3.10"] + docker_target: ["msodbc17", "msodbc18"] + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v2.0.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v3.1.1 + with: + context: devops + build-args: PYTHON_VERSION=${{ matrix.python_version }} + file: devops/CI.Dockerfile + push: true + platforms: linux/amd64 + target: ${{ matrix.docker_target }} + tags: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-${{ matrix.docker_target }} + + publish-docker-server: + strategy: + matrix: + mssql_version: ["2017", "2019", "2022"] + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v2.0.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v3.1.1 + with: + context: devops + build-args: MSSQL_VERSION=${{ matrix.mssql_version }} + file: devops/server.Dockerfile + push: true + platforms: linux/amd64 + tags: ghcr.io/${{ github.repository }}:server-${{ matrix.mssql_version }} diff --git a/.github/workflows/release-version.yml b/.github/workflows/release-version.yml new file mode 100644 index 00000000..e861be1b --- /dev/null +++ b/.github/workflows/release-version.yml @@ -0,0 +1,35 @@ +--- +name: Release new version + +on: # yamllint disable-line rule:truthy + push: + tags: + - 'v*' + +jobs: + release-version: + name: Release new version + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install dependencies + run: pip install -r dev_requirements.txt + + - name: Verify version match + run: python setup.py verify + + - name: Initialize .pypirc + run: | + echo -e "[pypi]" >> ~/.pypirc + echo -e "username = __token__" >> ~/.pypirc + echo -e "password = ${{ secrets.PYPI_DBT_SQLSERVER }}" >> ~/.pypirc + + - name: Build and publish package + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 00000000..dc6f2964 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,36 @@ +--- +name: Unit tests +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - v* + pull_request: + branches: + - master + - v* + +jobs: + unit-tests: + name: Unit tests + strategy: + matrix: + python_version: ["3.7", "3.8", "3.9", "3.10"] + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + container: + image: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-msodbc18 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.github_token }} + steps: + + - uses: actions/checkout@v3 + + - name: Install dependencies + run: pip install -r dev_requirements.txt + + - name: Run unit tests + run: pytest tests/unit diff --git a/.gitignore b/.gitignore index cf8a88eb..a10f0ec3 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,7 @@ htmlcov/ nosetests.xml coverage.xml *.cover +*.log.legacy # Translations *.mo @@ -90,7 +91,7 @@ target/ .mypy_cache/ # Environments -.env +*.env .venv env/ venv/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..a13b1619 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,96 @@ +default_language_version: + python: python3.9 +repos: + - repo: 'https://github.com/pre-commit/pre-commit-hooks' + rev: v4.3.0 + hooks: + - id: check-yaml + args: + - '--unsafe' + - id: check-json + - id: end-of-file-fixer + - id: trailing-whitespace + exclude_types: + - markdown + - id: check-case-conflict + - id: check-ast + - id: check-builtin-literals + - id: check-merge-conflict + - id: no-commit-to-branch + - id: fix-byte-order-marker + - id: mixed-line-ending + - id: check-docstring-first + - repo: 'https://github.com/adrienverge/yamllint' + rev: v1.28.0 + hooks: + - id: yamllint + args: + - '-d {extends: default, rules: {line-length: disable, document-start: disable}}' + - '-s' + - repo: 'https://github.com/MarcoGorelli/absolufy-imports' + rev: v0.3.1 + hooks: + - id: absolufy-imports + - repo: 'https://github.com/hadialqattan/pycln' + rev: v2.1.1 + hooks: + - id: pycln + args: + - '--all' + - repo: 'https://github.com/pycqa/isort' + rev: 5.10.1 + hooks: + - id: isort + args: + - '--profile' + - black + - '--atomic' + - '--line-length' + - '99' + - '--python-version' + - '39' + - repo: 'https://github.com/psf/black' + rev: 22.8.0 + hooks: + - id: black + args: + - '--line-length=99' + - '--target-version=py39' + - id: black + alias: black-check + stages: + - manual + args: + - '--line-length=99' + - '--target-version=py39' + - '--check' + - '--diff' + - repo: 'https://gitlab.com/pycqa/flake8' + rev: 3.9.2 + hooks: + - id: flake8 + args: + - '--max-line-length=99' + - id: flake8 + args: + - '--max-line-length=99' + alias: flake8-check + stages: + - manual + - repo: 'https://github.com/pre-commit/mirrors-mypy' + rev: v0.971 + hooks: + - id: mypy + args: + - '--show-error-codes' + - '--ignore-missing-imports' + files: '^dbt/adapters' + - id: mypy + alias: mypy-check + stages: + - manual + args: + - '--show-error-codes' + - '--pretty' + - '--ignore-missing-imports' + files: '^dbt/adapters' diff --git a/CHANGELOG.md b/CHANGELOG.md index cabbc459..785d22b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,84 @@ # Changelog +### v1.2.0 (pre-releases) + +#### Possibly breaking change: connection encryption + +For compatibility with MS ODBC Driver 18, the settings `Encrypt` and `TrustServerCertificate` are now always added to the connection string. +These are configured with the keys `encrypt` and `trust_cert` in your profile. +In previous versions, these settings were only added if they were set to `True`. + +The new version of the MS ODBC Driver sets `Encrypt` to `True` by default. +The adapter is following this change and also defaults to `True` for `Encrypt`. + +The default value for `TrustServerConnection` remains `False` as it would be a security risk otherwise. + +This means that connections made with this version of the adapter will now have `Encrypt=Yes` and `TrustServerCertificate=No` set if you are using the default settings. +You should change the settings `encrypt` or `trust_cert` to accommodate for your use case. + + +#### Features + +* Support for [dbt-core 1.2](https://github.com/dbt-labs/dbt-core/releases/tag/v1.2.0) +* Support for MS ODBC Driver 18 +* Support automatic retries with new `retries` setting introduced in core +* The correct owner of a table/view is now visible in generated documentation (and in catalog.json) +* A lot of features of dbt-utils & T-SQL utils are now available out-of-the-box in dbt-core and this adapter. A new release of T-SQL utils will follow. + * Support for all `type_*` macros + * Support for all [cross-database macros](https://docs.getdbt.com/reference/dbt-jinja-functions/cross-database-macros), except: + * `bool_or` + * `listagg` will only work in SQL Server 2017 or newer or the cloud versions. The `limit_num` option is unsupported. `DISTINCT` cannot be used in the measure. + +#### Fixes + +* In some cases the `TIMESTAMP` would be used as data type instead of `DATETIMEOFFSET`, fixed that + +#### Chores + +* Update adapter testing framework to 1.2.1 +* Update pre-commit, tox, pytest and pre-commit hooks +* Type hinting in connection class +* Automated testing with SQL Server 2017, 2019 and 2022 +* Automated testing with MS ODBC 17 and MS ODBC 18 + +#### Outstanding work for official release + +* Add documentation about new features to official dbt docs pages + +### v1.1.0 + +See changes included in v1.1.0rc1 below as well + +#### Fixes + +* [#251](https://github.com/dbt-msft/dbt-sqlserver/pull/251) fix incremental models with arrays for unique keys ([@sdebruyn](https://github.com/sdebruyn) & [@johnnytang24](https://github.com/johnnytang24)) +* [#214](https://github.com/dbt-msft/dbt-sqlserver/pull/214) fix for sources with spaces in the names ([@Freia3](https://github.com/Freia3)) +* [#238](https://github.com/dbt-msft/dbt-sqlserver/pull/238) fix snapshots breaking when new columns are added ([@jakemcaferty](https://github.com/jakemcaferty)) + +#### Chores + +* [#249](https://github.com/dbt-msft/dbt-sqlserver/pull/249) & [#250](https://github.com/dbt-msft/dbt-sqlserver/pull/251) add Python 3.10 to automated testing ([@sdebruyn](https://github.com/sdebruyn)) +* [#248](https://github.com/dbt-msft/dbt-sqlserver/pull/248) update all documentation, README and include on dbt docs ([@sdebruyn](https://github.com/sdebruyn)) +* [#252](https://github.com/dbt-msft/dbt-sqlserver/pull/252) add automated test for [#214](https://github.com/dbt-msft/dbt-sqlserver/pull/214) ([@sdebruyn](https://github.com/sdebruyn)) + +### v1.1.0.rc1 + +#### Features + +* update to dbt 1.1 + +#### Fixes + +* [#194](https://github.com/dbt-msft/dbt-sqlserver/pull/194) uppercased information_schema ([@TrololoLi](https://github.com/TrololoLi)) +* [#215](https://github.com/dbt-msft/dbt-sqlserver/pull/215) Escape schema names so they can contain strange characters ([@johnf](https://github.com/johnf)) + +#### Chores + +* Documentation on how to contribute to the adapter +* Automatic release process by adding a new tag +* Consistent code style with pre-commit +* [#201](https://github.com/dbt-msft/dbt-sqlserver/pull/201) use new dbt 1.0 logger ([@semcha](https://github.com/semcha)) +* [#216](https://github.com/dbt-msft/dbt-sqlserver/pull/216) use new dbt testing framework ([@dataders](https://github.com/dataders) & [@sdebruyn](https://github.com/sdebruyn)) ### v1.0.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..f5800a9a --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,93 @@ +# Development of the adapter + +Python 3.10 is used for developing the adapter. To get started, bootstrap your environment as follows: + +Create a virtual environment, [pyenv](https://github.com/pyenv/pyenv) is used in the example: + +```shell +pyenv install 3.10.7 +pyenv virtualenv 3.10.7 dbt-sqlserver +pyenv activate dbt-sqlserver +``` + +Install the development dependencies and pre-commit and get information about possible make commands: + +```shell +make dev +make help +``` + +[Pre-commit](https://pre-commit.com/) helps us to maintain a consistent style and code quality across the entire project. +After running `make dev`, pre-commit will automatically validate your commits and fix any formatting issues whenever possible. + +## Testing + +The functional tests require a running SQL Server instance. You can easily spin up a local instance with the following command: + +```shell +make server +``` + +This will use Docker Compose to spin up a local instance of SQL Server. Docker Compose is now bundled with Docker, so make sure to [install the latest version of Docker](https://docs.docker.com/get-docker/). + +Next, tell our tests how they should connect to the local instance by creating a file called `test.env` in the root of the project. +You can use the provided `test.env.sample` as a base and if you started the server with `make server`, then this matches the instance running on your local machine. + +```shell +cp test.env.sample test.env +``` + +You can tweak the contents of this file to test against a different database. + +Note that we need 3 users to be able to run tests related to the grants. +The 3 users are defined by the following environment variables containing their usernames. + +* `DBT_TEST_USER_1` +* `DBT_TEST_USER_2` +* `DBT_TEST_USER_3` + +You can use the following commands to run the unit and the functional tests respectively: + +```shell +make unit +make functional +``` + +## CI/CD + +We use Docker images that have all the things we need to test the adapter in the CI/CD workflows. +The Dockerfile is located in the *devops* directory and pushed to GitHub Packages to this repo. +There is one tag per supported Python version. + +All CI/CD pipelines are using GitHub Actions. The following pipelines are available: + +* `publish-docker`: publishes the image we use in all other pipelines. +* `unit-tests`: runs the unit tests for each supported Python version. +* `integration-tests-azure`: runs the integration tests for Azure SQL Server. +* `integration-tests-sqlserver`: runs the integration tests for SQL Server. +* `release-version`: publishes the adapter to PyPI. + +There is an additional [Pre-commit](https://pre-commit.ci/) pipeline that validates the code style. + +### Azure integration tests + +The following environment variables are available: + +* `DBT_AZURESQL_SERVER`: full hostname of the server hosting the Azure SQL database +* `DBT_AZURESQL_DB`: name of the Azure SQL database +* `DBT_AZURESQL_UID`: username of the SQL admin on the server hosting the Azure SQL database +* `DBT_AZURESQL_PWD`: password of the SQL admin on the server hosting the Azure SQL database +* `DBT_AZURE_TENANT`: Azure tenant ID +* `DBT_AZURE_SUBSCRIPTION_ID`: Azure subscription ID +* `DBT_AZURE_RESOURCE_GROUP_NAME`: Azure resource group name +* `DBT_AZURE_SP_NAME`: Client/application ID of the service principal used to connect to Azure AD +* `DBT_AZURE_SP_SECRET`: Password of the service principal used to connect to Azure AD +* `DBT_AZURE_USERNAME`: Username of the user to connect to Azure AD +* `DBT_AZURE_PASSWORD`: Password of the user to connect to Azure AD + +## Releasing a new version + +Make sure the version number is bumped in `__version__.py`. Then, create a git tag named `v` and push it to GitHub. +A GitHub Actions workflow will be triggered to build the package and push it to PyPI. + +If you're releasing support for a new version of `dbt-core`, also bump the `dbt_version` in `setup.py`. diff --git a/MANIFEST.in b/MANIFEST.in index 78412d5b..cfbc714e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -recursive-include dbt/include *.sql *.yml *.md \ No newline at end of file +recursive-include dbt/include *.sql *.yml *.md diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..b3ca2550 --- /dev/null +++ b/Makefile @@ -0,0 +1,71 @@ +.DEFAULT_GOAL:=help + +.PHONY: dev +dev: ## Installs adapter in develop mode along with development dependencies + @\ + pip install -r dev_requirements.txt && pre-commit install + +.PHONY: mypy +mypy: ## Runs mypy against staged changes for static type checking. + @\ + pre-commit run --hook-stage manual mypy-check | grep -v "INFO" + +.PHONY: flake8 +flake8: ## Runs flake8 against staged changes to enforce style guide. + @\ + pre-commit run --hook-stage manual flake8-check | grep -v "INFO" + +.PHONY: black +black: ## Runs black against staged changes to enforce style guide. + @\ + pre-commit run --hook-stage manual black-check -v | grep -v "INFO" + +.PHONY: lint +lint: ## Runs flake8 and mypy code checks against staged changes. + @\ + pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; \ + pre-commit run mypy-check --hook-stage manual | grep -v "INFO" + +.PHONY: all +all: ## Runs all checks against staged changes. + @\ + pre-commit run -a + +.PHONY: linecheck +linecheck: ## Checks for all Python lines 100 characters or more + @\ + find dbt -type f -name "*.py" -exec grep -I -r -n '.\{100\}' {} \; + +.PHONY: unit +unit: ## Runs unit tests. + @\ + pytest -v tests/unit + +.PHONY: functional +functional: ## Runs functional tests. + @\ + pytest -v tests/functional + +.PHONY: test +test: ## Runs unit tests and code checks against staged changes. + @\ + pytest -v tests/unit; \ + pre-commit run black-check --hook-stage manual | grep -v "INFO"; \ + pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; \ + pre-commit run mypy-check --hook-stage manual | grep -v "INFO" + +.PHONY: server +server: ## Spins up a local MS SQL Server instance for development. Docker-compose is required. + @\ + docker compose up -d + +.PHONY: clean + @echo "cleaning repo" + @git clean -f -X + +.PHONY: help +help: ## Show this help message. + @echo 'usage: make [target]' + @echo + @echo 'targets:' + @grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/README.md b/README.md index 381a36bc..5b6529bf 100644 --- a/README.md +++ b/README.md @@ -1,246 +1,68 @@ # dbt-sqlserver -[dbt](https://www.getdbt.com) adapter for sql server. -Passing all tests in [dbt-integration-tests](https://github.com/fishtown-analytics/dbt-integration-tests/). +[dbt](https://www.getdbt.com) adapter for Microsoft SQL Server and Azure SQL services. -Only supports dbt 0.14 and newer! -- For dbt 0.18.x use dbt-sqlserver 0.18.x -- dbt 0.17.x is unsupported -- dbt 0.16.x is unsupported -- For dbt 0.15.x use dbt-sqlserver 0.15.x -- For dbt 0.14.x use dbt-sqlserver 0.14.x +The adapter supports dbt-core 0.14 or newer and follows the same versioning scheme. +E.g. version 1.1.x of the adapter will be compatible with dbt-core 1.1.x. +## Documentation -Easiest install is to use pip: +We've bundled all documentation on the dbt docs site: - pip install dbt-sqlserver +* [Profile setup & authentication](https://docs.getdbt.com/reference/warehouse-profiles/mssql-profile) +* [Adapter-specific configuration](https://docs.getdbt.com/reference/resource-configs/mssql-configs) -On Ubuntu make sure you have the ODBC header files before installing +Join us on the [dbt Slack](https://getdbt.slack.com/archives/CMRMDDQ9W) to ask questions, get help, or to discuss the project. -``` -sudo apt install unixodbc-dev -``` - -## Authentication - -The following is needed for every target definition for both SQL Server and Azure SQL. The sections below details how to connect to SQL Server and Azure SQL specifically. -``` -type: sqlserver -driver: 'ODBC Driver 17 for SQL Server' (The ODBC Driver installed on your system) -server: server-host-name or ip -port: 1433 -schema: schemaname -``` - -### Security -Encryption is not enabled by default, unless you specify it. - -To enable encryption, add the following to your target definition. This is the default encryption strategy recommended by MSFT. For more information see [this docs page](https://docs.microsoft.com/en-us/dotnet/framework/data/adonet/connection-string-syntax#using-trustservercertificate?WT.mc_id=DP-MVP-5003930) -```yaml -encrypt: true # adds "Encrypt=Yes" to connection string -trust_cert: false -``` -For a fully-secure, encrypted connection, you must enable `trust_cert: false` because `"TrustServerCertificate=Yes"` is default for `dbt-sqlserver` in order to not break already defined targets. - -### standard SQL Server authentication -SQL Server credentials are supported for on-prem as well as cloud, and it is the default authentication method for `dbt-sqlsever` -``` -user: username -password: password -``` -### Windows Authentication (SQL Server-specific) - -``` -windows_login: True -``` -alternatively -``` -trusted_connection: True -``` -### Azure SQL-specific auth -The following [`pyodbc`-supported ActiveDirectory methods](https://docs.microsoft.com/en-us/sql/connect/odbc/using-azure-active-directory?view=sql-server-ver15#new-andor-modified-dsn-and-connection-string-keywords) are available to authenticate to Azure SQL: -- Auto -- Azure CLI -- Environment-based authentication -- ActiveDirectory Password -- ActiveDirectory Interactive -- ActiveDirectory Integrated -- Service Principal (a.k.a. AAD Application) -- Managed Identity - -Usually the automatic option is the easiest one to use since it will work with any configuration already present in your environment, as explained below. - -#### Auto - -This will try to authenticate by using the following methods one by one until it finds a valid way to authenticate: - -1. Read credentials from environment variables (see environment-based authentication below) -2. Use the managed identity of the system (see MSI below) -3. VS Code: use the account used to log in to the VS Code Azure extension if installed -4. Use the logged account in the Azure CLI if installed (see below) -5. Azure PowerShell: use the account used with `Connect-AzAccount` in the Azure PowerShell module if installed - -To use automatic authentication, set `authentication` in `profiles.yml` to `Auto`: - -```yaml -authentication: Auto -``` - -This is the recommended way for authenticating to databases on Azure because it avoids storing credentials in your -profile and can resort to different authentication mechanisms depending on the system you're running dbt on. - -#### Azure CLI -Use the authentication of the Azure command line interface (CLI). First, [install the Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli), then, log in: - -```bash -az login -``` - -Then, set `authentication` in `profiles.yml` to `CLI`: - -```yaml -authentication: CLI -``` - -#### Environment-based authentication -You can let dbt dynamically use credentials from your environment variables by configuring -your profile with environment-based authentication: - -```yaml -authenticaton: environment -``` +## Installation -You can configure the following environment variables: +This adapter requires the Microsoft ODBC driver to be installed: +[Windows](https://docs.microsoft.com/nl-be/sql/connect/odbc/download-odbc-driver-for-sql-server?view=sql-server-ver16#download-for-windows) | +[macOS](https://docs.microsoft.com/nl-be/sql/connect/odbc/linux-mac/install-microsoft-odbc-driver-sql-server-macos?view=sql-server-ver16) | +[Linux](https://docs.microsoft.com/nl-be/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server?view=sql-server-ver16) -**Service principal authentication**: -* AZURE_CLIENT_ID -* AZURE_TENANT_ID -* AZURE_CLIENT_SECRET or AZURE_CLIENT_CERTIFICATE_PATH +
Debian/Ubuntu +

-**User authenticaton**: -* AZURE_USERNAME -* AZURE_PASSWORD -* AZURE_CLIENT_ID +Make sure to install the ODBC headers as well as the driver linked above: -#### ActiveDirectory Password -Definitely not ideal, but available - -```yaml -authentication: ActiveDirectoryPassword -user: bill.gates@microsoft.com -password: i<3opensource? +```shell +sudo apt-get install -y unixodbc-dev ``` -#### ActiveDirectory Interactive (*Windows only*) -Brings up the Azure AD prompt so you can MFA if need be. The downside to this approach is that you must log in each time you run a dbt command! - -```yaml -authentication: ActiveDirectoryInteractive -user: bill.gates@microsoft.com -``` +

+
-#### ActiveDirectory Integrated (*Windows only*) -Uses your machine's credentials (might be disabled by your AAD admins), also requires that you have Active Directory Federation Services (ADFS) installed and running, which is only the case if you have an on-prem Active Directory linked to your Azure AD... +Latest version: ![PyPI](https://img.shields.io/pypi/v/dbt-sqlserver?label=latest%20stable&logo=pypi) -```yaml -authentication: ActiveDirectoryIntegrated +```shell +pip install -U dbt-sqlserver ``` -##### Service Principal -`client_*` and `app_*` can be used interchangeably. Again, it is not recommended to store a service principal secret in plain text in your `dbt_profile.yml`. The auto, environment or CLI auth methods are preferred over this one. - -```yaml -authentication: ServicePrincipal -tenant_id: tenatid -client_id: clientid -client_secret: clientsecret -``` - -#### Managed Identity - -If the system you're running dbt on has a [managed identity](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/overview), -then you can configure the authentication like so: - -```yaml -authentication: MSI -``` - -## Supported features - -### Materializations -- Table: - - Will be materialized as columns store index by default (requires SQL Server 2017 as least). - (For Azure SQL requires Service Tier greater than S2) - To override: -{{ - config( - as_columnstore = false, - ) -}} -- View -- Incremental -- Ephemeral +Latest pre-release: ![GitHub tag (latest SemVer pre-release)](https://img.shields.io/github/v/tag/dbt-msft/dbt-sqlserver?include_prereleases&label=latest%20pre-release&logo=pypi) -### Seeds - -By default, dbt-sqlserver will attempt to insert seed files in batches of 400 rows. If this exceeds SQL Server's 2100 parameter limit, the adapter will automatically limit to the highest safe value possible. - -To set a different default seed value, you can set the variable `max_batch_size` in your project configuration. - -```yaml -vars: - max_batch_size: 200 # Any integer less than or equal to 2100 will do. +```shell +pip install -U --pre dbt-sqlserver ``` -### Hooks - -### Custom schemas - -### Sources - - -### Testing & documentation -- Schema test supported -- Data tests supported from dbt 0.14.1 -- Docs - -### Snapshots -- Timestamp -- Check - -But, columns in source table can not have any constraints. If for example any column has a NOT NULL constraint, an error will be thrown. - -### DBT Utils -Many DBT utils macros are supported, but they require the addition of the `tsql_utils` dbt package. - -You can find the package and installation instructions in the [tsql-utils repo](https://github.com/dbt-msft/tsql-utils). +## Changelog -### Indexes -There is now possible to define a regular sql server index on a table. -This is best used when the default clustered columnstore index materialisation is not suitable. -One reason would be that you need a large table that usually is queried one row at a time. +See [the changelog](CHANGELOG.md) -Clusterad and non-clustered index are supported: -- create_clustered_index(columns, unique=False) -- create_nonclustered_index(columns, includes=False) -- drop_all_indexes_on_table(): Drops current indexex on a table. Only meaningfull if model is incremental. +## Contributing +[![Unit tests](https://github.com/dbt-msft/dbt-sqlserver/actions/workflows/unit-tests.yml/badge.svg)](https://github.com/dbt-msft/dbt-sqlserver/actions/workflows/unit-tests.yml) +[![Integration tests on SQL Server](https://github.com/dbt-msft/dbt-sqlserver/actions/workflows/integration-tests-sqlserver.yml/badge.svg)](https://github.com/dbt-msft/dbt-sqlserver/actions/workflows/integration-tests-sqlserver.yml) +[![Integration tests on Azure](https://github.com/dbt-msft/dbt-sqlserver/actions/workflows/integration-tests-azure.yml/badge.svg)](https://github.com/dbt-msft/dbt-sqlserver/actions/workflows/integration-tests-azure.yml) -Example of applying Unique clustered index on two columns, Ordinary index on one column, Ordinary index on one column with another column included +This adapter is community-maintained. +You are welcome to contribute by creating issues, opening or reviewing pull requests or helping other users in Slack channel. +If you're unsure how to get started, check out our [contributing guide](CONTRIBUTING.md). - {{ - config({ - "as_columnstore": false, - "materialized": 'table', - "post-hook": [ - "{{ create_clustered_index(columns = ['row_id', 'row_id_complement'], unique=True) }}", - "{{ create_nonclustered_index(columns = ['modified_date']) }}", - "{{ create_nonclustered_index(columns = ['row_id'], includes = ['modified_date']) }}", - ] - }) - }} +## License +[![PyPI - License](https://img.shields.io/pypi/l/dbt-sqlserver)](https://github.com/dbt-msft/dbt-sqlserver/blob/master/LICENSE) -## Changelog +## Code of Conduct -See [the changelog](CHANGELOG.md) +This project and everyone involved is expected to follow the [dbt Code of Conduct](https://community.getdbt.com/code-of-conduct). diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/dbt/__init__.py b/dbt/__init__.py index 07acae52..8db66d3d 100644 --- a/dbt/__init__.py +++ b/dbt/__init__.py @@ -1 +1 @@ -__path__ = __import__("pkgutil").extend_path(__path__, __name__) \ No newline at end of file +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/dbt/adapters/__init__.py b/dbt/adapters/__init__.py index 07acae52..8db66d3d 100644 --- a/dbt/adapters/__init__.py +++ b/dbt/adapters/__init__.py @@ -1 +1 @@ -__path__ = __import__("pkgutil").extend_path(__path__, __name__) \ No newline at end of file +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/dbt/adapters/sqlserver/__init__.py b/dbt/adapters/sqlserver/__init__.py index c1b2c56b..349d3a26 100644 --- a/dbt/adapters/sqlserver/__init__.py +++ b/dbt/adapters/sqlserver/__init__.py @@ -1,13 +1,21 @@ -from dbt.adapters.sqlserver.connections import SQLServerConnectionManager -from dbt.adapters.sqlserver.connections import SQLServerCredentials -from dbt.adapters.sqlserver.impl import SQLServerAdapter - from dbt.adapters.base import AdapterPlugin -from dbt.include import sqlserver +from dbt.adapters.sqlserver.sql_server_adapter import SQLServerAdapter +from dbt.adapters.sqlserver.sql_server_column import SQLServerColumn +from dbt.adapters.sqlserver.sql_server_connection_manager import SQLServerConnectionManager +from dbt.adapters.sqlserver.sql_server_credentials import SQLServerCredentials +from dbt.include import sqlserver Plugin = AdapterPlugin( adapter=SQLServerAdapter, credentials=SQLServerCredentials, include_path=sqlserver.PACKAGE_PATH, ) + +__all__ = [ + "Plugin", + "SQLServerConnectionManager", + "SQLServerColumn", + "SQLServerAdapter", + "SQLServerCredentials", +] diff --git a/dbt/adapters/sqlserver/__version__.py b/dbt/adapters/sqlserver/__version__.py index de43469d..1eb567ae 100644 --- a/dbt/adapters/sqlserver/__version__.py +++ b/dbt/adapters/sqlserver/__version__.py @@ -1 +1 @@ -version = '1.0.0' +version = "1.2.0b2" diff --git a/dbt/adapters/sqlserver/impl.py b/dbt/adapters/sqlserver/sql_server_adapter.py similarity index 75% rename from dbt/adapters/sqlserver/impl.py rename to dbt/adapters/sqlserver/sql_server_adapter.py index 21acb806..cf345fed 100644 --- a/dbt/adapters/sqlserver/impl.py +++ b/dbt/adapters/sqlserver/sql_server_adapter.py @@ -1,15 +1,16 @@ -from dbt.adapters.sql import SQLAdapter -from dbt.adapters.sqlserver import SQLServerConnectionManager -from dbt.adapters.base.relation import BaseRelation +from typing import List, Optional + import agate -from typing import ( - Optional, Tuple, Callable, Iterable, Type, Dict, Any, List, Mapping, - Iterator, Union, Set -) +from dbt.adapters.base.relation import BaseRelation +from dbt.adapters.sql import SQLAdapter + +from dbt.adapters.sqlserver.sql_server_column import SQLServerColumn +from dbt.adapters.sqlserver.sql_server_connection_manager import SQLServerConnectionManager class SQLServerAdapter(SQLAdapter): ConnectionManager = SQLServerConnectionManager + Column = SQLServerColumn @classmethod def date_function(cls): @@ -42,9 +43,7 @@ def convert_time_type(cls, agate_table, col_idx): return "datetime" # Methods used in adapter tests - def timestamp_add_sql( - self, add_to: str, number: int = 1, interval: str = "hour" - ) -> str: + def timestamp_add_sql(self, add_to: str, number: int = 1, interval: str = "hour") -> str: # note: 'interval' is not supported for T-SQL # for backwards compatibility, we're compelled to set some sort of # default. A lot of searching has lead me to believe that the @@ -53,19 +52,20 @@ def timestamp_add_sql( return f"DATEADD({interval},{number},{add_to})" def string_add_sql( - self, add_to: str, value: str, location='append', + self, + add_to: str, + value: str, + location="append", ) -> str: """ `+` is T-SQL's string concatenation operator """ - if location == 'append': + if location == "append": return f"{add_to} + '{value}'" - elif location == 'prepend': + elif location == "prepend": return f"'{value}' + {add_to}" else: - raise RuntimeException( - f'Got an unexpected location value of "{location}"' - ) + raise ValueError(f'Got an unexpected location value of "{location}"') def get_rows_different_sql( self, @@ -99,6 +99,26 @@ def get_rows_different_sql( return sql + # This is for use in the test suite + def run_sql_for_tests(self, sql, fetch, conn): + cursor = conn.handle.cursor() + try: + cursor.execute(sql) + if not fetch: + conn.handle.commit() + if fetch == "one": + return cursor.fetchone() + elif fetch == "all": + return cursor.fetchall() + else: + return + except BaseException: + if conn.handle and not getattr(conn.handle, "closed", True): + conn.handle.rollback() + raise + finally: + conn.transaction_open = False + COLUMNS_EQUAL_SQL = """ with diff_count as ( diff --git a/dbt/adapters/sqlserver/sql_server_column.py b/dbt/adapters/sqlserver/sql_server_column.py new file mode 100644 index 00000000..72dff888 --- /dev/null +++ b/dbt/adapters/sqlserver/sql_server_column.py @@ -0,0 +1,12 @@ +from typing import ClassVar, Dict + +from dbt.adapters.base import Column + + +class SQLServerColumn(Column): + TYPE_LABELS: ClassVar[Dict[str, str]] = { + "STRING": "VARCHAR(MAX)", + "TIMESTAMP": "DATETIMEOFFSET", + "FLOAT": "FLOAT", + "INTEGER": "INT", + } diff --git a/dbt/adapters/sqlserver/connections.py b/dbt/adapters/sqlserver/sql_server_connection_manager.py similarity index 64% rename from dbt/adapters/sqlserver/connections.py rename to dbt/adapters/sqlserver/sql_server_connection_manager.py index a66152a2..e981e63d 100644 --- a/dbt/adapters/sqlserver/connections.py +++ b/dbt/adapters/sqlserver/sql_server_connection_manager.py @@ -1,89 +1,32 @@ import struct import time from contextlib import contextmanager -from dataclasses import dataclass from itertools import chain, repeat -from typing import Callable, Dict, Mapping -from typing import Optional +from typing import Any, Callable, Dict, Mapping, Optional, Tuple +import agate import dbt.exceptions import pyodbc from azure.core.credentials import AccessToken from azure.identity import ( AzureCliCredential, - ManagedIdentityCredential, ClientSecretCredential, DefaultAzureCredential, EnvironmentCredential, + ManagedIdentityCredential, ) -from dbt.adapters.base import Credentials from dbt.adapters.sql import SQLConnectionManager -from dbt.contracts.connection import AdapterResponse -from dbt.logger import GLOBAL_LOGGER as logger +from dbt.clients.agate_helper import empty_table +from dbt.contracts.connection import AdapterResponse, Connection, ConnectionState +from dbt.events import AdapterLogger from dbt.adapters.sqlserver import __version__ +from dbt.adapters.sqlserver.sql_server_credentials import SQLServerCredentials AZURE_CREDENTIAL_SCOPE = "https://database.windows.net//.default" _TOKEN: Optional[AccessToken] = None - -@dataclass -class SQLServerCredentials(Credentials): - driver: str - host: str - database: str - schema: str - port: Optional[int] = 1433 - UID: Optional[str] = None - PWD: Optional[str] = None - windows_login: Optional[bool] = False - tenant_id: Optional[str] = None - client_id: Optional[str] = None - client_secret: Optional[str] = None - # "sql", "ActiveDirectoryPassword" or "ActiveDirectoryInteractive", or - # "ServicePrincipal" - authentication: Optional[str] = "sql" - encrypt: Optional[bool] = False - trust_cert: Optional[bool] = False - - _ALIASES = { - "user": "UID", - "username": "UID", - "pass": "PWD", - "password": "PWD", - "server": "host", - "trusted_connection": "windows_login", - "auth": "authentication", - "app_id": "client_id", - "app_secret": "client_secret", - "TrustServerCertificate": "trust_cert", - } - - @property - def type(self): - return "sqlserver" - - def _connection_keys(self): - # return an iterator of keys to pretty-print in 'dbt debug' - # raise NotImplementedError - if self.windows_login is True: - self.authentication = "Windows Login" - - return ( - "server", - "database", - "schema", - "port", - "UID", - "client_id", - "authentication", - "encrypt", - "trust_cert", - ) - - @property - def unique_field(self): - return self.host +logger = AdapterLogger("SQLServer") def convert_bytes_to_mswindows_byte_string(value: bytes) -> bytes: @@ -216,7 +159,7 @@ def get_sp_access_token(credentials: SQLServerCredentials) -> AccessToken: The access token. """ token = ClientSecretCredential( - credentials.tenant_id, credentials.client_id, credentials.client_secret + str(credentials.tenant_id), str(credentials.client_id), str(credentials.client_secret) ).get_token(AZURE_CREDENTIAL_SCOPE) return token @@ -253,11 +196,9 @@ def get_pyodbc_attrs_before(credentials: SQLServerCredentials) -> Dict: "environment": get_environment_access_token, } - authentication = credentials.authentication.lower() + authentication = str(credentials.authentication).lower() if authentication in azure_auth_functions: - time_remaining = ( - (_TOKEN.expires_on - time.time()) if _TOKEN else MAX_REMAINING_TIME - ) + time_remaining = (_TOKEN.expires_on - time.time()) if _TOKEN else MAX_REMAINING_TIME if _TOKEN is None or (time_remaining < MAX_REMAINING_TIME): azure_auth_function = azure_auth_functions[authentication] @@ -272,6 +213,25 @@ def get_pyodbc_attrs_before(credentials: SQLServerCredentials) -> Dict: return attrs_before +def bool_to_connection_string_arg(key: str, value: bool) -> str: + """ + Convert a boolean to a connection string argument. + + Parameters + ---------- + key : str + The key to use in the connection string. + value : bool + The boolean to convert. + + Returns + ------- + out : str + The connection string argument. + """ + return f'{key}={"Yes" if value else "No"}' + + class SQLServerConnectionManager(SQLConnectionManager): TYPE = "sqlserver" @@ -288,7 +248,6 @@ def exception_handler(self, sql): self.release() except pyodbc.Error: logger.debug("Failed to release connection!") - pass raise dbt.exceptions.DatabaseException(str(e).strip()) from e @@ -305,68 +264,74 @@ def exception_handler(self, sql): raise dbt.exceptions.RuntimeException(e) @classmethod - def open(cls, connection): + def open(cls, connection: Connection) -> Connection: - if connection.state == "open": + if connection.state == ConnectionState.OPEN: logger.debug("Connection is already open, skipping open.") return connection - credentials = connection.credentials + credentials = cls.get_credentials(connection.credentials) - try: - con_str = [] - con_str.append(f"DRIVER={{{credentials.driver}}}") + con_str = [f"DRIVER={{{credentials.driver}}}"] - if "\\" in credentials.host: - # if there is a backslash \ in the host name the host is a sql-server named instance - # in this case then port number has to be omitted - con_str.append(f"SERVER={credentials.host}") - else: - con_str.append(f"SERVER={credentials.host},{credentials.port}") + if "\\" in credentials.host: - con_str.append(f"Database={credentials.database}") + # If there is a backslash \ in the host name, the host is a + # SQL Server named instance. In this case then port number has to be omitted. + con_str.append(f"SERVER={credentials.host}") + else: + con_str.append(f"SERVER={credentials.host},{credentials.port}") - type_auth = getattr(credentials, "authentication", "sql") + con_str.append(f"Database={credentials.database}") - if "ActiveDirectory" in type_auth: - con_str.append(f"Authentication={credentials.authentication}") + assert credentials.authentication is not None - if type_auth == "ActiveDirectoryPassword": - con_str.append(f"UID={{{credentials.UID}}}") - con_str.append(f"PWD={{{credentials.PWD}}}") - elif type_auth == "ActiveDirectoryInteractive": - con_str.append(f"UID={{{credentials.UID}}}") + if "ActiveDirectory" in credentials.authentication: + con_str.append(f"Authentication={credentials.authentication}") - elif getattr(credentials, "windows_login", False): - con_str.append(f"trusted_connection=yes") - elif type_auth == "sql": + if credentials.authentication == "ActiveDirectoryPassword": con_str.append(f"UID={{{credentials.UID}}}") con_str.append(f"PWD={{{credentials.PWD}}}") + elif credentials.authentication == "ActiveDirectoryInteractive": + con_str.append(f"UID={{{credentials.UID}}}") + + elif credentials.windows_login: + con_str.append("trusted_connection=Yes") + elif credentials.authentication == "sql": + con_str.append(f"UID={{{credentials.UID}}}") + con_str.append(f"PWD={{{credentials.PWD}}}") + + # https://docs.microsoft.com/en-us/sql/relational-databases/native-client/features/using-encryption-without-validation?view=sql-server-ver15 + assert credentials.encrypt is not None + assert credentials.trust_cert is not None + + con_str.append(bool_to_connection_string_arg("encrypt", credentials.encrypt)) + con_str.append( + bool_to_connection_string_arg("TrustServerCertificate", credentials.trust_cert) + ) - # still confused whether to use "Yes", "yes", "True", or "true" - # to learn more visit - # https://docs.microsoft.com/en-us/sql/relational-databases/native-client/features/using-encryption-without-validation?view=sql-server-ver15 - if getattr(credentials, "encrypt", False) is True: - con_str.append(f"Encrypt=Yes") - if getattr(credentials, "trust_cert", False) is True: - con_str.append(f"TrustServerCertificate=Yes") + plugin_version = __version__.version + application_name = f"dbt-{credentials.type}/{plugin_version}" + con_str.append(f"Application Name={application_name}") - plugin_version = __version__.version - application_name = f"dbt-{credentials.type}/{plugin_version}" - con_str.append(f"Application Name={application_name}") + con_str_concat = ";".join(con_str) - con_str_concat = ";".join(con_str) + index = [] + for i, elem in enumerate(con_str): + if "pwd=" in elem.lower(): + index.append(i) - index = [] - for i, elem in enumerate(con_str): - if "pwd=" in elem.lower(): - index.append(i) + if len(index) != 0: + con_str[index[0]] = "PWD=***" - if len(index) != 0: - con_str[index[0]] = "PWD=***" + con_str_display = ";".join(con_str) - con_str_display = ";".join(con_str) + retryable_exceptions = [ # https://github.com/mkleehammer/pyodbc/wiki/Exceptions + pyodbc.InternalError, # not used according to docs, but defined in PEP-249 + pyodbc.OperationalError, + ] + def connect(): logger.debug(f"Using connection string: {con_str_display}") attrs_before = get_pyodbc_attrs_before(credentials) @@ -375,24 +340,19 @@ def open(cls, connection): attrs_before=attrs_before, autocommit=True, ) - - connection.state = "open" - connection.handle = handle logger.debug(f"Connected to db: {credentials.database}") + return handle + + return cls.retry_connection( + connection, + connect=connect, + logger=logger, + retry_limit=credentials.retries, + retryable_exceptions=retryable_exceptions, + ) - except pyodbc.Error as e: - logger.debug(f"Could not connect to db: {e}") - - connection.handle = None - connection.state = "fail" - - raise dbt.exceptions.FailedToConnectException(str(e)) - - return connection - - def cancel(self, connection): + def cancel(self, connection: Connection): logger.debug("Cancel query") - pass def add_begin_query(self): # return self.add_query('BEGIN TRANSACTION', auto_begin=False) @@ -402,7 +362,13 @@ def add_commit_query(self): # return self.add_query('COMMIT TRANSACTION', auto_begin=False) pass - def add_query(self, sql, auto_begin=True, bindings=None, abridge_sql_log=False): + def add_query( + self, + sql: str, + auto_begin: bool = True, + bindings: Optional[Any] = None, + abridge_sql_log: bool = False, + ) -> Tuple[Connection, Any]: connection = self.get_thread_connection() @@ -435,11 +401,11 @@ def add_query(self, sql, auto_begin=True, bindings=None, abridge_sql_log=False): return connection, cursor @classmethod - def get_credentials(cls, credentials): + def get_credentials(cls, credentials: SQLServerCredentials) -> SQLServerCredentials: return credentials @classmethod - def get_response(cls, cursor) -> AdapterResponse: + def get_response(cls, cursor: Any) -> AdapterResponse: # message = str(cursor.statusmessage) message = "OK" rows = cursor.rowcount @@ -456,7 +422,9 @@ def get_response(cls, cursor) -> AdapterResponse: rows_affected=rows, ) - def execute(self, sql, auto_begin=True, fetch=False): + def execute( + self, sql: str, auto_begin: bool = True, fetch: bool = False + ) -> Tuple[AdapterResponse, agate.Table]: _, cursor = self.add_query(sql, auto_begin) response = self.get_response(cursor) if fetch: @@ -466,7 +434,7 @@ def execute(self, sql, auto_begin=True, fetch=False): break table = self.get_result_from_cursor(cursor) else: - table = dbt.clients.agate_helper.empty_table() + table = empty_table() # Step through all result sets so we process all errors while cursor.nextset(): pass diff --git a/dbt/adapters/sqlserver/sql_server_credentials.py b/dbt/adapters/sqlserver/sql_server_credentials.py new file mode 100644 index 00000000..96412ac4 --- /dev/null +++ b/dbt/adapters/sqlserver/sql_server_credentials.py @@ -0,0 +1,62 @@ +from dataclasses import dataclass +from typing import Optional + +from dbt.contracts.connection import Credentials + + +@dataclass +class SQLServerCredentials(Credentials): + driver: str + host: str + database: str + schema: str + port: Optional[int] = 1433 + UID: Optional[str] = None + PWD: Optional[str] = None + windows_login: Optional[bool] = False + tenant_id: Optional[str] = None + client_id: Optional[str] = None + client_secret: Optional[str] = None + authentication: Optional[str] = "sql" + encrypt: Optional[bool] = True # default value in MS ODBC Driver 18 as well + trust_cert: Optional[bool] = False # default value in MS ODBC Driver 18 as well + retries: int = 1 + + _ALIASES = { + "user": "UID", + "username": "UID", + "pass": "PWD", + "password": "PWD", + "server": "host", + "trusted_connection": "windows_login", + "auth": "authentication", + "app_id": "client_id", + "app_secret": "client_secret", + "TrustServerCertificate": "trust_cert", + } + + @property + def type(self): + return "sqlserver" + + def _connection_keys(self): + # return an iterator of keys to pretty-print in 'dbt debug' + # raise NotImplementedError + if self.windows_login is True: + self.authentication = "Windows Login" + + return ( + "server", + "database", + "schema", + "port", + "UID", + "client_id", + "authentication", + "encrypt", + "trust_cert", + ) + + @property + def unique_field(self): + return self.host diff --git a/dbt/include/__init__.py b/dbt/include/__init__.py index 07acae52..8db66d3d 100644 --- a/dbt/include/__init__.py +++ b/dbt/include/__init__.py @@ -1 +1 @@ -__path__ = __import__("pkgutil").extend_path(__path__, __name__) \ No newline at end of file +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/dbt/include/sqlserver/dbt_project.yml b/dbt/include/sqlserver/dbt_project.yml index 08aa128f..8952ba41 100644 --- a/dbt/include/sqlserver/dbt_project.yml +++ b/dbt/include/sqlserver/dbt_project.yml @@ -1,4 +1,3 @@ - name: dbt_sqlserver version: 1.0 diff --git a/dbt/include/sqlserver/macros/adapters.sql b/dbt/include/sqlserver/macros/adapters.sql deleted file mode 100644 index 38fbb762..00000000 --- a/dbt/include/sqlserver/macros/adapters.sql +++ /dev/null @@ -1,3 +0,0 @@ -{# {% macro sqlserver__insert_into_from(to_relation, from_relation) -%} - SELECT * INTO {{ to_relation }} FROM {{ from_relation }} -{% endmacro %} #} diff --git a/dbt/include/sqlserver/macros/adapters/apply_grants.sql b/dbt/include/sqlserver/macros/adapters/apply_grants.sql new file mode 100644 index 00000000..2f00abcc --- /dev/null +++ b/dbt/include/sqlserver/macros/adapters/apply_grants.sql @@ -0,0 +1,9 @@ +{% macro sqlserver__get_show_grant_sql(relation) %} + select + GRANTEE as grantee, + PRIVILEGE_TYPE as privilege_type + from INFORMATION_SCHEMA.TABLE_PRIVILEGES + where TABLE_CATALOG = '{{ relation.database }}' + and TABLE_SCHEMA = '{{ relation.schema }}' + and TABLE_NAME = '{{ relation.identifier }}' + {% endmacro %} diff --git a/dbt/include/sqlserver/macros/adapters/columns.sql b/dbt/include/sqlserver/macros/adapters/columns.sql index df0f15e9..851df265 100644 --- a/dbt/include/sqlserver/macros/adapters/columns.sql +++ b/dbt/include/sqlserver/macros/adapters/columns.sql @@ -1,34 +1,49 @@ {% macro sqlserver__get_columns_in_relation(relation) -%} {% call statement('get_columns_in_relation', fetch_result=True) %} - SELECT - column_name, - data_type, - character_maximum_length, - numeric_precision, - numeric_scale - FROM - (select - ordinal_position, - column_name, - data_type, - character_maximum_length, - numeric_precision, - numeric_scale - from [{{ relation.database }}].INFORMATION_SCHEMA.COLUMNS - where table_name = '{{ relation.identifier }}' - and table_schema = '{{ relation.schema }}' - UNION ALL - select - ordinal_position, - column_name collate database_default, - data_type collate database_default, - character_maximum_length, - numeric_precision, - numeric_scale - from tempdb.INFORMATION_SCHEMA.COLUMNS - where table_name like '{{ relation.identifier }}%') cols - order by ordinal_position + with + regular_db_cols as ( + select + ordinal_position, + column_name, + data_type, + character_maximum_length, + numeric_precision, + numeric_scale + from [{{ relation.database }}].INFORMATION_SCHEMA.COLUMNS + where table_name = '{{ relation.identifier }}' + and table_schema = '{{ relation.schema }}' + ), + + temp_db_cols as ( + select + ordinal_position, + column_name collate database_default as column_name, + data_type collate database_default as data_type, + character_maximum_length, + numeric_precision, + numeric_scale + from tempdb.INFORMATION_SCHEMA.COLUMNS + where table_name like '{{ relation.identifier }}%' + ), + + all_cols as ( + select * + from regular_db_cols + union + select * + from temp_db_cols + ) + + select + column_name, + data_type, + character_maximum_length, + numeric_precision, + numeric_scale + from + all_cols + order by ordinal_position {% endcall %} {% set table = load_result('get_columns_in_relation').table %} @@ -60,4 +75,4 @@ {%- endcall -%} -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/dbt/include/sqlserver/macros/adapters/freshness.sql b/dbt/include/sqlserver/macros/adapters/freshness.sql index 60268d6e..84519e5b 100644 --- a/dbt/include/sqlserver/macros/adapters/freshness.sql +++ b/dbt/include/sqlserver/macros/adapters/freshness.sql @@ -1,3 +1,3 @@ {% macro sqlserver__current_timestamp() -%} SYSDATETIME() -{%- endmacro %} \ No newline at end of file +{%- endmacro %} diff --git a/dbt/include/sqlserver/macros/adapters/indexes.sql b/dbt/include/sqlserver/macros/adapters/indexes.sql index 9f385490..359d7f71 100644 --- a/dbt/include/sqlserver/macros/adapters/indexes.sql +++ b/dbt/include/sqlserver/macros/adapters/indexes.sql @@ -1,7 +1,7 @@ {% macro sqlserver__create_clustered_columnstore_index(relation) -%} - {%- set cci_name = relation.schema ~ '_' ~ relation.identifier ~ '_cci' -%} + {%- set cci_name = (relation.schema ~ '_' ~ relation.identifier ~ '_cci') | replace(".", "") | replace(" ", "") -%} {%- set relation_name = relation.schema ~ '_' ~ relation.identifier -%} - {%- set full_relation = relation.schema ~ '.' ~ relation.identifier -%} + {%- set full_relation = '"' ~ relation.schema ~ '"."' ~ relation.identifier ~ '"' -%} use [{{ relation.database }}]; if EXISTS ( SELECT * FROM @@ -132,9 +132,9 @@ select @drop_remaining_indexes_last = ( {% set idx_name = this.table + '__clustered_index_on_' + columns|join('_') %} -if not exists(select * from sys.indexes - where - name = '{{ idx_name }}' and +if not exists(select * from sys.indexes + where + name = '{{ idx_name }}' and object_id = OBJECT_ID('{{ this }}') ) begin @@ -156,9 +156,9 @@ end {% set idx_name = this.table + '__index_on_' + columns|join('_')|replace(" ", "_") %} -if not exists(select * from sys.indexes - where - name = '{{ idx_name }}' and +if not exists(select * from sys.indexes + where + name = '{{ idx_name }}' and object_id = OBJECT_ID('{{ this }}') ) begin diff --git a/dbt/include/sqlserver/macros/adapters/metadata.sql b/dbt/include/sqlserver/macros/adapters/metadata.sql index 8dee2beb..1182f81e 100644 --- a/dbt/include/sqlserver/macros/adapters/metadata.sql +++ b/dbt/include/sqlserver/macros/adapters/metadata.sql @@ -3,17 +3,84 @@ {%- call statement('catalog', fetch_result=True) -%} - with tabs as ( + with + principals as ( + select + name as principal_name, + principal_id as principal_id + from + sys.database_principals + ), + + schemas as ( + select + name as schema_name, + schema_id as schema_id, + principal_id as principal_id + from + sys.schemas + ), + + tables as ( + select + name as table_name, + schema_id as schema_id, + principal_id as principal_id, + 'BASE TABLE' as table_type + from + sys.tables + ), - select - TABLE_CATALOG as table_database, - TABLE_SCHEMA as table_schema, - TABLE_NAME as table_name, - TABLE_TYPE as table_type, - TABLE_SCHEMA as table_owner, - null as table_comment - from INFORMATION_SCHEMA.TABLES + tables_with_metadata as ( + select + table_name, + schema_name, + coalesce(tables.principal_id, schemas.principal_id) as owner_principal_id, + table_type + from + tables + join schemas on tables.schema_id = schemas.schema_id + ), + views as ( + select + name as table_name, + schema_id as schema_id, + principal_id as principal_id, + 'VIEW' as table_type + from + sys.views + ), + + views_with_metadata as ( + select + table_name, + schema_name, + coalesce(views.principal_id, schemas.principal_id) as owner_principal_id, + table_type + from + views + join schemas on views.schema_id = schemas.schema_id + ), + + tables_and_views as ( + select + table_name, + schema_name, + principal_name, + table_type + from + tables_with_metadata + join principals on tables_with_metadata.owner_principal_id = principals.principal_id + union all + select + table_name, + schema_name, + principal_name, + table_type + from + views_with_metadata + join principals on views_with_metadata.owner_principal_id = principals.principal_id ), cols as ( @@ -24,28 +91,27 @@ table_name, column_name, ordinal_position as column_index, - data_type as column_type, - null as column_comment - from information_schema.columns + data_type as column_type + from INFORMATION_SCHEMA.COLUMNS ) select - tabs.table_database, - tabs.table_schema, - tabs.table_name, - tabs.table_type, - tabs.table_comment, - tabs.table_owner, - cols.column_name, - cols.column_index, - cols.column_type, - cols.column_comment - from tabs - join cols on tabs.table_database = cols.table_database and tabs.table_schema = cols.table_schema and tabs.table_name = cols.table_name + cols.table_database, + tv.schema_name as table_schema, + tv.table_name, + tv.table_type, + null as table_comment, + tv.principal_name as table_owner, + cols.column_name, + cols.column_index, + cols.column_type, + null as column_comment + from tables_and_views tv + join cols on tv.schema_name = cols.table_schema and tv.table_name = cols.table_name order by column_index - {%- endcall -%} + {%- endcall -%} {{ return(load_result('catalog').table) }} @@ -83,8 +149,8 @@ else table_type end as table_type - from [{{ schema_relation.database }}].information_schema.tables + from [{{ schema_relation.database }}].INFORMATION_SCHEMA.TABLES where table_schema like '{{ schema_relation.schema }}' {% endcall %} {{ return(load_result('list_relations_without_caching').table) }} -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/dbt/include/sqlserver/macros/adapters/persist_docs.sql b/dbt/include/sqlserver/macros/adapters/persist_docs.sql index fb3f4fe8..38cd9b54 100644 --- a/dbt/include/sqlserver/macros/adapters/persist_docs.sql +++ b/dbt/include/sqlserver/macros/adapters/persist_docs.sql @@ -1,4 +1,27 @@ -{# we don't support "persist docs" today, but we'd like to! - https://github.com/dbt-msft/dbt-sqlserver/issues/134 - - #} \ No newline at end of file +{% macro sqlserver__alter_column_comment(relation, column_dict) -%} + {%- set existing_columns = adapter.get_columns_in_relation(relation)|map(attribute="name")|list %} + {%- for column_name in column_dict if (column_name in existing_columns) %} + {{ log('Alter extended property "MS_Description" to "' ~ column_dict[column_name]['description'] ~ '" for ' ~ relation ~ ' column "' ~ column_name ~ '"') }} + if not exists ( + select 1 + from + sys.extended_properties as ep + inner join sys.all_columns as cols + on cols.object_id = ep.major_id + and cols.column_id = ep.minor_id + where + ep.major_id = object_id('{{ relation }}') + and ep.name = N'MS_Description' + and cols.name = N'{{ column_name }}' + ) + execute sp_addextendedproperty @name = N'MS_Description', @value = N'{{ column_dict[column_name]['description'] }}' + , @level0type = N'SCHEMA', @level0name = N'{{ relation.schema }}' + , @level1type = N'{{ relation.type }}', @level1name = N'{{ relation.identifier }}' + , @level2type = N'COLUMN', @level2name = N'{{ column_name }}'; + else + execute sp_updateextendedproperty @name = N'MS_Description', @value = N'{{ column_dict[column_name]['description'] }}' + , @level0type = N'SCHEMA', @level0name = N'{{ relation.schema }}' + , @level1type = N'{{ relation.type }}', @level1name = N'{{ relation.identifier }}' + , @level2type = N'COLUMN', @level2name = N'{{ column_name }}'; + {%- endfor %} +{%- endmacro %} diff --git a/dbt/include/sqlserver/macros/adapters/relation.sql b/dbt/include/sqlserver/macros/adapters/relation.sql index e327b670..de043c22 100644 --- a/dbt/include/sqlserver/macros/adapters/relation.sql +++ b/dbt/include/sqlserver/macros/adapters/relation.sql @@ -36,4 +36,4 @@ WHERE name='{{ from_relation.schema }}_{{ from_relation.identifier }}_cci' and object_id = OBJECT_ID('{{ from_relation.schema }}.{{ to_relation.identifier }}')) EXEC sp_rename N'{{ from_relation.schema }}.{{ to_relation.identifier }}.{{ from_relation.schema }}_{{ from_relation.identifier }}_cci', N'{{ from_relation.schema }}_{{ to_relation.identifier }}_cci', N'INDEX' {%- endcall %} -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/dbt/include/sqlserver/macros/adapters/schema.sql b/dbt/include/sqlserver/macros/adapters/schema.sql index d18d3a00..3abf94ed 100644 --- a/dbt/include/sqlserver/macros/adapters/schema.sql +++ b/dbt/include/sqlserver/macros/adapters/schema.sql @@ -3,21 +3,20 @@ USE [{{ relation.database }}]; IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ relation.without_identifier().schema }}') BEGIN - EXEC('CREATE SCHEMA {{ relation.without_identifier().schema }}') + EXEC('CREATE SCHEMA [{{ relation.without_identifier().schema }}]') END {% endcall %} {% endmacro %} {% macro sqlserver__drop_schema(relation) -%} - {%- set tables_in_schema_query %} - SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES - WHERE TABLE_SCHEMA = '{{ relation.schema }}' - {% endset %} - {% set tables_to_drop = run_query(tables_in_schema_query).columns[0].values() %} - {% for table in tables_to_drop %} - {%- set schema_relation = adapter.get_relation(database=relation.database, + {%- set relations_in_schema = list_relations_without_caching(relation) %} + + {% for row in relations_in_schema %} + {%- set schema_relation = api.Relation.create(database=relation.database, schema=relation.schema, - identifier=table) -%} + identifier=row[1], + type=row[3] + ) -%} {% do drop_relation(schema_relation) %} {%- endfor %} @@ -27,6 +26,3 @@ EXEC('DROP SCHEMA {{ relation.schema }}') END {% endcall %} {% endmacro %} - - -{# there is no drop_schema... why? #} \ No newline at end of file diff --git a/dbt/include/sqlserver/macros/materializations/models/incremental/merge.sql b/dbt/include/sqlserver/macros/materializations/models/incremental/merge.sql index e1bb3b9a..9f8e3c5d 100644 --- a/dbt/include/sqlserver/macros/materializations/models/incremental/merge.sql +++ b/dbt/include/sqlserver/macros/materializations/models/incremental/merge.sql @@ -10,7 +10,39 @@ {% endmacro %} {% macro sqlserver__get_delete_insert_merge_sql(target, source, unique_key, dest_columns) %} - {{ default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns) }}; + + {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%} + + {% if unique_key %} + {% if unique_key is sequence and unique_key is not string %} + delete from {{ target }} + where exists ( + SELECT NULL + FROM + {{ source }} + WHERE + {% for key in unique_key %} + {{ source }}.{{ key }} = {{ target }}.{{ key }} + {{ "and " if not loop.last }} + {% endfor %} + ); + {% else %} + delete from {{ target }} + where ( + {{ unique_key }}) in ( + select ({{ unique_key }}) + from {{ source }} + ); + + {% endif %} + {% endif %} + + insert into {{ target }} ({{ dest_cols_csv }}) + ( + select {{ dest_cols_csv }} + from {{ source }} + ) + {% endmacro %} {% macro sqlserver__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) %} diff --git a/dbt/include/sqlserver/macros/materializations/models/table/create_table_as.sql b/dbt/include/sqlserver/macros/materializations/models/table/create_table_as.sql index dbf99531..53bf7221 100644 --- a/dbt/include/sqlserver/macros/materializations/models/table/create_table_as.sql +++ b/dbt/include/sqlserver/macros/materializations/models/table/create_table_as.sql @@ -18,7 +18,7 @@ {{ tmp_relation }} {{ sqlserver__drop_relation_script(tmp_relation) }} - + {% if not temporary and as_columnstore -%} {{ sqlserver__create_clustered_columnstore_index(relation) }} {% endif %} diff --git a/dbt/include/sqlserver/macros/materializations/seeds/helpers.sql b/dbt/include/sqlserver/macros/materializations/seeds/helpers.sql index 23035890..66bf829e 100644 --- a/dbt/include/sqlserver/macros/materializations/seeds/helpers.sql +++ b/dbt/include/sqlserver/macros/materializations/seeds/helpers.sql @@ -61,6 +61,6 @@ {% set max_batch_size = get_batch_size() %} {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %} {% set batch_size = calc_batch_size(cols_sql|length, max_batch_size) %} - + {{ return(basic_load_csv_rows(model, batch_size, agate_table) )}} {% endmacro %} diff --git a/dbt/include/sqlserver/macros/materializations/snapshots/snapshot.sql b/dbt/include/sqlserver/macros/materializations/snapshots/snapshot.sql index 9a94d7a5..928ac5fb 100644 --- a/dbt/include/sqlserver/macros/materializations/snapshots/snapshot.sql +++ b/dbt/include/sqlserver/macros/materializations/snapshots/snapshot.sql @@ -12,4 +12,8 @@ alter table {{ relation }} add "{{ column.name }}" {{ column.data_type }}; {% endcall %} {% endfor %} -{% endmacro %} \ No newline at end of file +{% endmacro %} + +{% macro sqlserver__get_true_sql() %} + {{ return('1=1') }} +{% endmacro %} diff --git a/dbt/include/sqlserver/macros/materializations/snapshots/snapshot_merge.sql b/dbt/include/sqlserver/macros/materializations/snapshots/snapshot_merge.sql index 5c006f9f..ff27ae31 100644 --- a/dbt/include/sqlserver/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt/include/sqlserver/macros/materializations/snapshots/snapshot_merge.sql @@ -1,3 +1,3 @@ {% macro sqlserver__snapshot_merge_sql(target, source, insert_cols) %} {{ default__snapshot_merge_sql(target, source, insert_cols) }}; -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/dbt/include/sqlserver/macros/utils/any_value.sql b/dbt/include/sqlserver/macros/utils/any_value.sql new file mode 100644 index 00000000..6dcf8ec2 --- /dev/null +++ b/dbt/include/sqlserver/macros/utils/any_value.sql @@ -0,0 +1,5 @@ +{% macro sqlserver__any_value(expression) -%} + + min({{ expression }}) + +{%- endmacro %} diff --git a/dbt/include/sqlserver/macros/utils/cast_bool_to_text.sql b/dbt/include/sqlserver/macros/utils/cast_bool_to_text.sql new file mode 100644 index 00000000..9771afbf --- /dev/null +++ b/dbt/include/sqlserver/macros/utils/cast_bool_to_text.sql @@ -0,0 +1,7 @@ +{% macro sqlserver__cast_bool_to_text(field) %} + case {{ field }} + when 1 then 'true' + when 0 then 'false' + else null + end +{% endmacro %} diff --git a/dbt/include/sqlserver/macros/utils/concat.sql b/dbt/include/sqlserver/macros/utils/concat.sql new file mode 100644 index 00000000..f5ca6c38 --- /dev/null +++ b/dbt/include/sqlserver/macros/utils/concat.sql @@ -0,0 +1,3 @@ +{% macro sqlserver__concat(fields) -%} + concat({{ fields|join(', ') }}, '') +{%- endmacro %} diff --git a/dbt/include/sqlserver/macros/utils/date_trunc.sql b/dbt/include/sqlserver/macros/utils/date_trunc.sql new file mode 100644 index 00000000..85b4ce32 --- /dev/null +++ b/dbt/include/sqlserver/macros/utils/date_trunc.sql @@ -0,0 +1,3 @@ +{% macro sqlserver__date_trunc(datepart, date) %} + CAST(DATEADD({{datepart}}, DATEDIFF({{datepart}}, 0, {{date}}), 0) AS DATE) +{% endmacro %} diff --git a/dbt/include/sqlserver/macros/utils/dateadd.sql b/dbt/include/sqlserver/macros/utils/dateadd.sql new file mode 100644 index 00000000..605379e3 --- /dev/null +++ b/dbt/include/sqlserver/macros/utils/dateadd.sql @@ -0,0 +1,9 @@ +{% macro sqlserver__dateadd(datepart, interval, from_date_or_timestamp) %} + + dateadd( + {{ datepart }}, + {{ interval }}, + cast({{ from_date_or_timestamp }} as datetime) + ) + +{% endmacro %} diff --git a/dbt/include/sqlserver/macros/utils/datediff.sql b/dbt/include/sqlserver/macros/utils/datediff.sql new file mode 100644 index 00000000..67093ce8 --- /dev/null +++ b/dbt/include/sqlserver/macros/utils/datediff.sql @@ -0,0 +1,7 @@ +{% macro synapse__datediff(first_date, second_date, datepart) %} + datediff( + {{ datepart }}, + cast({{first_date}} as datetime), + cast({{second_date}} as datetime) + ) +{% endmacro %} diff --git a/dbt/include/sqlserver/macros/utils/hash.sql b/dbt/include/sqlserver/macros/utils/hash.sql new file mode 100644 index 00000000..d3d2360d --- /dev/null +++ b/dbt/include/sqlserver/macros/utils/hash.sql @@ -0,0 +1,3 @@ +{% macro sqlserver__hash(field) %} + convert(varchar(50), hashbytes('md5', {{field}}), 2) +{% endmacro %} diff --git a/dbt/include/sqlserver/macros/utils/last_day.sql b/dbt/include/sqlserver/macros/utils/last_day.sql new file mode 100644 index 00000000..c523d944 --- /dev/null +++ b/dbt/include/sqlserver/macros/utils/last_day.sql @@ -0,0 +1,13 @@ +{% macro sqlserver__last_day(date, datepart) -%} + + {%- if datepart == 'quarter' -%} + CAST(DATEADD(QUARTER, DATEDIFF(QUARTER, 0, {{ date }}) + 1, -1) AS DATE) + {%- elif datepart == 'month' -%} + EOMONTH ( {{ date }}) + {%- elif datepart == 'year' -%} + CAST(DATEADD(YEAR, DATEDIFF(year, 0, {{ date }}) + 1, -1) AS DATE) + {%- else -%} + {{dbt_utils.default_last_day(date, datepart)}} + {%- endif -%} + +{%- endmacro %} diff --git a/dbt/include/sqlserver/macros/utils/length.sql b/dbt/include/sqlserver/macros/utils/length.sql new file mode 100644 index 00000000..ee9431ac --- /dev/null +++ b/dbt/include/sqlserver/macros/utils/length.sql @@ -0,0 +1,5 @@ +{% macro sqlserver__length(expression) %} + + len( {{ expression }} ) + +{%- endmacro -%} diff --git a/dbt/include/sqlserver/macros/utils/listagg.sql b/dbt/include/sqlserver/macros/utils/listagg.sql new file mode 100644 index 00000000..4d6ab215 --- /dev/null +++ b/dbt/include/sqlserver/macros/utils/listagg.sql @@ -0,0 +1,8 @@ +{% macro sqlserver__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} + + string_agg({{ measure }}, {{ delimiter_text }}) + {%- if order_by_clause != None %} + within group ({{ order_by_clause }}) + {%- endif %} + +{%- endmacro %} diff --git a/dbt/include/sqlserver/macros/utils/position.sql b/dbt/include/sqlserver/macros/utils/position.sql new file mode 100644 index 00000000..bd3f6577 --- /dev/null +++ b/dbt/include/sqlserver/macros/utils/position.sql @@ -0,0 +1,8 @@ +{% macro sqlserver__position(substring_text, string_text) %} + + CHARINDEX( + {{ substring_text }}, + {{ string_text }} + ) + +{%- endmacro -%} diff --git a/dbt/include/sqlserver/macros/utils/safe_cast.sql b/dbt/include/sqlserver/macros/utils/safe_cast.sql new file mode 100644 index 00000000..4ae065a7 --- /dev/null +++ b/dbt/include/sqlserver/macros/utils/safe_cast.sql @@ -0,0 +1,3 @@ +{% macro sqlserver__safe_cast(field, type) %} + try_cast({{field}} as {{type}}) +{% endmacro %} diff --git a/dbt/include/sqlserver/macros/utils/split_part.sql b/dbt/include/sqlserver/macros/utils/split_part.sql new file mode 100644 index 00000000..b92041db --- /dev/null +++ b/dbt/include/sqlserver/macros/utils/split_part.sql @@ -0,0 +1,9 @@ +{# + For more information on how this XML trick works with splitting strings, see https://www.mssqltips.com/sqlservertip/1771/splitting-delimited-strings-using-xml-in-sql-server/ +#} + +{% macro sqlserver__split_part(string_text, delimiter_text, part_number) %} + + LTRIM(CAST((''+REPLACE({{ string_text }},{{ delimiter_text }} ,'')+'') AS XML).value('(/X)[{{ part_number }}]', 'VARCHAR(128)')) + +{% endmacro %} diff --git a/dev_requirements.txt b/dev_requirements.txt index 6733bcac..4b4dd4b2 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,5 +1,7 @@ -black==20.8b1 -pytest-dbt-adapter~=0.6.0 -pytest==6.2.2 -tox==3.2.0 -flake8==3.5.0 +pytest==7.1.3 +twine==4.0.1 +wheel==0.37.1 +pre-commit==2.20.0 +pytest-dotenv==0.5.2 +dbt-tests-adapter==1.2.1 +-e . diff --git a/devops/CI.Dockerfile b/devops/CI.Dockerfile new file mode 100644 index 00000000..09e95f9a --- /dev/null +++ b/devops/CI.Dockerfile @@ -0,0 +1,59 @@ +ARG PYTHON_VERSION="3.10" +FROM python:${PYTHON_VERSION}-bullseye as base + +# Setup dependencies for pyodbc +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + apt-transport-https \ + curl \ + gnupg2 \ + unixodbc-dev \ + lsb-release && \ + apt-get autoremove -yqq --purge && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# enable Microsoft package repo +RUN curl -sL https://packages.microsoft.com/keys/microsoft.asc | apt-key add - +RUN curl -sL https://packages.microsoft.com/config/debian/$(lsb_release -sr)/prod.list | tee /etc/apt/sources.list.d/msprod.list +# enable Azure CLI package repo +RUN echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/azure-cli.list + +# install Azure CLI +ENV ACCEPT_EULA=Y +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + azure-cli && \ + apt-get autoremove -yqq --purge && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +FROM base as msodbc17 + +# install ODBC driver 17 +ENV ACCEPT_EULA=Y +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + msodbcsql17 \ + mssql-tools && \ + apt-get autoremove -yqq --purge && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# add sqlcmd to the path +ENV PATH="$PATH:/opt/mssql-tools/bin" + +FROM base as msodbc18 + +# install ODBC driver 18 +ENV ACCEPT_EULA=Y +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + msodbcsql18 \ + mssql-tools18 && \ + apt-get autoremove -yqq --purge && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# add sqlcmd to the path +ENV PATH="$PATH:/opt/mssql-tools18/bin" diff --git a/devops/scripts/create_sql_users.sql b/devops/scripts/create_sql_users.sql new file mode 100644 index 00000000..27c1a469 --- /dev/null +++ b/devops/scripts/create_sql_users.sql @@ -0,0 +1,8 @@ +IF NOT EXISTS(SELECT * FROM sys.database_principals WHERE name = '$(DBT_TEST_USER_1)') + CREATE USER [$(DBT_TEST_USER_1)] WITHOUT LOGIN; + +IF NOT EXISTS(SELECT * FROM sys.database_principals WHERE name = '$(DBT_TEST_USER_2)') + CREATE USER [$(DBT_TEST_USER_2)] WITHOUT LOGIN; + +IF NOT EXISTS(SELECT * FROM sys.database_principals WHERE name = '$(DBT_TEST_USER_3)') + CREATE USER [$(DBT_TEST_USER_3)] WITHOUT LOGIN; diff --git a/devops/scripts/entrypoint.sh b/devops/scripts/entrypoint.sh new file mode 100755 index 00000000..d2f40463 --- /dev/null +++ b/devops/scripts/entrypoint.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +/opt/init_scripts/init_db.sh & /opt/mssql/bin/sqlservr diff --git a/devops/scripts/init_db.sh b/devops/scripts/init_db.sh new file mode 100755 index 00000000..309c0816 --- /dev/null +++ b/devops/scripts/init_db.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +for i in {1..50}; +do + /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P "${SA_PASSWORD}" -d msdb -I -i create_sql_users.sql + if [ $? -eq 0 ] + then + echo "create_sql_users.sql completed" + break + else + echo "not ready yet..." + sleep 1 + fi +done diff --git a/devops/scripts/wakeup_azure.py b/devops/scripts/wakeup_azure.py new file mode 100755 index 00000000..23ae4adb --- /dev/null +++ b/devops/scripts/wakeup_azure.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +import os +import time + +import pyodbc + + +def resume_azsql(): + sql_server_name = os.getenv("DBT_AZURESQL_SERVER") + sql_server_port = 1433 + database_name = os.getenv("DBT_AZURESQL_DB") + username = os.getenv("DBT_AZURESQL_UID") + password = os.getenv("DBT_AZURESQL_PWD") + driver = f"ODBC Driver {os.getenv('MSODBC_VERSION')} for SQL Server" + + con_str = [ + f"DRIVER={{{driver}}}", + f"SERVER={sql_server_name},{sql_server_port}", + f"Database={database_name}", + "Encrypt=Yes", + f"UID={{{username}}}", + f"PWD={{{password}}}", + ] + + con_str_concat = ";".join(con_str) + print("Connecting with the following connection string:") + print(con_str_concat.replace(password, "***")) + + connected = False + attempts = 0 + while not connected and attempts < 20: + try: + attempts += 1 + handle = pyodbc.connect(con_str_concat, autocommit=True) + cursor = handle.cursor() + cursor.execute("SELECT 1") + connected = True + except pyodbc.Error as e: + print("Failed to connect to SQL Server. Retrying...") + print(e) + time.sleep(10) + + +def main(): + resume_azsql() + + +if __name__ == "__main__": + main() diff --git a/devops/server.Dockerfile b/devops/server.Dockerfile new file mode 100644 index 00000000..be3557f3 --- /dev/null +++ b/devops/server.Dockerfile @@ -0,0 +1,8 @@ +ARG MSSQL_VERSION="2022" +FROM mcr.microsoft.com/mssql/server:${MSSQL_VERSION}-latest + +RUN mkdir -p /opt/init_scripts +WORKDIR /opt/init_scripts +COPY scripts/* /opt/init_scripts/ + +ENTRYPOINT /bin/bash ./entrypoint.sh diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..34a189ad --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,14 @@ +services: + sqlserver: + build: + context: devops + dockerfile: server.Dockerfile + args: + MSSQL_VERSION: "2022" + environment: + SA_PASSWORD: "L0calTesting!" + ACCEPT_EULA: "Y" + env_file: + - test.env + ports: + - "1433:1433" diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..b3d74bc1 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,9 @@ +[pytest] +filterwarnings = + ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning + ignore:unclosed file .*:ResourceWarning +env_files = + test.env +testpaths = + tests/unit + tests/functional diff --git a/setup.py b/setup.py index acc0804e..78576f51 100644 --- a/setup.py +++ b/setup.py @@ -1,56 +1,87 @@ #!/usr/bin/env python -from setuptools import find_namespace_packages, setup import os import re +import sys -this_directory = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(this_directory, 'README.md')) as f: - long_description = f.read() - +from setuptools import find_namespace_packages, setup +from setuptools.command.install import install package_name = "dbt-sqlserver" +authors_list = ["Mikael Ene", "Anders Swanson", "Sam Debruyn", "Cor Zuurmond"] +dbt_version = "1.2" +description = """A Microsoft SQL Server adapter plugin for dbt (data build tool)""" + +this_directory = os.path.abspath(os.path.dirname(__file__)) +with open(os.path.join(this_directory, "README.md")) as f: + long_description = f.read() # get this from a separate file def _dbt_sqlserver_version(): - _version_path = os.path.join( - this_directory, 'dbt', 'adapters', 'sqlserver', '__version__.py' - ) - _version_pattern = r'''version\s*=\s*["'](.+)["']''' + _version_path = os.path.join(this_directory, "dbt", "adapters", "sqlserver", "__version__.py") + _version_pattern = r"""version\s*=\s*["'](.+)["']""" with open(_version_path) as f: match = re.search(_version_pattern, f.read().strip()) if match is None: - raise ValueError(f'invalid version at {_version_path}') + raise ValueError(f"invalid version at {_version_path}") return match.group(1) package_version = _dbt_sqlserver_version() -description = """A sqlserver adapter plugin for dbt (data build tool)""" -dbt_version = '1.0' # the package version should be the dbt version, with maybe some things on the # ends of it. (0.18.1 vs 0.18.1a1, 0.18.1.1, ...) if not package_version.startswith(dbt_version): raise ValueError( - f'Invalid setup.py: package_version={package_version} must start with ' - f'dbt_version={dbt_version}' + f"Invalid setup.py: package_version={package_version} must start with " + f"dbt_version={dbt_version}" ) + +class VerifyVersionCommand(install): + """Custom command to verify that the git tag matches our version""" + + description = "Verify that the git tag matches our version" + + def run(self): + tag = os.getenv("GITHUB_REF_NAME") + tag_without_prefix = tag[1:] + + if tag_without_prefix != package_version: + info = "Git tag: {0} does not match the version of this app: {1}".format( + tag_without_prefix, package_version + ) + sys.exit(info) + + setup( name=package_name, version=package_version, description=description, - long_description=description, + long_description=long_description, long_description_content_type="text/markdown", license="MIT", - author="Mikael Ene", - author_email="mikael.ene@eneanalytics.com", - url="https://github.com/mikaelene/dbt-sqlserver", - packages=find_namespace_packages(include=['dbt', 'dbt.*']), + author=", ".join(authors_list), + url="https://github.com/dbt-msft/dbt-sqlserver", + packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-core~=1.0.0", - "pyodbc~=4.0.32", - "azure-identity>=1.7.0", - ] + f"dbt-core~={dbt_version}.0", + "pyodbc==4.0.32", + "azure-identity>=1.10.0", + ], + cmdclass={ + "verify": VerifyVersionCommand, + }, + classifiers=[ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: MIT License", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS :: MacOS X", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + ], ) diff --git a/test.env.sample b/test.env.sample new file mode 100644 index 00000000..0238e75f --- /dev/null +++ b/test.env.sample @@ -0,0 +1,11 @@ +SQLSERVER_TEST_DRIVER=ODBC Driver 18 for SQL Server +SQLSERVER_TEST_HOST=localhost +SQLSERVER_TEST_USER=sa +SQLSERVER_TEST_PASS=L0calTesting! +SQLSERVER_TEST_PORT=1433 +SQLSERVER_TEST_DBNAME=msdb +SQLSERVER_TEST_ENCRYPT=True +SQLSERVER_TEST_TRUST_CERT=True +DBT_TEST_USER_1=DBT_TEST_USER_1 +DBT_TEST_USER_2=DBT_TEST_USER_2 +DBT_TEST_USER_3=DBT_TEST_USER_3 diff --git a/test/integration/azuresql.dbtspec b/test/integration/azuresql.dbtspec deleted file mode 100644 index 7618751c..00000000 --- a/test/integration/azuresql.dbtspec +++ /dev/null @@ -1,42 +0,0 @@ - -target: - type: sqlserver - driver: "ODBC Driver 17 for SQL Server" - port: 1433 - host: "{{ env_var('DBT_AZURESQL_SERVER') }}" - database: "{{ env_var('DBT_AZURESQL_DB') }}" - username: "{{ env_var('DBT_AZURESQL_UID') }}" - password: "{{ env_var('DBT_AZURESQL_PWD') }}" - schema: "dbt_test_azure_sql_{{ var('_dbt_random_suffix') }}" - encrypt: yes - trust_cert: yes - threads: 1 -projects: - - overrides: base - dbt_project_yml: &override-project - name: schema_tests - config-version: 2 - version: '1.0.0' - models: - dbt_test_project: - +as_columnstore: false - - overrides: ephemeral - dbt_project_yml: *override-project - - overrides: incremental - dbt_project_yml: *override-project - - overrides: snapshot_strategy_timestamp - dbt_project_yml: *override-project - - overrides: snapshot_strategy_check_cols - dbt_project_yml: *override-project - - overrides: schema_tests - dbt_project_yml: *override-project -sequences: - test_dbt_empty: empty - test_dbt_base: base - # test_dbt_ephemeral: ephemeral - test_dbt_incremental: incremental - test_dbt_snapshot_strategy_timestamp: snapshot_strategy_timestamp - test_dbt_snapshot_strategy_check_cols: snapshot_strategy_check_cols - test_dbt_data_test: data_test - test_dbt_schema_test: schema_test - # test_dbt_ephemeral_data_tests: data_test_ephemeral_models diff --git a/test/integration/dbt_project.yml b/test/integration/dbt_project.yml deleted file mode 100644 index f02f89c5..00000000 --- a/test/integration/dbt_project.yml +++ /dev/null @@ -1,6 +0,0 @@ - -name: 'sqlserver_integration_tests' -version: '1.0' -config-version: 2 - -profile: 'integration_tests' \ No newline at end of file diff --git a/test/integration/models/test.sql b/test/integration/models/test.sql deleted file mode 100644 index 90664768..00000000 --- a/test/integration/models/test.sql +++ /dev/null @@ -1,3 +0,0 @@ -{# inane comment #} -{% set col_name = 'foo' %} -SELECT 1 as {{ col_name }} \ No newline at end of file diff --git a/test/integration/sample.profiles.yml b/test/integration/sample.profiles.yml deleted file mode 100644 index 426805f1..00000000 --- a/test/integration/sample.profiles.yml +++ /dev/null @@ -1,48 +0,0 @@ -# HEY! This file is used in the tsql_utils integrations tests with CircleCI. -# You should __NEVER__ check credentials into version control. Thanks for reading :) - -config: - send_anonymous_usage_stats: False - use_colors: True - -defaults: - basic: &basic - type: sqlserver - driver: "ODBC Driver 17 for SQL Server" - schema: "dbt_cnxn_test" - port: 1433 - threads: 8 - basic-sqlserver: &basic-sqlserver - <<: *basic - host: localhost - database: msdb - username: SA - password: 5atyaNadella - azuresql: &azuresql-basic - <<: *basic - host: "{{ env_var('DBT_AZURESQL_SERVER') }}" - database: "{{ env_var('DBT_AZURESQL_DB') }}" - encrypt: yes - trust_cert: yes - -integration_tests: - target: sqlserver_local_userpass - outputs: - sqlserver_local_userpass: *basic-sqlserver - sqlserver_local_encrypt: - <<: *basic-sqlserver - encrypt: yes - trust_cert: yes - azuresql_sqlcred: - <<: *azuresql-basic - username: "{{ env_var('DBT_AZURESQL_UID') }}" - password: "{{ env_var('DBT_AZURESQL_PWD') }}" - azuresql_azcli: - <<: *azuresql-basic - authentication: CLI - azuresql_azauto: - <<: *azuresql-basic - authentication: auto - azuresql_azenv: - <<: *azuresql-basic - authentication: environment diff --git a/test/integration/sqlserver.dbtspec b/test/integration/sqlserver.dbtspec deleted file mode 100644 index 90842800..00000000 --- a/test/integration/sqlserver.dbtspec +++ /dev/null @@ -1,21 +0,0 @@ - -target: - type: sqlserver - driver: "ODBC Driver 17 for SQL Server" - schema: "dbt_test_{{ var('_dbt_random_suffix') }}" - host: localhost - database: msdb - username: SA - password: 5atyaNadella - port: 1433 - threads: 8 -sequences: - test_dbt_empty: empty - test_dbt_base: base - test_dbt_ephemeral: ephemeral - test_dbt_incremental: incremental - test_dbt_snapshot_strategy_timestamp: snapshot_strategy_timestamp - test_dbt_snapshot_strategy_check_cols: snapshot_strategy_check_cols - test_dbt_data_test: data_test - test_dbt_schema_test: schema_test - # test_dbt_ephemeral_data_tests: data_test_ephemeral_models diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..d7f267ed --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,140 @@ +import os + +import pytest + +pytest_plugins = ["dbt.tests.fixtures.project"] + + +def pytest_addoption(parser): + parser.addoption("--profile", action="store", default="user", type=str) + + +@pytest.fixture(scope="class") +def dbt_profile_target(request): + profile = request.config.getoption("--profile") + + if profile == "ci_sql_server": + return _profile_ci_sql_server() + if profile == "ci_azure_cli": + return _profile_ci_azure_cli() + if profile == "ci_azure_auto": + return _profile_ci_azure_auto() + if profile == "ci_azure_environment": + return _profile_ci_azure_environment() + if profile == "ci_azure_basic": + return _profile_ci_azure_basic() + if profile == "user": + return _profile_user() + if profile == "user_azure": + return _profile_user_azure() + + raise ValueError(f"Unknown profile: {profile}") + + +def _all_profiles_base(): + return { + "type": "sqlserver", + "threads": 1, + "driver": os.getenv("SQLSERVER_TEST_DRIVER", "ODBC Driver 18 for SQL Server"), + "port": int(os.getenv("SQLSERVER_TEST_PORT", "1433")), + "retries": 2, + } + + +def _profile_ci_azure_base(): + return { + **_all_profiles_base(), + **{ + "host": os.getenv("DBT_AZURESQL_SERVER"), + "database": os.getenv("DBT_AZURESQL_DB"), + "encrypt": True, + "trust_cert": True, + }, + } + + +def _profile_ci_azure_basic(): + return { + **_profile_ci_azure_base(), + **{ + "user": os.getenv("DBT_AZURESQL_UID"), + "pass": os.getenv("DBT_AZURESQL_PWD"), + }, + } + + +def _profile_ci_azure_cli(): + return { + **_profile_ci_azure_base(), + **{ + "authentication": "CLI", + }, + } + + +def _profile_ci_azure_auto(): + return { + **_profile_ci_azure_base(), + **{ + "authentication": "auto", + }, + } + + +def _profile_ci_azure_environment(): + return { + **_profile_ci_azure_base(), + **{ + "authentication": "environment", + }, + } + + +def _profile_ci_sql_server(): + return { + **_all_profiles_base(), + **{ + "host": "sqlserver", + "user": "SA", + "pass": "5atyaNadella", + "database": "msdb", + "encrypt": True, + "trust_cert": True, + }, + } + + +def _profile_user(): + return { + **_all_profiles_base(), + **{ + "host": os.getenv("SQLSERVER_TEST_HOST"), + "user": os.getenv("SQLSERVER_TEST_USER"), + "pass": os.getenv("SQLSERVER_TEST_PASS"), + "database": os.getenv("SQLSERVER_TEST_DBNAME"), + "encrypt": bool(os.getenv("SQLSERVER_TEST_ENCRYPT", "False")), + "trust_cert": bool(os.getenv("SQLSERVER_TEST_TRUST_CERT", "False")), + }, + } + + +def _profile_user_azure(): + return { + **_all_profiles_base(), + **{ + "host": os.getenv("SQLSERVER_TEST_HOST"), + "authentication": "auto", + "encrypt": True, + "trust_cert": True, + "database": os.getenv("SQLSERVER_TEST_DBNAME"), + }, + } + + +@pytest.fixture(autouse=True) +def skip_by_profile_type(request): + profile_type = request.config.getoption("--profile") + if request.node.get_closest_marker("skip_profile"): + for skip_profile_type in request.node.get_closest_marker("skip_profile").args: + if skip_profile_type == profile_type: + pytest.skip("Skipped on '{profile_type}' profile") diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py new file mode 100644 index 00000000..28cd5221 --- /dev/null +++ b/tests/functional/adapter/test_basic.py @@ -0,0 +1,57 @@ +import pytest +from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod +from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations +from dbt.tests.adapter.basic.test_empty import BaseEmpty +from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral +from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests +from dbt.tests.adapter.basic.test_incremental import BaseIncremental +from dbt.tests.adapter.basic.test_singular_tests import BaseSingularTests +from dbt.tests.adapter.basic.test_singular_tests_ephemeral import BaseSingularTestsEphemeral +from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols +from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp +from dbt.tests.adapter.basic.test_validate_connection import BaseValidateConnection + + +class TestSimpleMaterializationsSQLServer(BaseSimpleMaterializations): + pass + + +class TestSingularTestsSQLServer(BaseSingularTests): + pass + + +@pytest.mark.skip(reason="ephemeral not supported") +class TestSingularTestsEphemeralSQLServer(BaseSingularTestsEphemeral): + pass + + +class TestEmptySQLServer(BaseEmpty): + pass + + +class TestEphemeralSQLServer(BaseEphemeral): + pass + + +class TestIncrementalSQLServer(BaseIncremental): + pass + + +class TestGenericTestsSQLServer(BaseGenericTests): + pass + + +class TestSnapshotCheckColsSQLServer(BaseSnapshotCheckCols): + pass + + +class TestSnapshotTimestampSQLServer(BaseSnapshotTimestamp): + pass + + +class TestBaseCachingSQLServer(BaseAdapterMethod): + pass + + +class TestValidateConnectionSQLServer(BaseValidateConnection): + pass diff --git a/tests/functional/adapter/test_data_types.py b/tests/functional/adapter/test_data_types.py new file mode 100644 index 00000000..85b93779 --- /dev/null +++ b/tests/functional/adapter/test_data_types.py @@ -0,0 +1,54 @@ +import pytest +from dbt.tests.adapter.utils.data_types.test_type_bigint import BaseTypeBigInt +from dbt.tests.adapter.utils.data_types.test_type_float import BaseTypeFloat +from dbt.tests.adapter.utils.data_types.test_type_int import BaseTypeInt +from dbt.tests.adapter.utils.data_types.test_type_numeric import BaseTypeNumeric +from dbt.tests.adapter.utils.data_types.test_type_string import BaseTypeString +from dbt.tests.adapter.utils.data_types.test_type_timestamp import ( + BaseTypeTimestamp, + seeds__expected_csv, +) + + +@pytest.mark.skip(reason="SQL Server shows 'numeric' if you don't explicitly cast it to bigint") +class TestTypeBigIntSQLServer(BaseTypeBigInt): + pass + + +class TestTypeFloatSQLServer(BaseTypeFloat): + pass + + +class TestTypeIntSQLServer(BaseTypeInt): + pass + + +class TestTypeNumericSQLServer(BaseTypeNumeric): + pass + + +class TestTypeStringSQLServer(BaseTypeString): + def assert_columns_equal(self, project, expected_cols, actual_cols): + # ignore the size of the varchar since we do + # an optimization to not use varchar(max) all the time + assert ( + expected_cols[:-1] == actual_cols[:-1] + ), f"Type difference detected: {expected_cols} vs. {actual_cols}" + + +class TestTypeTimestampSQLServer(BaseTypeTimestamp): + @pytest.fixture(scope="class") + def seeds(self): + seeds__expected_yml = """ +version: 2 +seeds: + - name: expected + config: + column_types: + timestamp_col: "datetimeoffset" + """ + + return { + "expected.csv": seeds__expected_csv, + "expected.yml": seeds__expected_yml, + } diff --git a/tests/functional/adapter/test_docs.py b/tests/functional/adapter/test_docs.py new file mode 100644 index 00000000..9e06b0ea --- /dev/null +++ b/tests/functional/adapter/test_docs.py @@ -0,0 +1,94 @@ +import pytest +from dbt.tests.adapter.basic.expected_catalog import ( + base_expected_catalog, + expected_references_catalog, + no_stats, +) +from dbt.tests.adapter.basic.test_docs_generate import ( + BaseDocsGenerate, + BaseDocsGenReferences, + ref_models__docs_md, + ref_models__ephemeral_copy_sql, + ref_models__schema_yml, +) + + +class TestDocsGenerateSQLServer(BaseDocsGenerate): + @pytest.fixture(scope="class") + def expected_catalog(self, project): + return base_expected_catalog( + project, + role="dbo", + id_type="int", + text_type="varchar", + time_type="datetime", + view_type="VIEW", + table_type="BASE TABLE", + model_stats=no_stats(), + ) + + +class TestDocsGenReferencesSQLServer(BaseDocsGenReferences): + @pytest.fixture(scope="class") + def expected_catalog(self, project): + return expected_references_catalog( + project, + role="dbo", + id_type="int", + text_type="varchar", + time_type="datetime", + bigint_type="int", + view_type="VIEW", + table_type="BASE TABLE", + model_stats=no_stats(), + ) + + @pytest.fixture(scope="class") + def models(self): + ref_models__ephemeral_summary_sql_no_order_by = """ + {{ + config( + materialized = "table" + ) + }} + + select first_name, count(*) as ct from {{ref('ephemeral_copy')}} + group by first_name + """ + + ref_models__view_summary_sql_no_order_by = """ + {{ + config( + materialized = "view" + ) + }} + + select first_name, ct from {{ref('ephemeral_summary')}} + """ + + ref_sources__schema_yml = """ +version: 2 +sources: + - name: my_source + description: "{{ doc('source_info') }}" + loader: a_loader + schema: "{{ var('test_schema') }}" + tables: + - name: my_table + description: "{{ doc('table_info') }}" + identifier: seed + columns: + - name: id + description: "{{ doc('column_info') }}" + """ + + return { + "schema.yml": ref_models__schema_yml, + "sources.yml": ref_sources__schema_yml, + # order by not allowed in VIEWS + "view_summary.sql": ref_models__view_summary_sql_no_order_by, + # order by not allowed in CTEs + "ephemeral_summary.sql": ref_models__ephemeral_summary_sql_no_order_by, + "ephemeral_copy.sql": ref_models__ephemeral_copy_sql, + "docs.md": ref_models__docs_md, + } diff --git a/tests/functional/adapter/test_grants.py b/tests/functional/adapter/test_grants.py new file mode 100644 index 00000000..aaac1c4d --- /dev/null +++ b/tests/functional/adapter/test_grants.py @@ -0,0 +1,29 @@ +from dbt.tests.adapter.grants.test_incremental_grants import BaseIncrementalGrants +from dbt.tests.adapter.grants.test_invalid_grants import BaseInvalidGrants +from dbt.tests.adapter.grants.test_model_grants import BaseModelGrants +from dbt.tests.adapter.grants.test_seed_grants import BaseSeedGrants +from dbt.tests.adapter.grants.test_snapshot_grants import BaseSnapshotGrants + + +class TestIncrementalGrantsSQLServer(BaseIncrementalGrants): + pass + + +class TestInvalidGrantsSQLServer(BaseInvalidGrants): + def grantee_does_not_exist_error(self): + return "Cannot find the user" + + def privilege_does_not_exist_error(self): + return "Incorrect syntax near" + + +class TestModelGrantsSQLServer(BaseModelGrants): + pass + + +class TestSeedGrantsSQLServer(BaseSeedGrants): + pass + + +class TestSnapshotGrantsSQLServer(BaseSnapshotGrants): + pass diff --git a/tests/functional/adapter/test_incremental.py b/tests/functional/adapter/test_incremental.py new file mode 100644 index 00000000..873a6cfd --- /dev/null +++ b/tests/functional/adapter/test_incremental.py @@ -0,0 +1,5 @@ +from dbt.tests.adapter.incremental.test_incremental_unique_id import BaseIncrementalUniqueKey + + +class TestBaseIncrementalUniqueKeySQLServer(BaseIncrementalUniqueKey): + pass diff --git a/tests/functional/adapter/test_sources.py b/tests/functional/adapter/test_sources.py new file mode 100644 index 00000000..b5887c67 --- /dev/null +++ b/tests/functional/adapter/test_sources.py @@ -0,0 +1,71 @@ +import pytest +from dbt.tests.adapter.basic.files import config_materialized_table, config_materialized_view +from dbt.tests.util import run_dbt + +source_regular = """ +version: 2 +sources: +- name: regular + schema: INFORMATION_SCHEMA + tables: + - name: VIEWS + columns: + - name: TABLE_NAME + tests: + - not_null +""" + +source_space_in_name = """ +version: 2 +sources: +- name: 'space in name' + schema: INFORMATION_SCHEMA + tables: + - name: VIEWS + columns: + - name: TABLE_NAME + tests: + - not_null +""" + +select_from_source_regular = """ +select * from {{ source("regular", "VIEWS") }} +""" + +select_from_source_space_in_name = """ +select * from {{ source("space in name", "VIEWS") }} +""" + + +class TestSourcesSQLServer: + @pytest.fixture(scope="class") + def models(self): + return { + "source_regular.yml": source_regular, + "source_space_in_name.yml": source_space_in_name, + "v_select_from_source_regular.sql": config_materialized_view + + select_from_source_regular, + "v_select_from_source_space_in_name.sql": config_materialized_view + + select_from_source_space_in_name, + "t_select_from_source_regular.sql": config_materialized_table + + select_from_source_regular, + "t_select_from_source_space_in_name.sql": config_materialized_table + + select_from_source_space_in_name, + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "name": "test_sources", + } + + def test_dbt_run(self, project): + run_dbt(["compile"]) + + ls = run_dbt(["list"]) + assert len(ls) == 8 + ls_sources = [src for src in ls if src.startswith("source:")] + assert len(ls_sources) == 2 + + run_dbt(["run"]) + run_dbt(["test"]) diff --git a/tests/functional/adapter/test_utils.py b/tests/functional/adapter/test_utils.py new file mode 100644 index 00000000..71b23b28 --- /dev/null +++ b/tests/functional/adapter/test_utils.py @@ -0,0 +1,224 @@ +import pytest +from dbt.tests.adapter.utils.fixture_cast_bool_to_text import models__test_cast_bool_to_text_yml +from dbt.tests.adapter.utils.fixture_listagg import ( + models__test_listagg_yml, + seeds__data_listagg_csv, +) +from dbt.tests.adapter.utils.test_any_value import BaseAnyValue +from dbt.tests.adapter.utils.test_bool_or import BaseBoolOr +from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText +from dbt.tests.adapter.utils.test_concat import BaseConcat +from dbt.tests.adapter.utils.test_date_trunc import BaseDateTrunc +from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd +from dbt.tests.adapter.utils.test_datediff import BaseDateDiff +from dbt.tests.adapter.utils.test_escape_single_quotes import BaseEscapeSingleQuotesQuote +from dbt.tests.adapter.utils.test_except import BaseExcept +from dbt.tests.adapter.utils.test_hash import BaseHash +from dbt.tests.adapter.utils.test_intersect import BaseIntersect +from dbt.tests.adapter.utils.test_last_day import BaseLastDay +from dbt.tests.adapter.utils.test_length import BaseLength +from dbt.tests.adapter.utils.test_listagg import BaseListagg +from dbt.tests.adapter.utils.test_position import BasePosition +from dbt.tests.adapter.utils.test_replace import BaseReplace +from dbt.tests.adapter.utils.test_right import BaseRight +from dbt.tests.adapter.utils.test_safe_cast import BaseSafeCast +from dbt.tests.adapter.utils.test_split_part import BaseSplitPart +from dbt.tests.adapter.utils.test_string_literal import BaseStringLiteral + + +class TestAnyValueSQLServer(BaseAnyValue): + pass + + +@pytest.mark.skip("bool_or not supported in this adapter") +class TestBoolOrSQLServer(BaseBoolOr): + pass + + +class TestCastBoolToTextSQLServer(BaseCastBoolToText): + @pytest.fixture(scope="class") + def models(self): + models__test_cast_bool_to_text_sql = """ + with data as ( + + select 0 as input, 'false' as expected union all + select 1 as input, 'true' as expected union all + select null as input, null as expected + + ) + + select + + {{ cast_bool_to_text("input") }} as actual, + expected + + from data + """ + + return { + "test_cast_bool_to_text.yml": models__test_cast_bool_to_text_yml, + "test_cast_bool_to_text.sql": self.interpolate_macro_namespace( + models__test_cast_bool_to_text_sql, "cast_bool_to_text" + ), + } + + +class TestConcatSQLServer(BaseConcat): + pass + + +class TestDateTruncSQLServer(BaseDateTrunc): + pass + + +class TestHashSQLServer(BaseHash): + pass + + +class TestStringLiteralSQLServer(BaseStringLiteral): + pass + + +class TestSplitPartSQLServer(BaseSplitPart): + pass + + +class TestDateDiffSQLServer(BaseDateDiff): + pass + + +class TestEscapeSingleQuotesSQLServer(BaseEscapeSingleQuotesQuote): + pass + + +class TestIntersectSQLServer(BaseIntersect): + pass + + +class TestLastDaySQLServer(BaseLastDay): + pass + + +class TestLengthSQLServer(BaseLength): + pass + + +class TestListaggSQLServer(BaseListagg): + # Only supported in SQL Server 2017 and later or cloud versions + # DISTINCT not supported + # limit not supported + + @pytest.fixture(scope="class") + def seeds(self): + seeds__data_listagg_output_csv = """group_col,expected,version +1,"a_|_b_|_c",bottom_ordered +2,"1_|_a_|_p",bottom_ordered +3,"g_|_g_|_g",bottom_ordered +3,"g, g, g",comma_whitespace_unordered +3,"g,g,g",no_params + """ + + return { + "data_listagg.csv": seeds__data_listagg_csv, + "data_listagg_output.csv": seeds__data_listagg_output_csv, + } + + @pytest.fixture(scope="class") + def models(self): + models__test_listagg_sql = """ +with data as ( + + select * from {{ ref('data_listagg') }} + +), + +data_output as ( + + select * from {{ ref('data_listagg_output') }} + +), + +calculate as ( + + select + group_col, + {{ listagg('string_text', "'_|_'", "order by order_col") }} as actual, + 'bottom_ordered' as version + from data + group by group_col + + union all + + select + group_col, + {{ listagg('string_text', "', '") }} as actual, + 'comma_whitespace_unordered' as version + from data + where group_col = 3 + group by group_col + + union all + + select + group_col, + {{ listagg('string_text') }} as actual, + 'no_params' as version + from data + where group_col = 3 + group by group_col + +) + +select + calculate.actual, + data_output.expected +from calculate +left join data_output +on calculate.group_col = data_output.group_col +and calculate.version = data_output.version +""" + + return { + "test_listagg.yml": models__test_listagg_yml, + "test_listagg.sql": self.interpolate_macro_namespace( + models__test_listagg_sql, "listagg" + ), + } + + +class TestRightSQLServer(BaseRight): + pass + + +class TestSafeCastSQLServer(BaseSafeCast): + pass + + +class TestDateAddSQLServer(BaseDateAdd): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "name": "test", + "seeds": { + "test": { + "data_dateadd": { + "+column_types": { + "from_time": "datetimeoffset", + "result": "datetimeoffset", + }, + }, + }, + }, + } + + +class TestExceptSQLServer(BaseExcept): + pass + + +class TestPositionSQLServer(BasePosition): + pass + + +class TestReplaceSQLServer(BaseReplace): + pass diff --git a/test/unit/adapters/sqlserver/test_connections.py b/tests/unit/adapters/sqlserver/test_sql_server_connection_manager.py similarity index 60% rename from test/unit/adapters/sqlserver/test_connections.py rename to tests/unit/adapters/sqlserver/test_sql_server_connection_manager.py index 3580581d..f2ef0ffc 100644 --- a/test/unit/adapters/sqlserver/test_connections.py +++ b/tests/unit/adapters/sqlserver/test_sql_server_connection_manager.py @@ -5,10 +5,14 @@ import pytest from azure.identity import AzureCliCredential -from dbt.adapters.sqlserver import SQLServerCredentials, connections +from dbt.adapters.sqlserver.sql_server_connection_manager import ( + bool_to_connection_string_arg, + get_pyodbc_attrs_before, +) +from dbt.adapters.sqlserver.sql_server_credentials import SQLServerCredentials - -# See https://github.com/Azure/azure-sdk-for-python/blob/azure-identity_1.5.0/sdk/identity/azure-identity/tests/test_cli_credential.py +# See +# https://github.com/Azure/azure-sdk-for-python/blob/azure-identity_1.5.0/sdk/identity/azure-identity/tests/test_cli_credential.py CHECK_OUTPUT = AzureCliCredential.__module__ + ".subprocess.check_output" @@ -29,9 +33,9 @@ def mock_cli_access_token() -> str: expected_expires_on = 1602015811 successful_output = json.dumps( { - "expiresOn": dt.datetime.fromtimestamp( - expected_expires_on - ).strftime("%Y-%m-%d %H:%M:%S.%f"), + "expiresOn": dt.datetime.fromtimestamp(expected_expires_on).strftime( + "%Y-%m-%d %H:%M:%S.%f" + ), "accessToken": access_token, "subscription": "some-guid", "tenant": "some-guid", @@ -47,8 +51,8 @@ def test_get_pyodbc_attrs_before_empty_dict_when_service_principal( """ When the authentication is set to sql we expect an empty attrs before. """ - attrs_before = connections.get_pyodbc_attrs_before(credentials) - assert attrs_before == dict() + attrs_before = get_pyodbc_attrs_before(credentials) + assert attrs_before == {} @pytest.mark.parametrize("authentication", ["CLI", "cli", "cLi"]) @@ -62,8 +66,13 @@ def test_get_pyodbc_attrs_before_contains_access_token_key_for_cli_authenticatio access token key. """ credentials.authentication = authentication - with mock.patch( - CHECK_OUTPUT, mock.Mock(return_value=mock_cli_access_token) - ): - attrs_before = connections.get_pyodbc_attrs_before(credentials) + with mock.patch(CHECK_OUTPUT, mock.Mock(return_value=mock_cli_access_token)): + attrs_before = get_pyodbc_attrs_before(credentials) assert 1256 in attrs_before.keys() + + +@pytest.mark.parametrize( + "key, value, expected", [("somekey", False, "somekey=No"), ("somekey", True, "somekey=Yes")] +) +def test_bool_to_connection_string_arg(key: str, value: bool, expected: str) -> None: + assert bool_to_connection_string_arg(key, value) == expected diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 6d48e1f3..00000000 --- a/tox.ini +++ /dev/null @@ -1,13 +0,0 @@ -[tox] -envlist = py37 - -[testenv] -commands = pytest {posargs} -passenv = - DBT_AZURESQL_DB - DBT_AZURESQL_SERVER - DBT_AZURESQL_PWD - DBT_AZURESQL_UID -deps = - -rdev_requirements.txt - -e.