ibis-backends-cloud.yml

name: Cloud Backends

on:
  push:
    # Skip the backend suite if all changes are in the docs directory
    paths-ignore:
      - "docs/**"
      - "**/*.md"
      - "**/*.qmd"
      - "codecov.yml"
      - ".envrc"
    branches:
      - main
  pull_request_target:
    types:
      - labeled

permissions:
  # this allows extractions/setup-just to list releases for `just` at a higher
  # rate limit while restricting GITHUB_TOKEN permissions elsewhere
  contents: read

env:
  FORCE_COLOR: "1"
  SQLALCHEMY_WARN_20: "1"
  HYPOTHESIS_PROFILE: "ci"

jobs:
  test_backends:
    name: ${{ matrix.backend.title }} python-${{ matrix.python-version }}
    # only a single bigquery or snowflake run at a time, otherwise test data is
    # clobbered by concurrent runs
    concurrency:
      group: ${{ matrix.backend.title }}-${{ matrix.python-version }}-${{ github.event.label.name || 'ci-run-cloud' }}
      cancel-in-progress: false

    runs-on: ubuntu-latest
    if: github.event_name == 'push' || github.event.label.name == 'ci-run-cloud'
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - "3.10"
          - "3.12"
        backend:
          - name: snowflake
            title: Snowflake
            extras:
              - snowflake
          - name: databricks
            title: Databricks
            extras:
              - databricks
        include:
          - python-version: "3.10"
            backend:
              name: bigquery
              title: BigQuery
              extras:
                - bigquery
          - python-version: "3.12"
            backend:
              name: bigquery
              title: BigQuery
              extras:
                - bigquery
                - geospatial
          - python-version: "3.10"
            backend:
              name: snowflake
              title: Snowflake + Snowpark
              key: snowpark
              extras:
                - snowflake
          - python-version: "3.11"
            backend:
              name: snowflake
              title: Snowflake + Snowpark
              key: snowpark
              extras:
                - snowflake
    steps:
      - name: checkout
        uses: actions/checkout@v4
        if: github.event.label.name != 'ci-run-cloud'

      - name: checkout
        if: github.event.label.name == 'ci-run-cloud'
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha }}

      - uses: actions/create-github-app-token@v1.11.0
        id: generate_token
        with:
          app-id: ${{ secrets.DOCS_BOT_APP_ID }}
          private-key: ${{ secrets.DOCS_BOT_APP_PRIVATE_KEY }}

      - name: reset cloud ci run label
        uses: actions-ecosystem/action-remove-labels@v1
        if: github.event.label.name == 'ci-run-cloud'
        with:
          labels: ci-run-cloud
          github_token: ${{ steps.generate_token.outputs.token }}

      - name: install python
        uses: actions/setup-python@v5
        id: install_python
        with:
          python-version: ${{ matrix.python-version }}

      - name: install poetry
        run: pip install 'poetry==1.8.4'

      - name: install additional deps
        if: matrix.backend.key == 'snowpark'
        run: poetry add --lock snowflake-snowpark-python --python="==${{ steps.install_python.outputs.python-version }}"

      - name: install ibis
        run: poetry install --without dev --without docs --extras "${{ join(matrix.backend.extras, ' ') }} examples"

      - uses: extractions/setup-just@v2
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: download backend data
        run: just download-data

      - uses: google-github-actions/auth@v2
        with:
          credentials_json: ${{ secrets.GCP_CREDENTIALS }}

      - name: setup databricks credentials
        if: matrix.backend.name == 'databricks'
        run: |
          {
            echo "DATABRICKS_HTTP_PATH=${DATABRICKS_HTTP_PATH}"
            echo "DATABRICKS_SERVER_HOSTNAME=${DATABRICKS_SERVER_HOSTNAME}"
            echo "DATABRICKS_TOKEN=${DATABRICKS_TOKEN}"
          } >> "$GITHUB_ENV"
        env:
          DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH }}
          DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_SERVER_HOSTNAME }}
          DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}

      - name: setup snowflake credentials
        if: matrix.backend.name == 'snowflake'
        run: |
          pyversion="${{ matrix.python-version }}"
          {
            echo "SNOWFLAKE_USER=${SNOWFLAKE_USER}"
            echo "SNOWFLAKE_PASSWORD=${SNOWFLAKE_PASSWORD}"
            echo "SNOWFLAKE_ACCOUNT=${SNOWFLAKE_ACCOUNT}"
            echo "SNOWFLAKE_DATABASE=${SNOWFLAKE_DATABASE}"
            echo "SNOWFLAKE_SCHEMA=${SNOWFLAKE_SCHEMA}_python${pyversion//./}_${{ matrix.backend.key }}"
            echo "SNOWFLAKE_WAREHOUSE=${SNOWFLAKE_WAREHOUSE}"
          } >> "$GITHUB_ENV"
        env:
          SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
          SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
          SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
          SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }}
          SNOWFLAKE_SCHEMA: ${{ secrets.SNOWFLAKE_SCHEMA }}
          SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }}

      - name: enable snowpark testing
        if: matrix.backend.key == 'snowpark'
        run: echo "SNOWFLAKE_SNOWPARK=1" >> "$GITHUB_ENV"

      - name: "run parallel tests: ${{ matrix.backend.name }}"
        run: just ci-check -m ${{ matrix.backend.name }} --numprocesses auto --dist=loadgroup

      - name: upload code coverage
        if: success()
        continue-on-error: true
        uses: codecov/codecov-action@v4
        with:
          flags: backend,${{ matrix.backend.name }},${{ runner.os }},python-${{ steps.install_python.outputs.python-version }}
          token: ${{ secrets.CODECOV_TOKEN }}