Merge pull request #10 from pangpang20/master

Enhance Test and CI Compatibility with openGauss

Author: 5xuanwen

.flake8

@@ -17,3 +17,6 @@ per-file-ignores =
     # Pytest's importorskip() getting in the way
     tests/types/test_numpy.py: E402
     tests/types/test_shapely.py: E402
+
+    tools/async_to_sync.py: E999
+    tools/bump_version.py: E999

.github/workflows/3rd-party-tests.yml

@@ -1,15 +1,15 @@
 name: 3rd party tests
 
 on:
-  push:
-    branches:
-      - "master"
-      - "maint-3.1"
-      - "sqlalchemy_pipeline"
-      - "django_pipeline"
-    paths-ignore:
-      - "docs/*"
-      - "tools/*"
+  # push:
+  #   branches:
+  #     - "master"
+  #     - "maint-3.1"
+  #     - "sqlalchemy_pipeline"
+  #     - "django_pipeline"
+  #   paths-ignore:
+  #     - "docs/*"
+  #     - "tools/*"
   workflow_dispatch:
 
 concurrency:

.github/workflows/ci.yml

@@ -0,0 +1,104 @@
+name: CI for gaussdb-python
+
+on:
+  push:
+    branches:
+        - "*"
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_name }}
+  cancel-in-progress: true
+    
+jobs:
+  test:
+    runs-on: ubuntu-22.04
+
+    services:
+      opengauss:
+        image: opengauss/opengauss-server:latest
+        ports:
+          - 5432:5432
+        env:
+          GS_USERNAME: root
+          GS_USER_PASSWORD: ${{ secrets.OPENGAUSS_PASSWORD }}
+          GS_PASSWORD: ${{ secrets.OPENGAUSS_PASSWORD }}
+        options: >-
+          --privileged=true
+          --name opengauss-custom
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.9"
+          cache: pip
+
+      - name: Create and activate virtual environment
+        run: |
+          python -m venv venv
+          echo "VENV_PATH=$GITHUB_WORKSPACE/venv/bin" >> $GITHUB_ENV
+          source venv/bin/activate
+
+      - name: Install gaussdb libpq driver
+        run: |
+          sudo apt update
+          sudo apt install -y wget unzip
+          wget -O /tmp/GaussDB_driver.zip https://dbs-download.obs.cn-north-1.myhuaweicloud.com/GaussDB/1730887196055/GaussDB_driver.zip
+          unzip /tmp/GaussDB_driver.zip -d /tmp/ && rm -rf /tmp/GaussDB_driver.zip
+          \cp /tmp/GaussDB_driver/Centralized/Hce2_X86_64/GaussDB-Kernel*64bit_Python.tar.gz /tmp/
+          tar -zxvf /tmp/GaussDB-Kernel*64bit_Python.tar.gz -C /tmp/ && rm -rf /tmp/GaussDB-Kernel*64bit_Python.tar.gz && rm -rf /tmp/psycopg2 && rm -rf /tmp/GaussDB_driver
+          echo /tmp/lib | sudo tee /etc/ld.so.conf.d/gauss-libpq.conf
+          sudo sed -i '1s|^|/tmp/lib\n|' /etc/ld.so.conf
+          sudo ldconfig
+          ldconfig -p | grep pq
+           
+      - name: Install dependencies
+        run: |
+          source venv/bin/activate
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install -e "./psycopg[dev,test]"
+          pip install -e ./psycopg_pool
+
+          
+      - name: Wait for openGauss to be ready
+        env:
+          GSQL_PASSWORD: ${{ secrets.OPENGAUSS_PASSWORD }}
+        run: |
+          source venv/bin/activate
+          for i in {1..15}; do
+            pg_isready -h localhost -p 5432 -U root && break
+            sleep 5
+          done
+          if ! pg_isready -h localhost -p 5432 -U root; then
+            echo "openGauss is not ready"
+            exit 1
+          fi
+
+      - name: Create test database
+        run: |
+          docker exec opengauss-custom bash -c "su - omm -c 'gsql -d postgres -c \"CREATE DATABASE test DBCOMPATIBILITY '\''PG'\'';\"'"
+
+      - name: Create report directory
+        run: |
+          mkdir -p reports
+
+      - name: Run tests
+        env:
+          PYTHONPATH: ./psycopg:./psycopg_pool
+          PSYCOPG_IMPL: python
+          PSYCOPG_TEST_DSN: "host=127.0.0.1 port=5432 dbname=test user=root password=${{ secrets.OPENGAUSS_PASSWORD }} "
+        run: |
+          source venv/bin/activate
+          pytest -s -v
+
+      - name: Cleanup
+        if: always()
+        run: |
+          docker stop opengauss-custom
+          docker rm opengauss-custom

.github/workflows/docs.yml

@@ -1,10 +1,11 @@
 name: Build documentation
 
 on:
-  push:
-    branches:
-      # This should match the DOC3_BRANCH value in the psycopg-website Makefile
-      - master
+  # push:
+  #   branches:
+  #     # This should match the DOC3_BRANCH value in the psycopg-website Makefile
+  #     - master
+  pull_request:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref_name }}

.github/workflows/lint.yml

@@ -20,18 +20,32 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v5
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.9"
 
       - name: install packages to tests
-        run: pip install ./psycopg[dev,test]
+        run: |
+          pip install ./psycopg[dev,test]
+          pip install types-polib
+          pip install pre-commit
 
       - name: Lint codebase
         run: pre-commit run -a --color=always
 
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install packages for async_to_sync
+        run: |
+            pip install ./psycopg[dev,test]
+            pip install types-polib
+
       - name: Check for sync/async inconsistencies
-        run: ./tools/async_to_sync.py --check --all
+        run: ./tools/async_to_sync.py --check $(find tests -name "*_async.py" -type f ! -path "tests/pq/test_async.py")
 
       - name: Install requirements to generate docs
         run: sudo apt-get install -y libgeos-dev

.github/workflows/tests.yml

@@ -1,11 +1,11 @@
 name: Tests
 
 on:
-  push:
-    # This should disable running the workflow on tags, according to the
-    # on.<push|pull_request>.<branches|tags> GitHub Actions docs.
-    branches:
-      - "*"
+  # push:
+  #   # This should disable running the workflow on tags, according to the
+  #   # on.<push|pull_request>.<branches|tags> GitHub Actions docs.
+  #   branches:
+  #     - "*"
   pull_request:
   schedule:
     - cron: '48 6 * * *'

mypy.ini

@@ -0,0 +1,10 @@
+[mypy]
+python_version = 3.9
+ignore_missing_imports = True
+check_untyped_defs = True
+
+[mypy-tools.async_to_sync]
+ignore_errors = True
+
+[mypy-tools.bump_version]
+ignore_errors = True

psycopg/psycopg/_copy_base.py

@@ -10,7 +10,7 @@
 import sys
 import struct
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Any, Generic
+from typing import TYPE_CHECKING, Any, Generic, Optional, Tuple
 from collections.abc import Sequence
 
 from . import adapt
@@ -171,7 +171,7 @@ def _read_row_gen(self) -> PQGen[tuple[Any, ...] | None]:
 
         row = self.formatter.parse_row(data)
 
-        if row == IS_BINARY_SIGNATURE:
+        if isinstance(row, str) and row == IS_BINARY_SIGNATURE:
             row = yield from self._read_row_gen()
 
         if row is None:
@@ -267,21 +267,19 @@ def __init__(self, transformer: Transformer):
         super().__init__(transformer)
         self._signature_sent = False
 
-    def parse_row(self, data: Buffer) -> tuple[Any, ...] | str | None:
-        rv: tuple[Any, ...] | None = None
-
+    def parse_row(self, data: Buffer) -> Optional[Tuple[Any, ...]]:
         if not self._signature_sent:
             if data[: len(_binary_signature)] != _binary_signature:
                 raise e.DataError(
                     "binary copy doesn't start with the expected signature"
                 )
             self._signature_sent = True
-            return IS_BINARY_SIGNATURE
+            return None
 
         if data != _binary_trailer:
-            rv = parse_row_binary(data, self.transformer)
+            return parse_row_binary(data, self.transformer)
 
-        return rv
+        return None
 
     def write(self, buffer: Buffer | str) -> Buffer:
         data = self._ensure_bytes(buffer)

psycopg/psycopg/errors.py

@@ -440,7 +440,9 @@ def severity(self) -> str | None:
 
     @property
     def severity_nonlocalized(self) -> str | None:
-        raise NotSupportedError("This is present only in reports generated by libpq versions 9.6 and later.") 
+        raise NotSupportedError(
+            "This is present only in reports generated by libpq versions 9.6 and later."
+        )
 
     @property
     def sqlstate(self) -> str | None:

psycopg/psycopg/types/multirange.py

@@ -14,7 +14,7 @@
 
 from .. import _oids
 from .. import errors as e
-from .. import postgres, sql
+from .. import postgres
 from ..pq import Format
 from ..abc import AdaptContext, Buffer, Dumper, DumperKey, Query
 from .range import Range, T, dump_range_binary, dump_range_text, fail_dump
@@ -47,9 +47,7 @@ def __init__(
 
     @classmethod
     def _get_info_query(cls, conn: BaseConnection[Any]) -> Query:
-        raise e.NotSupportedError(
-                "multirange types are not supported in GaussDB"
-            )
+        raise e.NotSupportedError("multirange types are not supported in GaussDB")
 
     def _added(self, registry: TypesRegistry) -> None:
         # Map multiranges ranges and subtypes to info