Merge pull request #271 from microsoft/dev

Prepare for 1.3 release

Author: maikhanhbui

.github/workflows/codeql-analysis.yml

@@ -35,11 +35,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -50,7 +50,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -64,4 +64,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

README.md

@@ -10,8 +10,8 @@ We hope you enjoy using the MSSQL-Django 3rd party backend.
 
 ## Features
 
--  Supports Django 3.2 and 4.0
--  Tested on Microsoft SQL Server 2016, 2017, 2019
+-  Supports Django 3.2, 4.0, 4.1 and 4.2
+-  Tested on Microsoft SQL Server 2016, 2017, 2019, 2022
 -  Passes most of the tests of the Django test suite
 -  Compatible with
    [Micosoft ODBC Driver for SQL Server](https://docs.microsoft.com/en-us/sql/connect/odbc/microsoft-odbc-driver-for-sql-server),

azure-pipelines.yml

@@ -24,6 +24,22 @@ jobs:
 
     strategy:
       matrix:
+        Python3.11 - Django 4.2:
+          python.version: '3.11'
+          tox.env: 'py311-django42'
+        Python3.10 - Django 4.2:
+          python.version: '3.10'
+          tox.env: 'py310-django42'
+        Python 3.9 - Django 4.2:
+          python.version: '3.9'
+          tox.env: 'py39-django42'
+        Python 3.8 - Django 4.2:
+          python.version: '3.8'
+          tox.env: 'py38-django42'
+
+        Python3.11 - Django 4.1:
+          python.version: '3.11'
+          tox.env: 'py311-django41'
         Python3.10 - Django 4.1:
           python.version: '3.10'
           tox.env: 'py310-django41'
@@ -34,6 +50,9 @@ jobs:
           python.version: '3.8'
           tox.env: 'py38-django41'
 
+        Python3.11 - Django 4.0:
+          python.version: '3.11'
+          tox.env: 'py311-django40'
         Python3.10 - Django 4.0:
           python.version: '3.10'
           tox.env: 'py310-django40'
@@ -44,6 +63,9 @@ jobs:
           python.version: '3.8'
           tox.env: 'py38-django40'
 
+        Python3.11 - Django 3.2:
+          python.version: '3.11'
+          tox.env: 'py311-django32'
         Python 3.9 - Django 3.2:
           python.version: '3.9'
           tox.env: 'py39-django32'
@@ -53,9 +75,7 @@ jobs:
         Python 3.7 - Django 3.2:
           python.version: '3.7'
           tox.env: 'py37-django32'
-        Python 3.6 - Django 3.2:
-          python.version: '3.6'
-          tox.env: 'py36-django32'
+
 
     steps:
       - task: CredScan@2
@@ -111,6 +131,22 @@ jobs:
 
     strategy:
       matrix:
+        Python3.11 - Django 4.2:
+          python.version: '3.11'
+          tox.env: 'py311-django42'
+        Python3.10 - Django 4.2:
+          python.version: '3.10'
+          tox.env: 'py310-django42'
+        Python 3.9 - Django 4.2:
+          python.version: '3.9'
+          tox.env: 'py39-django42'
+        Python 3.8 - Django 4.2:
+          python.version: '3.8'
+          tox.env: 'py38-django42'
+
+        Python3.11 - Django 4.1:
+          python.version: '3.11'
+          tox.env: 'py311-django41'
         Python3.10 - Django 4.1:
           python.version: '3.10'
           tox.env: 'py310-django41'
@@ -121,6 +157,9 @@ jobs:
           python.version: '3.8'
           tox.env: 'py38-django41'
 
+        Python3.11 - Django 4.0:
+          python.version: '3.11'
+          tox.env: 'py311-django40'
         Python3.10 - Django 4.0:
           python.version: '3.10'
           tox.env: 'py310-django40'
@@ -131,6 +170,9 @@ jobs:
           python.version: '3.8'
           tox.env: 'py38-django40'
 
+        Python3.11 - Django 3.2:
+          python.version: '3.11'
+          tox.env: 'py311-django32'
         Python 3.9 - Django 3.2:
           python.version: '3.9'
           tox.env: 'py39-django32'
@@ -140,9 +182,6 @@ jobs:
         Python 3.7 - Django 3.2:
           python.version: '3.7'
           tox.env: 'py37-django32'
-        Python 3.6 - Django 3.2:
-          python.version: '3.6'
-          tox.env: 'py36-django32'
 
     steps:
       - task: UsePythonVersion@0

mssql/base.py

@@ -11,6 +11,7 @@
 import datetime
 
 from django.core.exceptions import ImproperlyConfigured
+from django.utils.functional import cached_property
 
 try:
     import pyodbc as Database
@@ -430,7 +431,7 @@ def init_connection_state(self):
         if (options.get('return_rows_bulk_insert', False)):
             self.features_class.can_return_rows_from_bulk_insert = True
 
-        val = self.get_system_datetime()
+        val = self.get_system_datetime
         if isinstance(val, str):
             raise ImproperlyConfigured(
                 "The database driver doesn't support modern datatime types.")
@@ -443,6 +444,7 @@ def is_usable(self):
         else:
             return True
 
+    @cached_property
     def get_system_datetime(self):
         # http://blogs.msdn.com/b/sqlnativeclient/archive/2008/02/27/microsoft-sql-server-native-client-and-microsoft-sql-server-2008-native-client.aspx
         with self.temporary_connection() as cursor:

mssql/compiler.py

@@ -15,6 +15,8 @@
 from django.db.utils import NotSupportedError
 if django.VERSION >= (3, 1):
     from django.db.models.fields.json import compile_json_path, KeyTransform as json_KeyTransform
+if django.VERSION >= (4, 2):
+    from django.core.exceptions import FullResultSet
 
 def _as_sql_agv(self, compiler, connection):
     return self.as_sql(compiler, connection, template='%(function)s(CONVERT(float, %(field)s))')
@@ -76,8 +78,9 @@ def _as_sql_lpad(self, compiler, connection):
     params.extend(length_arg)
     params.extend(expression_arg)
     params.extend(expression_arg)
-    template = ('LEFT(REPLICATE(%(fill_text)s, %(length)s), CASE WHEN %(length)s > LEN(%(expression)s) '
-                'THEN %(length)s - LEN(%(expression)s) ELSE 0 END) + %(expression)s')
+    params.extend(length_arg)
+    template = ('LEFT(LEFT(REPLICATE(%(fill_text)s, %(length)s), CASE WHEN %(length)s > LEN(%(expression)s) '
+                'THEN %(length)s - LEN(%(expression)s) ELSE 0 END) + %(expression)s, %(length)s)')
     return template % {'expression': expression, 'length': length, 'fill_text': fill_text}, params
 
 
@@ -227,13 +230,26 @@ def as_sql(self, with_limits=True, with_col_aliases=False):
                 if not getattr(features, 'supports_select_{}'.format(combinator)):
                     raise NotSupportedError('{} is not supported on this database backend.'.format(combinator))
                 result, params = self.get_combinator_sql(combinator, self.query.combinator_all)
+            elif django.VERSION >= (4, 2) and self.qualify:
+                result, params = self.get_qualify_sql()
+                order_by = None
             else:
                 distinct_fields, distinct_params = self.get_distinct()
                 # This must come after 'select', 'ordering', and 'distinct' -- see
                 # docstring of get_from_clause() for details.
                 from_, f_params = self.get_from_clause()
-                where, w_params = self.compile(self.where) if self.where is not None else ("", [])
-                having, h_params = self.compile(self.having) if self.having is not None else ("", [])
+                if django.VERSION >= (4, 2):
+                    try:
+                        where, w_params = self.compile(self.where) if self.where is not None else ("", [])
+                    except FullResultSet:
+                        where, w_params = "", []
+                    try:
+                        having, h_params = self.compile(self.having) if self.having is not None else ("", [])
+                    except FullResultSet:
+                        having, h_params = "", []
+                else:
+                    where, w_params = self.compile(self.where) if self.where is not None else ("", [])
+                    having, h_params = self.compile(self.having) if self.having is not None else ("", [])
                 params = []
                 result = ['SELECT']
 
@@ -364,7 +380,7 @@ def as_sql(self, with_limits=True, with_col_aliases=False):
                 # Django 2.x.  See https://github.com/microsoft/mssql-django/issues/12
                 # Add OFFSET for all Django versions.
                 # https://github.com/microsoft/mssql-django/issues/109
-                if not (do_offset or do_limit):
+                if not (do_offset or do_limit) and supports_offset_clause:
                     result.append("OFFSET 0 ROWS")
 
             # SQL Server requires the backend-specific emulation (2008 or earlier)

mssql/features.py

@@ -6,6 +6,7 @@
 
 
 class DatabaseFeatures(BaseDatabaseFeatures):
+    allows_group_by_select_index = False
     allow_sliced_subqueries_with_in = False
     can_introspect_autofield = True
     can_introspect_json_field = False
@@ -71,4 +72,4 @@ def introspected_field_types(self):
         return {
             **super().introspected_field_types,
             "DurationField": "BigIntegerField",
-        }
+        }

mssql/functions.py

@@ -10,7 +10,9 @@
 from django.db.models.expressions import Case, Exists, Expression, OrderBy, When, Window
 from django.db.models.fields import BinaryField, Field
 from django.db.models.functions import Cast, NthValue, MD5, SHA1, SHA224, SHA256, SHA384, SHA512
+from django.db.models.functions.datetime import Now
 from django.db.models.functions.math import ATan2, Ln, Log, Mod, Round, Degrees, Radians, Power
+from django.db.models.functions.text import Replace
 from django.db.models.lookups import In, Lookup
 from django.db.models.query import QuerySet
 from django.db.models.sql.query import Query
@@ -44,6 +46,19 @@ def sqlserver_log(self, compiler, connection, **extra_context):
 def sqlserver_ln(self, compiler, connection, **extra_context):
     return self.as_sql(compiler, connection, function='LOG', **extra_context)
 
+
+def sqlserver_replace(self, compiler, connection, **extra_context):
+    current_db = "CONVERT(varchar, (SELECT DB_NAME()))"
+    with connection.cursor() as cursor:
+        cursor.execute("SELECT CONVERT(varchar, DATABASEPROPERTYEX(%s, 'collation'))" % current_db)
+        default_collation = cursor.fetchone()[0]
+    current_collation = default_collation.replace('_CI', '_CS')
+    return self.as_sql(
+            compiler, connection, function='REPLACE',
+            template = 'REPLACE(%s COLLATE %s)' % ('%(expressions)s', current_collation),
+            **extra_context
+        )
+
 def sqlserver_degrees(self, compiler, connection, **extra_context):
     return self.as_sql(
             compiler, connection, function='DEGREES',
@@ -109,6 +124,10 @@ def sqlserver_exists(self, compiler, connection, template=None, **extra_context)
     sql = 'CASE WHEN {} THEN 1 ELSE 0 END'.format(sql)
     return sql, params
 
+def sqlserver_now(self, compiler, connection, **extra_context):
+        return self.as_sql(
+            compiler, connection, template="SYSDATETIME()", **extra_context
+        )
 
 def sqlserver_lookup(self, compiler, connection):
     # MSSQL doesn't allow EXISTS() to be compared to another expression
@@ -273,7 +292,7 @@ def bulk_update_with_default(self, objs, fields, batch_size=None, default=0):
         SQL Server require that at least one of the result expressions in a CASE specification must be an expression other than the NULL constant.
         Patched with a default value 0. The user can also pass a custom default value for CASE statement.
     """
-    if batch_size is not None and batch_size < 0:
+    if batch_size is not None and batch_size <= 0:
         raise ValueError('Batch size must be a positive integer.')
     if not fields:
         raise ValueError('Field names must be given to bulk_update().')
@@ -441,6 +460,8 @@ def sqlserver_sha512(self, compiler, connection, **extra_context):
 NthValue.as_microsoft = sqlserver_nth_value
 Round.as_microsoft = sqlserver_round
 Window.as_microsoft = sqlserver_window
+Replace.as_microsoft = sqlserver_replace
+Now.as_microsoft = sqlserver_now
 MD5.as_microsoft = sqlserver_md5
 SHA1.as_microsoft = sqlserver_sha1
 SHA224.as_microsoft = sqlserver_sha224

mssql/introspection.py

@@ -107,7 +107,7 @@ def get_table_description(self, cursor, table_name, identity_check=True):
         """
 
         # map pyodbc's cursor.columns to db-api cursor description
-        columns = [[c[3], c[4], None, c[6], c[6], c[8], c[10], c[12]] for c in cursor.columns(table=table_name)]
+        columns = [[c[3], c[4], c[6], c[6], c[6], c[8], c[10], c[12]] for c in cursor.columns(table=table_name)]
 
         if not columns:
             raise DatabaseError(f"Table {table_name} does not exist.")

mssql/operations.py

@@ -404,7 +404,7 @@ def limit_offset_sql(self, low_mark, high_mark):
         """Return LIMIT/OFFSET SQL clause."""
         limit, offset = self._get_limit_offset_params(low_mark, high_mark)
         return '%s%s' % (
-            (' OFFSET %d ROWS' % offset) if offset else '',
+            (' OFFSET %d ROWS' % offset) if offset else ' OFFSET 0 ROWS',
             (' FETCH FIRST %d ROWS ONLY' % limit) if limit else '',
         )
 
@@ -418,7 +418,13 @@ def last_executed_query(self, cursor, sql, params):
         exists for database backends to provide a better implementation
         according to their own quoting schemes.
         """
-        return super().last_executed_query(cursor, cursor.last_sql, cursor.last_params)
+        if params:
+            if isinstance(params, list):
+                params = tuple(params)
+            return sql % params
+        # Just return sql when there are no parameters.
+        else:
+            return sql
 
     def savepoint_create_sql(self, sid):
         """