Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions packages/django-google-spanner/README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,30 @@ A transaction is automatically started if you define an
To disable transactions in autocommit mode, set the flag `ALLOW_TRANSACTIONS_IN_AUTO_COMMIT` to False in your ``settings.py`` file.


DDL batching
~~~~~~~~~~~~

By default, ``DatabaseSchemaEditor`` executes each DDL statement individually.
To batch DDL statements generated within a ``schema_editor()`` context (such as
during ``manage.py migrate`` or test database creation) into batched
``UpdateDatabaseDdl`` requests, set ``'batch_ddl': True`` in ``OPTIONS``
(defaults to ``False``):

.. code:: python

DATABASES = {
'default': {
'ENGINE': 'django_spanner',
'PROJECT': '$PROJECT',
'INSTANCE': '$INSTANCE',
'NAME': '$DATABASE',
'OPTIONS': {
'batch_ddl': True,
},
}
}


Set credentials and project environment variables
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
You'll need to download a service account JSON key file and point to it using an environment variable:
Expand Down
12 changes: 12 additions & 0 deletions packages/django-google-spanner/django_spanner/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,7 @@ def get_new_connection(self, conn_params):
conn_params.pop("instance", None)
conn_params.pop("instance_id", None)
conn_params.pop("client", None)
conn_params.pop("batch_ddl", None)
# Ensure client is initialized
instance = self.instance
return self.Database.connect(
Expand Down Expand Up @@ -209,6 +210,17 @@ def create_cursor(self, name=None):
"""
return self.connection.cursor()

def set_autocommit(
self, autocommit, force_begin_transaction_with_broken_autocommit=False
):
editor = getattr(self, "_active_ddl_batch_editor", None)
if editor is not None and editor._ddl_batch_active:
editor._exit_ddl_batch(clean=True)
super().set_autocommit(
autocommit,
force_begin_transaction_with_broken_autocommit=force_begin_transaction_with_broken_autocommit,
)

def _set_autocommit(self, autocommit):
"""Set the Spanner transaction autocommit flag.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ def create_test_db(self, *args, **kwargs):
# by a developer running the tests locally.
if os.environ.get("RUNNING_SPANNER_BACKEND_TESTS") == "1":
self.mark_skips()
super().create_test_db(*args, **kwargs)
return super().create_test_db(*args, **kwargs)

def _create_test_db(self, verbosity, autoclobber, keepdb=False):
"""
Expand Down
105 changes: 104 additions & 1 deletion packages/django-google-spanner/django_spanner/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,24 @@
import uuid

import django
from django.db import NotSupportedError
from django.db import (
IntegrityError,
NotSupportedError,
OperationalError,
ProgrammingError,
)
from django.db.backends.base.schema import BaseDatabaseSchemaEditor
from django.db.models.fields import NOT_PROVIDED
from google.api_core.exceptions import (
AlreadyExists,
FailedPrecondition,
GoogleAPICallError,
InternalServerError,
InvalidArgument,
OutOfRange,
)
from google.cloud.spanner_dbapi import parse_utils
from google.cloud.spanner_dbapi.parsed_statement import StatementType

from django_spanner import USE_EMULATOR
from django_spanner._opentelemetry_tracing import trace_call
Expand Down Expand Up @@ -57,6 +72,94 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor):
# sql_create_inline_fk = "CONSTRAINT FK_%(to_table)s_%(to_column)s_%(from_table)s_%(from_column)s FOREIGN KEY (%(from_column_norm)s) REFERENCES %(to_table_norm)s (%(to_column_norm)s)" # noqa
sql_create_inline_fk = None

def __init__(self, connection, collect_sql=False, atomic=True):
self._atomic = atomic
super().__init__(connection, collect_sql=collect_sql, atomic=atomic)

def _commit_ddls(self):
try:
self.connection.connection.commit()
except (AlreadyExists, FailedPrecondition, OutOfRange) as exc:
raise IntegrityError(getattr(exc, "details", exc)) from exc
except InvalidArgument as exc:
raise ProgrammingError(getattr(exc, "details", exc)) from exc
except (InternalServerError, GoogleAPICallError) as exc:
raise OperationalError(getattr(exc, "details", exc)) from exc

def _enter_ddl_batch(self):
if not self._ddl_batch_active:
self.connection.ensure_connection()
self.connection._set_autocommit(False)
self._ddl_batch_active = True

def _exit_ddl_batch(self, clean):
if not self._ddl_batch_active:
return
try:
if clean:
self._commit_ddls()
else:
self.connection.connection.rollback()
finally:
self._ddl_batch_active = False
self.connection._set_autocommit(True)

def _ddl_execution_wrapper(self, execute, sql, params, many, context):
is_ddl = False
if (
not many
and sql is not None
and not self.connection.in_atomic_block
and self.connection.get_autocommit()
):
parsed = parse_utils.classify_statement(str(sql))
is_ddl = parsed is not None and parsed.statement_type == StatementType.DDL

if is_ddl:
self._enter_ddl_batch()
elif self._ddl_batch_active:
self._exit_ddl_batch(clean=True)

return execute(sql, params, many, context)

def __enter__(self):
super().__enter__()
self._ddl_batch_active = False
self._execute_wrapper_cm = None
batch_ddl = self.connection.settings_dict.get("OPTIONS", {}).get(
"batch_ddl", False
)
if (
batch_ddl
and getattr(self, "_atomic", True)
and not self.collect_sql
and not self.connection.in_atomic_block
and self.connection.get_autocommit()
and getattr(self.connection, "_active_ddl_batch_editor", None) is None
):
self.connection._active_ddl_batch_editor = self
self._execute_wrapper_cm = self.connection.execute_wrapper(
self._ddl_execution_wrapper
)
self._execute_wrapper_cm.__enter__()
return self

def __exit__(self, exc_type, exc_value, traceback):
clean_exit = False
try:
res = super().__exit__(exc_type, exc_value, traceback)
clean_exit = (exc_type is None) or bool(res)
return res
finally:
try:
self._exit_ddl_batch(clean=clean_exit)
finally:
if getattr(self, "_execute_wrapper_cm", None) is not None:
self._execute_wrapper_cm.__exit__(exc_type, exc_value, traceback)
self._execute_wrapper_cm = None
if getattr(self.connection, "_active_ddl_batch_editor", None) is self:
self.connection._active_ddl_batch_editor = None

def create_model(self, model):
"""
Create a table and any accompanying indexes or unique constraints for
Expand Down
6 changes: 6 additions & 0 deletions packages/django-google-spanner/django_test_suite_5.2.sh
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,18 @@ DATABASES = {
'PROJECT': "$PROJECT",
'INSTANCE': "$INSTANCE",
'NAME': "$TEST_DBNAME",
'OPTIONS': {
'batch_ddl': True,
},
},
'other': {
'ENGINE': 'django_spanner',
'PROJECT': "$PROJECT",
'INSTANCE': "$INSTANCE",
'NAME': "$TEST_DBNAME_OTHER",
'OPTIONS': {
'batch_ddl': True,
},
},
}
USE_TZ = False
Expand Down
6 changes: 6 additions & 0 deletions packages/django-google-spanner/django_test_suite_6.0.sh
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,18 @@ DATABASES = {
'PROJECT': "$PROJECT",
'INSTANCE': "$INSTANCE",
'NAME': "$TEST_DBNAME",
'OPTIONS': {
'batch_ddl': True,
},
},
'other': {
'ENGINE': 'django_spanner',
'PROJECT': "$PROJECT",
'INSTANCE': "$INSTANCE",
'NAME': "$TEST_DBNAME_OTHER",
'OPTIONS': {
'batch_ddl': True,
},
},
}
USE_TZ = False
Expand Down
24 changes: 24 additions & 0 deletions packages/django-google-spanner/docs/README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,30 @@ A transaction is automatically started if you define an
To disable transactions in autocommit mode, set the flag `ALLOW_TRANSACTIONS_IN_AUTO_COMMIT` to False in your ``settings.py`` file.


DDL batching
~~~~~~~~~~~~

By default, ``DatabaseSchemaEditor`` executes each DDL statement individually.
To batch DDL statements generated within a ``schema_editor()`` context (such as
during ``manage.py migrate`` or test database creation) into batched
``UpdateDatabaseDdl`` requests, set ``'batch_ddl': True`` in ``OPTIONS``
(defaults to ``False``):

.. code:: python

DATABASES = {
'default': {
'ENGINE': 'django_spanner',
'PROJECT': '$PROJECT',
'INSTANCE': '$INSTANCE',
'NAME': '$DATABASE',
'OPTIONS': {
'batch_ddl': True,
},
}
}


Set credentials and project environment variables
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
You'll need to download a service account JSON key file and point to it using an environment variable:
Expand Down
9 changes: 5 additions & 4 deletions packages/django-google-spanner/docs/limitations-spanner.rst
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,10 @@ it inherently. The workaround is to use python-spanner objects and run Mutations

Batch DDL
~~~~~~~~~
Migrations (creation and updation) of tables in Django are sequential operations.
Django doesn't inherently support doing batch DDL operations. However, customers
can use python-spanner objects to run batch DDL for unmanaged tables in Django.
By default, migrations (creation and update of tables) in Django execute DDL
statements sequentially. You can opt in to batching DDL statements within each
``schema_editor()`` context by setting ``'OPTIONS': {'batch_ddl': True}`` in
``DATABASES``, or use python-spanner objects directly for unmanaged tables.

Stale Reads
~~~~~~~~~~~
Expand Down Expand Up @@ -91,7 +92,7 @@ Workaround
* - Mutations
- Python-spanner database objects can be used to run Mutation queries. `Link <https://cloud.google.com/spanner/docs/modify-mutation-api#python>`__.
* - Batch DDL
- Using unmanaged tables in Django. `Link <https://docs.djangoproject.com/en/4.0/ref/models/options/#managed>`__ and Directly using Python spanner objects to execute batch DDL statements. `Link <https://cloud.google.com/spanner/docs/getting-started/python#create_a_database>`__.
- Set ``'OPTIONS': {'batch_ddl': True}`` in ``DATABASES`` to batch DDL statements within ``schema_editor()``, or use unmanaged tables (`Link <https://docs.djangoproject.com/en/5.2/ref/models/options/#managed>`__) with python-spanner objects directly (`Link <https://cloud.google.com/spanner/docs/getting-started/python#create_a_database>`__).
* - Stale Reads
- Python-spanner database objects can be used to perform stale reads. `Link <https://cloud.google.com/spanner/docs/reads#python>`__.
* - Interleaving
Expand Down
3 changes: 3 additions & 0 deletions packages/django-google-spanner/foreign_key_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ DATABASES = {
'PROJECT': '$GOOGLE_CLOUD_PROJECT',
'INSTANCE': '$SPANNER_TEST_INSTANCE',
'NAME': 'test_foreign_keys',
'OPTIONS': {
'batch_ddl': True,
},
}
}
" >> foreign_keys/settings.py
Expand Down
2 changes: 1 addition & 1 deletion packages/django-google-spanner/noxfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@
]

UNIT_TEST_MOCKSERVER_DEPENDENCIES = [
"google-cloud-spanner>=3.70.0",
"google-cloud-spanner>=3.72.0",
"sqlparse>=0.4.4",
]

Expand Down
2 changes: 1 addition & 1 deletion packages/django-google-spanner/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
# (django_spanner/__init__.py currently enforces SUPPORTED_DJANGO_VERSIONS = [(6, 0), (5, 2)])
dependencies = [
"sqlparse >= 0.3.0",
"google-cloud-spanner >= 3.70.0",
"google-cloud-spanner >= 3.72.0",
"django >= 5.2, < 6.1",
]
extras = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,14 @@
class MockSpanner:
def __init__(self):
self.results = {}
self.errors = {}

def add_result(self, sql: str, result: result_set.ResultSet):
self.results[sql.lower().strip()] = result

def add_error(self, sql: str, code: grpc.StatusCode, details: str):
self.errors[sql.lower().strip()] = (code, details)

def get_result(self, sql: str) -> result_set.ResultSet:
result = self.results.get(sql.lower().strip())
if result is None:
Expand Down Expand Up @@ -135,6 +139,9 @@ def ExecuteStreamingSql(self, request: ExecuteSqlRequest, context):
started_transaction = self.__create_transaction(
request.session, request.transaction.begin
)
error = self.mock_spanner.errors.get(request.sql.lower().strip())
if error is not None:
context.abort(error[0], error[1])
partials = self.mock_spanner.get_result_as_partial_result_sets(
request.sql, started_transaction
)
Expand Down
Loading
Loading