From b704e8979434edd605aa930eff30e133cf186cab Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Tue, 12 Apr 2022 09:17:19 +0800 Subject: [PATCH 01/15] jdbc backends: add support for named parameters --- dj_db_conn_pool/backends/jdbc/__init__.py | 35 +++++++++++++++++------ 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/dj_db_conn_pool/backends/jdbc/__init__.py b/dj_db_conn_pool/backends/jdbc/__init__.py index f6538f1..7d76469 100644 --- a/dj_db_conn_pool/backends/jdbc/__init__.py +++ b/dj_db_conn_pool/backends/jdbc/__init__.py @@ -1,12 +1,15 @@ # -*- coding: utf-8 -*- import types +import sqlparams import jaydebeapi from dj_db_conn_pool.core.mixins import PooledDatabaseWrapperMixin import logging logger = logging.getLogger(__name__) +sql_params = sqlparams.SQLParams('named', 'qmark') + class JDBCDatabaseWrapper(PooledDatabaseWrapperMixin): @property @@ -37,7 +40,7 @@ def _get_new_connection(self, conn_params): def create_cursor(self, name=None): """ create a cursor - just for compatibility + do some tricks here :param name: :return: """ @@ -45,19 +48,35 @@ def create_cursor(self, name=None): cursor = super().create_cursor(name) # just for compatibility - cursor.setinputsizes = types.MethodType(lambda *_: None, cursor) - - def _execute(_self, query, *_args): - # replace placeholder - query = query.replace('%s', '?') + # cursor.setinputsizes = types.MethodType(lambda *_: None, cursor) + + def _execute(_self, query, parameters=None): + """ + :param _self: django's cursor + :param query: SQL + :param parameters: SQL parameters + :return: + """ + if isinstance(parameters, dict): + # convert sql and parameters + _query, _parameters = sql_params.format(query, parameters) + + logger.debug( + 'SQL (%s), parameters(%s) has been converted to SQL(%s), parameters(%s)', + query, parameters, _query, _parameters) + + query, parameters = (_query, _parameters) + else: + # change paramstyle 'format' to 'qmark' + query = query.replace('%s', '?') # record last query cursor.statement = query # call jaydebeapi - _self.cursor.execute(query, *_args) + _self.cursor.execute(query, parameters) - # just for compatibility + # replace django cursor's execute method cursor.execute = types.MethodType(_execute, cursor) return cursor From dfdaceb91e85e90e5301543ea9ff7af7d73a56c1 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Wed, 25 May 2022 11:16:25 +0800 Subject: [PATCH 02/15] change pool's default parameters: --- dj_db_conn_pool/core/__init__.py | 72 ++++++++++++++++---------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/dj_db_conn_pool/core/__init__.py b/dj_db_conn_pool/core/__init__.py index 29617a4..a7b5817 100644 --- a/dj_db_conn_pool/core/__init__.py +++ b/dj_db_conn_pool/core/__init__.py @@ -1,36 +1,36 @@ -# -*- coding: utf-8 -*- - -import threading -from dj_db_conn_pool.compat import gettext_lazy as _ -from dj_db_conn_pool.core.exceptions import PoolDoesNotExist - - -class PoolContainer(dict): - # acquire this lock before modify pool_container - lock = threading.Lock() - - # the default parameters of pool - pool_default_params = { - 'pre_ping': True, - 'echo': True, - 'timeout': None, - 'recycle': 60 * 60, - 'pool_size': 10, - 'max_overflow': 10, - } - - def has(self, pool_name): - return pool_name in self - - def put(self, pool_name, pool): - self[pool_name] = pool - - def get(self, pool_name): - try: - return self[pool_name] - except KeyError: - raise PoolDoesNotExist(_('No such pool: {pool_name}').format(pool_name=pool_name)) - - -# the pool's container, for maintaining the pools -pool_container = PoolContainer() +# -*- coding: utf-8 -*- + +import threading +from dj_db_conn_pool.compat import gettext_lazy as _ +from dj_db_conn_pool.core.exceptions import PoolDoesNotExist + + +class PoolContainer(dict): + # acquire this lock before modify pool_container + lock = threading.Lock() + + # the default parameters of pool + pool_default_params = { + 'pre_ping': True, + 'echo': True, + 'timeout': 30, + 'recycle': 60 * 15, + 'pool_size': 10, + 'max_overflow': 10, + } + + def has(self, pool_name): + return pool_name in self + + def put(self, pool_name, pool): + self[pool_name] = pool + + def get(self, pool_name): + try: + return self[pool_name] + except KeyError: + raise PoolDoesNotExist(_('No such pool: {pool_name}').format(pool_name=pool_name)) + + +# the pool's container, for maintaining the pools +pool_container = PoolContainer() From 1910eb11d3c2a6a956de43b199535858b2f2f4e7 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Wed, 25 May 2022 11:19:44 +0800 Subject: [PATCH 03/15] remove `PooledDatabaseWrapperMixin._close` method --- dj_db_conn_pool/core/mixins.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/dj_db_conn_pool/core/mixins.py b/dj_db_conn_pool/core/mixins.py index c71b4ca..30710c8 100644 --- a/dj_db_conn_pool/core/mixins.py +++ b/dj_db_conn_pool/core/mixins.py @@ -77,11 +77,6 @@ def get_new_connection(self, conn_params): logger.debug(_("got %s's connection from its pool"), self.alias) return conn - def _close(self): - pass - def close(self, *args, **kwargs): - self._close() - logger.debug(_("release %s's connection to its pool"), self.alias) return super(PooledDatabaseWrapperMixin, self).close(*args, **kwargs) From 23f5f4e5b094589fb42190c4d5186a5fd5bb09be Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Wed, 25 May 2022 14:44:53 +0800 Subject: [PATCH 04/15] call `super()._close()` in jdbc --- dj_db_conn_pool/backends/jdbc/__init__.py | 200 +++++++++++----------- 1 file changed, 101 insertions(+), 99 deletions(-) diff --git a/dj_db_conn_pool/backends/jdbc/__init__.py b/dj_db_conn_pool/backends/jdbc/__init__.py index 7d76469..f19a29e 100644 --- a/dj_db_conn_pool/backends/jdbc/__init__.py +++ b/dj_db_conn_pool/backends/jdbc/__init__.py @@ -1,99 +1,101 @@ -# -*- coding: utf-8 -*- - -import types -import sqlparams -import jaydebeapi -from dj_db_conn_pool.core.mixins import PooledDatabaseWrapperMixin - -import logging -logger = logging.getLogger(__name__) - -sql_params = sqlparams.SQLParams('named', 'qmark') - - -class JDBCDatabaseWrapper(PooledDatabaseWrapperMixin): - @property - def jdbc_driver(self): - raise NotImplementedError() - - @property - def jdbc_url(self): - raise NotImplementedError() - - @property - def jdbc_options(self): - return self.settings_dict.get('JDBC_OPTIONS', {}) - - def _get_new_connection(self, conn_params): - conn = jaydebeapi.connect( - self.jdbc_driver, - self.jdbc_url, - { - 'user': self.settings_dict['USER'], - 'password': self.settings_dict['PASSWORD'], - **self.jdbc_options - } - ) - - return conn - - def create_cursor(self, name=None): - """ - create a cursor - do some tricks here - :param name: - :return: - """ - # get cursor from django - cursor = super().create_cursor(name) - - # just for compatibility - # cursor.setinputsizes = types.MethodType(lambda *_: None, cursor) - - def _execute(_self, query, parameters=None): - """ - :param _self: django's cursor - :param query: SQL - :param parameters: SQL parameters - :return: - """ - if isinstance(parameters, dict): - # convert sql and parameters - _query, _parameters = sql_params.format(query, parameters) - - logger.debug( - 'SQL (%s), parameters(%s) has been converted to SQL(%s), parameters(%s)', - query, parameters, _query, _parameters) - - query, parameters = (_query, _parameters) - else: - # change paramstyle 'format' to 'qmark' - query = query.replace('%s', '?') - - # record last query - cursor.statement = query - - # call jaydebeapi - _self.cursor.execute(query, parameters) - - # replace django cursor's execute method - cursor.execute = types.MethodType(_execute, cursor) - - return cursor - - def __str__(self): - return 'JDBC connection to {NAME}'.format(**self.settings_dict) - - __repr__ = __str__ - - def _close(self): - if self.connection is not None and self.connection.connection.jconn.getAutoCommit(): - # if jdbc connection's autoCommit is on - # jaydebeapi will throw an exception after rollback called - # we make a little dynamic patch here, make sure - # SQLAlchemy will not do rollback before recycling connection - self.connection._pool._reset_on_return = None - - logger.warning( - "current JDBC connection(to %s)'s autoCommit is on, won't do rollback before returning", - self.alias) +# -*- coding: utf-8 -*- + +import types +import sqlparams +import jaydebeapi +from dj_db_conn_pool.core.mixins import PooledDatabaseWrapperMixin + +import logging +logger = logging.getLogger(__name__) + +sql_params = sqlparams.SQLParams('named', 'qmark') + + +class JDBCDatabaseWrapper(PooledDatabaseWrapperMixin): + @property + def jdbc_driver(self): + raise NotImplementedError() + + @property + def jdbc_url(self): + raise NotImplementedError() + + @property + def jdbc_options(self): + return self.settings_dict.get('JDBC_OPTIONS', {}) + + def _get_new_connection(self, conn_params): + conn = jaydebeapi.connect( + self.jdbc_driver, + self.jdbc_url, + { + 'user': self.settings_dict['USER'], + 'password': self.settings_dict['PASSWORD'], + **self.jdbc_options + } + ) + + return conn + + def create_cursor(self, name=None): + """ + create a cursor + do some tricks here + :param name: + :return: + """ + # get cursor from django + cursor = super().create_cursor(name) + + # just for compatibility + # cursor.setinputsizes = types.MethodType(lambda *_: None, cursor) + + def _execute(_self, query, parameters=None): + """ + :param _self: django's cursor + :param query: SQL + :param parameters: SQL parameters + :return: + """ + if isinstance(parameters, dict): + # convert sql and parameters + _query, _parameters = sql_params.format(query, parameters) + + logger.debug( + 'SQL (%s), parameters(%s) has been converted to SQL(%s), parameters(%s)', + query, parameters, _query, _parameters) + + query, parameters = (_query, _parameters) + else: + # change paramstyle 'format' to 'qmark' + query = query.replace('%s', '?') + + # record last query + cursor.statement = query + + # call jaydebeapi + _self.cursor.execute(query, parameters) + + # replace django cursor's execute method + cursor.execute = types.MethodType(_execute, cursor) + + return cursor + + def __str__(self): + return 'JDBC connection to {NAME}'.format(**self.settings_dict) + + __repr__ = __str__ + + def _close(self): + if self.connection is not None and self.connection.connection.jconn.getAutoCommit(): + # if jdbc connection's autoCommit is on + # jaydebeapi will throw an exception after rollback called + # we make a little dynamic patch here, make sure + # SQLAlchemy will not do rollback before recycling connection + self.connection._pool._reset_on_return = None + + logger.warning( + "current JDBC connection(to %s)'s autoCommit is on, won't do rollback before returning", + self.alias) + + return super(JDBCDatabaseWrapper, self)._close() From 219b56063b9794ef3028a6226ac87998665b101e Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Wed, 25 May 2022 14:52:01 +0800 Subject: [PATCH 05/15] update --- dj_db_conn_pool/backends/jdbc/oracle/base.py | 13 ++++--------- dj_db_conn_pool/core/mixins.py | 8 ++++---- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/dj_db_conn_pool/backends/jdbc/oracle/base.py b/dj_db_conn_pool/backends/jdbc/oracle/base.py index b602432..688f6d2 100644 --- a/dj_db_conn_pool/backends/jdbc/oracle/base.py +++ b/dj_db_conn_pool/backends/jdbc/oracle/base.py @@ -6,7 +6,6 @@ import jpype import jaydebeapi from django.db.backends.oracle import base -from copy import deepcopy from sqlalchemy.dialects.oracle.base import OracleDialect from dj_db_conn_pool.backends.jdbc import JDBCDatabaseWrapper @@ -38,11 +37,7 @@ def jdbc_url(self): @property def jdbc_options(self): - # make a copy of default option, avoid side effects - jdbc_options = deepcopy(oracle_session_info) - # get super's jdbc_options - options = super(DatabaseWrapper, self).jdbc_options - # override default options - jdbc_options.update(**options) - - return jdbc_options + return { + **oracle_session_info, + **super(DatabaseWrapper, self).jdbc_options + } diff --git a/dj_db_conn_pool/core/mixins.py b/dj_db_conn_pool/core/mixins.py index 30710c8..b3b5265 100644 --- a/dj_db_conn_pool/core/mixins.py +++ b/dj_db_conn_pool/core/mixins.py @@ -31,9 +31,6 @@ def get_new_connection(self, conn_params): if not pool_container.has(self.alias): # self.alias's pool doesn't exist, time to create it - # make a copy of default parameters - pool_params = deepcopy(pool_container.pool_default_params) - # parse parameters of current database from self.settings_dict pool_setting = { # transform the keys in POOL_OPTIONS to upper case @@ -52,7 +49,10 @@ def get_new_connection(self, conn_params): # replace pool_params's items with pool_setting's items # to import custom parameters - pool_params.update(**pool_setting) + pool_params = { + **pool_container.pool_default_params, + **pool_setting + } # now we have all parameters of self.alias # create self.alias's pool From 5ef8afb1995f9fa0a1c1f2b8b08d858bc255a9f1 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Wed, 25 May 2022 15:34:08 +0800 Subject: [PATCH 06/15] update logging --- dj_db_conn_pool/backends/jdbc/__init__.py | 6 +++--- dj_db_conn_pool/core/__init__.py | 2 +- dj_db_conn_pool/core/mixins.py | 11 +++++++++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/dj_db_conn_pool/backends/jdbc/__init__.py b/dj_db_conn_pool/backends/jdbc/__init__.py index f19a29e..df3bc9b 100644 --- a/dj_db_conn_pool/backends/jdbc/__init__.py +++ b/dj_db_conn_pool/backends/jdbc/__init__.py @@ -94,8 +94,8 @@ def _close(self): # SQLAlchemy will not do rollback before recycling connection self.connection._pool._reset_on_return = None - logger.warning( - "current JDBC connection(to %s)'s autoCommit is on, won't do rollback before returning", - self.alias) + logger.debug( + "autoCommit of current JDBC connection to %s %s is on, won't do rollback before returning", + self.alias, self.connection.connection) return super(JDBCDatabaseWrapper, self)._close() diff --git a/dj_db_conn_pool/core/__init__.py b/dj_db_conn_pool/core/__init__.py index a7b5817..1d40e47 100644 --- a/dj_db_conn_pool/core/__init__.py +++ b/dj_db_conn_pool/core/__init__.py @@ -12,7 +12,7 @@ class PoolContainer(dict): # the default parameters of pool pool_default_params = { 'pre_ping': True, - 'echo': True, + 'echo': False, 'timeout': 30, 'recycle': 60 * 15, 'pool_size': 10, diff --git a/dj_db_conn_pool/core/mixins.py b/dj_db_conn_pool/core/mixins.py index b3b5265..21986bd 100644 --- a/dj_db_conn_pool/core/mixins.py +++ b/dj_db_conn_pool/core/mixins.py @@ -74,9 +74,16 @@ def get_new_connection(self, conn_params): db_pool = pool_container.get(self.alias) # get one connection from the pool conn = db_pool.connect() - logger.debug(_("got %s's connection from its pool"), self.alias) + + logger.debug( + _("got %s's connection %s from its pool"), + self.alias, conn.connection) + return conn def close(self, *args, **kwargs): - logger.debug(_("release %s's connection to its pool"), self.alias) + logger.debug( + _("release %s's connection %s to its pool"), + self.alias, getattr(self.connection, 'connection', None)) + return super(PooledDatabaseWrapperMixin, self).close(*args, **kwargs) From efde2027af398c97237a8864e5df6600dd6f9681 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Thu, 26 May 2022 09:28:04 +0800 Subject: [PATCH 07/15] updates --- dj_db_conn_pool/__init__.py | 2 +- dj_db_conn_pool/backends/jdbc/__init__.py | 8 -------- dj_db_conn_pool/backends/jdbc/oceanbase/base.py | 2 ++ dj_db_conn_pool/backends/mysql/base.py | 5 ----- dj_db_conn_pool/backends/oracle/base.py | 5 ----- dj_db_conn_pool/backends/postgresql/base.py | 5 ----- dj_db_conn_pool/compat/jdbc.py | 6 +----- dj_db_conn_pool/core/__init__.py | 1 + dj_db_conn_pool/core/mixins.py | 17 +++++++++++++---- 9 files changed, 18 insertions(+), 33 deletions(-) diff --git a/dj_db_conn_pool/__init__.py b/dj_db_conn_pool/__init__.py index 3da93aa..63a94ae 100644 --- a/dj_db_conn_pool/__init__.py +++ b/dj_db_conn_pool/__init__.py @@ -1,4 +1,4 @@ -__version__ = '1.1.0' +__version__ = '1.1.1' __author__ = 'Altair Bow' __author_email__ = 'altair.bow@foxmail.com' __description__ = 'Persistent database connection backends for Django' diff --git a/dj_db_conn_pool/backends/jdbc/__init__.py b/dj_db_conn_pool/backends/jdbc/__init__.py index df3bc9b..db8d455 100644 --- a/dj_db_conn_pool/backends/jdbc/__init__.py +++ b/dj_db_conn_pool/backends/jdbc/__init__.py @@ -47,9 +47,6 @@ def create_cursor(self, name=None): # get cursor from django cursor = super().create_cursor(name) - # just for compatibility - # cursor.setinputsizes = types.MethodType(lambda *_: None, cursor) - def _execute(_self, query, parameters=None): """ :param _self: django's cursor @@ -81,11 +78,6 @@ def _execute(_self, query, parameters=None): return cursor - def __str__(self): - return 'JDBC connection to {NAME}'.format(**self.settings_dict) - - __repr__ = __str__ - def _close(self): if self.connection is not None and self.connection.connection.jconn.getAutoCommit(): # if jdbc connection's autoCommit is on diff --git a/dj_db_conn_pool/backends/jdbc/oceanbase/base.py b/dj_db_conn_pool/backends/jdbc/oceanbase/base.py index 023d18f..1588c0b 100644 --- a/dj_db_conn_pool/backends/jdbc/oceanbase/base.py +++ b/dj_db_conn_pool/backends/jdbc/oceanbase/base.py @@ -12,6 +12,8 @@ class DatabaseWrapper(JDBCDatabaseWrapper, base.DatabaseWrapper): + vendor = 'OceanBase' + class SQLAlchemyDialect(OracleDialect): def do_ping(self, dbapi_connection): try: diff --git a/dj_db_conn_pool/backends/mysql/base.py b/dj_db_conn_pool/backends/mysql/base.py index 920d26f..e11b437 100644 --- a/dj_db_conn_pool/backends/mysql/base.py +++ b/dj_db_conn_pool/backends/mysql/base.py @@ -8,8 +8,3 @@ class DatabaseWrapper(PooledDatabaseWrapperMixin, base.DatabaseWrapper): class SQLAlchemyDialect(MySQLDialect_pymysql): server_version_info = (0, ) - - def __str__(self): - return 'MySQL connection to {HOST}/{NAME}'.format(**self.settings_dict) - - __repr__ = __str__ diff --git a/dj_db_conn_pool/backends/oracle/base.py b/dj_db_conn_pool/backends/oracle/base.py index cf223e2..f243917 100644 --- a/dj_db_conn_pool/backends/oracle/base.py +++ b/dj_db_conn_pool/backends/oracle/base.py @@ -13,8 +13,3 @@ def do_ping(self, dbapi_connection): return super(OracleDialect, self).do_ping(dbapi_connection) except DatabaseError: return False - - def __str__(self): - return 'Oracle connection to {NAME}'.format(**self.settings_dict) - - __repr__ = __str__ diff --git a/dj_db_conn_pool/backends/postgresql/base.py b/dj_db_conn_pool/backends/postgresql/base.py index 0335de9..7c82b53 100644 --- a/dj_db_conn_pool/backends/postgresql/base.py +++ b/dj_db_conn_pool/backends/postgresql/base.py @@ -12,8 +12,3 @@ class SQLAlchemyDialect(PGDialect_psycopg2): def _set_autocommit(self, autocommit): with self.wrap_database_errors: self.connection.dbapi_connection.autocommit = autocommit - - def __str__(self): - return 'PostgreSQL connection to {NAME}'.format(**self.settings_dict) - - __repr__ = __str__ diff --git a/dj_db_conn_pool/compat/jdbc.py b/dj_db_conn_pool/compat/jdbc.py index b711cb8..df2d14d 100644 --- a/dj_db_conn_pool/compat/jdbc.py +++ b/dj_db_conn_pool/compat/jdbc.py @@ -4,11 +4,7 @@ def patch_all(): def patch_converters(): """ - ['BIT', 'TINYINT', 'SMALLINT', 'INTEGER', 'BIGINT', 'FLOAT', 'REAL', 'DOUBLE', - 'NUMERIC', 'DECIMAL', 'CHAR', 'VARCHAR', 'LONGVARCHAR', 'DATE', 'TIME', 'TIMESTAMP', - 'BINARY', 'VARBINARY', 'LONGVARBINARY', 'NULL', 'OTHER', 'JAVA_OBJECT', 'DISTINCT', - 'STRUCT', 'ARRAY', 'BLOB', 'CLOB', 'REF', 'DATALINK', 'BOOLEAN', 'ROWID', 'NCHAR', - 'NVARCHAR', 'LONGNVARCHAR', 'NCLOB', 'SQLXML', 'REF_CURSOR', 'TIME_WITH_TIMEZONE', 'TIMESTAMP_WITH_TIMEZONE'] + patch jaydebeapi's converters """ from datetime import datetime import jaydebeapi diff --git a/dj_db_conn_pool/core/__init__.py b/dj_db_conn_pool/core/__init__.py index 1d40e47..dfc1cbe 100644 --- a/dj_db_conn_pool/core/__init__.py +++ b/dj_db_conn_pool/core/__init__.py @@ -33,4 +33,5 @@ def get(self, pool_name): # the pool's container, for maintaining the pools +# every process has it's own pool container pool_container = PoolContainer() diff --git a/dj_db_conn_pool/core/mixins.py b/dj_db_conn_pool/core/mixins.py index 21986bd..5a0061c 100644 --- a/dj_db_conn_pool/core/mixins.py +++ b/dj_db_conn_pool/core/mixins.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from copy import deepcopy from sqlalchemy import pool from dj_db_conn_pool.compat import gettext_lazy as _ from dj_db_conn_pool.core import pool_container @@ -11,6 +10,16 @@ class PooledDatabaseWrapperMixin(object): + def __str__(self): + try: + conn = repr(self.connection.connection) + except AttributeError: + conn = '' + + return f'{self.vendor} connection to {self.alias}: {conn}' + + __repr__ = __str__ + def _get_new_connection(self, conn_params): return super(PooledDatabaseWrapperMixin, self).get_new_connection(conn_params) @@ -33,7 +42,7 @@ def get_new_connection(self, conn_params): # parse parameters of current database from self.settings_dict pool_setting = { - # transform the keys in POOL_OPTIONS to upper case + # transform the keys in POOL_OPTIONS to lower case # to fit sqlalchemy.pool.QueuePool's arguments requirement key.lower(): value # traverse POOL_OPTIONS to get arguments @@ -43,12 +52,12 @@ def get_new_connection(self, conn_params): self.settings_dict.get('POOL_OPTIONS', {}).items() # There are some limits of self.alias's pool's option(POOL_OPTIONS): # the keys in POOL_OPTIONS must be capitalised - # and the keys's lowercase must be in pool_container.pool_default_params + # and the key's lowercase must be in pool_container.pool_default_params if key == key.upper() and key.lower() in pool_container.pool_default_params } # replace pool_params's items with pool_setting's items - # to import custom parameters + # to import custom settings pool_params = { **pool_container.pool_default_params, **pool_setting From 84be30cfca1e68e9835cba948145c1e6a0e162f0 Mon Sep 17 00:00:00 2001 From: kekessle Date: Thu, 30 Jun 2022 10:08:10 +0200 Subject: [PATCH 08/15] Add _set_autocommit to oracle/base.py as the autocommit was not set properly when using the oracle backend --- dj_db_conn_pool/backends/oracle/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dj_db_conn_pool/backends/oracle/base.py b/dj_db_conn_pool/backends/oracle/base.py index f243917..c5c7796 100644 --- a/dj_db_conn_pool/backends/oracle/base.py +++ b/dj_db_conn_pool/backends/oracle/base.py @@ -13,3 +13,7 @@ def do_ping(self, dbapi_connection): return super(OracleDialect, self).do_ping(dbapi_connection) except DatabaseError: return False + + def _set_autocommit(self, autocommit): + with self.wrap_database_errors: + self.connection.dbapi_connection.autocommit = autocommit \ No newline at end of file From 5fd410d73a58526268bd0ad382366e59bbcdb52a Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Thu, 30 Jun 2022 17:18:56 +0800 Subject: [PATCH 09/15] fix 'cx_Oracle.Connection' object has no attribute 'dbapi_connection' --- dj_db_conn_pool/backends/oracle/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dj_db_conn_pool/backends/oracle/base.py b/dj_db_conn_pool/backends/oracle/base.py index c5c7796..5dfd7b3 100644 --- a/dj_db_conn_pool/backends/oracle/base.py +++ b/dj_db_conn_pool/backends/oracle/base.py @@ -2,7 +2,7 @@ from django.db.backends.oracle import base from cx_Oracle import DatabaseError -from sqlalchemy.dialects.oracle.cx_oracle import OracleDialect +from sqlalchemy.dialects.oracle.base import OracleDialect from dj_db_conn_pool.core.mixins import PooledDatabaseWrapperMixin @@ -16,4 +16,4 @@ def do_ping(self, dbapi_connection): def _set_autocommit(self, autocommit): with self.wrap_database_errors: - self.connection.dbapi_connection.autocommit = autocommit \ No newline at end of file + self.connection.connection.autocommit = autocommit From 04be6044976fc20e273139164d8c65961bf305f6 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Thu, 30 Jun 2022 18:01:52 +0800 Subject: [PATCH 10/15] mv _set_autocommit(oracle&pg) to PooledDatabaseWrapperMixin --- dj_db_conn_pool/backends/oracle/base.py | 4 ---- dj_db_conn_pool/backends/postgresql/base.py | 4 ---- dj_db_conn_pool/core/mixins.py | 7 +++++++ 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/dj_db_conn_pool/backends/oracle/base.py b/dj_db_conn_pool/backends/oracle/base.py index 5dfd7b3..cc8a721 100644 --- a/dj_db_conn_pool/backends/oracle/base.py +++ b/dj_db_conn_pool/backends/oracle/base.py @@ -13,7 +13,3 @@ def do_ping(self, dbapi_connection): return super(OracleDialect, self).do_ping(dbapi_connection) except DatabaseError: return False - - def _set_autocommit(self, autocommit): - with self.wrap_database_errors: - self.connection.connection.autocommit = autocommit diff --git a/dj_db_conn_pool/backends/postgresql/base.py b/dj_db_conn_pool/backends/postgresql/base.py index 7c82b53..7f5a49a 100644 --- a/dj_db_conn_pool/backends/postgresql/base.py +++ b/dj_db_conn_pool/backends/postgresql/base.py @@ -8,7 +8,3 @@ class DatabaseWrapper(PooledDatabaseWrapperMixin, base.DatabaseWrapper): class SQLAlchemyDialect(PGDialect_psycopg2): pass - - def _set_autocommit(self, autocommit): - with self.wrap_database_errors: - self.connection.dbapi_connection.autocommit = autocommit diff --git a/dj_db_conn_pool/core/mixins.py b/dj_db_conn_pool/core/mixins.py index 5a0061c..8cd2c04 100644 --- a/dj_db_conn_pool/core/mixins.py +++ b/dj_db_conn_pool/core/mixins.py @@ -20,6 +20,13 @@ def __str__(self): __repr__ = __str__ + def _set_autocommit(self, autocommit): + with self.wrap_database_errors: + try: + self.connection.connection.autocommit = autocommit + except (Exception, ) as e: + logger.exception('unable to set database(%s) autocommit: %s', self.alias, e) + def _get_new_connection(self, conn_params): return super(PooledDatabaseWrapperMixin, self).get_new_connection(conn_params) From a2097d80dc344e35d07ffe14a81364add3b1ee11 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Thu, 30 Jun 2022 18:03:48 +0800 Subject: [PATCH 11/15] fix mysql autocommit --- dj_db_conn_pool/backends/mysql/base.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dj_db_conn_pool/backends/mysql/base.py b/dj_db_conn_pool/backends/mysql/base.py index e11b437..9600e0e 100644 --- a/dj_db_conn_pool/backends/mysql/base.py +++ b/dj_db_conn_pool/backends/mysql/base.py @@ -4,7 +4,17 @@ from sqlalchemy.dialects.mysql.pymysql import MySQLDialect_pymysql from dj_db_conn_pool.core.mixins import PooledDatabaseWrapperMixin +import logging +logger = logging.getLogger(__name__) + class DatabaseWrapper(PooledDatabaseWrapperMixin, base.DatabaseWrapper): class SQLAlchemyDialect(MySQLDialect_pymysql): server_version_info = (0, ) + + def _set_autocommit(self, autocommit): + with self.wrap_database_errors: + try: + self.connection.connection.autocommit(autocommit) + except (Exception, ) as e: + logger.exception('unable to set database(%s) autocommit: %s', self.alias, e) From 1c6d1b4caeb5e0f517bec4ab94c38c71e036ab5d Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Thu, 30 Jun 2022 18:10:37 +0800 Subject: [PATCH 12/15] add setAutoCommit support for jdbc backends --- dj_db_conn_pool/backends/jdbc/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dj_db_conn_pool/backends/jdbc/__init__.py b/dj_db_conn_pool/backends/jdbc/__init__.py index db8d455..fc52d80 100644 --- a/dj_db_conn_pool/backends/jdbc/__init__.py +++ b/dj_db_conn_pool/backends/jdbc/__init__.py @@ -12,6 +12,13 @@ class JDBCDatabaseWrapper(PooledDatabaseWrapperMixin): + def _set_autocommit(self, autocommit): + with self.wrap_database_errors: + try: + self.connection.connection.jconn.setAutoCommit(autocommit) + except (Exception, ) as e: + logger.exception('unable to set database(%s) autocommit: %s', self.alias, e) + @property def jdbc_driver(self): raise NotImplementedError() From fa313250361ce9d1dfcf993b2b226f1ff513b6a8 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Thu, 30 Jun 2022 18:21:01 +0800 Subject: [PATCH 13/15] increase version to 1.2.0 --- dj_db_conn_pool/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dj_db_conn_pool/__init__.py b/dj_db_conn_pool/__init__.py index 63a94ae..a441b0d 100644 --- a/dj_db_conn_pool/__init__.py +++ b/dj_db_conn_pool/__init__.py @@ -1,4 +1,4 @@ -__version__ = '1.1.1' +__version__ = '1.2.0' __author__ = 'Altair Bow' __author_email__ = 'altair.bow@foxmail.com' __description__ = 'Persistent database connection backends for Django' From f47279c1944061976336983d304daef634577cde Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Thu, 30 Jun 2022 22:25:12 +0800 Subject: [PATCH 14/15] re-raise if set_autocommit fails --- dj_db_conn_pool/backends/jdbc/__init__.py | 8 ++------ dj_db_conn_pool/backends/mysql/base.py | 8 ++------ dj_db_conn_pool/core/mixins.py | 11 ++++++++--- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/dj_db_conn_pool/backends/jdbc/__init__.py b/dj_db_conn_pool/backends/jdbc/__init__.py index fc52d80..1d56b7c 100644 --- a/dj_db_conn_pool/backends/jdbc/__init__.py +++ b/dj_db_conn_pool/backends/jdbc/__init__.py @@ -12,12 +12,8 @@ class JDBCDatabaseWrapper(PooledDatabaseWrapperMixin): - def _set_autocommit(self, autocommit): - with self.wrap_database_errors: - try: - self.connection.connection.jconn.setAutoCommit(autocommit) - except (Exception, ) as e: - logger.exception('unable to set database(%s) autocommit: %s', self.alias, e) + def _set_dbapi_autocommit(self, autocommit): + self.connection.connection.jconn.setAutoCommit(autocommit) @property def jdbc_driver(self): diff --git a/dj_db_conn_pool/backends/mysql/base.py b/dj_db_conn_pool/backends/mysql/base.py index 9600e0e..b9cb25b 100644 --- a/dj_db_conn_pool/backends/mysql/base.py +++ b/dj_db_conn_pool/backends/mysql/base.py @@ -12,9 +12,5 @@ class DatabaseWrapper(PooledDatabaseWrapperMixin, base.DatabaseWrapper): class SQLAlchemyDialect(MySQLDialect_pymysql): server_version_info = (0, ) - def _set_autocommit(self, autocommit): - with self.wrap_database_errors: - try: - self.connection.connection.autocommit(autocommit) - except (Exception, ) as e: - logger.exception('unable to set database(%s) autocommit: %s', self.alias, e) + def _set_dbapi_autocommit(self, autocommit): + self.connection.connection.autocommit(autocommit) diff --git a/dj_db_conn_pool/core/mixins.py b/dj_db_conn_pool/core/mixins.py index 8cd2c04..57359f5 100644 --- a/dj_db_conn_pool/core/mixins.py +++ b/dj_db_conn_pool/core/mixins.py @@ -20,12 +20,17 @@ def __str__(self): __repr__ = __str__ + def _set_dbapi_autocommit(self, autocommit): + self.connection.connection.autocommit = autocommit + def _set_autocommit(self, autocommit): with self.wrap_database_errors: try: - self.connection.connection.autocommit = autocommit - except (Exception, ) as e: - logger.exception('unable to set database(%s) autocommit: %s', self.alias, e) + self._set_dbapi_autocommit(autocommit) + except (Exception, ) as exc: + logger.exception('unable to set autocommit mode of %s(%s) to %s, caused by: %s', + self.vendor, self.alias, autocommit, exc) + raise exc from None def _get_new_connection(self, conn_params): return super(PooledDatabaseWrapperMixin, self).get_new_connection(conn_params) From 18dd52845fafe85427211804389ae637453b1b09 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Thu, 30 Jun 2022 22:31:11 +0800 Subject: [PATCH 15/15] v1.2.1 --- dj_db_conn_pool/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dj_db_conn_pool/__init__.py b/dj_db_conn_pool/__init__.py index a441b0d..96afbf1 100644 --- a/dj_db_conn_pool/__init__.py +++ b/dj_db_conn_pool/__init__.py @@ -1,4 +1,4 @@ -__version__ = '1.2.0' +__version__ = '1.2.1' __author__ = 'Altair Bow' __author_email__ = 'altair.bow@foxmail.com' __description__ = 'Persistent database connection backends for Django'