From 843aeb92a56d58451575d0c6ea77146e68ac66e1 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Thu, 10 Nov 2022 10:24:33 +0800 Subject: [PATCH 01/25] add OceanBase backends --- dj_db_conn_pool/backends/jdbc/oceanbase/mysql/__init__.py | 0 dj_db_conn_pool/backends/jdbc/oceanbase/mysql/base.py | 0 dj_db_conn_pool/backends/jdbc/oceanbase/oracle/__init__.py | 0 dj_db_conn_pool/backends/jdbc/oceanbase/oracle/base.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 dj_db_conn_pool/backends/jdbc/oceanbase/mysql/__init__.py create mode 100644 dj_db_conn_pool/backends/jdbc/oceanbase/mysql/base.py create mode 100644 dj_db_conn_pool/backends/jdbc/oceanbase/oracle/__init__.py create mode 100644 dj_db_conn_pool/backends/jdbc/oceanbase/oracle/base.py diff --git a/dj_db_conn_pool/backends/jdbc/oceanbase/mysql/__init__.py b/dj_db_conn_pool/backends/jdbc/oceanbase/mysql/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dj_db_conn_pool/backends/jdbc/oceanbase/mysql/base.py b/dj_db_conn_pool/backends/jdbc/oceanbase/mysql/base.py new file mode 100644 index 0000000..e69de29 diff --git a/dj_db_conn_pool/backends/jdbc/oceanbase/oracle/__init__.py b/dj_db_conn_pool/backends/jdbc/oceanbase/oracle/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dj_db_conn_pool/backends/jdbc/oceanbase/oracle/base.py b/dj_db_conn_pool/backends/jdbc/oceanbase/oracle/base.py new file mode 100644 index 0000000..e69de29 From 047fdd9024e66445161c885336d6e74917f13d2e Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Thu, 10 Nov 2022 10:42:05 +0800 Subject: [PATCH 02/25] split OceanBase backend to Oracle & MySql --- .../backends/jdbc/oceanbase/base.py | 50 ------------------- .../backends/jdbc/oceanbase/mixins.py | 14 ++++++ .../backends/jdbc/oceanbase/mysql/base.py | 20 ++++++++ .../backends/jdbc/oceanbase/oracle/base.py | 20 ++++++++ 4 files changed, 54 insertions(+), 50 deletions(-) delete mode 100644 dj_db_conn_pool/backends/jdbc/oceanbase/base.py create mode 100644 dj_db_conn_pool/backends/jdbc/oceanbase/mixins.py diff --git a/dj_db_conn_pool/backends/jdbc/oceanbase/base.py b/dj_db_conn_pool/backends/jdbc/oceanbase/base.py deleted file mode 100644 index 78cb114..0000000 --- a/dj_db_conn_pool/backends/jdbc/oceanbase/base.py +++ /dev/null @@ -1,50 +0,0 @@ -# -*- coding: utf-8 -*- - -import jpype -import jaydebeapi -from django.core.exceptions import ImproperlyConfigured -from django.db.backends.oracle import base -from sqlalchemy.dialects.mysql.base import MySQLDialect -from sqlalchemy.dialects.oracle.base import OracleDialect -from dj_db_conn_pool.backends.jdbc import JDBCDatabaseWrapperMixin - - -import logging -logger = logging.getLogger(__name__) - - -class DatabaseWrapper(JDBCDatabaseWrapperMixin, base.DatabaseWrapper): - vendor = 'OceanBase' - - OceanBase_Mode_Oracle = 'Oracle' - - OceanBase_Mode_MySQL = 'MySQL' - - def _get_dialect(self, *args, **kwargs): - ob_mode = self.settings_dict.get('OB_MODE', self.OceanBase_Mode_Oracle) - - if ob_mode == self.OceanBase_Mode_Oracle: - dialect_class = OracleDialect - elif ob_mode == self.OceanBase_Mode_MySQL: - dialect_class = MySQLDialect - else: - raise ImproperlyConfigured(f'Not supported mode of OceanBase: {ob_mode}') - - class Dialect(dialect_class): - def do_ping(self, dbapi_connection): - try: - return super(dialect_class, self).do_ping(dbapi_connection) - except (jaydebeapi.DatabaseError, jpype.JException): - return False - - return Dialect(dbapi=self.Database) - - jdbc_driver = 'com.alipay.oceanbase.jdbc.Driver' - - @property - def jdbc_url(self): - return 'jdbc:oceanbase://{NAME}'.format(**self.settings_dict) - - def init_connection_state(self): - # TODO: custom OceanBase connection initialization - pass diff --git a/dj_db_conn_pool/backends/jdbc/oceanbase/mixins.py b/dj_db_conn_pool/backends/jdbc/oceanbase/mixins.py new file mode 100644 index 0000000..a29588b --- /dev/null +++ b/dj_db_conn_pool/backends/jdbc/oceanbase/mixins.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- + +from django.db.backends.oracle import base +from dj_db_conn_pool.backends.jdbc import JDBCDatabaseWrapperMixin + + +class JDBCOceanBaseDatabaseWrapperMixin(JDBCDatabaseWrapperMixin, base.DatabaseWrapper): + vendor = 'OceanBase' + + jdbc_driver = 'com.alipay.oceanbase.jdbc.Driver' + + @property + def jdbc_url(self): + return 'jdbc:oceanbase://{NAME}'.format(**self.settings_dict) diff --git a/dj_db_conn_pool/backends/jdbc/oceanbase/mysql/base.py b/dj_db_conn_pool/backends/jdbc/oceanbase/mysql/base.py index e69de29..443d6d9 100644 --- a/dj_db_conn_pool/backends/jdbc/oceanbase/mysql/base.py +++ b/dj_db_conn_pool/backends/jdbc/oceanbase/mysql/base.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +import jpype +import jaydebeapi +from django.db.backends.mysql import base +from sqlalchemy.dialects.mysql.base import MySQLDialect +from dj_db_conn_pool.backends.jdbc.oceanbase.mixins import JDBCOceanBaseDatabaseWrapperMixin + + +class DatabaseWrapper(JDBCOceanBaseDatabaseWrapperMixin, base.DatabaseWrapper): + class Dialect(MySQLDialect): + def do_ping(self, dbapi_connection): + try: + return super(MySQLDialect, self).do_ping(dbapi_connection) + except (jaydebeapi.DatabaseError, jpype.JException): + return False + + def init_connection_state(self): + # TODO: custom OceanBase (MySQL mode) connection initialization + pass diff --git a/dj_db_conn_pool/backends/jdbc/oceanbase/oracle/base.py b/dj_db_conn_pool/backends/jdbc/oceanbase/oracle/base.py index e69de29..9b19e5e 100644 --- a/dj_db_conn_pool/backends/jdbc/oceanbase/oracle/base.py +++ b/dj_db_conn_pool/backends/jdbc/oceanbase/oracle/base.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +import jpype +import jaydebeapi +from django.db.backends.oracle import base +from sqlalchemy.dialects.oracle.base import OracleDialect +from dj_db_conn_pool.backends.jdbc.oceanbase.mixins import JDBCOceanBaseDatabaseWrapperMixin + + +class DatabaseWrapper(JDBCOceanBaseDatabaseWrapperMixin, base.DatabaseWrapper): + class Dialect(OracleDialect): + def do_ping(self, dbapi_connection): + try: + return super(OracleDialect, self).do_ping(dbapi_connection) + except (jaydebeapi.DatabaseError, jpype.JException): + return False + + def init_connection_state(self): + # TODO: custom OceanBase (Oracle mode) connection initialization + pass From f94342bee185041be306ff02769cfe4518117df7 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Thu, 10 Nov 2022 14:31:44 +0800 Subject: [PATCH 03/25] fix typo --- dj_db_conn_pool/backends/jdbc/oceanbase/mysql/base.py | 2 +- dj_db_conn_pool/backends/jdbc/oceanbase/oracle/base.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dj_db_conn_pool/backends/jdbc/oceanbase/mysql/base.py b/dj_db_conn_pool/backends/jdbc/oceanbase/mysql/base.py index 443d6d9..0688387 100644 --- a/dj_db_conn_pool/backends/jdbc/oceanbase/mysql/base.py +++ b/dj_db_conn_pool/backends/jdbc/oceanbase/mysql/base.py @@ -8,7 +8,7 @@ class DatabaseWrapper(JDBCOceanBaseDatabaseWrapperMixin, base.DatabaseWrapper): - class Dialect(MySQLDialect): + class SQLAlchemyDialect(MySQLDialect): def do_ping(self, dbapi_connection): try: return super(MySQLDialect, self).do_ping(dbapi_connection) diff --git a/dj_db_conn_pool/backends/jdbc/oceanbase/oracle/base.py b/dj_db_conn_pool/backends/jdbc/oceanbase/oracle/base.py index 9b19e5e..faf8000 100644 --- a/dj_db_conn_pool/backends/jdbc/oceanbase/oracle/base.py +++ b/dj_db_conn_pool/backends/jdbc/oceanbase/oracle/base.py @@ -8,7 +8,7 @@ class DatabaseWrapper(JDBCOceanBaseDatabaseWrapperMixin, base.DatabaseWrapper): - class Dialect(OracleDialect): + class SQLAlchemyDialect(OracleDialect): def do_ping(self, dbapi_connection): try: return super(OracleDialect, self).do_ping(dbapi_connection) From 8dd40636eb397d010ba41532e3ffaa87a59f595d Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Thu, 10 Nov 2022 15:42:35 +0800 Subject: [PATCH 04/25] add OceanBase(odbc, oracle mode) backend --- dj_db_conn_pool/backends/odbc/__init__.py | 0 dj_db_conn_pool/backends/odbc/mixins.py | 13 +++++++++++++ .../backends/odbc/oceanbase/__init__.py | 1 + .../backends/odbc/oceanbase/oracle/__init__.py | 0 .../backends/odbc/oceanbase/oracle/base.py | 18 ++++++++++++++++++ dj_db_conn_pool/backends/odbc/utils.py | 10 ++++++++++ 6 files changed, 42 insertions(+) create mode 100644 dj_db_conn_pool/backends/odbc/__init__.py create mode 100644 dj_db_conn_pool/backends/odbc/mixins.py create mode 100644 dj_db_conn_pool/backends/odbc/oceanbase/__init__.py create mode 100644 dj_db_conn_pool/backends/odbc/oceanbase/oracle/__init__.py create mode 100644 dj_db_conn_pool/backends/odbc/oceanbase/oracle/base.py create mode 100644 dj_db_conn_pool/backends/odbc/utils.py diff --git a/dj_db_conn_pool/backends/odbc/__init__.py b/dj_db_conn_pool/backends/odbc/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dj_db_conn_pool/backends/odbc/mixins.py b/dj_db_conn_pool/backends/odbc/mixins.py new file mode 100644 index 0000000..c067bb4 --- /dev/null +++ b/dj_db_conn_pool/backends/odbc/mixins.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- + +import pyodbc +from dj_db_conn_pool.core.mixins import PooledDatabaseWrapperMixin + + +class ODBCWrapperMixin(PooledDatabaseWrapperMixin): + def _get_new_connection(self, conn_params): + conn = pyodbc.connect( + '', **self.settings_dict + ) + + return conn diff --git a/dj_db_conn_pool/backends/odbc/oceanbase/__init__.py b/dj_db_conn_pool/backends/odbc/oceanbase/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/dj_db_conn_pool/backends/odbc/oceanbase/__init__.py @@ -0,0 +1 @@ + diff --git a/dj_db_conn_pool/backends/odbc/oceanbase/oracle/__init__.py b/dj_db_conn_pool/backends/odbc/oceanbase/oracle/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dj_db_conn_pool/backends/odbc/oceanbase/oracle/base.py b/dj_db_conn_pool/backends/odbc/oceanbase/oracle/base.py new file mode 100644 index 0000000..21d000e --- /dev/null +++ b/dj_db_conn_pool/backends/odbc/oceanbase/oracle/base.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +from django.db.backends.oracle import base +from sqlalchemy.dialects.oracle.base import OracleDialect +from dj_db_conn_pool.backends.odbc.mixins import ODBCWrapperMixin +from dj_db_conn_pool.backends.odbc.utils import CursorWrapper + + +class DatabaseWrapper(ODBCWrapperMixin, base.DatabaseWrapper): + class SQLAlchemyDialect(OracleDialect): + pass + + def init_connection_state(self): + pass + + def create_cursor(self, name=None): + cursor = self.connection.cursor() + return CursorWrapper(cursor) diff --git a/dj_db_conn_pool/backends/odbc/utils.py b/dj_db_conn_pool/backends/odbc/utils.py new file mode 100644 index 0000000..5169b26 --- /dev/null +++ b/dj_db_conn_pool/backends/odbc/utils.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- + +class CursorWrapper: + def __init__(self, cursor): + self._cursor = cursor + + self.statement = None + + def __getattr__(self, item): + return getattr(self._cursor, item) From 23f3ee455d5f661337a8b61f98781585922cb49c Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Thu, 10 Nov 2022 15:56:32 +0800 Subject: [PATCH 05/25] update jdbc.oceanbase.mysql --- dj_db_conn_pool/backends/jdbc/oceanbase/mysql/base.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dj_db_conn_pool/backends/jdbc/oceanbase/mysql/base.py b/dj_db_conn_pool/backends/jdbc/oceanbase/mysql/base.py index 0688387..d52e873 100644 --- a/dj_db_conn_pool/backends/jdbc/oceanbase/mysql/base.py +++ b/dj_db_conn_pool/backends/jdbc/oceanbase/mysql/base.py @@ -14,7 +14,3 @@ def do_ping(self, dbapi_connection): return super(MySQLDialect, self).do_ping(dbapi_connection) except (jaydebeapi.DatabaseError, jpype.JException): return False - - def init_connection_state(self): - # TODO: custom OceanBase (MySQL mode) connection initialization - pass From b5cd90798545c8fb0590cc33c4bbda3003972541 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Thu, 10 Nov 2022 16:10:21 +0800 Subject: [PATCH 06/25] update jdbc_url --- dj_db_conn_pool/backends/jdbc/__init__.py | 9 ++++++++- dj_db_conn_pool/backends/jdbc/oceanbase/mixins.py | 7 ++----- dj_db_conn_pool/backends/jdbc/oracle/base.py | 4 +--- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/dj_db_conn_pool/backends/jdbc/__init__.py b/dj_db_conn_pool/backends/jdbc/__init__.py index 540ebd3..17937f0 100644 --- a/dj_db_conn_pool/backends/jdbc/__init__.py +++ b/dj_db_conn_pool/backends/jdbc/__init__.py @@ -17,9 +17,16 @@ def jdbc_driver(self): raise NotImplementedError() @property - def jdbc_url(self): + def jdbc_url_prefix(self): raise NotImplementedError() + @property + def jdbc_url(self): + return '{prefix}//{HOST}:{PORT}/{NAME}'.format( + prefix=self.jdbc_url_prefix, + **self.settings_dict + ) + @property def jdbc_options(self): return self.settings_dict.get('JDBC_OPTIONS', {}) diff --git a/dj_db_conn_pool/backends/jdbc/oceanbase/mixins.py b/dj_db_conn_pool/backends/jdbc/oceanbase/mixins.py index a29588b..c34cae5 100644 --- a/dj_db_conn_pool/backends/jdbc/oceanbase/mixins.py +++ b/dj_db_conn_pool/backends/jdbc/oceanbase/mixins.py @@ -1,14 +1,11 @@ # -*- coding: utf-8 -*- -from django.db.backends.oracle import base from dj_db_conn_pool.backends.jdbc import JDBCDatabaseWrapperMixin -class JDBCOceanBaseDatabaseWrapperMixin(JDBCDatabaseWrapperMixin, base.DatabaseWrapper): +class JDBCOceanBaseDatabaseWrapperMixin(JDBCDatabaseWrapperMixin): vendor = 'OceanBase' jdbc_driver = 'com.alipay.oceanbase.jdbc.Driver' - @property - def jdbc_url(self): - return 'jdbc:oceanbase://{NAME}'.format(**self.settings_dict) + jdbc_url_prefix = 'jdbc:oceanbase:' diff --git a/dj_db_conn_pool/backends/jdbc/oracle/base.py b/dj_db_conn_pool/backends/jdbc/oracle/base.py index 469d418..eda8213 100644 --- a/dj_db_conn_pool/backends/jdbc/oracle/base.py +++ b/dj_db_conn_pool/backends/jdbc/oracle/base.py @@ -31,9 +31,7 @@ def do_ping(self, dbapi_connection): jdbc_driver = 'oracle.jdbc.OracleDriver' - @property - def jdbc_url(self): - return 'jdbc:oracle:thin:@//{NAME}'.format(**self.settings_dict) + jdbc_url_prefix = 'jdbc:oracle:thin:@' @property def jdbc_options(self): From 147a582034b4f1aac7d06eeab5105b088b057303 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Mon, 14 Nov 2022 10:57:38 +0800 Subject: [PATCH 07/25] ren jdbc_options to get_connection_params --- dj_db_conn_pool/backends/jdbc/__init__.py | 7 +++---- dj_db_conn_pool/backends/jdbc/oracle/base.py | 5 ++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/dj_db_conn_pool/backends/jdbc/__init__.py b/dj_db_conn_pool/backends/jdbc/__init__.py index 17937f0..9fd89c5 100644 --- a/dj_db_conn_pool/backends/jdbc/__init__.py +++ b/dj_db_conn_pool/backends/jdbc/__init__.py @@ -27,9 +27,8 @@ def jdbc_url(self): **self.settings_dict ) - @property - def jdbc_options(self): - return self.settings_dict.get('JDBC_OPTIONS', {}) + def get_connection_params(self): + return self.settings_dict.get('OPTIONS', {}) def _get_new_connection(self, conn_params): conn = jaydebeapi.connect( @@ -38,7 +37,7 @@ def _get_new_connection(self, conn_params): { 'user': self.settings_dict['USER'], 'password': self.settings_dict['PASSWORD'], - **self.jdbc_options + **conn_params } ) diff --git a/dj_db_conn_pool/backends/jdbc/oracle/base.py b/dj_db_conn_pool/backends/jdbc/oracle/base.py index eda8213..ac0c588 100644 --- a/dj_db_conn_pool/backends/jdbc/oracle/base.py +++ b/dj_db_conn_pool/backends/jdbc/oracle/base.py @@ -33,9 +33,8 @@ def do_ping(self, dbapi_connection): jdbc_url_prefix = 'jdbc:oracle:thin:@' - @property - def jdbc_options(self): + def get_connection_params(self): return { **oracle_session_info, - **super(DatabaseWrapper, self).jdbc_options + **super(DatabaseWrapper, self).get_connection_params() } From 2d78dd4c4d0fd75252c7bda71c70107041b9995e Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Wed, 15 Feb 2023 09:56:00 +0800 Subject: [PATCH 08/25] add support for postgis --- dj_db_conn_pool/backends/postgis/__init__.py | 0 dj_db_conn_pool/backends/postgis/base.py | 10 ++++++++++ 2 files changed, 10 insertions(+) create mode 100644 dj_db_conn_pool/backends/postgis/__init__.py create mode 100644 dj_db_conn_pool/backends/postgis/base.py diff --git a/dj_db_conn_pool/backends/postgis/__init__.py b/dj_db_conn_pool/backends/postgis/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dj_db_conn_pool/backends/postgis/base.py b/dj_db_conn_pool/backends/postgis/base.py new file mode 100644 index 0000000..7458a08 --- /dev/null +++ b/dj_db_conn_pool/backends/postgis/base.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- + +from django.contrib.gis.db.backends.postgis import base +from sqlalchemy.dialects.postgresql.base import PGDialect +from dj_db_conn_pool.core.mixins import PooledDatabaseWrapperMixin + + +class DatabaseWrapper(PooledDatabaseWrapperMixin, base.DatabaseWrapper): + class SQLAlchemyDialect(PGDialect): + pass From 827dc9001bd502a5ecafdeb0479e800b6f009512 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Fri, 31 Mar 2023 10:14:13 +0800 Subject: [PATCH 09/25] fix #44 --- dj_db_conn_pool/backends/jdbc/__init__.py | 6 +++--- dj_db_conn_pool/backends/mysql/base.py | 2 +- dj_db_conn_pool/core/mixins.py | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dj_db_conn_pool/backends/jdbc/__init__.py b/dj_db_conn_pool/backends/jdbc/__init__.py index 9fd89c5..50f36fa 100644 --- a/dj_db_conn_pool/backends/jdbc/__init__.py +++ b/dj_db_conn_pool/backends/jdbc/__init__.py @@ -10,7 +10,7 @@ class JDBCDatabaseWrapperMixin(PooledDatabaseWrapperMixin): def _set_dbapi_autocommit(self, autocommit): - self.connection.connection.jconn.setAutoCommit(autocommit) + self.connection.driver_connection.jconn.setAutoCommit(autocommit) @property def jdbc_driver(self): @@ -56,7 +56,7 @@ def create_cursor(self, name=None): return CursorWrapper(cursor) def _close(self): - if self.connection is not None and self.connection.connection.jconn.getAutoCommit(): + if self.connection is not None and self.connection.driver_connection.jconn.getAutoCommit(): # if jdbc connection's autoCommit is on # jaydebeapi will throw an exception after rollback called # we make a little dynamic patch here, make sure @@ -65,6 +65,6 @@ def _close(self): logger.debug( "autoCommit of current JDBC connection to %s %s is on, won't do rollback before returning", - self.alias, self.connection.connection) + self.alias, self.connection.driver_connection) return super(JDBCDatabaseWrapperMixin, self)._close() diff --git a/dj_db_conn_pool/backends/mysql/base.py b/dj_db_conn_pool/backends/mysql/base.py index 0238ca5..1ee83c6 100644 --- a/dj_db_conn_pool/backends/mysql/base.py +++ b/dj_db_conn_pool/backends/mysql/base.py @@ -13,4 +13,4 @@ class SQLAlchemyDialect(MySQLDialect_pymysql): pass def _set_dbapi_autocommit(self, autocommit): - self.connection.connection.autocommit(autocommit) + self.connection.driver_connection.autocommit(autocommit) diff --git a/dj_db_conn_pool/core/mixins.py b/dj_db_conn_pool/core/mixins.py index ce0ff2f..5da44a3 100644 --- a/dj_db_conn_pool/core/mixins.py +++ b/dj_db_conn_pool/core/mixins.py @@ -12,7 +12,7 @@ class PooledDatabaseWrapperMixin(object): def __str__(self): try: - conn = repr(self.connection.connection) + conn = repr(self.connection.driver_connection) except AttributeError: conn = '' @@ -21,7 +21,7 @@ def __str__(self): __repr__ = __str__ def _set_dbapi_autocommit(self, autocommit): - self.connection.connection.autocommit = autocommit + self.connection.driver_connection.autocommit = autocommit def _set_autocommit(self, autocommit): with self.wrap_database_errors: @@ -101,13 +101,13 @@ def get_new_connection(self, conn_params): logger.debug( _("got %s's connection %s from its pool"), - self.alias, conn.connection) + self.alias, conn.driver_connection) return conn def close(self, *args, **kwargs): logger.debug( _("release %s's connection %s to its pool"), - self.alias, getattr(self.connection, 'connection', None)) + self.alias, getattr(self.connection, 'driver_connection', None)) return super(PooledDatabaseWrapperMixin, self).close(*args, **kwargs) From 8d12a39ffb55608b4aad5f0e91f1287ce81171b9 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Fri, 31 Mar 2023 12:18:11 +0800 Subject: [PATCH 10/25] mv --- dj_db_conn_pool/core/mixins/__init__.py | 3 +++ dj_db_conn_pool/core/{mixins.py => mixins/core.py} | 0 2 files changed, 3 insertions(+) create mode 100644 dj_db_conn_pool/core/mixins/__init__.py rename dj_db_conn_pool/core/{mixins.py => mixins/core.py} (100%) diff --git a/dj_db_conn_pool/core/mixins/__init__.py b/dj_db_conn_pool/core/mixins/__init__.py new file mode 100644 index 0000000..cd4fcc2 --- /dev/null +++ b/dj_db_conn_pool/core/mixins/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +from .core import PooledDatabaseWrapperMixin diff --git a/dj_db_conn_pool/core/mixins.py b/dj_db_conn_pool/core/mixins/core.py similarity index 100% rename from dj_db_conn_pool/core/mixins.py rename to dj_db_conn_pool/core/mixins/core.py From 0b7cc1d1ffc492c8d1f081575a6ed63e9e5fc1bf Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Thu, 6 Apr 2023 11:18:58 +0800 Subject: [PATCH 11/25] ren --- dj_db_conn_pool/backends/jdbc/__init__.py | 4 ++-- dj_db_conn_pool/backends/mysql/base.py | 4 ++-- dj_db_conn_pool/backends/odbc/mixins.py | 4 ++-- dj_db_conn_pool/backends/oracle/base.py | 4 ++-- dj_db_conn_pool/backends/postgis/base.py | 4 ++-- dj_db_conn_pool/backends/postgresql/mixins.py | 4 ++-- dj_db_conn_pool/core/mixins/__init__.py | 2 +- dj_db_conn_pool/core/mixins/core.py | 6 +++--- 8 files changed, 16 insertions(+), 16 deletions(-) diff --git a/dj_db_conn_pool/backends/jdbc/__init__.py b/dj_db_conn_pool/backends/jdbc/__init__.py index 50f36fa..d5041d2 100644 --- a/dj_db_conn_pool/backends/jdbc/__init__.py +++ b/dj_db_conn_pool/backends/jdbc/__init__.py @@ -1,14 +1,14 @@ # -*- coding: utf-8 -*- import jaydebeapi -from dj_db_conn_pool.core.mixins import PooledDatabaseWrapperMixin +from dj_db_conn_pool.core.mixins import PersistentDatabaseWrapperMixin from dj_db_conn_pool.backends.jdbc.utils import CursorWrapper import logging logger = logging.getLogger(__name__) -class JDBCDatabaseWrapperMixin(PooledDatabaseWrapperMixin): +class JDBCDatabaseWrapperMixin(PersistentDatabaseWrapperMixin): def _set_dbapi_autocommit(self, autocommit): self.connection.driver_connection.jconn.setAutoCommit(autocommit) diff --git a/dj_db_conn_pool/backends/mysql/base.py b/dj_db_conn_pool/backends/mysql/base.py index 1ee83c6..91ae289 100644 --- a/dj_db_conn_pool/backends/mysql/base.py +++ b/dj_db_conn_pool/backends/mysql/base.py @@ -2,13 +2,13 @@ from django.db.backends.mysql import base from sqlalchemy.dialects.mysql.pymysql import MySQLDialect_pymysql -from dj_db_conn_pool.core.mixins import PooledDatabaseWrapperMixin +from dj_db_conn_pool.core.mixins import PersistentDatabaseWrapperMixin import logging logger = logging.getLogger(__name__) -class DatabaseWrapper(PooledDatabaseWrapperMixin, base.DatabaseWrapper): +class DatabaseWrapper(PersistentDatabaseWrapperMixin, base.DatabaseWrapper): class SQLAlchemyDialect(MySQLDialect_pymysql): pass diff --git a/dj_db_conn_pool/backends/odbc/mixins.py b/dj_db_conn_pool/backends/odbc/mixins.py index c067bb4..289b9e8 100644 --- a/dj_db_conn_pool/backends/odbc/mixins.py +++ b/dj_db_conn_pool/backends/odbc/mixins.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- import pyodbc -from dj_db_conn_pool.core.mixins import PooledDatabaseWrapperMixin +from dj_db_conn_pool.core.mixins import PersistentDatabaseWrapperMixin -class ODBCWrapperMixin(PooledDatabaseWrapperMixin): +class ODBCWrapperMixin(PersistentDatabaseWrapperMixin): def _get_new_connection(self, conn_params): conn = pyodbc.connect( '', **self.settings_dict diff --git a/dj_db_conn_pool/backends/oracle/base.py b/dj_db_conn_pool/backends/oracle/base.py index 71c3d26..6e306e0 100644 --- a/dj_db_conn_pool/backends/oracle/base.py +++ b/dj_db_conn_pool/backends/oracle/base.py @@ -2,9 +2,9 @@ from django.db.backends.oracle import base from sqlalchemy.dialects.oracle.base import OracleDialect -from dj_db_conn_pool.core.mixins import PooledDatabaseWrapperMixin +from dj_db_conn_pool.core.mixins import PersistentDatabaseWrapperMixin -class DatabaseWrapper(PooledDatabaseWrapperMixin, base.DatabaseWrapper): +class DatabaseWrapper(PersistentDatabaseWrapperMixin, base.DatabaseWrapper): class SQLAlchemyDialect(OracleDialect): pass diff --git a/dj_db_conn_pool/backends/postgis/base.py b/dj_db_conn_pool/backends/postgis/base.py index 7458a08..8455b35 100644 --- a/dj_db_conn_pool/backends/postgis/base.py +++ b/dj_db_conn_pool/backends/postgis/base.py @@ -2,9 +2,9 @@ from django.contrib.gis.db.backends.postgis import base from sqlalchemy.dialects.postgresql.base import PGDialect -from dj_db_conn_pool.core.mixins import PooledDatabaseWrapperMixin +from dj_db_conn_pool.core.mixins import PersistentDatabaseWrapperMixin -class DatabaseWrapper(PooledDatabaseWrapperMixin, base.DatabaseWrapper): +class DatabaseWrapper(PersistentDatabaseWrapperMixin, base.DatabaseWrapper): class SQLAlchemyDialect(PGDialect): pass diff --git a/dj_db_conn_pool/backends/postgresql/mixins.py b/dj_db_conn_pool/backends/postgresql/mixins.py index e38e332..69166e0 100644 --- a/dj_db_conn_pool/backends/postgresql/mixins.py +++ b/dj_db_conn_pool/backends/postgresql/mixins.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2 -from dj_db_conn_pool.core.mixins import PooledDatabaseWrapperMixin +from dj_db_conn_pool.core.mixins import PersistentDatabaseWrapperMixin -class PGDatabaseWrapperMixin(PooledDatabaseWrapperMixin): +class PGDatabaseWrapperMixin(PersistentDatabaseWrapperMixin): class SQLAlchemyDialect(PGDialect_psycopg2): pass diff --git a/dj_db_conn_pool/core/mixins/__init__.py b/dj_db_conn_pool/core/mixins/__init__.py index cd4fcc2..e8da3ed 100644 --- a/dj_db_conn_pool/core/mixins/__init__.py +++ b/dj_db_conn_pool/core/mixins/__init__.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- -from .core import PooledDatabaseWrapperMixin +from .core import PersistentDatabaseWrapperMixin diff --git a/dj_db_conn_pool/core/mixins/core.py b/dj_db_conn_pool/core/mixins/core.py index 5da44a3..7e8cef3 100644 --- a/dj_db_conn_pool/core/mixins/core.py +++ b/dj_db_conn_pool/core/mixins/core.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) -class PooledDatabaseWrapperMixin(object): +class PersistentDatabaseWrapperMixin(object): def __str__(self): try: conn = repr(self.connection.driver_connection) @@ -36,7 +36,7 @@ def _get_dialect(self): return self.SQLAlchemyDialect(dbapi=self.Database) def _get_new_connection(self, conn_params): - return super(PooledDatabaseWrapperMixin, self).get_new_connection(conn_params) + return super(PersistentDatabaseWrapperMixin, self).get_new_connection(conn_params) def get_new_connection(self, conn_params): """ @@ -110,4 +110,4 @@ def close(self, *args, **kwargs): _("release %s's connection %s to its pool"), self.alias, getattr(self.connection, 'driver_connection', None)) - return super(PooledDatabaseWrapperMixin, self).close(*args, **kwargs) + return super(PersistentDatabaseWrapperMixin, self).close(*args, **kwargs) From 09c0c52fe097d416e1b44b30f6b78ffa9fac3f2f Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Thu, 6 Apr 2023 16:30:34 +0800 Subject: [PATCH 12/25] fix #42: database xxx is being accessed by other users --- dj_db_conn_pool/core/__init__.py | 79 +++++++++++++------------ dj_db_conn_pool/core/mixins/core.py | 7 +++ dj_db_conn_pool/core/mixins/creation.py | 12 ++++ 3 files changed, 61 insertions(+), 37 deletions(-) create mode 100644 dj_db_conn_pool/core/mixins/creation.py diff --git a/dj_db_conn_pool/core/__init__.py b/dj_db_conn_pool/core/__init__.py index dfc1cbe..0e0881f 100644 --- a/dj_db_conn_pool/core/__init__.py +++ b/dj_db_conn_pool/core/__init__.py @@ -1,37 +1,42 @@ -# -*- coding: utf-8 -*- - -import threading -from dj_db_conn_pool.compat import gettext_lazy as _ -from dj_db_conn_pool.core.exceptions import PoolDoesNotExist - - -class PoolContainer(dict): - # acquire this lock before modify pool_container - lock = threading.Lock() - - # the default parameters of pool - pool_default_params = { - 'pre_ping': True, - 'echo': False, - 'timeout': 30, - 'recycle': 60 * 15, - 'pool_size': 10, - 'max_overflow': 10, - } - - def has(self, pool_name): - return pool_name in self - - def put(self, pool_name, pool): - self[pool_name] = pool - - def get(self, pool_name): - try: - return self[pool_name] - except KeyError: - raise PoolDoesNotExist(_('No such pool: {pool_name}').format(pool_name=pool_name)) - - -# the pool's container, for maintaining the pools -# every process has it's own pool container -pool_container = PoolContainer() +# -*- coding: utf-8 -*- + +import threading +from dj_db_conn_pool.compat import gettext_lazy as _ +from dj_db_conn_pool.core.exceptions import PoolDoesNotExist + + +class PoolContainer(dict): + # acquire this lock before modify pool_container + lock = threading.Lock() + + # the default parameters of pool + pool_default_params = { + 'pre_ping': True, + 'echo': False, + 'timeout': 30, + 'recycle': 60 * 15, + 'pool_size': 10, + 'max_overflow': 10, + } + + def has(self, pool_name): + return pool_name in self + + def put(self, pool_name, pool): + self[pool_name] = pool + + def get(self, pool_name): + try: + return self[pool_name] + except KeyError: + raise PoolDoesNotExist(_('No such pool: {pool_name}').format(pool_name=pool_name)) + + def dispose(self): + # dispose all pools + for name, pool in self.items(): + pool.dispose() + + +# the pool's container, for maintaining the pools +# every process has it's own pool container +pool_container = PoolContainer() diff --git a/dj_db_conn_pool/core/mixins/core.py b/dj_db_conn_pool/core/mixins/core.py index 7e8cef3..cd0a7f3 100644 --- a/dj_db_conn_pool/core/mixins/core.py +++ b/dj_db_conn_pool/core/mixins/core.py @@ -3,6 +3,7 @@ from sqlalchemy import pool from dj_db_conn_pool.compat import gettext_lazy as _ from dj_db_conn_pool.core import pool_container +from dj_db_conn_pool.core.mixins.creation import DatabaseCreationMixin import logging @@ -10,6 +11,12 @@ class PersistentDatabaseWrapperMixin(object): + def __init__(self, *args, **kwargs): + # override creation_class + self.creation_class = type('', (DatabaseCreationMixin, self.creation_class), {}) + + super(PersistentDatabaseWrapperMixin, self).__init__(*args, **kwargs) + def __str__(self): try: conn = repr(self.connection.driver_connection) diff --git a/dj_db_conn_pool/core/mixins/creation.py b/dj_db_conn_pool/core/mixins/creation.py new file mode 100644 index 0000000..cdf9686 --- /dev/null +++ b/dj_db_conn_pool/core/mixins/creation.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- + +from dj_db_conn_pool.core import pool_container + + +class DatabaseCreationMixin(object): + def _destroy_test_db(self, test_database_name, verbosity): + # dispose all pools before destroying testdb + pool_container.dispose() + + # destroy testdb + return super(DatabaseCreationMixin, self)._destroy_test_db(test_database_name, verbosity) From 77fae0015aa06bea9d67e3003b4be26539cf6ef7 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Mon, 17 Apr 2023 15:40:49 +0800 Subject: [PATCH 13/25] update jaydebeapi converters --- dj_db_conn_pool/compat/jdbc.py | 36 ++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/dj_db_conn_pool/compat/jdbc.py b/dj_db_conn_pool/compat/jdbc.py index df2d14d..f9566b9 100644 --- a/dj_db_conn_pool/compat/jdbc.py +++ b/dj_db_conn_pool/compat/jdbc.py @@ -1,3 +1,7 @@ +import jaydebeapi +from datetime import datetime + + def patch_all(): patch_converters() @@ -6,27 +10,33 @@ def patch_converters(): """ patch jaydebeapi's converters """ - from datetime import datetime - import jaydebeapi + def to_number(rs, col): + java_obj = rs.getBigDecimal(col) + + py_str = rs.getString(col) + + if java_obj.scale() > 0: + return float(py_str) + + return int(py_str) - def _to_str(rs, col): - return str(rs.getObject(col)) + def to_str(rs, col): + return rs.getString(col) - def _to_datetime(rs, col): - java_val = rs.getTimestamp(col) + def to_datetime(rs, col): + java_obj = rs.getTimestamp(col).getTime() - if not java_val: - return + time_stamp = int(str(java_obj)) // 1000 - dt = datetime.strptime( - str(java_val)[:19], '%Y-%m-%d %H:%M:%S') + dt = datetime.fromtimestamp(time_stamp) - dt = dt.replace(microsecond=int(str(java_val.getNanos())[:6])) return dt jaydebeapi._DEFAULT_CONVERTERS.update( { - 'TIMESTAMP': _to_datetime, - 'VARCHAR': _to_str + 'CHAR': to_str, + 'VARCHAR': to_str, + 'NUMERIC': to_number, + 'TIMESTAMP': to_datetime, } ) From 80333e6b4d23d0c9bad7f13744f851ddab3a81ef Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Mon, 17 Apr 2023 15:41:59 +0800 Subject: [PATCH 14/25] update oracle autocommit --- dj_db_conn_pool/backends/oracle/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dj_db_conn_pool/backends/oracle/base.py b/dj_db_conn_pool/backends/oracle/base.py index 6e306e0..4ae00e9 100644 --- a/dj_db_conn_pool/backends/oracle/base.py +++ b/dj_db_conn_pool/backends/oracle/base.py @@ -8,3 +8,6 @@ class DatabaseWrapper(PersistentDatabaseWrapperMixin, base.DatabaseWrapper): class SQLAlchemyDialect(OracleDialect): pass + + def _set_dbapi_autocommit(self, autocommit): + self.connection.driver_connection.autocommit = int(autocommit) From d65ce7aec6a4802a32a8ad9c8e126ae442fbf3af Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Mon, 17 Apr 2023 15:52:00 +0800 Subject: [PATCH 15/25] update odbc --- dj_db_conn_pool/backends/odbc/mixins.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/dj_db_conn_pool/backends/odbc/mixins.py b/dj_db_conn_pool/backends/odbc/mixins.py index 289b9e8..c617b75 100644 --- a/dj_db_conn_pool/backends/odbc/mixins.py +++ b/dj_db_conn_pool/backends/odbc/mixins.py @@ -1,13 +1,19 @@ # -*- coding: utf-8 -*- import pyodbc +from django.conf import ImproperlyConfigured from dj_db_conn_pool.core.mixins import PersistentDatabaseWrapperMixin class ODBCWrapperMixin(PersistentDatabaseWrapperMixin): def _get_new_connection(self, conn_params): - conn = pyodbc.connect( - '', **self.settings_dict - ) + try: + driver = self.settings_dict['ODBC_OPTIONS']['DRIVER'] + except KeyError: + raise ImproperlyConfigured('No odbc driver provided') - return conn + conn_str_template = 'DRIVER={DRIVER};SERVER={HOST}:{PORT};DATABASE={NAME};UID={USER};PWD={PASSWORD}' + + connection_string = conn_str_template.format(DRIVER=driver, **self.settings_dict) + + return pyodbc.connect(connection_string) From 27f7d7b24d60295787c0b03c9461cf76ff12f101 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Mon, 17 Apr 2023 18:05:00 +0800 Subject: [PATCH 16/25] switch to jpype --- dj_db_conn_pool/backends/jdbc/__init__.py | 28 +++++++------ .../backends/jdbc/oceanbase/mysql/base.py | 5 +-- .../backends/jdbc/oceanbase/oracle/base.py | 5 +-- dj_db_conn_pool/backends/jdbc/oracle/base.py | 5 +-- dj_db_conn_pool/backends/jdbc/utils.py | 1 - dj_db_conn_pool/compat/jdbc.py | 41 ++++++++----------- 6 files changed, 40 insertions(+), 45 deletions(-) diff --git a/dj_db_conn_pool/backends/jdbc/__init__.py b/dj_db_conn_pool/backends/jdbc/__init__.py index d5041d2..187c860 100644 --- a/dj_db_conn_pool/backends/jdbc/__init__.py +++ b/dj_db_conn_pool/backends/jdbc/__init__.py @@ -1,17 +1,17 @@ # -*- coding: utf-8 -*- -import jaydebeapi +import jpype +import jpype.dbapi2 from dj_db_conn_pool.core.mixins import PersistentDatabaseWrapperMixin from dj_db_conn_pool.backends.jdbc.utils import CursorWrapper import logging logger = logging.getLogger(__name__) +JDBC_TYPE_CONVERTERS = {} -class JDBCDatabaseWrapperMixin(PersistentDatabaseWrapperMixin): - def _set_dbapi_autocommit(self, autocommit): - self.connection.driver_connection.jconn.setAutoCommit(autocommit) +class JDBCDatabaseWrapperMixin(PersistentDatabaseWrapperMixin): @property def jdbc_driver(self): raise NotImplementedError() @@ -31,14 +31,18 @@ def get_connection_params(self): return self.settings_dict.get('OPTIONS', {}) def _get_new_connection(self, conn_params): - conn = jaydebeapi.connect( - self.jdbc_driver, + if not jpype.isJVMStarted(): + jpype.startJVM(ignoreUnrecognized=True) + + conn = jpype.dbapi2.connect( self.jdbc_url, - { - 'user': self.settings_dict['USER'], - 'password': self.settings_dict['PASSWORD'], + driver=self.jdbc_driver, + driver_args=dict( + user=self.settings_dict['USER'], + password=self.settings_dict['PASSWORD'], **conn_params - } + ), + converters=JDBC_TYPE_CONVERTERS, ) return conn @@ -56,9 +60,9 @@ def create_cursor(self, name=None): return CursorWrapper(cursor) def _close(self): - if self.connection is not None and self.connection.driver_connection.jconn.getAutoCommit(): + if self.connection is not None and self.connection.driver_connection.autocommit: # if jdbc connection's autoCommit is on - # jaydebeapi will throw an exception after rollback called + # jpype will throw NotSupportedError after rollback called # we make a little dynamic patch here, make sure # SQLAlchemy will not do rollback before recycling connection self.connection._pool._reset_on_return = None diff --git a/dj_db_conn_pool/backends/jdbc/oceanbase/mysql/base.py b/dj_db_conn_pool/backends/jdbc/oceanbase/mysql/base.py index d52e873..a3332b8 100644 --- a/dj_db_conn_pool/backends/jdbc/oceanbase/mysql/base.py +++ b/dj_db_conn_pool/backends/jdbc/oceanbase/mysql/base.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- -import jpype -import jaydebeapi +import jpype.dbapi2 from django.db.backends.mysql import base from sqlalchemy.dialects.mysql.base import MySQLDialect from dj_db_conn_pool.backends.jdbc.oceanbase.mixins import JDBCOceanBaseDatabaseWrapperMixin @@ -12,5 +11,5 @@ class SQLAlchemyDialect(MySQLDialect): def do_ping(self, dbapi_connection): try: return super(MySQLDialect, self).do_ping(dbapi_connection) - except (jaydebeapi.DatabaseError, jpype.JException): + except jpype.dbapi2.DatabaseError: return False diff --git a/dj_db_conn_pool/backends/jdbc/oceanbase/oracle/base.py b/dj_db_conn_pool/backends/jdbc/oceanbase/oracle/base.py index faf8000..58a38b1 100644 --- a/dj_db_conn_pool/backends/jdbc/oceanbase/oracle/base.py +++ b/dj_db_conn_pool/backends/jdbc/oceanbase/oracle/base.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- -import jpype -import jaydebeapi +import jpype.dbapi2 from django.db.backends.oracle import base from sqlalchemy.dialects.oracle.base import OracleDialect from dj_db_conn_pool.backends.jdbc.oceanbase.mixins import JDBCOceanBaseDatabaseWrapperMixin @@ -12,7 +11,7 @@ class SQLAlchemyDialect(OracleDialect): def do_ping(self, dbapi_connection): try: return super(OracleDialect, self).do_ping(dbapi_connection) - except (jaydebeapi.DatabaseError, jpype.JException): + except jpype.dbapi2.DatabaseError: return False def init_connection_state(self): diff --git a/dj_db_conn_pool/backends/jdbc/oracle/base.py b/dj_db_conn_pool/backends/jdbc/oracle/base.py index ac0c588..75c1671 100644 --- a/dj_db_conn_pool/backends/jdbc/oracle/base.py +++ b/dj_db_conn_pool/backends/jdbc/oracle/base.py @@ -3,8 +3,7 @@ import getpass import socket from multiprocessing import current_process -import jpype -import jaydebeapi +import jpype.dbapi2 from django.db.backends.oracle import base from sqlalchemy.dialects.oracle.base import OracleDialect from dj_db_conn_pool.backends.jdbc import JDBCDatabaseWrapperMixin @@ -26,7 +25,7 @@ class SQLAlchemyDialect(OracleDialect): def do_ping(self, dbapi_connection): try: return super(OracleDialect, self).do_ping(dbapi_connection) - except (jaydebeapi.DatabaseError, jpype.JException): + except jpype.dbapi2.DatabaseError: return False jdbc_driver = 'oracle.jdbc.OracleDriver' diff --git a/dj_db_conn_pool/backends/jdbc/utils.py b/dj_db_conn_pool/backends/jdbc/utils.py index a3240c1..01fe4c1 100644 --- a/dj_db_conn_pool/backends/jdbc/utils.py +++ b/dj_db_conn_pool/backends/jdbc/utils.py @@ -35,7 +35,6 @@ def execute(self, query, parameters=None): # record last query self.statement = query - # call jaydebeapi self._cursor.cursor.execute(query, parameters) def __getattr__(self, item): diff --git a/dj_db_conn_pool/compat/jdbc.py b/dj_db_conn_pool/compat/jdbc.py index f9566b9..d8482f4 100644 --- a/dj_db_conn_pool/compat/jdbc.py +++ b/dj_db_conn_pool/compat/jdbc.py @@ -1,4 +1,4 @@ -import jaydebeapi +import jpype from datetime import datetime @@ -8,35 +8,30 @@ def patch_all(): def patch_converters(): """ - patch jaydebeapi's converters + patch jpype's jdbc converters """ - def to_number(rs, col): - java_obj = rs.getBigDecimal(col) + def to_str(value): + return str(value.toString()) - py_str = rs.getString(col) + def to_number(value): + string = str(value.toString()) - if java_obj.scale() > 0: - return float(py_str) + if value.scale() > 0: + return float(string) - return int(py_str) + return int(string) - def to_str(rs, col): - return rs.getString(col) + def to_datetime(value): + timestamp = int(str(value.getTime())) // 1000 - def to_datetime(rs, col): - java_obj = rs.getTimestamp(col).getTime() + return datetime.fromtimestamp(timestamp) - time_stamp = int(str(java_obj)) // 1000 + @jpype.onJVMStart + def register_converters(): + from dj_db_conn_pool.backends.jdbc import JDBC_TYPE_CONVERTERS - dt = datetime.fromtimestamp(time_stamp) + JDBC_TYPE_CONVERTERS[jpype.java.lang.String] = to_str - return dt + JDBC_TYPE_CONVERTERS[jpype.java.math.BigDecimal] = to_number - jaydebeapi._DEFAULT_CONVERTERS.update( - { - 'CHAR': to_str, - 'VARCHAR': to_str, - 'NUMERIC': to_number, - 'TIMESTAMP': to_datetime, - } - ) + JDBC_TYPE_CONVERTERS[jpype.java.sql.Timestamp] = to_datetime From a8a369a4f4998724984cfa55d89acf20b3f99013 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Thu, 20 Apr 2023 21:10:35 +0800 Subject: [PATCH 17/25] update jdbc backend --- dj_db_conn_pool/backends/jdbc/__init__.py | 9 +++--- dj_db_conn_pool/compat/jdbc.py | 37 ----------------------- 2 files changed, 5 insertions(+), 41 deletions(-) delete mode 100644 dj_db_conn_pool/compat/jdbc.py diff --git a/dj_db_conn_pool/backends/jdbc/__init__.py b/dj_db_conn_pool/backends/jdbc/__init__.py index 187c860..4bd6643 100644 --- a/dj_db_conn_pool/backends/jdbc/__init__.py +++ b/dj_db_conn_pool/backends/jdbc/__init__.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import threading import jpype import jpype.dbapi2 from dj_db_conn_pool.core.mixins import PersistentDatabaseWrapperMixin @@ -8,7 +9,7 @@ import logging logger = logging.getLogger(__name__) -JDBC_TYPE_CONVERTERS = {} +LOCK_CHECK_JVM_STATUS = threading.Lock() class JDBCDatabaseWrapperMixin(PersistentDatabaseWrapperMixin): @@ -31,8 +32,9 @@ def get_connection_params(self): return self.settings_dict.get('OPTIONS', {}) def _get_new_connection(self, conn_params): - if not jpype.isJVMStarted(): - jpype.startJVM(ignoreUnrecognized=True) + with LOCK_CHECK_JVM_STATUS: + if not jpype.isJVMStarted(): + jpype.startJVM(ignoreUnrecognized=True) conn = jpype.dbapi2.connect( self.jdbc_url, @@ -42,7 +44,6 @@ def _get_new_connection(self, conn_params): password=self.settings_dict['PASSWORD'], **conn_params ), - converters=JDBC_TYPE_CONVERTERS, ) return conn diff --git a/dj_db_conn_pool/compat/jdbc.py b/dj_db_conn_pool/compat/jdbc.py deleted file mode 100644 index d8482f4..0000000 --- a/dj_db_conn_pool/compat/jdbc.py +++ /dev/null @@ -1,37 +0,0 @@ -import jpype -from datetime import datetime - - -def patch_all(): - patch_converters() - - -def patch_converters(): - """ - patch jpype's jdbc converters - """ - def to_str(value): - return str(value.toString()) - - def to_number(value): - string = str(value.toString()) - - if value.scale() > 0: - return float(string) - - return int(string) - - def to_datetime(value): - timestamp = int(str(value.getTime())) // 1000 - - return datetime.fromtimestamp(timestamp) - - @jpype.onJVMStart - def register_converters(): - from dj_db_conn_pool.backends.jdbc import JDBC_TYPE_CONVERTERS - - JDBC_TYPE_CONVERTERS[jpype.java.lang.String] = to_str - - JDBC_TYPE_CONVERTERS[jpype.java.math.BigDecimal] = to_number - - JDBC_TYPE_CONVERTERS[jpype.java.sql.Timestamp] = to_datetime From 353f59b8033739328bc800d9b7c19437ca2113de Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Thu, 20 Apr 2023 21:32:29 +0800 Subject: [PATCH 18/25] update requirements --- requirements.txt | 8 +++++--- setup.py | 20 +++++++++++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/requirements.txt b/requirements.txt index 83c75b7..05d4caa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,8 @@ -Django>=1.8 -SQLAlchemy>=1.2.16 +Django>=2.0 +SQLAlchemy>=1.4.24 +JPype1>=1.3.0 +sqlparams>=3.0.0 PyMySQL>=0.9.3 +pyodbc>=4.0.34 cx-Oracle>=6.4.1 psycopg2>=2.8.6 -JayDeBeApi>=1.2.3 diff --git a/setup.py b/setup.py index 9530692..e10e248 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,5 @@ +# encoding: utf-8 + import codecs import shutil import setuptools @@ -34,18 +36,26 @@ def setup(): packages=setuptools.find_packages(), include_package_data=True, install_requires=[ - 'Django', + 'Django>=2.0', 'SQLAlchemy>=1.4.24', ], extras_require={ 'all': [ - 'PyMySQL>=0.9.3', 'cx-Oracle>=6.4.1', 'psycopg2>=2.8.6', - 'JayDeBeApi>=1.2.3', 'sqlparams>=3.0.0' + 'JPype1>=1.3.0', + 'sqlparams>=3.0.0', + 'PyMySQL>=0.9.3', + 'pyodbc>=4.0.34', + 'cx-Oracle>=6.4.1', + 'psycopg2>=2.8.6', + ], + 'jdbc': [ + 'JPype1>=1.3.0', + 'sqlparams>=3.0.0', ], 'mysql': ['PyMySQL>=0.9.3'], + 'odbc': ['pyodbc>=4.0.34'], 'oracle': ['cx-Oracle>=6.4.1'], 'postgresql': ['psycopg2>=2.8.6'], - 'jdbc': ['JayDeBeApi>=1.2.3', 'sqlparams>=3.0.0'], }, classifiers=[ 'Development Status :: 5 - Production/Stable', @@ -57,7 +67,7 @@ def setup(): 'Programming Language :: Python :: 3', 'Topic :: Software Development :: Libraries :: Python Modules', ], - keywords=['django', 'db', 'database', 'persistent', 'connection', 'pool'], + keywords=['django', 'db', 'database', 'persistent', 'connection', 'pool', 'pooling'], ) From 1045d715eb98f38e7240f23ec955165817e67fec Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Thu, 20 Apr 2023 21:47:08 +0800 Subject: [PATCH 19/25] update readme --- README.md | 19 +++++++++---------- README_CN.md | 11 +---------- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index d984e57..b9d66cd 100644 --- a/README.md +++ b/README.md @@ -111,8 +111,16 @@ import dj_db_conn_pool dj_db_conn_pool.setup(pool_size=100, max_overflow=50) ``` +#### multiprocessing environment +In a multiprocessing environment, such as uWSGI, each process will have its own `dj_db_conn_pool.core:pool_container` object, +It means that each process has an independent connection pool, for example: +The `POOL_OPTIONS` configuration of database `db1` is`{ 'POOL_SIZE': 10, 'MAX_OVERFLOW': 20 }`, +If uWSGI starts 8 worker processes, then the total connection pool size of `db1` is `8 * 10`, +The maximum number of connections will not exceed `8 * 10 + 8 * 20` + + ## JDBC (experimental, NOT PRODUCTION READY) -Thanks to [JPype](https://github.com/jpype-project/jpype) [JayDeBeApi](https://github.com/baztian/jaydebeapi/) , +Thanks to [JPype](https://github.com/jpype-project/jpype), django-db-connection-pool can connect to database in jdbc way ### Usage @@ -122,15 +130,6 @@ export JAVA_HOME=$PATH_TO_JRE; export CLASSPATH=$PATH_RO_JDBC_DRIVER_JAR ``` -#### Start JVM -Start JVM before initialization of Django - -```python -import jpype -jvm_path = jpype.getDefaultJVMPath() -jpype.startJVM(jvm_path) -``` - #### Update settings.DATABASES ##### Oracle diff --git a/README_CN.md b/README_CN.md index 0d6edcd..0be0405 100644 --- a/README_CN.md +++ b/README_CN.md @@ -148,7 +148,7 @@ dj_db_conn_pool.setup(pool_size=10, max_overflow=20) `{ 'POOL_SIZE': 10, 'MAX_OVERFLOW': 20 }`,项目启动了8个进程,则该项目的`db1`连接池总大小是`8 * 10`,最大连接数是`8 * 10 + 8 * 20` ## JDBC(仍在完善中,不建议用于生产) -基于 [JPype](https://github.com/jpype-project/jpype) [JayDeBeApi](https://github.com/baztian/jaydebeapi/) ,django-db-connection-pool 现在可以通过 jdbc 连接到数据库并保持连接 +基于 [JPype](https://github.com/jpype-project/jpype),django-db-connection-pool 现在可以通过 jdbc 连接到数据库并保持连接 ### 使用方法 #### 设置环境变量 @@ -157,15 +157,6 @@ export JAVA_HOME=$PATH_TO_JRE; export CLASSPATH=$PATH_RO_JDBC_DRIVER_JAR ``` -#### 启动 JVM -在引用 `dj_db_conn_pool.backends.jdbc.xxx` 前,调用 jpype 启动 JVM - -```python -import jpype -jvm_path = jpype.getDefaultJVMPath() -jpype.startJVM(jvm_path) -``` - #### 更新 settings.DATABASES 配置 ##### Oracle From 2383593df44ed68298371c481611b22fdda21904 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Fri, 21 Apr 2023 20:11:02 +0800 Subject: [PATCH 20/25] updates: 1. add jdbc type converters 2. add named SQL parameter support for all backends --- dj_db_conn_pool/__init__.py | 2 +- dj_db_conn_pool/backends/jdbc/__init__.py | 24 ++++------ dj_db_conn_pool/backends/jdbc/utils.py | 41 ----------------- .../backends/odbc/oceanbase/oracle/base.py | 5 -- dj_db_conn_pool/backends/odbc/utils.py | 10 ---- dj_db_conn_pool/backends/oracle/base.py | 2 + dj_db_conn_pool/compat/jdbc.py | 33 +++++++++++++ dj_db_conn_pool/core/mixins/core.py | 11 +++++ dj_db_conn_pool/core/utils.py | 46 +++++++++++++++++++ setup.py | 6 +-- 10 files changed, 104 insertions(+), 76 deletions(-) delete mode 100644 dj_db_conn_pool/backends/jdbc/utils.py delete mode 100644 dj_db_conn_pool/backends/odbc/utils.py create mode 100644 dj_db_conn_pool/compat/jdbc.py create mode 100644 dj_db_conn_pool/core/utils.py diff --git a/dj_db_conn_pool/__init__.py b/dj_db_conn_pool/__init__.py index b573a49..7a4c349 100644 --- a/dj_db_conn_pool/__init__.py +++ b/dj_db_conn_pool/__init__.py @@ -1,4 +1,4 @@ -__version__ = '1.2.2' +__version__ = '1.2.3' __author__ = 'Altair Bow' __author_email__ = 'altair.bow@foxmail.com' __description__ = 'Persistent database connection backends for Django' diff --git a/dj_db_conn_pool/backends/jdbc/__init__.py b/dj_db_conn_pool/backends/jdbc/__init__.py index 4bd6643..44fd598 100644 --- a/dj_db_conn_pool/backends/jdbc/__init__.py +++ b/dj_db_conn_pool/backends/jdbc/__init__.py @@ -4,15 +4,20 @@ import jpype import jpype.dbapi2 from dj_db_conn_pool.core.mixins import PersistentDatabaseWrapperMixin -from dj_db_conn_pool.backends.jdbc.utils import CursorWrapper import logging logger = logging.getLogger(__name__) -LOCK_CHECK_JVM_STATUS = threading.Lock() +jdbc_type_converters = {} + +lock_check_jvm_status = threading.Lock() class JDBCDatabaseWrapperMixin(PersistentDatabaseWrapperMixin): + _sql_param_style = 'qmark' + + _sql_converter = staticmethod(lambda sql: sql.replace('%s', '?')) + @property def jdbc_driver(self): raise NotImplementedError() @@ -32,7 +37,7 @@ def get_connection_params(self): return self.settings_dict.get('OPTIONS', {}) def _get_new_connection(self, conn_params): - with LOCK_CHECK_JVM_STATUS: + with lock_check_jvm_status: if not jpype.isJVMStarted(): jpype.startJVM(ignoreUnrecognized=True) @@ -44,22 +49,11 @@ def _get_new_connection(self, conn_params): password=self.settings_dict['PASSWORD'], **conn_params ), + converters=jdbc_type_converters, ) return conn - def create_cursor(self, name=None): - """ - create a cursor - do some tricks here - :param name: - :return: - """ - # get cursor from django - cursor = super().create_cursor(name) - - return CursorWrapper(cursor) - def _close(self): if self.connection is not None and self.connection.driver_connection.autocommit: # if jdbc connection's autoCommit is on diff --git a/dj_db_conn_pool/backends/jdbc/utils.py b/dj_db_conn_pool/backends/jdbc/utils.py deleted file mode 100644 index 01fe4c1..0000000 --- a/dj_db_conn_pool/backends/jdbc/utils.py +++ /dev/null @@ -1,41 +0,0 @@ -import sqlparams - -import logging -logger = logging.getLogger(__name__) - -sql_params_converter = sqlparams.SQLParams('named', 'qmark') - - -class CursorWrapper: - def __init__(self, cursor): - self._cursor = cursor - - self.statement = None - - def execute(self, query, parameters=None): - """ - :param self: django's cursor - :param query: SQL - :param parameters: SQL parameters - :return: - """ - if isinstance(parameters, dict): - # convert sql and parameters - _query, _parameters = sql_params_converter.format(query, parameters) - - logger.debug( - 'SQL (%s), parameters(%s) has been converted to SQL(%s), parameters(%s)', - query, parameters, _query, _parameters) - - query, parameters = (_query, _parameters) - else: - # change paramstyle 'format' to 'qmark' - query = query.replace('%s', '?') - - # record last query - self.statement = query - - self._cursor.cursor.execute(query, parameters) - - def __getattr__(self, item): - return getattr(self._cursor, item) diff --git a/dj_db_conn_pool/backends/odbc/oceanbase/oracle/base.py b/dj_db_conn_pool/backends/odbc/oceanbase/oracle/base.py index 21d000e..95604bf 100644 --- a/dj_db_conn_pool/backends/odbc/oceanbase/oracle/base.py +++ b/dj_db_conn_pool/backends/odbc/oceanbase/oracle/base.py @@ -3,7 +3,6 @@ from django.db.backends.oracle import base from sqlalchemy.dialects.oracle.base import OracleDialect from dj_db_conn_pool.backends.odbc.mixins import ODBCWrapperMixin -from dj_db_conn_pool.backends.odbc.utils import CursorWrapper class DatabaseWrapper(ODBCWrapperMixin, base.DatabaseWrapper): @@ -12,7 +11,3 @@ class SQLAlchemyDialect(OracleDialect): def init_connection_state(self): pass - - def create_cursor(self, name=None): - cursor = self.connection.cursor() - return CursorWrapper(cursor) diff --git a/dj_db_conn_pool/backends/odbc/utils.py b/dj_db_conn_pool/backends/odbc/utils.py deleted file mode 100644 index 5169b26..0000000 --- a/dj_db_conn_pool/backends/odbc/utils.py +++ /dev/null @@ -1,10 +0,0 @@ -# -*- coding: utf-8 -*- - -class CursorWrapper: - def __init__(self, cursor): - self._cursor = cursor - - self.statement = None - - def __getattr__(self, item): - return getattr(self._cursor, item) diff --git a/dj_db_conn_pool/backends/oracle/base.py b/dj_db_conn_pool/backends/oracle/base.py index 4ae00e9..6cabf33 100644 --- a/dj_db_conn_pool/backends/oracle/base.py +++ b/dj_db_conn_pool/backends/oracle/base.py @@ -6,6 +6,8 @@ class DatabaseWrapper(PersistentDatabaseWrapperMixin, base.DatabaseWrapper): + _sql_param_style = 'numeric' + class SQLAlchemyDialect(OracleDialect): pass diff --git a/dj_db_conn_pool/compat/jdbc.py b/dj_db_conn_pool/compat/jdbc.py new file mode 100644 index 0000000..a3333c2 --- /dev/null +++ b/dj_db_conn_pool/compat/jdbc.py @@ -0,0 +1,33 @@ +import jpype + + +def patch_all(): + patch_converters() + + +def patch_converters(): + """ + patch jpype's jdbc converters + """ + def to_python(value): + return value._py() + + def to_number(value): + string = str(value.toString()) + + if value.scale() > 0: + return float(string) + + return int(string) + + @jpype.onJVMStart + def register_converters(): + from dj_db_conn_pool.backends.jdbc import jdbc_type_converters + + jdbc_type_converters[jpype.java.lang.String] = str + jdbc_type_converters[jpype.java.sql.Date] = to_python + jdbc_type_converters[jpype.java.sql.Time] = to_python + jdbc_type_converters[jpype.java.sql.Timestamp] = to_python + jdbc_type_converters[jpype.java.math.BigDecimal] = to_number + jdbc_type_converters[jpype.JArray(jpype.types.JByte)] = bytes + # jdbc_type_converters[type(None)] = lambda v: v diff --git a/dj_db_conn_pool/core/mixins/core.py b/dj_db_conn_pool/core/mixins/core.py index cd0a7f3..5ed2683 100644 --- a/dj_db_conn_pool/core/mixins/core.py +++ b/dj_db_conn_pool/core/mixins/core.py @@ -3,6 +3,7 @@ from sqlalchemy import pool from dj_db_conn_pool.compat import gettext_lazy as _ from dj_db_conn_pool.core import pool_container +from dj_db_conn_pool.core.utils import CursorWrapper from dj_db_conn_pool.core.mixins.creation import DatabaseCreationMixin @@ -42,6 +43,16 @@ def _set_autocommit(self, autocommit): def _get_dialect(self): return self.SQLAlchemyDialect(dbapi=self.Database) + _sql_param_style = 'format' + + _sql_converter = staticmethod(lambda sql: sql) + + def create_cursor(self, name=None): + # get cursor from django + cursor = super().create_cursor(name) + + return CursorWrapper(cursor, self._sql_param_style, self._sql_converter) + def _get_new_connection(self, conn_params): return super(PersistentDatabaseWrapperMixin, self).get_new_connection(conn_params) diff --git a/dj_db_conn_pool/core/utils.py b/dj_db_conn_pool/core/utils.py new file mode 100644 index 0000000..5de6dc9 --- /dev/null +++ b/dj_db_conn_pool/core/utils.py @@ -0,0 +1,46 @@ +import logging +import sqlparams + +logger = logging.getLogger(__name__) + + +class CursorWrapper: + def __init__(self, cursor, style, sql_converter): + self._cursor = cursor + + self._sql_params_converter = sqlparams.SQLParams('named', style, True, True) + + self._sql_converter = sql_converter + + self._statement = None + + def execute(self, query, parameters=None): + if isinstance(parameters, dict): + _query, _parameters = self._sql_params_converter.format(query, parameters) + else: + _query, _parameters = self._sql_converter(query), parameters + + if query != _query: + logger.debug( + 'SQL (%s), parameters(%s) has been converted to SQL(%s), parameters(%s)', + query, parameters, _query, _parameters + ) + + query, parameters = _query, _parameters + + try: + cursor = self._cursor.cursor + except AttributeError: + cursor = self._cursor + + self._statement = query + + return cursor.execute(query, parameters or []) + + def __getattr__(self, attr): + try: + return getattr(self._cursor, attr) + except AttributeError: + if attr == 'statement': + return self._statement + raise diff --git a/setup.py b/setup.py index e10e248..3d10ada 100644 --- a/setup.py +++ b/setup.py @@ -38,6 +38,7 @@ def setup(): install_requires=[ 'Django>=2.0', 'SQLAlchemy>=1.4.24', + 'sqlparams>=4.0.0', ], extras_require={ 'all': [ @@ -48,10 +49,7 @@ def setup(): 'cx-Oracle>=6.4.1', 'psycopg2>=2.8.6', ], - 'jdbc': [ - 'JPype1>=1.3.0', - 'sqlparams>=3.0.0', - ], + 'jdbc': ['JPype1>=1.3.0'], 'mysql': ['PyMySQL>=0.9.3'], 'odbc': ['pyodbc>=4.0.34'], 'oracle': ['cx-Oracle>=6.4.1'], From 7c4e509295242e6ca4eb2cc2e77d0147b928df71 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Thu, 25 May 2023 15:57:26 +0800 Subject: [PATCH 21/25] fix #47 --- dj_db_conn_pool/backends/postgresql/base.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dj_db_conn_pool/backends/postgresql/base.py b/dj_db_conn_pool/backends/postgresql/base.py index 62bc21f..2a88024 100644 --- a/dj_db_conn_pool/backends/postgresql/base.py +++ b/dj_db_conn_pool/backends/postgresql/base.py @@ -5,4 +5,10 @@ class DatabaseWrapper(PGDatabaseWrapperMixin, base.DatabaseWrapper): - pass + def get_new_connection(self, conn_params): + connection = super().get_new_connection(conn_params) + + if not connection.info: + connection.info = connection.connection.info + + return connection From 5be9152192ddb54002cbdca2c5fc00edb3ce63cb Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Mon, 5 Jun 2023 20:30:12 +0800 Subject: [PATCH 22/25] 1. fix #48 2. Remove support for `named` sql parameters, Use `pyformat` to handle SQL named parameters 3. JDBC backends: Add support for `pyformat` SQL named parameters --- dj_db_conn_pool/backends/jdbc/__init__.py | 9 +++-- dj_db_conn_pool/backends/jdbc/utils.py | 29 ++++++++++++++ dj_db_conn_pool/backends/oracle/base.py | 2 - dj_db_conn_pool/core/mixins/core.py | 11 ------ dj_db_conn_pool/core/utils.py | 46 ----------------------- 5 files changed, 34 insertions(+), 63 deletions(-) create mode 100644 dj_db_conn_pool/backends/jdbc/utils.py delete mode 100644 dj_db_conn_pool/core/utils.py diff --git a/dj_db_conn_pool/backends/jdbc/__init__.py b/dj_db_conn_pool/backends/jdbc/__init__.py index 44fd598..1775f7c 100644 --- a/dj_db_conn_pool/backends/jdbc/__init__.py +++ b/dj_db_conn_pool/backends/jdbc/__init__.py @@ -4,6 +4,7 @@ import jpype import jpype.dbapi2 from dj_db_conn_pool.core.mixins import PersistentDatabaseWrapperMixin +from dj_db_conn_pool.backends.jdbc.utils import CursorWrapper import logging logger = logging.getLogger(__name__) @@ -14,10 +15,6 @@ class JDBCDatabaseWrapperMixin(PersistentDatabaseWrapperMixin): - _sql_param_style = 'qmark' - - _sql_converter = staticmethod(lambda sql: sql.replace('%s', '?')) - @property def jdbc_driver(self): raise NotImplementedError() @@ -33,6 +30,10 @@ def jdbc_url(self): **self.settings_dict ) + def create_cursor(self, name=None): + cursor = self.connection.cursor() + return CursorWrapper(cursor) + def get_connection_params(self): return self.settings_dict.get('OPTIONS', {}) diff --git a/dj_db_conn_pool/backends/jdbc/utils.py b/dj_db_conn_pool/backends/jdbc/utils.py new file mode 100644 index 0000000..0773657 --- /dev/null +++ b/dj_db_conn_pool/backends/jdbc/utils.py @@ -0,0 +1,29 @@ +import logging +import sqlparams + +logger = logging.getLogger(__name__) + + +class CursorWrapper: + def __init__(self, cursor): + self.cursor = cursor + + self._sql_params_converter = sqlparams.SQLParams('pyformat', 'qmark', True, True) + + self.statement = None + + def execute(self, query, parameters=None): + if isinstance(parameters, dict): + query, parameters = self._sql_params_converter.format(query, parameters) + else: + query = query.replace('%s', '?') + + self.statement = query + + return self.cursor.execute(query, parameters) + + def __getattr__(self, attr): + return getattr(self.cursor, attr) + + def __iter__(self): + return iter(self.cursor) diff --git a/dj_db_conn_pool/backends/oracle/base.py b/dj_db_conn_pool/backends/oracle/base.py index 6cabf33..4ae00e9 100644 --- a/dj_db_conn_pool/backends/oracle/base.py +++ b/dj_db_conn_pool/backends/oracle/base.py @@ -6,8 +6,6 @@ class DatabaseWrapper(PersistentDatabaseWrapperMixin, base.DatabaseWrapper): - _sql_param_style = 'numeric' - class SQLAlchemyDialect(OracleDialect): pass diff --git a/dj_db_conn_pool/core/mixins/core.py b/dj_db_conn_pool/core/mixins/core.py index 5ed2683..cd0a7f3 100644 --- a/dj_db_conn_pool/core/mixins/core.py +++ b/dj_db_conn_pool/core/mixins/core.py @@ -3,7 +3,6 @@ from sqlalchemy import pool from dj_db_conn_pool.compat import gettext_lazy as _ from dj_db_conn_pool.core import pool_container -from dj_db_conn_pool.core.utils import CursorWrapper from dj_db_conn_pool.core.mixins.creation import DatabaseCreationMixin @@ -43,16 +42,6 @@ def _set_autocommit(self, autocommit): def _get_dialect(self): return self.SQLAlchemyDialect(dbapi=self.Database) - _sql_param_style = 'format' - - _sql_converter = staticmethod(lambda sql: sql) - - def create_cursor(self, name=None): - # get cursor from django - cursor = super().create_cursor(name) - - return CursorWrapper(cursor, self._sql_param_style, self._sql_converter) - def _get_new_connection(self, conn_params): return super(PersistentDatabaseWrapperMixin, self).get_new_connection(conn_params) diff --git a/dj_db_conn_pool/core/utils.py b/dj_db_conn_pool/core/utils.py deleted file mode 100644 index 5de6dc9..0000000 --- a/dj_db_conn_pool/core/utils.py +++ /dev/null @@ -1,46 +0,0 @@ -import logging -import sqlparams - -logger = logging.getLogger(__name__) - - -class CursorWrapper: - def __init__(self, cursor, style, sql_converter): - self._cursor = cursor - - self._sql_params_converter = sqlparams.SQLParams('named', style, True, True) - - self._sql_converter = sql_converter - - self._statement = None - - def execute(self, query, parameters=None): - if isinstance(parameters, dict): - _query, _parameters = self._sql_params_converter.format(query, parameters) - else: - _query, _parameters = self._sql_converter(query), parameters - - if query != _query: - logger.debug( - 'SQL (%s), parameters(%s) has been converted to SQL(%s), parameters(%s)', - query, parameters, _query, _parameters - ) - - query, parameters = _query, _parameters - - try: - cursor = self._cursor.cursor - except AttributeError: - cursor = self._cursor - - self._statement = query - - return cursor.execute(query, parameters or []) - - def __getattr__(self, attr): - try: - return getattr(self._cursor, attr) - except AttributeError: - if attr == 'statement': - return self._statement - raise From 3e195d7bf9081bcf728d6caa4f080343c13a679c Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Mon, 5 Jun 2023 20:35:25 +0800 Subject: [PATCH 23/25] Switch to `pyproject.toml` for packaging --- MANIFEST.in | 4 --- pyproject.toml | 53 +++++++++++++++++++++++++++++++++++ requirements.txt | 8 ------ setup.py | 73 ------------------------------------------------ 4 files changed, 53 insertions(+), 85 deletions(-) delete mode 100644 MANIFEST.in create mode 100644 pyproject.toml delete mode 100644 requirements.txt delete mode 100644 setup.py diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 5495b6a..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,4 +0,0 @@ -include LICENSE -include README.md -include README_cn.md -include requirements.txt diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..76ede3b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,53 @@ +[project] +name = "django-db-connection-pool" +description = "Database connection pool component library for Django" +readme = "README.md" +license = { file = "LICENSE" } +authors = [{ name = "Altair Bow", email = "altair.bow@foxmail.com" }] +keywords = ["django", "db", "database", "persistent", "connection", "pool", "pooling"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Web Environment", + "Framework :: Django", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Topic :: Software Development :: Libraries :: Python Modules", +] +requires-python = ">=3.4" +dependencies = [ + "Django>=2.0", + "SQLAlchemy>=1.4.24", + "sqlparams>=4.0.0", +] +dynamic = ["version"] + +[project.urls] +homepage = "https://github.com/altairbow/django-db-connection-pool" +repository = "https://github.com/altairbow/django-db-connection-pool" + +[project.optional-dependencies] +all = [ + "Django>=2.0", + "JPype1>=1.3.0", + "SQLAlchemy>=1.4.24", + "cx-Oracle>=6.4.1", + "mysqlclient>=1.3.0", + "psycopg2>=2.8.6", + "pyodbc>=4.0.34", + "sqlparams>=3.0.0", +] + +jdbc = ["JPype1>=1.3.0"] +mysql = ["mysqlclient>=1.3.0"] +odbc = ["pyodbc>=4.0.34"] +oracle = ["cx-Oracle>=6.4.1"] +postgresql = ["psycopg2>=2.8.6"] + +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.dynamic] +version = { attr = "dj_db_conn_pool.__version__" } diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 05d4caa..0000000 --- a/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -Django>=2.0 -SQLAlchemy>=1.4.24 -JPype1>=1.3.0 -sqlparams>=3.0.0 -PyMySQL>=0.9.3 -pyodbc>=4.0.34 -cx-Oracle>=6.4.1 -psycopg2>=2.8.6 diff --git a/setup.py b/setup.py deleted file mode 100644 index 3d10ada..0000000 --- a/setup.py +++ /dev/null @@ -1,73 +0,0 @@ -# encoding: utf-8 - -import codecs -import shutil -import setuptools - -from dj_db_conn_pool import ( - __version__, - __author__, - __author_email__, - __description__ -) - - -def clean(name): - def decorator(func): - def smash_the_egg(): - shutil.rmtree(name + '.egg-info', ignore_errors=True) - return lambda: [fn() for fn in [smash_the_egg, func, smash_the_egg]] - return decorator - - -@clean('django_db_connection_pool') -def setup(): - setuptools.setup( - name='django-db-connection-pool', - license='MIT', - version=__version__, - description=__description__, - long_description=codecs.open('README.md', encoding='UTF-8').read(), - long_description_content_type='text/markdown', - author=__author__, - author_email=__author_email__, - url='https://github.com/altairbow/django-db-connection-pool', - download_url='https://pypi.python.org/pypi/django-db-connection-pool/', - packages=setuptools.find_packages(), - include_package_data=True, - install_requires=[ - 'Django>=2.0', - 'SQLAlchemy>=1.4.24', - 'sqlparams>=4.0.0', - ], - extras_require={ - 'all': [ - 'JPype1>=1.3.0', - 'sqlparams>=3.0.0', - 'PyMySQL>=0.9.3', - 'pyodbc>=4.0.34', - 'cx-Oracle>=6.4.1', - 'psycopg2>=2.8.6', - ], - 'jdbc': ['JPype1>=1.3.0'], - 'mysql': ['PyMySQL>=0.9.3'], - 'odbc': ['pyodbc>=4.0.34'], - 'oracle': ['cx-Oracle>=6.4.1'], - 'postgresql': ['psycopg2>=2.8.6'], - }, - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Environment :: Web Environment', - 'Framework :: Django', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 3', - 'Topic :: Software Development :: Libraries :: Python Modules', - ], - keywords=['django', 'db', 'database', 'persistent', 'connection', 'pool', 'pooling'], - ) - - -if __name__ == '__main__': - setup() From db5b4796771fd4550c684db5b1e2b66290ad0b55 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Mon, 5 Jun 2023 21:02:18 +0800 Subject: [PATCH 24/25] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 8d99ef7..ce1ea88 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -10,9 +10,15 @@ assignees: '' **Describe the bug** A clear and concise description of what the bug is. +**django's backend** +Switch to django's backend (`django.db.backends.xxx`), does the problem reproduce? + **Environment** - Python Version: - Django Version: **Traceback** Post traceback here. + +**Optional: SQL and parameters** +Post SQL and parameters here, Pay attention to hiding private information. From d9aaf1a6f06efcad67a7df8133e592d7d3740e49 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Sat, 10 Jun 2023 09:56:26 +0800 Subject: [PATCH 25/25] v1.2.4 --- dj_db_conn_pool/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dj_db_conn_pool/__init__.py b/dj_db_conn_pool/__init__.py index 7a4c349..7fc1f65 100644 --- a/dj_db_conn_pool/__init__.py +++ b/dj_db_conn_pool/__init__.py @@ -1,4 +1,4 @@ -__version__ = '1.2.3' +__version__ = '1.2.4' __author__ = 'Altair Bow' __author_email__ = 'altair.bow@foxmail.com' __description__ = 'Persistent database connection backends for Django'