From f00f09295c1da2446da9160856a635f9648eb9a2 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Mon, 16 Sep 2019 09:42:33 +0800 Subject: [PATCH 1/7] rename BaseDatabaseWrapper to PooledDatabaseWrapperMixin (more pythonic) --- dj_db_conn_pool/backends/__init__.py | 6 +++--- dj_db_conn_pool/backends/mysql/base.py | 4 ++-- dj_db_conn_pool/backends/oracle/base.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dj_db_conn_pool/backends/__init__.py b/dj_db_conn_pool/backends/__init__.py index 1d5b5be..929a236 100644 --- a/dj_db_conn_pool/backends/__init__.py +++ b/dj_db_conn_pool/backends/__init__.py @@ -20,7 +20,7 @@ } -class BaseDatabaseWrapper(object): +class PooledDatabaseWrapperMixin(object): def get_new_connection(self, conn_params): """ 覆盖 Django 的 get_new_connection 方法 @@ -61,7 +61,7 @@ def get_new_connection(self, conn_params): # 我们利用 django 的 get_new_connection 方法 # “告诉” QueuePool 怎么去创建一个新的数据库连接 # 在获取一个新的数据库连接时，SQLAlchemy 会调用这个 lambda - lambda: super(BaseDatabaseWrapper, self).get_new_connection(conn_params), + lambda: super(PooledDatabaseWrapperMixin, self).get_new_connection(conn_params), # 数据库“方言” # 在这里的作用主要作用是给 SQLAlchemy 利用 # 判断数据库连接的状态，以便 SQLAlchemy 维护该连接池 @@ -84,4 +84,4 @@ def get_new_connection(self, conn_params): def close(self, *args, **kwargs): logger.debug('释放 %s 数据库连接到池中', self.alias) - return super(BaseDatabaseWrapper, self).close(*args, **kwargs) + return super(PooledDatabaseWrapperMixin, self).close(*args, **kwargs) diff --git a/dj_db_conn_pool/backends/mysql/base.py b/dj_db_conn_pool/backends/mysql/base.py index c301f54..fdb2eef 100644 --- a/dj_db_conn_pool/backends/mysql/base.py +++ b/dj_db_conn_pool/backends/mysql/base.py @@ -2,10 +2,10 @@ from django.db.backends.mysql import base from sqlalchemy.dialects.mysql.base import MySQLDialect -from dj_db_conn_pool.backends import BaseDatabaseWrapper +from dj_db_conn_pool.backends import PooledDatabaseWrapperMixin -class DatabaseWrapper(BaseDatabaseWrapper, base.DatabaseWrapper): +class DatabaseWrapper(PooledDatabaseWrapperMixin, base.DatabaseWrapper): class SQLAlchemyDialect(MySQLDialect): def _extract_error_code(self, exception): return exception.args[0] diff --git a/dj_db_conn_pool/backends/oracle/base.py b/dj_db_conn_pool/backends/oracle/base.py index b1ccd40..fc5c306 100644 --- a/dj_db_conn_pool/backends/oracle/base.py +++ b/dj_db_conn_pool/backends/oracle/base.py @@ -3,10 +3,10 @@ from django.db.backends.oracle import base from cx_Oracle import DatabaseError from sqlalchemy.dialects.oracle.cx_oracle import OracleDialect -from dj_db_conn_pool.backends import BaseDatabaseWrapper +from dj_db_conn_pool.backends import PooledDatabaseWrapperMixin -class DatabaseWrapper(BaseDatabaseWrapper, base.DatabaseWrapper): +class DatabaseWrapper(PooledDatabaseWrapperMixin, base.DatabaseWrapper): class SQLAlchemyDialect(OracleDialect): def do_ping(self, dbapi_connection): try: From c2d6957b609d3e74bd47f1628a15236caea2a0c2 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Mon, 16 Sep 2019 10:23:21 +0800 Subject: [PATCH 2/7] update logging, to support l10n --- dj_db_conn_pool/backends/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dj_db_conn_pool/backends/__init__.py b/dj_db_conn_pool/backends/__init__.py index 929a236..d75a305 100644 --- a/dj_db_conn_pool/backends/__init__.py +++ b/dj_db_conn_pool/backends/__init__.py @@ -3,6 +3,7 @@ import threading from copy import deepcopy from sqlalchemy import pool +from django.utils.translation import ugettext_lazy as _ import logging logger = logging.getLogger(__name__) @@ -70,8 +71,7 @@ def get_new_connection(self, conn_params): pre_ping=True, echo=False, timeout=None, **pool_params ) - logger.debug('%s 数据库连接池已创建, 参数: %s', - self.alias, pool_params) + logger.debug(_("%s's pool has been created, parameter: %s"), self.alias, pool_params) # 数据库连接池已创建 # 放到 database_pool_dict，以便重用 @@ -79,9 +79,9 @@ def get_new_connection(self, conn_params): # 调用 SQLAlchemy 从连接池内取一个连接 conn = database_pool_dict[self.alias].connect() - logger.debug('从池中获取到 %s 数据库的连接', self.alias) + logger.debug(_("got %s's connection from its pool"), self.alias) return conn def close(self, *args, **kwargs): - logger.debug('释放 %s 数据库连接到池中', self.alias) + logger.debug(_("release %s's connection to its pool"), self.alias) return super(PooledDatabaseWrapperMixin, self).close(*args, **kwargs) From 81ccb35f7fa005b95bf7df41edfe72253116de00 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Mon, 16 Sep 2019 11:00:36 +0800 Subject: [PATCH 3/7] move dj_db_conn_pool.backends.PooledDatabaseWrapperMixin to dj_db_conn_pool.core.mixins --- dj_db_conn_pool/backends/__init__.py | 87 ------------------------- dj_db_conn_pool/backends/mysql/base.py | 2 +- dj_db_conn_pool/backends/oracle/base.py | 2 +- dj_db_conn_pool/core/__init__.py | 27 ++++++++ dj_db_conn_pool/core/mixins.py | 74 +++++++++++++++++++++ 5 files changed, 103 insertions(+), 89 deletions(-) create mode 100644 dj_db_conn_pool/core/__init__.py create mode 100644 dj_db_conn_pool/core/mixins.py diff --git a/dj_db_conn_pool/backends/__init__.py b/dj_db_conn_pool/backends/__init__.py index d75a305..e69de29 100644 --- a/dj_db_conn_pool/backends/__init__.py +++ b/dj_db_conn_pool/backends/__init__.py @@ -1,87 +0,0 @@ -# -*- coding: utf-8 -*- - -import threading -from copy import deepcopy -from sqlalchemy import pool -from django.utils.translation import ugettext_lazy as _ - -import logging -logger = logging.getLogger(__name__) - -# 保存各个数据库的池（QueuePool）实例 -database_pool_dict = {} - -# 修改 database_pool_dict 时需要请求的锁 -pool_modify_mutex = threading.Lock() - -# 池的默认参数 -pool_default_params = { - 'pool_size': 10, - 'max_overflow': 10 -} - - -class PooledDatabaseWrapperMixin(object): - def get_new_connection(self, conn_params): - """ - 覆盖 Django 的 get_new_connection 方法 - 在 Django 调用 此方法时，检查 database_pool_dict 中是否有 self.alias 的连接池 - 如果没有，则初始化 self.alias 的连接池，然后从池中取出一个连接 - 如果有，则直接从池中取出一个连接返回 - :return: - """ - with pool_modify_mutex: - # 获取锁后，判断当前数据库（self.alias）的池是否存在 - # 不存在，开始初始化 - if self.alias not in database_pool_dict: - # 复制一份默认参数给当前数据库 - pool_params = deepcopy(pool_default_params) - # 开始解析、组装当前数据库的连接配置 - pool_setting = { - # 把 POOL_OPTIONS 内的参数名转换为小写 - # 与 QueuePool 的参数对应 - key.lower(): value - # 取每个 POOL_OPTIONS 内参数 - for key, value in - # self.settings_dict 由 Django 提供，是 self.alias 的连接参数 - self.settings_dict.get('POOL_OPTIONS', {}).items() - # 此处限制 POOL_OPTIONS 内的参数： - # POOL_OPTIONS 内的参数名必须是大写的 - # 而且其小写形式必须在 pool_default_params 内 - if key == key.upper() and key.lower() in pool_default_params - } - - # 现在 pool_setting 已经组装完成 - # 覆盖 pool_params 的参数（以输入用户的配置） - pool_params.update(**pool_setting) - - # 现在参数已经具备 - # 创建 self.alias 的连接池实例 - alias_pool = pool.QueuePool( - # QueuePool 的 creator 参数 - # 我们利用 django 的 get_new_connection 方法 - # “告诉” QueuePool 怎么去创建一个新的数据库连接 - # 在获取一个新的数据库连接时，SQLAlchemy 会调用这个 lambda - lambda: super(PooledDatabaseWrapperMixin, self).get_new_connection(conn_params), - # 数据库“方言” - # 在这里的作用主要作用是给 SQLAlchemy 利用 - # 判断数据库连接的状态，以便 SQLAlchemy 维护该连接池 - dialect=self.SQLAlchemyDialect(dbapi=self.Database), - # 一些固定的参数 - pre_ping=True, echo=False, timeout=None, **pool_params - ) - - logger.debug(_("%s's pool has been created, parameter: %s"), self.alias, pool_params) - - # 数据库连接池已创建 - # 放到 database_pool_dict，以便重用 - database_pool_dict[self.alias] = alias_pool - - # 调用 SQLAlchemy 从连接池内取一个连接 - conn = database_pool_dict[self.alias].connect() - logger.debug(_("got %s's connection from its pool"), self.alias) - return conn - - def close(self, *args, **kwargs): - logger.debug(_("release %s's connection to its pool"), self.alias) - return super(PooledDatabaseWrapperMixin, self).close(*args, **kwargs) diff --git a/dj_db_conn_pool/backends/mysql/base.py b/dj_db_conn_pool/backends/mysql/base.py index fdb2eef..5986d94 100644 --- a/dj_db_conn_pool/backends/mysql/base.py +++ b/dj_db_conn_pool/backends/mysql/base.py @@ -2,7 +2,7 @@ from django.db.backends.mysql import base from sqlalchemy.dialects.mysql.base import MySQLDialect -from dj_db_conn_pool.backends import PooledDatabaseWrapperMixin +from dj_db_conn_pool.core.mixins import PooledDatabaseWrapperMixin class DatabaseWrapper(PooledDatabaseWrapperMixin, base.DatabaseWrapper): diff --git a/dj_db_conn_pool/backends/oracle/base.py b/dj_db_conn_pool/backends/oracle/base.py index fc5c306..f243917 100644 --- a/dj_db_conn_pool/backends/oracle/base.py +++ b/dj_db_conn_pool/backends/oracle/base.py @@ -3,7 +3,7 @@ from django.db.backends.oracle import base from cx_Oracle import DatabaseError from sqlalchemy.dialects.oracle.cx_oracle import OracleDialect -from dj_db_conn_pool.backends import PooledDatabaseWrapperMixin +from dj_db_conn_pool.core.mixins import PooledDatabaseWrapperMixin class DatabaseWrapper(PooledDatabaseWrapperMixin, base.DatabaseWrapper): diff --git a/dj_db_conn_pool/core/__init__.py b/dj_db_conn_pool/core/__init__.py new file mode 100644 index 0000000..f861864 --- /dev/null +++ b/dj_db_conn_pool/core/__init__.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +import threading + + +class PoolContainer(dict): + # 修改池容器时需要请求的锁 + lock = threading.Lock() + + # 单个数据库连接池的默认参数 + pool_default_params = { + 'pool_size': 10, + 'max_overflow': 10 + } + + def has(self, pool_name): + return pool_name in self + + def put(self, pool_name, pool): + self[pool_name] = pool + + def get(self, pool_name): + return self[pool_name] + + +# 池容器，保存各个数据库的池（QueuePool）实例 +pool_container = PoolContainer() diff --git a/dj_db_conn_pool/core/mixins.py b/dj_db_conn_pool/core/mixins.py new file mode 100644 index 0000000..b0d667e --- /dev/null +++ b/dj_db_conn_pool/core/mixins.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- + +from copy import deepcopy +from sqlalchemy import pool +from django.utils.translation import ugettext_lazy as _ +from dj_db_conn_pool.core import pool_container + + +import logging +logger = logging.getLogger(__name__) + + +class PooledDatabaseWrapperMixin(object): + def get_new_connection(self, conn_params): + """ + 覆盖 Django 的 get_new_connection 方法 + 在 Django 调用此方法时，检查 pool_container 中是否有 self.alias 的连接池 + 如果没有，则初始化 self.alias 的连接池，然后从池中取出一个连接 + 如果有，则直接从池中取出一个连接返回 + :return: + """ + with pool_container.lock: + # 获取锁后，判断当前数据库（self.alias）的池是否存在 + # 不存在，开始初始化 + if not pool_container.has(self.alias): + # 复制一份默认参数给当前数据库 + pool_params = deepcopy(pool_container.pool_default_params) + + # 开始解析、组装当前数据库的连接配置 + pool_setting = { + # 把 POOL_OPTIONS 内的参数名转换为小写 + # 与 QueuePool 的参数对应 + key.lower(): value + # 取每个 POOL_OPTIONS 内参数 + for key, value in + # self.settings_dict 由 Django 提供，是 self.alias 的连接参数 + self.settings_dict.get('POOL_OPTIONS', {}).items() + # 此处限制 POOL_OPTIONS 内的参数： + # POOL_OPTIONS 内的参数名必须是大写的 + # 而且其小写形式必须在 pool_default_params 内 + if key == key.upper() and key.lower() in pool_container.pool_default_params + } + + # 现在 pool_setting 已经组装完成 + # 覆盖 pool_params 的参数（以输入用户的配置） + pool_params.update(**pool_setting) + + # 现在参数已经具备 + # 创建 self.alias 的连接池实例 + alias_pool = pool.QueuePool( + # QueuePool 的 creator 参数 + # 在获取一个新的数据库连接时，SQLAlchemy 会调用这个匿名函数 + lambda: super(PooledDatabaseWrapperMixin, self).get_new_connection(conn_params), + # 数据库方言 + # 用于 SQLAlchemy 维护该连接池 + dialect=self.SQLAlchemyDialect(dbapi=self.Database), + # 一些固定的参数 + pre_ping=True, echo=False, timeout=None, **pool_params + ) + + logger.debug(_("%s's pool has been created, parameter: %s"), self.alias, pool_params) + + # 数据库连接池已创建 + # 放到 pool_container，以便重用 + pool_container.put(self.alias, alias_pool) + + # 调用 SQLAlchemy 从连接池内取一个连接 + conn = pool_container.get(self.alias).connect() + logger.debug(_("got %s's connection from its pool"), self.alias) + return conn + + def close(self, *args, **kwargs): + logger.debug(_("release %s's connection to its pool"), self.alias) + return super(PooledDatabaseWrapperMixin, self).close(*args, **kwargs) From 5064903df9d8f7548c76904eb1ee6fff75fa9907 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Mon, 16 Sep 2019 11:10:22 +0800 Subject: [PATCH 4/7] add dj_db_conn_pool.setup, for setting up default parameters --- dj_db_conn_pool/__init__.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/dj_db_conn_pool/__init__.py b/dj_db_conn_pool/__init__.py index 13dd35f..a28c95d 100644 --- a/dj_db_conn_pool/__init__.py +++ b/dj_db_conn_pool/__init__.py @@ -1,4 +1,13 @@ -__version__ = '1.0.1' -__author__ = 'Altair Bow', +__version__ = '1.0.2' +__author__ = 'Altair Bow' __author_email__ = 'altair.bow@foxmail.com' __description__ = 'Persistent database connection backends for Django' + + +def setup(pool_size=10, max_overflow=10): + from dj_db_conn_pool.core import pool_container + + pool_container.pool_default_params.update( + pool_size=pool_size, + max_overflow=max_overflow + ) From 900e1b63a7ef221a1c4acdf6d26e08e2b121c197 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Mon, 16 Sep 2019 11:18:09 +0800 Subject: [PATCH 5/7] update misc --- MANIFEST.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index f6ec2d8..5495b6a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ include LICENSE include README.md -include README_en.md +include README_cn.md include requirements.txt diff --git a/requirements.txt b/requirements.txt index ad79d2b..1623cf3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django +Django>=1.8 SQLAlchemy>=1.2.16 PyMySQL>=0.9.3 cx-Oracle>=6.4.1 From c2d9f58e2d734f127f80cb0c9f4f87b0f94c51d2 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Mon, 16 Sep 2019 11:32:29 +0800 Subject: [PATCH 6/7] update usage --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f85a74a..a22c936 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ MySQL & Oracle connection pool backends of Django, Be based on SQLAlchemy. } } ``` - * ##### pool options + * ##### pool options(optional) you can provide additional options to pass to SQLAlchemy's pool creation, key's name is `POOL_OPTIONS`: ``` DATABASES = { @@ -71,3 +71,9 @@ MySQL & Oracle connection pool backends of Django, Be based on SQLAlchemy. can be set to -1 to indicate no overflow limit; no limit will be placed on the total number of concurrent connections. Defaults to 10. + + or you can use dj_db_conn_pool.setup to change default arguments(for each pool's creation), before using database pool: + ```python + import dj_db_conn_pool + dj_db_conn_pool.setup(pool_size=100, max_overflow=50) + ``` From ad53ca2ce5b2ab7e526e2cc5da5b186b637107d4 Mon Sep 17 00:00:00 2001 From: Altair Bow Date: Mon, 16 Sep 2019 14:31:15 +0800 Subject: [PATCH 7/7] add exception class: PoolDoesNotExist --- dj_db_conn_pool/core/__init__.py | 7 ++++++- dj_db_conn_pool/core/exceptions.py | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 dj_db_conn_pool/core/exceptions.py diff --git a/dj_db_conn_pool/core/__init__.py b/dj_db_conn_pool/core/__init__.py index f861864..9acefc0 100644 --- a/dj_db_conn_pool/core/__init__.py +++ b/dj_db_conn_pool/core/__init__.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- import threading +from django.utils.translation import ugettext_lazy as _ +from dj_db_conn_pool.core.exceptions import PoolDoesNotExist class PoolContainer(dict): @@ -20,7 +22,10 @@ def put(self, pool_name, pool): self[pool_name] = pool def get(self, pool_name): - return self[pool_name] + try: + return self[pool_name] + except KeyError: + raise PoolDoesNotExist(_('No such pool: {pool_name}').format(pool_name=pool_name)) # 池容器，保存各个数据库的池（QueuePool）实例 diff --git a/dj_db_conn_pool/core/exceptions.py b/dj_db_conn_pool/core/exceptions.py new file mode 100644 index 0000000..6eabb8d --- /dev/null +++ b/dj_db_conn_pool/core/exceptions.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- + + +class PoolDoesNotExist(Exception): + pass