From 3ad3773bee561de4da6db83520e949736cd7c94a Mon Sep 17 00:00:00 2001 From: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Date: Wed, 7 Sep 2022 13:57:15 -0700 Subject: [PATCH 01/17] update remove debug Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> --- Makefile | 21 +++++++++++++++++++++ sdk/python/feast/repo_config.py | 1 + setup.py | 1 + 3 files changed, 23 insertions(+) diff --git a/Makefile b/Makefile index 990604da575..146b4373aae 100644 --- a/Makefile +++ b/Makefile @@ -231,6 +231,27 @@ test-python-universal-postgres-online: not test_snowflake" \ sdk/python/tests + test-python-universal-mysql-online: + PYTHONPATH='.' \ + FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.mysql_repo_configuration \ + PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.mysql \ + FEAST_USAGE=False \ + IS_TEST=True \ + python -m pytest -n 8 --integration \ + -k "not test_universal_cli and \ + not test_go_feature_server and \ + not test_feature_logging and \ + not test_reorder_columns and \ + not test_logged_features_validation and \ + not test_lambda_materialization_consistency and \ + not test_offline_write and \ + not test_push_features_to_offline_store and \ + not gcs_registry and \ + not s3_registry and \ + not test_universal_types and \ + not test_snowflake" \ + sdk/python/tests + test-python-universal-cassandra: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.cassandra_repo_configuration \ diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 47a5ae321d9..f2450020238 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -50,6 +50,7 @@ "postgres": "feast.infra.online_stores.contrib.postgres.PostgreSQLOnlineStore", "hbase": "feast.infra.online_stores.contrib.hbase_online_store.hbase.HbaseOnlineStore", "cassandra": "feast.infra.online_stores.contrib.cassandra_online_store.cassandra_online_store.CassandraOnlineStore", + "mysql": "feast.infra.online_stores.contrib.mysql.MySQLOnlineStore", } OFFLINE_STORE_CLASS_FOR_TYPE = { diff --git a/setup.py b/setup.py index 37ed471cfa6..c078f2950a9 100644 --- a/setup.py +++ b/setup.py @@ -113,6 +113,7 @@ MYSQL_REQUIRED = [ "mysqlclient", + "pymysql" ] HBASE_REQUIRED = [ From 7d98a4a85b532c99f6ce74e59ee77380cd0b4e4b Mon Sep 17 00:00:00 2001 From: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Date: Wed, 7 Sep 2022 13:58:20 -0700 Subject: [PATCH 02/17] add interface Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> --- .../infra/online_stores/contrib/mysql.py | 215 ++++++++++++++++++ .../contrib/mysql_repo_configuration.py | 10 + .../universal/online_store/mysql.py | 27 +++ 3 files changed, 252 insertions(+) create mode 100644 sdk/python/feast/infra/online_stores/contrib/mysql.py create mode 100644 sdk/python/feast/infra/online_stores/contrib/mysql_repo_configuration.py create mode 100644 sdk/python/tests/integration/feature_repos/universal/online_store/mysql.py diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql.py b/sdk/python/feast/infra/online_stores/contrib/mysql.py new file mode 100644 index 00000000000..990c9c2494f --- /dev/null +++ b/sdk/python/feast/infra/online_stores/contrib/mysql.py @@ -0,0 +1,215 @@ +from __future__ import absolute_import + +from datetime import datetime +from typing import ( + Any, + Callable, + Dict, + List, + Optional, + Sequence, + Tuple, +) + +import pymysql +import pytz +from feast import ( + Entity, + FeatureView, + RepoConfig, +) +from feast.infra.key_encoding_utils import serialize_entity_key +from feast.infra.online_stores.online_store import OnlineStore +from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.repo_config import FeastConfigBaseModel +from pydantic import StrictStr +from pymysql.connections import Connection + + +class MySQLOnlineStoreConfig(FeastConfigBaseModel): + """ + Configuration for the MySQL online store. + NOTE: The class *must* end with the `OnlineStoreConfig` suffix. + """ + + type = "mysql" + + host: Optional[StrictStr] = None + user: Optional[StrictStr] = None + password: Optional[StrictStr] = None + database: Optional[StrictStr] = None + + +class MySQLOnlineStore(OnlineStore): + """ + An online store implementation that uses MySQL. + NOTE: The class *must* end with the `OnlineStore` suffix. + """ + + _conn: Connection + + def _get_conn(self, config: RepoConfig) -> Connection: + + online_store_config = config.online_store + assert isinstance(online_store_config, MySQLOnlineStoreConfig) + + if not self._conn: + self._conn = pymysql.connect( + host=online_store_config.host or "127.0.0.1", + user=online_store_config.user or "root", + password=online_store_config.password, + database=online_store_config.database or "feast", + autocommit=True, + ) + return self._conn + + def online_write_batch( + self, + config: RepoConfig, + table: FeatureView, + data: List[Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]]], + progress: Optional[Callable[[int], Any]], + ) -> None: + + conn = self._get_conn(config) + cur = conn.cursor() + + project = config.project + + for entity_key, values, timestamp, created_ts in data: + entity_key_bin = serialize_entity_key(entity_key).hex() + timestamp = _to_naive_utc(timestamp) + if created_ts is not None: + created_ts = _to_naive_utc(created_ts) + + for feature_name, val in values.items(): + self.write_to_table(created_ts, cur, entity_key_bin, feature_name, project, table, timestamp, val) + self._conn.commit() + if progress: + progress(1) + + @staticmethod + def write_to_table(created_ts, cur, entity_key_bin, feature_name, project, table, timestamp, val) -> None: + cur.execute( + f""" + UPDATE {_table_id(project, table)} + SET value = %s, event_ts = %s, created_ts = %s + WHERE (entity_key = %s AND feature_name = %s) + """, + ( + # SET + val.SerializeToString(), + timestamp, + created_ts, + # WHERE + entity_key_bin, + feature_name, + ), + ) + cur.execute( + f"""INSERT INTO {_table_id(project, table)} + (entity_key, feature_name, value, event_ts, created_ts) + VALUES (%s, %s, %s, %s, %s)""", + ( + entity_key_bin, + feature_name, + val.SerializeToString(), + timestamp, + created_ts, + ), + ) + + def online_read( + self, + config: RepoConfig, + table: FeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + conn = self._get_conn(config) + cur = conn.cursor() + + result: List[Tuple[Optional[datetime], Optional[Dict[str, Any]]]] = [] + + project = config.project + for entity_key in entity_keys: + entity_key_bin = serialize_entity_key(entity_key).hex() + + cur.execute( + f"SELECT feature_name, value, event_ts FROM {_table_id(project, table)} WHERE entity_key = %s", + (entity_key_bin,), + ) + + res = {} + res_ts: Optional[datetime] = None + records = cur.fetchall() + if records: + for feature_name, val_bin, ts in records: + val = ValueProto() + val.ParseFromString(val_bin) + res[feature_name] = val + res_ts = datetime.strptime(ts, '%Y-%m-%d %H:%M:%S') + + if not res: + result.append((None, None)) + else: + result.append((res_ts, res)) + return result + + def update( + self, + config: RepoConfig, + tables_to_delete: Sequence[FeatureView], + tables_to_keep: Sequence[FeatureView], + entities_to_delete: Sequence[Entity], + entities_to_keep: Sequence[Entity], + partial: bool, + ) -> None: + conn = self._get_conn(config) + cur = conn.cursor() + project = config.project + + # We don't create any special state for the entities in this implementation. + for table in tables_to_keep: + cur.execute( + f"""CREATE TABLE IF NOT EXISTS {_table_id(project, table)} (entity_key VARCHAR(512), + feature_name VARCHAR(256), + value BLOB, + event_ts, + created_ts timestamp NULL DEFAULT NULL, + PRIMARY KEY(entity_key, feature_name))""" + ) + + cur.execute( + f"ALTER TABLE {_table_id(project, table)} ADD INDEX {_table_id(project, table)}_ek (entity_key);" + ) + + for table in tables_to_delete: + cur.execute(f"DROP INDEX {_table_id(project, table)}_ek ON {_table_id(project, table)};") + cur.execute(f"DROP TABLE IF EXISTS {_table_id(project, table)}") + + def teardown( + self, + config: RepoConfig, + tables: Sequence[FeatureView], + entities: Sequence[Entity], + ) -> None: + conn = self._get_conn(config) + cur = conn.cursor() + project = config.project + + for table in tables: + cur.execute(f"DROP INDEX {_table_id(project, table)}_ek ON {_table_id(project, table)};") + cur.execute(f"DROP TABLE IF EXISTS {_table_id(project, table)}") + + +def _table_id(project: str, table: FeatureView) -> str: + return f"{project}_{table.name}" + + +def _to_naive_utc(ts: datetime) -> datetime: + if ts.tzinfo is None: + return ts + else: + return ts.astimezone(pytz.utc).replace(tzinfo=None) diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql_repo_configuration.py b/sdk/python/feast/infra/online_stores/contrib/mysql_repo_configuration.py new file mode 100644 index 00000000000..3e92ead2d0b --- /dev/null +++ b/sdk/python/feast/infra/online_stores/contrib/mysql_repo_configuration.py @@ -0,0 +1,10 @@ +from tests.integration.feature_repos.integration_test_repo_config import ( + IntegrationTestRepoConfig, +) +from tests.integration.feature_repos.universal.online_store.mysql import ( + MySQLOnlineStoreCreator, +) + +FULL_REPO_CONFIGS = [ + IntegrationTestRepoConfig(online_store_creator=MySQLOnlineStoreCreator), +] diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/mysql.py b/sdk/python/tests/integration/feature_repos/universal/online_store/mysql.py new file mode 100644 index 00000000000..060367f441f --- /dev/null +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/mysql.py @@ -0,0 +1,27 @@ +from typing import Dict + +from testcontainers.mysql import MySqlContainer +from testcontainers.core.waiting_utils import wait_for_logs + +from tests.integration.feature_repos.universal.online_store_creator import ( + OnlineStoreCreator, +) + + +class MySQLOnlineStoreCreator(OnlineStoreCreator): + def __init__(self, project_name: str, **kwargs): + super().__init__(project_name) + self.container = MySqlContainer('mysql:latest', platform='linux/amd64').with_exposed_ports("3306") \ + .with_env("MYSQL_USER", "root") \ + .with_env("MYSQL_DATABASE", "feast") + + def create_online_store(self) -> Dict[str, str]: + self.container.start() + log_string_to_wait_for = "Ready to accept connections" + wait_for_logs( + container=self.container, predicate=log_string_to_wait_for, timeout=10 + ) + return {"type": "mysql", "user": "root", "password": "test", "database": "feast"} + + def teardown(self): + self.container.stop() From cc13d24af8b17a77d9b229815113aef9b7ff5cfd Mon Sep 17 00:00:00 2001 From: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Date: Wed, 7 Sep 2022 14:14:39 -0700 Subject: [PATCH 03/17] fix lint Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c078f2950a9..ea166472e1e 100644 --- a/setup.py +++ b/setup.py @@ -113,7 +113,8 @@ MYSQL_REQUIRED = [ "mysqlclient", - "pymysql" + "pymysql", + "types-PyMySQL" ] HBASE_REQUIRED = [ From 0527d5dc8a087b95e3bedb37f94472bbd95ec3b2 Mon Sep 17 00:00:00 2001 From: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Date: Wed, 7 Sep 2022 18:25:58 -0700 Subject: [PATCH 04/17] fix lint Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> --- .../infra/online_stores/contrib/mysql.py | 35 +++++++------------ .../universal/online_store/mysql.py | 17 +++++---- 2 files changed, 23 insertions(+), 29 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql.py b/sdk/python/feast/infra/online_stores/contrib/mysql.py index 990c9c2494f..bf52f5310d6 100644 --- a/sdk/python/feast/infra/online_stores/contrib/mysql.py +++ b/sdk/python/feast/infra/online_stores/contrib/mysql.py @@ -1,30 +1,19 @@ from __future__ import absolute_import from datetime import datetime -from typing import ( - Any, - Callable, - Dict, - List, - Optional, - Sequence, - Tuple, -) +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple import pymysql import pytz -from feast import ( - Entity, - FeatureView, - RepoConfig, -) +from pydantic import StrictStr +from pymysql.connections import Connection + +from feast import Entity, FeatureView, RepoConfig from feast.infra.key_encoding_utils import serialize_entity_key from feast.infra.online_stores.online_store import OnlineStore from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel -from pydantic import StrictStr -from pymysql.connections import Connection class MySQLOnlineStoreConfig(FeastConfigBaseModel): @@ -39,6 +28,7 @@ class MySQLOnlineStoreConfig(FeastConfigBaseModel): user: Optional[StrictStr] = None password: Optional[StrictStr] = None database: Optional[StrictStr] = None + port: Optional[int] = None class MySQLOnlineStore(OnlineStore): @@ -47,7 +37,7 @@ class MySQLOnlineStore(OnlineStore): NOTE: The class *must* end with the `OnlineStore` suffix. """ - _conn: Connection + _conn: Optional[Connection] = None def _get_conn(self, config: RepoConfig) -> Connection: @@ -57,9 +47,10 @@ def _get_conn(self, config: RepoConfig) -> Connection: if not self._conn: self._conn = pymysql.connect( host=online_store_config.host or "127.0.0.1", - user=online_store_config.user or "root", - password=online_store_config.password, + user=online_store_config.user or "test", + password=online_store_config.password or "test", database=online_store_config.database or "feast", + port=online_store_config.port or 3306, autocommit=True, ) return self._conn @@ -85,7 +76,7 @@ def online_write_batch( for feature_name, val in values.items(): self.write_to_table(created_ts, cur, entity_key_bin, feature_name, project, table, timestamp, val) - self._conn.commit() + conn.commit() if progress: progress(1) @@ -149,7 +140,7 @@ def online_read( val = ValueProto() val.ParseFromString(val_bin) res[feature_name] = val - res_ts = datetime.strptime(ts, '%Y-%m-%d %H:%M:%S') + res_ts = ts if not res: result.append((None, None)) @@ -176,7 +167,7 @@ def update( f"""CREATE TABLE IF NOT EXISTS {_table_id(project, table)} (entity_key VARCHAR(512), feature_name VARCHAR(256), value BLOB, - event_ts, + event_ts timestamp NULL DEFAULT NULL, created_ts timestamp NULL DEFAULT NULL, PRIMARY KEY(entity_key, feature_name))""" ) diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/mysql.py b/sdk/python/tests/integration/feature_repos/universal/online_store/mysql.py index 060367f441f..6cba49b1581 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store/mysql.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/mysql.py @@ -1,7 +1,7 @@ from typing import Dict -from testcontainers.mysql import MySqlContainer from testcontainers.core.waiting_utils import wait_for_logs +from testcontainers.mysql import MySqlContainer from tests.integration.feature_repos.universal.online_store_creator import ( OnlineStoreCreator, @@ -11,17 +11,20 @@ class MySQLOnlineStoreCreator(OnlineStoreCreator): def __init__(self, project_name: str, **kwargs): super().__init__(project_name) - self.container = MySqlContainer('mysql:latest', platform='linux/amd64').with_exposed_ports("3306") \ + self.container = MySqlContainer('mysql:latest', platform='linux/amd64') \ + .with_exposed_ports(3306) \ .with_env("MYSQL_USER", "root") \ - .with_env("MYSQL_DATABASE", "feast") + .with_env("MYSQL_PASSWORD", "test") \ + .with_env("MYSQL_DATABASE", "test") def create_online_store(self) -> Dict[str, str]: self.container.start() log_string_to_wait_for = "Ready to accept connections" - wait_for_logs( - container=self.container, predicate=log_string_to_wait_for, timeout=10 - ) - return {"type": "mysql", "user": "root", "password": "test", "database": "feast"} + # wait_for_logs( + # container=self.container, predicate=log_string_to_wait_for, timeout=10 + # ) + exposed_port = self.container.get_exposed_port(3306) + return {"type": "mysql", "user": "root", "password": "test", "database": "test", "port": exposed_port} def teardown(self): self.container.stop() From 312af8b66befc237f9fc178b3cb982dbcc2743fd Mon Sep 17 00:00:00 2001 From: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Date: Wed, 7 Sep 2022 23:13:39 -0700 Subject: [PATCH 05/17] update sql Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> --- .../infra/online_stores/contrib/mysql.py | 31 ++++++++----------- .../universal/online_store/mysql.py | 5 --- 2 files changed, 13 insertions(+), 23 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql.py b/sdk/python/feast/infra/online_stores/contrib/mysql.py index bf52f5310d6..9e344c80d91 100644 --- a/sdk/python/feast/infra/online_stores/contrib/mysql.py +++ b/sdk/python/feast/infra/online_stores/contrib/mysql.py @@ -84,30 +84,25 @@ def online_write_batch( def write_to_table(created_ts, cur, entity_key_bin, feature_name, project, table, timestamp, val) -> None: cur.execute( f""" - UPDATE {_table_id(project, table)} - SET value = %s, event_ts = %s, created_ts = %s - WHERE (entity_key = %s AND feature_name = %s) - """, - ( - # SET - val.SerializeToString(), - timestamp, - created_ts, - # WHERE - entity_key_bin, - feature_name, - ), - ) - cur.execute( - f"""INSERT INTO {_table_id(project, table)} - (entity_key, feature_name, value, event_ts, created_ts) - VALUES (%s, %s, %s, %s, %s)""", + INSERT INTO {_table_id(project, table)} + (entity_key, feature_name, value, event_ts, created_ts) + values (%s, %s, %s, %s, %s) + ON DUPLICATE KEY UPDATE + value = %s, + event_ts = %s, + created_ts = %s; + """, ( + # Insert entity_key_bin, feature_name, val.SerializeToString(), timestamp, created_ts, + # Update on duplicate key + val.SerializeToString(), + timestamp, + created_ts ), ) diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/mysql.py b/sdk/python/tests/integration/feature_repos/universal/online_store/mysql.py index 6cba49b1581..84afd850f58 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store/mysql.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/mysql.py @@ -1,6 +1,5 @@ from typing import Dict -from testcontainers.core.waiting_utils import wait_for_logs from testcontainers.mysql import MySqlContainer from tests.integration.feature_repos.universal.online_store_creator import ( @@ -19,10 +18,6 @@ def __init__(self, project_name: str, **kwargs): def create_online_store(self) -> Dict[str, str]: self.container.start() - log_string_to_wait_for = "Ready to accept connections" - # wait_for_logs( - # container=self.container, predicate=log_string_to_wait_for, timeout=10 - # ) exposed_port = self.container.get_exposed_port(3306) return {"type": "mysql", "user": "root", "password": "test", "database": "test", "port": exposed_port} From 7bd0662380dd5bf2a6395faa1fc17da43487860b Mon Sep 17 00:00:00 2001 From: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Date: Thu, 8 Sep 2022 16:36:33 -0700 Subject: [PATCH 06/17] update ci reqs Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> --- .../requirements/py3.10-ci-requirements.txt | 96 +++++++++-------- .../requirements/py3.8-ci-requirements.txt | 96 +++++++++-------- .../requirements/py3.9-ci-requirements.txt | 102 +++++++++--------- 3 files changed, 156 insertions(+), 138 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 9d10b2c3132..85d7c9ca14f 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -4,6 +4,8 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.10-ci-requirements.txt # +--index-url https://registry.affirm-stage.com/artifactory/api/pypi/pypi/simple/ + absl-py==1.2.0 # via tensorflow-metadata adal==1.2.7 @@ -56,7 +58,7 @@ attrs==22.1.0 # pytest avro==1.10.0 # via feast (setup.py) -azure-core==1.25.0 +azure-core==1.25.1 # via # adlfs # azure-identity @@ -76,7 +78,7 @@ babel==2.10.3 # via sphinx backcall==0.2.0 # via ipython -black==22.6.0 +black==22.8.0 # via feast (setup.py) boto3==1.20.23 # via @@ -131,7 +133,7 @@ click==8.1.3 # moreorless # pip-tools # uvicorn -cloudpickle==2.1.0 +cloudpickle==2.2.0 # via dask colorama==0.4.5 # via @@ -169,7 +171,7 @@ dill==0.3.5.1 # via # feast (setup.py) # multiprocess -distlib==0.3.5 +distlib==0.3.6 # via virtualenv docker==6.0.0 # via @@ -183,9 +185,9 @@ entrypoints==0.4 # via altair execnet==1.9.0 # via pytest-xdist -executing==0.10.0 +executing==1.0.0 # via stack-data -fastapi==0.79.1 +fastapi==0.82.0 # via feast (setup.py) fastavro==1.6.0 # via @@ -195,7 +197,7 @@ fastjsonschema==2.16.1 # via nbformat filelock==3.8.0 # via virtualenv -firebase-admin==5.2.0 +firebase-admin==5.3.0 # via feast (setup.py) fissix==21.11.13 # via bowler @@ -215,7 +217,7 @@ gcsfs==2022.1.0 # via feast (setup.py) geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.8.2 +google-api-core[grpc]==2.10.0 # via # feast (setup.py) # firebase-admin @@ -226,9 +228,9 @@ google-api-core[grpc]==2.8.2 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.57.0 +google-api-python-client==2.60.0 # via firebase-admin -google-auth==2.10.0 +google-auth==2.11.0 # via # gcsfs # google-api-core @@ -244,7 +246,7 @@ google-auth-oauthlib==0.5.2 # via gcsfs google-cloud-bigquery[pandas]==3.3.2 # via feast (setup.py) -google-cloud-bigquery-storage==2.14.2 +google-cloud-bigquery-storage==2.15.0 # via # feast (setup.py) # google-cloud-bigquery @@ -263,7 +265,7 @@ google-cloud-storage==2.5.0 # feast (setup.py) # firebase-admin # gcsfs -google-crc32c==1.3.0 +google-crc32c==1.5.0 # via google-resumable-media google-resumable-media==2.3.3 # via @@ -277,9 +279,9 @@ googleapis-common-protos==1.56.4 # tensorflow-metadata great-expectations==0.14.13 # via feast (setup.py) -greenlet==1.1.2 +greenlet==1.1.3 # via sqlalchemy -grpcio==1.47.0 +grpcio==1.48.1 # via # feast (setup.py) # google-api-core @@ -288,13 +290,13 @@ grpcio==1.47.0 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-reflection==1.47.0 +grpcio-reflection==1.48.1 # via feast (setup.py) -grpcio-status==1.47.0 +grpcio-status==1.48.1 # via google-api-core -grpcio-testing==1.47.0 +grpcio-testing==1.48.1 # via feast (setup.py) -grpcio-tools==1.47.0 +grpcio-tools==1.48.1 # via feast (setup.py) h11==0.13.0 # via uvicorn @@ -308,7 +310,7 @@ httplib2==0.20.4 # google-auth-httplib2 httptools==0.4.0 # via uvicorn -identify==2.5.3 +identify==2.5.5 # via pre-commit idna==3.3 # via @@ -322,7 +324,7 @@ importlib-metadata==4.12.0 # via great-expectations iniconfig==1.1.1 # via pytest -ipython==8.4.0 +ipython==8.5.0 # via great-expectations isodate==0.6.1 # via msrest @@ -345,7 +347,7 @@ jsonpatch==1.32 # via great-expectations jsonpointer==2.3 # via jsonpatch -jsonschema==4.13.0 +jsonschema==4.15.0 # via # altair # feast (setup.py) @@ -375,7 +377,7 @@ mock==2.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -moto==3.1.18 +moto==4.0.2 # via feast (setup.py) msal==1.18.0 # via @@ -439,7 +441,7 @@ packaging==21.3 # pytest # redis # sphinx -pandas==1.4.3 +pandas==1.4.4 # via # altair # db-dtypes @@ -454,7 +456,7 @@ parso==0.8.3 # via jedi partd==1.3.0 # via dask -pathspec==0.9.0 +pathspec==0.10.1 # via black pbr==5.10.0 # via mock @@ -478,9 +480,9 @@ portalocker==2.5.1 # via msal-extensions pre-commit==2.20.0 # via feast (setup.py) -prompt-toolkit==3.0.30 +prompt-toolkit==3.0.31 # via ipython -proto-plus==1.22.0 +proto-plus==1.22.1 # via # feast (setup.py) # google-cloud-bigquery @@ -539,7 +541,7 @@ pycparser==2.21 # via cffi pycryptodomex==3.15.0 # via snowflake-connector-python -pydantic==1.9.2 +pydantic==1.10.2 # via # fastapi # feast (setup.py) @@ -557,6 +559,8 @@ pyjwt[crypto]==2.4.0 # snowflake-connector-python pymssql==2.2.5 # via feast (setup.py) +pymysql==1.0.2 + # via feast (setup.py) pyodbc==4.0.34 # via feast (setup.py) pyopenssl==22.0.0 @@ -570,7 +574,7 @@ pyrsistent==0.18.1 # via jsonschema pyspark==3.3.0 # via feast (setup.py) -pytest==7.1.2 +pytest==7.1.3 # via # feast (setup.py) # pytest-benchmark @@ -606,7 +610,7 @@ python-dateutil==2.8.2 # kubernetes # moto # pandas -python-dotenv==0.20.0 +python-dotenv==0.21.0 # via uvicorn pytz==2022.2.1 # via @@ -664,7 +668,7 @@ s3fs==2022.1.0 # via feast (setup.py) s3transfer==0.5.2 # via boto3 -scipy==1.9.0 +scipy==1.9.1 # via great-expectations six==1.16.0 # via @@ -682,7 +686,7 @@ six==1.16.0 # msrestazure # pandavro # python-dateutil -sniffio==1.2.0 +sniffio==1.3.0 # via anyio snowballstemmer==2.2.0 # via sphinx @@ -706,11 +710,11 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy[mypy]==1.4.40 +sqlalchemy[mypy]==1.4.41 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a25 +sqlalchemy2-stubs==0.0.2a27 # via sqlalchemy -stack-data==0.4.0 +stack-data==0.5.0 # via ipython starlette==0.19.1 # via fastapi @@ -718,7 +722,7 @@ tabulate==0.8.10 # via feast (setup.py) tenacity==8.0.1 # via feast (setup.py) -tensorflow-metadata==1.9.0 +tensorflow-metadata==1.10.0 # via feast (setup.py) termcolor==1.1.0 # via great-expectations @@ -743,7 +747,7 @@ toolz==0.12.0 # altair # dask # partd -tqdm==4.64.0 +tqdm==4.64.1 # via # feast (setup.py) # great-expectations @@ -757,25 +761,27 @@ trino==0.315.0 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -types-protobuf==3.19.22 +types-protobuf==3.20.2 # via # feast (setup.py) # mypy-protobuf +types-pymysql==1.0.19 + # via feast (setup.py) types-python-dateutil==2.8.19 # via feast (setup.py) types-pytz==2022.2.1.0 # via feast (setup.py) types-pyyaml==6.0.11 # via feast (setup.py) -types-redis==4.3.14 +types-redis==4.3.20 # via feast (setup.py) -types-requests==2.28.9 +types-requests==2.28.10 # via feast (setup.py) -types-setuptools==65.1.0 +types-setuptools==65.3.0 # via feast (setup.py) types-tabulate==0.8.11 # via feast (setup.py) -types-urllib3==1.26.23 +types-urllib3==1.26.24 # via types-requests typing-extensions==4.3.0 # via @@ -790,7 +796,7 @@ tzlocal==4.2 # via great-expectations uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.11 +urllib3==1.26.12 # via # botocore # docker @@ -800,11 +806,11 @@ urllib3==1.26.11 # minio # requests # responses -uvicorn[standard]==0.18.2 +uvicorn[standard]==0.18.3 # via feast (setup.py) uvloop==0.16.0 # via uvicorn -virtualenv==20.16.3 +virtualenv==20.16.5 # via pre-commit volatile==2.1.0 # via bowler @@ -812,7 +818,7 @@ watchfiles==0.16.1 # via uvicorn wcwidth==0.2.5 # via prompt-toolkit -websocket-client==1.3.3 +websocket-client==1.4.1 # via # docker # kubernetes diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index 93011cfdcf4..9fa441860c0 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -4,6 +4,8 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.8-ci-requirements.txt # +--index-url https://registry.affirm-stage.com/artifactory/api/pypi/pypi/simple/ + absl-py==1.2.0 # via tensorflow-metadata adal==1.2.7 @@ -56,7 +58,7 @@ attrs==22.1.0 # pytest avro==1.10.0 # via feast (setup.py) -azure-core==1.25.0 +azure-core==1.25.1 # via # adlfs # azure-identity @@ -80,7 +82,7 @@ backports-zoneinfo==0.2.1 # via # pytz-deprecation-shim # tzlocal -black==22.6.0 +black==22.8.0 # via feast (setup.py) boto3==1.20.23 # via @@ -135,7 +137,7 @@ click==8.1.3 # moreorless # pip-tools # uvicorn -cloudpickle==2.1.0 +cloudpickle==2.2.0 # via dask colorama==0.4.5 # via @@ -173,7 +175,7 @@ dill==0.3.5.1 # via # feast (setup.py) # multiprocess -distlib==0.3.5 +distlib==0.3.6 # via virtualenv docker==6.0.0 # via @@ -187,9 +189,9 @@ entrypoints==0.4 # via altair execnet==1.9.0 # via pytest-xdist -executing==0.10.0 +executing==1.0.0 # via stack-data -fastapi==0.79.1 +fastapi==0.82.0 # via feast (setup.py) fastavro==1.6.0 # via @@ -199,7 +201,7 @@ fastjsonschema==2.16.1 # via nbformat filelock==3.8.0 # via virtualenv -firebase-admin==5.2.0 +firebase-admin==5.3.0 # via feast (setup.py) fissix==21.11.13 # via bowler @@ -219,7 +221,7 @@ gcsfs==2022.1.0 # via feast (setup.py) geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.8.2 +google-api-core[grpc]==2.10.0 # via # feast (setup.py) # firebase-admin @@ -230,9 +232,9 @@ google-api-core[grpc]==2.8.2 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.57.0 +google-api-python-client==2.60.0 # via firebase-admin -google-auth==2.10.0 +google-auth==2.11.0 # via # gcsfs # google-api-core @@ -248,7 +250,7 @@ google-auth-oauthlib==0.5.2 # via gcsfs google-cloud-bigquery[pandas]==3.3.2 # via feast (setup.py) -google-cloud-bigquery-storage==2.14.2 +google-cloud-bigquery-storage==2.15.0 # via # feast (setup.py) # google-cloud-bigquery @@ -267,7 +269,7 @@ google-cloud-storage==2.5.0 # feast (setup.py) # firebase-admin # gcsfs -google-crc32c==1.3.0 +google-crc32c==1.5.0 # via google-resumable-media google-resumable-media==2.3.3 # via @@ -281,9 +283,9 @@ googleapis-common-protos==1.56.4 # tensorflow-metadata great-expectations==0.14.13 # via feast (setup.py) -greenlet==1.1.2 +greenlet==1.1.3 # via sqlalchemy -grpcio==1.47.0 +grpcio==1.48.1 # via # feast (setup.py) # google-api-core @@ -292,13 +294,13 @@ grpcio==1.47.0 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-reflection==1.47.0 +grpcio-reflection==1.48.1 # via feast (setup.py) -grpcio-status==1.47.0 +grpcio-status==1.48.1 # via google-api-core -grpcio-testing==1.47.0 +grpcio-testing==1.48.1 # via feast (setup.py) -grpcio-tools==1.47.0 +grpcio-tools==1.48.1 # via feast (setup.py) h11==0.13.0 # via uvicorn @@ -312,7 +314,7 @@ httplib2==0.20.4 # google-auth-httplib2 httptools==0.4.0 # via uvicorn -identify==2.5.3 +identify==2.5.5 # via pre-commit idna==3.3 # via @@ -328,7 +330,7 @@ importlib-resources==5.9.0 # via jsonschema iniconfig==1.1.1 # via pytest -ipython==8.4.0 +ipython==8.5.0 # via great-expectations isodate==0.6.1 # via msrest @@ -351,7 +353,7 @@ jsonpatch==1.32 # via great-expectations jsonpointer==2.3 # via jsonpatch -jsonschema==4.13.0 +jsonschema==4.15.0 # via # altair # feast (setup.py) @@ -381,7 +383,7 @@ mock==2.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -moto==3.1.18 +moto==4.0.2 # via feast (setup.py) msal==1.18.0 # via @@ -445,7 +447,7 @@ packaging==21.3 # pytest # redis # sphinx -pandas==1.4.3 +pandas==1.4.4 # via # altair # db-dtypes @@ -460,7 +462,7 @@ parso==0.8.3 # via jedi partd==1.3.0 # via dask -pathspec==0.9.0 +pathspec==0.10.1 # via black pbr==5.10.0 # via mock @@ -486,9 +488,9 @@ portalocker==2.5.1 # via msal-extensions pre-commit==2.20.0 # via feast (setup.py) -prompt-toolkit==3.0.30 +prompt-toolkit==3.0.31 # via ipython -proto-plus==1.22.0 +proto-plus==1.22.1 # via # feast (setup.py) # google-cloud-bigquery @@ -547,7 +549,7 @@ pycparser==2.21 # via cffi pycryptodomex==3.15.0 # via snowflake-connector-python -pydantic==1.9.2 +pydantic==1.10.2 # via # fastapi # feast (setup.py) @@ -565,6 +567,8 @@ pyjwt[crypto]==2.4.0 # snowflake-connector-python pymssql==2.2.5 # via feast (setup.py) +pymysql==1.0.2 + # via feast (setup.py) pyodbc==4.0.34 # via feast (setup.py) pyopenssl==22.0.0 @@ -578,7 +582,7 @@ pyrsistent==0.18.1 # via jsonschema pyspark==3.3.0 # via feast (setup.py) -pytest==7.1.2 +pytest==7.1.3 # via # feast (setup.py) # pytest-benchmark @@ -614,7 +618,7 @@ python-dateutil==2.8.2 # kubernetes # moto # pandas -python-dotenv==0.20.0 +python-dotenv==0.21.0 # via uvicorn pytz==2022.2.1 # via @@ -674,7 +678,7 @@ s3fs==2022.1.0 # via feast (setup.py) s3transfer==0.5.2 # via boto3 -scipy==1.9.0 +scipy==1.9.1 # via great-expectations six==1.16.0 # via @@ -692,7 +696,7 @@ six==1.16.0 # msrestazure # pandavro # python-dateutil -sniffio==1.2.0 +sniffio==1.3.0 # via anyio snowballstemmer==2.2.0 # via sphinx @@ -716,11 +720,11 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy[mypy]==1.4.40 +sqlalchemy[mypy]==1.4.41 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a25 +sqlalchemy2-stubs==0.0.2a27 # via sqlalchemy -stack-data==0.4.0 +stack-data==0.5.0 # via ipython starlette==0.19.1 # via fastapi @@ -728,7 +732,7 @@ tabulate==0.8.10 # via feast (setup.py) tenacity==8.0.1 # via feast (setup.py) -tensorflow-metadata==1.9.0 +tensorflow-metadata==1.10.0 # via feast (setup.py) termcolor==1.1.0 # via great-expectations @@ -753,7 +757,7 @@ toolz==0.12.0 # altair # dask # partd -tqdm==4.64.0 +tqdm==4.64.1 # via # feast (setup.py) # great-expectations @@ -767,25 +771,27 @@ trino==0.315.0 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -types-protobuf==3.19.22 +types-protobuf==3.20.2 # via # feast (setup.py) # mypy-protobuf +types-pymysql==1.0.19 + # via feast (setup.py) types-python-dateutil==2.8.19 # via feast (setup.py) types-pytz==2022.2.1.0 # via feast (setup.py) types-pyyaml==6.0.11 # via feast (setup.py) -types-redis==4.3.14 +types-redis==4.3.20 # via feast (setup.py) -types-requests==2.28.9 +types-requests==2.28.10 # via feast (setup.py) -types-setuptools==65.1.0 +types-setuptools==65.3.0 # via feast (setup.py) types-tabulate==0.8.11 # via feast (setup.py) -types-urllib3==1.26.23 +types-urllib3==1.26.24 # via types-requests typing-extensions==4.3.0 # via @@ -803,7 +809,7 @@ tzlocal==4.2 # via great-expectations uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.11 +urllib3==1.26.12 # via # botocore # docker @@ -813,11 +819,11 @@ urllib3==1.26.11 # minio # requests # responses -uvicorn[standard]==0.18.2 +uvicorn[standard]==0.18.3 # via feast (setup.py) uvloop==0.16.0 # via uvicorn -virtualenv==20.16.3 +virtualenv==20.16.5 # via pre-commit volatile==2.1.0 # via bowler @@ -825,7 +831,7 @@ watchfiles==0.16.1 # via uvicorn wcwidth==0.2.5 # via prompt-toolkit -websocket-client==1.3.3 +websocket-client==1.4.1 # via # docker # kubernetes diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index e13eee056bc..03ad2f76d1e 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -4,6 +4,8 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.9-ci-requirements.txt # +--index-url https://registry.affirm-stage.com/artifactory/api/pypi/pypi/simple/ + absl-py==1.2.0 # via tensorflow-metadata adal==1.2.7 @@ -56,7 +58,7 @@ attrs==22.1.0 # pytest avro==1.10.0 # via feast (setup.py) -azure-core==1.25.0 +azure-core==1.25.1 # via # adlfs # azure-identity @@ -76,7 +78,7 @@ babel==2.10.3 # via sphinx backcall==0.2.0 # via ipython -black==22.6.0 +black==22.8.0 # via feast (setup.py) boto3==1.20.23 # via @@ -131,7 +133,7 @@ click==8.1.3 # moreorless # pip-tools # uvicorn -cloudpickle==2.1.0 +cloudpickle==2.2.0 # via dask colorama==0.4.5 # via @@ -169,7 +171,7 @@ dill==0.3.5.1 # via # feast (setup.py) # multiprocess -distlib==0.3.5 +distlib==0.3.6 # via virtualenv docker==6.0.0 # via @@ -183,9 +185,9 @@ entrypoints==0.4 # via altair execnet==1.9.0 # via pytest-xdist -executing==0.10.0 +executing==1.0.0 # via stack-data -fastapi==0.79.1 +fastapi==0.82.0 # via feast (setup.py) fastavro==1.6.0 # via @@ -195,7 +197,7 @@ fastjsonschema==2.16.1 # via nbformat filelock==3.8.0 # via virtualenv -firebase-admin==5.2.0 +firebase-admin==5.3.0 # via feast (setup.py) fissix==21.11.13 # via bowler @@ -215,7 +217,7 @@ gcsfs==2022.1.0 # via feast (setup.py) geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.8.2 +google-api-core[grpc]==2.10.0 # via # feast (setup.py) # firebase-admin @@ -226,9 +228,9 @@ google-api-core[grpc]==2.8.2 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.57.0 +google-api-python-client==2.60.0 # via firebase-admin -google-auth==2.10.0 +google-auth==2.11.0 # via # gcsfs # google-api-core @@ -244,7 +246,7 @@ google-auth-oauthlib==0.5.2 # via gcsfs google-cloud-bigquery[pandas]==3.3.2 # via feast (setup.py) -google-cloud-bigquery-storage==2.14.2 +google-cloud-bigquery-storage==2.15.0 # via # feast (setup.py) # google-cloud-bigquery @@ -263,7 +265,7 @@ google-cloud-storage==2.5.0 # feast (setup.py) # firebase-admin # gcsfs -google-crc32c==1.3.0 +google-crc32c==1.5.0 # via google-resumable-media google-resumable-media==2.3.3 # via @@ -277,9 +279,9 @@ googleapis-common-protos==1.56.4 # tensorflow-metadata great-expectations==0.14.13 # via feast (setup.py) -greenlet==1.1.2 +greenlet==1.1.3 # via sqlalchemy -grpcio==1.47.0 +grpcio==1.48.1 # via # feast (setup.py) # google-api-core @@ -288,13 +290,13 @@ grpcio==1.47.0 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-reflection==1.47.0 +grpcio-reflection==1.48.1 # via feast (setup.py) -grpcio-status==1.47.0 +grpcio-status==1.48.1 # via google-api-core -grpcio-testing==1.47.0 +grpcio-testing==1.48.1 # via feast (setup.py) -grpcio-tools==1.47.0 +grpcio-tools==1.48.1 # via feast (setup.py) h11==0.13.0 # via uvicorn @@ -308,7 +310,7 @@ httplib2==0.20.4 # google-auth-httplib2 httptools==0.4.0 # via uvicorn -identify==2.5.3 +identify==2.5.5 # via pre-commit idna==3.3 # via @@ -322,7 +324,7 @@ importlib-metadata==4.12.0 # via great-expectations iniconfig==1.1.1 # via pytest -ipython==8.4.0 +ipython==8.5.0 # via great-expectations isodate==0.6.1 # via msrest @@ -345,7 +347,7 @@ jsonpatch==1.32 # via great-expectations jsonpointer==2.3 # via jsonpatch -jsonschema==4.13.0 +jsonschema==4.15.0 # via # altair # feast (setup.py) @@ -375,7 +377,7 @@ mock==2.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -moto==3.1.18 +moto==4.0.2 # via feast (setup.py) msal==1.18.0 # via @@ -439,7 +441,7 @@ packaging==21.3 # pytest # redis # sphinx -pandas==1.4.3 +pandas==1.4.4 # via # altair # db-dtypes @@ -454,7 +456,7 @@ parso==0.8.3 # via jedi partd==1.3.0 # via dask -pathspec==0.9.0 +pathspec==0.10.1 # via black pbr==5.10.0 # via mock @@ -478,9 +480,9 @@ portalocker==2.5.1 # via msal-extensions pre-commit==2.20.0 # via feast (setup.py) -prompt-toolkit==3.0.30 +prompt-toolkit==3.0.31 # via ipython -proto-plus==1.22.0 +proto-plus==1.22.1 # via # feast (setup.py) # google-cloud-bigquery @@ -539,7 +541,7 @@ pycparser==2.21 # via cffi pycryptodomex==3.15.0 # via snowflake-connector-python -pydantic==1.9.2 +pydantic==1.10.2 # via # fastapi # feast (setup.py) @@ -557,6 +559,8 @@ pyjwt[crypto]==2.4.0 # snowflake-connector-python pymssql==2.2.5 # via feast (setup.py) +pymysql==1.0.2 + # via feast (setup.py) pyodbc==4.0.34 # via feast (setup.py) pyopenssl==22.0.0 @@ -570,7 +574,7 @@ pyrsistent==0.18.1 # via jsonschema pyspark==3.3.0 # via feast (setup.py) -pytest==7.1.2 +pytest==7.1.3 # via # feast (setup.py) # pytest-benchmark @@ -606,7 +610,7 @@ python-dateutil==2.8.2 # kubernetes # moto # pandas -python-dotenv==0.20.0 +python-dotenv==0.21.0 # via uvicorn pytz==2022.2.1 # via @@ -658,15 +662,15 @@ responses==0.21.0 # via moto rsa==4.9 # via google-auth -ruamel.yaml==0.17.17 +ruamel-yaml==0.17.17 # via great-expectations -ruamel.yaml.clib==0.2.6 - # via ruamel.yaml +ruamel-yaml-clib==0.2.6 + # via ruamel-yaml s3fs==2022.1.0 # via feast (setup.py) s3transfer==0.5.2 # via boto3 -scipy==1.9.0 +scipy==1.9.1 # via great-expectations six==1.16.0 # via @@ -684,7 +688,7 @@ six==1.16.0 # msrestazure # pandavro # python-dateutil -sniffio==1.2.0 +sniffio==1.3.0 # via anyio snowballstemmer==2.2.0 # via sphinx @@ -708,11 +712,11 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy[mypy]==1.4.40 +sqlalchemy[mypy]==1.4.41 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a25 +sqlalchemy2-stubs==0.0.2a27 # via sqlalchemy -stack-data==0.4.0 +stack-data==0.5.0 # via ipython starlette==0.19.1 # via fastapi @@ -720,7 +724,7 @@ tabulate==0.8.10 # via feast (setup.py) tenacity==8.0.1 # via feast (setup.py) -tensorflow-metadata==1.9.0 +tensorflow-metadata==1.10.0 # via feast (setup.py) termcolor==1.1.0 # via great-expectations @@ -745,7 +749,7 @@ toolz==0.12.0 # altair # dask # partd -tqdm==4.64.0 +tqdm==4.64.1 # via # feast (setup.py) # great-expectations @@ -759,25 +763,27 @@ trino==0.315.0 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -types-protobuf==3.19.22 +types-protobuf==3.20.2 # via # feast (setup.py) # mypy-protobuf +types-pymysql==1.0.19 + # via feast (setup.py) types-python-dateutil==2.8.19 # via feast (setup.py) types-pytz==2022.2.1.0 # via feast (setup.py) types-pyyaml==6.0.11 # via feast (setup.py) -types-redis==4.3.14 +types-redis==4.3.20 # via feast (setup.py) -types-requests==2.28.9 +types-requests==2.28.10 # via feast (setup.py) -types-setuptools==65.1.0 +types-setuptools==65.3.0 # via feast (setup.py) types-tabulate==0.8.11 # via feast (setup.py) -types-urllib3==1.26.23 +types-urllib3==1.26.24 # via types-requests typing-extensions==4.3.0 # via @@ -795,7 +801,7 @@ tzlocal==4.2 # via great-expectations uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.11 +urllib3==1.26.12 # via # botocore # docker @@ -805,11 +811,11 @@ urllib3==1.26.11 # minio # requests # responses -uvicorn[standard]==0.18.2 +uvicorn[standard]==0.18.3 # via feast (setup.py) uvloop==0.16.0 # via uvicorn -virtualenv==20.16.3 +virtualenv==20.16.5 # via pre-commit volatile==2.1.0 # via bowler @@ -817,7 +823,7 @@ watchfiles==0.16.1 # via uvicorn wcwidth==0.2.5 # via prompt-toolkit -websocket-client==1.3.3 +websocket-client==1.4.1 # via # docker # kubernetes From 9a547bbcb04436c9ec2a5b6d32446604eacbd482 Mon Sep 17 00:00:00 2001 From: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Date: Thu, 8 Sep 2022 16:53:43 -0700 Subject: [PATCH 07/17] remove pip index Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> --- sdk/python/requirements/py3.10-ci-requirements.txt | 2 -- sdk/python/requirements/py3.8-ci-requirements.txt | 2 -- sdk/python/requirements/py3.9-ci-requirements.txt | 2 -- 3 files changed, 6 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 85d7c9ca14f..eb85866cd8f 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -4,8 +4,6 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.10-ci-requirements.txt # ---index-url https://registry.affirm-stage.com/artifactory/api/pypi/pypi/simple/ - absl-py==1.2.0 # via tensorflow-metadata adal==1.2.7 diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index 9fa441860c0..32b3953967c 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -4,8 +4,6 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.8-ci-requirements.txt # ---index-url https://registry.affirm-stage.com/artifactory/api/pypi/pypi/simple/ - absl-py==1.2.0 # via tensorflow-metadata adal==1.2.7 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 03ad2f76d1e..fc687f3b148 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -4,8 +4,6 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.9-ci-requirements.txt # ---index-url https://registry.affirm-stage.com/artifactory/api/pypi/pypi/simple/ - absl-py==1.2.0 # via tensorflow-metadata adal==1.2.7 From 5d84e0676fb0385eb14d6c37a0d0722daf5addba Mon Sep 17 00:00:00 2001 From: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Date: Thu, 8 Sep 2022 17:06:45 -0700 Subject: [PATCH 08/17] format Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> --- .../infra/online_stores/contrib/mysql.py | 29 +++++++++++++++---- .../universal/online_store/mysql.py | 18 ++++++++---- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql.py b/sdk/python/feast/infra/online_stores/contrib/mysql.py index 9e344c80d91..b68ff6171d9 100644 --- a/sdk/python/feast/infra/online_stores/contrib/mysql.py +++ b/sdk/python/feast/infra/online_stores/contrib/mysql.py @@ -59,7 +59,9 @@ def online_write_batch( self, config: RepoConfig, table: FeatureView, - data: List[Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]]], + data: List[ + Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + ], progress: Optional[Callable[[int], Any]], ) -> None: @@ -75,13 +77,24 @@ def online_write_batch( created_ts = _to_naive_utc(created_ts) for feature_name, val in values.items(): - self.write_to_table(created_ts, cur, entity_key_bin, feature_name, project, table, timestamp, val) + self.write_to_table( + created_ts, + cur, + entity_key_bin, + feature_name, + project, + table, + timestamp, + val, + ) conn.commit() if progress: progress(1) @staticmethod - def write_to_table(created_ts, cur, entity_key_bin, feature_name, project, table, timestamp, val) -> None: + def write_to_table( + created_ts, cur, entity_key_bin, feature_name, project, table, timestamp, val + ) -> None: cur.execute( f""" INSERT INTO {_table_id(project, table)} @@ -102,7 +115,7 @@ def write_to_table(created_ts, cur, entity_key_bin, feature_name, project, table # Update on duplicate key val.SerializeToString(), timestamp, - created_ts + created_ts, ), ) @@ -172,7 +185,9 @@ def update( ) for table in tables_to_delete: - cur.execute(f"DROP INDEX {_table_id(project, table)}_ek ON {_table_id(project, table)};") + cur.execute( + f"DROP INDEX {_table_id(project, table)}_ek ON {_table_id(project, table)};" + ) cur.execute(f"DROP TABLE IF EXISTS {_table_id(project, table)}") def teardown( @@ -186,7 +201,9 @@ def teardown( project = config.project for table in tables: - cur.execute(f"DROP INDEX {_table_id(project, table)}_ek ON {_table_id(project, table)};") + cur.execute( + f"DROP INDEX {_table_id(project, table)}_ek ON {_table_id(project, table)};" + ) cur.execute(f"DROP TABLE IF EXISTS {_table_id(project, table)}") diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/mysql.py b/sdk/python/tests/integration/feature_repos/universal/online_store/mysql.py index 84afd850f58..093295c86ba 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store/mysql.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/mysql.py @@ -10,16 +10,24 @@ class MySQLOnlineStoreCreator(OnlineStoreCreator): def __init__(self, project_name: str, **kwargs): super().__init__(project_name) - self.container = MySqlContainer('mysql:latest', platform='linux/amd64') \ - .with_exposed_ports(3306) \ - .with_env("MYSQL_USER", "root") \ - .with_env("MYSQL_PASSWORD", "test") \ + self.container = ( + MySqlContainer("mysql:latest", platform="linux/amd64") + .with_exposed_ports(3306) + .with_env("MYSQL_USER", "root") + .with_env("MYSQL_PASSWORD", "test") .with_env("MYSQL_DATABASE", "test") + ) def create_online_store(self) -> Dict[str, str]: self.container.start() exposed_port = self.container.get_exposed_port(3306) - return {"type": "mysql", "user": "root", "password": "test", "database": "test", "port": exposed_port} + return { + "type": "mysql", + "user": "root", + "password": "test", + "database": "test", + "port": exposed_port, + } def teardown(self): self.container.stop() From edceadf265e1a7c4c68b0267a9c5baae1c2dee5f Mon Sep 17 00:00:00 2001 From: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Date: Thu, 8 Sep 2022 17:20:15 -0700 Subject: [PATCH 09/17] fix unit test issue Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> --- .../requirements/py3.10-ci-requirements.txt | 94 ++++++++-------- .../requirements/py3.8-ci-requirements.txt | 94 ++++++++-------- .../requirements/py3.9-ci-requirements.txt | 100 +++++++++--------- 3 files changed, 138 insertions(+), 150 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index eb85866cd8f..9d10b2c3132 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -56,7 +56,7 @@ attrs==22.1.0 # pytest avro==1.10.0 # via feast (setup.py) -azure-core==1.25.1 +azure-core==1.25.0 # via # adlfs # azure-identity @@ -76,7 +76,7 @@ babel==2.10.3 # via sphinx backcall==0.2.0 # via ipython -black==22.8.0 +black==22.6.0 # via feast (setup.py) boto3==1.20.23 # via @@ -131,7 +131,7 @@ click==8.1.3 # moreorless # pip-tools # uvicorn -cloudpickle==2.2.0 +cloudpickle==2.1.0 # via dask colorama==0.4.5 # via @@ -169,7 +169,7 @@ dill==0.3.5.1 # via # feast (setup.py) # multiprocess -distlib==0.3.6 +distlib==0.3.5 # via virtualenv docker==6.0.0 # via @@ -183,9 +183,9 @@ entrypoints==0.4 # via altair execnet==1.9.0 # via pytest-xdist -executing==1.0.0 +executing==0.10.0 # via stack-data -fastapi==0.82.0 +fastapi==0.79.1 # via feast (setup.py) fastavro==1.6.0 # via @@ -195,7 +195,7 @@ fastjsonschema==2.16.1 # via nbformat filelock==3.8.0 # via virtualenv -firebase-admin==5.3.0 +firebase-admin==5.2.0 # via feast (setup.py) fissix==21.11.13 # via bowler @@ -215,7 +215,7 @@ gcsfs==2022.1.0 # via feast (setup.py) geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.10.0 +google-api-core[grpc]==2.8.2 # via # feast (setup.py) # firebase-admin @@ -226,9 +226,9 @@ google-api-core[grpc]==2.10.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.60.0 +google-api-python-client==2.57.0 # via firebase-admin -google-auth==2.11.0 +google-auth==2.10.0 # via # gcsfs # google-api-core @@ -244,7 +244,7 @@ google-auth-oauthlib==0.5.2 # via gcsfs google-cloud-bigquery[pandas]==3.3.2 # via feast (setup.py) -google-cloud-bigquery-storage==2.15.0 +google-cloud-bigquery-storage==2.14.2 # via # feast (setup.py) # google-cloud-bigquery @@ -263,7 +263,7 @@ google-cloud-storage==2.5.0 # feast (setup.py) # firebase-admin # gcsfs -google-crc32c==1.5.0 +google-crc32c==1.3.0 # via google-resumable-media google-resumable-media==2.3.3 # via @@ -277,9 +277,9 @@ googleapis-common-protos==1.56.4 # tensorflow-metadata great-expectations==0.14.13 # via feast (setup.py) -greenlet==1.1.3 +greenlet==1.1.2 # via sqlalchemy -grpcio==1.48.1 +grpcio==1.47.0 # via # feast (setup.py) # google-api-core @@ -288,13 +288,13 @@ grpcio==1.48.1 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-reflection==1.48.1 +grpcio-reflection==1.47.0 # via feast (setup.py) -grpcio-status==1.48.1 +grpcio-status==1.47.0 # via google-api-core -grpcio-testing==1.48.1 +grpcio-testing==1.47.0 # via feast (setup.py) -grpcio-tools==1.48.1 +grpcio-tools==1.47.0 # via feast (setup.py) h11==0.13.0 # via uvicorn @@ -308,7 +308,7 @@ httplib2==0.20.4 # google-auth-httplib2 httptools==0.4.0 # via uvicorn -identify==2.5.5 +identify==2.5.3 # via pre-commit idna==3.3 # via @@ -322,7 +322,7 @@ importlib-metadata==4.12.0 # via great-expectations iniconfig==1.1.1 # via pytest -ipython==8.5.0 +ipython==8.4.0 # via great-expectations isodate==0.6.1 # via msrest @@ -345,7 +345,7 @@ jsonpatch==1.32 # via great-expectations jsonpointer==2.3 # via jsonpatch -jsonschema==4.15.0 +jsonschema==4.13.0 # via # altair # feast (setup.py) @@ -375,7 +375,7 @@ mock==2.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -moto==4.0.2 +moto==3.1.18 # via feast (setup.py) msal==1.18.0 # via @@ -439,7 +439,7 @@ packaging==21.3 # pytest # redis # sphinx -pandas==1.4.4 +pandas==1.4.3 # via # altair # db-dtypes @@ -454,7 +454,7 @@ parso==0.8.3 # via jedi partd==1.3.0 # via dask -pathspec==0.10.1 +pathspec==0.9.0 # via black pbr==5.10.0 # via mock @@ -478,9 +478,9 @@ portalocker==2.5.1 # via msal-extensions pre-commit==2.20.0 # via feast (setup.py) -prompt-toolkit==3.0.31 +prompt-toolkit==3.0.30 # via ipython -proto-plus==1.22.1 +proto-plus==1.22.0 # via # feast (setup.py) # google-cloud-bigquery @@ -539,7 +539,7 @@ pycparser==2.21 # via cffi pycryptodomex==3.15.0 # via snowflake-connector-python -pydantic==1.10.2 +pydantic==1.9.2 # via # fastapi # feast (setup.py) @@ -557,8 +557,6 @@ pyjwt[crypto]==2.4.0 # snowflake-connector-python pymssql==2.2.5 # via feast (setup.py) -pymysql==1.0.2 - # via feast (setup.py) pyodbc==4.0.34 # via feast (setup.py) pyopenssl==22.0.0 @@ -572,7 +570,7 @@ pyrsistent==0.18.1 # via jsonschema pyspark==3.3.0 # via feast (setup.py) -pytest==7.1.3 +pytest==7.1.2 # via # feast (setup.py) # pytest-benchmark @@ -608,7 +606,7 @@ python-dateutil==2.8.2 # kubernetes # moto # pandas -python-dotenv==0.21.0 +python-dotenv==0.20.0 # via uvicorn pytz==2022.2.1 # via @@ -666,7 +664,7 @@ s3fs==2022.1.0 # via feast (setup.py) s3transfer==0.5.2 # via boto3 -scipy==1.9.1 +scipy==1.9.0 # via great-expectations six==1.16.0 # via @@ -684,7 +682,7 @@ six==1.16.0 # msrestazure # pandavro # python-dateutil -sniffio==1.3.0 +sniffio==1.2.0 # via anyio snowballstemmer==2.2.0 # via sphinx @@ -708,11 +706,11 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy[mypy]==1.4.41 +sqlalchemy[mypy]==1.4.40 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a27 +sqlalchemy2-stubs==0.0.2a25 # via sqlalchemy -stack-data==0.5.0 +stack-data==0.4.0 # via ipython starlette==0.19.1 # via fastapi @@ -720,7 +718,7 @@ tabulate==0.8.10 # via feast (setup.py) tenacity==8.0.1 # via feast (setup.py) -tensorflow-metadata==1.10.0 +tensorflow-metadata==1.9.0 # via feast (setup.py) termcolor==1.1.0 # via great-expectations @@ -745,7 +743,7 @@ toolz==0.12.0 # altair # dask # partd -tqdm==4.64.1 +tqdm==4.64.0 # via # feast (setup.py) # great-expectations @@ -759,27 +757,25 @@ trino==0.315.0 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -types-protobuf==3.20.2 +types-protobuf==3.19.22 # via # feast (setup.py) # mypy-protobuf -types-pymysql==1.0.19 - # via feast (setup.py) types-python-dateutil==2.8.19 # via feast (setup.py) types-pytz==2022.2.1.0 # via feast (setup.py) types-pyyaml==6.0.11 # via feast (setup.py) -types-redis==4.3.20 +types-redis==4.3.14 # via feast (setup.py) -types-requests==2.28.10 +types-requests==2.28.9 # via feast (setup.py) -types-setuptools==65.3.0 +types-setuptools==65.1.0 # via feast (setup.py) types-tabulate==0.8.11 # via feast (setup.py) -types-urllib3==1.26.24 +types-urllib3==1.26.23 # via types-requests typing-extensions==4.3.0 # via @@ -794,7 +790,7 @@ tzlocal==4.2 # via great-expectations uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.12 +urllib3==1.26.11 # via # botocore # docker @@ -804,11 +800,11 @@ urllib3==1.26.12 # minio # requests # responses -uvicorn[standard]==0.18.3 +uvicorn[standard]==0.18.2 # via feast (setup.py) uvloop==0.16.0 # via uvicorn -virtualenv==20.16.5 +virtualenv==20.16.3 # via pre-commit volatile==2.1.0 # via bowler @@ -816,7 +812,7 @@ watchfiles==0.16.1 # via uvicorn wcwidth==0.2.5 # via prompt-toolkit -websocket-client==1.4.1 +websocket-client==1.3.3 # via # docker # kubernetes diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index 32b3953967c..93011cfdcf4 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -56,7 +56,7 @@ attrs==22.1.0 # pytest avro==1.10.0 # via feast (setup.py) -azure-core==1.25.1 +azure-core==1.25.0 # via # adlfs # azure-identity @@ -80,7 +80,7 @@ backports-zoneinfo==0.2.1 # via # pytz-deprecation-shim # tzlocal -black==22.8.0 +black==22.6.0 # via feast (setup.py) boto3==1.20.23 # via @@ -135,7 +135,7 @@ click==8.1.3 # moreorless # pip-tools # uvicorn -cloudpickle==2.2.0 +cloudpickle==2.1.0 # via dask colorama==0.4.5 # via @@ -173,7 +173,7 @@ dill==0.3.5.1 # via # feast (setup.py) # multiprocess -distlib==0.3.6 +distlib==0.3.5 # via virtualenv docker==6.0.0 # via @@ -187,9 +187,9 @@ entrypoints==0.4 # via altair execnet==1.9.0 # via pytest-xdist -executing==1.0.0 +executing==0.10.0 # via stack-data -fastapi==0.82.0 +fastapi==0.79.1 # via feast (setup.py) fastavro==1.6.0 # via @@ -199,7 +199,7 @@ fastjsonschema==2.16.1 # via nbformat filelock==3.8.0 # via virtualenv -firebase-admin==5.3.0 +firebase-admin==5.2.0 # via feast (setup.py) fissix==21.11.13 # via bowler @@ -219,7 +219,7 @@ gcsfs==2022.1.0 # via feast (setup.py) geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.10.0 +google-api-core[grpc]==2.8.2 # via # feast (setup.py) # firebase-admin @@ -230,9 +230,9 @@ google-api-core[grpc]==2.10.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.60.0 +google-api-python-client==2.57.0 # via firebase-admin -google-auth==2.11.0 +google-auth==2.10.0 # via # gcsfs # google-api-core @@ -248,7 +248,7 @@ google-auth-oauthlib==0.5.2 # via gcsfs google-cloud-bigquery[pandas]==3.3.2 # via feast (setup.py) -google-cloud-bigquery-storage==2.15.0 +google-cloud-bigquery-storage==2.14.2 # via # feast (setup.py) # google-cloud-bigquery @@ -267,7 +267,7 @@ google-cloud-storage==2.5.0 # feast (setup.py) # firebase-admin # gcsfs -google-crc32c==1.5.0 +google-crc32c==1.3.0 # via google-resumable-media google-resumable-media==2.3.3 # via @@ -281,9 +281,9 @@ googleapis-common-protos==1.56.4 # tensorflow-metadata great-expectations==0.14.13 # via feast (setup.py) -greenlet==1.1.3 +greenlet==1.1.2 # via sqlalchemy -grpcio==1.48.1 +grpcio==1.47.0 # via # feast (setup.py) # google-api-core @@ -292,13 +292,13 @@ grpcio==1.48.1 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-reflection==1.48.1 +grpcio-reflection==1.47.0 # via feast (setup.py) -grpcio-status==1.48.1 +grpcio-status==1.47.0 # via google-api-core -grpcio-testing==1.48.1 +grpcio-testing==1.47.0 # via feast (setup.py) -grpcio-tools==1.48.1 +grpcio-tools==1.47.0 # via feast (setup.py) h11==0.13.0 # via uvicorn @@ -312,7 +312,7 @@ httplib2==0.20.4 # google-auth-httplib2 httptools==0.4.0 # via uvicorn -identify==2.5.5 +identify==2.5.3 # via pre-commit idna==3.3 # via @@ -328,7 +328,7 @@ importlib-resources==5.9.0 # via jsonschema iniconfig==1.1.1 # via pytest -ipython==8.5.0 +ipython==8.4.0 # via great-expectations isodate==0.6.1 # via msrest @@ -351,7 +351,7 @@ jsonpatch==1.32 # via great-expectations jsonpointer==2.3 # via jsonpatch -jsonschema==4.15.0 +jsonschema==4.13.0 # via # altair # feast (setup.py) @@ -381,7 +381,7 @@ mock==2.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -moto==4.0.2 +moto==3.1.18 # via feast (setup.py) msal==1.18.0 # via @@ -445,7 +445,7 @@ packaging==21.3 # pytest # redis # sphinx -pandas==1.4.4 +pandas==1.4.3 # via # altair # db-dtypes @@ -460,7 +460,7 @@ parso==0.8.3 # via jedi partd==1.3.0 # via dask -pathspec==0.10.1 +pathspec==0.9.0 # via black pbr==5.10.0 # via mock @@ -486,9 +486,9 @@ portalocker==2.5.1 # via msal-extensions pre-commit==2.20.0 # via feast (setup.py) -prompt-toolkit==3.0.31 +prompt-toolkit==3.0.30 # via ipython -proto-plus==1.22.1 +proto-plus==1.22.0 # via # feast (setup.py) # google-cloud-bigquery @@ -547,7 +547,7 @@ pycparser==2.21 # via cffi pycryptodomex==3.15.0 # via snowflake-connector-python -pydantic==1.10.2 +pydantic==1.9.2 # via # fastapi # feast (setup.py) @@ -565,8 +565,6 @@ pyjwt[crypto]==2.4.0 # snowflake-connector-python pymssql==2.2.5 # via feast (setup.py) -pymysql==1.0.2 - # via feast (setup.py) pyodbc==4.0.34 # via feast (setup.py) pyopenssl==22.0.0 @@ -580,7 +578,7 @@ pyrsistent==0.18.1 # via jsonschema pyspark==3.3.0 # via feast (setup.py) -pytest==7.1.3 +pytest==7.1.2 # via # feast (setup.py) # pytest-benchmark @@ -616,7 +614,7 @@ python-dateutil==2.8.2 # kubernetes # moto # pandas -python-dotenv==0.21.0 +python-dotenv==0.20.0 # via uvicorn pytz==2022.2.1 # via @@ -676,7 +674,7 @@ s3fs==2022.1.0 # via feast (setup.py) s3transfer==0.5.2 # via boto3 -scipy==1.9.1 +scipy==1.9.0 # via great-expectations six==1.16.0 # via @@ -694,7 +692,7 @@ six==1.16.0 # msrestazure # pandavro # python-dateutil -sniffio==1.3.0 +sniffio==1.2.0 # via anyio snowballstemmer==2.2.0 # via sphinx @@ -718,11 +716,11 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy[mypy]==1.4.41 +sqlalchemy[mypy]==1.4.40 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a27 +sqlalchemy2-stubs==0.0.2a25 # via sqlalchemy -stack-data==0.5.0 +stack-data==0.4.0 # via ipython starlette==0.19.1 # via fastapi @@ -730,7 +728,7 @@ tabulate==0.8.10 # via feast (setup.py) tenacity==8.0.1 # via feast (setup.py) -tensorflow-metadata==1.10.0 +tensorflow-metadata==1.9.0 # via feast (setup.py) termcolor==1.1.0 # via great-expectations @@ -755,7 +753,7 @@ toolz==0.12.0 # altair # dask # partd -tqdm==4.64.1 +tqdm==4.64.0 # via # feast (setup.py) # great-expectations @@ -769,27 +767,25 @@ trino==0.315.0 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -types-protobuf==3.20.2 +types-protobuf==3.19.22 # via # feast (setup.py) # mypy-protobuf -types-pymysql==1.0.19 - # via feast (setup.py) types-python-dateutil==2.8.19 # via feast (setup.py) types-pytz==2022.2.1.0 # via feast (setup.py) types-pyyaml==6.0.11 # via feast (setup.py) -types-redis==4.3.20 +types-redis==4.3.14 # via feast (setup.py) -types-requests==2.28.10 +types-requests==2.28.9 # via feast (setup.py) -types-setuptools==65.3.0 +types-setuptools==65.1.0 # via feast (setup.py) types-tabulate==0.8.11 # via feast (setup.py) -types-urllib3==1.26.24 +types-urllib3==1.26.23 # via types-requests typing-extensions==4.3.0 # via @@ -807,7 +803,7 @@ tzlocal==4.2 # via great-expectations uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.12 +urllib3==1.26.11 # via # botocore # docker @@ -817,11 +813,11 @@ urllib3==1.26.12 # minio # requests # responses -uvicorn[standard]==0.18.3 +uvicorn[standard]==0.18.2 # via feast (setup.py) uvloop==0.16.0 # via uvicorn -virtualenv==20.16.5 +virtualenv==20.16.3 # via pre-commit volatile==2.1.0 # via bowler @@ -829,7 +825,7 @@ watchfiles==0.16.1 # via uvicorn wcwidth==0.2.5 # via prompt-toolkit -websocket-client==1.4.1 +websocket-client==1.3.3 # via # docker # kubernetes diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index fc687f3b148..e13eee056bc 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -56,7 +56,7 @@ attrs==22.1.0 # pytest avro==1.10.0 # via feast (setup.py) -azure-core==1.25.1 +azure-core==1.25.0 # via # adlfs # azure-identity @@ -76,7 +76,7 @@ babel==2.10.3 # via sphinx backcall==0.2.0 # via ipython -black==22.8.0 +black==22.6.0 # via feast (setup.py) boto3==1.20.23 # via @@ -131,7 +131,7 @@ click==8.1.3 # moreorless # pip-tools # uvicorn -cloudpickle==2.2.0 +cloudpickle==2.1.0 # via dask colorama==0.4.5 # via @@ -169,7 +169,7 @@ dill==0.3.5.1 # via # feast (setup.py) # multiprocess -distlib==0.3.6 +distlib==0.3.5 # via virtualenv docker==6.0.0 # via @@ -183,9 +183,9 @@ entrypoints==0.4 # via altair execnet==1.9.0 # via pytest-xdist -executing==1.0.0 +executing==0.10.0 # via stack-data -fastapi==0.82.0 +fastapi==0.79.1 # via feast (setup.py) fastavro==1.6.0 # via @@ -195,7 +195,7 @@ fastjsonschema==2.16.1 # via nbformat filelock==3.8.0 # via virtualenv -firebase-admin==5.3.0 +firebase-admin==5.2.0 # via feast (setup.py) fissix==21.11.13 # via bowler @@ -215,7 +215,7 @@ gcsfs==2022.1.0 # via feast (setup.py) geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.10.0 +google-api-core[grpc]==2.8.2 # via # feast (setup.py) # firebase-admin @@ -226,9 +226,9 @@ google-api-core[grpc]==2.10.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.60.0 +google-api-python-client==2.57.0 # via firebase-admin -google-auth==2.11.0 +google-auth==2.10.0 # via # gcsfs # google-api-core @@ -244,7 +244,7 @@ google-auth-oauthlib==0.5.2 # via gcsfs google-cloud-bigquery[pandas]==3.3.2 # via feast (setup.py) -google-cloud-bigquery-storage==2.15.0 +google-cloud-bigquery-storage==2.14.2 # via # feast (setup.py) # google-cloud-bigquery @@ -263,7 +263,7 @@ google-cloud-storage==2.5.0 # feast (setup.py) # firebase-admin # gcsfs -google-crc32c==1.5.0 +google-crc32c==1.3.0 # via google-resumable-media google-resumable-media==2.3.3 # via @@ -277,9 +277,9 @@ googleapis-common-protos==1.56.4 # tensorflow-metadata great-expectations==0.14.13 # via feast (setup.py) -greenlet==1.1.3 +greenlet==1.1.2 # via sqlalchemy -grpcio==1.48.1 +grpcio==1.47.0 # via # feast (setup.py) # google-api-core @@ -288,13 +288,13 @@ grpcio==1.48.1 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-reflection==1.48.1 +grpcio-reflection==1.47.0 # via feast (setup.py) -grpcio-status==1.48.1 +grpcio-status==1.47.0 # via google-api-core -grpcio-testing==1.48.1 +grpcio-testing==1.47.0 # via feast (setup.py) -grpcio-tools==1.48.1 +grpcio-tools==1.47.0 # via feast (setup.py) h11==0.13.0 # via uvicorn @@ -308,7 +308,7 @@ httplib2==0.20.4 # google-auth-httplib2 httptools==0.4.0 # via uvicorn -identify==2.5.5 +identify==2.5.3 # via pre-commit idna==3.3 # via @@ -322,7 +322,7 @@ importlib-metadata==4.12.0 # via great-expectations iniconfig==1.1.1 # via pytest -ipython==8.5.0 +ipython==8.4.0 # via great-expectations isodate==0.6.1 # via msrest @@ -345,7 +345,7 @@ jsonpatch==1.32 # via great-expectations jsonpointer==2.3 # via jsonpatch -jsonschema==4.15.0 +jsonschema==4.13.0 # via # altair # feast (setup.py) @@ -375,7 +375,7 @@ mock==2.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -moto==4.0.2 +moto==3.1.18 # via feast (setup.py) msal==1.18.0 # via @@ -439,7 +439,7 @@ packaging==21.3 # pytest # redis # sphinx -pandas==1.4.4 +pandas==1.4.3 # via # altair # db-dtypes @@ -454,7 +454,7 @@ parso==0.8.3 # via jedi partd==1.3.0 # via dask -pathspec==0.10.1 +pathspec==0.9.0 # via black pbr==5.10.0 # via mock @@ -478,9 +478,9 @@ portalocker==2.5.1 # via msal-extensions pre-commit==2.20.0 # via feast (setup.py) -prompt-toolkit==3.0.31 +prompt-toolkit==3.0.30 # via ipython -proto-plus==1.22.1 +proto-plus==1.22.0 # via # feast (setup.py) # google-cloud-bigquery @@ -539,7 +539,7 @@ pycparser==2.21 # via cffi pycryptodomex==3.15.0 # via snowflake-connector-python -pydantic==1.10.2 +pydantic==1.9.2 # via # fastapi # feast (setup.py) @@ -557,8 +557,6 @@ pyjwt[crypto]==2.4.0 # snowflake-connector-python pymssql==2.2.5 # via feast (setup.py) -pymysql==1.0.2 - # via feast (setup.py) pyodbc==4.0.34 # via feast (setup.py) pyopenssl==22.0.0 @@ -572,7 +570,7 @@ pyrsistent==0.18.1 # via jsonschema pyspark==3.3.0 # via feast (setup.py) -pytest==7.1.3 +pytest==7.1.2 # via # feast (setup.py) # pytest-benchmark @@ -608,7 +606,7 @@ python-dateutil==2.8.2 # kubernetes # moto # pandas -python-dotenv==0.21.0 +python-dotenv==0.20.0 # via uvicorn pytz==2022.2.1 # via @@ -660,15 +658,15 @@ responses==0.21.0 # via moto rsa==4.9 # via google-auth -ruamel-yaml==0.17.17 +ruamel.yaml==0.17.17 # via great-expectations -ruamel-yaml-clib==0.2.6 - # via ruamel-yaml +ruamel.yaml.clib==0.2.6 + # via ruamel.yaml s3fs==2022.1.0 # via feast (setup.py) s3transfer==0.5.2 # via boto3 -scipy==1.9.1 +scipy==1.9.0 # via great-expectations six==1.16.0 # via @@ -686,7 +684,7 @@ six==1.16.0 # msrestazure # pandavro # python-dateutil -sniffio==1.3.0 +sniffio==1.2.0 # via anyio snowballstemmer==2.2.0 # via sphinx @@ -710,11 +708,11 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy[mypy]==1.4.41 +sqlalchemy[mypy]==1.4.40 # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a27 +sqlalchemy2-stubs==0.0.2a25 # via sqlalchemy -stack-data==0.5.0 +stack-data==0.4.0 # via ipython starlette==0.19.1 # via fastapi @@ -722,7 +720,7 @@ tabulate==0.8.10 # via feast (setup.py) tenacity==8.0.1 # via feast (setup.py) -tensorflow-metadata==1.10.0 +tensorflow-metadata==1.9.0 # via feast (setup.py) termcolor==1.1.0 # via great-expectations @@ -747,7 +745,7 @@ toolz==0.12.0 # altair # dask # partd -tqdm==4.64.1 +tqdm==4.64.0 # via # feast (setup.py) # great-expectations @@ -761,27 +759,25 @@ trino==0.315.0 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -types-protobuf==3.20.2 +types-protobuf==3.19.22 # via # feast (setup.py) # mypy-protobuf -types-pymysql==1.0.19 - # via feast (setup.py) types-python-dateutil==2.8.19 # via feast (setup.py) types-pytz==2022.2.1.0 # via feast (setup.py) types-pyyaml==6.0.11 # via feast (setup.py) -types-redis==4.3.20 +types-redis==4.3.14 # via feast (setup.py) -types-requests==2.28.10 +types-requests==2.28.9 # via feast (setup.py) -types-setuptools==65.3.0 +types-setuptools==65.1.0 # via feast (setup.py) types-tabulate==0.8.11 # via feast (setup.py) -types-urllib3==1.26.24 +types-urllib3==1.26.23 # via types-requests typing-extensions==4.3.0 # via @@ -799,7 +795,7 @@ tzlocal==4.2 # via great-expectations uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.12 +urllib3==1.26.11 # via # botocore # docker @@ -809,11 +805,11 @@ urllib3==1.26.12 # minio # requests # responses -uvicorn[standard]==0.18.3 +uvicorn[standard]==0.18.2 # via feast (setup.py) uvloop==0.16.0 # via uvicorn -virtualenv==20.16.5 +virtualenv==20.16.3 # via pre-commit volatile==2.1.0 # via bowler @@ -821,7 +817,7 @@ watchfiles==0.16.1 # via uvicorn wcwidth==0.2.5 # via prompt-toolkit -websocket-client==1.4.1 +websocket-client==1.3.3 # via # docker # kubernetes From 28090e5d21ceeeee65ba0193cdb67a80786dae7f Mon Sep 17 00:00:00 2001 From: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Date: Thu, 8 Sep 2022 17:31:29 -0700 Subject: [PATCH 10/17] fix lint issue Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> --- sdk/python/requirements/py3.10-ci-requirements.txt | 4 ++++ sdk/python/requirements/py3.8-ci-requirements.txt | 4 ++++ sdk/python/requirements/py3.9-ci-requirements.txt | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 9d10b2c3132..02d11ab5cd0 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -557,6 +557,8 @@ pyjwt[crypto]==2.4.0 # snowflake-connector-python pymssql==2.2.5 # via feast (setup.py) +pymysql==1.0.2 + # via feast (setup.py) pyodbc==4.0.34 # via feast (setup.py) pyopenssl==22.0.0 @@ -761,6 +763,8 @@ types-protobuf==3.19.22 # via # feast (setup.py) # mypy-protobuf +types-pymysql==1.0.19 + # via feast (setup.py) types-python-dateutil==2.8.19 # via feast (setup.py) types-pytz==2022.2.1.0 diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index 93011cfdcf4..dc891475598 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -565,6 +565,8 @@ pyjwt[crypto]==2.4.0 # snowflake-connector-python pymssql==2.2.5 # via feast (setup.py) +pymysql==1.0.2 + # via feast (setup.py) pyodbc==4.0.34 # via feast (setup.py) pyopenssl==22.0.0 @@ -771,6 +773,8 @@ types-protobuf==3.19.22 # via # feast (setup.py) # mypy-protobuf +types-pymysql==1.0.19 + # via feast (setup.py) types-python-dateutil==2.8.19 # via feast (setup.py) types-pytz==2022.2.1.0 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index e13eee056bc..b7d7c3c06e8 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -557,6 +557,8 @@ pyjwt[crypto]==2.4.0 # snowflake-connector-python pymssql==2.2.5 # via feast (setup.py) +pymysql==1.0.2 + # via feast (setup.py) pyodbc==4.0.34 # via feast (setup.py) pyopenssl==22.0.0 @@ -763,6 +765,8 @@ types-protobuf==3.19.22 # via # feast (setup.py) # mypy-protobuf +types-pymysql==1.0.19 + # via feast (setup.py) types-python-dateutil==2.8.19 # via feast (setup.py) types-pytz==2022.2.1.0 From 7f2afd685158c05feb58c755cddec43873941403 Mon Sep 17 00:00:00 2001 From: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Date: Thu, 8 Sep 2022 19:42:06 -0700 Subject: [PATCH 11/17] add entity_key_serialization_version Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> --- sdk/python/feast/infra/online_stores/contrib/mysql.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql.py b/sdk/python/feast/infra/online_stores/contrib/mysql.py index b68ff6171d9..d74d5ecfcbe 100644 --- a/sdk/python/feast/infra/online_stores/contrib/mysql.py +++ b/sdk/python/feast/infra/online_stores/contrib/mysql.py @@ -71,7 +71,10 @@ def online_write_batch( project = config.project for entity_key, values, timestamp, created_ts in data: - entity_key_bin = serialize_entity_key(entity_key).hex() + entity_key_bin = serialize_entity_key( + entity_key, + entity_key_serialization_version=config.entity_key_serialization_version, + ).hex() timestamp = _to_naive_utc(timestamp) if created_ts is not None: created_ts = _to_naive_utc(created_ts) @@ -133,7 +136,10 @@ def online_read( project = config.project for entity_key in entity_keys: - entity_key_bin = serialize_entity_key(entity_key).hex() + entity_key_bin = serialize_entity_key( + entity_key, + entity_key_serialization_version=config.entity_key_serialization_version + ).hex() cur.execute( f"SELECT feature_name, value, event_ts FROM {_table_id(project, table)} WHERE entity_key = %s", From 84b52469700b7541423e2992c64e2ff10486e9b6 Mon Sep 17 00:00:00 2001 From: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Date: Thu, 8 Sep 2022 19:58:39 -0700 Subject: [PATCH 12/17] update doc Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> --- Makefile | 2 +- docs/SUMMARY.md | 1 + docs/reference/online-stores/README.md | 4 ++++ .../docs/source/feast.infra.online_stores.contrib.rst | 9 +++++++++ .../contrib/{ => mysql_online_store}/mysql.py | 0 sdk/python/feast/repo_config.py | 2 +- 6 files changed, 16 insertions(+), 2 deletions(-) rename sdk/python/feast/infra/online_stores/contrib/{ => mysql_online_store}/mysql.py (100%) diff --git a/Makefile b/Makefile index 146b4373aae..28f98c90788 100644 --- a/Makefile +++ b/Makefile @@ -233,7 +233,7 @@ test-python-universal-postgres-online: test-python-universal-mysql-online: PYTHONPATH='.' \ - FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.mysql_repo_configuration \ + FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.mysql_online_store.mysql_repo_configuration \ PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.mysql \ FEAST_USAGE=False \ IS_TEST=True \ diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 430f9a745d7..2b58fb277e3 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -94,6 +94,7 @@ * [DynamoDB](reference/online-stores/dynamodb.md) * [PostgreSQL (contrib)](reference/online-stores/postgres.md) * [Cassandra + Astra DB (contrib)](reference/online-stores/cassandra.md) + * [MySQL (contrib)](reference/online-stores/mysql.md) * [Providers](reference/providers/README.md) * [Local](reference/providers/local.md) * [Google Cloud Platform](reference/providers/google-cloud-platform.md) diff --git a/docs/reference/online-stores/README.md b/docs/reference/online-stores/README.md index 58c8705e9ee..2ade4f0de9b 100644 --- a/docs/reference/online-stores/README.md +++ b/docs/reference/online-stores/README.md @@ -34,3 +34,7 @@ Please see [Online Store](../../getting-started/architecture-and-components/onli [cassandra.md](cassandra.md) {% endcontent-ref %} +{% content-ref url="mysql.md" %} +[mysql.md](mysql.md) +{% endcontent-ref %} + diff --git a/sdk/python/docs/source/feast.infra.online_stores.contrib.rst b/sdk/python/docs/source/feast.infra.online_stores.contrib.rst index 6afe9071ace..6b175f4584f 100644 --- a/sdk/python/docs/source/feast.infra.online_stores.contrib.rst +++ b/sdk/python/docs/source/feast.infra.online_stores.contrib.rst @@ -9,6 +9,7 @@ Subpackages feast.infra.online_stores.contrib.cassandra_online_store feast.infra.online_stores.contrib.hbase_online_store + feast.infra.online_stores.contrib.mysql_online_store Submodules ---------- @@ -29,6 +30,14 @@ feast.infra.online\_stores.contrib.hbase\_repo\_configuration module :undoc-members: :show-inheritance: +feast.infra.online\_stores.contrib.mysql\_repo\_configuration module +-------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.contrib.mysql_repo_configuration + :members: + :undoc-members: + :show-inheritance: + feast.infra.online\_stores.contrib.postgres module -------------------------------------------------- diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql.py b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py similarity index 100% rename from sdk/python/feast/infra/online_stores/contrib/mysql.py rename to sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index f2450020238..65d59a8eb59 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -50,7 +50,7 @@ "postgres": "feast.infra.online_stores.contrib.postgres.PostgreSQLOnlineStore", "hbase": "feast.infra.online_stores.contrib.hbase_online_store.hbase.HbaseOnlineStore", "cassandra": "feast.infra.online_stores.contrib.cassandra_online_store.cassandra_online_store.CassandraOnlineStore", - "mysql": "feast.infra.online_stores.contrib.mysql.MySQLOnlineStore", + "mysql": "feast.infra.online_stores.contrib.mysql_online_store.mysql.MySQLOnlineStore", } OFFLINE_STORE_CLASS_FOR_TYPE = { From d17ffe616825d940ab556008376260e89bd41f01 Mon Sep 17 00:00:00 2001 From: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Date: Thu, 8 Sep 2022 19:58:44 -0700 Subject: [PATCH 13/17] format Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> --- docs/reference/online-stores/mysql.md | 53 ++++++++++++ ...line_stores.contrib.mysql_online_store.rst | 21 +++++ .../contrib/mysql_online_store/README.md | 81 +++++++++++++++++++ .../contrib/mysql_online_store/__init__.py | 0 4 files changed, 155 insertions(+) create mode 100644 docs/reference/online-stores/mysql.md create mode 100644 sdk/python/docs/source/feast.infra.online_stores.contrib.mysql_online_store.rst create mode 100644 sdk/python/feast/infra/online_stores/contrib/mysql_online_store/README.md create mode 100644 sdk/python/feast/infra/online_stores/contrib/mysql_online_store/__init__.py diff --git a/docs/reference/online-stores/mysql.md b/docs/reference/online-stores/mysql.md new file mode 100644 index 00000000000..cb90c7561c5 --- /dev/null +++ b/docs/reference/online-stores/mysql.md @@ -0,0 +1,53 @@ +# MySQL online store (contrib) + +## Description + +The MySQL online store provides support for materializing feature values into a MySQL database for serving online features. + +* Only the latest feature values are persisted + + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: mysql + host: DB_HOST + port: DB_PORT + database: DB_NAME + user: DB_USERNAME + password: DB_PASSWORD +``` +{% endcode %} + +The full set of configuration options is available in [MySQLOnlineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.online_stores.contrib.mysql.MySQLOnlineStoreConfig). + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Mys online store. + +| | Mys | +| :-------------------------------------------------------- | :-- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/sdk/python/docs/source/feast.infra.online_stores.contrib.mysql_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.contrib.mysql_online_store.rst new file mode 100644 index 00000000000..036922d658f --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.contrib.mysql_online_store.rst @@ -0,0 +1,21 @@ +feast.infra.online\_stores.contrib.mysql\_online\_store package +=============================================================== + +Submodules +---------- + +feast.infra.online\_stores.contrib.mysql\_online\_store.mysql module +-------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.contrib.mysql_online_store.mysql + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.contrib.mysql_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/README.md b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/README.md new file mode 100644 index 00000000000..ac38237cd11 --- /dev/null +++ b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/README.md @@ -0,0 +1,81 @@ +# Mysql Online Store +Mysql is not included in current [Feast](https://github.com/feast-dev/feast) roadmap, this project intends to add Mysql support for Online Store. +We create a table _ which gets updated with data on every materialize call + + +#### Create a feature repository + +```shell +feast init feature_repo +cd feature_repo +``` + +#### Edit `feature_store.yaml` + +set `online_store` type to be `mysql` + +```yaml +project: feature_repo +registry: data/registry.db +provider: local +online_store: + type: mysql + host: 127.0.0.1 # mysql endpoint, default to 127.0.0.1 + port: 3306 # mysql port, default to 3306 + user: test # mysql user, default to test + password: test # mysql password, default to test + database: feast # mysql database, default to feast +``` + +#### Apply the feature definitions in `example.py` + +```shell +feast -c feature_repo apply +``` +##### Output +``` +Registered entity driver_id +Registered feature view driver_hourly_stats_view +Deploying infrastructure for driver_hourly_stats_view +``` + +### Materialize Latest Data to Online Feature Store (Mysql) +``` +$ CURRENT_TIME=$(date -u +"%Y-%m-%dT%H:%M:%S") +$ feast -c feature_repo materialize-incremental $CURRENT_TIME +``` +#### Output +``` +Materializing 1 feature views from 2022-04-16 15:30:39+05:30 to 2022-04-19 15:31:04+05:30 into the mysql online store. + +driver_hourly_stats_view from 2022-04-16 15:30:39+05:30 to 2022-04-19 15:31:04+05:30: +100%|████████████████████████████████████████████████████████████████| 5/5 [00:00<00:00, 120.59it/s] +``` + +### Fetch the latest features for some entity id +```python +from pprint import pprint +from feast import FeatureStore + +store = FeatureStore(repo_path=".") +feature_vector = store.get_online_features( + features=[ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", + ], + entity_rows=[ + {"driver_id": 1004}, + {"driver_id": 1005}, + ], +).to_dict() +pprint(feature_vector) + +``` +#### Output +``` +{'acc_rate': [0.01390857808291912, 0.4063614010810852], + 'avg_daily_trips': [69, 706], + 'conv_rate': [0.6624961495399475, 0.7595928311347961], + 'driver_id': [1004, 1005]} +``` diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/__init__.py b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From a1aed889b210d52d5152fde8a2108962a7db7004 Mon Sep 17 00:00:00 2001 From: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Date: Thu, 8 Sep 2022 20:00:52 -0700 Subject: [PATCH 14/17] format Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> --- .../infra/online_stores/contrib/mysql_online_store/mysql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py index d74d5ecfcbe..3315c9211d8 100644 --- a/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py +++ b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py @@ -138,7 +138,7 @@ def online_read( for entity_key in entity_keys: entity_key_bin = serialize_entity_key( entity_key, - entity_key_serialization_version=config.entity_key_serialization_version + entity_key_serialization_version=config.entity_key_serialization_version, ).hex() cur.execute( From 5717d8cdd04ca792d414e96a0690ddd563082ba4 Mon Sep 17 00:00:00 2001 From: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Date: Thu, 8 Sep 2022 20:03:45 -0700 Subject: [PATCH 15/17] fix makefile typo Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 28f98c90788..146b4373aae 100644 --- a/Makefile +++ b/Makefile @@ -233,7 +233,7 @@ test-python-universal-postgres-online: test-python-universal-mysql-online: PYTHONPATH='.' \ - FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.mysql_online_store.mysql_repo_configuration \ + FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.mysql_repo_configuration \ PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.mysql \ FEAST_USAGE=False \ IS_TEST=True \ From 9f64788d334d86777f37efe5d7faa1a381a86e79 Mon Sep 17 00:00:00 2001 From: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Date: Fri, 9 Sep 2022 19:24:00 -0700 Subject: [PATCH 16/17] move to func Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> --- .../contrib/mysql_online_store/mysql.py | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py index 3315c9211d8..37c0ee8426b 100644 --- a/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py +++ b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py @@ -31,6 +31,7 @@ class MySQLOnlineStoreConfig(FeastConfigBaseModel): port: Optional[int] = None + class MySQLOnlineStore(OnlineStore): """ An online store implementation that uses MySQL. @@ -73,7 +74,7 @@ def online_write_batch( for entity_key, values, timestamp, created_ts in data: entity_key_bin = serialize_entity_key( entity_key, - entity_key_serialization_version=config.entity_key_serialization_version, + entity_key_serialization_version=2, ).hex() timestamp = _to_naive_utc(timestamp) if created_ts is not None: @@ -138,7 +139,7 @@ def online_read( for entity_key in entity_keys: entity_key_bin = serialize_entity_key( entity_key, - entity_key_serialization_version=config.entity_key_serialization_version, + entity_key_serialization_version=2, ).hex() cur.execute( @@ -191,10 +192,7 @@ def update( ) for table in tables_to_delete: - cur.execute( - f"DROP INDEX {_table_id(project, table)}_ek ON {_table_id(project, table)};" - ) - cur.execute(f"DROP TABLE IF EXISTS {_table_id(project, table)}") + _drop_table_and_index(cur, project, table) def teardown( self, @@ -207,10 +205,15 @@ def teardown( project = config.project for table in tables: - cur.execute( - f"DROP INDEX {_table_id(project, table)}_ek ON {_table_id(project, table)};" - ) - cur.execute(f"DROP TABLE IF EXISTS {_table_id(project, table)}") + _drop_table_and_index(cur, project, table) + + +def _drop_table_and_index(cur, project: str, table: FeatureView) -> None: + table_name = _table_id(project, table) + cur.execute( + f"DROP INDEX {table_name}_ek ON {table_name};" + ) + cur.execute(f"DROP TABLE IF EXISTS {table_name}") def _table_id(project: str, table: FeatureView) -> str: From 0d7d8319cbba6001e804cb2f77ef2bb0db09e06b Mon Sep 17 00:00:00 2001 From: hao-affirm <104030690+hao-affirm@users.noreply.github.com> Date: Fri, 9 Sep 2022 19:32:28 -0700 Subject: [PATCH 17/17] format Signed-off-by: hao-affirm <104030690+hao-affirm@users.noreply.github.com> --- .../online_stores/contrib/mysql_online_store/mysql.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py index 37c0ee8426b..fa7dd2c2a49 100644 --- a/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py +++ b/sdk/python/feast/infra/online_stores/contrib/mysql_online_store/mysql.py @@ -7,6 +7,7 @@ import pytz from pydantic import StrictStr from pymysql.connections import Connection +from pymysql.cursors import Cursor from feast import Entity, FeatureView, RepoConfig from feast.infra.key_encoding_utils import serialize_entity_key @@ -31,7 +32,6 @@ class MySQLOnlineStoreConfig(FeastConfigBaseModel): port: Optional[int] = None - class MySQLOnlineStore(OnlineStore): """ An online store implementation that uses MySQL. @@ -208,11 +208,9 @@ def teardown( _drop_table_and_index(cur, project, table) -def _drop_table_and_index(cur, project: str, table: FeatureView) -> None: +def _drop_table_and_index(cur: Cursor, project: str, table: FeatureView) -> None: table_name = _table_id(project, table) - cur.execute( - f"DROP INDEX {table_name}_ek ON {table_name};" - ) + cur.execute(f"DROP INDEX {table_name}_ek ON {table_name};") cur.execute(f"DROP TABLE IF EXISTS {table_name}")