From 2a9fad0542b33bbcff08a0f92b40dc6eb6f61ad8 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 4 Mar 2022 15:41:59 -0800 Subject: [PATCH 1/7] Add make file Signed-off-by: Kevin Zhang --- Makefile | 8 +++++++- .../feast/infra/offline_stores/contrib/__init__.py | 0 .../contrib/contrib_repo_configuration.py | 12 ++++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 sdk/python/feast/infra/offline_stores/contrib/__init__.py create mode 100644 sdk/python/feast/infra/offline_stores/contrib/contrib_repo_configuration.py diff --git a/Makefile b/Makefile index 164f6afb54d..4ed1bc77a14 100644 --- a/Makefile +++ b/Makefile @@ -72,6 +72,8 @@ test-python: test-python-integration: FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests +test-python-contrib-universal: setup_contrib_env test-python-universal + test-python-universal-local: FEAST_USAGE=False IS_TEST=True FEAST_IS_LOCAL_TEST=True python -m pytest -n 8 --integration --universal sdk/python/tests @@ -198,4 +200,8 @@ build-sphinx: compile-protos-python cd $(ROOT_DIR)/sdk/python/docs && $(MAKE) build-api-source build-templates: - python infra/scripts/compile-templates.py \ No newline at end of file + python infra/scripts/compile-templates.py + +setup_contrib_env: + export PYTHONPATH='.' + export FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.contrib_repo_configuration \ No newline at end of file diff --git a/sdk/python/feast/infra/offline_stores/contrib/__init__.py b/sdk/python/feast/infra/offline_stores/contrib/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/infra/offline_stores/contrib/contrib_repo_configuration.py b/sdk/python/feast/infra/offline_stores/contrib/contrib_repo_configuration.py new file mode 100644 index 00000000000..518180cb5cb --- /dev/null +++ b/sdk/python/feast/infra/offline_stores/contrib/contrib_repo_configuration.py @@ -0,0 +1,12 @@ +from tests.integration.feature_repos.integration_test_repo_config import ( + IntegrationTestRepoConfig, +) + +from tests.integration.feature_repos.universal.data_sources.spark_data_source_creator import SparkDataSourceCreator + +print("Importing new repo_configs") +FULL_REPO_CONFIGS = [ + IntegrationTestRepoConfig( + offline_store_creator=SparkDataSourceCreator, + ) +] \ No newline at end of file From a881e18776212d9610050684f061011128e183ce Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 4 Mar 2022 16:01:53 -0800 Subject: [PATCH 2/7] Fix makefile Signed-off-by: Kevin Zhang --- Makefile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 4ed1bc77a14..376e8e0a793 100644 --- a/Makefile +++ b/Makefile @@ -72,7 +72,8 @@ test-python: test-python-integration: FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests -test-python-contrib-universal: setup_contrib_env test-python-universal +test-python-contrib-universal: + PYTHONPATH='.' FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.contrib_repo_configuration FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration --universal sdk/python/tests test-python-universal-local: FEAST_USAGE=False IS_TEST=True FEAST_IS_LOCAL_TEST=True python -m pytest -n 8 --integration --universal sdk/python/tests @@ -201,7 +202,3 @@ build-sphinx: compile-protos-python build-templates: python infra/scripts/compile-templates.py - -setup_contrib_env: - export PYTHONPATH='.' - export FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.contrib_repo_configuration \ No newline at end of file From ad0bad058d1126fd78fbf8fd310f197453f3ff7b Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 4 Mar 2022 16:03:11 -0800 Subject: [PATCH 3/7] Clean up Signed-off-by: Kevin Zhang --- .../contrib/contrib_repo_configuration.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/contrib_repo_configuration.py b/sdk/python/feast/infra/offline_stores/contrib/contrib_repo_configuration.py index 518180cb5cb..320fcf8030d 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/contrib_repo_configuration.py +++ b/sdk/python/feast/infra/offline_stores/contrib/contrib_repo_configuration.py @@ -1,12 +1,10 @@ from tests.integration.feature_repos.integration_test_repo_config import ( IntegrationTestRepoConfig, ) +from tests.integration.feature_repos.universal.data_sources.spark_data_source_creator import ( + SparkDataSourceCreator, +) -from tests.integration.feature_repos.universal.data_sources.spark_data_source_creator import SparkDataSourceCreator - -print("Importing new repo_configs") FULL_REPO_CONFIGS = [ - IntegrationTestRepoConfig( - offline_store_creator=SparkDataSourceCreator, - ) -] \ No newline at end of file + IntegrationTestRepoConfig(offline_store_creator=SparkDataSourceCreator,) +] From 72c393835716dabdbea0f8376f17f052787b6051 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 4 Mar 2022 16:09:47 -0800 Subject: [PATCH 4/7] Add documentation Signed-off-by: Kevin Zhang --- docs/how-to-guides/adding-or-reusing-tests.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/how-to-guides/adding-or-reusing-tests.md b/docs/how-to-guides/adding-or-reusing-tests.md index bf89345a456..43602a04097 100644 --- a/docs/how-to-guides/adding-or-reusing-tests.md +++ b/docs/how-to-guides/adding-or-reusing-tests.md @@ -161,6 +161,13 @@ The key fixtures are the `environment` and `universal_data_sources` fixtures, wh * In `repo_configuration.py` add a new`IntegrationTestRepoConfig` or two (depending on how many online stores you want to test). * Run the full test suite with `make test-python-integration.` +### Including a new offline / online store in the main Feast repo as part of the contributions. + +* This is for plugins that are not 100% stable. Place the APIs in feast/infra/offline_stores/contrib/ +* Extend `data_source_creator.py` for your offline store. +* In `contrib_repo_configuration.py` add a new `IntegrationTestRepoConfig` or two (depending on how many online stores you want to test). +* Run the test suite on the contrib test suite with `make test-python-contrib-universal`. + ### To include a new online store * In `repo_configuration.py` add a new config that maps to a serialized version of configuration you need in `feature_store.yaml` to setup the online store. From 2d241156d87fd041c494a591ae9f3acfd25913e4 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 4 Mar 2022 16:12:32 -0800 Subject: [PATCH 5/7] Clean up Signed-off-by: Kevin Zhang --- .../infra/offline_stores/contrib/contrib_repo_configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/offline_stores/contrib/contrib_repo_configuration.py b/sdk/python/feast/infra/offline_stores/contrib/contrib_repo_configuration.py index 320fcf8030d..bce640ad641 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/contrib_repo_configuration.py +++ b/sdk/python/feast/infra/offline_stores/contrib/contrib_repo_configuration.py @@ -6,5 +6,5 @@ ) FULL_REPO_CONFIGS = [ - IntegrationTestRepoConfig(offline_store_creator=SparkDataSourceCreator,) + IntegrationTestRepoConfig(offline_store_creator=SparkDataSourceCreator) ] From c59fd055cafe7df5de4e3af0272da26a5cd56c4c Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 7 Mar 2022 12:26:14 -0800 Subject: [PATCH 6/7] Update documentation Signed-off-by: Kevin Zhang --- docs/how-to-guides/adding-or-reusing-tests.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/how-to-guides/adding-or-reusing-tests.md b/docs/how-to-guides/adding-or-reusing-tests.md index 43602a04097..86c116442f2 100644 --- a/docs/how-to-guides/adding-or-reusing-tests.md +++ b/docs/how-to-guides/adding-or-reusing-tests.md @@ -161,11 +161,11 @@ The key fixtures are the `environment` and `universal_data_sources` fixtures, wh * In `repo_configuration.py` add a new`IntegrationTestRepoConfig` or two (depending on how many online stores you want to test). * Run the full test suite with `make test-python-integration.` -### Including a new offline / online store in the main Feast repo as part of the contributions. +### Including a new offline / online store in the main Feast repo from external plugins with community maintainers. -* This is for plugins that are not 100% stable. Place the APIs in feast/infra/offline_stores/contrib/ -* Extend `data_source_creator.py` for your offline store. -* In `contrib_repo_configuration.py` add a new `IntegrationTestRepoConfig` or two (depending on how many online stores you want to test). +* This folder is for plugins that are officially maintained with community owners. Place the APIs in feast/infra/offline_stores/contrib/. +* Extend `data_source_creator.py` for your offline store and implement the required APIs. +* In `contrib_repo_configuration.py` add a new `IntegrationTestRepoConfig` (depending on how many online stores you want to test). * Run the test suite on the contrib test suite with `make test-python-contrib-universal`. ### To include a new online store From 5c318973547ddc76e76515ebf6764006e3e3d8ca Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 7 Mar 2022 15:59:18 -0800 Subject: [PATCH 7/7] update make target Signed-off-by: Kevin Zhang --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 376e8e0a793..bfea5e28265 100644 --- a/Makefile +++ b/Makefile @@ -72,7 +72,7 @@ test-python: test-python-integration: FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests -test-python-contrib-universal: +test-python-universal-contrib: PYTHONPATH='.' FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.contrib_repo_configuration FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration --universal sdk/python/tests test-python-universal-local: