From 36514989e7e3398c9134bc952aea8b02d797e132 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 28 Feb 2022 11:43:53 -0800 Subject: [PATCH 01/22] Roll back redis config and add tests back Signed-off-by: Kevin Zhang --- .gitignore | 3 ++- sdk/python/feast/infra/online_stores/redis.py | 6 ++---- sdk/python/tests/conftest.py | 8 ++++++-- .../tests/integration/feature_repos/repo_configuration.py | 1 + 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index e4becb3c311..97a9d2c975a 100644 --- a/.gitignore +++ b/.gitignore @@ -54,7 +54,7 @@ vendor .terraform/ *.tfvars -# python +# python # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -107,6 +107,7 @@ coverage.xml *.cover .hypothesis/ .pytest_cache/ +infra/scripts/*.conf # Translations *.mo diff --git a/sdk/python/feast/infra/online_stores/redis.py b/sdk/python/feast/infra/online_stores/redis.py index d32740adf32..6225f2d1d1e 100644 --- a/sdk/python/feast/infra/online_stores/redis.py +++ b/sdk/python/feast/infra/online_stores/redis.py @@ -41,7 +41,7 @@ try: from redis import Redis - from redis.cluster import ClusterNode, RedisCluster + from rediscluster import RedisCluster except ImportError as e: from feast.errors import FeastExtrasDependencyImportError @@ -163,9 +163,7 @@ def _get_client(self, online_store_config: RedisOnlineStoreConfig): online_store_config.connection_string ) if online_store_config.redis_type == RedisType.redis_cluster: - kwargs["startup_nodes"] = [ - ClusterNode(**node) for node in startup_nodes - ] + kwargs["startup_nodes"] = startup_nodes self._client = RedisCluster(**kwargs) else: kwargs["host"] = startup_nodes[0]["host"] diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index a3428844680..e2bdf77e6c0 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -37,6 +37,8 @@ construct_universal_test_data, ) +from sdk.python.tests.integration.feature_repos.repo_configuration import REDIS_CLUSTER_CONFIG + logger = logging.getLogger(__name__) @@ -169,10 +171,12 @@ def cleanup(): return e -@pytest.fixture() +@pytest.fixture( + params=[REDIS_CONFIG, REDIS_CLUSTER_CONFIG], scope="session", ids=[str(c) for c in [REDIS_CONFIG, REDIS_CLUSTER_CONFIG]] +) def local_redis_environment(request, worker_id): e = construct_test_environment( - IntegrationTestRepoConfig(online_store=REDIS_CONFIG), worker_id=worker_id + IntegrationTestRepoConfig(online_store=request.param), worker_id=worker_id ) def cleanup(): diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 83fc3e03f02..77b7c0cd46d 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -72,6 +72,7 @@ [ # Redis configurations IntegrationTestRepoConfig(online_store=REDIS_CONFIG), + IntegrationTestRepoConfig(online_store=REDIS_CLUSTER_CONFIG), # GCP configurations IntegrationTestRepoConfig( provider="gcp", From 196912f830b7bc66fa6c7fc692cb3e6508d54889 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 28 Feb 2022 11:44:55 -0800 Subject: [PATCH 02/22] lint Signed-off-by: Kevin Zhang --- sdk/python/tests/conftest.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index e2bdf77e6c0..6c02af7fc6e 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -22,6 +22,9 @@ import pandas as pd import pytest from _pytest.nodes import Item +from sdk.python.tests.integration.feature_repos.repo_configuration import ( + REDIS_CLUSTER_CONFIG, +) from feast import FeatureStore from tests.data.data_creator import create_dataset @@ -37,8 +40,6 @@ construct_universal_test_data, ) -from sdk.python.tests.integration.feature_repos.repo_configuration import REDIS_CLUSTER_CONFIG - logger = logging.getLogger(__name__) @@ -172,7 +173,9 @@ def cleanup(): @pytest.fixture( - params=[REDIS_CONFIG, REDIS_CLUSTER_CONFIG], scope="session", ids=[str(c) for c in [REDIS_CONFIG, REDIS_CLUSTER_CONFIG]] + params=[REDIS_CONFIG, REDIS_CLUSTER_CONFIG], + scope="session", + ids=[str(c) for c in [REDIS_CONFIG, REDIS_CLUSTER_CONFIG]], ) def local_redis_environment(request, worker_id): e = construct_test_environment( From 0513d61eb7491722d5d2d41196cce8bf9f70ee64 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 28 Feb 2022 11:59:41 -0800 Subject: [PATCH 03/22] Fix autogenerated import Signed-off-by: Kevin Zhang --- sdk/python/tests/conftest.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index 6c02af7fc6e..7ba0795a194 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -22,9 +22,6 @@ import pandas as pd import pytest from _pytest.nodes import Item -from sdk.python.tests.integration.feature_repos.repo_configuration import ( - REDIS_CLUSTER_CONFIG, -) from feast import FeatureStore from tests.data.data_creator import create_dataset @@ -34,6 +31,7 @@ from tests.integration.feature_repos.repo_configuration import ( FULL_REPO_CONFIGS, REDIS_CONFIG, + REDIS_CLUSTER_CONFIG, Environment, TestData, construct_test_environment, From 551ea5d1a70391c5b206f680a09f67785af987a1 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 28 Feb 2022 12:00:38 -0800 Subject: [PATCH 04/22] Fix lint Signed-off-by: Kevin Zhang --- sdk/python/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index 7ba0795a194..6121e02d62a 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -30,8 +30,8 @@ ) from tests.integration.feature_repos.repo_configuration import ( FULL_REPO_CONFIGS, - REDIS_CONFIG, REDIS_CLUSTER_CONFIG, + REDIS_CONFIG, Environment, TestData, construct_test_environment, From 9f55a2a587eab1ad7ca93043e6832f5acc67e693 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 28 Feb 2022 12:34:06 -0800 Subject: [PATCH 05/22] Add redis-py-cluster back Signed-off-by: Kevin Zhang --- .../requirements/py3.7-ci-requirements.txt | 60 ++++++----------- .../requirements/py3.8-ci-requirements.txt | 62 +++++++----------- .../requirements/py3.9-ci-requirements.txt | 64 ++++++++----------- sdk/python/setup.py | 3 +- 4 files changed, 69 insertions(+), 120 deletions(-) diff --git a/sdk/python/requirements/py3.7-ci-requirements.txt b/sdk/python/requirements/py3.7-ci-requirements.txt index 1b953a5707c..d4994dbc613 100644 --- a/sdk/python/requirements/py3.7-ci-requirements.txt +++ b/sdk/python/requirements/py3.7-ci-requirements.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with python 3.7 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # -# pip-compile --extra=ci --output-file=requirements/py3.7-ci-requirements.txt +# pip-compile --extra=ci --output-file=requirements/py3.7-ci-requirements.txt setup.py # absl-py==1.0.0 # via tensorflow-metadata @@ -44,8 +44,6 @@ assertpy==1.1 # via feast (setup.py) async-timeout==4.0.2 # via aiohttp -asynctest==0.13.0 - # via aiohttp attrs==21.4.0 # via # aiohttp @@ -133,7 +131,7 @@ cryptography==3.3.2 # pyjwt # pyopenssl # snowflake-connector-python -dask==2022.1.1 +dask==2022.2.1 # via feast (setup.py) debugpy==1.5.1 # via ipykernel @@ -143,8 +141,6 @@ decorator==5.1.1 # ipython defusedxml==0.7.1 # via nbconvert -deprecated==1.2.13 - # via redis deprecation==2.1.0 # via testcontainers dill==0.3.4 @@ -282,18 +278,7 @@ idna==3.3 imagesize==1.3.0 # via sphinx importlib-metadata==4.2.0 - # via - # click - # flake8 - # great-expectations - # jsonschema - # moto - # pep517 - # pluggy - # pre-commit - # pytest - # redis - # virtualenv + # via great-expectations importlib-resources==5.4.0 # via jsonschema iniconfig==1.1.1 @@ -358,10 +343,12 @@ jupyterlab-pygments==0.1.2 jupyterlab-widgets==1.0.2 # via ipywidgets libcst==0.4.1 - # via google-cloud-datastore + # via + # google-cloud-bigquery-storage + # google-cloud-datastore locket==0.2.1 # via partd -markupsafe==2.1.0 +markupsafe==2.0.1 # via # jinja2 # moto @@ -450,7 +437,6 @@ packaging==21.3 # google-cloud-bigquery # google-cloud-firestore # pytest - # redis # sphinx pandas==1.3.5 # via @@ -621,7 +607,11 @@ pyzmq==22.3.0 # via # jupyter-client # notebook -redis==4.1.4 +redis==3.5.3 + # via + # feast (setup.py) + # redis-py-cluster +redis-py-cluster==2.1.3 # via feast (setup.py) regex==2022.1.18 # via black @@ -762,10 +752,10 @@ traitlets==5.1.1 # nbformat # notebook typed-ast==1.5.2 - # via - # black - # mypy -types-protobuf==3.19.12 + # via black +types-futures==3.3.8 + # via types-protobuf +types-protobuf==3.19.2 # via # feast (setup.py) # mypy-protobuf @@ -787,22 +777,11 @@ types-urllib3==1.26.9 # via types-requests typing-extensions==4.1.1 # via - # aiohttp - # anyio - # argon2-cffi - # asgiref - # async-timeout # great-expectations - # h11 - # importlib-metadata - # jsonschema # libcst # mypy # pydantic - # starlette # typing-inspect - # uvicorn - # yarl typing-inspect==0.7.1 # via libcst tzdata==2021.5 @@ -841,9 +820,7 @@ wheel==0.37.1 widgetsnbextension==3.5.2 # via ipywidgets wrapt==1.13.3 - # via - # deprecated - # testcontainers + # via testcontainers xmltodict==0.12.0 # via moto yarl==1.7.2 @@ -852,7 +829,6 @@ zipp==3.7.0 # via # importlib-metadata # importlib-resources - # pep517 # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index 7a94294c956..39b3af3f32b 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with python 3.8 # To update, run: # -# pip-compile --extra=ci --output-file=requirements/py3.8-ci-requirements.txt +# pip-compile --extra=ci --output-file=requirements/py3.8-ci-requirements.txt setup.py # absl-py==1.0.0 # via tensorflow-metadata @@ -44,8 +44,6 @@ assertpy==1.1 # via feast (setup.py) async-timeout==4.0.2 # via aiohttp -asynctest==0.13.0 - # via aiohttp attrs==21.4.0 # via # aiohttp @@ -116,6 +114,8 @@ click==8.0.3 # great-expectations # pip-tools # uvicorn +cloudpickle==2.0.0 + # via dask colorama==0.4.4 # via feast (setup.py) coverage[toml]==6.3 @@ -131,6 +131,8 @@ cryptography==3.3.2 # pyjwt # pyopenssl # snowflake-connector-python +dask==2022.2.1 + # via feast (setup.py) debugpy==1.5.1 # via ipykernel decorator==5.1.1 @@ -139,8 +141,6 @@ decorator==5.1.1 # ipython defusedxml==0.7.1 # via nbconvert -deprecated==1.2.13 - # via redis deprecation==2.1.0 # via testcontainers dill==0.3.4 @@ -181,6 +181,7 @@ frozenlist==1.3.0 fsspec==2022.1.0 # via # adlfs + # dask # gcsfs gcsfs==2022.1.0 # via feast (setup.py) @@ -277,18 +278,7 @@ idna==3.3 imagesize==1.3.0 # via sphinx importlib-metadata==4.2.0 - # via - # click - # flake8 - # great-expectations - # jsonschema - # moto - # pep517 - # pluggy - # pre-commit - # pytest - # redis - # virtualenv + # via great-expectations importlib-resources==5.4.0 # via jsonschema iniconfig==1.1.1 @@ -356,6 +346,8 @@ libcst==0.4.1 # via # google-cloud-bigquery-storage # google-cloud-datastore +locket==0.2.1 + # via partd markupsafe==2.0.1 # via # jinja2 @@ -439,12 +431,12 @@ oscrypto==1.2.1 packaging==21.3 # via # bleach + # dask # deprecation # google-api-core # google-cloud-bigquery # google-cloud-firestore # pytest - # redis # sphinx pandas==1.3.5 # via @@ -459,6 +451,8 @@ pandocfilters==1.5.0 # via nbconvert parso==0.8.3 # via jedi +partd==1.2.0 + # via dask pathspec==0.9.0 # via black pbr==5.8.0 @@ -604,6 +598,7 @@ pytz-deprecation-shim==0.1.0.post0 # via tzlocal pyyaml==6.0 # via + # dask # feast (setup.py) # libcst # pre-commit @@ -612,7 +607,11 @@ pyzmq==22.3.0 # via # jupyter-client # notebook -redis==4.1.2 +redis==3.5.3 + # via + # feast (setup.py) + # redis-py-cluster +redis-py-cluster==2.1.3 # via feast (setup.py) regex==2022.1.18 # via black @@ -727,7 +726,10 @@ tomli==2.0.0 # mypy # pep517 toolz==0.11.2 - # via altair + # via + # altair + # dask + # partd tornado==6.1 # via # ipykernel @@ -751,9 +753,7 @@ traitlets==5.1.1 # nbformat # notebook typed-ast==1.5.2 - # via - # black - # mypy + # via black types-futures==3.3.8 # via types-protobuf types-protobuf==3.19.7 @@ -778,22 +778,11 @@ types-urllib3==1.26.8 # via types-requests typing-extensions==4.0.1 # via - # aiohttp - # anyio - # argon2-cffi - # asgiref - # async-timeout # great-expectations - # h11 - # importlib-metadata - # jsonschema # libcst # mypy # pydantic - # starlette # typing-inspect - # uvicorn - # yarl typing-inspect==0.7.1 # via libcst tzdata==2021.5 @@ -832,9 +821,7 @@ wheel==0.37.1 widgetsnbextension==3.5.2 # via ipywidgets wrapt==1.13.3 - # via - # deprecated - # testcontainers + # via testcontainers xmltodict==0.12.0 # via moto yarl==1.7.2 @@ -843,7 +830,6 @@ zipp==3.7.0 # via # importlib-metadata # importlib-resources - # pep517 # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 1421d7e3c31..45899c1b7f2 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # -# pip-compile --extra=ci --output-file=requirements/py3.9-ci-requirements.txt +# pip-compile --extra=ci --output-file=requirements/py3.9-ci-requirements.txt setup.py # absl-py==1.0.0 # via tensorflow-metadata @@ -44,8 +44,6 @@ assertpy==1.1 # via feast (setup.py) async-timeout==4.0.2 # via aiohttp -asynctest==0.13.0 - # via aiohttp attrs==21.4.0 # via # aiohttp @@ -116,6 +114,8 @@ click==8.0.3 # great-expectations # pip-tools # uvicorn +cloudpickle==2.0.0 + # via dask colorama==0.4.4 # via feast (setup.py) coverage[toml]==6.3 @@ -131,6 +131,8 @@ cryptography==3.3.2 # pyjwt # pyopenssl # snowflake-connector-python +dask==2022.2.1 + # via feast (setup.py) debugpy==1.5.1 # via ipykernel decorator==5.1.1 @@ -139,8 +141,6 @@ decorator==5.1.1 # ipython defusedxml==0.7.1 # via nbconvert -deprecated==1.2.13 - # via redis deprecation==2.1.0 # via testcontainers dill==0.3.4 @@ -181,6 +181,7 @@ frozenlist==1.3.0 fsspec==2022.1.0 # via # adlfs + # dask # gcsfs gcsfs==2022.1.0 # via feast (setup.py) @@ -277,18 +278,7 @@ idna==3.3 imagesize==1.3.0 # via sphinx importlib-metadata==4.2.0 - # via - # click - # flake8 - # great-expectations - # jsonschema - # moto - # pep517 - # pluggy - # pre-commit - # pytest - # redis - # virtualenv + # via great-expectations importlib-resources==5.4.0 # via jsonschema iniconfig==1.1.1 @@ -356,6 +346,8 @@ libcst==0.4.1 # via # google-cloud-bigquery-storage # google-cloud-datastore +locket==0.2.1 + # via partd markupsafe==2.0.1 # via # jinja2 @@ -439,12 +431,12 @@ oscrypto==1.2.1 packaging==21.3 # via # bleach + # dask # deprecation # google-api-core # google-cloud-bigquery # google-cloud-firestore # pytest - # redis # sphinx pandas==1.3.5 # via @@ -459,6 +451,8 @@ pandocfilters==1.5.0 # via nbconvert parso==0.8.3 # via jedi +partd==1.2.0 + # via dask pathspec==0.9.0 # via black pbr==5.8.0 @@ -604,6 +598,7 @@ pytz-deprecation-shim==0.1.0.post0 # via tzlocal pyyaml==6.0 # via + # dask # feast (setup.py) # libcst # pre-commit @@ -612,7 +607,11 @@ pyzmq==22.3.0 # via # jupyter-client # notebook -redis==4.1.2 +redis==3.5.3 + # via + # feast (setup.py) + # redis-py-cluster +redis-py-cluster==2.1.3 # via feast (setup.py) regex==2022.1.18 # via black @@ -727,7 +726,10 @@ tomli==2.0.0 # mypy # pep517 toolz==0.11.2 - # via altair + # via + # altair + # dask + # partd tornado==6.1 # via # ipykernel @@ -751,9 +753,7 @@ traitlets==5.1.1 # nbformat # notebook typed-ast==1.5.2 - # via - # black - # mypy + # via black types-futures==3.3.8 # via types-protobuf types-protobuf==3.19.7 @@ -778,22 +778,11 @@ types-urllib3==1.26.8 # via types-requests typing-extensions==4.0.1 # via - # aiohttp - # anyio - # argon2-cffi - # asgiref - # async-timeout # great-expectations - # h11 - # importlib-metadata - # jsonschema # libcst # mypy # pydantic - # starlette # typing-inspect - # uvicorn - # yarl typing-inspect==0.7.1 # via libcst tzdata==2021.5 @@ -832,9 +821,7 @@ wheel==0.37.1 widgetsnbextension==3.5.2 # via ipywidgets wrapt==1.13.3 - # via - # deprecated - # testcontainers + # via testcontainers xmltodict==0.12.0 # via moto yarl==1.7.2 @@ -843,7 +830,6 @@ zipp==3.7.0 # via # importlib-metadata # importlib-resources - # pep517 # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/sdk/python/setup.py b/sdk/python/setup.py index 337b4378002..7c74e1e1129 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -78,7 +78,8 @@ ] REDIS_REQUIRED = [ - "redis>=4.1.0", + "redis>=3.5.3", + "redis-py-cluster>=2.1.3", "hiredis>=2.0.0", ] From 2e16f1e74c6d85abb13eef24e3a81d9ea467629d Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 28 Feb 2022 12:40:14 -0800 Subject: [PATCH 06/22] Fix dask ci Signed-off-by: Kevin Zhang --- sdk/python/requirements/py3.7-ci-requirements.txt | 2 +- sdk/python/requirements/py3.8-ci-requirements.txt | 2 +- sdk/python/requirements/py3.9-ci-requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/python/requirements/py3.7-ci-requirements.txt b/sdk/python/requirements/py3.7-ci-requirements.txt index d4994dbc613..5b35e4df332 100644 --- a/sdk/python/requirements/py3.7-ci-requirements.txt +++ b/sdk/python/requirements/py3.7-ci-requirements.txt @@ -131,7 +131,7 @@ cryptography==3.3.2 # pyjwt # pyopenssl # snowflake-connector-python -dask==2022.2.1 +dask==2022.1.1 # via feast (setup.py) debugpy==1.5.1 # via ipykernel diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index 39b3af3f32b..5a8aa355323 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -131,7 +131,7 @@ cryptography==3.3.2 # pyjwt # pyopenssl # snowflake-connector-python -dask==2022.2.1 +dask==2022.1.1 # via feast (setup.py) debugpy==1.5.1 # via ipykernel diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 45899c1b7f2..f94bfa2dea6 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -131,7 +131,7 @@ cryptography==3.3.2 # pyjwt # pyopenssl # snowflake-connector-python -dask==2022.2.1 +dask==2022.1.1 # via feast (setup.py) debugpy==1.5.1 # via ipykernel From d70c1a56d328955d4dcde33ac3961ddf4ca2c666 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 28 Feb 2022 13:52:56 -0800 Subject: [PATCH 07/22] Regenerate using respective python 3 versions Signed-off-by: Kevin Zhang --- .../requirements/py3.7-ci-requirements.txt | 2 +- .../requirements/py3.9-ci-requirements.txt | 18 +++++------------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/sdk/python/requirements/py3.7-ci-requirements.txt b/sdk/python/requirements/py3.7-ci-requirements.txt index 5b35e4df332..d8f4eed9ce3 100644 --- a/sdk/python/requirements/py3.7-ci-requirements.txt +++ b/sdk/python/requirements/py3.7-ci-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.7 # To update, run: # # pip-compile --extra=ci --output-file=requirements/py3.7-ci-requirements.txt setup.py diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index f94bfa2dea6..403aab70aa7 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile --extra=ci --output-file=requirements/py3.9-ci-requirements.txt setup.py @@ -67,10 +67,6 @@ babel==2.9.1 # via sphinx backcall==0.2.0 # via ipython -backports.zoneinfo==0.2.1 - # via - # pytz-deprecation-shim - # tzlocal black==19.10b0 # via feast (setup.py) bleach==4.1.0 @@ -279,8 +275,6 @@ imagesize==1.3.0 # via sphinx importlib-metadata==4.2.0 # via great-expectations -importlib-resources==5.4.0 - # via jsonschema iniconfig==1.1.1 # via pytest ipykernel==6.7.0 @@ -643,10 +637,10 @@ responses==0.17.0 # via moto rsa==4.8 # via google-auth -ruamel.yaml==0.17.17 +ruamel-yaml==0.17.17 # via great-expectations -ruamel.yaml.clib==0.2.6 - # via ruamel.yaml +ruamel-yaml-clib==0.2.6 + # via ruamel-yaml s3transfer==0.5.0 # via boto3 scipy==1.7.3 @@ -827,9 +821,7 @@ xmltodict==0.12.0 yarl==1.7.2 # via aiohttp zipp==3.7.0 - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip From f4160ade756fc012bbcb3bfdb768a80a95c03d7d Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 28 Feb 2022 16:24:41 -0800 Subject: [PATCH 08/22] Change test_entity_ttl test Signed-off-by: Kevin Zhang --- sdk/python/tests/conftest.py | 17 +++++++++++++++++ .../online_store/test_universal_online.py | 9 ++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index 6121e02d62a..437924cb3ec 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -196,6 +196,23 @@ def cleanup(): request.addfinalizer(cleanup) return construct_universal_test_data(environment) +@pytest.fixture(scope="session") +def redis_universal_data_sources(request, local_redis_environment): + entities = construct_universal_entities() + datasets = construct_universal_datasets( + entities, local_redis_environment.start_date, local_redis_environment.end_date + ) + datasources = construct_universal_data_sources( + datasets, local_redis_environment.data_source_creator + ) + + def cleanup(): + # logger.info("Running cleanup in %s, Request: %s", worker_id, request.param) + local_redis_environment.data_source_creator.teardown() + + request.addfinalizer(cleanup) + return entities, datasets, datasources + @pytest.fixture(scope="session") def e2e_data_sources(environment: Environment, request): diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index 8d0c251cef1..386ac021505 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -36,14 +36,16 @@ @pytest.mark.integration -def test_entity_ttl_online_store(local_redis_environment, universal_data_sources): +def test_entity_ttl_online_store(local_redis_environment, redis_universal_data_sources): if os.getenv("FEAST_IS_LOCAL_TEST", "False") == "True": return fs = local_redis_environment.feature_store # setting ttl setting in online store to 1 second fs.config.online_store.key_ttl_seconds = 1 - entities, datasets, data_sources = universal_data_sources - driver_hourly_stats = create_driver_hourly_stats_feature_view(data_sources.driver) + entities, datasets, data_sources = redis_universal_data_sources + driver_hourly_stats = create_driver_hourly_stats_feature_view( + data_sources["driver"] + ) driver_entity = driver() # Register Feature View and Entity @@ -94,6 +96,7 @@ def test_entity_ttl_online_store(local_redis_environment, universal_data_sources assertpy.assert_that(df["conv_rate"].iloc[0]).is_none() + # TODO: make this work with all universal (all online store types) @pytest.mark.integration def test_write_to_online_store_event_check(local_redis_environment): From c5425d7dc93f1d095fc81bc7c0a54fc514b165c6 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 28 Feb 2022 16:25:18 -0800 Subject: [PATCH 09/22] Fix lint Signed-off-by: Kevin Zhang --- sdk/python/tests/conftest.py | 1 + .../tests/integration/online_store/test_universal_online.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index 437924cb3ec..d5bea0f82e2 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -196,6 +196,7 @@ def cleanup(): request.addfinalizer(cleanup) return construct_universal_test_data(environment) + @pytest.fixture(scope="session") def redis_universal_data_sources(request, local_redis_environment): entities = construct_universal_entities() diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index 386ac021505..28b4ca96426 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -96,7 +96,6 @@ def test_entity_ttl_online_store(local_redis_environment, redis_universal_data_s assertpy.assert_that(df["conv_rate"].iloc[0]).is_none() - # TODO: make this work with all universal (all online store types) @pytest.mark.integration def test_write_to_online_store_event_check(local_redis_environment): From e3268205bf264ba72db7d6df7f992e42dd4bde7c Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 1 Mar 2022 11:59:56 -0800 Subject: [PATCH 10/22] Change integration workflow Signed-off-by: Kevin Zhang --- .github/workflows/pr_integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 1a914473c1d..868734a9fdf 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -153,7 +153,7 @@ jobs: - name: Setup Redis Cluster run: | docker pull vishnunair/docker-redis-cluster:latest - docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster + docker run -d -p 32001:6380 -p 32002:6381 -p 32003:6382 -p 32004:6383 -p 32005:6384 -p 32006:6385 --name redis-cluster vishnunair/docker-redis-cluster - name: Test python if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak env: From 00fddbed2f3748be81757f59ebae417408528739 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 1 Mar 2022 13:45:08 -0800 Subject: [PATCH 11/22] Temporarily disable for testing Signed-off-by: Kevin Zhang --- .github/workflows/pr_integration_tests.yml | 194 ++++++++++----------- 1 file changed, 97 insertions(+), 97 deletions(-) diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 868734a9fdf..2c413bdf994 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -13,68 +13,68 @@ on: # cancel-in-progress: true jobs: - build-docker-image: - # all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. - if: (github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test')) - || (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved'))) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - # pull_request_target runs the workflow in the context of the base repo - # as such actions/checkout needs to be explicit configured to retrieve - # code from the PR. - ref: refs/pull/${{ github.event.pull_request.number }}/merge - submodules: recursive - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Set up AWS SDK - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - name: Set ECR image tag - id: image-tag - run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" - - name: Cache Public ECR Image - id: lambda_python_3_9 - uses: actions/cache@v2 - with: - path: ~/cache - key: lambda_python_3_9 - - name: Handle Cache Miss (pull public ECR image & save it to tar file) - if: steps.cache-primes.outputs.cache-hit != 'true' - run: | - mkdir -p ~/cache - docker pull public.ecr.aws/lambda/python:3.9 - docker save public.ecr.aws/lambda/python:3.9 -o ~/cache/lambda_python_3_9.tar - - name: Handle Cache Hit (load docker image from tar file) - if: steps.cache-primes.outputs.cache-hit == 'true' - run: | - docker load -i ~/cache/lambda_python_3_9.tar - - name: Build and push - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: feast-python-server - run: | - docker build \ - --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \ - --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \ - . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} - outputs: - DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} + # build-docker-image: + # # all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. + # if: (github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test')) + # || (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved'))) + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v2 + # with: + # # pull_request_target runs the workflow in the context of the base repo + # # as such actions/checkout needs to be explicit configured to retrieve + # # code from the PR. + # ref: refs/pull/${{ github.event.pull_request.number }}/merge + # submodules: recursive + # - name: Set up QEMU + # uses: docker/setup-qemu-action@v1 + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v1 + # - name: Set up AWS SDK + # uses: aws-actions/configure-aws-credentials@v1 + # with: + # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws-region: us-west-2 + # - name: Login to Amazon ECR + # id: login-ecr + # uses: aws-actions/amazon-ecr-login@v1 + # - name: Set ECR image tag + # id: image-tag + # run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" + # - name: Cache Public ECR Image + # id: lambda_python_3_9 + # uses: actions/cache@v2 + # with: + # path: ~/cache + # key: lambda_python_3_9 + # - name: Handle Cache Miss (pull public ECR image & save it to tar file) + # if: steps.cache-primes.outputs.cache-hit != 'true' + # run: | + # mkdir -p ~/cache + # docker pull public.ecr.aws/lambda/python:3.9 + # docker save public.ecr.aws/lambda/python:3.9 -o ~/cache/lambda_python_3_9.tar + # - name: Handle Cache Hit (load docker image from tar file) + # if: steps.cache-primes.outputs.cache-hit == 'true' + # run: | + # docker load -i ~/cache/lambda_python_3_9.tar + # - name: Build and push + # env: + # ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + # ECR_REPOSITORY: feast-python-server + # run: | + # docker build \ + # --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \ + # --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \ + # . + # docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} + # outputs: + # DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} integration-test-python: # all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. - if: (github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test')) - || (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved'))) - needs: build-docker-image + # if: (github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test')) + # || (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved'))) + # needs: build-docker-image runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -100,7 +100,7 @@ jobs: # pull_request_target runs the workflow in the context of the base repo # as such actions/checkout needs to be explicit configured to retrieve # code from the PR. - ref: refs/pull/${{ github.event.pull_request.number }}/merge + ref: refs/pull/3/merge submodules: recursive - name: Start Redis uses: supercharge/redis-github-action@1.4.0 @@ -113,23 +113,23 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: x64 - - name: Set up gcloud SDK - uses: google-github-actions/setup-gcloud@v0 - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - export_default_credentials: true - - name: Use gcloud CLI - run: gcloud info - - name: Set up AWS SDK - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - name: Use AWS CLI - run: aws sts get-caller-identity - - name: Upgrade pip version + # - name: Set up gcloud SDK + # uses: google-github-actions/setup-gcloud@v0 + # with: + # project_id: ${{ secrets.GCP_PROJECT_ID }} + # service_account_key: ${{ secrets.GCP_SA_KEY }} + # export_default_credentials: true + # - name: Use gcloud CLI + # run: gcloud info + # - name: Set up AWS SDK + # uses: aws-actions/configure-aws-credentials@v1 + # with: + # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws-region: us-west-2 + # - name: Use AWS CLI + # run: aws sts get-caller-identity + # - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1" - name: Get pip cache dir @@ -155,23 +155,23 @@ jobs: docker pull vishnunair/docker-redis-cluster:latest docker run -d -p 32001:6380 -p 32002:6381 -p 32003:6382 -p 32004:6383 -p 32005:6384 -p 32006:6385 --name redis-cluster vishnunair/docker-redis-cluster - name: Test python - if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak - env: - FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }} - FEAST_USAGE: "False" - IS_TEST: "True" - SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }} - SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }} - SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }} - SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }} - SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }} + # if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak + # env: + # FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }} + # FEAST_USAGE: "False" + # IS_TEST: "True" + # SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }} + # SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }} + # SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }} + # SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }} + # SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }} run: pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5 - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.xml - flags: integrationtests - env_vars: OS,PYTHON - fail_ci_if_error: true - verbose: true + # - name: Upload coverage to Codecov + # uses: codecov/codecov-action@v1 + # with: + # token: ${{ secrets.CODECOV_TOKEN }} + # files: ./coverage.xml + # flags: integrationtests + # env_vars: OS,PYTHON + # fail_ci_if_error: true + # verbose: true From 35232c3ee350cc5c5615f9f6ddc48348f376517d Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 1 Mar 2022 14:15:56 -0800 Subject: [PATCH 12/22] Continue making work Signed-off-by: Kevin Zhang --- .github/workflows/pr_integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 2c413bdf994..8a3982e2f3e 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -165,7 +165,7 @@ jobs: # SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }} # SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }} # SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }} - run: pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5 + run: pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests/integration/registration/test_cli.py::test_universal_cli --integration --universal --durations=5 # - name: Upload coverage to Codecov # uses: codecov/codecov-action@v1 # with: From 247ecb3f583f8bb465ae4ae20ee7324ea7847fc5 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 1 Mar 2022 14:18:31 -0800 Subject: [PATCH 13/22] Continue making work Signed-off-by: Kevin Zhang --- .github/workflows/pr_integration_tests.yml | 86 +--------------------- 1 file changed, 2 insertions(+), 84 deletions(-) diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 8a3982e2f3e..b37768b113f 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -1,11 +1,7 @@ name: pr-integration-tests -on: - pull_request_target: - types: - - opened - - synchronize - - labeled +on: [push, pull_request] + # concurrency is currently broken, see details https://github.com/actions/runner/issues/1532 #concurrency: @@ -13,68 +9,7 @@ on: # cancel-in-progress: true jobs: - # build-docker-image: - # # all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. - # if: (github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test')) - # || (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved'))) - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v2 - # with: - # # pull_request_target runs the workflow in the context of the base repo - # # as such actions/checkout needs to be explicit configured to retrieve - # # code from the PR. - # ref: refs/pull/${{ github.event.pull_request.number }}/merge - # submodules: recursive - # - name: Set up QEMU - # uses: docker/setup-qemu-action@v1 - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v1 - # - name: Set up AWS SDK - # uses: aws-actions/configure-aws-credentials@v1 - # with: - # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws-region: us-west-2 - # - name: Login to Amazon ECR - # id: login-ecr - # uses: aws-actions/amazon-ecr-login@v1 - # - name: Set ECR image tag - # id: image-tag - # run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" - # - name: Cache Public ECR Image - # id: lambda_python_3_9 - # uses: actions/cache@v2 - # with: - # path: ~/cache - # key: lambda_python_3_9 - # - name: Handle Cache Miss (pull public ECR image & save it to tar file) - # if: steps.cache-primes.outputs.cache-hit != 'true' - # run: | - # mkdir -p ~/cache - # docker pull public.ecr.aws/lambda/python:3.9 - # docker save public.ecr.aws/lambda/python:3.9 -o ~/cache/lambda_python_3_9.tar - # - name: Handle Cache Hit (load docker image from tar file) - # if: steps.cache-primes.outputs.cache-hit == 'true' - # run: | - # docker load -i ~/cache/lambda_python_3_9.tar - # - name: Build and push - # env: - # ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - # ECR_REPOSITORY: feast-python-server - # run: | - # docker build \ - # --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \ - # --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \ - # . - # docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} - # outputs: - # DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} integration-test-python: - # all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. - # if: (github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test')) - # || (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved'))) - # needs: build-docker-image runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -113,23 +48,6 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: x64 - # - name: Set up gcloud SDK - # uses: google-github-actions/setup-gcloud@v0 - # with: - # project_id: ${{ secrets.GCP_PROJECT_ID }} - # service_account_key: ${{ secrets.GCP_SA_KEY }} - # export_default_credentials: true - # - name: Use gcloud CLI - # run: gcloud info - # - name: Set up AWS SDK - # uses: aws-actions/configure-aws-credentials@v1 - # with: - # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws-region: us-west-2 - # - name: Use AWS CLI - # run: aws sts get-caller-identity - # - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1" - name: Get pip cache dir From 9d921e8c6da0b23219a350d3610cc6183349ee8c Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 1 Mar 2022 14:29:33 -0800 Subject: [PATCH 14/22] Fix workflow Signed-off-by: Kevin Zhang --- .github/workflows/pr_integration_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index b37768b113f..39bd8f5463d 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -48,6 +48,7 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: x64 + - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1" - name: Get pip cache dir From c7ae586e7b3ba019cda5310b32c0180e01149152 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 1 Mar 2022 14:33:55 -0800 Subject: [PATCH 15/22] Fix workflow Signed-off-by: Kevin Zhang --- .../feature_repos/repo_configuration.py | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 77b7c0cd46d..5ac3d7da197 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -71,37 +71,37 @@ DEFAULT_FULL_REPO_CONFIGS.extend( [ # Redis configurations - IntegrationTestRepoConfig(online_store=REDIS_CONFIG), + #IntegrationTestRepoConfig(online_store=REDIS_CONFIG), IntegrationTestRepoConfig(online_store=REDIS_CLUSTER_CONFIG), # GCP configurations - IntegrationTestRepoConfig( - provider="gcp", - offline_store_creator=BigQueryDataSourceCreator, - online_store="datastore", - ), - IntegrationTestRepoConfig( - provider="gcp", - offline_store_creator=BigQueryDataSourceCreator, - online_store=REDIS_CONFIG, - ), - # AWS configurations - IntegrationTestRepoConfig( - provider="aws", - offline_store_creator=RedshiftDataSourceCreator, - online_store=DYNAMO_CONFIG, - python_feature_server=True, - ), - IntegrationTestRepoConfig( - provider="aws", - offline_store_creator=RedshiftDataSourceCreator, - online_store=REDIS_CONFIG, - ), - # Snowflake configurations - IntegrationTestRepoConfig( - provider="aws", # no list features, no feature server - offline_store_creator=SnowflakeDataSourceCreator, - online_store=REDIS_CONFIG, - ), + # IntegrationTestRepoConfig( + # provider="gcp", + # offline_store_creator=BigQueryDataSourceCreator, + # online_store="datastore", + # ), + # IntegrationTestRepoConfig( + # provider="gcp", + # offline_store_creator=BigQueryDataSourceCreator, + # online_store=REDIS_CONFIG, + # ), + # # AWS configurations + # IntegrationTestRepoConfig( + # provider="aws", + # offline_store_creator=RedshiftDataSourceCreator, + # online_store=DYNAMO_CONFIG, + # python_feature_server=True, + # ), + # IntegrationTestRepoConfig( + # provider="aws", + # offline_store_creator=RedshiftDataSourceCreator, + # online_store=REDIS_CONFIG, + # ), + # # Snowflake configurations + # IntegrationTestRepoConfig( + # provider="aws", # no list features, no feature server + # offline_store_creator=SnowflakeDataSourceCreator, + # online_store=REDIS_CONFIG, + # ), ] ) full_repo_configs_module = os.environ.get(FULL_REPO_CONFIGS_MODULE_ENV_NAME) From 659a3dbbe66687612631782abe230913e5dfd790 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 1 Mar 2022 15:08:19 -0800 Subject: [PATCH 16/22] Update port numbers Signed-off-by: Kevin Zhang --- .../tests/integration/feature_repos/repo_configuration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 5ac3d7da197..7ed50fce1fe 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -47,12 +47,12 @@ DYNAMO_CONFIG = {"type": "dynamodb", "region": "us-west-2"} # Port 12345 will chosen as default for redis node configuration because Redis Cluster is started off of nodes 6379 -> 6384. This causes conflicts in cli integration tests so we manually keep them separate. -REDIS_CONFIG = {"type": "redis", "connection_string": "localhost:6379,db=0"} +REDIS_CONFIG = {"type": "redis", "connection_string": "localhost:12345,db=0"} REDIS_CLUSTER_CONFIG = { "type": "redis", "redis_type": "redis_cluster", # Redis Cluster Port Forwarding is setup in "pr_integration_tests.yaml" under "Setup Redis Cluster". - "connection_string": "127.0.0.1:6001,127.0.0.1:6002,127.0.0.1:6003", + "connection_string": "127.0.0.1:32001,127.0.0.1:32002,127.0.0.1:32003", } # FULL_REPO_CONFIGS contains the repo configurations (e.g. provider, offline store, From ce380715510cfbbe6e1f3288b07e4f1fcfee8522 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 1 Mar 2022 17:20:59 -0800 Subject: [PATCH 17/22] Update pr integration tests Signed-off-by: Kevin Zhang --- .github/workflows/pr_integration_tests.yml | 129 ++++++++++++++---- .../feature_repos/repo_configuration.py | 2 +- 2 files changed, 106 insertions(+), 25 deletions(-) diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 39bd8f5463d..38df0b2d4f1 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -1,7 +1,11 @@ name: pr-integration-tests -on: [push, pull_request] - +on: + pull_request_target: + types: + - opened + - synchronize + - labeled # concurrency is currently broken, see details https://github.com/actions/runner/issues/1532 #concurrency: @@ -9,7 +13,68 @@ on: [push, pull_request] # cancel-in-progress: true jobs: + build-docker-image: + # all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. + if: (github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test')) + || (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved'))) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + # pull_request_target runs the workflow in the context of the base repo + # as such actions/checkout needs to be explicit configured to retrieve + # code from the PR. + ref: refs/pull/${{ github.event.pull_request.number }}/merge + submodules: recursive + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Set up AWS SDK + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + - name: Set ECR image tag + id: image-tag + run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" + - name: Cache Public ECR Image + id: lambda_python_3_9 + uses: actions/cache@v2 + with: + path: ~/cache + key: lambda_python_3_9 + - name: Handle Cache Miss (pull public ECR image & save it to tar file) + if: steps.cache-primes.outputs.cache-hit != 'true' + run: | + mkdir -p ~/cache + docker pull public.ecr.aws/lambda/python:3.9 + docker save public.ecr.aws/lambda/python:3.9 -o ~/cache/lambda_python_3_9.tar + - name: Handle Cache Hit (load docker image from tar file) + if: steps.cache-primes.outputs.cache-hit == 'true' + run: | + docker load -i ~/cache/lambda_python_3_9.tar + - name: Build and push + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: feast-python-server + run: | + docker build \ + --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \ + --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \ + . + docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} + outputs: + DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} integration-test-python: + # all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. + if: (github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test')) + || (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved'))) + needs: build-docker-image runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -35,7 +100,7 @@ jobs: # pull_request_target runs the workflow in the context of the base repo # as such actions/checkout needs to be explicit configured to retrieve # code from the PR. - ref: refs/pull/3/merge + ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive - name: Start Redis uses: supercharge/redis-github-action@1.4.0 @@ -48,6 +113,22 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: x64 + - name: Set up gcloud SDK + uses: google-github-actions/setup-gcloud@v0 + with: + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + export_default_credentials: true + - name: Use gcloud CLI + run: gcloud info + - name: Set up AWS SDK + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - name: Use AWS CLI + run: aws sts get-caller-identity - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1" @@ -72,25 +153,25 @@ jobs: - name: Setup Redis Cluster run: | docker pull vishnunair/docker-redis-cluster:latest - docker run -d -p 32001:6380 -p 32002:6381 -p 32003:6382 -p 32004:6383 -p 32005:6384 -p 32006:6385 --name redis-cluster vishnunair/docker-redis-cluster + docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster - name: Test python - # if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak - # env: - # FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }} - # FEAST_USAGE: "False" - # IS_TEST: "True" - # SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }} - # SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }} - # SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }} - # SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }} - # SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }} - run: pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests/integration/registration/test_cli.py::test_universal_cli --integration --universal --durations=5 - # - name: Upload coverage to Codecov - # uses: codecov/codecov-action@v1 - # with: - # token: ${{ secrets.CODECOV_TOKEN }} - # files: ./coverage.xml - # flags: integrationtests - # env_vars: OS,PYTHON - # fail_ci_if_error: true - # verbose: true + if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak + env: + FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }} + FEAST_USAGE: "False" + IS_TEST: "True" + SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }} + SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }} + SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }} + SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }} + SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }} + run: pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5 + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml + flags: integrationtests + env_vars: OS,PYTHON + fail_ci_if_error: true + verbose: true \ No newline at end of file diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 7ed50fce1fe..f29cc062404 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -52,7 +52,7 @@ "type": "redis", "redis_type": "redis_cluster", # Redis Cluster Port Forwarding is setup in "pr_integration_tests.yaml" under "Setup Redis Cluster". - "connection_string": "127.0.0.1:32001,127.0.0.1:32002,127.0.0.1:32003", + "connection_string": "127.0.0.1:6001,127.0.0.1:6002,127.0.0.1:6003", } # FULL_REPO_CONFIGS contains the repo configurations (e.g. provider, offline store, From fbe94f19c7d4c2c774b2850b2a0c4d29a600ca10 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 1 Mar 2022 17:23:15 -0800 Subject: [PATCH 18/22] Reenable rest of tests Signed-off-by: Kevin Zhang --- .../feature_repos/repo_configuration.py | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index f29cc062404..89aea727a6e 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -71,37 +71,37 @@ DEFAULT_FULL_REPO_CONFIGS.extend( [ # Redis configurations - #IntegrationTestRepoConfig(online_store=REDIS_CONFIG), + IntegrationTestRepoConfig(online_store=REDIS_CONFIG), IntegrationTestRepoConfig(online_store=REDIS_CLUSTER_CONFIG), # GCP configurations - # IntegrationTestRepoConfig( - # provider="gcp", - # offline_store_creator=BigQueryDataSourceCreator, - # online_store="datastore", - # ), - # IntegrationTestRepoConfig( - # provider="gcp", - # offline_store_creator=BigQueryDataSourceCreator, - # online_store=REDIS_CONFIG, - # ), - # # AWS configurations - # IntegrationTestRepoConfig( - # provider="aws", - # offline_store_creator=RedshiftDataSourceCreator, - # online_store=DYNAMO_CONFIG, - # python_feature_server=True, - # ), - # IntegrationTestRepoConfig( - # provider="aws", - # offline_store_creator=RedshiftDataSourceCreator, - # online_store=REDIS_CONFIG, - # ), - # # Snowflake configurations - # IntegrationTestRepoConfig( - # provider="aws", # no list features, no feature server - # offline_store_creator=SnowflakeDataSourceCreator, - # online_store=REDIS_CONFIG, - # ), + IntegrationTestRepoConfig( + provider="gcp", + offline_store_creator=BigQueryDataSourceCreator, + online_store="datastore", + ), + IntegrationTestRepoConfig( + provider="gcp", + offline_store_creator=BigQueryDataSourceCreator, + online_store=REDIS_CONFIG, + ), + # AWS configurations + IntegrationTestRepoConfig( + provider="aws", + offline_store_creator=RedshiftDataSourceCreator, + online_store=DYNAMO_CONFIG, + python_feature_server=True, + ), + IntegrationTestRepoConfig( + provider="aws", + offline_store_creator=RedshiftDataSourceCreator, + online_store=REDIS_CONFIG, + ), + # Snowflake configurations + IntegrationTestRepoConfig( + provider="aws", # no list features, no feature server + offline_store_creator=SnowflakeDataSourceCreator, + online_store=REDIS_CONFIG, + ), ] ) full_repo_configs_module = os.environ.get(FULL_REPO_CONFIGS_MODULE_ENV_NAME) From 43fe9138484944993c7de263a6bf01abb9c2a70b Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 1 Mar 2022 17:26:57 -0800 Subject: [PATCH 19/22] Update data source creator Signed-off-by: Kevin Zhang --- sdk/python/tests/conftest.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index d5bea0f82e2..9de146ac12a 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -199,20 +199,12 @@ def cleanup(): @pytest.fixture(scope="session") def redis_universal_data_sources(request, local_redis_environment): - entities = construct_universal_entities() - datasets = construct_universal_datasets( - entities, local_redis_environment.start_date, local_redis_environment.end_date - ) - datasources = construct_universal_data_sources( - datasets, local_redis_environment.data_source_creator - ) - def cleanup(): # logger.info("Running cleanup in %s, Request: %s", worker_id, request.param) local_redis_environment.data_source_creator.teardown() request.addfinalizer(cleanup) - return entities, datasets, datasources + return construct_universal_test_data(local_redis_environment) @pytest.fixture(scope="session") From e5fedcc9dd5702d8b11147ce932a771d943cca0d Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 1 Mar 2022 18:00:03 -0800 Subject: [PATCH 20/22] Fix integration test Signed-off-by: Kevin Zhang --- sdk/python/setup.py | 2 +- .../tests/integration/feature_repos/repo_configuration.py | 2 +- .../tests/integration/online_store/test_universal_online.py | 4 +--- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/sdk/python/setup.py b/sdk/python/setup.py index 7c74e1e1129..80daff00ebe 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -78,7 +78,7 @@ ] REDIS_REQUIRED = [ - "redis>=3.5.3", + "redis==3.5.3", "redis-py-cluster>=2.1.3", "hiredis>=2.0.0", ] diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 89aea727a6e..d7e4231d7fa 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -52,7 +52,7 @@ "type": "redis", "redis_type": "redis_cluster", # Redis Cluster Port Forwarding is setup in "pr_integration_tests.yaml" under "Setup Redis Cluster". - "connection_string": "127.0.0.1:6001,127.0.0.1:6002,127.0.0.1:6003", + "connection_string": "127.0.0.1:7001,127.0.0.1:7002,127.0.0.1:7003", } # FULL_REPO_CONFIGS contains the repo configurations (e.g. provider, offline store, diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index 28b4ca96426..90a06bb3477 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -43,9 +43,7 @@ def test_entity_ttl_online_store(local_redis_environment, redis_universal_data_s # setting ttl setting in online store to 1 second fs.config.online_store.key_ttl_seconds = 1 entities, datasets, data_sources = redis_universal_data_sources - driver_hourly_stats = create_driver_hourly_stats_feature_view( - data_sources["driver"] - ) + driver_hourly_stats = create_driver_hourly_stats_feature_view(data_sources.driver) driver_entity = driver() # Register Feature View and Entity From a321d54a101b09d533af2d1c34cbadc41e3ba562 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 1 Mar 2022 18:03:58 -0800 Subject: [PATCH 21/22] Clean up code Signed-off-by: Kevin Zhang --- .github/workflows/pr_integration_tests.yml | 11 +---------- .../integration/feature_repos/repo_configuration.py | 2 +- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 38df0b2d4f1..e04b78ec320 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -102,11 +102,6 @@ jobs: # code from the PR. ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive - - name: Start Redis - uses: supercharge/redis-github-action@1.4.0 - with: - redis-version: ${{ matrix.redis-version }} - redis-port: 12345 - name: Setup Python uses: actions/setup-python@v2 id: setup-python @@ -150,10 +145,6 @@ jobs: run: pip install pip-tools - name: Install dependencies run: make install-python-ci-dependencies - - name: Setup Redis Cluster - run: | - docker pull vishnunair/docker-redis-cluster:latest - docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster - name: Test python if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak env: @@ -174,4 +165,4 @@ jobs: flags: integrationtests env_vars: OS,PYTHON fail_ci_if_error: true - verbose: true \ No newline at end of file + verbose: true diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index d7e4231d7fa..89aea727a6e 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -52,7 +52,7 @@ "type": "redis", "redis_type": "redis_cluster", # Redis Cluster Port Forwarding is setup in "pr_integration_tests.yaml" under "Setup Redis Cluster". - "connection_string": "127.0.0.1:7001,127.0.0.1:7002,127.0.0.1:7003", + "connection_string": "127.0.0.1:6001,127.0.0.1:6002,127.0.0.1:6003", } # FULL_REPO_CONFIGS contains the repo configurations (e.g. provider, offline store, From 4a0d350b8c9a4e559b0fdbc8811a87466dc5e6b9 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 1 Mar 2022 18:50:22 -0800 Subject: [PATCH 22/22] Somehow removed the key components -,- Signed-off-by: Kevin Zhang --- .github/workflows/pr_integration_tests.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index e04b78ec320..4c758d7601a 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -84,16 +84,6 @@ jobs: env: OS: ${{ matrix.os }} PYTHON: ${{ matrix.python-version }} - services: - redis: - image: redis - ports: - - 6379:6379 - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 steps: - uses: actions/checkout@v2 with: @@ -102,6 +92,11 @@ jobs: # code from the PR. ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive + - name: Start Redis + uses: supercharge/redis-github-action@1.4.0 + with: + redis-version: ${{ matrix.redis-version }} + redis-port: 12345 - name: Setup Python uses: actions/setup-python@v2 id: setup-python @@ -145,6 +140,10 @@ jobs: run: pip install pip-tools - name: Install dependencies run: make install-python-ci-dependencies + - name: Setup Redis Cluster + run: | + docker pull vishnunair/docker-redis-cluster:latest + docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster - name: Test python if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak env: