diff --git a/.github/workflows/registry-rest-api-tests.yml b/.github/workflows/registry-rest-api-tests.yml index 5eddd68a539..cd679c7044b 100644 --- a/.github/workflows/registry-rest-api-tests.yml +++ b/.github/workflows/registry-rest-api-tests.yml @@ -145,8 +145,8 @@ jobs: - name: Setup and Run Registry Rest API tests run: | echo "Running Registry REST API tests..." - cd sdk/python/tests/registry_rest_api_tests/ - pytest test_feast_registry.py -s + cd sdk/python + pytest tests/integration/registration/rest_api/test_registry_rest_api.py --integration -s - name: Clean up docker images if: always() diff --git a/docs/reference/feature-servers/registry-server.md b/docs/reference/feature-servers/registry-server.md index dbcf661fc86..496eaa8badc 100644 --- a/docs/reference/feature-servers/registry-server.md +++ b/docs/reference/feature-servers/registry-server.md @@ -1207,28 +1207,33 @@ Please refer the [page](./../../../docs/getting-started/concepts/permission.md) "name": "user_id", "description": "Primary identifier for users", "project": "project1", - "match_score": 100 + "match_score": 100, + "matched_tags": {} }, { "type": "featureView", "name": "user_features", "description": "User demographic and behavioral features", "project": "project1", - "match_score": 100 + "match_score": 100, + "matched_tags": {"team": "user_analytics"} }, { "type": "feature", "name": "user_age", "description": "Age of the user in years", "project": "project1", - "match_score": 80 + "featureView": "user_features", + "match_score": 80, + "matched_tags": {} }, { "type": "dataSource", "name": "user_analytics", "description": "Analytics data for user behavior tracking", "project": "project2", - "match_score": 80 + "match_score": 80, + "matched_tags": {"source": "user_data"} } ], "pagination": { diff --git a/go.mod b/go.mod index a097aa67719..5b2592536df 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.24.0 toolchain go1.24.4 require ( + cloud.google.com/go/storage v1.58.0 github.com/apache/arrow/go/v17 v17.0.0 github.com/aws/aws-sdk-go-v2 v1.36.4 github.com/aws/aws-sdk-go-v2/config v1.29.14 @@ -25,13 +26,23 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 go.opentelemetry.io/otel/sdk v1.38.0 go.opentelemetry.io/otel/trace v1.38.0 - golang.org/x/sync v0.17.0 - google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 - google.golang.org/grpc v1.75.0 - google.golang.org/protobuf v1.36.8 + golang.org/x/sync v0.18.0 + google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba + google.golang.org/grpc v1.76.0 + google.golang.org/protobuf v1.36.10 ) require ( + cel.dev/expr v0.24.0 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.17.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/monitoring v1.24.2 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.54.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.54.0 // indirect github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/apache/thrift v0.21.0 // indirect @@ -53,13 +64,21 @@ require ( github.com/aws/smithy-go v1.22.2 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect + github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-jose/go-jose/v4 v4.1.2 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v24.3.25+incompatible // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect + github.com/googleapis/gax-go/v2 v2.15.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.17.9 // indirect @@ -69,20 +88,33 @@ require ( github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect github.com/stretchr/objx v0.5.2 // indirect + github.com/zeebo/errs v1.4.0 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect go.opentelemetry.io/proto/otlp v1.7.1 // indirect + golang.org/x/crypto v0.45.0 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect - golang.org/x/mod v0.26.0 // indirect - golang.org/x/net v0.43.0 // indirect - golang.org/x/sys v0.35.0 // indirect - golang.org/x/text v0.28.0 // indirect - golang.org/x/tools v0.35.0 // indirect + golang.org/x/mod v0.29.0 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/oauth2 v0.33.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 // indirect + golang.org/x/text v0.31.0 // indirect + golang.org/x/time v0.14.0 // indirect + golang.org/x/tools v0.38.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/api v0.256.0 // indirect + google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index f2530758915..aff636cad1b 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,33 @@ +cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= +cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.17.0 h1:74yCm7hCj2rUyyAocqnFzsAYXgJhrG26XCFimrc/Kz4= +cloud.google.com/go/auth v0.17.0/go.mod h1:6wv/t5/6rOPAX4fJiRjKkJCvswLwdet7G8+UGXt7nCQ= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= +cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= +cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= +cloud.google.com/go/longrunning v0.7.0 h1:FV0+SYF1RIj59gyoWDRi45GiYUMM3K1qO51qoboQT1E= +cloud.google.com/go/longrunning v0.7.0/go.mod h1:ySn2yXmjbK9Ba0zsQqunhDkYi0+9rlXIwnoAf+h+TPY= +cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM= +cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U= +cloud.google.com/go/storage v1.58.0 h1:PflFXlmFJjG/nBeR9B7pKddLQWaFaRWx4uUi/LyNxxo= +cloud.google.com/go/storage v1.58.0/go.mod h1:cMWbtM+anpC74gn6qjLh+exqYcfmB9Hqe5z6adx+CLI= +cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4= +cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0/go.mod h1:Cz6ft6Dkn3Et6l2v2a9/RpN7epQ1GtDlO6lj8bEcOvw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.54.0 h1:lhhYARPUu3LmHysQ/igznQphfzynnqI3D75oUyw1HXk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.54.0/go.mod h1:l9rva3ApbBpEJxSNYnwT9N4CDLrWgtq3u8736C5hyJw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.54.0 h1:xfK3bbi6F2RDtaZFtUdKO3osOBIhNb+xTs8lFW6yx9o= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.54.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.54.0 h1:s0WlVbf9qpvkh1c/uDAPElam0WrL7fHRIidgZJ7UqZI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.54.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= @@ -54,13 +84,27 @@ github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1x github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= +github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= +github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= +github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-jose/go-jose/v4 v4.1.2 h1:TK/7NqRQZfgAh+Td8AlsrvtPoUyiHh0LqVvokh+1vHI= +github.com/go-jose/go-jose/v4 v4.1.2/go.mod h1:22cg9HWM1pOlnRiY+9cQYJ9XHmya1bYW8OeDM6Ku6Oo= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -77,8 +121,16 @@ github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81A github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ= +github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo= +github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= @@ -107,35 +159,49 @@ github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4= github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA= github.com/roberson-io/mmh3 v0.0.0-20190729202758-fdfce3ba6225 h1:ZMsPCp7oYgjoIFt1c+sM2qojxZXotSYcMF8Ur9/LJlM= github.com/roberson-io/mmh3 v0.0.0-20190729202758-fdfce3ba6225/go.mod h1:XEESr+X1SY8ZSuc3jqsTlb3clCkqQJ4DcF3Qxv1N3PM= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= +github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= +github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0 h1:F7q2tNlCaHY9nMKHR6XH9/qkp8FktLnIcy6jJNyOCQw= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0/go.mod h1:IbBN8uAIIx734PTonTPxAxnjc2pQTxWNkwfstZ+6H2k= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0 h1:wm/Q0GAAykXv83wzcKzGGqAnnfLFyFe7RslekZuv+VI= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0/go.mod h1:ra3Pa40+oKjvYh+ZD3EdxFZZB0xdMfuileHAm4nNN7w= go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= @@ -148,36 +214,48 @@ go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOV go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= -golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= -golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/oauth2 v0.33.0 h1:4Q+qn+E5z8gPRJfmRy7C2gGG3T4jIprK6aSYgTXGRpo= +golang.org/x/oauth2 v0.33.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= -golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= -golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 h1:LvzTn0GQhWuvKH/kVRS3R3bVAsdQWI7hvfLHGgh9+lU= +golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8/go.mod h1:Pi4ztBfryZoJEkyFTI5/Ocsu2jXyDr6iSdgJiYE/uwE= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= -google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= -google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= -google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= -google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= -google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +google.golang.org/api v0.256.0 h1:u6Khm8+F9sxbCTYNoBHg6/Hwv0N/i+V94MvkOSor6oI= +google.golang.org/api v0.256.0/go.mod h1:KIgPhksXADEKJlnEoRa9qAII4rXcy40vfI8HRqcU964= +google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 h1:LvZVVaPE0JSqL+ZWb6ErZfnEOKIqqFWUJE2D0fObSmc= +google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9/go.mod h1:QFOrLhdAe2PsTp3vQY4quuLKTi9j3XG3r6JPPaw7MSc= +google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba h1:B14OtaXuMaCQsl2deSvNkyPKIzq3BjfxQp8d00QyWx4= +google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:G5IanEx8/PgI9w6CFcYQf7jMtHQhZruvfM1i3qOqk5U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba h1:UKgtfRM7Yh93Sya0Fo8ZzhDP4qBckrrxEr2oF5UIVb8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A= +google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/go/internal/feast/registry/gcs.go b/go/internal/feast/registry/gcs.go new file mode 100644 index 00000000000..00968a55ae3 --- /dev/null +++ b/go/internal/feast/registry/gcs.go @@ -0,0 +1,114 @@ +package registry + +import ( + "context" + "errors" + "io" + "net/url" + "strings" + "time" + + "cloud.google.com/go/storage" + "google.golang.org/protobuf/proto" + + "github.com/feast-dev/feast/go/protos/feast/core" +) + +// GCSObjectReader defines the interface for reading GCS objects to allow mocking in tests. +type GCSObjectReader interface { + GetObject(ctx context.Context, bucket string, object string) (io.ReadCloser, error) + DeleteObject(ctx context.Context, bucket string, object string) error +} + +// GCSClient implements GCSObjectReader using the real GCS SDK. +type GCSClient struct { + client *storage.Client +} + +func (g *GCSClient) GetObject(ctx context.Context, bucket string, object string) (io.ReadCloser, error) { + return g.client.Bucket(bucket).Object(object).NewReader(ctx) +} + +func (g *GCSClient) DeleteObject(ctx context.Context, bucket string, object string) error { + return g.client.Bucket(bucket).Object(object).Delete(ctx) +} + +// GCSRegistryStore is a GCS bucket-based implementation of the RegistryStore interface. +type GCSRegistryStore struct { + registryPath string + client GCSObjectReader +} + +// NewGCSRegistryStore creates a GCSRegistryStore with the given configuration. +func NewGCSRegistryStore(config *RegistryConfig, repoPath string) *GCSRegistryStore { + var rs GCSRegistryStore + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + client, err := storage.NewClient(ctx) + if err != nil { + rs = GCSRegistryStore{ + registryPath: config.Path, + } + } else { + rs = GCSRegistryStore{ + registryPath: config.Path, + client: &GCSClient{client: client}, + } + } + return &rs +} + +// GetRegistryProto reads and parses the registry proto from the GCS bucket object. +func (g *GCSRegistryStore) GetRegistryProto() (*core.Registry, error) { + bucket, object, err := g.parseGCSPath() + if err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + reader, err := g.client.GetObject(ctx, bucket, object) + if err != nil { + return nil, err + } + defer reader.Close() + + data, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + + registry := &core.Registry{} + if err := proto.Unmarshal(data, registry); err != nil { + return nil, err + } + return registry, nil +} + +func (g *GCSRegistryStore) UpdateRegistryProto(rp *core.Registry) error { + return errors.New("not implemented in GCSRegistryStore") +} + +func (g *GCSRegistryStore) Teardown() error { + bucket, object, err := g.parseGCSPath() + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + return g.client.DeleteObject(ctx, bucket, object) +} + +func (g *GCSRegistryStore) parseGCSPath() (string, string, error) { + uri, err := url.Parse(g.registryPath) + if err != nil { + return "", "", errors.New("invalid GCS registry path format") + } + bucket := uri.Host + object := strings.TrimPrefix(uri.Path, "/") + return bucket, object, nil +} diff --git a/go/internal/feast/registry/registry.go b/go/internal/feast/registry/registry.go index 160dda94fd6..9cd0febe5d3 100644 --- a/go/internal/feast/registry/registry.go +++ b/go/internal/feast/registry/registry.go @@ -357,7 +357,7 @@ func getRegistryStoreFromScheme(registryPath string, registryConfig *RegistryCon if registryStoreType, ok := REGISTRY_STORE_CLASS_FOR_SCHEME[uri.Scheme]; ok { return getRegistryStoreFromType(registryStoreType, registryConfig, repoPath, project) } - return nil, fmt.Errorf("registry path %s has unsupported scheme %s. Supported schemes are file, s3 and gs", registryPath, uri.Scheme) + return nil, fmt.Errorf("registry path %s has unsupported scheme %s. Supported schemes are file, s3 and gcs", registryPath, uri.Scheme) } func getRegistryStoreFromType(registryStoreType string, registryConfig *RegistryConfig, repoPath string, project string) (RegistryStore, error) { @@ -366,6 +366,8 @@ func getRegistryStoreFromType(registryStoreType string, registryConfig *Registry return NewFileRegistryStore(registryConfig, repoPath), nil case "S3RegistryStore": return NewS3RegistryStore(registryConfig, repoPath), nil + case "GCSRegistryStore": + return NewGCSRegistryStore(registryConfig, repoPath), nil } - return nil, errors.New("only FileRegistryStore as a RegistryStore is supported at this moment") + return nil, errors.New("only FileRegistryStore, S3RegistryStore, and GCSRegistryStore are supported at this moment") } diff --git a/go/internal/feast/registry/registry_test.go b/go/internal/feast/registry/registry_test.go index 3e544d486e5..6f75dbbbeb2 100644 --- a/go/internal/feast/registry/registry_test.go +++ b/go/internal/feast/registry/registry_test.go @@ -12,7 +12,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" ) -func TestGetOnlineFeaturesS3Registry(t *testing.T) { +func TestCloudRegistryStores(t *testing.T) { mockS3Client := &MockS3Client{ GetObjectFn: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) { return &s3.GetObjectOutput{ @@ -24,56 +24,82 @@ func TestGetOnlineFeaturesS3Registry(t *testing.T) { }, } + mockGCSClient := &MockGCSClient{ + GetObjectFn: func(ctx context.Context, bucket string, object string) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader("mock data")), nil + }, + DeleteObjectFn: func(ctx context.Context, bucket string, object string) error { + return nil + }, + } + tests := []struct { name string config *RepoConfig }{ { - name: "redis with simple features", + name: "s3 registry store", config: &RepoConfig{ Project: "feature_repo", - Registry: map[string]interface{}{ + Registry: map[string]any{ "path": "s3://test-bucket/path/to/registry.db", }, Provider: "aws", }, }, + { + name: "gcs registry store", + config: &RepoConfig{ + Project: "feature_repo", + Registry: map[string]any{ + "path": "gs://test-bucket/path/to/registry.db", + }, + Provider: "gcp", + }, + }, } + for _, test := range tests { - registryConfig, err := test.config.GetRegistryConfig() - if err != nil { - t.Errorf("Error getting registry config. msg: %s", err.Error()) - } - r := &Registry{ - project: test.config.Project, - cachedRegistryProtoTtl: time.Duration(registryConfig.CacheTtlSeconds) * time.Second, - } - _ = registryConfig.RegistryStoreType - registryPath := registryConfig.Path - uri, err := url.Parse(registryPath) - if err != nil { - t.Errorf("Error parsing registry path. msg: %s", err.Error()) - } - if registryStoreType, ok := REGISTRY_STORE_CLASS_FOR_SCHEME[uri.Scheme]; ok { - switch registryStoreType { - case "S3RegistryStore": - registryStore := &S3RegistryStore{ - filePath: registryConfig.Path, - s3Client: mockS3Client, + t.Run(test.name, func(t *testing.T) { + registryConfig, err := test.config.GetRegistryConfig() + if err != nil { + t.Errorf("Error getting registry config. msg: %s", err.Error()) + } + r := &Registry{ + project: test.config.Project, + cachedRegistryProtoTtl: time.Duration(registryConfig.CacheTtlSeconds) * time.Second, + } + registryPath := registryConfig.Path + uri, err := url.Parse(registryPath) + if err != nil { + t.Errorf("Error parsing registry path. msg: %s", err.Error()) + } + if registryStoreType, ok := REGISTRY_STORE_CLASS_FOR_SCHEME[uri.Scheme]; ok { + switch registryStoreType { + case "S3RegistryStore": + r.registryStore = &S3RegistryStore{ + filePath: registryConfig.Path, + s3Client: mockS3Client, + } + case "GCSRegistryStore": + r.registryStore = &GCSRegistryStore{ + registryPath: registryConfig.Path, + client: mockGCSClient, + } + default: + t.Errorf("Unsupported registry store type: %s", registryStoreType) + return } - r.registryStore = registryStore err := r.InitializeRegistry() if err != nil { t.Errorf("Error initializing registry. msg: %s. registry path=%q", err.Error(), registryPath) } - default: - t.Errorf("Only S3RegistryStore is supported on this testing. got=%s", registryStoreType) } - } + }) } } -// MockS3Client is mock client for testing s3 registry store +// MockS3Client is mock client for testing S3 registry store type MockS3Client struct { GetObjectFn func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) DeleteObjectFn func(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) @@ -92,3 +118,23 @@ func (m *MockS3Client) DeleteObject(ctx context.Context, params *s3.DeleteObject } return nil, errors.New("not implemented") } + +// MockGCSClient is mock client for testing GCS registry store +type MockGCSClient struct { + GetObjectFn func(ctx context.Context, bucket string, object string) (io.ReadCloser, error) + DeleteObjectFn func(ctx context.Context, bucket string, object string) error +} + +func (m *MockGCSClient) GetObject(ctx context.Context, bucket string, object string) (io.ReadCloser, error) { + if m.GetObjectFn != nil { + return m.GetObjectFn(ctx, bucket, object) + } + return nil, errors.New("not implemented") +} + +func (m *MockGCSClient) DeleteObject(ctx context.Context, bucket string, object string) error { + if m.DeleteObjectFn != nil { + return m.DeleteObjectFn(ctx, bucket, object) + } + return errors.New("not implemented") +} diff --git a/infra/feast-operator/test/e2e_rhoai/e2e_suite_test.go b/infra/feast-operator/test/e2e_rhoai/e2e_suite_test.go deleted file mode 100644 index 86750f36e4f..00000000000 --- a/infra/feast-operator/test/e2e_rhoai/e2e_suite_test.go +++ /dev/null @@ -1,32 +0,0 @@ -/* -Copyright 2025 Feast Community. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2erhoai - -import ( - "fmt" - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -// Run e2e feast Notebook tests using the Ginkgo runner. -func TestNotebookRunE2E(t *testing.T) { - RegisterFailHandler(Fail) - _, _ = fmt.Fprintf(GinkgoWriter, "Feast Jupyter Notebook Test suite\n") - RunSpecs(t, "e2erhoai Feast Notebook test suite") -} diff --git a/infra/feast-operator/test/e2e_rhoai/feast_postupgrade_test.go b/infra/feast-operator/test/e2e_rhoai/feast_postupgrade_test.go deleted file mode 100644 index d8d091a44b8..00000000000 --- a/infra/feast-operator/test/e2e_rhoai/feast_postupgrade_test.go +++ /dev/null @@ -1,56 +0,0 @@ -/* -Copyright 2025 Feast Community. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2erhoai - -import ( - "fmt" - - . "github.com/feast-dev/feast/infra/feast-operator/test/utils" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("Feast PostUpgrade scenario Testing", Ordered, func() { - const ( - namespace = "test-ns-feast-upgrade" - testDir = "/test/e2e_rhoai" - feastDeploymentName = FeastPrefix + "credit-scoring" - feastCRName = "credit-scoring" - ) - - AfterAll(func() { - By(fmt.Sprintf("Deleting test namespace: %s", namespace)) - Expect(DeleteNamespace(namespace, testDir)).To(Succeed()) - fmt.Printf("Namespace %s deleted successfully\n", namespace) - }) - runPostUpgradeTest := func() { - By("Verify Feature Store CR is in Ready state") - ValidateFeatureStoreCRStatus(namespace, feastCRName) - - By("Running `feast apply` and `feast materialize-incremental` to validate registry definitions") - VerifyApplyFeatureStoreDefinitions(namespace, feastCRName, feastDeploymentName) - - By("Validating Feast entity, feature, and feature view presence") - VerifyFeastMethods(namespace, feastDeploymentName, testDir) - } - - // This context verifies that a pre-created Feast FeatureStore CR continues to function as expected - // after an upgrade. It validates `feast apply`, registry sync, feature retrieval, and model execution. - Context("Feast post Upgrade Test", func() { - It("Should create and run a feastPostUpgrade test scenario feast apply and materialize functionality successfully", runPostUpgradeTest) - }) -}) diff --git a/infra/feast-operator/test/e2e_rhoai/feast_preupgrade_test.go b/infra/feast-operator/test/e2e_rhoai/feast_preupgrade_test.go deleted file mode 100644 index 680d79812fe..00000000000 --- a/infra/feast-operator/test/e2e_rhoai/feast_preupgrade_test.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2025 Feast Community. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2erhoai - -import ( - "fmt" - - . "github.com/feast-dev/feast/infra/feast-operator/test/utils" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("Feast PreUpgrade scenario Testing", Ordered, func() { - const ( - namespace = "test-ns-feast-upgrade" - replaceNamespace = "test-ns-feast" - testDir = "/test/e2e_rhoai" - feastDeploymentName = FeastPrefix + "credit-scoring" - feastCRName = "credit-scoring" - ) - - filesToUpdateNamespace := []string{ - "test/testdata/feast_integration_test_crs/postgres.yaml", - "test/testdata/feast_integration_test_crs/redis.yaml", - "test/testdata/feast_integration_test_crs/feast.yaml", - } - - BeforeAll(func() { - By(fmt.Sprintf("Creating test namespace: %s", namespace)) - Expect(CreateNamespace(namespace, testDir)).To(Succeed()) - fmt.Printf("Namespace %s created successfully\n", namespace) - - By("Replacing placeholder namespace in CR YAMLs for test setup") - Expect(ReplaceNamespaceInYamlFilesInPlace(filesToUpdateNamespace, replaceNamespace, namespace)).To(Succeed()) - }) - - AfterAll(func() { - By("Restoring original namespace in CR YAMLs") - Expect(ReplaceNamespaceInYamlFilesInPlace(filesToUpdateNamespace, namespace, replaceNamespace)).To(Succeed()) - - if CurrentSpecReport().Failed() { - By(fmt.Sprintf("Deleting test namespace: %s", namespace)) - Expect(DeleteNamespace(namespace, testDir)).To(Succeed()) - fmt.Printf("Namespace %s deleted successfully\n", namespace) - } - }) - - runPreUpgradeTest := func() { - By("Applying Feast infra manifests and verifying setup") - ApplyFeastInfraManifestsAndVerify(namespace, testDir) - - By("Applying and validating the credit-scoring FeatureStore CR") - ApplyFeastYamlAndVerify(namespace, testDir, feastDeploymentName, feastCRName, "test/testdata/feast_integration_test_crs/feast.yaml") - } - - // This context ensures the Feast CR setup is functional prior to any upgrade - Context("Feast Pre Upgrade Test", func() { - It("Should create and run a feastPreUpgrade test scenario feast credit-scoring CR setup successfully", runPreUpgradeTest) - }) -}) diff --git a/infra/feast-operator/test/e2e_rhoai/feast_wb_connection_integration_test.go b/infra/feast-operator/test/e2e_rhoai/feast_wb_connection_integration_test.go deleted file mode 100644 index 43bfff17c2f..00000000000 --- a/infra/feast-operator/test/e2e_rhoai/feast_wb_connection_integration_test.go +++ /dev/null @@ -1,168 +0,0 @@ -/* -Copyright 2025 Feast Community. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package e2erhoai provides end-to-end (E2E) test coverage for Feast integration with -// Red Hat OpenShift AI (RHOAI) environments. -// This specific test validates the functionality -// of executing a Feast workbench integration connection with kubernetes auth and without auth successfully -package e2erhoai - -import ( - "fmt" - "time" - - utils "github.com/feast-dev/feast/infra/feast-operator/test/utils" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("Feast Workbench Integration Connection Testing", Ordered, func() { - const ( - namespace = "test-ns-feast" - configMapName = "feast-wb-cm" - rolebindingName = "rb-feast-test" - notebookFile = "test/e2e_rhoai/resources/feast-wb-connection-credit-scoring.ipynb" - pvcFile = "test/e2e_rhoai/resources/pvc.yaml" - permissionFile = "test/e2e_rhoai/resources/permissions.py" - notebookPVC = "jupyterhub-nb-kube-3aadmin-pvc" - testDir = "/test/e2e_rhoai" - notebookName = "feast-wb-connection-credit-scoring.ipynb" - feastDeploymentName = utils.FeastPrefix + "credit-scoring" - feastCRName = "credit-scoring" - ) - - // Verify feast ConfigMap - verifyFeastConfigMap := func(authEnabled bool) { - feastConfigMapName := "jupyter-nb-kube-3aadmin-feast-config" - configMapKey := "credit_scoring_local" - By(fmt.Sprintf("Listing ConfigMaps and verifying %s exists with correct content", feastConfigMapName)) - - // Build expected content based on auth type - expectedContent := []string{ - "project: credit_scoring_local", - } - if authEnabled { - expectedContent = append(expectedContent, "type: kubernetes") - } else { - expectedContent = append(expectedContent, "type: no_auth") - } - - // First, list ConfigMaps and check if target ConfigMap exists - // Retry with polling since the ConfigMap may be created asynchronously - const maxRetries = 5 - const retryInterval = 5 * time.Second - var configMapExists bool - var err error - - for i := 0; i < maxRetries; i++ { - exists, listErr := utils.VerifyConfigMapExistsInList(namespace, feastConfigMapName) - if listErr != nil { - err = listErr - if i < maxRetries-1 { - fmt.Printf("Failed to list ConfigMaps, retrying in %v... (attempt %d/%d)\n", retryInterval, i+1, maxRetries) - time.Sleep(retryInterval) - continue - } - } else if exists { - configMapExists = true - fmt.Printf("ConfigMap %s found in ConfigMap list\n", feastConfigMapName) - break - } - - if i < maxRetries-1 { - fmt.Printf("ConfigMap %s not found in list yet, retrying in %v... (attempt %d/%d)\n", feastConfigMapName, retryInterval, i+1, maxRetries) - time.Sleep(retryInterval) - } - } - - if !configMapExists { - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to find ConfigMap %s in ConfigMap list after %d attempts: %v", feastConfigMapName, maxRetries, err)) - } - - // Once ConfigMap exists in list, verify content (project name and auth type) - err = utils.VerifyFeastConfigMapContent(namespace, feastConfigMapName, configMapKey, expectedContent) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to verify Feast ConfigMap %s content: %v", feastConfigMapName, err)) - fmt.Printf("Feast ConfigMap %s verified successfully with project and auth type\n", feastConfigMapName) - } - - // Parameterized test function that handles both auth and non-auth scenarios - runFeastWorkbenchIntegration := func(authEnabled bool) { - // Apply permissions only if auth is enabled - if authEnabled { - By("Applying Feast permissions for kubernetes authenticated scenario") - utils.ApplyFeastPermissions(permissionFile, "/feast-data/credit_scoring_local/feature_repo/permissions.py", namespace, feastDeploymentName) - } - - // Create notebook with all setup steps - // Pass feastProject parameter to set the opendatahub.io/feast-config annotation - utils.CreateNotebookTest(namespace, configMapName, notebookFile, "test/e2e_rhoai/resources/feature_repo", pvcFile, rolebindingName, notebookPVC, notebookName, testDir, "credit_scoring_local") - - // Verify Feast ConfigMap was created with correct auth type - verifyFeastConfigMap(authEnabled) - - // Monitor notebook execution - utils.MonitorNotebookTest(namespace, notebookName) - } - - BeforeAll(func() { - By(fmt.Sprintf("Creating test namespace: %s", namespace)) - Expect(utils.CreateNamespace(namespace, testDir)).To(Succeed()) - fmt.Printf("Namespace %s created successfully\n", namespace) - - By("Applying Feast infra manifests and verifying setup") - utils.ApplyFeastInfraManifestsAndVerify(namespace, testDir) - }) - - AfterAll(func() { - By(fmt.Sprintf("Deleting test namespace: %s", namespace)) - Expect(utils.DeleteNamespace(namespace, testDir)).To(Succeed()) - fmt.Printf("Namespace %s deleted successfully\n", namespace) - }) - - Context("Feast Workbench Integration Tests - Without Auth", func() { - BeforeEach(func() { - By("Applying and validating the credit-scoring FeatureStore CR without auth") - utils.ApplyFeastYamlAndVerify(namespace, testDir, feastDeploymentName, feastCRName, "test/testdata/feast_integration_test_crs/feast.yaml") - - By("Verify Feature Store CR is in Ready state") - utils.ValidateFeatureStoreCRStatus(namespace, feastCRName) - - By("Running `feast apply` and `feast materialize-incremental` to validate registry definitions") - utils.VerifyApplyFeatureStoreDefinitions(namespace, feastCRName, feastDeploymentName) - }) - - It("Should create and run a FeastWorkbenchIntegrationWithoutAuth scenario successfully", func() { - runFeastWorkbenchIntegration(false) - }) - }) - - Context("Feast Workbench Integration Tests - With Auth", func() { - BeforeEach(func() { - By("Applying and validating the credit-scoring FeatureStore CR (with auth)") - utils.ApplyFeastYamlAndVerify(namespace, testDir, feastDeploymentName, feastCRName, "test/e2e_rhoai/resources/feast_kube_auth.yaml") - - By("Verify Feature Store CR is in Ready state") - utils.ValidateFeatureStoreCRStatus(namespace, feastCRName) - - By("Running `feast apply` and `feast materialize-incremental` to validate registry definitions") - utils.VerifyApplyFeatureStoreDefinitions(namespace, feastCRName, feastDeploymentName) - }) - - It("Should create and run a FeastWorkbenchIntegrationWithAuth scenario successfully", func() { - runFeastWorkbenchIntegration(true) - }) - }) -}) diff --git a/infra/feast-operator/test/e2e_rhoai/feast_wb_milvus_test.go b/infra/feast-operator/test/e2e_rhoai/feast_wb_milvus_test.go deleted file mode 100644 index b4e1d37b827..00000000000 --- a/infra/feast-operator/test/e2e_rhoai/feast_wb_milvus_test.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Copyright 2025 Feast Community. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package e2erhoai provides end-to-end (E2E) test coverage for Feast integration with -// Red Hat OpenShift AI (RHOAI) environments. This specific test validates the functionality -// of executing a Feast Jupyter notebook within a fully configured OpenShift namespace -package e2erhoai - -import ( - "fmt" - - utils "github.com/feast-dev/feast/infra/feast-operator/test/utils" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("Feast Jupyter Notebook Testing", Ordered, func() { - const ( - namespace = "test-ns-feast-wb" - configMapName = "feast-wb-cm" - rolebindingName = "rb-feast-test" - notebookFile = "test/e2e_rhoai/resources/feast-wb-milvus-test.ipynb" - pvcFile = "test/e2e_rhoai/resources/pvc.yaml" - notebookPVC = "jupyterhub-nb-kube-3aadmin-pvc" - testDir = "/test/e2e_rhoai" - notebookName = "feast-wb-milvus-test.ipynb" - feastMilvusTest = "TestFeastMilvusNotebook" - ) - - BeforeAll(func() { - By(fmt.Sprintf("Creating test namespace: %s", namespace)) - Expect(utils.CreateNamespace(namespace, testDir)).To(Succeed()) - fmt.Printf("Namespace %s created successfully\n", namespace) - }) - - AfterAll(func() { - By(fmt.Sprintf("Deleting test namespace: %s", namespace)) - Expect(utils.DeleteNamespace(namespace, testDir)).To(Succeed()) - fmt.Printf("Namespace %s deleted successfully\n", namespace) - }) - - Context("Feast Jupyter Notebook Test", func() { - It("Should create and run a "+feastMilvusTest+" successfully", func() { - // Create notebook with all setup steps - // Pass empty string for feastProject to keep annotation empty - utils.CreateNotebookTest(namespace, configMapName, notebookFile, "test/e2e_rhoai/resources/feature_repo", pvcFile, rolebindingName, notebookPVC, notebookName, testDir, "") - - // Monitor notebook execution - utils.MonitorNotebookTest(namespace, notebookName) - }) - }) -}) diff --git a/infra/feast-operator/test/e2e_rhoai/feast_wb_ray_offline_store_test.go b/infra/feast-operator/test/e2e_rhoai/feast_wb_ray_offline_store_test.go deleted file mode 100644 index eb467ad07ff..00000000000 --- a/infra/feast-operator/test/e2e_rhoai/feast_wb_ray_offline_store_test.go +++ /dev/null @@ -1,82 +0,0 @@ -/* -Copyright 2025 Feast Community. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package e2erhoai provides end-to-end (E2E) test coverage for Feast integration with -// Red Hat OpenShift AI (RHOAI) environments. This specific test validates the functionality -// of executing a Feast Jupyter notebook with Ray offline store within a fully configured OpenShift namespace -package e2erhoai - -import ( - "fmt" - "os/exec" - - utils "github.com/feast-dev/feast/infra/feast-operator/test/utils" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("Feast Jupyter Notebook Testing with Ray Offline Store", Ordered, func() { - const ( - namespace = "test-ns-feast-wb-ray" - configMapName = "feast-wb-ray-cm" - rolebindingName = "rb-feast-ray-test" - notebookFile = "test/e2e_rhoai/resources/feast-wb-ray-test.ipynb" - pvcFile = "test/e2e_rhoai/resources/pvc.yaml" - kueueResourcesFile = "test/e2e_rhoai/resources/kueue_resources_setup.yaml" - notebookPVC = "jupyterhub-nb-kube-3aadmin-pvc" - testDir = "/test/e2e_rhoai" - notebookName = "feast-wb-ray-test.ipynb" - feastRayTest = "TestFeastRayOfflineStoreNotebook" - ) - - BeforeAll(func() { - By(fmt.Sprintf("Creating test namespace: %s", namespace)) - Expect(utils.CreateNamespace(namespace, testDir)).To(Succeed()) - fmt.Printf("Namespace %s created successfully\n", namespace) - - By("Applying Kueue resources setup") - // Apply with namespace flag - cluster-scoped resources (ResourceFlavor, ClusterQueue) will be applied at cluster level, - // and namespace-scoped resources (LocalQueue) will be applied in the specified namespace - cmd := exec.Command("kubectl", "apply", "-f", kueueResourcesFile, "-n", namespace) - output, err := utils.Run(cmd, testDir) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to apply Kueue resources: %v\nOutput: %s", err, output)) - fmt.Printf("Kueue resources applied successfully\n") - }) - - AfterAll(func() { - By("Deleting Kueue resources") - // Delete with namespace flag - will delete namespace-scoped resources from the namespace - // and cluster-scoped resources from the cluster - cmd := exec.Command("kubectl", "delete", "-f", kueueResourcesFile, "-n", namespace, "--ignore-not-found=true") - _, _ = utils.Run(cmd, testDir) - fmt.Printf("Kueue resources cleanup completed\n") - - By(fmt.Sprintf("Deleting test namespace: %s", namespace)) - Expect(utils.DeleteNamespace(namespace, testDir)).To(Succeed()) - fmt.Printf("Namespace %s deleted successfully\n", namespace) - }) - - Context("Feast Jupyter Notebook Test with Ray Offline store", func() { - It("Should create and run a "+feastRayTest+" successfully", func() { - // Create notebook with all setup steps - // Pass empty string for feastProject to keep annotation empty - utils.CreateNotebookTest(namespace, configMapName, notebookFile, "test/e2e_rhoai/resources/feature_repo", pvcFile, rolebindingName, notebookPVC, notebookName, testDir, "") - - // Monitor notebook execution - utils.MonitorNotebookTest(namespace, notebookName) - }) - }) -}) diff --git a/infra/feast-operator/test/e2e_rhoai/resources/custom-nb.yaml b/infra/feast-operator/test/e2e_rhoai/resources/custom-nb.yaml deleted file mode 100644 index 6dd9304e4b9..00000000000 --- a/infra/feast-operator/test/e2e_rhoai/resources/custom-nb.yaml +++ /dev/null @@ -1,92 +0,0 @@ -# This template maybe used to spin up a custom notebook image -# i.e.: sed s/{{.IngressDomain}}/$(oc get ingresses.config/cluster -o jsonpath={.spec.domain})/g tests/resources/custom-nb.template | oc apply -f - -# resources generated: -# pod/jupyter-nb-kube-3aadmin-0 -# service/jupyter-nb-kube-3aadmin -# route.route.openshift.io/jupyter-nb-kube-3aadmin (jupyter-nb-kube-3aadmin-opendatahub.apps.tedbig412.cp.fyre.ibm.com) -# service/jupyter-nb-kube-3aadmin-tls -apiVersion: kubeflow.org/v1 -kind: Notebook -metadata: - annotations: - notebooks.opendatahub.io/inject-auth: "true" - notebooks.opendatahub.io/last-size-selection: Small - opendatahub.io/link: https://jupyter-nb-kube-3aadmin-{{.Namespace}}.{{.IngressDomain}}/notebook/{{.Namespace}}/jupyter-nb-kube-3aadmin - opendatahub.io/username: {{.Username}} - opendatahub.io/feast-config: {{.FeastProject}} - generation: 1 - labels: - app: jupyter-nb-kube-3aadmin - opendatahub.io/dashboard: "true" - opendatahub.io/odh-managed: "true" - opendatahub.io/user: {{.Username}} - opendatahub.io/feast-integration: 'true' - name: jupyter-nb-kube-3aadmin - namespace: {{.Namespace}} -spec: - template: - spec: - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - preference: - matchExpressions: - - key: nvidia.com/gpu.present - operator: NotIn - values: - - "true" - weight: 1 - containers: - - env: - - name: NOTEBOOK_ARGS - value: |- - --ServerApp.port=8888 - --ServerApp.token='' - --ServerApp.password='' - --ServerApp.base_url=/notebook/test-feast-wb/jupyter-nb-kube-3aadmin - --ServerApp.quit_button=False - --ServerApp.tornado_settings={"user":"{{.Username}}","hub_host":"https://odh-dashboard-{{.OpenDataHubNamespace}}.{{.IngressDomain}}","hub_prefix":"/notebookController/{{.Username}}"} - - name: JUPYTER_IMAGE - value: {{.NotebookImage}} - - name: JUPYTER_NOTEBOOK_PORT - value: "8888" - - name: PIP_INDEX_URL - value: {{.PipIndexUrl}} - - name: PIP_TRUSTED_HOST - value: {{.PipTrustedHost}} - - name: FEAST_VERSION - value: {{.FeastVerison}} - - name: OPENAI_API_KEY - value: {{.OpenAIAPIKey}} - - name: NAMESPACE - value: {{.Namespace}} - image: {{.NotebookImage}} - command: {{.Command}} - imagePullPolicy: Always - name: jupyter-nb-kube-3aadmin - ports: - - containerPort: 8888 - name: notebook-port - protocol: TCP - resources: - limits: - cpu: "2" - memory: 3Gi - requests: - cpu: "1" - memory: 3Gi - volumeMounts: - - mountPath: /opt/app-root/src - name: jupyterhub-nb-kube-3aadmin-pvc - - mountPath: /opt/app-root/notebooks - name: {{.NotebookConfigMapName}} - workingDir: /opt/app-root/src - enableServiceLinks: false - serviceAccountName: default - volumes: - - name: jupyterhub-nb-kube-3aadmin-pvc - persistentVolumeClaim: - claimName: {{.NotebookPVC}} - - name: {{.NotebookConfigMapName}} - configMap: - name: {{.NotebookConfigMapName}} diff --git a/infra/feast-operator/test/e2e_rhoai/resources/feast-wb-connection-credit-scoring.ipynb b/infra/feast-operator/test/e2e_rhoai/resources/feast-wb-connection-credit-scoring.ipynb deleted file mode 100755 index 39e1f9c6e37..00000000000 --- a/infra/feast-operator/test/e2e_rhoai/resources/feast-wb-connection-credit-scoring.ipynb +++ /dev/null @@ -1,416 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import feast\n", - "\n", - "actual_version = feast.__version__\n", - "assert actual_version == os.environ.get(\"FEAST_VERSION\"), (\n", - " f\"❌ Feast version mismatch. Expected: {os.environ.get('FEAST_VERSION')}, Found: {actual_version}\"\n", - ")\n", - "print(f\"✅ Found Expected Feast version: {actual_version} in workbench\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# --- Configuration Variables ---\n", - "import os \n", - "\n", - "# Fetch token and server directly from oc CLI\n", - "import subprocess\n", - "\n", - "def oc(cmd):\n", - " return subprocess.check_output(cmd, shell=True).decode(\"utf-8\").strip()\n", - "\n", - "token = oc(\"oc whoami -t\")\n", - "server = oc(\"oc whoami --show-server\")\n", - "namespace = os.environ.get(\"NAMESPACE\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!oc login --token=$token --server=$server" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Add user permission to namespace\n", - "!oc adm policy add-role-to-user admin $(oc whoami) -n $namespace" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "namespace = os.environ.get(\"NAMESPACE\") # read namespace from env\n", - "if not namespace:\n", - " raise ValueError(\"NAMESPACE environment variable is not set\")\n", - "\n", - "yaml_content = os.popen(\n", - " f\"oc get configmap feast-credit-scoring-client -n {namespace} \"\n", - " \"-o jsonpath='{.data.feature_store\\\\.yaml}' | sed 's/\\\\\\\\n/\\\\n/g'\"\n", - ").read()\n", - "\n", - "# Save the configmap data into an environment variable (if needed)\n", - "os.environ[\"CONFIGMAP_DATA\"] = yaml_content" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from feast import FeatureStore\n", - "fs_credit_scoring_local = FeatureStore(fs_yaml_file='/opt/app-root/src/feast-config/credit_scoring_local')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "project_name = \"credit_scoring_local\"\n", - "project = fs_credit_scoring_local.get_project(project_name)\n", - "\n", - "# 1. Assert object returned\n", - "assert project is not None, f\"❌ get_project('{project_name}') returned None\"\n", - "\n", - "# 2. Extract project name (works for dict or Feast object)\n", - "if isinstance(project, dict):\n", - " returned_name = project.get(\"spec\", {}).get(\"name\")\n", - "else:\n", - " # Feast Project object\n", - " returned_name = getattr(project, \"name\", None)\n", - " if not returned_name and hasattr(project, \"spec\") and hasattr(project.spec, \"name\"):\n", - " returned_name = project.spec.name\n", - "\n", - "# 3. Assert that name exists\n", - "assert returned_name, f\"❌ Returned project does not contain a valid name: {project}\"\n", - "\n", - "print(\"• Project Name Returned:\", returned_name)\n", - "\n", - "# 4. Assert the name matches expected\n", - "assert returned_name == project_name, (\n", - " f\"❌ Expected project '{project_name}', but got '{returned_name}'\"\n", - ")\n", - "\n", - "print(f\"\\n✓ get_project('{project_name}') validation passed!\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "feast_list_functions = [\n", - " \"list_projects\",\n", - " \"list_entities\",\n", - " \"list_feature_views\",\n", - " \"list_all_feature_views\",\n", - " \"list_batch_feature_views\",\n", - " \"list_on_demand_feature_views\",\n", - "]\n", - "\n", - "# validates feast list methods returns data and method type\n", - "def validate_list_method(fs_obj, method_name):\n", - " assert hasattr(fs_obj, method_name), f\"Method not found: {method_name}\"\n", - "\n", - " method = getattr(fs_obj, method_name)\n", - " result = method()\n", - "\n", - " assert isinstance(result, list), (\n", - " f\"{method_name}() must return a list, got {type(result)}\"\n", - " )\n", - " assert len(result) > 0, (\n", - " f\"{method_name}() returned an empty list — expected data\"\n", - " )\n", - "\n", - " print(f\"✓ {method_name}() returned {len(result)} items\")\n", - "\n", - "for m in feast_list_functions:\n", - " validate_list_method(fs_credit_scoring_local, m)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "feast_list_functions = [\n", - " \"list_feature_services\",\n", - " # \"list_permissions\",\n", - " \"list_saved_datasets\",\n", - "]\n", - "\n", - "# validates feast methods exists and type is valid\n", - "def validate_list_func(fs_obj, method_name):\n", - " assert hasattr(fs_obj, method_name), f\"Method not found: {method_name}\"\n", - "\n", - " method = getattr(fs_obj, method_name)\n", - "\n", - " result = method()\n", - "\n", - " assert isinstance(result, list), (\n", - " f\"{method_name}() must return a list, got {type(result)}\"\n", - " )\n", - "\n", - "for m in feast_list_functions:\n", - " validate_list_func(fs_credit_scoring_local, m)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# validate_list_data_sources for with and without permissions \n", - "\n", - "import os\n", - "from feast.errors import FeastPermissionError\n", - "\n", - "def validate_list_data_sources(fs_obj):\n", - " \"\"\"\n", - " Validates list_data_sources() with special handling for Kubernetes auth mode.\n", - " If CONFIGMAP_DATA indicates auth=kubernetes, expect FeastPermissionError.\n", - " Otherwise validate output type normally.\n", - " \"\"\"\n", - " auth_mode = os.getenv(\"CONFIGMAP_DATA\")\n", - "\n", - " # Case 1: Kubernetes auth → expect permission error\n", - " if \"kubernetes\" in auth_mode.lower():\n", - " try:\n", - " fs_obj.list_data_sources()\n", - " raise AssertionError(\n", - " \"Expected FeastPermissionError due to Kubernetes auth, but the call succeeded.\"\n", - " )\n", - " except FeastPermissionError as e:\n", - " # Correct, this is expected\n", - " return\n", - " except Exception as e:\n", - " raise AssertionError(\n", - " f\"Expected FeastPermissionError, but got different exception: {type(e)} - {e}\"\n", - " )\n", - "\n", - " # Case 2: Non-Kubernetes auth → normal path\n", - " assert hasattr(fs_obj, \"list_data_sources\"), \"Method not found: list_data_sources\"\n", - " result = fs_obj.list_data_sources()\n", - " assert isinstance(result, list), (\n", - " f\"list_data_sources() must return a list, got {type(result)}\"\n", - " )\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "entity = fs_credit_scoring_local.get_entity(\"dob_ssn\")\n", - "\n", - "assert entity is not None, \"❌ Entity 'dob_ssn' not found!\"\n", - "assert entity.name == \"dob_ssn\", f\"❌ Entity name mismatch: {entity.name}\"\n", - "\n", - "print(\"✓ Entity validation successful!\\n\", entity.name)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "fv = fs_credit_scoring_local.get_feature_view(\"credit_history\")\n", - "\n", - "assert fv is not None, \"❌ FeatureView 'credit_history' not found!\"\n", - "assert fv.name == \"credit_history\", f\"❌ Name mismatch: {fv.name}\"\n", - "\n", - "print(\"• FeatureView : validation successful!\", fv.name)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from feast.errors import FeastPermissionError\n", - "\n", - "def validate_get_data_source(fs_obj, name: str):\n", - " auth_mode = os.getenv(\"CONFIGMAP_DATA\", \"\")\n", - "\n", - " print(\"📌 CONFIGMAP_DATA:\", auth_mode)\n", - "\n", - " # If Kubernetes auth is enabled → expect permission error\n", - " if \"auth\" in \"kubernetes\" in auth_mode.lower():\n", - " print(f\"🔒 Kubernetes auth detected, expecting permission error for get_data_source('{name}')\")\n", - "\n", - " try:\n", - " fs_obj.get_data_source(name)\n", - " raise AssertionError(\n", - " f\"❌ Expected FeastPermissionError when accessing data source '{name}', but call succeeded\"\n", - " )\n", - "\n", - " except FeastPermissionError as e:\n", - " print(f\"✅ Correctly blocked with FeastPermissionError: {e}\")\n", - " return\n", - "\n", - " except Exception as e:\n", - " raise AssertionError(\n", - " f\"❌ Expected FeastPermissionError but got {type(e)}: {e}\"\n", - " )\n", - "\n", - " # Otherwise → normal validation\n", - " print(f\"🔍 Fetching data source '{name}'...\")\n", - "\n", - " ds = fs_obj.get_data_source(name)\n", - "\n", - " print(\"\\n📌 Data Source Object:\")\n", - " print(ds)\n", - "\n", - " assert ds.name == name, (\n", - " f\"❌ Expected name '{name}', got '{ds.name}'\"\n", - " )\n", - "\n", - " print(f\"✅ Data source '{name}' exists and is correctly configured.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "feast_features = [\n", - " \"zipcode_features:city\",\n", - " \"zipcode_features:state\",\n", - "]\n", - "\n", - "entity_rows = [{\n", - " \"zipcode\": 1463,\n", - " \"dob_ssn\": \"19530219_5179\"\n", - "}]\n", - "\n", - "response = fs_credit_scoring_local.get_online_features(\n", - " features=feast_features,\n", - " entity_rows=entity_rows,\n", - ").to_dict()\n", - "\n", - "print(\"Actual response:\", response)\n", - "\n", - "expected = {\n", - " 'zipcode': [1463],\n", - " 'dob_ssn': ['19530219_5179'],\n", - " 'city': ['PEPPERELL'],\n", - " 'state': ['MA'],\n", - "}\n", - "\n", - "assert response == expected" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "\n", - "# Input entity dataframe\n", - "entity_df = pd.DataFrame({\n", - " \"dob_ssn\": [\"19530219_5179\"],\n", - " \"zipcode\": [1463],\n", - " \"event_timestamp\": [pd.Timestamp(\"2020-04-26 18:01:04\")]\n", - "})\n", - "\n", - "feast_features = [\n", - " \"zipcode_features:city\",\n", - " \"zipcode_features:state\",\n", - " \"credit_history:credit_card_due\",\n", - " \"credit_history:mortgage_due\",\n", - "]\n", - "\n", - "# Retrieve historical features\n", - "historical_df = fs_credit_scoring_local.get_historical_features(\n", - " entity_df=entity_df,\n", - " features=feast_features,\n", - ").to_df()\n", - "\n", - "print(\"Historical DF:\\n\", historical_df)\n", - "\n", - "# Validate dataframe is not empty\n", - "assert not historical_df.empty, \" Historical features dataframe is empty!\"\n", - "\n", - "# 2. Validate required columns exist\n", - "expected_cols = {\n", - " \"dob_ssn\", \"zipcode\", \"event_timestamp\",\n", - " \"city\", \"state\",\n", - " \"credit_card_due\", \"mortgage_due\"\n", - "}\n", - "\n", - "missing_cols = expected_cols - set(historical_df.columns)\n", - "assert not missing_cols, f\" Missing columns in result: {missing_cols}\"\n", - "\n", - "# 3. Validate city/state are non-null (critical features)\n", - "assert pd.notna(historical_df.loc[0, \"city\"]), \" 'city' value is null!\"\n", - "assert pd.notna(historical_df.loc[0, \"state\"]), \" 'state' value is null!\"\n", - "\n", - "# 4. Validate entity matches input\n", - "assert historical_df.loc[0, \"zipcode\"] == 1463, \" zipcode mismatch!\"\n", - "assert historical_df.loc[0, \"dob_ssn\"] == \"19530219_5179\", \"❌ dob_ssn mismatch!\"\n", - "\n", - "print(\"✅ All validations passed successfully!\")\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.5" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/infra/feast-operator/test/e2e_rhoai/resources/feast-wb-milvus-test.ipynb b/infra/feast-operator/test/e2e_rhoai/resources/feast-wb-milvus-test.ipynb deleted file mode 100755 index e2838a4f33e..00000000000 --- a/infra/feast-operator/test/e2e_rhoai/resources/feast-wb-milvus-test.ipynb +++ /dev/null @@ -1,481 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import feast\n", - "\n", - "actual_version = feast.__version__\n", - "assert actual_version == os.environ.get(\"FEAST_VERSION\"), (\n", - " f\"❌ Feast version mismatch. Expected: {os.environ.get('FEAST_VERSION')}, Found: {actual_version}\"\n", - ")\n", - "print(f\"✅ Successfully installed Feast version: {actual_version}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%cd /opt/app-root/src/feature_repo\n", - "!ls -l" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!cat /opt/app-root/src/feature_repo/feature_store.yaml" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!mkdir -p data\n", - "!wget -O data/city_wikipedia_summaries_with_embeddings.parquet https://raw.githubusercontent.com/opendatahub-io/feast/master/examples/rag/feature_repo/data/city_wikipedia_summaries_with_embeddings.parquet" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd \n", - "\n", - "df = pd.read_parquet(\"./data/city_wikipedia_summaries_with_embeddings.parquet\")\n", - "df['vector'] = df['vector'].apply(lambda x: x.tolist())\n", - "embedding_length = len(df['vector'][0])\n", - "assert embedding_length == 384, f\"❌ Expected vector length 384, but got {embedding_length}\"\n", - "print(f'embedding length = {embedding_length}')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from IPython.display import display\n", - "\n", - "display(df.head())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pip install -q pymilvus[milvus_lite] transformers torch" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import subprocess\n", - "\n", - "# Run `feast apply` and capture output\n", - "result = subprocess.run([\"feast\", \"apply\"], capture_output=True, text=True)\n", - "\n", - "# Combine stdout and stderr in case important info is in either\n", - "output = result.stdout + result.stderr\n", - "\n", - "# Print full output for debugging (optional)\n", - "print(output)\n", - "\n", - "# Expected substrings to validate\n", - "expected_messages = [\n", - " \"Applying changes for project rag\",\n", - " \"Connecting to Milvus in local mode\",\n", - " \"Deploying infrastructure for city_embeddings\"\n", - "]\n", - "\n", - "# Validate all expected messages are in output\n", - "for msg in expected_messages:\n", - " assert msg in output, f\"❌ Expected message not found: '{msg}'\"\n", - "\n", - "print(\"✅ All expected messages were found in the output.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from datetime import datetime\n", - "from feast import FeatureStore\n", - "\n", - "store = FeatureStore(repo_path=\".\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import io\n", - "import sys\n", - "\n", - "# Capture stdout\n", - "captured_output = io.StringIO()\n", - "sys_stdout_backup = sys.stdout\n", - "sys.stdout = captured_output\n", - "\n", - "# Call the function\n", - "store.write_to_online_store(feature_view_name='city_embeddings', df=df)\n", - "\n", - "# Restore stdout\n", - "sys.stdout = sys_stdout_backup\n", - "\n", - "# Get the output\n", - "output_str = captured_output.getvalue()\n", - "\n", - "# Expected message\n", - "expected_msg = \"Connecting to Milvus in local mode using data/online_store.db\"\n", - "\n", - "# Validate\n", - "assert expected_msg in output_str, f\"❌ Expected message not found.\\nExpected: {expected_msg}\\nActual Output:\\n{output_str}\"\n", - "\n", - "print(\"✅ Output message validated successfully.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# List batch feature views\n", - "batch_fvs = store.list_batch_feature_views()\n", - "\n", - "# Print the number of batch feature views\n", - "print(\"Number of batch feature views:\", len(batch_fvs))\n", - "\n", - "# Assert that the result is an integer and non-negative\n", - "assert isinstance(len(batch_fvs), int), \"Result is not an integer\"\n", - "assert len(batch_fvs) >= 0, \"Feature view count is negative\"\n", - "\n", - "print(\"Feature views listed correctly ✅\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from feast import FeatureStore\n", - "\n", - "# Initialize store (if not already)\n", - "store = FeatureStore(repo_path=\".\") # Adjust path if necessary\n", - "\n", - "# Retrieve the feature view\n", - "fv = store.get_feature_view(\"city_embeddings\")\n", - "\n", - "# Assert name\n", - "assert fv.name == \"city_embeddings\", \"Feature view name mismatch\"\n", - "\n", - "# Assert entities\n", - "assert fv.entities == [\"item_id\"], f\"Expected entities ['item_id'], got {fv.entities}\"\n", - "\n", - "# Assert feature names and vector index settings\n", - "feature_names = [f.name for f in fv.features]\n", - "assert \"vector\" in feature_names, \"Missing 'vector' feature\"\n", - "assert \"state\" in feature_names, \"Missing 'state' feature\"\n", - "assert \"sentence_chunks\" in feature_names, \"Missing 'sentence_chunks' feature\"\n", - "assert \"wiki_summary\" in feature_names, \"Missing 'wiki_summary' feature\"\n", - "\n", - "# Assert 'vector' feature is a vector index with COSINE metric\n", - "vector_feature = next(f for f in fv.features if f.name == \"vector\")\n", - "assert vector_feature.vector_index, \"'vector' feature is not indexed\"\n", - "assert vector_feature.vector_search_metric == \"COSINE\", \"Expected COSINE search metric for 'vector'\"\n", - "\n", - "print(\"All assertions passed ✅\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from feast.entity import Entity\n", - "from feast.types import ValueType\n", - "entity = Entity(\n", - " name=\"item_id1\",\n", - " value_type=ValueType.INT64,\n", - " description=\"test id\",\n", - " tags={\"team\": \"feast\"},\n", - ")\n", - "store.apply(entity)\n", - "assert any(e.name == \"item_id1\" for e in store.list_entities())\n", - "print(\"Entity added ✅\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "entity_to_delete = store.get_entity(\"item_id1\")\n", - "\n", - "store.apply(\n", - " objects=[],\n", - " objects_to_delete=[entity_to_delete],\n", - " partial=False\n", - ")\n", - "\n", - "# Validation after deletion\n", - "assert not any(e.name == \"item_id1\" for e in store.list_entities())\n", - "print(\"Entity deleted ✅\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# List batch feature views\n", - "batch_fvs = store.list_batch_feature_views()\n", - "assert len(batch_fvs) == 1\n", - "\n", - "# Print count\n", - "print(f\"Found {len(batch_fvs)} batch feature view(s) ✅\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pymilvus_client = store._provider._online_store._connect(store.config)\n", - "COLLECTION_NAME = pymilvus_client.list_collections()[0]\n", - "\n", - "milvus_query_result = pymilvus_client.query(\n", - " collection_name=COLLECTION_NAME,\n", - " filter=\"item_id == '0'\",\n", - ")\n", - "pd.DataFrame(milvus_query_result[0]).head()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn.functional as F\n", - "from feast import FeatureStore\n", - "from pymilvus import MilvusClient, DataType, FieldSchema\n", - "from transformers import AutoTokenizer, AutoModel\n", - "from example_repo import city_embeddings_feature_view, item\n", - "\n", - "TOKENIZER = \"sentence-transformers/all-MiniLM-L6-v2\"\n", - "MODEL = \"sentence-transformers/all-MiniLM-L6-v2\"\n", - "\n", - "def mean_pooling(model_output, attention_mask):\n", - " token_embeddings = model_output[\n", - " 0\n", - " ] # First element of model_output contains all token embeddings\n", - " input_mask_expanded = (\n", - " attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()\n", - " )\n", - " return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(\n", - " input_mask_expanded.sum(1), min=1e-9\n", - " )\n", - "\n", - "def run_model(sentences, tokenizer, model):\n", - " encoded_input = tokenizer(\n", - " sentences, padding=True, truncation=True, return_tensors=\"pt\"\n", - " )\n", - " # Compute token embeddings\n", - " with torch.no_grad():\n", - " model_output = model(**encoded_input)\n", - "\n", - " sentence_embeddings = mean_pooling(model_output, encoded_input[\"attention_mask\"])\n", - " sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1)\n", - " return sentence_embeddings" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "question = \"Which city has the largest population in New York?\"\n", - "\n", - "tokenizer = AutoTokenizer.from_pretrained(TOKENIZER)\n", - "model = AutoModel.from_pretrained(MODEL)\n", - "query_embedding = run_model(question, tokenizer, model)\n", - "query = query_embedding.detach().cpu().numpy().tolist()[0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from IPython.display import display\n", - "\n", - "# Retrieve top k documents\n", - "context_data = store.retrieve_online_documents_v2(\n", - " features=[\n", - " \"city_embeddings:vector\",\n", - " \"city_embeddings:item_id\",\n", - " \"city_embeddings:state\",\n", - " \"city_embeddings:sentence_chunks\",\n", - " \"city_embeddings:wiki_summary\",\n", - " ],\n", - " query=query,\n", - " top_k=3,\n", - " distance_metric='COSINE',\n", - ").to_df()\n", - "display(context_data)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def format_documents(context_df):\n", - " output_context = \"\"\n", - " unique_documents = context_df.drop_duplicates().apply(\n", - " lambda x: \"City & State = {\" + x['state'] +\"}\\nSummary = {\" + x['wiki_summary'].strip()+\"}\",\n", - " axis=1,\n", - " )\n", - " for i, document_text in enumerate(unique_documents):\n", - " output_context+= f\"****START DOCUMENT {i}****\\n{document_text.strip()}\\n****END DOCUMENT {i}****\"\n", - " return output_context" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "RAG_CONTEXT = format_documents(context_data[['state', 'wiki_summary']])\n", - "print(RAG_CONTEXT)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "FULL_PROMPT = f\"\"\"\n", - "You are an assistant for answering questions about states. You will be provided documentation from Wikipedia. Provide a conversational answer.\n", - "If you don't know the answer, just say \"I do not know.\" Don't make up an answer.\n", - "\n", - "Here are document(s) you should use when answer the users question:\n", - "{RAG_CONTEXT}\n", - "\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pip install openai" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from openai import OpenAI\n", - "\n", - "client = OpenAI(\n", - " api_key=os.environ.get(\"OPENAI_API_KEY\"),\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = client.chat.completions.create(\n", - " model=\"gpt-4o-mini\",\n", - " messages=[\n", - " {\"role\": \"system\", \"content\": FULL_PROMPT},\n", - " {\"role\": \"user\", \"content\": question}\n", - " ],\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# The expected output\n", - "expected_output = (\n", - " \"New York City\"\n", - ")\n", - "\n", - "# Actual output from response\n", - "actual_output = '\\n'.join([c.message.content.strip() for c in response.choices])\n", - "\n", - "# Validate\n", - "assert expected_output in actual_output, f\"❌ Output mismatch:\\nExpected: {expected_output}\\nActual: {actual_output}\"\n", - "\n", - "print(\"✅ Output matches expected response.\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.5" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/infra/feast-operator/test/e2e_rhoai/resources/feast-wb-ray-test.ipynb b/infra/feast-operator/test/e2e_rhoai/resources/feast-wb-ray-test.ipynb deleted file mode 100644 index 3b91bcccd8e..00000000000 --- a/infra/feast-operator/test/e2e_rhoai/resources/feast-wb-ray-test.ipynb +++ /dev/null @@ -1,516 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# --- Configuration Variables ---\n", - "import os \n", - "\n", - "# Namespace where your resources exist\n", - "namespace = os.environ.get(\"NAMESPACE\")\n", - "\n", - "fsconfigmap = \"cm-fs-data\"\n", - "\n", - "# Fetch token and server directly from oc CLI\n", - "import subprocess\n", - "\n", - "def oc(cmd):\n", - " return subprocess.check_output(cmd, shell=True).decode(\"utf-8\").strip()\n", - "\n", - "token = oc(\"oc whoami -t\")\n", - "server = oc(\"oc whoami --show-server\")\n", - "\n", - "os.environ[\"CLUSTER_TOKEN\"] = token\n", - "os.environ[\"CLUSTER_SERVER\"] = server\n", - "\n", - "\n", - "# RayCluster name\n", - "raycluster = \"feastraytest\"\n", - "os.environ[\"RAY_CLUSTER\"] = raycluster\n", - "\n", - "# Show configured values\n", - "print(\"Configuration Variables:\")\n", - "print(f\" Namespace: {namespace}\")\n", - "print(f\" Server: {server}\")\n", - "print(f\" Token: {'*' * 20}\") # hide actual token\n", - "print(f\" Ray Cluster: {raycluster}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "! git clone https://github.com/Srihari1192/feast-rag-ray.git" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%cd feast-rag-ray/feature_repo" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!oc login --token=$token --server=$server" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!oc create configmap $fsconfigmap --from-file=data/customer_daily_profile.parquet --from-file=data/driver_stats.parquet -n $namespace" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Import pieces from codeflare-sdk\n", - "from codeflare_sdk import Cluster, ClusterConfiguration, TokenAuthentication\n", - "\n", - "# Create authentication with token and server from oc\n", - "auth = TokenAuthentication(\n", - " token=token,\n", - " server=server,\n", - " skip_tls=True\n", - ")\n", - "auth.login()\n", - "print(\"✓ Authentication successful\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from kubernetes.client import (\n", - " V1Volume,\n", - " V1ConfigMapVolumeSource,\n", - " V1VolumeMount,\n", - ") \n", - "\n", - "data_volume = V1Volume(\n", - " name=\"data\",\n", - " config_map=V1ConfigMapVolumeSource(name=fsconfigmap)\n", - ")\n", - "\n", - "data_mount = V1VolumeMount(\n", - " name=\"data\",\n", - " mount_path=\"/opt/app-root/src/feast-rag-ray/feature_repo/data\",\n", - " read_only=True\n", - ")\n", - "\n", - "cluster = Cluster(ClusterConfiguration(\n", - " name=raycluster,\n", - " head_cpu_requests=1,\n", - " head_cpu_limits=1,\n", - " head_memory_requests=4,\n", - " head_memory_limits=4,\n", - " head_extended_resource_requests={'nvidia.com/gpu':0}, # For GPU enabled workloads set the head_extended_resource_requests and worker_extended_resource_requests\n", - " worker_extended_resource_requests={'nvidia.com/gpu':0},\n", - " num_workers=2,\n", - " worker_cpu_requests='250m',\n", - " worker_cpu_limits=1,\n", - " worker_memory_requests=4,\n", - " worker_memory_limits=4,\n", - " # image=\"\", # Optional Field \n", - " write_to_file=False, # When enabled Ray Cluster yaml files are written to /HOME/.codeflare/resources\n", - " local_queue=\"fs-user-queue\", # Specify the local queue manually\n", - " # ⭐ Best method: Use secretKeyRef to expose AWS credentials safely\n", - " volumes=[data_volume],\n", - " volume_mounts=[data_mount],\n", - " \n", - "))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cluster.apply()\n", - "# cluster.wait_ready()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import time\n", - "\n", - "MAX_WAIT = 180 # 3 minutes\n", - "INTERVAL = 5 # check every 5 seconds\n", - "elapsed = 0\n", - "\n", - "print(\"⏳ Waiting up to 3 minutes for RayCluster to be READY...\\n\")\n", - "\n", - "while elapsed < MAX_WAIT:\n", - " details = cluster.details()\n", - " status = details.status.value\n", - "\n", - " print(details)\n", - " print(\"Cluster Status:\", status)\n", - "\n", - " if status == \"ready\":\n", - " print(\"✅ RayCluster is READY!\")\n", - " break\n", - " \n", - " print(f\"⏳ RayCluster is NOT ready yet: {status} ... checking again in {INTERVAL}s\\n\")\n", - " time.sleep(INTERVAL)\n", - " elapsed += INTERVAL\n", - "\n", - "else:\n", - " print(\"❌ Timeout: RayCluster did NOT become READY within 3 minutes.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "! feast apply" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "from pathlib import Path\n", - "from feast import FeatureStore\n", - "\n", - "# Add feature repo to PYTHONPATH\n", - "repo_path = Path(\".\")\n", - "sys.path.append(str(repo_path))\n", - "\n", - "# Initialize Feature Store\n", - "print(\"Initializing Feast with Ray configuration...\")\n", - "store = FeatureStore(repo_path=\".\")\n", - "\n", - "# Assertions: Verify store is initialized correctly\n", - "assert store is not None, \"FeatureStore should be initialized\"\n", - "assert store.config is not None, \"Store config should be available\"\n", - "assert store.config.offline_store is not None, \"Offline store should be configured\"\n", - "\n", - "print(f\"✓ Offline store: {store.config.offline_store.type}\")\n", - "if hasattr(store.config, \"batch_engine\") and store.config.batch_engine:\n", - " print(f\"✓ Compute engine: {store.config.batch_engine.type}\")\n", - " # Assertion: Verify batch engine is configured if present\n", - " assert store.config.batch_engine.type is not None, \"Batch engine type should be set\"\n", - "else:\n", - " print(\"⚠ No compute engine configured\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. Create Entity DataFrame\n", - "\n", - "Create an entity DataFrame for historical feature retrieval with point-in-time timestamps.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from datetime import datetime, timedelta\n", - "import pandas as pd\n", - "\n", - "# --- Create time window ---\n", - "end_date = datetime.now().replace(microsecond=0, second=0, minute=0)\n", - "start_date = end_date - timedelta(days=2)\n", - "\n", - "\n", - "entity_df = pd.DataFrame(\n", - " {\n", - " \"driver_id\": [1001, 1002, 1003],\n", - " \"customer_id\": [2001, 2002, 2003],\n", - " \"event_timestamp\": [\n", - " pd.Timestamp(end_date - timedelta(hours=24), tz=\"UTC\"),\n", - " pd.Timestamp(end_date - timedelta(hours=12), tz=\"UTC\"),\n", - " pd.Timestamp(end_date - timedelta(hours=6), tz=\"UTC\"),\n", - " ],\n", - " }\n", - ")\n", - "\n", - "# Assertions: Verify entity DataFrame is created correctly\n", - "assert len(entity_df) == 3, f\"Expected 3 rows, got {len(entity_df)}\"\n", - "assert \"driver_id\" in entity_df.columns, \"driver_id column should be present\"\n", - "assert \"customer_id\" in entity_df.columns, \"customer_id column should be present\"\n", - "assert \"event_timestamp\" in entity_df.columns, \"event_timestamp column should be present\"\n", - "assert all(entity_df[\"driver_id\"].isin([1001, 1002, 1003])), \"driver_id values should match expected\"\n", - "assert all(entity_df[\"customer_id\"].isin([2001, 2002, 2003])), \"customer_id values should match expected\"\n", - "assert entity_df[\"event_timestamp\"].notna().all(), \"All event_timestamp values should be non-null\"\n", - "\n", - "print(f\"✓ Created entity DataFrame with {len(entity_df)} rows\")\n", - "print(f\"✓ Time range: {start_date} to {end_date}\")\n", - "print(\"\\nEntity DataFrame:\")\n", - "print(entity_df)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3. Retrieve Historical Features\n", - "\n", - "Retrieve historical features using Ray compute engine for distributed point-in-time joins.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Cell 4: Retrieve Historical Features\n", - "print(\"Retrieving historical features with Ray compute engine...\")\n", - "print(\"(This demonstrates distributed point-in-time joins)\")\n", - "\n", - "try:\n", - " # Get historical features - this uses Ray compute engine for distributed processing\n", - " historical_features = store.get_historical_features(\n", - " entity_df=entity_df,\n", - " features=[\n", - " \"driver_hourly_stats:conv_rate\",\n", - " \"driver_hourly_stats:acc_rate\",\n", - " \"driver_hourly_stats:avg_daily_trips\",\n", - " \"customer_daily_profile:current_balance\",\n", - " \"customer_daily_profile:avg_passenger_count\",\n", - " \"customer_daily_profile:lifetime_trip_count\",\n", - " ],\n", - " )\n", - "\n", - " # Convert to DataFrame - Ray processes this efficiently\n", - " historical_df = historical_features.to_df()\n", - " \n", - " # Assertions: Verify historical features are retrieved correctly\n", - " assert historical_df is not None, \"Historical features DataFrame should not be None\"\n", - " assert len(historical_df) > 0, \"Should retrieve at least one row of historical features\"\n", - " assert \"driver_id\" in historical_df.columns, \"driver_id should be in the result\"\n", - " assert \"customer_id\" in historical_df.columns, \"customer_id should be in the result\"\n", - " \n", - " # Verify expected feature columns are present (some may be None if data doesn't exist)\n", - " expected_features = [\n", - " \"conv_rate\", \"acc_rate\", \"avg_daily_trips\",\n", - " \"current_balance\", \"avg_passenger_count\", \"lifetime_trip_count\"\n", - " ]\n", - " feature_columns = [col for col in historical_df.columns if col in expected_features]\n", - " assert len(feature_columns) > 0, f\"Should have at least one feature column, got: {historical_df.columns.tolist()}\"\n", - " \n", - " print(f\"✓ Retrieved {len(historical_df)} historical feature rows\")\n", - " print(f\"✓ Features: {list(historical_df.columns)}\")\n", - " \n", - " # Display the results\n", - " print(\"\\nHistorical Features DataFrame:\")\n", - " display(historical_df.head(10))\n", - "\n", - "except Exception as e:\n", - " print(f\"⚠ Historical features retrieval failed: {e}\")\n", - " print(\"This might be due to missing Ray dependencies or data\")\n", - " raise\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4. Test On-Demand Feature Transformations\n", - "\n", - "Demonstrate on-demand feature transformations that are computed at request time.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Cell 5: Test On-Demand Features\n", - "print(\"Testing on-demand feature transformations...\")\n", - "\n", - "try:\n", - " # Get features including on-demand transformations\n", - " features_with_odfv = store.get_historical_features(\n", - " entity_df=entity_df.head(1),\n", - " features=[\n", - " \"driver_hourly_stats:conv_rate\",\n", - " \"driver_hourly_stats:acc_rate\",\n", - " \"driver_hourly_stats:avg_daily_trips\",\n", - " \"driver_activity_v2:conv_rate_plus_acc_rate\",\n", - " \"driver_activity_v2:trips_per_day_normalized\",\n", - " ],\n", - " )\n", - "\n", - " odfv_df = features_with_odfv.to_df()\n", - " \n", - " # Assertions: Verify on-demand features are computed correctly\n", - " assert odfv_df is not None, \"On-demand features DataFrame should not be None\"\n", - " assert len(odfv_df) > 0, \"Should retrieve at least one row with on-demand features\"\n", - " assert \"driver_id\" in odfv_df.columns, \"driver_id should be in the result\"\n", - " \n", - " # Verify on-demand feature columns if they exist\n", - " if \"conv_rate_plus_acc_rate\" in odfv_df.columns:\n", - " # Assertion: Verify the on-demand feature is computed\n", - " assert odfv_df[\"conv_rate_plus_acc_rate\"].notna().any(), \"conv_rate_plus_acc_rate should have non-null values\"\n", - " print(\"✓ On-demand feature 'conv_rate_plus_acc_rate' is computed\")\n", - " \n", - " if \"trips_per_day_normalized\" in odfv_df.columns:\n", - " assert odfv_df[\"trips_per_day_normalized\"].notna().any(), \"trips_per_day_normalized should have non-null values\"\n", - " print(\"✓ On-demand feature 'trips_per_day_normalized' is computed\")\n", - " \n", - " print(f\"✓ Retrieved {len(odfv_df)} rows with on-demand transformations\")\n", - " \n", - " # Display results\n", - " print(\"\\nFeatures with On-Demand Transformations:\")\n", - " display(odfv_df)\n", - " \n", - " # Show specific transformed features\n", - " if \"conv_rate_plus_acc_rate\" in odfv_df.columns:\n", - " print(\"\\nSample with on-demand features:\")\n", - " display(\n", - " odfv_df[[\"driver_id\", \"conv_rate\", \"acc_rate\", \"conv_rate_plus_acc_rate\"]]\n", - " )\n", - "\n", - "except Exception as e:\n", - " print(f\"⚠ On-demand features failed: {e}\")\n", - " raise\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 5. Materialize Features to Online Store\n", - "\n", - "Materialize features to the online store using Ray compute engine for efficient batch processing.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from datetime import timezone\n", - "print(\"Materializing features to online store...\")\n", - "store.materialize(\n", - "\tstart_date=datetime(2025, 1, 1, tzinfo=timezone.utc),\n", - "\tend_date=end_date,\n", - ")\n", - "\n", - "# Minimal output assertion: materialization succeeded if no exception\n", - "assert True, \"Materialization completed successfully\"\n", - "print(\"✓ Initial materialization successful\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 6. Test Online Feature Serving\n", - "\n", - "Retrieve features from the online store for low-latency serving.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Cell 7: Test Online Feature Serving\n", - "print(\"Testing online feature serving...\")\n", - "\n", - "try:\n", - " entity_rows = [\n", - " {\"driver_id\": 1001, \"customer_id\": 2001},\n", - " {\"driver_id\": 1002, \"customer_id\": 2002},\n", - " ]\n", - " \n", - " # Assertion: Verify entity rows are valid\n", - " assert len(entity_rows) == 2, \"Should have 2 entity rows\"\n", - " assert all(\"driver_id\" in row for row in entity_rows), \"All entity rows should have driver_id\"\n", - " assert all(\"customer_id\" in row for row in entity_rows), \"All entity rows should have customer_id\"\n", - " \n", - " online_features = store.get_online_features(\n", - " features=[\n", - " \"driver_hourly_stats:conv_rate\",\n", - " \"driver_hourly_stats:acc_rate\",\n", - " \"customer_daily_profile:current_balance\",\n", - " ],\n", - " entity_rows=entity_rows,\n", - " )\n", - "\n", - " online_df = online_features.to_df()\n", - " \n", - " # Assertions: Verify online features are retrieved correctly\n", - " assert online_df is not None, \"Online features DataFrame should not be None\"\n", - " assert len(online_df) == len(entity_rows), f\"Should retrieve {len(entity_rows)} rows, got {len(online_df)}\"\n", - " assert \"driver_id\" in online_df.columns, \"driver_id should be in the result\"\n", - " assert \"customer_id\" in online_df.columns, \"customer_id should be in the result\"\n", - " \n", - " # Verify expected feature columns are present\n", - " expected_features = [\"conv_rate\", \"acc_rate\", \"current_balance\"]\n", - " feature_columns = [col for col in online_df.columns if col in expected_features]\n", - " assert len(feature_columns) > 0, f\"Should have at least one feature column, got: {online_df.columns.tolist()}\"\n", - " \n", - " # Verify entity IDs match\n", - " assert all(online_df[\"driver_id\"].isin([1001, 1002])), \"driver_id values should match entity rows\"\n", - " assert all(online_df[\"customer_id\"].isin([2001, 2002])), \"customer_id values should match entity rows\"\n", - " \n", - " print(f\"✓ Retrieved {len(online_df)} online feature rows\")\n", - " print(f\"✓ Features retrieved: {feature_columns}\")\n", - " \n", - " print(\"\\nOnline Features DataFrame:\")\n", - " display(online_df)\n", - "\n", - "except Exception as e:\n", - " print(f\"⚠ Online serving failed: {e}\")\n", - " raise\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cluster.down()" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/infra/feast-operator/test/e2e_rhoai/resources/feast_kube_auth.yaml b/infra/feast-operator/test/e2e_rhoai/resources/feast_kube_auth.yaml deleted file mode 100644 index fae126b528a..00000000000 --- a/infra/feast-operator/test/e2e_rhoai/resources/feast_kube_auth.yaml +++ /dev/null @@ -1,74 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: feast-data-stores - namespace: test-ns-feast -stringData: - redis: | - connection_string: redis.test-ns-feast.svc.cluster.local:6379 - sql: | - path: postgresql+psycopg://${POSTGRESQL_USER}:${POSTGRESQL_PASSWORD}@postgres.test-ns-feast.svc.cluster.local:5432/${POSTGRESQL_DATABASE} - cache_ttl_seconds: 60 - sqlalchemy_config_kwargs: - echo: false - pool_pre_ping: true ---- -apiVersion: feast.dev/v1 -kind: FeatureStore -metadata: - name: credit-scoring - namespace: test-ns-feast -spec: - authz: - kubernetes: - roles: [] - feastProject: credit_scoring_local - feastProjectDir: - git: - url: https://github.com/feast-dev/feast-credit-score-local-tutorial - ref: 598a270 - services: - offlineStore: - persistence: - file: - type: duckdb - server: - envFrom: - - secretRef: - name: postgres-secret - env: - - name: MPLCONFIGDIR - value: /tmp - resources: - requests: - cpu: 150m - memory: 128Mi - onlineStore: - persistence: - store: - type: redis - secretRef: - name: feast-data-stores - server: - envFrom: - - secretRef: - name: postgres-secret - env: - - name: MPLCONFIGDIR - value: /tmp - resources: - requests: - cpu: 150m - memory: 128Mi - registry: - local: - persistence: - store: - type: sql - secretRef: - name: feast-data-stores - server: - envFrom: - - secretRef: - name: postgres-secret - restAPI: true diff --git a/infra/feast-operator/test/e2e_rhoai/resources/feature_repo/__init__.py b/infra/feast-operator/test/e2e_rhoai/resources/feature_repo/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/infra/feast-operator/test/e2e_rhoai/resources/feature_repo/example_repo.py b/infra/feast-operator/test/e2e_rhoai/resources/feature_repo/example_repo.py deleted file mode 100755 index 7a37d99d495..00000000000 --- a/infra/feast-operator/test/e2e_rhoai/resources/feature_repo/example_repo.py +++ /dev/null @@ -1,42 +0,0 @@ -from datetime import timedelta - -from feast import ( - FeatureView, - Field, - FileSource, -) -from feast.data_format import ParquetFormat -from feast.types import Float32, Array, String, ValueType -from feast import Entity - -item = Entity( - name="item_id", - description="Item ID", - value_type=ValueType.INT64, -) - -parquet_file_path = "./data/city_wikipedia_summaries_with_embeddings.parquet" - -source = FileSource( - file_format=ParquetFormat(), - path=parquet_file_path, - timestamp_field="event_timestamp", -) - -city_embeddings_feature_view = FeatureView( - name="city_embeddings", - entities=[item], - schema=[ - Field( - name="vector", - dtype=Array(Float32), - vector_index=True, - vector_search_metric="COSINE", - ), - Field(name="state", dtype=String), - Field(name="sentence_chunks", dtype=String), - Field(name="wiki_summary", dtype=String), - ], - source=source, - ttl=timedelta(hours=2), -) diff --git a/infra/feast-operator/test/e2e_rhoai/resources/feature_repo/feature_store.yaml b/infra/feast-operator/test/e2e_rhoai/resources/feature_repo/feature_store.yaml deleted file mode 100755 index f8f9cc293dc..00000000000 --- a/infra/feast-operator/test/e2e_rhoai/resources/feature_repo/feature_store.yaml +++ /dev/null @@ -1,16 +0,0 @@ -project: rag -provider: local -registry: data/registry.db -online_store: - type: milvus - path: data/online_store.db - vector_enabled: true - embedding_dim: 384 - index_type: "FLAT" - metric_type: "COSINE" -offline_store: - type: file -entity_key_serialization_version: 3 -auth: - type: no_auth - diff --git a/infra/feast-operator/test/e2e_rhoai/resources/kueue_resources_setup.yaml b/infra/feast-operator/test/e2e_rhoai/resources/kueue_resources_setup.yaml deleted file mode 100644 index ebcac54f4a0..00000000000 --- a/infra/feast-operator/test/e2e_rhoai/resources/kueue_resources_setup.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: kueue.x-k8s.io/v1beta1 -kind: ResourceFlavor -metadata: - name: "fs-resource-flavor" ---- -apiVersion: kueue.x-k8s.io/v1beta1 -kind: ClusterQueue -metadata: - name: "fs-cluster-queue" -spec: - namespaceSelector: {} # match all. - resourceGroups: - - coveredResources: ["cpu", "memory","nvidia.com/gpu"] - flavors: - - name: "fs-resource-flavor" - resources: - - name: "cpu" - nominalQuota: 9 - - name: "memory" - nominalQuota: 36Gi - - name: "nvidia.com/gpu" - nominalQuota: 0 ---- -apiVersion: kueue.x-k8s.io/v1beta1 -kind: LocalQueue -metadata: - name: "fs-user-queue" - annotations: - "kueue.x-k8s.io/default-queue": "true" -spec: - clusterQueue: "fs-cluster-queue" diff --git a/infra/feast-operator/test/e2e_rhoai/resources/permissions.py b/infra/feast-operator/test/e2e_rhoai/resources/permissions.py deleted file mode 100644 index 7b48a7b4c56..00000000000 --- a/infra/feast-operator/test/e2e_rhoai/resources/permissions.py +++ /dev/null @@ -1,19 +0,0 @@ -from feast.feast_object import ALL_FEATURE_VIEW_TYPES -from feast.permissions.permission import Permission -from feast.permissions.action import READ, AuthzedAction -from feast.permissions.policy import NamespaceBasedPolicy -from feast.project import Project -from feast.entity import Entity -from feast.feature_service import FeatureService -from feast.saved_dataset import SavedDataset - -perm_namespace = ["test-ns-feast"] - -WITHOUT_DATA_SOURCE = [Project, Entity, FeatureService, SavedDataset] + ALL_FEATURE_VIEW_TYPES - -test_perm = Permission( - name="feast-auth", - types=WITHOUT_DATA_SOURCE, - policy=NamespaceBasedPolicy(namespaces=perm_namespace), - actions=[AuthzedAction.DESCRIBE] + READ -) diff --git a/infra/feast-operator/test/e2e_rhoai/resources/pvc.yaml b/infra/feast-operator/test/e2e_rhoai/resources/pvc.yaml deleted file mode 100644 index a9e8c1be299..00000000000 --- a/infra/feast-operator/test/e2e_rhoai/resources/pvc.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: jupyterhub-nb-kube-3aadmin-pvc -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi diff --git a/infra/feast-operator/test/utils/notebook_util.go b/infra/feast-operator/test/utils/notebook_util.go deleted file mode 100644 index 8652b481889..00000000000 --- a/infra/feast-operator/test/utils/notebook_util.go +++ /dev/null @@ -1,387 +0,0 @@ -package utils - -import ( - "bytes" - "fmt" - "os" - "os/exec" - "strings" - "text/template" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -type NotebookTemplateParams struct { - Namespace string - IngressDomain string - OpenDataHubNamespace string - NotebookImage string - NotebookConfigMapName string - NotebookPVC string - Username string - OC_TOKEN string - OC_SERVER string - NotebookFile string - Command string - PipIndexUrl string - PipTrustedHost string - FeastVerison string - OpenAIAPIKey string - FeastProject string -} - -// CreateNotebook renders a notebook manifest from a template and applies it using kubectl. -func CreateNotebook(params NotebookTemplateParams) error { - content, err := os.ReadFile("test/e2e_rhoai/resources/custom-nb.yaml") - if err != nil { - return fmt.Errorf("failed to read template file: %w", err) - } - - tmpl, err := template.New("notebook").Parse(string(content)) - if err != nil { - return fmt.Errorf("failed to parse template: %w", err) - } - - var rendered bytes.Buffer - if err := tmpl.Execute(&rendered, params); err != nil { - return fmt.Errorf("failed to substitute template: %w", err) - } - - tmpFile, err := os.CreateTemp("", "notebook-*.yaml") - if err != nil { - return fmt.Errorf("failed to create temp file: %w", err) - } - - // Defer cleanup of temp file - defer func() { - if err := os.Remove(tmpFile.Name()); err != nil { - fmt.Printf("warning: failed to remove temp file %s: %v", tmpFile.Name(), err) - } - }() - - if _, err := tmpFile.Write(rendered.Bytes()); err != nil { - return fmt.Errorf("failed to write to temp file: %w", err) - } - - if err := tmpFile.Close(); err != nil { - return fmt.Errorf("failed to close temp file: %w", err) - } - - // fmt.Println("Notebook manifest applied successfully") - cmd := exec.Command("kubectl", "apply", "-f", tmpFile.Name(), "-n", params.Namespace) - output, err := Run(cmd, "/test/e2e_rhoai") - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( - "Failed to create Notebook %s.\nError: %v\nOutput: %s\n", - tmpFile.Name(), err, output, - )) - fmt.Printf("Notebook %s created successfully\n", tmpFile.Name()) - return nil -} - -// MonitorNotebookPod waits for a notebook pod to reach Running state and verifies execution logs. -func MonitorNotebookPod(namespace, podPrefix string, notebookName string) error { - const successMarker = "Notebook executed successfully" - const failureMarker = "Notebook execution failed" - const pollInterval = 5 * time.Second - var pod *PodInfo - - fmt.Println("🔄 Waiting for notebook pod to reach Running & Ready state...") - - foundRunningReady := false - for i := 0; i < 36; i++ { - var err error - pod, err = getPodByPrefix(namespace, podPrefix) - if err != nil { - fmt.Printf("⏳ Pod not created yet: %v\n", err) - time.Sleep(pollInterval) - continue - } - if pod.Status == "Running" { - fmt.Printf("✅ Pod %s is Running and Ready.\n", pod.Name) - foundRunningReady = true - break - } - fmt.Printf("⏳ Pod %s not ready yet. Phase: %s\n", pod.Name, pod.Status) - time.Sleep(pollInterval) - } - - if !foundRunningReady { - return fmt.Errorf("❌ Pod %s did not reach Running & Ready state within 3 minutes", podPrefix) - } - - // Start monitoring notebook logs - fmt.Printf("⏳ Monitoring Notebook pod %s Logs for Jupyter Notebook %s execution status\n", pod.Name, notebookName) - - for i := 0; i < 60; i++ { - logs, err := getPodLogs(namespace, pod.Name) - if err != nil { - fmt.Printf("⏳ Failed to get logs for pod %s: %v\n", pod.Name, err) - time.Sleep(pollInterval) - continue - } - - if strings.Contains(logs, successMarker) { - Expect(logs).To(ContainSubstring(successMarker)) - fmt.Printf("✅ Jupyter Notebook pod %s executed successfully.\n", pod.Name) - return nil - } - - if strings.Contains(logs, failureMarker) { - fmt.Printf("❌ Notebook pod %s failed: failure marker found.\n", pod.Name) - return fmt.Errorf("Notebook failed in execution. Logs:\n%s", logs) - } - - time.Sleep(pollInterval) - } - - return fmt.Errorf("❌ Timed out waiting for notebook pod %s to complete", podPrefix) -} - -type PodInfo struct { - Name string - Status string -} - -// returns the first pod matching a name prefix in the given namespace. -func getPodByPrefix(namespace, prefix string) (*PodInfo, error) { - cmd := exec.Command( - "kubectl", "get", "pods", "-n", namespace, - "-o", "jsonpath={range .items[*]}{.metadata.name} {.status.phase}{\"\\n\"}{end}", - ) - output, err := Run(cmd, "/test/e2e_rhoai") - if err != nil { - return nil, fmt.Errorf("failed to get pods: %w", err) - } - - lines := strings.Split(strings.TrimSpace(string(output)), "\n") - for _, line := range lines { - parts := strings.Fields(line) - if len(parts) < 2 { - continue - } - name := parts[0] - status := parts[1] - - if strings.HasPrefix(name, prefix) { - return &PodInfo{ - Name: name, - Status: status, - }, nil - } - } - - return nil, fmt.Errorf("no pod found with prefix %q in namespace %q", prefix, namespace) -} - -// retrieves the logs of a specified pod in the given namespace. -func getPodLogs(namespace, podName string) (string, error) { - cmd := exec.Command("kubectl", "logs", "-n", namespace, podName) - var out bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &stderr - - err := cmd.Run() - if err != nil { - return "", fmt.Errorf("error getting pod logs: %v - %s", err, stderr.String()) - } - - return out.String(), nil -} - -// returns the OpenShift cluster ingress domain. -func GetIngressDomain(testDir string) string { - cmd := exec.Command("oc", "get", "ingresses.config.openshift.io", "cluster", "-o", "jsonpath={.spec.domain}") - output, _ := Run(cmd, testDir) - return string(output) -} - -// returns the current OpenShift user authentication token. -func GetOCToken(testDir string) string { - cmd := exec.Command("oc", "whoami", "--show-token") - output, _ := Run(cmd, testDir) - return string(output) -} - -// returns the OpenShift API server URL for the current user. -func GetOCServer(testDir string) string { - cmd := exec.Command("oc", "whoami", "--show-server") - output, _ := Run(cmd, testDir) - return string(output) -} - -// returns the OpenShift cluster logged in Username -func GetOCUser(testDir string) string { - cmd := exec.Command("oc", "whoami") - output, _ := Run(cmd, testDir) - return strings.TrimSpace(string(output)) -} - -// SetNamespaceContext sets the kubectl namespace context to the specified namespace -func SetNamespaceContext(namespace, testDir string) error { - cmd := exec.Command("kubectl", "config", "set-context", "--current", "--namespace", namespace) - output, err := Run(cmd, testDir) - if err != nil { - return fmt.Errorf("failed to set namespace context to %s: %w\nOutput: %s", namespace, err, output) - } - return nil -} - -// CreateNotebookConfigMap creates a ConfigMap containing the notebook file and feature repo -func CreateNotebookConfigMap(namespace, configMapName, notebookFile, featureRepoPath, testDir string) error { - cmd := exec.Command("kubectl", "create", "configmap", configMapName, - "--from-file="+notebookFile, - "--from-file="+featureRepoPath) - output, err := Run(cmd, testDir) - if err != nil { - return fmt.Errorf("failed to create ConfigMap %s: %w\nOutput: %s", configMapName, err, output) - } - return nil -} - -// CreateNotebookPVC creates a PersistentVolumeClaim for the notebook -func CreateNotebookPVC(pvcFile, testDir string) error { - cmd := exec.Command("kubectl", "apply", "-f", pvcFile) - _, err := Run(cmd, testDir) - if err != nil { - return fmt.Errorf("failed to create PVC from %s: %w", pvcFile, err) - } - return nil -} - -// CreateNotebookRoleBinding creates a rolebinding for the user in the specified namespace -func CreateNotebookRoleBinding(namespace, rolebindingName, username, testDir string) error { - cmd := exec.Command("kubectl", "create", "rolebinding", rolebindingName, - "-n", namespace, - "--role=admin", - "--user="+username) - _, err := Run(cmd, testDir) - if err != nil { - return fmt.Errorf("failed to create rolebinding %s: %w", rolebindingName, err) - } - return nil -} - -// BuildNotebookCommand builds the command array for executing a notebook with papermill -func BuildNotebookCommand(notebookName, testDir string) []string { - return []string{ - "/bin/sh", - "-c", - fmt.Sprintf( - "pip install papermill && "+ - "mkdir -p /opt/app-root/src/feature_repo && "+ - "cp -rL /opt/app-root/notebooks/* /opt/app-root/src/feature_repo/ && "+ - "oc login --token=%s --server=%s --insecure-skip-tls-verify=true && "+ - "(papermill /opt/app-root/notebooks/%s /opt/app-root/src/output.ipynb --kernel python3 && "+ - "echo '✅ Notebook executed successfully' || "+ - "(echo '❌ Notebook execution failed' && "+ - "cp /opt/app-root/src/output.ipynb /opt/app-root/src/failed_output.ipynb && "+ - "echo '📄 Copied failed notebook to failed_output.ipynb')) && "+ - "jupyter nbconvert --to notebook --stdout /opt/app-root/src/output.ipynb || echo '⚠️ nbconvert failed' && "+ - "sleep 100; exit 0", - GetOCToken(testDir), - GetOCServer(testDir), - notebookName, - ), - } -} - -// GetNotebookParams builds and returns NotebookTemplateParams from environment variables and configuration -// feastProject is optional - if provided, it will be set in the notebook annotation, otherwise it will be empty -func GetNotebookParams(namespace, configMapName, notebookPVC, notebookName, testDir string, feastProject string) NotebookTemplateParams { - username := GetOCUser(testDir) - command := BuildNotebookCommand(notebookName, testDir) - - getEnv := func(key string) string { - val, _ := os.LookupEnv(key) - return val - } - - return NotebookTemplateParams{ - Namespace: namespace, - IngressDomain: GetIngressDomain(testDir), - OpenDataHubNamespace: getEnv("APPLICATIONS_NAMESPACE"), - NotebookImage: getEnv("NOTEBOOK_IMAGE"), - NotebookConfigMapName: configMapName, - NotebookPVC: notebookPVC, - Username: username, - OC_TOKEN: GetOCToken(testDir), - OC_SERVER: GetOCServer(testDir), - NotebookFile: notebookName, - Command: "[\"" + strings.Join(command, "\",\"") + "\"]", - PipIndexUrl: getEnv("PIP_INDEX_URL"), - PipTrustedHost: getEnv("PIP_TRUSTED_HOST"), - FeastVerison: getEnv("FEAST_VERSION"), - OpenAIAPIKey: getEnv("OPENAI_API_KEY"), - FeastProject: feastProject, - } -} - -// SetupNotebookEnvironment performs all the setup steps required for notebook testing -func SetupNotebookEnvironment(namespace, configMapName, notebookFile, featureRepoPath, pvcFile, rolebindingName, testDir string) error { - // Set namespace context - if err := SetNamespaceContext(namespace, testDir); err != nil { - return fmt.Errorf("failed to set namespace context: %w", err) - } - - // Create config map - if err := CreateNotebookConfigMap(namespace, configMapName, notebookFile, featureRepoPath, testDir); err != nil { - return fmt.Errorf("failed to create config map: %w", err) - } - - // Create PVC - if err := CreateNotebookPVC(pvcFile, testDir); err != nil { - return fmt.Errorf("failed to create PVC: %w", err) - } - - // Create rolebinding - username := GetOCUser(testDir) - if err := CreateNotebookRoleBinding(namespace, rolebindingName, username, testDir); err != nil { - return fmt.Errorf("failed to create rolebinding: %w", err) - } - - return nil -} - -// CreateNotebookTest performs all the setup steps and creates a notebook. -// This function handles namespace context, ConfigMap, PVC, rolebinding, and notebook creation. -// feastProject is optional - if provided, it will be set in the notebook annotation, otherwise it will be empty -func CreateNotebookTest(namespace, configMapName, notebookFile, featureRepoPath, pvcFile, rolebindingName, notebookPVC, notebookName, testDir string, feastProject string) { - // Execute common setup steps - By(fmt.Sprintf("Setting namespace context to : %s", namespace)) - Expect(SetNamespaceContext(namespace, testDir)).To(Succeed()) - fmt.Printf("Successfully set namespace context to: %s\n", namespace) - - By(fmt.Sprintf("Creating Config map: %s", configMapName)) - Expect(CreateNotebookConfigMap(namespace, configMapName, notebookFile, featureRepoPath, testDir)).To(Succeed()) - fmt.Printf("ConfigMap %s created successfully\n", configMapName) - - By(fmt.Sprintf("Creating Persistent volume claim: %s", notebookPVC)) - Expect(CreateNotebookPVC(pvcFile, testDir)).To(Succeed()) - fmt.Printf("Persistent Volume Claim %s created successfully\n", notebookPVC) - - By(fmt.Sprintf("Creating rolebinding %s for the user", rolebindingName)) - Expect(CreateNotebookRoleBinding(namespace, rolebindingName, GetOCUser(testDir), testDir)).To(Succeed()) - fmt.Printf("Created rolebinding %s successfully\n", rolebindingName) - - // Build notebook parameters and create notebook - nbParams := GetNotebookParams(namespace, configMapName, notebookPVC, notebookName, testDir, feastProject) - By("Creating Jupyter Notebook") - Expect(CreateNotebook(nbParams)).To(Succeed(), "Failed to create notebook") -} - -// MonitorNotebookTest monitors the notebook execution and verifies completion. -func MonitorNotebookTest(namespace, notebookName string) { - By("Monitoring notebook logs") - Expect(MonitorNotebookPod(namespace, "jupyter-nb-", notebookName)).To(Succeed(), "Notebook execution failed") -} - -// RunNotebookTest performs all the setup steps, creates a notebook, and monitors its execution. -// This function is kept for backward compatibility. For new tests, use CreateNotebookTest and MonitorNotebookTest separately. -// feastProject is optional - if provided, it will be set in the notebook annotation, otherwise it will be empty -func RunNotebookTest(namespace, configMapName, notebookFile, featureRepoPath, pvcFile, rolebindingName, notebookPVC, notebookName, testDir string, feastProject string) { - CreateNotebookTest(namespace, configMapName, notebookFile, featureRepoPath, pvcFile, rolebindingName, notebookPVC, notebookName, testDir, feastProject) - MonitorNotebookTest(namespace, notebookName) -} diff --git a/infra/feast-operator/test/utils/test_util.go b/infra/feast-operator/test/utils/test_util.go index a883efc020d..7b5f0f8d6a0 100644 --- a/infra/feast-operator/test/utils/test_util.go +++ b/infra/feast-operator/test/utils/test_util.go @@ -152,104 +152,6 @@ func checkIfConfigMapExists(namespace, configMapName string) error { return nil } -// ListConfigMaps lists all ConfigMaps in the given namespace -func ListConfigMaps(namespace string) ([]string, error) { - cmd := exec.Command("kubectl", "get", "cm", "-n", namespace, "-o", "jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}") - var out bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &stderr - - if err := cmd.Run(); err != nil { - return nil, fmt.Errorf("failed to list config maps in namespace %s. Error: %v. Stderr: %s", - namespace, err, stderr.String()) - } - - configMaps := strings.Split(strings.TrimSpace(out.String()), "\n") - // Filter out empty strings - var result []string - for _, cm := range configMaps { - if cm != "" { - result = append(result, cm) - } - } - return result, nil -} - -// VerifyConfigMapExistsInList checks if a ConfigMap exists in the list of ConfigMaps -func VerifyConfigMapExistsInList(namespace, configMapName string) (bool, error) { - configMaps, err := ListConfigMaps(namespace) - if err != nil { - return false, err - } - - for _, cm := range configMaps { - if cm == configMapName { - return true, nil - } - } - - return false, nil -} - -// VerifyFeastConfigMapExists verifies that a ConfigMap exists and contains the specified key/file -func VerifyFeastConfigMapExists(namespace, configMapName, expectedKey string) error { - // First verify the ConfigMap exists - if err := checkIfConfigMapExists(namespace, configMapName); err != nil { - return fmt.Errorf("config map %s does not exist: %w", configMapName, err) - } - - // Get the ConfigMap data to verify the key exists - cmd := exec.Command("kubectl", "get", "cm", configMapName, "-n", namespace, "-o", "jsonpath={.data."+expectedKey+"}") - var out bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to get config map data for %s in namespace %s. Error: %v. Stderr: %s", - configMapName, namespace, err, stderr.String()) - } - - configContent := out.String() - if configContent == "" { - return fmt.Errorf("config map %s does not contain key %s", configMapName, expectedKey) - } - - return nil -} - -// VerifyFeastConfigMapContent verifies that a ConfigMap contains the expected feast configuration content -// This assumes the ConfigMap and key already exist (use VerifyFeastConfigMapExists first) -func VerifyFeastConfigMapContent(namespace, configMapName, expectedKey string, expectedContent []string) error { - // Get the ConfigMap data - cmd := exec.Command("kubectl", "get", "cm", configMapName, "-n", namespace, "-o", "jsonpath={.data."+expectedKey+"}") - var out bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to get config map data for %s in namespace %s. Error: %v. Stderr: %s", - configMapName, namespace, err, stderr.String()) - } - - configContent := out.String() - if configContent == "" { - return fmt.Errorf("config map %s does not contain key %s", configMapName, expectedKey) - } - - // Verify all expected content strings are present - for _, expected := range expectedContent { - if !strings.Contains(configContent, expected) { - return fmt.Errorf("config map %s content does not contain expected string: %s. Content:\n%s", - configMapName, expected, configContent) - } - } - - return nil -} - // validates if a kubernetes service exists using the kubectl CLI. func checkIfKubernetesServiceExists(namespace, serviceName string) error { cmd := exec.Command("kubectl", "get", "service", serviceName, "-n", namespace) @@ -794,77 +696,3 @@ func ApplyFeastYamlAndVerify(namespace string, testDir string, feastDeploymentNa By("Verifying client feature_store.yaml for expected store types") validateFeatureStoreYaml(namespace, feastDeploymentName) } - -// ReplaceNamespaceInYaml reads a YAML file, replaces all existingNamespace with the actual namespace -func ReplaceNamespaceInYamlFilesInPlace(filePaths []string, existingNamespace string, actualNamespace string) error { - for _, filePath := range filePaths { - data, err := os.ReadFile(filePath) - if err != nil { - return fmt.Errorf("failed to read YAML file %s: %w", filePath, err) - } - updated := strings.ReplaceAll(string(data), existingNamespace, actualNamespace) - - err = os.WriteFile(filePath, []byte(updated), 0644) - if err != nil { - return fmt.Errorf("failed to write updated YAML file %s: %w", filePath, err) - } - } - return nil -} - -func ApplyFeastPermissions(fileName string, registryFilePath string, namespace string, podNamePrefix string) { - By("Applying Feast permissions to the Feast registry pod") - - // 1. Get the pod by prefix - By(fmt.Sprintf("Finding pod with prefix %q in namespace %q", podNamePrefix, namespace)) - pod, err := getPodByPrefix(namespace, podNamePrefix) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - ExpectWithOffset(1, pod).NotTo(BeNil()) - - podName := pod.Name - fmt.Printf("Found pod: %s\n", podName) - - cmd := exec.Command( - "oc", "cp", - fileName, // local source file - fmt.Sprintf("%s/%s:%s", namespace, podName, registryFilePath), // remote destination - "-c", "registry", - ) - - _, err = Run(cmd, "/test/e2e_rhoai") - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - fmt.Printf("Successfully copied file to pod: %s\n", podName) - - // Run `feast apply` inside the pod to apply updated permissions - By("Running feast apply inside the Feast registry pod") - cmd = exec.Command( - "oc", "exec", podName, - "-n", namespace, - "-c", "registry", - "--", - "bash", "-c", - "cd /feast-data/credit_scoring_local/feature_repo && feast apply", - ) - _, err = Run(cmd, "/test/e2e_rhoai") - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - fmt.Println("Feast permissions apply executed successfully") - - By("Validating that Feast permission has been applied") - - cmd = exec.Command( - "oc", "exec", podName, - "-n", namespace, - "-c", "registry", - "--", - "feast", "permissions", "list", - ) - - output, err := Run(cmd, "/test/e2e_rhoai") - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - // Change "feast-auth" if your permission name is different - ExpectWithOffset(1, output).To(ContainSubstring("feast-auth"), "Expected permission 'feast-auth' to exist") - - fmt.Println("Verified: Feast permission 'feast-auth' exists") -} diff --git a/sdk/python/feast/api/registry/rest/rest_utils.py b/sdk/python/feast/api/registry/rest/rest_utils.py index dac6b7ccf7b..4c517b0abdf 100644 --- a/sdk/python/feast/api/registry/rest/rest_utils.py +++ b/sdk/python/feast/api/registry/rest/rest_utils.py @@ -18,7 +18,6 @@ MATCH_SCORE_NAME = 100 MATCH_SCORE_DESCRIPTION = 80 MATCH_SCORE_TAGS = 60 -MATCH_SCORE_PARTIAL = 40 def grpc_call(handler_fn, request): @@ -537,12 +536,42 @@ def filter_search_results_and_match_score( ) -> List[Dict]: """Filter search results based on query string""" if not query: + # Add all tags as matched_tags when no query (all tags match) + for result in results: + result["matched_tags"] = result.get("tags", {}) return results query_lower = query.lower() filtered_results = [] for result in results: + matched_tags = {} + best_fuzzy_tag_score = 0.0 + + # Collect all matching tags (exact and fuzzy) upfront + tags = result.get("tags", {}) + has_exact_tag_match = False + + for key, value in tags.items(): + key_lower = str(key).lower() + value_str = str(value).lower() + tag_combined = f"{key_lower}={value_str}" + + # Exact match in key or value + if query_lower in tag_combined: + has_exact_tag_match = True + matched_tags[key] = value + else: + # Fuzzy match for tags (on combined "key:value" string) + tag_fuzzy_score = fuzzy_match(query_lower, tag_combined) + + if tag_fuzzy_score >= MATCH_SCORE_DEFAULT_THRESHOLD: + matched_tags[key] = value + if tag_fuzzy_score > best_fuzzy_tag_score: + best_fuzzy_tag_score = tag_fuzzy_score + + result["matched_tags"] = matched_tags + # Search in name if query_lower in result.get("name", "").lower(): result["match_score"] = MATCH_SCORE_NAME @@ -555,42 +584,15 @@ def filter_search_results_and_match_score( filtered_results.append(result) continue - # Search in tags - tags = result.get("tags", {}) - tag_match = False - matched_tag = None - best_fuzzy_score = 0.0 - best_fuzzy_tag = None - - for key, value in tags.items(): - key_lower = key.lower() - value_str = str(value).lower() - - # Exact match in key or value - if query_lower in key_lower or query_lower in value_str: - tag_match = True - # Store the matched tag as a dictionary - matched_tag = {key: value} - break - - # Fuzzy match for tags (on combined "key:value" string) - tag_combined = f"{key_lower}={value_str}" - tag_fuzzy_score = fuzzy_match(query_lower, tag_combined) - - if tag_fuzzy_score > best_fuzzy_score: - best_fuzzy_score = tag_fuzzy_score - best_fuzzy_tag = {key: value} - - if tag_match: + # Exact tag match + if has_exact_tag_match: result["match_score"] = MATCH_SCORE_TAGS - result["matched_tag"] = matched_tag filtered_results.append(result) continue # Fuzzy tag match - if best_fuzzy_score >= MATCH_SCORE_DEFAULT_THRESHOLD: - result["match_score"] = best_fuzzy_score * 100 - result["matched_tag"] = best_fuzzy_tag + if best_fuzzy_tag_score >= MATCH_SCORE_DEFAULT_THRESHOLD: + result["match_score"] = best_fuzzy_tag_score * 100 filtered_results.append(result) continue diff --git a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres_source.py b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres_source.py index 0d3045e4aa7..272b8ad0474 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres_source.py @@ -145,11 +145,45 @@ def get_table_column_names_and_types( ) def get_table_query_string(self) -> str: + """Returns a string that can be used to reference this table in SQL. + + For query-based sources, returns the query wrapped in parentheses. + + Note: + When using the returned string directly in a FROM clause with PostgreSQL, + you may need to add an alias if this is a query-based source. PostgreSQL + requires all subqueries in FROM clauses to have aliases. Consider using + get_table_query_string_with_alias() for automatic aliasing. + """ if self._postgres_options._table: return f"{self._postgres_options._table}" else: return f"({self._postgres_options._query})" + def get_table_query_string_with_alias(self, alias: str = "subquery") -> str: + """Returns a string for use in FROM clause with alias for PostgreSQL compatibility. + + PostgreSQL requires all subqueries in FROM clauses to have aliases. This method + automatically adds an alias when the source is query-based. + + Args: + alias: The alias to use for query-based sources. Defaults to "subquery". + + Returns: + For table-based sources: the table name (no alias needed). + For query-based sources: "(query) AS alias". + + Example:: + + source = PostgreSQLSource(query="SELECT * FROM my_table", ...) + entity_sql = f"SELECT id, ts FROM {source.get_table_query_string_with_alias()}" + # Results in: "SELECT id, ts FROM (SELECT * FROM my_table) AS subquery" + """ + if self._postgres_options._table: + return f"{self._postgres_options._table}" + else: + return f"({self._postgres_options._query}) AS {alias}" + class PostgreSQLOptions: def __init__( diff --git a/sdk/python/feast/infra/online_stores/redis.py b/sdk/python/feast/infra/online_stores/redis.py index 59892fcbe0f..9a4e908810d 100644 --- a/sdk/python/feast/infra/online_stores/redis.py +++ b/sdk/python/feast/infra/online_stores/redis.py @@ -304,22 +304,25 @@ def online_write_batch( for redis_key_bin, prev_event_time, (_, values, timestamp, _) in zip( keys, prev_event_timestamps, data ): - event_time_seconds = int(utils.make_tzaware(timestamp).timestamp()) - - # ignore if event_timestamp is before the event features that are currently in the feature store + # Convert incoming timestamp to millisecond-aware datetime + aware_ts = utils.make_tzaware(timestamp) + # Build protobuf timestamp with nanos + ts = Timestamp() + ts.FromDatetime(aware_ts) + # New timestamp in nanoseconds + new_total_nanos = ts.seconds * 1_000_000_000 + ts.nanos + # Compare against existing timestamp (nanosecond precision) if prev_event_time: prev_ts = Timestamp() prev_ts.ParseFromString(prev_event_time) - if prev_ts.seconds and event_time_seconds <= prev_ts.seconds: - # TODO: somehow signal that it's not overwriting the current record? + prev_total_nanos = prev_ts.seconds * 1_000_000_000 + prev_ts.nanos + # Skip only if older OR exact same instant + if prev_total_nanos and new_total_nanos <= prev_total_nanos: if progress: progress(1) continue - - ts = Timestamp() - ts.seconds = event_time_seconds - entity_hset = dict() - entity_hset[ts_key] = ts.SerializeToString() + # Store full timestamp (seconds + nanos) + entity_hset = {ts_key: ts.SerializeToString()} for feature_name, val in values.items(): f_key = _mmh3(f"{feature_view}:{feature_name}") @@ -456,5 +459,7 @@ def _get_features_for_entity( if not res: return None, None else: - timestamp = datetime.fromtimestamp(res_ts.seconds, tz=timezone.utc) + # reconstruct full timestamp including nanos + total_seconds = res_ts.seconds + res_ts.nanos / 1_000_000_000.0 + timestamp = datetime.fromtimestamp(total_seconds, tz=timezone.utc) return timestamp, res diff --git a/sdk/python/feast/permissions/auth_model.py b/sdk/python/feast/permissions/auth_model.py index 179a8bc0c0e..3690d62b728 100644 --- a/sdk/python/feast/permissions/auth_model.py +++ b/sdk/python/feast/permissions/auth_model.py @@ -33,25 +33,23 @@ class OidcClientAuthConfig(OidcAuthConfig): token: Optional[str] = None # pre-issued `token` @model_validator(mode="after") - def _validate_credentials(cls, values): + def _validate_credentials(self): """Enforce exactly one valid credential set.""" - d = values.__dict__ if hasattr(values, "__dict__") else values - - has_user_pass = bool(d.get("username")) and bool(d.get("password")) - has_secret = bool(d.get("client_secret")) - has_token = bool(d.get("token")) + has_user_pass = bool(self.username) and bool(self.password) + has_secret = bool(self.client_secret) + has_token = bool(self.token) # 1 static token if has_token and not (has_user_pass or has_secret): - return values + return self # 2 client_credentials if has_secret and not has_user_pass and not has_token: - return values + return self # 3 ROPG if has_user_pass and has_secret and not has_token: - return values + return self raise ValueError( "Invalid OIDC client auth combination: " diff --git a/sdk/python/feast/permissions/enforcer.py b/sdk/python/feast/permissions/enforcer.py index 2cb6608a2d8..4db0241ef5a 100644 --- a/sdk/python/feast/permissions/enforcer.py +++ b/sdk/python/feast/permissions/enforcer.py @@ -43,9 +43,9 @@ def enforce_policy( # If no permissions are defined, deny access to all resources # This is a security measure to prevent unauthorized access logger.warning("No permissions defined - denying access to all resources") - if not filter_only: - raise FeastPermissionError("No permissions defined - access denied") - return [] + raise FeastPermissionError( + "Permissions are not defined - access denied for all resources" + ) _permitted_resources: list[FeastObject] = [] for resource in resources: @@ -71,17 +71,42 @@ def enforce_policy( if evaluator.is_decided(): grant, explanations = evaluator.grant() - if not grant and not filter_only: + if not grant: + if filter_only and p.name_patterns: + continue logger.error(f"Permission denied: {','.join(explanations)}") raise FeastPermissionError(",".join(explanations)) - if grant: - logger.debug( - f"Permission granted for {type(resource).__name__}:{resource.name}" - ) - _permitted_resources.append(resource) + logger.debug( + f"Permission granted for {type(resource).__name__}:{resource.name}" + ) + _permitted_resources.append(resource) break else: - message = f"No permissions defined to manage {actions} on {type(resource)}/{resource.name}." - logger.exception(f"**PERMISSION NOT GRANTED**: {message}") - raise FeastPermissionError(message) + if not filter_only: + message = f"No permissions defined to manage {actions} on {type(resource)}/{resource.name}." + logger.exception(f"**PERMISSION NOT GRANTED**: {message}") + raise FeastPermissionError(message) + else: + # filter_only=True: Check if there are permissions for this resource type + resource_type_permissions = [ + p + for p in permissions + if any(isinstance(resource, t) for t in p.types) # type: ignore + ] + if not resource_type_permissions: + # No permissions exist for this resource type - should raise error + message = f"No permissions defined to manage {actions} on {type(resource)}/{resource.name}." + logger.exception(f"**PERMISSION NOT GRANTED**: {message}") + raise FeastPermissionError(message) + elif not any(p.name_patterns for p in resource_type_permissions): + # Permissions exist for this resource type but no name_patterns - should raise error + message = f"No permissions defined to manage {actions} on {type(resource)}/{resource.name}." + logger.exception(f"**PERMISSION NOT GRANTED**: {message}") + raise FeastPermissionError(message) + else: + # Permissions exist for this resource type with name_patterns - filter out this resource + logger.debug( + f"Filtering out {type(resource).__name__}:{resource.name} - no matching permissions" + ) + continue return _permitted_resources diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 2c448173938..554c9eba60f 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -5988,9 +5988,9 @@ uri-template==1.3.0 \ --hash=sha256:0e00f8eb65e18c7de20d595a14336e9f337ead580c70934141624b6d1ffdacc7 \ --hash=sha256:a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363 # via jsonschema -urllib3==2.6.2 \ - --hash=sha256:016f9c98bb7e98085cb2b4b17b87d2c702975664e4f060c6532e64d1c1a5e797 \ - --hash=sha256:ec21cddfe7724fc7cb4ba4bea7aa8e2ef36f607a4bab81aa6ce42a13dc3f03dd +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 # via # feast (setup.py) # botocore diff --git a/sdk/python/requirements/py3.10-minimal-requirements.txt b/sdk/python/requirements/py3.10-minimal-requirements.txt index 66d34c10570..49c47aba783 100644 --- a/sdk/python/requirements/py3.10-minimal-requirements.txt +++ b/sdk/python/requirements/py3.10-minimal-requirements.txt @@ -2812,9 +2812,9 @@ ujson==5.11.0 \ --hash=sha256:fa79fdb47701942c2132a9dd2297a1a85941d966d8c87bfd9e29b0cf423f26cc \ --hash=sha256:fac6c0649d6b7c3682a0a6e18d3de6857977378dce8d419f57a0b20e3d775b39 # via pymilvus -urllib3==2.6.2 \ - --hash=sha256:016f9c98bb7e98085cb2b4b17b87d2c702975664e4f060c6532e64d1c1a5e797 \ - --hash=sha256:ec21cddfe7724fc7cb4ba4bea7aa8e2ef36f607a4bab81aa6ce42a13dc3f03dd +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 # via # botocore # kubernetes diff --git a/sdk/python/requirements/py3.10-minimal-sdist-requirements.txt b/sdk/python/requirements/py3.10-minimal-sdist-requirements.txt index 1c92409c888..159a6a266ae 100644 --- a/sdk/python/requirements/py3.10-minimal-sdist-requirements.txt +++ b/sdk/python/requirements/py3.10-minimal-sdist-requirements.txt @@ -3092,9 +3092,9 @@ ujson==5.11.0 \ --hash=sha256:fa79fdb47701942c2132a9dd2297a1a85941d966d8c87bfd9e29b0cf423f26cc \ --hash=sha256:fac6c0649d6b7c3682a0a6e18d3de6857977378dce8d419f57a0b20e3d775b39 # via pymilvus -urllib3==2.6.2 \ - --hash=sha256:016f9c98bb7e98085cb2b4b17b87d2c702975664e4f060c6532e64d1c1a5e797 \ - --hash=sha256:ec21cddfe7724fc7cb4ba4bea7aa8e2ef36f607a4bab81aa6ce42a13dc3f03dd +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 # via # botocore # kubernetes diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 6f38b0c16c6..bf89b08b92a 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -1326,9 +1326,9 @@ tzdata==2025.3 \ --hash=sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1 \ --hash=sha256:de39c2ca5dc7b0344f2eba86f49d614019d29f060fc4ebc8a417896a620b56a7 # via pandas -urllib3==2.6.2 \ - --hash=sha256:016f9c98bb7e98085cb2b4b17b87d2c702975664e4f060c6532e64d1c1a5e797 \ - --hash=sha256:ec21cddfe7724fc7cb4ba4bea7aa8e2ef36f607a4bab81aa6ce42a13dc3f03dd +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 # via requests uvicorn[standard]==0.34.0 \ --hash=sha256:023dc038422502fa28a09c7a30bf2b6991512da7dcdb8fd35fe57cfc154126f4 \ diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 091236521ff..b3bbdb5ad4d 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -6229,9 +6229,9 @@ uri-template==1.3.0 \ --hash=sha256:0e00f8eb65e18c7de20d595a14336e9f337ead580c70934141624b6d1ffdacc7 \ --hash=sha256:a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363 # via jsonschema -urllib3==2.6.2 \ - --hash=sha256:016f9c98bb7e98085cb2b4b17b87d2c702975664e4f060c6532e64d1c1a5e797 \ - --hash=sha256:ec21cddfe7724fc7cb4ba4bea7aa8e2ef36f607a4bab81aa6ce42a13dc3f03dd +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 # via # feast (setup.py) # botocore diff --git a/sdk/python/requirements/py3.11-minimal-requirements.txt b/sdk/python/requirements/py3.11-minimal-requirements.txt index 93613cb6e50..04ac9943f7b 100644 --- a/sdk/python/requirements/py3.11-minimal-requirements.txt +++ b/sdk/python/requirements/py3.11-minimal-requirements.txt @@ -2816,9 +2816,9 @@ ujson==5.11.0 \ --hash=sha256:fa79fdb47701942c2132a9dd2297a1a85941d966d8c87bfd9e29b0cf423f26cc \ --hash=sha256:fac6c0649d6b7c3682a0a6e18d3de6857977378dce8d419f57a0b20e3d775b39 # via pymilvus -urllib3==2.6.2 \ - --hash=sha256:016f9c98bb7e98085cb2b4b17b87d2c702975664e4f060c6532e64d1c1a5e797 \ - --hash=sha256:ec21cddfe7724fc7cb4ba4bea7aa8e2ef36f607a4bab81aa6ce42a13dc3f03dd +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 # via # botocore # kubernetes diff --git a/sdk/python/requirements/py3.11-minimal-sdist-requirements.txt b/sdk/python/requirements/py3.11-minimal-sdist-requirements.txt index 3fb8dbf6562..56fa579fe39 100644 --- a/sdk/python/requirements/py3.11-minimal-sdist-requirements.txt +++ b/sdk/python/requirements/py3.11-minimal-sdist-requirements.txt @@ -3092,9 +3092,9 @@ ujson==5.11.0 \ --hash=sha256:fa79fdb47701942c2132a9dd2297a1a85941d966d8c87bfd9e29b0cf423f26cc \ --hash=sha256:fac6c0649d6b7c3682a0a6e18d3de6857977378dce8d419f57a0b20e3d775b39 # via pymilvus -urllib3==2.6.2 \ - --hash=sha256:016f9c98bb7e98085cb2b4b17b87d2c702975664e4f060c6532e64d1c1a5e797 \ - --hash=sha256:ec21cddfe7724fc7cb4ba4bea7aa8e2ef36f607a4bab81aa6ce42a13dc3f03dd +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 # via # botocore # kubernetes diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index 8253a7a0800..fcae25c341f 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -1293,9 +1293,9 @@ tzdata==2025.3 \ --hash=sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1 \ --hash=sha256:de39c2ca5dc7b0344f2eba86f49d614019d29f060fc4ebc8a417896a620b56a7 # via pandas -urllib3==2.6.2 \ - --hash=sha256:016f9c98bb7e98085cb2b4b17b87d2c702975664e4f060c6532e64d1c1a5e797 \ - --hash=sha256:ec21cddfe7724fc7cb4ba4bea7aa8e2ef36f607a4bab81aa6ce42a13dc3f03dd +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 # via requests uvicorn[standard]==0.34.0 \ --hash=sha256:023dc038422502fa28a09c7a30bf2b6991512da7dcdb8fd35fe57cfc154126f4 \ diff --git a/sdk/python/requirements/py3.12-ci-requirements.txt b/sdk/python/requirements/py3.12-ci-requirements.txt index 50efe8231a9..48c1fc21371 100644 --- a/sdk/python/requirements/py3.12-ci-requirements.txt +++ b/sdk/python/requirements/py3.12-ci-requirements.txt @@ -6217,9 +6217,9 @@ uri-template==1.3.0 \ --hash=sha256:0e00f8eb65e18c7de20d595a14336e9f337ead580c70934141624b6d1ffdacc7 \ --hash=sha256:a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363 # via jsonschema -urllib3==2.6.2 \ - --hash=sha256:016f9c98bb7e98085cb2b4b17b87d2c702975664e4f060c6532e64d1c1a5e797 \ - --hash=sha256:ec21cddfe7724fc7cb4ba4bea7aa8e2ef36f607a4bab81aa6ce42a13dc3f03dd +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 # via # feast (setup.py) # botocore diff --git a/sdk/python/requirements/py3.12-minimal-requirements.txt b/sdk/python/requirements/py3.12-minimal-requirements.txt index d2064d4ed78..707d8cdddad 100644 --- a/sdk/python/requirements/py3.12-minimal-requirements.txt +++ b/sdk/python/requirements/py3.12-minimal-requirements.txt @@ -2808,9 +2808,9 @@ ujson==5.11.0 \ --hash=sha256:fa79fdb47701942c2132a9dd2297a1a85941d966d8c87bfd9e29b0cf423f26cc \ --hash=sha256:fac6c0649d6b7c3682a0a6e18d3de6857977378dce8d419f57a0b20e3d775b39 # via pymilvus -urllib3==2.6.2 \ - --hash=sha256:016f9c98bb7e98085cb2b4b17b87d2c702975664e4f060c6532e64d1c1a5e797 \ - --hash=sha256:ec21cddfe7724fc7cb4ba4bea7aa8e2ef36f607a4bab81aa6ce42a13dc3f03dd +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 # via # botocore # kubernetes diff --git a/sdk/python/requirements/py3.12-minimal-sdist-requirements.txt b/sdk/python/requirements/py3.12-minimal-sdist-requirements.txt index ac6f755be87..b787047fe86 100644 --- a/sdk/python/requirements/py3.12-minimal-sdist-requirements.txt +++ b/sdk/python/requirements/py3.12-minimal-sdist-requirements.txt @@ -3084,9 +3084,9 @@ ujson==5.11.0 \ --hash=sha256:fa79fdb47701942c2132a9dd2297a1a85941d966d8c87bfd9e29b0cf423f26cc \ --hash=sha256:fac6c0649d6b7c3682a0a6e18d3de6857977378dce8d419f57a0b20e3d775b39 # via pymilvus -urllib3==2.6.2 \ - --hash=sha256:016f9c98bb7e98085cb2b4b17b87d2c702975664e4f060c6532e64d1c1a5e797 \ - --hash=sha256:ec21cddfe7724fc7cb4ba4bea7aa8e2ef36f607a4bab81aa6ce42a13dc3f03dd +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 # via # botocore # kubernetes diff --git a/sdk/python/requirements/py3.12-requirements.txt b/sdk/python/requirements/py3.12-requirements.txt index 697d4f25b0c..cefe5dbf8c7 100644 --- a/sdk/python/requirements/py3.12-requirements.txt +++ b/sdk/python/requirements/py3.12-requirements.txt @@ -1289,9 +1289,9 @@ tzdata==2025.3 \ --hash=sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1 \ --hash=sha256:de39c2ca5dc7b0344f2eba86f49d614019d29f060fc4ebc8a417896a620b56a7 # via pandas -urllib3==2.6.2 \ - --hash=sha256:016f9c98bb7e98085cb2b4b17b87d2c702975664e4f060c6532e64d1c1a5e797 \ - --hash=sha256:ec21cddfe7724fc7cb4ba4bea7aa8e2ef36f607a4bab81aa6ce42a13dc3f03dd +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 # via requests uvicorn[standard]==0.34.0 \ --hash=sha256:023dc038422502fa28a09c7a30bf2b6991512da7dcdb8fd35fe57cfc154126f4 \ diff --git a/sdk/python/tests/registry_rest_api_tests/conftest.py b/sdk/python/tests/integration/registration/rest_api/conftest.py similarity index 78% rename from sdk/python/tests/registry_rest_api_tests/conftest.py rename to sdk/python/tests/integration/registration/rest_api/conftest.py index 2a128785fd8..36c358a9aa6 100644 --- a/sdk/python/tests/registry_rest_api_tests/conftest.py +++ b/sdk/python/tests/integration/registration/rest_api/conftest.py @@ -1,9 +1,11 @@ import os +from pathlib import Path import pytest import requests from kubernetes import client, config -from support import ( + +from tests.integration.registration.rest_api.support import ( applyFeastProject, create_feast_project, create_namespace, @@ -41,6 +43,10 @@ def feast_rest_client(): config.load_kube_config() api_instance = client.CoreV1Api() + # Get the directory containing this conftest.py file + test_dir = Path(__file__).parent + resource_dir = test_dir / "resource" + # Constants and environment values namespace = "test-ns-feast-rest" credit_scoring = "credit-scoring" @@ -54,23 +60,37 @@ def feast_rest_client(): try: if not run_on_openshift: # Deploy dependencies - deploy_and_validate_pod(namespace, "resource/redis.yaml", "app=redis") - deploy_and_validate_pod(namespace, "resource/postgres.yaml", "app=postgres") + deploy_and_validate_pod( + namespace, str(resource_dir / "redis.yaml"), "app=redis" + ) + deploy_and_validate_pod( + namespace, str(resource_dir / "postgres.yaml"), "app=postgres" + ) # Create and validate FeatureStore CRs create_feast_project( - "resource/feast_config_credit_scoring.yaml", namespace, credit_scoring + str(resource_dir / "feast_config_credit_scoring.yaml"), + namespace, + credit_scoring, ) validate_feature_store_cr_status(namespace, credit_scoring) create_feast_project( - "resource/feast_config_driver_ranking.yaml", namespace, driver_ranking + str(resource_dir / "feast_config_driver_ranking.yaml"), + namespace, + driver_ranking, ) validate_feature_store_cr_status(namespace, driver_ranking) # Deploy ingress and get route URL run_kubectl_command( - ["apply", "-f", "resource/feast-registry-nginx.yaml", "-n", namespace] + [ + "apply", + "-f", + str(resource_dir / "feast-registry-nginx.yaml"), + "-n", + namespace, + ] ) ingress_host = run_kubectl_command( [ @@ -114,7 +134,7 @@ def feast_rest_client(): aws_secret_key, aws_bucket, registry_path, - "resource/feast_config_rhoai.yaml", + str(resource_dir / "feast_config_rhoai.yaml"), namespace, ) validate_feature_store_cr_status(namespace, "test-s3") diff --git a/sdk/python/tests/registry_rest_api_tests/resource/feast-registry-nginx.yaml b/sdk/python/tests/integration/registration/rest_api/resource/feast-registry-nginx.yaml similarity index 100% rename from sdk/python/tests/registry_rest_api_tests/resource/feast-registry-nginx.yaml rename to sdk/python/tests/integration/registration/rest_api/resource/feast-registry-nginx.yaml diff --git a/sdk/python/tests/registry_rest_api_tests/resource/feast_config_credit_scoring.yaml b/sdk/python/tests/integration/registration/rest_api/resource/feast_config_credit_scoring.yaml similarity index 100% rename from sdk/python/tests/registry_rest_api_tests/resource/feast_config_credit_scoring.yaml rename to sdk/python/tests/integration/registration/rest_api/resource/feast_config_credit_scoring.yaml diff --git a/sdk/python/tests/registry_rest_api_tests/resource/feast_config_driver_ranking.yaml b/sdk/python/tests/integration/registration/rest_api/resource/feast_config_driver_ranking.yaml similarity index 100% rename from sdk/python/tests/registry_rest_api_tests/resource/feast_config_driver_ranking.yaml rename to sdk/python/tests/integration/registration/rest_api/resource/feast_config_driver_ranking.yaml diff --git a/sdk/python/tests/registry_rest_api_tests/resource/feast_config_rhoai.yaml b/sdk/python/tests/integration/registration/rest_api/resource/feast_config_rhoai.yaml similarity index 100% rename from sdk/python/tests/registry_rest_api_tests/resource/feast_config_rhoai.yaml rename to sdk/python/tests/integration/registration/rest_api/resource/feast_config_rhoai.yaml diff --git a/sdk/python/tests/registry_rest_api_tests/resource/postgres.yaml b/sdk/python/tests/integration/registration/rest_api/resource/postgres.yaml similarity index 100% rename from sdk/python/tests/registry_rest_api_tests/resource/postgres.yaml rename to sdk/python/tests/integration/registration/rest_api/resource/postgres.yaml diff --git a/sdk/python/tests/registry_rest_api_tests/resource/redis.yaml b/sdk/python/tests/integration/registration/rest_api/resource/redis.yaml similarity index 100% rename from sdk/python/tests/registry_rest_api_tests/resource/redis.yaml rename to sdk/python/tests/integration/registration/rest_api/resource/redis.yaml diff --git a/sdk/python/tests/registry_rest_api_tests/support.py b/sdk/python/tests/integration/registration/rest_api/support.py similarity index 100% rename from sdk/python/tests/registry_rest_api_tests/support.py rename to sdk/python/tests/integration/registration/rest_api/support.py diff --git a/sdk/python/tests/registry_rest_api_tests/test_feast_registry.py b/sdk/python/tests/integration/registration/rest_api/test_registry_rest_api.py similarity index 84% rename from sdk/python/tests/registry_rest_api_tests/test_feast_registry.py rename to sdk/python/tests/integration/registration/rest_api/test_registry_rest_api.py index 04e86618cf7..5d37d700a64 100644 --- a/sdk/python/tests/registry_rest_api_tests/test_feast_registry.py +++ b/sdk/python/tests/integration/registration/rest_api/test_registry_rest_api.py @@ -13,7 +13,7 @@ # Test Configuration Constants @dataclass(frozen=True) -class TestConfig: +class RegistryTestConfig: """Configuration constants for registry REST API tests.""" CREDIT_SCORING_PROJECT = "credit_scoring_local" @@ -103,7 +103,7 @@ def validate_entity_structure(entity: Dict[str, Any]) -> None: assert "lastUpdatedTimestamp" in meta assert isinstance(entity["project"], str) - assert entity["project"] in TestConfig.PROJECT_NAMES + assert entity["project"] in RegistryTestConfig.PROJECT_NAMES @staticmethod def validate_feature_structure(feature: Dict[str, Any]) -> None: @@ -132,6 +132,7 @@ def validate_batch_source(batch_source: Dict[str, Any]) -> None: assert batch_source.get("type") == "BATCH_FILE" +@pytest.mark.integration @pytest.mark.skipif( not os.path.exists(os.path.expanduser("~/.kube/config")), reason="Kube config not available in this environment", @@ -143,7 +144,7 @@ class TestRegistryServerRest: def test_list_entities(self, feast_rest_client): """Test listing entities for a specific project.""" response = feast_rest_client.get( - f"/entities/?project={TestConfig.CREDIT_SCORING_PROJECT}" + f"/entities/?project={RegistryTestConfig.CREDIT_SCORING_PROJECT}" ) data = APITestHelpers.validate_response_success(response) @@ -151,21 +152,21 @@ def test_list_entities(self, feast_rest_client): assert "entities" in data entities = data["entities"] assert isinstance(entities, list) - assert len(entities) == TestConfig.CREDIT_SCORING_ENTITIES_COUNT + assert len(entities) == RegistryTestConfig.CREDIT_SCORING_ENTITIES_COUNT # Validate entity names actual_entity_names = {entity["spec"]["name"] for entity in entities} - assert actual_entity_names == TestConfig.ENTITY_NAMES + assert actual_entity_names == RegistryTestConfig.ENTITY_NAMES # Validate pagination APITestHelpers.validate_pagination( - data, TestConfig.CREDIT_SCORING_ENTITIES_COUNT + data, RegistryTestConfig.CREDIT_SCORING_ENTITIES_COUNT ) def test_get_entity(self, feast_rest_client): """Test getting a specific entity with detailed validation.""" response = feast_rest_client.get( - f"/entities/zipcode/?project={TestConfig.CREDIT_SCORING_PROJECT}" + f"/entities/zipcode/?project={RegistryTestConfig.CREDIT_SCORING_PROJECT}" ) data = APITestHelpers.validate_response_success(response) @@ -178,7 +179,7 @@ def test_get_entity(self, feast_rest_client): spec["description"] == "ZIP code identifier for geographic location-based features" ) - assert spec["tags"] == TestConfig.ZIPCODE_SPEC_TAGS + assert spec["tags"] == RegistryTestConfig.ZIPCODE_SPEC_TAGS # Validate meta meta = data["meta"] @@ -215,22 +216,22 @@ def test_entities_all(self, feast_rest_client): def test_list_data_sources(self, feast_rest_client): """Test listing data sources for a specific project.""" response = feast_rest_client.get( - f"/data_sources/?project={TestConfig.CREDIT_SCORING_PROJECT}" + f"/data_sources/?project={RegistryTestConfig.CREDIT_SCORING_PROJECT}" ) data = APITestHelpers.validate_response_success(response) assert "dataSources" in data data_sources = data["dataSources"] - assert len(data_sources) == TestConfig.CREDIT_SCORING_DATA_SOURCES_COUNT + assert len(data_sources) == RegistryTestConfig.CREDIT_SCORING_DATA_SOURCES_COUNT APITestHelpers.validate_pagination( - data, TestConfig.CREDIT_SCORING_DATA_SOURCES_COUNT + data, RegistryTestConfig.CREDIT_SCORING_DATA_SOURCES_COUNT ) def test_get_data_sources(self, feast_rest_client): """Test getting a specific data source.""" response = feast_rest_client.get( - f"/data_sources/Zipcode source/?project={TestConfig.CREDIT_SCORING_PROJECT}" + f"/data_sources/Zipcode source/?project={RegistryTestConfig.CREDIT_SCORING_PROJECT}" ) data = APITestHelpers.validate_response_success(response) @@ -254,7 +255,7 @@ def test_data_sources_all(self, feast_rest_client): # Validate project associations for relevant data source types for ds in data_sources: if ds["type"] in ("BATCH_FILE", "REQUEST_SOURCE"): - assert ds["project"] in TestConfig.PROJECT_NAMES + assert ds["project"] in RegistryTestConfig.PROJECT_NAMES pagination = data.get("pagination", {}) assert pagination.get("page") == 1 @@ -266,12 +267,15 @@ def test_data_sources_all(self, feast_rest_client): def test_list_feature_services(self, feast_rest_client): """Test listing feature services for a specific project.""" response = feast_rest_client.get( - f"/feature_services/?project={TestConfig.DRIVER_RANKING_PROJECT}" + f"/feature_services/?project={RegistryTestConfig.DRIVER_RANKING_PROJECT}" ) data = APITestHelpers.validate_response_success(response) feature_services = data.get("featureServices", []) - assert len(feature_services) == TestConfig.DRIVER_RANKING_FEATURE_SERVICES_COUNT + assert ( + len(feature_services) + == RegistryTestConfig.DRIVER_RANKING_FEATURE_SERVICES_COUNT + ) # Validate batch sources in features for fs in feature_services: @@ -288,7 +292,7 @@ def test_feature_services_all(self, feast_rest_client): assert len(feature_services) >= 1 for fs in feature_services: - assert fs.get("project") in TestConfig.PROJECT_NAMES + assert fs.get("project") in RegistryTestConfig.PROJECT_NAMES # Validate features structure spec = fs.get("spec", {}) @@ -299,7 +303,7 @@ def test_feature_services_all(self, feast_rest_client): def test_get_feature_services(self, feast_rest_client): """Test getting a specific feature service.""" response = feast_rest_client.get( - f"/feature_services/driver_activity_v2/?project={TestConfig.DRIVER_RANKING_PROJECT}" + f"/feature_services/driver_activity_v2/?project={RegistryTestConfig.DRIVER_RANKING_PROJECT}" ) data = APITestHelpers.validate_response_success(response) @@ -313,21 +317,22 @@ def test_get_feature_services(self, feast_rest_client): def test_list_feature_views(self, feast_rest_client): """Test listing feature views for a specific project.""" response = feast_rest_client.get( - f"/feature_views/?project={TestConfig.CREDIT_SCORING_PROJECT}" + f"/feature_views/?project={RegistryTestConfig.CREDIT_SCORING_PROJECT}" ) data = APITestHelpers.validate_response_success(response) assert ( - len(data["featureViews"]) == TestConfig.CREDIT_SCORING_FEATURE_VIEWS_COUNT + len(data["featureViews"]) + == RegistryTestConfig.CREDIT_SCORING_FEATURE_VIEWS_COUNT ) APITestHelpers.validate_pagination( - data, TestConfig.CREDIT_SCORING_FEATURE_VIEWS_COUNT + data, RegistryTestConfig.CREDIT_SCORING_FEATURE_VIEWS_COUNT ) def test_get_feature_view(self, feast_rest_client): """Test getting a specific feature view.""" response = feast_rest_client.get( - f"/feature_views/credit_history/?project={TestConfig.CREDIT_SCORING_PROJECT}" + f"/feature_views/credit_history/?project={RegistryTestConfig.CREDIT_SCORING_PROJECT}" ) data = APITestHelpers.validate_response_success(response) @@ -351,26 +356,26 @@ def test_feature_views_all(self, feast_rest_client): def test_list_features(self, feast_rest_client): """Test listing features for a specific project.""" response = feast_rest_client.get( - f"/features/?project={TestConfig.CREDIT_SCORING_PROJECT}&include_relationships=true" + f"/features/?project={RegistryTestConfig.CREDIT_SCORING_PROJECT}&include_relationships=true" ) data = APITestHelpers.validate_response_success(response) features = data.get("features") assert isinstance(features, list) - assert len(features) == TestConfig.CREDIT_SCORING_FEATURES_COUNT + assert len(features) == RegistryTestConfig.CREDIT_SCORING_FEATURES_COUNT # Validate each feature structure for feature in features: APITestHelpers.validate_feature_structure(feature) APITestHelpers.validate_pagination( - data, TestConfig.CREDIT_SCORING_FEATURES_COUNT + data, RegistryTestConfig.CREDIT_SCORING_FEATURES_COUNT ) def test_get_feature(self, feast_rest_client): """Test getting a specific feature.""" response = feast_rest_client.get( - f"/features/zipcode_features/city/?project={TestConfig.CREDIT_SCORING_PROJECT}&include_relationships=false" + f"/features/zipcode_features/city/?project={RegistryTestConfig.CREDIT_SCORING_PROJECT}&include_relationships=false" ) data = APITestHelpers.validate_response_success(response) @@ -396,14 +401,17 @@ def test_features_all(self, feast_rest_client): # Validate expected projects are present actual_projects = set(f["project"] for f in features) - assert TestConfig.PROJECT_NAMES.issubset(actual_projects) + assert RegistryTestConfig.PROJECT_NAMES.issubset(actual_projects) APITestHelpers.validate_pagination_all_endpoint(data, "features") # Project Tests @pytest.mark.parametrize( "project_name", - [TestConfig.CREDIT_SCORING_PROJECT, TestConfig.DRIVER_RANKING_PROJECT], + [ + RegistryTestConfig.CREDIT_SCORING_PROJECT, + RegistryTestConfig.DRIVER_RANKING_PROJECT, + ], ) def test_get_project_by_name(self, feast_rest_client, project_name): """Test getting a project by name.""" @@ -420,13 +428,13 @@ def test_get_projects_list(self, feast_rest_client): assert len(projects) == 2 actual_project_names = [project["spec"]["name"] for project in projects] - assert set(actual_project_names) == TestConfig.PROJECT_NAMES + assert set(actual_project_names) == RegistryTestConfig.PROJECT_NAMES # Lineage Tests def test_get_registry_lineage(self, feast_rest_client): """Test getting registry lineage for a specific project.""" response = feast_rest_client.get( - f"/lineage/registry?project={TestConfig.CREDIT_SCORING_PROJECT}" + f"/lineage/registry?project={RegistryTestConfig.CREDIT_SCORING_PROJECT}" ) data = APITestHelpers.validate_response_success(response) @@ -448,11 +456,11 @@ def test_get_registry_lineage(self, feast_rest_client): def test_get_lineage_complete(self, feast_rest_client): """Test getting complete lineage for a specific project.""" response = feast_rest_client.get( - f"/lineage/complete?project={TestConfig.CREDIT_SCORING_PROJECT}" + f"/lineage/complete?project={RegistryTestConfig.CREDIT_SCORING_PROJECT}" ) data = APITestHelpers.validate_response_success(response) - assert data.get("project") == TestConfig.CREDIT_SCORING_PROJECT + assert data.get("project") == RegistryTestConfig.CREDIT_SCORING_PROJECT assert "objects" in data objects = data["objects"] @@ -511,12 +519,12 @@ def test_get_registry_complete_all(self, feast_rest_client): assert len(data["projects"]) > 0 project_names = [project["project"] for project in data.get("projects", [])] - assert TestConfig.CREDIT_SCORING_PROJECT in project_names + assert RegistryTestConfig.CREDIT_SCORING_PROJECT in project_names def test_get_lineage_object_path(self, feast_rest_client): """Test getting lineage for a specific object.""" response = feast_rest_client.get( - f"/lineage/objects/entity/dob_ssn?project={TestConfig.CREDIT_SCORING_PROJECT}" + f"/lineage/objects/entity/dob_ssn?project={RegistryTestConfig.CREDIT_SCORING_PROJECT}" ) data = APITestHelpers.validate_response_success(response) @@ -541,7 +549,7 @@ def test_get_lineage_object_path(self, feast_rest_client): def test_saved_datasets_endpoints(self, feast_rest_client, endpoint, key): """Test saved datasets endpoints with parameterization.""" if endpoint == "/saved_datasets": - url = f"{endpoint}?project={TestConfig.CREDIT_SCORING_PROJECT}&include_relationships=false" + url = f"{endpoint}?project={RegistryTestConfig.CREDIT_SCORING_PROJECT}&include_relationships=false" else: url = f"{endpoint}?allow_cache=true&page=1&limit=50&sort_order=asc&include_relationships=false" @@ -555,11 +563,13 @@ def test_saved_datasets_endpoints(self, feast_rest_client, endpoint, key): # Extract and validate names actual_names = [ds["spec"]["name"] for ds in saved_datasets] APITestHelpers.validate_names_match( - actual_names, TestConfig.SAVED_DATASET_NAMES + actual_names, RegistryTestConfig.SAVED_DATASET_NAMES ) # Validate pagination - APITestHelpers.validate_pagination(data, TestConfig.SAVED_DATASETS_COUNT) + APITestHelpers.validate_pagination( + data, RegistryTestConfig.SAVED_DATASETS_COUNT + ) if endpoint == "/saved_datasets/all": assert data["pagination"]["page"] == 1 assert data["pagination"]["limit"] == 50 @@ -568,7 +578,7 @@ def test_get_saved_datasets_by_name(self, feast_rest_client): """Test getting a specific saved dataset by name.""" dataset_name = "comprehensive_credit_dataset_v1" response = feast_rest_client.get( - f"/saved_datasets/{dataset_name}?project={TestConfig.CREDIT_SCORING_PROJECT}&include_relationships=false" + f"/saved_datasets/{dataset_name}?project={RegistryTestConfig.CREDIT_SCORING_PROJECT}&include_relationships=false" ) data = APITestHelpers.validate_response_success(response) @@ -580,14 +590,14 @@ def test_get_saved_datasets_by_name(self, feast_rest_client): def test_get_permission_by_name(self, feast_rest_client): """Test getting a specific permission by name.""" response = feast_rest_client.get( - f"/permissions/feast_admin_permission?project={TestConfig.CREDIT_SCORING_PROJECT}&include_relationships=false" + f"/permissions/feast_admin_permission?project={RegistryTestConfig.CREDIT_SCORING_PROJECT}&include_relationships=false" ) APITestHelpers.validate_response_success(response) def test_list_permissions(self, feast_rest_client): """Test listing permissions for a specific project.""" response = feast_rest_client.get( - f"/permissions?project={TestConfig.CREDIT_SCORING_PROJECT}&include_relationships=false" + f"/permissions?project={RegistryTestConfig.CREDIT_SCORING_PROJECT}&include_relationships=false" ) data = APITestHelpers.validate_response_success(response) @@ -595,9 +605,9 @@ def test_list_permissions(self, feast_rest_client): # Extract and validate names actual_names = [ds["spec"]["name"] for ds in data["permissions"]] - assert len(actual_names) == len(TestConfig.PERMISSION_NAMES) + assert len(actual_names) == len(RegistryTestConfig.PERMISSION_NAMES) - for name in TestConfig.PERMISSION_NAMES: + for name in RegistryTestConfig.PERMISSION_NAMES: assert name in actual_names - APITestHelpers.validate_pagination(data, TestConfig.PERMISSIONS_COUNT) + APITestHelpers.validate_pagination(data, RegistryTestConfig.PERMISSIONS_COUNT) diff --git a/sdk/python/tests/unit/api/test_search_api.py b/sdk/python/tests/unit/api/test_search_api.py index 9116db1c59b..f0d7c3942e8 100644 --- a/sdk/python/tests/unit/api/test_search_api.py +++ b/sdk/python/tests/unit/api/test_search_api.py @@ -734,8 +734,8 @@ def test_search_by_tags(self, shared_search_responses): f"Expected to find some of {expected_resources} but found none in {found_resources}" ) - def test_search_matched_tag_exact_match(self, search_test_app): - """Test that matched_tag field is present when a tag matches exactly""" + def test_search_matched_tags_exact_match(self, search_test_app): + """Test that matched_tags field is present when a tag matches exactly""" # Search for "data" which should match tag key "team" with value "data" response = search_test_app.get("/search?query=data") assert response.status_code == 200 @@ -745,34 +745,75 @@ def test_search_matched_tag_exact_match(self, search_test_app): # Find results that matched via tags (match_score = 60) tag_matched_results = [ - r for r in results if r.get("match_score") == 60 and "matched_tag" in r + r for r in results if r.get("match_score") == 60 and "matched_tags" in r ] assert len(tag_matched_results) > 0, ( - "Expected to find at least one result with matched_tag from tag matching" + "Expected to find at least one result with matched_tags from tag matching" ) - # Verify matched_tag is present and has a valid dictionary value + # Verify matched_tags is present and has a valid dictionary value for result in tag_matched_results: - matched_tag = result.get("matched_tag") - assert matched_tag is not None, ( - f"matched_tag should not be None for result {result['name']}" + matched_tags = result.get("matched_tags") + assert matched_tags is not None, ( + f"matched_tags should not be None for result {result['name']}" ) - assert isinstance(matched_tag, dict), ( - f"matched_tag should be a dictionary, got {type(matched_tag)}" + assert isinstance(matched_tags, dict), ( + f"matched_tags should be a dictionary, got {type(matched_tags)}" ) - # matched_tag should be a dictionary with key:value format - assert len(matched_tag) > 0, "matched_tag should not be empty" - assert len(matched_tag) == 1, ( - f"matched_tag should contain exactly one key-value pair, got {len(matched_tag)}" + # matched_tags should be a non-empty dict for tag-matched results + assert len(matched_tags) > 0, ( + "matched_tags should not be empty for tag matches" ) logger.debug( - f"Found {len(tag_matched_results)} results with matched_tag: {[r['name'] + ' -> ' + str(r.get('matched_tag', 'N/A')) for r in tag_matched_results]}" + f"Found {len(tag_matched_results)} results with matched_tags: {[r['name'] + ' -> ' + str(r.get('matched_tags', 'N/A')) for r in tag_matched_results]}" ) - def test_search_matched_tag_fuzzy_match(self, search_test_app): - """Test that matched_tag field is present when a tag matches via fuzzy matching""" + def test_search_matched_tags_multiple_tags(self, search_test_app): + """Test that multiple matching tags are returned in matched_tags""" + # Search for "a" which should match: + # - Names containing "a" (e.g., user_training_dataset, data sources) + # - Tags where key/value contains "a": "team" (key), "data" (value), "training" (value) + response = search_test_app.get("/search?query=a") + logger.info(response.json()) + assert response.status_code == 200 + + data = response.json() + results = data["results"] + + # Find user_training_dataset which has tags: {"environment": "test", "purpose": "training", "team": "data"} + # "team" contains "a", "data" contains "a", "training" contains "a" + # So matched_tags should have at least 2 entries: "purpose" and "team" + dataset_results = [ + r for r in results if r.get("name") == "user_training_dataset" + ] + + assert len(dataset_results) > 0, ( + "Expected to find user_training_dataset in results" + ) + + dataset_result = dataset_results[0] + matched_tags = dataset_result.get("matched_tags", {}) + + assert isinstance(matched_tags, dict), ( + f"matched_tags should be a dictionary, got {type(matched_tags)}" + ) + + # Should have multiple matching tags: "purpose" and "team" + assert len(matched_tags) >= 2, ( + f"Expected at least 2 matching tags for 'a' query, got {len(matched_tags)}: {matched_tags}" + ) + + # Verify the expected tags are present + assert "team" in matched_tags and "purpose" in matched_tags, ( + f"Expected 'team' and 'purpose' in matched_tags, got: {matched_tags}" + ) + + logger.debug(f"user_training_dataset matched_tags: {matched_tags}") + + def test_search_matched_tags_fuzzy_match(self, search_test_app): + """Test that matched_tags field is present when a tag matches via fuzzy matching""" # Search for "te" which should fuzzy match tag key "team" # "te" vs "team": overlap={'t','e'}/union={'t','e','a','m'} = 2/4 = 50% (below threshold) # Try "tea" which should fuzzy match "team" better @@ -789,7 +830,7 @@ def test_search_matched_tag_fuzzy_match(self, search_test_app): for r in results if r.get("match_score", 0) >= 40 and r.get("match_score", 0) < 60 - and "matched_tag" in r + and "matched_tags" in r ] # If we don't find fuzzy matches, try a different query that's more likely to match @@ -805,22 +846,21 @@ def test_search_matched_tag_fuzzy_match(self, search_test_app): for r in results if r.get("match_score", 0) >= 40 and r.get("match_score", 0) < 60 - and "matched_tag" in r + and "matched_tags" in r ] if len(fuzzy_tag_matched_results) > 0: - # Verify matched_tag is present for fuzzy matches + # Verify matched_tags is present for fuzzy matches for result in fuzzy_tag_matched_results: - matched_tag = result.get("matched_tag") - assert matched_tag is not None, ( - f"matched_tag should not be None for fuzzy-matched result {result['name']}" + matched_tags = result.get("matched_tags") + assert matched_tags is not None, ( + f"matched_tags should not be None for fuzzy-matched result {result['name']}" ) - assert isinstance(matched_tag, dict), ( - f"matched_tag should be a dictionary, got {type(matched_tag)}" + assert isinstance(matched_tags, dict), ( + f"matched_tags should be a dictionary, got {type(matched_tags)}" ) - assert len(matched_tag) > 0, "matched_tag should not be empty" - assert len(matched_tag) == 1, ( - f"matched_tag should contain exactly one key-value pair, got {len(matched_tag)}" + assert len(matched_tags) > 0, ( + "matched_tags should not be empty for fuzzy tag matches" ) # Verify the match_score is in the fuzzy range assert 40 <= result.get("match_score", 0) < 60, ( @@ -828,7 +868,7 @@ def test_search_matched_tag_fuzzy_match(self, search_test_app): ) logger.debug( - f"Found {len(fuzzy_tag_matched_results)} results with fuzzy matched_tag: {[r['name'] + ' -> ' + str(r.get('matched_tag', 'N/A')) + ' (score: ' + str(r.get('match_score', 'N/A')) + ')' for r in fuzzy_tag_matched_results]}" + f"Found {len(fuzzy_tag_matched_results)} results with fuzzy matched_tags: {[r['name'] + ' -> ' + str(r.get('matched_tags', 'N/A')) + ' (score: ' + str(r.get('match_score', 'N/A')) + ')' for r in fuzzy_tag_matched_results]}" ) def test_search_sorting_functionality(self, shared_search_responses): diff --git a/sdk/python/tests/unit/infra/offline_stores/contrib/postgres_offline_store/test_postgres.py b/sdk/python/tests/unit/infra/offline_stores/contrib/postgres_offline_store/test_postgres.py index ce98315eef4..fad837e4c16 100644 --- a/sdk/python/tests/unit/infra/offline_stores/contrib/postgres_offline_store/test_postgres.py +++ b/sdk/python/tests/unit/infra/offline_stores/contrib/postgres_offline_store/test_postgres.py @@ -950,3 +950,138 @@ def test_cli_date_combinations(self): # Should not fail on parameter validation stderr_output = result.stderr.decode() assert "must be provided" not in stderr_output + + +class TestPostgreSQLSourceQueryStringAlias: + """Test suite for get_table_query_string_with_alias() method. + + This addresses GitHub issue #5605: PostgreSQL requires all subqueries + in FROM clauses to have aliases. + """ + + def test_table_source_get_table_query_string(self): + """Test get_table_query_string() with table-based source""" + source = PostgreSQLSource( + name="test_source", + table="my_schema.my_table", + timestamp_field="event_timestamp", + ) + result = source.get_table_query_string() + assert result == "my_schema.my_table" + + def test_query_source_get_table_query_string(self): + """Test get_table_query_string() with query-based source""" + source = PostgreSQLSource( + name="test_source", + query="SELECT * FROM my_table WHERE active = true", + timestamp_field="event_timestamp", + ) + result = source.get_table_query_string() + assert result == "(SELECT * FROM my_table WHERE active = true)" + + def test_table_source_with_alias(self): + """Test get_table_query_string_with_alias() with table-based source returns table without alias""" + source = PostgreSQLSource( + name="test_source", + table="my_schema.my_table", + timestamp_field="event_timestamp", + ) + result = source.get_table_query_string_with_alias() + # Table sources don't need aliases + assert result == "my_schema.my_table" + + def test_query_source_with_default_alias(self): + """Test get_table_query_string_with_alias() with query-based source uses default alias""" + source = PostgreSQLSource( + name="test_source", + query="SELECT * FROM my_table WHERE active = true", + timestamp_field="event_timestamp", + ) + result = source.get_table_query_string_with_alias() + assert result == "(SELECT * FROM my_table WHERE active = true) AS subquery" + + def test_query_source_with_custom_alias(self): + """Test get_table_query_string_with_alias() with custom alias""" + source = PostgreSQLSource( + name="test_source", + query="SELECT id, name FROM users", + timestamp_field="event_timestamp", + ) + result = source.get_table_query_string_with_alias(alias="user_data") + assert result == "(SELECT id, name FROM users) AS user_data" + + def test_table_source_with_custom_alias_ignored(self): + """Test get_table_query_string_with_alias() ignores alias for table-based sources""" + source = PostgreSQLSource( + name="test_source", + table="events", + timestamp_field="event_timestamp", + ) + result = source.get_table_query_string_with_alias(alias="ignored_alias") + # Alias should be ignored for table sources + assert result == "events" + + def test_sql_query_with_alias_is_valid(self): + """Test that SQL using get_table_query_string_with_alias() is syntactically valid""" + source = PostgreSQLSource( + name="test_source", + query="SELECT id, ts FROM raw_data", + timestamp_field="ts", + ) + + # Construct a SQL query using the new method + entity_sql = f"SELECT id, ts FROM {source.get_table_query_string_with_alias()}" + + # Verify SQL is valid using sqlglot + parsed = sqlglot.parse(entity_sql, dialect="postgres") + assert len(parsed) == 1 + assert parsed[0] is not None + + def test_sql_query_without_alias_fails_in_postgres(self): + """Test that SQL using get_table_query_string() for query source produces invalid PostgreSQL + + This demonstrates the issue that get_table_query_string_with_alias() fixes: + PostgreSQL requires all subqueries in FROM clauses to have aliases. + """ + source = PostgreSQLSource( + name="test_source", + query="SELECT id, ts FROM raw_data", + timestamp_field="ts", + ) + + # Using the old method (without alias) for query-based source + entity_sql_without_alias = ( + f"SELECT id, ts FROM {source.get_table_query_string()}" + ) + + # This produces: SELECT id, ts FROM (SELECT id, ts FROM raw_data) + # which is invalid in PostgreSQL (subquery needs alias) + # sqlglot is lenient and may parse it, but PostgreSQL would reject it + assert "AS" not in entity_sql_without_alias, ( + "get_table_query_string() should not add alias" + ) + + # Using the new method (with alias) produces valid SQL + entity_sql_with_alias = ( + f"SELECT id, ts FROM {source.get_table_query_string_with_alias()}" + ) + assert "AS subquery" in entity_sql_with_alias + + def test_complex_query_with_alias(self): + """Test get_table_query_string_with_alias() with complex nested query""" + complex_query = """ + SELECT u.id, u.name, o.total + FROM users u + JOIN orders o ON u.id = o.user_id + WHERE o.created_at > '2023-01-01' + """ + source = PostgreSQLSource( + name="test_source", + query=complex_query, + timestamp_field="created_at", + ) + + result = source.get_table_query_string_with_alias(alias="user_orders") + assert result.startswith("(") + assert result.endswith(") AS user_orders") + assert "SELECT u.id" in result diff --git a/sdk/python/tests/unit/permissions/auth/server/test_auth_registry_server.py b/sdk/python/tests/unit/permissions/auth/server/test_auth_registry_server.py index 71a883e1933..e0f75d1d3d8 100644 --- a/sdk/python/tests/unit/permissions/auth/server/test_auth_registry_server.py +++ b/sdk/python/tests/unit/permissions/auth/server/test_auth_registry_server.py @@ -142,6 +142,11 @@ def _test_get_historical_features(client_fs: FeatureStore): def _test_get_entity(client_fs: FeatureStore, permissions: list[Permission]): + if _is_auth_enabled(client_fs) and len(permissions) == 0: + with pytest.raises(FeastPermissionError): + client_fs.get_entity("driver") + return + if not _is_auth_enabled(client_fs) or _is_permission_enabled( client_fs, permissions, read_entities_perm ): @@ -156,6 +161,18 @@ def _test_get_entity(client_fs: FeatureStore, permissions: list[Permission]): def _test_list_entities(client_fs: FeatureStore, permissions: list[Permission]): + if _is_auth_enabled(client_fs) and len(permissions) == 0: + with pytest.raises(FeastPermissionError): + client_fs.list_entities() + return + + if _is_auth_enabled(client_fs) and _permissions_exist_in_permission_list( + [invalid_list_entities_perm], permissions + ): + with pytest.raises(FeastPermissionError): + client_fs.list_entities() + return + entities = client_fs.list_entities() if not _is_auth_enabled(client_fs) or _is_permission_enabled( @@ -183,6 +200,10 @@ def _test_list_permissions( with pytest.raises(Exception): client_fs.list_permissions() return [] + elif _is_auth_enabled(client_fs) and len(applied_permissions) == 0: + with pytest.raises(FeastPermissionError): + client_fs.list_permissions() + return [] else: permissions = client_fs.list_permissions() @@ -229,6 +250,11 @@ def _is_auth_enabled(client_fs: FeatureStore) -> bool: def _test_get_fv(client_fs: FeatureStore, permissions: list[Permission]): + if _is_auth_enabled(client_fs) and len(permissions) == 0: + with pytest.raises(FeastPermissionError): + client_fs.get_feature_view("driver_hourly_stats") + return + if not _is_auth_enabled(client_fs) or _is_permission_enabled( client_fs, permissions, read_fv_perm ): @@ -249,6 +275,10 @@ def _test_list_fvs(client_fs: FeatureStore, permissions: list[Permission]): with pytest.raises(Exception): client_fs.list_feature_views() return [] + elif _is_auth_enabled(client_fs) and len(permissions) == 0: + with pytest.raises(FeastPermissionError): + client_fs.list_feature_views() + return [] else: fvs = client_fs.list_feature_views() for fv in fvs: diff --git a/sdk/python/tests/unit/permissions/test_security_manager.py b/sdk/python/tests/unit/permissions/test_security_manager.py index ee0ec9e079a..34d8e4962e9 100644 --- a/sdk/python/tests/unit/permissions/test_security_manager.py +++ b/sdk/python/tests/unit/permissions/test_security_manager.py @@ -15,7 +15,7 @@ @pytest.mark.parametrize( "username, requested_actions, allowed, allowed_single, raise_error_in_assert, raise_error_in_permit, intra_communication_flag", [ - (None, [], False, [False, False], [True, True], False, False), + (None, [], False, [False, False], [True, True], True, False), (None, [], True, [True, True], [False, False], False, True), ( "r", @@ -42,7 +42,7 @@ False, [False, False], [True, True], - False, + True, False, ), ("r", [AuthzedAction.UPDATE], True, [True, True], [False, False], False, True), @@ -52,7 +52,7 @@ False, [False, False], [True, True], - False, + True, False, ), ( @@ -116,7 +116,7 @@ False, [False, False], [True, True], - True, + False, False, ), ( @@ -134,7 +134,7 @@ False, [False, True], [True, False], - True, + False, False, ), ( @@ -152,7 +152,7 @@ False, [False, False], [True, True], - True, + False, False, ), (