From 0693f5fd0025ba2986575097239491876cebde7d Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Jul 2022 23:32:37 -0700 Subject: [PATCH 01/10] Fix Signed-off-by: Kevin Zhang --- .github/workflows/nightly-ci.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index f125ebdc233..15717014de6 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -4,11 +4,6 @@ on: schedule: - cron: '00 08 * * *' # early morning 08:00 AM UTC, which is 1 AM PST/4 AM EST. -# concurrency is currently broken, see details https://github.com/actions/runner/issues/1532 -#concurrency: -# group: pr-integration-tests-${{ github.event.pull_request.number }} -# cancel-in-progress: true - jobs: check_date: runs-on: ubuntu-latest From 57e5a1bef026ca52fa4ec8433046c32fa5388c7f Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Jul 2022 23:33:36 -0700 Subject: [PATCH 02/10] Fix Signed-off-by: Kevin Zhang --- .github/workflows/nightly-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 15717014de6..f125ebdc233 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -4,6 +4,11 @@ on: schedule: - cron: '00 08 * * *' # early morning 08:00 AM UTC, which is 1 AM PST/4 AM EST. +# concurrency is currently broken, see details https://github.com/actions/runner/issues/1532 +#concurrency: +# group: pr-integration-tests-${{ github.event.pull_request.number }} +# cancel-in-progress: true + jobs: check_date: runs-on: ubuntu-latest From 37b4a972d957c8ba01ab446ef45cd59be29f8f5a Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 14 Jul 2022 09:52:16 -0700 Subject: [PATCH 03/10] Fix Signed-off-by: Kevin Zhang --- .github/workflows/nightly-ci.yml | 66 ++++++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index f125ebdc233..a8279a7005c 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -23,9 +23,69 @@ jobs: name: Check if there were commits in the last day if: ${{ github.event_name == 'schedule' }} run: echo '::set-output name=WAS_EDITED::'$(test -n "$(git log --format=%H --since='24 hours ago')" && echo 'true' || echo 'false') - + # build-docker-image: + # # all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. + # if: + # (github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) || + # (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm'))) + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + # with: + # # pull_request_target runs the workflow in the context of the base repo + # # as such actions/checkout needs to be explicit configured to retrieve + # # code from the PR. + # ref: refs/pull/${{ github.event.pull_request.number }}/merge + # submodules: recursive + # - name: Set up QEMU + # uses: docker/setup-qemu-action@v1 + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v2 + # with: + # install: true + # - name: Set up AWS SDK + # uses: aws-actions/configure-aws-credentials@v1 + # with: + # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws-region: us-west-2 + # - name: Login to Amazon ECR + # id: login-ecr + # uses: aws-actions/amazon-ecr-login@v1 + # - name: Set ECR image tag + # id: image-tag + # run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" + # - name: Cache Public ECR Image + # id: lambda_python_3_9 + # uses: actions/cache@v2 + # with: + # path: ~/cache + # key: lambda_python_3_9 + # - name: Handle Cache Miss (pull public ECR image & save it to tar file) + # if: steps.cache-primes.outputs.cache-hit != 'true' + # run: | + # mkdir -p ~/cache + # docker pull public.ecr.aws/lambda/python:3.9 + # docker save public.ecr.aws/lambda/python:3.9 -o ~/cache/lambda_python_3_9.tar + # - name: Handle Cache Hit (load docker image from tar file) + # if: steps.cache-primes.outputs.cache-hit == 'true' + # run: | + # docker load -i ~/cache/lambda_python_3_9.tar + # - name: Build and push + # env: + # ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + # ECR_REPOSITORY: feast-python-server + # run: | + # docker build \ + # --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \ + # --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \ + # --load \ + # . + # docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} + # outputs: + # DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} integration-test-python: - needs: check_date + needs: [check_date, build-docker-image] runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -57,7 +117,7 @@ jobs: uses: actions/setup-python@v2 id: setup-python with: - python-version: ${{ os.PYTHON }} + python-version: ${{ matrix.python-version }} architecture: x64 - name: Setup Go id: setup-go From 46a116cb07f88a1f18d03b8a6889c2170e137ef8 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 14 Jul 2022 09:53:04 -0700 Subject: [PATCH 04/10] Fix Signed-off-by: Kevin Zhang --- .github/workflows/nightly-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index a8279a7005c..5f1e456bdee 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -1,9 +1,9 @@ name: nightly-ci -on: - schedule: - - cron: '00 08 * * *' # early morning 08:00 AM UTC, which is 1 AM PST/4 AM EST. - +# on: +# schedule: +# - cron: '00 08 * * *' # early morning 08:00 AM UTC, which is 1 AM PST/4 AM EST. +on: workflow_dispatch # concurrency is currently broken, see details https://github.com/actions/runner/issues/1532 #concurrency: # group: pr-integration-tests-${{ github.event.pull_request.number }} From e517bca466c4a3b1ca0338885eca333c0dc482a7 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 14 Jul 2022 09:57:35 -0700 Subject: [PATCH 05/10] Fix Signed-off-by: Kevin Zhang --- .github/workflows/nightly-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 5f1e456bdee..917d5245475 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -85,7 +85,7 @@ jobs: # outputs: # DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} integration-test-python: - needs: [check_date, build-docker-image] + needs: [check_date] runs-on: ${{ matrix.os }} strategy: fail-fast: false From 1907de7c4483dd136b2eb14fa522b88db5e09f2e Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 14 Jul 2022 10:06:17 -0700 Subject: [PATCH 06/10] Fix Signed-off-by: Kevin Zhang --- .github/workflows/nightly-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 917d5245475..226714a72aa 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -18,7 +18,7 @@ jobs: steps: - uses: actions/checkout@v2 with: - ref: develop + ref: master - id: check_date name: Check if there were commits in the last day if: ${{ github.event_name == 'schedule' }} From 98b064bcf572a0128272c699f04b8ef3564f7d07 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 14 Jul 2022 10:08:34 -0700 Subject: [PATCH 07/10] Fix Signed-off-by: Kevin Zhang --- .github/workflows/nightly-ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 226714a72aa..470daebc8c7 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -108,10 +108,7 @@ jobs: steps: - uses: actions/checkout@v2 with: - # pull_request_target runs the workflow in the context of the base repo - # as such actions/checkout needs to be explicit configured to retrieve - # code from the PR. - ref: refs/pull/${{ github.event.pull_request.number }}/merge + ref: master submodules: recursive - name: Setup Python uses: actions/setup-python@v2 From 745b0132eca743abde9957a5a2568804109c2cba Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 14 Jul 2022 10:16:59 -0700 Subject: [PATCH 08/10] Fix Signed-off-by: Kevin Zhang --- .github/workflows/nightly-ci.yml | 56 ++++++++++++++------------------ 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 470daebc8c7..769998e38c0 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -24,18 +24,12 @@ jobs: if: ${{ github.event_name == 'schedule' }} run: echo '::set-output name=WAS_EDITED::'$(test -n "$(git log --format=%H --since='24 hours ago')" && echo 'true' || echo 'false') # build-docker-image: - # # all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. - # if: - # (github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) || - # (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm'))) + # needs: [check_date] # runs-on: ubuntu-latest # steps: # - uses: actions/checkout@v3 # with: - # # pull_request_target runs the workflow in the context of the base repo - # # as such actions/checkout needs to be explicit configured to retrieve - # # code from the PR. - # ref: refs/pull/${{ github.event.pull_request.number }}/merge + # ref: master # submodules: recursive # - name: Set up QEMU # uses: docker/setup-qemu-action@v1 @@ -121,29 +115,29 @@ jobs: uses: actions/setup-go@v2 with: go-version: 1.18.0 - - name: Set up gcloud SDK - uses: google-github-actions/setup-gcloud@v0 - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - export_default_credentials: true - - name: Use gcloud CLI - run: gcloud info - - name: Set up AWS SDK - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - name: Use AWS CLI - run: aws sts get-caller-identity - - name: Upgrade pip version - run: | - pip install --upgrade "pip>=21.3.1,<22.1" - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" + # - name: Set up gcloud SDK + # uses: google-github-actions/setup-gcloud@v0 + # with: + # project_id: ${{ secrets.GCP_PROJECT_ID }} + # service_account_key: ${{ secrets.GCP_SA_KEY }} + # export_default_credentials: true + # - name: Use gcloud CLI + # run: gcloud info + # - name: Set up AWS SDK + # uses: aws-actions/configure-aws-credentials@v1 + # with: + # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws-region: us-west-2 + # - name: Use AWS CLI + # run: aws sts get-caller-identity + # - name: Upgrade pip version + # run: | + # pip install --upgrade "pip>=21.3.1,<22.1" + # - name: Get pip cache dir + # id: pip-cache + # run: | + # echo "::set-output name=dir::$(pip cache dir)" - name: pip cache uses: actions/cache@v2 with: From a4ec95ed050632d338ad3543718feee30440f90c Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 14 Jul 2022 10:31:16 -0700 Subject: [PATCH 09/10] Fix Signed-off-by: Kevin Zhang --- .github/workflows/nightly-ci.yml | 158 +++++++++++++++---------------- 1 file changed, 79 insertions(+), 79 deletions(-) diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 769998e38c0..cf032bff07a 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -23,63 +23,63 @@ jobs: name: Check if there were commits in the last day if: ${{ github.event_name == 'schedule' }} run: echo '::set-output name=WAS_EDITED::'$(test -n "$(git log --format=%H --since='24 hours ago')" && echo 'true' || echo 'false') - # build-docker-image: - # needs: [check_date] - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v3 - # with: - # ref: master - # submodules: recursive - # - name: Set up QEMU - # uses: docker/setup-qemu-action@v1 - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v2 - # with: - # install: true - # - name: Set up AWS SDK - # uses: aws-actions/configure-aws-credentials@v1 - # with: - # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws-region: us-west-2 - # - name: Login to Amazon ECR - # id: login-ecr - # uses: aws-actions/amazon-ecr-login@v1 - # - name: Set ECR image tag - # id: image-tag - # run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" - # - name: Cache Public ECR Image - # id: lambda_python_3_9 - # uses: actions/cache@v2 - # with: - # path: ~/cache - # key: lambda_python_3_9 - # - name: Handle Cache Miss (pull public ECR image & save it to tar file) - # if: steps.cache-primes.outputs.cache-hit != 'true' - # run: | - # mkdir -p ~/cache - # docker pull public.ecr.aws/lambda/python:3.9 - # docker save public.ecr.aws/lambda/python:3.9 -o ~/cache/lambda_python_3_9.tar - # - name: Handle Cache Hit (load docker image from tar file) - # if: steps.cache-primes.outputs.cache-hit == 'true' - # run: | - # docker load -i ~/cache/lambda_python_3_9.tar - # - name: Build and push - # env: - # ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - # ECR_REPOSITORY: feast-python-server - # run: | - # docker build \ - # --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \ - # --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \ - # --load \ - # . - # docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} - # outputs: - # DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} - integration-test-python: + build-docker-image: needs: [check_date] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + ref: master + submodules: recursive + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + install: true + - name: Set up AWS SDK + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + - name: Set ECR image tag + id: image-tag + run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" + - name: Cache Public ECR Image + id: lambda_python_3_9 + uses: actions/cache@v2 + with: + path: ~/cache + key: lambda_python_3_9 + - name: Handle Cache Miss (pull public ECR image & save it to tar file) + if: steps.cache-primes.outputs.cache-hit != 'true' + run: | + mkdir -p ~/cache + docker pull public.ecr.aws/lambda/python:3.9 + docker save public.ecr.aws/lambda/python:3.9 -o ~/cache/lambda_python_3_9.tar + - name: Handle Cache Hit (load docker image from tar file) + if: steps.cache-primes.outputs.cache-hit == 'true' + run: | + docker load -i ~/cache/lambda_python_3_9.tar + - name: Build and push + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: feast-python-server + run: | + docker build \ + --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \ + --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \ + --load \ + . + docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} + outputs: + DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} + integration-test-python: + needs: [check_date, build-docker-image] runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -115,29 +115,29 @@ jobs: uses: actions/setup-go@v2 with: go-version: 1.18.0 - # - name: Set up gcloud SDK - # uses: google-github-actions/setup-gcloud@v0 - # with: - # project_id: ${{ secrets.GCP_PROJECT_ID }} - # service_account_key: ${{ secrets.GCP_SA_KEY }} - # export_default_credentials: true - # - name: Use gcloud CLI - # run: gcloud info - # - name: Set up AWS SDK - # uses: aws-actions/configure-aws-credentials@v1 - # with: - # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws-region: us-west-2 - # - name: Use AWS CLI - # run: aws sts get-caller-identity - # - name: Upgrade pip version - # run: | - # pip install --upgrade "pip>=21.3.1,<22.1" - # - name: Get pip cache dir - # id: pip-cache - # run: | - # echo "::set-output name=dir::$(pip cache dir)" + - name: Set up gcloud SDK + uses: google-github-actions/setup-gcloud@v0 + with: + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + export_default_credentials: true + - name: Use gcloud CLI + run: gcloud info + - name: Set up AWS SDK + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - name: Use AWS CLI + run: aws sts get-caller-identity + - name: Upgrade pip version + run: | + pip install --upgrade "pip>=21.3.1,<22.1" + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" - name: pip cache uses: actions/cache@v2 with: From 7dc835eb022fb7c7d78ce1e534448b23ec64b5a0 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 14 Jul 2022 10:42:24 -0700 Subject: [PATCH 10/10] Fix Signed-off-by: Kevin Zhang --- .github/workflows/nightly-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index cf032bff07a..fead5124081 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -1,9 +1,9 @@ name: nightly-ci -# on: -# schedule: -# - cron: '00 08 * * *' # early morning 08:00 AM UTC, which is 1 AM PST/4 AM EST. -on: workflow_dispatch +on: + schedule: + - cron: '00 08 * * *' # early morning 08:00 AM UTC, which is 1 AM PST/4 AM EST. + # concurrency is currently broken, see details https://github.com/actions/runner/issues/1532 #concurrency: # group: pr-integration-tests-${{ github.event.pull_request.number }}