From 1530d93fc14fbbd8abefeb14e3358fd1e4f73bc8 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Tue, 19 Dec 2023 15:27:04 +0100 Subject: [PATCH] Update localstack services + loki (dev and CI) (#2649) --- .github/workflows/go-tests.yml | 20 ++++++++++++-------- Makefile | 4 ++-- test/localstack/docker-compose.yml | 15 ++++++--------- test/localstack/scripts/init_script.sh | 6 ++++++ 4 files changed, 26 insertions(+), 19 deletions(-) create mode 100755 test/localstack/scripts/init_script.sh diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 4cc118e66..87f574f30 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -24,10 +24,9 @@ env: RICHGO_FORCE_COLOR: 1 AWS_HOST: localstack # these are to mimic aws config - AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE - AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test AWS_REGION: us-east-1 - KINESIS_INITIALIZE_STREAMS: "stream-1-shard:1,stream-2-shards:2" CROWDSEC_FEATURE_DISABLE_HTTP_RETRY_BACKOFF: true jobs: @@ -36,7 +35,7 @@ jobs: runs-on: ubuntu-latest services: localstack: - image: localstack/localstack:1.3.0 + image: localstack/localstack:3.0 ports: - 4566:4566 # Localstack exposes all services on the same port env: @@ -45,7 +44,7 @@ jobs: KINESIS_ERROR_PROBABILITY: "" DOCKER_HOST: unix:///var/run/docker.sock KINESIS_INITIALIZE_STREAMS: ${{ env.KINESIS_INITIALIZE_STREAMS }} - HOSTNAME_EXTERNAL: ${{ env.AWS_HOST }} # Required so that resource urls are provided properly + LOCALSTACK_HOST: ${{ env.AWS_HOST }} # Required so that resource urls are provided properly # e.g sqs url will get localhost if we don't set this env to map our service options: >- --name=localstack @@ -54,7 +53,7 @@ jobs: --health-timeout=5s --health-retries=3 zoo1: - image: confluentinc/cp-zookeeper:7.3.0 + image: confluentinc/cp-zookeeper:7.4.3 ports: - "2181:2181" env: @@ -105,12 +104,12 @@ jobs: --health-retries 5 loki: - image: grafana/loki:2.8.0 + image: grafana/loki:2.9.1 ports: - "3100:3100" options: >- --name=loki1 - --health-cmd "wget -q -O - http://localhost:3100/ready | grep 'ready'" + --health-cmd "wget -q -O - http://localhost:3100/ready | grep 'ready'" --health-interval 30s --health-timeout 10s --health-retries 5 @@ -129,6 +128,11 @@ jobs: with: go-version: "1.21.5" + - name: Create localstack streams + run: | + aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-1-shard --shard-count 1 + aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-2-shards --shard-count 2 + - name: Build and run tests, static run: | sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev diff --git a/Makefile b/Makefile index 66ad2e025..5d656165f 100644 --- a/Makefile +++ b/Makefile @@ -212,8 +212,8 @@ testclean: bats-clean ## Remove test artifacts # for the tests with localstack export AWS_ENDPOINT_FORCE=http://localhost:4566 -export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE -export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +export AWS_ACCESS_KEY_ID=test +export AWS_SECRET_ACCESS_KEY=test testenv: @echo 'NOTE: You need Docker, docker-compose and run "make localstack" in a separate shell ("make localstack-stop" to terminate it)' diff --git a/test/localstack/docker-compose.yml b/test/localstack/docker-compose.yml index 65dd1ca8e..f58f3c7f2 100644 --- a/test/localstack/docker-compose.yml +++ b/test/localstack/docker-compose.yml @@ -3,7 +3,7 @@ version: "3.8" services: localstack: container_name: localstack_main - image: localstack/localstack:1.3.0 + image: localstack/localstack:3.0 network_mode: bridge ports: - "127.0.0.1:53:53" # only required for Pro (DNS) @@ -14,21 +14,18 @@ services: environment: AWS_HOST: localstack DEBUG: "" - LAMBDA_EXECUTOR: "" KINESYS_ERROR_PROBABILITY: "" DOCKER_HOST: "unix://var/run/docker.sock" - KINESIS_INITIALIZE_STREAMS: "stream-1-shard:1,stream-2-shards:2" - HOSTNAME_EXTERNAL: "localstack" - AWS_ACCESS_KEY_ID: "AKIAIOSFODNN7EXAMPLE" - AWS_SECRET_ACCESS_KEY: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + LOCALSTACK_HOST: "localstack" AWS_REGION: "us-east-1" volumes: - "${TMPDIR:-/tmp}/localstack:/var/lib/localstack" - "/var/run/docker.sock:/var/run/docker.sock" + - "./scripts/init_script.sh:/etc/localstack/init/ready.d/init_script.sh" zoo1: - image: confluentinc/cp-zookeeper:7.3.0 + image: confluentinc/cp-zookeeper:7.4.3 ports: - "2181:2181" environment: @@ -79,6 +76,6 @@ services: timeout: 10s loki: - image: grafana/loki:2.8.0 + image: grafana/loki:2.9.1 ports: - - "3100:3100" + - "127.0.0.1:3100:3100" diff --git a/test/localstack/scripts/init_script.sh b/test/localstack/scripts/init_script.sh new file mode 100755 index 000000000..808ae4eb0 --- /dev/null +++ b/test/localstack/scripts/init_script.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# Create Kinesis streams +aws --endpoint-url=http://localstack:4566 --region us-east-1 kinesis create-stream --stream-name stream-1-shard --shard-count 1 +aws --endpoint-url=http://localstack:4566 --region us-east-1 kinesis create-stream --stream-name stream-2-shards --shard-count 2 +