Update localstack services + loki (dev and CI) (#2649)

.github/workflows/go-tests.yml

@@ -24,10 +24,9 @@ env:
   RICHGO_FORCE_COLOR: 1
   AWS_HOST: localstack
   # these are to mimic aws config
-  AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE
-  AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+  AWS_ACCESS_KEY_ID: test
+  AWS_SECRET_ACCESS_KEY: test
   AWS_REGION: us-east-1
-  KINESIS_INITIALIZE_STREAMS: "stream-1-shard:1,stream-2-shards:2"
   CROWDSEC_FEATURE_DISABLE_HTTP_RETRY_BACKOFF: true
 
 jobs:
@@ -36,7 +35,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       localstack:
-        image: localstack/localstack:1.3.0
+        image: localstack/localstack:3.0
         ports:
         - 4566:4566  # Localstack exposes all services on the same port
         env:
@@ -45,7 +44,7 @@ jobs:
           KINESIS_ERROR_PROBABILITY: ""
           DOCKER_HOST: unix:///var/run/docker.sock
           KINESIS_INITIALIZE_STREAMS: ${{ env.KINESIS_INITIALIZE_STREAMS }}
-          HOSTNAME_EXTERNAL: ${{ env.AWS_HOST }}  # Required so that resource urls are provided properly
+          LOCALSTACK_HOST: ${{ env.AWS_HOST }}  # Required so that resource urls are provided properly
           # e.g sqs url will get localhost if we don't set this env to map our service
         options: >-
           --name=localstack
@@ -54,7 +53,7 @@ jobs:
           --health-timeout=5s
           --health-retries=3
       zoo1:
-        image: confluentinc/cp-zookeeper:7.3.0
+        image: confluentinc/cp-zookeeper:7.4.3
         ports:
           - "2181:2181"
         env:
@@ -105,12 +104,12 @@ jobs:
           --health-retries 5
 
       loki:
-        image: grafana/loki:2.8.0
+        image: grafana/loki:2.9.1
         ports:
           - "3100:3100"
         options: >-
           --name=loki1
-          --health-cmd "wget -q -O -  http://localhost:3100/ready | grep 'ready'"
+          --health-cmd "wget -q -O - http://localhost:3100/ready | grep 'ready'"
           --health-interval 30s
           --health-timeout 10s
           --health-retries 5
@@ -129,6 +128,11 @@ jobs:
       with:
         go-version: "1.21.5"
 
+    - name: Create localstack streams
+      run: |
+          aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-1-shard --shard-count 1
+          aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-2-shards --shard-count 2
+
     - name: Build and run tests, static
       run: |
         sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev

Makefile

@@ -212,8 +212,8 @@ testclean: bats-clean  ## Remove test artifacts
 
 # for the tests with localstack
 export AWS_ENDPOINT_FORCE=http://localhost:4566
-export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
-export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+export AWS_ACCESS_KEY_ID=test
+export AWS_SECRET_ACCESS_KEY=test
 
 testenv:
 	@echo 'NOTE: You need Docker, docker-compose and run "make localstack" in a separate shell ("make localstack-stop" to terminate it)'

test/localstack/docker-compose.yml

@@ -3,7 +3,7 @@ version: "3.8"
 services:
   localstack:
     container_name: localstack_main
-    image: localstack/localstack:1.3.0
+    image: localstack/localstack:3.0
     network_mode: bridge
     ports:
       - "127.0.0.1:53:53"                # only required for Pro (DNS)
@@ -14,21 +14,18 @@ services:
     environment:
       AWS_HOST: localstack
       DEBUG: ""
-      LAMBDA_EXECUTOR: ""
       KINESYS_ERROR_PROBABILITY: ""
       DOCKER_HOST: "unix://var/run/docker.sock"
-      KINESIS_INITIALIZE_STREAMS: "stream-1-shard:1,stream-2-shards:2"
-      HOSTNAME_EXTERNAL: "localstack"
-      AWS_ACCESS_KEY_ID: "AKIAIOSFODNN7EXAMPLE"
-      AWS_SECRET_ACCESS_KEY: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+      LOCALSTACK_HOST: "localstack"
       AWS_REGION: "us-east-1"
 
     volumes:
       - "${TMPDIR:-/tmp}/localstack:/var/lib/localstack"
       - "/var/run/docker.sock:/var/run/docker.sock"
+      - "./scripts/init_script.sh:/etc/localstack/init/ready.d/init_script.sh"
 
   zoo1:
-    image: confluentinc/cp-zookeeper:7.3.0
+    image: confluentinc/cp-zookeeper:7.4.3
     ports:
       - "2181:2181"
     environment:
@@ -79,6 +76,6 @@ services:
       timeout: 10s
 
   loki:
-    image: grafana/loki:2.8.0
+    image: grafana/loki:2.9.1
     ports:
-      - "3100:3100"
+      - "127.0.0.1:3100:3100"

test/localstack/scripts/init_script.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Create Kinesis streams
+aws --endpoint-url=http://localstack:4566 --region us-east-1 kinesis create-stream --stream-name stream-1-shard --shard-count 1
+aws --endpoint-url=http://localstack:4566 --region us-east-1 kinesis create-stream --stream-name stream-2-shards --shard-count 2
+