From 979d1b4ca3af2ac2458e998ac6b4ae4540fa3fec Mon Sep 17 00:00:00 2001 From: sabban Date: Thu, 6 Jul 2023 14:40:16 +0200 Subject: [PATCH] test specifically the Ci failing test --- .github/workflows/bats-hub.yml | 77 --------- .github/workflows/bats-mysql.yml | 100 ------------ .github/workflows/bats-postgres.yml | 85 ---------- .github/workflows/bats-sqlite-coverage.yml | 89 ---------- .github/workflows/bats.yml | 53 ------ .github/workflows/cache-cleanup.yaml | 35 ---- .github/workflows/ci-windows-build-msi.yml | 51 ------ .github/workflows/ci_release-drafter.yml | 21 --- .github/workflows/codeql-analysis.yml | 76 --------- .github/workflows/docker-tests.yml | 103 ------------ .github/workflows/go-tests-windows.yml | 70 -------- .github/workflows/go-tests.yml | 154 ------------------ .github/workflows/governance-bot.yaml | 30 ---- .../publish_docker-image_on_master-debian.yml | 70 -------- .../publish_docker-image_on_master.yml | 70 -------- .github/workflows/release_publish-package.yml | 45 ----- .../release_publish_docker-image-debian.yml | 61 ------- .../release_publish_docker-image.yml | 86 ---------- .github/workflows/tmp.yaml | 34 ++++ .github/workflows/update_docker_hub_doc.yml | 26 --- 20 files changed, 34 insertions(+), 1302 deletions(-) delete mode 100644 .github/workflows/bats-hub.yml delete mode 100644 .github/workflows/bats-mysql.yml delete mode 100644 .github/workflows/bats-postgres.yml delete mode 100644 .github/workflows/bats-sqlite-coverage.yml delete mode 100644 .github/workflows/bats.yml delete mode 100644 .github/workflows/cache-cleanup.yaml delete mode 100644 .github/workflows/ci-windows-build-msi.yml delete mode 100644 .github/workflows/ci_release-drafter.yml delete mode 100644 .github/workflows/codeql-analysis.yml delete mode 100644 .github/workflows/docker-tests.yml delete mode 100644 .github/workflows/go-tests-windows.yml delete mode 100644 .github/workflows/go-tests.yml delete mode 100644 .github/workflows/governance-bot.yaml delete mode 100644 .github/workflows/publish_docker-image_on_master-debian.yml delete mode 100644 .github/workflows/publish_docker-image_on_master.yml delete mode 100644 .github/workflows/release_publish-package.yml delete mode 100644 .github/workflows/release_publish_docker-image-debian.yml delete mode 100644 .github/workflows/release_publish_docker-image.yml create mode 100644 .github/workflows/tmp.yaml delete mode 100644 .github/workflows/update_docker_hub_doc.yml diff --git a/.github/workflows/bats-hub.yml b/.github/workflows/bats-hub.yml deleted file mode 100644 index c62229cbe..000000000 --- a/.github/workflows/bats-hub.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Hub tests - -on: - workflow_call: - secrets: - GIST_BADGES_SECRET: - required: true - GIST_BADGES_ID: - required: true - -env: - PREFIX_TEST_NAMES_WITH_FILE: true - -jobs: - build: - strategy: - matrix: - go-version: ["1.20.5"] - - name: "Build + tests" - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - - name: "Force machineid" - run: | - sudo chmod +w /etc/machine-id - echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id - - - name: "Check out CrowdSec repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: true - - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 - with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" - - - name: "Install bats dependencies" - env: - GOBIN: /usr/local/bin - run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev - - - name: "Build crowdsec and fixture" - run: make bats-clean bats-build bats-fixture BUILD_STATIC=1 - - - name: "Run hub tests" - run: make bats-test-hub - - - name: "Collect hub coverage" - run: ./test/bin/collect-hub-coverage >> $GITHUB_ENV - - - name: "Create Parsers badge" - uses: schneegans/dynamic-badges-action@v1.6.0 - if: ${{ github.ref == 'refs/heads/master' && github.repository_owner == 'crowdsecurity' }} - with: - auth: ${{ secrets.GIST_BADGES_SECRET }} - gistID: ${{ secrets.GIST_BADGES_ID }} - filename: crowdsec_parsers_badge.json - label: Hub Parsers - message: ${{ env.PARSERS_COV }} - color: ${{ env.SCENARIO_BADGE_COLOR }} - - - name: "Create Scenarios badge" - uses: schneegans/dynamic-badges-action@v1.6.0 - if: ${{ github.ref == 'refs/heads/master' && github.repository_owner == 'crowdsecurity' }} - with: - auth: ${{ secrets.GIST_BADGES_SECRET }} - gistID: ${{ secrets.GIST_BADGES_ID }} - filename: crowdsec_scenarios_badge.json - label: Hub Scenarios - message: ${{ env.SCENARIOS_COV }} - color: ${{ env.SCENARIO_BADGE_COLOR }} diff --git a/.github/workflows/bats-mysql.yml b/.github/workflows/bats-mysql.yml deleted file mode 100644 index 529c3f521..000000000 --- a/.github/workflows/bats-mysql.yml +++ /dev/null @@ -1,100 +0,0 @@ -name: Functional tests (MySQL) - -on: - workflow_call: - inputs: - database_image: - required: true - type: string - -env: - PREFIX_TEST_NAMES_WITH_FILE: true - -jobs: - build: - strategy: - matrix: - go-version: ["1.20.5"] - - name: "Build + tests" - runs-on: ubuntu-latest - timeout-minutes: 30 - services: - database: - image: ${{ inputs.database_image }} - env: - MYSQL_ROOT_PASSWORD: "secret" - ports: - - 3306:3306 - - steps: - - - name: "Force machineid" - run: | - sudo chmod +w /etc/machine-id - echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id - - - name: "Check out CrowdSec repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: true - - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 - with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" - - - name: "Install bats dependencies" - env: - GOBIN: /usr/local/bin - run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev - - - name: "Build crowdsec and fixture" - run: | - make clean bats-build bats-fixture BUILD_STATIC=1 - env: - DB_BACKEND: mysql - MYSQL_HOST: 127.0.0.1 - MYSQL_PORT: 3306 - MYSQL_PASSWORD: "secret" - MYSQL_USER: root - - - name: "Run tests" - run: make bats-test - env: - DB_BACKEND: mysql - MYSQL_HOST: 127.0.0.1 - MYSQL_PORT: 3306 - MYSQL_PASSWORD: "secret" - MYSQL_USER: root - - # - # In case you need to inspect the database status after the failure of a given test - # - # - name: "Run specified tests" - # run: ./test/run-tests test/bats/.bats -f "" - - - name: Show database dump - run: ./test/instance-db dump /dev/fd/1 - env: - DB_BACKEND: mysql - MYSQL_HOST: 127.0.0.1 - MYSQL_PORT: 3306 - MYSQL_PASSWORD: "secret" - MYSQL_USER: root - if: ${{ always() }} - - - name: "Show stack traces" - run: for file in $(find /tmp/crowdsec-crash.*.txt); do echo ">>>>> $file"; cat $file; echo; done - if: ${{ always() }} - - - name: "Show crowdsec logs" - run: for file in $(find ./test/local/var/log -type f); do echo ">>>>> $file"; cat $file; echo; done - if: ${{ always() }} - - - name: "Show database logs" - run: docker logs "${{ job.services.database.id }}" - if: ${{ always() }} diff --git a/.github/workflows/bats-postgres.yml b/.github/workflows/bats-postgres.yml deleted file mode 100644 index 91e7ac103..000000000 --- a/.github/workflows/bats-postgres.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: Functional tests (Postgres) - -on: - workflow_call: - -env: - PREFIX_TEST_NAMES_WITH_FILE: true - -jobs: - build: - strategy: - matrix: - go-version: ["1.20.5"] - - name: "Build + tests" - runs-on: ubuntu-latest - timeout-minutes: 30 - services: - database: - image: postgres:14 - env: - POSTGRES_PASSWORD: "secret" - ports: - - 5432:5432 - options: >- - --health-cmd pg_isready -u postgres - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - - name: "Force machineid" - run: | - sudo chmod +w /etc/machine-id - echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id - - - name: "Check out CrowdSec repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: true - - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 - with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" - - - name: "Install bats dependencies" - env: - GOBIN: /usr/local/bin - run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev - - - name: "Build crowdsec and fixture (DB_BACKEND: pgx)" - run: | - make clean bats-build bats-fixture BUILD_STATIC=1 - env: - DB_BACKEND: pgx - PGHOST: 127.0.0.1 - PGPORT: 5432 - PGPASSWORD: "secret" - PGUSER: postgres - - - name: "Run tests (DB_BACKEND: pgx)" - run: make bats-test - env: - DB_BACKEND: pgx - PGHOST: 127.0.0.1 - PGPORT: 5432 - PGPASSWORD: "secret" - PGUSER: postgres - - - name: "Show stack traces" - run: for file in $(find /tmp/crowdsec-crash.*.txt); do echo ">>>>> $file"; cat $file; echo; done - if: ${{ always() }} - - - name: "Show crowdsec logs" - run: for file in $(find ./test/local/var/log -type f); do echo ">>>>> $file"; cat $file; echo; done - if: ${{ always() }} - - - name: "Show database logs" - run: docker logs "${{ job.services.database.id }}" - if: ${{ always() }} diff --git a/.github/workflows/bats-sqlite-coverage.yml b/.github/workflows/bats-sqlite-coverage.yml deleted file mode 100644 index 17b2aac09..000000000 --- a/.github/workflows/bats-sqlite-coverage.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Functional tests (sqlite) - -on: - workflow_call: - -env: - PREFIX_TEST_NAMES_WITH_FILE: true - TEST_COVERAGE: true - -jobs: - build: - strategy: - matrix: - go-version: ["1.20.5"] - - name: "Build + tests" - runs-on: ubuntu-latest - timeout-minutes: 20 - - steps: - - - name: "Force machineid" - run: | - sudo chmod +w /etc/machine-id - echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id - - - name: "Check out CrowdSec repository" - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: true - - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 - with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" - - - name: "Install bats dependencies" - env: - GOBIN: /usr/local/bin - run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev - - - name: "Build crowdsec and fixture" - run: | - make clean bats-build bats-fixture BUILD_STATIC=1 - - - name: "Run tests" - run: make bats-test - - - name: "Collect coverage data" - run: | - go tool covdata textfmt -i test/coverage -o coverage-bats-raw.out - # filter out unwanted packages, should match the argument to "go-acc --ignore" - grep -v \ - -e '/pkg/database' \ - -e '/plugins/notifications' \ - -e '/pkg/protobufs' \ - -e '/pkg/cwversions' \ - -e '/pkg/models' \ - < coverage-bats-raw.out \ - > coverage-bats.out - - # - # In case you need to inspect the database status after the failure of a given test - # - # - name: "Run specified tests" - # run: ./test/run-tests test/bats/.bats -f "" - - - name: "Show database dump" - run: | - ./test/instance-crowdsec stop - sqlite3 ./test/local/var/lib/crowdsec/data/crowdsec.db '.dump' - if: ${{ always() }} - - - name: "Show stack traces" - run: for file in $(find /tmp/crowdsec-crash.*.txt); do echo ">>>>> $file"; cat $file; echo; done - if: ${{ always() }} - - - name: "Show crowdsec logs" - run: for file in $(find ./test/local/var/log -type f); do echo ">>>>> $file"; cat $file; echo; done - if: ${{ always() }} - - - name: Upload crowdsec coverage to codecov - uses: codecov/codecov-action@v3 - with: - files: ./coverage-bats.out - flags: bats diff --git a/.github/workflows/bats.yml b/.github/workflows/bats.yml deleted file mode 100644 index 46b1414ef..000000000 --- a/.github/workflows/bats.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -# This workflow is actually running -# only functional tests, but the -# name is used for the badge in README.md - -name: Tests - -# Main workflow for functional tests, it calls all the others through parallel jobs. -# -# https://docs.github.com/en/actions/using-workflows/reusing-workflows -# -# There is no need to merge coverage output because codecov.io should take care of that. - -on: - push: - branches: - - master - - releases/** - paths-ignore: - - "README.md" - pull_request: - branches: - - master - - releases/** - paths-ignore: - - "README.md" - -jobs: - sqlite: - uses: ./.github/workflows/bats-sqlite-coverage.yml - - # Jobs for Postgres (and sometimes MySQL) can have failing tests on GitHub - # CI, but they pass when run on devs' machines or in the release checks. We - # disable them here by default. Remove the if..false to enable them. - - mariadb: - uses: ./.github/workflows/bats-mysql.yml - with: - database_image: mariadb:latest - - mysql: - uses: ./.github/workflows/bats-mysql.yml - with: - database_image: mysql:latest - - postgres: - uses: ./.github/workflows/bats-postgres.yml - - hub: - uses: ./.github/workflows/bats-hub.yml - secrets: - GIST_BADGES_ID: ${{ secrets.GIST_BADGES_ID }} - GIST_BADGES_SECRET: ${{ secrets.GIST_BADGES_SECRET }} diff --git a/.github/workflows/cache-cleanup.yaml b/.github/workflows/cache-cleanup.yaml deleted file mode 100644 index d19365024..000000000 --- a/.github/workflows/cache-cleanup.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#managing-caches - -name: cleanup caches by a branch -on: - pull_request: - types: - - closed - -jobs: - cleanup: - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v3 - - - name: Cleanup - run: | - gh extension install actions/gh-actions-cache - - REPO=${{ github.repository }} - BRANCH="refs/pull/${{ github.event.pull_request.number }}/merge" - - echo "Fetching list of cache key" - cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH | cut -f 1 ) - - ## Setting this to not fail the workflow while deleting cache keys. - set +e - echo "Deleting caches..." - for cacheKey in $cacheKeysForPR - do - gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm - done - echo "Done" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-windows-build-msi.yml b/.github/workflows/ci-windows-build-msi.yml deleted file mode 100644 index 3b66b0c1f..000000000 --- a/.github/workflows/ci-windows-build-msi.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: build-msi (windows) - -on: - release: - types: - - prereleased - push: - branches: - - master - - releases/** - paths: - - windows/installer/*.wxs - - .github/workflows/ci-windows-build-msi.yml - pull_request: - branches: - - master - - releases/** - paths: - - windows/installer/*.wxs - - .github/workflows/ci-windows-build-msi.yml - -jobs: - build: - strategy: - matrix: - go-version: ["1.20.5"] - - name: Build - runs-on: windows-2019 - - steps: - - - name: Check out code into the Go module directory - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: false - - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 - with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" - - - name: Build - run: make windows_installer BUILD_RE2_WASM=1 - - name: Upload MSI - uses: actions/upload-artifact@v3 - with: - path: crowdsec*msi - name: crowdsec.msi diff --git a/.github/workflows/ci_release-drafter.yml b/.github/workflows/ci_release-drafter.yml deleted file mode 100644 index 2ccb6977c..000000000 --- a/.github/workflows/ci_release-drafter.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Release Drafter - -on: - push: - # branches to consider in the event; optional, defaults to all - branches: - - master - - releases/** - -jobs: - update_release_draft: - runs-on: ubuntu-latest - steps: - # Drafts your next Release notes as Pull Requests are merged into "master" - - uses: release-drafter/release-drafter@v5 - with: - config-name: release-drafter.yml - # (Optional) specify config name to use, relative to .github/. Default: release-drafter.yml - # config-name: my-config.yml - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index c1995cd8d..000000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,76 +0,0 @@ -# yamllint disable rule:comments -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - push: - branches: - - master - - releases/** - pull_request: - # The branches below must be a subset of the branches above - branches: - - master - - releases/** - schedule: - - cron: '15 16 * * 2' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'go' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] - # Learn more: - # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v2 - - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml deleted file mode 100644 index 913c47662..000000000 --- a/.github/workflows/docker-tests.yml +++ /dev/null @@ -1,103 +0,0 @@ -name: Test Docker images - -on: - push: - branches: - - master - - releases/** - paths-ignore: - - 'README.md' - pull_request: - branches: - - master - - releases/** - paths-ignore: - - 'README.md' - -jobs: - test_docker_image: - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - config: .github/buildkit.toml - - - name: "Build flavor: slim" - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile - tags: crowdsecurity/crowdsec:test-slim - target: slim - platforms: linux/amd64 - load: true - cache-from: type=gha - cache-to: type=gha,mode=min - - - name: "Build flavor: full" - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile - tags: crowdsecurity/crowdsec:test - target: full - platforms: linux/amd64 - load: true - cache-from: type=gha - cache-to: type=gha,mode=min - - - name: "Build flavor: full (debian)" - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile.debian - tags: crowdsecurity/crowdsec:test-debian - target: full - platforms: linux/amd64 - load: true - cache-from: type=gha - cache-to: type=gha,mode=min - - - name: "Setup Python" - uses: actions/setup-python@v4 - with: - python-version: "3.x" - - - name: "Install pipenv" - run: | - cd docker/test - python -m pip install --upgrade pipenv wheel - - - name: "Cache virtualenvs" - id: cache-pipenv - uses: actions/cache@v3 - with: - path: ~/.local/share/virtualenvs - key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }} - - - name: "Install dependencies" - if: steps.cache-pipenv.outputs.cache-hit != 'true' - run: | - cd docker/test - pipenv install --deploy - - - name: "Create Docker network" - run: docker network create net-test - - - name: "Run tests" - env: - CROWDSEC_TEST_VERSION: test - CROWDSEC_TEST_FLAVORS: slim,debian - CROWDSEC_TEST_NETWORK: net-test - CROWDSEC_TEST_TIMEOUT: 90 - run: | - cd docker/test - pipenv run pytest -n 2 --durations=0 --color=yes diff --git a/.github/workflows/go-tests-windows.yml b/.github/workflows/go-tests-windows.yml deleted file mode 100644 index 500fc58ce..000000000 --- a/.github/workflows/go-tests-windows.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: Go tests (windows) - -on: - push: - branches: - - master - - releases/** - paths-ignore: - - 'README.md' - pull_request: - branches: - - master - - releases/** - paths-ignore: - - 'README.md' - -env: - RICHGO_FORCE_COLOR: 1 - CROWDSEC_FEATURE_DISABLE_HTTP_RETRY_BACKOFF: true - -jobs: - build: - strategy: - matrix: - go-version: ["1.20.5"] - - name: "Build + tests" - runs-on: windows-2022 - - steps: - - - name: Check out CrowdSec repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: false - - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 - with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" - - - name: Build - run: | - make build BUILD_RE2_WASM=1 - - - name: Run tests - run: | - go install github.com/kyoh86/richgo@v0.3.10 - go test -coverprofile coverage.out -covermode=atomic ./... > out.txt - if(!$?) { cat out.txt | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter; Exit 1 } - cat out.txt | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter - - - name: Upload unit coverage to Codecov - uses: codecov/codecov-action@v3 - with: - files: coverage.out - flags: unit-windows - - - name: golangci-lint - uses: golangci/golangci-lint-action@v3 - with: - version: v1.51 - args: --issues-exit-code=1 --timeout 10m - only-new-issues: false - # the cache is already managed above, enabling it here - # gives errors when extracting - skip-pkg-cache: true - skip-build-cache: true diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml deleted file mode 100644 index 079f6c827..000000000 --- a/.github/workflows/go-tests.yml +++ /dev/null @@ -1,154 +0,0 @@ ---- -# This workflow is actually running -# tests (with localstack) but the -# name is used for the badge in README.md - -name: Build - -on: - push: - branches: - - master - - releases/** - paths-ignore: - - 'README.md' - pull_request: - branches: - - master - - releases/** - paths-ignore: - - 'README.md' - -# these env variables are for localstack, so we can emulate aws services -env: - RICHGO_FORCE_COLOR: 1 - AWS_HOST: localstack - # these are to mimic aws config - AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE - AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY - AWS_REGION: us-east-1 - KINESIS_INITIALIZE_STREAMS: "stream-1-shard:1,stream-2-shards:2" - CROWDSEC_FEATURE_DISABLE_HTTP_RETRY_BACKOFF: true - -jobs: - build: - strategy: - matrix: - go-version: ["1.20.5"] - - name: "Build + tests" - runs-on: ubuntu-latest - services: - localstack: - image: localstack/localstack:1.3.0 - ports: - - 4566:4566 # Localstack exposes all services on the same port - env: - DEBUG: "" - LAMBDA_EXECUTOR: "" - KINESIS_ERROR_PROBABILITY: "" - DOCKER_HOST: unix:///var/run/docker.sock - KINESIS_INITIALIZE_STREAMS: ${{ env.KINESIS_INITIALIZE_STREAMS }} - HOSTNAME_EXTERNAL: ${{ env.AWS_HOST }} # Required so that resource urls are provided properly - # e.g sqs url will get localhost if we don't set this env to map our service - options: >- - --name=localstack - --health-cmd="curl -sS 127.0.0.1:4566 || exit 1" - --health-interval=10s - --health-timeout=5s - --health-retries=3 - zoo1: - image: confluentinc/cp-zookeeper:7.3.0 - ports: - - "2181:2181" - env: - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_SERVER_ID: 1 - ZOOKEEPER_SERVERS: zoo1:2888:3888 - options: >- - --name=zoo1 - --health-cmd "jps -l | grep zookeeper" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - kafka1: - image: crowdsecurity/kafka-ssl - ports: - - "9093:9093" - - "9092:9092" - - "9999:9999" - env: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://127.0.0.1:19092,LISTENER_DOCKER_EXTERNAL://127.0.0.1:9092,LISTENER_DOCKER_EXTERNAL_SSL://127.0.0.1:9093 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_SSL:SSL - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL - KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" - KAFKA_BROKER_ID: 1 - KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_JMX_PORT: 9999 - KAFKA_JMX_HOSTNAME: "127.0.0.1" - KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" - KAFKA_SSL_KEYSTORE_FILENAME: kafka.kafka1.keystore.jks - KAFKA_SSL_KEYSTORE_CREDENTIALS: kafka1_keystore_creds - KAFKA_SSL_KEY_CREDENTIALS: kafka1_sslkey_creds - KAFKA_SSL_TRUSTSTORE_FILENAME: kafka.kafka1.truststore.jks - KAFKA_SSL_TRUSTSTORE_CREDENTIALS: kafka1_truststore_creds - KAFKA_SSL_ENABLED_PROTOCOLS: TLSv1.2 - KAFKA_SSL_PROTOCOL: TLSv1.2 - KAFKA_SSL_CLIENT_AUTH: none - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" - options: >- - --name=kafka1 - --health-cmd "kafka-broker-api-versions --version" - --health-interval 10s - --health-timeout 10s - --health-retries 5 - - steps: - - - name: Check out CrowdSec repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: false - - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 - with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" - - - name: Build and run tests, static - run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev - go install github.com/ory/go-acc@v0.2.8 - go install github.com/kyoh86/richgo@v0.3.10 - set -o pipefail - make build BUILD_STATIC=1 - make go-acc | richgo testfilter - - - name: Run tests again, dynamic - run: | - make clean build - make go-acc | richgo testfilter - - - name: Upload unit coverage to Codecov - uses: codecov/codecov-action@v3 - with: - files: coverage.out - flags: unit-linux - - - name: golangci-lint - uses: golangci/golangci-lint-action@v3 - with: - version: v1.51 - args: --issues-exit-code=1 --timeout 10m - only-new-issues: false - # the cache is already managed above, enabling it here - # gives errors when extracting - skip-pkg-cache: true - skip-build-cache: true diff --git a/.github/workflows/governance-bot.yaml b/.github/workflows/governance-bot.yaml deleted file mode 100644 index 5c08cabf5..000000000 --- a/.github/workflows/governance-bot.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# .github/workflow/governance.yml - -on: - pull_request_target: - types: [ synchronize, opened, labeled, unlabeled ] - issues: - types: [ opened, labeled, unlabeled ] - issue_comment: - types: [ created ] - -# You can use permissions to modify the default permissions granted to the GITHUB_TOKEN, -# adding or removing access as required, so that you only allow the minimum required access. -permissions: - contents: read - issues: write - pull-requests: write - statuses: write - checks: write - -jobs: - governance: - name: Governance - runs-on: ubuntu-latest - steps: - # Semantic versioning, lock to different version: v2, v2.0 or a commit hash. - - uses: BirthdayResearch/oss-governance-bot@v3 - with: - # You can use a PAT to post a comment/label/status so that it shows up as a user instead of github-actions - github-token: ${{secrets.GITHUB_TOKEN}} # optional, default to '${{ github.token }}' - config-path: .github/governance.yml # optional, default to '.github/governance.yml' diff --git a/.github/workflows/publish_docker-image_on_master-debian.yml b/.github/workflows/publish_docker-image_on_master-debian.yml deleted file mode 100644 index 88076157c..000000000 --- a/.github/workflows/publish_docker-image_on_master-debian.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: Publish Debian Docker image on Push to Master - -on: - push: - branches: [ master ] - paths: - - 'pkg/**' - - 'cmd/**' - - 'plugins/**' - - 'docker/docker_start.sh' - - 'docker/config.yaml' - - '.github/workflows/publish_docker-image_on_master-debian.yml' - - 'Dockerfile.debian' - - 'go.mod' - - 'go.sum' - - 'Makefile' - -jobs: - push_to_registry: - name: Push Debian Docker image to Docker Hub - runs-on: ubuntu-latest - steps: - - - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Prepare - id: prep - run: | - DOCKER_IMAGE=crowdsecurity/crowdsec - GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec - VERSION=dev-debian - TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}" - echo "tags=${TAGS}" >> $GITHUB_OUTPUT - echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - config: .github/buildkit.toml - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push full image - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile.debian - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.prep.outputs.tags }} - platforms: linux/amd64 - labels: | - org.opencontainers.image.source=${{ github.event.repository.html_url }} - org.opencontainers.image.created=${{ steps.prep.outputs.created }} - org.opencontainers.image.revision=${{ github.sha }} - cache-from: type=gha - cache-to: type=gha,mode=min diff --git a/.github/workflows/publish_docker-image_on_master.yml b/.github/workflows/publish_docker-image_on_master.yml deleted file mode 100644 index 6cab486b0..000000000 --- a/.github/workflows/publish_docker-image_on_master.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: Publish Docker image on Push to Master - -on: - push: - branches: [ master ] - paths: - - 'pkg/**' - - 'cmd/**' - - 'plugins/**' - - 'docker/docker_start.sh' - - 'docker/config.yaml' - - '.github/workflows/publish_docker-image_on_master.yml' - - 'Dockerfile' - - 'go.mod' - - 'go.sum' - - 'Makefile' - -jobs: - push_to_registry: - name: Push Docker image to Docker Hub - runs-on: ubuntu-latest - steps: - - - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Prepare - id: prep - run: | - DOCKER_IMAGE=crowdsecurity/crowdsec - GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec - VERSION=dev - TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}" - echo "tags=${TAGS}" >> $GITHUB_OUTPUT - echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - config: .github/buildkit.toml - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push full image - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.prep.outputs.tags }} - platforms: linux/amd64 - labels: | - org.opencontainers.image.source=${{ github.event.repository.html_url }} - org.opencontainers.image.created=${{ steps.prep.outputs.created }} - org.opencontainers.image.revision=${{ github.sha }} - cache-from: type=gha - cache-to: type=gha,mode=min diff --git a/.github/workflows/release_publish-package.yml b/.github/workflows/release_publish-package.yml deleted file mode 100644 index 54419cc6e..000000000 --- a/.github/workflows/release_publish-package.yml +++ /dev/null @@ -1,45 +0,0 @@ -# .github/workflows/build-docker-image.yml -name: build - -on: - release: - types: - - prereleased - -permissions: - # Use write for: hub release edit - contents: write - -jobs: - build: - strategy: - matrix: - go-version: ["1.20.5"] - - name: Build and upload binary package - runs-on: ubuntu-latest - steps: - - - name: Check out code into the Go module directory - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: false - - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v4 - with: - go-version: ${{ matrix.go-version }} - cache-dependency-path: "**/go.sum" - - - name: Build the binaries - run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev - make vendor release BUILD_STATIC=1 - - - name: Upload to release - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - tag_name="${GITHUB_REF##*/}" - hub release edit -a crowdsec-release.tgz -a vendor.tgz -m "" "$tag_name" diff --git a/.github/workflows/release_publish_docker-image-debian.yml b/.github/workflows/release_publish_docker-image-debian.yml deleted file mode 100644 index e766dae09..000000000 --- a/.github/workflows/release_publish_docker-image-debian.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Publish Docker Debian image - -on: - release: - types: - - released - - prereleased - workflow_dispatch: - -jobs: - push_to_registry: - name: Push Docker debian image to Docker Hub - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Prepare - id: prep - run: | - DOCKER_IMAGE=crowdsecurity/crowdsec - VERSION=bullseye - if [[ $GITHUB_REF == refs/tags/* ]]; then - VERSION=${GITHUB_REF#refs/tags/} - elif [[ $GITHUB_REF == refs/heads/* ]]; then - VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -E 's#/+#-#g') - elif [[ $GITHUB_REF == refs/pull/* ]]; then - VERSION=pr-${{ github.event.number }} - fi - TAGS="${DOCKER_IMAGE}:${VERSION}-debian" - if [[ "${{ github.event.action }}" == "released" ]]; then - TAGS=$TAGS,${DOCKER_IMAGE}:latest-debian - fi - echo "version=${VERSION}" >> $GITHUB_OUTPUT - echo "tags=${TAGS}" >> $GITHUB_OUTPUT - echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - config: .github/buildkit.toml - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and push - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile.debian - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.prep.outputs.tags }} - platforms: linux/amd64,linux/arm64,linux/386 - labels: | - org.opencontainers.image.source=${{ github.event.repository.html_url }} - org.opencontainers.image.created=${{ steps.prep.outputs.created }} - org.opencontainers.image.revision=${{ github.sha }} diff --git a/.github/workflows/release_publish_docker-image.yml b/.github/workflows/release_publish_docker-image.yml deleted file mode 100644 index db344f549..000000000 --- a/.github/workflows/release_publish_docker-image.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Publish Docker image - -on: - release: - types: - - released - - prereleased - -jobs: - push_to_registry: - name: Push Docker image to Docker Hub - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Prepare - id: prep - run: | - DOCKER_IMAGE=crowdsecurity/crowdsec - GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec - VERSION=edge - if [[ $GITHUB_REF == refs/tags/* ]]; then - VERSION=${GITHUB_REF#refs/tags/} - elif [[ $GITHUB_REF == refs/heads/* ]]; then - VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -E 's#/+#-#g') - elif [[ $GITHUB_REF == refs/pull/* ]]; then - VERSION=pr-${{ github.event.number }} - fi - TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}" - TAGS_SLIM="${DOCKER_IMAGE}:${VERSION}-slim" - if [[ ${{ github.event.action }} == released ]]; then - TAGS=$TAGS,${DOCKER_IMAGE}:latest,${GHCR_IMAGE}:latest - TAGS_SLIM=$TAGS_SLIM,${DOCKER_IMAGE}:slim - fi - echo "version=${VERSION}" >> $GITHUB_OUTPUT - echo "tags=${TAGS}" >> $GITHUB_OUTPUT - echo "tags_slim=${TAGS_SLIM}" >> $GITHUB_OUTPUT - echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - config: .github/buildkit.toml - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push slim image - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.prep.outputs.tags_slim }} - target: slim - platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6,linux/386 - labels: | - org.opencontainers.image.source=${{ github.event.repository.html_url }} - org.opencontainers.image.created=${{ steps.prep.outputs.created }} - org.opencontainers.image.revision=${{ github.sha }} - - - name: Build and push full image - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.prep.outputs.tags }} - platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6,linux/386 - labels: | - org.opencontainers.image.source=${{ github.event.repository.html_url }} - org.opencontainers.image.created=${{ steps.prep.outputs.created }} - org.opencontainers.image.revision=${{ github.sha }} diff --git a/.github/workflows/tmp.yaml b/.github/workflows/tmp.yaml new file mode 100644 index 000000000..9a29f74ae --- /dev/null +++ b/.github/workflows/tmp.yaml @@ -0,0 +1,34 @@ +name: HubTest +on: + pull_request: + branches: [ master ] + +jobs: + run-hub-tests: + runs-on: ubuntu-latest + steps: + - name: "Set up Go 1.20" + uses: actions/setup-go@v4 + with: + go-version: 1.20.4 + - name: checkout repo + uses: actions/checkout@v3 + - name: checkout hub + uses: actions/checkout@v3 + with: + repository: 'crowdsecurity/hub' + path: hub + - run: go version + - name: run tests on last crowdsec tag + run: | + sudo apt-get install -y make + make && cp cmd/crowdsec/crowdsec /usr/local/bin && cp cmd/crowdsec-cli/cscli /usr/local/bin + sudo mkdir -p /etc/crowdsec && sudo cp -a config/* /etc/crowdsec + cd hub + cscli hubtest run modsecurity --debug || cat .tests/modsecurity/results/bucket-dump.yaml + cscli hubtest run modsecurity --debug + cscli hubtest run modsecurity --debug + cscli hubtest run modsecurity --debug + cscli hubtest run modsecurity --debug + cscli hubtest run modsecurity --debug + cscli hubtest run modsecurity --debug diff --git a/.github/workflows/update_docker_hub_doc.yml b/.github/workflows/update_docker_hub_doc.yml deleted file mode 100644 index 0a5047ddc..000000000 --- a/.github/workflows/update_docker_hub_doc.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Update Docker Hub README - -on: - push: - branches: - - master - paths: - - 'docker/README.md' - -jobs: - update-docker-hub-readme: - runs-on: ubuntu-latest - steps: - - - name: Check out the repo - uses: actions/checkout@v3 - if: ${{ github.repository_owner == 'crowdsecurity' }} - - - name: Update docker hub README - uses: ms-jpq/sync-dockerhub-readme@v1 - if: ${{ github.repository_owner == 'crowdsecurity' }} - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - repository: crowdsecurity/crowdsec - readme: "./docker/README.md"