diff --git a/README.md b/README.md index d89c826c9..eb112195c 100644 --- a/README.md +++ b/README.md @@ -128,3 +128,12 @@ Or look directly at [installation documentation](https://doc.crowdsec.net/Crowds This repository contains the code for the two main components of crowdsec : - `crowdsec` : the daemon a-la-fail2ban that can read, parse, enrich and apply heuristics to logs. This is the component in charge of "detecting" the attacks - `cscli` : the cli tool mainly used to interact with crowdsec : ban/unban/view current bans, enable/disable parsers and scenarios. + + +## Contributing + +If you wish to contribute to the core of crowdsec, you are welcome to open a PR in this repository. + +If you wish to add a new parser, scenario or collection, please open a PR in the [hub repository](https://github.com/crowdsecurity/hub). + +If you wish to contribute to the documentation, please open a PR in the [documentation repository](http://github.com/crowdsecurity/crowdsec-docs). \ No newline at end of file diff --git a/docs/assets/images/CS-simplified-treatment.png b/docs/assets/images/CS-simplified-treatment.png deleted file mode 100644 index 219d3164b..000000000 Binary files a/docs/assets/images/CS-simplified-treatment.png and /dev/null differ diff --git a/docs/assets/images/blocker-installation.gif b/docs/assets/images/blocker-installation.gif deleted file mode 100644 index 9e87fd21d..000000000 Binary files a/docs/assets/images/blocker-installation.gif and /dev/null differ diff --git a/docs/assets/images/crowdsec2.png b/docs/assets/images/crowdsec2.png deleted file mode 100644 index bbf619a73..000000000 Binary files a/docs/assets/images/crowdsec2.png and /dev/null differ diff --git a/docs/assets/images/crowdsec_architecture.png b/docs/assets/images/crowdsec_architecture.png deleted file mode 100644 index 4764c0d2c..000000000 Binary files a/docs/assets/images/crowdsec_architecture.png and /dev/null differ diff --git a/docs/assets/images/crowdsec_install.gif b/docs/assets/images/crowdsec_install.gif deleted file mode 100644 index 1f58be84b..000000000 Binary files a/docs/assets/images/crowdsec_install.gif and /dev/null differ diff --git a/docs/assets/images/crowdsec_logo.png b/docs/assets/images/crowdsec_logo.png deleted file mode 100644 index 6b2953dac..000000000 Binary files a/docs/assets/images/crowdsec_logo.png and /dev/null differ diff --git a/docs/assets/images/crowdsec_logo1.png b/docs/assets/images/crowdsec_logo1.png deleted file mode 100644 index c9142c134..000000000 Binary files a/docs/assets/images/crowdsec_logo1.png and /dev/null differ diff --git a/docs/assets/images/cscli-metabase.gif b/docs/assets/images/cscli-metabase.gif deleted file mode 100644 index 605105511..000000000 Binary files a/docs/assets/images/cscli-metabase.gif and /dev/null differ diff --git a/docs/assets/images/dashboard_view.png b/docs/assets/images/dashboard_view.png deleted file mode 100644 index 6db945c8c..000000000 Binary files a/docs/assets/images/dashboard_view.png and /dev/null differ diff --git a/docs/assets/images/dashboard_view2.png b/docs/assets/images/dashboard_view2.png deleted file mode 100644 index 6a91381eb..000000000 Binary files a/docs/assets/images/dashboard_view2.png and /dev/null differ diff --git a/docs/assets/images/forensic-mode.gif b/docs/assets/images/forensic-mode.gif deleted file mode 100644 index 7875996bd..000000000 Binary files a/docs/assets/images/forensic-mode.gif and /dev/null differ diff --git a/docs/assets/images/grafana_details.png b/docs/assets/images/grafana_details.png deleted file mode 100644 index bf6b504f5..000000000 Binary files a/docs/assets/images/grafana_details.png and /dev/null differ diff --git a/docs/assets/images/grafana_insight.png b/docs/assets/images/grafana_insight.png deleted file mode 100644 index 8a1c6af85..000000000 Binary files a/docs/assets/images/grafana_insight.png and /dev/null differ diff --git a/docs/assets/images/grafana_overview.png b/docs/assets/images/grafana_overview.png deleted file mode 100644 index 52de69b81..000000000 Binary files a/docs/assets/images/grafana_overview.png and /dev/null differ diff --git a/docs/assets/images/out-of-the-box-protection.gif b/docs/assets/images/out-of-the-box-protection.gif deleted file mode 100644 index cbf59f73e..000000000 Binary files a/docs/assets/images/out-of-the-box-protection.gif and /dev/null differ diff --git a/docs/contributing.md b/docs/contributing.md deleted file mode 100644 index 73f6fa4d6..000000000 --- a/docs/contributing.md +++ /dev/null @@ -1,152 +0,0 @@ -# Contributing - -You have an idea, a suggestion or you spotted a mistake ? -Help us improve the software and the user experience, to make the internet a safer place together ! - - - -## Contributing to the documentation - -If you spotted some mistakes in the documentation or have improvement suggestions, you can : - - - open a {{v1X.doc.new_issue}} if you are comfortable with github - - let us know on {{v1X.doc.discourse}} if you want to discuss about it - -Let us as well know if you have some improvement suggestions ! - - -
- Preview your documentation changes locally - -```bash -python3 -m venv cs-env -source cs-env/bin/activate -pip install -r docs/requirements.txt -mkdocs serve -``` - -
- - -## Contributing to the code - - - If you want to report a bug, you can use [the github bugtracker]({{v1X.crowdsec.bugreport}}) - - If you want to suggest an improvement you can use either [the github bugtracker]({{v1X.crowdsec.bugreport}}) or the {{v1X.doc.discourse}} if you want to discuss - - -## Contributing to the parsers/scenarios - -If you want to contribute your parser or scenario to the community and have them appear on the {{v1X.hub.htmlname}}, you should [open a merge request](https://github.com/crowdsecurity/hub/pulls) on the hub. - -We are currently working on a proper [CI](https://en.wikipedia.org/wiki/Continuous_integration) for the {{v1X.hub.htmlname}}, so for now all contribution are subject to peer-review, please bear with us ! - -## Contacting the team - -If you want to contact us using non-public media, you can contact us on `support` AT `crowdsec` DOT `net` with the following gpg-key : - -``` ------BEGIN PGP PUBLIC KEY BLOCK----- -mQGNBF+VOSUBDADP6bxKDv88CdLBNhQMFNI37LE82vyfIAQmrGszON1m1EtL/LSQ -b/vC9mmlkUmJHM+bdxJ0BSl/xlWwrXjHVpaZNoluQDngVUe62cybN4tpFCvtVTMr -lo4Y0UhETgOmBFxaQLVd7Xc/jaSZGoHtSzh9hpGHg9pKrcYviG0MR173JYQfilw3 -L8yJ+K/oUUpvh2MRRwXiCNUVLtTppb7oxlcdExb0Px2PcaC34e/M30xFwiu7VJFj -0D7IIdKs6gvZuqwkNSUBF8/jtuzzM/YGzJHIdvOj15z+81/o/e6p3xvY/IKmyXC/ -1FMD8f4g5T/5fNDVq6QgJLel/g0bJ+kG75ccXfY45xKFo/YhdQ2Wg9JQX5Yjc5k7 -5AI0iuJjatXlym2Ek1niPEqR5H0C/KXFG4mPyCu9wzJu11jtY34e5TNYl9DA31F6 -81BbMmVFg4EbhYSN/2DuxpCvt2qQpk33bmdT7tFWcd2hYB/bSq2f8+K6ho50Sqwk -PK68LNZzi5ZXqGEAEQEAAbQnQ3Jvd2RTZWMgc3VwcG9ydCA8c3VwcG9ydEBjcm93 -ZHNlYy5uZXQ+iQHUBBMBCgA+FiEEpRXNfWM+DON/Satp2MpQXYwzLTEFAl+VOSUC -GwMFCQPCZwAFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQ2MpQXYwzLTEhuwwA -wWdsuSrTC4ryVOYnfHRcT2b/rfbJXIUYXqAy75qsdUGwvueYdYSBMCMXqRB65J+J -juofCF0kTQKuhjtyJezwUfr5C+Sd08JWlZwf9F7CO83/ztLOPIUUp69H3m9heW7C -+A/Lpq3epALytC/QSkDHYnKBBZbLhoR/7WXhdLFvh+A475/ggn4GAOnZMg8WULpR -Kisu1GbEBPcVr1Xl6VTYVX5ghA/1W2WTY/NxAcLhCiJO/ENeka7xy4EKdCE5pDxM -QO/fnpCHsWDIHTxpCx+JAhdkb2BIvzSiF2+o+9y+vwzcPxdGemx7y8MjSGXIp1xp -TJparq309nljh+wqI6w/K+NjzNn/qJL0tpGqiHQXtYDbi86KaAXT9IYCGAIP36w8 -XUHYGgo0s6zMEP1NEFHWAgGy5elO403vm+NO5vpHv59FTjgoK2UcjeSjqtAYwzvc -bWQ6wZHwhoqD0WevFcAMmgdbebyOdPoA7+8eCPnkjER4eKxE23ffFU75HDuQNRYk -uQGNBF+VOSUBDADNHEm33IcwhO+uJQxjKtcF0DdAMqbjU5cXxeryo1i7A1WkTH5/ -wHfyJAmtLrY4abkQ1LEJ4bMYKdJz2vmvWq0fKCAXC18yLnxU+l0Ld4tWME8hJ/Wh -p+aePsW5BdLpHQeqmQ5MCsw1cZllbURcee22hLJ/PIM2bRsZp7goSj4wXBFjhJyq -EepVmasI17dBbIBFWBSSIJW4UnSBk+Zqbj6C6PDmsket68qcEebsqduWXPxegAzh -IIFD2qhC5t+nn5i+hPwKZN5ZYLQJeAjI4Z7wi3FIBZCzZ214421BbohxPo+GKkFp -mUQ7ZrIa+goHXAcj6ZHMeNNP0lsJRl91lK6NVu3p+Ygl0+wbMOAqDRguMfFdbnV8 -gcoYpAyk4YFCfgVQLuKGaYcGjcMP8+nZnPsbaTwbUKkjDAUo+JGmrB4XyAQPugZq -TiUN+lYgTs0cJALEQkKTh2w10TPyV6/YsYDSSnwJeVDIpNCQVg5EB0eRvhaCs9fd -dVni1C5RMcb+Q4MAEQEAAYkBvAQYAQoAJhYhBKUVzX1jPgzjf0mradjKUF2MMy0x -BQJflTklAhsMBQkDwmcAAAoJENjKUF2MMy0xkIcL/johqZbyHskQIaTfQUgASbbu -bdLXSrIkB8Ort9WULxdqs8hveFy6RjXFJWFitFHk46Bj6FJ1ZykfozL+k9uOGrL9 -lBk1e3bhqMVhW1o00DufgawNU2FU9NuH/rCuGpum9DE0cc1fFmQ3pjeiHV55GYxr -BGuyyals1ORwK06h+1VFMHrGB12SR7Imgo7FWuexhgLyOK4t1MXg3E4h72qaowpj -5B45qG9jUXgIFKR1D8G8tPeDYLbd37pskNDFozzfAe/H2fqmEjQxMLHrk7J8I3wQ -FPvKIvUF8M3NqZjyaFSiisOn32AS3RAsI8RuD4T2XgpE2L6e29u3RpJkvhPbcAN6 -w0W8yw3z1/2uHSvYbwoH1cn4akAikYR9aVVHv86AvNlr0BguqWdzEfiGT6mcJ/hH -2sGQJ1nJRgGpAlx/2HpsLJxhJwLVbXSDSk6Bu2T9G/VIda95niVgq6MfE9GSS+MS -ucVcwqjIXn/9V6+pFZ11soXNKuTk4Wx+uO2r/i5bVA== -=Edl+ ------END PGP PUBLIC KEY BLOCK----- -``` - - -## Publishing bouncers - -We do welcome bouncers from the community, and will gladly publish them on the hub. - -### Why ? - -Sharing your bouncer on the hub allows other users to find it and use it. While increasing your code's visibility, it ensures as well a benevolent look from the community and the team over it. - -### How ? - -To have your bouncer published on the hub, please simply [open a new issue on the hub](https://github.com/crowdsecurity/hub/issues/new), requesting "bouncer inclusion". The bouncer will then be reviewed by the team, and then will be published directly on the hub, for everyone to find & use it ! - - -The information that should be stated in your issue are : - - - The source repository of your bouncer (for example `https://github.com/crowdsecurity/cs-firewall-bouncer/`) - - The software licence used - - The current status of the bouncer (stage : dev/unstable/stable) - - Documentation (can be simply in the README.md) : - - must contains : installing, uninstalling - - should contains : configuration documentation - - Link to existing tests if applicable (functional tests or unit tests) - -Please take care of the following : - - - Ensure your repository has a About/Short description meaningful enough : it will be displayed in the hub - - Ensure your repository has a decent README.md file : it will be displayed in the hub - - Ensure your repository has *at least* one release : this is what users will be looking for - - (ideally) Have a "social preview image" on your repository : this will be displayed in the hub when available - - (ideally) A Howto or link to guide that provides a hands-on experience with the bouncer - - -Please find below a template : - -```markdown -Hello, - -I would like to suggest the addition of the `XXXX` to the hub : - - - Source repository: https://github.com/xxx/xxx/ - - Licence : MIT - - Current status : stable (has been used in production for a while) - - README/doc : https://github.com/xxx/xxx/blob/main/README.md - - Existing tests : - - functional tests : https://github.com/xxx/xxx/blob/main/.github/workflows/tests.yml - - - Short/Long description : OK - - Howto : in README - - At least one release : yes - -``` - -## Publishing parsers, scenarios and collections - -### Why ? - -Sharing your parsers, scenarios and collections on the hub allows other users to find it and use it. While increasing your code's visibility, it ensures as well a benevolent look from the community and the team over it. - -### How ? - -To have your parser/scenario published on the hub, please simply [open a new issue on the hub](https://github.com/crowdsecurity/hub/issues/new), requesting "parser/scenario inclusion". The configurations will then be reviewed by the team, and then will be published directly on the hub, for everyone to find & use it ! - diff --git a/docs/faq.md b/docs/faq.md deleted file mode 100644 index bdc4c34db..000000000 --- a/docs/faq.md +++ /dev/null @@ -1,232 +0,0 @@ -# FREQUENTLY ASKED QUESTIONS - -## What is {{v1X.crowdsec.name}} ? - -{{v1X.crowdsec.Name}} is a security open-source software. See the [overview](/#what-is-crowdsec). - -## I've installed crowdsec, it detects attacks but doesn't block anything ?! - -Yes, {{v1X.crowdsec.Name}} is in charge of detecting attacks, and {{v1X.bouncers.htmlname}} are applying decisions. -If you want to block the detected IPs, you should deploy a bouncer, such as the ones found on the [hub](https://hub.crowdsec.net/browse/#bouncers) ! - - -## What language is it written in ? - -{{v1X.crowdsec.Name}} is written in [Golang](https://golang.org/). - -## What licence is {{v1X.crowdsec.name}} released under ? - -{{v1X.crowdsec.Name}} is under [MIT license]({{v1X.crowdsec.url}}/blob/master/LICENSE). - -## Which information is sent to the APIs ? - -Our aim is to build a strong community that can share malevolent attackers IPs, for that we need to collect the bans triggered locally by each user. - -The signal sent by your {{v1X.crowdsec.name}} to the central API only contains only meta-data about the attack : - - - Attacker IP - - Scenario name - - Time of start/end of attack - -Your logs are not sent to our central API, only meta-data about blocked attacks will be. - - -When pulling block-lists from the platform, the following information is shared as well : - - - list of [upstream installed scenarios](https://crowdsecurity.github.io/api_doc/index.html?urls.primaryName=CAPI#/watchers/post_metrics) - - list of [bouncers & number of machines](https://crowdsecurity.github.io/api_doc/index.html?urls.primaryName=CAPI#/watchers/post_metrics) - -## What is the performance impact ? - -As {{v1X.crowdsec.name}} only works on logs, it shouldn't impact your production. -When it comes to {{v1X.bouncers.name}}, it should perform **one** request to the database when a **new** IP is discovered thus have minimal performance impact. - -## How fast is it ? - -{{v1X.crowdsec.name}} can easily handle several thousands of events per second on a rich pipeline (multiple parsers, geoip enrichment, scenarios and so on). Logs are a good fit for sharding by default, so it is definitely the way to go if you need to handle higher throughput. - -If you need help for large scale deployment, please get in touch with us on the {{v1X.doc.discourse}}, we love challenges ;) - -## What backend database does {{v1X.crowdsec.Name}} supports and how to switch ? - -{{v1X.crowdsec.name}} versions (under v0.3.X) supports SQLite (default) and MySQL databases. -See [backend configuration](/Crowdsec/v0/references/output/#switching-backend-database) for relevant configuration. MySQL here is more suitable for distributed architectures where bouncers across the applicative stack need to access a centralized ban database. - -{{v1X.crowdsec.name}} versions (after v1) supports SQLite (default), MySQL and PostgreSQL databases. -See [databases configuration](/Crowdsec/v1/user_guide/database/) for relevant configuration. Thanks to the {{v1X.lapi.Htmlname}}, distributed architectures are resolved even with sqlite database. - -SQLite by default as it's suitable for standalone/single-machine setups. - -## How to control granularity of actions ? (whitelists, simulation etc.) - -{{v1X.crowdsec.name}} support both [whitelists](/Crowdsec/v1/write_configurations/whitelist/) and [simulation](/Crowdsec/v1/references/simulation/) : - - - Whitelists allows you to "discard" events or overflows - - Simulation allows you to simply cancel the decision that is going to be taken, but keep track of it - - {{v1X.profiles.htmlname}} allows you to control which decision will be applied to which alert. - -## How to know if my setup is working correctly ? Some of my logs are unparsed, is it normal ? - -Yes, crowdsec parsers only parse the logs that are relevant for scenarios :) - -Take a look at `cscli metrics` [and understand what do they mean](/Crowdsec/v1/getting_started/crowdsec-tour/#reading-metrics) to know if your setup is correct. - - -## How to add whitelists ? - -You can follow this [guide](/Crowdsec/v1/write_configurations/whitelist/) - -## How to set up proxy ? - -Setting up a proxy works out of the box, the [net/http golang library](https://golang.org/src/net/http/transport.go) can handle those environment variables: - -* `HTTP_PROXY` -* `HTTPS_PROXY` -* `NO_PROXY` - -For example: - -``` -export HTTP_PROXY=http://: -``` -### Systemd variable -On Systemd devices you have to set the proxy variable in the environment section for the CrowdSec service. To avoid overwriting the service file during an update, a folder is created in `/etc/systemd/system/crowdsec.service.d` and a file in it named `http-proxy.conf`. The content for this file should look something like this: -``` -[Service] -Environment=HTTP_PROXY=http://myawesomeproxy.com:8080 -Environment=HTTPS_PROXY=https://myawesomeproxy.com:443 -``` -After this change you need to reload the systemd daemon using: -`systemctl daemon-reload` - -Then you can restart CrowdSec like this: -`systemctl restart crowdsec` - -### Sudo -If you use `sudo` {{v1X.cli.name}}, just add this line in `visudo` after setting up the previous environment variables: - -``` -Defaults env_keep += "HTTP_PROXY HTTPS_PROXY NO_PROXY" -``` - -## How to report a bug ? - -To report a bug, please open an issue on the [repository]({{v1X.crowdsec.bugreport}}). - -## What about false positives ? - -Several initiatives have been taken to tackle the false positives approach as early as possible : - - - The scenarios published on the hub are tailored to favor low false positive rates - - You can find [generic whitelists](https://hub.crowdsec.net/author/crowdsecurity/collections/whitelist-good-actors) that should allow to cover most common cases (SEO whitelists, CDN whitelists etc.) - - The [simulation configuration](/Crowdsec/v1/references/simulation/) allows you to keep a tight control over scenario and their false positives - - -## I need some help - -Feel free to ask for some help to the {{v1X.doc.discourse}} or directly in the {{v1X.doc.gitter}} chat. - -## How to use crowdsec on raspberry pi OS (formerly known as rasbian) - -Please keep in mind that raspberry pi OS is designed to work on all -raspberry pi versions. Even if the port target is known as armhf, it's -not exactly the same target as the debian named armhf port. - -The best way to have a crowdsec version for such an architecture is to -do: - -1. install golang (all versions from 1.13 will do) -2. `export GOARCH=arm` -3. `export CGO=1` -4. Update the GOARCH variable in the Makefile to `arm` -5. install the arm gcc cross compilator (On debian the package is gcc-arm-linux-gnueabihf) -6. Compile crowdsec using the usual `make` command - - -## How to have a dashboard without docker - -`cscli dashboard` rely on [`docker`](https://docs.docker.com/) to launch the `metabase` image. If `docker` is not installed on your machine, here are the step to follow to get crowdsec dashboards without docker: - -- Download Metabase `jar` file. See [metabase documentation](https://www.metabase.com/docs/latest/operations-guide/running-the-metabase-jar-file.html). -- Download the `metabase.db` folder from Crowdsec [here](https://crowdsec-statics-assets.s3-eu-west-1.amazonaws.com/metabase_sqlite.zip). -- Unzip the `zip` file: - -```bash -unzip metabase_sqlite.zip -``` - -- Make crowdsec database reachable from metabase : - -```bash -sudo mkdir /metabase-data/ -sudo ln -s /var/lib/crowdsec/data/crowdsec.db /metabase-data/crowdsec.db -``` - -- Launch Metabase: - -```bash -sudo MB_DB_TYPE=h2 MB_DB_FILE=/metabase.db/metabase.db java -jar metabase.jar -``` - -!!! warning - The default username is `crowdsec@crowdsec.net` and the default password is `!!Cr0wdS3c_M3t4b4s3??`. Please update the password when you will connect to metabase for the first time - -You can as well check [liberodark's helper script for it](https://github.com/liberodark/crowdsec-dashboard). - -## How to configure crowdsec/cscli to use Tor - - -It is possible to configure `cscli` and `crowdsec` to use [tor](https://www.torproject.org/) to anonymously interact with our API. -All (http) requests made to the central API to go through the [tor network](https://www.torproject.org/). - - -With tor installed, setting `HTTP_PROXY` and `HTTPS_PROXY` environment variables to your socks5 proxy will do the trick. - - -### Running the wizard with tor - -```bash -$ sudo HTTPS_PROXY=socks5://127.0.0.1:9050 HTTP_PROXY=socks5://127.0.0.1:9050 ./wizard.sh --bininstall -``` - -!!! warning - Do not use the wizard in interactive (`-i`) mode if you're concerned, as it will start the service at the end of the setup, leaking your IP address. - - -### Edit crowdsec systemd unit to push/pull via tor - -```bash -[Service] -Environment="HTTPS_PROXY=socks5://127.0.0.1:9050" -Environment="HTTP_PROXY=socks5://127.0.0.1:9050" -... -``` -### Using cscli via tor - -```bash -$ sudo HTTP_PROXY=socks5://127.0.0.1:9050 HTTPS_PROXY=socks5://127.0.0.1:9050 cscli capi register -``` - - - - - diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index f26b02c6b..000000000 --- a/docs/index.md +++ /dev/null @@ -1,41 +0,0 @@ -
[[Hub]]({{v1X.hub.url}}) [[Releases]]({{v1X.crowdsec.download_url}})
- - -!!! warning - For crowdsec versions `<= 1.0` please refer to [v0.3.X](/Crowdsec/v0/) - - For crowdsec versions `>= 1.0` please refer to [v1.X](/Crowdsec/v1/) - -# What is {{v1X.crowdsec.Name}} ? - -[{{v1X.crowdsec.Name}}]({{v1X.crowdsec.url}}) is an open-source and lightweight software that allows you to detect peers with malevolent behaviors and block them from accessing your systems at various level (infrastructural, system, applicative). - -To achieve this, {{v1X.crowdsec.Name}} reads logs from different sources (files, streams ...) to parse, normalize and enrich them before matching them to threats patterns called scenarios. - -{{v1X.crowdsec.Name}} is a modular and plug-able framework, it ships a large variety of [well known popular scenarios](https://hub.crowdsec.net/browse/#configurations); users can choose what scenarios they want to be protected from as well as easily adding new custom ones to better fit their environment. - -Detected malevolent peers can then be prevented from accessing your resources by deploying [bouncers]({{v1X.hub.bouncers_url}}) at various levels (applicative, system, infrastructural) of your stack. - -One of the advantages of Crowdsec when compared to other solutions is its crowd-sourced aspect : Meta information about detected attacks (source IP, time and triggered scenario) are sent to a central API and then shared amongst all users. - -Thanks to this, besides detecting and stopping attacks in real time based on your logs, it allows you to preemptively block known bad actors from accessing your information system. - - -## Main features - -{{v0X.crowdsec.Name}}, besides the core "detect and react" mechanism, is committed to a few other key points : - - - **Easy Installation** : The provided wizard allows a [trivial deployment](/Crowdsec/v1/getting_started/installation/#using-the-interactive-wizard) on most standard setups - - **Easy daily operations** : Using [cscli](/Crowdsec/v1/cscli/cscli_upgrade/) and the {{v0X.hub.htmlname}}, keeping your detection mechanisms up-to-date is trivial - - **Reproducibility** : Crowdsec can run not only against live logs, but as well against cold logs. It makes it a lot easier to detect potential false-positives, perform forensic ou generate reporting - - **Observability** : Providing strongs insights on what is going on and what {{v0X.crowdsec.name}} is doing : - - Humans have [access to a trivially deployable web interface](/Crowdsec/v1/observability/dashboard/) - - OPs have [access to detailed prometheus metrics](/Crowdsec/v1/observability/prometheus/) - - Admins have [a friendly command-line interface tool](/Crowdsec/v1/observability/command_line/) - -## About this documentation - -This document is split according to major {{v1X.crowdsec.Name}} versions : - - - [Crowdsec v0](/Crowdsec/v0/) Refers to versions `0.3.X`, before the local API was introduced. (_note: this is going to be deprecated and your are strongly incited to migrate to versions 1.X_) - - [Crowdsec v1](/Crowdsec/v1/) Refers to versions `1.X`, it is the current version \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 3f8aa881b..000000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,27 +0,0 @@ -click==7.1.1 -future==0.18.2 -Jinja2==2.11.3 -joblib==0.14.1 -livereload==2.6.1 -lunr==0.5.6 -Markdown==3.2.1 -MarkupSafe==1.1.1 -mkdocs==1.1 -mkdocs-macros-plugin==0.4.18 -mkdocs-material==6.1.0 -mkdocs-material-extensions==1.0.1 -mkdocs-monorepo-plugin==0.4.11 -mkdocs-redirects==1.0.1 -nltk==3.5b1 -prompt-toolkit==2.0.10 -Pygments==2.7.4 -pymdown-extensions==7.0 -python-markdown-math==0.6 -PyYAML==5.4 -regex==2020.2.20 -repackage==0.7.3 -six==1.14.0 -termcolor==1.1.0 -tornado==6.0.4 -tqdm==4.43.0 -wcwidth==0.1.9 diff --git a/docs/v0.3.X/docs/assets/images/blocker-installation.gif b/docs/v0.3.X/docs/assets/images/blocker-installation.gif deleted file mode 100644 index 9846e97fc..000000000 Binary files a/docs/v0.3.X/docs/assets/images/blocker-installation.gif and /dev/null differ diff --git a/docs/v0.3.X/docs/assets/images/crowdsec2.png b/docs/v0.3.X/docs/assets/images/crowdsec2.png deleted file mode 100644 index bbf619a73..000000000 Binary files a/docs/v0.3.X/docs/assets/images/crowdsec2.png and /dev/null differ diff --git a/docs/v0.3.X/docs/assets/images/crowdsec_architecture.png b/docs/v0.3.X/docs/assets/images/crowdsec_architecture.png deleted file mode 100644 index 5e5e6184d..000000000 Binary files a/docs/v0.3.X/docs/assets/images/crowdsec_architecture.png and /dev/null differ diff --git a/docs/v0.3.X/docs/assets/images/crowdsec_install.gif b/docs/v0.3.X/docs/assets/images/crowdsec_install.gif deleted file mode 100644 index ceddd7f1d..000000000 Binary files a/docs/v0.3.X/docs/assets/images/crowdsec_install.gif and /dev/null differ diff --git a/docs/v0.3.X/docs/assets/images/crowdsec_logo1.png b/docs/v0.3.X/docs/assets/images/crowdsec_logo1.png deleted file mode 100644 index c9142c134..000000000 Binary files a/docs/v0.3.X/docs/assets/images/crowdsec_logo1.png and /dev/null differ diff --git a/docs/v0.3.X/docs/assets/images/cscli-metabase.gif b/docs/v0.3.X/docs/assets/images/cscli-metabase.gif deleted file mode 100644 index b21d41191..000000000 Binary files a/docs/v0.3.X/docs/assets/images/cscli-metabase.gif and /dev/null differ diff --git a/docs/v0.3.X/docs/assets/images/dashboard_view.png b/docs/v0.3.X/docs/assets/images/dashboard_view.png deleted file mode 100644 index 6db945c8c..000000000 Binary files a/docs/v0.3.X/docs/assets/images/dashboard_view.png and /dev/null differ diff --git a/docs/v0.3.X/docs/assets/images/dashboard_view2.png b/docs/v0.3.X/docs/assets/images/dashboard_view2.png deleted file mode 100644 index 6a91381eb..000000000 Binary files a/docs/v0.3.X/docs/assets/images/dashboard_view2.png and /dev/null differ diff --git a/docs/v0.3.X/docs/assets/images/grafana_details.png b/docs/v0.3.X/docs/assets/images/grafana_details.png deleted file mode 100644 index bf6b504f5..000000000 Binary files a/docs/v0.3.X/docs/assets/images/grafana_details.png and /dev/null differ diff --git a/docs/v0.3.X/docs/assets/images/grafana_insight.png b/docs/v0.3.X/docs/assets/images/grafana_insight.png deleted file mode 100644 index 8a1c6af85..000000000 Binary files a/docs/v0.3.X/docs/assets/images/grafana_insight.png and /dev/null differ diff --git a/docs/v0.3.X/docs/assets/images/grafana_overview.png b/docs/v0.3.X/docs/assets/images/grafana_overview.png deleted file mode 100644 index 52de69b81..000000000 Binary files a/docs/v0.3.X/docs/assets/images/grafana_overview.png and /dev/null differ diff --git a/docs/v0.3.X/docs/assets/images/out-of-the-box-protection.gif b/docs/v0.3.X/docs/assets/images/out-of-the-box-protection.gif deleted file mode 100644 index a309f794c..000000000 Binary files a/docs/v0.3.X/docs/assets/images/out-of-the-box-protection.gif and /dev/null differ diff --git a/docs/v0.3.X/docs/bouncers/index.md b/docs/v0.3.X/docs/bouncers/index.md deleted file mode 100644 index 97c3b0152..000000000 --- a/docs/v0.3.X/docs/bouncers/index.md +++ /dev/null @@ -1,12 +0,0 @@ -# bouncers - - -{{v0X.bouncers.Name}} are standalone software pieces in charge of acting upon blocked IPs. - -They can either within the applicative stack, or work out of band : - -[nginx blocker](https://github.com/crowdsecurity/cs-nginx-blocker) will check every unknown IP against the database before letting go through or serving a *403* to the user, while a [netfilter blocker](https://github.com/crowdsecurity/cs-netfilter-blocker) will simply "add" malevolent IPs to nftables/ipset set of blacklisted IPs. - - -You can explore [available {{v0X.bouncers.name}} on the hub]({{v0X.hub.plugins_url}}), and find below a few of the "main" {{v0X.bouncers.name}} : - diff --git a/docs/v0.3.X/docs/cheat_sheets/ban-mgmt.md b/docs/v0.3.X/docs/cheat_sheets/ban-mgmt.md deleted file mode 100644 index e074fbceb..000000000 --- a/docs/v0.3.X/docs/cheat_sheets/ban-mgmt.md +++ /dev/null @@ -1,87 +0,0 @@ -!!! info - - Please see your local `{{v0X.cli.bin}} help ban` for up-to-date documentation. - -## List bans - -```bash -{{v0X.cli.bin}} ban list -``` - -
- example -```bash -bui@sd:~$ cli ban list -4 local decisions: -+--------+----------------+----------------------+------+--------+---------+--------------------------------+--------+------------+ -| SOURCE | IP | REASON | BANS | ACTION | COUNTRY | AS | EVENTS | EXPIRATION | -+--------+----------------+----------------------+------+--------+---------+--------------------------------+--------+------------+ -| cli | 1.1.1.1 | spammer | 1 | ban | | | 0 | 23h59m58s | -| local | 2.2.2.2 | crowdsecurity/ssh-bf | 1 | ban | FR | 3215 Orange | 6 | 3h7m30s | -| local | 3.3.3.3 | crowdsecurity/ssh-bf | 1 | ban | US | 3266 Joao Carlos de Almeida | 6 | 57m17s | -| | | | | | | Silveira trading as Bitcanal | | | -| local | 4.4.4.4 | crowdsecurity/ssh-bf | 1 | ban | FR | 15557 SFR SA | 6 | 5m11s | -+--------+----------------+----------------------+------+--------+---------+--------------------------------+--------+------------+ -And 64 records from API, 32 distinct AS, 19 distinct countries - -``` -
- - - `SOURCE` is the source of the decision : - - "local" : the decision has been taken by {{v0X.crowdsec.name}} - - "cli" : the decision has been made with {{v0X.cli.name}} (ie. `{{v0X.cli.name}} ban ip 1.2.3.4 24h "because"`) - - "api" : the decision has been pushed to you by the API (because there is a consensus about this ip) - - `IP` is the IP or the IP range impacted by the decision - - `REASON` is the scenario that was triggered (or human-supplied reason) - - `BANS` is the number of "active" remediation against this IP - - `COUNTRY` and `AS` are provided by GeoIP enrichment if present - - `EXPIRATION` is the time left on remediation - - -Check [command usage](/Crowdsec/v0/cscli/cscli_ban_list/) for additional filtering and output control flags. - - -## Delete a ban - -> delete the ban on IP `1.2.3.4` - -```bash -{{v0X.cli.bin}} ban del ip 1.2.3.4 -``` - -> delete the ban on range 1.2.3.0/24 - -```bash -{{v0X.cli.bin}} ban del range 1.2.3.0/24 -``` - - -## Add a ban manually - -> Add a ban on IP `1.2.3.4` for 24 hours, with reason 'web bruteforce' - -```bash -{{v0X.cli.bin}} ban add ip 1.2.3.4 24h "web bruteforce" -``` - -> Add a ban on range `1.2.3.0/24` for 24 hours, with reason 'web bruteforce' - -```bash -{{v0X.cli.bin}} ban add range 1.2.3.0/24 "web bruteforce" -``` - - - -## Flush all existing bans - -> Flush all the existing bans - -```bash -{{v0X.cli.bin}} ban flush -``` - -!!! warning - This will as well remove any existing ban - - - diff --git a/docs/v0.3.X/docs/cheat_sheets/config-mgmt.md b/docs/v0.3.X/docs/cheat_sheets/config-mgmt.md deleted file mode 100644 index c5eaf848e..000000000 --- a/docs/v0.3.X/docs/cheat_sheets/config-mgmt.md +++ /dev/null @@ -1,115 +0,0 @@ -{{v0X.cli.bin}} allows you install, list, upgrade and remove configurations : parsers, enrichment, scenarios. - -!!! warning - If you're not running the latest CrowdSec version, configurations might not be the latest available. `cscli` will use the branch of the corresponding CrowdSec version to download and install configurations from the hub (it will use the `master` branch if you are on the latest CrowdSec version). - -The various parsers, enrichers and scenarios installed on your machine makes a coherent ensemble to provide detection capabilities. - -_Parsers, Scenarios and Enrichers are often bundled together in "collections" to facilitate configuration._ - -Parsers, scenarios, enrichers and collections all follow the same principle : - - - `{{v0X.cli.bin}} install parser crowdsec/nginx-logs` - - `{{v0X.cli.bin}} update collection crowdsec/base-http-scenarios` - - `{{v0X.cli.bin}} remove scenario crowdsec/mysql-bf` - -> Please see your local `{{v0X.cli.bin}} help` for up-to-date documentation - - -## List configurations - -``` -{{v0X.cli.bin}} list -``` - -**note** `-a` allows for listing of uninstalled configurations as well - -
- {{v0X.cli.name}} list example - -```bash -$ {{v0X.cli.bin}} list -INFO[0000] Loaded 9 collecs, 14 parsers, 12 scenarios, 1 post-overflow parsers -INFO[0000] PARSERS: --------------------------------------------------------------------------------------------------------------------- - NAME 📦 STATUS VERSION LOCAL PATH --------------------------------------------------------------------------------------------------------------------- - crowdsec/nginx-logs ✔️ enabled 0.3 /etc/crowdsec/config/parsers/s01-parse/nginx-logs.yaml - crowdsec/geoip-enrich ✔️ enabled 0.4 /etc/crowdsec/config/parsers/s02-enrich/geoip-enrich.yaml - crowdsec/syslog-logs ✔️ enabled 0.4 /etc/crowdsec/config/parsers/s00-raw/syslog-logs.yaml - crowdsec/whitelists ✔️ enabled 0.4 /etc/crowdsec/config/parsers/s02-enrich/whitelists.yaml - crowdsec/http-logs ✔️ enabled 0.4 /etc/crowdsec/config/parsers/s02-enrich/http-logs.yaml - crowdsec/dateparse-enrich ✔️ enabled 0.4 /etc/crowdsec/config/parsers/s02-enrich/dateparse-enrich.yaml --------------------------------------------------------------------------------------------------------------------- -INFO[0000] SCENARIOS: ------------------------------------------------------------------------------------------------------------------------ - NAME 📦 STATUS VERSION LOCAL PATH ------------------------------------------------------------------------------------------------------------------------ - crowdsec/http-scan-uniques_404 ✔️ enabled 0.4 /etc/crowdsec/config/scenarios/http-scan-uniques_404.yaml - crowdsec/http-crawl-non_statics ✔️ enabled 0.4 /etc/crowdsec/config/scenarios/http-crawl-non_statics.yaml ------------------------------------------------------------------------------------------------------------------------ -INFO[0000] COLLECTIONS: -------------------------------------------------------------------------------------------------------------------- - NAME 📦 STATUS VERSION LOCAL PATH -------------------------------------------------------------------------------------------------------------------- - crowdsec/linux ✔️ enabled 0.2 /etc/crowdsec/config/collections/linux.yaml - crowdsec/nginx ✔️ enabled 0.2 /etc/crowdsec/config/collections/nginx.yaml - crowdsec/base-http-scenarios ✔️ enabled 0.1 /etc/crowdsec/config/collections/base-http-scenarios.yaml -------------------------------------------------------------------------------------------------------------------- -INFO[0000] POSTOVERFLOWS: --------------------------------------- - NAME 📦 STATUS VERSION LOCAL PATH --------------------------------------- --------------------------------------- - -``` -
- - - -For {{v0X.parsers.htmlname}}, {{v0X.scenarios.htmlname}}, {{v0X.collections.htmlname}} the outputs include, beside the version, the path and the name, a `STATUS` column : - - - `✔️ enabled` : configuration is up-to-date - - `⚠️ enabled,outdated` : a newer version is available - - `🚫 enabled,local` : configuration is not managed by {{v0X.cli.name}} - - `⚠️ enabled,tainted` : configuration has been locally modified - -(see `{{v0X.cli.name}} upgrade` to upgrade/sync your configurations with {{v0X.hub.htmlname}}) - -## Install new configurations - - -`{{v0X.cli.bin}} install parser|scenario|postoverflow [--force]` - - - - `{{v0X.cli.bin}} install parser crowdsec/nginx-logs` - - `{{v0X.cli.bin}} install scenario crowdsec/http-scan-uniques_404` - - -## Remove configurations - - -`{{v0X.cli.bin}} remove parser|scenario|postoverflow [--force]` - - -## Upgrade configurations - -> upgrade a specific scenario - -``` -{{v0X.cli.bin}} upgrade scenario crowdsec/http-scan-uniques_404 -``` - - -> upgrade **all** scenarios - -``` -{{v0X.cli.bin}} upgrade scenario --all -``` - -> upgrade **all** configurations (parsers, scenarios, collections, postoverflows) - -``` -{{v0X.cli.bin}} upgrade --all -``` - diff --git a/docs/v0.3.X/docs/cheat_sheets/debugging_configs.md b/docs/v0.3.X/docs/cheat_sheets/debugging_configs.md deleted file mode 100644 index be0489235..000000000 --- a/docs/v0.3.X/docs/cheat_sheets/debugging_configs.md +++ /dev/null @@ -1,127 +0,0 @@ - - - -# Debugging Scenarios and Parsers - -## General Advice - -When trying to debug a parser or a scenario : - - - Work on "cold logs" (with the `-file` and `-type` options) rather than live ones - - Use the `/etc/crowdsec/config/user.yaml` configuration files to have logs on stdout - -## Using user-mode configuration - -```bash -crowdsec -c /etc/crowdsec/config/user.yaml -file mylogs.log.gz -type syslog -INFO[05-08-2020 16:15:47] Crowdsec v0.3.0-rc3-7525f11975a0107746213862dc41c69e00122ac7 -INFO[05-08-2020 16:15:47] Loading grok library -... -WARN[05-08-2020 16:16:12] 182.x.x.x triggered a 4h0m0s ip ban remediation for [crowdsecurity/http-probing] bucket_id=misty-moon event_time="2019-01-01 22:58:32 +0100 CET" scenario=crowdsecurity/http-probing source_ip=182.x.x.x -... -``` - - - `/etc/crowdsec/config/user.yaml` disables demonization and push logs to stdout/stderr - - `-type` must respect expected log type (ie. `nginx` `syslog` etc.) - - `-file` must point to a flat file or a gzip file - -When processing logs like this, {{v0X.crowdsec.name}} runs in "time machine" mode, and relies on the timestamps *in* the logs to evaluate scenarios. You will most likely need the `crowdsecurity/dateparse-enrich` parser for this. - - -## Testing configurations on live system - -If you're playing around with parser/scenarios on a live system, you can use the `-t` (lint) option of {{v0X.crowdsec.Name}} to check your configurations validity before restarting/reloading services : - -```bash -$ emacs /etc/crowdsec/config/scenarios/ssh-bf.yaml -... -$ crowdsec -c /etc/crowdsec/config/user.yaml -t -INFO[06-08-2020 13:36:04] Crowdsec v0.3.0-rc3-4cffef42732944d4b81b3e62a03d4040ad74f185 -... -ERRO[06-08-2020 13:36:05] Bad yaml in /etc/crowdsec/config/scenarios/ssh-bf.yaml : yaml: unmarshal errors: - line 2: field typex not found in type leakybucket.BucketFactory -FATA[06-08-2020 13:36:05] Failed to load scenarios: Scenario loading failed : bad yaml in /etc/crowdsec/config/scenarios/ssh-bf.yaml : yaml: unmarshal errors: - line 2: field typex not found in type leakybucket.BucketFactory -``` - -Using this, you won't have to kill your running service before you know the scenarios/parsers are at least syntactically correct. - - -## Using debug - -Both scenarios and parsers support a `debug: true|false` option which produce useful debug. - -
- Debug parsing output (expand) -```bash -DEBU[05-08-2020 15:25:36] eval(evt.Parsed.program == 'nginx') = TRUE id=lively-smoke name=crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] eval variables: id=lively-smoke name=crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] evt.Parsed.program = 'nginx' id=lively-smoke name=crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] Event entering node id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] + Grok 'NGINXACCESS' returned 10 entries to merge in Parsed id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] .Parsed['request'] = '/data.php' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] .Parsed['http_user_agent'] = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] .Parsed['http_referer'] = '-' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] .Parsed['remote_addr'] = '123.x.x.x' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] .Parsed['remote_user'] = '-' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] .Parsed['time_local'] = '01/Jan/2019:01:39:06 +0100' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] .Parsed['method'] = 'POST' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] .Parsed['body_bytes_sent'] = '162' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] .Parsed['http_version'] = '1.1' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] .Parsed['status'] = '404' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] .Meta[log_type] = 'http_access-log' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] evt.StrTime = '01/Jan/2019:01:39:06 +0100' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] Event leaving node : ok id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] child is success, OnSuccess=next_stage, skip id=lively-smoke name=crowdsecurity/nginx-logs stage=s01-parse -``` -
- - -
- Debug scenario output (expand) -```bash -DEBU[05-08-2020 16:02:26] eval(evt.Meta.service == 'http' && evt.Meta.http_status in ['404', '403', '400'] && evt.Parsed.static_ressource == 'false') = TRUE cfg=black-wave file=config/scenarios/http-probing.yaml name=crowdsecurity/http-probing -DEBU[05-08-2020 16:02:26] eval variables: cfg=black-wave file=config/scenarios/http-probing.yaml name=crowdsecurity/http-probing -DEBU[05-08-2020 16:02:26] evt.Meta.service = 'http' cfg=black-wave file=config/scenarios/http-probing.yaml name=crowdsecurity/http-probing -DEBU[05-08-2020 16:02:26] evt.Meta.http_status = '404' cfg=black-wave file=config/scenarios/http-probing.yaml name=crowdsecurity/http-probing -DEBU[05-08-2020 16:02:26] evt.Parsed.static_ressource = 'false' cfg=black-wave file=config/scenarios/http-probing.yaml name=crowdsecurity/http-probing -``` -
- - -# Test environments - -From a [{{v0X.crowdsec.name}} release archive]({{v0X.crowdsec.download_url}}), you can deploy a test (non-root) environment that is very suitable to write/debug/test parsers and scenarios. Environment is deployed using `./test_env.sh` script from tgz directory, and creates a test environment in `./tests` : - -```bash -$ cd crowdsec-v0.3.0/ -$ ./test_env.sh -... -[08/05/2020:04:19:18 PM][INFO] Setting up configurations -INFO[0000] Wrote new 75065 bytes index to config/crowdsec-cli/.index.json -INFO[0000] crowdsecurity/syslog-logs : OK -INFO[0000] crowdsecurity/geoip-enrich : OK -... -INFO[0007] Enabled collections : crowdsecurity/linux -INFO[0007] Enabled crowdsecurity/linux -[08/05/2020:04:19:26 PM][INFO] Environment is ready in /home/bui/github/crowdsec/crowdsec/crowdsec-v0.3.0/tests -$ cd tests -$ ./cscli -c dev.yaml list -... -INFO[0000] PARSERS: -------------------------------------------------------------------------------------------------------- - NAME 📦 STATUS VERSION LOCAL PATH -------------------------------------------------------------------------------------------------------- - crowdsecurity/geoip-enrich ✔️ enabled 0.2 config/parsers/s02-enrich/geoip-enrich.yaml - crowdsecurity/syslog-logs ✔️ enabled 0.3 config/parsers/s00-raw/syslog-logs.yaml - crowdsecurity/sshd-logs ✔️ enabled 0.2 config/parsers/s01-parse/sshd-logs.yaml - crowdsecurity/dateparse-enrich ✔️ enabled 0.1 config/parsers/s02-enrich/dateparse-enrich.yaml -------------------------------------------------------------------------------------------------------- -... -$ ./crowdsec -c dev.yaml -file sshd.log -type syslog -INFO[05-08-2020 16:23:32] Crowdsec v0.3.0-rc3-7525f11975a0107746213862dc41c69e00122ac7 -INFO[05-08-2020 16:23:32] Loading grok library -... -``` - - diff --git a/docs/v0.3.X/docs/cscli/cscli.md b/docs/v0.3.X/docs/cscli/cscli.md deleted file mode 100644 index 02c1f51f3..000000000 --- a/docs/v0.3.X/docs/cscli/cscli.md +++ /dev/null @@ -1,58 +0,0 @@ -## cscli - -cscli allows you to manage crowdsec - -### Synopsis - -cscli is the main command to interact with your crowdsec service, scenarios & db. -It is meant to allow you to manage bans, parsers/scenarios/etc, api and generally manage you crowdsec setup. - -### Examples - -``` -View/Add/Remove bans: - - cscli ban list - - cscli ban add ip 1.2.3.4 24h 'go away' - - cscli ban del 1.2.3.4 - -View/Add/Upgrade/Remove scenarios and parsers: - - cscli list - - cscli install collection crowdsec/linux-web - - cscli remove scenario crowdsec/ssh_enum - - cscli upgrade --all - -API interaction: - - cscli api pull - - cscli api register - -``` - -### Options - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - -o, --output string Output format : human, json, raw. (default "human") - --debug Set logging to debug. - --info Set logging to info. - --warning Set logging to warning. - --error Set logging to error. - -h, --help help for cscli -``` - -### SEE ALSO - -* [cscli api](cscli_api.md) - Crowdsec API interaction -* [cscli backup](cscli_backup.md) - Backup or restore configuration (api, parsers, scenarios etc.) to/from directory -* [cscli ban](cscli_ban.md) - Manage bans/mitigations -* [cscli config](cscli_config.md) - Allows to view/edit cscli config -* [cscli dashboard](cscli_dashboard.md) - Start a dashboard (metabase) container. -* [cscli inspect](cscli_inspect.md) - Inspect configuration(s) -* [cscli install](cscli_install.md) - Install configuration(s) from hub -* [cscli list](cscli_list.md) - List enabled configs -* [cscli metrics](cscli_metrics.md) - Display crowdsec prometheus metrics. -* [cscli remove](cscli_remove.md) - Remove/disable configuration(s) -* [cscli simulation](cscli_simulation.md) - -* [cscli update](cscli_update.md) - Fetch available configs from hub -* [cscli upgrade](cscli_upgrade.md) - Upgrade configuration(s) - - diff --git a/docs/v0.3.X/docs/cscli/cscli_api.md b/docs/v0.3.X/docs/cscli/cscli_api.md deleted file mode 100644 index e8b0843ed..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_api.md +++ /dev/null @@ -1,49 +0,0 @@ -## cscli api - -Crowdsec API interaction - -### Synopsis - - -Allow to register your machine into crowdsec API to send and receive signal. - - -### Examples - -``` - -cscli api register # Register to Crowdsec API -cscli api pull # Pull malevolant IPs from Crowdsec API -cscli api reset # Reset your machines credentials -cscli api enroll # Enroll your machine to the user account you created on Crowdsec backend -cscli api credentials # Display your API credentials - -``` - -### Options - -``` - -h, --help help for api -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli api credentials](cscli_api_credentials.md) - Display api credentials -* [cscli api enroll](cscli_api_enroll.md) - Associate your machine to an existing crowdsec user -* [cscli api pull](cscli_api_pull.md) - Pull crowdsec API TopX -* [cscli api register](cscli_api_register.md) - Register on Crowdsec API -* [cscli api reset](cscli_api_reset.md) - Reset password on CrowdSec API - - diff --git a/docs/v0.3.X/docs/cscli/cscli_api_credentials.md b/docs/v0.3.X/docs/cscli/cscli_api_credentials.md deleted file mode 100644 index f628c8a54..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_api_credentials.md +++ /dev/null @@ -1,40 +0,0 @@ -## cscli api credentials - -Display api credentials - -### Synopsis - -Display api credentials - -``` -cscli api credentials [flags] -``` - -### Examples - -``` -cscli api credentials -``` - -### Options - -``` - -h, --help help for credentials -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli api](cscli_api.md) - Crowdsec API interaction - - diff --git a/docs/v0.3.X/docs/cscli/cscli_api_enroll.md b/docs/v0.3.X/docs/cscli/cscli_api_enroll.md deleted file mode 100644 index e2c2588f5..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_api_enroll.md +++ /dev/null @@ -1,41 +0,0 @@ -## cscli api enroll - -Associate your machine to an existing crowdsec user - -### Synopsis - -Enrolling your machine into your user account will allow for more accurate lists and threat detection. See website to create user account. - -``` -cscli api enroll [flags] -``` - -### Examples - -``` -cscli api enroll -u 1234567890ffff -``` - -### Options - -``` - -h, --help help for enroll - -u, --user string User ID (required) -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli api](cscli_api.md) - Crowdsec API interaction - - diff --git a/docs/v0.3.X/docs/cscli/cscli_api_pull.md b/docs/v0.3.X/docs/cscli/cscli_api_pull.md deleted file mode 100644 index 528e48f5a..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_api_pull.md +++ /dev/null @@ -1,40 +0,0 @@ -## cscli api pull - -Pull crowdsec API TopX - -### Synopsis - -Pulls a list of malveolent IPs relevant to your situation and add them into the local ban database. - -``` -cscli api pull [flags] -``` - -### Examples - -``` -cscli api pull -``` - -### Options - -``` - -h, --help help for pull -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli api](cscli_api.md) - Crowdsec API interaction - - diff --git a/docs/v0.3.X/docs/cscli/cscli_api_register.md b/docs/v0.3.X/docs/cscli/cscli_api_register.md deleted file mode 100644 index 4cf61776e..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_api_register.md +++ /dev/null @@ -1,41 +0,0 @@ -## cscli api register - -Register on Crowdsec API - -### Synopsis - -This command will register your machine to crowdsec API to allow you to receive list of malveolent IPs. - The printed machine_id and password should be added to your api.yaml file. - -``` -cscli api register [flags] -``` - -### Examples - -``` -cscli api register -``` - -### Options - -``` - -h, --help help for register -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli api](cscli_api.md) - Crowdsec API interaction - - diff --git a/docs/v0.3.X/docs/cscli/cscli_api_reset.md b/docs/v0.3.X/docs/cscli/cscli_api_reset.md deleted file mode 100644 index a85e2cd7f..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_api_reset.md +++ /dev/null @@ -1,40 +0,0 @@ -## cscli api reset - -Reset password on CrowdSec API - -### Synopsis - -Attempts to reset your credentials to the API. - -``` -cscli api reset [flags] -``` - -### Examples - -``` -cscli api reset -``` - -### Options - -``` - -h, --help help for reset -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli api](cscli_api.md) - Crowdsec API interaction - - diff --git a/docs/v0.3.X/docs/cscli/cscli_backup.md b/docs/v0.3.X/docs/cscli/cscli_backup.md deleted file mode 100644 index e7a442340..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_backup.md +++ /dev/null @@ -1,39 +0,0 @@ -## cscli backup - -Backup or restore configuration (api, parsers, scenarios etc.) to/from directory - -### Synopsis - -This command is here to help you save and/or restore crowdsec configurations to simple replication - -### Examples - -``` -cscli backup save ./my-backup -cscli backup restore ./my-backup -``` - -### Options - -``` - -h, --help help for backup -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli backup restore](cscli_backup_restore.md) - Restore configuration (api, parsers, scenarios etc.) from directory -* [cscli backup save](cscli_backup_save.md) - Backup configuration (api, parsers, scenarios etc.) to directory - - diff --git a/docs/v0.3.X/docs/cscli/cscli_backup_restore.md b/docs/v0.3.X/docs/cscli/cscli_backup_restore.md deleted file mode 100644 index f219eb65f..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_backup_restore.md +++ /dev/null @@ -1,49 +0,0 @@ -## cscli backup restore - -Restore configuration (api, parsers, scenarios etc.) from directory - -### Synopsis - -restore command will try to restore all saved information from to yor local setup, including : - -- Installation of up-to-date scenarios/parsers/... via cscli - -- Restauration of tainted/local/out-of-date scenarios/parsers/... file - -- Restauration of API credentials (if the existing ones aren't working) - -- Restauration of acqusition configuration - - -``` -cscli backup restore [flags] -``` - -### Examples - -``` -cscli backup restore ./my-backup -``` - -### Options - -``` - -h, --help help for restore -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli backup](cscli_backup.md) - Backup or restore configuration (api, parsers, scenarios etc.) to/from directory - - diff --git a/docs/v0.3.X/docs/cscli/cscli_backup_save.md b/docs/v0.3.X/docs/cscli/cscli_backup_save.md deleted file mode 100644 index 37eaf8ce1..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_backup_save.md +++ /dev/null @@ -1,50 +0,0 @@ -## cscli backup save - -Backup configuration (api, parsers, scenarios etc.) to directory - -### Synopsis - -backup command will try to save all relevant informations to crowdsec config, including : - -- List of scenarios, parsers, postoverflows and collections that are up-to-date - -- Actual backup of tainted/local/out-of-date scenarios, parsers, postoverflows and collections - -- Backup of API credentials - -- Backup of acquisition configuration - - - -``` -cscli backup save [flags] -``` - -### Examples - -``` -cscli backup save ./my-backup -``` - -### Options - -``` - -h, --help help for save -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli backup](cscli_backup.md) - Backup or restore configuration (api, parsers, scenarios etc.) to/from directory - - diff --git a/docs/v0.3.X/docs/cscli/cscli_ban.md b/docs/v0.3.X/docs/cscli/cscli_ban.md deleted file mode 100644 index 70ee7e4c6..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_ban.md +++ /dev/null @@ -1,37 +0,0 @@ -## cscli ban - -Manage bans/mitigations - -### Synopsis - -This is the main interaction point with local ban database for humans. - -You can add/delete/list or flush current bans in your local ban DB. - -### Options - -``` - --remediation string Set specific remediation type : ban|slow|captcha (default "ban") - -h, --help help for ban -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli ban add](cscli_ban_add.md) - Adds a ban against a given ip/range for the provided duration -* [cscli ban del](cscli_ban_del.md) - Delete bans from db -* [cscli ban flush](cscli_ban_flush.md) - Fush ban DB -* [cscli ban list](cscli_ban_list.md) - List local or api bans/remediations - - diff --git a/docs/v0.3.X/docs/cscli/cscli_ban_add.md b/docs/v0.3.X/docs/cscli/cscli_ban_add.md deleted file mode 100644 index 7391fddba..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_ban_add.md +++ /dev/null @@ -1,45 +0,0 @@ -## cscli ban add - -Adds a ban against a given ip/range for the provided duration - -### Synopsis - - -Allows to add a ban against a specific ip or range target for a specific duration. - -The duration argument can be expressed in seconds(s), minutes(m) or hours (h). - -See [time.ParseDuration](https://golang.org/pkg/time/#ParseDuration) for more informations. - -### Examples - -``` -cscli ban add ip 1.2.3.4 24h "scan" -cscli ban add range 1.2.3.0/24 24h "the whole range" -``` - -### Options - -``` - -h, --help help for add -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --remediation string Set specific remediation type : ban|slow|captcha (default "ban") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli ban](cscli_ban.md) - Manage bans/mitigations -* [cscli ban add ip](cscli_ban_add_ip.md) - Adds the specific ip to the ban db -* [cscli ban add range](cscli_ban_add_range.md) - Adds the specific ip to the ban db - - diff --git a/docs/v0.3.X/docs/cscli/cscli_ban_add_ip.md b/docs/v0.3.X/docs/cscli/cscli_ban_add_ip.md deleted file mode 100644 index 3e312f133..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_ban_add_ip.md +++ /dev/null @@ -1,41 +0,0 @@ -## cscli ban add ip - -Adds the specific ip to the ban db - -### Synopsis - -Duration must be [time.ParseDuration](https://golang.org/pkg/time/#ParseDuration), expressed in s/m/h. - -``` -cscli ban add ip [flags] -``` - -### Examples - -``` -cscli ban add ip 1.2.3.4 12h "the scan" -``` - -### Options - -``` - -h, --help help for ip -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --remediation string Set specific remediation type : ban|slow|captcha (default "ban") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli ban add](cscli_ban_add.md) - Adds a ban against a given ip/range for the provided duration - - diff --git a/docs/v0.3.X/docs/cscli/cscli_ban_add_range.md b/docs/v0.3.X/docs/cscli/cscli_ban_add_range.md deleted file mode 100644 index a362535b3..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_ban_add_range.md +++ /dev/null @@ -1,41 +0,0 @@ -## cscli ban add range - -Adds the specific ip to the ban db - -### Synopsis - -Duration must be [time.ParseDuration](https://golang.org/pkg/time/#ParseDuration) compatible, expressed in s/m/h. - -``` -cscli ban add range [flags] -``` - -### Examples - -``` -cscli ban add range 1.2.3.0/24 12h "the whole range" -``` - -### Options - -``` - -h, --help help for range -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --remediation string Set specific remediation type : ban|slow|captcha (default "ban") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli ban add](cscli_ban_add.md) - Adds a ban against a given ip/range for the provided duration - - diff --git a/docs/v0.3.X/docs/cscli/cscli_ban_del.md b/docs/v0.3.X/docs/cscli/cscli_ban_del.md deleted file mode 100644 index 2dc026e50..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_ban_del.md +++ /dev/null @@ -1,40 +0,0 @@ -## cscli ban del - -Delete bans from db - -### Synopsis - -The removal of the bans can be applied on a single IP address or directly on a IP range. - -### Examples - -``` -cscli ban del ip 1.2.3.4 -cscli ban del range 1.2.3.0/24 -``` - -### Options - -``` - -h, --help help for del -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --remediation string Set specific remediation type : ban|slow|captcha (default "ban") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli ban](cscli_ban.md) - Manage bans/mitigations -* [cscli ban del ip](cscli_ban_del_ip.md) - Delete bans for given ip from db -* [cscli ban del range](cscli_ban_del_range.md) - Delete bans for given ip from db - - diff --git a/docs/v0.3.X/docs/cscli/cscli_ban_del_ip.md b/docs/v0.3.X/docs/cscli/cscli_ban_del_ip.md deleted file mode 100644 index d059e9125..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_ban_del_ip.md +++ /dev/null @@ -1,41 +0,0 @@ -## cscli ban del ip - -Delete bans for given ip from db - -### Synopsis - -Delete bans for given ip from db - -``` -cscli ban del ip [flags] -``` - -### Examples - -``` -cscli ban del ip 1.2.3.4 -``` - -### Options - -``` - -h, --help help for ip -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --remediation string Set specific remediation type : ban|slow|captcha (default "ban") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli ban del](cscli_ban_del.md) - Delete bans from db - - diff --git a/docs/v0.3.X/docs/cscli/cscli_ban_del_range.md b/docs/v0.3.X/docs/cscli/cscli_ban_del_range.md deleted file mode 100644 index 53debb369..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_ban_del_range.md +++ /dev/null @@ -1,41 +0,0 @@ -## cscli ban del range - -Delete bans for given ip from db - -### Synopsis - -Delete bans for given ip from db - -``` -cscli ban del range [flags] -``` - -### Examples - -``` -cscli ban del range 1.2.3.0/24 -``` - -### Options - -``` - -h, --help help for range -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --remediation string Set specific remediation type : ban|slow|captcha (default "ban") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli ban del](cscli_ban_del.md) - Delete bans from db - - diff --git a/docs/v0.3.X/docs/cscli/cscli_ban_flush.md b/docs/v0.3.X/docs/cscli/cscli_ban_flush.md deleted file mode 100644 index e0a2ba732..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_ban_flush.md +++ /dev/null @@ -1,41 +0,0 @@ -## cscli ban flush - -Fush ban DB - -### Synopsis - -Fush ban DB - -``` -cscli ban flush [flags] -``` - -### Examples - -``` -cscli ban flush -``` - -### Options - -``` - -h, --help help for flush -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --remediation string Set specific remediation type : ban|slow|captcha (default "ban") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli ban](cscli_ban.md) - Manage bans/mitigations - - diff --git a/docs/v0.3.X/docs/cscli/cscli_ban_list.md b/docs/v0.3.X/docs/cscli/cscli_ban_list.md deleted file mode 100644 index 8ced4a8b6..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_ban_list.md +++ /dev/null @@ -1,66 +0,0 @@ -## cscli ban list - -List local or api bans/remediations - -### Synopsis - -List the bans, by default only local decisions. - -If --all/-a is specified, bans will be displayed without limit (--limit). -Default limit is 50. - -Time can be specified with --at and support a variety of date formats: - - Jan 2 15:04:05 - - Mon Jan 02 15:04:05.000000 2006 - - 2006-01-02T15:04:05Z07:00 - - 2006/01/02 - - 2006/01/02 15:04 - - 2006-01-02 - - 2006-01-02 15:04 - - -``` -cscli ban list [flags] -``` - -### Examples - -``` -ban list --range 0.0.0.0/0 : will list all - ban list --country CN - ban list --reason crowdsecurity/http-probing - ban list --as OVH -``` - -### Options - -``` - -a, --all List bans without limit - --api List as well bans received from API - --as string List bans belonging to given AS name - --at string List bans at given time - --country string List bans belonging to given country code - -h, --help help for list - --ip string List bans for given IP - --limit int Limit of bans to display (default 50) (default 50) - --range string List bans belonging to given range - --reason string List bans containing given reason -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --remediation string Set specific remediation type : ban|slow|captcha (default "ban") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli ban](cscli_ban.md) - Manage bans/mitigations - - diff --git a/docs/v0.3.X/docs/cscli/cscli_config.md b/docs/v0.3.X/docs/cscli/cscli_config.md deleted file mode 100644 index 4a470c96a..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_config.md +++ /dev/null @@ -1,39 +0,0 @@ -## cscli config - -Allows to view/edit cscli config - -### Synopsis - -Allow to configure database plugin path and installation directory. -If no commands are specified, config is in interactive mode. - -### Examples - -``` - - cscli config show -- cscli config prompt -``` - -### Options - -``` - -h, --help help for config -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli config show](cscli_config_show.md) - Displays current config - - diff --git a/docs/v0.3.X/docs/cscli/cscli_config_show.md b/docs/v0.3.X/docs/cscli/cscli_config_show.md deleted file mode 100644 index 16aadd3e0..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_config_show.md +++ /dev/null @@ -1,34 +0,0 @@ -## cscli config show - -Displays current config - -### Synopsis - -Displays the current cli configuration. - -``` -cscli config show [flags] -``` - -### Options - -``` - -h, --help help for show -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli config](cscli_config.md) - Allows to view/edit cscli config - - diff --git a/docs/v0.3.X/docs/cscli/cscli_dashboard.md b/docs/v0.3.X/docs/cscli/cscli_dashboard.md deleted file mode 100644 index f8fccb167..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_dashboard.md +++ /dev/null @@ -1,42 +0,0 @@ -## cscli dashboard - -Start a dashboard (metabase) container. - -### Synopsis - -Start a metabase container exposing dashboards and metrics. - -### Examples - -``` -cscli dashboard setup -cscli dashboard start -cscli dashboard stop -cscli dashboard setup --force -``` - -### Options - -``` - -h, --help help for dashboard -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli dashboard setup](cscli_dashboard_setup.md) - Setup a metabase container. -* [cscli dashboard start](cscli_dashboard_start.md) - Start the metabase container. -* [cscli dashboard stop](cscli_dashboard_stop.md) - Stops the metabase container. - - diff --git a/docs/v0.3.X/docs/cscli/cscli_dashboard_setup.md b/docs/v0.3.X/docs/cscli/cscli_dashboard_setup.md deleted file mode 100644 index 54f377729..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_dashboard_setup.md +++ /dev/null @@ -1,47 +0,0 @@ -## cscli dashboard setup - -Setup a metabase container. - -### Synopsis - -Perform a metabase docker setup, download standard dashboards, create a fresh user and start the container - -``` -cscli dashboard setup [flags] -``` - -### Examples - -``` -cscli dashboard setup -cscli dashboard setup --force -cscli dashboard setup -l 0.0.0.0 -p 443 - -``` - -### Options - -``` - -d, --dir string Shared directory with metabase container. (default "/var/lib/crowdsec/data") - -f, --force Force setup : override existing files. - -h, --help help for setup - -l, --listen string Listen address of container (default "127.0.0.1") - -p, --port string Listen port of container (default "3000") -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli dashboard](cscli_dashboard.md) - Start a dashboard (metabase) container. - - diff --git a/docs/v0.3.X/docs/cscli/cscli_dashboard_start.md b/docs/v0.3.X/docs/cscli/cscli_dashboard_start.md deleted file mode 100644 index f34167b0c..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_dashboard_start.md +++ /dev/null @@ -1,34 +0,0 @@ -## cscli dashboard start - -Start the metabase container. - -### Synopsis - -Stats the metabase container using docker. - -``` -cscli dashboard start [flags] -``` - -### Options - -``` - -h, --help help for start -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli dashboard](cscli_dashboard.md) - Start a dashboard (metabase) container. - - diff --git a/docs/v0.3.X/docs/cscli/cscli_dashboard_stop.md b/docs/v0.3.X/docs/cscli/cscli_dashboard_stop.md deleted file mode 100644 index efd5c9acc..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_dashboard_stop.md +++ /dev/null @@ -1,35 +0,0 @@ -## cscli dashboard stop - -Stops the metabase container. - -### Synopsis - -Stops the metabase container using docker. - -``` -cscli dashboard stop [flags] -``` - -### Options - -``` - -h, --help help for stop - -r, --remove remove (docker rm) container as well. -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli dashboard](cscli_dashboard.md) - Start a dashboard (metabase) container. - - diff --git a/docs/v0.3.X/docs/cscli/cscli_inspect.md b/docs/v0.3.X/docs/cscli/cscli_inspect.md deleted file mode 100644 index 8719796a1..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_inspect.md +++ /dev/null @@ -1,47 +0,0 @@ -## cscli inspect - -Inspect configuration(s) - -### Synopsis - - -Inspect give you full detail about local installed configuration. - -[type] must be parser, scenario, postoverflow, collection. - -[config_name] must be a valid config name from [Crowdsec Hub](https://hub.crowdsec.net) or locally installed. - - -### Examples - -``` -cscli inspect parser crowdsec/xxx -cscli inspect collection crowdsec/xxx -``` - -### Options - -``` - -h, --help help for inspect -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli inspect collection](cscli_inspect_collection.md) - Inspect given collection -* [cscli inspect parser](cscli_inspect_parser.md) - Inspect given log parser -* [cscli inspect postoverflow](cscli_inspect_postoverflow.md) - Inspect given postoverflow parser -* [cscli inspect scenario](cscli_inspect_scenario.md) - Inspect given scenario - - diff --git a/docs/v0.3.X/docs/cscli/cscli_inspect_collection.md b/docs/v0.3.X/docs/cscli/cscli_inspect_collection.md deleted file mode 100644 index 3b347c9e8..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_inspect_collection.md +++ /dev/null @@ -1,40 +0,0 @@ -## cscli inspect collection - -Inspect given collection - -### Synopsis - -Inspect given collection from hub - -``` -cscli inspect collection [config] [flags] -``` - -### Examples - -``` -cscli inspect collection crowdsec/xxx -``` - -### Options - -``` - -h, --help help for collection -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli inspect](cscli_inspect.md) - Inspect configuration(s) - - diff --git a/docs/v0.3.X/docs/cscli/cscli_inspect_parser.md b/docs/v0.3.X/docs/cscli/cscli_inspect_parser.md deleted file mode 100644 index 5cdd7a479..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_inspect_parser.md +++ /dev/null @@ -1,40 +0,0 @@ -## cscli inspect parser - -Inspect given log parser - -### Synopsis - -Inspect given parser from hub - -``` -cscli inspect parser [config] [flags] -``` - -### Examples - -``` -cscli inspect parser crowdsec/xxx -``` - -### Options - -``` - -h, --help help for parser -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli inspect](cscli_inspect.md) - Inspect configuration(s) - - diff --git a/docs/v0.3.X/docs/cscli/cscli_inspect_postoverflow.md b/docs/v0.3.X/docs/cscli/cscli_inspect_postoverflow.md deleted file mode 100644 index 63c2d632d..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_inspect_postoverflow.md +++ /dev/null @@ -1,40 +0,0 @@ -## cscli inspect postoverflow - -Inspect given postoverflow parser - -### Synopsis - -Inspect given postoverflow from hub. - -``` -cscli inspect postoverflow [config] [flags] -``` - -### Examples - -``` -cscli inspect postoverflow crowdsec/xxx -``` - -### Options - -``` - -h, --help help for postoverflow -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli inspect](cscli_inspect.md) - Inspect configuration(s) - - diff --git a/docs/v0.3.X/docs/cscli/cscli_inspect_scenario.md b/docs/v0.3.X/docs/cscli/cscli_inspect_scenario.md deleted file mode 100644 index b09a309fa..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_inspect_scenario.md +++ /dev/null @@ -1,40 +0,0 @@ -## cscli inspect scenario - -Inspect given scenario - -### Synopsis - -Inspect given scenario from hub - -``` -cscli inspect scenario [config] [flags] -``` - -### Examples - -``` -cscli inspect scenario crowdsec/xxx -``` - -### Options - -``` - -h, --help help for scenario -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli inspect](cscli_inspect.md) - Inspect configuration(s) - - diff --git a/docs/v0.3.X/docs/cscli/cscli_install.md b/docs/v0.3.X/docs/cscli/cscli_install.md deleted file mode 100644 index 6ef7bb1c9..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_install.md +++ /dev/null @@ -1,51 +0,0 @@ -## cscli install - -Install configuration(s) from hub - -### Synopsis - - -Install configuration from the CrowdSec Hub. - -In order to download latest versions of configuration, -you should [update cscli](./cscli_update.md). - -[type] must be parser, scenario, postoverflow, collection. - -[config_name] must be a valid config name from [Crowdsec Hub](https://hub.crowdsec.net). - - -### Examples - -``` -cscli install [type] [config_name] -``` - -### Options - -``` - -d, --download-only Only download packages, don't enable - --force Force install : Overwrite tainted and outdated files - -h, --help help for install -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli install collection](cscli_install_collection.md) - Install given collection -* [cscli install parser](cscli_install_parser.md) - Install given parser -* [cscli install postoverflow](cscli_install_postoverflow.md) - Install given postoverflow parser -* [cscli install scenario](cscli_install_scenario.md) - Install given scenario - - diff --git a/docs/v0.3.X/docs/cscli/cscli_install_collection.md b/docs/v0.3.X/docs/cscli/cscli_install_collection.md deleted file mode 100644 index fc1fe25b0..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_install_collection.md +++ /dev/null @@ -1,42 +0,0 @@ -## cscli install collection - -Install given collection - -### Synopsis - -Fetch and install given collection from hub - -``` -cscli install collection [config] [flags] -``` - -### Examples - -``` -cscli install collection crowdsec/xxx -``` - -### Options - -``` - -h, --help help for collection -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - -d, --download-only Only download packages, don't enable - --error Set logging to error. - --force Force install : Overwrite tainted and outdated files - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli install](cscli_install.md) - Install configuration(s) from hub - - diff --git a/docs/v0.3.X/docs/cscli/cscli_install_parser.md b/docs/v0.3.X/docs/cscli/cscli_install_parser.md deleted file mode 100644 index 861a1d2eb..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_install_parser.md +++ /dev/null @@ -1,42 +0,0 @@ -## cscli install parser - -Install given parser - -### Synopsis - -Fetch and install given parser from hub - -``` -cscli install parser [config] [flags] -``` - -### Examples - -``` -cscli install parser crowdsec/xxx -``` - -### Options - -``` - -h, --help help for parser -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - -d, --download-only Only download packages, don't enable - --error Set logging to error. - --force Force install : Overwrite tainted and outdated files - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli install](cscli_install.md) - Install configuration(s) from hub - - diff --git a/docs/v0.3.X/docs/cscli/cscli_install_postoverflow.md b/docs/v0.3.X/docs/cscli/cscli_install_postoverflow.md deleted file mode 100644 index 80fbfcc85..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_install_postoverflow.md +++ /dev/null @@ -1,43 +0,0 @@ -## cscli install postoverflow - -Install given postoverflow parser - -### Synopsis - -Fetch and install given postoverflow from hub. -As a reminder, postoverflows are parsing configuration that will occur after the overflow (before a decision is applied). - -``` -cscli install postoverflow [config] [flags] -``` - -### Examples - -``` -cscli install collection crowdsec/xxx -``` - -### Options - -``` - -h, --help help for postoverflow -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - -d, --download-only Only download packages, don't enable - --error Set logging to error. - --force Force install : Overwrite tainted and outdated files - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli install](cscli_install.md) - Install configuration(s) from hub - - diff --git a/docs/v0.3.X/docs/cscli/cscli_install_scenario.md b/docs/v0.3.X/docs/cscli/cscli_install_scenario.md deleted file mode 100644 index 5d50c8a36..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_install_scenario.md +++ /dev/null @@ -1,42 +0,0 @@ -## cscli install scenario - -Install given scenario - -### Synopsis - -Fetch and install given scenario from hub - -``` -cscli install scenario [config] [flags] -``` - -### Examples - -``` -cscli install scenario crowdsec/xxx -``` - -### Options - -``` - -h, --help help for scenario -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - -d, --download-only Only download packages, don't enable - --error Set logging to error. - --force Force install : Overwrite tainted and outdated files - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli install](cscli_install.md) - Install configuration(s) from hub - - diff --git a/docs/v0.3.X/docs/cscli/cscli_list.md b/docs/v0.3.X/docs/cscli/cscli_list.md deleted file mode 100644 index 015c4c654..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_list.md +++ /dev/null @@ -1,54 +0,0 @@ -## cscli list - -List enabled configs - -### Synopsis - - -List enabled configurations (parser/scenarios/collections) on your host. - -It is possible to list also configuration from [Crowdsec Hub](https://hub.crowdsec.net) with the '-a' options. - -[type] must be parsers, scenarios, postoverflows, collections - - -``` -cscli list [-a] [flags] -``` - -### Examples - -``` -cscli list # List all local configurations -cscli list [type] # List all local configuration of type [type] -cscli list -a # List all local and remote configurations - -``` - -### Options - -``` - -a, --all List as well disabled items - -h, --help help for list -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli list collections](cscli_list_collections.md) - List enabled collections -* [cscli list parsers](cscli_list_parsers.md) - List enabled parsers -* [cscli list postoverflows](cscli_list_postoverflows.md) - List enabled postoverflow parsers -* [cscli list scenarios](cscli_list_scenarios.md) - List enabled scenarios - - diff --git a/docs/v0.3.X/docs/cscli/cscli_list_collections.md b/docs/v0.3.X/docs/cscli/cscli_list_collections.md deleted file mode 100644 index b7da2236f..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_list_collections.md +++ /dev/null @@ -1,35 +0,0 @@ -## cscli list collections - -List enabled collections - -### Synopsis - -List enabled collections - -``` -cscli list collections [-a] [flags] -``` - -### Options - -``` - -h, --help help for collections -``` - -### Options inherited from parent commands - -``` - -a, --all List as well disabled items - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli list](cscli_list.md) - List enabled configs - - diff --git a/docs/v0.3.X/docs/cscli/cscli_list_parsers.md b/docs/v0.3.X/docs/cscli/cscli_list_parsers.md deleted file mode 100644 index 89b808293..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_list_parsers.md +++ /dev/null @@ -1,35 +0,0 @@ -## cscli list parsers - -List enabled parsers - -### Synopsis - -List enabled parsers - -``` -cscli list parsers [-a] [flags] -``` - -### Options - -``` - -h, --help help for parsers -``` - -### Options inherited from parent commands - -``` - -a, --all List as well disabled items - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli list](cscli_list.md) - List enabled configs - - diff --git a/docs/v0.3.X/docs/cscli/cscli_list_postoverflows.md b/docs/v0.3.X/docs/cscli/cscli_list_postoverflows.md deleted file mode 100644 index 39cdf5a02..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_list_postoverflows.md +++ /dev/null @@ -1,35 +0,0 @@ -## cscli list postoverflows - -List enabled postoverflow parsers - -### Synopsis - -List enabled postoverflow parsers - -``` -cscli list postoverflows [-a] [flags] -``` - -### Options - -``` - -h, --help help for postoverflows -``` - -### Options inherited from parent commands - -``` - -a, --all List as well disabled items - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli list](cscli_list.md) - List enabled configs - - diff --git a/docs/v0.3.X/docs/cscli/cscli_list_scenarios.md b/docs/v0.3.X/docs/cscli/cscli_list_scenarios.md deleted file mode 100644 index c43e5f9b9..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_list_scenarios.md +++ /dev/null @@ -1,35 +0,0 @@ -## cscli list scenarios - -List enabled scenarios - -### Synopsis - -List enabled scenarios - -``` -cscli list scenarios [-a] [flags] -``` - -### Options - -``` - -h, --help help for scenarios -``` - -### Options inherited from parent commands - -``` - -a, --all List as well disabled items - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli list](cscli_list.md) - List enabled configs - - diff --git a/docs/v0.3.X/docs/cscli/cscli_metrics.md b/docs/v0.3.X/docs/cscli/cscli_metrics.md deleted file mode 100644 index bf43ed02d..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_metrics.md +++ /dev/null @@ -1,35 +0,0 @@ -## cscli metrics - -Display crowdsec prometheus metrics. - -### Synopsis - -Fetch metrics from the prometheus server and display them in a human-friendly way - -``` -cscli metrics [flags] -``` - -### Options - -``` - -h, --help help for metrics - -u, --url string Prometheus url (default "http://127.0.0.1:6060/metrics") -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec - - diff --git a/docs/v0.3.X/docs/cscli/cscli_remove.md b/docs/v0.3.X/docs/cscli/cscli_remove.md deleted file mode 100644 index c89ff2419..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_remove.md +++ /dev/null @@ -1,48 +0,0 @@ -## cscli remove - -Remove/disable configuration(s) - -### Synopsis - - - Remove local configuration. - -[type] must be parser, scenario, postoverflow, collection - -[config_name] must be a valid config name from [Crowdsec Hub](https://hub.crowdsec.net) or locally installed. - - -### Examples - -``` -cscli remove [type] [config_name] -``` - -### Options - -``` - --all Delete all the files in selected scope - -h, --help help for remove - --purge Delete source file in ~/.cscli/hub/ too -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli remove collection](cscli_remove_collection.md) - Remove/disable collection -* [cscli remove parser](cscli_remove_parser.md) - Remove/disable parser -* [cscli remove postoverflow](cscli_remove_postoverflow.md) - Remove/disable postoverflow parser -* [cscli remove scenario](cscli_remove_scenario.md) - Remove/disable scenario - - diff --git a/docs/v0.3.X/docs/cscli/cscli_remove_collection.md b/docs/v0.3.X/docs/cscli/cscli_remove_collection.md deleted file mode 100644 index 1e5d8ff5f..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_remove_collection.md +++ /dev/null @@ -1,36 +0,0 @@ -## cscli remove collection - -Remove/disable collection - -### Synopsis - - must be a valid collection. - -``` -cscli remove collection [config] [flags] -``` - -### Options - -``` - -h, --help help for collection -``` - -### Options inherited from parent commands - -``` - --all Delete all the files in selected scope - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --purge Delete source file in ~/.cscli/hub/ too - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli remove](cscli_remove.md) - Remove/disable configuration(s) - - diff --git a/docs/v0.3.X/docs/cscli/cscli_remove_parser.md b/docs/v0.3.X/docs/cscli/cscli_remove_parser.md deleted file mode 100644 index c037b0478..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_remove_parser.md +++ /dev/null @@ -1,36 +0,0 @@ -## cscli remove parser - -Remove/disable parser - -### Synopsis - - must be a valid parser. - -``` -cscli remove parser [flags] -``` - -### Options - -``` - -h, --help help for parser -``` - -### Options inherited from parent commands - -``` - --all Delete all the files in selected scope - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --purge Delete source file in ~/.cscli/hub/ too - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli remove](cscli_remove.md) - Remove/disable configuration(s) - - diff --git a/docs/v0.3.X/docs/cscli/cscli_remove_postoverflow.md b/docs/v0.3.X/docs/cscli/cscli_remove_postoverflow.md deleted file mode 100644 index fb02aae47..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_remove_postoverflow.md +++ /dev/null @@ -1,36 +0,0 @@ -## cscli remove postoverflow - -Remove/disable postoverflow parser - -### Synopsis - - must be a valid collection. - -``` -cscli remove postoverflow [config] [flags] -``` - -### Options - -``` - -h, --help help for postoverflow -``` - -### Options inherited from parent commands - -``` - --all Delete all the files in selected scope - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --purge Delete source file in ~/.cscli/hub/ too - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli remove](cscli_remove.md) - Remove/disable configuration(s) - - diff --git a/docs/v0.3.X/docs/cscli/cscli_remove_scenario.md b/docs/v0.3.X/docs/cscli/cscli_remove_scenario.md deleted file mode 100644 index 1eb641584..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_remove_scenario.md +++ /dev/null @@ -1,36 +0,0 @@ -## cscli remove scenario - -Remove/disable scenario - -### Synopsis - - must be a valid scenario. - -``` -cscli remove scenario [config] [flags] -``` - -### Options - -``` - -h, --help help for scenario -``` - -### Options inherited from parent commands - -``` - --all Delete all the files in selected scope - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --purge Delete source file in ~/.cscli/hub/ too - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli remove](cscli_remove.md) - Remove/disable configuration(s) - - diff --git a/docs/v0.3.X/docs/cscli/cscli_simulation.md b/docs/v0.3.X/docs/cscli/cscli_simulation.md deleted file mode 100644 index 17e02dbd1..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_simulation.md +++ /dev/null @@ -1,33 +0,0 @@ -## cscli simulation - - - -### Synopsis - - - -### Options - -``` - -h, --help help for simulation -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli simulation disable](cscli_simulation_disable.md) - Disable the simulation mode. Disable only specified scenarios -* [cscli simulation enable](cscli_simulation_enable.md) - Enable the simulation, globally or on specified scenarios -* [cscli simulation status](cscli_simulation_status.md) - Show simulation mode status - - diff --git a/docs/v0.3.X/docs/cscli/cscli_simulation_disable.md b/docs/v0.3.X/docs/cscli/cscli_simulation_disable.md deleted file mode 100644 index 462df2e59..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_simulation_disable.md +++ /dev/null @@ -1,40 +0,0 @@ -## cscli simulation disable - -Disable the simulation mode. Disable only specified scenarios - -### Synopsis - -Disable the simulation mode. Disable only specified scenarios - -``` -cscli simulation disable [scenario_name] [flags] -``` - -### Examples - -``` -cscli simulation disable -``` - -### Options - -``` - -h, --help help for disable -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli simulation](cscli_simulation.md) - - - diff --git a/docs/v0.3.X/docs/cscli/cscli_simulation_enable.md b/docs/v0.3.X/docs/cscli/cscli_simulation_enable.md deleted file mode 100644 index 0b0e560d4..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_simulation_enable.md +++ /dev/null @@ -1,40 +0,0 @@ -## cscli simulation enable - -Enable the simulation, globally or on specified scenarios - -### Synopsis - -Enable the simulation, globally or on specified scenarios - -``` -cscli simulation enable [scenario_name] [flags] -``` - -### Examples - -``` -cscli simulation enable -``` - -### Options - -``` - -h, --help help for enable -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli simulation](cscli_simulation.md) - - - diff --git a/docs/v0.3.X/docs/cscli/cscli_simulation_status.md b/docs/v0.3.X/docs/cscli/cscli_simulation_status.md deleted file mode 100644 index 237771ec4..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_simulation_status.md +++ /dev/null @@ -1,40 +0,0 @@ -## cscli simulation status - -Show simulation mode status - -### Synopsis - -Show simulation mode status - -``` -cscli simulation status [flags] -``` - -### Examples - -``` -cscli simulation status -``` - -### Options - -``` - -h, --help help for status -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli simulation](cscli_simulation.md) - - - diff --git a/docs/v0.3.X/docs/cscli/cscli_update.md b/docs/v0.3.X/docs/cscli/cscli_update.md deleted file mode 100644 index a9a893585..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_update.md +++ /dev/null @@ -1,36 +0,0 @@ -## cscli update - -Fetch available configs from hub - -### Synopsis - - -Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.index.json) file from hub, containing the list of available configs. - - -``` -cscli update [flags] -``` - -### Options - -``` - -h, --help help for update -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec - - diff --git a/docs/v0.3.X/docs/cscli/cscli_upgrade.md b/docs/v0.3.X/docs/cscli/cscli_upgrade.md deleted file mode 100644 index d874532ce..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_upgrade.md +++ /dev/null @@ -1,62 +0,0 @@ -## cscli upgrade - -Upgrade configuration(s) - -### Synopsis - - -Upgrade configuration from the CrowdSec Hub. - -In order to upgrade latest versions of configuration, -the Hub cache should be [updated](./cscli_update.md). - -Tainted configuration will not be updated (use --force to update them). - -[type] must be parser, scenario, postoverflow, collection. - -[config_name] must be a valid config name from [Crowdsec Hub](https://hub.crowdsec.net). - - - - -``` -cscli upgrade [type] [config] [flags] -``` - -### Examples - -``` -cscli upgrade [type] [config_name] -cscli upgrade --all # Upgrade all configurations types -cscli upgrade --force # Overwrite tainted configuration - -``` - -### Options - -``` - --all Upgrade all configuration in scope - --force Overwrite existing files, even if tainted - -h, --help help for upgrade -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli upgrade collection](cscli_upgrade_collection.md) - Upgrade collection configuration(s) -* [cscli upgrade parser](cscli_upgrade_parser.md) - Upgrade parser configuration(s) -* [cscli upgrade postoverflow](cscli_upgrade_postoverflow.md) - Upgrade postoverflow parser configuration(s) -* [cscli upgrade scenario](cscli_upgrade_scenario.md) - Upgrade scenario configuration(s) - - diff --git a/docs/v0.3.X/docs/cscli/cscli_upgrade_collection.md b/docs/v0.3.X/docs/cscli/cscli_upgrade_collection.md deleted file mode 100644 index 27ab014bd..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_upgrade_collection.md +++ /dev/null @@ -1,44 +0,0 @@ -## cscli upgrade collection - -Upgrade collection configuration(s) - -### Synopsis - -Upgrade one or more collection configurations - -``` -cscli upgrade collection [config] [flags] -``` - -### Examples - -``` - - cscli upgrade collection crowdsec/apache-lamp - - cscli upgrade collection -all - - cscli upgrade collection crowdsec/apache-lamp --force -``` - -### Options - -``` - -h, --help help for collection -``` - -### Options inherited from parent commands - -``` - --all Upgrade all configuration in scope - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --force Overwrite existing files, even if tainted - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli upgrade](cscli_upgrade.md) - Upgrade configuration(s) - - diff --git a/docs/v0.3.X/docs/cscli/cscli_upgrade_parser.md b/docs/v0.3.X/docs/cscli/cscli_upgrade_parser.md deleted file mode 100644 index 74ab724da..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_upgrade_parser.md +++ /dev/null @@ -1,44 +0,0 @@ -## cscli upgrade parser - -Upgrade parser configuration(s) - -### Synopsis - -Upgrade one or more parser configurations - -``` -cscli upgrade parser [config] [flags] -``` - -### Examples - -``` - - cscli upgrade parser crowdsec/apache-logs - - cscli upgrade parser -all - - cscli upgrade parser crowdsec/apache-logs --force -``` - -### Options - -``` - -h, --help help for parser -``` - -### Options inherited from parent commands - -``` - --all Upgrade all configuration in scope - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --force Overwrite existing files, even if tainted - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli upgrade](cscli_upgrade.md) - Upgrade configuration(s) - - diff --git a/docs/v0.3.X/docs/cscli/cscli_upgrade_postoverflow.md b/docs/v0.3.X/docs/cscli/cscli_upgrade_postoverflow.md deleted file mode 100644 index 46c6c624f..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_upgrade_postoverflow.md +++ /dev/null @@ -1,44 +0,0 @@ -## cscli upgrade postoverflow - -Upgrade postoverflow parser configuration(s) - -### Synopsis - -Upgrade one or more postoverflow parser configurations - -``` -cscli upgrade postoverflow [config] [flags] -``` - -### Examples - -``` - - cscli upgrade postoverflow crowdsec/enrich-rdns - - cscli upgrade postoverflow -all - - cscli upgrade postoverflow crowdsec/enrich-rdns --force -``` - -### Options - -``` - -h, --help help for postoverflow -``` - -### Options inherited from parent commands - -``` - --all Upgrade all configuration in scope - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --force Overwrite existing files, even if tainted - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli upgrade](cscli_upgrade.md) - Upgrade configuration(s) - - diff --git a/docs/v0.3.X/docs/cscli/cscli_upgrade_scenario.md b/docs/v0.3.X/docs/cscli/cscli_upgrade_scenario.md deleted file mode 100644 index 4b20097d2..000000000 --- a/docs/v0.3.X/docs/cscli/cscli_upgrade_scenario.md +++ /dev/null @@ -1,43 +0,0 @@ -## cscli upgrade scenario - -Upgrade scenario configuration(s) - -### Synopsis - -Upgrade one or more scenario configurations - -``` -cscli upgrade scenario [config] [flags] -``` - -### Examples - -``` - - cscli upgrade scenario -all - - cscli upgrade scenario crowdsec/http-404 --force -``` - -### Options - -``` - -h, --help help for scenario -``` - -### Options inherited from parent commands - -``` - --all Upgrade all configuration in scope - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config/default.yaml") - --debug Set logging to debug. - --error Set logging to error. - --force Overwrite existing files, even if tainted - --info Set logging to info. - -o, --output string Output format : human, json, raw. (default "human") - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli upgrade](cscli_upgrade.md) - Upgrade configuration(s) - - diff --git a/docs/v0.3.X/docs/getting_started/concepts.md b/docs/v0.3.X/docs/getting_started/concepts.md deleted file mode 100644 index 475a96bb1..000000000 --- a/docs/v0.3.X/docs/getting_started/concepts.md +++ /dev/null @@ -1,134 +0,0 @@ -{{v0X.crowdsec.Name}}'s main goal is to crunch logs to detect things (duh). -You will find below an introduction to the concepts that are frequently used within the documentation. - -## Acquisition - -[Acquistion configuration](/Crowdsec/v0/guide/crowdsec/acquisition/) defines which streams of information {{v0X.crowdsec.name}} is going to process. - -At the time of writing, it's mostly files, but it should be more or less any kind of stream, such as a kafka topic or a cloudtrail. - -Acquisition configuration always contains a stream (ie. a file to tail) and a tag (ie. "these are in syslog format" "these are non-syslog nginx logs"). - -File acquisition configuration is defined as : - -```yaml -filenames: #a list of file or regexp to read from (supports regular expressions) - - /var/log/nginx/http_access.log - - /var/log/nginx/https_access.log - - /var/log/nginx/error.log -labels: - type: nginx ---- -filenames: - - /var/log/auth.log -labels: - type: syslog -``` - -The `labels` part is here to tag the incoming logs with a type. `labels.type` are used by the parsers to know which logs to process. - -## Parsers [[reference](/Crowdsec/v0/references/parsers/)] - -For logs to be able to be exploited and analyzed, they need to be parsed and normalized, and this is where parsers are used. - -A parser is a YAML configuration file that describes how a string is being parsed. Said string can be a log line, or a field extracted from a previous parser. While a lot of parsers rely on the **GROK** approach (a.k.a regular expression named capture groups), parsers can as well reference enrichment modules to allow specific data processing. - -A parser usually has a specific scope. For example, if you are using [nginx](https://nginx.org), you will probably want to use the `crowdsecurity/nginx-logs` which allows your {{v0X.crowdsec.name}} setup to parse nginx's access and error logs. - -Parsers are organized into stages to allow pipelines and branching in parsing. - -See the [{{v0X.hub.name}}]({{v0X.hub.url}}) to explore parsers, or see below some examples : - - - [apache2 access/error log parser](https://github.com/crowdsecurity/hub/blob/master/parsers/s01-parse/crowdsecurity/apache2-logs.yaml) - - [iptables logs parser](https://github.com/crowdsecurity/hub/blob/master/parsers/s01-parse/crowdsecurity/iptables-logs.yaml) - - [http logs post-processing](https://github.com/crowdsecurity/hub/blob/master/parsers/s02-enrich/crowdsecurity/http-logs.yaml) - -You can as well [write your own](/Crowdsec/v0/write_configurations/parsers/) ! - - -## Stages - -Parsers are organized into "stages" to allow pipelines and branching in parsing. Each parser belongs to a stage, and can trigger next stage when successful. At the time of writing, the parsers are organized around 3 stages : - - - `s00-raw` : low level parser, such as syslog - - `s01-parse` : most of the services parsers (ssh, nginx etc.) - - `s02-enrich` : enrichment that requires parsed events (ie. geoip-enrichment) or generic parsers that apply on parsed logs (ie. second stage http parser) - -The number and structure of stages can be altered by the user, the directory structure and their alphabetical order dictates in which order stages and parsers are processed. - -Every event starts in the first stage, and will move to the next stage once it has been successfully processed by a parser that has the `onsuccess` directive set to `next_stage`, and so on until it reaches the last stage, when it's going to start to be matched against scenarios. Thus a sshd log might follow this pipeline : - - - `s00-raw` : be parsed by `crowdsecurity/syslog-logs` (will move event to the next stage) - - `s01-raw` : be parsed by `crowdsecurity/sshd-logs` (will move event to the next stage) - - `s02-enrich` : will be parsed by `crowdsecurity/geoip-enrich` and `crowdsecurity/dateparse-enrich` - - - -## Enrichers - -Enrichment is the action of adding extra context to an event based on the information we already have, so that better decision can later be taken. In most cases, you should be able to find the relevant enrichers on our {{v0X.hub.htmlname}}. - -A common/simple type of enrichment would be [geoip-enrich](https://github.com/crowdsecurity/hub/blob/master/parsers/s02-enrich/crowdsecurity/geoip-enrich.yaml) of an event (adding information such as : origin country, origin AS and origin IP range to an event). - -Once again, you should be able to find the ones you're looking for on the {{v0X.hub.htmlname}} ! - -## Scenarios [[reference](/Crowdsec/v0/references/scenarios/)] - -Scenarios is the expression of a heuristic that allows you to qualify a specific event (usually an attack).It is a YAML file that describes a set of events characterizing a scenario. Scenarios in {{v0X.crowdsec.name}} gravitate around the [leaky bucket](https://en.wikipedia.org/wiki/Leaky_bucket) principle. - -A scenario description includes at least : - - - Event eligibility rules. (For example if we're writing a ssh bruteforce detection we only focus on logs of type `ssh_failed_auth`) - - Bucket configuration such as the leak speed or its capacity (in our same ssh bruteforce example, we might allow 1 failed auth per 10s and no more than 5 in a short amount of time: `leakspeed: 10s` `capacity: 5`) - - Aggregation rules : per source ip or per other criterias (in our ssh bruteforce example, we will group per source ip) - -The description allows for many other rules to be specified (blackhole, distinct filters etc.), to allow rather complex scenarios. - -See the [{{v0X.hub.name}}]({{v0X.hub.url}}) to explore scenarios and their capabilities, or see below some examples : - - - [ssh bruteforce detection](https://github.com/crowdsecurity/hub/blob/master/scenarios/crowdsecurity/ssh-bf.yaml) - - [distinct http-404 scan](https://github.com/crowdsecurity/hub/blob/master/scenarios/crowdsecurity/http-scan-uniques_404.yaml) - - [iptables port scan](https://github.com/crowdsecurity/hub/blob/master/scenarios/crowdsecurity/iptables-scan-multi_ports.yaml) - -You can as well [write your own](/Crowdsec/v0/write_configurations/scenarios/) ! - - -## Collections - -To make user's life easier, "collections" are available, which are just a bundle of parsers and scenarios. -In this way, if you want to cover basic use-cases of let's say "nginx", you can just install the `crowdsecurity/nginx` collection that is composed of `crowdsecurity/nginx-logs` parser, as well as generic http scenarios such as `crowdsecurity/base-http-scenarios`. - -As usual, those can be found on the {{v0X.hub.htmlname}} ! - -## Event - -The objects that are processed within {{v0X.crowdsec.name}} are named "Events". -An Event can be a log line, or an overflow result. This object layout evolves around a few important items : - - - `Parsed` is an associative array that will be used during parsing to store temporary variables or processing results. - - `Enriched`, very similar to `Parsed`, is an associative array but is intended to be used for enrichment process. - - `Overflow` is a `SignalOccurence` structure that represents information about a triggered scenario, when applicable. - - `Meta` is an associative array that will be used to keep track of meta information about the event. - -_Other fields omitted for clarity, see [`pkg/types/event.go`](https://github.com/crowdsecurity/crowdsec/blob/master/pkg/types/event.go) for detailed definition_ - -## Overflow or SignalOccurence - -This object holds the relevant information about a scenario that happened : who / when / where / what etc. -Its most relevant fields are : - - - `Scenario` : name of the scenario - - `Alert_message` : a humanly readable message about what happened - - `Events_count` : the number of individual events that lead to said overflow - - `Start_at` + `Stop_at` : timestamp of the first and last events that triggered the scenario - - `Source` : a binary representation of the source of the attack - - `Source_[ip,range,AutonomousSystemNumber,AutonomousSystemOrganization,Country]` : string representation of source information - - `Labels` : an associative array representing the scenario "labels" (see scenario definition) - -_Other fields omitted for clarity, see [`pkg/types/signal_occurence.go`](https://github.com/crowdsecurity/crowdsec/blob/master/pkg/types/signal_occurence.go) for detailed definition_ - -### PostOverflow - -A postoverflow is a parser that will be applied on overflows (scenario results) before the decision is written to local DB or pushed to API. Parsers in postoverflows are meant to be used for "expensive" enrichment/parsing process that you do not want to perform on all incoming events, but rather on decision that are about to be taken. - -An example could be slack/mattermost enrichment plugin that requires human confirmation before applying the decision or reverse-dns lookup operations. diff --git a/docs/v0.3.X/docs/getting_started/crowdsec-tour.md b/docs/v0.3.X/docs/getting_started/crowdsec-tour.md deleted file mode 100644 index 527550565..000000000 --- a/docs/v0.3.X/docs/getting_started/crowdsec-tour.md +++ /dev/null @@ -1,204 +0,0 @@ - -## List installed configurations - -> List installed parsers/scenarios/collections/enricher - -```bash -{{v0X.cli.bin}} list -``` - -On the machine where you deployed {{v0X.crowdsec.name}}, type `{{v0X.cli.bin}} list` to see deployed configurations. -This list represents the parsers, scenarios and/or collections that you deployed. They represent what your {{v0X.crowdsec.name}} setup can read (logs) and detect (scenarios). - -Check [{{v0X.cli.name}} configuration](/Crowdsec/v0/guide/cscli/) management for more ! - -
- output example -```bash -bui@sd:~$ {{v0X.cli.bin}} list -INFO[0000] Loaded 9 collecs, 14 parsers, 12 scenarios, 1 post-overflow parsers -INFO[0000] PARSERS: --------------------------------------------------------------------------------------------------------------------- - NAME 📦 STATUS VERSION LOCAL PATH --------------------------------------------------------------------------------------------------------------------- - crowdsec/nginx-logs ✔️ enabled 0.3 /etc/crowdsec/config/parsers/s01-parse/nginx-logs.yaml - crowdsec/sshd-logs ✔️ enabled 0.3 /etc/crowdsec/config/parsers/s01-parse/sshd-logs.yaml - crowdsec/syslog-logs ✔️ enabled 0.4 /etc/crowdsec/config/parsers/s00-raw/syslog-logs.yaml - crowdsec/whitelists ✔️ enabled 0.4 /etc/crowdsec/config/parsers/s02-enrich/whitelists.yaml - crowdsec/dateparse-enrich ✔️ enabled 0.4 /etc/crowdsec/config/parsers/s02-enrich/dateparse-enrich.yaml - crowdsec/iptables-logs ✔️ enabled 0.3 /etc/crowdsec/config/parsers/s01-parse/iptables-logs.yaml - crowdsec/naxsi-logs ✔️ enabled 0.4 /etc/crowdsec/config/parsers/s02-enrich/naxsi-logs.yaml - crowdsec/http-logs ✔️ enabled 0.4 /etc/crowdsec/config/parsers/s02-enrich/http-logs.yaml - crowdsec/geoip-enrich ✔️ enabled 0.4 /etc/crowdsec/config/parsers/s02-enrich/geoip-enrich.yaml --------------------------------------------------------------------------------------------------------------------- -INFO[0000] SCENARIOS: ------------------------------------------------------------------------------------------------------------------------------ - NAME 📦 STATUS VERSION LOCAL PATH ------------------------------------------------------------------------------------------------------------------------------ - crowdsec/http-crawl-non_statics ✔️ enabled 0.4 /etc/crowdsec/config/scenarios/http-crawl-non_statics.yaml - crowdsec/iptables-scan-multi_ports ✔️ enabled 0.4 /etc/crowdsec/config/scenarios/iptables-scan-multi_ports.yaml - crowdsec/http-scan-uniques_404 ✔️ enabled 0.4 /etc/crowdsec/config/scenarios/http-scan-uniques_404.yaml - crowdsec/ssh-bf ✔️ enabled 0.8 /etc/crowdsec/config/scenarios/ssh-bf.yaml ------------------------------------------------------------------------------------------------------------------------------ -INFO[0000] COLLECTIONS: -------------------------------------------------------------------------------------------------------------------- - NAME 📦 STATUS VERSION LOCAL PATH -------------------------------------------------------------------------------------------------------------------- - crowdsec/base-http-scenarios ✔️ enabled 0.1 /etc/crowdsec/config/collections/base-http-scenarios.yaml - crowdsec/iptables ✔️ enabled 0.2 /etc/crowdsec/config/collections/iptables.yaml - crowdsec/nginx ✔️ enabled 0.2 /etc/crowdsec/config/collections/nginx.yaml - crowdsec/sshd ✔️ enabled 0.2 /etc/crowdsec/config/collections/sshd.yaml - crowdsec/linux ✔️ enabled 0.2 /etc/crowdsec/config/collections/linux.yaml -------------------------------------------------------------------------------------------------------------------- -INFO[0000] POSTOVERFLOWS: --------------------------------------- - NAME 📦 STATUS VERSION LOCAL PATH --------------------------------------- --------------------------------------- -``` -
- - - - -## Finding configurations - -{{v0X.crowdsec.Name}} efficiency is dictated by installed parsers and scenarios, so [take a look at the {{v0X.hub.name}}]({{v0X.hub.url}}) to find the appropriated ones ! - -If you didn't perform the setup with the wizard, or if you are reading logs from other machines, you will have to pick the right {{v0X.collections.htmlname}}. This will ensure that {{v0X.crowdsec.name}} can parse the logs and has the corresponding scenarios. - -For example, if you're processing [nginx](http://nginx.org) logs, you might want to install the [nginx collection](https://hub.crowdsec.net/author/crowdsecurity/collections/nginx). - -A collection can be installed by typing `cscli install collection crowdsecurity/nginx`, and provides all the necessary parsers and scenarios to handle said log source. `systemctl reload crowdsec` to ensure the new scenarios are loaded. - -In the same spirit, the [crowdsecurity/sshd](https://hub.crowdsec.net/author/crowdsecurity/collections/sshd)'s collection will fit most sshd setups ! - -While {{v0X.crowdsec.name}} is running, a quick look at [`cscli metrics`](/Crowdsec/v0/observability/command_line/) should help you ensure that your log sources are correctly parsed. - - -## List existing bans - -> List current bans - -```bash -{{v0X.cli.bin}} ban list -``` - - -On the machine where you deployed {{v0X.crowdsec.name}}, type `{{v0X.cli.bin}} ban list` to see existing bans. -If you just deployed {{v0X.crowdsec.name}}, the list might be empty, but don't worry, it simply means you haven't yet been attacked, congrats! - -Check [{{v0X.cli.name}} ban](/Crowdsec/v0/cheat_sheets/ban-mgmt/) management for more ! - - -
- output example -```bash -bui@sd:~$ {{v0X.cli.bin}} ban list -7 local decisions: -+--------+----------------+--------------------------------+------+--------+---------+--------------------------------+--------+------------+ -| SOURCE | IP | REASON | BANS | ACTION | COUNTRY | AS | EVENTS | EXPIRATION | -+--------+----------------+--------------------------------+------+--------+---------+--------------------------------+--------+------------+ -| local | 103.218.xxx.xx | crowdsecurity/ssh-bf | 4 | ban | HK | 59077 Shanghai UCloud | 24 | 3h28m24s | -| | | | | | | Information Technology Company | | | -| | | | | | | Limited | | | -| local | 176.174.x.xx | crowdsecurity/ssh-bf | 11 | ban | FR | 5410 Bouygues Telecom SA | 66 | 2h48m6s | -| local | 37.49.xxx.xxx | crowdsecurity/ssh-bf | 4 | ban | NL | 0 | 37 | 2h16m35s | -| local | 37.49.xxx.xx | crowdsecurity/ssh-bf_user-enum | 5 | ban | NL | 0 | 59 | 2h16m21s | -| local | 92.246.xx.xxx | crowdsecurity/ssh-bf_user-enum | 2 | ban | | 0 | 12 | 1h42m2s | -| local | 23.237.x.xx | crowdsecurity/ssh-bf | 8 | ban | US | 174 Cogent Communications | 48 | 1h7m48s | -| local | 185.153.xxx.xx | crowdsecurity/ssh-bf_user-enum | 59 | ban | MD | 49877 RM Engineering LLC | 449 | 12m54s | -+--------+----------------+--------------------------------+------+--------+---------+--------------------------------+--------+------------+ -And 64 records from API, 32 distinct AS, 19 distinct countries -``` -
- -There are different bans sources: - - - local : bans triggered locally - - api : bans fetched from the API as part of the global consensus - - cli : bans added via `{{v0X.cli.bin}} ban add` - -## Monitor on-going activity (prometheus) - -> List metrics - -```bash -{{v0X.cli.bin}} metrics -``` - -The metrics displayed are extracted from {{v0X.crowdsec.name}} prometheus. -The indicators are grouped by scope : - - - Buckets : Know which buckets are created and/or overflew (scenario efficiency) - - Acquisition : Know which file produce logs and if thy are parsed (or end up in bucket) - - Parser : Know how frequently the individual parsers are triggered and their success rate - -
- output example - -```bash -bui@sd:~$ {{v0X.cli.bin}} metrics -INFO[0000] Buckets Metrics: -+---------------------------------+-----------+--------------+--------+---------+ -| BUCKET | OVERFLOWS | INSTANTIATED | POURED | EXPIRED | -+---------------------------------+-----------+--------------+--------+---------+ -| crowdsec/http-scan-uniques_404 | 69 | 77 | 424 | 8 | -| crowdsec/ssh-bf | 4 | 23 | 53 | 18 | -| crowdsec/ssh-bf_user-enum | - | 21 | 23 | 20 | -| crowdsec/http-crawl-non_statics | 9 | 14 | 425 | 5 | -+---------------------------------+-----------+--------------+--------+---------+ -INFO[0000] Acquisition Metrics: -+------------------------------------------+------------+--------------+----------------+------------------------+ -| SOURCE | LINES READ | LINES PARSED | LINES UNPARSED | LINES POURED TO BUCKET | -+------------------------------------------+------------+--------------+----------------+------------------------+ -| /var/log/nginx/error.log | 496 | 496 | - | - | -| /var/log/nginx/http.access.log | 472 | 465 | 7 | 847 | -| /var/log/nginx/https.access.log | 1 | 1 | - | 2 | -| /var/log/auth.log | 357 | 53 | 304 | 76 | -| /var/log/kern.log | 2292 | - | 2292 | - | -| /var/log/syslog | 2358 | - | 2358 | - | -+------------------------------------------+------------+--------------+----------------+------------------------+ -INFO[0000] Parser Metrics: -+---------------------------+------+--------+----------+ -| PARSERS | HITS | PARSED | UNPARSED | -+---------------------------+------+--------+----------+ -| crowdsec/syslog-logs | 5007 | 5007 | 0 | -| crowdsec/whitelists | 1015 | 1015 | 0 | -| crowdsec/dateparse-enrich | 1015 | 1015 | 0 | -| crowdsec/geoip-enrich | 519 | 519 | 0 | -| crowdsec/http-logs | 962 | 427 | 535 | -| crowdsec/nginx-logs | 973 | 962 | 11 | -| crowdsec/non-syslog | 969 | 969 | 0 | -| crowdsec/sshd-logs | 350 | 53 | 297 | -+---------------------------+------+--------+----------+ - -``` - -
- -## Monitor on-going activity (log files) - -The {{v0X.crowdsec.main_log}} file will tell you what is going on and when an IP is blocked. - -Check [{{v0X.crowdsec.name}} monitoring](/Crowdsec/v0/observability/overview/) for more ! - - -
- output example - - -```bash -bui@sd:~$ tail -f /var/log/crowdsec-agent.log -time="14-04-2020 16:06:21" level=warning msg="40 existing LeakyRoutine" -time="14-04-2020 16:14:07" level=warning msg="1.2.3.4 triggered a 4h0m0s ip ban remediation for [crowdsec/ssh-bf]" bucket_id=throbbing-forest event_time="2020-04-14 16:14:07.215101505 +0200 CEST m=+359659.646220115" scenario=crowdsec/ssh-bf source_ip=1.2.3.4 -time="14-04-2020 16:15:52" level=info msg="api push signal: token renewed. Pushing signals" -time="14-04-2020 16:15:53" level=info msg="api push signal: pushed 1 signals successfully" -time="14-04-2020 16:21:10" level=warning msg="18 existing LeakyRoutine" -time="14-04-2020 16:30:01" level=info msg="Flushed 1 expired entries from Ban Application" -time="14-04-2020 16:33:23" level=warning msg="33 existing LeakyRoutine" -time="14-04-2020 16:35:58" level=info msg="Flushed 1 expired entries from Ban Application" - -``` - -
diff --git a/docs/v0.3.X/docs/getting_started/installation.md b/docs/v0.3.X/docs/getting_started/installation.md deleted file mode 100644 index ac99cd301..000000000 --- a/docs/v0.3.X/docs/getting_started/installation.md +++ /dev/null @@ -1,83 +0,0 @@ -# Installation - -Fetch {{v0X.crowdsec.name}}'s latest version [here]({{v0X.crowdsec.download_url}}). - -```bash -tar xvzf crowdsec-release.tgz -``` -```bash -cd crowdsec-v0.X.X -``` - -A {{v0X.wizard.name}} is provided to help you deploy {{v0X.crowdsec.name}} and {{v0X.cli.name}}. - -## Using the interactive wizard - -``` -sudo {{v0X.wizard.bin}} -i -``` - -![crowdsec](../assets/images/crowdsec_install.gif) - -The {{v0X.wizard.name}} is going to guide you through the following steps : - - - detect services that are present on your machine - - detect selected services logs - - suggest collections (parsers and scenarios) to deploy - - deploy & configure {{v0X.crowdsec.name}} in order to watch selected logs for selected scenarios - -The process should take less than a minute, [please report if there are any issues]({{v0X.wizard.bugreport}}). - -You are then ready to [take a tour](/Crowdsec/v0/getting_started/crowdsec-tour/) of your freshly deployed {{v0X.crowdsec.name}} ! - -## Binary installation - -> you of little faith - -``` -sudo {{v0X.wizard.bin}} --bininstall -``` - -This will deploy a valid/empty {{v0X.crowdsec.name}} configuration files and binaries. -Beware, in this state, {{v0X.crowdsec.name}} won't monitor/detect anything unless configured. - -``` -cscli install collection crowdsecurity/linux -``` - - -Installing at least the `crowdsecurity/linux` collection will provide you : - - - syslog parser - - geoip enrichment - - date parsers - - -You will need as well to configure your {{v0X.ref.acquis}} file to feed {{v0X.crowdsec.name}} some logs. - - - - - -## From source - -!!! warning "Requirements" - - * [Go](https://golang.org/doc/install) v1.13+ - * `git clone {{v0X.crowdsec.url}}` - * [jq](https://stedolan.github.io/jq/download/) - - -Go in {{v0X.crowdsec.name}} folder and build the binaries : - -```bash -cd crowdsec -``` -```bash -make build -``` - - -{{v0X.crowdsec.name}} bin will be located in `./cmd/crowdsec/crowdsec` and {{v0X.cli.name}} bin in `cmd/crowdsec-cli/{{v0X.cli.bin}}` - -Now, you can install either with [interactive wizard](#using-the-interactive-wizard) or the [unattended mode](#using-unattended-mode). \ No newline at end of file diff --git a/docs/v0.3.X/docs/guide/crowdsec/acquisition.md b/docs/v0.3.X/docs/guide/crowdsec/acquisition.md deleted file mode 100644 index a08a0c701..000000000 --- a/docs/v0.3.X/docs/guide/crowdsec/acquisition.md +++ /dev/null @@ -1,83 +0,0 @@ -!!! info - - Please note that the `{{v0X.config.acquis_path}}` should be auto generated by the {{v0X.wizard.name}} in most case. - -The acquisition configuration specifies lists of files to monitor and associated "labels". - -The `type` label is mandatory as it's later used in the process to determine which parser(s) can handle lines coming from this source. - -Acquisition can be found in `{{v0X.config.acquis_path}}`, for example : -
- Acquisition example -```yaml -filenames: - - /var/log/nginx/access-*.log - - /var/log/nginx/error.log -labels: - type: nginx ---- -filenames: - - /var/log/auth.log -labels: - type: syslog - -``` -
- - -## Testing and viewing acquisition - -### At startup - -At startup, you will see the monitored files in `{{v0X.crowdsec.main_log}}` : - -``` -... -time="30-04-2020 08:57:25" level=info msg="Opening file '/var/log/nginx/http.access.log' (pattern:/var/log/nginx/http.access.log)" -time="30-04-2020 08:57:25" level=info msg="Opening file '/var/log/nginx/https.access.log' (pattern:/var/log/nginx/https.access.log)" -time="30-04-2020 08:57:25" level=info msg="Opening file '/var/log/nginx/error.log' (pattern:/var/log/nginx/error.log)" -time="30-04-2020 08:57:25" level=info msg="Opening file '/var/log/auth.log' (pattern:/var/log/auth.log)" -time="30-04-2020 08:57:25" level=info msg="Opening file '/var/log/syslog' (pattern:/var/log/syslog)" -time="30-04-2020 08:57:25" level=info msg="Opening file '/var/log/kern.log' (pattern:/var/log/kern.log)" -... -``` - -### At runtime - -{{v0X.cli.name}} allows you to view {{v0X.crowdsec.name}} metrics info via the `metrics` command. -This allows you to see how many lines are coming from each source, and if they are parsed correctly. - -You can see those metrics with the following command: -``` -{{v0X.cli.bin}} metrics -``` - - -
- {{v0X.cli.name}} metrics example - -```bash -## {{v0X.cli.bin}} metrics -... -INFO[0000] Acquisition Metrics: -+------------------------------------------+------------+--------------+----------------+------------------------+ -| SOURCE | LINES READ | LINES PARSED | LINES UNPARSED | LINES POURED TO BUCKET | -+------------------------------------------+------------+--------------+----------------+------------------------+ -| /var/log/nginx/http.access.log | 47 | 46 | 1 | 10 | -| /var/log/nginx/https.access.log | 25 | 25 | - | 18 | -| /var/log/kern.log | 297948 | 297948 | - | 69421 | -| /var/log/syslog | 303868 | 297947 | 5921 | 71539 | -| /var/log/auth.log | 63419 | 12896 | 50523 | 20463 | -| /var/log/nginx/error.log | 65 | 65 | - | - | -+------------------------------------------+------------+--------------+----------------+------------------------+ -... -``` - -
- - -!!! info - - All these metrics are actually coming from {{v0X.crowdsec.name}}'s prometheus agent. See [prometheus](/Crowdsec/v0/observability/prometheus/) directly for more insights. - - diff --git a/docs/v0.3.X/docs/guide/crowdsec/enrichers.md b/docs/v0.3.X/docs/guide/crowdsec/enrichers.md deleted file mode 100644 index 3bb6f4512..000000000 --- a/docs/v0.3.X/docs/guide/crowdsec/enrichers.md +++ /dev/null @@ -1,21 +0,0 @@ -Enrichers are basically {{v0X.parsers.htmlname}} that can rely on external methods to provide extra contextual information to the event. The enrichers are usually in the `s02-enrich` {{v0X.stage.htmlname}} (after most of the parsing happened). - -Enrichers functions should all accept a string as a parameter, and return an associative string array, that will be automatically merged into the `Enriched` map of the {{v0X.event.htmlname}}. - -!!! warning - At the time of writing, enrichers plugin mechanism implementation is still ongoing (read: the list of available enrichment methods is currently hardcoded). - - -As an example let's look into the geoip-enrich parser/enricher : - -It relies on [the geolite2 data created by maxmind](https://www.maxmind.com) and the [geoip2 golang module](https://github.com/oschwald/geoip2-golang) to provide the actual data. - - -It exposes three methods : `GeoIpCity` `GeoIpASN` and `IpToRange` that are used by the `crowdsecurity/geoip-enrich`. -Enrichers can be installed as any other parsers with the following command: - -``` -{{v0X.cli.bin}} install parser crowdsecurity/geoip-enrich -``` - -Take a tour at the {{v0X.hub.htmlname}} to find them ! diff --git a/docs/v0.3.X/docs/guide/crowdsec/overview.md b/docs/v0.3.X/docs/guide/crowdsec/overview.md deleted file mode 100644 index 7e0e51d32..000000000 --- a/docs/v0.3.X/docs/guide/crowdsec/overview.md +++ /dev/null @@ -1,142 +0,0 @@ - -{{v0X.crowdsec.name}} configuration lives under `{{v0X.config.crowdsec_dir}}` and should be as : - -## default.yaml - -This is the 'main' configuration file, it allows to specify parameters such as : - - - logging (level and media) - - directories (config, data, runtime) - - API flag (on/off) - - prometheus (on/off) - - etc. - -
- Default configuration -```yaml -working_dir: /tmp/ -data_dir: /var/lib/crowdsec/data -config_dir: /etc/crowdsec/config -pid_dir: /var/run -log_dir: /var/log/ -log_mode: file -log_level: info -profiling: false -apimode: true -daemon: true -prometheus: true -#for prometheus agent / golang debugging -http_listen: 127.0.0.1:6060 -plugin: - backend: "/etc/crowdsec/plugins/backend" -``` -
- -#### `working_dir:` -The working directory where Prometheus will write metrics in text file. - -#### `data_dir:` -Directory where {{v0X.crowdsec.Name}} will install its data ({{v0X.crowdsec.Name}} database for example). - -#### `pid_dir:` -To specify where {{v0X.crowdsec.Name}} PID file will be stored. - -#### `config_dir:` -To specify where {{v0X.crowdsec.Name}} configuration will be stored. - -#### `log_dir:` -To specify where the logs should be stored. - -#### `log_mode:` -To specify your selected logging mode, available modes are : - -* `file` : to write logs in a file -* `stdout` : to write logs in STDOUT - -#### `log_level:` -To specify the logging level, available levels: - -* `debug` -* `info` -* `warning` -* `error` - -#### `profiling:` -To enable or disable the profiling in {{v0X.crowdsec.Name}}. - -#### `apimode:` -To enable or disable signals sending to the {{v0X.api.htmlname}}. - -#### `daemon:` -To enable or disable {{v0X.crowdsec.Name}} daemon mode. - -#### `prometheus:` -To enable or disable Prometheus metrics. - -### `prometheus_mode:` -If `prometheus` is enabled, and is set to `aggregated`, will restrict prometheus metrics to global ones. All metrics containing a source as a label will be unregistered. Meant to keep cardinality low when relevant. - -#### `http_listen:` -To configure the Prometheus service listening `address:port` or {{v0X.crowdsec.Name}} profiling - -#### `plugin:` -To specify the directories where {{v0X.ref.output}} plugins will be stored : -* `backend:` : the path where all {{v0X.crowdsec.Name}} backend plugins (database output, ...) will be located. - -## acquis.yaml - -This is the file that tells which streams (or files) {{v0X.crowdsec.name}} is reading, and their types (so that it knows how to parse them). If you're lucky, this file should be auto-generated by the wizard. - -You can find details on the configuration file format [here](/Crowdsec/v0/guide/crowdsec/acquisition/). - - -## api.yaml - -Name is self-explanatory : it holds API configuration. - -This file should never be edited by a human : the wizard will deploy safe default for it, and {{v0X.cli.name}} will alter it on your behalf when you register or enroll your machine. - -You can look into it, and you should see : - - - url endpoints - - login and password (auto-generated by your machine upon registration) - -To get new credentials : -```bash -{{v0X.cli.name}} api register -``` -Or if you loose your credentials: -```bash -{{v0X.cli.name}} api reset -``` - - -## profiles.yaml - -The profiles is what allows you to decide how do you react when a scenario is triggered : - - - do you notify yourself on mattermost/slack ? - - do you push the signal to a database so that your bouncers can stop the IP from continuing its attack ? - - do you want to avoid pushing this signal to the API ? - -Behind the scenes, the "profiles" system actually allows you to dispatch an event/overflow to various output plugins. - -You can find details on the configuration file format of {{v0X.ref.output}}. - -## parsers/ - -This directory holds all the {{v0X.parsers.htmlname}} that are enabled on your system. - -The parsers are organized in {{v0X.stage.htmlname}} (which are just folders) and the {{v0X.parsers.htmlname}} themselves are yaml files. - - -## scenarios/ - -This directory holds all the {{v0X.scenarios.htmlname}} that are enabled on your system. - -The {{v0X.scenarios.htmlname}} are yaml files. - - - - - diff --git a/docs/v0.3.X/docs/guide/crowdsec/parsers.md b/docs/v0.3.X/docs/guide/crowdsec/parsers.md deleted file mode 100644 index 788e73bb7..000000000 --- a/docs/v0.3.X/docs/guide/crowdsec/parsers.md +++ /dev/null @@ -1,99 +0,0 @@ - -## Listing installed parsers - -{{v0X.parsers.Htmlname}} are yaml files in `{{v0X.config.crowdsec_dir}}parsers//parser.yaml`. - -!!! info - - Alphabetical file order dictates the order of {{v0X.stage.htmlname}} and the orders of parsers within stage. - -You can use the following command to view installed parsers: - -``` -{{v0X.cli.bin}} list parsers -``` - -
- {{v0X.cli.name}} list example - -```bash -# {{v0X.cli.name}} list parsers -INFO[0000] Loaded 9 collecs, 14 parsers, 12 scenarios, 1 post-overflow parsers --------------------------------------------------------------------------------------------------------------------- - NAME 📦 STATUS VERSION LOCAL PATH --------------------------------------------------------------------------------------------------------------------- - crowdsec/iptables-logs ✔️ enabled 0.3 /etc/crowdsec/config/parsers/s01-parse/iptables-logs.yaml - crowdsec/dateparse-enrich ✔️ enabled 0.4 /etc/crowdsec/config/parsers/s02-enrich/dateparse-enrich.yaml - crowdsec/sshd-logs ✔️ enabled 0.3 /etc/crowdsec/config/parsers/s01-parse/sshd-logs.yaml - crowdsec/whitelists ✔️ enabled 0.4 /etc/crowdsec/config/parsers/s02-enrich/whitelists.yaml - crowdsec/http-logs ✔️ enabled 0.4 /etc/crowdsec/config/parsers/s02-enrich/http-logs.yaml - crowdsec/nginx-logs ✔️ enabled 0.3 /etc/crowdsec/config/parsers/s01-parse/nginx-logs.yaml - crowdsec/syslog-logs ✔️ enabled 0.4 /etc/crowdsec/config/parsers/s00-raw/syslog-logs.yaml - crowdsec/geoip-enrich ✔️ enabled 0.4 /etc/crowdsec/config/parsers/s02-enrich/geoip-enrich.yaml --------------------------------------------------------------------------------------------------------------------- -``` - -
- - -## Installing parsers - -### From the hub - -{{v0X.hub.htmlname}} allows you to find needed parsers. - -```bash -# {{v0X.cli.name}} install parser crowdsec/nginx-logs -INFO[0000] Loaded 9 collecs, 14 parsers, 12 scenarios, 1 post-overflow parsers -INFO[0000] crowdsec/nginx-logs : OK -INFO[0000] Enabled parsers : crowdsec/nginx-logs -INFO[0000] Enabled crowdsec/nginx-logs -# systemctl reload crowdsec -``` - -### Your own parsers - -[Write your parser configuration](/Crowdsec/v0/write_configurations/parsers/) and deploy yaml file in `{{v0X.config.crowdsec_dir}}parsers//`. - - - -## Monitoring parsers behavior - -{{v0X.cli.name}} allows you to view {{v0X.crowdsec.name}} metrics info via the `metrics` command. -This allows you to see how many logs were ingested and then parsed or unparsed by said parser. - -You can see those metrics with the following command: -``` -cscli metrics -``` - -
- {{v0X.cli.name}} metrics example - -```bash -# {{v0X.cli.name}} metrics -... -INFO[0000] Parser Metrics: -+---------------------------+--------+--------+----------+ -| PARSERS | HITS | PARSED | UNPARSED | -+---------------------------+--------+--------+----------+ -| crowdsec/sshd-logs | 62424 | 12922 | 49502 | -| crowdsec/syslog-logs | 667417 | 667417 | 0 | -| crowdsec/whitelists | 610901 | 610901 | 0 | -| crowdsec/http-logs | 136 | 21 | 115 | -| crowdsec/iptables-logs | 597843 | 597843 | 0 | -| crowdsec/nginx-logs | 137 | 136 | 1 | -| crowdsec/dateparse-enrich | 610901 | 610901 | 0 | -| crowdsec/geoip-enrich | 610836 | 610836 | 0 | -| crowdsec/non-syslog | 137 | 137 | 0 | -+---------------------------+--------+--------+----------+ - -``` - -
- - -## Going further - -If you're interested into [understanding how parsers are made](/Crowdsec/v0/references/parsers/) or writing your own, please have a look at [this page](/Crowdsec/v0/write_configurations/parsers/). - diff --git a/docs/v0.3.X/docs/guide/crowdsec/scenarios.md b/docs/v0.3.X/docs/guide/crowdsec/scenarios.md deleted file mode 100644 index 3316a2ceb..000000000 --- a/docs/v0.3.X/docs/guide/crowdsec/scenarios.md +++ /dev/null @@ -1,90 +0,0 @@ -Scenarios are yaml files that define "buckets". -Most of the scenarios currently rely on the [leaky bucket](https://en.wikipedia.org/wiki/Leaky_bucket) concept. -Scenarios lead to the instantiation, and sometime the overflow, of buckets. - - -When a bucket "overflows", the scenario is considered as having been realized. -This event leads to the creation of a new {{v0X.event.htmlname}} that describes the scenario that just happened (via a {{v0X.signal.htmlname}}). - - -## Listing installed scenarios - -scenarios are yaml files in `{{v0X.config.crowdsec_dir}}scenarios/.yaml`. - -You can view installed scenarios with the following command: -``` -{{v0X.cli.bin}} list scenarios -``` - - -
- {{v0X.cli.name}} list example - -```bash -# {{v0X.cli.name}} list scenarios -INFO[0000] Loaded 9 collecs, 14 parsers, 12 scenarios, 1 post-overflow parsers ------------------------------------------------------------------------------------------------------------------------------ - NAME 📦 STATUS VERSION LOCAL PATH ------------------------------------------------------------------------------------------------------------------------------ - crowdsec/http-scan-uniques_404 ✔️ enabled 0.4 /etc/crowdsec/config/scenarios/http-scan-uniques_404.yaml - crowdsec/ssh-bf ✔️ enabled 0.8 /etc/crowdsec/config/scenarios/ssh-bf.yaml - crowdsec/http-crawl-non_statics ✔️ enabled 0.4 /etc/crowdsec/config/scenarios/http-crawl-non_statics.yaml - crowdsec/iptables-scan-multi_ports ✔️ enabled 0.4 /etc/crowdsec/config/scenarios/iptables-scan-multi_ports.yaml ------------------------------------------------------------------------------------------------------------------------------ -``` - -
- -## Installing scenarios - -### From the hub - -{{v0X.hub.htmlname}} allows you to find needed scenarios. - - -```bash -# {{v0X.cli.name}} install scenario crowdsec/ssh-bf -INFO[0000] Loaded 9 collecs, 14 parsers, 12 scenarios, 1 post-overflow parsers -INFO[0000] crowdsec/ssh-bf : OK -INFO[0000] Enabled scenarios : crowdsec/ssh-bf -INFO[0000] Enabled crowdsec/ssh-bf -# systemctl reload crowdsec -``` - -### Your own scenarios - -[Write your scenario configuration](/Crowdsec/v0/write_configurations/scenarios/) and deploy yaml file in `{{v0X.config.crowdsec_dir}}scenarios/`. - - - - -## Monitoring scenarios behavior - -{{v0X.cli.name}} allows you to view {{v0X.crowdsec.name}} metrics info via the `metrics` command. -This allows you to see how many "buckets" associated to each scenario have been created (an event eligible from said scenario has arrived), poured (how many subsequent events have been pushed to said bucket), overflowed (the scenario happened) or underflow (there was not enough event to make the bucket overflow, and it thus expired after a while). - -You can see those metrics with the following command: -``` -{{v0X.cli.bin}} metrics -``` - - -
- {{v0X.cli.name}} metrics example - -```bash -# {{v0X.cli.name}} metrics -INFO[0000] Buckets Metrics: -+------------------------------------+-----------+--------------+--------+---------+ -| BUCKET | OVERFLOWS | INSTANTIATED | POURED | EXPIRED | -+------------------------------------+-----------+--------------+--------+---------+ -| crowdsec/http-crawl-non_statics | - | 9 | 14 | 9 | -| crowdsec/http-scan-uniques_404 | - | 11 | 14 | 11 | -| crowdsec/iptables-scan-multi_ports | 13 | 125681 | 141601 | 125650 | -| crowdsec/ssh-bf | 669 | 3721 | 12925 | 3046 | -| crowdsec/ssh-bf_user-enum | 136 | 4093 | 7587 | 3956 | -+------------------------------------+-----------+--------------+--------+---------+ -``` - -
- diff --git a/docs/v0.3.X/docs/guide/crowdsec/simulation.md b/docs/v0.3.X/docs/guide/crowdsec/simulation.md deleted file mode 100644 index 5dfd27241..000000000 --- a/docs/v0.3.X/docs/guide/crowdsec/simulation.md +++ /dev/null @@ -1,55 +0,0 @@ -You can tag some (or all) scenarios as being in **simulation mode**, which is especially useful if : - - - You have one/multiple scenario that might trigger false positives : You can keep track of decisions while not applying automated counter-measures - - You want *only* specific scenarios to trigger counter-measures - - - -!!! warning "Simulation vs [Whitelists](/Crowdsec/v0/write_configurations/whitelist/)" - Simulation and [Whitelists](/Crowdsec/v0/write_configurations/whitelist/) are not to be mixed. [Whitelists](/Crowdsec/v0/write_configurations/whitelist/) allows you to purely discard an overflow or a log, while simulation will only "cancel" the action against a peer, while keeping track of events and overflows. - - -When this happens, the scenarios are still triggered, but the action is prefixed with `simulation:`, which means that bouncers won't take action against the peer(s) that triggered the scenario. - -Simulation can be managed with [cscli simulation](/Crowdsec/v0/cscli/cscli_simulation/) command, and allows you to have settings such as _"all in simulation except scenarios X,Y,Z"_ or _"only scenarios X,Y,Z in simulation mode"_ : - - - `cscli simulation enable` : Globally enables simulation (all scenarios will be in simulation mode) - - `cscli simulation enable author/my_scenario` : Enables simulation only for a specific scenario - - -```bash -$ cscli simulation enable crowdsecurity/http-probing -INFO[0000] simulation mode for 'crowdsecurity/http-probing' enabled - -$ cscli simulation status -INFO[0000] global simulation: disabled -INFO[0000] Scenarios in simulation mode : -INFO[0000] - crowdsecurity/http-probing - -$ tail -f /var/log/crowdsec.log -... -WARN[21-07-2020 11:29:01] 127.0.0.1 triggered a 4h0m0s ip simulation:ban remediation for [crowdsecurity/http-probing] bucket_id=restless-sound event_time="2020-07-21 11:29:01.817545253 +0200 CEST m=+3.794547062" scenario=crowdsecurity/http-probing source_ip=127.0.0.1 - -$ cscliban list -1 local decisions: -+--------+-----------+----------------------------+------+----------------+---------+----+--------+------------+ -| SOURCE | IP | REASON | BANS | ACTION | COUNTRY | AS | EVENTS | EXPIRATION | -+--------+-----------+----------------------------+------+----------------+---------+----+--------+------------+ -| local | 127.0.0.1 | crowdsecurity/http-probing | 2 | simulation:ban | | 0 | 22 | 3h59m24s | -+--------+-----------+----------------------------+------+----------------+---------+----+--------+------------+ - -``` - -The simulation settings can be found in the `simulation.yaml` file of your configuration directory : - - -```yaml -#if simulation is false, exclusions are the only ones in learning, -#if simulation is true, exclusions are the only ones *not* in learning -simulation: false -exclusions: -- crowdsecurity/http-crawl-non_statics -- crowdsecurity/http-probing -``` - - diff --git a/docs/v0.3.X/docs/guide/cscli.md b/docs/v0.3.X/docs/guide/cscli.md deleted file mode 100644 index 12ef513d1..000000000 --- a/docs/v0.3.X/docs/guide/cscli.md +++ /dev/null @@ -1,39 +0,0 @@ -`{{v0X.cli.bin}}` is the utility that will help you to manage {{v0X.crowdsec.name}}. This tools has the following functionalities: - - - [manage bans]({{v0X. cli.ban_doc }}) - - [backup and restore configuration]({{v0X. cli.backup_doc }}) - - [display metrics]({{v0X. cli.metrics_doc }}) - - [install configurations]({{v0X. cli.install_doc }}) - - [remove configurations]({{v0X. cli.remove_doc }}) - - [update configurations]({{v0X. cli.update_doc }}) - - [upgrade configurations]({{v0X. cli.upgrade_doc }}) - - [list configurations]({{v0X. cli.list_doc }}) - - [interact with CrowdSec API]({{v0X. cli.api_doc }}) - - [manage simulation]({{v0X.cli.simulation_doc}}) - - Take a look at the [dedicated documentation]({{v0X.cli.main_doc}}) - -## Overview - -{{v0X.cli.name}} configuration location is `/etc/crowdsec/cscli/`. - -In this folder, we store the {{v0X.cli.name}} configuration and the hub cache files. - -## Config - -The {{v0X.cli.name}} configuration is light for now, stored in `/etc/crowdsec/cscli/config`. - -```yaml -installdir: /etc/crowdsec/config # {{v0X.crowdsec.name}} configuration location -backend: /etc/crowdsec/plugins/backend # path to the backend plugin used -``` - -For {{v0X.cli.name}} to be able to pull the {{v0X.api.topX.htmlname}}, you need a valid API configuration in [api.yaml](/Crowdsec/v0/guide/crowdsec/overview/#apiyaml). - - -## Hub cache - -- `.index.json`: The file containing the metadata of all the existing {{v0X.collections.htmlname}}, {{v0X.parsers.htmlname}} and {{v0X.scenarios.htmlname}} stored in the {{v0X.hub.htmlname}}. -- `hub/*`: Folder containing all the {{v0X.collections.htmlname}}, {{v0X.parsers.htmlname}} and {{v0X.scenarios.htmlname}} stored in the {{v0X.hub.htmlname}}. - -This is used to manage configurations from the {{v0X.cli.name}} \ No newline at end of file diff --git a/docs/v0.3.X/docs/guide/overview.md b/docs/v0.3.X/docs/guide/overview.md deleted file mode 100644 index 744025dff..000000000 --- a/docs/v0.3.X/docs/guide/overview.md +++ /dev/null @@ -1,10 +0,0 @@ - -When talking about {{v0X.crowdsec.name}} or {{v0X.cli.name}} configurations, most of things are going to gravitate around {{v0X.parsers.htmlname}}, {{v0X.scenarios.htmlname}} and {{v0X.collections.htmlname}}. - -In most common setup, all these configurations should be found on the {{v0X.hub.htmlname}} and installed with {{v0X.cli.name}}. - -It is important to keep those configurations up-to-date via the `{{v0X.cli.name}} upgrade` command. - -See the [{{v0X.cli.name}} list](/Crowdsec/v0/cheat_sheets/cscli-collections-tour/) command to view the state of your deployed configurations. - - diff --git a/docs/v0.3.X/docs/index.md b/docs/v0.3.X/docs/index.md deleted file mode 100644 index 9222d5b83..000000000 --- a/docs/v0.3.X/docs/index.md +++ /dev/null @@ -1,31 +0,0 @@ -
[[Hub]]({{v0X.hub.url}}) [[Releases]]({{v0X.crowdsec.download_url}})
- -# Architecture - -![Architecture](assets/images/crowdsec_architecture.png) - -## Components - -{{v0X.crowdsec.name}} ecosystem is based on the following components : - - - [{{v0X.crowdsec.Name}}]({{v0X.crowdsec.url}}) is the lightweight service that processes logs and keeps track of attacks. - - [{{v0X.cli.name}}]({{v0X.cli.main_doc}}) is the command line interface for humans, it allows you to view, add, or remove bans as well as to install, find, or update scenarios and parsers - - [{{v0X.bouncers.name}}]({{v0X.hub.plugins_url}}) are the components that block malevolent traffic, and can be deployed anywhere in your stack - -## Moving forward - -To learn more about {{v0X.crowdsec.name}} and give it a try, please see : - - - [How to install {{v0X.crowdsec.name}}](/Crowdsec/v0/getting_started/installation/) - - [Take a quick tour of {{v0X.crowdsec.name}} and {{v0X.cli.name}} features](/Crowdsec/v0/getting_started/crowdsec-tour/) - - [Observability of {{v0X.crowdsec.name}}](/Crowdsec/v0/observability/overview/) - - [Understand {{v0X.crowdsec.name}} configuration](/Crowdsec/v0/getting_started/concepts/) - - [Deploy {{v0X.bouncers.name}} to stop malevolent peers](/Crowdsec/v0/bouncers/) - - [FAQ](/faq/) - -Don't hesitate to reach out if you're facing issues : - - - [report a bug](https://github.com/crowdsecurity/crowdsec/issues/new?assignees=&labels=bug&template=bug_report.md&title=Bug%2F) - - [suggest an improvement](https://github.com/crowdsecurity/crowdsec/issues/new?assignees=&labels=enhancement&template=feature_request.md&title=Improvment%2F) - - [ask for help on the forums](https://discourse.crowdsec.net) - diff --git a/docs/v0.3.X/docs/migration.md b/docs/v0.3.X/docs/migration.md deleted file mode 100644 index eb7a78d48..000000000 --- a/docs/v0.3.X/docs/migration.md +++ /dev/null @@ -1,74 +0,0 @@ -# Migration from v0.X to v1.X - -!!! warning - Migrating to V1.X will impact (any change you made will be lost and must be adapted to the new configuration) : - - - Database model : your existing database will be lost, a new one will be created in the V1. - - - {{v1X.crowdsec.Name}} configuration : - - `/etc/crowdsec/config/default.yaml` : check [new format](/Crowdsec/v1/references/crowdsec-config/#configuration-format) - - `/etc/crowdsec/config/profiles.yaml` : check [new format](/Crowdsec/v1/references/profiles/#profiles-configurations) - -To upgrade {{v0X.crowdsec.name}} from v0.X to v1, we'll follow those steps - -#### Backup up configuration - -``` -sudo cscli backup save /tmp/crowdsec_backup -sudo cp -R /etc/crowdsec/config/patterns /tmp/crowdsec_backup -``` - -#### Uninstall old version & install new - -Download latest V1 {{v0X.crowdsec.name}} version [here]({{v0X.crowdsec.download_url}}) - -``` -tar xvzf crowdsec-release.tgz -cd crowdsec-v1*/ -sudo ./wizard.sh --uninstall -sudo ./wizard.sh --bininstall -``` - -!!! warning - Don't forget to remove {{v0X.metabase.name}} dashboard if you installed it manually (without {{v0X.cli.name}}). - -#### Restore configuration - -!!! warning - Before restoring old backup, if you have `local` or `tainted` postoverflows, be aware that they are no longer compatible. You should update the syntax (the community and us are available to help you doing this part). -``` -sudo cscli hub update -sudo cscli config restore --old-backup /tmp/crowdsec_backup/ -sudo cp -R /tmp/crowdsec_backup/patterns /etc/crowdsec/ -``` - -#### Start & health check - -Finally, you will be able to start {{v0X.crowdsec.name}} service. Before that, just check if {{v1X.lapi.name}} and {{v0X.api.name}} are correctly configured. - -``` -ubuntu@ip-:~$ sudo cscli lapi status -INFO[0000] Loaded credentials from /etc/crowdsec/local_api_credentials.yaml -INFO[0000] Trying to authenticate with username 941c3fxxxxxxxxxxxxxxxxxxxxxx on http://localhost:8080/ -INFO[0000] You can successfully interact with Local API (LAPI) - -ubuntu@ip-:~$ sudo cscli capi status -INFO[0000] Loaded credentials from /etc/crowdsec/online_api_credentials.yaml -INFO[0000] Trying to authenticate with username 941c3fxxxxxxxxxxxxxxxxxxxxxxx on https://api.crowdsec.net/ -INFO[0000] You can successfully interact with Central API (CAPI) - -ubuntu@ip-:~$ sudo systemctl start crowdsec.service -ubuntu@ip-:~$ sudo systemctl status crowdsec.service -``` - -You can even check logs (located by default here: `/var/log/crowdsec.log` & `/var/log/crowdsec_api.log`). - -You can now navigate documentation to learn new {{v0X.cli.name}} commands to interact with crowdsec. - -#### Upgrade {{v0X.bouncers.name}} - -If you were using **{{v0X.bouncers.name}}** (formerly called **blocker(s)**), you need to replace them by the new compatibles {{v0X.bouncers.name}}, available on the [hub](https://hub.crowdsec.net/browse/#bouncers) (selecting `agent version` to `v1`). - -Following your bouncer type (netfilter, nginx, wordpress etc...), you need to replace them by the new available {{v0X.bouncers.name}} on the hub, please follow the {{v0X.bouncers.name}} documentation that will help you to install easily. - -We're also available to help (on [discourse](https://discourse.crowdsec.net/) or [gitter](https://gitter.im/crowdsec-project/community)) upgrading your {{v0X.bouncers.name}}. \ No newline at end of file diff --git a/docs/v0.3.X/docs/observability/command_line.md b/docs/v0.3.X/docs/observability/command_line.md deleted file mode 100644 index 85ee51ebf..000000000 --- a/docs/v0.3.X/docs/observability/command_line.md +++ /dev/null @@ -1,61 +0,0 @@ -```bash -{{v0X.cli.name}} metrics -``` - -This command provides an overview of {{v0X.crowdsec.name}} statistics provided by [prometheus client](/Crowdsec/v0/observability/prometheus/). By default it assumes that the {{v0X.crowdsec.name}} is installed on the same machine. - -The metrics are split in 3 main sections : - - - Acquisition metrics : How many lines were read from which sources, how many were successfully or unsuccessfully parsed, and how many of those lines ultimately ended up being poured to a bucket. - - Parser metrics : How many lines were fed (eligible) to each parser, and how many of those were successfully or unsuccessfully parsed. - - Bucket metrics : How many time each scenario lead to a bucket instantiation, and for each of those : - - how many times it overflowed - - how many times it expired (underflows) - - how many subsequent events were poured to said bucket - -!!! hint - These metrics should help you identify potential configuration errors. - - For example, if you have a source that has mostly unparsed logs, you know you might be missing some parsers. - As well, if you have scenarios that are never instantiated, it might be a hint that they are not relevant to your configuration. - -
- {{v0X.cli.name}} metrics example -```bash -INFO[0000] Buckets Metrics: -+-----------------------------------------+-----------+--------------+--------+---------+ -| BUCKET | OVERFLOWS | INSTANTIATED | POURED | EXPIRED | -+-----------------------------------------+-----------+--------------+--------+---------+ -| crowdsecurity/http-scan-uniques_404 | - | 8 | 9 | 8 | -| crowdsecurity/iptables-scan-multi_ports | 1 | 8306 | 9097 | 8288 | -| crowdsecurity/ssh-bf | 42 | 281 | 1434 | 238 | -| crowdsecurity/ssh-bf_user-enum | 13 | 659 | 777 | 646 | -| crowdsecurity/http-crawl-non_statics | - | 10 | 12 | 10 | -+-----------------------------------------+-----------+--------------+--------+---------+ -INFO[0000] Acquisition Metrics: -+------------------------------------------+------------+--------------+----------------+------------------------+ -| SOURCE | LINES READ | LINES PARSED | LINES UNPARSED | LINES POURED TO BUCKET | -+------------------------------------------+------------+--------------+----------------+------------------------+ -| /var/log/nginx/https.access.log | 25 | 25 | - | 7 | -| /var/log/kern.log | 18078 | 18078 | - | 4066 | -| /var/log/syslog | 18499 | 18078 | 421 | 5031 | -| /var/log/auth.log | 6086 | 1434 | 4652 | 2211 | -| /var/log/nginx/error.log | 170243 | 169632 | 611 | - | -| /var/log/nginx/http.access.log | 44 | 44 | - | 14 | -+------------------------------------------+------------+--------------+----------------+------------------------+ -INFO[0000] Parser Metrics: -+--------------------------------+--------+--------+----------+ -| PARSERS | HITS | PARSED | UNPARSED | -+--------------------------------+--------+--------+----------+ -| crowdsecurity/geoip-enrich | 37659 | 37659 | 0 | -| crowdsecurity/http-logs | 169701 | 27 | 169674 | -| crowdsecurity/iptables-logs | 36156 | 36156 | 0 | -| crowdsecurity/nginx-logs | 170316 | 169701 | 615 | -| crowdsecurity/non-syslog | 170312 | 170312 | 0 | -| crowdsecurity/sshd-logs | 6053 | 1434 | 4619 | -| crowdsecurity/syslog-logs | 42663 | 42663 | 0 | -| crowdsecurity/dateparse-enrich | 207291 | 207291 | 0 | -+--------------------------------+--------+--------+----------+ - -``` -
\ No newline at end of file diff --git a/docs/v0.3.X/docs/observability/dashboard.md b/docs/v0.3.X/docs/observability/dashboard.md deleted file mode 100644 index 4a4909a23..000000000 --- a/docs/v0.3.X/docs/observability/dashboard.md +++ /dev/null @@ -1,61 +0,0 @@ - -!!! warning "SQLite & MySQL" - The default database of {{v0X.crowdsec.Name}} is SQLite. While MySQL is supported as well (>= 0.3.0), it is not in the scope of this documentation. - - - -The {{v0X.cli.name}} command `{{v0X.cli.bin}} dashboard setup` will use [docker](https://docs.docker.com/get-docker/) to install [metabase docker image](https://hub.docker.com/r/metabase/metabase/) and fetch our metabase template to have a configured and ready dashboard. - - -## Deployment - - -The metabase dashboard can be setup with : -```bash -{{v0X.cli.bin}} dashboard setup -``` - - -`--listen` and `--port` options allow you to control on which address / port will the docker be binded. - - -
- {{v0X.cli.name}} dashboard setup output - -```bash -INFO[0000] /var/lib/crowdsec/data/metabase.db exists, skip. -INFO[0000] Downloaded metabase DB -INFO[0000] Pulling docker image metabase/metabase -... -INFO[0001] Creating container -INFO[0001] Starting container -INFO[0002] Started metabase -INFO[0002] Waiting for metabase API to be up (can take up to a minute) -........... -INFO[0034] Metabase API is up -INFO[0034] Successfully authenticated -INFO[0034] Changed password ! -INFO[0034] Setup finished -INFO[0034] url : http://127.0.0.1:3000 -INFO[0034] username: metabase@crowdsec.net -INFO[0034] password: W1XJb8iw1A02U5nW7xxxxXXXxxXXXxXXxxXXXxxxXxXxXxXPdbvQdLlshqqPg8pf -``` -
- -!!! tip "Protip" - the `dashboard setup` command will output generated credentials for metabase. - Don't forget to save those ! - -Now you can connect to your dashboard, sign-in with your saved credentials then click on {{v0X.crowdsec.Name}} Dashboard to get this: - -![Dashboard_view](../assets/images/dashboard_view.png) - -![Dashboard_view2](../assets/images/dashboard_view2.png) - - - -Dashboard docker image can be managed by {{v0X.cli.name}} and docker cli also. Look at the {{v0X.cli.name}} help command using - -```bash -{{v0X.cli.bin}} dashboard -h -``` \ No newline at end of file diff --git a/docs/v0.3.X/docs/observability/logs.md b/docs/v0.3.X/docs/observability/logs.md deleted file mode 100644 index a7691d5a3..000000000 --- a/docs/v0.3.X/docs/observability/logs.md +++ /dev/null @@ -1,30 +0,0 @@ -Logs concern everything that happens to {{v0X.crowdsec.Name}} itself (startup, configuration, events like IP ban or an alert, shutdown, and so on). - -By default, logs are written to `{{v0X.crowdsec.main_log}}`, in text format. - -
- Logs example - - -```bash -time="12-05-2020 15:34:21" level=info msg="setting loglevel to info" -time="12-05-2020 15:34:21" level=info msg="Crowdsec v0.0.19-9ae496aa9cfd008513976a096accc7cfc43f2d9b" -time="12-05-2020 15:34:21" level=warning msg="Loading prometheus collectors" -time="12-05-2020 15:34:23" level=warning msg="no version in /etc/crowdsec/config/parsers/s00-raw/syslog-logs.yaml, assuming '1.0'" -time="12-05-2020 15:34:23" level=warning msg="Starting profiling and http server" -time="12-05-2020 15:34:24" level=warning msg="no version in /etc/crowdsec/config/parsers/s00-raw/syslog-logs.yaml, assuming '1.0'" -time="12-05-2020 15:34:24" level=info msg="Node has no name,author or description. Skipping." -time="12-05-2020 15:34:24" level=info msg="Loading 2 parser nodes" file=/etc/crowdsec/config/parsers/s00-raw/syslog-logs.yaml -time="12-05-2020 15:34:24" level=warning msg="no version in /etc/crowdsec/config/parsers/s01-parse/nginx-logs.yaml, assuming '1.0'" -time="12-05-2020 15:34:24" level=info msg="Loading 3 parser nodes" file=/etc/crowdsec/config/parsers/s01-parse/nginx-logs.yaml -time="12-05-2020 15:34:24" level=warning msg="no version in /etc/crowdsec/config/parsers/s01-parse/sshd-logs.yaml, assuming '1.0'" -time="13-05-2020 17:42:53" level=warning msg="24 existing LeakyRoutine" -time="13-05-2020 18:02:51" level=info msg="Flushed 1 expired entries from Ban Application" -time="13-05-2020 18:12:46" level=info msg="Flushed 1 expired entries from Ban Application" -time="13-05-2020 18:20:29" level=warning msg="11.11.11.11 triggered a 4h0m0s ip ban remediation for [crowdsecurity/ssh-bf]" bucket_id=winter-shadow event_time="2020-05-13 18:20:29.855776892 +0200 CEST m=+96112.558589990" scenario=crowdsecurity/ssh-bf source_ip=11.11.11.11 -time="13-05-2020 18:31:26" level=warning msg="22.22.22.22 triggered a 4h0m0s ip ban remediation for [crowdsecurity/ssh-bf]" bucket_id=dry-mountain event_time="2020-05-13 18:31:26.989769738 +0200 CEST m=+96769.692582872" scenario=crowdsecurity/ssh-bf source_ip=22.22.22.22 -time="13-05-2020 18:41:10" level=warning msg="16 existing LeakyRoutine" -time="13-05-2020 18:46:19" level=warning msg="33.33.33.33 triggered a 4h0m0s ip ban remediation for [crowdsecurity/iptables-scan-multi_ports]" bucket_id=holy-paper event_time="2020-05-13 18:46:19.825693323 +0200 CEST m=+97662.528506421" scenario=crowdsecurity/iptables-scan-multi_ports source_ip=33.33.33.33 -``` - -
\ No newline at end of file diff --git a/docs/v0.3.X/docs/observability/overview.md b/docs/v0.3.X/docs/observability/overview.md deleted file mode 100644 index 9763a03c8..000000000 --- a/docs/v0.3.X/docs/observability/overview.md +++ /dev/null @@ -1,13 +0,0 @@ -# Observability Overview - -Observability in security software is crucial, especially when this software might take important decision such as blocking IP addresses. - -We attempt to provide good observability of {{v0X.crowdsec.name}}'s behavior : - - - {{v0X.crowdsec.name}} itself exposes a [prometheus instrumentation](/Crowdsec/v0/observability/prometheus/) - - {{v0X.cli.Name}} allows you to view part of prometheus metrics in [cli (`{{v0X.cli.bin}} metrics`)](/Crowdsec/v0/observability/command_line/) - - {{v0X.crowdsec.name}} logging is contextualized for easy processing - - for **humans**, {{v0X.cli.name}} allows you to trivially start a service [exposing dashboards](/Crowdsec/v0/observability/dashboard/) (using [metabase](https://www.metabase.com/)) - -Furthermore, most of {{v0X.crowdsec.name}} configuration should allow you to enable partial debug (ie. per-scenario, per-parser etc.) - diff --git a/docs/v0.3.X/docs/observability/prometheus.md b/docs/v0.3.X/docs/observability/prometheus.md deleted file mode 100644 index a43dd185e..000000000 --- a/docs/v0.3.X/docs/observability/prometheus.md +++ /dev/null @@ -1,85 +0,0 @@ -{{v0X.crowdsec.name}} can expose a {{v0X.prometheus.htmlname}} endpoint for collection (on `http://127.0.0.1:6060/metrics` by default). - -The goal of this endpoint, besides the usual resources consumption monitoring, aims at offering a view of {{v0X.crowdsec.name}} "applicative" behavior : - - - is it processing a lot of logs ? is it parsing them successfully ? - - are a lot of scenarios being triggered ? - - are a lot of IPs banned ? - - etc. - -All the counters are "since {{v0X.crowdsec.name}} start". - -### Metrics details - -#### Scenarios - - - `cs_buckets` : number of scenario that currently exist - - `cs_bucket_created_total` : total number of instantiation of each scenario - - `cs_bucket_overflowed_total` : total number of overflow of each scenario - - `cs_bucket_underflowed_total` : total number of underflow of each scenario (bucket was created but expired because of lack of events) - - `cs_bucket_poured_total` : total number of event poured to each scenario with source as complementary key - -
- example - - -``` -#2030 lines from `/var/log/nginx/access.log` were poured to `crowdsecurity/http-scan-uniques_404` scenario -cs_bucket_poured_total{name="crowdsecurity/http-scan-uniques_404",source="/var/log/nginx/access.log"} 2030 -``` - -
- - -#### Parsers - - `cs_node_hits_total` : how many time an event from a specific source was processed by a parser node : - - -
- example - - -``` -# 235 lines from `auth.log` were processed by the `crowdsecurity/dateparse-enrich` parser -cs_node_hits_total{name="crowdsecurity/dateparse-enrich",source="/var/log/auth.log"} 235 -``` - -
- - - `cs_node_hits_ko_total` : how many times an event from a specific was unsuccessfully parsed by a specific parser - -
- example - - -``` -# 2112 lines from `error.log` failed to be parsed by `crowdsecurity/http-logs` -cs_node_hits_ko_total{name="crowdsecurity/http-logs",source="/var/log/nginx/error.log"} 2112 -``` - -
- - - `cs_node_hits_ok_total` : how many times an event from a specific source was successfully parsed by a specific parser - - - `cs_parser_hits_total` : how many times an event from a source has hit the parser - - `cs_parser_hits_ok_total` : how many times an event from a source was successfully parsed - - `cs_parser_hits_ko_total` : how many times an event from a source was unsuccessfully parsed - - -#### Acquisition - - - `cs_reader_hits_total` : how many events were read from a specific source - -#### Info - - - `cs_info` : Information about {{v0X.crowdsec.name}} (software version) - -### Exploitation with prometheus server & grafana - -Those metrics can be scaped by [prometheus server](https://prometheus.io/docs/introduction/overview/#architecture) and visualized with [grafana](https://grafana.com/). They [can be downloaded here](https://github.com/crowdsecurity/grafana-dashboards) : - -![Overview](../assets/images/grafana_overview.png) - -![Insight](../assets/images/grafana_insight.png) - -![Details](../assets/images/grafana_details.png) \ No newline at end of file diff --git a/docs/v0.3.X/docs/references/output.md b/docs/v0.3.X/docs/references/output.md deleted file mode 100644 index 658ace32f..000000000 --- a/docs/v0.3.X/docs/references/output.md +++ /dev/null @@ -1,144 +0,0 @@ -# Output - -The output mechanism is composed of plugins. In order to store the bans for {{v0X.bouncers.htmlname}}, at least one backend plugin must be loaded. Else, bans will not be stored and decisions cannot be applied. - - -Currently the supported backends are SQLite (default) and MySQL. - -In order to filter which signals will be sent to which plugin, {{v0X.crowdsec.name}} use a system of `profile` that can allow to granularly process your bans and signals. - -## Profile - -Here is a sample of a profile configuration: - -```yaml -profile: -filter: "" -api: true # default true : send signal to crowdsec API -outputs: # here choose your output plugins for this profile - - plugin: plugin1 - custom_config: - - plugin: plugin2 - -``` - -The default configuration that is deployed with {{v0X.crowdsec.name}} is the following one: -```yaml -profile: default_remediation -filter: "sig.Labels.remediation == 'true'" -api: true # If no api: specified, will use the default config in default.yaml -remediation: - ban: true - slow: true - captcha: true - duration: 4h -outputs: - - plugin: database ---- -profile: default_notification -filter: "sig.Labels.remediation != 'true'" -#remediation is empty, it means non taken -api: false -outputs: - - plugin: database # If we do not want to push, we can remove this line and the next one - store: false -``` - -Here we can use {{v0X.filter.htmlname}} like in parsers and scenarios with the {{v0X.signal.htmlname}} object to choose which signal will be process by which plugin. - - - -# Backend database configuration - -The `/etc/crowdsec/plugins/backend/database.yaml` file allows you to configure to which backend database you'd like to write. {{v0X.crowdsec.Name}} support SQLite and MySQL via [gorm](https://gorm.io/docs/). - -```yaml -name: database -path: /usr/local/lib/crowdsec/plugins/backend/database.so -config: - ## DB type supported (mysql, sqlite) - ## By default it using sqlite - type: sqlite - - ## mysql options - # db_host: localhost - # db_username: crowdsec - # db_password: password - # db_name: crowdsec - - ## sqlite options - db_path: /var/lib/crowdsec/data/crowdsec.db - - ## Other options - flush: true - # debug: true - -``` - -## SQLite - -SQLite is the default backend database, so you don't have to touch anything. - -## MySQL - -If you want to use MySQL as a backend database (which is suitable to distributed architectures), you need to have root privileges (ie. `mysql -u root -p`) on you MySQL database to type the following commands : - -```bash -#create the database for crowdsec -CREATE database crowdsec -#create the dedicated user -CREATE USER 'crowdsec'@'localhost' IDENTIFIED BY 'verybadpassword'; -#grant the privileges -GRANT ALL PRIVILEGES ON crowdsec.* TO 'crowdsec'@'localhost'; -#allow backward compatibility for mysql >= 5.7 -SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY','')); -``` - -Then, configure accordingly your `/etc/crowdsec/plugins/backend/database.yaml` : - -```yaml -name: database -path: /usr/local/lib/crowdsec/plugins/backend/database.so -config: - ## DB type supported (mysql, sqlite) - ## By default it using sqlite - type: mysql - - ## mysql options - db_host: localhost - db_username: crowdsec - db_password: verybadpassword - db_name: crowdsec - - ## sqlite options - #db_path: /var/lib/crowdsec/data/crowdsec.db - - ## Other options - flush: true - # debug: true -``` - - -# Plugins - -Plugins configuration file are stored in `{{v0X.plugins.configpath}}`. {{v0X.crowdsec.Name}} will scan this folder to load all the plugins. Each configuration file should provide the path to the plugin binary. By default they are stored in `{{v0X.plugins.binpath}}`. - -!!! info - If you want crowdsec to not load a plugin, `mv` or `rm` its configuration file in `{{v0X.plugins.configpath}}` - -Here is a sample of a plugin configuration file stored in `{{v0X.plugins.configpath}}`: -```yaml -name: -path: # -config: # in a form of key(string)/value(string) -``` - -For the plugin database, here is its configuration file: -```yaml -name: database -path: /usr/local/lib/crowdsec/plugins/backend/database.so -config: - db_path: /var/lib/crowdsec/data/crowdsec.db - flush: true -``` - diff --git a/docs/v0.3.X/docs/references/parsers.md b/docs/v0.3.X/docs/references/parsers.md deleted file mode 100644 index abca9a9c8..000000000 --- a/docs/v0.3.X/docs/references/parsers.md +++ /dev/null @@ -1,370 +0,0 @@ -## Understanding parsers - - -A parser is a YAML configuration file that describes how a string is being parsed. Said string can be a log line, or a field extracted from a previous parser. While a lot of parsers rely on the **GROK** approach (a.k.a regular expression named capture groups), parsers can as well reference enrichment modules to allow specific data processing, or use specific {{v0X.expr.htmlname}} feature to perform parsing on specific data, such as JSON. - -Parsers are organized into stages to allow pipelines and branching in parsing. - -See the [{{v0X.hub.name}}]({{v0X.hub.url}}) to explore parsers, or see below some examples : - - - [apache2 access/error log parser](https://github.com/crowdsecurity/hub/blob/master/parsers/s01-parse/crowdsecurity/apache2-logs.yaml) - - [iptables logs parser](https://github.com/crowdsecurity/hub/blob/master/parsers/s01-parse/crowdsecurity/iptables-logs.yaml) - - [http logs post-processing](https://github.com/crowdsecurity/hub/blob/master/parsers/s02-enrich/crowdsecurity/http-logs.yaml) - - -## Stages - -Stages concept is central to data parsing in {{v0X.crowdsec.name}}, as it allows to have various "steps" of parsing. All parsers belong to a given stage. While users can add or modify the stages order, the following stages exist : - - - `s00-raw` : low level parser, such as syslog - - `s01-parse` : most of the services parsers (ssh, nginx etc.) - - `s02-enrich` : enrichment that requires parsed events (ie. geoip-enrichment) or generic parsers that apply on parsed logs (ie. second stage http parser) - - -Every event starts in the first stage, and will move to the next stage once it has been successfully processed by a parser that has the `onsuccess` directive set to `next_stage`, and so on until it reaches the last stage, when it's going to start to be matched against scenarios. Thus a sshd log might follow this pipeline : - - - `s00-raw` : be parsed by `crowdsecurity/syslog-logs` (will move event to the next stage) - - `s01-raw` : be parsed by `crowdsecurity/sshd-logs` (will move event to the next stage) - - `s02-enrich` : will be parsed by `crowdsecurity/geoip-enrich` and `crowdsecurity/dateparse-enrich` - - -## Parser configuration format - - -A parser node might look like : -```yaml -#if 'onsuccess' is 'next_stage', the event will make it to next stage if this node succeed -onsuccess: next_stage -#a 'debug' (bool) flag allow to enable node level debug in any node to enable local debug -debug: true -#a filter to decide if the Event is elligible for this parser node -filter: "evt.Parsed.program == 'kernel'" -#a unique name to allow easy debug & logging -name: crowdsecurity/demo-iptables -#this is for humans -description: "Parse iptables drop logs" -#we can define named capture groups (a-la-grok) -pattern_syntax: - MYCAP: ".*" -#an actual grok pattern (regular expression with named capture groupe) -grok: - pattern: ^xxheader %{MYCAP:extracted_value} trailing stuff$ - #we define on which field the regular expression must be applied - apply_on: evt.Parsed.some_field -#statics are transformations that are applied on the event if the node is considered "successfull" -statics: - - parsed: something - expression: JsonExtract(evt.Event.extracted_value, "nested.an_array[0]") - #to which field the value will be written (here -> evt.Meta.log_type) - - meta: log_type - #and here a static value - value: parsed_testlog - #another one - - meta: source_ip - #here the value stored is the result of a dynamic expression - expression: "evt.Parsed.src_ip" -``` - -The parser nodes are processed sequentially based on the alphabetical order of {{v0X.stage.htmlname}} and subsequent files. -If the node is considered successful (grok is present and returned data or no grok is present) and "onsuccess" equals to `next_stage`, then the {{v0X.event.name}} is moved to the next stage. - -## Parser trees - -A parser node can contain sub-nodes, to provide proper branching (on top of stages). -It can be useful when you want to apply different parsing based on different criterias, or when you have a set of candidates parsers that you want to apply to an event : - -```yaml -#This first node will capture/extract some value -filter: "evt.Line.Labels.type == 'type1'" -name: tests/base-grok-root -pattern_syntax: - MYCAP: ".*" -grok: - pattern: ^... %{MYCAP:extracted_value} ...$ - apply_on: Line.Raw -statics: - - meta: state - value: root-done - - meta: state_sub - expression: evt.Parsed.extracted_value ---- -#and this node will apply different patterns to it -filter: "evt.Line.Labels.type == 'type1' && evt.Meta.state == 'root-done'" -name: tests/base-grok-leafs -onsuccess: next_stage -#the sub-nodes will process the result of the master node -nodes: - - filter: "evt.Parsed.extracted_value == 'VALUE1'" - debug: true - statics: - - meta: final_state - value: leaf1 - - filter: "evt.Parsed.extracted_value == 'VALUE2'" - debug: true - statics: - - meta: final_state - value: leaf2 -``` - -The logic is that the `tests/base-grok-root` node will be processed first and will alter the event (here mostly by extracting some text from the `Line.Raw` field into `Parsed` thanks to the `grok` pattern and the `statics` directive). - -The event will then continue its life and be parsed by the the following `tests/base-grok-leafs` node. -This node has `onsuccess` set to `next_stage` which means that if the node is successful, the event will be moved to the next stage. - -This node consists actually of two sub-nodes that have different conditions (branching) to allow differential treatment of said event. - -A real-life example can be seen when it comes to parsing HTTP logs. -HTTP ACCESS and ERROR logs often have different formats, and thus our "nginx" parser needs to handle both formats - -```yaml -filter: "evt.Parsed.program == 'nginx'" -onsuccess: next_stage -name: crowdsecurity/nginx-logs -nodes: - - grok: - #this is the access log - name: NGINXACCESS - apply_on: message - statics: - - meta: log_type - value: http_access-log - - target: evt.StrTime - expression: evt.Parsed.time_local - - grok: - # and this one the error log - name: NGINXERROR - apply_on: message - statics: - - meta: log_type - value: http_error-log - - target: evt.StrTime - expression: evt.Parsed.time -# these ones apply for both grok patterns -statics: - - meta: service - value: http - - meta: source_ip - expression: "evt.Parsed.remote_addr" - - meta: http_status - expression: "evt.Parsed.status" - - meta: http_path - expression: "evt.Parsed.request" -``` - -## Parser directives - -### debug - -```yaml -debug: true|false -``` -_default: false_ - -If set to to `true`, enabled node level debugging. -It is meant to help understanding parser node behavior by providing contextual logging : - -
- assignments made by statics -``` -DEBU[31-07-2020 16:36:28] + Processing 4 statics id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse -DEBU[31-07-2020 16:36:28] .Meta[service] = 'http' id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse -DEBU[31-07-2020 16:36:28] .Meta[source_ip] = '127.0.0.1' id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse -DEBU[31-07-2020 16:36:28] .Meta[http_status] = '200' id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse -DEBU[31-07-2020 16:36:28] .Meta[http_path] = '/' id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse -``` -
-
- assignments made by grok pattern -``` -DEBU[31-07-2020 16:36:28] + Grok 'NGINXACCESS' returned 10 entries to merge in Parsed id=dark-glitter name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[31-07-2020 16:36:28] .Parsed['time_local'] = '21/Jul/2020:16:13:05 +0200' id=dark-glitter name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[31-07-2020 16:36:28] .Parsed['method'] = 'GET' id=dark-glitter name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[31-07-2020 16:36:28] .Parsed['request'] = '/' id=dark-glitter name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[31-07-2020 16:36:28] .Parsed['http_user_agent'] = 'curl/7.58.0' id=dark-glitter name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[31-07-2020 16:36:28] .Parsed['remote_addr'] = '127.0.0.1' id=dark-glitter name=child-crowdsecurity/nginx-logs stage=s01-parse -``` -
-
- debug of filters and expression results -``` -DEBU[31-07-2020 16:36:28] eval(evt.Parsed.program == 'nginx') = TRUE id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse -DEBU[31-07-2020 16:36:28] eval variables: id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse -DEBU[31-07-2020 16:36:28] evt.Parsed.program = 'nginx' id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse -``` -
- - -### filter - -```yaml -filter: expression -``` - -`filter` must be a valid {{v0X.expr.htmlname}} expression that will be evaluated against the {{v0X.event.htmlname}}. - -If `filter` evaluation returns true or is absent, node will be processed. - -If `filter` returns `false` or a non-boolean, node won't be processed. - -Here is the [expr documentation](https://github.com/antonmedv/expr/tree/master/docs). - -Examples : - - - `filter: "evt.Meta.foo == 'test'"` - - `filter: "evt.Meta.bar == 'test' && evt.Meta.foo == 'test2'` - - -### grok - -```yaml -grok: - name: NAMED_EXISTING_PATTERN - apply_on: source_field -``` - -```yaml -grok: - pattern: ^a valid RE2 expression with %{CAPTURE:field}$ - apply_on: source_field -``` - -The `grok` structure in a node represent a regular expression with capture group (grok pattern) that must be applied on a field of {{v0X.event.name}}. - -The pattern can : - - - be imported by name (if present within the core of {{v0X.crowdsec.name}}) - - defined in place - -In both case, the pattern must be a valid RE2 expression. -The field(s) returned by the regular expression are going to be merged into the `Parsed` associative array of the `Event`. - - - -### name - -```yaml -name: explicit_string -``` - -The *mandatory* name of the node. If not present, node will be skipped at runtime. -It is used for example in debug log to help you track things. - -### nodes - -```yaml -nodes: - - filter: ... - grok: ... -``` - -`nodes` is a list of parser nodes, allowing you to build trees. -Each subnode must be valid, and if any of the subnodes succeed, the whole node is considered successful. - -### onsuccess - -``` -onsuccess: next_stage|continue -``` - -_default: continue_ - -if set to `next_stage` and the node is considered successful, the {{v0X.event.name}} will be moved directly to next stage without processing other nodes in the current stage. - -### pattern_syntax - -```yaml -pattern_syntax: - CAPTURE_NAME: VALID_RE2_EXPRESSION -``` - -`pattern_syntax` allows user to define named capture group expressions for future use in grok patterns. -Regexp must be a valid RE2 expression. - -```yaml -pattern_syntax: - MYCAP: ".*" -grok: - pattern: ^xxheader %{MYCAP:extracted_value} trailing stuff$ - apply_on: Line.Raw -``` - - -### statics - -```yaml -statics: - - target: evt.Meta.target_field - value: static_value - - meta: target_field - expression: evt.Meta.target_field + ' this_is' + ' a dynamic expression' - - enriched: target_field - value: static_value -``` - -`statics` is a list of directives that will be executed when the node is considered successful. -Each entry of the list is composed of a target (where to write) and a source (what data to write). - -**Target** - -The target aims at being any part of the {{v0X.event.htmlname}} object, and can be expressed in different ways : - - - `meta: ` - - `parsed: ` - - `enriched: ` - - a dynamic target (please note that the **current** event is accessible via the `evt.` variable) : - - `target: evt.Meta.foobar` - - `target: Meta.foobar` - - `target: evt.StrTime` - - - **Source** - - The source itself can be either a static value, or an {{v0X.expr.htmlname}} result : - -```yaml -statics: - - meta: target_field - value: static_value - - meta: target_field - expression: evt.Meta.another_field - - meta: target_field - expression: evt.Meta.target_field + ' this_is' + ' a dynamic expression' -``` - -### data - -``` -data: - - source_url: https://URL/TO/FILE - dest_file: LOCAL_FILENAME - [type: (regexp|string)] -``` - -`data` allows user to specify an external source of data. -This section is only relevant when `cscli` is used to install parser from hub, as it will download the `source_url` and store it to `dest_file`. When the parser is not installed from the hub, {{v0X.crowdsec.name}} won't download the URL, but the file must exist for the parser to be loaded correctly. - -The `type` is mandatory if you want to evaluate the data in the file, and should be `regex` for valid (re2) regular expression per line or `string` for string per line. -The regexps will be compiled, the strings will be loaded into a list and both will be kept in memory. -Without specifying a `type`, the file will be downloaded and stored as file and not in memory. - - -```yaml -name: crowdsecurity/cdn-whitelist -... -data: - - source_url: https://www.cloudflare.com/ips-v4 - dest_file: cloudflare_ips.txt - type: string -``` - - -## Parser concepts - -### Success and failure - -A parser is considered "successful" if : - - - A grok pattern was present and successfully matched - - No grok pattern was present - - diff --git a/docs/v0.3.X/docs/references/plugins_api.md b/docs/v0.3.X/docs/references/plugins_api.md deleted file mode 100644 index a807c32bd..000000000 --- a/docs/v0.3.X/docs/references/plugins_api.md +++ /dev/null @@ -1,178 +0,0 @@ -## Foreword - -Output plugins handle Signal Occurences resulting from bucket overflows. -This allows to either make a simple notification/alerting plugin or fully manage a backend (this is what {{v0X.crowdsec.name}} uses to manage SQLite and MySQL). - -You can create your own plugins to perform specific actions when a scenario is triggered. - -The plugin itself will be compiled into a `.so` and will have its dedicated configuration. - -## Interface - -Plugins are created in golang and must conform to the following interface : - -```go -type Backend interface { - Insert(types.SignalOccurence) error - ReadAT(time.Time) ([]map[string]string, error) - Delete(string) (int, error) - Init(map[string]string) error - Flush() error - Shutdown() error - DeleteAll() error - StartAutoCommit() error -} -``` - -> Startup/shutdown methods - - - `Init` : called at startup time and receives the custom configuration as a string map. Errors aren't fatal, but plugin will be discarded. - - `Shutdown` : called when {{v0X.crowdsec.Name}} is shutting down or restarting - - -> Writing/Deleting events - - - `Insert` : called every time an overflow happens, receives the `SignalOccurence` as a single parameter. Returned errors are non-fatal and will be logged in warning level. - - `Delete` : called to delete existing bans. Receives the exact `ip_text` (ban target) to delete. Only used by `cscli ban del`, only relevant for read/write plugins such as database ones. - - `DeleteAll` : called to delete *all* existing bans. Only used by `cscli ban flush`, only relevant for read/write plugins such as database ones) - -> Reading events - - - `ReadAT` : returns the list of bans that where active at the given time. The following keys are relevant in the list returned : source, iptext, reason, bancount, action, cn, as, events_count, until. Only used by `cscli ban list`, only relevant for read/write plugins such as database ones) - -> Backend - - - `Flush` is called regulary by crowdsec for each plugin that received events. For example it will be called after each write in `cscli` (as it's one-shot) and every few hundreds of ms / few events in {{v0X.crowdsec.name}} itself. It might be a good place to deal with slower write operations. - - -## Configurations - -Each plugin has its own configuration file : - -```bash -$ cat config/plugins/backend/dummy.yaml -# name of the plugin, is used by profiles.yaml -name: dummy -# path to the .so -path: ./plugins/backend/dummy.so -# your plugin specific configuration -config: - some_parameter: some value - other_parameter: more data - token: fooobarjajajajaja -``` - - -## Dummy plugin - -```go -package main - -import ( - "time" - - "github.com/crowdsecurity/crowdsec/pkg/types" - log "github.com/sirupsen/logrus" -) - -//This is where you would hold your plugin-specific context -type pluginDummy struct { - //some persistent data -} - -func (p *pluginDummy) Shutdown() error { - return nil -} - -func (p *pluginDummy) StartAutoCommit() error { - return nil -} - -func (p *pluginDummy) Init(config map[string]string) error { - log.Infof("pluginDummy config : %+v ", config) - return nil -} - -func (p *pluginDummy) Delete(target string) (int, error) { - return 0, nil -} - -func (p *pluginDummy) DeleteAll() error { - return nil -} - -func (p *pluginDummy) Insert(sig types.SignalOccurence) error { - log.Infof("insert signal : %+v", sig) - return nil -} - -func (p *pluginDummy) Flush() error { - return nil -} - -func (p *pluginDummy) ReadAT(timeAT time.Time) ([]map[string]string, error) { - return nil, nil -} - -// New is used by the plugin system to get the context -func New() interface{} { - return &pluginDummy - {} -} - -// empty main function is mandatory since we are in a main package -func main() {} -``` - - -## Building plugin - -```bash -$ go build -buildmode=plugin -o dummy.so -``` - - -## Testing plugin - - -
- Get a test env from fresh crowdsec release - -```bash -$ cd crowdsec-v0.3.0 -$ ./test_env.sh -$ cd tests -``` -
- - - - -```bash -$ cp ../../plugins/backend/dummy/dummy.so ./plugins/backend/ -$ cat > config/plugins/backend/dummy.yaml -name: dummy -path: ./plugins/backend/dummy.so -config: - some_parameter: some value - other_parameter: more data - token: fooobarjajajajaja -$ ./crowdsec -c dev.yaml -file test.log -type mylog -... -INFO[06-08-2020 17:21:30] pluginDummy config : map[flush:false max_records:10000 max_records_age:720h other_parameter:more data some_parameter:some value token:fooobarjajajajaja] -... -INFO[06-08-2020 17:21:30] Starting processing routines -... -INFO[06-08-2020 17:21:30] Processing Overflow ... -INFO[06-08-2020 17:21:30] insert signal : {Model:{ID:0 CreatedAt:0001-01-01 00:00:00 +0000 UTC UpdatedAt:0001-01-01 00:00:00 +0000 UTC DeletedAt:} MapKey:97872dfae02c523577eff8ec8e19706eec5fa21e Scenario:trigger on stuff Bucket_id:summer-field Alert_message:0.0.0.0 performed 'trigger on stuff' (1 events over 59ns) at 2020-08-06 17:21:30.491000439 +0200 CEST m=+0.722674306 Events_count:1 Events_sequence:[{Model:{ID:0 CreatedAt:0001-01-01 00:00:00 +0000 UTC UpdatedAt:0001-01-01 00:00:00 +0000 UTC DeletedAt:} Time:2020-08-06 17:21:30.491000368 +0200 CEST m=+0.722674247 Source:{Model:{ID:0 CreatedAt:0001-01-01 00:00:00 +0000 UTC UpdatedAt:0001-01-01 00:00:00 +0000 UTC DeletedAt:} Ip:0.0.0.0 Range:{IP: Mask:} AutonomousSystemNumber:0 AutonomousSystemOrganization: Country: Latitude:0 Longitude:0 Flags:map[]} Source_ip:0.0.0.0 Source_range: Source_AutonomousSystemNumber:0 Source_AutonomousSystemOrganization: Source_Country: SignalOccurenceID:0 Serialized:{"ASNNumber":"0","IsInEU":"false","command":"...","cwd":"...":"...","orig_uid":"...","orig_user":"...","parent":"bash","service":"...","source_ip":"...","user":"..."}}] Start_at:2020-08-06 17:21:30.491000368 +0200 CEST m=+0.722674247 BanApplications:[] Stop_at:2020-08-06 17:21:30.491000439 +0200 CEST m=+0.722674306 Source:0xc000248410 Source_ip:0.0.0.0 Source_range: Source_AutonomousSystemNumber:0 Source_AutonomousSystemOrganization: Source_Country: Source_Latitude:0 Source_Longitude:0 Sources:map[0.0.0.0:{Model:{ID:0 CreatedAt:0001-01-01 00:00:00 +0000 UTC UpdatedAt:0001-01-01 00:00:00 +0000 UTC DeletedAt:} Ip:0.0.0.0 Range:{IP: Mask:} AutonomousSystemNumber:0 AutonomousSystemOrganization: Country: Latitude:0 Longitude:0 Flags:map[]}] Dest_ip: Capacity:0 Leak_speed:0s Whitelisted:false Simulation:false Reprocess:false Labels:map[type:foobar]} -... -``` - - -## Notes - - - All the calls to the plugin methods are blocking. If you need to perform long running operations, it's the plugin's task to handle the background processing with [tombs](https://godoc.org/gopkg.in/tomb.v2) or such. - - Due to [a golang limitation](https://github.com/golang/go/issues/31354) you might have to build crowdsec in the same environment as the plugins. - - - diff --git a/docs/v0.3.X/docs/references/scenarios.md b/docs/v0.3.X/docs/references/scenarios.md deleted file mode 100644 index c23ca77bc..000000000 --- a/docs/v0.3.X/docs/references/scenarios.md +++ /dev/null @@ -1,384 +0,0 @@ -## Understanding scenarios - - -Scenarios are YAML files that allow to detect and qualify a specific behavior, usually an attack. - -Scenarios receive {{v0X.event.htmlname}}(s) and can produce {{v0X.overflow.htmlname}}(s) using the [leaky bucket](https://en.wikipedia.org/wiki/Leaky_bucket) algorithm. - -As an {{v0X.event.htmlname}} can be the representation of a log line, or an overflow, it allows scenarios to process both logs or overflows to allow inference. - -Scenarios can be of different types (leaky, trigger, counter), and are based on various factors, such as : - - - the speed/frequency of the [leaky bucket](https://en.wikipedia.org/wiki/Leaky_bucket) - - the capacity of the [leaky bucket](https://en.wikipedia.org/wiki/Leaky_bucket) - - the characteristic(s) of eligible {{v0X.event.htmlname}}(s) : "log type XX with field YY set to ZZ" - - various filters/directives that can alter the bucket's behavior, such as [groupby](/Crowdsec/v0/references/scenarios/#groupby), [distinct](references/scenarios/#distinct) or [blackhole](/Crowdsec/v0/references/scenarios/#blackhole) - -Behind the scenes, {{v0X.crowdsec.name}} is going to create one or more buckets when events with matching characteristics arrive to the scenario. When any of these buckets overflows, the scenario has been triggered. - -_Bucket partitioning_ : One scenario usually leads to many buckets creation, as each bucket is only tracking a specific subset of events. For example, if we are tracking brute-force, each "offending peer" get its own bucket. - - -A way to detect a http scanner might be to track the number of distinct non-existing pages it's requesting, and the scenario might look like this : - - -```yaml -#the bucket type : leaky, trigger, counter -type: leaky -#name and description for humans -name: crowdsecurity/http-scan-uniques_404 -description: "Detect multiple unique 404 from a single ip" -#a filter to know which events are eligible -filter: "evt.Meta.service == 'http' && evt.Meta.http_status in ['404', '403', '400']" -#how we are going to partition buckets -groupby: "evt.Meta.source_ip" -#we are only interested into counting UNIQUE/DISTINCT requested URLs -distinct: "evt.Meta.http_path" -#we specify the bucket capacity and leak speed -capacity: 5 -leakspeed: "10s" -#this will prevent the same bucket from overflowing more often than every 5 minutes -blackhole: 5m -#some labels to give context to the overflow -labels: - service: http - type: scan - #yes we want to ban people triggering this - remediation: true -``` - - -## Scenario concepts - -### TimeMachine - -{{v0X.crowdsec.name}} can be used not only to process live logs, but as well to process "cold" logs (think forensics). - -For this to be able to work, the date/time from the log must have been properly parsed for the scenario temporal aspect to be able to work properly. This relies on the [dateparser enrichment](https://github.com/crowdsecurity/hub/blob/master/parsers/s02-enrich/crowdsecurity/dateparse-enrich.yaml) - - -## Scenario directives - -### type - - -```yaml -type: leaky|trigger|counter -``` - -Defines the type of the bucket. Currently three types are supported : - - - `leaky` : a [leaky bucket](https://en.wikipedia.org/wiki/Leaky_bucket) that must be configured with a {{v0X.capacity.htmlname}} and a {{v0X.leakspeed.htmlname}} - - `trigger` : a bucket that overflows as soon as an event is poured (it's like a leaky bucket is a capacity of 0) - - `counter` : a bucket that only overflows every {{v0X.duration.htmlname}}. It's especially useful to count things. - -### name & description - -```yaml -name: my_author_name/my_scenario_name -description: A scenario that detect XXXX behavior -``` - - -Mandatory `name` and `description` for said scenario. -The name must be unique (and will define the scenario's name in the hub), and the description must be a quick sentence describing what it detects. - - -### filter - -```yaml -filter: expression -``` - -`filter` must be a valid {{v0X.expr.htmlname}} expression that will be evaluated against the {{v0X.event.htmlname}}. - -If `filter` evaluation returns true or is absent, event will be pour in the bucket. - -If `filter` returns `false` or a non-boolean, the event will be skip for this bucket. - -Here is the [expr documentation](https://github.com/antonmedv/expr/tree/master/docs). - -Examples : - - - `evt.Meta.log_type == 'telnet_new_session'` - - `evt.Meta.log_type in ['http_access-log', 'http_error-log'] && evt.Parsed.static_ressource == 'false'` - - `evt.Meta.log_type == 'ssh_failed-auth'` - - -### duration - -```yaml -duration: 45s -duration: 10m -``` - -(applicable to `counter` buckets only) - -A duration after which the bucket will overflow. -The format must be compatible with [golang ParseDuration format](https://golang.org/pkg/time/#ParseDuration) - -Examples : - -```yaml -type: counter -name: crowdsecurity/ban-reports-ssh_bf_report -description: "Count unique ips performing ssh bruteforce" -filter: "evt.Overflow.Scenario == 'ssh_bruteforce'" -distinct: "evt.Overflow.Source_ip" -capacity: -1 -duration: 10m -labels: - service: ssh -``` - - -### groupby - -```yaml -groupby: evt.Meta.source_ip -``` - - -an {{v0X.expr.htmlname}} that must return a string. This string will be used as to partition the buckets. - - -Examples : - -Here, each `source_ip` will get its own bucket. - -```yaml -type: leaky -... -groupby: evt.Meta.source_ip -... -``` - - - -Here, each unique combo of `source_ip` + `target_username` will get its own bucket. - -```yaml -type: leaky -... -groupby: evt.Meta.source_ip + '--' + evt.Parsed.target_username -... -``` - - - -### distinct - - -```yaml -distinct: evt.Meta.http_path -``` - - -an {{v0X.expr.htmlname}} that must return a string. The event will be poured **only** if the string is not already present in the bucket. - -Examples : - -This will ensure that events that keep triggering the same `.Meta.http_path` will be poured only once. - -```yaml -type: leaky -... -distinct: "evt.Meta.http_path" -... -``` - -In the logs, you can see it like this (for example from the iptables-logs portscan detection) : - -```bash -DEBU[2020-05-13T11:29:51+02:00] Uniq(7681) : ok buck.. -DEBU[2020-05-13T11:29:51+02:00] Uniq(7681) : ko, discard event buck.. -``` - -The first event has been poured (value `7681`) was not yet present in the events, while the second time, the event got discarded because the value was already present in the bucket. - - -### capacity - -```yaml -capacity: 5 -``` - - -(Applies only to `leaky` buckets) - -A positive integer representing the bucket capacity. -If there are more than `capacity` item in the bucket, it will overflow. - - -### leakspeed - -```yaml -leakspeed: "10s" -``` - -(Applies only to `leaky` buckets) - -A duration that represent how often an event will be leaking from the bucket. - -Must be compatible with [golang ParseDuration format](https://golang.org/pkg/time/#ParseDuration). - - -Example: - -Here the bucket will leak one item every 10 seconds, and can hold up to 5 items before overflowing. - -```yaml -type: leaky -... -leakspeed: "10s" -capacity: 5 -... -``` - - -### labels - -```yaml -labels: - service: ssh - type: bruteforce - remediation: true -``` - -Labels is a list of `label: values` that provide context to an overflow. -The labels are (currently) not stored in the database, nor they are sent to the API. - -Special labels : - - - The **remediation** label, if set to `true` indicate the the originating IP should be ban. - - The **scope** label, can be set to `ip` or `range` when **remediation** is set to true, and indicate to which scope should the decision apply. If you set a scenario with **remediation** to true and **scope** to `range` and the range of the IP could have been determined by the GeoIP library, the whole range to which the IP belongs will be banned. - - -Example : - -The IP that triggered the overflow (`.Meta.source_ip`) will be banned. -```yaml -type: leaky -... -labels: - service: ssh - type: bruteforce - remediation: true -``` - -The range to which the offending IP belong (`.Meta.source_ip`) will be banned. -```yaml -type: leaky -... -labels: - type: distributed_attack - remediation: true - scope: range -``` - -### blackhole - -```yaml -blackhole: 10m -``` - -A duration for which a bucket will be "silenced" after overflowing. -This is intended to limit / avoid spam of buckets that might be very rapidly triggered. - -The blackhole only applies to the individual bucket rather than the whole scenario. - -Must be compatible with [golang ParseDuration format](https://golang.org/pkg/time/#ParseDuration). - -Example : - -The same `source_ip` won't be able to trigger this overflow more than once every 10 minutes. -The potential overflows in the meanwhile will be discarded (but will still appear in logs as being blackholed). - -```yaml -type: trigger -... -blackhole: 10m -groupby: evt.Meta.source_ip -``` - -### debug - -```yaml -debug: true|false -``` - -_default: false_ - - -If set to to `true`, enabled scenario level debugging. -It is meant to help understanding scenario behavior by providing contextual logging : - -debug of filters and expression results -``` -DEBU[31-07-2020 16:34:58] eval(evt.Meta.log_type in ["http_access-log", "http_error-log"] && any(File("bad_user_agents.txt"), {evt.Parsed.http_user_agent contains #})) = TRUE cfg=still-feather file=config/scenarios/http-bad-user-agent.yaml name=crowdsecurity/http-bad-user-agent -DEBU[31-07-2020 16:34:58] eval variables: cfg=still-feather file=config/scenarios/http-bad-user-agent.yaml name=crowdsecurity/http-bad-user-agent -DEBU[31-07-2020 16:34:58] evt.Meta.log_type = 'http_access-log' cfg=still-feather file=config/scenarios/http-bad-user-agent.yaml name=crowdsecurity/http-bad-user-agent -DEBU[31-07-2020 16:34:58] evt.Parsed.http_user_agent = 'Mozilla/5.00 (Nikto/2.1.5) (Evasions:None) (Test:002810)' cfg=still-feather file=config/scenarios/http-bad-user-agent.yaml name=crowdsecurity/http-bad-user-agent -``` - - -### reprocess - -```yaml -reprocess: true|false -``` - -_default: false_ - -If set to `true`, the resulting overflow will be sent again in the scenario/parsing pipeline. -It is useful when you want to have further scenarios that will rely on past-overflows to take decisions. - - -### cache_size - -```yaml -cache_size: 5 -``` - -By default, a bucket holds {{v0X.capacity.htmlname}} events "in memory". -However, for a number of cases, you don't want this, as it might lead to excessive memory consumption. - -By setting `cache_size` to a positive integer, we can control the maximum in-memory cache size of the bucket, without changing its capacity and such. This is especially useful when using `counter` buckets on long duration that might end up counting (and this storing in memory) an important number of events. - - -### overflow_filter - -```yaml -overflow_filter: any(queue.Queue, { .Enriched.IsInEU == "true" }) -``` - -`overflow_filter` is an {{v0X.expr.htmlname}} that is run when the bucket overflows. -If this expression is present and returns false, the overflow will be discarded. - - -### data - -``` -data: - - source_url: https://URL/TO/FILE - dest_file: LOCAL_FILENAME - [type: (regexp|string)] -``` - -`data` allows user to specify an external source of data. -This section is only relevant when `cscli` is used to install scenario from hub, as ill download the `source_url` and store it to `dest_file`. When the scenario is not installed from the hub, {{v0X.crowdsec.name}} won't download the URL, but the file must exist for the scenario to be loaded correctly. -The `type` is mandatory if you want to evaluate the data in the file, and should be `regex` for valid (re2) regular expression per line or `string` for string per line. -The regexps will be compiled, the strings will be loaded into a list and both will be kept in memory. -Without specifying a `type`, the file will be downloaded and stored as file and not in memory. - - -```yaml -name: crowdsecurity/cdn-whitelist -... -data: - - source_url: https://www.cloudflare.com/ips-v4 - dest_file: cloudflare_ips.txt - type: string -``` - - diff --git a/docs/v0.3.X/docs/write_configurations/acquisition.md b/docs/v0.3.X/docs/write_configurations/acquisition.md deleted file mode 100644 index ad49db74d..000000000 --- a/docs/v0.3.X/docs/write_configurations/acquisition.md +++ /dev/null @@ -1,41 +0,0 @@ -# Write the acquisition file (optional for test) - -In order for your log to be processed by the good parser, it must match the filter that you will configure in your parser file. -There are two options: - - - Your logs are written by a syslog server, so you just have to install the [syslog parser](https://master.d3padiiorjhf1k.amplifyapp.com/author/crowdsecurity/configurations/syslog-logs) - - Your logs are read from a log file. Please add this kind of configuration in your `acquis.yaml` file: - -ⓘ the `type` will be matched by the parsers's `filter` in stage `s01-parse`. - - -```yaml ---- -filename: -labels: - type: - -``` -Here an example: - -
- Nginx acquisition - -```yaml ---- -filename: /var/log/nginx/access.log -labels: - type: nginx -``` - -
- -
- Nginx parser filter - -```yaml ---- -filter: evt.Parsed.program == 'nginx' -``` - -
diff --git a/docs/v0.3.X/docs/write_configurations/expressions.md b/docs/v0.3.X/docs/write_configurations/expressions.md deleted file mode 100644 index 17a9a18db..000000000 --- a/docs/v0.3.X/docs/write_configurations/expressions.md +++ /dev/null @@ -1,62 +0,0 @@ -# Expressions - -> [antonmedv/expr](https://github.com/antonmedv/expr) - Expression evaluation engine for Go: fast, non-Turing complete, dynamic typing, static typing - -Several places of {{v0X.crowdsec.name}}'s configuration use [expr](https://github.com/antonmedv/expr), notably : - - - {{v0X.filter.Htmlname}} that are used to determine events eligibility in {{v0X.parsers.htmlname}} and {{v0X.scenarios.htmlname}} or `profiles` - - {{v0X.statics.Htmlname}} use expr in the `expression` directive, to compute complex values - - {{v0X.whitelists.Htmlname}} rely on `expression` directive to allow more complex whitelists filters - -To learn more about [expr](https://github.com/antonmedv/expr), [check the github page of the project](https://github.com/antonmedv/expr/blob/master/docs/Language-Definition.md). - - -When {{v0X.crowdsec.name}} relies on `expr`, a context is provided to let the expression access relevant objects : - - - `evt.` is the representation of the current {{v0X.event.htmlname}} and is the most relevant object - - in [profiles](/Crowdsec/v0/references/output/#profile), {{v0X.signal.htmlname}} is accessible via the `sig.` object - -If the `debug` is enabled (in the scenario or parser where expr is used), additional debug will be displayed regarding evaluated expressions. - - -# Helpers - -In order to makes its use in {{v0X.crowdsec.name}} more efficient, we added a few helpers that are documented bellow. - -## Atof(string) float64 - -Parses a string representation of a float number to an actual float number (binding on `strconv.ParseFloat`) - -> Atof(evt.Parsed.tcp_port) - - -## JsonExtract(JsonBlob, FieldName) string - -Extract the `FieldName` from the `JsonBlob` and returns it as a string. (binding on [jsonparser](https://github.com/buger/jsonparser/)) - -> JsonExtract(evt.Parsed.some_json_blob, "foo.bar[0].one_item") - -## File(FileName) []string - -Returns the content of `FileName` as an array of string, while providing cache mechanism. - -> evt.Parsed.some_field in File('some_patterns.txt') -> any(File('rdns_seo_bots.txt'), { evt.Enriched.reverse_dns endsWith #}) - -## RegexpInFile(StringToMatch, FileName) bool - -Returns `true` if the `StringToMatch` is matched by one of the expressions contained in `FileName` (uses RE2 regexp engine). - -> RegexpInFile( evt.Enriched.reverse_dns, 'my_legit_seo_whitelists.txt') - -## Upper(string) string - -Returns the uppercase version of the string - -> Upper("yop") - -## IpInRange(IPStr, RangeStr) bool - -Returns true if the IP `IPStr` is contained in the IP range `RangeStr` (uses `net.ParseCIDR`) - -> IpInRange("1.2.3.4", "1.2.3.0/24") diff --git a/docs/v0.3.X/docs/write_configurations/parsers.md b/docs/v0.3.X/docs/write_configurations/parsers.md deleted file mode 100644 index 3846b89c1..000000000 --- a/docs/v0.3.X/docs/write_configurations/parsers.md +++ /dev/null @@ -1,259 +0,0 @@ -# Writing {{v0X.crowdsec.Name}} parser - -!!! warning "Parser dependency" - The crowdsecurity/syslog-logs parsers is needed by the core parsing - engine. Deletion or modification of this could result of {{v0X.crowdsec.name}} - being unable to parse logs, so this should be done very carefully. - -> In the current example, we'll write a parser for the logs produced by `iptables` (netfilter) with the `-j LOG` target. -> This document aims at detailing the process of writing and testing new parsers. - -## Base parser file - -The most simple parser can be defined as : - - -```yaml -filter: 1 == 1 -debug: true -onsuccess: next_stage -name: me/myparser -description: a cool parser for my service -grok: -#our grok pattern : capture .* - pattern: ^%{DATA:some_data}$ -#the field to which we apply the grok pattern : the log message itself - apply_on: message -statics: - - parsed: is_my_service - value: yes -``` - - - a {{v0X.filter.htmlname}} : if the expression is `true`, the event will enter the parser, otherwise, it won't - - a {{v0X.onsuccess.htmlname}} : defines what happens when the {{v0X.event.htmlname}} was successfully parsed : shall we continue ? shall we move to next stage ? etc. - - a name & a description - - some {{v0X.statics.htmlname}} that will modify the {{v0X.event.htmlname}} - - a `debug` flag that allows to enable local debugging information. - - -We are going to use to following sample log as an example : -```bash -May 11 16:23:43 sd-126005 kernel: [47615895.771900] IN=enp1s0 OUT= MAC=00:08:a2:0c:1f:12:00:c8:8b:e2:d6:87:08:00 SRC=99.99.99.99 DST=127.0.0.1 LEN=40 TOS=0x00 PREC=0x00 TTL=245 ID=51006 PROTO=TCP SPT=45225 DPT=8888 WINDOW=1024 RES=0x00 SYN URGP=0 -May 11 16:23:50 sd-126005 kernel: [47615902.763137] IN=enp1s0 OUT= MAC=00:08:a2:0c:1f:12:00:c8:8b:e2:d6:87:08:00 SRC=44.44.44.44 DST=127.0.0.1 LEN=60 TOS=0x00 PREC=0x00 TTL=49 ID=17451 DF PROTO=TCP SPT=53668 DPT=80 WINDOW=14600 RES=0x00 SYN URGP=0 -``` - -## Trying our mock parser - -!!! warning - Your yaml file must be in the `config/parsers/s01-parser/` directory. - - For example it can be `~/crowdsec-v0.0.19/tests/config/parsers/s01-parser/myparser.yaml`, or `/etc/crowdsec/config/parsers/s01-parser/myparser.yaml`. - - The {{v0X.stage.htmlname}} directory might not exist, don't forget to create it. - -(deployment is assuming [you're using a test environment](/Crowdsec/v0/write_configurations/requirements/)) - -Setting up our new parser : -```bash -cd crowdsec-v0.X.Y/tests -mkdir -p config/parsers/s01-parser -cp myparser.yaml config/parsers/s01-parser/ -./crowdsec -c ./dev.yaml -file ./x.log -type foobar -``` - -
- Expected output - -```bash -INFO[0000] setting loglevel to info -INFO[11-05-2020 15:48:28] Crowdsec v0.0.18-6b1281ba76819fed4b89247a5a673c592a3a9f88 -... -DEBU[0000] Event entering node id=dark-water name=me/myparser stage=s01-parser -DEBU[0000] eval(TRUE) '1 == 1' id=dark-water name=me/myparser stage=s01-parser -DEBU[0000] no ip in event, cidr/ip whitelists not checked id=dark-water name=me/myparser stage=s01-parser -DEBU[0000] + Grok '' returned 1 entries to merge in Parsed id=dark-water name=me/myparser stage=s01-parser -DEBU[0000] .Parsed['some_data'] = 'May 11 16:23:41 sd-126005 kernel: [47615893.721616] IN=enp1s0 OUT= MAC=00:08:a2:0c:1f:12:00:c8:8b:e2:d6:87:08:00 SRC=99.99.99.99 DST=127.0.0.1 LEN=40 TOS=0x00 PREC=0x00 TTL=245 ID=54555 PROTO=TCP SPT=45225 DPT=8080 WINDOW=1024 RES=0x00 SYN URGP=0 ' id=dark-water name=me/myparser stage=s01-parser -DEBU[0000] + Processing 1 statics id=dark-water name=me/myparser stage=s01-parser -DEBU[0000] .Parsed[is_my_service] = 'yes' id=dark-water name=me/myparser stage=s01-parser -DEBU[0000] Event leaving node : ok id=dark-water name=me/myparser stage=s01-parser -DEBU[0000] move Event from stage s01-parser to s02-enrich id=dark-water name=me/myparser stage=s01-parser -... -``` -
- - -We can see our "mock" parser is working, let's see what happened : - - - The event enter the node - - The `filter` returned true (`1 == 1`) so the {{v0X.event.htmlname}} will be processed - - Our grok pattern (just a `.*` capture) "worked" and captured data (the whole line actually) - - The grok captures (under the name "some_data") are merged into the `.Parsed` map of the {{v0X.event.htmlname}} - - The {{v0X.statics.htmlname}} section is processed, and `.Parsed[is_my_service]` is set to `yes` - - The {{v0X.event.htmlname}} leaves the parser successfully, and because "next_stage" is set, we move the event to the next "stage" - -## Writing the GROK pattern - -We are going to write a parser for `iptables` logs, they look like this : - -``` -May 11 16:23:43 sd-126005 kernel: [47615895.771900] IN=enp1s0 OUT= MAC=00:08:a2:0c:1f:12:00:c8:8b:e2:d6:87:08:00 SRC=99.99.99.99 DST=127.0.0.1 LEN=40 TOS=0x00 PREC=0x00 TTL=245 ID=51006 PROTO=TCP SPT=45225 DPT=8888 WINDOW=1024 RES=0x00 SYN URGP=0 -May 11 16:23:50 sd-126005 kernel: [47615902.763137] IN=enp1s0 OUT= MAC=00:08:a2:0c:1f:12:00:c8:8b:e2:d6:87:08:00 SRC=44.44.44.44 DST=127.0.0.1 LEN=60 TOS=0x00 PREC=0x00 TTL=49 ID=17451 DF PROTO=TCP SPT=53668 DPT=80 WINDOW=14600 RES=0x00 SYN URGP=0 - -``` - -Using an [online grok debugger](https://grokdebug.herokuapp.com/) or an [online regex debugger](https://www.debuggex.com/), we come up with the following grok pattern : - -`\[%{DATA}\]+.*(%{WORD:action})? IN=%{WORD:int_eth} OUT= MAC=%{IP}:%{MAC} SRC=%{IP:src_ip} DST=%{IP:dst_ip} LEN=%{INT:length}.*PROTO=%{WORD:proto} SPT=%{INT:src_port} DPT=%{INT:dst_port}.*` - -!!! warning - Check if the pattern you are looking for is not already present in [patterns configuration](https://github.com/crowdsecurity/crowdsec/tree/master/config/patterns). - - -## Test our new pattern - -Now, let's integrate our GROK pattern within our YAML : - -```yaml -#let's set onsuccess to "next_stage" : if the log is parsed, we can consider it has been dealt with -onsuccess: next_stage -#debug, for reasons (don't do this in production) -debug: true -#as seen in our sample log, those logs are processed by the system and have a progname set to 'kernel' -filter: "1 == 1" -#name and description: -name: crowdsecurity/iptables-logs -description: "Parse iptables drop logs" -grok: -#our grok pattern - pattern: \[%{DATA}\]+.*(%{WORD:action})? IN=%{WORD:int_eth} OUT= MAC=%{IP}:%{MAC} SRC=%{IP:src_ip} DST=%{IP:dst_ip} LEN=%{INT:length}.*PROTO=%{WORD:proto} SPT=%{INT:src_port} DPT=%{INT:dst_port}.* -#the field to which we apply the grok pattern : the log message itself - apply_on: message -statics: - - parsed: is_my_service - value: yes -``` - - -```bash -./crowdsec -c ./dev.yaml -file ./x.log -type foobar -``` - - -
- Expected output - -```bash -INFO[0000] setting loglevel to info -INFO[11-05-2020 16:18:58] Crowdsec v0.0.18-6b1281ba76819fed4b89247a5a673c592a3a9f88 -... -DEBU[0000] Event entering node id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] eval(TRUE) '1 == 1' id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] no ip in event, cidr/ip whitelists not checked id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] + Grok '' returned 8 entries to merge in Parsed id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] .Parsed['dst_port'] = '8080' id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] .Parsed['action'] = '' id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] .Parsed['int_eth'] = 'enp1s0' id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] .Parsed['src_ip'] = '99.99.99.99' id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] .Parsed['dst_ip'] = '127.0.0.1' id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] .Parsed['length'] = '40' id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] .Parsed['proto'] = 'TCP' id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] .Parsed['src_port'] = '45225' id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] + Processing 1 statics id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] .Parsed[is_my_service] = 'yes' id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] Event leaving node : ok id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] move Event from stage s01-parser to s02-enrich id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parser -... -``` - -
- -What changed ? We can now see that the fragment captured by the GROK pattern are merged in the `Parsed` array ! -We now have parsed data, only a few more changes and we will be done :) - -## Finalizing our parser - -```yaml -#let's set onsuccess to "next_stage" : if the log is parsed, we can consider it has been dealt with -onsuccess: next_stage -#debug, for reasons (don't do this in production) -debug: true -#as seen in our sample log, those logs are processed by the system and have a progname set to 'kernel' -filter: "evt.Parsed.program == 'kernel'" -#name and description: -name: crowdsecurity/iptables-logs -description: "Parse iptables drop logs" -grok: -#our grok pattern - pattern: \[%{DATA}\]+.*(%{WORD:action})? IN=%{WORD:int_eth} OUT= MAC=%{IP}:%{MAC} SRC=%{IP:src_ip} DST=%{IP:dst_ip} LEN=%{INT:length}.*PROTO=%{WORD:proto} SPT=%{INT:src_port} DPT=%{INT:dst_port}.* -#the field to which we apply the grok pattern : the log message itself - apply_on: message -statics: - - meta: log_type - value: iptables_drop - - meta: service - expression: "evt.Parsed.proto == 'TCP' ? 'tcp' : 'unknown'" - - meta: source_ip - expression: "evt.Parsed.src_ip" -``` - -### filter - -We changed the {{v0X.filter.htmlname}} to correctly filter on the program name. -In the current example, our logs are produced by the kernel (netfilter), and thus the program is `kernel` : - -```bash -tail -f /var/log/kern.log -May 11 16:23:50 sd-126005 kernel: [47615902.763137] IN=enp1s0 OUT= MAC=00:08:a2:0c:1f:12:00:c8:8b:e2:d6:87:08:00 SRC=44.44.44.44 DST=127.0.0.1 LEN=60 TOS=0x00 PREC=0x00 TTL=49 ID=17451 DF PROTO=TCP SPT=53668 DPT=80 WINDOW=14600 RES=0x00 SYN URGP=0 -``` - -### statics - -We are setting various entries to static or dynamic values to give "context" to the log : - - - `.Meta.log_type` is set to `iptables_drop` (so that we later can filter events coming from this) - - `.Meta.source_ip` is set the the source ip captured `.Parsed.src_ip` - - `.Meta.service` is set the the result of an expression that relies on the GROK output (`proto` field) - -Look into dedicated {{v0X.statics.htmlname}} documentation to know more about its possibilities. - - -### Testing our finalized parser - - -```bash -./crowdsec -c ./dev.yaml -file ./x.log -type kernel -``` - -
- Expected output -```bash -... -DEBU[0000] Event entering node id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] eval(TRUE) 'evt.Parsed.program == 'kernel'' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] no ip in event, cidr/ip whitelists not checked id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] + Grok '' returned 8 entries to merge in Parsed id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] .Parsed['src_port'] = '45225' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] .Parsed['dst_port'] = '8118' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] .Parsed['action'] = '' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] .Parsed['int_eth'] = 'enp1s0' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] .Parsed['src_ip'] = '44.44.44.44' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] .Parsed['dst_ip'] = '127.0.0.1' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] .Parsed['length'] = '40' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] .Parsed['proto'] = 'TCP' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] + Processing 3 statics id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] .Meta[log_type] = 'iptables_drop' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] .Meta[service] = 'tcp' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] .Meta[source_ip] = '44.44.44.44' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] Event leaving node : ok id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parser -DEBU[0000] move Event from stage s01-parser to s02-enrich id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parser -... -``` -
- -## Closing word - -We have now a fully functional parser for {{v0X.crowdsec.name}} ! -We can either deploy it to our production systems to do stuff, or even better, contribute to the {{v0X.hub.htmlname}} ! - -If you want to know more about directives and possibilities, take a look at [the parser reference documentation](/Crowdsec/v0/references/parsers/) ! - diff --git a/docs/v0.3.X/docs/write_configurations/requirements.md b/docs/v0.3.X/docs/write_configurations/requirements.md deleted file mode 100644 index 68f695c00..000000000 --- a/docs/v0.3.X/docs/write_configurations/requirements.md +++ /dev/null @@ -1,94 +0,0 @@ -# Requirements - ->Some requirements are needed in order to be able to write your own end-to-end configurations. ->During all this documentation, we are going to show as an exemple how we wrote a full port scan detection scenario (from acqusition to scenario, including parser) - -## Create the test environment - -First of all, please [download the latest release of {{v0X.crowdsec.name}}](https://github.com/crowdsecurity/crowdsec/releases). - -Then run the following commands: - -```bash -tar xzvf crowdsec-release.tgz -``` -```bash -cd ./crowdsec-vX.Y/ -``` -```bash -./test_env.sh # the -o is facultative, default is "./tests/" -``` -```bash -cd ./tests/ -``` - -The `./test_env.sh` script creates a local (non privileged) working environement for {{v0X.crowdsec.name}} and {{v0X.cli.name}}. -The deployed environment is intended to write and test parsers and scenarios easily. - - -
- Example - -```bash -$ tar xzvf ./crowdsec-release.tgz -$ cd ./crowdsec-v0.0.18/ -$ ./test_env.sh -[09/05/2020:20:02:19][INFO] Creating test arboresence in /tmp/crowdsec-v0.0.18/tests -[09/05/2020:20:02:19][INFO] Arboresence created -[09/05/2020:20:02:19][INFO] Copying needed files for tests environment -[09/05/2020:20:02:19][INFO] Files copied -[09/05/2020:20:02:19][INFO] Setting up configurations -INFO[0000] Failed to open config /tmp/crowdsec-v0.0.18/tests/config/crowdsec-cli/config : open /tmp/crowdsec-v0.0.18/tests/config/crowdsec-cli/config: no such file or directory -WARN[0000] creating skeleton! -INFO[0000] wrote config to /tmp/crowdsec-v0.0.18/tests/config/crowdsec-cli/config -INFO[0000] wrote config to /tmp/crowdsec-v0.0.18/tests/config/crowdsec-cli/config -INFO[0000] Wrote new 45625 bytes index to /tmp/crowdsec-v0.0.18/tests/config/crowdsec-cli/.index.json -INFO[0000] crowdsecurity/syslog-logs : OK -INFO[0000] crowdsecurity/geoip-enrich : OK -INFO[0000] crowdsecurity/dateparse-enrich : OK -INFO[0001] crowdsecurity/linux : OK -INFO[0001] /tmp/crowdsec-v0.0.18/tests/config/collections doesn\'t exist, create -INFO[0001] /tmp/crowdsec-v0.0.18/tests/config/parsers/s00-raw doesn\'t exist, create -INFO[0001] Enabled parsers : crowdsecurity/syslog-logs -INFO[0001] /tmp/crowdsec-v0.0.18/tests/config/parsers/s02-enrich doesn\'t exist, create -INFO[0001] Enabled parsers : crowdsecurity/geoip-enrich -INFO[0001] Enabled parsers : crowdsecurity/dateparse-enrich -INFO[0001] Enabled collections : crowdsecurity/linux -INFO[0001] Enabled crowdsecurity/linux -[09/05/2020:20:02:20][INFO] Environment is ready in /tmp/crowdsec-v0.0.18/tests -``` - -
- -## ⓘ Reminder - -Logs parsing is divided into stage, and each stage can contain one or more parser. Stages are named using a "sXX-" convention, and are processed in the alphabetical order. When a log is successfully parsed by a node that is configured to go in `next_stage`, the event is forwarded to the next stage (and the remaining parsers of the current stage aren't parsed). - -Stages and parsers are being processed alphabetically, thus the expected order would be : - -``` -s00-raw/syslog.yaml - -s01-parse/apache.yaml -s01-parse/nginx.yaml - -s02-enrich/geoip.yaml -s02-enrich/rdns.yaml -``` - -### Default stages - -- The preliminary stage (`s00-raw`) is mostly the one that will parse the structure of the log. This is where [syslog-logs](https://hub.crowdsec.net/author/crowdsecurity/configurations/syslog-logs) are parsed for example. Such a parser will parse the syslog header to detect the program source. - -- The main stage (`s01-parse`) is the one that will parse actual applications logs and output parsed data and static assigned values. There is one parser for each type of software. To parse the logs, regexp or GROK pattern are used. If the parser is configured to go to the [`next_stage`](/Crowdsec/v0/references/parsers/#onsuccess), then it will be process by the `enrichment` stage. - -- The enrichment (`s02-enrich`) stage is the one that will enrich the normalized log (we call it an event now that it is normalized) in order to get more information for the heuristic process. This stage can be composed of grok patterns and so on, but as well of plugins that can be writen by the community (geiop enrichment, rdns ...) for example [geoip-enrich](https://hub.crowdsec.net/author/crowdsecurity/configurations/geoip-enrich). - - -You can now jump to the next step : [writing our own parser !](/Crowdsec/v0/write_configurations/parsers/) - - -### Custom stage - -It is possible to write custom stage. If you want some specific parsing or enrichment to be done after the `s02-enrich` stage, it is possible by creating a new folder `s03-` (and so on). The configuration that will be created in this folder will process the logs configured to go to `next_stage` in the `s02-enrich` stage. - diff --git a/docs/v0.3.X/docs/write_configurations/scenarios.md b/docs/v0.3.X/docs/write_configurations/scenarios.md deleted file mode 100644 index 7769289af..000000000 --- a/docs/v0.3.X/docs/write_configurations/scenarios.md +++ /dev/null @@ -1,346 +0,0 @@ -# Writing {{v0X.crowdsec.Name}} scenarios - -!!! info - Please ensure that you have working env or setup test environment before writing your scenario. - - Ensure that [your logs are properly parsed](/Crowdsec/v0/write_configurations/parsers/). - - Have some sample logs at hand reach to test your scenario as you progress. - - -> In the current example, we'll write a scenario to detect port scans relying on the logs produced by `iptables` (netfilter) with the `-j LOG` target. - -> This document aims at detailing the process of writing and testing new scenarios. - -> If you're writing scenario for existing logs, [take a look at the taxonomy](https://hub.crowdsec.net/fields) to find your way ! - - -## Base scenario file - - -A rudimentary scenario can be defined as : - -```yaml -type: leaky -debug: true -name: me/my-cool-scenario -description: "detect cool stuff" -filter: evt.Meta.log_type == 'iptables_drop' -capacity: 1 -leakspeed: 1m -blackhole: 1m -labels: - type: my_test -``` - - - a {{v0X.filter.htmlname}} : if the expression is `true`, the event will enter the scenario, otherwise, it won't - - a name & a description - - a capacity for our [Leaky Bucket](https://en.wikipedia.org/wiki/Leaky_bucket) - - a leak speed for our [Leaky Bucket](https://en.wikipedia.org/wiki/Leaky_bucket) - - a blackhole duration (it will prevent the same bucket from overflowing too often to limit spam) - - some labels to qualify the events that just happen - - a `debug` flag that allows to enable local debugging information. - - -We are going to use the following sample log in our example : - -```bash -May 12 09:40:15 sd-126005 kernel: [47678084.929208] IN=enp1s0 OUT= MAC=00:08:a2:0c:1f:12:00:c8:8b:e2:d6:87:08:00 SRC=66.66.66.66 DST=127.0.0.1 LEN=40 TOS=0x08 PREC=0x20 TTL=244 ID=54321 PROTO=TCP SPT=42403 DPT=7681 WINDOW=65535 RES=0x00 SYN URGP=0 -May 12 09:40:15 sd-126005 kernel: [47678084.929245] IN=enp1s0 OUT= MAC=00:08:a2:0c:1f:12:00:c8:8b:e2:d6:87:08:00 SRC=99.99.99.99 DST=127.0.0.1 LEN=40 TOS=0x08 PREC=0x20 TTL=244 ID=54321 PROTO=TCP SPT=42403 DPT=7681 WINDOW=65535 RES=0x00 SYN URGP=0 -May 12 09:40:16 sd-126005 kernel: [47678084.929208] IN=enp1s0 OUT= MAC=00:08:a2:0c:1f:12:00:c8:8b:e2:d6:87:08:00 SRC=99.99.99.99 DST=127.0.0.1 LEN=40 TOS=0x08 PREC=0x20 TTL=244 ID=54321 PROTO=TCP SPT=42403 DPT=7681 WINDOW=65535 RES=0x00 SYN URGP=0 -May 12 09:40:16 sd-126005 kernel: [47678084.929208] IN=enp1s0 OUT= MAC=00:08:a2:0c:1f:12:00:c8:8b:e2:d6:87:08:00 SRC=44.44.44.44 DST=127.0.0.1 LEN=40 TOS=0x08 PREC=0x20 TTL=244 ID=54321 PROTO=TCP SPT=42403 DPT=7681 WINDOW=65535 RES=0x00 SYN URGP=0 -``` - -## Let's try our mock scenario - -!!! info - This assumes that you've followed the previous tutorial and that your iptables logs are properly parsed - - -```bash -./crowdsec -c ./dev.yaml -file ./x.log -type syslog -``` - - -
- Expected output -```bash -DEBU[04-08-2020 10:44:26] eval(evt.Meta.log_type == 'iptables_drop') = TRUE cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario -DEBU[04-08-2020 10:44:26] eval variables: cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario -DEBU[04-08-2020 10:44:26] evt.Meta.log_type = 'iptables_drop' cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario -... -DEBU[04-08-2020 10:44:26] eval(evt.Meta.log_type == 'iptables_drop') = TRUE cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario -DEBU[04-08-2020 10:44:26] eval variables: cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario -DEBU[04-08-2020 10:44:26] evt.Meta.log_type = 'iptables_drop' cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario -... -DEBU[04-08-2020 10:44:26] Overflow (start: 2020-05-12 09:40:15 +0000 UTC, end: 2020-05-12 09:40:15 +0000 UTC) bucket_id=sparkling-thunder capacity=1 cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario partition=ea2fed6bf8bb70d462ef8acacc4c96f5f8754413 -DEBU[04-08-2020 10:44:26] Adding overflow to blackhole (2020-05-12 09:40:15 +0000 UTC) bucket_id=sparkling-thunder capacity=1 cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario partition=ea2fed6bf8bb70d462ef8acacc4c96f5f8754413 -DEBU[04-08-2020 10:44:26] eval(evt.Meta.log_type == 'iptables_drop') = TRUE cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario -DEBU[04-08-2020 10:44:26] eval variables: cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario -DEBU[04-08-2020 10:44:26] evt.Meta.log_type = 'iptables_drop' cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario -DEBU[04-08-2020 10:44:26] Bucket ea2fed6bf8bb70d462ef8acacc4c96f5f8754413 found dead, cleanup the body bucket_id=sparkling-thunder capacity=1 cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario partition=ea2fed6bf8bb70d462ef8acacc4c96f5f8754413 -WARN[04-08-2020 10:44:26] read 4 lines file=./x.log -... -INFO[04-08-2020 10:44:26] Processing Overflow with no decisions 2 IPs performed 'me/my-cool-scenario' (2 events over 0s) at 2020-05-12 09:40:15 +0000 UTC bucket_id=sparkling-thunder event_time="2020-05-12 09:40:15 +0000 UTC" scenario=me/my-cool-scenario source_ip=66.66.66.66 -... -DEBU[04-08-2020 10:44:26] Overflow discarded, still blackholed for 59s bucket_id=long-pine capacity=1 cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario partition=ea2fed6bf8bb70d462ef8acacc4c96f5f8754413 -DEBU[04-08-2020 10:44:26] Overflow has been discard (*leakybucket.Blackhole) bucket_id=long-pine capacity=1 cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario partition=ea2fed6bf8bb70d462ef8acacc4c96f5f8754413 -... -``` -
- - -We can see our "mock" scenario is working, let's see what happened : - -- The first event (parsed line) is processed : - - - The `filter` returned true (`evt.Meta.log_type == 'iptables_drop'`) so the {{v0X.event.htmlname}} will be processed by our bucket - - The bucket is instantiated in {{v0X.timeMachine.htmlname}} mode, and its creation date is set to the timestamp from the first log - - The {{v0X.event.htmlname}} is poured in the actual bucket - -- The second event is processed - - The `filter` is still true, and the event is poured - - As our bucket's capacity is `1`, pouring this second overflow leads to an {{v0X.overflow.htmlname}} - - Because we set a blackhole directive of `1 minute`, we remember to prevent this bucket to overflowing again for the next minute - -The overflow itself is produced and we get this message : - -``` -INFO[12-05-2020 11:22:17] Processing Overflow with no decisions 2 IPs performed 'me/my-cool-scenario' (2 events over 0s) at 2020-05-12 09:40:15 +0000 UTC bucket_id=withered-brook event_time="2020-05-12 09:40:15 +0000 UTC" scenario=me/my-cool-scenario source_ip=66.66.66.66 - -``` - -!!! warning - While it "worked" we can see the first issue : the offending IP is reported to be `66.66.66.66` but there are actually 3 IPs involved (`66.66.66.66`, `99.99.99.99` and `44.44.44.44`). To make sense our "detect port scans" should detect events coming from a single IP ! - - -## One step forward : peer attribution - -Let's evolve our scenario to be closer to something meaningful : - - -```yaml -type: leaky -debug: true -name: me/my-cool-scenario -description: "detect cool stuff" -filter: "evt.Meta.log_type == 'iptables_drop' && evt.Meta.service == 'tcp'" -groupby: evt.Meta.source_ip -capacity: 1 -leakspeed: 1m -blackhole: 1m -labels: - type: my_test -``` - -What did we change ? - - - we added a meaningful filter : we are only going to look into `iptables_drop` events, and only take care of `tcp` ones (see the parser we wrote in the [previous step](/Crowdsec/v0/write_configurations/parsers/)) - - we added a `groupby` directive : it's going to ensure that each offending peer get its own bucket - - -Let's try again ! - -```bash -./crowdsec -c ./dev.yaml -file ./x.log -type syslog -``` - -
- Expected output -```bash -... -DEBU[2020-05-12T11:25:20+02:00] eval(TRUE) evt.Meta.log_type == 'iptables_drop' && evt.Meta.service == 'tcp' cfg=holy-breeze file=config/scenarios/mytest.yaml name=me/my-cool-scenario -DEBU[2020-05-12T11:25:20+02:00] Leaky routine starting, lifetime : 2m0s bucket_id=cold-lake capacity=1 cfg=holy-breeze file=config/scenarios/mytest.yaml name=me/my-cool-scenario partition=2308799e2cc5b57331df10eb93a495aff7725922 -... -DEBU[2020-05-12T11:25:20+02:00] eval(TRUE) evt.Meta.log_type == 'iptables_drop' && evt.Meta.service == 'tcp' cfg=holy-breeze file=config/scenarios/mytest.yaml name=me/my-cool-scenario -DEBU[2020-05-12T11:25:20+02:00] Instanciating TimeMachine bucket cfg=holy-breeze file=config/scenarios/mytest.yaml name=me/my-cool-scenario -DEBU[2020-05-12T11:25:20+02:00] Leaky routine starting, lifetime : 2m0s bucket_id=muddy-haze capacity=1 cfg=holy-breeze file=config/scenarios/mytest.yaml name=me/my-cool-scenario partition=6236f134d0f34d0061748c065bdcb64d8ac6dc54 -... -INFO[12-05-2020 11:25:20] node warning : no remediation bucket_id=muddy-haze event_time="2020-05-12 09:40:16 +0000 UTC" scenario=me/my-cool-scenario source_ip=99.99.99.99 -INFO[12-05-2020 11:25:20] Processing Overflow with no decisions 99.99.99.99 performed 'me/my-cool-scenario' (2 events over 1s) at 2020-05-12 09:40:16 +0000 UTC bucket_id=muddy-haze event_time="2020-05-12 09:40:16 +0000 UTC" scenario=me/my-cool-scenario source_ip=99.99.99.99 -... - -``` -
- -Let's see what happened : - - - Thanks to our `groupby` key, we now see two different partition keys appearing (`partition=...`). - It means that each peer will get its own bucket, and a "unique key" is derived from the groupby field value (here : the source IP) - - - We see that we only have one overflow, and it correctly concerns `99.99.99.99` (it's the one that actually triggered two events). This is again thanks to the groupby key - - -## One step forward : unique ports - - - -Is it done ? not yet, but we're getting close ! - -To really qualify a port-scan, we want to rely on the number of unique probed ports. Let's arbitrarily decide that a port-scan is : "One peer trying to probe AT LEAST 15 different ports within a few seconds" - -Our evolved scenario is now : - -```yaml -type: leaky -debug: true -name: me/my-cool-scenario -description: "detect cool stuff" -filter: "evt.Meta.log_type == 'iptables_drop' && evt.Meta.service == 'tcp'" -groupby: evt.Meta.source_ip -distinct: evt.Parsed.dst_port -capacity: 15 -leakspeed: 5s -blackhole: 1m -labels: - type: scan - service: tcp - -``` - -What did we changed : - - - We add a `distinct` directive on the `evt.Parsed.dst_port`. It allows the bucket to discard any event with an already seen `evt.Parsed.dst_port`. (yes, like in SQL) - - We changed `capacity` and `leakspeed` to be more relevant to our target - - We fixed the `labels` so that the event makes sense ! - - -Let's see what it changes : - -```bash -./crowdsec -c ./dev.yaml -file ./x.log -type syslog -``` - -
- Expected output -```bash -... -DEBU[2020-05-12T11:49:01+02:00] eval(TRUE) evt.Meta.log_type == 'iptables_drop' && evt.Meta.service == 'tcp' cfg=dark-pond file=config/scenarios/mytest.yaml name=me/my-cool-scenario -DEBU[2020-05-12T11:49:01+02:00] Instantiating TimeMachine bucket cfg=dark-pond file=config/scenarios/mytest.yaml name=me/my-cool-scenario -DEBU[2020-05-12T11:49:01+02:00] Leaky routine starting, lifetime : 1m20s bucket_id=nameless-feather capacity=15 cfg=dark-pond file=config/scenarios/mytest.yaml name=me/my-cool-scenario partition=2308799e2cc5b57331df10eb93a495aff7725922 -DEBU[2020-05-12T11:49:01+02:00] Uniq 'evt.Parsed.dst_port' -> '7681' bucket_id=nameless-feather capacity=15 cfg=dark-pond file=config/scenarios/mytest.yaml name=me/my-cool-scenario partition=2308799e2cc5b57331df10eb93a495aff7725922 -DEBU[2020-05-12T11:49:01+02:00] Uniq(7681) : false, discard bucket_id=nameless-feather capacity=15 cfg=dark-pond file=config/scenarios/mytest.yaml name=me/my-cool-scenario partition=2308799e2cc5b57331df10eb93a495aff7725922 -DEBU[2020-05-12T11:49:01+02:00] Pouring event bucket_id=nameless-feather capacity=15 cfg=dark-pond file=config/scenarios/mytest.yaml name=me/my-cool-scenario partition=2308799e2cc5b57331df10eb93a495aff7725922 -... - -``` -
- - - We can see that the second event was discarded, because it had a destination port similar to the first one - - No overflow were produced - - -## Is it really working - -Ok, **fingers crossed** our thing should be working. - -Let's grab some real-life logs ! - -```bash -$ wc -l kern.log -78215 kern.log -$ head -n1 kern.log -May 11 06:25:20 sd-126005 kernel: ... -$ tail -n1 kern.log -May 12 12:09:00 sd-126005 kernel: ... -``` - -We have around 80k lines averaging about 24h of logs, let's try ! - -```bash -./crowdsec -c ./dev.yaml -file ./kern.log -type syslog -``` - -
- Expected output -```bash -INFO[0000] setting loglevel to info -INFO[12-05-2020 11:50:38] Crowdsec v0.0.18-f672dbb4aec29ca2b24080a33d4d92eb9d4441cc -... -INFO[12-05-2020 11:50:42] node warning : no remediation bucket_id=sparkling-violet event_time="2020-05-11 10:41:45 +0000 UTC" scenario=me/my-cool-scenario source_ip=xx.xx.xx.xx -INFO[12-05-2020 11:50:42] Processing Overflow with no decisions xx.xx.xx.xx performed 'me/my-cool-scenario' (16 events over 0s) at 2020-05-11 10:41:45 +0000 UTC bucket_id=sparkling-violet event_time="2020-05-11 10:41:45 +0000 UTC" scenario=me/my-cool-scenario source_ip=xx.xx.xx.xx -... -INFO[12-05-2020 11:50:43] node warning : no remediation bucket_id=quiet-leaf event_time="2020-05-11 11:34:11 +0000 UTC" scenario=me/my-cool-scenario source_ip=yy.yy.yy.yy -INFO[12-05-2020 11:50:43] Processing Overflow with no decisions yy.yy.yy.yy performed 'me/my-cool-scenario' (16 events over 2s) at 2020-05-11 11:34:11 +0000 UTC bucket_id=quiet-leaf event_time="2020-05-11 11:34:11 +0000 UTC" scenario=me/my-cool-scenario source_ip=yy.yy.yy.yy -... -WARN[12-05-2020 11:51:05] read 78215 lines file=./kern.log -... -``` -
- -It seems to work correctly ! - - -## Hold my beer and watch this - - -Once I have acquire confidence in my scenario and I want it to trigger some bans, we can simply add : - - -```yaml -type: leaky -debug: true -name: me/my-cool-scenario -description: "detect cool stuff" -filter: "evt.Meta.log_type == 'iptables_drop' && evt.Meta.service == 'tcp'" -groupby: evt.Meta.source_ip -distinct: evt.Parsed.dst_port -capacity: 15 -leakspeed: 5s -blackhole: 1m -labels: - type: scan - service: tcp - remediation: true - scope: ip -``` - - -Adding `remediation: true` into the labels tells {{v0X.crowdsec.name}} that we should write a ban for the IP when the scenario is triggered ! - -Let's try : - - - I copied the yaml file to a production system (`/etc/crowdsec/crowdsec/scenarios/mytest.yaml`) - - I restart {{v0X.crowdsec.name}} (`systemctl reload crowdsec`) - -Let's check if it seems correctly enabled : - -```bash -$ {{v0X.cli.bin}} list -... -INFO[0000] SCENARIOS: ----------------------------------------------------------------------------------------------------------------------------------- - NAME 📦 STATUS VERSION LOCAL PATH ----------------------------------------------------------------------------------------------------------------------------------- -... - mytest.yaml 🚫 enabled,local /etc/crowdsec/config/scenarios/mytest.yaml -... -``` - - -Let's launch (from an external machine, as {{v0X.crowdsec.name}} ignores events from private IPs by default) a real port-scan with a good old `nmap` : - -```bash -sudo nmap -sS xx.xx.xx.xx -``` - - -and on our server : - -```bash -$ tail -f /var/log/crowdsec.log -... -time="12-05-2020 12:31:43" level=warning msg="xx.xx.16.6 triggered a 4h0m0s ip ban remediation for [me/my-cool-scenario]" bucket_id=wispy-breeze event_time="2020-05-12 12:31:43.953498645 +0200 CEST m=+64.533521568" scenario=me/my-cool-scenario source_ip=xx.xx.16.6 -... -^C -$ {{v0X.cli.bin}} ban list -INFO[0000] backend plugin 'database' loaded -8 local decisions: -+--------+-----------------+----------------------+------+--------+---------+--------------------------+--------+------------+ -| SOURCE | IP | REASON | BANS | ACTION | COUNTRY | AS | EVENTS | EXPIRATION | -+--------+-----------------+----------------------+------+--------+---------+--------------------------+--------+------------+ -| local | xx.xx.xx.xx | me/my-cool-scenario | 4 | ban | FR | 21502 SFR SA | 79 | 3h58m27s | -... -``` - -It worked !!! diff --git a/docs/v0.3.X/docs/write_configurations/whitelist.md b/docs/v0.3.X/docs/write_configurations/whitelist.md deleted file mode 100644 index 7ea157aea..000000000 --- a/docs/v0.3.X/docs/write_configurations/whitelist.md +++ /dev/null @@ -1,184 +0,0 @@ -# What are whitelists - -Whitelists are special parsers that allow you to "discard" events, and can exist at two different steps : - - - *Parser whitelists* : Allows you to discard an event at parse time, so that it never hits the buckets. - - *PostOverflow whitelists* : Those are whitelists that are checked *after* the overflow happens. It is usually best for whitelisting process that can be expensive (such as performing reverse DNS on an IP, or performing a `whois` of an IP). - -!!! info - While the whitelists are the same for parser or postoverflows, beware that field names might change. - Source ip is usually in `evt.Meta.source_ip` when it's a log, but `evt.Overflow.Source_ip` when it's an overflow - - -The whitelist can be based on several criteria : - - - specific ip address : if the event/overflow IP is the same, event is whitelisted - - ip ranges : if the event/overflow IP belongs to this range, event is whitelisted - - a list of {{v0X.expr.htmlname}} expressions : if any expression returns true, event is whitelisted - -Here is an example showcasing configuration : - -```yaml -name: crowdsecurity/my-whitelists -description: "Whitelist events from my ipv4 addresses" -#it's a normal parser, so we can restrict its scope with filter -filter: "1 == 1" -whitelist: - reason: "my ipv4 ranges" - ip: - - "127.0.0.1" - cidr: - - "192.168.0.0/16" - - "10.0.0.0/8" - - "172.16.0.0/12" - expression: - #beware, this one will work *only* if you enabled the reverse dns (crowdsecurity/rdns) enrichment postoverflow parser - - evt.Enriched.reverse_dns endsWith ".mycoolorg.com." - #this one will work *only* if you enabled the geoip (crowdsecurity/geoip-enrich) enrichment parser - - evt.Enriched.IsoCode == 'FR' -``` - - -# Whitelists in parsing - -When a whitelist is present in parsing `/etc/crowdsec/config/parsers/...`, it will be checked/discarded before being poured to any bucket. These whitelists intentionally generate no logs and are useful to discard noisy false positive sources. - -## Whitelist by ip - -Let's assume we have a setup with a `crowdsecurity/nginx` collection enabled and no whitelists. - -Thus, if I "attack" myself : - -```bash -nikto -host myfqdn.com -``` - -my own IP will be flagged as being an attacker : - -```bash -$ tail -f /var/log/crowdsec.log -ime="07-07-2020 16:13:16" level=warning msg="80.x.x.x triggered a 4h0m0s ip ban remediation for [crowdsecurity/http-bad-user-agent]" bucket_id=cool-smoke event_time="2020-07-07 16:13:16.579581642 +0200 CEST m=+358819.413561109" scenario=crowdsecurity/http-bad-user-agent source_ip=80.x.x.x -time="07-07-2020 16:13:16" level=warning msg="80.x.x.x triggered a 4h0m0s ip ban remediation for [crowdsecurity/http-probing]" bucket_id=green-silence event_time="2020-07-07 16:13:16.737579458 +0200 CEST m=+358819.571558901" scenario=crowdsecurity/http-probing source_ip=80.x.x.x -time="07-07-2020 16:13:17" level=warning msg="80.x.x.x triggered a 4h0m0s ip ban remediation for [crowdsecurity/http-crawl-non_statics]" bucket_id=purple-snowflake event_time="2020-07-07 16:13:17.353641625 +0200 CEST m=+358820.187621068" scenario=crowdsecurity/http-crawl-non_statics source_ip=80.x.x.x -time="07-07-2020 16:13:18" level=warning msg="80.x.x.x triggered a 4h0m0s ip ban remediation for [crowdsecurity/http-sensitive-files]" bucket_id=small-hill event_time="2020-07-07 16:13:18.005919055 +0200 CEST m=+358820.839898498" scenario=crowdsecurity/http-sensitive-files source_ip=80.x.x.x -^C -$ {{v0X.cli.bin}} ban list -4 local decisions: -+--------+---------------+-----------------------------------+------+--------+---------+---------------------------+--------+------------+ -| SOURCE | IP | REASON | BANS | ACTION | COUNTRY | AS | EVENTS | EXPIRATION | -+--------+---------------+-----------------------------------+------+--------+---------+---------------------------+--------+------------+ -| local | 80.x.x.x | crowdsecurity/http-bad-user-agent | 4 | ban | FR | 21502 SFR SA | 60 | 3h59m3s | -... - -``` - - -### Create the whitelist by IP - -Let's create a `/etc/crowdsec/crowdsec/parsers/s02-enrich/mywhitelists.yaml` file with the following content : - -```yaml -name: crowdsecurity/whitelists -description: "Whitelist events from my ip addresses" -whitelist: - reason: "my ip ranges" - ip: - - "80.x.x.x" -``` - -and reload {{v0X.crowdsec.name}} : `sudo systemctl reload crowdsec` - -### Test the whitelist - -Thus, if we restart our attack : - -```bash -nikto -host myfqdn.com -``` - -And we don't get bans : - -```bash -$ tail -f /var/log/crowdsec.log -... -^C -$ {{v0X.cli.bin}} ban list -No local decisions. -And 21 records from API, 15 distinct AS, 12 distinct countries - -``` - -Here, we don't get *any* logs, as the event have been discarded at parsing time. - - -## Create whitelist by expression - -Now, let's make something more tricky : let's whitelist a **specific** user-agent (of course, it's just an example, don't do this at home !). The [hub's taxonomy](https://hub.crowdsec.net/fields) will helps us to find which data is present in which field. - -Let's change our whitelist to : - -```yaml -name: crowdsecurity/whitelists -description: "Whitelist events from private ipv4 addresses" -whitelist: - reason: "private ipv4 ranges" - expression: - - evt.Parsed.http_user_agent == 'MySecretUserAgent' -``` - -again, let's restart {{v0X.crowdsec.name}} ! - -For the record, I edited nikto's configuration to use 'MySecretUserAgent' as user-agent, and thus : - -```bash -nikto -host myfqdn.com -``` - -```bash -$ tail -f /var/log/crowdsec.log -... -time="07-05-2020 09:39:09" level=info msg="Event is whitelisted by Expr !" filter= name=solitary-leaf stage=s02-enrich -... -``` - - -# Whitelist in PostOverflows - -Whitelists in PostOverflows are applied *after* the bucket overflow happens. -It has the advantage of being triggered only once we are about to take decision about an IP or Range, and thus happens a lot less often. - -A good example is the [crowdsecurity/whitelist-good-actors](https://hub.crowdsec.net/author/crowdsecurity/collections/whitelist-good-actors) collection. - -But let's craft ours based on our previous example ! -First of all, install the [crowdsecurity/rdns postoverflow](https://hub.crowdsec.net/author/crowdsecurity/configurations/rdns) : it will be in charge of enriching overflows with reverse dns information of the offending IP. - -Let's put the following file in `/etc/crowdsec/config/postoverflows/s01-whitelists/mywhitelists.yaml` : - -```yaml -name: me/my_cool_whitelist -description: lets whitelist our own reverse dns -whitelist: - reason: dont ban my ISP - expression: - #this is the reverse of my ip, you can get it by performing a "host" command on your public IP for example - - evt.Enriched.reverse_dns endsWith '.asnieres.rev.numericable.fr.' -``` - -After reloading {{v0X.crowdsec.name}}, and launching (again!) nikto : - -```bash -nikto -host myfqdn.com -``` - - -```bash -$ tail -f /var/log/crowdsec.log -ime="07-07-2020 17:11:09" level=info msg="Ban for 80.x.x.x whitelisted, reason [dont ban my ISP]" id=cold-sunset name=me/my_cool_whitelist stage=s01 -time="07-07-2020 17:11:09" level=info msg="node warning : no remediation" bucket_id=blue-cloud event_time="2020-07-07 17:11:09.175068053 +0200 CEST m=+2308.040825320" scenario=crowdsecurity/http-probing source_ip=80.x.x.x -time="07-07-2020 17:11:09" level=info msg="Processing Overflow with no decisions 80.x.x.x performed 'crowdsecurity/http-probing' (11 events over 313.983994ms) at 2020-07-07 17:11:09.175068053 +0200 CEST m=+2308.040825320" bucket_id=blue-cloud event_time="2020-07-07 17:11:09.175068053 +0200 CEST m=+2308.040825320" scenario=crowdsecurity/http-probing source_ip=80.x.x.x -... - -``` - -This time, we can see that logs are being produced when the event is discarded. - diff --git a/docs/v0.3.X/mkdocs.yml b/docs/v0.3.X/mkdocs.yml deleted file mode 100644 index 3af333e7f..000000000 --- a/docs/v0.3.X/mkdocs.yml +++ /dev/null @@ -1,55 +0,0 @@ -site_name: Crowdsec/v0 -nav: - - Home: index.md - - Getting Started: - - Concepts & Glossary : getting_started/concepts.md - - Install Crowdsec : getting_started/installation.md - - Crowdsec Tour: getting_started/crowdsec-tour.md - - Guide: - - Overview: guide/crowdsec/overview.md - - Acquisition: guide/crowdsec/acquisition.md - - Parsers: guide/crowdsec/parsers.md - - Enrichers: guide/crowdsec/enrichers.md - - Scenarios: guide/crowdsec/scenarios.md - - Cscli: guide/cscli.md - - Simulation Mode: guide/crowdsec/simulation.md - - Cheat Sheets: - - Ban Management: cheat_sheets/ban-mgmt.md - - Configuration Management: cheat_sheets/config-mgmt.md - - Debugging Parsers & Scenarios: cheat_sheets/debugging_configs.md - - Observability: - - Overview: observability/overview.md - - Logs: observability/logs.md - - Metrics: - - Prometheus: observability/prometheus.md - - Command line: observability/command_line.md - - Dashboard: observability/dashboard.md - - References: - - Parsers format: references/parsers.md - - Scenarios format: references/scenarios.md - - Outputs format: references/output.md - - Write Configurations: - - Requirements: write_configurations/requirements.md - - Acquisition: write_configurations/acquisition.md - - Parsers: write_configurations/parsers.md - - Scenarios: write_configurations/scenarios.md - - Whitelists: write_configurations/whitelist.md - - Expressions: write_configurations/expressions.md - - bouncers: bouncers/index.md - - Contributing: - - Writing Output Plugins: references/plugins_api.md - - Cscli commands: - - Cscli: cscli/cscli.md - - API: cscli/cscli_api.md - - Backup: cscli/cscli_backup.md - - Bans: cscli/cscli_ban.md - - Metrics: cscli/cscli_metrics.md - - Update: cscli/cscli_update.md - - Install configurations: cscli/cscli_install.md - - Remove configurations: cscli/cscli_remove.md - - Upgrade configurations: cscli/cscli_upgrade.md - - List configurations: cscli/cscli_list.md - - Inspect configurations: cscli/cscli_inspect.md - - Manage simulation: cscli/cscli_simulation.md - - Dashboard: cscli/cscli_dashboard.md - - Upgrade V0.X to V1.X: migration.md diff --git a/docs/v1.X/docs/assets/images/blocker-installation.gif b/docs/v1.X/docs/assets/images/blocker-installation.gif deleted file mode 100644 index 9846e97fc..000000000 Binary files a/docs/v1.X/docs/assets/images/blocker-installation.gif and /dev/null differ diff --git a/docs/v1.X/docs/assets/images/crowdsec2.png b/docs/v1.X/docs/assets/images/crowdsec2.png deleted file mode 100644 index bbf619a73..000000000 Binary files a/docs/v1.X/docs/assets/images/crowdsec2.png and /dev/null differ diff --git a/docs/v1.X/docs/assets/images/crowdsec_architecture.png b/docs/v1.X/docs/assets/images/crowdsec_architecture.png deleted file mode 100644 index 94426bdfb..000000000 Binary files a/docs/v1.X/docs/assets/images/crowdsec_architecture.png and /dev/null differ diff --git a/docs/v1.X/docs/assets/images/crowdsec_install.gif b/docs/v1.X/docs/assets/images/crowdsec_install.gif deleted file mode 100644 index ceddd7f1d..000000000 Binary files a/docs/v1.X/docs/assets/images/crowdsec_install.gif and /dev/null differ diff --git a/docs/v1.X/docs/assets/images/crowdsec_logo1.png b/docs/v1.X/docs/assets/images/crowdsec_logo1.png deleted file mode 100644 index c9142c134..000000000 Binary files a/docs/v1.X/docs/assets/images/crowdsec_logo1.png and /dev/null differ diff --git a/docs/v1.X/docs/assets/images/cscli-metabase.gif b/docs/v1.X/docs/assets/images/cscli-metabase.gif deleted file mode 100644 index b21d41191..000000000 Binary files a/docs/v1.X/docs/assets/images/cscli-metabase.gif and /dev/null differ diff --git a/docs/v1.X/docs/assets/images/dashboard_view.png b/docs/v1.X/docs/assets/images/dashboard_view.png deleted file mode 100644 index 6db945c8c..000000000 Binary files a/docs/v1.X/docs/assets/images/dashboard_view.png and /dev/null differ diff --git a/docs/v1.X/docs/assets/images/dashboard_view2.png b/docs/v1.X/docs/assets/images/dashboard_view2.png deleted file mode 100644 index 6a91381eb..000000000 Binary files a/docs/v1.X/docs/assets/images/dashboard_view2.png and /dev/null differ diff --git a/docs/v1.X/docs/assets/images/grafana_details.png b/docs/v1.X/docs/assets/images/grafana_details.png deleted file mode 100644 index bf6b504f5..000000000 Binary files a/docs/v1.X/docs/assets/images/grafana_details.png and /dev/null differ diff --git a/docs/v1.X/docs/assets/images/grafana_insight.png b/docs/v1.X/docs/assets/images/grafana_insight.png deleted file mode 100644 index 8a1c6af85..000000000 Binary files a/docs/v1.X/docs/assets/images/grafana_insight.png and /dev/null differ diff --git a/docs/v1.X/docs/assets/images/grafana_overview.png b/docs/v1.X/docs/assets/images/grafana_overview.png deleted file mode 100644 index 52de69b81..000000000 Binary files a/docs/v1.X/docs/assets/images/grafana_overview.png and /dev/null differ diff --git a/docs/v1.X/docs/assets/images/out-of-the-box-protection.gif b/docs/v1.X/docs/assets/images/out-of-the-box-protection.gif deleted file mode 100644 index a309f794c..000000000 Binary files a/docs/v1.X/docs/assets/images/out-of-the-box-protection.gif and /dev/null differ diff --git a/docs/v1.X/docs/bouncers/index.md b/docs/v1.X/docs/bouncers/index.md deleted file mode 100644 index 199982fa2..000000000 --- a/docs/v1.X/docs/bouncers/index.md +++ /dev/null @@ -1,27 +0,0 @@ -# Bouncers - - -{{v1X.bouncers.Name}} are standalone software pieces in charge of acting upon a decision taken by crowdsec : block an IP, present a captcha, enforce MFA on a given user, etc. - -They can either be within the applicative stack, or work out of band : - -[nginx bouncer](https://github.com/crowdsecurity/cs-nginx-bouncer) will check every unknown IP against the local API before letting go through or serving a *403* to the user, while a [firewall bouncer](https://github.com/crowdsecurity/cs-firewall-bouncer) will simply "add" malevolent IPs to nftables/ipset set of blacklisted IPs. - -Bouncers rely on [crowdsec's Local API](/Crowdsec/v1/localAPI/) to be able to get informations about a given IP or such. - - -You can explore [available {{v1X.bouncers.name}} on the hub]({{v1X.hub.bouncers_url}}). - - -To be able for your {{v1X.bouncers.Name}} to communicate with the local API, you have to generate an API token with `cscli` and put it in your {{v1X.bouncers.Name}} configuration file: - -```bash -$ sudo cscli bouncers add testBouncer -Api key for 'testBouncer': - - 6dcfe93f18675265e905aef390330a35 - -Please keep this key since you will not be able to retrive it! -``` - -Note: this command must be run on the server where the local API is installed (or at least with a cscli that has valid credentials to communicate with the database used by the API). \ No newline at end of file diff --git a/docs/v1.X/docs/cscli/cscli.md b/docs/v1.X/docs/cscli/cscli.md deleted file mode 100644 index d17418f3c..000000000 --- a/docs/v1.X/docs/cscli/cscli.md +++ /dev/null @@ -1,43 +0,0 @@ -## cscli - -cscli allows you to manage crowdsec - -### Synopsis - -cscli is the main command to interact with your crowdsec service, scenarios & db. -It is meant to allow you to manage bans, parsers/scenarios/etc, api and generally manage you crowdsec setup. - -### Options - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - -o, --output string Output format : human, json, raw. - --debug Set logging to debug. - --info Set logging to info. - --warning Set logging to warning. - --error Set logging to error. - --trace Set logging to trace. - -h, --help help for cscli -``` - -### SEE ALSO - -* [cscli alerts](cscli_alerts.md) - Manage alerts -* [cscli bouncers](cscli_bouncers.md) - Manage bouncers [requires local API] -* [cscli capi](cscli_capi.md) - Manage interaction with Central API (CAPI) -* [cscli collections](cscli_collections.md) - Manage collections from hub -* [cscli completion](cscli_completion.md) - Generate completion script -* [cscli config](cscli_config.md) - Allows to view current config -* [cscli dashboard](cscli_dashboard.md) - Manage your metabase dashboard container [requires local API] -* [cscli decisions](cscli_decisions.md) - Manage decisions -* [cscli hub](cscli_hub.md) - Manage Hub -* [cscli lapi](cscli_lapi.md) - Manage interaction with Local API (LAPI) -* [cscli machines](cscli_machines.md) - Manage local API machines [requires local API] -* [cscli metrics](cscli_metrics.md) - Display crowdsec prometheus metrics. -* [cscli parsers](cscli_parsers.md) - Install/Remove/Upgrade/Inspect parser(s) from hub -* [cscli postoverflows](cscli_postoverflows.md) - Install/Remove/Upgrade/Inspect postoverflow(s) from hub -* [cscli scenarios](cscli_scenarios.md) - Install/Remove/Upgrade/Inspect scenario(s) from hub -* [cscli simulation](cscli_simulation.md) - Manage simulation status of scenarios -* [cscli version](cscli_version.md) - Display version and exit. - - diff --git a/docs/v1.X/docs/cscli/cscli_alerts.md b/docs/v1.X/docs/cscli/cscli_alerts.md deleted file mode 100644 index 51aac6ca5..000000000 --- a/docs/v1.X/docs/cscli/cscli_alerts.md +++ /dev/null @@ -1,31 +0,0 @@ -## cscli alerts - -Manage alerts - -### Options - -``` - -h, --help help for alerts -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli alerts delete](cscli_alerts_delete.md) - Delete alerts -/!\ This command can be use only on the same machine than the local API. -* [cscli alerts inspect](cscli_alerts_inspect.md) - Show info about an alert -* [cscli alerts list](cscli_alerts_list.md) - List alerts - - diff --git a/docs/v1.X/docs/cscli/cscli_alerts_delete.md b/docs/v1.X/docs/cscli/cscli_alerts_delete.md deleted file mode 100644 index 103716391..000000000 --- a/docs/v1.X/docs/cscli/cscli_alerts_delete.md +++ /dev/null @@ -1,47 +0,0 @@ -## cscli alerts delete - -Delete alerts -/!\ This command can be use only on the same machine than the local API. - -``` -cscli alerts delete [filters] [--all] [flags] -``` - -### Examples - -``` -cscli alerts delete --ip 1.2.3.4 -cscli alerts delete --range 1.2.3.0/24 -cscli alerts delete -s crowdsecurity/ssh-bf" -``` - -### Options - -``` - --scope string the scope (ie. ip,range) - -v, --value string the value to match for in the specified scope - -s, --scenario string the scenario (ie. crowdsecurity/ssh-bf) - -i, --ip string Source ip (shorthand for --scope ip --value ) - -r, --range string Range source ip (shorthand for --scope range --value ) - -a, --all delete all alerts - --contained query decisions contained by range - -h, --help help for delete -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli alerts](cscli_alerts.md) - Manage alerts - - diff --git a/docs/v1.X/docs/cscli/cscli_alerts_inspect.md b/docs/v1.X/docs/cscli/cscli_alerts_inspect.md deleted file mode 100644 index 71cad63ed..000000000 --- a/docs/v1.X/docs/cscli/cscli_alerts_inspect.md +++ /dev/null @@ -1,38 +0,0 @@ -## cscli alerts inspect - -Show info about an alert - -``` -cscli alerts inspect [flags] -``` - -### Examples - -``` -cscli alerts inspect 123 -``` - -### Options - -``` - -d, --details show alerts with events - -h, --help help for inspect -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli alerts](cscli_alerts.md) - Manage alerts - - diff --git a/docs/v1.X/docs/cscli/cscli_alerts_list.md b/docs/v1.X/docs/cscli/cscli_alerts_list.md deleted file mode 100644 index 5d5ef7c45..000000000 --- a/docs/v1.X/docs/cscli/cscli_alerts_list.md +++ /dev/null @@ -1,52 +0,0 @@ -## cscli alerts list - -List alerts - -``` -cscli alerts list [filters] [flags] -``` - -### Examples - -``` -cscli alerts list -cscli alerts list --ip 1.2.3.4 -cscli alerts list --range 1.2.3.0/24 -cscli alerts list -s crowdsecurity/ssh-bf -cscli alerts list --type ban -``` - -### Options - -``` - --until string restrict to alerts older than until (ie. 4h, 30d) - --since string restrict to alerts newer than since (ie. 4h, 30d) - -i, --ip string restrict to alerts from this source ip (shorthand for --scope ip --value ) - -s, --scenario string the scenario (ie. crowdsecurity/ssh-bf) - -r, --range string restrict to alerts from this range (shorthand for --scope range --value ) - --type string restrict to alerts with given decision type (ie. ban, captcha) - --scope string restrict to alerts of this scope (ie. ip,range) - -v, --value string the value to match for in the specified scope - --contained query decisions contained by range - -m, --machine print machines that sended alerts - -l, --limit int limit size of alerts list table (0 to view all alerts) (default 50) - -h, --help help for list -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli alerts](cscli_alerts.md) - Manage alerts - - diff --git a/docs/v1.X/docs/cscli/cscli_bouncers.md b/docs/v1.X/docs/cscli/cscli_bouncers.md deleted file mode 100644 index 1dc772a43..000000000 --- a/docs/v1.X/docs/cscli/cscli_bouncers.md +++ /dev/null @@ -1,36 +0,0 @@ -## cscli bouncers - -Manage bouncers [requires local API] - -### Synopsis - -To list/add/delete bouncers. -Note: This command requires database direct access, so is intended to be run on Local API/master. - - -### Options - -``` - -h, --help help for bouncers -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli bouncers add](cscli_bouncers_add.md) - add bouncer -* [cscli bouncers delete](cscli_bouncers_delete.md) - delete bouncer -* [cscli bouncers list](cscli_bouncers_list.md) - List bouncers - - diff --git a/docs/v1.X/docs/cscli/cscli_bouncers_add.md b/docs/v1.X/docs/cscli/cscli_bouncers_add.md deleted file mode 100644 index b2d4eaa45..000000000 --- a/docs/v1.X/docs/cscli/cscli_bouncers_add.md +++ /dev/null @@ -1,43 +0,0 @@ -## cscli bouncers add - -add bouncer - -### Synopsis - -add bouncer - -``` -cscli bouncers add MyBouncerName [--length 16] [flags] -``` - -### Examples - -``` -cscli bouncers add MyBouncerName -cscli bouncers add MyBouncerName -l 24 -``` - -### Options - -``` - -h, --help help for add - -l, --length int length of the api key (default 16) -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli bouncers](cscli_bouncers.md) - Manage bouncers [requires local API] - - diff --git a/docs/v1.X/docs/cscli/cscli_bouncers_delete.md b/docs/v1.X/docs/cscli/cscli_bouncers_delete.md deleted file mode 100644 index ab93a0ae4..000000000 --- a/docs/v1.X/docs/cscli/cscli_bouncers_delete.md +++ /dev/null @@ -1,31 +0,0 @@ -## cscli bouncers delete - -delete bouncer - -``` -cscli bouncers delete MyBouncerName [flags] -``` - -### Options - -``` - -h, --help help for delete -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli bouncers](cscli_bouncers.md) - Manage bouncers [requires local API] - - diff --git a/docs/v1.X/docs/cscli/cscli_bouncers_list.md b/docs/v1.X/docs/cscli/cscli_bouncers_list.md deleted file mode 100644 index c2460dc5f..000000000 --- a/docs/v1.X/docs/cscli/cscli_bouncers_list.md +++ /dev/null @@ -1,41 +0,0 @@ -## cscli bouncers list - -List bouncers - -### Synopsis - -List bouncers - -``` -cscli bouncers list [flags] -``` - -### Examples - -``` -cscli bouncers list -``` - -### Options - -``` - -h, --help help for list -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli bouncers](cscli_bouncers.md) - Manage bouncers [requires local API] - - diff --git a/docs/v1.X/docs/cscli/cscli_capi.md b/docs/v1.X/docs/cscli/cscli_capi.md deleted file mode 100644 index 4b535b5c0..000000000 --- a/docs/v1.X/docs/cscli/cscli_capi.md +++ /dev/null @@ -1,29 +0,0 @@ -## cscli capi - -Manage interaction with Central API (CAPI) - -### Options - -``` - -h, --help help for capi -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli capi register](cscli_capi_register.md) - Register to Central API (CAPI) -* [cscli capi status](cscli_capi_status.md) - Check status with the Central API (CAPI) - - diff --git a/docs/v1.X/docs/cscli/cscli_capi_register.md b/docs/v1.X/docs/cscli/cscli_capi_register.md deleted file mode 100644 index 2ea7fd0a3..000000000 --- a/docs/v1.X/docs/cscli/cscli_capi_register.md +++ /dev/null @@ -1,32 +0,0 @@ -## cscli capi register - -Register to Central API (CAPI) - -``` -cscli capi register [flags] -``` - -### Options - -``` - -f, --file string output file destination - -h, --help help for register -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli capi](cscli_capi.md) - Manage interaction with Central API (CAPI) - - diff --git a/docs/v1.X/docs/cscli/cscli_capi_status.md b/docs/v1.X/docs/cscli/cscli_capi_status.md deleted file mode 100644 index 783f64b1b..000000000 --- a/docs/v1.X/docs/cscli/cscli_capi_status.md +++ /dev/null @@ -1,31 +0,0 @@ -## cscli capi status - -Check status with the Central API (CAPI) - -``` -cscli capi status [flags] -``` - -### Options - -``` - -h, --help help for status -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli capi](cscli_capi.md) - Manage interaction with Central API (CAPI) - - diff --git a/docs/v1.X/docs/cscli/cscli_collections.md b/docs/v1.X/docs/cscli/cscli_collections.md deleted file mode 100644 index 61646547b..000000000 --- a/docs/v1.X/docs/cscli/cscli_collections.md +++ /dev/null @@ -1,36 +0,0 @@ -## cscli collections - -Manage collections from hub - -### Synopsis - -Install/Remove/Upgrade/Inspect collections from the CrowdSec Hub. - -### Options - -``` - -h, --help help for collections -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli collections inspect](cscli_collections_inspect.md) - Inspect given collection -* [cscli collections install](cscli_collections_install.md) - Install given collection(s) -* [cscli collections list](cscli_collections_list.md) - List all collections or given one -* [cscli collections remove](cscli_collections_remove.md) - Remove given collection(s) -* [cscli collections upgrade](cscli_collections_upgrade.md) - Upgrade given collection(s) - - diff --git a/docs/v1.X/docs/cscli/cscli_collections_inspect.md b/docs/v1.X/docs/cscli/cscli_collections_inspect.md deleted file mode 100644 index 873479baf..000000000 --- a/docs/v1.X/docs/cscli/cscli_collections_inspect.md +++ /dev/null @@ -1,42 +0,0 @@ -## cscli collections inspect - -Inspect given collection - -### Synopsis - -Inspect given collection - -``` -cscli collections inspect collection [flags] -``` - -### Examples - -``` -cscli collections inspect crowdsec/xxx crowdsec/xyz -``` - -### Options - -``` - -h, --help help for inspect - -u, --url string Prometheus url -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli collections](cscli_collections.md) - Manage collections from hub - - diff --git a/docs/v1.X/docs/cscli/cscli_collections_install.md b/docs/v1.X/docs/cscli/cscli_collections_install.md deleted file mode 100644 index 0dd11d460..000000000 --- a/docs/v1.X/docs/cscli/cscli_collections_install.md +++ /dev/null @@ -1,43 +0,0 @@ -## cscli collections install - -Install given collection(s) - -### Synopsis - -Fetch and install given collection(s) from hub - -``` -cscli collections install collection [flags] -``` - -### Examples - -``` -cscli collections install crowdsec/xxx crowdsec/xyz -``` - -### Options - -``` - -d, --download-only Only download packages, don't enable - --force Force install : Overwrite tainted and outdated files - -h, --help help for install -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli collections](cscli_collections.md) - Manage collections from hub - - diff --git a/docs/v1.X/docs/cscli/cscli_collections_list.md b/docs/v1.X/docs/cscli/cscli_collections_list.md deleted file mode 100644 index 09425245d..000000000 --- a/docs/v1.X/docs/cscli/cscli_collections_list.md +++ /dev/null @@ -1,42 +0,0 @@ -## cscli collections list - -List all collections or given one - -### Synopsis - -List all collections or given one - -``` -cscli collections list collection [-a] [flags] -``` - -### Examples - -``` -cscli collections list -``` - -### Options - -``` - -a, --all List as well disabled items - -h, --help help for list -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli collections](cscli_collections.md) - Manage collections from hub - - diff --git a/docs/v1.X/docs/cscli/cscli_collections_remove.md b/docs/v1.X/docs/cscli/cscli_collections_remove.md deleted file mode 100644 index bb6d6b985..000000000 --- a/docs/v1.X/docs/cscli/cscli_collections_remove.md +++ /dev/null @@ -1,44 +0,0 @@ -## cscli collections remove - -Remove given collection(s) - -### Synopsis - -Remove given collection(s) from hub - -``` -cscli collections remove collection [flags] -``` - -### Examples - -``` -cscli collections remove crowdsec/xxx crowdsec/xyz -``` - -### Options - -``` - --all Delete all the collections - --force Force remove : Remove tainted and outdated files - -h, --help help for remove - --purge Delete source file too -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli collections](cscli_collections.md) - Manage collections from hub - - diff --git a/docs/v1.X/docs/cscli/cscli_collections_upgrade.md b/docs/v1.X/docs/cscli/cscli_collections_upgrade.md deleted file mode 100644 index d4d415122..000000000 --- a/docs/v1.X/docs/cscli/cscli_collections_upgrade.md +++ /dev/null @@ -1,43 +0,0 @@ -## cscli collections upgrade - -Upgrade given collection(s) - -### Synopsis - -Fetch and upgrade given collection(s) from hub - -``` -cscli collections upgrade collection [flags] -``` - -### Examples - -``` -cscli collections upgrade crowdsec/xxx crowdsec/xyz -``` - -### Options - -``` - -a, --all Upgrade all the collections - --force Force upgrade : Overwrite tainted and outdated files - -h, --help help for upgrade -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli collections](cscli_collections.md) - Manage collections from hub - - diff --git a/docs/v1.X/docs/cscli/cscli_completion.md b/docs/v1.X/docs/cscli/cscli_completion.md deleted file mode 100644 index 48c38841e..000000000 --- a/docs/v1.X/docs/cscli/cscli_completion.md +++ /dev/null @@ -1,64 +0,0 @@ -## cscli completion - -Generate completion script - -### Synopsis - -To load completions: - -### Bash: - - $ source <(cscli completion bash) - - # To load completions for each session, execute once: - - - # Linux: - - $ cscli completion bash | sudo tee /etc/bash_completion.d/cscli - - # macOS: - - $ cscli completion bash | sudo tee /usr/local/etc/bash_completion.d/cscli - -### Zsh: - - # If shell completion is not already enabled in your environment, - # you will need to enable it. You can execute the following once: - - $ echo "autoload -U compinit; compinit" >> ~/.zshrc - - # To load completions for each session, execute once: - - $ cscli completion zsh > "${fpath[1]}/_cscli" - - # You will need to start a new shell for this setup to take effect. - - -``` -cscli completion [bash|zsh] -``` - -### Options - -``` - -h, --help help for completion -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec - - diff --git a/docs/v1.X/docs/cscli/cscli_config.md b/docs/v1.X/docs/cscli/cscli_config.md deleted file mode 100644 index 84361b975..000000000 --- a/docs/v1.X/docs/cscli/cscli_config.md +++ /dev/null @@ -1,30 +0,0 @@ -## cscli config - -Allows to view current config - -### Options - -``` - -h, --help help for config -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli config backup](cscli_config_backup.md) - Backup current config -* [cscli config restore](cscli_config_restore.md) - Restore config in backup -* [cscli config show](cscli_config_show.md) - Displays current config - - diff --git a/docs/v1.X/docs/cscli/cscli_config_backup.md b/docs/v1.X/docs/cscli/cscli_config_backup.md deleted file mode 100644 index e59c219f3..000000000 --- a/docs/v1.X/docs/cscli/cscli_config_backup.md +++ /dev/null @@ -1,48 +0,0 @@ -## cscli config backup - -Backup current config - -### Synopsis - -Backup the current crowdsec configuration including : - -- Main config (config.yaml) -- Simulation config (simulation.yaml) -- Profiles config (profiles.yaml) -- List of scenarios, parsers, postoverflows and collections that are up-to-date -- Tainted/local/out-of-date scenarios, parsers, postoverflows and collections -- Backup of API credentials (local API and online API) - -``` -cscli config backup [flags] -``` - -### Examples - -``` -cscli config backup ./my-backup -``` - -### Options - -``` - -h, --help help for backup -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli config](cscli_config.md) - Allows to view current config - - diff --git a/docs/v1.X/docs/cscli/cscli_config_restore.md b/docs/v1.X/docs/cscli/cscli_config_restore.md deleted file mode 100644 index 17d18b78a..000000000 --- a/docs/v1.X/docs/cscli/cscli_config_restore.md +++ /dev/null @@ -1,43 +0,0 @@ -## cscli config restore - -Restore config in backup - -### Synopsis - -Restore the crowdsec configuration from specified backup including: - -- Main config (config.yaml) -- Simulation config (simulation.yaml) -- Profiles config (profiles.yaml) -- List of scenarios, parsers, postoverflows and collections that are up-to-date -- Tainted/local/out-of-date scenarios, parsers, postoverflows and collections -- Backup of API credentials (local API and online API) - -``` -cscli config restore [flags] -``` - -### Options - -``` - -h, --help help for restore - --old-backup To use when you are upgrading crowdsec v0.X to v1.X and you need to restore backup from v0.X -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli config](cscli_config.md) - Allows to view current config - - diff --git a/docs/v1.X/docs/cscli/cscli_config_show.md b/docs/v1.X/docs/cscli/cscli_config_show.md deleted file mode 100644 index 0b2711ce5..000000000 --- a/docs/v1.X/docs/cscli/cscli_config_show.md +++ /dev/null @@ -1,35 +0,0 @@ -## cscli config show - -Displays current config - -### Synopsis - -Displays the current cli configuration. - -``` -cscli config show [flags] -``` - -### Options - -``` - -h, --help help for show -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli config](cscli_config.md) - Allows to view current config - - diff --git a/docs/v1.X/docs/cscli/cscli_dashboard.md b/docs/v1.X/docs/cscli/cscli_dashboard.md deleted file mode 100644 index ac936e798..000000000 --- a/docs/v1.X/docs/cscli/cscli_dashboard.md +++ /dev/null @@ -1,48 +0,0 @@ -## cscli dashboard - -Manage your metabase dashboard container [requires local API] - -### Synopsis - -Install/Start/Stop/Remove a metabase container exposing dashboard and metrics. -Note: This command requires database direct access, so is intended to be run on Local API/master. - - -### Examples - -``` - -cscli dashboard setup -cscli dashboard start -cscli dashboard stop -cscli dashboard remove - -``` - -### Options - -``` - -h, --help help for dashboard -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli dashboard remove](cscli_dashboard_remove.md) - removes the metabase container. -* [cscli dashboard setup](cscli_dashboard_setup.md) - Setup a metabase container. -* [cscli dashboard start](cscli_dashboard_start.md) - Start the metabase container. -* [cscli dashboard stop](cscli_dashboard_stop.md) - Stops the metabase container. - - diff --git a/docs/v1.X/docs/cscli/cscli_dashboard_remove.md b/docs/v1.X/docs/cscli/cscli_dashboard_remove.md deleted file mode 100644 index ab62e7452..000000000 --- a/docs/v1.X/docs/cscli/cscli_dashboard_remove.md +++ /dev/null @@ -1,46 +0,0 @@ -## cscli dashboard remove - -removes the metabase container. - -### Synopsis - -removes the metabase container using docker. - -``` -cscli dashboard remove [flags] -``` - -### Examples - -``` - -cscli dashboard remove -cscli dashboard remove --force - -``` - -### Options - -``` - -f, --force Remove also the metabase image - -h, --help help for remove - -y, --yes force yes -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli dashboard](cscli_dashboard.md) - Manage your metabase dashboard container [requires local API] - - diff --git a/docs/v1.X/docs/cscli/cscli_dashboard_setup.md b/docs/v1.X/docs/cscli/cscli_dashboard_setup.md deleted file mode 100644 index 20ce9b748..000000000 --- a/docs/v1.X/docs/cscli/cscli_dashboard_setup.md +++ /dev/null @@ -1,51 +0,0 @@ -## cscli dashboard setup - -Setup a metabase container. - -### Synopsis - -Perform a metabase docker setup, download standard dashboards, create a fresh user and start the container - -``` -cscli dashboard setup [flags] -``` - -### Examples - -``` - -cscli dashboard setup -cscli dashboard setup --listen 0.0.0.0 -cscli dashboard setup -l 0.0.0.0 -p 443 --password - -``` - -### Options - -``` - -d, --dir string Shared directory with metabase container. - -f, --force Force setup : override existing files. - -h, --help help for setup - -l, --listen string Listen address of container (default "127.0.0.1") - --password string metabase password - -p, --port string Listen port of container (default "3000") - -y, --yes force yes -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli dashboard](cscli_dashboard.md) - Manage your metabase dashboard container [requires local API] - - diff --git a/docs/v1.X/docs/cscli/cscli_dashboard_start.md b/docs/v1.X/docs/cscli/cscli_dashboard_start.md deleted file mode 100644 index 004dff993..000000000 --- a/docs/v1.X/docs/cscli/cscli_dashboard_start.md +++ /dev/null @@ -1,35 +0,0 @@ -## cscli dashboard start - -Start the metabase container. - -### Synopsis - -Stats the metabase container using docker. - -``` -cscli dashboard start [flags] -``` - -### Options - -``` - -h, --help help for start -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli dashboard](cscli_dashboard.md) - Manage your metabase dashboard container [requires local API] - - diff --git a/docs/v1.X/docs/cscli/cscli_dashboard_stop.md b/docs/v1.X/docs/cscli/cscli_dashboard_stop.md deleted file mode 100644 index d5360c05e..000000000 --- a/docs/v1.X/docs/cscli/cscli_dashboard_stop.md +++ /dev/null @@ -1,35 +0,0 @@ -## cscli dashboard stop - -Stops the metabase container. - -### Synopsis - -Stops the metabase container using docker. - -``` -cscli dashboard stop [flags] -``` - -### Options - -``` - -h, --help help for stop -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli dashboard](cscli_dashboard.md) - Manage your metabase dashboard container [requires local API] - - diff --git a/docs/v1.X/docs/cscli/cscli_decisions.md b/docs/v1.X/docs/cscli/cscli_decisions.md deleted file mode 100644 index cd7798792..000000000 --- a/docs/v1.X/docs/cscli/cscli_decisions.md +++ /dev/null @@ -1,40 +0,0 @@ -## cscli decisions - -Manage decisions - -### Synopsis - -Add/List/Delete decisions from LAPI - -### Examples - -``` -cscli decisions [action] [filter] -``` - -### Options - -``` - -h, --help help for decisions -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli decisions add](cscli_decisions_add.md) - Add decision to LAPI -* [cscli decisions delete](cscli_decisions_delete.md) - Delete decisions -* [cscli decisions list](cscli_decisions_list.md) - List decisions from LAPI - - diff --git a/docs/v1.X/docs/cscli/cscli_decisions_add.md b/docs/v1.X/docs/cscli/cscli_decisions_add.md deleted file mode 100644 index 34bc9567f..000000000 --- a/docs/v1.X/docs/cscli/cscli_decisions_add.md +++ /dev/null @@ -1,48 +0,0 @@ -## cscli decisions add - -Add decision to LAPI - -``` -cscli decisions add [options] [flags] -``` - -### Examples - -``` -cscli decisions add --ip 1.2.3.4 -cscli decisions add --range 1.2.3.0/24 -cscli decisions add --ip 1.2.3.4 --duration 24h --type captcha -cscli decisions add --scope username --value foobar - -``` - -### Options - -``` - -i, --ip string Source ip (shorthand for --scope ip --value ) - -r, --range string Range source ip (shorthand for --scope range --value ) - -d, --duration string Decision duration (ie. 1h,4h,30m) (default "4h") - -v, --value string The value (ie. --scope username --value foobar) - --scope string Decision scope (ie. ip,range,username) (default "Ip") - -R, --reason string Decision reason (ie. scenario-name) - -t, --type string Decision type (ie. ban,captcha,throttle) (default "ban") - -h, --help help for add -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli decisions](cscli_decisions.md) - Manage decisions - - diff --git a/docs/v1.X/docs/cscli/cscli_decisions_delete.md b/docs/v1.X/docs/cscli/cscli_decisions_delete.md deleted file mode 100644 index e7747ecfe..000000000 --- a/docs/v1.X/docs/cscli/cscli_decisions_delete.md +++ /dev/null @@ -1,49 +0,0 @@ -## cscli decisions delete - -Delete decisions - -``` -cscli decisions delete [options] [flags] -``` - -### Examples - -``` -cscli decisions delete -r 1.2.3.0/24 -cscli decisions delete -i 1.2.3.4 -cscli decisions delete -s crowdsecurity/ssh-bf -cscli decisions delete --id 42 -cscli decisions delete --type captcha - -``` - -### Options - -``` - -i, --ip string Source ip (shorthand for --scope ip --value ) - -r, --range string Range source ip (shorthand for --scope range --value ) - --id string decision id - -t, --type string the decision type (ie. ban,captcha) - -v, --value string the value to match for in the specified scope - --all delete all decisions - --contained query decisions contained by range - -h, --help help for delete -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli decisions](cscli_decisions.md) - Manage decisions - - diff --git a/docs/v1.X/docs/cscli/cscli_decisions_list.md b/docs/v1.X/docs/cscli/cscli_decisions_list.md deleted file mode 100644 index 1c1b16981..000000000 --- a/docs/v1.X/docs/cscli/cscli_decisions_list.md +++ /dev/null @@ -1,52 +0,0 @@ -## cscli decisions list - -List decisions from LAPI - -``` -cscli decisions list [options] [flags] -``` - -### Examples - -``` -cscli decisions list -i 1.2.3.4 -cscli decisions list -r 1.2.3.0/24 -cscli decisions list -s crowdsecurity/ssh-bf -cscli decisions list -t ban - -``` - -### Options - -``` - -a, --all Include decisions from Central API - --since string restrict to alerts newer than since (ie. 4h, 30d) - --until string restrict to alerts older than until (ie. 4h, 30d) - -t, --type string restrict to this decision type (ie. ban,captcha) - --scope string restrict to this scope (ie. ip,range,session) - -v, --value string restrict to this value (ie. 1.2.3.4,userName) - -s, --scenario string restrict to this scenario (ie. crowdsecurity/ssh-bf) - -i, --ip string restrict to alerts from this source ip (shorthand for --scope ip --value ) - -r, --range string restrict to alerts from this source range (shorthand for --scope range --value ) - --no-simu exclude decisions in simulation mode - --contained query decisions contained by range - -h, --help help for list -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli decisions](cscli_decisions.md) - Manage decisions - - diff --git a/docs/v1.X/docs/cscli/cscli_hub.md b/docs/v1.X/docs/cscli/cscli_hub.md deleted file mode 100644 index 18739f7e6..000000000 --- a/docs/v1.X/docs/cscli/cscli_hub.md +++ /dev/null @@ -1,48 +0,0 @@ -## cscli hub - -Manage Hub - -### Synopsis - - -Hub management - -List/update parsers/scenarios/postoverflows/collections from [Crowdsec Hub](https://hub.crowdsec.net). -Hub is manage by cscli, to get latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update. - - -### Examples - -``` - -cscli hub list # List all installed configurations -cscli hub update # Download list of available configurations from the hub - -``` - -### Options - -``` - -h, --help help for hub -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli hub list](cscli_hub_list.md) - List installed configs -* [cscli hub update](cscli_hub_update.md) - Fetch available configs from hub -* [cscli hub upgrade](cscli_hub_upgrade.md) - Upgrade all configs installed from hub - - diff --git a/docs/v1.X/docs/cscli/cscli_hub_list.md b/docs/v1.X/docs/cscli/cscli_hub_list.md deleted file mode 100644 index 223e843a2..000000000 --- a/docs/v1.X/docs/cscli/cscli_hub_list.md +++ /dev/null @@ -1,33 +0,0 @@ -## cscli hub list - -List installed configs - -``` -cscli hub list [-a] [flags] -``` - -### Options - -``` - -a, --all List as well disabled items - -h, --help help for list -``` - -### Options inherited from parent commands - -``` - -b, --branch string Use given branch from hub - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli hub](cscli_hub.md) - Manage Hub - - diff --git a/docs/v1.X/docs/cscli/cscli_hub_update.md b/docs/v1.X/docs/cscli/cscli_hub_update.md deleted file mode 100644 index d3423d722..000000000 --- a/docs/v1.X/docs/cscli/cscli_hub_update.md +++ /dev/null @@ -1,38 +0,0 @@ -## cscli hub update - -Fetch available configs from hub - -### Synopsis - - -Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.index.json) file from hub, containing the list of available configs. - - -``` -cscli hub update [flags] -``` - -### Options - -``` - -h, --help help for update -``` - -### Options inherited from parent commands - -``` - -b, --branch string Use given branch from hub - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli hub](cscli_hub.md) - Manage Hub - - diff --git a/docs/v1.X/docs/cscli/cscli_hub_upgrade.md b/docs/v1.X/docs/cscli/cscli_hub_upgrade.md deleted file mode 100644 index d10b572f7..000000000 --- a/docs/v1.X/docs/cscli/cscli_hub_upgrade.md +++ /dev/null @@ -1,39 +0,0 @@ -## cscli hub upgrade - -Upgrade all configs installed from hub - -### Synopsis - - -Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if you want the latest versions available. - - -``` -cscli hub upgrade [flags] -``` - -### Options - -``` - --force Force upgrade : Overwrite tainted and outdated files - -h, --help help for upgrade -``` - -### Options inherited from parent commands - -``` - -b, --branch string Use given branch from hub - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli hub](cscli_hub.md) - Manage Hub - - diff --git a/docs/v1.X/docs/cscli/cscli_lapi.md b/docs/v1.X/docs/cscli/cscli_lapi.md deleted file mode 100644 index 43e2d5d05..000000000 --- a/docs/v1.X/docs/cscli/cscli_lapi.md +++ /dev/null @@ -1,29 +0,0 @@ -## cscli lapi - -Manage interaction with Local API (LAPI) - -### Options - -``` - -h, --help help for lapi -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli lapi register](cscli_lapi_register.md) - Register a machine to Local API (LAPI) -* [cscli lapi status](cscli_lapi_status.md) - Check authentication to Local API (LAPI) - - diff --git a/docs/v1.X/docs/cscli/cscli_lapi_register.md b/docs/v1.X/docs/cscli/cscli_lapi_register.md deleted file mode 100644 index 24e640f4a..000000000 --- a/docs/v1.X/docs/cscli/cscli_lapi_register.md +++ /dev/null @@ -1,39 +0,0 @@ -## cscli lapi register - -Register a machine to Local API (LAPI) - -### Synopsis - -Register you machine to the Local API (LAPI). -Keep in mind the machine needs to be validated by an administrator on LAPI side to be effective. - -``` -cscli lapi register [flags] -``` - -### Options - -``` - -f, --file string output file destination - -h, --help help for register - --machine string Name of the machine to register with - -u, --url string URL of the API (ie. http://127.0.0.1) -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli lapi](cscli_lapi.md) - Manage interaction with Local API (LAPI) - - diff --git a/docs/v1.X/docs/cscli/cscli_lapi_status.md b/docs/v1.X/docs/cscli/cscli_lapi_status.md deleted file mode 100644 index 61785b52b..000000000 --- a/docs/v1.X/docs/cscli/cscli_lapi_status.md +++ /dev/null @@ -1,31 +0,0 @@ -## cscli lapi status - -Check authentication to Local API (LAPI) - -``` -cscli lapi status [flags] -``` - -### Options - -``` - -h, --help help for status -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli lapi](cscli_lapi.md) - Manage interaction with Local API (LAPI) - - diff --git a/docs/v1.X/docs/cscli/cscli_machines.md b/docs/v1.X/docs/cscli/cscli_machines.md deleted file mode 100644 index 6af6bd969..000000000 --- a/docs/v1.X/docs/cscli/cscli_machines.md +++ /dev/null @@ -1,43 +0,0 @@ -## cscli machines - -Manage local API machines [requires local API] - -### Synopsis - -To list/add/delete/validate machines. -Note: This command requires database direct access, so is intended to be run on the local API machine. - - -### Examples - -``` -cscli machines [action] -``` - -### Options - -``` - -h, --help help for machines -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli machines add](cscli_machines_add.md) - add machine to the database. -* [cscli machines delete](cscli_machines_delete.md) - delete machines -* [cscli machines list](cscli_machines_list.md) - List machines -* [cscli machines validate](cscli_machines_validate.md) - validate a machine to access the local API - - diff --git a/docs/v1.X/docs/cscli/cscli_machines_add.md b/docs/v1.X/docs/cscli/cscli_machines_add.md deleted file mode 100644 index 41907d67c..000000000 --- a/docs/v1.X/docs/cscli/cscli_machines_add.md +++ /dev/null @@ -1,51 +0,0 @@ -## cscli machines add - -add machine to the database. - -### Synopsis - -Register a new machine in the database. cscli should be on the same machine as LAPI. - -``` -cscli machines add [flags] -``` - -### Examples - -``` - -cscli machines add --auto -cscli machines add MyTestMachine --auto -cscli machines add MyTestMachine --password MyPassword - -``` - -### Options - -``` - -a, --auto automatically generate password (and username if not provided) - -f, --file string output file destination (defaults to /etc/crowdsec/local_api_credentials.yaml) - --force will force add the machine if it already exist - -h, --help help for add - -i, --interactive interfactive mode to enter the password - -p, --password string machine password to login to the API - -u, --url string URL of the local API -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli machines](cscli_machines.md) - Manage local API machines [requires local API] - - diff --git a/docs/v1.X/docs/cscli/cscli_machines_delete.md b/docs/v1.X/docs/cscli/cscli_machines_delete.md deleted file mode 100644 index 3fa431cc9..000000000 --- a/docs/v1.X/docs/cscli/cscli_machines_delete.md +++ /dev/null @@ -1,38 +0,0 @@ -## cscli machines delete - -delete machines - -``` -cscli machines delete --machine MyTestMachine [flags] -``` - -### Examples - -``` -cscli machines delete -``` - -### Options - -``` - -h, --help help for delete - -m, --machine string machine to delete -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli machines](cscli_machines.md) - Manage local API machines [requires local API] - - diff --git a/docs/v1.X/docs/cscli/cscli_machines_list.md b/docs/v1.X/docs/cscli/cscli_machines_list.md deleted file mode 100644 index e2b793d58..000000000 --- a/docs/v1.X/docs/cscli/cscli_machines_list.md +++ /dev/null @@ -1,41 +0,0 @@ -## cscli machines list - -List machines - -### Synopsis - -List - -``` -cscli machines list [flags] -``` - -### Examples - -``` -cscli machines list -``` - -### Options - -``` - -h, --help help for list -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli machines](cscli_machines.md) - Manage local API machines [requires local API] - - diff --git a/docs/v1.X/docs/cscli/cscli_machines_register.md b/docs/v1.X/docs/cscli/cscli_machines_register.md deleted file mode 100644 index e95d90dc6..000000000 --- a/docs/v1.X/docs/cscli/cscli_machines_register.md +++ /dev/null @@ -1,44 +0,0 @@ -## cscli machines register - -register a machine to a remote API - -### Synopsis - -register a machine to a remote API. -/!\ The machine will not be validated. You have to connect on the remote API server and run 'cscli machine validate -m ' - -``` -cscli machines register -u http://127.0.0.1:8080/ [flags] -``` - -### Examples - -``` -cscli machine register -``` - -### Options - -``` - -f, --file string output file destination - -h, --help help for register - -u, --url string URL of the API -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli machines](cscli_machines.md) - Manage local API machines - - diff --git a/docs/v1.X/docs/cscli/cscli_machines_validate.md b/docs/v1.X/docs/cscli/cscli_machines_validate.md deleted file mode 100644 index e5194d1d5..000000000 --- a/docs/v1.X/docs/cscli/cscli_machines_validate.md +++ /dev/null @@ -1,41 +0,0 @@ -## cscli machines validate - -validate a machine to access the local API - -### Synopsis - -validate a machine to access the local API. - -``` -cscli machines validate [flags] -``` - -### Examples - -``` -cscli machines validate -``` - -### Options - -``` - -h, --help help for validate -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli machines](cscli_machines.md) - Manage local API machines [requires local API] - - diff --git a/docs/v1.X/docs/cscli/cscli_metrics.md b/docs/v1.X/docs/cscli/cscli_metrics.md deleted file mode 100644 index 48b47993a..000000000 --- a/docs/v1.X/docs/cscli/cscli_metrics.md +++ /dev/null @@ -1,36 +0,0 @@ -## cscli metrics - -Display crowdsec prometheus metrics. - -### Synopsis - -Fetch metrics from the prometheus server and display them in a human-friendly way - -``` -cscli metrics [flags] -``` - -### Options - -``` - -h, --help help for metrics - -u, --url string Prometheus url (http://:/metrics) -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec - - diff --git a/docs/v1.X/docs/cscli/cscli_parsers.md b/docs/v1.X/docs/cscli/cscli_parsers.md deleted file mode 100644 index 045465518..000000000 --- a/docs/v1.X/docs/cscli/cscli_parsers.md +++ /dev/null @@ -1,43 +0,0 @@ -## cscli parsers - -Install/Remove/Upgrade/Inspect parser(s) from hub - -### Examples - -``` -cscli parsers install crowdsecurity/sshd-logs -cscli parsers inspect crowdsecurity/sshd-logs -cscli parsers upgrade crowdsecurity/sshd-logs -cscli parsers list -cscli parsers remove crowdsecurity/sshd-logs - -``` - -### Options - -``` - -h, --help help for parsers -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli parsers inspect](cscli_parsers_inspect.md) - Inspect given parser -* [cscli parsers install](cscli_parsers_install.md) - Install given parser(s) -* [cscli parsers list](cscli_parsers_list.md) - List all parsers or given one -* [cscli parsers remove](cscli_parsers_remove.md) - Remove given parser(s) -* [cscli parsers upgrade](cscli_parsers_upgrade.md) - Upgrade given parser(s) - - diff --git a/docs/v1.X/docs/cscli/cscli_parsers_inspect.md b/docs/v1.X/docs/cscli/cscli_parsers_inspect.md deleted file mode 100644 index 8f9318f24..000000000 --- a/docs/v1.X/docs/cscli/cscli_parsers_inspect.md +++ /dev/null @@ -1,42 +0,0 @@ -## cscli parsers inspect - -Inspect given parser - -### Synopsis - -Inspect given parser - -``` -cscli parsers inspect [name] [flags] -``` - -### Examples - -``` -cscli parsers inspect crowdsec/xxx -``` - -### Options - -``` - -h, --help help for inspect - -u, --url string Prometheus url -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli parsers](cscli_parsers.md) - Install/Remove/Upgrade/Inspect parser(s) from hub - - diff --git a/docs/v1.X/docs/cscli/cscli_parsers_install.md b/docs/v1.X/docs/cscli/cscli_parsers_install.md deleted file mode 100644 index c60062f8a..000000000 --- a/docs/v1.X/docs/cscli/cscli_parsers_install.md +++ /dev/null @@ -1,43 +0,0 @@ -## cscli parsers install - -Install given parser(s) - -### Synopsis - -Fetch and install given parser(s) from hub - -``` -cscli parsers install [config] [flags] -``` - -### Examples - -``` -cscli parsers install crowdsec/xxx crowdsec/xyz -``` - -### Options - -``` - -d, --download-only Only download packages, don't enable - --force Force install : Overwrite tainted and outdated files - -h, --help help for install -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli parsers](cscli_parsers.md) - Install/Remove/Upgrade/Inspect parser(s) from hub - - diff --git a/docs/v1.X/docs/cscli/cscli_parsers_list.md b/docs/v1.X/docs/cscli/cscli_parsers_list.md deleted file mode 100644 index 29897b3fe..000000000 --- a/docs/v1.X/docs/cscli/cscli_parsers_list.md +++ /dev/null @@ -1,43 +0,0 @@ -## cscli parsers list - -List all parsers or given one - -### Synopsis - -List all parsers or given one - -``` -cscli parsers list [name] [flags] -``` - -### Examples - -``` -cscli parsers list -cscli parser list crowdsecurity/xxx -``` - -### Options - -``` - -a, --all List as well disabled items - -h, --help help for list -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli parsers](cscli_parsers.md) - Install/Remove/Upgrade/Inspect parser(s) from hub - - diff --git a/docs/v1.X/docs/cscli/cscli_parsers_remove.md b/docs/v1.X/docs/cscli/cscli_parsers_remove.md deleted file mode 100644 index f077964d7..000000000 --- a/docs/v1.X/docs/cscli/cscli_parsers_remove.md +++ /dev/null @@ -1,44 +0,0 @@ -## cscli parsers remove - -Remove given parser(s) - -### Synopsis - -Remove given parse(s) from hub - -``` -cscli parsers remove [config] [flags] -``` - -### Examples - -``` -cscli parsers remove crowdsec/xxx crowdsec/xyz -``` - -### Options - -``` - --all Delete all the parsers - --force Force remove : Remove tainted and outdated files - -h, --help help for remove - --purge Delete source file too -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli parsers](cscli_parsers.md) - Install/Remove/Upgrade/Inspect parser(s) from hub - - diff --git a/docs/v1.X/docs/cscli/cscli_parsers_upgrade.md b/docs/v1.X/docs/cscli/cscli_parsers_upgrade.md deleted file mode 100644 index 307653cf9..000000000 --- a/docs/v1.X/docs/cscli/cscli_parsers_upgrade.md +++ /dev/null @@ -1,43 +0,0 @@ -## cscli parsers upgrade - -Upgrade given parser(s) - -### Synopsis - -Fetch and upgrade given parser(s) from hub - -``` -cscli parsers upgrade [config] [flags] -``` - -### Examples - -``` -cscli parsers upgrade crowdsec/xxx crowdsec/xyz -``` - -### Options - -``` - --all Upgrade all the parsers - --force Force upgrade : Overwrite tainted and outdated files - -h, --help help for upgrade -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli parsers](cscli_parsers.md) - Install/Remove/Upgrade/Inspect parser(s) from hub - - diff --git a/docs/v1.X/docs/cscli/cscli_postoverflows.md b/docs/v1.X/docs/cscli/cscli_postoverflows.md deleted file mode 100644 index 8b7242661..000000000 --- a/docs/v1.X/docs/cscli/cscli_postoverflows.md +++ /dev/null @@ -1,42 +0,0 @@ -## cscli postoverflows - -Install/Remove/Upgrade/Inspect postoverflow(s) from hub - -### Examples - -``` -cscli postoverflows install crowdsecurity/cdn-whitelist - cscli postoverflows inspect crowdsecurity/cdn-whitelist - cscli postoverflows upgrade crowdsecurity/cdn-whitelist - cscli postoverflows list - cscli postoverflows remove crowdsecurity/cdn-whitelist -``` - -### Options - -``` - -h, --help help for postoverflows -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli postoverflows inspect](cscli_postoverflows_inspect.md) - Inspect given postoverflow -* [cscli postoverflows install](cscli_postoverflows_install.md) - Install given postoverflow(s) -* [cscli postoverflows list](cscli_postoverflows_list.md) - List all postoverflows or given one -* [cscli postoverflows remove](cscli_postoverflows_remove.md) - Remove given postoverflow(s) -* [cscli postoverflows upgrade](cscli_postoverflows_upgrade.md) - Upgrade given postoverflow(s) - - diff --git a/docs/v1.X/docs/cscli/cscli_postoverflows_inspect.md b/docs/v1.X/docs/cscli/cscli_postoverflows_inspect.md deleted file mode 100644 index 9e2e90500..000000000 --- a/docs/v1.X/docs/cscli/cscli_postoverflows_inspect.md +++ /dev/null @@ -1,41 +0,0 @@ -## cscli postoverflows inspect - -Inspect given postoverflow - -### Synopsis - -Inspect given postoverflow - -``` -cscli postoverflows inspect [config] [flags] -``` - -### Examples - -``` -cscli postoverflows inspect crowdsec/xxx crowdsec/xyz -``` - -### Options - -``` - -h, --help help for inspect -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli postoverflows](cscli_postoverflows.md) - Install/Remove/Upgrade/Inspect postoverflow(s) from hub - - diff --git a/docs/v1.X/docs/cscli/cscli_postoverflows_install.md b/docs/v1.X/docs/cscli/cscli_postoverflows_install.md deleted file mode 100644 index f1baa75e9..000000000 --- a/docs/v1.X/docs/cscli/cscli_postoverflows_install.md +++ /dev/null @@ -1,43 +0,0 @@ -## cscli postoverflows install - -Install given postoverflow(s) - -### Synopsis - -Fetch and install given postoverflow(s) from hub - -``` -cscli postoverflows install [config] [flags] -``` - -### Examples - -``` -cscli postoverflows install crowdsec/xxx crowdsec/xyz -``` - -### Options - -``` - -d, --download-only Only download packages, don't enable - --force Force install : Overwrite tainted and outdated files - -h, --help help for install -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli postoverflows](cscli_postoverflows.md) - Install/Remove/Upgrade/Inspect postoverflow(s) from hub - - diff --git a/docs/v1.X/docs/cscli/cscli_postoverflows_list.md b/docs/v1.X/docs/cscli/cscli_postoverflows_list.md deleted file mode 100644 index 4f6ce6d41..000000000 --- a/docs/v1.X/docs/cscli/cscli_postoverflows_list.md +++ /dev/null @@ -1,43 +0,0 @@ -## cscli postoverflows list - -List all postoverflows or given one - -### Synopsis - -List all postoverflows or given one - -``` -cscli postoverflows list [config] [flags] -``` - -### Examples - -``` -cscli postoverflows list -cscli postoverflows list crowdsecurity/xxx -``` - -### Options - -``` - -a, --all List as well disabled items - -h, --help help for list -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli postoverflows](cscli_postoverflows.md) - Install/Remove/Upgrade/Inspect postoverflow(s) from hub - - diff --git a/docs/v1.X/docs/cscli/cscli_postoverflows_postoverflows.md b/docs/v1.X/docs/cscli/cscli_postoverflows_postoverflows.md deleted file mode 100644 index c1afa8986..000000000 --- a/docs/v1.X/docs/cscli/cscli_postoverflows_postoverflows.md +++ /dev/null @@ -1,44 +0,0 @@ -## cscli postoverflows postoverflows - -Install given postoverflow(s) - -### Synopsis - -Fetch and install given postoverflow(s) from hub - -``` -cscli postoverflows postoverflows [config] [flags] -``` - -### Examples - -``` -cscli postoverflows install crowdsec/xxx crowdsec/xyz -``` - -### Options - -``` - -d, --download-only Only download packages, don't enable - --force Force install : Overwrite tainted and outdated files - -h, --help help for postoverflows -``` - -### Options inherited from parent commands - -``` - -b, --branch string Use given branch from hub - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli postoverflows](cscli_postoverflows.md) - Install/Remove/Upgrade/Inspect postoverflow(s) from hub - - diff --git a/docs/v1.X/docs/cscli/cscli_postoverflows_remove.md b/docs/v1.X/docs/cscli/cscli_postoverflows_remove.md deleted file mode 100644 index 226cdbb7d..000000000 --- a/docs/v1.X/docs/cscli/cscli_postoverflows_remove.md +++ /dev/null @@ -1,44 +0,0 @@ -## cscli postoverflows remove - -Remove given postoverflow(s) - -### Synopsis - -remove given postoverflow(s) - -``` -cscli postoverflows remove [config] [flags] -``` - -### Examples - -``` -cscli postoverflows remove crowdsec/xxx crowdsec/xyz -``` - -### Options - -``` - --all Delete all the postoverflows - --force Force remove : Remove tainted and outdated files - -h, --help help for remove - --purge Delete source file too -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli postoverflows](cscli_postoverflows.md) - Install/Remove/Upgrade/Inspect postoverflow(s) from hub - - diff --git a/docs/v1.X/docs/cscli/cscli_postoverflows_upgrade.md b/docs/v1.X/docs/cscli/cscli_postoverflows_upgrade.md deleted file mode 100644 index 37aec0ed7..000000000 --- a/docs/v1.X/docs/cscli/cscli_postoverflows_upgrade.md +++ /dev/null @@ -1,43 +0,0 @@ -## cscli postoverflows upgrade - -Upgrade given postoverflow(s) - -### Synopsis - -Fetch and Upgrade given postoverflow(s) from hub - -``` -cscli postoverflows upgrade [config] [flags] -``` - -### Examples - -``` -cscli postoverflows upgrade crowdsec/xxx crowdsec/xyz -``` - -### Options - -``` - -a, --all Upgrade all the postoverflows - --force Force upgrade : Overwrite tainted and outdated files - -h, --help help for upgrade -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli postoverflows](cscli_postoverflows.md) - Install/Remove/Upgrade/Inspect postoverflow(s) from hub - - diff --git a/docs/v1.X/docs/cscli/cscli_scenarios.md b/docs/v1.X/docs/cscli/cscli_scenarios.md deleted file mode 100644 index cac728b7f..000000000 --- a/docs/v1.X/docs/cscli/cscli_scenarios.md +++ /dev/null @@ -1,43 +0,0 @@ -## cscli scenarios - -Install/Remove/Upgrade/Inspect scenario(s) from hub - -### Examples - -``` -cscli scenarios list [-a] -cscli scenarios install crowdsecurity/ssh-bf -cscli scenarios inspect crowdsecurity/ssh-bf -cscli scenarios upgrade crowdsecurity/ssh-bf -cscli scenarios remove crowdsecurity/ssh-bf - -``` - -### Options - -``` - -h, --help help for scenarios -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli scenarios inspect](cscli_scenarios_inspect.md) - Inspect given scenario -* [cscli scenarios install](cscli_scenarios_install.md) - Install given scenario(s) -* [cscli scenarios list](cscli_scenarios_list.md) - List all scenario(s) or given one -* [cscli scenarios remove](cscli_scenarios_remove.md) - Remove given scenario(s) -* [cscli scenarios upgrade](cscli_scenarios_upgrade.md) - Upgrade given scenario(s) - - diff --git a/docs/v1.X/docs/cscli/cscli_scenarios_inspect.md b/docs/v1.X/docs/cscli/cscli_scenarios_inspect.md deleted file mode 100644 index c480d585f..000000000 --- a/docs/v1.X/docs/cscli/cscli_scenarios_inspect.md +++ /dev/null @@ -1,42 +0,0 @@ -## cscli scenarios inspect - -Inspect given scenario - -### Synopsis - -Inspect given scenario - -``` -cscli scenarios inspect [config] [flags] -``` - -### Examples - -``` -cscli scenarios inspect crowdsec/xxx -``` - -### Options - -``` - -h, --help help for inspect - -u, --url string Prometheus url -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli scenarios](cscli_scenarios.md) - Install/Remove/Upgrade/Inspect scenario(s) from hub - - diff --git a/docs/v1.X/docs/cscli/cscli_scenarios_install.md b/docs/v1.X/docs/cscli/cscli_scenarios_install.md deleted file mode 100644 index e6243bd40..000000000 --- a/docs/v1.X/docs/cscli/cscli_scenarios_install.md +++ /dev/null @@ -1,43 +0,0 @@ -## cscli scenarios install - -Install given scenario(s) - -### Synopsis - -Fetch and install given scenario(s) from hub - -``` -cscli scenarios install [config] [flags] -``` - -### Examples - -``` -cscli scenarios install crowdsec/xxx crowdsec/xyz -``` - -### Options - -``` - -d, --download-only Only download packages, don't enable - --force Force install : Overwrite tainted and outdated files - -h, --help help for install -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli scenarios](cscli_scenarios.md) - Install/Remove/Upgrade/Inspect scenario(s) from hub - - diff --git a/docs/v1.X/docs/cscli/cscli_scenarios_list.md b/docs/v1.X/docs/cscli/cscli_scenarios_list.md deleted file mode 100644 index b204574e3..000000000 --- a/docs/v1.X/docs/cscli/cscli_scenarios_list.md +++ /dev/null @@ -1,43 +0,0 @@ -## cscli scenarios list - -List all scenario(s) or given one - -### Synopsis - -List all scenario(s) or given one - -``` -cscli scenarios list [config] [flags] -``` - -### Examples - -``` -cscli scenarios list -cscli scenarios list crowdsecurity/xxx -``` - -### Options - -``` - -a, --all List as well disabled items - -h, --help help for list -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli scenarios](cscli_scenarios.md) - Install/Remove/Upgrade/Inspect scenario(s) from hub - - diff --git a/docs/v1.X/docs/cscli/cscli_scenarios_remove.md b/docs/v1.X/docs/cscli/cscli_scenarios_remove.md deleted file mode 100644 index 45cb3cd3e..000000000 --- a/docs/v1.X/docs/cscli/cscli_scenarios_remove.md +++ /dev/null @@ -1,44 +0,0 @@ -## cscli scenarios remove - -Remove given scenario(s) - -### Synopsis - -remove given scenario(s) - -``` -cscli scenarios remove [config] [flags] -``` - -### Examples - -``` -cscli scenarios remove crowdsec/xxx crowdsec/xyz -``` - -### Options - -``` - --all Delete all the scenarios - --force Force remove : Remove tainted and outdated files - -h, --help help for remove - --purge Delete source file too -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli scenarios](cscli_scenarios.md) - Install/Remove/Upgrade/Inspect scenario(s) from hub - - diff --git a/docs/v1.X/docs/cscli/cscli_scenarios_upgrade.md b/docs/v1.X/docs/cscli/cscli_scenarios_upgrade.md deleted file mode 100644 index 9cf1713f3..000000000 --- a/docs/v1.X/docs/cscli/cscli_scenarios_upgrade.md +++ /dev/null @@ -1,43 +0,0 @@ -## cscli scenarios upgrade - -Upgrade given scenario(s) - -### Synopsis - -Fetch and Upgrade given scenario(s) from hub - -``` -cscli scenarios upgrade [config] [flags] -``` - -### Examples - -``` -cscli scenarios upgrade crowdsec/xxx crowdsec/xyz -``` - -### Options - -``` - -a, --all Upgrade all the scenarios - --force Force upgrade : Overwrite tainted and outdated files - -h, --help help for upgrade -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli scenarios](cscli_scenarios.md) - Install/Remove/Upgrade/Inspect scenario(s) from hub - - diff --git a/docs/v1.X/docs/cscli/cscli_simulation.md b/docs/v1.X/docs/cscli/cscli_simulation.md deleted file mode 100644 index ff085a7a7..000000000 --- a/docs/v1.X/docs/cscli/cscli_simulation.md +++ /dev/null @@ -1,38 +0,0 @@ -## cscli simulation - -Manage simulation status of scenarios - -### Examples - -``` -cscli simulation status -cscli simulation enable crowdsecurity/ssh-bf -cscli simulation disable crowdsecurity/ssh-bf -``` - -### Options - -``` - -h, --help help for simulation -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec -* [cscli simulation disable](cscli_simulation_disable.md) - Disable the simulation mode. Disable only specified scenarios -* [cscli simulation enable](cscli_simulation_enable.md) - Enable the simulation, globally or on specified scenarios -* [cscli simulation status](cscli_simulation_status.md) - Show simulation mode status - - diff --git a/docs/v1.X/docs/cscli/cscli_simulation_disable.md b/docs/v1.X/docs/cscli/cscli_simulation_disable.md deleted file mode 100644 index cb33b192b..000000000 --- a/docs/v1.X/docs/cscli/cscli_simulation_disable.md +++ /dev/null @@ -1,38 +0,0 @@ -## cscli simulation disable - -Disable the simulation mode. Disable only specified scenarios - -``` -cscli simulation disable [scenario] [flags] -``` - -### Examples - -``` -cscli simulation disable -``` - -### Options - -``` - -g, --global Disable global simulation (reverse mode) - -h, --help help for disable -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli simulation](cscli_simulation.md) - Manage simulation status of scenarios - - diff --git a/docs/v1.X/docs/cscli/cscli_simulation_enable.md b/docs/v1.X/docs/cscli/cscli_simulation_enable.md deleted file mode 100644 index e52c67503..000000000 --- a/docs/v1.X/docs/cscli/cscli_simulation_enable.md +++ /dev/null @@ -1,38 +0,0 @@ -## cscli simulation enable - -Enable the simulation, globally or on specified scenarios - -``` -cscli simulation enable [scenario] [-global] [flags] -``` - -### Examples - -``` -cscli simulation enable -``` - -### Options - -``` - -g, --global Enable global simulation (reverse mode) - -h, --help help for enable -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli simulation](cscli_simulation.md) - Manage simulation status of scenarios - - diff --git a/docs/v1.X/docs/cscli/cscli_simulation_status.md b/docs/v1.X/docs/cscli/cscli_simulation_status.md deleted file mode 100644 index a6d29ebc0..000000000 --- a/docs/v1.X/docs/cscli/cscli_simulation_status.md +++ /dev/null @@ -1,37 +0,0 @@ -## cscli simulation status - -Show simulation mode status - -``` -cscli simulation status [flags] -``` - -### Examples - -``` -cscli simulation status -``` - -### Options - -``` - -h, --help help for status -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli simulation](cscli_simulation.md) - Manage simulation status of scenarios - - diff --git a/docs/v1.X/docs/cscli/cscli_version.md b/docs/v1.X/docs/cscli/cscli_version.md deleted file mode 100644 index 38aacae7f..000000000 --- a/docs/v1.X/docs/cscli/cscli_version.md +++ /dev/null @@ -1,31 +0,0 @@ -## cscli version - -Display version and exit. - -``` -cscli version [flags] -``` - -### Options - -``` - -h, --help help for version -``` - -### Options inherited from parent commands - -``` - -c, --config string path to crowdsec config file (default "/etc/crowdsec/config.yaml") - --debug Set logging to debug. - --error Set logging to error. - --info Set logging to info. - -o, --output string Output format : human, json, raw. - --trace Set logging to trace. - --warning Set logging to warning. -``` - -### SEE ALSO - -* [cscli](cscli.md) - cscli allows you to manage crowdsec - - diff --git a/docs/v1.X/docs/getting_started/concepts.md b/docs/v1.X/docs/getting_started/concepts.md deleted file mode 100644 index 7a435113c..000000000 --- a/docs/v1.X/docs/getting_started/concepts.md +++ /dev/null @@ -1,180 +0,0 @@ - -# Global overview - -{{v1X.crowdsec.Name}} runtime revolves around a few simple concepts : - - - It read logs (defined via {{v1X.ref.acquis}} config) - - Those logs are parsed via {{v1X.ref.parsers}} and eventually enriched - - Those normalized logs are then matched against the {{v1X.ref.scenarios}} that the user deployed - - When a scenario is "triggered", {{v1X.crowdsec.Name}} generates an {{v1X.alert.Htmlname}} and eventually one or more associated {{v1X.decision.Htmlname}} : - - The alert is here mostly for tracability, and will stay even after the decision expires - - The decision on the other hand, is short lived, and tells *what* action should be taken against the offending ip/range/user... - - Those information (the signal, the associated decisions) are then sent to crowdsec's {{v1X.lapi.htmlname}} and stored in the database - -As you might have guessed by now, {{v1X.crowdsec.Name}} itself does the detection part and stores those decisions. -Then, {{v1X.bouncers.htmlname}} can "consume" those decisions (via the very same {{v1X.lapi.htmlname}}) and apply some actual remediation. - -## Crowd sourced aspect - - [[References](https://crowdsecurity.github.io/api_doc/index.html?urls.primaryName=CAPI)] - -Whenever the {{v1X.lapi.htmlname}} receives an alert with associated decisions, the meta information about the alert are shared with our central api : - - - The source ip that triggered the alert - - The scenario that was triggered - - The timestamp of the attack - -These are the only information that are sent to our API. Those are then processed on our side to be able to redistribute relevant blocklists to all the participants. You can check the central API documentation in the references link to have an exhaustive view of what might be shared between your instance and our services. - -## Bouncers - -[[References](/Crowdsec/v1/bouncers/)] - -Bouncers are standalone software pieces in charge of acting upon IPs that triggered alerts. -To do so, bouncers are going to query the local API to know if there is an existing decisions against a given IP, range, username etc. [You can find a list of existing bouncers on the hub]({{v1X.bouncers.url}}) - - - -# Configuration items - -## Acquisition - -[[References](/Crowdsec/v1/references/acquisition/)] - -Acquistion configuration defines which streams of information {{v1X.crowdsec.name}} is going to process. - -At the time of writing, it's mostly files or journald, but it should be more or less any kind of stream, such as a kafka topic or a cloudtrail. - -Acquisition configuration always contains a stream (ie. a file to tail) and a tag (ie. "these are in syslog format" "these are non-syslog nginx logs"). - -File acquisition configuration is defined as : - -```yaml -filenames: - - /var/log/auth.log -labels: - type: syslog -``` - -The `labels` part is here to tag the incoming logs with a type. `labels.type` are used by the parsers to know which logs to process. - -## Stages - -[[References](/Crowdsec/v1/references/parsers/#stages)] - -Stages concept is central to data parsing in {{v1X.crowdsec.name}}, as it allows to have various "steps" of parsing. All parsers belong to a given stage. While users can add or modify the stages order, the following stages exist : - - - `s00-raw` : low level parser, such as syslog - - `s01-parse` : most of the services parsers (ssh, nginx etc.) - - `s02-enrich` : enrichment that requires parsed events (ie. geoip-enrichment) or generic parsers that apply on parsed logs (ie. second stage http parser) - - -Every event starts in the first stage, and will move to the next stage once it has been successfully processed by a parser that has the `onsuccess` directive set to `next_stage`, and so on until it reaches the last stage, when it's going to start to be matched against scenarios. Thus a sshd log might follow this pipeline : - - - `s00-raw` : be parsed by `crowdsecurity/syslog-logs` (will move event to the next stage) - - `s01-raw` : be parsed by `crowdsecurity/sshd-logs` (will move event to the next stage) - - `s02-enrich` : will be parsed by `crowdsecurity/geoip-enrich` and `crowdsecurity/dateparse-enrich` - -## Parsers - -[[References](/Crowdsec/v1/references/parsers/)] - -For logs to be able to be exploited and analyzed, they need to be parsed and normalized, and this is where parsers are used. - -A parser is a YAML configuration file that describes how a string is being parsed. Said string can be a log line, or a field extracted from a previous parser. While a lot of parsers rely on the **GROK** approach (a.k.a regular expression named capture groups), parsers can as well reference enrichment modules to allow specific data processing. - -A parser usually has a specific scope. For example, if you are using [nginx](https://nginx.org), you will probably want to use the `crowdsecurity/nginx-logs` which allows your {{v1X.crowdsec.name}} setup to parse nginx's access and error logs. - -Parsers are organized into stages to allow pipelines and branching in parsing. - -See the [{{v1X.hub.name}}]({{v1X.hub.url}}) to explore parsers, or see below some examples : - - - [apache2 access/error log parser](https://github.com/crowdsecurity/hub/blob/master/parsers/s01-parse/crowdsecurity/apache2-logs.yaml) - - [iptables logs parser](https://github.com/crowdsecurity/hub/blob/master/parsers/s01-parse/crowdsecurity/iptables-logs.yaml) - - [http logs post-processing](https://github.com/crowdsecurity/hub/blob/master/parsers/s02-enrich/crowdsecurity/http-logs.yaml) - -You can as well [write your own](/Crowdsec/v1/write_configurations/parsers/) ! - - - - - -## Enrichers - -[[References](/Crowdsec/v1/references/enrichers/)] - -Enrichment is the action of adding extra context to an event based on the information we already have, so that better decision can later be taken. In most cases, you should be able to find the relevant enrichers on our {{v1X.hub.htmlname}}. - -A common/simple type of enrichment would be [geoip-enrich](https://github.com/crowdsecurity/hub/blob/master/parsers/s02-enrich/crowdsecurity/geoip-enrich.yaml) of an event (adding information such as : origin country, origin AS and origin IP range to an event). - -Once again, you should be able to find the ones you're looking for on the {{v1X.hub.htmlname}} ! - -## Scenarios - -[[References](/Crowdsec/v1/references/scenarios/)] - -Scenarios is the expression of a heuristic that allows you to qualify a specific event (usually an attack).It is a YAML file that describes a set of events characterizing a scenario. Scenarios in {{v1X.crowdsec.name}} gravitate around the [leaky bucket](https://en.wikipedia.org/wiki/Leaky_bucket) principle. - -A scenario description includes at least : - - - Event eligibility rules. (For example if we're writing a ssh bruteforce detection we only focus on logs of type `ssh_failed_auth`) - - Bucket configuration such as the leak speed or its capacity (in our same ssh bruteforce example, we might allow 1 failed auth per 10s and no more than 5 in a short amount of time: `leakspeed: 10s` `capacity: 5`) - - Aggregation rules : per source ip or per other criterias (in our ssh bruteforce example, we will group per source ip) - -The description allows for many other rules to be specified (blackhole, distinct filters etc.), to allow rather complex scenarios. - -See the [{{v1X.hub.name}}]({{v1X.hub.url}}) to explore scenarios and their capabilities, or see below some examples : - - - [ssh bruteforce detection](https://github.com/crowdsecurity/hub/blob/master/scenarios/crowdsecurity/ssh-bf.yaml) - - [distinct http-404 scan](https://github.com/crowdsecurity/hub/blob/master/scenarios/crowdsecurity/http-scan-uniques_404.yaml) - - [iptables port scan](https://github.com/crowdsecurity/hub/blob/master/scenarios/crowdsecurity/iptables-scan-multi_ports.yaml) - -You can as well [write your own](/Crowdsec/v1/write_configurations/scenarios/) ! - - -## Collections - -[[References](/Crowdsec/v1/references/collections/)] - -To make user's life easier, "collections" are available, which are just a bundle of parsers and scenarios. -In this way, if you want to cover basic use-cases of let's say "nginx", you can just install the `crowdsecurity/nginx` collection that is composed of `crowdsecurity/nginx-logs` parser, as well as generic http scenarios such as `crowdsecurity/base-http-scenarios`. - -As usual, those can be found on the {{v1X.hub.htmlname}} ! - -## PostOverflows - -[[References](/Crowdsec/v1/references/postoverflows)] - -A postoverflow is a parser that will be applied on overflows (scenario results) before the decision is written to local DB or pushed to API. Parsers in postoverflows are meant to be used for "expensive" enrichment/parsing process that you do not want to perform on all incoming events, but rather on decision that are about to be taken. - -An example could be slack/mattermost enrichment plugin that requires human confirmation before applying the decision or reverse-dns lookup operations. - - -# Runtime items - -## Events - -[[References](/Crowdsec/v1/references/events)] - -An `Event` is the runtime representation of an item being processed by crowdsec : It be a Log line being parsed, or an Overflow being reprocessed. - -The `Event` object is modified by parses, scenarios, and directly via user [statics expressions](/Crowdsec/v1/references/parsers/#statics) (for example). - - - - -## Alerts - -[[References](/Crowdsec/v1/references/alerts)] - -An `Alert` is the runtime representation of a bucket overflow being processed by crowdsec : It is embedded in an Event. - -The `Alert` object is modified by post-overflows and {{v1X.profiles.htmlname}}. - -## Decisions - -[[References](/Crowdsec/v1/references/decisions)] - -A `Decision` is the representation of the consequence of a bucket overflow : a decision against an IP, a range, an AS, a Country, a User, a Session etc. - -`Decisions` are generated by Local API (LAPI) when an `Alert` is received, according to the existing {{v1X.profiles.htmlname}} \ No newline at end of file diff --git a/docs/v1.X/docs/getting_started/crowdsec-tour.md b/docs/v1.X/docs/getting_started/crowdsec-tour.md deleted file mode 100644 index f3962c126..000000000 --- a/docs/v1.X/docs/getting_started/crowdsec-tour.md +++ /dev/null @@ -1,278 +0,0 @@ - -## List installed configurations - -```bash -sudo {{v1X.cli.bin}} hub list -``` - -On the machine where you deployed {{v1X.crowdsec.name}}, type `sudo {{v1X.cli.bin}} hub list` to see install configurations. -This list represents the parsers, scenarios and/or collections that you deployed. They represent what your {{v1X.crowdsec.name}} setup can read (logs) and detect (scenarios). `sudo {{v1X.cli.bin}} hub list -a` will list all available configurations in the hub. - - -Check [{{v1X.cli.name}} configuration](/Crowdsec/v1/user_guide/cscli/) management for more ! - -
- output example -```bash -$ sudo cscli hub list -INFO[0000] Loaded 13 collecs, 17 parsers, 21 scenarios, 3 post-overflow parsers -INFO[0000] unmanaged items : 23 local, 0 tainted -INFO[0000] PARSERS: --------------------------------------------------------------------------------------------------------------- - NAME 📦 STATUS VERSION LOCAL PATH --------------------------------------------------------------------------------------------------------------- - crowdsecurity/mysql-logs ✔️ enabled 0.1 /etc/crowdsec/parsers/s01-parse/mysql-logs.yaml - crowdsecurity/sshd-logs ✔️ enabled 0.1 /etc/crowdsec/parsers/s01-parse/sshd-logs.yaml - crowdsecurity/dateparse-enrich ✔️ enabled 0.1 /etc/crowdsec/parsers/s02-enrich/dateparse-enrich.yaml - crowdsecurity/whitelists ✔️ enabled 0.1 /etc/crowdsec/parsers/s02-enrich/whitelists.yaml - crowdsecurity/geoip-enrich ✔️ enabled 0.2 /etc/crowdsec/parsers/s02-enrich/geoip-enrich.yaml - crowdsecurity/syslog-logs ✔️ enabled 0.1 /etc/crowdsec/parsers/s00-raw/syslog-logs.yaml --------------------------------------------------------------------------------------------------------------- -INFO[0000] SCENARIOS: -------------------------------------------------------------------------------------- - NAME 📦 STATUS VERSION LOCAL PATH -------------------------------------------------------------------------------------- - crowdsecurity/mysql-bf ✔️ enabled 0.1 /etc/crowdsec/scenarios/mysql-bf.yaml - crowdsecurity/ssh-bf ✔️ enabled 0.1 /etc/crowdsec/scenarios/ssh-bf.yaml -------------------------------------------------------------------------------------- -INFO[0000] COLLECTIONS: ---------------------------------------------------------------------------------- - NAME 📦 STATUS VERSION LOCAL PATH ---------------------------------------------------------------------------------- - crowdsecurity/mysql ✔️ enabled 0.1 /etc/crowdsec/collections/mysql.yaml - crowdsecurity/sshd ✔️ enabled 0.1 /etc/crowdsec/collections/sshd.yaml - crowdsecurity/linux ✔️ enabled 0.2 /etc/crowdsec/collections/linux.yaml ---------------------------------------------------------------------------------- -INFO[0000] POSTOVERFLOWS: --------------------------------------- - NAME 📦 STATUS VERSION LOCAL PATH --------------------------------------- --------------------------------------- - -``` -
- -## Installing configurations - -```bash -sudo cscli install -``` - -`configuration_type` can be: - - - [`parsers`](https://docs.crowdsec.net/Crowdsec/v1/references/parsers/) - - - [`scenarios`](https://docs.crowdsec.net/Crowdsec/v1/references/scenarios/) - - - [`postoverflows`](https://docs.crowdsec.net/Crowdsec/v1/references/postoverflows/) - - - [`collections`](https://docs.crowdsec.net/Crowdsec/v1/references/collections/) - - -They can be found and browsed on the {{v1X.hub.htmlname}} - -## Upgrading configurations - -```bash -sudo cscli upgrade -``` - -`configuration_type` can be: - - - [`parsers`](https://docs.crowdsec.net/Crowdsec/v1/references/parsers/) - - - [`scenarios`](https://docs.crowdsec.net/Crowdsec/v1/references/scenarios/) - - - [`postoverflows`](https://docs.crowdsec.net/Crowdsec/v1/references/postoverflows/) - - - [`collections`](https://docs.crowdsec.net/Crowdsec/v1/references/collections/) - -They can be found and browsed on the {{v1X.hub.htmlname}} - -## List active decisions - - -```bash -sudo {{v1X.cli.bin}} decisions list -``` - -If you just deployed {{v1X.crowdsec.name}}, the list might be empty, but don't worry, it simply means you haven't yet been attacked, congrats! - -Check [{{v1X.cli.name}} decisions](/Crowdsec/v1/user_guide/decision_management/) management for more ! - - -
- output example -```bash -$ sudo cscli decisions list -+-----+-----------+-------------+------------------------------------+--------+---------+----+--------+--------------------+----------+ -| ID | SOURCE | SCOPE:VALUE | REASON | ACTION | COUNTRY | AS | EVENTS | EXPIRATION | ALERT ID | -+-----+-----------+-------------+------------------------------------+--------+---------+----+--------+--------------------+----------+ -| 802 | cscli | Ip:1.2.3.5 | manual 'ban' from | ban | | | 1 | 3h50m58.10039043s | 802 | -| | | | 'b76cc7b1bbdc489e93909d2043031de8' | | | | | | | -| 801 | crowdsec | Ip:1.2.3.4 | crowdsecurity/ssh-bf | ban | | | 6 | 3h59m45.100387557s | 801 | -+-----+-----------+-------------+------------------------------------+--------+---------+----+--------+--------------------+----------+ -``` -
- -There are different decisions `SOURCE`: - - - crowdsec : decisions triggered locally by the crowdsec agent - - CAPI : decisions fetched from the Crowdsec Central API - - csli : decisions added via `sudo {{v1X.cli.bin}} decisions add` - - -## List alerts - - -```bash -sudo {{v1X.cli.bin}} alerts list -``` - -While decisions won't be shown anymore once they expire (or are manually deleted), the alerts will stay visible, allowing you to keep track of past decisions. -You will here see the alerts, even if the associated decisions expired. - -
- output example -```bash -$ sudo cscli alerts list --since 1h -+----+-------------+----------------------------+---------+----+-----------+---------------------------+ -| ID | SCOPE:VALUE | REASON | COUNTRY | AS | DECISIONS | CREATED AT | -+----+-------------+----------------------------+---------+----+-----------+---------------------------+ -| 5 | Ip:1.2.3.6 | crowdsecurity/ssh-bf (0.1) | US | | ban:1 | 2020-10-29T11:33:36+01:00 | -+----+-------------+----------------------------+---------+----+-----------+---------------------------+ -``` -
- - -## Monitor on-going activity (prometheus) - -```bash -sudo {{v1X.cli.bin}} metrics -``` - -The metrics displayed are extracted from {{v1X.crowdsec.name}} prometheus. -The indicators are grouped by scope : - - - Buckets : Know which buckets are created and/or overflew (scenario efficiency) - - Acquisition : Know which file produce logs and if thy are parsed (or end up in bucket) - - Parser : Know how frequently the individual parsers are triggered and their success rate - - Local Api Metrics : Know how often each endpoint of crowdsec's local API has been used - -
- output example - -```bash -$ sudo {{v1X.cli.bin}} metrics -INFO[0000] Buckets Metrics: -+--------------------------------------+---------------+-----------+--------------+--------+---------+ -| BUCKET | CURRENT COUNT | OVERFLOWS | INSTANCIATED | POURED | EXPIRED | -+--------------------------------------+---------------+-----------+--------------+--------+---------+ -| crowdsecurity/http-bad-user-agent | - | - | 7 | 7 | 7 | -| crowdsecurity/http-crawl-non_statics | - | - | 82 | 107 | 82 | -| crowdsecurity/http-probing | - | - | 2 | 2 | 2 | -| crowdsecurity/http-sensitive-files | - | - | 1 | 1 | 1 | -| crowdsecurity/ssh-bf | 16 | 5562 | 7788 | 41542 | 2210 | -| crowdsecurity/ssh-bf_user-enum | 8 | - | 6679 | 12571 | 6671 | -+--------------------------------------+---------------+-----------+--------------+--------+---------+ -INFO[0000] Acquisition Metrics: -+---------------------------+------------+--------------+----------------+------------------------+ -| SOURCE | LINES READ | LINES PARSED | LINES UNPARSED | LINES POURED TO BUCKET | -+---------------------------+------------+--------------+----------------+------------------------+ -| /var/log/auth.log | 92978 | 41542 | 51436 | 54113 | -| /var/log/messages | 2 | - | 2 | - | -| /var/log/nginx/access.log | 124 | 99 | 25 | 88 | -| /var/log/nginx/error.log | 287 | 63 | 224 | 29 | -| /var/log/syslog | 27271 | - | 27271 | - | -+---------------------------+------------+--------------+----------------+------------------------+ -INFO[0000] Parser Metrics: -+--------------------------------+--------+--------+----------+ -| PARSERS | HITS | PARSED | UNPARSED | -+--------------------------------+--------+--------+----------+ -| child-crowdsecurity/http-logs | 486 | 232 | 254 | -| child-crowdsecurity/nginx-logs | 723 | 162 | 561 | -| child-crowdsecurity/sshd-logs | 381792 | 41542 | 340250 | -| crowdsecurity/dateparse-enrich | 41704 | 41704 | - | -| crowdsecurity/geoip-enrich | 41641 | 41641 | - | -| crowdsecurity/http-logs | 162 | 59 | 103 | -| crowdsecurity/nginx-logs | 411 | 162 | 249 | -| crowdsecurity/non-syslog | 411 | 411 | - | -| crowdsecurity/sshd-logs | 92126 | 41542 | 50584 | -| crowdsecurity/syslog-logs | 120251 | 120249 | 2 | -| crowdsecurity/whitelists | 41704 | 41704 | - | -+--------------------------------+--------+--------+----------+ -INFO[0000] Local Api Metrics: -+----------------------+--------+------+ -| ROUTE | METHOD | HITS | -+----------------------+--------+------+ -| /v1/alerts | GET | 3 | -| /v1/alerts | POST | 4673 | -| /v1/decisions/stream | GET | 6498 | -| /v1/watchers/login | POST | 23 | -+----------------------+--------+------+ -INFO[0000] Local Api Machines Metrics: -+----------------------------------+------------+--------+------+ -| MACHINE | ROUTE | METHOD | HITS | -+----------------------------------+------------+--------+------+ -| 7f0607a3469243139699bf2f30321fc4 | /v1/alerts | POST | 4673 | -| 7f0607a3469243139699bf2f30321fc4 | /v1/alerts | GET | 3 | -+----------------------------------+------------+--------+------+ -INFO[0000] Local Api Bouncers Metrics: -+------------------------------+----------------------+--------+------+ -| BOUNCER | ROUTE | METHOD | HITS | -+------------------------------+----------------------+--------+------+ -| cs-firewall-bouncer-n3W19Qua | /v1/decisions/stream | GET | 6498 | -+------------------------------+----------------------+--------+------+ -``` - -
- -### Reading metrics - -Those metrics are a great way to know if your configuration is correct : - -The `Acquisition Metrics` is a great way to know if your parsers are setup correctly : - - - If you have 0 **LINES PARSED** for a source : You are probably *missing* a parser, or you have a custom log format that prevents the parser from understanding your logs. - - However, it's perfectly OK to have a lot of **LINES UNPARSED** : Crowdsec is not a SIEM, and only parses the logs that are relevant to its scenarios. For example, [ssh parser](https://hub.crowdsec.net/author/crowdsecurity/configurations/sshd-logs), only cares about failed authentication events (at the time of writting). - - **LINES POURED TO BUCKET** tell you that your scenarios are matching your log sources : it means that some events from this log source made all their way to an actual scenario - - -The `Parser Metrics` will let you troubleshoot eventual parser misconfigurations : - - - **HITS** is how many events where fed to this specific parser - - - **PARSED** and **UNPARSED** indicate how many events successfully come out of the parser - - -For example, if you have a custom log format in nginx that is not supported by the default parser, you will end up seeing a lot of **UNPARSED** for this specific parser, and 0 for **PARSED**. - -For more advanced metrics understanding, [take a look at the dedicated prometheus documentation](/Crowdsec/v1/observability/prometheus/). - - -## Deploy dashboard - -```bash -sudo cscli dashboard setup --listen 0.0.0.0 -``` - -A docker metabase {{v1X.metabase.Htmlname}} container can be deployed with `cscli dashboard`. -It requires docker, [installation instructions are available here](https://docs.docker.com/engine/install/). - -## Logs - -```bash -sudo tail -f /var/log/crowdsec.log -``` - - - `/var/log/crowdsec.log` is the main log, it shows ongoing decisions and acquisition/parsing/scenario errors. - - `/var/log/crowdsec_api.log` is the access log of the local api (LAPI) - - -## Scalability - -Crowdsec uses go-routines for parsing and enriching logs, pouring events to buckets and manage outputs. - -By default, one routine of each exists (should be enough to handle ~1K EP/s), and can be changed in `crowdsec_service` of the main configuration file via the [parser_routines](/Crowdsec/v1/references/crowdsec-config/#parser_routines), [buckets_routines](/Crowdsec/v1/references/crowdsec-config/#buckets_routines) and [output_routines](/Crowdsec/v1/references/crowdsec-config/#output_routines) directives. - -Please keep in mind that thanks to the [http API]({{v1X.lapi.swagger}}), the workload of log parsing can be splitted amongst several agents pushing to a single {{v1X.lapi.Htmlname}}. diff --git a/docs/v1.X/docs/getting_started/installation.md b/docs/v1.X/docs/getting_started/installation.md deleted file mode 100644 index 0a5daedf6..000000000 --- a/docs/v1.X/docs/getting_started/installation.md +++ /dev/null @@ -1,159 +0,0 @@ - -# Installation methods - -You can install crowdsec in different ways : - - - Most users [set up crowdsec's repositories](/Crowdsec/v1/getting_started/installation/#install-using-crowdsec-repository) and install from them, for ease of installation and upgrade - - Some users [use debian's official crowdsec packages](/Crowdsec/v1/getting_started/installation/#install-using-debian-official-packages) - - Some users download the DEB package directly and [install it manually](/Crowdsec/v1/getting_started/installation/#manually-install-the-debian-package) - - Some users download the tarball directly and [install it manually](/Crowdsec/v1/getting_started/installation/#install-from-the-release-tarball) - - Some users use the [docker hub image](https://hub.docker.com/r/crowdsecurity/crowdsec) - - And the most adventurous might want to [build & install from source](/Crowdsec/v1/getting_started/installation/#install-from-source) - - And some might even want to [build their own docker image](/Crowdsec/v1/getting_started/installation/#build-docker-image) - - Or use it with [docker-compose](https://github.com/crowdsecurity/example-docker-compose) - - -!!! info - Packaging for FreeBSD and RedHat/CentOS are WIP at the time of writing. Documentation will be updated once those packages are published & functional. - -# Install using crowdsec repository - -Crowdsec distributes their own pragmatic debian packages that closely follow the development stream (packages are automatically published on release), and are suitable for those that want to keep up with the latest changes of crowdsec. - -## setup the repository - -```bash -wget -qO - https://s3-eu-west-1.amazonaws.com/crowdsec.debian.pragmatic/crowdsec.asc |sudo apt-key add - && echo "deb https://s3-eu-west-1.amazonaws.com/crowdsec.debian.pragmatic/$(lsb_release -cs) $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/crowdsec.list > /dev/null -sudo apt-get update -``` - -The following debian suites / architectures are available : - -## install crowdsec - -```bash -sudo apt-get install crowdsec -``` - -## compatibility matrix - -
- -| Suite | Architecture | -| :------------- | :----------: | -| bionic | amd64, arm64, i386 | -| bullseye | amd64, i386 | -| buster | amd64, arm64, i386 | -| focal | amd64, arm64, i386 | -| sid | amd64, i386 | -| stretch | amd64, arm64, i386 | -| xenial | amd64, arm64, i386 | - -
- -# Manually install the debian package - -Fetch your package from the [public repository](https://s3-eu-west-1.amazonaws.com/crowdsec.debian.pragmatic/), and install it manually : - -```bash -sudo dpkg -i ./crowdsec_1.0.7-4_amd64.deb -``` - -# Install using debian official packages - -Crowdsec is available for [bullseye & sid](https://packages.debian.org/search?searchon=names&keywords=crowdsec) and can be installed simply : - -```bash -sudo apt-get install crowdsec -``` - -# Install from the release tarball - -Fetch {{v1X.crowdsec.name}}'s latest version [here]({{v1X.crowdsec.download_url}}). - -```bash -tar xvzf crowdsec-release.tgz -``` -```bash -cd crowdsec-v1.X.X -``` - -A {{v1X.wizard.name}} is provided to help you deploy {{v1X.crowdsec.name}} and {{v1X.cli.name}}. - -## Using the interactive wizard - -``` -sudo {{v1X.wizard.bin}} -i -``` - -![crowdsec](../assets/images/crowdsec_install.gif) - -The {{v1X.wizard.name}} is going to guide you through the following steps : - - - detect services that are present on your machine - - detect selected services logs - - suggest collections (parsers and scenarios) to deploy - - deploy & configure {{v1X.crowdsec.name}} in order to watch selected logs for selected scenarios - -The process should take less than a minute, [please report if there are any issues]({{v1X.wizard.bugreport}}). - -You are then ready to [take a tour](/Crowdsec/v1/getting_started/crowdsec-tour/) of your freshly deployed {{v1X.crowdsec.name}} ! - -!!! info - Keep in mind the {{v1X.crowdsec.name}} is only in charge of the "detection", and won't block anything on its own. You need to deploy a {{v1X.bouncers.Htmlname}} to "apply" decisions. - -## Binary installation - -> you of little faith - -``` -sudo {{v1X.wizard.bin}} --bininstall -``` - -This will only deploy the binaries, and some extra installation steps need to be completed for the software to be functional : - - - `sudo cscli hub update` : update the hub index - - `sudo cscli machines add -a` : register crowdsec to the local API - - `sudo cscli capi register` : register to the central API - - `sudo cscli collections install crowdsecurity/linux` : install essential configs (syslog parser, geoip enrichment, date parsers) - - configure your sources in your {{v1X.ref.acquis}} : `/etc/crowdsec/acquis.yaml` - -You can now start & enable the crowdsec service : - - - `sudo systemctl start crowdsec` - - `sudo systemctl enable crowdsec` - -## Using the unattended wizard - -If your setup is standard and you've walked through the default installation without issues, you can win some time in case you need to perform a new install : `sudo ./wizard.sh --unattended` - -This mode will emulate the interactive mode of the wizard where you answer **yes** to everything and stick with the default options. - -# Install from source - -!!! warning "Requirements" - - * [Go](https://golang.org/doc/install) v1.13+ - * `git clone {{v1X.crowdsec.url}}` - * [jq](https://stedolan.github.io/jq/download/) - - -Go in {{v1X.crowdsec.name}} folder and build the binaries : - -```bash -cd crowdsec -make release -``` - -This will create you a directory (`crowdsec-vXXX/`) and an archive (`crowdsec-release.tgz`) that are release built from your local code source. - -Now, you can install either with [interactive wizard](#using-the-interactive-wizard) or the [unattended mode](#using-unattended-mode). - -# Build docker image - -Crowdsec provides a docker image and can simply built like this : - -```bash -git clone https://github.com/crowdsecurity/crowdsec.git && cd crowdsec -docker build -t crowdsec . -``` diff --git a/docs/v1.X/docs/getting_started/upgrades.md b/docs/v1.X/docs/getting_started/upgrades.md deleted file mode 100644 index b4420dfd9..000000000 --- a/docs/v1.X/docs/getting_started/upgrades.md +++ /dev/null @@ -1,62 +0,0 @@ -# Upgrade notes - -Crowdsec does it best not to break existing setups, and the following rules generally applies : - - - patches (`X.X.Y` to `X.X.Z`) can be applied blindly and are for bugfixes and backward compatible changes - - minor (`X.Y.X` to `X.Z.X`) can be applied blindly but might introduce some features that are not backward compatible - - major (`Y.X.X` to `Z.X.X`) must be applied with caution as they might break existing installation - - -!!! warning - - We **strongly** advise you against running crowdsec and LAPI in different versions. - When upgrading existing setup, we suggest you to upgrade both crowdsec, cscli and LAPI. - -# Upgrades from debian packages (official or pragmatic) - -```bash -apt-get update -apt-get install crowdsec -``` - -# Upgrades from release tarball - -## Patch upgrade - -`wizard.sh --binupgrade` - -When doing a minor/patch upgrade (ie. `1.0.0` to `1.0.1`), the `--binupgrade` feature should be the more appropriate : It will simply upgrade the existing binaries, letting all configurations untouched. - -## Minor upgrade - -`wizard.sh --upgrade` - -When doing a minor upgrade (ie. `1.0.4` to `1.1.0`), the `--upgrade` feature should be used : It will attempt to migrate and upgrade any existing configurations, include tainted/custom ones. The ambition is to be able to upgrade scenarios, parsers etc to the latest version when relevant, while keeping custom/tainted ones untouched. - -It's using `cscli config backup`, creating a directory (usually `/tmp/tmp.`) in which it's going to dump all relevant configurations before performing an upgrade : - - - configuration files : `acquis.yaml` `*_credentials.yaml` `profiles.yaml` `simulation.yaml` `config.yaml` - - one directory for **parsers**, **scenarios**, **postoverflows** and **collections**, where it's going to store both reference to upstream configurations, and your custom/tainted ones - -It is then going to cleanup crowdsec configuration, `/etc/crowdsec/` content (except bouncers configuration), before deploying the new binaries. Once this is done, configuration will be restored from our temp directory using `cscli config restore`. - - -## Major upgrade - -For major upgrades (ie. `0.3.X` to `1.0.X`), `wizard` won't do the trick, and you'll have to rely on documentation to do so : - - - Migrating from `0.3.X` to `1.0.X` : [documentation](Crowdsec/v1/migration/) - - -# Manual operations - -[`cscli config`](/Crowdsec/v1/cscli/cscli_config/) is your friend here, with [`backup`](/Crowdsec/v1/cscli/cscli_config_backup/) and [`restore`](/Crowdsec/v1/cscli/cscli_config_backup/) subcommands allowing you to backup and restore all of the configuration files. - - -# Upgrading collections/parsers/scenarios - -[`cscli hub`](/Crowdsec/v1/cscli/cscli_hub/) allows you to view, update and upgrade configurations : - - - [`cscli hub update`](/Crowdsec/v1/cscli/cscli_hub_update/) downloads the latest list of available scenarios/parsers/etc - - [`cscli hub list`](/Crowdsec/v1/cscli/cscli_hub_list/) lists all the installed configurations, their versions and status - - [`cscli hub upgrade`](/Crowdsec/v1/cscli/cscli_hub_upgrade/) upgrades existing configurations to the latest available version in said list diff --git a/docs/v1.X/docs/index.md b/docs/v1.X/docs/index.md deleted file mode 100644 index 01d5e76d4..000000000 --- a/docs/v1.X/docs/index.md +++ /dev/null @@ -1,32 +0,0 @@ -
[[Hub]]({{v1X.hub.url}}) [[Releases]]({{v1X.crowdsec.download_url}})
- -# Architecture - -![Architecture](assets/images/crowdsec_architecture.png) - - -## Components - -{{v1X.crowdsec.name}} ecosystem is based on the following components : - - - [{{v1X.crowdsec.Name}}]({{v1X.crowdsec.url}}) is the lightweight service that processes logs and keeps track of attacks. - - [{{v1X.lapi.Name}}]({{v1X.lapi.url}}) is a core component of crowdsec-agent that exposes a local API to interact with crowdsec-agent. - - [{{v1X.cli.name}}]({{v1X.cli.main_doc}}) is the command line interface for humans, it allows you to view, add, or remove bans as well as to install, find, or update scenarios and parsers - - [{{v1X.bouncers.name}}]({{v1X.hub.bouncers_url}}) are the components that block malevolent traffic, and can be deployed anywhere in your stack - -## Moving forward - -To learn more about {{v1X.crowdsec.name}} and give it a try, please see : - - - [How to install {{v1X.crowdsec.name}}](/Crowdsec/v1/getting_started/installation/) - - [Take a quick tour of {{v1X.crowdsec.name}} and {{v1X.cli.name}} features](/Crowdsec/v1/getting_started/crowdsec-tour/) - - [Observability of {{v1X.crowdsec.name}}](/Crowdsec/v1/observability/overview/) - - [Understand {{v1X.crowdsec.name}} configuration](/Crowdsec/v1/getting_started/concepts/) - - [Deploy {{v1X.bouncers.name}} to stop malevolent peers](/Crowdsec/v1/bouncers/) - - [FAQ](/faq/) - -Don't hesitate to reach out if you're facing issues : - - - [report a bug](https://github.com/crowdsecurity/crowdsec/issues/new?assignees=&labels=bug&template=bug_report.md&title=Bug%2F) - - [suggest an improvement](https://github.com/crowdsecurity/crowdsec/issues/new?assignees=&labels=enhancement&template=feature_request.md&title=Improvment%2F) - - [ask for help on the forums](https://discourse.crowdsec.net) \ No newline at end of file diff --git a/docs/v1.X/docs/localAPI/howto.md b/docs/v1.X/docs/localAPI/howto.md deleted file mode 100644 index 4d0c57ad6..000000000 --- a/docs/v1.X/docs/localAPI/howto.md +++ /dev/null @@ -1,275 +0,0 @@ - - -!!! info - - This page explains how to interact with the local API exposed by crowdsec. - - It's meant to be useful for system administrators, or users that want to create their own bouncers. - - - -## Introduction - -This documentation only covers the API usage from the bouncer POV : - - - Authentication via API token (rather than JWT as crowdsec/cscli) - - Reading decisions - -This guide will assume that you already have crowdsec running locally. - -## Authentication - -Existing tokens can be viewed with `cscli bouncers list` : - -``` -# cscli bouncers list -------------------------------------------------------------------------------------------- - NAME IP ADDRESS VALID LAST API PULL TYPE VERSION -------------------------------------------------------------------------------------------- - cs-firewall-bouncer-hPrueCas ✔️ 2021-02-25T19:54:46+01:00 -------------------------------------------------------------------------------------------- -``` - -Let's create a new token with `cscli bouncers add MyTestClient` : - -``` -# cscli bouncers add MyTestClient -Api key for 'MyTestClient': - - 837be58e22a28738066de1be8f53636b - -Please keep this key since you will not be able to retrive it! - -``` - -This is the token that we will use to authenticate with the API : - -```bash -▶ curl -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" -I localhost:8080/v1/decisions -HTTP/1.1 200 OK -Content-Type: text/plain; charset=utf-8 -Date: Fri, 26 Feb 2021 12:35:37 GMT -``` - -Note: if the token is missing or incorrect, you will get a **403** answer. - -## API Usage - -As stated in the [swagger documentation](https://crowdsecurity.github.io/api_doc/index.html?urls.primaryName=LAPI), bouncer's method are restricted to the `/decisions` path. They allow to query the local decisions in two modes : - - - stream mode : Intended for bouncers that will - on a regular basis - query the local api for new and expired/decisions - - query mode : Intended for bouncers that want to query the local api about a specific ip/range/username etc. - - -## Query Mode - -To have some data to query for, let's add two decisions to our local API - -```bash -▶ sudo cscli decisions add -i 1.2.3.4 -INFO[0000] Decision successfully added -▶ sudo cscli decisions add -r 2.2.3.0/24 -INFO[0000] Decision successfully added -▶ sudo cscli decisions list -+------+--------+------------------+----------------------------------------------------+--------+---------+----+--------+--------------------+----------+ -| ID | SOURCE | SCOPE:VALUE | REASON | ACTION | COUNTRY | AS | EVENTS | EXPIRATION | ALERT ID | -+------+--------+------------------+----------------------------------------------------+--------+---------+----+--------+--------------------+----------+ -| 2337 | cscli | Range:2.2.3.0/24 | manual 'ban' from | ban | | | 1 | 3h59m18.079301785s | 1164 | -| | | | '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA' | | | | | | | -| 2336 | cscli | Ip:1.2.3.4 | manual 'ban' from | ban | | | 1 | 3h59m11.079297437s | 1163 | -| | | | '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA' | | | | | | | -+------+--------+------------------+----------------------------------------------------+--------+---------+----+--------+--------------------+----------+ - -``` - -#### Query mode : IP - -We can now try to query the API : - -> Query a single banned IP - -```bash -▶ curl -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" http://localhost:8080/v1/decisions\?ip=1.2.3.4 -[{"duration":"3h51m57.363171728s","id":2336,"origin":"cscli","scenario":"manual 'ban' from '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA'","scope":"Ip","type":"ban","value":"1.2.3.4"}] -``` - -> Query a single IP - -```bash -▶ curl -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" http://localhost:8080/v1/decisions\?ip=1.2.3.5 -null -``` - -> Query an IP contained in an existing ban - -```bash -▶ curl -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" http://localhost:8080/v1/decisions\?ip\=2.2.3.42 -[{"duration":"3h38m32.349736035s","id":2337,"origin":"cscli","scenario":"manual 'ban' from '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA'","scope":"Range","type":"ban","value":"2.2.3.0/24"}] -``` -_note: notice that the decision returned is the range that we banned earlier and that contains query ip_ - -#### Query mode : Range - -> Query a range in which one of the ban is contained - -```bash -▶ curl -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" http://localhost:8080/v1/decisions\?range=1.2.3.0/24\&contains\=false -[{"duration":"3h48m7.676653651s","id":2336,"origin":"cscli","scenario":"manual 'ban' from '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA'","scope":"Ip","type":"ban","value":"1.2.3.4"}] -``` -_note: notice the `contains` flag that is set to false_ - -```bash -▶ curl -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" http://localhost:8080/v1/decisions\?range=1.2.3.0/24\&contains\=true -null -``` - -> Query a range which is contained by an existing ban - -```bash -▶ curl -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" http://localhost:8080/v1/decisions\?range\=2.2.3.1/25 -[{"duration":"3h30m24.773063133s","id":2337,"origin":"cscli","scenario":"manual 'ban' from '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA'","scope":"Range","type":"ban","value":"2.2.3.0/24"}] -``` - -### Query mode : non IP centric decisions - -While most people will use crowdsec to ban IPs or ranges, decisions can target other scopes and other decisions : - -```bash -▶ sudo cscli decisions add --scope username --value myuser --type enforce_mfa -INFO[0000] Decision successfully added -▶ sudo cscli decisions list -+------+--------+------------------+----------------------------------------------------+-------------+---------+----+--------+--------------------+----------+ -| ID | SOURCE | SCOPE:VALUE | REASON | ACTION | COUNTRY | AS | EVENTS | EXPIRATION | ALERT ID | -+------+--------+------------------+----------------------------------------------------+-------------+---------+----+--------+--------------------+----------+ -| 2338 | cscli | username:myuser | manual 'enforce_mfa' from | enforce_mfa | | | 1 | 3h59m55.384975175s | 1165 | -| | | | '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA' | | | | | | | -| 2337 | cscli | Range:2.2.3.0/24 | manual 'ban' from | ban | | | 1 | 3h27m1.384972861s | 1164 | -| | | | '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA' | | | | | | | -| 2336 | cscli | Ip:1.2.3.4 | manual 'ban' from | ban | | | 1 | 3h26m54.384971268s | 1163 | -| | | | '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA' | | | | | | | -+------+--------+------------------+----------------------------------------------------+-------------+---------+----+--------+--------------------+----------+ -``` - - - -> Query a decision on a given user - -```bash -▶ curl -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" http://localhost:8080/v1/decisions\?scope\=username\&value\=myuser -[{"duration":"3h57m59.021170481s","id":2338,"origin":"cscli","scenario":"manual 'enforce_mfa' from '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA'","scope":"username","type":"enforce_mfa","value":"myuser"}] -``` - -> Query a decision on a given user - -```bash -▶ curl -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" http://localhost:8080/v1/decisions\?scope\=username\&value\=myuser -[{"duration":"3h57m59.021170481s","id":2338,"origin":"cscli","scenario":"manual 'enforce_mfa' from '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA'","scope":"username","type":"enforce_mfa","value":"myuser"}] -``` - - -> Query all decisions of a given type - -```bash -▶ curl -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" http://localhost:8080/v1/decisions\?type\=enforce_mfa -[{"duration":"3h57m21.050290118s","id":2338,"origin":"cscli","scenario":"manual 'enforce_mfa' from '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA'","scope":"username","type":"enforce_mfa","value":"myuser"}] - -``` - -## Stream mode - -The "streaming mode" of the API (which is actually more like polling) allows for bouncers that are going to fetch on a regular basis an update of the existing decisions. The endpoint is `/decisions/stream` with a single `startup` (boolean) argument. The argument allows to indicate if the bouncer wants the full state of decisions, or only an update since it last pulled. - - -Given the our state looks like : - -```bash -▶ sudo cscli decisions list -+------+--------+------------------+----------------------------------------------------+--------+---------+----+--------+--------------------+----------+ -| ID | SOURCE | SCOPE:VALUE | REASON | ACTION | COUNTRY | AS | EVENTS | EXPIRATION | ALERT ID | -+------+--------+------------------+----------------------------------------------------+--------+---------+----+--------+--------------------+----------+ -| 2337 | cscli | Range:2.2.3.0/24 | manual 'ban' from | ban | | | 1 | 2h55m26.05271136s | 1164 | -| | | | '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA' | | | | | | | -| 2336 | cscli | Ip:1.2.3.4 | manual 'ban' from | ban | | | 1 | 2h55m19.052706441s | 1163 | -| | | | '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA' | | | | | | | -+------+--------+------------------+----------------------------------------------------+--------+---------+----+--------+--------------------+----------+ - -``` - -The first call to `/decisions/stream` will look like : - -```bash -▶ curl -s -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" http://localhost:8080/v1/decisions/stream\?startup\=true | jq . -{ - "deleted": [ - { - "duration": "-18897h25m52.809576151s", - "id": 1, - "origin": "crowdsec", - "scenario": "crowdsecurity/http-probing", - "scope": "Ip", - "type": "ban", - "value": "123.206.50.249" - }, - ... - ], - "new": [ - { - "duration": "22h20m11.909761348s", - "id": 2266, - "origin": "CAPI", - "scenario": "crowdsecurity/http-sensitive-files", - "scope": "ip", - "type": "ban", - "value": "91.241.19.122/32" - }, - ... - ] -} -``` -_note: the initial state will contained passed deleted events (to account for crashes/services restart for example), and the current decisions, both local and those fed from the central API_ - - -!!! info - - You might notice that even you are requesting for the initial state, you receive a lot of "deleted" decisions. - This is intended to allow you to easily restart the local API without having a desynchronized state with the bouncers. - -```bash -▶ curl -s -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" http://localhost:8080/v1/decisions/stream\?startup\=false | jq . -{ - "deleted": null, - "new": null -} -``` -_note: Calling the decisions/stream just after will lead to empty results, as no decisions have been added or deleted_ - - - -Let's now add a new decision : - -```bash -▶ sudo cscli decisions add -i 3.3.3.4 -INFO[0000] Decision successfully added -``` - -And call our endpoint again : - -```bash -▶ curl -s -H "X-Api-Key: 837be58e22a28738066de1be8f53636b" http://localhost:8080/v1/decisions/stream\?startup\=false | jq . -{ - "deleted": null, - "new": [ - { - "duration": "3h59m57.641708614s", - "id": 2410, - "origin": "cscli", - "scenario": "manual 'ban' from '939972095cf1459c8b22cc608eff85daEb4yoi2wiTD7Y3fA'", - "scope": "Ip", - "type": "ban", - "value": "3.3.3.4" - } - ] -} -``` - - diff --git a/docs/v1.X/docs/localAPI/index.md b/docs/v1.X/docs/localAPI/index.md deleted file mode 100644 index d0a238542..000000000 --- a/docs/v1.X/docs/localAPI/index.md +++ /dev/null @@ -1,106 +0,0 @@ -# Local API - -The Local API (LAPI) is a core component of {{v1X.crowdsec.name}} and has a few essential missions : - - - Allow crowdsec machines to push alerts & decisions to a database - - Allow bouncers to consume said alerts & decisions from database - - Allow `cscli` to view add or delete decisions - - -You can find the swagger documentation [here](https://crowdsecurity.github.io/api_doc/index.html?urls.primaryName=LAPI). - -## Authentication - -There are two kinds of authentication to the local API : - - - {{v1X.bouncers.Name}} : they authenticate with a simple API key and can only read decisions - - - Machines : they authenticate with a login&password and can not only read decisions, but create new ones too - - -### {{v1X.bouncers.Name}} - -To register a bouncer to your API, you need to run the following command on the server where the API is installed: - -```bash -$ sudo cscli bouncers add testBouncer -``` - -and keep the generated API token to use it in your {{v1X.bouncers.Name}} configuration file. - -### Machines - -To allow a machine to communicate with the local API, the machine needs to be validated by an administrator of the local API. - -There are two ways to register a crowdsec to a local API. - -* You can create a machine directly on the API server that will be automatically validated by running the following command on the server where the API is installed: - -```bash -$ sudo cscli machines add testMachine -``` - -If your crowdsec runs on the same server as the local API, then your credentials file will be generated automatically, otherwise you will have to copy/paste them in your remote crowdsec credentials file (`/etc/crowdsec/local_api_credentials.yaml`) - -* You can use `cscli` to register to the API server: - -``` -sudo cscli lapi register -u -``` - -And validate it with `cscli` on the server where the API is installed: - -``` -sudo cscli machines validate -``` - -!!! tips - You can use `cscli machines list` to list all the machines registered to the API and view the ones that are not validated yet. - -## Configuration - -### Client - -By default, `crowdsec` and `cscli` use `127.0.0.1:8080` as the default local API. However you might want to use a remote API and configure a different endpoint for your api client. - -#### Register to a remote API server - -* On the remote crowdsec server, run: - -``` -$ sudo cscli lapi register -u http://: -``` - -* On the local API server, validate the machine by running the command: - - -```bash -$ sudo cscli machines list # to get the name of the new registered machine -``` - -``` -$ sudo cscli machines validate -``` - - -### Server - -#### Configure listen URL - -If you would like your local API to be used by a remote crowdsec you will need to modify the URL it listens on. -Modify the [`listen_uri` option](/Crowdsec/v1/references/crowdsec-config/#listen_uri) in the main configuration file. -Then see [how to configure your crowdsec to use a remote API](/Crowdsec/v1/localAPI/#register-to-a-remote-api-server). - - -#### Enable SSL - -The most common use case of the local API is to listen on 127.0.0.1. In that case there's no need for -configuring any ssl layer. In some cases, the local API will listen for other crowdsec installations that -will report their triggered scenarios. In that case the endpoint may be configured with ssl. -You can see how to configure SSL on your local API [here](/Crowdsec/v1/references/crowdsec-config/#tls). - - -See the [Local API public documentation]({{v1X.lapi.swagger}}). - - - diff --git a/docs/v1.X/docs/migration.md b/docs/v1.X/docs/migration.md deleted file mode 100644 index bde27eda0..000000000 --- a/docs/v1.X/docs/migration.md +++ /dev/null @@ -1,96 +0,0 @@ -# Migration from v0.X to v1.X - -!!! warning - Migrating to V1.X will impact (any change you made will be lost and must be adapted to the new configuration) : - - - Database model : your existing database will be lost, a new one will be created in the V1. - - - {{v1X.crowdsec.Name}} configuration : - - `/etc/crowdsec/config/default.yaml` : check [new format](/Crowdsec/v1/references/crowdsec-config/#configuration-format) - - `/etc/crowdsec/config/profiles.yaml` : check [new format](/Crowdsec/v1/references/profiles/#profiles-configurations) - - -To upgrade {{v1X.crowdsec.name}} from v0.X to v1, we'll follow those steps - -#### Backup up configuration - -```bash -sudo cscli backup save /tmp/crowdsec_backup -sudo cp -R /etc/crowdsec/config/patterns /tmp/crowdsec_backup -``` - -#### Uninstall old version & install new - -Download latest V1 {{v1X.crowdsec.name}} version [here]({{v1X.crowdsec.download_url}}) - -```bash -tar xvzf crowdsec-release.tgz -cd crowdsec-v1*/ -sudo ./wizard.sh --uninstall -sudo rm /etc/cron.d/crowdsec_pull -sudo ./wizard.sh --bininstall -``` - -!!! warning - Don't forget to remove {{v1X.metabase.name}} dashboard if you installed it manually (without {{v1X.cli.name}}). - -#### Restore configuration - -!!! warning - Before restoring old backup, if you have `local` or `tainted` postoverflows, be aware that they are no longer compatible. You should update the syntax (the community and us are available to help you doing this part). -```bash -sudo cscli hub update -sudo cscli config restore --old-backup /tmp/crowdsec_backup/ -sudo cp -R /tmp/crowdsec_backup/patterns /etc/crowdsec/ -``` - -### Register crowdsec to local & central API - -```bash -$ sudo cscli machines add -a -INFO[0000] Machine '...' created successfully -INFO[0000] API credentials dumped to '/etc/crowdsec/local_api_credentials.yaml' -``` - -Before starting the services, let's check that we're properly registered : - -```bash -$ sudo cscli capi status -INFO[0000] Loaded credentials from /etc/crowdsec/online_api_credentials.yaml -INFO[0000] Trying to authenticate with username ... on https://api.crowdsec.net/ -INFO[0000] You can successfully interact with Central API (CAPI) -``` - -#### Start & health check - -Finally, you will be able to start {{v1X.crowdsec.name}} service. Before that, just check if {{v1X.lapi.name}} and {{v1X.api.name}} are correctly configured. - -```bash -$ sudo systemctl enable crowdsec -$ sudo systemctl start crowdsec -$ sudo cscli lapi status -INFO[0000] Loaded credentials from /etc/crowdsec/local_api_credentials.yaml -INFO[0000] Trying to authenticate with username ... on http://127.0.0.1:8080/ -INFO[0000] You can successfully interact with Local API (LAPI) -$ sudo cscli capi status -INFO[0000] Loaded credentials from /etc/crowdsec/online_api_credentials.yaml -INFO[0000] Trying to authenticate with username ... on https://api.crowdsec.net/ -INFO[0000] You can successfully interact with Central API (CAPI) -``` - -!!! warning - If you're facing issues with `cscli lapi status`, just re-run `cscli machines add -a`. - If you're facing issues with `cscli capi status`, just re-run `cscli capi register` - - -You can check logs (located by default here: `/var/log/crowdsec.log` & `/var/log/crowdsec_api.log`). - -You can now navigate documentation to learn new {{v1X.cli.name}} commands to interact with crowdsec. - -#### Upgrade {{v1X.bouncers.name}} - -If you were using **{{v1X.bouncers.name}}** (formerly called **blocker(s)**), you need to replace them by the new compatibles {{v1X.bouncers.name}}, available on the [hub](https://hub.crowdsec.net/browse/#bouncers) (selecting `agent version` to `v1`). - -Following your bouncer type (netfilter, nginx, wordpress etc...), you need to replace them by the new available {{v1X.bouncers.name}} on the hub, please follow the {{v1X.bouncers.name}} documentation that will help you to install easily. - -We're also available to help (on [discourse](https://discourse.crowdsec.net/) or [gitter](https://gitter.im/crowdsec-project/community)) upgrading your {{v1X.bouncers.name}}. \ No newline at end of file diff --git a/docs/v1.X/docs/observability/command_line.md b/docs/v1.X/docs/observability/command_line.md deleted file mode 100644 index 856d94ef9..000000000 --- a/docs/v1.X/docs/observability/command_line.md +++ /dev/null @@ -1,92 +0,0 @@ -```bash -sudo {{v1X.cli.name}} metrics -``` - -This command provides an overview of {{v1X.crowdsec.name}} statistics provided by [prometheus client](/Crowdsec/v1/observability/prometheus/). By default it assumes that the {{v1X.crowdsec.name}} is installed on the same machine. - -The metrics are split in 3 main sections : - - - Acquisition metrics : How many lines were read from which sources, how many were successfully or unsuccessfully parsed, and how many of those lines ultimately ended up being poured to a bucket. - - Parser metrics : How many lines were fed (eligible) to each parser, and how many of those were successfully or unsuccessfully parsed. - - Bucket metrics : How many time each scenario lead to a bucket instantiation, and for each of those : - - how many times it overflowed - - how many times it expired (underflows) - - how many subsequent events were poured to said bucket - -!!! hint - These metrics should help you identify potential configuration errors. - - For example, if you have a source that has mostly unparsed logs, you know you might be missing some parsers. - As well, if you have scenarios that are never instantiated, it might be a hint that they are not relevant to your configuration. - - Furthermore, you might see parsers called `child-` while calling `cscli metrics`. This correspond to all nodes belonging to a parser. Their metrics - (HITS, PARSED, UNPARSED) are gather by default. If you want to identify metrics for a specific parser node, you just have to set a name for this node in your parser configuration. - - -
- {{v1X.cli.name}} metrics example -```bash -$ sudo cscli metrics - -INFO[0000] Buckets Metrics: -+--------------------------------------+---------------+-----------+--------------+--------+---------+ -| BUCKET | CURRENT COUNT | OVERFLOWS | INSTANCIATED | POURED | EXPIRED | -+--------------------------------------+---------------+-----------+--------------+--------+---------+ -| crowdsecurity/http-bad-user-agent | - | - | 10 | 10 | 10 | -| crowdsecurity/http-crawl-non_statics | - | - | 91 | 119 | 91 | -| crowdsecurity/http-probing | - | - | 2 | 2 | 2 | -| crowdsecurity/http-sensitive-files | - | - | 1 | 1 | 1 | -| crowdsecurity/ssh-bf | 13 | 6314 | 8768 | 46772 | 2441 | -| crowdsecurity/ssh-bf_user-enum | 6 | - | 7646 | 14406 | 7640 | -+--------------------------------------+---------------+-----------+--------------+--------+---------+ -INFO[0000] Acquisition Metrics: -+---------------------------+------------+--------------+----------------+------------------------+ -| SOURCE | LINES READ | LINES PARSED | LINES UNPARSED | LINES POURED TO BUCKET | -+---------------------------+------------+--------------+----------------+------------------------+ -| /var/log/auth.log | 105476 | 46772 | 58704 | 61178 | -| /var/log/messages | 2 | - | 2 | - | -| /var/log/nginx/access.log | 138 | 111 | 27 | 100 | -| /var/log/nginx/error.log | 312 | 68 | 244 | 32 | -| /var/log/syslog | 31919 | - | 31919 | - | -+---------------------------+------------+--------------+----------------+------------------------+ -INFO[0000] Parser Metrics: -+--------------------------------+--------+--------+----------+ -| PARSERS | HITS | PARSED | UNPARSED | -+--------------------------------+--------+--------+----------+ -| child-crowdsecurity/http-logs | 537 | 257 | 280 | -| child-crowdsecurity/nginx-logs | 789 | 179 | 610 | -| child-crowdsecurity/sshd-logs | 436048 | 46772 | 389276 | -| crowdsecurity/dateparse-enrich | 46951 | 46951 | - | -| crowdsecurity/geoip-enrich | 46883 | 46883 | - | -| crowdsecurity/http-logs | 179 | 66 | 113 | -| crowdsecurity/nginx-logs | 450 | 179 | 271 | -| crowdsecurity/non-syslog | 450 | 450 | - | -| crowdsecurity/sshd-logs | 104386 | 46772 | 57614 | -| crowdsecurity/syslog-logs | 137397 | 137395 | 2 | -| crowdsecurity/whitelists | 46951 | 46951 | - | -+--------------------------------+--------+--------+----------+ -INFO[0000] Local Api Metrics: -+----------------------+--------+------+ -| ROUTE | METHOD | HITS | -+----------------------+--------+------+ -| /v1/alerts | GET | 4 | -| /v1/alerts | POST | 5400 | -| /v1/decisions/stream | GET | 7694 | -| /v1/watchers/login | POST | 27 | -+----------------------+--------+------+ -INFO[0000] Local Api Machines Metrics: -+----------------------------------+------------+--------+------+ -| MACHINE | ROUTE | METHOD | HITS | -+----------------------------------+------------+--------+------+ -| 7f0607a3469243139699bf2f30321fc4 | /v1/alerts | GET | 4 | -| 7f0607a3469243139699bf2f30321fc4 | /v1/alerts | POST | 5400 | -+----------------------------------+------------+--------+------+ -INFO[0000] Local Api Bouncers Metrics: -+------------------------------+----------------------+--------+------+ -| BOUNCER | ROUTE | METHOD | HITS | -+------------------------------+----------------------+--------+------+ -| cs-firewall-bouncer-n3W19Qua | /v1/decisions/stream | GET | 7694 | -+------------------------------+----------------------+--------+------+ - -``` -
\ No newline at end of file diff --git a/docs/v1.X/docs/observability/dashboard.md b/docs/v1.X/docs/observability/dashboard.md deleted file mode 100644 index e81b01ddd..000000000 --- a/docs/v1.X/docs/observability/dashboard.md +++ /dev/null @@ -1,83 +0,0 @@ - -!!! warning "MySQL & PostgreSQL" - MySQL and PostgreSQL are currently not supported by `cscli dashboard`. It means that you can run cscli dashboard only if you use `SQLite` (default) as storage database with your local API. - - - -The {{v1X.cli.name}} command `{{v1X.cli.bin}} dashboard setup` will use [docker](https://docs.docker.com/get-docker/) to install [metabase docker image](https://hub.docker.com/r/metabase/metabase/) and fetch our metabase template to have a configured and ready dashboard. - - -## Setup -> Setup and Start crowdsec metabase dashboard - -```bash -sudo {{v1X.cli.bin}} dashboard setup -``` - -Optional arguments: - - - `-l` |`--listen` : ip address to listen on for docker (default is `127.0.0.1`) - - `-p` |`--port` : port to listen on for docker (default is `8080`) - - `--password` : password for metabase user (default is generated randomly) - - `-f` | `--force` : override existing setup - - - -
- {{v1X.cli.name}} dashboard setup - -```bash -INFO[0000] Pulling docker image metabase/metabase -........... -INFO[0002] creating container '/crowdsec-metabase' -INFO[0002] Waiting for metabase API to be up (can take up to a minute) -.............. -INFO[0051] Metabase is ready - - URL : 'http://127.0.0.1:3000' - username : 'crowdsec@crowdsec.net' - password : '' - -``` -
- -!!! tip "Protip" - the `dashboard setup` command will output generated credentials for metabase. - Those are stored in `/etc/crowdsec/metabase/metabase.yaml` - -Now you can connect to your dashboard, sign-in with your saved credentials then click on {{v1X.crowdsec.Name}} Dashboard to get this: - - -Dashboard docker image can be managed by {{v1X.cli.name}} and docker cli also. Look at the {{v1X.cli.name}} help command using - -```bash -sudo {{v1X.cli.bin}} dashboard -h -``` - -## Remove the dashboard -> Remove crowdsec metabase dashboard - -```bash -sudo {{v1X.cli.bin}} dashboard remove [-f] -``` -Optional arguments: - -- `-f` | `--force` : will force remove the dashboard - -## Stop the dashboard -> Stop crowdsec metabase dashboard - -```bash -sudo {{v1X.cli.bin}} dashboard stop -``` - -## Start the dashboard -> Start crowdsec metabase dashboard - -```bash -sudo {{v1X.cli.bin}} dashboard start -``` - -**Note:** Please look [at this documentation](https://doc.crowdsec.net/faq/#how-to-have-a-dashboard-without-docker) for those of you that would like to deploy metabase without using docker. - - diff --git a/docs/v1.X/docs/observability/logs.md b/docs/v1.X/docs/observability/logs.md deleted file mode 100644 index d846264b0..000000000 --- a/docs/v1.X/docs/observability/logs.md +++ /dev/null @@ -1,30 +0,0 @@ -Logs concern everything that happens to {{v1X.crowdsec.Name}} itself (startup, configuration, events like IP ban or an alert, shutdown, and so on). - -By default, logs are written to `/var/log/crowdsec.log`, in text format. - -
- Logs example - - -```bash -time="12-05-2020 15:34:21" level=info msg="setting loglevel to info" -time="12-05-2020 15:34:21" level=info msg="Crowdsec v0.0.19-9ae496aa9cfd008513976a096accc7cfc43f2d9b" -time="12-05-2020 15:34:21" level=warning msg="Loading prometheus collectors" -time="12-05-2020 15:34:23" level=warning msg="no version in /etc/crowdsec/parsers/s00-raw/syslog-logs.yaml, assuming '1.0'" -time="12-05-2020 15:34:23" level=warning msg="Starting profiling and http server" -time="12-05-2020 15:34:24" level=warning msg="no version in /etc/crowdsec/parsers/s00-raw/syslog-logs.yaml, assuming '1.0'" -time="12-05-2020 15:34:24" level=info msg="Node has no name,author or description. Skipping." -time="12-05-2020 15:34:24" level=info msg="Loading 2 parser nodes" file=/etc/crowdsec/parsers/s00-raw/syslog-logs.yaml -time="12-05-2020 15:34:24" level=warning msg="no version in /etc/crowdsec/parsers/s01-parse/nginx-logs.yaml, assuming '1.0'" -time="12-05-2020 15:34:24" level=info msg="Loading 3 parser nodes" file=/etc/crowdsec/parsers/s01-parse/nginx-logs.yaml -time="12-05-2020 15:34:24" level=warning msg="no version in /etc/crowdsec/parsers/s01-parse/sshd-logs.yaml, assuming '1.0'" -time="13-05-2020 17:42:53" level=warning msg="24 existing LeakyRoutine" -time="13-05-2020 18:02:51" level=info msg="Flushed 1 expired entries from Ban Application" -time="13-05-2020 18:12:46" level=info msg="Flushed 1 expired entries from Ban Application" -time="13-05-2020 18:20:29" level=warning msg="11.11.11.11 triggered a 4h0m0s ip ban remediation for [crowdsecurity/ssh-bf]" bucket_id=winter-shadow event_time="2020-05-13 18:20:29.855776892 +0200 CEST m=+96112.558589990" scenario=crowdsecurity/ssh-bf source_ip=11.11.11.11 -time="13-05-2020 18:31:26" level=warning msg="22.22.22.22 triggered a 4h0m0s ip ban remediation for [crowdsecurity/ssh-bf]" bucket_id=dry-mountain event_time="2020-05-13 18:31:26.989769738 +0200 CEST m=+96769.692582872" scenario=crowdsecurity/ssh-bf source_ip=22.22.22.22 -time="13-05-2020 18:41:10" level=warning msg="16 existing LeakyRoutine" -time="13-05-2020 18:46:19" level=warning msg="33.33.33.33 triggered a 4h0m0s ip ban remediation for [crowdsecurity/iptables-scan-multi_ports]" bucket_id=holy-paper event_time="2020-05-13 18:46:19.825693323 +0200 CEST m=+97662.528506421" scenario=crowdsecurity/iptables-scan-multi_ports source_ip=33.33.33.33 -``` - -
\ No newline at end of file diff --git a/docs/v1.X/docs/observability/overview.md b/docs/v1.X/docs/observability/overview.md deleted file mode 100644 index 8cd9565ec..000000000 --- a/docs/v1.X/docs/observability/overview.md +++ /dev/null @@ -1,13 +0,0 @@ -# Observability Overview - -Observability in security software is crucial, especially when this software might take important decision such as blocking IP addresses. - -We attempt to provide good observability of {{v1X.crowdsec.name}}'s behavior : - - - {{v1X.crowdsec.name}} itself exposes a [prometheus instrumentation](/Crowdsec/v1/observability/prometheus/) - - {{v1X.cli.Name}} allows you to view part of prometheus metrics in [cli (`{{v1X.cli.bin}} metrics`)](/Crowdsec/v1/observability/command_line/) - - {{v1X.crowdsec.name}} logging is contextualized for easy processing - - for **humans**, {{v1X.cli.name}} allows you to trivially start a service [exposing dashboards](/Crowdsec/v1/observability/dashboard/) (using [metabase](https://www.metabase.com/)) - -Furthermore, most of {{v1X.crowdsec.name}} configuration should allow you to enable partial debug (ie. per-scenario, per-parser etc.) - diff --git a/docs/v1.X/docs/observability/prometheus.md b/docs/v1.X/docs/observability/prometheus.md deleted file mode 100644 index 3ca013309..000000000 --- a/docs/v1.X/docs/observability/prometheus.md +++ /dev/null @@ -1,93 +0,0 @@ -{{v1X.crowdsec.name}} can expose a {{v1X.prometheus.htmlname}} endpoint for collection (on `http://127.0.0.1:6060/metrics` by default). - -The goal of this endpoint, besides the usual resources consumption monitoring, aims at offering a view of {{v1X.crowdsec.name}} "applicative" behavior : - - - is it processing a lot of logs ? is it parsing them successfully ? - - are a lot of scenarios being triggered ? - - are a lot of IPs banned ? - - etc. - -All the counters are "since {{v1X.crowdsec.name}} start". - -### Metrics details - -#### Scenarios - - - `cs_buckets` : number of scenario that currently exist - - `cs_bucket_created_total` : total number of instantiation of each scenario - - `cs_bucket_overflowed_total` : total number of overflow of each scenario - - `cs_bucket_underflowed_total` : total number of underflow of each scenario (bucket was created but expired because of lack of events) - - `cs_bucket_poured_total` : total number of event poured to each scenario with source as complementary key - -
- example - - -``` -#2030 lines from `/var/log/nginx/access.log` were poured to `crowdsecurity/http-scan-uniques_404` scenario -cs_bucket_poured_total{name="crowdsecurity/http-scan-uniques_404",source="/var/log/nginx/access.log"} 2030 -``` - -
- - -#### Parsers - - `cs_node_hits_total` : how many time an event from a specific source was processed by a parser node : - - -
- example - - -``` -# 235 lines from `auth.log` were processed by the `crowdsecurity/dateparse-enrich` parser -cs_node_hits_total{name="crowdsecurity/dateparse-enrich",source="/var/log/auth.log"} 235 -``` - -
- - - `cs_node_hits_ko_total` : how many times an event from a specific was unsuccessfully parsed by a specific parser - -
- example - - -``` -# 2112 lines from `error.log` failed to be parsed by `crowdsecurity/http-logs` -cs_node_hits_ko_total{name="crowdsecurity/http-logs",source="/var/log/nginx/error.log"} 2112 -``` - -
- - - `cs_node_hits_ok_total` : how many times an event from a specific source was successfully parsed by a specific parser - - - `cs_parser_hits_total` : how many times an event from a source has hit the parser - - `cs_parser_hits_ok_total` : how many times an event from a source was successfully parsed - - `cs_parser_hits_ko_total` : how many times an event from a source was unsuccessfully parsed - - -#### Acquisition - - - `cs_reader_hits_total` : how many events were read from a specific source - -#### Local API - - - `cs_lapi_route_requests_total` : number of calls to each route per method - - `cs_lapi_machine_requests_total` : number of calls to each route per method grouped by machines - - `cs_lapi_bouncer_requests_total` : number of calls to each route per method grouped by bouncers - - `cs_lapi_decisions_ko_total` : number of unsuccessfully responses when bouncers ask for an IP. - - `cs_lapi_decisions_ok_total` : number of successfully responses when bouncers ask for an IP. - -#### Info - - - `cs_info` : Information about {{v1X.crowdsec.name}} (software version) - -### Exploitation with prometheus server & grafana - -Those metrics can be scaped by [prometheus server](https://prometheus.io/docs/introduction/overview/#architecture) and visualized with [grafana](https://grafana.com/). They [can be downloaded here](https://github.com/crowdsecurity/grafana-dashboards) : - -![Overview](../assets/images/grafana_overview.png) - -![Insight](../assets/images/grafana_insight.png) - -![Details](../assets/images/grafana_details.png) \ No newline at end of file diff --git a/docs/v1.X/docs/references/acquisition.md b/docs/v1.X/docs/references/acquisition.md deleted file mode 100644 index d37927078..000000000 --- a/docs/v1.X/docs/references/acquisition.md +++ /dev/null @@ -1,57 +0,0 @@ -# Acquisition format - -The `crowdsec_service` section of configuration supports `acquisition_path` and `acquisition_dir` (>1.0.7). - -The default setting is to have `acquisition_path` pointing to `/etc/crowdsec/acquis.yaml`. - -`acquisition_dir` can be set to point to a directory where every `.yaml` file is considered as a valid acquisition configuration file. - - - -The acquisition file(s) define which source of information (ie. files or journald streams) are read by crowdsec at runtime. -The file is a list of object representing groups of files to read, with the following properties. - -A least one of : - - - filename: a string representing the path to a file (globbing supported) - - filenames: a list of string represent paths to files (globbing supported) - - journalctl_filter: a list of string passed as arguments to `journalctl` - -And a `labels` object with a field `type` indicating the log's type : -```yaml -filenames: - - /var/log/nginx/access-*.log - - /var/log/nginx/error.log -labels: - type: nginx ---- -filenames: - - /var/log/auth.log -labels: - type: syslog ---- -journalctl_filter: - - "_SYSTEMD_UNIT=ssh.service" -labels: - type: syslog - -``` - -The `labels.type` is *important* as it is what will determine which parser will try to process the logs. - -The log won't be processed by the syslog parser if its type is not syslog : -```bash -$ cat /etc/crowdsec/parsers/s00-raw/syslog-logs.yaml -filter: "evt.Line.Labels.type == 'syslog'" -... -``` - -On the other hand, nginx tends to write its own logs without using syslog : -```bash -$ cat /etc/crowdsec/parsers/s01-parse/nginx-logs.yaml -filter: "evt.Parsed.program startsWith 'nginx'" -... -``` - -If for example your nginx was logging via syslog, you need to set its `labels.type` to `syslog` so that it's first parsed by the syslog parser, and *then* by the nginx parser (notice they are in different stages). - diff --git a/docs/v1.X/docs/references/alerts.md b/docs/v1.X/docs/references/alerts.md deleted file mode 100644 index 6102bcab3..000000000 --- a/docs/v1.X/docs/references/alerts.md +++ /dev/null @@ -1,8 +0,0 @@ -# Alerts - -An `Alert` is the runtime representation of a bucket overflow. - -The representation of the object can be found here : - -[Alert object documentation](https://pkg.go.dev/github.com/crowdsecurity/crowdsec/pkg/types#RuntimeAlert) - diff --git a/docs/v1.X/docs/references/collections.md b/docs/v1.X/docs/references/collections.md deleted file mode 100644 index 48a78b3e0..000000000 --- a/docs/v1.X/docs/references/collections.md +++ /dev/null @@ -1,29 +0,0 @@ -# Collections - -Collections are bundle of parsers, scenarios, postoverflows that form a coherent package. -Collections are present in `/etc/crowdsec/collections/` and follow this format : - -> `/etc/crowdsec/collections/linux.yaml` - -```yaml -#the list of parsers it contains -parsers: - - crowdsecurity/syslog-logs - - crowdsecurity/geoip-enrich - - crowdsecurity/dateparse-enrich -#the list of collections it contains -collections: - - crowdsecurity/sshd -# the list of postoverflows it contains -# postoverflows: -# - crowdsecurity/seo-bots-whitelist -# the list of scenarios it contains -# scenarios: -# - crowdsecurity/http-crawl-non_statics -description: "core linux support : syslog+geoip+ssh" -author: crowdsecurity -tags: - - linux -``` - -It mostly exists as a convenience for the user when using the hub. diff --git a/docs/v1.X/docs/references/crowdsec-config.md b/docs/v1.X/docs/references/crowdsec-config.md deleted file mode 100644 index 64421fb52..000000000 --- a/docs/v1.X/docs/references/crowdsec-config.md +++ /dev/null @@ -1,449 +0,0 @@ -# Crowdsec configuration - -{{v1X.crowdsec.Name}} has a main `yaml` configuration file, usually located in `/etc/crowdsec/config.yaml`. - -## Configuration example - - -
- Default configuration - -```yaml -common: - daemonize: true - pid_dir: /var/run/ - log_media: file - log_level: info - log_dir: /var/log/ - working_dir: . -config_paths: - config_dir: /etc/crowdsec/ - data_dir: /var/lib/crowdsec/data/ - simulation_path: /etc/crowdsec/simulation.yaml - hub_dir: /etc/crowdsec/hub/ - index_path: /etc/crowdsec/hub/.index.json -crowdsec_service: - acquisition_path: /etc/crowdsec/acquis.yaml - #acquisition_dir: /etc/crowdsec/acquis/ - parser_routines: 1 - buckets_routines: 1 - output_routines: 1 -cscli: - output: human - hub_branch: wip_lapi -db_config: - log_level: info - type: sqlite - db_path: /var/lib/crowdsec/data/crowdsec.db - #user: - #password: - #db_name: - #host: - #port: - flush: - max_items: 5000 - max_age: 7d -api: - client: - insecure_skip_verify: false - credentials_path: /etc/crowdsec/local_api_credentials.yaml - server: - log_level: info - listen_uri: 127.0.0.1:8080 - profiles_path: /etc/crowdsec/profiles.yaml - use_forwarded_for_headers: false - online_client: # Crowdsec API - credentials_path: /etc/crowdsec/online_api_credentials.yaml -# tls: -# cert_file: /etc/crowdsec/ssl/cert.pem -# key_file: /etc/crowdsec/ssl/key.pem -prometheus: - enabled: true - level: full - listen_addr: 127.0.0.1 - listen_port: 6060 -``` - -
- -## Environment variable - -It is possible to set a configuration value based on an enrivonement variables. - -For example, if you don't want to store your database password in the configuration file, you can do this: - -```yaml -db_config: - type: mysql - user: database_user - password: ${DB_PASSWORD} - db_name: db_name - host: 192.168.0.2 - port: 3306 -``` - -And export the environment variable such as: - -```bash -export DB_PASSWORD="" -``` - -!!! warning - **Note**: you need to be `root` or put the environment variable in `/etc/environement` - -## Configuration format - -```yaml -common: - daemonize: (true|false) - pid_dir: - log_media: (file|stdout) - log_level: (error|info|debug|trace) - log_dir: - working_dir: -config_paths: - config_dir: - data_dir: - simulation_path: - hub_dir: - index_path: -crowdsec_service: - acquisition_path: - acquisition_dir: - parser_routines: - buckets_routines: - output_routines: -cscli: - output: (human|json|raw) - hub_branch: -db_config: - type: - db_path: - user: # for mysql/pgsql - password: # for mysql/pgsql - db_name: # for mysql/pgsql - host: # for mysql/pgsql - port: # for mysql/pgsql - sslmode: # for pgsql - flush: - max_items: - max_age: -api: - client: - insecure_skip_verify: (true|false) - credentials_path: - server: - log_level: (error|info|debug|trace>) - listen_uri: # host:port - profiles_path: - use_forwarded_for_headers: - online_client: - credentials_path: - tls: - cert_file: - key_file: -prometheus: - enabled: (true|false) - level: (full|aggregated) - listen_addr: - listen_port: -``` - -## Configuration directives - -### `common` - -```yaml -common: - daemonize: (true|false) - pid_dir: - log_media: (file|stdout) - log_level: (error|info|debug|trace) - log_dir: - working_dir: -``` - -#### `daemonize` -> bool - -Daemonize or not the crowdsec daemon. - -#### `pid_dir` -> string - -Folder to store PID file. - -#### `log_media` -> string - -Log media. Can be `stdout` or `file`. - -#### `log_level` -> string - -Log level. Can be `error`, `info`, `debug`, `trace`. - -#### `log_folder` -> string - -Folder to write log file. - -!!! warning - Works only with `log_media = file`. - -#### `working_dir` -> string - -Current working directory. - - -### `config_paths` - -This section contains most paths to various sub configuration items. - - -```yaml -config_paths: - config_dir: - data_dir: - simulation_path: - hub_dir: - index_path: -``` - -#### `config_dir` -> string - -The main configuration directory of crowdsec. - -#### `data_dir` -> string - -This is where crowdsec is going to store data, such as files downloaded by scenarios, geolocalisation database, metabase configuration database, or even SQLite database. - -#### `simulation_path` -> string - -The path to the {{v1X.simulation.htmlname}} configuration. - -#### `hub_dir` -> string - -The directory where `cscli` will store parsers, scenarios, collections and such. - -#### `index_path` -> string - -Tath to the `.index.json` file downloaded by `cscli` to know the list of available configurations. - - -### `crowdsec_service` - -This section is only used by crowdsec agent. - - -```yaml -crowdsec_service: - acquisition_path: - acquisition_dir: - parser_routines: - buckets_routines: - output_routines: -``` - - -#### `parser_routines` -> int - -Number of dedicated goroutines for parsing files. - -#### `buckets_routines` -> int - -Number of dedicated goroutines for managing live buckets. - -#### `output_routines` -> int - -Number of dedicated goroutines for pushing data to local api. - -#### `acquisition_path` -> string - -Path to the yaml file containing logs that needs to be read. - -#### `acquisition_dir` -> string - -(>1.0.7) Path to a directory where each yaml is considered as a acquisition configuration file containing logs that needs to be read. - - -### `cscli` - -This section is only used by `cscli`. - -```yaml -cscli: - output: (human|json|raw) - hub_branch: - prometheus_uri: -``` - -#### `output` -> string - -The default output format (human, json or raw). - -#### `hub_branch` -> string - -The git branch on which `cscli` is going to fetch configurations. - -#### `prometheus_uri` -> uri - -(>1.0.7) An uri (without the trailing `/metrics`) that will be used by `cscli metrics` command, ie. `http://127.0.0.1:6060/` - -## `db_config` - -Please refer to the [database configuration](/Crowdsec/v1/references/database). - -## `api` - -The api section is used by both `cscli`, `crowdsec` and the local API. - -```yaml -api: - client: - insecure_skip_verify: (true|false) - credentials_path: - server: - log_level: (error|info|debug|trace>) - listen_uri: # host:port - profiles_path: - use_forwarded_for_headers: (true|false) - online_client: - credentials_path: - tls: - cert_file: - key_file: -``` - -### `client` - -The client subsection is used by `crowdsec` and `cscli` to read and write decisions to the local API. - -```yaml -client: - insecure_skip_verify: (true|false) - credentials_path: -``` - -#### `insecure_skip_verify` ->bool - -Allows the use of https with self-signed certificates. - -#### `credentials_path` ->string - -Path to the credential files (contains API url + login/password). - -### `server` - -The `server` subsection is the local API configuration. - -```yaml -server: - log_level: (error|info|debug|trace) - listen_uri: # host:port - profiles_path: - use_forwarded_for_headers: (true|false) - online_client: - credentials_path: - tls: - cert_file: - key_file: -``` - -#### `listen_uri` -> string - -Address and port listen configuration, the form `host:port`. - -#### `profiles_path` -> string - -The path to the {{v1X.profiles.htmlname}} configuration. - -#### `use_forwarded_for_headers` -> string - -Allow the usage of `X-Forwarded-For` or `X-Real-IP` to get the client IP address. Do not enable if you are not running the LAPI behind a trusted reverse-proxy or LB. - -#### `online_client` - -Configuration to push signals and receive bad IPs from Crowdsec API. - -```yaml -online_client: - credentials_path: -``` - -##### `credentials_path` -> string - -Path to a file containing credentials for the Central API. - -#### `tls` - -if present, holds paths to certs and key files. - -```yaml -tls: - cert_file: - key_file: -``` - -##### `cert_file` -> string - -Path to certificate file. - -##### `key_file` -> string - -Path to certficate key file. - -### `prometheus` - -This section is used by local API and crowdsec. - -```yaml -prometheus: - enabled: (true|false) - level: (full|aggregated) - listen_addr: - listen_port: -``` - - -#### `enabled` -> bool - -Allows to enable/disable prometheus instrumentation. - -#### `level` -> string - -Can be `full` (all metrics) or `aggregated` (to allow minimal metrics that will keep cardinality low). - -#### `listen_addr` -> string - -Prometheus listen url. - -#### `listen_port` -> int - -Prometheus listen port. diff --git a/docs/v1.X/docs/references/database.md b/docs/v1.X/docs/references/database.md deleted file mode 100644 index 234a65636..000000000 --- a/docs/v1.X/docs/references/database.md +++ /dev/null @@ -1,347 +0,0 @@ - -# Database - -The database is mostly used by the {{v1X.lapi.htmlname}} but also by {{v1X.cli.user_guide}} for some tasks. - -Currently, 3 databases are supported: - -- `sqlite` (default database) - -- `mysql` - -- `postgresql` - - -!!! warning - It is recommanded to use `mysql` or `postgresql` if you expect to have a lot of traffic on the API. - - -The database configuration can be found in the `crowdsec` configuration file (default: {{v1X.config.crowdsec_config_file}}). - -Its located under the `db_config` block. - -## Configuration Examples - -
- SQLite - -```yaml -db_config: - type: sqlite - db_path: /var/lib/crowdsec/data/crowdsec.db - flush: - max_items: 5000 - max_age: 7d -``` -
-
-MySQL - -```yaml -db_config: - type: mysql - user: crowdsec - password: crowdsecpassword - db_name: crowdsec - host: "127.0.0.1" - port: 3306 - flush: - max_items: 5000 - max_age: 7d -``` -
-
-PostgreSQL - -```yaml -db_config: - type: postgresql - user: crowdsec - password: crowdsecpassword - db_name: crowdsec - host: "127.0.0.1" - port: 5432 - sslmode: disable - flush: - max_items: 5000 - max_age: 7d -``` - -
- -## Configuration Format - -### `db_config` -> Contains the configuration of the database - -```yaml -db_config: - type: - - db_path: # for sqlite - - user: # for mysql/pgsql - password: # for mysql/pgsql - db_name: # for mysql/pgsql - host: # for mysql/pgsql - port: # for mysql/pgsql - sslmode: # for pgsql - flush: - max_items: - max_age: -``` - - -## Configuration Directives - -### `type` - -```yaml -db_config: - type: sqlite -``` - -The `type` of database to use. It can be: - -- `sqlite` -- `mysql` -- `postgresql` - -### `db_path` - -```yaml -db_config: - type: sqlite - db_path: "/var/lib/crowdsec/data/crowdsec.db -``` - -The path to the database file (only if the type of database is `sqlite`) - -### `user` - -```yaml -db_config: - type: mysql|postgresql - - user: foo -``` -The username to connect to the database (only if the type of database is `mysql` or `postgresql`) - -### `password` - -```yaml -db_config: - type: mysql|postgresql - - password: foobar -``` -The password to connect to the database (only if the type of database is `mysql` or `postgresql`) - -### `db_name` - -```yaml -db_config: - type: mysql|postgresql - - db_name: crowdsec -``` -The database name to connect to (only if the type of database is `mysql` or `postgresql`) - -### `db_host` - -```yaml -db_config: - type: mysql|postgresql - - user: foo -``` -The host to connect to (only if the type of database is `mysql` or `postgresql`) - -### `db_port` - -```yaml -db_config: - type: mysql|postgresql - - user: foo -``` -The port to connect to (only if the type of database is `mysql` or `postgresql`) - -```yaml -db_config: - type: postgresql - - sslmode: required -``` -Required or disable ssl connection to database (only if the type of database is `postgresql`) - -### `flush` - -```yaml -flush: - max_items: - max_age: -``` - -#### `max_items` -> int - -Number max of alerts in database. - -#### `max_age` -> string - -Alerts retention time. - -Supported units: - - - `s`: seconds - - - `m`: minutes - - - `h`: hours - - - `d`: days - - -## Database schema - -{{v1X.crowdsec.name}} uses the [ent framework](https://entgo.io/) to manage the database. - -This is the schema of the database (as seen by `entc describe`) - -``` -Alert: - +-----------------+-----------+--------+----------+----------+---------+---------------+-----------+----------------------------------+------------+ - | Field | Type | Unique | Optional | Nillable | Default | UpdateDefault | Immutable | StructTag | Validators | - +-----------------+-----------+--------+----------+----------+---------+---------------+-----------+----------------------------------+------------+ - | id | int | false | false | false | false | false | false | json:"id,omitempty" | 0 | - | created_at | time.Time | false | false | false | true | false | false | json:"created_at,omitempty" | 0 | - | updated_at | time.Time | false | false | false | true | false | false | json:"updated_at,omitempty" | 0 | - | scenario | string | false | false | false | false | false | false | json:"scenario,omitempty" | 0 | - | bucketId | string | false | true | false | true | false | false | json:"bucketId,omitempty" | 0 | - | message | string | false | true | false | true | false | false | json:"message,omitempty" | 0 | - | eventsCount | int32 | false | true | false | true | false | false | json:"eventsCount,omitempty" | 0 | - | startedAt | time.Time | false | true | false | true | false | false | json:"startedAt,omitempty" | 0 | - | stoppedAt | time.Time | false | true | false | true | false | false | json:"stoppedAt,omitempty" | 0 | - | sourceIp | string | false | true | false | false | false | false | json:"sourceIp,omitempty" | 0 | - | sourceRange | string | false | true | false | false | false | false | json:"sourceRange,omitempty" | 0 | - | sourceAsNumber | string | false | true | false | false | false | false | json:"sourceAsNumber,omitempty" | 0 | - | sourceAsName | string | false | true | false | false | false | false | json:"sourceAsName,omitempty" | 0 | - | sourceCountry | string | false | true | false | false | false | false | json:"sourceCountry,omitempty" | 0 | - | sourceLatitude | float32 | false | true | false | false | false | false | json:"sourceLatitude,omitempty" | 0 | - | sourceLongitude | float32 | false | true | false | false | false | false | json:"sourceLongitude,omitempty" | 0 | - | sourceScope | string | false | true | false | false | false | false | json:"sourceScope,omitempty" | 0 | - | sourceValue | string | false | true | false | false | false | false | json:"sourceValue,omitempty" | 0 | - | capacity | int32 | false | true | false | false | false | false | json:"capacity,omitempty" | 0 | - | leakSpeed | string | false | true | false | false | false | false | json:"leakSpeed,omitempty" | 0 | - | scenarioVersion | string | false | true | false | false | false | false | json:"scenarioVersion,omitempty" | 0 | - | scenarioHash | string | false | true | false | false | false | false | json:"scenarioHash,omitempty" | 0 | - | simulated | bool | false | false | false | true | false | false | json:"simulated,omitempty" | 0 | - +-----------------+-----------+--------+----------+----------+---------+---------------+-----------+----------------------------------+------------+ - +-----------+----------+---------+---------+----------+--------+----------+ - | Edge | Type | Inverse | BackRef | Relation | Unique | Optional | - +-----------+----------+---------+---------+----------+--------+----------+ - | owner | Machine | true | alerts | M2O | true | true | - | decisions | Decision | false | | O2M | false | true | - | events | Event | false | | O2M | false | true | - | metas | Meta | false | | O2M | false | true | - +-----------+----------+---------+---------+----------+--------+----------+ - -Bouncer: - +------------+-----------+--------+----------+----------+---------+---------------+-----------+-----------------------------+------------+ - | Field | Type | Unique | Optional | Nillable | Default | UpdateDefault | Immutable | StructTag | Validators | - +------------+-----------+--------+----------+----------+---------+---------------+-----------+-----------------------------+------------+ - | id | int | false | false | false | false | false | false | json:"id,omitempty" | 0 | - | created_at | time.Time | false | false | false | true | false | false | json:"created_at,omitempty" | 0 | - | updated_at | time.Time | false | false | false | true | false | false | json:"updated_at,omitempty" | 0 | - | name | string | true | false | false | false | false | false | json:"name,omitempty" | 0 | - | api_key | string | false | false | false | false | false | false | json:"api_key,omitempty" | 0 | - | revoked | bool | false | false | false | false | false | false | json:"revoked,omitempty" | 0 | - | ip_address | string | false | true | false | true | false | false | json:"ip_address,omitempty" | 0 | - | type | string | false | true | false | false | false | false | json:"type,omitempty" | 0 | - | version | string | false | true | false | false | false | false | json:"version,omitempty" | 0 | - | until | time.Time | false | true | false | true | false | false | json:"until,omitempty" | 0 | - | last_pull | time.Time | false | false | false | true | false | false | json:"last_pull,omitempty" | 0 | - +------------+-----------+--------+----------+----------+---------+---------------+-----------+-----------------------------+------------+ - -Decision: - +--------------+-----------+--------+----------+----------+---------+---------------+-----------+-------------------------------+------------+ - | Field | Type | Unique | Optional | Nillable | Default | UpdateDefault | Immutable | StructTag | Validators | - +--------------+-----------+--------+----------+----------+---------+---------------+-----------+-------------------------------+------------+ - | id | int | false | false | false | false | false | false | json:"id,omitempty" | 0 | - | created_at | time.Time | false | false | false | true | false | false | json:"created_at,omitempty" | 0 | - | updated_at | time.Time | false | false | false | true | false | false | json:"updated_at,omitempty" | 0 | - | until | time.Time | false | false | false | false | false | false | json:"until,omitempty" | 0 | - | scenario | string | false | false | false | false | false | false | json:"scenario,omitempty" | 0 | - | type | string | false | false | false | false | false | false | json:"type,omitempty" | 0 | - | start_ip | int64 | false | true | false | false | false | false | json:"start_ip,omitempty" | 0 | - | end_ip | int64 | false | true | false | false | false | false | json:"end_ip,omitempty" | 0 | - | start_suffix | int64 | false | true | false | false | false | false | json:"start_suffix,omitempty" | 0 | - | end_suffix | int64 | false | true | false | false | false | false | json:"end_suffix,omitempty" | 0 | - | ip_size | int64 | false | true | false | false | false | false | json:"ip_size,omitempty" | 0 | - | scope | string | false | false | false | false | false | false | json:"scope,omitempty" | 0 | - | value | string | false | false | false | false | false | false | json:"value,omitempty" | 0 | - | origin | string | false | false | false | false | false | false | json:"origin,omitempty" | 0 | - | simulated | bool | false | false | false | true | false | false | json:"simulated,omitempty" | 0 | - +--------------+-----------+--------+----------+----------+---------+---------------+-----------+-------------------------------+------------+ - +-------+-------+---------+-----------+----------+--------+----------+ - | Edge | Type | Inverse | BackRef | Relation | Unique | Optional | - +-------+-------+---------+-----------+----------+--------+----------+ - | owner | Alert | true | decisions | M2O | true | true | - +-------+-------+---------+-----------+----------+--------+----------+ - -Event: - +------------+-----------+--------+----------+----------+---------+---------------+-----------+-----------------------------+------------+ - | Field | Type | Unique | Optional | Nillable | Default | UpdateDefault | Immutable | StructTag | Validators | - +------------+-----------+--------+----------+----------+---------+---------------+-----------+-----------------------------+------------+ - | id | int | false | false | false | false | false | false | json:"id,omitempty" | 0 | - | created_at | time.Time | false | false | false | true | false | false | json:"created_at,omitempty" | 0 | - | updated_at | time.Time | false | false | false | true | false | false | json:"updated_at,omitempty" | 0 | - | time | time.Time | false | false | false | false | false | false | json:"time,omitempty" | 0 | - | serialized | string | false | false | false | false | false | false | json:"serialized,omitempty" | 1 | - +------------+-----------+--------+----------+----------+---------+---------------+-----------+-----------------------------+------------+ - +-------+-------+---------+---------+----------+--------+----------+ - | Edge | Type | Inverse | BackRef | Relation | Unique | Optional | - +-------+-------+---------+---------+----------+--------+----------+ - | owner | Alert | true | events | M2O | true | true | - +-------+-------+---------+---------+----------+--------+----------+ - -Machine: - +-------------+-----------+--------+----------+----------+---------+---------------+-----------+------------------------------+------------+ - | Field | Type | Unique | Optional | Nillable | Default | UpdateDefault | Immutable | StructTag | Validators | - +-------------+-----------+--------+----------+----------+---------+---------------+-----------+------------------------------+------------+ - | id | int | false | false | false | false | false | false | json:"id,omitempty" | 0 | - | created_at | time.Time | false | false | false | true | false | false | json:"created_at,omitempty" | 0 | - | updated_at | time.Time | false | false | false | true | false | false | json:"updated_at,omitempty" | 0 | - | machineId | string | true | false | false | false | false | false | json:"machineId,omitempty" | 0 | - | password | string | false | false | false | false | false | false | json:"password,omitempty" | 0 | - | ipAddress | string | false | false | false | false | false | false | json:"ipAddress,omitempty" | 0 | - | scenarios | string | false | true | false | false | false | false | json:"scenarios,omitempty" | 1 | - | version | string | false | true | false | false | false | false | json:"version,omitempty" | 0 | - | isValidated | bool | false | false | false | true | false | false | json:"isValidated,omitempty" | 0 | - | status | string | false | true | false | false | false | false | json:"status,omitempty" | 0 | - +-------------+-----------+--------+----------+----------+---------+---------------+-----------+------------------------------+------------+ - +--------+-------+---------+---------+----------+--------+----------+ - | Edge | Type | Inverse | BackRef | Relation | Unique | Optional | - +--------+-------+---------+---------+----------+--------+----------+ - | alerts | Alert | false | | O2M | false | true | - +--------+-------+---------+---------+----------+--------+----------+ - -Meta: - +------------+-----------+--------+----------+----------+---------+---------------+-----------+-----------------------------+------------+ - | Field | Type | Unique | Optional | Nillable | Default | UpdateDefault | Immutable | StructTag | Validators | - +------------+-----------+--------+----------+----------+---------+---------------+-----------+-----------------------------+------------+ - | id | int | false | false | false | false | false | false | json:"id,omitempty" | 0 | - | created_at | time.Time | false | false | false | true | false | false | json:"created_at,omitempty" | 0 | - | updated_at | time.Time | false | false | false | true | false | false | json:"updated_at,omitempty" | 0 | - | key | string | false | false | false | false | false | false | json:"key,omitempty" | 0 | - | value | string | false | false | false | false | false | false | json:"value,omitempty" | 1 | - +------------+-----------+--------+----------+----------+---------+---------------+-----------+-----------------------------+------------+ - +-------+-------+---------+---------+----------+--------+----------+ - | Edge | Type | Inverse | BackRef | Relation | Unique | Optional | - +-------+-------+---------+---------+----------+--------+----------+ - | owner | Alert | true | metas | M2O | true | true | - +-------+-------+---------+---------+----------+--------+----------+ - -``` diff --git a/docs/v1.X/docs/references/decisions.md b/docs/v1.X/docs/references/decisions.md deleted file mode 100644 index 7eeea7707..000000000 --- a/docs/v1.X/docs/references/decisions.md +++ /dev/null @@ -1,10 +0,0 @@ -# Decisions - -A `Decision` is the runtime representation of a bucket overflow consequence : an action being taken against an IP, a Range, a User etc. - -The representation of the object can be found here : - -[Decision object documentation](https://pkg.go.dev/github.com/crowdsecurity/crowdsec/pkg/models#Decision) - -Those objects are not meant to be manipulated directly by parsers and such, but rather be consumed by the {{v1X.bouncers.htmlname}} via the {{v1X.lapi.htmlname}}. - diff --git a/docs/v1.X/docs/references/enrichers.md b/docs/v1.X/docs/references/enrichers.md deleted file mode 100644 index 3883542ee..000000000 --- a/docs/v1.X/docs/references/enrichers.md +++ /dev/null @@ -1,23 +0,0 @@ -# Enrichers - -Enrichers are {{v1X.parsers.htmlname}} that can rely on external methods to provide extra contextual information to the event. The enrichers are usually in the `s02-enrich` {{v1X.stage.htmlname}} (after most of the parsing happened). - -Enrichers functions should all accept a string as a parameter, and return an associative string array, that will be automatically merged into the `Enriched` map of the {{v1X.event.htmlname}}. - -!!! warning - At the time of writing, enrichers plugin mechanism implementation is still ongoing (read: the list of available enrichment methods is currently hardcoded). - - -As an example let's look into the geoip-enrich parser/enricher : - -It relies on [the geolite2 data created by maxmind](https://www.maxmind.com) and the [geoip2 golang module](https://github.com/oschwald/geoip2-golang) to provide the actual data. - - -It exposes three methods : `GeoIpCity` `GeoIpASN` and `IpToRange` that are used by the `crowdsecurity/geoip-enrich`. -Enrichers can be installed as any other parsers with the following command: - -``` -sudo {{v1X.cli.bin}} parsers install crowdsecurity/geoip-enrich -``` - -Take a tour at the {{v1X.hub.htmlname}} to find them ! diff --git a/docs/v1.X/docs/references/events.md b/docs/v1.X/docs/references/events.md deleted file mode 100644 index 5672fc9f3..000000000 --- a/docs/v1.X/docs/references/events.md +++ /dev/null @@ -1,47 +0,0 @@ -# Events - -An `Event` is the runtime representation of an item being processed by crowdsec, it can be: - - - a log line being parsed - - - an overflow being reprocessed - - -The `Event` object is modified by parsers, scenarios, and directly via user [statics expressions](/Crowdsec/v1/references/parsers/#statics) (for example). - -The representation of the object can be found here : - -[Event object documentation](https://pkg.go.dev/github.com/crowdsecurity/crowdsec/pkg/types#Event) - -## LOG relevant fields - - - `Type` is `types.LOG` - - `Whitelisted` : if `true` the LOG or OVFLW will be dropped - - `Line` : representation of the raw line - - `Raw` : the raw line representation - - `Src` : a label for the source - - `Time` : acquisition timestamp - - `Labels` : the static labels (from acquis.yaml) associated to the source - - `Process`: if set to false, processing of line will stop - - `Parsed` : a `map[string]string` that can be used during parsing and enrichment. This is where GROK patterns will output their captures by default - - `Enriched` : a `map[string]string` that can be used during parsing and enrichment. This is where enrichment functions will output their captures by default - - `Meta` : a `map[string]string` that can be used to store *important* information about a log. This map is serialized into DB when storing event. - - `Overflow` : representation of an Overflow if `Type` is set to `OVFLW` - - `Time` : processing timestamp - - `StrTime` : string representation of log timestamp. Can be set by parsers that capture timestamp in logs. Will be automatically processed by `crowdsecurity/dateparse-enrich` when processing logs in forensic mode to set `MarshaledTime` - - `MarshaledTime` : if non-empty, the event's timestamp that will be used when processing buckets (for forensic mode) - -## OVERFLOW relevant fields - - - `Type` is `types.OVFLW` - - `Whitelisted` : if `true` the LOG or OVFLW will be dropped - - `Overflow` : representation of an Overflow if `Type` is set to `OVFLW` - - `Time` : processing timestamp - - `StrTime` : string representation of log timestamp. Can be set by parsers that capture timestamp in logs. Will be automatically processed by `crowdsecurity/dateparse-enrich` when processing logs in forensic mode to set `MarshaledTime` - - `MarshaledTime` : if non-empty, the event's timestamp that will be used when processing buckets (for forensic mode) - - `Overflow` : - - `Whitelisted` : if true the OVFLW will be dropped - - `Reprocess` : if true, the OVFLOW will be reprocessed (inference) - - `Sources` : a `map[string]models.Source` representing the distinct sources that triggered the overflow, with their types and values. - - `Alert` and `APIAlerts` : representation of the signals that will be sent to LAPI. - diff --git a/docs/v1.X/docs/references/expressions.md b/docs/v1.X/docs/references/expressions.md deleted file mode 100644 index 68f925009..000000000 --- a/docs/v1.X/docs/references/expressions.md +++ /dev/null @@ -1,68 +0,0 @@ -# Expressions - -> [antonmedv/expr](https://github.com/antonmedv/expr) - Expression evaluation engine for Go: fast, non-Turing complete, dynamic typing, static typing - -Several places of {{v1X.crowdsec.name}}'s configuration use [expr](https://github.com/antonmedv/expr), notably : - - - {{v1X.filter.Htmlname}} that are used to determine events eligibility in {{v1X.parsers.htmlname}} and {{v1X.scenarios.htmlname}} or `profiles` - - {{v1X.statics.Htmlname}} use expr in the `expression` directive, to compute complex values - - {{v1X.whitelists.Htmlname}} rely on `expression` directive to allow more complex whitelists filters - -To learn more about [expr](https://github.com/antonmedv/expr), [check the github page of the project](https://github.com/antonmedv/expr/blob/master/docs/Language-Definition.md). - - -When {{v1X.crowdsec.name}} relies on `expr`, a context is provided to let the expression access relevant objects : - - - `evt.` is the representation of the current {{v1X.event.htmlname}} and is the most relevant object - - in [profiles](/Crowdsec/v1/references/profiles/), {{v1X.alert.htmlname}} is accessible via the `sig.` object - -If the `debug` is enabled (in the scenario or parser where expr is used), additional debug will be displayed regarding evaluated expressions. - - -# Helpers - -In order to makes its use in {{v1X.crowdsec.name}} more efficient, we added a few helpers that are documented bellow. - -## `Atof(string) float64` - -Parses a string representation of a float number to an actual float number (binding on `strconv.ParseFloat`) - -> Atof(evt.Parsed.tcp_port) - - -## `JsonExtract(JsonBlob, FieldName) string` - -Extract the `FieldName` from the `JsonBlob` and returns it as a string. (binding on [jsonparser](https://github.com/buger/jsonparser/)) - -> JsonExtract(evt.Parsed.some_json_blob, "foo.bar[0].one_item") - -## `File(FileName) []string` - -Returns the content of `FileName` as an array of string, while providing cache mechanism. - -> evt.Parsed.some_field in File('some_patterns.txt') -> any(File('rdns_seo_bots.txt'), { evt.Enriched.reverse_dns endsWith #}) - -## `RegexpInFile(StringToMatch, FileName) bool` - -Returns `true` if the `StringToMatch` is matched by one of the expressions contained in `FileName` (uses RE2 regexp engine). - -> RegexpInFile( evt.Enriched.reverse_dns, 'my_legit_seo_whitelists.txt') - -## `Upper(string) string` - -Returns the uppercase version of the string - -> Upper("yop") - -## `IpInRange(IPStr, RangeStr) bool` - -Returns true if the IP `IPStr` is contained in the IP range `RangeStr` (uses `net.ParseCIDR`) - -> IpInRange("1.2.3.4", "1.2.3.0/24") - -## `TimeNow() string` - -Return RFC3339 formatted time - -> TimeNow() diff --git a/docs/v1.X/docs/references/parsers.md b/docs/v1.X/docs/references/parsers.md deleted file mode 100644 index fbac6f324..000000000 --- a/docs/v1.X/docs/references/parsers.md +++ /dev/null @@ -1,360 +0,0 @@ -## Understanding parsers - - -A parser is a YAML configuration file that describes how a string is being parsed. Said string can be a log line, or a field extracted from a previous parser. While a lot of parsers rely on the **GROK** approach (a.k.a regular expression named capture groups), parsers can as well reference enrichment modules to allow specific data processing, or use specific {{v1X.expr.htmlname}} feature to perform parsing on specific data, such as JSON. - -Parsers are organized into stages to allow pipelines and branching in parsing. - -See the [{{v1X.hub.name}}]({{v1X.hub.url}}) to explore parsers, or see below some examples : - - - [apache2 access/error log parser](https://github.com/crowdsecurity/hub/blob/master/parsers/s01-parse/crowdsecurity/apache2-logs.yaml) - - [iptables logs parser](https://github.com/crowdsecurity/hub/blob/master/parsers/s01-parse/crowdsecurity/iptables-logs.yaml) - - [http logs post-processing](https://github.com/crowdsecurity/hub/blob/master/parsers/s02-enrich/crowdsecurity/http-logs.yaml) - -The parsers usually reside in `/etc/crowdsec/parsers//`. - - -## Parser configuration format - -A parser node might look like : -```yaml -onsuccess: next_stage -debug: true -filter: "evt.Parsed.program == 'kernel'" -name: crowdsecurity/demo-iptables -description: "Parse iptables drop logs" -pattern_syntax: - MYCAP: ".*" -grok: - pattern: ^xxheader %{MYCAP:extracted_value} trailing stuff$ - apply_on: evt.Parsed.some_field -statics: - - parsed: something - expression: JsonExtract(evt.Event.extracted_value, "nested.an_array[0]") - - meta: log_type - value: parsed_testlog - - meta: source_ip - expression: "evt.Parsed.src_ip" -``` - -The parser nodes are processed sequentially based on the alphabetical order of {{v1X.stage.htmlname}} and subsequent files. -If the node is considered successful (grok is present and returned data or no grok is present) and "onsuccess" equals to `next_stage`, then the {{v1X.event.name}} is moved to the next stage. - -## Parser trees - -A parser node can contain sub-nodes, to provide proper branching (on top of stages). -It can be useful when you want to apply different parsing based on different criterias, or when you have a set of candidates parsers that you want to apply to an event : - -```yaml -#This first node will capture/extract some value -filter: "evt.Line.Labels.type == 'type1'" -name: tests/base-grok-root -pattern_syntax: - MYCAP: ".*" -grok: - pattern: ^... %{MYCAP:extracted_value} ...$ - apply_on: Line.Raw -statics: - - meta: state - value: root-done - - meta: state_sub - expression: evt.Parsed.extracted_value ---- -#and this node will apply different patterns to it -filter: "evt.Line.Labels.type == 'type1' && evt.Meta.state == 'root-done'" -name: tests/base-grok-leafs -onsuccess: next_stage -#the sub-nodes will process the result of the master node -nodes: - - filter: "evt.Parsed.extracted_value == 'VALUE1'" - debug: true - statics: - - meta: final_state - value: leaf1 - - filter: "evt.Parsed.extracted_value == 'VALUE2'" - debug: true - statics: - - meta: final_state - value: leaf2 -``` - -The logic is that the `tests/base-grok-root` node will be processed first and will alter the event (here mostly by extracting some text from the `Line.Raw` field into `Parsed` thanks to the `grok` pattern and the `statics` directive). - -The event will then continue its life and be parsed by the the following `tests/base-grok-leafs` node. -This node has `onsuccess` set to `next_stage` which means that if the node is successful, the event will be moved to the next stage. - -This node consists actually of two sub-nodes that have different conditions (branching) to allow differential treatment of said event. - -A real-life example can be seen when it comes to parsing HTTP logs. -HTTP ACCESS and ERROR logs often have different formats, and thus our "nginx" parser needs to handle both formats -
- Nginx parser - -```yaml -filter: "evt.Parsed.program == 'nginx'" -onsuccess: next_stage -name: crowdsecurity/nginx-logs -nodes: - - grok: - #this is the access log - name: NGINXACCESS - apply_on: message - statics: - - meta: log_type - value: http_access-log - - target: evt.StrTime - expression: evt.Parsed.time_local - - grok: - # and this one the error log - name: NGINXERROR - apply_on: message - statics: - - meta: log_type - value: http_error-log - - target: evt.StrTime - expression: evt.Parsed.time -# these ones apply for both grok patterns -statics: - - meta: service - value: http - - meta: source_ip - expression: "evt.Parsed.remote_addr" - - meta: http_status - expression: "evt.Parsed.status" - - meta: http_path - expression: "evt.Parsed.request" -``` -
- -## Parser directives - -### `debug` - -```yaml -debug: true|false -``` -_default: false_ - -If set to to `true`, enabled node level debugging. -It is meant to help understanding parser node behavior by providing contextual logging : - -
- assignments made by statics -``` -DEBU[31-07-2020 16:36:28] + Processing 4 statics id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse -DEBU[31-07-2020 16:36:28] .Meta[service] = 'http' id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse -DEBU[31-07-2020 16:36:28] .Meta[source_ip] = '127.0.0.1' id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse -DEBU[31-07-2020 16:36:28] .Meta[http_status] = '200' id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse -DEBU[31-07-2020 16:36:28] .Meta[http_path] = '/' id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse -``` -
-
- assignments made by grok pattern -``` -DEBU[31-07-2020 16:36:28] + Grok 'NGINXACCESS' returned 10 entries to merge in Parsed id=dark-glitter name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[31-07-2020 16:36:28] .Parsed['time_local'] = '21/Jul/2020:16:13:05 +0200' id=dark-glitter name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[31-07-2020 16:36:28] .Parsed['method'] = 'GET' id=dark-glitter name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[31-07-2020 16:36:28] .Parsed['request'] = '/' id=dark-glitter name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[31-07-2020 16:36:28] .Parsed['http_user_agent'] = 'curl/7.58.0' id=dark-glitter name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[31-07-2020 16:36:28] .Parsed['remote_addr'] = '127.0.0.1' id=dark-glitter name=child-crowdsecurity/nginx-logs stage=s01-parse -``` -
-
- debug of filters and expression results -``` -DEBU[31-07-2020 16:36:28] eval(evt.Parsed.program == 'nginx') = TRUE id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse -DEBU[31-07-2020 16:36:28] eval variables: id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse -DEBU[31-07-2020 16:36:28] evt.Parsed.program = 'nginx' id=withered-rain name=crowdsecurity/nginx-logs stage=s01-parse -``` -
- - -### `filter` - -```yaml -filter: expression -``` - -`filter` must be a valid {{v1X.expr.htmlname}} expression that will be evaluated against the {{v1X.event.htmlname}}. - -If `filter` evaluation returns true or is absent, node will be processed. - -If `filter` returns `false` or a non-boolean, node won't be processed. - -Here is the [expr documentation](https://github.com/antonmedv/expr/tree/master/docs). - -Examples : - - - `filter: "evt.Meta.foo == 'test'"` - - `filter: "evt.Meta.bar == 'test' && evt.Meta.foo == 'test2'` - - -### `grok` - -```yaml -grok: - name: NAMED_EXISTING_PATTERN - apply_on: source_field -``` - -```yaml -grok: - pattern: ^a valid RE2 expression with %{CAPTURE:field}$ - apply_on: source_field -``` - -The `grok` structure in a node represent a regular expression with capture group (grok pattern) that must be applied on a field of {{v1X.event.name}}. - -The pattern can : - - - be imported by name (if present within the core of {{v1X.crowdsec.name}}) - - defined in place - -In both case, the pattern must be a valid RE2 expression. -The field(s) returned by the regular expression are going to be merged into the `Parsed` associative array of the `Event`. - - - -### `name` - -```yaml -name: explicit_string -``` - -The *mandatory* name of the node. If not present, node will be skipped at runtime. -It is used for example in debug log to help you track things. - -### `nodes` - -```yaml -nodes: - - filter: ... - grok: ... -``` - -`nodes` is a list of parser nodes, allowing you to build trees. -Each subnode must be valid, and if any of the subnodes succeed, the whole node is considered successful. - -### `onsuccess` - -``` -onsuccess: next_stage|continue -``` - -_default: continue_ - -if set to `next_stage` and the node is considered successful, the {{v1X.event.name}} will be moved directly to the next stage without processing other nodes in the current stage. _note: if it's a parser tree, and a "leaf" node succeeds, it is the parent's "onsuccess" that is evaluated._ - -### `pattern_syntax` - -```yaml -pattern_syntax: - CAPTURE_NAME: VALID_RE2_EXPRESSION -``` - -`pattern_syntax` allows user to define named capture group expressions for future use in grok patterns. -Regexp must be a valid RE2 expression. - -```yaml -pattern_syntax: - MYCAP: ".*" -grok: - pattern: ^xxheader %{MYCAP:extracted_value} trailing stuff$ - apply_on: Line.Raw -``` - - -### `statics` - -```yaml -statics: - - target: evt.Meta.target_field - value: static_value - - meta: target_field - expression: evt.Meta.target_field + ' this_is' + ' a dynamic expression' - - enriched: target_field - value: static_value -``` - -`statics` is a list of directives that will be executed when the node is considered successful. -Each entry of the list is composed of a target (where to write) and a source (what data to write). - -#### `target` - -The target aims at being any part of the {{v1X.event.htmlname}} object, and can be expressed in different ways : - -- `meta: ` -- `parsed: ` -- `enriched: ` -- a dynamic target (please note that the **current** event is accessible via the `evt.` variable) : - - `target: evt.Meta.foobar` - - `target: Meta.foobar` - - `target: evt.StrTime` - - -#### `value` - - The source itself can be either a static value, or an {{v1X.expr.htmlname}} result : - -```yaml -statics: - - meta: target_field - value: static_value - - meta: target_field - expression: evt.Meta.another_field - - meta: target_field - expression: evt.Meta.target_field + ' this_is' + ' a dynamic expression' -``` - -##### `value` -> string - -A static value - -##### `expression` -> string - -A valid [`expr`](https://github.com/antonmedv/expr) expression to eval. -The result of the evaluation will be set in the target field. - -### `data` - -```yaml -data: - - source_url: https://URL/TO/FILE - dest_file: LOCAL_FILENAME - type: (regexp|string) -``` - -`data` allows user to specify an external source of data. -This section is only relevant when `cscli` is used to install parser from hub, as it will download the `source_url` and store it to `dest_file`. When the parser is not installed from the hub, {{v1X.crowdsec.name}} won't download the URL, but the file must exist for the parser to be loaded correctly. - -The `type` is mandatory if you want to evaluate the data in the file, and should be `regex` for valid (re2) regular expression per line or `string` for string per line. -The regexps will be compiled, the strings will be loaded into a list and both will be kept in memory. -Without specifying a `type`, the file will be downloaded and stored as file and not in memory. - - -```yaml -name: crowdsecurity/cdn-whitelist -... -data: - - source_url: https://www.cloudflare.com/ips-v4 - dest_file: cloudflare_ips.txt - type: string -``` - - -## Parser concepts - -### Success and failure - -A parser is considered "successful" if : - - - A grok pattern was present and successfully matched - - No grok pattern was present - - -### Patterns documentation - - -You can find [exhaustive patterns documentation here](/Crowdsec/v1/references/patterns-documentation). diff --git a/docs/v1.X/docs/references/patterns-documentation.md b/docs/v1.X/docs/references/patterns-documentation.md deleted file mode 100644 index 2c3af424e..000000000 --- a/docs/v1.X/docs/references/patterns-documentation.md +++ /dev/null @@ -1,2795 +0,0 @@ -# Patterns documentation - -You will find here a generated documentation of all the patterns loaded by crowdsec. -They are sorted by pattern length, and are meant to be used in parsers, in the form %{PATTERN_NAME}. - - -## MONGO3_SEVERITY - -Pattern : -``` -\w -``` - -## GREEDYDATA - -Pattern : -``` -.* -``` - -## DATA - -Pattern : -``` -.*? -``` - -## NOTSPACE - -Pattern : -``` -\S+ -``` - -## SPACE - -Pattern : -``` -\s* -``` - -## RAIL_ACTION - -Pattern : -``` -\w+ -``` - -## JAVALOGMESSAGE - -Pattern : -``` -(.*) -``` - -## DAY2 - -Pattern : -``` -\d{2} -``` - -## NOTDQUOTE - -Pattern : -``` -[^"]* -``` - -## RAILS_CONSTROLLER - -Pattern : -``` -[^#]+ -``` - -## RUUID - -Pattern : -``` -\s{32} -``` - -## SYSLOG5424PRINTASCII - -Pattern : -``` -[!-~]+ -``` - -## BACULA_VERSION - -Pattern : -``` -%{USER} -``` - -## WORD - -Pattern : -``` -\b\w+\b -``` - -## BACULA_JOB - -Pattern : -``` -%{USER} -``` - -## CRON_ACTION - -Pattern : -``` -[A-Z ]+ -``` - -## BACULA_VOLUME - -Pattern : -``` -%{USER} -``` - -## BACULA_DEVICE - -Pattern : -``` -%{USER} -``` - -## TZ - -Pattern : -``` -[A-Z]{3} -``` - -## NUMTZ - -Pattern : -``` -[+-]\d{4} -``` - -## MONGO3_COMPONENT - -Pattern : -``` -%{WORD}|- -``` - -## MONGO_WORDDASH - -Pattern : -``` -\b[\w-]+\b -``` - -## NAGIOS_TYPE_HOST_ALERT - -Pattern : -``` -HOST ALERT -``` - -## NONNEGINT - -Pattern : -``` -\b[0-9]+\b -``` - -## MINUTE - -Pattern : -``` -[0-5][0-9] -``` - -## BACULA_DEVICEPATH - -Pattern : -``` -%{UNIXPATH} -``` - -## SYSLOGHOST - -Pattern : -``` -%{IPORHOST} -``` - -## REDISLOG1 - -Pattern : -``` -%{REDISLOG} -``` - -## USER - -Pattern : -``` -%{USERNAME} -``` - -## NUMBER - -Pattern : -``` -%{BASE10NUM} -``` - -## SYSLOG5424SD - -Pattern : -``` -\[%{DATA}\]+ -``` - -## ISO8601_SECOND - -Pattern : -``` -%{SECOND}|60 -``` - -## NGUSER - -Pattern : -``` -%{NGUSERNAME} -``` - -## MONTHNUM2 - -Pattern : -``` -0[1-9]|1[0-2] -``` - -## BACULA_HOST - -Pattern : -``` -[a-zA-Z0-9-]+ -``` - -## EXIM_PID - -Pattern : -``` -\[%{POSINT}\] -``` - -## NAGIOS_TYPE_SERVICE_ALERT - -Pattern : -``` -SERVICE ALERT -``` - -## YEAR - -Pattern : -``` -(?:\d\d){1,2} -``` - -## MONTHNUM - -Pattern : -``` -0?[1-9]|1[0-2] -``` - -## CISCO_XLATE_TYPE - -Pattern : -``` -static|dynamic -``` - -## RAILS_CONTEXT - -Pattern : -``` -(?:%{DATA}\n)* -``` - -## BACULA_LOG_ENDPRUNE - -Pattern : -``` -End auto prune. -``` - -## POSINT - -Pattern : -``` -\b[1-9][0-9]*\b -``` - -## INT - -Pattern : -``` -[+-]?(?:[0-9]+) -``` - -## USERNAME - -Pattern : -``` -[a-zA-Z0-9._-]+ -``` - -## IP - -Pattern : -``` -%{IPV6}|%{IPV4} -``` - -## QS - -Pattern : -``` -%{QUOTEDSTRING} -``` - -## MODSECRULEVERS - -Pattern : -``` -\[ver "[^"]+"\] -``` - -## NAGIOS_TYPE_EXTERNAL_COMMAND - -Pattern : -``` -EXTERNAL COMMAND -``` - -## NAGIOS_EC_ENABLE_SVC_CHECK - -Pattern : -``` -ENABLE_SVC_CHECK -``` - -## IPORHOST - -Pattern : -``` -%{IP}|%{HOSTNAME} -``` - -## NAGIOS_EC_ENABLE_HOST_CHECK - -Pattern : -``` -ENABLE_HOST_CHECK -``` - -## NAGIOS_TYPE_HOST_NOTIFICATION - -Pattern : -``` -HOST NOTIFICATION -``` - -## NAGIOS_EC_DISABLE_SVC_CHECK - -Pattern : -``` -DISABLE_SVC_CHECK -``` - -## NAGIOS_TYPE_PASSIVE_HOST_CHECK - -Pattern : -``` -PASSIVE HOST CHECK -``` - -## NAGIOS_TYPE_HOST_EVENT_HANDLER - -Pattern : -``` -HOST EVENT HANDLER -``` - -## HOUR - -Pattern : -``` -2[0123]|[01]?[0-9] -``` - -## DATESTAMP - -Pattern : -``` -%{DATE}[- ]%{TIME} -``` - -## NAGIOS_TYPE_CURRENT_HOST_STATE - -Pattern : -``` -CURRENT HOST STATE -``` - -## NAGIOS_EC_DISABLE_HOST_CHECK - -Pattern : -``` -DISABLE_HOST_CHECK -``` - -## NGUSERNAME - -Pattern : -``` -[a-zA-Z\.\@\-\+_%]+ -``` - -## NAGIOS_TYPE_HOST_FLAPPING_ALERT - -Pattern : -``` -HOST FLAPPING ALERT -``` - -## NAGIOS_TYPE_HOST_DOWNTIME_ALERT - -Pattern : -``` -HOST DOWNTIME ALERT -``` - -## JAVAFILE - -Pattern : -``` -(?:[A-Za-z0-9_. -]+) -``` - -## NAGIOS_TYPE_SERVICE_NOTIFICATION - -Pattern : -``` -SERVICE NOTIFICATION -``` - -## BACULA_LOG_BEGIN_PRUNE_FILES - -Pattern : -``` -Begin pruning Files. -``` - -## NAGIOS_TYPE_CURRENT_SERVICE_STATE - -Pattern : -``` -CURRENT SERVICE STATE -``` - -## NAGIOS_TYPE_PASSIVE_SERVICE_CHECK - -Pattern : -``` -PASSIVE SERVICE CHECK -``` - -## NAGIOS_TYPE_TIMEPERIOD_TRANSITION - -Pattern : -``` -TIMEPERIOD TRANSITION -``` - -## HOSTPORT - -Pattern : -``` -%{IPORHOST}:%{POSINT} -``` - -## NAGIOS_TYPE_SERVICE_EVENT_HANDLER - -Pattern : -``` -SERVICE EVENT HANDLER -``` - -## NAGIOS_EC_SCHEDULE_HOST_DOWNTIME - -Pattern : -``` -SCHEDULE_HOST_DOWNTIME -``` - -## EXIM_FLAGS - -Pattern : -``` -(<=|[-=>*]>|[*]{2}|==) -``` - -## NAGIOS_TYPE_SERVICE_DOWNTIME_ALERT - -Pattern : -``` -SERVICE DOWNTIME ALERT -``` - -## EXIM_SUBJECT - -Pattern : -``` -(T=%{QS:exim_subject}) -``` - -## PATH - -Pattern : -``` -%{UNIXPATH}|%{WINPATH} -``` - -## NAGIOS_TYPE_SERVICE_FLAPPING_ALERT - -Pattern : -``` -SERVICE FLAPPING ALERT -``` - -## SSHD_CORRUPT_MAC - -Pattern : -``` -Corrupted MAC on input -``` - -## BACULA_LOG_NOPRUNE_JOBS - -Pattern : -``` -No Jobs found to prune. -``` - -## HTTPDUSER - -Pattern : -``` -%{EMAILADDRESS}|%{USER} -``` - -## BACULA_LOG_NOPRUNE_FILES - -Pattern : -``` -No Files found to prune. -``` - -## NAGIOS_EC_ENABLE_SVC_NOTIFICATIONS - -Pattern : -``` -ENABLE_SVC_NOTIFICATIONS -``` - -## BACULA_CAPACITY - -Pattern : -``` -%{INT}{1,3}(,%{INT}{3})* -``` - -## EXIM_PROTOCOL - -Pattern : -``` -(P=%{NOTSPACE:protocol}) -``` - -## URIPROTO - -Pattern : -``` -[A-Za-z]+(\+[A-Za-z+]+)? -``` - -## PROG - -Pattern : -``` -[\x21-\x5a\x5c\x5e-\x7e]+ -``` - -## NAGIOS_EC_ENABLE_HOST_NOTIFICATIONS - -Pattern : -``` -ENABLE_HOST_NOTIFICATIONS -``` - -## NAGIOS_EC_PROCESS_HOST_CHECK_RESULT - -Pattern : -``` -PROCESS_HOST_CHECK_RESULT -``` - -## BACULA_LOG_VSS - -Pattern : -``` -(Generate )?VSS (Writer)? -``` - -## NAGIOS_EC_DISABLE_SVC_NOTIFICATIONS - -Pattern : -``` -DISABLE_SVC_NOTIFICATIONS -``` - -## NAGIOS_EC_SCHEDULE_SERVICE_DOWNTIME - -Pattern : -``` -SCHEDULE_SERVICE_DOWNTIME -``` - -## MONGO_QUERY - -Pattern : -``` -\{ \{ .* \} ntoreturn: \} -``` - -## URIPATHPARAM - -Pattern : -``` -%{URIPATH}(?:%{URIPARAM})? -``` - -## NAGIOS_EC_DISABLE_HOST_NOTIFICATIONS - -Pattern : -``` -DISABLE_HOST_NOTIFICATIONS -``` - -## UNIXPATH - -Pattern : -``` -(/([\w_%!$@:.,~-]+|\\.)*)+ -``` - -## KITCHEN - -Pattern : -``` -\d{1,2}:\d{2}(AM|PM|am|pm) -``` - -## NAGIOSTIME - -Pattern : -``` -\[%{NUMBER:nagios_epoch}\] -``` - -## EMAILLOCALPART - -Pattern : -``` -[a-zA-Z][a-zA-Z0-9_.+-=:]+ -``` - -## JAVATHREAD - -Pattern : -``` -(?:[A-Z]{2}-Processor[\d]+) -``` - -## TIME - -Pattern : -``` -%{HOUR}:%{MINUTE}:%{SECOND} -``` - -## EXIM_MSG_SIZE - -Pattern : -``` -(S=%{NUMBER:exim_msg_size}) -``` - -## RUBY_LOGLEVEL - -Pattern : -``` -DEBUG|FATAL|ERROR|WARN|INFO -``` - -## BASE16NUM - -Pattern : -``` -[+-]?(?:0x)?(?:[0-9A-Fa-f]+) -``` - -## ISO8601_TIMEZONE - -Pattern : -``` -Z|[+-]%{HOUR}(?::?%{MINUTE}) -``` - -## REDISTIMESTAMP - -Pattern : -``` -%{MONTHDAY} %{MONTH} %{TIME} -``` - -## NAGIOS_EC_PROCESS_SERVICE_CHECK_RESULT - -Pattern : -``` -PROCESS_SERVICE_CHECK_RESULT -``` - -## SSHD_PACKET_CORRUPT - -Pattern : -``` -Disconnecting: Packet corrupt -``` - -## SYSLOG5424PRI - -Pattern : -``` -<%{NONNEGINT:syslog5424_pri}> -``` - -## EMAILADDRESS - -Pattern : -``` -%{EMAILLOCALPART}@%{HOSTNAME} -``` - -## MODSECRULEID - -Pattern : -``` -\[id %{QUOTEDSTRING:ruleid}\] -``` - -## SYSLOGTIMESTAMP - -Pattern : -``` -%{MONTH} +%{MONTHDAY} %{TIME} -``` - -## NAGIOS_EC_ENABLE_HOST_SVC_NOTIFICATIONS - -Pattern : -``` -ENABLE_HOST_SVC_NOTIFICATIONS -``` - -## NAGIOS_EC_DISABLE_HOST_SVC_NOTIFICATIONS - -Pattern : -``` -DISABLE_HOST_SVC_NOTIFICATIONS -``` - -## EXIM_HEADER_ID - -Pattern : -``` -(id=%{NOTSPACE:exim_header_id}) -``` - -## URIHOST - -Pattern : -``` -%{IPORHOST}(?::%{POSINT:port})? -``` - -## DATE - -Pattern : -``` -%{DATE_US}|%{DATE_EU}|%{DATE_X} -``` - -## SSHD_TUNN_TIMEOUT - -Pattern : -``` -Timeout, client not responding. -``` - -## MCOLLECTIVEAUDIT - -Pattern : -``` -%{TIMESTAMP_ISO8601:timestamp}: -``` - -## CISCOTAG - -Pattern : -``` -[A-Z0-9]+-%{INT}-(?:[A-Z0-9_]+) -``` - -## MODSECRULEREV - -Pattern : -``` -\[rev %{QUOTEDSTRING:rulerev}\] -``` - -## HAPROXYCAPTUREDREQUESTHEADERS - -Pattern : -``` -%{DATA:captured_request_headers} -``` - -## CISCO_INTERVAL - -Pattern : -``` -first hit|%{INT}-second interval -``` - -## DATE_X - -Pattern : -``` -%{YEAR}/%{MONTHNUM2}/%{MONTHDAY} -``` - -## SSHD_INIT - -Pattern : -``` -%{SSHD_LISTEN}|%{SSHD_TERMINATE} -``` - -## WINPATH - -Pattern : -``` -(?:[A-Za-z]+:|\\)(?:\\[^\\?*]*)+ -``` - -## HAPROXYCAPTUREDRESPONSEHEADERS - -Pattern : -``` -%{DATA:captured_response_headers} -``` - -## MODSECURI - -Pattern : -``` -\[uri ["']%{DATA:targeturi}["']\] -``` - -## CISCO_DIRECTION - -Pattern : -``` -Inbound|inbound|Outbound|outbound -``` - -## MODSECRULEDATA - -Pattern : -``` -\[data %{QUOTEDSTRING:ruledata}\] -``` - -## MODSECRULELINE - -Pattern : -``` -\[line %{QUOTEDSTRING:ruleline}\] -``` - -## MODSECRULEFILE - -Pattern : -``` -\[file %{QUOTEDSTRING:rulefile}\] -``` - -## SECOND - -Pattern : -``` -(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)? -``` - -## BACULA_LOG_CANCELLING - -Pattern : -``` -Cancelling duplicate JobId=%{INT}. -``` - -## MODSECRULEMSG - -Pattern : -``` -\[msg %{QUOTEDSTRING:rulemessage}\] -``` - -## SSHD_TUNN_ERR3 - -Pattern : -``` -error: bind: Address already in use -``` - -## BACULA_LOG_STARTRESTORE - -Pattern : -``` -Start Restore Job %{BACULA_JOB:job} -``` - -## SYSLOGLINE - -Pattern : -``` -%{SYSLOGBASE2} %{GREEDYDATA:message} -``` - -## COMMONMAC - -Pattern : -``` -(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2} -``` - -## WINDOWSMAC - -Pattern : -``` -(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2} -``` - -## SYSLOGPROG - -Pattern : -``` -%{PROG:program}(?:\[%{POSINT:pid}\])? -``` - -## JAVAMETHOD - -Pattern : -``` -(?:()|[a-zA-Z$_][a-zA-Z$_0-9]*) -``` - -## DATE_US - -Pattern : -``` -%{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR} -``` - -## CISCOMAC - -Pattern : -``` -(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4} -``` - -## MODSECUID - -Pattern : -``` -\[unique_id %{QUOTEDSTRING:uniqueid}\] -``` - -## MAC - -Pattern : -``` -%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC} -``` - -## ELB_URIPATHPARAM - -Pattern : -``` -%{URIPATH:path}(?:%{URIPARAM:params})? -``` - -## BACULA_LOG_NOPRIOR - -Pattern : -``` -No prior Full backup Job record found. -``` - -## MODSECMATCHOFFSET - -Pattern : -``` -\[offset %{QUOTEDSTRING:matchoffset}\] -``` - -## BACULA_TIMESTAMP - -Pattern : -``` -%{MONTHDAY}-%{MONTH} %{HOUR}:%{MINUTE} -``` - -## MODSECHOSTNAME - -Pattern : -``` -\[hostname ['"]%{DATA:targethost}["']\] -``` - -## TTY - -Pattern : -``` -/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+) -``` - -## DATE_EU - -Pattern : -``` -%{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR} -``` - -## URIPATH - -Pattern : -``` -(?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+ -``` - -## HTTPD_ERRORLOG - -Pattern : -``` -%{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG} -``` - -## MONTHDAY - -Pattern : -``` -(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9] -``` - -## BACULA_LOG_USEDEVICE - -Pattern : -``` -Using Device \"%{BACULA_DEVICE:device}\" -``` - -## RFC822Z - -Pattern : -``` -[0-3]\d %{MONTH} %{YEAR} %{TIME} %{NUMTZ} -``` - -## MODSECRULESEVERITY - -Pattern : -``` -\[severity ["']%{WORD:ruleseverity}["']\] -``` - -## ANSIC - -Pattern : -``` -%{DAY} %{MONTH} [_123]\d %{TIME} %{YEAR}" -``` - -## GENERICAPACHEERROR - -Pattern : -``` -%{APACHEERRORPREFIX} %{GREEDYDATA:message} -``` - -## SSHD_CONN_CLOSE - -Pattern : -``` -Connection closed by %{IP:sshd_client_ip}$ -``` - -## CISCOTIMESTAMP - -Pattern : -``` -%{MONTH} +%{MONTHDAY}(?: %{YEAR})? %{TIME} -``` - -## APACHEERRORTIME - -Pattern : -``` -%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR} -``` - -## CISCOFW104004 - -Pattern : -``` -\((?:Primary|Secondary)\) Switching to OK\. -``` - -## HTTPDATE - -Pattern : -``` -%{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT} -``` - -## HTTPDERROR_DATE - -Pattern : -``` -%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR} -``` - -## EXIM_QT - -Pattern : -``` -((\d+y)?(\d+w)?(\d+d)?(\d+h)?(\d+m)?(\d+s)?) -``` - -## BACULA_LOG_NOJOBSTAT - -Pattern : -``` -Fatal error: No Job status returned from FD. -``` - -## NAGIOS_WARNING - -Pattern : -``` -Warning:%{SPACE}%{GREEDYDATA:nagios_message} -``` - -## EXIM_MSGID - -Pattern : -``` -[0-9A-Za-z]{6}-[0-9A-Za-z]{6}-[0-9A-Za-z]{2} -``` - -## BASE10NUM - -Pattern : -``` -[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)) -``` - -## REDISLOG - -Pattern : -``` -\[%{POSINT:pid}\] %{REDISTIMESTAMP:time} \*\s -``` - -## URIPARAM - -Pattern : -``` -\?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]<>]* -``` - -## COMBINEDAPACHELOG - -Pattern : -``` -%{COMMONAPACHELOG} %{QS:referrer} %{QS:agent} -``` - -## SYSLOGFACILITY - -Pattern : -``` -<%{NONNEGINT:facility}.%{NONNEGINT:priority}> -``` - -## RFC1123 - -Pattern : -``` -%{DAY}, [0-3]\d %{MONTH} %{YEAR} %{TIME} %{TZ} -``` - -## UNIXDATE - -Pattern : -``` -%{DAY} %{MONTH} [_123]\d %{TIME} %{TZ} %{YEAR} -``` - -## RFC850 - -Pattern : -``` -%{DAY}, [0-3]\d-%{MONTH}-%{YEAR} %{TIME} %{TZ} -``` - -## SYSLOG5424LINE - -Pattern : -``` -%{SYSLOG5424BASE} +%{GREEDYDATA:syslog5424_msg} -``` - -## CISCOFW104003 - -Pattern : -``` -\((?:Primary|Secondary)\) Switching to FAILED\. -``` - -## RUBYDATE - -Pattern : -``` -%{DAY} %{MONTH} [0-3]\d %{TIME} %{NUMTZ} %{YEAR} -``` - -## BACULA_LOG_NOOPEN - -Pattern : -``` -\s+Cannot open %{DATA}: ERR=%{GREEDYDATA:berror} -``` - -## BACULA_LOG_STARTJOB - -Pattern : -``` -Start Backup JobId %{INT}, Job=%{BACULA_JOB:job} -``` - -## DATESTAMP_RFC822 - -Pattern : -``` -%{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ} -``` - -## DATESTAMP_OTHER - -Pattern : -``` -%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR} -``` - -## RFC3339 - -Pattern : -``` -%{YEAR}-[01]\d-[0-3]\dT%{TIME}%{ISO8601_TIMEZONE} -``` - -## RFC1123Z - -Pattern : -``` -%{DAY}, [0-3]\d %{MONTH} %{YEAR} %{TIME} %{NUMTZ} -``` - -## BACULA_LOG_NOSTAT - -Pattern : -``` -\s+Could not stat %{DATA}: ERR=%{GREEDYDATA:berror} -``` - -## SSHD_TERMINATE - -Pattern : -``` -Received signal %{NUMBER:sshd_signal}; terminating. -``` - -## UUID - -Pattern : -``` -[A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12} -``` - -## SSHD_LOGOUT_ERR - -Pattern : -``` -syslogin_perform_logout: logout\(\) returned an error -``` - -## RCONTROLLER - -Pattern : -``` -%{RAILS_CONSTROLLER:controller}#%{RAIL_ACTION:action} -``` - -## JAVACLASS - -Pattern : -``` -(?:[a-zA-Z$_][a-zA-Z$_0-9]*\.)*[a-zA-Z$_][a-zA-Z$_0-9]* -``` - -## DATESTAMP_EVENTLOG - -Pattern : -``` -%{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND} -``` - -## NGINXERRTIME - -Pattern : -``` -%{YEAR}/%{MONTHNUM2}/%{DAY2} %{HOUR}:%{MINUTE}:%{SECOND} -``` - -## BACULA_LOG_BEGIN_PRUNE_JOBS - -Pattern : -``` -Begin pruning Jobs older than %{INT} month %{INT} days . -``` - -## RFC3339NANO - -Pattern : -``` -%{YEAR}-[01]\d-[0-3]\dT%{TIME}\.\d{9}%{ISO8601_TIMEZONE} -``` - -## BACULA_LOG_MARKCANCEL - -Pattern : -``` -JobId %{INT}, Job %{BACULA_JOB:job} marked to be canceled. -``` - -## BACULA_LOG_NEW_VOLUME - -Pattern : -``` -Created new Volume \"%{BACULA_VOLUME:volume}\" in catalog. -``` - -## SSHD_TCPWRAP_FAIL5 - -Pattern : -``` -warning: can't get client address: Connection reset by peer -``` - -## EXIM_INTERFACE - -Pattern : -``` -(I=\[%{IP:exim_interface}\](:%{NUMBER:exim_interface_port})) -``` - -## BACULA_LOG_NOOPENDIR - -Pattern : -``` -\s+Could not open directory %{DATA}: ERR=%{GREEDYDATA:berror} -``` - -## BACULA_LOG_CLIENT_RBJ - -Pattern : -``` -shell command: run ClientRunBeforeJob \"%{GREEDYDATA:runjob}\" -``` - -## SSHD_IDENT_FAIL - -Pattern : -``` -Did not receive identification string from %{IP:sshd_client_ip} -``` - -## DATESTAMP_RFC2822 - -Pattern : -``` -%{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE} -``` - -## BACULA_LOG_MAXSTART - -Pattern : -``` -Fatal error: Job canceled because max start delay time exceeded. -``` - -## QUOTEDSTRING - -Pattern : -``` -("(\\.|[^\\"]+)+")|""|('(\\.|[^\\']+)+')|''|(`(\\.|[^\\`]+)+`)|`` -``` - -## REDISLOG2 - -Pattern : -``` -%{POSINT:pid}:M %{REDISTIMESTAMP:time} [*#] %{GREEDYDATA:message} -``` - -## BACULA_LOG_PRUNED_JOBS - -Pattern : -``` -Pruned %{INT} Jobs* for client %{BACULA_HOST:client} from catalog. -``` - -## RT_FLOW_EVENT - -Pattern : -``` -(RT_FLOW_SESSION_CREATE|RT_FLOW_SESSION_CLOSE|RT_FLOW_SESSION_DENY) -``` - -## BACULA_LOG_NOSUIT - -Pattern : -``` -No prior or suitable Full backup found in catalog. Doing FULL backup. -``` - -## CISCOFW302010 - -Pattern : -``` -%{INT:connection_count} in use, %{INT:connection_count_max} most used -``` - -## SSHD_INVAL_USER - -Pattern : -``` -Invalid user\s*%{USERNAME:sshd_invalid_user}? from %{IP:sshd_client_ip} -``` - -## SSHD_SESSION_CLOSE - -Pattern : -``` -pam_unix\(sshd:session\): session closed for user %{USERNAME:sshd_user} -``` - -## MONGO_LOG - -Pattern : -``` -%{SYSLOGTIMESTAMP:timestamp} \[%{WORD:component}\] %{GREEDYDATA:message} -``` - -## BACULA_LOG_READYAPPEND - -Pattern : -``` -Ready to append to end of Volume \"%{BACULA_VOLUME:volume}\" size=%{INT} -``` - -## CRONLOG - -Pattern : -``` -%{SYSLOGBASE} \(%{USER:user}\) %{CRON_ACTION:action} \(%{DATA:message}\) -``` - -## BACULA_LOG_JOB - -Pattern : -``` -(Error: )?Bacula %{BACULA_HOST} %{BACULA_VERSION} \(%{BACULA_VERSION}\): -``` - -## SSHD_LISTEN - -Pattern : -``` -Server listening on %{IP:sshd_listen_ip} port %{NUMBER:sshd_listen_port}. -``` - -## URI - -Pattern : -``` -%{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})? -``` - -## RAILS3 - -Pattern : -``` -%{RAILS3HEAD}(?:%{RPROCESSING})?%{RAILS_CONTEXT:context}(?:%{RAILS3FOOT})? -``` - -## BASE16FLOAT - -Pattern : -``` -\b[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+))\b -``` - -## HAPROXYTIME - -Pattern : -``` -%{HOUR:haproxy_hour}:%{MINUTE:haproxy_minute}(?::%{SECOND:haproxy_second}) -``` - -## CISCOFW104001 - -Pattern : -``` -\((?:Primary|Secondary)\) Switching to ACTIVE - %{GREEDYDATA:switch_reason} -``` - -## CATALINA_DATESTAMP - -Pattern : -``` -%{MONTH} %{MONTHDAY}, 20%{YEAR} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) (?:AM|PM) -``` - -## CISCOFW105008 - -Pattern : -``` -\((?:Primary|Secondary)\) Testing [Ii]nterface %{GREEDYDATA:interface_name} -``` - -## HOSTNAME - -Pattern : -``` -\b[0-9A-Za-z][0-9A-Za-z-]{0,62}(?:\.[0-9A-Za-z][0-9A-Za-z-]{0,62})*(\.?|\b) -``` - -## CISCOFW104002 - -Pattern : -``` -\((?:Primary|Secondary)\) Switching to STANDBY - %{GREEDYDATA:switch_reason} -``` - -## BACULA_LOG_VOLUME_PREVWRITTEN - -Pattern : -``` -Volume \"%{BACULA_VOLUME:volume}\" previously written, moving to end of data. -``` - -## SSHD_BAD_VERSION - -Pattern : -``` -Bad protocol version identification '%{GREEDYDATA}' from %{IP:sshd_client_ip} -``` - -## BACULA_LOG_PRUNED_FILES - -Pattern : -``` -Pruned Files from %{INT} Jobs* for client %{BACULA_HOST:client} from catalog. -``` - -## SSHD_BADL_PREAUTH - -Pattern : -``` -Bad packet length %{NUMBER:sshd_packet_length}. \[%{GREEDYDATA:sshd_privsep}\] -``` - -## CATALINALOG - -Pattern : -``` -%{CATALINA_DATESTAMP:timestamp} %{JAVACLASS:class} %{JAVALOGMESSAGE:logmessage} -``` - -## RAILS_TIMESTAMP - -Pattern : -``` -%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} %{ISO8601_TIMEZONE} -``` - -## SSHD_TUNN_ERR1 - -Pattern : -``` -error: connect_to %{IP:sshd_listen_ip} port %{NUMBER:sshd_listen_port}: failed. -``` - -## EXIM_DATE - -Pattern : -``` -%{YEAR:exim_year}-%{MONTHNUM:exim_month}-%{MONTHDAY:exim_day} %{TIME:exim_time} -``` - -## BACULA_LOG_DUPLICATE - -Pattern : -``` -Fatal error: JobId %{INT:duplicate} already running. Duplicate job not allowed. -``` - -## SSHD_REFUSE_CONN - -Pattern : -``` -refused connect from %{DATA:sshd_client_hostname} \(%{IPORHOST:sshd_client_ip}\) -``` - -## SSHD_TOOMANY_AUTH - -Pattern : -``` -Disconnecting: Too many authentication failures for %{USERNAME:sshd_invalid_user} -``` - -## BACULA_LOG_ALL_RECORDS_PRUNED - -Pattern : -``` -All records pruned from Volume \"%{BACULA_VOLUME:volume}\"; marking it \"Purged\" -``` - -## SSHD_DISR_PREAUTH - -Pattern : -``` -Disconnecting: %{GREEDYDATA:sshd_disconnect_status} \[%{GREEDYDATA:sshd_privsep}\] -``` - -## MCOLLECTIVE - -Pattern : -``` -., \[%{TIMESTAMP_ISO8601:timestamp} #%{POSINT:pid}\]%{SPACE}%{LOGLEVEL:event_level} -``` - -## BACULA_LOG_DIFF_FS - -Pattern : -``` -\s+%{UNIXPATH} is a different filesystem. Will not descend from %{UNIXPATH} into it. -``` - -## SSHD_TUNN_ERR2 - -Pattern : -``` -error: channel_setup_fwd_listener: cannot listen to port: %{NUMBER:sshd_listen_port} -``` - -## CISCOFW321001 - -Pattern : -``` -Resource '%{WORD:resource_name}' limit of %{POSINT:resource_limit} reached for system -``` - -## BACULA_LOG_NO_AUTH - -Pattern : -``` -Fatal error: Unable to authenticate with File daemon at %{HOSTNAME}. Possible causes: -``` - -## POSTGRESQL - -Pattern : -``` -%{DATESTAMP:timestamp} %{TZ} %{DATA:user_id} %{GREEDYDATA:connection_id} %{POSINT:pid} -``` - -## ELB_REQUEST_LINE - -Pattern : -``` -(?:%{WORD:verb} %{ELB_URI:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest}) -``` - -## SSHD_SESSION_OPEN - -Pattern : -``` -pam_unix\(sshd:session\): session opened for user %{USERNAME:sshd_user} by \(uid=\d+\) -``` - -## TOMCAT_DATESTAMP - -Pattern : -``` -20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) %{ISO8601_TIMEZONE} -``` - -## S3_REQUEST_LINE - -Pattern : -``` -(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest}) -``` - -## RAILS3FOOT - -Pattern : -``` -Completed %{NUMBER:response}%{DATA} in %{NUMBER:totalms}ms %{RAILS3PROFILE}%{GREEDYDATA} -``` - -## CISCOFW105004 - -Pattern : -``` -\((?:Primary|Secondary)\) Monitoring on [Ii]nterface %{GREEDYDATA:interface_name} normal -``` - -## CISCOFW105003 - -Pattern : -``` -\((?:Primary|Secondary)\) Monitoring on [Ii]nterface %{GREEDYDATA:interface_name} waiting -``` - -## BACULA_LOG_JOBEND - -Pattern : -``` -Job write elapsed time = %{DATA:elapsed}, Transfer rate = %{NUMBER} (K|M|G)? Bytes/second -``` - -## TIMESTAMP_ISO8601 - -Pattern : -``` -%{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? -``` - -## SYSLOGBASE - -Pattern : -``` -%{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: -``` - -## SSHD_TUNN_ERR4 - -Pattern : -``` -error: channel_setup_fwd_listener_tcpip: cannot listen to port: %{NUMBER:sshd_listen_port} -``` - -## MODSECPREFIX - -Pattern : -``` -%{APACHEERRORPREFIX} ModSecurity: %{NOTSPACE:modsecseverity}\. %{GREEDYDATA:modsecmessage} -``` - -## JAVASTACKTRACEPART - -Pattern : -``` -%{SPACE}at %{JAVACLASS:class}\.%{JAVAMETHOD:method}\(%{JAVAFILE:file}(?::%{NUMBER:line})?\) -``` - -## EXIM_REMOTE_HOST - -Pattern : -``` -(H=(%{NOTSPACE:remote_hostname} )?(\(%{NOTSPACE:remote_heloname}\) )?\[%{IP:remote_host}\]) -``` - -## ELB_URI - -Pattern : -``` -%{URIPROTO:proto}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST:urihost})?(?:%{ELB_URIPATHPARAM})? -``` - -## DAY - -Pattern : -``` -Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)? -``` - -## SSHD_TUNN - -Pattern : -``` -%{SSHD_TUNN_ERR1}|%{SSHD_TUNN_ERR2}|%{SSHD_TUNN_ERR3}|%{SSHD_TUNN_ERR4}|%{SSHD_TUNN_TIMEOUT} -``` - -## SSHD_SESSION_FAIL - -Pattern : -``` -pam_systemd\(sshd:session\): Failed to release session: %{GREEDYDATA:sshd_disconnect_status} -``` - -## BACULA_LOG_NOJOBS - -Pattern : -``` -There are no more Jobs associated with Volume \"%{BACULA_VOLUME:volume}\". Marking it purged. -``` - -## RPROCESSING - -Pattern : -``` -\W*Processing by %{RCONTROLLER} as %{NOTSPACE:format}(?:\W*Parameters: \{\%\{DATA:params}}\W*)? -``` - -## CISCOFW105009 - -Pattern : -``` -\((?:Primary|Secondary)\) Testing on [Ii]nterface %{GREEDYDATA:interface_name} (?:Passed|Failed) -``` - -## SSHD_LOG - -Pattern : -``` -%{SSHD_INIT}|%{SSHD_NORMAL_LOG}|%{SSHD_PROBE_LOG}|%{SSHD_CORRUPTED}|%{SSHD_TUNN}|%{SSHD_PREAUTH} -``` - -## SSHD_DISC_PREAUTH - -Pattern : -``` -Disconnected from %{IP:sshd_client_ip} port %{NUMBER:sshd_port}\s*(?:\[%{GREEDYDATA:sshd_privsep}\]|) -``` - -## SSHD_REST_PREAUTH - -Pattern : -``` -Connection reset by %{IP:sshd_client_ip} port %{NUMBER:sshd_port}\s*(?:\[%{GREEDYDATA:sshd_privsep}\]|) -``` - -## TOMCATLOG - -Pattern : -``` -%{TOMCAT_DATESTAMP:timestamp} \| %{LOGLEVEL:level} \| %{JAVACLASS:class} - %{JAVALOGMESSAGE:logmessage} -``` - -## SSHD_CLOS_PREAUTH - -Pattern : -``` -Connection closed by %{IP:sshd_client_ip} port %{NUMBER:sshd_port}\s*(?:\[%{GREEDYDATA:sshd_privsep}\]|) -``` - -## CISCO_TAGGED_SYSLOG - -Pattern : -``` -^<%{POSINT:syslog_pri}>%{CISCOTIMESTAMP:timestamp}( %{SYSLOGHOST:sysloghost})? ?: %%{CISCOTAG:ciscotag}: -``` - -## SSHD_INVA_PREAUTH - -Pattern : -``` -input_userauth_request: invalid user %{USERNAME:sshd_invalid_user}?\s*(?:\[%{GREEDYDATA:sshd_privsep}\]|) -``` - -## RAILS3HEAD - -Pattern : -``` -(?m)Started %{WORD:verb} "%{URIPATHPARAM:request}" for %{IPORHOST:clientip} at %{RAILS_TIMESTAMP:timestamp} -``` - -## CISCOFW105005 - -Pattern : -``` -\((?:Primary|Secondary)\) Lost Failover communications with mate on [Ii]nterface %{GREEDYDATA:interface_name} -``` - -## BACULA_LOG_NEW_LABEL - -Pattern : -``` -Labeled new Volume \"%{BACULA_VOLUME:volume}\" on device \"%{BACULA_DEVICE:device}\" \(%{BACULA_DEVICEPATH}\). -``` - -## NAGIOS_EC_LINE_ENABLE_HOST_CHECK - -Pattern : -``` -%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_HOST_CHECK:nagios_command};%{DATA:nagios_hostname} -``` - -## COWRIE_NEW_CO - -Pattern : -``` -New connection: %{IPV4:source_ip}:[0-9]+ \(%{IPV4:dest_ip}:%{INT:dest_port}\) \[session: %{DATA:telnet_session}\]$ -``` - -## CISCO_ACTION - -Pattern : -``` -Built|Teardown|Deny|Denied|denied|requested|permitted|denied by ACL|discarded|est-allowed|Dropping|created|deleted -``` - -## NAGIOS_EC_LINE_DISABLE_HOST_CHECK - -Pattern : -``` -%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_HOST_CHECK:nagios_command};%{DATA:nagios_hostname} -``` - -## CISCOFW402117 - -Pattern : -``` -%{WORD:protocol}: Received a non-IPSec packet \(protocol= %{WORD:orig_protocol}\) from %{IP:src_ip} to %{IP:dst_ip} -``` - -## BACULA_LOG_WROTE_LABEL - -Pattern : -``` -Wrote label to prelabeled Volume \"%{BACULA_VOLUME:volume}\" on device \"%{BACULA_DEVICE}\" \(%{BACULA_DEVICEPATH}\) -``` - -## RAILS3PROFILE - -Pattern : -``` -(?:\(Views: %{NUMBER:viewms}ms \| ActiveRecord: %{NUMBER:activerecordms}ms|\(ActiveRecord: %{NUMBER:activerecordms}ms)? -``` - -## CISCOFW500004 - -Pattern : -``` -%{CISCO_REASON:reason} for protocol=%{WORD:protocol}, from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} -``` - -## NAGIOS_TIMEPERIOD_TRANSITION - -Pattern : -``` -%{NAGIOS_TYPE_TIMEPERIOD_TRANSITION:nagios_type}: %{DATA:nagios_service};%{DATA:nagios_unknown1};%{DATA:nagios_unknown2} -``` - -## NAGIOS_PASSIVE_HOST_CHECK - -Pattern : -``` -%{NAGIOS_TYPE_PASSIVE_HOST_CHECK:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment} -``` - -## NAGIOS_HOST_DOWNTIME_ALERT - -Pattern : -``` -%{NAGIOS_TYPE_HOST_DOWNTIME_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment} -``` - -## NAGIOS_HOST_FLAPPING_ALERT - -Pattern : -``` -%{NAGIOS_TYPE_HOST_FLAPPING_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_message} -``` - -## HTTPD20_ERRORLOG - -Pattern : -``` -\[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg} -``` - -## NGINXERROR - -Pattern : -``` -%{NGINXERRTIME:time} \[%{LOGLEVEL:loglevel}\] %{NONNEGINT:pid}#%{NONNEGINT:tid}: (\*%{NONNEGINT:cid} )?%{GREEDYDATA:message} -``` - -## MYSQL_AUTH_FAIL - -Pattern : -``` -%{TIMESTAMP_ISO8601:time} %{NUMBER} \[Note\] Access denied for user '%{DATA:user}'@'%{IP:source_ip}' \(using password: YES\) -``` - -## BACULA_LOG_MAX_CAPACITY - -Pattern : -``` -User defined maximum volume capacity %{BACULA_CAPACITY} exceeded on device \"%{BACULA_DEVICE:device}\" \(%{BACULA_DEVICEPATH}\) -``` - -## NAGIOS_EC_LINE_ENABLE_HOST_NOTIFICATIONS - -Pattern : -``` -%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_HOST_NOTIFICATIONS:nagios_command};%{GREEDYDATA:nagios_hostname} -``` - -## HAPROXYDATE - -Pattern : -``` -%{MONTHDAY:haproxy_monthday}/%{MONTH:haproxy_month}/%{YEAR:haproxy_year}:%{HAPROXYTIME:haproxy_time}.%{INT:haproxy_milliseconds} -``` - -## CISCOFW106021 - -Pattern : -``` -%{CISCO_ACTION:action} %{WORD:protocol} reverse path check from %{IP:src_ip} to %{IP:dst_ip} on interface %{GREEDYDATA:interface} -``` - -## RUBY_LOGGER - -Pattern : -``` -[DFEWI], \[%{TIMESTAMP_ISO8601:timestamp} #%{POSINT:pid}\] *%{RUBY_LOGLEVEL:loglevel} -- +%{DATA:progname}: %{GREEDYDATA:message} -``` - -## NAGIOS_EC_LINE_DISABLE_HOST_NOTIFICATIONS - -Pattern : -``` -%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_HOST_NOTIFICATIONS:nagios_command};%{GREEDYDATA:nagios_hostname} -``` - -## CISCOFW110002 - -Pattern : -``` -%{CISCO_REASON:reason} for %{WORD:protocol} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} -``` - -## NAGIOS_EC_LINE_ENABLE_HOST_SVC_NOTIFICATIONS - -Pattern : -``` -%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_HOST_SVC_NOTIFICATIONS:nagios_command};%{GREEDYDATA:nagios_hostname} -``` - -## NAGIOS_EC_LINE_DISABLE_HOST_SVC_NOTIFICATIONS - -Pattern : -``` -%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_HOST_SVC_NOTIFICATIONS:nagios_command};%{GREEDYDATA:nagios_hostname} -``` - -## SSHD_RMAP_FAIL - -Pattern : -``` -reverse mapping checking getaddrinfo for %{HOSTNAME:sshd_client_hostname} \[%{IP:sshd_client_ip}\] failed - POSSIBLE BREAK-IN ATTEMPT! -``` - -## HAPROXYHTTP - -Pattern : -``` -(?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{HAPROXYHTTPBASE} -``` - -## SSHD_USER_FAIL - -Pattern : -``` -Failed password for invalid user %{USERNAME:sshd_invalid_user} from %{IP:sshd_client_ip} port %{NUMBER:sshd_port} %{WORD:sshd_protocol} -``` - -## SYSLOGBASE2 - -Pattern : -``` -(?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource}+(?: %{SYSLOGPROG}:|) -``` - -## SSHD_NORMAL_LOG - -Pattern : -``` -%{SSHD_SUCCESS}|%{SSHD_DISCONNECT}|%{SSHD_CONN_CLOSE}|%{SSHD_SESSION_OPEN}|%{SSHD_SESSION_CLOSE}|%{SSHD_SESSION_FAIL}|%{SSHD_LOGOUT_ERR} -``` - -## SSHD_FAIL - -Pattern : -``` -Failed %{WORD:sshd_auth_type} for %{USERNAME:sshd_invalid_user} from %{IP:sshd_client_ip} port %{NUMBER:sshd_port} %{WORD:sshd_protocol} -``` - -## NAGIOS_EC_LINE_ENABLE_SVC_CHECK - -Pattern : -``` -%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_SVC_CHECK:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_service} -``` - -## NAGIOS_EC_LINE_DISABLE_SVC_CHECK - -Pattern : -``` -%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_SVC_CHECK:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_service} -``` - -## CISCO_REASON - -Pattern : -``` -Duplicate TCP SYN|Failed to locate egress interface|Invalid transport field|No matching connection|DNS Response|DNS Query|(?:%{WORD}\s*)* -``` - -## SSHD_CORRUPTED - -Pattern : -``` -%{SSHD_IDENT_FAIL}|%{SSHD_MAPB_FAIL}|%{SSHD_RMAP_FAIL}|%{SSHD_TOOMANY_AUTH}|%{SSHD_CORRUPT_MAC}|%{SSHD_PACKET_CORRUPT}|%{SSHD_BAD_VERSION} -``` - -## BACULA_LOG_NO_CONNECT - -Pattern : -``` -Warning: bsock.c:127 Could not connect to (Client: %{BACULA_HOST:client}|Storage daemon) on %{HOSTNAME}:%{POSINT}. ERR=%{GREEDYDATA:berror} -``` - -## SSHD_DISCONNECT - -Pattern : -``` -Received disconnect from %{IP:sshd_client_ip} port %{NUMBER:sshd_port}:%{NUMBER:sshd_disconnect_code}: %{GREEDYDATA:sshd_disconnect_status} -``` - -## SSHD_MAPB_FAIL - -Pattern : -``` -Address %{IP:sshd_client_ip} maps to %{HOSTNAME:sshd_client_hostname}, but this does not map back to the address - POSSIBLE BREAK-IN ATTEMPT! -``` - -## SSHD_TCPWRAP_FAIL2 - -Pattern : -``` -warning: %{DATA:sshd_tcpd_file}, line %{NUMBER}: host name/address mismatch: %{IPORHOST:sshd_client_ip} != %{HOSTNAME:sshd_paranoid_hostname} -``` - -## MONGO3_LOG - -Pattern : -``` -%{TIMESTAMP_ISO8601:timestamp} %{MONGO3_SEVERITY:severity} %{MONGO3_COMPONENT:component}%{SPACE}(?:\[%{DATA:context}\])? %{GREEDYDATA:message} -``` - -## BACULA_LOG_FATAL_CONN - -Pattern : -``` -Fatal error: bsock.c:133 Unable to connect to (Client: %{BACULA_HOST:client}|Storage daemon) on %{HOSTNAME}:%{POSINT}. ERR=%{GREEDYDATA:berror} -``` - -## SSHD_TCPWRAP_FAIL4 - -Pattern : -``` -warning: %{DATA:sshd_tcpd_file}, line %{NUMBER}: host name/name mismatch: reverse lookup results in non-FQDN %{HOSTNAME:sshd_paranoid_hostname} -``` - -## CISCOFW710001_710002_710003_710005_710006 - -Pattern : -``` -%{WORD:protocol} (?:request|access) %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port} -``` - -## NAGIOS_PASSIVE_SERVICE_CHECK - -Pattern : -``` -%{NAGIOS_TYPE_PASSIVE_SERVICE_CHECK:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment} -``` - -## NAGIOS_SERVICE_FLAPPING_ALERT - -Pattern : -``` -%{NAGIOS_TYPE_SERVICE_FLAPPING_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_message} -``` - -## NAGIOS_SERVICE_DOWNTIME_ALERT - -Pattern : -``` -%{NAGIOS_TYPE_SERVICE_DOWNTIME_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment} -``` - -## TCPDUMP_OUTPUT - -Pattern : -``` -%{GREEDYDATA:timestamp} IP %{IPORHOST:source_ip}\.%{INT:source_port} > %{IPORHOST:dest_ip}\.%{INT:dest_port}: Flags \[%{GREEDYDATA:tcpflags}\], seq -``` - -## SSHD_TCPWRAP_FAIL1 - -Pattern : -``` -warning: %{DATA:sshd_tcpd_file}, line %{NUMBER}: can't verify hostname: getaddrinfo\(%{DATA:sshd_paranoid_hostname}, %{DATA:sshd_sa_family}\) failed -``` - -## SSHD_FAIL_PREAUTH - -Pattern : -``` -fatal: Unable to negotiate with %{IP:sshd_client_ip} port %{NUMBER:sshd_port}:\s*%{GREEDYDATA:sshd_disconnect_status}? \[%{GREEDYDATA:sshd_privsep}\] -``` - -## NAGIOS_EC_LINE_ENABLE_SVC_NOTIFICATIONS - -Pattern : -``` -%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_SVC_NOTIFICATIONS:nagios_command};%{DATA:nagios_hostname};%{GREEDYDATA:nagios_service} -``` - -## SSHD_TCPWRAP_FAIL3 - -Pattern : -``` -warning: %{DATA:sshd_tcpd_file}, line %{NUMBER}: host name/name mismatch: %{HOSTNAME:sshd_paranoid_hostname_1} != %{HOSTNAME:sshd_paranoid_hostname_2} -``` - -## NAGIOS_EC_LINE_DISABLE_SVC_NOTIFICATIONS - -Pattern : -``` -%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_SVC_NOTIFICATIONS:nagios_command};%{DATA:nagios_hostname};%{GREEDYDATA:nagios_service} -``` - -## NAGIOS_HOST_EVENT_HANDLER - -Pattern : -``` -%{NAGIOS_TYPE_HOST_EVENT_HANDLER:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{DATA:nagios_event_handler_name} -``` - -## CISCOFW313001_313004_313008 - -Pattern : -``` -%{CISCO_ACTION:action} %{WORD:protocol} type=%{INT:icmp_type}, code=%{INT:icmp_code} from %{IP:src_ip} on interface %{DATA:interface}( to %{IP:dst_ip})? -``` - -## BACULA_LOG_END_VOLUME - -Pattern : -``` -End of medium on Volume \"%{BACULA_VOLUME:volume}\" Bytes=%{BACULA_CAPACITY} Blocks=%{BACULA_CAPACITY} at %{MONTHDAY}-%{MONTH}-%{YEAR} %{HOUR}:%{MINUTE}. -``` - -## SSHD_SUCCESS - -Pattern : -``` -Accepted %{WORD:sshd_auth_type} for %{USERNAME:sshd_user} from %{IP:sshd_client_ip} port %{NUMBER:sshd_port} %{WORD:sshd_protocol}: %{GREEDYDATA:sshd_cipher} -``` - -## SMB_AUTH_FAIL - -Pattern : -``` -Auth:%{GREEDYDATA} user \[%{DATA:smb_domain}\]\\\[%{DATA:user}\]%{GREEDYDATA} status \[NT_STATUS_NO_SUCH_USER\]%{GREEDYDATA} remote host \[ipv4:%{IP:ip_source} -``` - -## BACULA_LOG_NEW_MOUNT - -Pattern : -``` -New volume \"%{BACULA_VOLUME:volume}\" mounted on device \"%{BACULA_DEVICE:device}\" \(%{BACULA_DEVICEPATH}\) at %{MONTHDAY}-%{MONTH}-%{YEAR} %{HOUR}:%{MINUTE}. -``` - -## NAGIOS_HOST_ALERT - -Pattern : -``` -%{NAGIOS_TYPE_HOST_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{NUMBER:nagios_attempt};%{GREEDYDATA:nagios_message} -``` - -## NAGIOS_HOST_NOTIFICATION - -Pattern : -``` -%{NAGIOS_TYPE_HOST_NOTIFICATION:nagios_type}: %{DATA:nagios_notifyname};%{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_contact};%{GREEDYDATA:nagios_message} -``` - -## SYSLOGPAMSESSION - -Pattern : -``` -%{SYSLOGBASE} %{GREEDYDATA:message}%{WORD:pam_module}\(%{DATA:pam_caller}\): session %{WORD:pam_session_state} for user %{USERNAME:username}(?: by %{GREEDYDATA:pam_by})? -``` - -## NAGIOS_CURRENT_HOST_STATE - -Pattern : -``` -%{NAGIOS_TYPE_CURRENT_HOST_STATE:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statetype};%{DATA:nagios_statecode};%{GREEDYDATA:nagios_message} -``` - -## CISCOFW419002 - -Pattern : -``` -%{CISCO_REASON:reason} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port} with different initial sequence number -``` - -## IPV4 - -Pattern : -``` -(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])) -``` - -## SSHD_FAI2_PREAUTH - -Pattern : -``` -fatal: %{GREEDYDATA:sshd_fatal_status}: Connection from %{IP:sshd_client_ip} port %{NUMBER:sshd_port}:\s*%{GREEDYDATA:sshd_disconnect_status}? \[%{GREEDYDATA:sshd_privsep}\] -``` - -## APACHEERRORPREFIX - -Pattern : -``` -\[%{APACHEERRORTIME:timestamp}\] \[%{NOTSPACE:apacheseverity}\] (\[pid %{INT}:tid %{INT}\] )?\[client %{IPORHOST:sourcehost}(:%{INT:source_port})?\] (\[client %{IPORHOST}\])? -``` - -## NAGIOS_SERVICE_EVENT_HANDLER - -Pattern : -``` -%{NAGIOS_TYPE_SERVICE_EVENT_HANDLER:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{DATA:nagios_event_handler_name} -``` - -## NAGIOS_EC_LINE_PROCESS_HOST_CHECK_RESULT - -Pattern : -``` -%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_PROCESS_HOST_CHECK_RESULT:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_check_result} -``` - -## SSHD_PROBE_LOG - -Pattern : -``` -%{SSHD_REFUSE_CONN}|%{SSHD_TCPWRAP_FAIL1}|%{SSHD_TCPWRAP_FAIL2}|%{SSHD_TCPWRAP_FAIL3}|%{SSHD_TCPWRAP_FAIL4}|%{SSHD_TCPWRAP_FAIL5}|%{SSHD_FAIL}|%{SSHD_USER_FAIL}|%{SSHD_INVAL_USER} -``` - -## NAXSI_EXLOG - -Pattern : -``` -^NAXSI_EXLOG: ip=%{IPORHOST:naxsi_src_ip}&server=%{IPORHOST:naxsi_dst_ip}&uri=%{PATH:http_path}&id=%{INT:naxsi_id}&zone=%{WORD:naxsi_zone}&var_name=%{DATA:naxsi_var_name}&content= -``` - -## SSHD_RECE_PREAUTH - -Pattern : -``` -(?:error: |)Received disconnect from %{IP:sshd_client_ip} port %{NUMBER:sshd_port}:%{NUMBER:sshd_disconnect_code}: %{GREEDYDATA:sshd_disconnect_status}? \[%{GREEDYDATA:sshd_privsep}\] -``` - -## MONTH - -Pattern : -``` -\bJan(?:uary|uar)?|Feb(?:ruary|ruar)?|M(?:a|ä)?r(?:ch|z)?|Apr(?:il)?|Ma(?:y|i)?|Jun(?:e|i)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|O(?:c|k)?t(?:ober)?|Nov(?:ember)?|De(?:c|z)(?:ember)?\b -``` - -## CISCOFW419001 - -Pattern : -``` -%{CISCO_ACTION:action} %{WORD:protocol} packet from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port}, reason: %{GREEDYDATA:reason} -``` - -## NAGIOS_SERVICE_ALERT - -Pattern : -``` -%{NAGIOS_TYPE_SERVICE_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{NUMBER:nagios_attempt};%{GREEDYDATA:nagios_message} -``` - -## CISCOFW106015 - -Pattern : -``` -%{CISCO_ACTION:action} %{WORD:protocol} \(%{DATA:policy_id}\) from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{DATA:tcp_flags} on interface %{GREEDYDATA:interface} -``` - -## CISCOFW602303_602304 - -Pattern : -``` -%{WORD:protocol}: An %{CISCO_DIRECTION:direction} %{GREEDYDATA:tunnel_type} SA \(SPI= %{DATA:spi}\) between %{IP:src_ip} and %{IP:dst_ip} \(user= %{DATA:user}\) has been %{CISCO_ACTION:action} -``` - -## NAGIOS_SERVICE_NOTIFICATION - -Pattern : -``` -%{NAGIOS_TYPE_SERVICE_NOTIFICATION:nagios_type}: %{DATA:nagios_notifyname};%{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_contact};%{GREEDYDATA:nagios_message} -``` - -## RT_FLOW3 - -Pattern : -``` -%{RT_FLOW_EVENT:event}: session denied %{IP:src-ip}/%{INT:src-port}->%{IP:dst-ip}/%{INT:dst-port} %{DATA:service} %{INT:protocol-id}\(\d\) %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} .* -``` - -## NAGIOS_CURRENT_SERVICE_STATE - -Pattern : -``` -%{NAGIOS_TYPE_CURRENT_SERVICE_STATE:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statetype};%{DATA:nagios_statecode};%{GREEDYDATA:nagios_message} -``` - -## CISCOFW713172 - -Pattern : -``` -Group = %{GREEDYDATA:group}, IP = %{IP:src_ip}, Automatic NAT Detection Status:\s+Remote end\s*%{DATA:is_remote_natted}\s*behind a NAT device\s+This\s+end\s*%{DATA:is_local_natted}\s*behind a NAT device -``` - -## NAGIOS_EC_LINE_PROCESS_SERVICE_CHECK_RESULT - -Pattern : -``` -%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_PROCESS_SERVICE_CHECK_RESULT:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_check_result} -``` - -## CISCOFW402119 - -Pattern : -``` -%{WORD:protocol}: Received an %{WORD:orig_protocol} packet \(SPI= %{DATA:spi}, sequence number= %{DATA:seq_num}\) from %{IP:src_ip} \(user= %{DATA:user}\) to %{IP:dst_ip} that failed anti-replay checking -``` - -## SSHD_PREAUTH - -Pattern : -``` -%{SSHD_DISC_PREAUTH}|%{SSHD_RECE_PREAUTH}|%{SSHD_MAXE_PREAUTH}|%{SSHD_DISR_PREAUTH}|%{SSHD_INVA_PREAUTH}|%{SSHD_REST_PREAUTH}|%{SSHD_FAIL_PREAUTH}|%{SSHD_CLOS_PREAUTH}|%{SSHD_FAI2_PREAUTH}|%{SSHD_BADL_PREAUTH} -``` - -## COMMONAPACHELOG - -Pattern : -``` -%{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) -``` - -## SSHD_MAXE_PREAUTH - -Pattern : -``` -error: maximum authentication attempts exceeded for (?:invalid user |)%{USERNAME:sshd_invalid_user} from %{IP:sshd_client_ip} port %{NUMBER:sshd_port} %{WORD:sshd_protocol}\s*(?:\[%{GREEDYDATA:sshd_privsep}\]|) -``` - -## CISCOFW106001 - -Pattern : -``` -%{CISCO_DIRECTION:direction} %{WORD:protocol} connection %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{GREEDYDATA:tcp_flags} on interface %{GREEDYDATA:interface} -``` - -## LOGLEVEL - -Pattern : -``` -[Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)? -``` - -## CISCOFW305011 - -Pattern : -``` -%{CISCO_ACTION:action} %{CISCO_XLATE_TYPE:xlate_type} %{WORD:protocol} translation from %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? to %{DATA:src_xlated_interface}:%{IP:src_xlated_ip}/%{DATA:src_xlated_port} -``` - -## MONGO_SLOWQUERY - -Pattern : -``` -%{WORD} %{MONGO_WORDDASH:database}\.%{MONGO_WORDDASH:collection} %{WORD}: %{MONGO_QUERY:query} %{WORD}:%{NONNEGINT:ntoreturn} %{WORD}:%{NONNEGINT:ntoskip} %{WORD}:%{NONNEGINT:nscanned}.*nreturned:%{NONNEGINT:nreturned}..+ %{POSINT:duration}ms -``` - -## NAXSI_FMT - -Pattern : -``` -^NAXSI_FMT: ip=%{IPORHOST:src_ip}&server=%{IPORHOST:target_ip}&uri=%{PATH:http_path}&learning=\d&vers=%{DATA:naxsi_version}&total_processed=\d+&total_blocked=\d+&block=\d+(&cscore\d=%{WORD:score_label}&score\d=%{INT:score})+&zone0=%{WORD:zone} -``` - -## CISCOFW106014 - -Pattern : -``` -%{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{IP:dst_ip}(\(%{DATA:dst_fwuser}\))? \(type %{INT:icmp_type}, code %{INT:icmp_code}\) -``` - -## NGINXACCESS - -Pattern : -``` -%{IPORHOST:remote_addr} - %{NGUSER:remote_user} \[%{HTTPDATE:time_local}\] "%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}" %{NUMBER:status} %{NUMBER:body_bytes_sent} "%{NOTDQUOTE:http_referer}" "%{NOTDQUOTE:http_user_agent}" -``` - -## EXIM_EXCLUDE_TERMS - -Pattern : -``` -(Message is frozen|(Start|End) queue run| Warning: | retry time not reached | no (IP address|host name) found for (IP address|host) | unexpected disconnection while reading SMTP command | no immediate delivery: |another process is handling this message) -``` - -## CISCOFW302020_302021 - -Pattern : -``` -%{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection for faddr %{IP:dst_ip}/%{INT:icmp_seq_num}(?:\(%{DATA:fwuser}\))? gaddr %{IP:src_xlated_ip}/%{INT:icmp_code_xlated} laddr %{IP:src_ip}/%{INT:icmp_code}( \(%{DATA:user}\))? -``` - -## CISCOFW106006_106007_106010 - -Pattern : -``` -%{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} (?:from|src) %{IP:src_ip}/%{INT:src_port}(\(%{DATA:src_fwuser}\))? (?:to|dst) %{IP:dst_ip}/%{INT:dst_port}(\(%{DATA:dst_fwuser}\))? (?:on interface %{DATA:interface}|due to %{CISCO_REASON:reason}) -``` - -## HTTPD24_ERRORLOG - -Pattern : -``` -\[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message} -``` - -## MODSECAPACHEERROR - -Pattern : -``` -%{MODSECPREFIX} %{MODSECRULEFILE} %{MODSECRULELINE} (?:%{MODSECMATCHOFFSET} )?(?:%{MODSECRULEID} )?(?:%{MODSECRULEREV} )?(?:%{MODSECRULEMSG} )?(?:%{MODSECRULEDATA} )?(?:%{MODSECRULESEVERITY} )?(?:%{MODSECRULEVERS} )?%{MODSECRULETAGS}%{MODSECHOSTNAME} %{MODSECURI} %{MODSECUID} -``` - -## NAGIOS_EC_LINE_SCHEDULE_HOST_DOWNTIME - -Pattern : -``` -%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_SCHEDULE_HOST_DOWNTIME:nagios_command};%{DATA:nagios_hostname};%{NUMBER:nagios_start_time};%{NUMBER:nagios_end_time};%{NUMBER:nagios_fixed};%{NUMBER:nagios_trigger_id};%{NUMBER:nagios_duration};%{DATA:author};%{DATA:comment} -``` - -## SYSLOG5424BASE - -Pattern : -``` -%{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP_ISO8601:syslog5424_ts}|-) +(?:%{HOSTNAME:syslog5424_host}|-) +(-|%{SYSLOG5424PRINTASCII:syslog5424_app}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_proc}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_msgid}) +(?:%{SYSLOG5424SD:syslog5424_sd}|-|) -``` - -## CISCOFW106100_2_3 - -Pattern : -``` -access-list %{NOTSPACE:policy_id} %{CISCO_ACTION:action} %{WORD:protocol} for user '%{DATA:src_fwuser}' %{DATA:src_interface}/%{IP:src_ip}\(%{INT:src_port}\) -> %{DATA:dst_interface}/%{IP:dst_ip}\(%{INT:dst_port}\) hit-cnt %{INT:hit_count} %{CISCO_INTERVAL:interval} \[%{DATA:hashcode1}, %{DATA:hashcode2}\] -``` - -## CISCOFW106100 - -Pattern : -``` -access-list %{NOTSPACE:policy_id} %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}/%{IP:src_ip}\(%{INT:src_port}\)(\(%{DATA:src_fwuser}\))? -> %{DATA:dst_interface}/%{IP:dst_ip}\(%{INT:dst_port}\)(\(%{DATA:src_fwuser}\))? hit-cnt %{INT:hit_count} %{CISCO_INTERVAL:interval} \[%{DATA:hashcode1}, %{DATA:hashcode2}\] -``` - -## RT_FLOW2 - -Pattern : -``` -%{RT_FLOW_EVENT:event}: session created %{IP:src-ip}/%{INT:src-port}->%{IP:dst-ip}/%{INT:dst-port} %{DATA:service} %{IP:nat-src-ip}/%{INT:nat-src-port}->%{IP:nat-dst-ip}/%{INT:nat-dst-port} %{DATA:src-nat-rule-name} %{DATA:dst-nat-rule-name} %{INT:protocol-id} %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} %{INT:session-id} .* -``` - -## CISCOFW733100 - -Pattern : -``` -\[\s*%{DATA:drop_type}\s*\] drop %{DATA:drop_rate_id} exceeded. Current burst rate is %{INT:drop_rate_current_burst} per second, max configured rate is %{INT:drop_rate_max_burst}; Current average rate is %{INT:drop_rate_current_avg} per second, max configured rate is %{INT:drop_rate_max_avg}; Cumulative total count is %{INT:drop_total_count} -``` - -## CISCOFW106023 - -Pattern : -``` -%{CISCO_ACTION:action}( protocol)? %{WORD:protocol} src %{DATA:src_interface}:%{DATA:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{DATA:dst_ip}(/%{INT:dst_port})?(\(%{DATA:dst_fwuser}\))?( \(type %{INT:icmp_type}, code %{INT:icmp_code}\))? by access-group "?%{DATA:policy_id}"? \[%{DATA:hashcode1}, %{DATA:hashcode2}\] -``` - -## ELB_ACCESS_LOG - -Pattern : -``` -%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:elb} %{IP:clientip}:%{INT:clientport:int} (?:(%{IP:backendip}:?:%{INT:backendport:int})|-) %{NUMBER:request_processing_time:float} %{NUMBER:backend_processing_time:float} %{NUMBER:response_processing_time:float} %{INT:response:int} %{INT:backend_response:int} %{INT:received_bytes:int} %{INT:bytes:int} "%{ELB_REQUEST_LINE}" -``` - -## MODSECRULETAGS - -Pattern : -``` -(?:\[tag %{QUOTEDSTRING:ruletag0}\] )?(?:\[tag %{QUOTEDSTRING:ruletag1}\] )?(?:\[tag %{QUOTEDSTRING:ruletag2}\] )?(?:\[tag %{QUOTEDSTRING:ruletag3}\] )?(?:\[tag %{QUOTEDSTRING:ruletag4}\] )?(?:\[tag %{QUOTEDSTRING:ruletag5}\] )?(?:\[tag %{QUOTEDSTRING:ruletag6}\] )?(?:\[tag %{QUOTEDSTRING:ruletag7}\] )?(?:\[tag %{QUOTEDSTRING:ruletag8}\] )?(?:\[tag %{QUOTEDSTRING:ruletag9}\] )?(?:\[tag %{QUOTEDSTRING}\] )* -``` - -## RT_FLOW1 - -Pattern : -``` -%{RT_FLOW_EVENT:event}: %{GREEDYDATA:close-reason}: %{IP:src-ip}/%{INT:src-port}->%{IP:dst-ip}/%{INT:dst-port} %{DATA:service} %{IP:nat-src-ip}/%{INT:nat-src-port}->%{IP:nat-dst-ip}/%{INT:nat-dst-port} %{DATA:src-nat-rule-name} %{DATA:dst-nat-rule-name} %{INT:protocol-id} %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} %{INT:session-id} \d+\(%{DATA:sent}\) \d+\(%{DATA:received}\) %{INT:elapsed-time} .* -``` - -## BRO_CONN - -Pattern : -``` -%{NUMBER:ts}\t%{NOTSPACE:uid}\t%{IP:orig_h}\t%{INT:orig_p}\t%{IP:resp_h}\t%{INT:resp_p}\t%{WORD:proto}\t%{GREEDYDATA:service}\t%{NUMBER:duration}\t%{NUMBER:orig_bytes}\t%{NUMBER:resp_bytes}\t%{GREEDYDATA:conn_state}\t%{GREEDYDATA:local_orig}\t%{GREEDYDATA:missed_bytes}\t%{GREEDYDATA:history}\t%{GREEDYDATA:orig_pkts}\t%{GREEDYDATA:orig_ip_bytes}\t%{GREEDYDATA:resp_pkts}\t%{GREEDYDATA:resp_ip_bytes}\t%{GREEDYDATA:tunnel_parents} -``` - -## S3_ACCESS_LOG - -Pattern : -``` -%{WORD:owner} %{NOTSPACE:bucket} \[%{HTTPDATE:timestamp}\] %{IP:clientip} %{NOTSPACE:requester} %{NOTSPACE:request_id} %{NOTSPACE:operation} %{NOTSPACE:key} (?:"%{S3_REQUEST_LINE}"|-) (?:%{INT:response:int}|-) (?:-|%{NOTSPACE:error_code}) (?:%{INT:bytes:int}|-) (?:%{INT:object_size:int}|-) (?:%{INT:request_time_ms:int}|-) (?:%{INT:turnaround_time_ms:int}|-) (?:%{QS:referrer}|-) (?:"?%{QS:agent}"?|-) (?:-|%{NOTSPACE:version_id}) -``` - -## BRO_DNS - -Pattern : -``` -%{NUMBER:ts}\t%{NOTSPACE:uid}\t%{IP:orig_h}\t%{INT:orig_p}\t%{IP:resp_h}\t%{INT:resp_p}\t%{WORD:proto}\t%{INT:trans_id}\t%{GREEDYDATA:query}\t%{GREEDYDATA:qclass}\t%{GREEDYDATA:qclass_name}\t%{GREEDYDATA:qtype}\t%{GREEDYDATA:qtype_name}\t%{GREEDYDATA:rcode}\t%{GREEDYDATA:rcode_name}\t%{GREEDYDATA:AA}\t%{GREEDYDATA:TC}\t%{GREEDYDATA:RD}\t%{GREEDYDATA:RA}\t%{GREEDYDATA:Z}\t%{GREEDYDATA:answers}\t%{GREEDYDATA:TTLs}\t%{GREEDYDATA:rejected} -``` - -## CISCOFW302013_302014_302015_302016 - -Pattern : -``` -%{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection %{INT:connection_id} for %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port}( \(%{IP:src_mapped_ip}/%{INT:src_mapped_port}\))?(\(%{DATA:src_fwuser}\))? to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port}( \(%{IP:dst_mapped_ip}/%{INT:dst_mapped_port}\))?(\(%{DATA:dst_fwuser}\))?( duration %{TIME:duration} bytes %{INT:bytes})?(?: %{CISCO_REASON:reason})?( \(%{DATA:user}\))? -``` - -## SHOREWALL - -Pattern : -``` -(%{SYSLOGTIMESTAMP:timestamp}) (%{WORD:nf_host}) kernel:.*Shorewall:(%{WORD:nf_action1})?:(%{WORD:nf_action2})?.*IN=(%{USERNAME:nf_in_interface})?.*(OUT= *MAC=(%{COMMONMAC:nf_dst_mac}):(%{COMMONMAC:nf_src_mac})?|OUT=%{USERNAME:nf_out_interface}).*SRC=(%{IPV4:nf_src_ip}).*DST=(%{IPV4:nf_dst_ip}).*LEN=(%{WORD:nf_len}).*?TOS=(%{WORD:nf_tos}).*?PREC=(%{WORD:nf_prec}).*?TTL=(%{INT:nf_ttl}).*?ID=(%{INT:nf_id}).*?PROTO=(%{WORD:nf_protocol}).*?SPT=(%{INT:nf_src_port}?.*DPT=%{INT:nf_dst_port}?.*) -``` - -## HAPROXYTCP - -Pattern : -``` -(?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{IP:client_ip}:%{INT:client_port} \[%{HAPROXYDATE:accept_date}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_queue}/%{INT:time_backend_connect}/%{NOTSPACE:time_duration} %{NOTSPACE:bytes_read} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} -``` - -## CISCOFW313005 - -Pattern : -``` -%{CISCO_REASON:reason} for %{WORD:protocol} error message: %{WORD:err_protocol} src %{DATA:err_src_interface}:%{IP:err_src_ip}(\(%{DATA:err_src_fwuser}\))? dst %{DATA:err_dst_interface}:%{IP:err_dst_ip}(\(%{DATA:err_dst_fwuser}\))? \(type %{INT:err_icmp_type}, code %{INT:err_icmp_code}\) on %{DATA:interface} interface\. Original IP payload: %{WORD:protocol} src %{IP:orig_src_ip}/%{INT:orig_src_port}(\(%{DATA:orig_src_fwuser}\))? dst %{IP:orig_dst_ip}/%{INT:orig_dst_port}(\(%{DATA:orig_dst_fwuser}\))? -``` - -## BRO_FILES - -Pattern : -``` -%{NUMBER:ts}\t%{NOTSPACE:fuid}\t%{IP:tx_hosts}\t%{IP:rx_hosts}\t%{NOTSPACE:conn_uids}\t%{GREEDYDATA:source}\t%{GREEDYDATA:depth}\t%{GREEDYDATA:analyzers}\t%{GREEDYDATA:mime_type}\t%{GREEDYDATA:filename}\t%{GREEDYDATA:duration}\t%{GREEDYDATA:local_orig}\t%{GREEDYDATA:is_orig}\t%{GREEDYDATA:seen_bytes}\t%{GREEDYDATA:total_bytes}\t%{GREEDYDATA:missing_bytes}\t%{GREEDYDATA:overflow_bytes}\t%{GREEDYDATA:timedout}\t%{GREEDYDATA:parent_fuid}\t%{GREEDYDATA:md5}\t%{GREEDYDATA:sha1}\t%{GREEDYDATA:sha256}\t%{GREEDYDATA:extracted} -``` - -## BRO_HTTP - -Pattern : -``` -%{NUMBER:ts}\t%{NOTSPACE:uid}\t%{IP:orig_h}\t%{INT:orig_p}\t%{IP:resp_h}\t%{INT:resp_p}\t%{INT:trans_depth}\t%{GREEDYDATA:method}\t%{GREEDYDATA:domain}\t%{GREEDYDATA:uri}\t%{GREEDYDATA:referrer}\t%{GREEDYDATA:user_agent}\t%{NUMBER:request_body_len}\t%{NUMBER:response_body_len}\t%{GREEDYDATA:status_code}\t%{GREEDYDATA:status_msg}\t%{GREEDYDATA:info_code}\t%{GREEDYDATA:info_msg}\t%{GREEDYDATA:filename}\t%{GREEDYDATA:bro_tags}\t%{GREEDYDATA:username}\t%{GREEDYDATA:password}\t%{GREEDYDATA:proxied}\t%{GREEDYDATA:orig_fuids}\t%{GREEDYDATA:orig_mime_types}\t%{GREEDYDATA:resp_fuids}\t%{GREEDYDATA:resp_mime_types} -``` - -## NETSCREENSESSIONLOG - -Pattern : -``` -%{SYSLOGTIMESTAMP:date} %{IPORHOST:device} %{IPORHOST}: NetScreen device_id=%{WORD:device_id}%{DATA}: start_time=%{QUOTEDSTRING:start_time} duration=%{INT:duration} policy_id=%{INT:policy_id} service=%{DATA:service} proto=%{INT:proto} src zone=%{WORD:src_zone} dst zone=%{WORD:dst_zone} action=%{WORD:action} sent=%{INT:sent} rcvd=%{INT:rcvd} src=%{IPORHOST:src_ip} dst=%{IPORHOST:dst_ip} src_port=%{INT:src_port} dst_port=%{INT:dst_port} src-xlated ip=%{IPORHOST:src_xlated_ip} port=%{INT:src_xlated_port} dst-xlated ip=%{IPORHOST:dst_xlated_ip} port=%{INT:dst_xlated_port} session_id=%{INT:session_id} reason=%{GREEDYDATA:reason} -``` - -## HAPROXYHTTPBASE - -Pattern : -``` -%{IP:client_ip}:%{INT:client_port} \[%{HAPROXYDATE:accept_date}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_request}/%{INT:time_queue}/%{INT:time_backend_connect}/%{INT:time_backend_response}/%{NOTSPACE:time_duration} %{INT:http_status_code} %{NOTSPACE:bytes_read} %{DATA:captured_request_cookie} %{DATA:captured_response_cookie} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} (\\{\%\{HAPROXYCAPTUREDREQUESTHEADERS}\})?( )?(\\{\%\{HAPROXYCAPTUREDRESPONSEHEADERS}\})?( )?"(|(%{WORD:http_verb} (%{URIPROTO:http_proto}://)?(?:%{USER:http_user}(?::[^@]*)?@)?(?:%{URIHOST:http_host})?(?:%{URIPATHPARAM:http_request})?( HTTP/%{NUMBER:http_version})?))?" -``` - -## BACULA_LOGLINE - -Pattern : -``` -%{BACULA_TIMESTAMP:bts} %{BACULA_HOST:hostname} JobId %{INT:jobid}: (%{BACULA_LOG_MAX_CAPACITY}|%{BACULA_LOG_END_VOLUME}|%{BACULA_LOG_NEW_VOLUME}|%{BACULA_LOG_NEW_LABEL}|%{BACULA_LOG_WROTE_LABEL}|%{BACULA_LOG_NEW_MOUNT}|%{BACULA_LOG_NOOPEN}|%{BACULA_LOG_NOOPENDIR}|%{BACULA_LOG_NOSTAT}|%{BACULA_LOG_NOJOBS}|%{BACULA_LOG_ALL_RECORDS_PRUNED}|%{BACULA_LOG_BEGIN_PRUNE_JOBS}|%{BACULA_LOG_BEGIN_PRUNE_FILES}|%{BACULA_LOG_PRUNED_JOBS}|%{BACULA_LOG_PRUNED_FILES}|%{BACULA_LOG_ENDPRUNE}|%{BACULA_LOG_STARTJOB}|%{BACULA_LOG_STARTRESTORE}|%{BACULA_LOG_USEDEVICE}|%{BACULA_LOG_DIFF_FS}|%{BACULA_LOG_JOBEND}|%{BACULA_LOG_NOPRUNE_JOBS}|%{BACULA_LOG_NOPRUNE_FILES}|%{BACULA_LOG_VOLUME_PREVWRITTEN}|%{BACULA_LOG_READYAPPEND}|%{BACULA_LOG_CANCELLING}|%{BACULA_LOG_MARKCANCEL}|%{BACULA_LOG_CLIENT_RBJ}|%{BACULA_LOG_VSS}|%{BACULA_LOG_MAXSTART}|%{BACULA_LOG_DUPLICATE}|%{BACULA_LOG_NOJOBSTAT}|%{BACULA_LOG_FATAL_CONN}|%{BACULA_LOG_NO_CONNECT}|%{BACULA_LOG_NO_AUTH}|%{BACULA_LOG_NOSUIT}|%{BACULA_LOG_JOB}|%{BACULA_LOG_NOPRIOR}) -``` - -## NAGIOSLOGLINE - -Pattern : -``` -%{NAGIOSTIME} (?:%{NAGIOS_WARNING}|%{NAGIOS_CURRENT_SERVICE_STATE}|%{NAGIOS_CURRENT_HOST_STATE}|%{NAGIOS_SERVICE_NOTIFICATION}|%{NAGIOS_HOST_NOTIFICATION}|%{NAGIOS_SERVICE_ALERT}|%{NAGIOS_HOST_ALERT}|%{NAGIOS_SERVICE_FLAPPING_ALERT}|%{NAGIOS_HOST_FLAPPING_ALERT}|%{NAGIOS_SERVICE_DOWNTIME_ALERT}|%{NAGIOS_HOST_DOWNTIME_ALERT}|%{NAGIOS_PASSIVE_SERVICE_CHECK}|%{NAGIOS_PASSIVE_HOST_CHECK}|%{NAGIOS_SERVICE_EVENT_HANDLER}|%{NAGIOS_HOST_EVENT_HANDLER}|%{NAGIOS_TIMEPERIOD_TRANSITION}|%{NAGIOS_EC_LINE_DISABLE_SVC_CHECK}|%{NAGIOS_EC_LINE_ENABLE_SVC_CHECK}|%{NAGIOS_EC_LINE_DISABLE_HOST_CHECK}|%{NAGIOS_EC_LINE_ENABLE_HOST_CHECK}|%{NAGIOS_EC_LINE_PROCESS_HOST_CHECK_RESULT}|%{NAGIOS_EC_LINE_PROCESS_SERVICE_CHECK_RESULT}|%{NAGIOS_EC_LINE_SCHEDULE_HOST_DOWNTIME}|%{NAGIOS_EC_LINE_DISABLE_HOST_SVC_NOTIFICATIONS}|%{NAGIOS_EC_LINE_ENABLE_HOST_SVC_NOTIFICATIONS}|%{NAGIOS_EC_LINE_DISABLE_HOST_NOTIFICATIONS}|%{NAGIOS_EC_LINE_ENABLE_HOST_NOTIFICATIONS}|%{NAGIOS_EC_LINE_DISABLE_SVC_NOTIFICATIONS}|%{NAGIOS_EC_LINE_ENABLE_SVC_NOTIFICATIONS}) -``` - -## IPV6 - -Pattern : -``` -((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? -``` - - -# Documentation generation -This documentation is generated by `pkg/parser` : `GO_WANT_TEST_DOC=1 go test -run TestGeneratePatternsDoc` diff --git a/docs/v1.X/docs/references/postoverflows.md b/docs/v1.X/docs/references/postoverflows.md deleted file mode 100644 index 4c6119d3d..000000000 --- a/docs/v1.X/docs/references/postoverflows.md +++ /dev/null @@ -1,7 +0,0 @@ -# Post Overflows - -PostOverflows is secondary parsing phase that happens *after* a bucket overflowed. -It behaves exactly like a [Normal Parsing](/Crowdsec/v1/references/parsers/). However, instead of receiving {{v1X.event.htmlname}} with logs, the parser receive events with {{v1X.alert.htmlname}} representing the overflows. - -The configuration resides in `/etc/crowdsec/postoverflows/`. - diff --git a/docs/v1.X/docs/references/profiles.md b/docs/v1.X/docs/references/profiles.md deleted file mode 100644 index f8f7fc5a1..000000000 --- a/docs/v1.X/docs/references/profiles.md +++ /dev/null @@ -1,81 +0,0 @@ -# Profiles configurations - -The profiles configuration (`/etc/crowdsec/profiles.yaml`) allow to configure what kind of remediation needs to be applied when a scenario is triggered : - -The configuration file is a yaml file that looks like : - -```yaml -name: default_ip_remediation -#debug: true -filters: - - Alert.Remediation == true && Alert.GetScope() == "Ip" -decisions: - - type: ban - duration: 4h -on_success: break -``` - -Each YAML object in the file contains a list of `models.Decision` that contains : - -## `name` - -```yaml -name: foobar -``` - -A label for the profile (used in logging) - -## `debug` - -```yaml -debug: true -``` - -A boolean flag that provides contextual debug. - -## `filters` - -```yaml -filters: - - Alert.Remediation == true && Alert.GetScope() == "Session" - - Alert.Remediation == true && Alert.GetScope() == "Ip" -``` - -If any `filter` of the list returns `true`, the profile is elligible and the `decisions` will be applied. - -## `decisions` - -```yaml -decisions: - - type: captcha - duration: 1h - scope: custom_app1_captcha - - type: ban - duration: 2h -``` - -If the profile applies, decisions objects will be created for each of the sources that triggered the scenario. - -It is a list of `models.Decision` objects. The following fields, when present, allows to alter the resulting decision : - - - `scope` : defines the scope of the resulting decision - - `duration` : defines for how long will the decision be valid - - `type` : defines the type of the remediation that will be applied by available {{v1X.bouncers.htmlname}}, for example `ban`, `captcha` - - `value` : define a hardcoded value for the decision (ie. `1.2.3.4`) - -## `on_success` - -```yaml -on_success: break -``` - -If the profile applies and `on_success` is set to `break`, decisions processing will stop here and it won't evaluate against following profiles. - -## `on_failure` - -```yaml -on_failure: break -``` - -If the profile didn't apply and `on_failure` is set to `break`, decisions processing will stop here and it won't evaluate against following profiles. - diff --git a/docs/v1.X/docs/references/scenarios.md b/docs/v1.X/docs/references/scenarios.md deleted file mode 100644 index 0819fddd5..000000000 --- a/docs/v1.X/docs/references/scenarios.md +++ /dev/null @@ -1,477 +0,0 @@ -## Understanding scenarios - - -Scenarios are YAML files that allow to detect and qualify a specific behavior, usually an attack. - -Scenarios receive {{v1X.event.htmlname}}(s) and can produce {{v1X.alert.htmlname}}(s) using the [leaky bucket](https://en.wikipedia.org/wiki/Leaky_bucket) algorithm. - -As an {{v1X.event.htmlname}} can be the representation of a log line, or an overflow, it allows scenarios to process both logs or overflows to allow inference. - -Scenarios can be of different types (leaky, trigger, counter), and are based on various factors, such as : - - - the speed/frequency of the [leaky bucket](https://en.wikipedia.org/wiki/Leaky_bucket) - - the capacity of the [leaky bucket](https://en.wikipedia.org/wiki/Leaky_bucket) - - the characteristic(s) of eligible {{v1X.event.htmlname}}(s) : "log type XX with field YY set to ZZ" - - various filters/directives that can alter the bucket's behavior, such as [groupby](/Crowdsec/v1/references/scenarios/#groupby), [distinct](/Crowdsec/v1/references/scenarios/#distinct) or [blackhole](/Crowdsec/v1/references/scenarios/#blackhole) - -Behind the scenes, {{v1X.crowdsec.name}} is going to create one or more buckets when events with matching characteristics arrive to the scenario. When any of these buckets overflows, the scenario has been triggered. - -_Bucket partitioning_ : One scenario usually leads to many buckets creation, as each bucket is only tracking a specific subset of events. For example, if we are tracking brute-force, each "offending peer" get its own bucket. - - -A way to detect a http scanner might be to track the number of distinct non-existing pages it's requesting, and the scenario might look like this : - - -```yaml -#the bucket type : leaky, trigger, counter -type: leaky -#name and description for humans -name: crowdsecurity/http-scan-uniques_404 -description: "Detect multiple unique 404 from a single ip" -#a filter to know which events are eligible -filter: "evt.Meta.service == 'http' && evt.Meta.http_status in ['404', '403', '400']" -#how we are going to partition buckets -groupby: "evt.Meta.source_ip" -#we are only interested into counting UNIQUE/DISTINCT requested URLs -distinct: "evt.Meta.http_path" -#we specify the bucket capacity and leak speed -capacity: 5 -leakspeed: "10s" -#this will prevent the same bucket from overflowing more often than every 5 minutes -blackhole: 5m -#some labels to give context to the overflow -labels: - service: http - type: scan - #yes we want to ban people triggering this - remediation: true -``` - - -## Scenario concepts - -### TimeMachine - -{{v1X.crowdsec.name}} can be used not only to process live logs, but as well to process "cold" logs (think forensics). - -For this to be able to work, the date/time from the log must have been properly parsed for the scenario temporal aspect to be able to work properly. This relies on the [date-parse enrichment](https://github.com/crowdsecurity/hub/blob/master/parsers/s02-enrich/crowdsecurity/dateparse-enrich.yaml). For this to work the field `evt.StrTime` must have been filled with a string that represents the date & time. the date-parse enrichment support a large variety of formats. - - -## Scenario directives - -### type - - -```yaml -type: leaky|trigger|counter -``` - -Defines the type of the bucket. Currently three types are supported : - - - `leaky` : a [leaky bucket](https://en.wikipedia.org/wiki/Leaky_bucket) that must be configured with a {{v1X.capacity.htmlname}} and a {{v1X.leakspeed.htmlname}} - - `trigger` : a bucket that overflows as soon as an event is poured (it's like a leaky bucket is a capacity of 0) - - `counter` : a bucket that only overflows every {{v1X.duration.htmlname}}. It's especially useful to count things. - -### name & description - -```yaml -name: my_author_name/my_scenario_name -description: A scenario that detect XXXX behavior -``` - - -Mandatory `name` and `description` for said scenario. -The name must be unique (and will define the scenario's name in the hub), and the description must be a quick sentence describing what it detects. - - -### filter - -```yaml -filter: expression -``` - -`filter` must be a valid {{v1X.expr.htmlname}} expression that will be evaluated against the {{v1X.event.htmlname}}. - -If `filter` evaluation returns true or is absent, event will be pour in the bucket. - -If `filter` returns `false` or a non-boolean, the event will be skip for this bucket. - -Here is the [expr documentation](https://github.com/antonmedv/expr/tree/master/docs). - -Examples : - - - `evt.Meta.log_type == 'telnet_new_session'` - - `evt.Meta.log_type in ['http_access-log', 'http_error-log'] && evt.Parsed.static_ressource == 'false'` - - `evt.Meta.log_type == 'ssh_failed-auth'` - - -### duration - -```yaml -duration: 45s -duration: 10m -``` - -(applicable to `counter` buckets only) - -A duration after which the bucket will overflow. -The format must be compatible with [golang ParseDuration format](https://golang.org/pkg/time/#ParseDuration) - -Examples : - -```yaml -type: counter -name: crowdsecurity/ban-reports-ssh_bf_report -description: "Count unique ips performing ssh bruteforce" -filter: "evt.Overflow.Scenario == 'ssh_bruteforce'" -distinct: "evt.Overflow.Source_ip" -capacity: -1 -duration: 10m -labels: - service: ssh -``` - - -### groupby - -```yaml -groupby: evt.Meta.source_ip -``` - - -an {{v1X.expr.htmlname}} that must return a string. This string will be used as to partition the buckets. - - -Examples : - -Here, each `source_ip` will get its own bucket. - -```yaml -type: leaky -... -groupby: evt.Meta.source_ip -... -``` - - - -Here, each unique combo of `source_ip` + `target_username` will get its own bucket. - -```yaml -type: leaky -... -groupby: evt.Meta.source_ip + '--' + evt.Parsed.target_username -... -``` - - - -### distinct - - -```yaml -distinct: evt.Meta.http_path -``` - - -an {{v1X.expr.htmlname}} that must return a string. The event will be poured **only** if the string is not already present in the bucket. - -Examples : - -This will ensure that events that keep triggering the same `.Meta.http_path` will be poured only once. - -```yaml -type: leaky -... -distinct: "evt.Meta.http_path" -... -``` - -In the logs, you can see it like this (for example from the iptables-logs portscan detection) : - -```bash -DEBU[2020-05-13T11:29:51+02:00] Uniq(7681) : ok buck.. -DEBU[2020-05-13T11:29:51+02:00] Uniq(7681) : ko, discard event buck.. -``` - -The first event has been poured (value `7681`) was not yet present in the events, while the second time, the event got discarded because the value was already present in the bucket. - - -### capacity - -```yaml -capacity: 5 -``` - - -(Applies only to `leaky` buckets) - -A positive integer representing the bucket capacity. -If there are more than `capacity` item in the bucket, it will overflow. - - -### cache_size - -```yaml -cache_size: 10 -``` - -A positive integer representing the number of events we hold in the -bucket. All events are still accounted for the bucket, it only -prevents the bucket event history to grow larger than -`cache_size`. When the bucket reachs `cache_size` events, then events -are discarded on an first-in first-out basis. - - -### leakspeed - -```yaml -leakspeed: "10s" -``` - -(Applies only to `leaky` buckets) - -A duration that represent how often an event will be leaking from the bucket. - -Must be compatible with [golang ParseDuration format](https://golang.org/pkg/time/#ParseDuration). - - -Example: - -Here the bucket will leak one item every 10 seconds, and can hold up to 5 items before overflowing. - -```yaml -type: leaky -... -leakspeed: "10s" -capacity: 5 -... -``` - - -### labels - -```yaml -labels: - service: ssh - type: bruteforce - remediation: true -``` - -Labels is a list of `label: values` that provide context to an overflow. -The labels are (currently) not stored in the database, nor they are sent to the API. - -Special labels : - - - The **remediation** label, if set to `true` indicate the the originating IP should be ban. - - The **scope** label, can be set to `ip` or `range` when **remediation** is set to true, and indicate to which scope should the decision apply. If you set a scenario with **remediation** to true and **scope** to `range` and the range of the IP could have been determined by the GeoIP library, the whole range to which the IP belongs will be banned. - - -Example : - -The IP that triggered the overflow (`.Meta.source_ip`) will be banned. -```yaml -type: leaky -... -labels: - service: ssh - type: bruteforce - remediation: true -``` - -The range to which the offending IP belong (`.Meta.source_ip`) will be banned. -```yaml -type: leaky -... -labels: - type: distributed_attack - remediation: true - scope: range -``` - -### blackhole - -```yaml -blackhole: 10m -``` - -A duration for which a bucket will be "silenced" after overflowing. -This is intended to limit / avoid spam of buckets that might be very rapidly triggered. - -The blackhole only applies to the individual bucket rather than the whole scenario. - -Must be compatible with [golang ParseDuration format](https://golang.org/pkg/time/#ParseDuration). - -Example : - -The same `source_ip` won't be able to trigger this overflow more than once every 10 minutes. -The potential overflows in the meanwhile will be discarded (but will still appear in logs as being blackholed). - -```yaml -type: trigger -... -blackhole: 10m -groupby: evt.Meta.source_ip -``` - -### debug - -```yaml -debug: true|false -``` - -_default: false_ - - -If set to to `true`, enabled scenario level debugging. -It is meant to help understanding scenario behavior by providing contextual logging : - -debug of filters and expression results -``` -DEBU[31-07-2020 16:34:58] eval(evt.Meta.log_type in ["http_access-log", "http_error-log"] && any(File("bad_user_agents.txt"), {evt.Parsed.http_user_agent contains #})) = TRUE cfg=still-feather file=config/scenarios/http-bad-user-agent.yaml name=crowdsecurity/http-bad-user-agent -DEBU[31-07-2020 16:34:58] eval variables: cfg=still-feather file=config/scenarios/http-bad-user-agent.yaml name=crowdsecurity/http-bad-user-agent -DEBU[31-07-2020 16:34:58] evt.Meta.log_type = 'http_access-log' cfg=still-feather file=config/scenarios/http-bad-user-agent.yaml name=crowdsecurity/http-bad-user-agent -DEBU[31-07-2020 16:34:58] evt.Parsed.http_user_agent = 'Mozilla/5.00 (Nikto/2.1.5) (Evasions:None) (Test:002810)' cfg=still-feather file=config/scenarios/http-bad-user-agent.yaml name=crowdsecurity/http-bad-user-agent -``` - - -### reprocess - -```yaml -reprocess: true|false -``` - -_default: false_ - -If set to `true`, the resulting overflow will be sent again in the scenario/parsing pipeline. -It is useful when you want to have further scenarios that will rely on past-overflows to take decisions. - - -### cache_size - -```yaml -cache_size: 5 -``` - -By default, a bucket holds {{v1X.capacity.htmlname}} events "in memory". -However, for a number of cases, you don't want this, as it might lead to excessive memory consumption. - -By setting `cache_size` to a positive integer, we can control the maximum in-memory cache size of the bucket, without changing its capacity and such. This is especially useful when using `counter` buckets on long duration that might end up counting (and this storing in memory) an important number of events. - - -### overflow_filter - -```yaml -overflow_filter: any(queue.Queue, { .Enriched.IsInEU == "true" }) -``` - -`overflow_filter` is an {{v1X.expr.htmlname}} that is run when the bucket overflows. -If this expression is present and returns false, the overflow will be discarded. - - -### data - -```yaml -data: - - source_url: https://URL/TO/FILE - dest_file: LOCAL_FILENAME - [type: (regexp|string)] -``` - -`data` allows user to specify an external source of data. -This section is only relevant when `cscli` is used to install scenario from hub, as ill download the `source_url` and store it to `dest_file`. When the scenario is not installed from the hub, {{v1X.crowdsec.name}} won't download the URL, but the file must exist for the scenario to be loaded correctly. -The `type` is mandatory if you want to evaluate the data in the file, and should be `regex` for valid (re2) regular expression per line or `string` for string per line. -The regexps will be compiled, the strings will be loaded into a list and both will be kept in memory. -Without specifying a `type`, the file will be downloaded and stored as file and not in memory. - - -```yaml -name: crowdsecurity/cdn-whitelist -... -data: - - source_url: https://www.cloudflare.com/ips-v4 - dest_file: cloudflare_ips.txt - type: string -``` - - -### format - -```yaml -format: 2.0 -``` - -{{v1X.crowdsec.name}} has a notion of format support for parsers & scenarios for compatibility management. -Running `cscli version` will show you such compatibility matrix : - -```bash -$ sudo cscli version -2020/11/05 09:35:05 version: v0.3.6-183e34c966c475e0d2cdb3c60d0b7426499aa573 -2020/11/05 09:35:05 Codename: beta -2020/11/05 09:35:05 BuildDate: 2020-11-04_17:56:46 -2020/11/05 09:35:05 GoVersion: 1.13 -2020/11/05 09:35:05 Constraint_parser: >= 1.0, < 2.0 -2020/11/05 09:35:05 Constraint_scenario: >= 1.0, < 3.0 -2020/11/05 09:35:05 Constraint_api: v1 -2020/11/05 09:35:05 Constraint_acquis: >= 1.0, < 2.0 -``` - -### Scope - -```yaml -scope: - type: Range - expression: evt.Parsed.mySourceRange -``` - -While most scenarios might focus on Ips, {{v1X.crowdsec.name}} and {{v1X.bouncers.name}} can work with any scope. -The `scope` directive allows you to override the default scope : - - - `type` is a string representing the scope name - - `expression` is an `expr` expression that will be evaluated to fetch the value - - -let's imagine a scenario such as : - -```yaml -# ssh bruteforce -type: leaky -name: crowdsecurity/ssh-enforce-mfa -description: "Enforce mfa on users that have been bruteforced" -filter: "evt.Meta.log_type == 'ssh_failed-auth'" -leakspeed: "10s" -capacity: 5 -groupby: evt.Meta.source_ip -blackhole: 1m -labels: - service: ssh - type: bruteforce - remediation: true -scope: - type: username - expression: evt.Meta.target_user -``` - -and a profile such as : - -```yaml -name: enforce_mfa -filters: - - 'Alert.Remediation == true && Alert.GetScope() == "username"' -decisions: - - type: enforce_mfa - scope: "username" - duration: 1h -on_success: continue -``` - -the resulting overflow will be : - -```bash -$ ./cscli -c dev.yaml decisions list -+----+----------+---------------+-------------------------------+-------------+---------+----+--------+------------------+ -| ID | SOURCE | SCOPE:VALUE | REASON | ACTION | COUNTRY | AS | EVENTS | EXPIRATION | -+----+----------+---------------+-------------------------------+-------------+---------+----+--------+------------------+ -| 2 | crowdsec | username:rura | crowdsecurity/ssh-enforce-mfa | enforce_mfa | | | 6 | 59m46.121840343s | -``` - diff --git a/docs/v1.X/docs/references/simulation.md b/docs/v1.X/docs/references/simulation.md deleted file mode 100644 index 828e7cfe3..000000000 --- a/docs/v1.X/docs/references/simulation.md +++ /dev/null @@ -1,13 +0,0 @@ -# Simulation - -Simulation config is in `/etc/crowdsec/simulation.yaml` and looks like : - -```yaml -#if simulation is set to 'true' here, *all* scenarios will be in simulation unless in exclusion list -simulation: false -#exclusion to the policy - here, the scenarios that are in simulation mode -exclusions: -- crowdsecurity/ssh-bf - -``` - diff --git a/docs/v1.X/docs/references/stages.md b/docs/v1.X/docs/references/stages.md deleted file mode 100644 index 205348492..000000000 --- a/docs/v1.X/docs/references/stages.md +++ /dev/null @@ -1,24 +0,0 @@ -# Stages - -Parsers are organized into "stages" (named using a "sXX-" convention) to allow pipelines and branching in parsing. Each parser belongs to a stage, and can trigger next stage when successful. At the time of writing, the parsers are organized around 3 stages : - - - `s00-raw` : low level parser, such as syslog - - `s01-parse` : most of the services parsers (ssh, nginx etc.) - - `s02-enrich` : enrichment that requires parsed events (ie. geoip-enrichment) or generic parsers that apply on parsed logs (ie. second stage http parser) - -The number and structure of stages can be altered by the user, the directory structure and their alphabetical order dictates in which order stages and parsers are processed. - -Every event starts in the first stage, and will move to the next stage once it has been successfully processed by a parser that has the `onsuccess` directive set to `next_stage`, and so on until it reaches the last stage, when it's going to start to be matched against scenarios. - -## Default stages - -- The preliminary stage (`s00-raw`) is mostly the one that will parse the structure of the log. This is where [syslog-logs](https://hub.crowdsec.net/author/crowdsecurity/configurations/syslog-logs) are parsed for example. Such a parser will parse the syslog header to detect the program source. - -- The main stage (`s01-parse`) is the one that will parse actual applications logs and output parsed data and static assigned values. There is one parser for each type of software. To parse the logs, regexp or GROK pattern are used. If the parser is configured to go to the [`next_stage`](/Crowdsec/v1/references/parsers/#onsuccess), then it will be process by the `enrichment` stage. - -- The enrichment (`s02-enrich`) stage is the one that will enrich the normalized log (we call it an event now that it is normalized) in order to get more information for the heuristic process. This stage can be composed of grok patterns and so on, but as well of plugins that can be writen by the community (geiop enrichment, rdns ...) for example [geoip-enrich](https://hub.crowdsec.net/author/crowdsecurity/configurations/geoip-enrich). - - -## Custom stage - -It is possible to write custom stage. If you want some specific parsing or enrichment to be done after the `s02-enrich` stage, it is possible by creating a new folder `s03-` (and so on). The configuration that will be created in this folder will process the logs configured to go to `next_stage` in the `s02-enrich` stage. \ No newline at end of file diff --git a/docs/v1.X/docs/user_guide/bouncer_machine_management.md b/docs/v1.X/docs/user_guide/bouncer_machine_management.md deleted file mode 100644 index 15bafcb8b..000000000 --- a/docs/v1.X/docs/user_guide/bouncer_machine_management.md +++ /dev/null @@ -1,119 +0,0 @@ -# Bouncers & Machines management - -Crowdsec is composed of different components that communicate via a local API. -To access this API, the various components (crowdsec agent, cscli and bouncers) need to be authenticated. - -!!! info - This documentation should be relevant mostly for administrators that would like to setup distributed architectures. Single machine setup users can likely skip this part. - - -There are two kind of access to the local api : - - - `machines` : it's a login/password authentication used by {{v1X.cli.name}} and {{v1X.crowdsec.name}}, this one allows to post, get and delete decisions and alerts. - - `bouncers` : it's a token authentication used by {{v1X.bouncers.htmlname}} to query the decisions, and only allows to perform get on decisions and alerts. - -## Bouncers authentication - -!!! warning - The `cscli bouncers` command interacts directly with the database (bouncers add and delete are not implemented in the API), and thus it must have the correct database configuration. - -```bash -$ sudo cscli bouncers list -``` - - -You can view the registered bouncers with `list`, as well as add or delete them : - -```bash -$ sudo cscli bouncers add mybouncersname -Api key for 'mybouncersname': - - 23........b5a0c - -Please keep this key since will not be able to retrive it! -$ sudo cscli bouncers delete mybouncersname -``` - -The API KEY must be kept and given to the {{v1X.bouncers.htmlname}}. - -
- cscli bouncers example -```bash -$ sudo cscli bouncers add mybouncersname -Api key for 'mybouncersname': - - 23........b5a0c - -Please keep this key since will not be able to retrive it! -$ sudo cscli bouncers list ------------------------------------------------------------------------------ - NAME IP ADDRESS VALID LAST API PULL TYPE VERSION ------------------------------------------------------------------------------ - mybouncersname ✔️ 2020-11-01T11:45:05+01:00 ------------------------------------------------------------------------------ -$ sudo cscli bouncers add jlkqweq -Api key for 'jlkqweq': - - a7........efdc9c - -Please keep this key since will not be able to retrive it! -$ sudo cscli bouncers delete mybouncersname -$ sudo cscli bouncers list ----------------------------------------------------------------------- - NAME IP ADDRESS VALID LAST API PULL TYPE VERSION ----------------------------------------------------------------------- - jlkqweq ✔️ 2020-11-01T11:49:32+01:00 ----------------------------------------------------------------------- -``` - -
- -## Machines authentication - -!!! warning - The `cscli machines` command interacts directly with the database (machines add and delete are not implemented in the API), and thus it must have the correct database configuration. - -```bash -$ cscli machines list -``` - -You can view the registered machines with `list`, as well as add or delete them : - -```bash -$ sudo cscli machines add mytestmachine -a -INFO[0004] Machine 'mytestmachine' created successfully -INFO[0004] API credentials dumped to '/etc/crowdsec/local_api_credentials.yaml' -$ sudo cscli machines delete 82929df7ee394b73b81252fe3b4e5020 -``` - - -
- cscli machines example - -```bash -$ sudo cscli machines list ----------------------------------------------------------------------------------------------------------------------------------- - NAME IP ADDRESS LAST UPDATE STATUS VERSION ----------------------------------------------------------------------------------------------------------------------------------- - 82929df7ee394b73b81252fe3b4e5020 127.0.0.1 2020-10-31T14:06:32+01:00 ✔️ v0.3.6-3d6ce33908409f2a830af6551a7f5e37f2a4728f ----------------------------------------------------------------------------------------------------------------------------------- -$ sudo cscli machines add -m mytestmachine -a -INFO[0004] Machine 'mytestmachine' created successfully -INFO[0004] API credentials dumped to '/etc/crowdsec/local_api_credentials.yaml' -$ sudo cscli machines list ----------------------------------------------------------------------------------------------------------------------------------- - NAME IP ADDRESS LAST UPDATE STATUS VERSION ----------------------------------------------------------------------------------------------------------------------------------- - 82929df7ee394b73b81252fe3b4e5020 127.0.0.1 2020-10-31T14:06:32+01:00 ✔️ v0.3.6-3d6ce33908409f2a830af6551a7f5e37f2a4728f - mytestmachine 127.0.0.1 2020-11-01T11:37:19+01:00 ✔️ v0.3.6-6a18458badf8ae5fed8d5f1bb96fc7a59c96163c ----------------------------------------------------------------------------------------------------------------------------------- -$ sudo cscli machines delete -m 82929df7ee394b73b81252fe3b4e5020 -$ sudo cscli machines list ---------------------------------------------------------------------------------------------------------- - NAME IP ADDRESS LAST UPDATE STATUS VERSION ---------------------------------------------------------------------------------------------------------- - mytestmachine 127.0.0.1 2020-11-01T11:37:19+01:00 ✔️ v0.3.6-6a18458badf8ae5fed8d5f1bb96fc7a59c96163c ---------------------------------------------------------------------------------------------------------- -``` - -
diff --git a/docs/v1.X/docs/user_guide/configurations_management/acquisition.md b/docs/v1.X/docs/user_guide/configurations_management/acquisition.md deleted file mode 100644 index e423cf845..000000000 --- a/docs/v1.X/docs/user_guide/configurations_management/acquisition.md +++ /dev/null @@ -1,89 +0,0 @@ -!!! info - - Please note that the `{{v1X.config.acquis_path}}` should be auto generated by the {{v1X.wizard.name}} in most case. - -The acquisition configuration specifies lists of logs that {{v1X.crowdsec.name}} will ingest and feed to parsers. -Acquisition provides two information about a given log : - - - its source (a path to a file, or a journalctl filter) - - its type, given in the form of a label - -The `type` label is crucial as it's later used in the process to determine which parser(s) can handle lines coming from this source. - -Acquisition can be found in `{{v1X.config.acquis_path}}`, for example : -
- Acquisition example -```yaml -filenames: - - /var/log/nginx/access*.log - - /var/log/nginx/error.log -labels: - type: nginx ---- -filenames: - - /var/log/auth.log -labels: - type: syslog ---- -journalctl_filter: - - "_SYSTEMD_UNIT=ssh.service" -labels: - type: syslog -``` -
- - -## Testing and viewing acquisition - -### At startup - -At startup, you will see the monitored files in `/var/log/crowdsec.log` : - -``` -... -INFO[23-11-2020 15:21:17] [file datasource] opening file '/tmp/test.log' -WARN[23-11-2020 15:21:17] [file datasource] no results for /tmp/ratata.log -INFO[23-11-2020 15:21:17] [journald datasource] Configured with filters : [--follow _SYSTEMD_UNIT=ssh.service] -... -``` - -### At runtime - -{{v1X.cli.name}} allows you to view {{v1X.crowdsec.name}} metrics info via the `metrics` command. -This allows you to see how many lines are coming from each source, and if they are parsed correctly. - -You can see those metrics with the following command: -``` -sudo {{v1X.cli.bin}} metrics -``` - - -
- {{v1X.cli.name}} metrics example - -```bash -$ sudo {{v1X.cli.bin}} metrics -... -... -INFO[0000] Acquisition Metrics: -+--------------------------------------+------------+--------------+----------------+------------------------+ -| SOURCE | LINES READ | LINES PARSED | LINES UNPARSED | LINES POURED TO BUCKET | -+--------------------------------------+------------+--------------+----------------+------------------------+ -| /tmp/test.log | 10 | 10 | - | 11 | -| journalctl-_SYSTEMD_UNIT=ssh.service | 36 | 12 | 24 | 17 | -+--------------------------------------+------------+--------------+----------------+------------------------+ -... -... -``` - -
- - -!!! info - - All these metrics are actually coming from {{v1X.crowdsec.name}}'s prometheus agent. See [prometheus](/Crowdsec/v1/observability/prometheus/) directly for more insights. - - -## Reference documentation - -[Link to acquisition reference documentation](/Crowdsec/v1/references/acquisition/) diff --git a/docs/v1.X/docs/user_guide/configurations_management/collections.md b/docs/v1.X/docs/user_guide/configurations_management/collections.md deleted file mode 100644 index 85ef81e48..000000000 --- a/docs/v1.X/docs/user_guide/configurations_management/collections.md +++ /dev/null @@ -1,138 +0,0 @@ - -{{v1X.hub.htmlname}} allows you to find needed collections. - -## Installing collections - -```bash -$ sudo cscli collections install crowdsecurity/whitelist-good-actors -``` - -
- {{v1X.cli.name}} collection install example - -```bash -$ sudo cscli collections install crowdsecurity/whitelist-good-actors -INFO[0000] crowdsecurity/seo-bots-whitelist : OK -INFO[0000] downloading data 'https://raw.githubusercontent.com/crowdsecurity/sec-lists/master/whitelists/benign_bots/search_engine_crawlers/rdns_seo_bots.txt' in '/var/lib/crowdsec/data/rdns_seo_bots.txt' -INFO[0001] downloading data 'https://raw.githubusercontent.com/crowdsecurity/sec-lists/master/whitelists/benign_bots/search_engine_crawlers/rnds_seo_bots.regex' in '/var/lib/crowdsec/data/rdns_seo_bots.regex' -INFO[0002] downloading data 'https://raw.githubusercontent.com/crowdsecurity/sec-lists/master/whitelists/benign_bots/search_engine_crawlers/ip_seo_bots.txt' in '/var/lib/crowdsec/data/ip_seo_bots.txt' -INFO[0002] crowdsecurity/cdn-whitelist : OK -INFO[0002] downloading data 'https://www.cloudflare.com/ips-v4' in '/var/lib/crowdsec/data/cloudflare_ips.txt' -INFO[0003] crowdsecurity/rdns : OK -INFO[0003] crowdsecurity/whitelist-good-actors : OK -INFO[0003] /etc/crowdsec/postoverflows/s01-whitelist doesn't exist, create -INFO[0003] Enabled postoverflows : crowdsecurity/seo-bots-whitelist -INFO[0003] Enabled postoverflows : crowdsecurity/cdn-whitelist -INFO[0003] /etc/crowdsec/postoverflows/s00-enrich doesn't exist, create -INFO[0003] Enabled postoverflows : crowdsecurity/rdns -INFO[0003] Enabled collections : crowdsecurity/whitelist-good-actors -INFO[0003] Enabled crowdsecurity/whitelist-good-actors -INFO[0003] Run 'systemctl reload crowdsec' for the new configuration to be effective. -$ systemctl reload crowdsec -``` -
- - -## Listing installed collections - -```bash -$ sudo {{v1X.cli.bin}} collections list -``` - -
- cscli collections list example - -```bash -$ sudo cscli collections list -------------------------------------------------------------------------------------------------------------- - NAME 📦 STATUS VERSION LOCAL PATH -------------------------------------------------------------------------------------------------------------- - crowdsecurity/nginx ✔️ enabled 0.1 /etc/crowdsec/collections/nginx.yaml - crowdsecurity/base-http-scenarios ✔️ enabled 0.1 /etc/crowdsec/collections/base-http-scenarios.yaml - crowdsecurity/sshd ✔️ enabled 0.1 /etc/crowdsec/collections/sshd.yaml - crowdsecurity/linux ✔️ enabled 0.2 /etc/crowdsec/collections/linux.yaml -------------------------------------------------------------------------------------------------------------- -``` - -
- -## Upgrading installed collections - -```bash -$ sudo {{v1X.cli.bin}} hub update -$ sudo {{v1X.cli.bin}} collections upgrade crowdsecurity/sshd -``` - -Collection upgrade allows you to upgrade an existing collection (and its items) to the latest version. - - -
- cscli collections upgrade example - -```bash -$ sudo cscli collections upgrade crowdsecurity/sshd -INFO[0000] crowdsecurity/sshd : up-to-date -WARN[0000] crowdsecurity/sshd-logs : overwrite -WARN[0000] crowdsecurity/ssh-bf : overwrite -WARN[0000] crowdsecurity/sshd : overwrite -INFO[0000] 📦 crowdsecurity/sshd : updated -INFO[0000] Upgraded 1 items -INFO[0000] Run 'systemctl reload crowdsec' for the new configuration to be effective. -$ systemctl reload crowdsec - -``` - -
- -## Monitoring collections - -```bash -$ sudo cscli collections inspect crowdsecurity/sshd -``` - -Collections inspect will give you detailed information about a given collection, including versioning information *and* runtime metrics (fetched from prometheus). - -
- cscli collections inspect example - -```bash -$ sudo cscli collections inspect crowdsecurity/sshd -type: collections -name: crowdsecurity/sshd -filename: sshd.yaml -description: 'sshd support : parser and brute-force detection' -author: crowdsecurity -belongs_to_collections: -- crowdsecurity/linux -- crowdsecurity/linux -remote_path: collections/crowdsecurity/sshd.yaml -version: "0.1" -local_path: /etc/crowdsec/collections/sshd.yaml -localversion: "0.1" -localhash: 21159aeb87529efcf1a5033f720413d5321a6451bab679a999f7f01a7aa972b3 -installed: true -downloaded: true -uptodate: true -tainted: false -local: false -parsers: -- crowdsecurity/sshd-logs -scenarios: -- crowdsecurity/ssh-bf - -Current metrics : - - - (Scenario) crowdsecurity/ssh-bf: -+---------------+-----------+--------------+--------+---------+ -| CURRENT COUNT | OVERFLOWS | INSTANCIATED | POURED | EXPIRED | -+---------------+-----------+--------------+--------+---------+ -| 0 | 1 | 2 | 10 | 1 | -+---------------+-----------+--------------+--------+---------+ - -``` - -
- -## Reference documentation - -[Link to collections reference documentation](/Crowdsec/v1/references/collections/) diff --git a/docs/v1.X/docs/user_guide/configurations_management/enrichers.md b/docs/v1.X/docs/user_guide/configurations_management/enrichers.md deleted file mode 100644 index 4da053e47..000000000 --- a/docs/v1.X/docs/user_guide/configurations_management/enrichers.md +++ /dev/null @@ -1,26 +0,0 @@ -Enrichers are basically {{v1X.parsers.htmlname}} that can rely on external methods to provide extra contextual information to the event. The enrichers are usually in the `s02-enrich` {{v1X.stage.htmlname}} (after most of the parsing happened). - -Enrichers functions should all accept a string as a parameter, and return an associative string array, that will be automatically merged into the `Enriched` map of the {{v1X.event.htmlname}}. - -!!! warning - At the time of writing, enrichers plugin mechanism implementation is still ongoing (read: the list of available enrichment methods is currently hardcoded). - - -As an example let's look into the geoip-enrich parser/enricher : - -It relies on [the geolite2 data created by maxmind](https://www.maxmind.com) and the [geoip2 golang module](https://github.com/oschwald/geoip2-golang) to provide the actual data. - - -It exposes three methods : `GeoIpCity` `GeoIpASN` and `IpToRange` that are used by the `crowdsecurity/geoip-enrich`. -Enrichers can be installed as any other parsers with the following command: - -``` -sudo {{v1X.cli.bin}} parsers install crowdsecurity/geoip-enrich -``` - -Take a tour at the {{v1X.hub.htmlname}} to find them ! - -## Reference documentation - -[Link to enrichers reference documentation](/Crowdsec/v1/references/enrichers/) - diff --git a/docs/v1.X/docs/user_guide/configurations_management/parsers.md b/docs/v1.X/docs/user_guide/configurations_management/parsers.md deleted file mode 100644 index 83a2a6e00..000000000 --- a/docs/v1.X/docs/user_guide/configurations_management/parsers.md +++ /dev/null @@ -1,127 +0,0 @@ -{{v1X.hub.htmlname}} allows you to find needed parsers. - -## Installing parsers - -```bash -$ sudo cscli parsers install crowdsecurity/sshd-logs -``` - -
- cscli parsers install example - -```bash -$ sudo cscli parsers install crowdsecurity/iptables-logs -INFO[0000] crowdsecurity/iptables-logs : OK -INFO[0000] Enabled parsers : crowdsecurity/iptables-logs -INFO[0000] Enabled crowdsecurity/iptables-logs -INFO[0000] Run 'systemctl reload crowdsec' for the new configuration to be effective. -``` -
- -## Listing installed parsers - -```bash -sudo cscli parsers list -``` - -{{v1X.parsers.Htmlname}} are yaml files in `{{v1X.config.crowdsec_dir}}parsers//parser.yaml`. - - -
- cscli parsers list example - -```bash -$ sudo cscli parsers list --------------------------------------------------------------------------------------------------------------- - NAME 📦 STATUS VERSION LOCAL PATH --------------------------------------------------------------------------------------------------------------- - crowdsecurity/whitelists ✔️ enabled 0.1 /etc/crowdsec/parsers/s02-enrich/whitelists.yaml - crowdsecurity/dateparse-enrich ✔️ enabled 0.1 /etc/crowdsec/parsers/s02-enrich/dateparse-enrich.yaml - crowdsecurity/iptables-logs ✔️ enabled 0.1 /etc/crowdsec/parsers/s01-parse/iptables-logs.yaml - crowdsecurity/syslog-logs ✔️ enabled 0.1 /etc/crowdsec/parsers/s00-raw/syslog-logs.yaml - crowdsecurity/sshd-logs ✔️ enabled 0.1 /etc/crowdsec/parsers/s01-parse/sshd-logs.yaml - crowdsecurity/geoip-enrich ✔️ enabled 0.2 /etc/crowdsec/parsers/s02-enrich/geoip-enrich.yaml - crowdsecurity/http-logs ✔️ enabled 0.2 /etc/crowdsec/parsers/s02-enrich/http-logs.yaml - crowdsecurity/nginx-logs ✔️ enabled 0.1 /etc/crowdsec/parsers/s01-parse/nginx-logs.yaml --------------------------------------------------------------------------------------------------------------- - -``` - -
- - -## Upgrading installed parsers - -```bash -$ sudo {{v1X.cli.bin}} parsers upgrade crowdsecurity/sshd-logs -``` - -Parsers upgrade allows you to upgrade an existing parser to the latest version. - -
- cscli parsers upgrade example - -```bash -$ sudo cscli parsers upgrade crowdsecurity/sshd-logs -INFO[0000] crowdsecurity/sshd : up-to-date -WARN[0000] crowdsecurity/sshd-logs : overwrite -WARN[0000] crowdsecurity/ssh-bf : overwrite -WARN[0000] crowdsecurity/sshd : overwrite -INFO[0000] 📦 crowdsecurity/sshd : updated -INFO[0000] Upgraded 1 items -INFO[0000] Run 'systemctl reload crowdsec' for the new configuration to be effective. - -``` - -
- -## Monitoring parsers - -```bash -$ sudo cscli parsers inspect crowdsecurity/sshd-logs -``` - -Parsers inspect will give you detailed information about a given parser, including versioning information *and* runtime metrics (fetched from prometheus). - - -
- cscli parsers inspect example - -```bash -$ sudo cscli parsers inspect crowdsecurity/sshd-logs -type: parsers -stage: s01-parse -name: crowdsecurity/sshd-logs -filename: sshd-logs.yaml -description: Parse openSSH logs -author: crowdsecurity -belongs_to_collections: -- crowdsecurity/sshd -remote_path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml -version: "0.1" -local_path: /etc/crowdsec/parsers/s01-parse/sshd-logs.yaml -localversion: "0.1" -localhash: ecd40cb8cd95e2bad398824ab67b479362cdbf0e1598b8833e2f537ae3ce2f93 -installed: true -downloaded: true -uptodate: true -tainted: false -local: false - -Current metrics : - - - (Parser) crowdsecurity/sshd-logs: -+-------------------+-------+--------+----------+ -| PARSERS | HITS | PARSED | UNPARSED | -+-------------------+-------+--------+----------+ -| /var/log/auth.log | 94138 | 42404 | 51734 | -+-------------------+-------+--------+----------+ - -``` - -
- -## Reference documentation - -[Link to parsers reference documentation](/Crowdsec/v1/references/parsers/) - diff --git a/docs/v1.X/docs/user_guide/configurations_management/scenarios.md b/docs/v1.X/docs/user_guide/configurations_management/scenarios.md deleted file mode 100644 index e18e1a067..000000000 --- a/docs/v1.X/docs/user_guide/configurations_management/scenarios.md +++ /dev/null @@ -1,127 +0,0 @@ -{{v1X.hub.htmlname}} allows you to find needed scenarios. - -## Installing scenarios - -```bash -$ sudo cscli scenarios install crowdsecurity/http-bf-wordpress_bf -``` - -
- cscli scenarios install example - -```bash -$ sudo cscli scenarios install crowdsecurity/http-bf-wordpress_bf -INFO[0000] crowdsecurity/http-bf-wordpress_bf : OK -INFO[0000] Enabled scenarios : crowdsecurity/http-bf-wordpress_bf -INFO[0000] Enabled crowdsecurity/http-bf-wordpress_bf -INFO[0000] Run 'systemctl reload crowdsec' for the new configuration to be effective. -$ systemctl reload crowdsec -``` - -
- - -## Listing installed scenarios - -```bash -sudo cscli scenarios list -``` - -{{v1X.scenarios.Htmlname}} are yaml files in `{{v1X.config.crowdsec_dir}}scenarios/`. - - -
- cscli scenarios list example - -```bash -$ sudo cscli scenarios list ---------------------------------------------------------------------------------------------------------------------------- - NAME 📦 STATUS VERSION LOCAL PATH ---------------------------------------------------------------------------------------------------------------------------- - crowdsecurity/ssh-bf ✔️ enabled 0.1 /etc/crowdsec/scenarios/ssh-bf.yaml - crowdsecurity/http-bf-wordpress_bf ✔️ enabled 0.1 /etc/crowdsec/scenarios/http-bf-wordpress_bf.yaml - crowdsecurity/http-crawl-non_statics ✔️ enabled 0.2 /etc/crowdsec/scenarios/http-crawl-non_statics.yaml - crowdsecurity/http-probing ✔️ enabled 0.1 /etc/crowdsec/scenarios/http-probing.yaml - crowdsecurity/http-sensitive-files ✔️ enabled 0.2 /etc/crowdsec/scenarios/http-sensitive-files.yaml - crowdsecurity/http-bad-user-agent ✔️ enabled 0.2 /etc/crowdsec/scenarios/http-bad-user-agent.yaml - crowdsecurity/http-path-traversal-probing ✔️ enabled 0.2 /etc/crowdsec/scenarios/http-path-traversal-probing.yaml - crowdsecurity/http-sqli-probing ✔️ enabled 0.2 /etc/crowdsec/scenarios/http-sqli-probing.yaml - crowdsecurity/http-backdoors-attempts ✔️ enabled 0.2 /etc/crowdsec/scenarios/http-backdoors-attempts.yaml - crowdsecurity/http-xss-probing ✔️ enabled 0.2 /etc/crowdsec/scenarios/http-xss-probing.yaml ---------------------------------------------------------------------------------------------------------------------------- - -``` - -
- - -## Upgrading installed scenarios - -```bash -$ sudo cscli scenarios upgrade crowdsecurity/ssh-bf -``` - -Scenarios upgrade allows you to upgrade an existing scenario to the latest version. - -
- cscli scenarios upgrade example - -```bash -$ sudo cscli scenarios upgrade crowdsecurity/ssh-bf -INFO[0000] crowdsecurity/ssh-bf : up-to-date -WARN[0000] crowdsecurity/ssh-bf : overwrite -INFO[0000] 📦 crowdsecurity/ssh-bf : updated -INFO[0000] Upgraded 1 items -INFO[0000] Run 'systemctl reload crowdsec' for the new configuration to be effective. -``` - -
- -## Monitoring scenarios - -```bash -$ sudo cscli scenarios inspect crowdsecurity/ssh-bf -``` - -Scenarios inspect will give you detailed information about a given scenario, including versioning information *and* runtime metrics (fetched from prometheus). - -
- cscli scenarios inspect example - -```bash -$ sudo cscli scenarios inspect crowdsecurity/ssh-bf -type: scenarios -name: crowdsecurity/ssh-bf -filename: ssh-bf.yaml -description: Detect ssh bruteforce -author: crowdsecurity -references: -- http://wikipedia.com/ssh-bf-is-bad -belongs_to_collections: -- crowdsecurity/sshd -remote_path: scenarios/crowdsecurity/ssh-bf.yaml -version: "0.1" -local_path: /etc/crowdsec/scenarios/ssh-bf.yaml -localversion: "0.1" -localhash: 4441dcff07020f6690d998b7101e642359ba405c2abb83565bbbdcee36de280f -installed: true -downloaded: true -uptodate: true -tainted: false -local: false - -Current metrics : - - - (Scenario) crowdsecurity/ssh-bf: -+---------------+-----------+--------------+--------+---------+ -| CURRENT COUNT | OVERFLOWS | INSTANCIATED | POURED | EXPIRED | -+---------------+-----------+--------------+--------+---------+ -| 14 | 5700 | 7987 | 42572 | 2273 | -+---------------+-----------+--------------+--------+---------+ -``` - -
- -## Reference documentation - -[Link to scenarios reference documentation](/Crowdsec/v1/references/scenarios/) diff --git a/docs/v1.X/docs/user_guide/cscli.md b/docs/v1.X/docs/user_guide/cscli.md deleted file mode 100644 index 0d5294474..000000000 --- a/docs/v1.X/docs/user_guide/cscli.md +++ /dev/null @@ -1,22 +0,0 @@ -# Overview - -`{{v1X.cli.name}}` is the utility that will help you to manage {{v1X.crowdsec.name}}. This tool has the following functionalities: - - - manage [decisions](/Crowdsec/v1/cscli/cscli_decisions/) and [alerts](/Crowdsec/v1/cscli/cscli_alerts/) : This is how you monitor ongoing remediation and detections - - manage configurations such as [collections](/Crowdsec/v1/cscli/cscli_collections/), [parsers](/Crowdsec/v1/cscli/cscli_parsers/), [scenarios](/Crowdsec/v1/cscli/cscli_scenarios/) : This is how you install/update {{v1X.crowdsec.htmname}}'s detection capabilities and manage whitelists - - interact with the [hub](/Crowdsec/v1/cscli/cscli_hub/) to find new configurations or update existing ones - - manage local api (LAPI) [bouncers](/Crowdsec/v1/cscli/cscli_bouncers/) and [machines](/Crowdsec/v1/cscli/cscli_machines/) : This allows you to manage LAPI credentials, this is how you make {{v1X.crowdsec.htmname}} and bouncers comunicate - - observe crowdsec via [metrics](/Crowdsec/v1/cscli/cscli_metrics/) or the [dashboard](/Crowdsec/v1/cscli/cscli_dashboard/) : This is how you gain real-time observability - - manage [simulation](/Crowdsec/v1/cscli/cscli_simulation/) configurations, allowing you to disable/modify remediation triggered by specific scenarios - - -Take a look at the [dedicated documentation](/Crowdsec/v1/cscli/cscli) - -!!! tips - You can enable `cscli` auto completion in `bash` or `zsh`. - - You can find `cscli completion` documentation [here](/Crowdsec/v1/cscli/cscli_completion/). - -# Configuration - -`{{v1X.cli.name}}` shares the configuration file of {{v1X.crowdsec.name}}, usually in `/etc/crowdsec/config.yaml` diff --git a/docs/v1.X/docs/user_guide/database.md b/docs/v1.X/docs/user_guide/database.md deleted file mode 100644 index ade460fab..000000000 --- a/docs/v1.X/docs/user_guide/database.md +++ /dev/null @@ -1,47 +0,0 @@ -# Databases - -By default, the crowdsec Local API use `SQLite` as backend storage. But in case you expect a lot of traffic on your local API, you should use `MySQL` or `PostgreSQL`. - -For `SQLite`, there is nothing to do to make it work with crowdsec. But for `MySQL` and `PostgreSQL` , you have to create the database and the user. - -Please refer to [ent.](https://entgo.io/) [supported database](https://entgo.io/docs/dialects/). At the time of writting : - - - MySQL `5.6.35`, `5.7.26` and `8` - - MariaDB `10.2` and latest - - PostgreSQL `10`, `11` and `12` - - SQLite - - Gremlin - - -!!! warning - When switching an existing instance of crowdsec to a new database backend, you need to register your machine(s) (ie. `cscli machines add -a`) and bouncer(s) to the new database, as data is not migrated. - - -## MySQL - -Connect to your `MySQL` server and run the following commands: - -``` -mysql> CREATE DATABASE crowdsec; -mysql> CREATE USER 'crowdsec'@'%' IDENTIFIED BY ''; -mysql> GRANT ALL PRIVILEGES ON crowdsec.* TO 'crowdsec'@'%'; -mysql> FLUSH PRIVILEGES; -``` - -Then edit `{{v1X.config.crowdsec_config_file}}` to update the [`db_config`](/Crowdsec/v1/references/database/#db_config) part. - -You can now start/restart crowdsec. - -## PostgreSQL - -Connect to your `PostgreSQL` server and run the following commands: - -``` -postgres=# CREATE DATABASE crowdsec; -postgres=# CREATE USER crowdsec WITH PASSWORD ''; -postgres=# GRANT ALL PRIVILEGES ON DATABASE crowdsec TO crowdsec; -``` - -Then edit `{{v1X.config.crowdsec_config_file}}` to update the [`db_config`](/Crowdsec/v1/references/database/#db_config) part. - -You can now start/restart crowdsec. \ No newline at end of file diff --git a/docs/v1.X/docs/user_guide/debugging_configs.md b/docs/v1.X/docs/user_guide/debugging_configs.md deleted file mode 100644 index 496e6b0f8..000000000 --- a/docs/v1.X/docs/user_guide/debugging_configs.md +++ /dev/null @@ -1,124 +0,0 @@ -# Debugging Scenarios and Parsers - -## General Advice - -When trying to debug a parser or a scenario : - - - Work on "cold logs" (with the `-file` and `-type` options) rather than live ones - - Use the `/etc/crowdsec/user.yaml` configuration files to have logs on stdout - -## Using user-mode configuration - -```bash -crowdsec -c /etc/crowdsec/user.yaml -file mylogs.log.gz -type syslog -INFO[05-08-2020 16:15:47] Crowdsec v0.3.0-rc3-7525f11975a0107746213862dc41c69e00122ac7 -INFO[05-08-2020 16:15:47] Loading grok library -... -WARN[05-08-2020 16:16:12] 182.x.x.x triggered a 4h0m0s ip ban remediation for [crowdsecurity/http-probing] bucket_id=misty-moon event_time="2019-01-01 22:58:32 +0100 CET" scenario=crowdsecurity/http-probing source_ip=182.x.x.x -... -``` - - - `/etc/crowdsec/user.yaml` disables demonization and push logs to stdout/stderr - - `-type` must respect expected log type (ie. `nginx` `syslog` etc.) - - `-file` must point to a flat file or a gzip file - -When processing logs like this, {{v1X.crowdsec.name}} runs in "time machine" mode, and relies on the timestamps *in* the logs to evaluate scenarios. You will most likely need the `crowdsecurity/dateparse-enrich` parser for this. - - -## Testing configurations on live system - -If you're playing around with parser/scenarios on a live system, you can use the `-t` (lint) option of {{v1X.crowdsec.Name}} to check your configurations validity before restarting/reloading services : - -```bash -$ emacs /etc/crowdsec/scenarios/ssh-bf.yaml -... -$ crowdsec -c /etc/crowdsec/user.yaml -t -INFO[06-08-2020 13:36:04] Crowdsec v0.3.0-rc3-4cffef42732944d4b81b3e62a03d4040ad74f185 -... -ERRO[06-08-2020 13:36:05] Bad yaml in /etc/crowdsec/scenarios/ssh-bf.yaml : yaml: unmarshal errors: - line 2: field typex not found in type leakybucket.BucketFactory -FATA[06-08-2020 13:36:05] Failed to load scenarios: Scenario loading failed : bad yaml in /etc/crowdsec/scenarios/ssh-bf.yaml : yaml: unmarshal errors: - line 2: field typex not found in type leakybucket.BucketFactory -``` - -Using this, you won't have to kill your running service before you know the scenarios/parsers are at least syntactically correct. - - -## Using debug - -Both scenarios and parsers support a `debug: true|false` option which produce useful debug. - -
- Debug parsing output (expand) -```bash -DEBU[05-08-2020 15:25:36] eval(evt.Parsed.program == 'nginx') = TRUE id=lively-smoke name=crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] eval variables: id=lively-smoke name=crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] evt.Parsed.program = 'nginx' id=lively-smoke name=crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] Event entering node id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] + Grok 'NGINXACCESS' returned 10 entries to merge in Parsed id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] .Parsed['request'] = '/data.php' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] .Parsed['http_user_agent'] = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] .Parsed['http_referer'] = '-' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] .Parsed['remote_addr'] = '123.x.x.x' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] .Parsed['remote_user'] = '-' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] .Parsed['time_local'] = '01/Jan/2019:01:39:06 +0100' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] .Parsed['method'] = 'POST' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] .Parsed['body_bytes_sent'] = '162' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] .Parsed['http_version'] = '1.1' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] .Parsed['status'] = '404' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] .Meta[log_type] = 'http_access-log' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] evt.StrTime = '01/Jan/2019:01:39:06 +0100' id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] Event leaving node : ok id=icy-dew name=child-crowdsecurity/nginx-logs stage=s01-parse -DEBU[05-08-2020 15:25:36] child is success, OnSuccess=next_stage, skip id=lively-smoke name=crowdsecurity/nginx-logs stage=s01-parse -``` -
- - -
- Debug scenario output (expand) -```bash -DEBU[05-08-2020 16:02:26] eval(evt.Meta.service == 'http' && evt.Meta.http_status in ['404', '403', '400'] && evt.Parsed.static_ressource == 'false') = TRUE cfg=black-wave file=config/scenarios/http-probing.yaml name=crowdsecurity/http-probing -DEBU[05-08-2020 16:02:26] eval variables: cfg=black-wave file=config/scenarios/http-probing.yaml name=crowdsecurity/http-probing -DEBU[05-08-2020 16:02:26] evt.Meta.service = 'http' cfg=black-wave file=config/scenarios/http-probing.yaml name=crowdsecurity/http-probing -DEBU[05-08-2020 16:02:26] evt.Meta.http_status = '404' cfg=black-wave file=config/scenarios/http-probing.yaml name=crowdsecurity/http-probing -DEBU[05-08-2020 16:02:26] evt.Parsed.static_ressource = 'false' cfg=black-wave file=config/scenarios/http-probing.yaml name=crowdsecurity/http-probing -``` -
- - -# Test environments - -From a [{{v1X.crowdsec.name}} release archive]({{v1X.crowdsec.download_url}}), you can deploy a test (non-root) environment that is very suitable to write/debug/test parsers and scenarios. Environment is deployed using `./test_env.sh` script from tgz directory, and creates a test environment in `./tests` : - -```bash -$ cd crowdsec-v0.3.0/ -$ ./test_env.sh -... -[08/05/2020:04:19:18 PM][INFO] Setting up configurations -INFO[0000] Wrote new 75065 bytes index to config/crowdsec-cli/.index.json -INFO[0000] crowdsecurity/syslog-logs : OK -INFO[0000] crowdsecurity/geoip-enrich : OK -... -INFO[0007] Enabled collections : crowdsecurity/linux -INFO[0007] Enabled crowdsecurity/linux -[08/05/2020:04:19:26 PM][INFO] Environment is ready in /home/bui/github/crowdsec/crowdsec/crowdsec-v0.3.0/tests -$ cd tests -$ ./cscli -c dev.yaml list -... -INFO[0000] PARSERS: -------------------------------------------------------------------------------------------------------- - NAME 📦 STATUS VERSION LOCAL PATH -------------------------------------------------------------------------------------------------------- - crowdsecurity/geoip-enrich ✔️ enabled 0.2 config/parsers/s02-enrich/geoip-enrich.yaml - crowdsecurity/syslog-logs ✔️ enabled 0.3 config/parsers/s00-raw/syslog-logs.yaml - crowdsecurity/sshd-logs ✔️ enabled 0.2 config/parsers/s01-parse/sshd-logs.yaml - crowdsecurity/dateparse-enrich ✔️ enabled 0.1 config/parsers/s02-enrich/dateparse-enrich.yaml -------------------------------------------------------------------------------------------------------- -... -$ ./crowdsec -c dev.yaml -file sshd.log -type syslog -INFO[05-08-2020 16:23:32] Crowdsec v0.3.0-rc3-7525f11975a0107746213862dc41c69e00122ac7 -INFO[05-08-2020 16:23:32] Loading grok library -... -``` - - diff --git a/docs/v1.X/docs/user_guide/decision_management.md b/docs/v1.X/docs/user_guide/decision_management.md deleted file mode 100644 index cf56526b4..000000000 --- a/docs/v1.X/docs/user_guide/decision_management.md +++ /dev/null @@ -1,103 +0,0 @@ -!!! info - - Please see your local `sudo {{v1X.cli.bin}} help decisions` for up-to-date documentation. - -## List active decisions - -```bash -sudo {{v1X.cli.bin}} decisions list -``` - -
- example -```bash -$ sudo cscli decisions list -+-----+-----------+-------------+------------------------------------+--------+---------+----+--------+--------------------+----------+ -| ID | SOURCE | SCOPE:VALUE | REASON | ACTION | COUNTRY | AS | EVENTS | EXPIRATION | ALERT ID | -+-----+-----------+-------------+------------------------------------+--------+---------+----+--------+--------------------+----------+ -| 802 | cscli | Ip:1.2.3.5 | manual 'ban' from | ban | | | 1 | 3h50m58.10039043s | 802 | -| | | | 'b76cc7b1bbdc489e93909d2043031de8' | | | | | | | -| 801 | crowdsec | Ip:1.2.3.4 | crowdsecurity/ssh-bf | ban | | | 6 | 3h59m45.100387557s | 801 | -+-----+-----------+-------------+------------------------------------+--------+---------+----+--------+--------------------+----------+ -``` - -
- - `SOURCE` : the source of the decisions: - - `crowdsec` : decision from crowdsec agent - - `cscli` : decision from `cscli` (manual decision) - - `api` : decision from crowdsec API - - `SCOPE:VALUE` is the target of the decisions : - - "scope" : the scope of the decisions (`ip`, `range`, `user` ...) - - "value" : the value to apply on the decisions (, , ...) - - `REASON` is the scenario that was triggered (or human-supplied reason) - - `ACTION` is the type of the decision (`ban`, `captcha` ...) - - `COUNTRY` and `AS` are provided by GeoIP enrichment if present - - `EVENTS` number of event that triggered this decison - - `EXPIRATION` is the time left on remediation - - `ALERT ID` is the ID of the corresponding alert - - -Check [command usage](/Crowdsec/v1/cscli/cscli_decisions_list/) for additional filtering and output control flags. - - -## Add a decision - * default `duration`: `4h` - * default `type` : `ban` - - -> Add a decision (ban) on IP `1.2.3.4` for 24 hours, with reason 'web bruteforce' - -```bash -sudo {{v1X.cli.bin}} decisions add --ip 1.2.3.4 --duration 24h --reason "web bruteforce" -``` - -> Add a decision (ban) on range `1.2.3.0/24` for 4 hours, with reason 'web bruteforce' - -```bash -sudo {{v1X.cli.bin}} decisions add --range 1.2.3.0/24 --reason "web bruteforce" -``` - - -> Add a decision (captcha) on ip `1.2.3.4` for 4hours (default duration), with reason 'web bruteforce' - -```bash -sudo {{v1X.cli.bin}} decisions add --ip 1.2.3.4 --reason "web bruteforce" --type captcha -``` - - - -## Delete a decision - -> delete the decision on IP `1.2.3.4` - -```bash -sudo {{v1X.cli.bin}} decisions delete --ip 1.2.3.4 -``` - -> delete the decision on range 1.2.3.0/24 - -```bash -sudo {{v1X.cli.bin}} decisions delete --range 1.2.3.0/24 -``` - -!!! warning - Please note that `cscli decisions list` will show you only the latest alert per given ip/scope. - However, several decisions targeting the same IP can exist. If you want to be sure to clear all decisions for a given ip/scope, use `cscli decisions delete -i x.x.x.x` - - - - - -## Delete all existing bans - -> Flush all the existing bans - -```bash -sudo {{v1X.cli.bin}} decisions delete --all -``` - -!!! warning - This will as well remove any existing ban - - - diff --git a/docs/v1.X/docs/user_guide/forensic_mode.md b/docs/v1.X/docs/user_guide/forensic_mode.md deleted file mode 100644 index 555e85389..000000000 --- a/docs/v1.X/docs/user_guide/forensic_mode.md +++ /dev/null @@ -1,195 +0,0 @@ -## Forensic mode - -While {{v1X.crowdsec.name}} can be used to monitor "live" logs, it can as well be used on cold logs. -It is a *great* way to test scenario, detect false positives & false negatives or simply generate reporting on a past time period. - -When doing so, {{v1X.crowdsec.name}} will read the logs, extract timestamps from those, so that the scenarios/buckets can be evaluated with the log's timestamps. The resulting overflows will be pushed to the API as any other alert, but the timestamp will be the timestamps of the logs, properly allowing you to view the alerts in their original time line. - - -you can run : - -```bash -sudo crowdsec -c /etc/crowdsec/user.yaml -file /path/to/your/log/file.log -type log_file_type -``` - -Where `-file` points to the log file you want to process, and the `-type` is similar to what you would put in your acquisition's label field, for example : - -```bash -sudo crowdsec -c /etc/crowdsec/user.yaml -file /var/log/nginx/2019.log -type nginx -sudo crowdsec -c /etc/crowdsec/user.yaml -file /var/log/sshd-2019.log -type syslog -sudo crowdsec -c /etc/crowdsec/user.yaml -jfilter "_SYSTEMD_UNIT=ssh.service --since yesterday" -type syslog -``` - -When running crowdsec in forensic mode, the alerts will be displayed to stdout, and as well pushed to database : - -```bash -$ sudo crowdsec -c /etc/crowdsec/user.yaml -file /var/log/nginx/nginx-2019.log.1 -type nginx -... -INFO[13-11-2020 13:05:23] Ip 123.206.50.249 performed 'crowdsecurity/http-probing' (11 events over 6s) at 2019-01-01 01:37:32 +0100 CET -INFO[13-11-2020 13:05:23] Ip 123.206.50.249 performed 'crowdsecurity/http-backdoors-attempts' (2 events over 1s) at 2019-01-01 01:37:33 +0100 CET -INFO[13-11-2020 13:05:24] (14baeedafc1e44c08b806fc0c1cd92c4/crowdsec) crowdsecurity/http-probing by ip 123.206.50.249 (CN) : 1h ban on Ip 123.206.50.249 -INFO[13-11-2020 13:05:24] (14baeedafc1e44c08b806fc0c1cd92c4/crowdsec) crowdsecurity/http-backdoors-attempts by ip 123.206.50.249 (CN) : 1h ban on Ip 123.206.50.249 -... -``` - -And as these alerts are as well pushed to database, it mean you can view them in metabase, or using cscli ! - - -!!! warning - To work in forensic mode, crowdsec-agent relies on [crowdsecurity/dateparse-enrich](https://hub.crowdsec.net/author/crowdsecurity/configurations/dateparse-enrich) to parse date formats. See dedicated hub page for supported formats. - - -## Injecting alerts into existing database - -If you already have a running crowdsec/Local API running and want to inject events into existing database, you can run crowdsec directly : - -```bash -sudo crowdsec -file ~/logs/nginx/access.log -type nginx --no-api -``` - -Crowdsec will process `~/logs/nginx/access.log` and push alerts to the Local API configured in your default configuration file (`/etc/crowdsec/config.yaml`, see `api.client.credentials_path`) - -## Injection alerts into new database - no local instance running - -If you don't have a service currently running, you can run crowdsec directly : - -```bash -sudo crowdsec -file ~/logs/nginx/access.log -type nginx -``` - -Crowdsec will start a Local API and process `~/logs/nginx/access.log`. - - -## Injection alerts into new database - while local instance is running - -If you have a local instance running and you don't want to pollute your existing database, we are going to configure a separate instance of Local API & database. - -Let's copy the existing configuration to edit it : - -```bash -$ sudo cp /etc/crowdsec/config.yaml ./forensic.yaml -$ emacs ./forensic.yaml -``` - -In our file, let's edit the local API & database config to ensure we're not going to pollute existing data : - -```bash -$ emacs ./forensic.yaml -... -db_config: - type: sqlite - # we edit the db_path to point to a different SQLite database - db_path: /var/lib/crowdsec/data/crowdsec_alt.db - # let's comment out the auto-flush (database garbage collection) - #flush: - # max_items: 5000 - # max_age: 7d - -... -api: - client: - # we edit credentials_path to point to a local file - credentials_path: /tmp/local_api_credentials.yaml - server: - # we edit the listen_uri so that it doesn't try to listen on the same port as the existing Local API - listen_uri: 127.0.0.1:8081 -``` - -With the following edits, we ensure that : - - - The SQLite database path will be different : it avoids conflicts if you already had one running locally - - Edit the local api credentials path : we're going to register our machine to the ephemeral Local API - - Edit the listen uri of the local api : it avoids conflicts for listen port in case you already had one running locally - - Comment out the `flush` section : it ensure the database garbage collector won't run and delete your old events you're injecting ;) - - - -Let's create the new database and register a machine to it : - -```bash -$ touch /tmp/local_api_credentials.yaml -$ cscli -c forensic.yaml machines add --auto -INFO[0000] Machine '...' created successfully -INFO[0000] API credentials dumped to '/tmp/local_api_credentials.yaml' -$ cat /tmp/local_api_credentials.yaml -url: http://127.0.0.1:8081 -login: ... -password: ... -``` - -Now we can start the new Local API and crowdsec : - -```bash -$ crowdsec -c ./forensic.yaml -file ~/github/crowdsec/OLDS/LOGS/nginx/10k_ACCESS_LOGS.log -type nginx -... -INFO[15-11-2020 10:09:20] Ip x.x.x.x performed 'crowdsecurity/http-bad-user-agent' (2 events over 0s) at 2017-10-21 13:58:38 +0200 CEST -INFO[15-11-2020 10:09:20] Ip y.y.y.y performed 'crowdsecurity/http-probing' (11 events over 0s) at 2017-10-23 12:00:34 +0200 CEST -... -``` - -And we can even fire a dedicated dashboard to view the data : - -```bash -$ cscli -c forensic.yaml dashboard setup -INFO[0000] /var/lib/crowdsec/data/metabase.db exists, skip. -INFO[0000] Pulling docker image metabase/metabase:v0.37.0.2 -... -INFO[0001] creating container '/crowdsec-metabase' -INFO[0002] waiting for metabase to be up (can take up to a minute) -......... -INFO[0040] Metabase is ready - - URL : 'http://127.0.0.1:3000' - username : 'crowdsec@crowdsec.net' - password : ... -``` - -## Injection alerts into new database - dev env - -From a fresh release : - -```bash -$ tar xvzf crowdsec-release.tgz -$ cd crowdsec-v1.0.0-rc -$ ./test_env.sh -$ cd tests -``` - -Install the needed collection(s) : - -```bash -$ ./cscli -c dev.yaml collections install crowdsecurity/nginx -``` - -And we can process logs : - -```bash -$ ./crowdsec -c dev.yaml -file ~/github/crowdsec/OLDS/LOGS/nginx/10k_ACCESS_LOGS.log -type nginx -INFO[0000] single file mode : log_media=stdout daemonize=true -INFO[15-11-2020 11:18:27] Crowdsec v1.0.0-rc-0ecb142dfffc89b019b6d9044cb7cc5569d12c70 -INFO[15-11-2020 11:18:38] Ip x.x.x.x performed 'crowdsecurity/http-sensitive-files' (5 events over 4s) at 2017-10-23 12:35:54 +0200 CEST -INFO[15-11-2020 11:18:39] (test/crowdsec) crowdsecurity/http-probing by ip x.x.x.x (DE) : 1h ban on Ip x.x.x.x -``` - -And we can then query the local api (while letting the {{v1X.crowdsec.name}} running) : -```bash -$ ./cscli -c dev.yaml alerts list -+----+--------------------+---------------------------------------+---------+--------------+-----------+--------------------------------+ -| ID | VALUE | REASON | COUNTRY | AS | DECISIONS | CREATED AT | -+----+--------------------+---------------------------------------+---------+--------------+-----------+--------------------------------+ -| 28 | Ip:x.x.x.x | crowdsecurity/http-crawl-non_statics | DE | Linode, LLC | ban:1 | 2017-10-23 12:36:48 +0200 | -| | | | | | | +0200 | -| 27 | Ip:x.x.x.x | crowdsecurity/http-sensitive-files | DE | Linode, LLC | ban:1 | 2017-10-23 12:35:50 +0200 | -| | | | | | | +0200 | - -``` - -Or even start a dashboard to view data : - -```bash -$ sudo ./cscli dashboard setup -... -INFO[0002] waiting for metabase to be up (can take up to a minute) -........ - -``` \ No newline at end of file diff --git a/docs/v1.X/docs/user_guide/network.md b/docs/v1.X/docs/user_guide/network.md deleted file mode 100644 index 463ef9a4e..000000000 --- a/docs/v1.X/docs/user_guide/network.md +++ /dev/null @@ -1,43 +0,0 @@ - -# Ports inventory - - - `tcp/8080` exposes a [REST API](https://crowdsecurity.github.io/api_doc/index.html?urls.primaryName=LAPI) for bouncers, `cscli` and comunication between crowdsec agent and local api - - `tcp/6060` (endpoint `/metrics`) exposes [prometheus metrics](https://doc.crowdsec.net/Crowdsec/v1/observability/prometheus/) - - `tcp/6060` (endpoint `/debug`) exposes pprof debugging metrics - -# Outgoing connections - - - Local API connects to `tcp/443` on `api.crowdsec.net` (signal push and blocklists pull) - - `cscli` connects to `tcp/443` on `raw.githubusercontent.com` to fetch scenarios, parsers etc. - - `cscli dashboard` fetches metabase configuration from a s3 bucket (`https://crowdsec-statics-assets.s3-eu-west-1.amazonaws.com/`) - - - -# Comunication between components - -## Bouncers -> Local API - - - Bouncers are using Local API on `tcp/8080` by default - -## Agents -> Local API - - - Agents connect to local API on port `tcp/8080` (only relevant ) - -!!! warning - If there is an error in the agent configuration, it will also cause the Local API to fail if both of them are running in the same machine ! - Both components need proper configuration to run (we decide to keep this behavior to detect agent or local API errors on start). - -## Local API -> Central API - - - Central API is reached on port `tcp/443` by Local API. The FQDN is `api.crowdsec.net` - -## Local API -> Database - - - When using a networked database (PostgreSQL or MySQL), only the local API needs to access the database, agents don't have to be able to comunicate with it. - -## Prometheus -> Agents - - - If you're scrapping prometheus metrics from your agents or your local API, you need to allow inbound connections to `tcp/6060` - - - diff --git a/docs/v1.X/docs/user_guide/simulation_mode.md b/docs/v1.X/docs/user_guide/simulation_mode.md deleted file mode 100644 index f16967b58..000000000 --- a/docs/v1.X/docs/user_guide/simulation_mode.md +++ /dev/null @@ -1,34 +0,0 @@ -# Simulation - -```bash -$ sudo cscli simulation status -INFO[0000] global simulation: disabled -INFO[0000] Scenarios in simulation mode : -INFO[0000] - crowdsecurity/ssh-bf -``` - -`cscli simulation` allows to manage a list of scenarios that have their remediation "simulated" : they won't be effective (but will still be showed by `cscli decisions list`). This configuration file is present in `/etc/crowdsec/simulation.yaml`. - -You can add and remove scenarios to the simulation list : - -```bash -$ sudo cscli simulation enable crowdsecurity/ssh-bf -INFO[0000] simulation mode for 'crowdsecurity/ssh-bf' enabled -INFO[0000] Run 'sudo systemctl reload crowdsec' for the new configuration to be effective. -$ sudo systemctl reload crowdsec -$ sudo tail -f /var/log/crowdsec.log - .... -time="01-11-2020 14:08:58" level=info msg="Ip 1.2.3.6 performed 'crowdsecurity/ssh-bf' (6 events over 986.769µs) at 2020-11-01 14:08:58.575885389 +0100 CET m=+437.524832750" -time="01-11-2020 14:08:58" level=info msg="Ip 1.2.3.6 decision : 1h (simulation) ban" - .... - -$ cscli decisions list -+----+----------+--------------+-----------------------------------+------------+---------+----+--------+------------------+ -| ID | SOURCE | SCOPE:VALUE | REASON | ACTION | COUNTRY | AS | EVENTS | EXPIRATION | -+----+----------+--------------+-----------------------------------+------------+---------+----+--------+------------------+ -| 4 | crowdsec | Ip:1.2.3.6 | crowdsecurity/ssh-bf | (simul)ban | US | | 6 | 59m38.293036072s | -+----+----------+--------------+-----------------------------------+------------+---------+----+--------+------------------+ - -``` - -But as well turn on "global simulation" : in this case, only scenarios in the exclusion list will have their decisions applied. \ No newline at end of file diff --git a/docs/v1.X/docs/write_configurations/acquisition.md b/docs/v1.X/docs/write_configurations/acquisition.md deleted file mode 100644 index c361fc6bf..000000000 --- a/docs/v1.X/docs/write_configurations/acquisition.md +++ /dev/null @@ -1,34 +0,0 @@ -# Write the acquisition file (optional for test) - -In order for your log to be processed by the good parser, it must match the filter that you will configure in your parser file. - -The filters of the parsers in the first (`s00-raw`) stage will usually check `evt.Line.Labels.type`, which is the label of your acquisition file : - -With an acquisition file like this : - -```yaml -filename: /path/to/log/file.log -labels: - type: my_program -``` - - - The log line will enter the parsing pipeline with `evt.Line.Labels.type` set to `my_program` - - The parsers in the 1st stage (`s00-raw`) are dealing with the raw format, and the program name will end up in `evt.Parsed.program` - - When the log line arrive the main parsing stage (`s01-parse`), `evt.Parsed.program` will be `my_program` - - -For example, this file line(s) : - -```yaml -filename: /var/log/nginx/access.log -labels: - type: nginx -``` - -will be read by this parser : - -```yaml -filter: "evt.Parsed.program startsWith 'nginx'" -onsuccess: next_stage -... -``` diff --git a/docs/v1.X/docs/write_configurations/parsers.md b/docs/v1.X/docs/write_configurations/parsers.md deleted file mode 100644 index edfb01501..000000000 --- a/docs/v1.X/docs/write_configurations/parsers.md +++ /dev/null @@ -1,272 +0,0 @@ -# Writing {{v1X.crowdsec.Name}} parser - -!!! warning "Parser dependency" - The crowdsecurity/syslog-logs parsers is needed by the core parsing - engine. Deletion or modification of this could result of {{v1X.crowdsec.name}} - being unable to parse logs, so this should be done very carefully. - -> In the current example, we'll write a parser for the logs produced by `iptables` (netfilter) with the `-j LOG` target. -> This document aims at detailing the process of writing and testing new parsers. - -!!! tips "Exported fields" - You can view some of the extracted fields of existing parsers in the [Hub](https://hub.crowdsec.net/fields) - -## Base parser file - -The most simple parser can be defined as : - - -```yaml -filter: 1 == 1 -debug: true -onsuccess: next_stage -name: me/myparser -description: a cool parser for my service -grok: -#our grok pattern : capture .* - pattern: ^%{DATA:some_data}$ -#the field to which we apply the grok pattern : the log message itself - apply_on: message -statics: - - parsed: is_my_service - value: yes -``` - - - a {{v1X.filter.htmlname}} : if the expression is `true`, the event will enter the parser, otherwise, it won't - - a {{v1X.onsuccess.htmlname}} : defines what happens when the {{v1X.event.htmlname}} was successfully parsed : shall we continue ? shall we move to next stage ? etc. - - a name & a description - - some {{v1X.statics.htmlname}} that will modify the {{v1X.event.htmlname}} - - a `debug` flag that allows to enable local debugging information. - - -We are going to use to following sample log as an example : -```bash -May 11 16:23:43 sd-126005 kernel: [47615895.771900] IN=enp1s0 OUT= MAC=00:08:a2:0c:1f:12:00:c8:8b:e2:d6:87:08:00 SRC=99.99.99.99 DST=127.0.0.1 LEN=40 TOS=0x00 PREC=0x00 TTL=245 ID=51006 PROTO=TCP SPT=45225 DPT=8888 WINDOW=1024 RES=0x00 SYN URGP=0 -May 11 16:23:50 sd-126005 kernel: [47615902.763137] IN=enp1s0 OUT= MAC=00:08:a2:0c:1f:12:00:c8:8b:e2:d6:87:08:00 SRC=44.44.44.44 DST=127.0.0.1 LEN=60 TOS=0x00 PREC=0x00 TTL=49 ID=17451 DF PROTO=TCP SPT=53668 DPT=80 WINDOW=14600 RES=0x00 SYN URGP=0 -``` - -## Trying our mock parser - -!!! warning - Your yaml file must be in the `config/parsers/s01-parse/` directory. - - For example it can be `~/crowdsec-v0.0.19/tests/config/parsers/s01-parse/myparser.yaml`, or `/etc/crowdsec/parsers/s01-parse/myparser.yaml`. - - The {{v1X.stage.htmlname}} directory might not exist, don't forget to create it. - -(deployment is assuming [you're using a test environment](/Crowdsec/v1/write_configurations/requirements/)) - -Setting up our new parser : - -- if `config/parsers/s01-parse` doesn't exist, create it: - -```bash -cd crowdsec-v0.X.Y/tests -mkdir -p config/parsers/s01-parse -``` - -- Then copy your parser in `config/parsers/s01-parse` and try it: - -``` -cp myparser.yaml config/parsers/s01-parse/ -./crowdsec -c ./dev.yaml -file ./x.log -type foobar -``` - -
- Expected output - -```bash -INFO[0000] setting loglevel to info -INFO[11-05-2020 15:48:28] Crowdsec v0.0.18-6b1281ba76819fed4b89247a5a673c592a3a9f88 -... -DEBU[0000] Event entering node id=dark-water name=me/myparser stage=s01-parse -DEBU[0000] eval(TRUE) '1 == 1' id=dark-water name=me/myparser stage=s01-parse -DEBU[0000] no ip in event, cidr/ip whitelists not checked id=dark-water name=me/myparser stage=s01-parse -DEBU[0000] + Grok '' returned 1 entries to merge in Parsed id=dark-water name=me/myparser stage=s01-parse -DEBU[0000] .Parsed['some_data'] = 'May 11 16:23:41 sd-126005 kernel: [47615893.721616] IN=enp1s0 OUT= MAC=00:08:a2:0c:1f:12:00:c8:8b:e2:d6:87:08:00 SRC=99.99.99.99 DST=127.0.0.1 LEN=40 TOS=0x00 PREC=0x00 TTL=245 ID=54555 PROTO=TCP SPT=45225 DPT=8080 WINDOW=1024 RES=0x00 SYN URGP=0 ' id=dark-water name=me/myparser stage=s01-parse -DEBU[0000] + Processing 1 statics id=dark-water name=me/myparser stage=s01-parse -DEBU[0000] .Parsed[is_my_service] = 'yes' id=dark-water name=me/myparser stage=s01-parse -DEBU[0000] Event leaving node : ok id=dark-water name=me/myparser stage=s01-parse -DEBU[0000] move Event from stage s01-parse to s02-enrich id=dark-water name=me/myparser stage=s01-parse -... -``` -
- - -We can see our "mock" parser is working, let's see what happened : - - - The event enter the node - - The `filter` returned true (`1 == 1`) so the {{v1X.event.htmlname}} will be processed - - Our grok pattern (just a `.*` capture) "worked" and captured data (the whole line actually) - - The grok captures (under the name "some_data") are merged into the `.Parsed` map of the {{v1X.event.htmlname}} - - The {{v1X.statics.htmlname}} section is processed, and `.Parsed[is_my_service]` is set to `yes` - - The {{v1X.event.htmlname}} leaves the parser successfully, and because "next_stage" is set, we move the event to the next "stage" - -## Writing the GROK pattern - -We are going to write a parser for `iptables` logs, they look like this : - -``` -May 11 16:23:43 sd-126005 kernel: [47615895.771900] IN=enp1s0 OUT= MAC=00:08:a2:0c:1f:12:00:c8:8b:e2:d6:87:08:00 SRC=99.99.99.99 DST=127.0.0.1 LEN=40 TOS=0x00 PREC=0x00 TTL=245 ID=51006 PROTO=TCP SPT=45225 DPT=8888 WINDOW=1024 RES=0x00 SYN URGP=0 -May 11 16:23:50 sd-126005 kernel: [47615902.763137] IN=enp1s0 OUT= MAC=00:08:a2:0c:1f:12:00:c8:8b:e2:d6:87:08:00 SRC=44.44.44.44 DST=127.0.0.1 LEN=60 TOS=0x00 PREC=0x00 TTL=49 ID=17451 DF PROTO=TCP SPT=53668 DPT=80 WINDOW=14600 RES=0x00 SYN URGP=0 - -``` - -Using an [online grok debugger](https://grokdebug.herokuapp.com/) or an [online regex debugger](https://www.debuggex.com/), we come up with the following grok pattern : - -``` -\[%{DATA}\]+.*(%{WORD:action})? IN=%{WORD:int_eth} OUT= MAC=%{IP}:%{MAC} SRC=%{IP:src_ip} DST=%{IP:dst_ip} LEN=%{INT:length}.*PROTO=%{WORD:proto} SPT=%{INT:src_port} DPT=%{INT:dst_port}.* -``` - -!!! warning - Check if the pattern you are looking for is not already present in [patterns configuration](https://github.com/crowdsecurity/crowdsec/tree/master/config/patterns). - - -## Test our new pattern - -Now, let's integrate our GROK pattern within our YAML : - -```yaml -#let's set onsuccess to "next_stage" : if the log is parsed, we can consider it has been dealt with -onsuccess: next_stage -#debug, for reasons (don't do this in production) -debug: true -#as seen in our sample log, those logs are processed by the system and have a progname set to 'kernel' -filter: "1 == 1" -#name and description: -name: crowdsecurity/iptables-logs -description: "Parse iptables drop logs" -grok: -#our grok pattern - pattern: \[%{DATA}\]+.*(%{WORD:action})? IN=%{WORD:int_eth} OUT= MAC=%{IP}:%{MAC} SRC=%{IP:src_ip} DST=%{IP:dst_ip} LEN=%{INT:length}.*PROTO=%{WORD:proto} SPT=%{INT:src_port} DPT=%{INT:dst_port}.* -#the field to which we apply the grok pattern : the log message itself - apply_on: message -statics: - - parsed: is_my_service - value: yes -``` - - -```bash -./crowdsec -c ./dev.yaml -file ./x.log -type foobar -``` - - -
- Expected output - -```bash -INFO[0000] setting loglevel to info -INFO[11-05-2020 16:18:58] Crowdsec v0.0.18-6b1281ba76819fed4b89247a5a673c592a3a9f88 -... -DEBU[0000] Event entering node id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] eval(TRUE) '1 == 1' id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] no ip in event, cidr/ip whitelists not checked id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] + Grok '' returned 8 entries to merge in Parsed id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] .Parsed['dst_port'] = '8080' id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] .Parsed['action'] = '' id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] .Parsed['int_eth'] = 'enp1s0' id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] .Parsed['src_ip'] = '99.99.99.99' id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] .Parsed['dst_ip'] = '127.0.0.1' id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] .Parsed['length'] = '40' id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] .Parsed['proto'] = 'TCP' id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] .Parsed['src_port'] = '45225' id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] + Processing 1 statics id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] .Parsed[is_my_service] = 'yes' id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] Event leaving node : ok id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] move Event from stage s01-parse to s02-enrich id=lingering-breeze name=crowdsecurity/iptables-logs stage=s01-parse -... -``` - -
- -What changed ? We can now see that the fragment captured by the GROK pattern are merged in the `Parsed` array ! -We now have parsed data, only a few more changes and we will be done :) - -## Finalizing our parser - -```yaml -#let's set onsuccess to "next_stage" : if the log is parsed, we can consider it has been dealt with -onsuccess: next_stage -#debug, for reasons (don't do this in production) -debug: true -#as seen in our sample log, those logs are processed by the system and have a progname set to 'kernel' -filter: "evt.Parsed.program == 'kernel'" -#name and description: -name: crowdsecurity/iptables-logs -description: "Parse iptables drop logs" -grok: -#our grok pattern - pattern: \[%{DATA}\]+.*(%{WORD:action})? IN=%{WORD:int_eth} OUT= MAC=%{IP}:%{MAC} SRC=%{IP:src_ip} DST=%{IP:dst_ip} LEN=%{INT:length}.*PROTO=%{WORD:proto} SPT=%{INT:src_port} DPT=%{INT:dst_port}.* -#the field to which we apply the grok pattern : the log message itself - apply_on: message -statics: - - meta: log_type - value: iptables_drop - - meta: service - expression: "evt.Parsed.proto == 'TCP' ? 'tcp' : 'unknown'" - - meta: source_ip - expression: "evt.Parsed.src_ip" -``` - -### filter - -We changed the {{v1X.filter.htmlname}} to correctly filter on the program name. -In the current example, our logs are produced by the kernel (netfilter), and thus the program is `kernel` : - -```bash -tail -f /var/log/kern.log -May 11 16:23:50 sd-126005 kernel: [47615902.763137] IN=enp1s0 OUT= MAC=00:08:a2:0c:1f:12:00:c8:8b:e2:d6:87:08:00 SRC=44.44.44.44 DST=127.0.0.1 LEN=60 TOS=0x00 PREC=0x00 TTL=49 ID=17451 DF PROTO=TCP SPT=53668 DPT=80 WINDOW=14600 RES=0x00 SYN URGP=0 -``` - -### statics - -We are setting various entries to static or dynamic values to give "context" to the log : - - - `.Meta.log_type` is set to `iptables_drop` (so that we later can filter events coming from this) - - `.Meta.source_ip` is set the the source ip captured `.Parsed.src_ip` - - `.Meta.service` is set the the result of an expression that relies on the GROK output (`proto` field) - -Look into dedicated {{v1X.statics.htmlname}} documentation to know more about its possibilities. - - -### Testing our finalized parser - - -```bash -./crowdsec -c ./dev.yaml -file ./x.log -type kernel -``` - -
- Expected output -```bash -... -DEBU[0000] Event entering node id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] eval(TRUE) 'evt.Parsed.program == 'kernel'' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] no ip in event, cidr/ip whitelists not checked id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] + Grok '' returned 8 entries to merge in Parsed id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] .Parsed['src_port'] = '45225' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] .Parsed['dst_port'] = '8118' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] .Parsed['action'] = '' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] .Parsed['int_eth'] = 'enp1s0' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] .Parsed['src_ip'] = '44.44.44.44' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] .Parsed['dst_ip'] = '127.0.0.1' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] .Parsed['length'] = '40' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] .Parsed['proto'] = 'TCP' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] + Processing 3 statics id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] .Meta[log_type] = 'iptables_drop' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] .Meta[service] = 'tcp' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] .Meta[source_ip] = '44.44.44.44' id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] Event leaving node : ok id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parse -DEBU[0000] move Event from stage s01-parse to s02-enrich id=shy-forest name=crowdsecurity/iptables-logs stage=s01-parse -... -``` -
- -## Closing word - -We have now a fully functional parser for {{v1X.crowdsec.name}} ! -We can either deploy it to our production systems to do stuff, or even better, contribute to the {{v1X.hub.htmlname}} ! - -If you want to know more about directives and possibilities, take a look at [the parser reference documentation](/Crowdsec/v1/references/parsers/) ! - diff --git a/docs/v1.X/docs/write_configurations/requirements.md b/docs/v1.X/docs/write_configurations/requirements.md deleted file mode 100644 index f9321c1db..000000000 --- a/docs/v1.X/docs/write_configurations/requirements.md +++ /dev/null @@ -1,72 +0,0 @@ -# Requirements - -> - Having read and understood [`crowdsec` concepts](/Crowdsec/v1/getting_started/concepts/) - -> - Some requirements are needed in order to be able to write your own end-to-end configurations. - -> - During all this documentation, we are going to show as an exemple how we wrote a full port scan detection scenario (from acqusition to scenario, including parser) - - -## Create the test environment - -First of all, please [download the latest release of {{v1X.crowdsec.name}}](https://github.com/crowdsecurity/crowdsec/releases). - -Then run the following commands: - -```bash -tar xzvf crowdsec-release.tgz -``` -```bash -cd ./crowdsec-vX.Y/ -``` -```bash -./test_env.sh # the -o is facultative, default is "./tests/" -``` -```bash -cd ./tests/ -``` - -The `./test_env.sh` script creates a local (non privileged) working environement for {{v1X.crowdsec.name}} and {{v1X.cli.name}}. -The deployed environment is intended to write and test parsers and scenarios easily. - - -
- Example - -```bash -$ tar xzvf ./crowdsec-release.tgz -$ cd ./crowdsec-v*/ -$ ./test_env.sh -[12/11/2020:11:45:19][INFO] Creating test arboresence in /tmp/crowdsec-v1.0.0/tests -[12/11/2020:11:45:19][INFO] Arboresence created -[12/11/2020:11:45:19][INFO] Copying needed files for tests environment -[12/11/2020:11:45:19][INFO] Files copied -[12/11/2020:11:45:19][INFO] Setting up configurations -INFO[0000] Machine 'test' created successfully -INFO[0000] API credentials dumped to '/tmp/crowdsec-v1.0.0/tests/config/local_api_credentials.yaml' -INFO[0000] Wrote new 73826 bytes index to /tmp/crowdsec-v1.0.0/tests/config/hub/.index.json -INFO[0000] crowdsecurity/syslog-logs : OK -INFO[0000] crowdsecurity/geoip-enrich : OK -INFO[0000] downloading data 'https://crowdsec-statics-assets.s3-eu-west-1.amazonaws.com/GeoLite2-City.mmdb' in '/tmp/crowdsec-v1.0.0/tests/data/GeoLite2-City.mmdb' -INFO[0002] downloading data 'https://crowdsec-statics-assets.s3-eu-west-1.amazonaws.com/GeoLite2-ASN.mmdb' in '/tmp/crowdsec-v1.0.0/tests/data/GeoLite2-ASN.mmdb' -INFO[0003] crowdsecurity/dateparse-enrich : OK -INFO[0003] crowdsecurity/sshd-logs : OK -INFO[0004] crowdsecurity/ssh-bf : OK -INFO[0004] crowdsecurity/sshd : OK -WARN[0004] crowdsecurity/sshd : overwrite -INFO[0004] crowdsecurity/linux : OK -INFO[0004] /tmp/crowdsec-v1.0.0/tests/config/collections doesn't exist, create -INFO[0004] Enabled parsers : crowdsecurity/syslog-logs -INFO[0004] Enabled parsers : crowdsecurity/geoip-enrich -INFO[0004] Enabled parsers : crowdsecurity/dateparse-enrich -INFO[0004] Enabled parsers : crowdsecurity/sshd-logs -INFO[0004] Enabled scenarios : crowdsecurity/ssh-bf -INFO[0004] Enabled collections : crowdsecurity/sshd -INFO[0004] Enabled collections : crowdsecurity/linux -INFO[0004] Enabled crowdsecurity/linux -INFO[0004] Run 'systemctl reload crowdsec' for the new configuration to be effective. -[12/11/2020:11:45:25][INFO] Environment is ready in /tmp/crowdsec-v1.0.0/tests - -``` - - diff --git a/docs/v1.X/docs/write_configurations/scenarios.md b/docs/v1.X/docs/write_configurations/scenarios.md deleted file mode 100644 index 5f91591a6..000000000 --- a/docs/v1.X/docs/write_configurations/scenarios.md +++ /dev/null @@ -1,351 +0,0 @@ -# Writing {{v1X.crowdsec.Name}} scenarios - -!!! info - Please ensure that you have working env or [setup test environment](/Crowdsec/v1/write_configurations/requirements/) before writing your scenario. - - Ensure that [your logs are properly parsed](/Crowdsec/v1/write_configurations/parsers/). - - Have some sample logs at hand reach to test your scenario as you progress. - - -> In the current example, we'll write a scenario to detect port scans relying on the logs produced by `iptables` (netfilter) with the `-j LOG` target. - -> This document aims at detailing the process of writing and testing new scenarios. - -> If you're writing scenario for existing logs, [take a look at the taxonomy](https://hub.crowdsec.net/fields) to find your way ! - - -!!! tips "Exported fields" - You can view some of the extracted fields of existing parsers in the [Hub](https://hub.crowdsec.net/fields) - -## Base scenario file - - -A rudimentary scenario can be defined as : -!!! warning - Your yaml file must be in the `config/scenarios/` directory. - -```yaml -type: leaky -debug: true -name: me/my-cool-scenario -description: "detect cool stuff" -filter: evt.Meta.log_type == 'iptables_drop' -capacity: 1 -leakspeed: 1m -blackhole: 1m -labels: - type: my_test -``` - - - a {{v1X.filter.htmlname}} : if the expression is `true`, the event will enter the scenario, otherwise, it won't - - a name & a description - - a capacity for our [Leaky Bucket](https://en.wikipedia.org/wiki/Leaky_bucket) - - a leak speed for our [Leaky Bucket](https://en.wikipedia.org/wiki/Leaky_bucket) - - a blackhole duration (it will prevent the same bucket from overflowing too often to limit spam) - - some labels to qualify the events that just happen - - a `debug` flag that allows to enable local debugging information. - - -We are going to use the following sample log in our example : - -```bash -May 12 09:40:15 sd-126005 kernel: [47678084.929208] IN=enp1s0 OUT= MAC=00:08:a2:0c:1f:12:00:c8:8b:e2:d6:87:08:00 SRC=66.66.66.66 DST=127.0.0.1 LEN=40 TOS=0x08 PREC=0x20 TTL=244 ID=54321 PROTO=TCP SPT=42403 DPT=7681 WINDOW=65535 RES=0x00 SYN URGP=0 -May 12 09:40:15 sd-126005 kernel: [47678084.929245] IN=enp1s0 OUT= MAC=00:08:a2:0c:1f:12:00:c8:8b:e2:d6:87:08:00 SRC=99.99.99.99 DST=127.0.0.1 LEN=40 TOS=0x08 PREC=0x20 TTL=244 ID=54321 PROTO=TCP SPT=42403 DPT=7681 WINDOW=65535 RES=0x00 SYN URGP=0 -May 12 09:40:16 sd-126005 kernel: [47678084.929208] IN=enp1s0 OUT= MAC=00:08:a2:0c:1f:12:00:c8:8b:e2:d6:87:08:00 SRC=99.99.99.99 DST=127.0.0.1 LEN=40 TOS=0x08 PREC=0x20 TTL=244 ID=54321 PROTO=TCP SPT=42403 DPT=7681 WINDOW=65535 RES=0x00 SYN URGP=0 -May 12 09:40:16 sd-126005 kernel: [47678084.929208] IN=enp1s0 OUT= MAC=00:08:a2:0c:1f:12:00:c8:8b:e2:d6:87:08:00 SRC=44.44.44.44 DST=127.0.0.1 LEN=40 TOS=0x08 PREC=0x20 TTL=244 ID=54321 PROTO=TCP SPT=42403 DPT=7681 WINDOW=65535 RES=0x00 SYN URGP=0 -``` - -## Let's try our mock scenario - -!!! info - This assumes that you've followed the previous tutorial and that your iptables logs are properly parsed - - -```bash -./crowdsec -c ./dev.yaml -file ./x.log -type syslog -``` - - -
- Expected output -```bash -DEBU[04-08-2020 10:44:26] eval(evt.Meta.log_type == 'iptables_drop') = TRUE cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario -DEBU[04-08-2020 10:44:26] eval variables: cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario -DEBU[04-08-2020 10:44:26] evt.Meta.log_type = 'iptables_drop' cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario -... -DEBU[04-08-2020 10:44:26] eval(evt.Meta.log_type == 'iptables_drop') = TRUE cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario -DEBU[04-08-2020 10:44:26] eval variables: cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario -DEBU[04-08-2020 10:44:26] evt.Meta.log_type = 'iptables_drop' cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario -... -DEBU[04-08-2020 10:44:26] Overflow (start: 2020-05-12 09:40:15 +0000 UTC, end: 2020-05-12 09:40:15 +0000 UTC) bucket_id=sparkling-thunder capacity=1 cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario partition=ea2fed6bf8bb70d462ef8acacc4c96f5f8754413 -DEBU[04-08-2020 10:44:26] Adding overflow to blackhole (2020-05-12 09:40:15 +0000 UTC) bucket_id=sparkling-thunder capacity=1 cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario partition=ea2fed6bf8bb70d462ef8acacc4c96f5f8754413 -DEBU[04-08-2020 10:44:26] eval(evt.Meta.log_type == 'iptables_drop') = TRUE cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario -DEBU[04-08-2020 10:44:26] eval variables: cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario -DEBU[04-08-2020 10:44:26] evt.Meta.log_type = 'iptables_drop' cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario -DEBU[04-08-2020 10:44:26] Bucket ea2fed6bf8bb70d462ef8acacc4c96f5f8754413 found dead, cleanup the body bucket_id=sparkling-thunder capacity=1 cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario partition=ea2fed6bf8bb70d462ef8acacc4c96f5f8754413 -WARN[04-08-2020 10:44:26] read 4 lines file=./x.log -... -INFO[04-08-2020 10:44:26] Processing Overflow with no decisions 2 IPs performed 'me/my-cool-scenario' (2 events over 0s) at 2020-05-12 09:40:15 +0000 UTC bucket_id=sparkling-thunder event_time="2020-05-12 09:40:15 +0000 UTC" scenario=me/my-cool-scenario source_ip=66.66.66.66 -... -DEBU[04-08-2020 10:44:26] Overflow discarded, still blackholed for 59s bucket_id=long-pine capacity=1 cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario partition=ea2fed6bf8bb70d462ef8acacc4c96f5f8754413 -DEBU[04-08-2020 10:44:26] Overflow has been discard (*leakybucket.Blackhole) bucket_id=long-pine capacity=1 cfg=shy-dust file=config/scenarios/iptables-scan.yaml name=me/my-cool-scenario partition=ea2fed6bf8bb70d462ef8acacc4c96f5f8754413 -... -``` -
- - -We can see our "mock" scenario is working, let's see what happened : - -- The first event (parsed line) is processed : - - - The `filter` returned true (`evt.Meta.log_type == 'iptables_drop'`) so the {{v1X.event.htmlname}} will be processed by our bucket - - The bucket is instantiated in {{v1X.timeMachine.htmlname}} mode, and its creation date is set to the timestamp from the first log - - The {{v1X.event.htmlname}} is poured in the actual bucket - -- The second event is processed - - The `filter` is still true, and the event is poured - - As our bucket's capacity is `1`, pouring this second overflow leads to an {{v1X.alert.htmlname}} - - Because we set a blackhole directive of `1 minute`, we remember to prevent this bucket to overflowing again for the next minute - -The overflow itself is produced and we get this message : - -``` -INFO[12-05-2020 11:22:17] Processing Overflow with no decisions 2 IPs performed 'me/my-cool-scenario' (2 events over 0s) at 2020-05-12 09:40:15 +0000 UTC bucket_id=withered-brook event_time="2020-05-12 09:40:15 +0000 UTC" scenario=me/my-cool-scenario source_ip=66.66.66.66 - -``` - -!!! warning - While it "worked" we can see the first issue : the offending IP is reported to be `66.66.66.66` but there are actually 3 IPs involved (`66.66.66.66`, `99.99.99.99` and `44.44.44.44`). To make sense our "detect port scans" should detect events coming from a single IP ! - - -## One step forward : peer attribution - -Let's evolve our scenario to be closer to something meaningful : - - -```yaml -type: leaky -debug: true -name: me/my-cool-scenario -description: "detect cool stuff" -filter: "evt.Meta.log_type == 'iptables_drop' && evt.Meta.service == 'tcp'" -groupby: evt.Meta.source_ip -capacity: 1 -leakspeed: 1m -blackhole: 1m -labels: - type: my_test -``` - -What did we change ? - - - we added a meaningful filter : we are only going to look into `iptables_drop` events, and only take care of `tcp` ones (see the parser we wrote in the [previous step](/Crowdsec/v1/write_configurations/parsers/)) - - we added a `groupby` directive : it's going to ensure that each offending peer get its own bucket - - -Let's try again ! - -```bash -./crowdsec -c ./dev.yaml -file ./x.log -type syslog -``` - -
- Expected output -```bash -... -DEBU[2020-05-12T11:25:20+02:00] eval(TRUE) evt.Meta.log_type == 'iptables_drop' && evt.Meta.service == 'tcp' cfg=holy-breeze file=config/scenarios/mytest.yaml name=me/my-cool-scenario -DEBU[2020-05-12T11:25:20+02:00] Leaky routine starting, lifetime : 2m0s bucket_id=cold-lake capacity=1 cfg=holy-breeze file=config/scenarios/mytest.yaml name=me/my-cool-scenario partition=2308799e2cc5b57331df10eb93a495aff7725922 -... -DEBU[2020-05-12T11:25:20+02:00] eval(TRUE) evt.Meta.log_type == 'iptables_drop' && evt.Meta.service == 'tcp' cfg=holy-breeze file=config/scenarios/mytest.yaml name=me/my-cool-scenario -DEBU[2020-05-12T11:25:20+02:00] Instanciating TimeMachine bucket cfg=holy-breeze file=config/scenarios/mytest.yaml name=me/my-cool-scenario -DEBU[2020-05-12T11:25:20+02:00] Leaky routine starting, lifetime : 2m0s bucket_id=muddy-haze capacity=1 cfg=holy-breeze file=config/scenarios/mytest.yaml name=me/my-cool-scenario partition=6236f134d0f34d0061748c065bdcb64d8ac6dc54 -... -INFO[12-05-2020 11:25:20] node warning : no remediation bucket_id=muddy-haze event_time="2020-05-12 09:40:16 +0000 UTC" scenario=me/my-cool-scenario source_ip=99.99.99.99 -INFO[12-05-2020 11:25:20] Processing Overflow with no decisions 99.99.99.99 performed 'me/my-cool-scenario' (2 events over 1s) at 2020-05-12 09:40:16 +0000 UTC bucket_id=muddy-haze event_time="2020-05-12 09:40:16 +0000 UTC" scenario=me/my-cool-scenario source_ip=99.99.99.99 -... - -``` -
- -Let's see what happened : - - - Thanks to our `groupby` key, we now see two different partition keys appearing (`partition=...`). - It means that each peer will get its own bucket, and a "unique key" is derived from the groupby field value (here : the source IP) - - - We see that we only have one overflow, and it correctly concerns `99.99.99.99` (it's the one that actually triggered two events). This is again thanks to the groupby key - - -## One step forward : unique ports - - - -Is it done ? not yet, but we're getting close ! - -To really qualify a port-scan, we want to rely on the number of unique probed ports. Let's arbitrarily decide that a port-scan is : "One peer trying to probe AT LEAST 15 different ports within a few seconds" - -Our evolved scenario is now : - -```yaml -type: leaky -debug: true -name: me/my-cool-scenario -description: "detect cool stuff" -filter: "evt.Meta.log_type == 'iptables_drop' && evt.Meta.service == 'tcp'" -groupby: evt.Meta.source_ip -distinct: evt.Parsed.dst_port -capacity: 15 -leakspeed: 5s -blackhole: 1m -labels: - type: scan - service: tcp - -``` - -What did we changed : - - - We add a `distinct` directive on the `evt.Parsed.dst_port`. It allows the bucket to discard any event with an already seen `evt.Parsed.dst_port`. (yes, like in SQL) - - We changed `capacity` and `leakspeed` to be more relevant to our target - - We fixed the `labels` so that the event makes sense ! - - -Let's see what it changes : - -```bash -./crowdsec -c ./dev.yaml -file ./x.log -type syslog -``` - -
- Expected output -```bash -... -DEBU[2020-05-12T11:49:01+02:00] eval(TRUE) evt.Meta.log_type == 'iptables_drop' && evt.Meta.service == 'tcp' cfg=dark-pond file=config/scenarios/mytest.yaml name=me/my-cool-scenario -DEBU[2020-05-12T11:49:01+02:00] Instantiating TimeMachine bucket cfg=dark-pond file=config/scenarios/mytest.yaml name=me/my-cool-scenario -DEBU[2020-05-12T11:49:01+02:00] Leaky routine starting, lifetime : 1m20s bucket_id=nameless-feather capacity=15 cfg=dark-pond file=config/scenarios/mytest.yaml name=me/my-cool-scenario partition=2308799e2cc5b57331df10eb93a495aff7725922 -DEBU[2020-05-12T11:49:01+02:00] Uniq 'evt.Parsed.dst_port' -> '7681' bucket_id=nameless-feather capacity=15 cfg=dark-pond file=config/scenarios/mytest.yaml name=me/my-cool-scenario partition=2308799e2cc5b57331df10eb93a495aff7725922 -DEBU[2020-05-12T11:49:01+02:00] Uniq(7681) : false, discard bucket_id=nameless-feather capacity=15 cfg=dark-pond file=config/scenarios/mytest.yaml name=me/my-cool-scenario partition=2308799e2cc5b57331df10eb93a495aff7725922 -DEBU[2020-05-12T11:49:01+02:00] Pouring event bucket_id=nameless-feather capacity=15 cfg=dark-pond file=config/scenarios/mytest.yaml name=me/my-cool-scenario partition=2308799e2cc5b57331df10eb93a495aff7725922 -... - -``` -
- - - We can see that the second event was discarded, because it had a destination port similar to the first one - - No overflow were produced - - -## Is it really working - -Ok, **fingers crossed** our thing should be working. - -Let's grab some real-life logs ! - -```bash -$ wc -l kern.log -78215 kern.log -$ head -n1 kern.log -May 11 06:25:20 sd-126005 kernel: ... -$ tail -n1 kern.log -May 12 12:09:00 sd-126005 kernel: ... -``` - -We have around 80k lines averaging about 24h of logs, let's try ! - -```bash -./crowdsec -c ./dev.yaml -file ./kern.log -type syslog -``` - -
- Expected output -```bash -INFO[0000] setting loglevel to info -INFO[12-05-2020 11:50:38] Crowdsec v0.0.18-f672dbb4aec29ca2b24080a33d4d92eb9d4441cc -... -INFO[12-05-2020 11:50:42] node warning : no remediation bucket_id=sparkling-violet event_time="2020-05-11 10:41:45 +0000 UTC" scenario=me/my-cool-scenario source_ip=xx.xx.xx.xx -INFO[12-05-2020 11:50:42] Processing Overflow with no decisions xx.xx.xx.xx performed 'me/my-cool-scenario' (16 events over 0s) at 2020-05-11 10:41:45 +0000 UTC bucket_id=sparkling-violet event_time="2020-05-11 10:41:45 +0000 UTC" scenario=me/my-cool-scenario source_ip=xx.xx.xx.xx -... -INFO[12-05-2020 11:50:43] node warning : no remediation bucket_id=quiet-leaf event_time="2020-05-11 11:34:11 +0000 UTC" scenario=me/my-cool-scenario source_ip=yy.yy.yy.yy -INFO[12-05-2020 11:50:43] Processing Overflow with no decisions yy.yy.yy.yy performed 'me/my-cool-scenario' (16 events over 2s) at 2020-05-11 11:34:11 +0000 UTC bucket_id=quiet-leaf event_time="2020-05-11 11:34:11 +0000 UTC" scenario=me/my-cool-scenario source_ip=yy.yy.yy.yy -... -WARN[12-05-2020 11:51:05] read 78215 lines file=./kern.log -... -``` -
- -It seems to work correctly ! - - -## Hold my beer and watch this - - -Once I have acquire confidence in my scenario and I want it to trigger some bans, we can simply add : - - -```yaml -type: leaky -debug: true -name: me/my-cool-scenario -description: "detect cool stuff" -filter: "evt.Meta.log_type == 'iptables_drop' && evt.Meta.service == 'tcp'" -groupby: evt.Meta.source_ip -distinct: evt.Parsed.dst_port -capacity: 15 -leakspeed: 5s -blackhole: 1m -labels: - type: scan - service: tcp - remediation: true - scope: ip -``` - - -Adding `remediation: true` into the labels tells {{v1X.crowdsec.name}} that we should write a ban for the IP when the scenario is triggered ! - -Let's try : - - - I copied the yaml file to a production system (`/etc/crowdsec/scenarios/mytest.yaml`) - - I restart {{v1X.crowdsec.name}} (`systemctl reload crowdsec`) - -Let's check if it seems correctly enabled : - -```bash -$ {{v1X.cli.bin}} list -... -INFO[0000] SCENARIOS: ----------------------------------------------------------------------------------------------------------------------------------- - NAME 📦 STATUS VERSION LOCAL PATH ----------------------------------------------------------------------------------------------------------------------------------- -... - mytest.yaml 🚫 enabled,local /etc/crowdsec/scenarios/mytest.yaml -... -``` - - -Let's launch (from an external machine, as {{v1X.crowdsec.name}} ignores events from private IPs by default) a real port-scan with a good old `nmap` : - -```bash -sudo nmap -sS xx.xx.xx.xx -``` - - -and on our server : - -```bash -$ tail -f /var/log/crowdsec.log -... -time="12-05-2020 12:31:43" level=warning msg="xx.xx.16.6 triggered a 4h0m0s ip ban remediation for [me/my-cool-scenario]" bucket_id=wispy-breeze event_time="2020-05-12 12:31:43.953498645 +0200 CEST m=+64.533521568" scenario=me/my-cool-scenario source_ip=xx.xx.16.6 -... -^C -$ {{v1X.cli.bin}} ban list -INFO[0000] backend plugin 'database' loaded -8 local decisions: -+--------+-----------------+----------------------+------+--------+---------+--------------------------+--------+------------+ -| SOURCE | IP | REASON | BANS | ACTION | COUNTRY | AS | EVENTS | EXPIRATION | -+--------+-----------------+----------------------+------+--------+---------+--------------------------+--------+------------+ -| local | xx.xx.xx.xx | me/my-cool-scenario | 4 | ban | FR | 21502 SFR SA | 79 | 3h58m27s | -... -``` - -It worked !!! diff --git a/docs/v1.X/docs/write_configurations/whitelist.md b/docs/v1.X/docs/write_configurations/whitelist.md deleted file mode 100644 index 068d2f188..000000000 --- a/docs/v1.X/docs/write_configurations/whitelist.md +++ /dev/null @@ -1,184 +0,0 @@ -# What are whitelists - -Whitelists are special parsers that allow you to "discard" events, and can exist at two different steps : - - - *Parser whitelists* : Allows you to discard an event at parse time, so that it never hits the buckets. - - *PostOverflow whitelists* : Those are whitelists that are checked *after* the overflow happens. It is usually best for whitelisting process that can be expensive (such as performing reverse DNS on an IP, or performing a `whois` of an IP). - -!!! info - While the whitelists are the same for parser or postoverflows, beware that field names might change. - Source ip is usually in `evt.Meta.source_ip` when it's a log, but `evt.Overflow.Source_ip` when it's an overflow - - -The whitelist can be based on several criteria : - - - specific ip address : if the event/overflow IP is the same, event is whitelisted - - ip ranges : if the event/overflow IP belongs to this range, event is whitelisted - - a list of {{v1X.expr.htmlname}} expressions : if any expression returns true, event is whitelisted - -Here is an example showcasing configuration : - -```yaml -name: crowdsecurity/my-whitelists -description: "Whitelist events from my ipv4 addresses" -#it's a normal parser, so we can restrict its scope with filter -filter: "1 == 1" -whitelist: - reason: "my ipv4 ranges" - ip: - - "127.0.0.1" - cidr: - - "192.168.0.0/16" - - "10.0.0.0/8" - - "172.16.0.0/12" - expression: - #beware, this one will work *only* if you enabled the reverse dns (crowdsecurity/rdns) enrichment postoverflow parser - - evt.Enriched.reverse_dns endsWith ".mycoolorg.com." - #this one will work *only* if you enabled the geoip (crowdsecurity/geoip-enrich) enrichment parser - - evt.Enriched.IsoCode == 'FR' -``` - - -# Whitelists in parsing - -When a whitelist is present in parsing `/etc/crowdsec/parsers/...`, it will be checked/discarded before being poured to any bucket. These whitelists intentionally generate no logs and are useful to discard noisy false positive sources. - -## Whitelist by ip - -Let's assume we have a setup with a `crowdsecurity/nginx` collection enabled and no whitelists. - -Thus, if I "attack" myself : - -```bash -nikto -host myfqdn.com -``` - -my own IP will be flagged as being an attacker : - -```bash -$ tail -f /var/log/crowdsec.log -ime="07-07-2020 16:13:16" level=warning msg="80.x.x.x triggered a 4h0m0s ip ban remediation for [crowdsecurity/http-bad-user-agent]" bucket_id=cool-smoke event_time="2020-07-07 16:13:16.579581642 +0200 CEST m=+358819.413561109" scenario=crowdsecurity/http-bad-user-agent source_ip=80.x.x.x -time="07-07-2020 16:13:16" level=warning msg="80.x.x.x triggered a 4h0m0s ip ban remediation for [crowdsecurity/http-probing]" bucket_id=green-silence event_time="2020-07-07 16:13:16.737579458 +0200 CEST m=+358819.571558901" scenario=crowdsecurity/http-probing source_ip=80.x.x.x -time="07-07-2020 16:13:17" level=warning msg="80.x.x.x triggered a 4h0m0s ip ban remediation for [crowdsecurity/http-crawl-non_statics]" bucket_id=purple-snowflake event_time="2020-07-07 16:13:17.353641625 +0200 CEST m=+358820.187621068" scenario=crowdsecurity/http-crawl-non_statics source_ip=80.x.x.x -time="07-07-2020 16:13:18" level=warning msg="80.x.x.x triggered a 4h0m0s ip ban remediation for [crowdsecurity/http-sensitive-files]" bucket_id=small-hill event_time="2020-07-07 16:13:18.005919055 +0200 CEST m=+358820.839898498" scenario=crowdsecurity/http-sensitive-files source_ip=80.x.x.x -^C -$ {{v1X.cli.bin}} ban list -4 local decisions: -+--------+---------------+-----------------------------------+------+--------+---------+---------------------------+--------+------------+ -| SOURCE | IP | REASON | BANS | ACTION | COUNTRY | AS | EVENTS | EXPIRATION | -+--------+---------------+-----------------------------------+------+--------+---------+---------------------------+--------+------------+ -| local | 80.x.x.x | crowdsecurity/http-bad-user-agent | 4 | ban | FR | 21502 SFR SA | 60 | 3h59m3s | -... - -``` - - -### Create the whitelist by IP - -Let's create a `/etc/crowdsec/parsers/s02-enrich/mywhitelists.yaml` file with the following content : - -```yaml -name: crowdsecurity/whitelists -description: "Whitelist events from my ip addresses" -whitelist: - reason: "my ip ranges" - ip: - - "80.x.x.x" -``` - -and reload {{v1X.crowdsec.name}} : `sudo systemctl reload crowdsec` - -### Test the whitelist - -Thus, if we restart our attack : - -```bash -nikto -host myfqdn.com -``` - -And we don't get bans : - -```bash -$ tail -f /var/log/crowdsec.log -... -^C -$ {{v1X.cli.bin}} ban list -No local decisions. -And 21 records from API, 15 distinct AS, 12 distinct countries - -``` - -Here, we don't get *any* logs, as the event have been discarded at parsing time. - - -## Create whitelist by expression - -Now, let's make something more tricky : let's whitelist a **specific** user-agent (of course, it's just an example, don't do this at home !). The [hub's taxonomy](https://hub.crowdsec.net/fields) will helps us to find which data is present in which field. - -Let's change our whitelist to : - -```yaml -name: crowdsecurity/whitelists -description: "Whitelist events from private ipv4 addresses" -whitelist: - reason: "private ipv4 ranges" - expression: - - evt.Parsed.http_user_agent == 'MySecretUserAgent' -``` - -again, let's restart {{v1X.crowdsec.name}} ! - -For the record, I edited nikto's configuration to use 'MySecretUserAgent' as user-agent, and thus : - -```bash -nikto -host myfqdn.com -``` - -```bash -$ tail -f /var/log/crowdsec.log -... -time="07-05-2020 09:39:09" level=info msg="Event is whitelisted by Expr !" filter= name=solitary-leaf stage=s02-enrich -... -``` - - -# Whitelist in PostOverflows - -Whitelists in PostOverflows are applied *after* the bucket overflow happens. -It has the advantage of being triggered only once we are about to take decision about an IP or Range, and thus happens a lot less often. - -A good example is the [crowdsecurity/whitelist-good-actors](https://hub.crowdsec.net/author/crowdsecurity/collections/whitelist-good-actors) collection. - -But let's craft ours based on our previous example ! -First of all, install the [crowdsecurity/rdns postoverflow](https://hub.crowdsec.net/author/crowdsecurity/configurations/rdns) : it will be in charge of enriching overflows with reverse dns information of the offending IP. - -Let's put the following file in `/etc/crowdsec/postoverflows/s01-whitelists/mywhitelists.yaml` : - -```yaml -name: me/my_cool_whitelist -description: lets whitelist our own reverse dns -whitelist: - reason: dont ban my ISP - expression: - #this is the reverse of my ip, you can get it by performing a "host" command on your public IP for example - - evt.Enriched.reverse_dns endsWith '.asnieres.rev.numericable.fr.' -``` - -After reloading {{v1X.crowdsec.name}}, and launching (again!) nikto : - -```bash -nikto -host myfqdn.com -``` - - -```bash -$ tail -f /var/log/crowdsec.log -ime="07-07-2020 17:11:09" level=info msg="Ban for 80.x.x.x whitelisted, reason [dont ban my ISP]" id=cold-sunset name=me/my_cool_whitelist stage=s01 -time="07-07-2020 17:11:09" level=info msg="node warning : no remediation" bucket_id=blue-cloud event_time="2020-07-07 17:11:09.175068053 +0200 CEST m=+2308.040825320" scenario=crowdsecurity/http-probing source_ip=80.x.x.x -time="07-07-2020 17:11:09" level=info msg="Processing Overflow with no decisions 80.x.x.x performed 'crowdsecurity/http-probing' (11 events over 313.983994ms) at 2020-07-07 17:11:09.175068053 +0200 CEST m=+2308.040825320" bucket_id=blue-cloud event_time="2020-07-07 17:11:09.175068053 +0200 CEST m=+2308.040825320" scenario=crowdsecurity/http-probing source_ip=80.x.x.x -... - -``` - -This time, we can see that logs are being produced when the event is discarded. - diff --git a/docs/v1.X/mkdocs.yml b/docs/v1.X/mkdocs.yml deleted file mode 100644 index 9ef3ecc43..000000000 --- a/docs/v1.X/mkdocs.yml +++ /dev/null @@ -1,80 +0,0 @@ -site_name: Crowdsec/v1 -nav: - - Home: index.md - - Getting Started: -# - Glossary: getting_started/glossary.md - - Concepts : getting_started/concepts.md - - Install Crowdsec : getting_started/installation.md - - Upgrade Crowdsec: getting_started/upgrades.md - - Crowdsec Tour: getting_started/crowdsec-tour.md - - User guide: - - CLI: user_guide/cscli.md - - Configurations management: - - Acquisition: user_guide/configurations_management/acquisition.md - - Collections: user_guide/configurations_management/collections.md - - Parsers: user_guide/configurations_management/parsers.md - - Enrichers: user_guide/configurations_management/enrichers.md - - Scenarios: user_guide/configurations_management/scenarios.md - - Decisions management: user_guide/decision_management.md - - Bouncers & machines management: user_guide/bouncer_machine_management.md - - Databases: user_guide/database.md - - Network management: user_guide/network.md - - Simulation management: user_guide/simulation_mode.md - - Crowdsec forensic mode: user_guide/forensic_mode.md - - Debugging: user_guide/debugging_configs.md - - CLI: - - Cscli: cscli/cscli.md - - Alerts: cscli/cscli_alerts.md - - Bouncers: cscli/cscli_bouncers.md - - Collections: cscli/cscli_collections.md - - Completion: cscli/cscli_completion.md - - Config: cscli/cscli_config.md - - Dashboard: cscli/cscli_dashboard.md - - Decisions: cscli/cscli_decisions.md - - Hub: cscli/cscli_hub.md - - Machines: cscli/cscli_machines.md - - Metrics: cscli/cscli_metrics.md - - Parsers: cscli/cscli_parsers.md - - Postoverflows: cscli/cscli_postoverflows.md - - Scenarios: cscli/cscli_scenarios.md - - Simulation: cscli/cscli_simulation.md - - Local API: - - Introduction: localAPI/index.md - - API Guide: localAPI/howto.md - - Swagger: https://crowdsecurity.github.io/api_doc/index.html?urls.primaryName=LAPI" target="_blank - - Observability: - - Overview: observability/overview.md - - Logs: observability/logs.md - - Metrics: - - Prometheus: observability/prometheus.md - - Command line: observability/command_line.md - - Dashboard: observability/dashboard.md - - Bouncers: bouncers/index.md - - References: - - Parsers, Scenarios etc.: - - Stages format: references/stages.md - - Parsers format: references/parsers.md - - Scenarios format: references/scenarios.md - - PostOverlows format: references/postoverflows.md - - Enrichers format: references/enrichers.md - - Collections format: references/collections.md - - Expressions helpers: references/expressions.md - - Patterns references: references/patterns-documentation.md - - Configuration files: - - Configuration format: references/crowdsec-config.md - - Database format: references/database.md - - Acquisition format: references/acquisition.md - - Profiles format: references/profiles.md - - Simulation configuration: references/simulation.md - - Runtime objects: - - Event object: references/events.md - - Alert object: references/alerts.md - - Decision object: references/decisions.md - - Write Configurations: - - Requirements: write_configurations/requirements.md - - Acquisition: write_configurations/acquisition.md - - Parsers: write_configurations/parsers.md - - Scenarios: write_configurations/scenarios.md - - Whitelists: write_configurations/whitelist.md - - Upgrade V0.X to V1.X: migration.md - diff --git a/mkdocs.yml b/mkdocs.yml deleted file mode 100644 index d4fcd3ead..000000000 --- a/mkdocs.yml +++ /dev/null @@ -1,363 +0,0 @@ -site_name: Crowdsec -nav: - - Home: index.md - - Crowdsec v0: "!include ./docs/v0.3.X/mkdocs.yml" - - Crowdsec v1 : "!include ./docs/v1.X/mkdocs.yml" - - API Swagger : https://crowdsecurity.github.io/api_doc/index.html?urls.primaryName=LAPI" target="_blank - - Hub : https://hub.crowdsec.net/" target="_blank - - Releases : https://github.com/crowdsecurity/crowdsec/releases" target="_blank - - Contributing: - - Guide: contributing.md - - FAQ: - - Questions: faq.md - - -markdown_extensions: - - codehilite: - guess_lang: false - - toc: - permalink: true - - admonition -theme: - name: material - logo: assets/images/crowdsec2.png - favicon: assets/images/crowdsec2.png - features: - - navigation.tabs - - navigation.expand - - navigation.instant - - navigation.section - palette: - primary: "#3d85c6" - language: en - font: - text: Montserrat - highlightjs: true - hljs_languages: - - yaml -repo_url: https://github.com/crowdsecurity/crowdsec -repo_name: GitHub -plugins: - - search - - macros - - monorepo - -extra: - swagger_url: "https://raw.githubusercontent.com/crowdsecurity/crowdsec/master/pkg/models/localapi_swagger.yaml" - v0X: - doc: - new_issue: "[new documentation issue](https://github.com/crowdsecurity/crowdsec/issues/new)" - discourse: "[CrowdSecurity discourse](http://discourse.crowdsec.net)" - gitter: "[Crowdsec gitter](http://discourse.crowdsec.net)" - hub: - name: Crowdsec Hub - htmlname: "[Crowdsec Hub](https://hub.crowdsec.net/)" - url: "https://hub.crowdsec.net/" - plugins_url: "https://hub.crowdsec.net/browse/#bouncers" - scenarios_url: "https://hub.crowdsec.net/browse/#configurations" - parsers_url: "https://hub.crowdsec.net/browse/#configurations" - collections_url: "https://hub.crowdsec.net/browse/#collections" - crowdsec: - name: Crowdsec - Name: Crowdsec - bin: crowdsec-agent - path: /usr/bin/crowdsec-agent - url: https://github.com/crowdsecurity/crowdsec - bugreport: "https://github.com/crowdsecurity/crowdsec/issues" - main_log: "/var/log/crowdsec.log" - download_url: https://github.com/crowdsecurity/crowdsec/releases - cli: - name: cscli - Name: cscli - main_doc: /Crowdsec/v0/cscli/cscli/ - url: "https://github.com/crowdsecurity/crowdsec" - bugreport: "https://github.com/crowdsecurity/crowdsec/issues" - # alerts_doc: /Crowdsec/v1/cscli/cscli_alerts/ - # decisions_doc: /Crowdsec/v1/cscli/cscli_decisions/ - # collections_doc: /Crowdsec/v1/cscli/cscli_collections/ - # parsers_doc: /Crowdsec/v1/cscli/cscli_parsers/ - # scenarios_doc: /Crowdsec/v1/cscli/cscli_scenarios/ - - # api_doc: /Crowdsec/v0/cscli/cscli_api/ - # ban_doc: /Crowdsec/v0/cscli/cscli_ban/ - # metrics_doc: /Crowdsec/v0/cscli/cscli_metrics/ - # remove_doc: /Crowdsec/v0/cscli/cscli_remove/ - # install_doc: /Crowdsec/v0/cscli/cscli_install/ - # list_doc: /Crowdsec/v0/cscli/cscli_list/ - # update_doc: /Crowdsec/v0/cscli/cscli_update/ - # upgrade_doc: /Crowdsec/v0/cscli/cscli_upgrade/ - # backup_doc: /Crowdsec/v0/cscli/cscli_backup/ - # simulation_doc: /Crowdsec/v0/cscli/cscli_simulation/ - config: - cli_dir: /etc/crowdsec/cscli/ - crowdsec_dir: "/etc/crowdsec/config/" - acquis_path: "/etc/crowdsec/config/acquis.yaml" - bouncers: - name: bouncers - Name: bouncers - url: "https://hub.crowdsec.net/browse/#bouncers" - htmlname: "[bouncers](/Crowdsec/v0/bouncers/)" - Htmlname: "[Bouncers](/Crowdsec/v0/bouncers/)" - plugins: - name: backend plugins - configpath: "/etc/crowdsec/plugins/backend/" - binpath: "/usr/local/lib/crowdsec/plugins/" - metabase: - name: metabase - url: https://github.com/crowdsecurity/crowdsec - wizard: - name: wizard - url: "https://github.com/crowdsecurity/crowdsec" - bin: "./wizard.sh" - bugreport: "https://github.com/crowdsecurity/crowdsec/issues" - ref: - parser: "[parser](/Crowdsec/v0/references/parsers/)" - Parser: "[Parser](/Crowdsec/v0/references/parsers/)" - scenario: "[scenarios](/Crowdsec/v0/references/scenarios/)" - Scenario: "[Scenarios](/Crowdsec/v0/references/scenarios/)" - acquis: "[acquisition](/Crowdsec/v0/guide/crowdsec/acquisition/)" - Acquis: "[Acquisition](/Crowdsec/v0/guide/crowdsec/acquisition/)" - output: "[output](/Crowdsec/v0/references/output/)" - Output: "[Output](/Crowdsec/v0/references/output/)" - # All the technical terms - event: - name: event - Name: Event - htmlname: "[event](/Crowdsec/v0/getting_started/concepts/#event)" - Htmlname: "[Event](/Crowdsec/v0/getting_started/concepts/#event)" - expr: - name: expr - Name: Expr - htmlname: "[expr](/Crowdsec/v0/write_configurations/expressions/)" - Htmlname: "[Expr](/Crowdsec/v0/write_configurations/expressions/)" - filter: - name: filter - Name: Filter - htmlname: "[filter](/Crowdsec/v0/references/parsers/#filter)" - Htmlname: "[Filter](/Crowdsec/v0/references/parsers/#filter)" - onsuccess: - name: onsuccess - Name: Onsuccess - htmlname: "[onsuccess](/Crowdsec/v0/references/parsers/#onsuccess)" - Htmlname: "[Onsuccess](/Crowdsec/v0/references/parsers/#onsuccess)" - statics: - name: statics - Name: Statics - htmlname: "[statics](/Crowdsec/v0/references/parsers/#statics)" - Htmlname: "[Statics](/Crowdsec/v0/references/parsers/#statics)" - parsers: - name: parsers - Name: Parsers - htmlname: "[parsers](/Crowdsec/v0/getting_started/concepts/#parsers)" - Htmlname: "[Parsers](/Crowdsec/v0/getting_started/concepts/#parsers)" - scenarios: - name: scenarios - Name: Scenarios - htmlname: "[scenarios](/Crowdsec/v0/getting_started/concepts/#scenarios)" - Htmlname: "[Scenarios](/Crowdsec/v0/getting_started/concepts/#scenarios)" - collections: - name: collections - Name: Collections - htmlname: "[collections](/Crowdsec/v0/getting_started/concepts/#collections)" - Htmlname: "[Collections](/Crowdsec/v0/getting_started/concepts/#collections)" - timeMachine: - name: timeMachine - Name: TimeMachine - htmlname: "[timeMachine](/Crowdsec/v0/getting_started/concepts/#timemachine)" - Htmlname: "[TimeMachine](/Crowdsec/v0/getting_started/concepts/#timemachine)" - overflow: - name: overflow - Name: Overflow - htmlname: "[overflow](/Crowdsec/v0/getting_started/concepts/#overflow-or-signaloccurence)" - Htmlname: "[Overflow](/Crowdsec/v0/getting_started/concepts/#overflow-or-signaloccurence)" - whitelists: - name: whitelists - Name: Whitelists - htmlname: "[whitelists](/Crowdsec/v0/write_configurations/whitelist/)" - Htmlname: "[Whitelists](/Crowdsec/v0/write_configurations/whitelist/)" - signal: - name: signal - Name: Signal - htmlname: "[signal](/Crowdsec/v0/getting_started/concepts/#overflow-or-signaloccurence)" - Htmlname: "[Signal](/Crowdsec/v0/getting_started/concepts/#overflow-or-signaloccurence)" - #scenario stuff - stage: - name: stage - Name: Stage - htmlname: "[stage](/Crowdsec/v0/getting_started/concepts/#stages)" - Htmlname: "[Stage](/Crowdsec/v0/getting_started/concepts/#stages)" - leakspeed: - name: leakspeed - Name: Leakspeed - htmlname: "[leakspeed](/Crowdsec/v0/references/scenarios/#leakspeed)" - Htmlname: "[Leakspeed](/Crowdsec/v0/references/scenarios/#leakspeed)" - capacity: - name: capacity - Name: Capacity - htmlname: "[capacity](/Crowdsec/v0/references/scenarios/#capacity)" - Htmlname: "[Capacity](/Crowdsec/v0/references/scenarios/#capacity)" - duration: - name: duration - Name: Duration - htmlname: "[duration](/Crowdsec/v0/references/scenarios/#duration)" - Htmlname: "[Duration](/Crowdsec/v0/references/scenarios/#duration)" - prometheus: - name: prometheus - htmlname: "[prometheus](https://github.com/prometheus/client_golang)" - api: - name: API - htmlname: "[API](TBD)" - topX: - name: topX - htmlname: "[topX](TBD)" -############################################################### -### Bellow are defines for V1 and later of the documentation ## -############################################################### - v1X: - doc: - new_issue: "[new documentation issue](https://github.com/crowdsecurity/crowdsec/issues/new)" - discourse: "[CrowdSecurity discourse](http://discourse.crowdsec.net)" - community: "[community](http://discourse.crowdsec.net)" - hub: - name: Crowdsec Hub - htmlname: "[Crowdsec Hub](https://hub.crowdsec.net/)" - url: "https://hub.crowdsec.net/" - bouncers_url: "https://hub.crowdsec.net/browse/#bouncers" - scenarios_url: "https://hub.crowdsec.net/browse/#configurations" - parsers_url: "https://hub.crowdsec.net/browse/#configurations" - collections_url: "https://hub.crowdsec.net/browse/#collections" - crowdsec: - name: crowdsec-agent - Name: Crowdsec-agent - url: https://github.com/crowdsecurity/crowdsec - bugreport: "https://github.com/crowdsecurity/crowdsec/issues" - download_url: https://github.com/crowdsecurity/crowdsec/releases - lapi: - name: local API (LAPI) - Name: Local API (LAPI) - Htmlname: "[Local API](/Crowdsec/v1/localAPI/)" - htmlname: "[local API](/Crowdsec/v1/localAPI/)" - url: /Crowdsec/v1/localAPI/ - swagger: https://crowdsecurity.github.io/api_doc/index.html?urls.primaryName=LAPI - cli: - name: cscli - Name: Cscli - bin: cscli - user_guide: "[cscli](/Crowdsec/v1/user_guide/cscli/)" - config: - crowdsec_dir: "/etc/crowdsec/" - acquis_path: "/etc/crowdsec/acquis.yaml" - crowdsec_config_file: "/etc/crowdsec/config.yaml" - bouncers: - name: bouncers - Name: Bouncers - url: "https://hub.crowdsec.net/browse/#bouncers" - htmlname: "[bouncers](/Crowdsec/v1/bouncers/)" - Htmlname: "[bouncers](/Crowdsec/v1/bouncers/)" - metabase: - name: metabase - htmlName: "[dashboard](/Crowdsec/v1/observability/dashboard)" - wizard: - name: wizard - bin: "./wizard.sh" - bugreport: "https://github.com/crowdsecurity/crowdsec/issues" - ref: - acquis: "[acquisition](/Crowdsec/v1/user_guide/configurations_management/acquisition/)" - parsers: "[parsers](/Crowdsec/v1/references/parsers/)" - scenarios: "[scenarios](/Crowdsec/v1/references/scenarios/)" - # All the technical terms - event: - name: event - Name: Event - htmlname: "[event](/Crowdsec/v1/getting_started/concepts/#events)" - Htmlname: "[Event](/Crowdsec/v1/getting_started/concepts/#events)" - expr: - name: expr - Name: Expr - htmlname: "[expr](/Crowdsec/v1/references/expressions/)" - Htmlname: "[Expr](/Crowdsec/v1/references/expressions/)" - filter: - name: filter - Name: Filter - htmlname: "[filter](/Crowdsec/v1/references/parsers/#filter)" - Htmlname: "[Filter](/Crowdsec/v1/references/parsers/#filter)" - onsuccess: - name: onsuccess - Name: Onsuccess - htmlname: "[onsuccess](/Crowdsec/v1/references/parsers/#onsuccess)" - Htmlname: "[Onsuccess](/Crowdsec/v1/references/parsers/#onsuccess)" - profiles: - htmlname: "[profiles](/Crowdsec/v1/references/profiles/)" - simulation: - htmlname: "[profiles](/Crowdsec/v1/references/simulation/)" - statics: - name: statics - Name: Statics - htmlname: "[statics](/Crowdsec/v1/references/parsers/#statics)" - Htmlname: "[Statics](/Crowdsec/v1/references/parsers/#statics)" - parsers: - name: parsers - Name: Parsers - htmlname: "[parsers](/Crowdsec/v1/getting_started/concepts/#parsers)" - Htmlname: "[Parsers](/Crowdsec/v1/getting_started/concepts/#parsers)" - scenarios: - name: scenarios - Name: Scenarios - htmlname: "[scenarios](/Crowdsec/v1/getting_started/concepts/#scenarios)" - Htmlname: "[Scenarios](/Crowdsec/v1/getting_started/concepts/#scenarios)" - collections: - name: collections - Name: Collections - htmlname: "[collections](/Crowdsec/v1/getting_started/concepts/#collections)" - Htmlname: "[Collections](/Crowdsec/v1/getting_started/concepts/#collections)" - timeMachine: - name: timeMachine - Name: TimeMachine - htmlname: "[timeMachine](/Crowdsec/v1/getting_started/concepts/#timemachine)" - Htmlname: "[TimeMachine](/Crowdsec/v1/getting_started/concepts/#timemachine)" - alert: - name: alert - Name: Alert - htmlname: "[alert](/Crowdsec/v1/getting_started/concepts/#alerts)" - Htmlname: "[Alert](/Crowdsec/v1/getting_started/concepts/#alerts)" - decision: - name: decision - Name: Decision - htmlname: "[decision](/Crowdsec/v1/getting_started/concepts/#decisions)" - Htmlname: "[Decision](/Crowdsec/v1/getting_started/concepts/#decisions)" - whitelists: - name: whitelists - Name: Whitelists - htmlname: "[whitelists](/Crowdsec/v1/write_configurations/whitelist/)" - Htmlname: "[Whitelists](/Crowdsec/v1/write_configurations/whitelist/)" - #scenario stuff - stage: - name: stage - Name: Stage - htmlname: "[stage](/Crowdsec/v1/getting_started/concepts/#stages)" - Htmlname: "[Stage](/Crowdsec/v1/getting_started/concepts/#stages)" - leakspeed: - name: leakspeed - Name: Leakspeed - htmlname: "[leakspeed](/Crowdsec/v1/references/scenarios/#leakspeed)" - Htmlname: "[Leakspeed](/Crowdsec/v1/references/scenarios/#leakspeed)" - capacity: - name: capacity - Name: Capacity - htmlname: "[capacity](/Crowdsec/v1/references/scenarios/#capacity)" - Htmlname: "[Capacity](/Crowdsec/v1/references/scenarios/#capacity)" - duration: - name: duration - Name: Duration - htmlname: "[duration](/Crowdsec/v1/references/scenarios/#duration)" - Htmlname: "[Duration](/Crowdsec/v1/references/scenarios/#duration)" - prometheus: - name: prometheus - htmlname: "[prometheus](https://github.com/prometheus/client_golang)" - api: - name: API - htmlname: "[API](TBD)" - topX: - name: topX - htmlname: "[topX](TBD)" -