From 3355c192988ee8ef0d838af07e89f03c5b9f6a47 Mon Sep 17 00:00:00 2001 From: Jeroen Date: Wed, 26 Aug 2026 14:27:44 +0200 Subject: [PATCH 01/18] add logstash and influxdb docker roles, fix strings in stats role --- roles/influxdb_docker/defaults/main.yml | 3 + roles/influxdb_docker/meta/main.yml | 35 ++++++ roles/influxdb_docker/tasks/main.yml | 68 ++++++++++++ roles/logstash_docker/defaults/main.yml | 3 + .../files/core/02-filebeat-input.conf | 6 ++ .../files/core/13-core-ebauth.conf | 33 ++++++ .../files/core/33-core-influxdb-output.conf | 48 +++++++++ roles/logstash_docker/files/jvm.options | 77 +++++++++++++ roles/logstash_docker/files/log4j2.properties | 101 ++++++++++++++++++ roles/logstash_docker/files/logstash.yml | 9 ++ roles/logstash_docker/files/pipelines.yml | 8 ++ roles/logstash_docker/meta/main.yml | 35 ++++++ roles/logstash_docker/tasks/main.yml | 56 ++++++++++ roles/stats/tasks/main.yml | 4 +- 14 files changed, 484 insertions(+), 2 deletions(-) create mode 100644 roles/influxdb_docker/defaults/main.yml create mode 100644 roles/influxdb_docker/meta/main.yml create mode 100644 roles/influxdb_docker/tasks/main.yml create mode 100644 roles/logstash_docker/defaults/main.yml create mode 100644 roles/logstash_docker/files/core/02-filebeat-input.conf create mode 100644 roles/logstash_docker/files/core/13-core-ebauth.conf create mode 100644 roles/logstash_docker/files/core/33-core-influxdb-output.conf create mode 100644 roles/logstash_docker/files/jvm.options create mode 100644 roles/logstash_docker/files/log4j2.properties create mode 100644 roles/logstash_docker/files/logstash.yml create mode 100644 roles/logstash_docker/files/pipelines.yml create mode 100644 roles/logstash_docker/meta/main.yml create mode 100644 roles/logstash_docker/tasks/main.yml diff --git a/roles/influxdb_docker/defaults/main.yml b/roles/influxdb_docker/defaults/main.yml new file mode 100644 index 000000000..923fcc3f2 --- /dev/null +++ b/roles/influxdb_docker/defaults/main.yml @@ -0,0 +1,3 @@ +#SPDX-License-Identifier: MIT-0 +--- +# defaults file for influxdbcontainer diff --git a/roles/influxdb_docker/meta/main.yml b/roles/influxdb_docker/meta/main.yml new file mode 100644 index 000000000..6f91fd335 --- /dev/null +++ b/roles/influxdb_docker/meta/main.yml @@ -0,0 +1,35 @@ +#SPDX-License-Identifier: MIT-0 +galaxy_info: + author: your name + description: your role description + company: your company (optional) + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Choose a valid license ID from https://spdx.org - some suggested licenses: + # - BSD-3-Clause (default) + # - MIT + # - GPL-2.0-or-later + # - GPL-3.0-only + # - Apache-2.0 + # - CC-BY-4.0 + license: license (GPL-2.0-or-later, MIT, etc) + + min_ansible_version: 2.2 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. diff --git a/roles/influxdb_docker/tasks/main.yml b/roles/influxdb_docker/tasks/main.yml new file mode 100644 index 000000000..b6886caf6 --- /dev/null +++ b/roles/influxdb_docker/tasks/main.yml @@ -0,0 +1,68 @@ +--- +- name: Create influx user + ansible.builtin.user: + name: influxdb + uid: 1500 + +- name: Ensure required host directories exists + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: "influxdb" + group: "influxdb" + mode: "0755" + with_items: + - "/etc/influxdb" + - "/var/lib/influxdb" + +- name: Create influx container + community.docker.docker_container: + name: "influxdb" + image: "influxdb:1.12" + state: started + restart_policy: "always" + ports: + - "8086:8086" + mounts: + - source: /var/lib/influxdb + target: /var/lib/influxdb + type: bind + - source: /etc/influxdb + target: /etc/influxdb + type: bind + env: + INFLUXDB_DB: "{{ influxdb_stats_db }}" + INFLUXDB_ADMIN_USER: "{{ influxdb_admin_user }}" + INFLUXDB_ADMIN_PASSWORD: "{{ influxdb_admin_password }}" + INFLUXDB_READ_USER: "{{ influxdb_stats_user }}" + INFLUXDB_READ_USER_PASSWORD: "{{ influxdb_stats_password }}" + INFLUXDB_WRITE_USER: "{{ influxdb_stats_user }}" + INFLUXDB_WRITE_USER_PASSWORD: "{{ influxdb_stats_password }}" + INFLUXDB_HTTP_AUTH_ENABLED: "True" + +- name: Wait for influx to start + ansible.builtin.wait_for: + port: 8086 + host: localhost + state: started + delay: 5 + +- name: Wait for infludb init script to run + ansible.builtin.pause: + seconds: 20 + +- name: Grant stats user required access to InfluxDB database + ansible.builtin.uri: + url: "http://{{ influxdb_host }}:8086/query" + method: POST + url_username: "{{ influxdb_admin_user }}" + url_password: "{{ influxdb_admin_password }}" + force_basic_auth: true + body_format: form-urlencoded + body: + q: 'GRANT ALL PRIVILEGES TO "{{ influxdb_stats_user }}"' + status_code: 200 + return_content: true + register: influxdb_grant + changed_when: false + #no_log: true diff --git a/roles/logstash_docker/defaults/main.yml b/roles/logstash_docker/defaults/main.yml new file mode 100644 index 000000000..9cc6ec2cb --- /dev/null +++ b/roles/logstash_docker/defaults/main.yml @@ -0,0 +1,3 @@ +#SPDX-License-Identifier: MIT-0 +--- +# defaults file for logstash diff --git a/roles/logstash_docker/files/core/02-filebeat-input.conf b/roles/logstash_docker/files/core/02-filebeat-input.conf new file mode 100644 index 000000000..daeb4a214 --- /dev/null +++ b/roles/logstash_docker/files/core/02-filebeat-input.conf @@ -0,0 +1,6 @@ +input { + beats { + port => 5044 + type => "log" + } +} diff --git a/roles/logstash_docker/files/core/13-core-ebauth.conf b/roles/logstash_docker/files/core/13-core-ebauth.conf new file mode 100644 index 000000000..5d25ca794 --- /dev/null +++ b/roles/logstash_docker/files/core/13-core-ebauth.conf @@ -0,0 +1,33 @@ +filter { + if [prog] == "EBAUTH" { + grok { + patterns_dir => "/etc/logstash/patterns" + match => { "message" => "%{SYSLOGBASE:syslogbase} %{GREEDYDATA:ebauth}" } + } +} + json { + source => "ebauth" + target => "ebauth" +} + date { + match => [ "[ebauth][context][login_stamp]", "yyyy-MM-dd'T'HH:mm:ssZZ" ] +} + de_dot { +} + mutate { + remove_field => [ "source" ] + remove_field => [ "syslogbase" ] + add_field => {"[month]" => "%{+MM}"} + add_field => {"[year]" => "%{+YYYY}"} +} + fingerprint { + key => eighiePhie3xiu3boChaekoomai6iiYilux1zoohoh6ahgh9iaphuthaero4ahme + source => [ "[ebauth][context][user_id]" ] + method => SHA256 + target => [ "[ebauth][context][user_id_hashed]" ] +} + ruby { + code => 'quarter = (event.get("month").to_i / 3.0).ceil; + event.set("quarter",quarter)' +} +} diff --git a/roles/logstash_docker/files/core/33-core-influxdb-output.conf b/roles/logstash_docker/files/core/33-core-influxdb-output.conf new file mode 100644 index 000000000..15950c952 --- /dev/null +++ b/roles/logstash_docker/files/core/33-core-influxdb-output.conf @@ -0,0 +1,48 @@ +filter { + if [prog] == "EBAUTH" { + ruby { + code => 'require "date"; + d = DateTime.parse(event.get("[ebauth][context][login_stamp]")); + ds = ((d.strftime("%Q").to_i * 1000) + d.strftime("%6N")[3,3].to_i).to_s; + event.set("[ebauth][context][micro_seconds]", ds); + if (event.get("[ebauth][context][proxied_sp_entity_ids][0]")) ; + proxied_sp_entity_id = (event.get("[ebauth][context][proxied_sp_entity_ids][0]")); + event.set("[ebauth][context][sp_entity_id]", proxied_sp_entity_id) + end' + } + } +} +filter { + if [prog] == "EBAUTH" and [ebauth][context][authncontextclassref] not in ["http://surfconext.nl/assurance/loa1_5","http://surfconext.nl/assurance/loa2","http://surfconext.nl/assurance/loa3"] { + mutate + { replace => { "[ebauth][context][authncontextclassref]" => "http://surfconext.nl/assurance/loa1"} + } + } +} + +output { + if [prog] == "EBAUTH" { + influxdb { + host => "localhost" + port => 8086 + db => "log_logins" + measurement => "EBAUTH" + retention_policy => "autogen" + use_event_fields_for_data_points => false + allow_time_override => true + time_precision => "u" + data_points => { + "idp_entity_id" => "%{[ebauth][context][idp_entity_id]}" + "user_id" => "%{[ebauth][context][user_id_hashed]}" + "sp_entity_id" => "%{[ebauth][context][sp_entity_id]}" + "state" => "%{[ebauth][context][workflow_state]}" + "authncontextclassref" => "%{[ebauth][context][authncontextclassref]}" + "time" => "%{[ebauth][context][micro_seconds]}" + "month" => "%{+MM}" + "year" => "%{+YYYY}" + "quarter" => "%{[quarter]}" + } + send_as_tags => ["idp_entity_id","sp_entity_id","state","month","year","quarter","authncontextclassref"] + } + } +} diff --git a/roles/logstash_docker/files/jvm.options b/roles/logstash_docker/files/jvm.options new file mode 100644 index 000000000..458f0ec68 --- /dev/null +++ b/roles/logstash_docker/files/jvm.options @@ -0,0 +1,77 @@ +# JVM configuration + +# Xms represents the initial size of total heap space +# Xmx represents the maximum size of total heap space + +-Xms2g +-Xmx2g + +################################################################ +## Expert settings +################################################################ +## +## All settings below this section are considered +## expert settings. Don't tamper with them unless +## you understand what you are doing +## +################################################################ + +## GC configuration +# -XX:+UseParNewGC #Option does not exist +# -XX:+UseConcMarkSweepGC +# -XX:CMSInitiatingOccupancyFraction=75 +# -XX:+UseCMSInitiatingOccupancyOnly + +## Locale +# Set the locale language +#-Duser.language=en + +# Set the locale country +#-Duser.country=US + +# Set the locale variant, if any +#-Duser.variant= + +## basic + +# set the I/O temp directory +#-Djava.io.tmpdir=$HOME + +# set to headless, just in case +-Djava.awt.headless=true + +# ensure UTF-8 encoding by default (e.g. filenames) +-Dfile.encoding=UTF-8 + +# use our provided JNA always versus the system one +#-Djna.nosys=true + +# Turn on JRuby invokedynamic +-Djruby.compile.invokedynamic=true +# Force Compilation +-Djruby.jit.threshold=0 + +## heap dumps + +# generate a heap dump when an allocation from the Java heap fails +# heap dumps are created in the working directory of the JVM +-XX:+HeapDumpOnOutOfMemoryError + +# specify an alternative path for heap dumps +# ensure the directory exists and has sufficient space +#-XX:HeapDumpPath=${LOGSTASH_HOME}/heapdump.hprof + +## GC logging +#-XX:+PrintGCDetails +#-XX:+PrintGCTimeStamps +#-XX:+PrintGCDateStamps +#-XX:+PrintClassHistogram +#-XX:+PrintTenuringDistribution +#-XX:+PrintGCApplicationStoppedTime + +# log GC status to a file with time stamps +# ensure the directory exists +#-Xloggc:${LS_GC_LOG_FILE} + +# Entropy source for randomness +-Djava.security.egd=file:/dev/urandom diff --git a/roles/logstash_docker/files/log4j2.properties b/roles/logstash_docker/files/log4j2.properties new file mode 100644 index 000000000..683a51e0a --- /dev/null +++ b/roles/logstash_docker/files/log4j2.properties @@ -0,0 +1,101 @@ +tatus = error +name = LogstashPropertiesConfig + +appender.console.type = Console +appender.console.name = plain_console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n + +appender.json_console.type = Console +appender.json_console.name = json_console +appender.json_console.layout.type = JSONLayout +appender.json_console.layout.compact = true +appender.json_console.layout.eventEol = true + +appender.rolling.type = RollingFile +appender.rolling.name = plain_rolling +appender.rolling.fileName = ${sys:ls.logs}/logstash-${sys:ls.log.format}.log +appender.rolling.filePattern = ${sys:ls.logs}/logstash-${sys:ls.log.format}-%d{yyyy-MM-dd}-%i.log.gz +appender.rolling.policies.type = Policies +appender.rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling.policies.time.interval = 1 +appender.rolling.policies.time.modulate = true +appender.rolling.layout.type = PatternLayout +appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %-.10000m%n +appender.rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling.policies.size.size = 100MB +appender.rolling.strategy.type = DefaultRolloverStrategy +appender.rolling.strategy.max = 30 + +appender.json_rolling.type = RollingFile +appender.json_rolling.name = json_rolling +appender.json_rolling.fileName = ${sys:ls.logs}/logstash-${sys:ls.log.format}.log +appender.json_rolling.filePattern = ${sys:ls.logs}/logstash-${sys:ls.log.format}-%d{yyyy-MM-dd}-%i.log.gz +appender.json_rolling.policies.type = Policies +appender.json_rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.json_rolling.policies.time.interval = 1 +appender.json_rolling.policies.time.modulate = true +appender.json_rolling.layout.type = JSONLayout +appender.json_rolling.layout.compact = true +appender.json_rolling.layout.eventEol = true +appender.json_rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.json_rolling.policies.size.size = 100MB +appender.json_rolling.strategy.type = DefaultRolloverStrategy +appender.json_rolling.strategy.max = 30 + +rootLogger.level = ${sys:ls.log.level} +rootLogger.appenderRef.console.ref = ${sys:ls.log.format}_console +rootLogger.appenderRef.rolling.ref = ${sys:ls.log.format}_rolling + +# Slowlog + +appender.console_slowlog.type = Console +appender.console_slowlog.name = plain_console_slowlog +appender.console_slowlog.layout.type = PatternLayout +appender.console_slowlog.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n + +appender.json_console_slowlog.type = Console +appender.json_console_slowlog.name = json_console_slowlog +appender.json_console_slowlog.layout.type = JSONLayout +appender.json_console_slowlog.layout.compact = true +appender.json_console_slowlog.layout.eventEol = true + +appender.rolling_slowlog.type = RollingFile +appender.rolling_slowlog.name = plain_rolling_slowlog +appender.rolling_slowlog.fileName = ${sys:ls.logs}/logstash-slowlog-${sys:ls.log.format}.log +appender.rolling_slowlog.filePattern = ${sys:ls.logs}/logstash-slowlog-${sys:ls.log.format}-%d{yyyy-MM-dd}-%i.log.gz +appender.rolling_slowlog.policies.type = Policies +appender.rolling_slowlog.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling_slowlog.policies.time.interval = 1 +appender.rolling_slowlog.policies.time.modulate = true +appender.rolling_slowlog.layout.type = PatternLayout +appender.rolling_slowlog.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %.10000m%n +appender.rolling_slowlog.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling_slowlog.policies.size.size = 100MB +appender.rolling_slowlog.strategy.type = DefaultRolloverStrategy +appender.rolling_slowlog.strategy.max = 30 + +appender.json_rolling_slowlog.type = RollingFile +appender.json_rolling_slowlog.name = json_rolling_slowlog +appender.json_rolling_slowlog.fileName = ${sys:ls.logs}/logstash-slowlog-${sys:ls.log.format}.log +appender.json_rolling_slowlog.filePattern = ${sys:ls.logs}/logstash-slowlog-${sys:ls.log.format}-%d{yyyy-MM-dd}-%i.log.gz +appender.json_rolling_slowlog.policies.type = Policies +appender.json_rolling_slowlog.policies.time.type = TimeBasedTriggeringPolicy +appender.json_rolling_slowlog.policies.time.interval = 1 +appender.json_rolling_slowlog.policies.time.modulate = true +appender.json_rolling_slowlog.layout.type = JSONLayout +appender.json_rolling_slowlog.layout.compact = true +appender.json_rolling_slowlog.layout.eventEol = true +appender.json_rolling_slowlog.policies.size.type = SizeBasedTriggeringPolicy +appender.json_rolling_slowlog.policies.size.size = 100MB +appender.json_rolling_slowlog.strategy.type = DefaultRolloverStrategy +appender.json_rolling_slowlog.strategy.max = 30 + +logger.slowlog.name = slowlog +logger.slowlog.level = trace +logger.slowlog.appenderRef.console_slowlog.ref = ${sys:ls.log.format}_console_slowlog +logger.slowlog.appenderRef.rolling_slowlog.ref = ${sys:ls.log.format}_rolling_slowlog +logger.slowlog.additivity = false + +logger.licensereader.name = logstash.licensechecker.licensereader +logger.licensereader.level = error diff --git a/roles/logstash_docker/files/logstash.yml b/roles/logstash_docker/files/logstash.yml new file mode 100644 index 000000000..086fd5073 --- /dev/null +++ b/roles/logstash_docker/files/logstash.yml @@ -0,0 +1,9 @@ +api.http.host: 0.0.0.0 +monitoring.elasticsearch.hosts: http://elasticsearch:9200 +pipeline.workers: 1 +queue.type: persisted +queue.max_bytes: 4gb +dead_letter_queue.enable: true +dead_letter_queue.max_bytes: 1024mb +path.logs: /var/log/logstash +path.data: /data/logstash diff --git a/roles/logstash_docker/files/pipelines.yml b/roles/logstash_docker/files/pipelines.yml new file mode 100644 index 000000000..5d53e7fe0 --- /dev/null +++ b/roles/logstash_docker/files/pipelines.yml @@ -0,0 +1,8 @@ +- pipeline.id: core + path.config: "/etc/logstash/conf.d/core" + pipeline.workers: 2 + queue.type: persisted +#- pipeline.id: sa +# path.config: "/etc/logstash/conf.d/sa" +# pipeline.workers: 2 +# queue.type: persisted diff --git a/roles/logstash_docker/meta/main.yml b/roles/logstash_docker/meta/main.yml new file mode 100644 index 000000000..6f91fd335 --- /dev/null +++ b/roles/logstash_docker/meta/main.yml @@ -0,0 +1,35 @@ +#SPDX-License-Identifier: MIT-0 +galaxy_info: + author: your name + description: your role description + company: your company (optional) + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Choose a valid license ID from https://spdx.org - some suggested licenses: + # - BSD-3-Clause (default) + # - MIT + # - GPL-2.0-or-later + # - GPL-3.0-only + # - Apache-2.0 + # - CC-BY-4.0 + license: license (GPL-2.0-or-later, MIT, etc) + + min_ansible_version: 2.2 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. diff --git a/roles/logstash_docker/tasks/main.yml b/roles/logstash_docker/tasks/main.yml new file mode 100644 index 000000000..f9feb6de5 --- /dev/null +++ b/roles/logstash_docker/tasks/main.yml @@ -0,0 +1,56 @@ +--- +- name: Create required directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: 1000 + group: 1000 + mode: 0750 + with_items: + - /opt/logstash/config + - /opt/logstash/conf.d/core + - /data/logstash + +- name: Place logstash settings files + ansible.builtin.copy: + src: "files/{{ item }}" + dest: "/opt/logstash/config/{{ item }}" + owner: 1000 + group: 1000 + mode: 0644 + with_items: + - logstash.yml + - jvm.options + - pipelines.yml + - log4j2.properties + +- name: Place core config files + ansible.builtin.copy: + src: "files/core/{{ item }}" + dest: "/opt/logstash/conf.d/core/{{ item }}" + owner: 1000 + group: 1000 + mode: 0644 + with_items: + - 02-filebeat-input.conf + - 13-core-ebauth.conf + - 33-core-influxdb-output.conf + +- name: Create logstash container + community.docker.docker_container: + name: "logstash" + image: "ghcr.io/openconext/openconext-basecontainers/logstash8:latest" + state: started + restart_policy: "always" + ports: + - 5044:5044 + mounts: + - source: /opt/logstash/config/ + target: /usr/share/logstash/config/ + type: bind + - source: /opt/logstash/conf.d/ + target: /etc/logstash/conf.d/ + type: bind + - source: /data/logstash/ + target: /data/logstash/ + type: bind diff --git a/roles/stats/tasks/main.yml b/roles/stats/tasks/main.yml index f6e1d159f..9f9e80c7d 100644 --- a/roles/stats/tasks/main.yml +++ b/roles/stats/tasks/main.yml @@ -70,8 +70,8 @@ restart_policy: "always" state: "started" env: - RUNAS_UID: "{{ stats_uid.uid }}" - RUNAS_GID: "{{ stats_gid.gid }}" + RUNAS_UID: "{{ stats_uid.uid | string }}" + RUNAS_GID: "{{ stats_gid.gid | string }}" networks: - name: "loadbalancer" mounts: From 41af5313fb5601e82843aada8340b5377fadc434 Mon Sep 17 00:00:00 2001 From: Jeroen Date: Wed, 26 Aug 2026 14:47:20 +0200 Subject: [PATCH 02/18] correct stats db variable name --- roles/influxdb_docker/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/influxdb_docker/tasks/main.yml b/roles/influxdb_docker/tasks/main.yml index b6886caf6..2944e7e1c 100644 --- a/roles/influxdb_docker/tasks/main.yml +++ b/roles/influxdb_docker/tasks/main.yml @@ -31,7 +31,7 @@ target: /etc/influxdb type: bind env: - INFLUXDB_DB: "{{ influxdb_stats_db }}" + INFLUXDB_DB: "{{ influx_stats_db }}" INFLUXDB_ADMIN_USER: "{{ influxdb_admin_user }}" INFLUXDB_ADMIN_PASSWORD: "{{ influxdb_admin_password }}" INFLUXDB_READ_USER: "{{ influxdb_stats_user }}" From 1c123e113cf221ef33f2608a46ff4d3cb8788162 Mon Sep 17 00:00:00 2001 From: Jeroen Date: Wed, 26 Aug 2026 14:51:04 +0200 Subject: [PATCH 03/18] set default for influxdb host --- roles/influxdb_docker/defaults/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/influxdb_docker/defaults/main.yml b/roles/influxdb_docker/defaults/main.yml index 923fcc3f2..30bf7d9f8 100644 --- a/roles/influxdb_docker/defaults/main.yml +++ b/roles/influxdb_docker/defaults/main.yml @@ -1,3 +1,2 @@ -#SPDX-License-Identifier: MIT-0 --- -# defaults file for influxdbcontainer +influxdb_host: "localhost" From e4a83cb8f805c543d8d51851d7ca7ae49472d7c6 Mon Sep 17 00:00:00 2001 From: Jeroen Date: Thu, 27 Aug 2026 10:10:56 +0200 Subject: [PATCH 04/18] add container to the correct network --- roles/influxdb_docker/tasks/main.yml | 2 ++ roles/logstash_docker/tasks/main.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/roles/influxdb_docker/tasks/main.yml b/roles/influxdb_docker/tasks/main.yml index 2944e7e1c..32df1f236 100644 --- a/roles/influxdb_docker/tasks/main.yml +++ b/roles/influxdb_docker/tasks/main.yml @@ -21,6 +21,8 @@ image: "influxdb:1.12" state: started restart_policy: "always" + networks: + - name: "loadbalancer" ports: - "8086:8086" mounts: diff --git a/roles/logstash_docker/tasks/main.yml b/roles/logstash_docker/tasks/main.yml index f9feb6de5..aa5cba5a2 100644 --- a/roles/logstash_docker/tasks/main.yml +++ b/roles/logstash_docker/tasks/main.yml @@ -42,6 +42,8 @@ image: "ghcr.io/openconext/openconext-basecontainers/logstash8:latest" state: started restart_policy: "always" + networks: + - name: "loadbalancer" ports: - 5044:5044 mounts: From f28c396b2d9794f6ee9db8a84befa833a05abafa Mon Sep 17 00:00:00 2001 From: Jeroen Date: Thu, 27 Aug 2026 10:30:01 +0200 Subject: [PATCH 05/18] change logstash mountpoints --- roles/logstash_docker/tasks/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/roles/logstash_docker/tasks/main.yml b/roles/logstash_docker/tasks/main.yml index aa5cba5a2..2ae2ede47 100644 --- a/roles/logstash_docker/tasks/main.yml +++ b/roles/logstash_docker/tasks/main.yml @@ -7,14 +7,14 @@ group: 1000 mode: 0750 with_items: - - /opt/logstash/config - - /opt/logstash/conf.d/core + - /var/lib/logstash/config + - /var/lib/logstash/conf.d/core - /data/logstash - name: Place logstash settings files ansible.builtin.copy: src: "files/{{ item }}" - dest: "/opt/logstash/config/{{ item }}" + dest: "/var/lib/logstash/config/{{ item }}" owner: 1000 group: 1000 mode: 0644 @@ -27,7 +27,7 @@ - name: Place core config files ansible.builtin.copy: src: "files/core/{{ item }}" - dest: "/opt/logstash/conf.d/core/{{ item }}" + dest: "/var/lib/logstash/conf.d/core/{{ item }}" owner: 1000 group: 1000 mode: 0644 @@ -47,10 +47,10 @@ ports: - 5044:5044 mounts: - - source: /opt/logstash/config/ + - source: /var/lib/logstash/config/ target: /usr/share/logstash/config/ type: bind - - source: /opt/logstash/conf.d/ + - source: /var/lib/logstash/conf.d/ target: /etc/logstash/conf.d/ type: bind - source: /data/logstash/ From f2aa893a6b78a85cd20e26da9b7fb2694de539c7 Mon Sep 17 00:00:00 2001 From: Jeroen Date: Thu, 27 Aug 2026 10:40:11 +0200 Subject: [PATCH 06/18] make influxdb_docker role idempotent and remove unneeded options --- roles/influxdb_docker/tasks/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/influxdb_docker/tasks/main.yml b/roles/influxdb_docker/tasks/main.yml index 32df1f236..2482ae838 100644 --- a/roles/influxdb_docker/tasks/main.yml +++ b/roles/influxdb_docker/tasks/main.yml @@ -21,6 +21,7 @@ image: "influxdb:1.12" state: started restart_policy: "always" + recreate: false networks: - name: "loadbalancer" ports: @@ -38,8 +39,6 @@ INFLUXDB_ADMIN_PASSWORD: "{{ influxdb_admin_password }}" INFLUXDB_READ_USER: "{{ influxdb_stats_user }}" INFLUXDB_READ_USER_PASSWORD: "{{ influxdb_stats_password }}" - INFLUXDB_WRITE_USER: "{{ influxdb_stats_user }}" - INFLUXDB_WRITE_USER_PASSWORD: "{{ influxdb_stats_password }}" INFLUXDB_HTTP_AUTH_ENABLED: "True" - name: Wait for influx to start From 5c6c9934f31fd9bd3e5ee6cb2e1aa02c62f20ac2 Mon Sep 17 00:00:00 2001 From: Jeroen Date: Thu, 27 Aug 2026 11:10:07 +0200 Subject: [PATCH 07/18] refactor logstash file locations --- roles/logstash_docker/tasks/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/roles/logstash_docker/tasks/main.yml b/roles/logstash_docker/tasks/main.yml index 2ae2ede47..75a6ed346 100644 --- a/roles/logstash_docker/tasks/main.yml +++ b/roles/logstash_docker/tasks/main.yml @@ -8,7 +8,7 @@ mode: 0750 with_items: - /var/lib/logstash/config - - /var/lib/logstash/conf.d/core + - /etc/logstash/conf.d/core - /data/logstash - name: Place logstash settings files @@ -27,7 +27,7 @@ - name: Place core config files ansible.builtin.copy: src: "files/core/{{ item }}" - dest: "/var/lib/logstash/conf.d/core/{{ item }}" + dest: "/etc/logstash/conf.d/core/{{ item }}" owner: 1000 group: 1000 mode: 0644 @@ -47,10 +47,10 @@ ports: - 5044:5044 mounts: - - source: /var/lib/logstash/config/ - target: /usr/share/logstash/config/ + - source: /var/lib/logstash/ + target: /usr/share/logstash/ type: bind - - source: /var/lib/logstash/conf.d/ + - source: /etc/logstash/conf.d/ target: /etc/logstash/conf.d/ type: bind - source: /data/logstash/ From 5efa1abcd1583158c4bfefaab15c82bee3735809 Mon Sep 17 00:00:00 2001 From: Jeroen Date: Thu, 27 Aug 2026 11:26:35 +0200 Subject: [PATCH 08/18] fix logstash config dir location --- roles/logstash_docker/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/logstash_docker/tasks/main.yml b/roles/logstash_docker/tasks/main.yml index 75a6ed346..1586b15e4 100644 --- a/roles/logstash_docker/tasks/main.yml +++ b/roles/logstash_docker/tasks/main.yml @@ -47,8 +47,8 @@ ports: - 5044:5044 mounts: - - source: /var/lib/logstash/ - target: /usr/share/logstash/ + - source: /var/lib/logstash/config + target: /usr/share/logstash/config type: bind - source: /etc/logstash/conf.d/ target: /etc/logstash/conf.d/ From 59280a89614d908a893e0f485f91c46635c958cc Mon Sep 17 00:00:00 2001 From: Jeroen Date: Thu, 27 Aug 2026 12:53:53 +0200 Subject: [PATCH 09/18] make logstash amount of memory variable --- roles/logstash_docker/defaults/main.yml | 3 +-- roles/logstash_docker/tasks/main.yml | 9 ++++++++- .../{files/logstash.yml => templates/logstash.yml.j2} | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) rename roles/logstash_docker/{files/logstash.yml => templates/logstash.yml.j2} (84%) diff --git a/roles/logstash_docker/defaults/main.yml b/roles/logstash_docker/defaults/main.yml index 9cc6ec2cb..1aec105a9 100644 --- a/roles/logstash_docker/defaults/main.yml +++ b/roles/logstash_docker/defaults/main.yml @@ -1,3 +1,2 @@ -#SPDX-License-Identifier: MIT-0 --- -# defaults file for logstash +logstash_memory_gb: 4 diff --git a/roles/logstash_docker/tasks/main.yml b/roles/logstash_docker/tasks/main.yml index 1586b15e4..e48384eac 100644 --- a/roles/logstash_docker/tasks/main.yml +++ b/roles/logstash_docker/tasks/main.yml @@ -19,11 +19,18 @@ group: 1000 mode: 0644 with_items: - - logstash.yml - jvm.options - pipelines.yml - log4j2.properties +- name: Place logstash config file + ansible.builtin.template: + src: templates/logstash.yml.j2 + dest: /var/lib/logstash/config/logstash.yml + owner: 1000 + group: 1000 + mode: 0644 + - name: Place core config files ansible.builtin.copy: src: "files/core/{{ item }}" diff --git a/roles/logstash_docker/files/logstash.yml b/roles/logstash_docker/templates/logstash.yml.j2 similarity index 84% rename from roles/logstash_docker/files/logstash.yml rename to roles/logstash_docker/templates/logstash.yml.j2 index 086fd5073..9619f6bf7 100644 --- a/roles/logstash_docker/files/logstash.yml +++ b/roles/logstash_docker/templates/logstash.yml.j2 @@ -2,7 +2,7 @@ api.http.host: 0.0.0.0 monitoring.elasticsearch.hosts: http://elasticsearch:9200 pipeline.workers: 1 queue.type: persisted -queue.max_bytes: 4gb +queue.max_bytes: {{ logstash_memory_gb }}gb dead_letter_queue.enable: true dead_letter_queue.max_bytes: 1024mb path.logs: /var/log/logstash From 4543a38636c25be2990300f405dc10183a1f40a6 Mon Sep 17 00:00:00 2001 From: Jeroen Date: Thu, 27 Aug 2026 13:32:22 +0200 Subject: [PATCH 10/18] give correct path for files directory --- roles/filebeat/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/filebeat/tasks/main.yml b/roles/filebeat/tasks/main.yml index 41c6c2a09..0a1249eaa 100644 --- a/roles/filebeat/tasks/main.yml +++ b/roles/filebeat/tasks/main.yml @@ -38,7 +38,7 @@ - name: "Copy Elasticsearch GPG key" ansible.builtin.copy: - src: "/files/GPG-KEY-elasticsearch" + src: "files/GPG-KEY-elasticsearch" dest: "/tmp/GPG-KEY-elasticsearch" mode: '0644' From 98929d00074357ab65a9b0a7dc6dc5dbfadf9d8e Mon Sep 17 00:00:00 2001 From: Jeroen Date: Thu, 27 Aug 2026 14:23:09 +0200 Subject: [PATCH 11/18] fix hostname for influxdb endpoint in logstash --- roles/logstash_docker/tasks/main.yml | 9 ++++++++- .../33-core-influxdb-output.conf.j2} | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) rename roles/logstash_docker/{files/core/33-core-influxdb-output.conf => templates/33-core-influxdb-output.conf.j2} (97%) diff --git a/roles/logstash_docker/tasks/main.yml b/roles/logstash_docker/tasks/main.yml index e48384eac..c7406fa00 100644 --- a/roles/logstash_docker/tasks/main.yml +++ b/roles/logstash_docker/tasks/main.yml @@ -41,7 +41,14 @@ with_items: - 02-filebeat-input.conf - 13-core-ebauth.conf - - 33-core-influxdb-output.conf + +- name: Place core config file for influxdb + ansible.builtin.template: + src: templates/33-core-influxdb-output.conf.j2 + dest: /etc/logstash/conf.d/core/33-core-influxdb-output.conf + owner: 1000 + group: 1000 + mode: 0644 - name: Create logstash container community.docker.docker_container: diff --git a/roles/logstash_docker/files/core/33-core-influxdb-output.conf b/roles/logstash_docker/templates/33-core-influxdb-output.conf.j2 similarity index 97% rename from roles/logstash_docker/files/core/33-core-influxdb-output.conf rename to roles/logstash_docker/templates/33-core-influxdb-output.conf.j2 index 15950c952..17468da31 100644 --- a/roles/logstash_docker/files/core/33-core-influxdb-output.conf +++ b/roles/logstash_docker/templates/33-core-influxdb-output.conf.j2 @@ -23,7 +23,7 @@ filter { output { if [prog] == "EBAUTH" { influxdb { - host => "localhost" + host => "{{ influx_stats_dbhost }}" port => 8086 db => "log_logins" measurement => "EBAUTH" From a6a066b45d1fde4a3de5076d5bcbd129faaa6fc7 Mon Sep 17 00:00:00 2001 From: Jeroen Date: Thu, 27 Aug 2026 14:39:09 +0200 Subject: [PATCH 12/18] add credentials to logstash config --- roles/logstash_docker/templates/33-core-influxdb-output.conf.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/logstash_docker/templates/33-core-influxdb-output.conf.j2 b/roles/logstash_docker/templates/33-core-influxdb-output.conf.j2 index 17468da31..8e1a2c445 100644 --- a/roles/logstash_docker/templates/33-core-influxdb-output.conf.j2 +++ b/roles/logstash_docker/templates/33-core-influxdb-output.conf.j2 @@ -26,6 +26,8 @@ output { host => "{{ influx_stats_dbhost }}" port => 8086 db => "log_logins" + user => "{{ influxdb_stats_user }}" + password => "{{ influxdb_stats_password }}" measurement => "EBAUTH" retention_policy => "autogen" use_event_fields_for_data_points => false From fc33aecde724f9dcecfa7ae4960c8df8afc40b74 Mon Sep 17 00:00:00 2001 From: Jeroen Date: Thu, 27 Aug 2026 16:06:05 +0200 Subject: [PATCH 13/18] change default file locations to the standard paths --- roles/influxdb_docker/tasks/main.yml | 14 +++++--------- roles/logstash_docker/tasks/main.yml | 16 ++++++++-------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/roles/influxdb_docker/tasks/main.yml b/roles/influxdb_docker/tasks/main.yml index 2482ae838..27612985f 100644 --- a/roles/influxdb_docker/tasks/main.yml +++ b/roles/influxdb_docker/tasks/main.yml @@ -12,8 +12,8 @@ group: "influxdb" mode: "0755" with_items: - - "/etc/influxdb" - - "/var/lib/influxdb" + - "/opt/openconext/influxdb/etc" + - "/opt/openconext/influxdb/varlib" - name: Create influx container community.docker.docker_container: @@ -27,10 +27,10 @@ ports: - "8086:8086" mounts: - - source: /var/lib/influxdb + - source: /opt/openconext/influxdb/varlib target: /var/lib/influxdb type: bind - - source: /etc/influxdb + - source: /opt/openconext/influxdb/etc target: /etc/influxdb type: bind env: @@ -48,10 +48,6 @@ state: started delay: 5 -- name: Wait for infludb init script to run - ansible.builtin.pause: - seconds: 20 - - name: Grant stats user required access to InfluxDB database ansible.builtin.uri: url: "http://{{ influxdb_host }}:8086/query" @@ -66,4 +62,4 @@ return_content: true register: influxdb_grant changed_when: false - #no_log: true + no_log: true diff --git a/roles/logstash_docker/tasks/main.yml b/roles/logstash_docker/tasks/main.yml index c7406fa00..3ef1b3ea4 100644 --- a/roles/logstash_docker/tasks/main.yml +++ b/roles/logstash_docker/tasks/main.yml @@ -7,14 +7,14 @@ group: 1000 mode: 0750 with_items: - - /var/lib/logstash/config - - /etc/logstash/conf.d/core + - /opt/openconext/logstash/config + - /opt/openconext/logstash/conf.d/core - /data/logstash - name: Place logstash settings files ansible.builtin.copy: src: "files/{{ item }}" - dest: "/var/lib/logstash/config/{{ item }}" + dest: "/opt/openconext/logstash/config/{{ item }}" owner: 1000 group: 1000 mode: 0644 @@ -26,7 +26,7 @@ - name: Place logstash config file ansible.builtin.template: src: templates/logstash.yml.j2 - dest: /var/lib/logstash/config/logstash.yml + dest: /opt/openconext/logstash/config/logstash.yml owner: 1000 group: 1000 mode: 0644 @@ -34,7 +34,7 @@ - name: Place core config files ansible.builtin.copy: src: "files/core/{{ item }}" - dest: "/etc/logstash/conf.d/core/{{ item }}" + dest: "/opt/openconext/logstash/conf.d/core/{{ item }}" owner: 1000 group: 1000 mode: 0644 @@ -45,7 +45,7 @@ - name: Place core config file for influxdb ansible.builtin.template: src: templates/33-core-influxdb-output.conf.j2 - dest: /etc/logstash/conf.d/core/33-core-influxdb-output.conf + dest: /opt/openconext/logstash/conf.d/core/33-core-influxdb-output.conf owner: 1000 group: 1000 mode: 0644 @@ -61,10 +61,10 @@ ports: - 5044:5044 mounts: - - source: /var/lib/logstash/config + - source: /opt/openconext/logstash/config target: /usr/share/logstash/config type: bind - - source: /etc/logstash/conf.d/ + - source: /opt/openconext/logstash/conf.d/ target: /etc/logstash/conf.d/ type: bind - source: /data/logstash/ From cfdc4349308372884b8d41ebfcf9d067c0a7f11c Mon Sep 17 00:00:00 2001 From: Jeroen Date: Tue, 1 Sep 2026 11:44:32 +0200 Subject: [PATCH 14/18] add documentation to new roles --- roles/influxdb_docker/README.md | 32 +++++++++++++++++++++++++ roles/influxdb_docker/defaults/main.yml | 6 +++++ roles/logstash_docker/README.md | 29 ++++++++++++++++++++++ roles/logstash_docker/defaults/main.yml | 4 ++++ 4 files changed, 71 insertions(+) create mode 100644 roles/influxdb_docker/README.md create mode 100644 roles/logstash_docker/README.md diff --git a/roles/influxdb_docker/README.md b/roles/influxdb_docker/README.md new file mode 100644 index 000000000..3a41a5ede --- /dev/null +++ b/roles/influxdb_docker/README.md @@ -0,0 +1,32 @@ +influxdb_docker +========= + +This role deploys a influxdb docker container + +Requirements +------------ + +Requires a logstash instance to fill the influxdb. The influxdb instance is used by the stats container. The OpenConext-deploy roles "logstash_docker" and "stats" can provide this. + +Role Variables +-------------- + +The following variables must be set: + - influxdb_host (defaults to localhost, is used for setting user permissions within the database) + - influx_stats_db (name of the database to be created) + - influxdb_admin_user (admin user for influxdb) + - influxdb_admin_password (admin password for influxdb, make sure this is securely stored for example in a vault) + - influxdb_stats_user (user for accessing the database) + - influxdb_stats_password (user password for influxdb, make sure this is securely stored for example in a vault) + + +License +-------------- + +These files are licensed under version 2.0 of the Apache License, as described in the file [LICENSE](LICENSE). + +Support +-------------- + +* You can ask questions on the [OpenConext mailing list](https://openconext.org/get-involved/mailing-lists/) +* Or you can join our [Slack Workspace](https://edu.nl/ocslk) diff --git a/roles/influxdb_docker/defaults/main.yml b/roles/influxdb_docker/defaults/main.yml index 30bf7d9f8..ab9326751 100644 --- a/roles/influxdb_docker/defaults/main.yml +++ b/roles/influxdb_docker/defaults/main.yml @@ -1,2 +1,8 @@ --- influxdb_host: "localhost" +# The following variables are not provided but required to run this role. Define these in your inventory or vault: +# influx_stats_db: +# influxdb_admin_user: +# influxdb_admin_password: +# influxdb_stats_user: +# influxdb_stats_password: diff --git a/roles/logstash_docker/README.md b/roles/logstash_docker/README.md new file mode 100644 index 000000000..06db993f3 --- /dev/null +++ b/roles/logstash_docker/README.md @@ -0,0 +1,29 @@ +logstash_docker +========= + +This role deploys a logstash docker container + +Requirements +------------ + +Requires some form of input, the filebeat role from OpenConext-deploy can be used for this. Requires a influxdb instance to write to, the OpenConext-deploy role influxdb_container can be used for this. The database must be named "log_logins" for this combination to work. + +Role Variables +-------------- + +The following variables must be set: + - logstash_memory_gb: (defaults to 4, can be set to 1 for development/test environments) + - influx_stats_dbhost: (the host where influxdb runs on, if this runs in a container on the same host provide the name of the container.) + - influxdb_stats_user: (user that can access the influxdb database) + - influxdb_stats_password: (password of the user that can access the database, store this securely for example in a vault.) + +License +-------------- + +These files are licensed under version 2.0 of the Apache License, as described in the file [LICENSE](LICENSE). + +Support +-------------- + +* You can ask questions on the [OpenConext mailing list](https://openconext.org/get-involved/mailing-lists/) +* Or you can join our [Slack Workspace](https://edu.nl/ocslk) diff --git a/roles/logstash_docker/defaults/main.yml b/roles/logstash_docker/defaults/main.yml index 1aec105a9..1db02e731 100644 --- a/roles/logstash_docker/defaults/main.yml +++ b/roles/logstash_docker/defaults/main.yml @@ -1,2 +1,6 @@ --- logstash_memory_gb: 4 +# The following variables are not provided but required to run this role. Define these in your inventory or vault: +# influx_stats_dbhost: +# influxdb_stats_user: +# influxdb_stats_password: From 89c57b782076fe06d8515605ce9c1978192e5ea2 Mon Sep 17 00:00:00 2001 From: Jeroen Date: Tue, 1 Sep 2026 14:09:01 +0200 Subject: [PATCH 15/18] remove unused meta files --- roles/influxdb_docker/meta/main.yml | 35 ----------------------------- roles/logstash_docker/meta/main.yml | 35 ----------------------------- 2 files changed, 70 deletions(-) delete mode 100644 roles/influxdb_docker/meta/main.yml delete mode 100644 roles/logstash_docker/meta/main.yml diff --git a/roles/influxdb_docker/meta/main.yml b/roles/influxdb_docker/meta/main.yml deleted file mode 100644 index 6f91fd335..000000000 --- a/roles/influxdb_docker/meta/main.yml +++ /dev/null @@ -1,35 +0,0 @@ -#SPDX-License-Identifier: MIT-0 -galaxy_info: - author: your name - description: your role description - company: your company (optional) - - # If the issue tracker for your role is not on github, uncomment the - # next line and provide a value - # issue_tracker_url: http://example.com/issue/tracker - - # Choose a valid license ID from https://spdx.org - some suggested licenses: - # - BSD-3-Clause (default) - # - MIT - # - GPL-2.0-or-later - # - GPL-3.0-only - # - Apache-2.0 - # - CC-BY-4.0 - license: license (GPL-2.0-or-later, MIT, etc) - - min_ansible_version: 2.2 - - # If this a Container Enabled role, provide the minimum Ansible Container version. - # min_ansible_container_version: - - galaxy_tags: [] - # List tags for your role here, one per line. A tag is a keyword that describes - # and categorizes the role. Users find roles by searching for tags. Be sure to - # remove the '[]' above, if you add tags to this list. - # - # NOTE: A tag is limited to a single word comprised of alphanumeric characters. - # Maximum 20 tags per role. - -dependencies: [] - # List your role dependencies here, one per line. Be sure to remove the '[]' above, - # if you add dependencies to this list. diff --git a/roles/logstash_docker/meta/main.yml b/roles/logstash_docker/meta/main.yml deleted file mode 100644 index 6f91fd335..000000000 --- a/roles/logstash_docker/meta/main.yml +++ /dev/null @@ -1,35 +0,0 @@ -#SPDX-License-Identifier: MIT-0 -galaxy_info: - author: your name - description: your role description - company: your company (optional) - - # If the issue tracker for your role is not on github, uncomment the - # next line and provide a value - # issue_tracker_url: http://example.com/issue/tracker - - # Choose a valid license ID from https://spdx.org - some suggested licenses: - # - BSD-3-Clause (default) - # - MIT - # - GPL-2.0-or-later - # - GPL-3.0-only - # - Apache-2.0 - # - CC-BY-4.0 - license: license (GPL-2.0-or-later, MIT, etc) - - min_ansible_version: 2.2 - - # If this a Container Enabled role, provide the minimum Ansible Container version. - # min_ansible_container_version: - - galaxy_tags: [] - # List tags for your role here, one per line. A tag is a keyword that describes - # and categorizes the role. Users find roles by searching for tags. Be sure to - # remove the '[]' above, if you add tags to this list. - # - # NOTE: A tag is limited to a single word comprised of alphanumeric characters. - # Maximum 20 tags per role. - -dependencies: [] - # List your role dependencies here, one per line. Be sure to remove the '[]' above, - # if you add dependencies to this list. From be28197fdebf2e8aff5818f48cbfeda6b42b0dc7 Mon Sep 17 00:00:00 2001 From: Jeroen Date: Thu, 3 Sep 2026 15:18:48 +0200 Subject: [PATCH 16/18] make logstash uid variable, add influxdb backups and continuous queries. --- .../files/continuous_queries.sql | 216 ++++++++++++++++++ roles/influxdb_docker/files/influxdb_backup | 2 + roles/influxdb_docker/tasks/main.yml | 26 ++- roles/logstash_docker/tasks/main.yml | 59 ++++- 4 files changed, 292 insertions(+), 11 deletions(-) create mode 100644 roles/influxdb_docker/files/continuous_queries.sql create mode 100644 roles/influxdb_docker/files/influxdb_backup diff --git a/roles/influxdb_docker/files/continuous_queries.sql b/roles/influxdb_docker/files/continuous_queries.sql new file mode 100644 index 000000000..1315a114a --- /dev/null +++ b/roles/influxdb_docker/files/continuous_queries.sql @@ -0,0 +1,216 @@ +CREATE CONTINUOUS QUERY sp_idp_pa_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.sp_idp_pa_users_day FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY sp_entity_id, idp_entity_id, year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY idp_pa_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.idp_pa_users_day FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY idp_entity_id, year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY sp_pa_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.sp_pa_users_day FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY sp_entity_id, year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY total_pa_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.total_pa_users_day FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY sp_idp_ta_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.sp_idp_ta_users_day FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY sp_entity_id, idp_entity_id, year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY idp_ta_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.idp_ta_users_day FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY idp_entity_id, year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY sp_ta_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.sp_ta_users_day FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY sp_entity_id, year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY total_ta_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.total_ta_users_day FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY sp_idp_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.sp_idp_users_day FROM log_logins.autogen.EBAUTH GROUP BY sp_entity_id, idp_entity_id, year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY idp_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.idp_users_day FROM log_logins.autogen.EBAUTH GROUP BY idp_entity_id, year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY sp_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.sp_users_day FROM log_logins.autogen.EBAUTH GROUP BY sp_entity_id, year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY total_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.total_users_day FROM log_logins.autogen.EBAUTH GROUP BY year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY sp_idp_pa_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_pa_users_month_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY idp_pa_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_pa_users_month_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_pa_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_pa_users_month_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY sp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY total_pa_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_pa_users_month_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_idp_ta_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_ta_users_month_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY idp_ta_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_ta_users_month_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_ta_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_ta_users_month_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY sp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY total_ta_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_ta_users_month_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_idp_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_users_month_unique FROM log_logins.autogen.EBAUTH GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY idp_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_users_month_unique FROM log_logins.autogen.EBAUTH GROUP BY idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_users_month_unique FROM log_logins.autogen.EBAUTH GROUP BY sp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY total_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_users_month_unique FROM log_logins.autogen.EBAUTH GROUP BY time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_idp_pa_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_pa_users_quarter_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY idp_pa_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_pa_users_quarter_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_pa_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_pa_users_quarter_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY sp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY total_pa_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_pa_users_quarter_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_idp_ta_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_ta_users_quarter_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY idp_ta_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_ta_users_quarter_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_ta_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_ta_users_quarter_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY sp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY total_ta_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_ta_users_quarter_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_idp_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_users_quarter_unique FROM log_logins.autogen.EBAUTH GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY idp_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_users_quarter_unique FROM log_logins.autogen.EBAUTH GROUP BY idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_users_quarter_unique FROM log_logins.autogen.EBAUTH GROUP BY sp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY total_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_users_quarter_unique FROM log_logins.autogen.EBAUTH GROUP BY time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_idp_pa_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_pa_users_year_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY sp_entity_id, idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY idp_pa_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_pa_users_year_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_pa_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_pa_users_year_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY sp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY total_pa_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_pa_users_year_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_idp_ta_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_ta_users_year_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY sp_entity_id, idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY idp_ta_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_ta_users_year_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_ta_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_ta_users_year_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY sp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY total_ta_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_ta_users_year_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_idp_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_users_year_unique FROM log_logins.autogen.EBAUTH GROUP BY sp_entity_id, idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY idp_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_users_year_unique FROM log_logins.autogen.EBAUTH GROUP BY idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_users_year_unique FROM log_logins.autogen.EBAUTH GROUP BY sp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY total_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_users_year_unique FROM log_logins.autogen.EBAUTH GROUP BY time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_idp_pa_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_pa_users_week FROM log_logins.autogen.sp_idp_pa_users_day GROUP BY sp_entity_id, idp_entity_id, year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY idp_pa_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_pa_users_week FROM log_logins.autogen.idp_pa_users_day GROUP BY idp_entity_id, year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY sp_pa_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_pa_users_week FROM log_logins.autogen.sp_pa_users_day GROUP BY sp_entity_id, year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY total_pa_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_pa_users_week FROM log_logins.autogen.total_pa_users_day GROUP BY year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY sp_idp_ta_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_ta_users_week FROM log_logins.autogen.sp_idp_ta_users_day GROUP BY sp_entity_id, idp_entity_id, year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY idp_ta_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_ta_users_week FROM log_logins.autogen.idp_ta_users_day GROUP BY idp_entity_id, year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY sp_ta_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_ta_users_week FROM log_logins.autogen.sp_ta_users_day GROUP BY sp_entity_id, year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY total_ta_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_ta_users_week FROM log_logins.autogen.total_ta_users_day GROUP BY year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY sp_idp_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_users_week FROM log_logins.autogen.sp_idp_users_day GROUP BY sp_entity_id, idp_entity_id, year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY idp_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_users_week FROM log_logins.autogen.idp_users_day GROUP BY idp_entity_id, year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY sp_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_users_week FROM log_logins.autogen.sp_users_day GROUP BY sp_entity_id, year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY total_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_users_week FROM log_logins.autogen.total_users_day GROUP BY year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY sp_idp_pa_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_pa_users_month FROM log_logins.autogen.sp_idp_pa_users_week GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY idp_pa_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_pa_users_month FROM log_logins.autogen.idp_pa_users_week GROUP BY idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_pa_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_pa_users_month FROM log_logins.autogen.sp_pa_users_week GROUP BY sp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY total_pa_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_pa_users_month FROM log_logins.autogen.total_pa_users_week GROUP BY time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_idp_ta_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_ta_users_month FROM log_logins.autogen.sp_idp_ta_users_week GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY idp_ta_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_ta_users_month FROM log_logins.autogen.idp_ta_users_week GROUP BY idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_ta_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_ta_users_month FROM log_logins.autogen.sp_ta_users_week GROUP BY sp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY total_ta_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_ta_users_month FROM log_logins.autogen.total_ta_users_week GROUP BY time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_idp_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_users_month FROM log_logins.autogen.sp_idp_users_week GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY idp_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_users_month FROM log_logins.autogen.idp_users_week GROUP BY idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_users_month FROM log_logins.autogen.sp_users_week GROUP BY sp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY total_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_users_month FROM log_logins.autogen.total_users_week GROUP BY time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_idp_pa_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_pa_users_quarter FROM log_logins.autogen.sp_idp_pa_users_week GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY idp_pa_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_pa_users_quarter FROM log_logins.autogen.idp_pa_users_week GROUP BY idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_pa_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_pa_users_quarter FROM log_logins.autogen.sp_pa_users_week GROUP BY sp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY total_pa_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_pa_users_quarter FROM log_logins.autogen.total_pa_users_week GROUP BY time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_idp_ta_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_ta_users_quarter FROM log_logins.autogen.sp_idp_ta_users_week GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY idp_ta_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_ta_users_quarter FROM log_logins.autogen.idp_ta_users_week GROUP BY idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_ta_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_ta_users_quarter FROM log_logins.autogen.sp_ta_users_week GROUP BY sp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY total_ta_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_ta_users_quarter FROM log_logins.autogen.total_ta_users_week GROUP BY time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_idp_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_users_quarter FROM log_logins.autogen.sp_idp_users_week GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY idp_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_users_quarter FROM log_logins.autogen.idp_users_week GROUP BY idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_users_quarter FROM log_logins.autogen.sp_users_week GROUP BY sp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY total_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_users_quarter FROM log_logins.autogen.total_users_week GROUP BY time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_idp_pa_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_pa_users_year FROM log_logins.autogen.sp_idp_pa_users_week GROUP BY sp_entity_id, idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY idp_pa_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_pa_users_year FROM log_logins.autogen.idp_pa_users_week GROUP BY idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_pa_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_pa_users_year FROM log_logins.autogen.sp_pa_users_week GROUP BY sp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY total_pa_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_pa_users_year FROM log_logins.autogen.total_pa_users_week GROUP BY time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_idp_ta_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_ta_users_year FROM log_logins.autogen.sp_idp_ta_users_week GROUP BY sp_entity_id, idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY idp_ta_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_ta_users_year FROM log_logins.autogen.idp_ta_users_week GROUP BY idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_ta_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_ta_users_year FROM log_logins.autogen.sp_ta_users_week GROUP BY sp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY total_ta_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_ta_users_year FROM log_logins.autogen.total_ta_users_week GROUP BY time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_idp_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_users_year FROM log_logins.autogen.sp_idp_users_week GROUP BY sp_entity_id, idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY idp_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_users_year FROM log_logins.autogen.idp_users_week GROUP BY idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_users_year FROM log_logins.autogen.sp_users_week GROUP BY sp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY total_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_users_year FROM log_logins.autogen.total_users_week GROUP BY time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_idp_pa_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_pa_users_week_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY sp_entity_id, idp_entity_id, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY idp_pa_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_pa_users_week_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY idp_entity_id, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY sp_pa_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_pa_users_week_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY sp_entity_id, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY total_pa_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_pa_users_week_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY time(1w, 4d) END; + +CREATE CONTINUOUS QUERY sp_idp_ta_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_ta_users_week_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY sp_entity_id, idp_entity_id, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY idp_ta_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_ta_users_week_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY idp_entity_id, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY sp_ta_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_ta_users_week_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY sp_entity_id, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY total_ta_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_ta_users_week_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY time(1w, 4d) END; + +CREATE CONTINUOUS QUERY sp_idp_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_users_week_unique FROM log_logins.autogen.EBAUTH GROUP BY sp_entity_id, idp_entity_id, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY idp_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_users_week_unique FROM log_logins.autogen.EBAUTH GROUP BY idp_entity_id, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY sp_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_users_week_unique FROM log_logins.autogen.EBAUTH GROUP BY sp_entity_id, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY total_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_users_week_unique FROM log_logins.autogen.EBAUTH GROUP BY time(1w, 4d) END; + diff --git a/roles/influxdb_docker/files/influxdb_backup b/roles/influxdb_docker/files/influxdb_backup new file mode 100644 index 000000000..b7e1f8a1e --- /dev/null +++ b/roles/influxdb_docker/files/influxdb_backup @@ -0,0 +1,2 @@ +0 3 * * * root docker exec influxdb influxd backup -portable /data/influxdb-backup/$(date +\%F) +0 4 * * * root find /data/influxdb-backup/ -type f -mtime +1 -exec rm {} \; diff --git a/roles/influxdb_docker/tasks/main.yml b/roles/influxdb_docker/tasks/main.yml index 27612985f..8791c3d85 100644 --- a/roles/influxdb_docker/tasks/main.yml +++ b/roles/influxdb_docker/tasks/main.yml @@ -2,7 +2,7 @@ - name: Create influx user ansible.builtin.user: name: influxdb - uid: 1500 + uid: "1500" - name: Ensure required host directories exists ansible.builtin.file: @@ -14,6 +14,7 @@ with_items: - "/opt/openconext/influxdb/etc" - "/opt/openconext/influxdb/varlib" + - "/data/influxdb-backup" - name: Create influx container community.docker.docker_container: @@ -33,6 +34,9 @@ - source: /opt/openconext/influxdb/etc target: /etc/influxdb type: bind + - source: /data/influxdb-backup + target: /data/influxdb-backup + type: bind env: INFLUXDB_DB: "{{ influx_stats_db }}" INFLUXDB_ADMIN_USER: "{{ influxdb_admin_user }}" @@ -40,6 +44,9 @@ INFLUXDB_READ_USER: "{{ influxdb_stats_user }}" INFLUXDB_READ_USER_PASSWORD: "{{ influxdb_stats_password }}" INFLUXDB_HTTP_AUTH_ENABLED: "True" + INFLUXDB_META_DIR: "/var/lib/influxdb/meta" + INFLUXDB_DATA_DIR: "/var/lib/influxdb/data" + INFLUXDB_COORDINATOR_WAL_DIR: "/var/lib/influxdb/wal" - name: Wait for influx to start ansible.builtin.wait_for: @@ -63,3 +70,20 @@ register: influxdb_grant changed_when: false no_log: true + +- name: Import continuous queries through the InfluxDB Docker container + community.docker.docker_container_exec: + container: "influxdb" + command: > + influx + -host {{ influxdb_host }} + -port 8086 + -username {{ influxdb_admin_user }} + -password {{ influxdb_admin_password }} + stdin: "{{ lookup('ansible.builtin.file', 'files/continuous_queries.sql') }}" + no_log: true + +- name: Create cron file for making the backups and cleaning them up after 7 days + ansible.builtin.copy: + src: files/influxdb_backup + dest: /etc/cron.d/influxdb_backup diff --git a/roles/logstash_docker/tasks/main.yml b/roles/logstash_docker/tasks/main.yml index 3ef1b3ea4..471075e1b 100644 --- a/roles/logstash_docker/tasks/main.yml +++ b/roles/logstash_docker/tasks/main.yml @@ -3,11 +3,13 @@ ansible.builtin.file: path: "{{ item }}" state: directory - owner: 1000 - group: 1000 + owner: "{{ container_ids.logstash }}" + group: "{{ container_ids.logstash }}" mode: 0750 with_items: + - /opt/openconext/logstash - /opt/openconext/logstash/config + - /opt/openconext/logstash/conf.d - /opt/openconext/logstash/conf.d/core - /data/logstash @@ -15,8 +17,8 @@ ansible.builtin.copy: src: "files/{{ item }}" dest: "/opt/openconext/logstash/config/{{ item }}" - owner: 1000 - group: 1000 + owner: "{{ container_ids.logstash }}" + group: "{{ container_ids.logstash }}" mode: 0644 with_items: - jvm.options @@ -27,16 +29,16 @@ ansible.builtin.template: src: templates/logstash.yml.j2 dest: /opt/openconext/logstash/config/logstash.yml - owner: 1000 - group: 1000 + owner: "{{ container_ids.logstash }}" + group: "{{ container_ids.logstash }}" mode: 0644 - name: Place core config files ansible.builtin.copy: src: "files/core/{{ item }}" dest: "/opt/openconext/logstash/conf.d/core/{{ item }}" - owner: 1000 - group: 1000 + owner: "{{ container_ids.logstash }}" + group: "{{ container_ids.logstash }}" mode: 0644 with_items: - 02-filebeat-input.conf @@ -46,8 +48,8 @@ ansible.builtin.template: src: templates/33-core-influxdb-output.conf.j2 dest: /opt/openconext/logstash/conf.d/core/33-core-influxdb-output.conf - owner: 1000 - group: 1000 + owner: "{{ container_ids.logstash }}" + group: "{{ container_ids.logstash }}" mode: 0644 - name: Create logstash container @@ -70,3 +72,40 @@ - source: /data/logstash/ target: /data/logstash/ type: bind + +- name: Change logstash UID + community.docker.docker_container_exec: + container: logstash + user: "root" + command: + sh -c ' + groupmod -g {{ container_ids.logstash }} logstash && + usermod -u {{ container_ids.logstash }} -g {{ container_ids.logstash }} logstash' + +- name: Check if running user needs to be changed + community.docker.docker_container_info: + name: "logstash" + register: logstash_container + +- name: Update logstash container to run as non root + community.docker.docker_container: + name: "logstash" + image: "ghcr.io/openconext/openconext-basecontainers/logstash8:latest" + state: started + restart_policy: "always" + networks: + - name: "loadbalancer" + ports: + - 5044:5044 + user: "{{ container_ids.logstash }}" + mounts: + - source: /opt/openconext/logstash/config + target: /usr/share/logstash/config + type: bind + - source: /opt/openconext/logstash/conf.d/ + target: /etc/logstash/conf.d/ + type: bind + - source: /data/logstash/ + target: /data/logstash/ + type: bind + when: logstash_container.container.Config.User | default('') != container_ids.logstash From 810206f82f4123707c6f0ee423c36822a0dd2e55 Mon Sep 17 00:00:00 2001 From: Jeroen Date: Thu, 3 Sep 2026 15:31:22 +0200 Subject: [PATCH 17/18] fix idempotency --- roles/logstash_docker/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/logstash_docker/tasks/main.yml b/roles/logstash_docker/tasks/main.yml index 471075e1b..b6b7291fa 100644 --- a/roles/logstash_docker/tasks/main.yml +++ b/roles/logstash_docker/tasks/main.yml @@ -72,6 +72,7 @@ - source: /data/logstash/ target: /data/logstash/ type: bind + register: logstash_container_created - name: Change logstash UID community.docker.docker_container_exec: @@ -81,6 +82,7 @@ sh -c ' groupmod -g {{ container_ids.logstash }} logstash && usermod -u {{ container_ids.logstash }} -g {{ container_ids.logstash }} logstash' + when: logstash_container_created.changed - name: Check if running user needs to be changed community.docker.docker_container_info: From 6e06dc1d576babe2ba3a9ef71e45f17649a0c383 Mon Sep 17 00:00:00 2001 From: Jeroen Date: Thu, 3 Sep 2026 15:41:04 +0200 Subject: [PATCH 18/18] cleanup unneeded parts of config file --- roles/logstash_docker/files/pipelines.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/roles/logstash_docker/files/pipelines.yml b/roles/logstash_docker/files/pipelines.yml index 5d53e7fe0..c471a91d3 100644 --- a/roles/logstash_docker/files/pipelines.yml +++ b/roles/logstash_docker/files/pipelines.yml @@ -2,7 +2,3 @@ path.config: "/etc/logstash/conf.d/core" pipeline.workers: 2 queue.type: persisted -#- pipeline.id: sa -# path.config: "/etc/logstash/conf.d/sa" -# pipeline.workers: 2 -# queue.type: persisted