diff --git a/roles/filebeat/tasks/main.yml b/roles/filebeat/tasks/main.yml index 41c6c2a09..0a1249eaa 100644 --- a/roles/filebeat/tasks/main.yml +++ b/roles/filebeat/tasks/main.yml @@ -38,7 +38,7 @@ - name: "Copy Elasticsearch GPG key" ansible.builtin.copy: - src: "/files/GPG-KEY-elasticsearch" + src: "files/GPG-KEY-elasticsearch" dest: "/tmp/GPG-KEY-elasticsearch" mode: '0644' diff --git a/roles/influxdb_docker/README.md b/roles/influxdb_docker/README.md new file mode 100644 index 000000000..3a41a5ede --- /dev/null +++ b/roles/influxdb_docker/README.md @@ -0,0 +1,32 @@ +influxdb_docker +========= + +This role deploys a influxdb docker container + +Requirements +------------ + +Requires a logstash instance to fill the influxdb. The influxdb instance is used by the stats container. The OpenConext-deploy roles "logstash_docker" and "stats" can provide this. + +Role Variables +-------------- + +The following variables must be set: + - influxdb_host (defaults to localhost, is used for setting user permissions within the database) + - influx_stats_db (name of the database to be created) + - influxdb_admin_user (admin user for influxdb) + - influxdb_admin_password (admin password for influxdb, make sure this is securely stored for example in a vault) + - influxdb_stats_user (user for accessing the database) + - influxdb_stats_password (user password for influxdb, make sure this is securely stored for example in a vault) + + +License +-------------- + +These files are licensed under version 2.0 of the Apache License, as described in the file [LICENSE](LICENSE). + +Support +-------------- + +* You can ask questions on the [OpenConext mailing list](https://openconext.org/get-involved/mailing-lists/) +* Or you can join our [Slack Workspace](https://edu.nl/ocslk) diff --git a/roles/influxdb_docker/defaults/main.yml b/roles/influxdb_docker/defaults/main.yml new file mode 100644 index 000000000..ab9326751 --- /dev/null +++ b/roles/influxdb_docker/defaults/main.yml @@ -0,0 +1,8 @@ +--- +influxdb_host: "localhost" +# The following variables are not provided but required to run this role. Define these in your inventory or vault: +# influx_stats_db: +# influxdb_admin_user: +# influxdb_admin_password: +# influxdb_stats_user: +# influxdb_stats_password: diff --git a/roles/influxdb_docker/files/continuous_queries.sql b/roles/influxdb_docker/files/continuous_queries.sql new file mode 100644 index 000000000..1315a114a --- /dev/null +++ b/roles/influxdb_docker/files/continuous_queries.sql @@ -0,0 +1,216 @@ +CREATE CONTINUOUS QUERY sp_idp_pa_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.sp_idp_pa_users_day FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY sp_entity_id, idp_entity_id, year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY idp_pa_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.idp_pa_users_day FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY idp_entity_id, year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY sp_pa_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.sp_pa_users_day FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY sp_entity_id, year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY total_pa_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.total_pa_users_day FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY sp_idp_ta_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.sp_idp_ta_users_day FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY sp_entity_id, idp_entity_id, year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY idp_ta_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.idp_ta_users_day FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY idp_entity_id, year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY sp_ta_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.sp_ta_users_day FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY sp_entity_id, year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY total_ta_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.total_ta_users_day FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY sp_idp_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.sp_idp_users_day FROM log_logins.autogen.EBAUTH GROUP BY sp_entity_id, idp_entity_id, year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY idp_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.idp_users_day FROM log_logins.autogen.EBAUTH GROUP BY idp_entity_id, year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY sp_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.sp_users_day FROM log_logins.autogen.EBAUTH GROUP BY sp_entity_id, year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY total_users_day_cq ON log_logins RESAMPLE EVERY 1h FOR 2d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id, count(user_id) AS count_user_id INTO log_logins.autogen.total_users_day FROM log_logins.autogen.EBAUTH GROUP BY year, month, quarter, time(1d) END; + +CREATE CONTINUOUS QUERY sp_idp_pa_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_pa_users_month_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY idp_pa_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_pa_users_month_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_pa_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_pa_users_month_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY sp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY total_pa_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_pa_users_month_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_idp_ta_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_ta_users_month_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY idp_ta_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_ta_users_month_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_ta_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_ta_users_month_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY sp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY total_ta_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_ta_users_month_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_idp_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_users_month_unique FROM log_logins.autogen.EBAUTH GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY idp_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_users_month_unique FROM log_logins.autogen.EBAUTH GROUP BY idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_users_month_unique FROM log_logins.autogen.EBAUTH GROUP BY sp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY total_users_month_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_users_month_unique FROM log_logins.autogen.EBAUTH GROUP BY time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_idp_pa_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_pa_users_quarter_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY idp_pa_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_pa_users_quarter_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_pa_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_pa_users_quarter_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY sp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY total_pa_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_pa_users_quarter_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_idp_ta_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_ta_users_quarter_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY idp_ta_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_ta_users_quarter_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_ta_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_ta_users_quarter_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY sp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY total_ta_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_ta_users_quarter_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_idp_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_users_quarter_unique FROM log_logins.autogen.EBAUTH GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY idp_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_users_quarter_unique FROM log_logins.autogen.EBAUTH GROUP BY idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_users_quarter_unique FROM log_logins.autogen.EBAUTH GROUP BY sp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY total_users_quarter_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_users_quarter_unique FROM log_logins.autogen.EBAUTH GROUP BY time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_idp_pa_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_pa_users_year_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY sp_entity_id, idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY idp_pa_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_pa_users_year_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_pa_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_pa_users_year_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY sp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY total_pa_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_pa_users_year_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_idp_ta_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_ta_users_year_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY sp_entity_id, idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY idp_ta_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_ta_users_year_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_ta_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_ta_users_year_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY sp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY total_ta_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_ta_users_year_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_idp_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_users_year_unique FROM log_logins.autogen.EBAUTH GROUP BY sp_entity_id, idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY idp_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_users_year_unique FROM log_logins.autogen.EBAUTH GROUP BY idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_users_year_unique FROM log_logins.autogen.EBAUTH GROUP BY sp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY total_users_year_unique_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_users_year_unique FROM log_logins.autogen.EBAUTH GROUP BY time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_idp_pa_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_pa_users_week FROM log_logins.autogen.sp_idp_pa_users_day GROUP BY sp_entity_id, idp_entity_id, year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY idp_pa_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_pa_users_week FROM log_logins.autogen.idp_pa_users_day GROUP BY idp_entity_id, year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY sp_pa_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_pa_users_week FROM log_logins.autogen.sp_pa_users_day GROUP BY sp_entity_id, year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY total_pa_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_pa_users_week FROM log_logins.autogen.total_pa_users_day GROUP BY year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY sp_idp_ta_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_ta_users_week FROM log_logins.autogen.sp_idp_ta_users_day GROUP BY sp_entity_id, idp_entity_id, year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY idp_ta_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_ta_users_week FROM log_logins.autogen.idp_ta_users_day GROUP BY idp_entity_id, year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY sp_ta_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_ta_users_week FROM log_logins.autogen.sp_ta_users_day GROUP BY sp_entity_id, year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY total_ta_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_ta_users_week FROM log_logins.autogen.total_ta_users_day GROUP BY year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY sp_idp_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_users_week FROM log_logins.autogen.sp_idp_users_day GROUP BY sp_entity_id, idp_entity_id, year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY idp_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_users_week FROM log_logins.autogen.idp_users_day GROUP BY idp_entity_id, year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY sp_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_users_week FROM log_logins.autogen.sp_users_day GROUP BY sp_entity_id, year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY total_users_week_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_users_week FROM log_logins.autogen.total_users_day GROUP BY year, month, quarter, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY sp_idp_pa_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_pa_users_month FROM log_logins.autogen.sp_idp_pa_users_week GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY idp_pa_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_pa_users_month FROM log_logins.autogen.idp_pa_users_week GROUP BY idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_pa_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_pa_users_month FROM log_logins.autogen.sp_pa_users_week GROUP BY sp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY total_pa_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_pa_users_month FROM log_logins.autogen.total_pa_users_week GROUP BY time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_idp_ta_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_ta_users_month FROM log_logins.autogen.sp_idp_ta_users_week GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY idp_ta_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_ta_users_month FROM log_logins.autogen.idp_ta_users_week GROUP BY idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_ta_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_ta_users_month FROM log_logins.autogen.sp_ta_users_week GROUP BY sp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY total_ta_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_ta_users_month FROM log_logins.autogen.total_ta_users_week GROUP BY time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_idp_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_users_month FROM log_logins.autogen.sp_idp_users_week GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY idp_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_users_month FROM log_logins.autogen.idp_users_week GROUP BY idp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_users_month FROM log_logins.autogen.sp_users_week GROUP BY sp_entity_id, time(12600w), year, month END; + +CREATE CONTINUOUS QUERY total_users_month_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_users_month FROM log_logins.autogen.total_users_week GROUP BY time(12600w), year, month END; + +CREATE CONTINUOUS QUERY sp_idp_pa_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_pa_users_quarter FROM log_logins.autogen.sp_idp_pa_users_week GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY idp_pa_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_pa_users_quarter FROM log_logins.autogen.idp_pa_users_week GROUP BY idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_pa_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_pa_users_quarter FROM log_logins.autogen.sp_pa_users_week GROUP BY sp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY total_pa_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_pa_users_quarter FROM log_logins.autogen.total_pa_users_week GROUP BY time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_idp_ta_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_ta_users_quarter FROM log_logins.autogen.sp_idp_ta_users_week GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY idp_ta_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_ta_users_quarter FROM log_logins.autogen.idp_ta_users_week GROUP BY idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_ta_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_ta_users_quarter FROM log_logins.autogen.sp_ta_users_week GROUP BY sp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY total_ta_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_ta_users_quarter FROM log_logins.autogen.total_ta_users_week GROUP BY time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_idp_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_users_quarter FROM log_logins.autogen.sp_idp_users_week GROUP BY sp_entity_id, idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY idp_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_users_quarter FROM log_logins.autogen.idp_users_week GROUP BY idp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_users_quarter FROM log_logins.autogen.sp_users_week GROUP BY sp_entity_id, time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY total_users_quarter_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_users_quarter FROM log_logins.autogen.total_users_week GROUP BY time(12600w), year, quarter END; + +CREATE CONTINUOUS QUERY sp_idp_pa_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_pa_users_year FROM log_logins.autogen.sp_idp_pa_users_week GROUP BY sp_entity_id, idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY idp_pa_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_pa_users_year FROM log_logins.autogen.idp_pa_users_week GROUP BY idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_pa_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_pa_users_year FROM log_logins.autogen.sp_pa_users_week GROUP BY sp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY total_pa_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_pa_users_year FROM log_logins.autogen.total_pa_users_week GROUP BY time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_idp_ta_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_ta_users_year FROM log_logins.autogen.sp_idp_ta_users_week GROUP BY sp_entity_id, idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY idp_ta_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_ta_users_year FROM log_logins.autogen.idp_ta_users_week GROUP BY idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_ta_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_ta_users_year FROM log_logins.autogen.sp_ta_users_week GROUP BY sp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY total_ta_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_ta_users_year FROM log_logins.autogen.total_ta_users_week GROUP BY time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_idp_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_idp_users_year FROM log_logins.autogen.sp_idp_users_week GROUP BY sp_entity_id, idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY idp_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.idp_users_year FROM log_logins.autogen.idp_users_week GROUP BY idp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.sp_users_year FROM log_logins.autogen.sp_users_week GROUP BY sp_entity_id, time(12600w), year END; + +CREATE CONTINUOUS QUERY total_users_year_cq ON log_logins RESAMPLE EVERY 1d BEGIN SELECT sum(count_user_id) AS count_user_id INTO log_logins.autogen.total_users_year FROM log_logins.autogen.total_users_week GROUP BY time(12600w), year END; + +CREATE CONTINUOUS QUERY sp_idp_pa_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_pa_users_week_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY sp_entity_id, idp_entity_id, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY idp_pa_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_pa_users_week_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY idp_entity_id, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY sp_pa_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_pa_users_week_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY sp_entity_id, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY total_pa_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_pa_users_week_unique FROM log_logins.autogen.EBAUTH WHERE state = 'prodaccepted' GROUP BY time(1w, 4d) END; + +CREATE CONTINUOUS QUERY sp_idp_ta_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_ta_users_week_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY sp_entity_id, idp_entity_id, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY idp_ta_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_ta_users_week_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY idp_entity_id, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY sp_ta_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_ta_users_week_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY sp_entity_id, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY total_ta_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_ta_users_week_unique FROM log_logins.autogen.EBAUTH WHERE state = 'testaccepted' GROUP BY time(1w, 4d) END; + +CREATE CONTINUOUS QUERY sp_idp_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_idp_users_week_unique FROM log_logins.autogen.EBAUTH GROUP BY sp_entity_id, idp_entity_id, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY idp_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.idp_users_week_unique FROM log_logins.autogen.EBAUTH GROUP BY idp_entity_id, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY sp_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.sp_users_week_unique FROM log_logins.autogen.EBAUTH GROUP BY sp_entity_id, time(1w, 4d) END; + +CREATE CONTINUOUS QUERY total_users_week_unique_cq ON log_logins RESAMPLE EVERY 6h FOR 2w BEGIN SELECT count(distinct(user_id)) AS distinct_count_user_id INTO log_logins.autogen.total_users_week_unique FROM log_logins.autogen.EBAUTH GROUP BY time(1w, 4d) END; + diff --git a/roles/influxdb_docker/files/influxdb_backup b/roles/influxdb_docker/files/influxdb_backup new file mode 100644 index 000000000..b7e1f8a1e --- /dev/null +++ b/roles/influxdb_docker/files/influxdb_backup @@ -0,0 +1,2 @@ +0 3 * * * root docker exec influxdb influxd backup -portable /data/influxdb-backup/$(date +\%F) +0 4 * * * root find /data/influxdb-backup/ -type f -mtime +1 -exec rm {} \; diff --git a/roles/influxdb_docker/tasks/main.yml b/roles/influxdb_docker/tasks/main.yml new file mode 100644 index 000000000..8791c3d85 --- /dev/null +++ b/roles/influxdb_docker/tasks/main.yml @@ -0,0 +1,89 @@ +--- +- name: Create influx user + ansible.builtin.user: + name: influxdb + uid: "1500" + +- name: Ensure required host directories exists + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: "influxdb" + group: "influxdb" + mode: "0755" + with_items: + - "/opt/openconext/influxdb/etc" + - "/opt/openconext/influxdb/varlib" + - "/data/influxdb-backup" + +- name: Create influx container + community.docker.docker_container: + name: "influxdb" + image: "influxdb:1.12" + state: started + restart_policy: "always" + recreate: false + networks: + - name: "loadbalancer" + ports: + - "8086:8086" + mounts: + - source: /opt/openconext/influxdb/varlib + target: /var/lib/influxdb + type: bind + - source: /opt/openconext/influxdb/etc + target: /etc/influxdb + type: bind + - source: /data/influxdb-backup + target: /data/influxdb-backup + type: bind + env: + INFLUXDB_DB: "{{ influx_stats_db }}" + INFLUXDB_ADMIN_USER: "{{ influxdb_admin_user }}" + INFLUXDB_ADMIN_PASSWORD: "{{ influxdb_admin_password }}" + INFLUXDB_READ_USER: "{{ influxdb_stats_user }}" + INFLUXDB_READ_USER_PASSWORD: "{{ influxdb_stats_password }}" + INFLUXDB_HTTP_AUTH_ENABLED: "True" + INFLUXDB_META_DIR: "/var/lib/influxdb/meta" + INFLUXDB_DATA_DIR: "/var/lib/influxdb/data" + INFLUXDB_COORDINATOR_WAL_DIR: "/var/lib/influxdb/wal" + +- name: Wait for influx to start + ansible.builtin.wait_for: + port: 8086 + host: localhost + state: started + delay: 5 + +- name: Grant stats user required access to InfluxDB database + ansible.builtin.uri: + url: "http://{{ influxdb_host }}:8086/query" + method: POST + url_username: "{{ influxdb_admin_user }}" + url_password: "{{ influxdb_admin_password }}" + force_basic_auth: true + body_format: form-urlencoded + body: + q: 'GRANT ALL PRIVILEGES TO "{{ influxdb_stats_user }}"' + status_code: 200 + return_content: true + register: influxdb_grant + changed_when: false + no_log: true + +- name: Import continuous queries through the InfluxDB Docker container + community.docker.docker_container_exec: + container: "influxdb" + command: > + influx + -host {{ influxdb_host }} + -port 8086 + -username {{ influxdb_admin_user }} + -password {{ influxdb_admin_password }} + stdin: "{{ lookup('ansible.builtin.file', 'files/continuous_queries.sql') }}" + no_log: true + +- name: Create cron file for making the backups and cleaning them up after 7 days + ansible.builtin.copy: + src: files/influxdb_backup + dest: /etc/cron.d/influxdb_backup diff --git a/roles/logstash_docker/README.md b/roles/logstash_docker/README.md new file mode 100644 index 000000000..06db993f3 --- /dev/null +++ b/roles/logstash_docker/README.md @@ -0,0 +1,29 @@ +logstash_docker +========= + +This role deploys a logstash docker container + +Requirements +------------ + +Requires some form of input, the filebeat role from OpenConext-deploy can be used for this. Requires a influxdb instance to write to, the OpenConext-deploy role influxdb_container can be used for this. The database must be named "log_logins" for this combination to work. + +Role Variables +-------------- + +The following variables must be set: + - logstash_memory_gb: (defaults to 4, can be set to 1 for development/test environments) + - influx_stats_dbhost: (the host where influxdb runs on, if this runs in a container on the same host provide the name of the container.) + - influxdb_stats_user: (user that can access the influxdb database) + - influxdb_stats_password: (password of the user that can access the database, store this securely for example in a vault.) + +License +-------------- + +These files are licensed under version 2.0 of the Apache License, as described in the file [LICENSE](LICENSE). + +Support +-------------- + +* You can ask questions on the [OpenConext mailing list](https://openconext.org/get-involved/mailing-lists/) +* Or you can join our [Slack Workspace](https://edu.nl/ocslk) diff --git a/roles/logstash_docker/defaults/main.yml b/roles/logstash_docker/defaults/main.yml new file mode 100644 index 000000000..1db02e731 --- /dev/null +++ b/roles/logstash_docker/defaults/main.yml @@ -0,0 +1,6 @@ +--- +logstash_memory_gb: 4 +# The following variables are not provided but required to run this role. Define these in your inventory or vault: +# influx_stats_dbhost: +# influxdb_stats_user: +# influxdb_stats_password: diff --git a/roles/logstash_docker/files/core/02-filebeat-input.conf b/roles/logstash_docker/files/core/02-filebeat-input.conf new file mode 100644 index 000000000..daeb4a214 --- /dev/null +++ b/roles/logstash_docker/files/core/02-filebeat-input.conf @@ -0,0 +1,6 @@ +input { + beats { + port => 5044 + type => "log" + } +} diff --git a/roles/logstash_docker/files/core/13-core-ebauth.conf b/roles/logstash_docker/files/core/13-core-ebauth.conf new file mode 100644 index 000000000..5d25ca794 --- /dev/null +++ b/roles/logstash_docker/files/core/13-core-ebauth.conf @@ -0,0 +1,33 @@ +filter { + if [prog] == "EBAUTH" { + grok { + patterns_dir => "/etc/logstash/patterns" + match => { "message" => "%{SYSLOGBASE:syslogbase} %{GREEDYDATA:ebauth}" } + } +} + json { + source => "ebauth" + target => "ebauth" +} + date { + match => [ "[ebauth][context][login_stamp]", "yyyy-MM-dd'T'HH:mm:ssZZ" ] +} + de_dot { +} + mutate { + remove_field => [ "source" ] + remove_field => [ "syslogbase" ] + add_field => {"[month]" => "%{+MM}"} + add_field => {"[year]" => "%{+YYYY}"} +} + fingerprint { + key => eighiePhie3xiu3boChaekoomai6iiYilux1zoohoh6ahgh9iaphuthaero4ahme + source => [ "[ebauth][context][user_id]" ] + method => SHA256 + target => [ "[ebauth][context][user_id_hashed]" ] +} + ruby { + code => 'quarter = (event.get("month").to_i / 3.0).ceil; + event.set("quarter",quarter)' +} +} diff --git a/roles/logstash_docker/files/jvm.options b/roles/logstash_docker/files/jvm.options new file mode 100644 index 000000000..458f0ec68 --- /dev/null +++ b/roles/logstash_docker/files/jvm.options @@ -0,0 +1,77 @@ +# JVM configuration + +# Xms represents the initial size of total heap space +# Xmx represents the maximum size of total heap space + +-Xms2g +-Xmx2g + +################################################################ +## Expert settings +################################################################ +## +## All settings below this section are considered +## expert settings. Don't tamper with them unless +## you understand what you are doing +## +################################################################ + +## GC configuration +# -XX:+UseParNewGC #Option does not exist +# -XX:+UseConcMarkSweepGC +# -XX:CMSInitiatingOccupancyFraction=75 +# -XX:+UseCMSInitiatingOccupancyOnly + +## Locale +# Set the locale language +#-Duser.language=en + +# Set the locale country +#-Duser.country=US + +# Set the locale variant, if any +#-Duser.variant= + +## basic + +# set the I/O temp directory +#-Djava.io.tmpdir=$HOME + +# set to headless, just in case +-Djava.awt.headless=true + +# ensure UTF-8 encoding by default (e.g. filenames) +-Dfile.encoding=UTF-8 + +# use our provided JNA always versus the system one +#-Djna.nosys=true + +# Turn on JRuby invokedynamic +-Djruby.compile.invokedynamic=true +# Force Compilation +-Djruby.jit.threshold=0 + +## heap dumps + +# generate a heap dump when an allocation from the Java heap fails +# heap dumps are created in the working directory of the JVM +-XX:+HeapDumpOnOutOfMemoryError + +# specify an alternative path for heap dumps +# ensure the directory exists and has sufficient space +#-XX:HeapDumpPath=${LOGSTASH_HOME}/heapdump.hprof + +## GC logging +#-XX:+PrintGCDetails +#-XX:+PrintGCTimeStamps +#-XX:+PrintGCDateStamps +#-XX:+PrintClassHistogram +#-XX:+PrintTenuringDistribution +#-XX:+PrintGCApplicationStoppedTime + +# log GC status to a file with time stamps +# ensure the directory exists +#-Xloggc:${LS_GC_LOG_FILE} + +# Entropy source for randomness +-Djava.security.egd=file:/dev/urandom diff --git a/roles/logstash_docker/files/log4j2.properties b/roles/logstash_docker/files/log4j2.properties new file mode 100644 index 000000000..683a51e0a --- /dev/null +++ b/roles/logstash_docker/files/log4j2.properties @@ -0,0 +1,101 @@ +tatus = error +name = LogstashPropertiesConfig + +appender.console.type = Console +appender.console.name = plain_console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n + +appender.json_console.type = Console +appender.json_console.name = json_console +appender.json_console.layout.type = JSONLayout +appender.json_console.layout.compact = true +appender.json_console.layout.eventEol = true + +appender.rolling.type = RollingFile +appender.rolling.name = plain_rolling +appender.rolling.fileName = ${sys:ls.logs}/logstash-${sys:ls.log.format}.log +appender.rolling.filePattern = ${sys:ls.logs}/logstash-${sys:ls.log.format}-%d{yyyy-MM-dd}-%i.log.gz +appender.rolling.policies.type = Policies +appender.rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling.policies.time.interval = 1 +appender.rolling.policies.time.modulate = true +appender.rolling.layout.type = PatternLayout +appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %-.10000m%n +appender.rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling.policies.size.size = 100MB +appender.rolling.strategy.type = DefaultRolloverStrategy +appender.rolling.strategy.max = 30 + +appender.json_rolling.type = RollingFile +appender.json_rolling.name = json_rolling +appender.json_rolling.fileName = ${sys:ls.logs}/logstash-${sys:ls.log.format}.log +appender.json_rolling.filePattern = ${sys:ls.logs}/logstash-${sys:ls.log.format}-%d{yyyy-MM-dd}-%i.log.gz +appender.json_rolling.policies.type = Policies +appender.json_rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.json_rolling.policies.time.interval = 1 +appender.json_rolling.policies.time.modulate = true +appender.json_rolling.layout.type = JSONLayout +appender.json_rolling.layout.compact = true +appender.json_rolling.layout.eventEol = true +appender.json_rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.json_rolling.policies.size.size = 100MB +appender.json_rolling.strategy.type = DefaultRolloverStrategy +appender.json_rolling.strategy.max = 30 + +rootLogger.level = ${sys:ls.log.level} +rootLogger.appenderRef.console.ref = ${sys:ls.log.format}_console +rootLogger.appenderRef.rolling.ref = ${sys:ls.log.format}_rolling + +# Slowlog + +appender.console_slowlog.type = Console +appender.console_slowlog.name = plain_console_slowlog +appender.console_slowlog.layout.type = PatternLayout +appender.console_slowlog.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n + +appender.json_console_slowlog.type = Console +appender.json_console_slowlog.name = json_console_slowlog +appender.json_console_slowlog.layout.type = JSONLayout +appender.json_console_slowlog.layout.compact = true +appender.json_console_slowlog.layout.eventEol = true + +appender.rolling_slowlog.type = RollingFile +appender.rolling_slowlog.name = plain_rolling_slowlog +appender.rolling_slowlog.fileName = ${sys:ls.logs}/logstash-slowlog-${sys:ls.log.format}.log +appender.rolling_slowlog.filePattern = ${sys:ls.logs}/logstash-slowlog-${sys:ls.log.format}-%d{yyyy-MM-dd}-%i.log.gz +appender.rolling_slowlog.policies.type = Policies +appender.rolling_slowlog.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling_slowlog.policies.time.interval = 1 +appender.rolling_slowlog.policies.time.modulate = true +appender.rolling_slowlog.layout.type = PatternLayout +appender.rolling_slowlog.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %.10000m%n +appender.rolling_slowlog.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling_slowlog.policies.size.size = 100MB +appender.rolling_slowlog.strategy.type = DefaultRolloverStrategy +appender.rolling_slowlog.strategy.max = 30 + +appender.json_rolling_slowlog.type = RollingFile +appender.json_rolling_slowlog.name = json_rolling_slowlog +appender.json_rolling_slowlog.fileName = ${sys:ls.logs}/logstash-slowlog-${sys:ls.log.format}.log +appender.json_rolling_slowlog.filePattern = ${sys:ls.logs}/logstash-slowlog-${sys:ls.log.format}-%d{yyyy-MM-dd}-%i.log.gz +appender.json_rolling_slowlog.policies.type = Policies +appender.json_rolling_slowlog.policies.time.type = TimeBasedTriggeringPolicy +appender.json_rolling_slowlog.policies.time.interval = 1 +appender.json_rolling_slowlog.policies.time.modulate = true +appender.json_rolling_slowlog.layout.type = JSONLayout +appender.json_rolling_slowlog.layout.compact = true +appender.json_rolling_slowlog.layout.eventEol = true +appender.json_rolling_slowlog.policies.size.type = SizeBasedTriggeringPolicy +appender.json_rolling_slowlog.policies.size.size = 100MB +appender.json_rolling_slowlog.strategy.type = DefaultRolloverStrategy +appender.json_rolling_slowlog.strategy.max = 30 + +logger.slowlog.name = slowlog +logger.slowlog.level = trace +logger.slowlog.appenderRef.console_slowlog.ref = ${sys:ls.log.format}_console_slowlog +logger.slowlog.appenderRef.rolling_slowlog.ref = ${sys:ls.log.format}_rolling_slowlog +logger.slowlog.additivity = false + +logger.licensereader.name = logstash.licensechecker.licensereader +logger.licensereader.level = error diff --git a/roles/logstash_docker/files/pipelines.yml b/roles/logstash_docker/files/pipelines.yml new file mode 100644 index 000000000..c471a91d3 --- /dev/null +++ b/roles/logstash_docker/files/pipelines.yml @@ -0,0 +1,4 @@ +- pipeline.id: core + path.config: "/etc/logstash/conf.d/core" + pipeline.workers: 2 + queue.type: persisted diff --git a/roles/logstash_docker/tasks/main.yml b/roles/logstash_docker/tasks/main.yml new file mode 100644 index 000000000..b6b7291fa --- /dev/null +++ b/roles/logstash_docker/tasks/main.yml @@ -0,0 +1,113 @@ +--- +- name: Create required directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: "{{ container_ids.logstash }}" + group: "{{ container_ids.logstash }}" + mode: 0750 + with_items: + - /opt/openconext/logstash + - /opt/openconext/logstash/config + - /opt/openconext/logstash/conf.d + - /opt/openconext/logstash/conf.d/core + - /data/logstash + +- name: Place logstash settings files + ansible.builtin.copy: + src: "files/{{ item }}" + dest: "/opt/openconext/logstash/config/{{ item }}" + owner: "{{ container_ids.logstash }}" + group: "{{ container_ids.logstash }}" + mode: 0644 + with_items: + - jvm.options + - pipelines.yml + - log4j2.properties + +- name: Place logstash config file + ansible.builtin.template: + src: templates/logstash.yml.j2 + dest: /opt/openconext/logstash/config/logstash.yml + owner: "{{ container_ids.logstash }}" + group: "{{ container_ids.logstash }}" + mode: 0644 + +- name: Place core config files + ansible.builtin.copy: + src: "files/core/{{ item }}" + dest: "/opt/openconext/logstash/conf.d/core/{{ item }}" + owner: "{{ container_ids.logstash }}" + group: "{{ container_ids.logstash }}" + mode: 0644 + with_items: + - 02-filebeat-input.conf + - 13-core-ebauth.conf + +- name: Place core config file for influxdb + ansible.builtin.template: + src: templates/33-core-influxdb-output.conf.j2 + dest: /opt/openconext/logstash/conf.d/core/33-core-influxdb-output.conf + owner: "{{ container_ids.logstash }}" + group: "{{ container_ids.logstash }}" + mode: 0644 + +- name: Create logstash container + community.docker.docker_container: + name: "logstash" + image: "ghcr.io/openconext/openconext-basecontainers/logstash8:latest" + state: started + restart_policy: "always" + networks: + - name: "loadbalancer" + ports: + - 5044:5044 + mounts: + - source: /opt/openconext/logstash/config + target: /usr/share/logstash/config + type: bind + - source: /opt/openconext/logstash/conf.d/ + target: /etc/logstash/conf.d/ + type: bind + - source: /data/logstash/ + target: /data/logstash/ + type: bind + register: logstash_container_created + +- name: Change logstash UID + community.docker.docker_container_exec: + container: logstash + user: "root" + command: + sh -c ' + groupmod -g {{ container_ids.logstash }} logstash && + usermod -u {{ container_ids.logstash }} -g {{ container_ids.logstash }} logstash' + when: logstash_container_created.changed + +- name: Check if running user needs to be changed + community.docker.docker_container_info: + name: "logstash" + register: logstash_container + +- name: Update logstash container to run as non root + community.docker.docker_container: + name: "logstash" + image: "ghcr.io/openconext/openconext-basecontainers/logstash8:latest" + state: started + restart_policy: "always" + networks: + - name: "loadbalancer" + ports: + - 5044:5044 + user: "{{ container_ids.logstash }}" + mounts: + - source: /opt/openconext/logstash/config + target: /usr/share/logstash/config + type: bind + - source: /opt/openconext/logstash/conf.d/ + target: /etc/logstash/conf.d/ + type: bind + - source: /data/logstash/ + target: /data/logstash/ + type: bind + when: logstash_container.container.Config.User | default('') != container_ids.logstash diff --git a/roles/logstash_docker/templates/33-core-influxdb-output.conf.j2 b/roles/logstash_docker/templates/33-core-influxdb-output.conf.j2 new file mode 100644 index 000000000..8e1a2c445 --- /dev/null +++ b/roles/logstash_docker/templates/33-core-influxdb-output.conf.j2 @@ -0,0 +1,50 @@ +filter { + if [prog] == "EBAUTH" { + ruby { + code => 'require "date"; + d = DateTime.parse(event.get("[ebauth][context][login_stamp]")); + ds = ((d.strftime("%Q").to_i * 1000) + d.strftime("%6N")[3,3].to_i).to_s; + event.set("[ebauth][context][micro_seconds]", ds); + if (event.get("[ebauth][context][proxied_sp_entity_ids][0]")) ; + proxied_sp_entity_id = (event.get("[ebauth][context][proxied_sp_entity_ids][0]")); + event.set("[ebauth][context][sp_entity_id]", proxied_sp_entity_id) + end' + } + } +} +filter { + if [prog] == "EBAUTH" and [ebauth][context][authncontextclassref] not in ["http://surfconext.nl/assurance/loa1_5","http://surfconext.nl/assurance/loa2","http://surfconext.nl/assurance/loa3"] { + mutate + { replace => { "[ebauth][context][authncontextclassref]" => "http://surfconext.nl/assurance/loa1"} + } + } +} + +output { + if [prog] == "EBAUTH" { + influxdb { + host => "{{ influx_stats_dbhost }}" + port => 8086 + db => "log_logins" + user => "{{ influxdb_stats_user }}" + password => "{{ influxdb_stats_password }}" + measurement => "EBAUTH" + retention_policy => "autogen" + use_event_fields_for_data_points => false + allow_time_override => true + time_precision => "u" + data_points => { + "idp_entity_id" => "%{[ebauth][context][idp_entity_id]}" + "user_id" => "%{[ebauth][context][user_id_hashed]}" + "sp_entity_id" => "%{[ebauth][context][sp_entity_id]}" + "state" => "%{[ebauth][context][workflow_state]}" + "authncontextclassref" => "%{[ebauth][context][authncontextclassref]}" + "time" => "%{[ebauth][context][micro_seconds]}" + "month" => "%{+MM}" + "year" => "%{+YYYY}" + "quarter" => "%{[quarter]}" + } + send_as_tags => ["idp_entity_id","sp_entity_id","state","month","year","quarter","authncontextclassref"] + } + } +} diff --git a/roles/logstash_docker/templates/logstash.yml.j2 b/roles/logstash_docker/templates/logstash.yml.j2 new file mode 100644 index 000000000..9619f6bf7 --- /dev/null +++ b/roles/logstash_docker/templates/logstash.yml.j2 @@ -0,0 +1,9 @@ +api.http.host: 0.0.0.0 +monitoring.elasticsearch.hosts: http://elasticsearch:9200 +pipeline.workers: 1 +queue.type: persisted +queue.max_bytes: {{ logstash_memory_gb }}gb +dead_letter_queue.enable: true +dead_letter_queue.max_bytes: 1024mb +path.logs: /var/log/logstash +path.data: /data/logstash diff --git a/roles/stats/tasks/main.yml b/roles/stats/tasks/main.yml index f6e1d159f..9f9e80c7d 100644 --- a/roles/stats/tasks/main.yml +++ b/roles/stats/tasks/main.yml @@ -70,8 +70,8 @@ restart_policy: "always" state: "started" env: - RUNAS_UID: "{{ stats_uid.uid }}" - RUNAS_GID: "{{ stats_gid.gid }}" + RUNAS_UID: "{{ stats_uid.uid | string }}" + RUNAS_GID: "{{ stats_gid.gid | string }}" networks: - name: "loadbalancer" mounts: