From 6e74ddd17fe3a352ce5d0e9ff59730f622390b4f Mon Sep 17 00:00:00 2001 From: Sergey Shubin Date: Thu, 12 May 2022 14:05:33 +0300 Subject: [PATCH 1/7] Update dashboards.yml. Fix owner and permissions for home folder Signed-off-by: Sergey Shubin --- roles/linux/dashboards/tasks/dashboards.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/roles/linux/dashboards/tasks/dashboards.yml b/roles/linux/dashboards/tasks/dashboards.yml index 0be1588..c39c9f7 100644 --- a/roles/linux/dashboards/tasks/dashboards.yml +++ b/roles/linux/dashboards/tasks/dashboards.yml @@ -34,6 +34,20 @@ mode: 0644 backup: yes +- name: Dashboards Install | Set the file ownerships + file: + dest: "{{ os_dashboards_home }}" + owner: "{{ os_dashboards_user }}" + group: "{{ os_dashboards_user }}" + recurse: yes + +- name: Dashboards Install | Set the folder permission + file: + dest: "{{ os_conf_dir }}" + owner: "{{ os_dashboards_user }}" + group: "{{ os_dashboards_user }}" + mode: 0700 + - name: Dashboards Install | create systemd service template: src: dashboards.service From 9925a0cad8f6fa7bd2d7b1943dab6941a6c8def9 Mon Sep 17 00:00:00 2001 From: Sergey Shubin Date: Thu, 12 May 2022 14:30:09 +0300 Subject: [PATCH 2/7] The validity period of certificates is set to a variable Signed-off-by: Sergey Shubin --- inventories/opensearch/group_vars/all/all.yml | 3 +++ roles/linux/opensearch/templates/tlsconfig.yml | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/inventories/opensearch/group_vars/all/all.yml b/inventories/opensearch/group_vars/all/all.yml index 4975cd5..f3e4175 100644 --- a/inventories/opensearch/group_vars/all/all.yml +++ b/inventories/opensearch/group_vars/all/all.yml @@ -34,3 +34,6 @@ cluster_type: multi-node os_user: opensearch os_dashboards_user: opensearch-dashboards + +# Number of days that certificates are valid +cert_valid_days: 730 diff --git a/roles/linux/opensearch/templates/tlsconfig.yml b/roles/linux/opensearch/templates/tlsconfig.yml index 2125ed6..5b7408f 100644 --- a/roles/linux/opensearch/templates/tlsconfig.yml +++ b/roles/linux/opensearch/templates/tlsconfig.yml @@ -2,13 +2,13 @@ ca: root: dn: CN=root.ca.{{ domain_name }},OU=CA,O={{ domain_name }}\, Inc.,DC={{ domain_name }} keysize: 2048 - validityDays: 730 + validityDays: {{ cert_valid_days }} pkPassword: none file: root-ca.pem ### Default values and global settings defaults: - validityDays: 730 + validityDays: {{ cert_valid_days }} pkPassword: none # Set this to true in order to generate config and certificates for # the HTTP interface of nodes From 0a5885e3e7d27caa6fec0839b80ae646ba28402b Mon Sep 17 00:00:00 2001 From: Sergey Shubin Date: Fri, 13 May 2022 16:21:00 +0300 Subject: [PATCH 3/7] change HOME directory for {{ os_user }} and {{ os_dashboards_user }} and set them /bin/false shell Signed-off-by: Sergey Shubin --- roles/linux/dashboards/tasks/dashboards.yml | 6 ++++-- roles/linux/opensearch/tasks/opensearch.yml | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/roles/linux/dashboards/tasks/dashboards.yml b/roles/linux/dashboards/tasks/dashboards.yml index c39c9f7..fa6e90b 100644 --- a/roles/linux/dashboards/tasks/dashboards.yml +++ b/roles/linux/dashboards/tasks/dashboards.yml @@ -6,11 +6,13 @@ dest: "/tmp/opensearch-dashboards.tar.gz" register: download -- name: Dashboards Install | Create opensearch user +- name: Dashboards Install | Create opensearch dashboard user user: name: "{{ os_dashboards_user }}" state: present - shell: /bin/bash + shell: /bin/false + create_home: true + home: "{{ os_dashboards_home }}" when: download.changed - name: Dashboards Install | Create home directory diff --git a/roles/linux/opensearch/tasks/opensearch.yml b/roles/linux/opensearch/tasks/opensearch.yml index d54fbfd..7dd25f7 100644 --- a/roles/linux/opensearch/tasks/opensearch.yml +++ b/roles/linux/opensearch/tasks/opensearch.yml @@ -10,7 +10,9 @@ user: name: "{{ os_user }}" state: present - shell: /bin/bash + shell: /bin/false + create_home: true + home: "{{ os_home }}" when: download.changed - name: OpenSearch Install | Create home directory From abf770d05363dbc7c6bb681020e67ac3b0affd6e Mon Sep 17 00:00:00 2001 From: Sergey Shubin Date: Tue, 17 May 2022 18:51:33 +0300 Subject: [PATCH 4/7] auth_type (internal, openid). Custom configs, IaC 1. Added the ability to log in via OpenID 2. Added the ability to install custom configuration files for the cluster 3. Added the ability to reconfigure the cluster (in particular, update certificates) when expanding it 4. Added the ability not to change certificates if the cluster composition has not changed, but only the settings have changed. Signed-off-by: Sergey Shubin --- files/internal_users.yml | 26 ++ files/roles.yml | 54 ++++ files/roles_mapping.yml | 59 ++++ files/tenants.yml | 12 + inventories/opensearch/group_vars/all/all.yml | 59 ++++ roles/linux/dashboards/defaults/main.yml | 3 + roles/linux/dashboards/tasks/dashboards.yml | 6 +- .../templates/opensearch_dashboards.yml | 12 + roles/linux/opensearch/defaults/main.yml | 3 + roles/linux/opensearch/tasks/opensearch.yml | 6 +- roles/linux/opensearch/tasks/security.yml | 150 ++++++++- .../templates/security_plugin_conf.yml | 287 ++++++++++++++++++ 12 files changed, 659 insertions(+), 18 deletions(-) create mode 100644 files/internal_users.yml create mode 100644 files/roles.yml create mode 100644 files/roles_mapping.yml create mode 100644 files/tenants.yml create mode 100644 roles/linux/opensearch/templates/security_plugin_conf.yml diff --git a/files/internal_users.yml b/files/internal_users.yml new file mode 100644 index 0000000..e19bb07 --- /dev/null +++ b/files/internal_users.yml @@ -0,0 +1,26 @@ +--- +# This is the internal user database +# The hash value is a bcrypt hash and can be generated with plugin/tools/hash.sh + +_meta: + type: "internalusers" + config_version: 2 + +# Define your internal users here + +admin: + hash: "{{ admin_password }}" + reserved: true + backend_roles: + - "admin" + description: "admin user" + +kibanaserver: + hash: "{{ kibanaserver_password }}" + reserved: true + description: "kibanaserver user" + +logstash: + hash: "{{ logstash_password }}" + reserved: true + description: "logstash user" \ No newline at end of file diff --git a/files/roles.yml b/files/roles.yml new file mode 100644 index 0000000..36a2af7 --- /dev/null +++ b/files/roles.yml @@ -0,0 +1,54 @@ +--- +_meta: + type: "roles" + config_version: 2 + + +indexes_full_access: + reserved: false + index_permissions: + - index_patterns: + - "*" + allowed_actions: + - "*" + tenant_permissions: + - tenant_patterns: + - "*" + allowed_actions: + - "kibana_all_write" +# ---------------------------------------------------- +indexes_security_search_full_access: + reserved: true + index_permissions: + - index_patterns: + - "kube-apiserver-audit-*" + - "syslog-*" + allowed_actions: + - "indices:data/read/search*" + - "read" + - "view_index_metadata" + tenant_permissions: + - tenant_patterns: + - "SECURITY" + allowed_actions: + - "kibana_all_write" +# ---------------------------------------------------- +indexes_web_search_full_access: + reserved: true + index_permissions: + - index_patterns: + - "ingress-nginx-*" + - "mywebapp-*" + allowed_actions: + - "indices:data/read/search*" + - "read" + - "view_index_metadata" + tenant_permissions: + - tenant_patterns: + - "WEB" + allowed_actions: + - "kibana_all_write" +# ---------------------------------------------------- +# Restrict users so they can only view visualization and dashboard on OpenSearchDashboards +kibana_read_only: + reserved: true diff --git a/files/roles_mapping.yml b/files/roles_mapping.yml new file mode 100644 index 0000000..acd807c --- /dev/null +++ b/files/roles_mapping.yml @@ -0,0 +1,59 @@ +--- +# In this file users, backendroles and hosts can be mapped to Security roles. +# Permissions for OpenSearch roles are configured in roles.yml + +_meta: + type: "rolesmapping" + config_version: 2 + +kibana_server: + reserved: true + users: + - "kibanaserver" + +logstash: + reserved: true + users: + - "logstash" + +# Define your roles mapping here +all_access: + reserved: false + backend_roles: + - "admin" + - "opensearch_admin" + description: "Maps admin to all_access" +# ---------------------------------------------------- +indexes_full_access: + reserved: false + backend_roles: + - "opensearch_admin" + description: "Maps admin to indexes_full_access" +# ---------------------------------------------------- +own_index: + reserved: false + users: + - "*" + description: "Allow full access to an index named like the username" +# ---------------------------------------------------- +readall: + reserved: false + backend_roles: + - "opensearch_index_read_all" +# ---------------------------------------------------- +indexes_security_search_full_access: + reserved: true + backend_roles: + - "opensearch_index_read_all" + - "opensearch_index_read_security" + description: "Maps users to indexes_security_search_full_access" +# ---------------------------------------------------- +indexes_web_search_full_access: + reserved: true + backend_roles: + - "opensearch_index_read_all" + - "opensearch_index_read_web" + description: "Maps users to indexes_web_search_full_access" + + + diff --git a/files/tenants.yml b/files/tenants.yml new file mode 100644 index 0000000..da8ce76 --- /dev/null +++ b/files/tenants.yml @@ -0,0 +1,12 @@ +--- +_meta: + type: "tenants" + config_version: 2 + +# Define your tenants here +SECURITY: + reserved: false + description: "Tenant for security logs (e.g. kubernetes audit or opensearch audit)" +WEB: + reserved: false + description: "Tenant for web-app logs" diff --git a/inventories/opensearch/group_vars/all/all.yml b/inventories/opensearch/group_vars/all/all.yml index f3e4175..34c79fd 100644 --- a/inventories/opensearch/group_vars/all/all.yml +++ b/inventories/opensearch/group_vars/all/all.yml @@ -37,3 +37,62 @@ os_dashboards_user: opensearch-dashboards # Number of days that certificates are valid cert_valid_days: 730 + +# Auth type: 'internal' or 'oidc' (OpenID). Default: internal +auth_type: internal + +# OIDC settings +oidc: + description: "Authenticate via IdP" + # OpenID server URI + connect_url: https://oidc.example.com/auth/realms//.well-known/openid-configuration + # The JWT token field that contains the user name + subject_key: preferred_username + # the JWT token field that contains a list of user roles + roles_key: roles + # Scopes + scopes: "openid profile email" + # The address of Dashboards to redirect the user to after successful authentication + dashboards_url: http(s)://.example.com + # IdP client ID + client_id: opensearch + # IdP client secret + client_secret: "00000000-0000-0000-0000-000000000000" + +# Overwrite demo configurations with your own +copy_custom_security_configs: false + +# To override demo configurations, you can use your own configuration files. +# Place them in the "files" directory. Specify the path to the files +custom_security_plugin_configs: + - files/tenants.yml + - files/roles.yml + - files/roles_mapping.yml + - files/internal_users.yml + +# By default, if the /tmp/opensearch-nodecerts directory with certificates +# exists on the server from which the playbook is launched, it is assumed +# that the configuration has not changed and some settings are not copied +# to the target servers. +# +# Conversely, if the /tmp/opensearch-nodecerts directory does not exist on +# the server from which the playbook is launched, then new certificates and +# settings are generated and they are copied to the target servers. +# +# If you use this repository not only for the initial deployment of the +# cluster, but also for its automatic configuration via CI/CD, then new +# certificates will be generated every time the pipeline is launched, +# overwriting existing ones, which is not always necessary if the cluster is +# already in production. +# +# When iac_enable enabling, and all the cluster servers have all the necessary +# certificates, they will not be copied again. If at least on one server (for +# example, when adding a new server to the cluster) if there is not at least one +# certificate from the list, then all certificates on all cluster servers will +# be updated +# +# Also, if the option is enabled, the settings files will be updated with each +# execution (previously, the settings were updated only if the +# /tmp/opensearch-nodecerts directory was missing on the server from which the +# playbook was launched and new certificates were generated) +iac_enable: false \ No newline at end of file diff --git a/roles/linux/dashboards/defaults/main.yml b/roles/linux/dashboards/defaults/main.yml index d257a22..13c224d 100644 --- a/roles/linux/dashboards/defaults/main.yml +++ b/roles/linux/dashboards/defaults/main.yml @@ -17,3 +17,6 @@ os_nodes_dashboards: |- {%- endfor %} systemctl_path: /etc/systemd/system + +# Auth type: 'internal' or 'oidc' (OpenID). Default: internal +auth_type: internal diff --git a/roles/linux/dashboards/tasks/dashboards.yml b/roles/linux/dashboards/tasks/dashboards.yml index fa6e90b..0999a4a 100644 --- a/roles/linux/dashboards/tasks/dashboards.yml +++ b/roles/linux/dashboards/tasks/dashboards.yml @@ -13,7 +13,7 @@ shell: /bin/false create_home: true home: "{{ os_dashboards_home }}" - when: download.changed + when: download.changed or iac_enable - name: Dashboards Install | Create home directory file: @@ -21,11 +21,11 @@ state: directory owner: "{{ os_dashboards_user }}" group: "{{ os_dashboards_user }}" - when: download.changed + when: download.changed or iac_enable - name: Dashboards Install | Extract the tar file command: chdir=/tmp/ tar -xvzf opensearch-dashboards.tar.gz -C "{{ os_dashboards_home }}" --strip-components=1 - when: download.changed + when: download.changed or iac_enable - name: Dashboards Install | Copy Configuration File template: diff --git a/roles/linux/dashboards/templates/opensearch_dashboards.yml b/roles/linux/dashboards/templates/opensearch_dashboards.yml index ebcf8ff..b7b06a5 100644 --- a/roles/linux/dashboards/templates/opensearch_dashboards.yml +++ b/roles/linux/dashboards/templates/opensearch_dashboards.yml @@ -11,3 +11,15 @@ opensearch_security.multitenancy.tenants.preferred: ["Private", "Global"] opensearch_security.readonly_mode.roles: ["kibana_read_only"] # Use this setting if you are running dashboards without https opensearch_security.cookie.secure: false + + +# OpenID settings +{% if auth_type == 'oidc' %} +opensearch_security.auth.type: openid +opensearch_security.openid.base_redirect_url: "{{ oidc.dashboards_url }}" +opensearch_security.openid.client_id: "{{ oidc.client_id }}" +opensearch_security.openid.scope: "{{ oidc.scopes }}" +opensearch_security.openid.client_secret: "{{ oidc.client_secret }}" +opensearch_security.openid.connect_url: "{{ oidc.connect_url }}" +opensearch_security.openid.verify_hostnames: true +{% endif %} \ No newline at end of file diff --git a/roles/linux/opensearch/defaults/main.yml b/roles/linux/opensearch/defaults/main.yml index d35701e..c2ccd98 100644 --- a/roles/linux/opensearch/defaults/main.yml +++ b/roles/linux/opensearch/defaults/main.yml @@ -20,3 +20,6 @@ os_sec_plugin_tools_path: /usr/share/opensearch/plugins/opensearch-security/tool os_api_port: 9200 systemctl_path: /etc/systemd/system + +# Auth type: 'internal' or 'oidc' (OpenID). Default: internal +auth_type: internal diff --git a/roles/linux/opensearch/tasks/opensearch.yml b/roles/linux/opensearch/tasks/opensearch.yml index 7dd25f7..4c44988 100644 --- a/roles/linux/opensearch/tasks/opensearch.yml +++ b/roles/linux/opensearch/tasks/opensearch.yml @@ -13,7 +13,7 @@ shell: /bin/false create_home: true home: "{{ os_home }}" - when: download.changed + when: download.changed or iac_enable - name: OpenSearch Install | Create home directory file: @@ -21,11 +21,11 @@ state: directory owner: "{{ os_user }}" group: "{{ os_user }}" - when: download.changed + when: download.changed or iac_enable - name: OpenSearch Install | Extract the tar file command: chdir=/tmp/ tar -xvzf opensearch.tar.gz -C "{{ os_home }}" --strip-components=1 - when: download.changed + when: download.changed or iac_enable - name: OpenSearch Install | Copy Configuration File blockinfile: diff --git a/roles/linux/opensearch/tasks/security.yml b/roles/linux/opensearch/tasks/security.yml index 9aca582..c8f839e 100644 --- a/roles/linux/opensearch/tasks/security.yml +++ b/roles/linux/opensearch/tasks/security.yml @@ -2,6 +2,16 @@ ## Here we are going to use self-signed certificates for Transport (Node-Node communication) & REST API layer ## Using searchguard offline TLS tool to create node & root certificates +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +- name: Security Plugin configuration | Force remove local temporary directory for certificates generation + local_action: + module: file + path: /tmp/opensearch-nodecerts + state: absent + run_once: true + become: false + when: iac_enable + - name: Security Plugin configuration | Create local temporary directory for certificates generation local_action: module: file @@ -10,6 +20,7 @@ run_once: true register: configuration become: false +# <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< - name: Security Plugin configuration | Download certificates generation tool local_action: @@ -51,7 +62,61 @@ when: configuration.changed become: false -- name: Security Plugin configuration | Copy the node & admin certificates to opensearch nodes +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +- name: Security Plugin configuration | IaC enabled - Check certificate + block: + - name: Security Plugin configuration | Check cert exists + stat: + path: "{{ item }}" + get_attributes: no + get_checksum: no + get_mime: no + register: cert_stat_result + with_items: + - "{{ os_conf_dir }}/root-ca.pem" + - "{{ os_conf_dir }}/root-ca.key" + - "{{ os_conf_dir }}/{{ inventory_hostname }}.key" + - "{{ os_conf_dir }}/{{ inventory_hostname }}.pem" + - "{{ os_conf_dir }}/{{ inventory_hostname }}_http.key" + - "{{ os_conf_dir }}/{{ inventory_hostname }}_http.pem" + - "{{ os_conf_dir }}/admin.key" + - "{{ os_conf_dir }}/admin.pem" + + - name: Security Plugin configuration | Set fact. The initial value "Don't update certs" + set_fact: + force_update_cert: false + + - name: Security Plugin configuration | Set fact. Update certificates if at least one certificate is not found + set_fact: + force_update_cert: true + with_items: "{{ cert_stat_result.results }}" + when: item.stat.exists == False + + - name: Security Plugin configuration | debug 1 - force_update_cert + debug: + msg: "force_update_cert: {{ force_update_cert }}" + + - name: Security Plugin configuration | Count force_update_cert nodes + set_fact: + force_update_cert_nodes_count: "{{ hostvars | dict2items | selectattr('value.force_update_cert', 'defined') | rejectattr('value.force_update_cert', 'equalto', false) | map(attribute='value.force_update_cert') | list | length }}" + + - name: Security Plugin configuration | debug 2 - force_update_cert_nodes_count + debug: + msg: "force_update_cert_nodes_count: {{ force_update_cert_nodes_count }}" + + - name: Security Plugin configuration | Do need to update certificates + debug: + msg: "Need to update certificates..." + when: force_update_cert_nodes_count | int > 0 + when: iac_enable + +- name: Security Plugin configuration | IaC disabled - Count force_update_cert nodes + set_fact: + force_update_cert_nodes_count: 0 + when: not iac_enable +# <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + +- name: Security Plugin configuration | Copy the node & admin certificates to opensearch nodes if at least one certificate is not found on at least one server copy: src: "/tmp/opensearch-nodecerts/config/{{ item }}" dest: "{{ os_conf_dir }}" @@ -65,7 +130,7 @@ - "{{ inventory_hostname }}_http.pem" - admin.key - admin.pem - when: configuration.changed + when: (configuration.changed and not iac_enable) or (iac_enable and force_update_cert_nodes_count | int > 0) - name: Security Plugin configuration | Copy the security configuration file 1 to cluster blockinfile: @@ -74,7 +139,7 @@ backup: yes insertafter: EOF marker: "## {mark} OpenSearch Security common configuration ##" - when: configuration.changed + when: configuration.changed or iac_enable - name: Security Plugin configuration | Copy the security configuration file 2 to cluster blockinfile: @@ -83,11 +148,22 @@ backup: yes insertafter: EOF marker: "## {mark} opensearch Security Node & Admin certificates configuration ##" - when: configuration.changed + when: configuration.changed or iac_enable + +- name: Security Plugin configuration | Copy the security configuration file 3 to cluster + template: + src: security_plugin_conf.yml + dest: "{{ os_sec_plugin_conf_path }}/config.yml" + backup: yes + owner: "{{ os_user }}" + group: "{{ os_user }}" + mode: 0600 + force: yes + when: auth_type == 'oidc' - name: Security Plugin configuration | Prepare the opensearch security configuration file command: sed -i 's/searchguard/plugins.security/g' {{ os_conf_dir }}/opensearch.yml - when: configuration.changed + when: configuration.changed or iac_enable - name: Security Plugin configuration | Set the file ownerships file: @@ -109,9 +185,9 @@ state: restarted enabled: yes -- name: Pause for 3 seconds to provide sometime for OpenSearch start +- name: Pause for 10 seconds to provide sometime for OpenSearch start pause: - seconds: 3 + seconds: 10 - name: Security Plugin configuration | Copy the opensearch security internal users template template: @@ -119,7 +195,19 @@ dest: "{{ os_sec_plugin_conf_path }}/internal_users.yml" mode: 0644 run_once: true - when: configuration.changed + when: configuration.changed or iac_enable + +- name: Security Plugin configuration | Copy custom configuration files to cluster + template: + src: "{{ item }}" + dest: "{{ os_sec_plugin_conf_path }}/" + owner: "{{ os_user }}" + group: "{{ os_user }}" + backup: yes + mode: 0640 + force: yes + with_items: "{{ custom_security_plugin_configs }}" + when: copy_custom_security_configs - name: Security Plugin configuration | Set the Admin user password shell: > @@ -128,7 +216,7 @@ environment: JAVA_HOME: "{{ os_home }}/jdk" run_once: true - when: configuration.changed + when: configuration.changed or iac_enable - name: Security Plugin configuration | Set the kibanaserver user pasword shell: > @@ -137,7 +225,45 @@ environment: JAVA_HOME: "{{ os_home }}/jdk" run_once: true - when: configuration.changed + when: configuration.changed or iac_enable + +- name: Security Plugin configuration | Check that the files/internal_users.yml exists + stat: + path: files/internal_users.yml + register: custom_users_result + delegate_to: localhost + run_once: true + +- name: Security Plugin configuration | Check for a custom configuration for internal users and hash passwords for them + block: + + - name: Security Plugin configuration | Load custom internal users configuration + include_vars: + file: files/internal_users.yml + name: custom_users + run_once: true + + - name: Security Plugin configuration | Filter service keys from the list of users + set_fact: + custom_users_filtered: '{{ custom_users|dict2items|rejectattr("key", "equalto", "_meta")|list|items2dict }}' + +# - name: Security Plugin configuration | debug +# debug: +# msg: "{{ item }} password: {{ lookup('vars', item + '_password') }}" +# with_items: "{{ custom_users_filtered }}" +# run_once: true + + - name: Security Plugin configuration | Set passwords for all users from custom config + shell: > + sed -i '/hash: / s,{{ lookup('vars', item + '_password') }},'$(bash {{ os_sec_plugin_tools_path }}/hash.sh -p {{ lookup('vars', item + '_password') }} | tail -1)',' + {{ os_sec_plugin_conf_path }}/internal_users.yml + environment: + JAVA_HOME: "{{ os_home }}/jdk" + run_once: true + when: configuration.changed or copy_custom_security_configs + with_items: "{{ custom_users_filtered }}" + + when: custom_users_result.stat.exists - name: Security Plugin configuration | Initialize the opensearch security index in opensearch shell: > @@ -145,13 +271,13 @@ -cacert {{ os_conf_dir }}/root-ca.pem -cert {{ os_conf_dir }}/admin.pem -key {{ os_conf_dir }}/admin.key - -f {{ os_sec_plugin_conf_path }}/internal_users.yml + -cd {{ os_sec_plugin_conf_path }} -nhnv -icl -h {{ hostvars[inventory_hostname]['ip'] }} environment: JAVA_HOME: "{{ os_home }}/jdk" run_once: true - when: configuration.changed + when: configuration.changed or copy_custom_security_configs - name: Security Plugin configuration | Cleanup local temporary directory local_action: diff --git a/roles/linux/opensearch/templates/security_plugin_conf.yml b/roles/linux/opensearch/templates/security_plugin_conf.yml new file mode 100644 index 0000000..249058c --- /dev/null +++ b/roles/linux/opensearch/templates/security_plugin_conf.yml @@ -0,0 +1,287 @@ +--- + +# This is the main OpenSearch Security configuration file where authentication +# and authorization is defined. +# +# You need to configure at least one authentication domain in the authc of this file. +# An authentication domain is responsible for extracting the user credentials from +# the request and for validating them against an authentication backend like Active Directory for example. +# +# If more than one authentication domain is configured the first one which succeeds wins. +# If all authentication domains fail then the request is unauthenticated. +# In this case an exception is thrown and/or the HTTP status is set to 401. +# +# After authentication authorization (authz) will be applied. There can be zero or more authorizers which collect +# the roles from a given backend for the authenticated user. +# +# Both, authc and auth can be enabled/disabled separately for REST and TRANSPORT layer. Default is true for both. +# http_enabled: true +# transport_enabled: true +# +# For HTTP it is possible to allow anonymous authentication. If that is the case then the HTTP authenticators try to +# find user credentials in the HTTP request. If credentials are found then the user gets regularly authenticated. +# If none can be found the user will be authenticated as an "anonymous" user. This user has always the username "anonymous" +# and one role named "anonymous_backendrole". +# If you enable anonymous authentication all HTTP authenticators will not challenge. +# +# +# Note: If you define more than one HTTP authenticators make sure to put non-challenging authenticators like "proxy" or "clientcert" +# first and the challenging one last. +# Because it's not possible to challenge a client with two different authentication methods (for example +# Kerberos and Basic) only one can have the challenge flag set to true. You can cope with this situation +# by using pre-authentication, e.g. sending a HTTP Basic authentication header in the request. +# +# Default value of the challenge flag is true. +# +# +# HTTP +# basic (challenging) +# proxy (not challenging, needs xff) +# kerberos (challenging) +# clientcert (not challenging, needs https) +# jwt (not challenging) +# host (not challenging) #DEPRECATED, will be removed in a future version. +# host based authentication is configurable in roles_mapping + +# Authc +# internal +# noop +# ldap + +# Authz +# ldap +# noop + + + +_meta: + type: "config" + config_version: 2 + +config: + dynamic: + # Set filtered_alias_mode to 'disallow' to forbid more than 2 filtered aliases per index + # Set filtered_alias_mode to 'warn' to allow more than 2 filtered aliases per index but warns about it (default) + # Set filtered_alias_mode to 'nowarn' to allow more than 2 filtered aliases per index silently + #filtered_alias_mode: warn + #do_not_fail_on_forbidden: false + #kibana: + # Kibana multitenancy + #multitenancy_enabled: true + #server_username: kibanaserver + #index: '.kibana' +# OpenID settings +{% if auth_type == 'oidc' %} + http: + anonymous_auth_enabled: false + xff: + enabled: false + internalProxies: ".*" + remoteIpHeader: "x-forwarded-for" + authc: + # In order for Dashboards to access OpenSearch, you must first use + # authentication_backend.type: internal + basic_internal_auth_domain: + description: "Authenticate via HTTP Basic against internal users database" + http_enabled: true + transport_enabled: false + order: 0 + http_authenticator: + type: basic + challenge: false + authentication_backend: + type: internal + openid_auth_domain: + description: "Authenticate via OpenID" + http_enabled: true + transport_enabled: true + order: 1 + http_authenticator: + type: openid + challenge: false + config: + enable_ssl: false + verify_hostnames: false + subject_key: {{ oidc.subject_key}} + roles_key: {{ oidc.roles_key}} + openid_connect_url: {{ oidc.connect_url}} + kibana_url: {{ oidc.dashboards_url}} + authentication_backend: + type: noop + authz: {} +{% else %} + http: + anonymous_auth_enabled: false + xff: + enabled: false + internalProxies: '192\.168\.0\.10|192\.168\.0\.11' # regex pattern + #internalProxies: '.*' # trust all internal proxies, regex pattern + #remoteIpHeader: 'x-forwarded-for' + ###### see https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html for regex help + ###### more information about XFF https://en.wikipedia.org/wiki/X-Forwarded-For + ###### and here https://tools.ietf.org/html/rfc7239 + ###### and https://tomcat.apache.org/tomcat-8.0-doc/config/valve.html#Remote_IP_Valve + authc: + kerberos_auth_domain: + http_enabled: false + transport_enabled: false + order: 6 + http_authenticator: + type: kerberos + challenge: true + config: + # If true a lot of kerberos/security related debugging output will be logged to standard out + krb_debug: false + # If true then the realm will be stripped from the user name + strip_realm_from_principal: true + authentication_backend: + type: noop + basic_internal_auth_domain: + description: "Authenticate via HTTP Basic against internal users database" + http_enabled: true + transport_enabled: true + order: 4 + http_authenticator: + type: basic + challenge: true + authentication_backend: + type: intern + proxy_auth_domain: + description: "Authenticate via proxy" + http_enabled: false + transport_enabled: false + order: 3 + http_authenticator: + type: proxy + challenge: false + config: + user_header: "x-proxy-user" + roles_header: "x-proxy-roles" + authentication_backend: + type: noop + jwt_auth_domain: + description: "Authenticate via Json Web Token" + http_enabled: false + transport_enabled: false + order: 0 + http_authenticator: + type: jwt + challenge: false + config: + signing_key: "base64 encoded HMAC key or public RSA/ECDSA pem key" + jwt_header: "Authorization" + jwt_url_parameter: null + roles_key: null + subject_key: null + authentication_backend: + type: noop + clientcert_auth_domain: + description: "Authenticate via SSL client certificates" + http_enabled: false + transport_enabled: false + order: 2 + http_authenticator: + type: clientcert + config: + username_attribute: cn #optional, if omitted DN becomes username + challenge: false + authentication_backend: + type: noop + ldap: + description: "Authenticate via LDAP or Active Directory" + http_enabled: false + transport_enabled: false + order: 5 + http_authenticator: + type: basic + challenge: false + authentication_backend: + # LDAP authentication backend (authenticate users against a LDAP or Active Directory) + type: ldap + config: + # enable ldaps + enable_ssl: false + # enable start tls, enable_ssl should be false + enable_start_tls: false + # send client certificate + enable_ssl_client_auth: false + # verify ldap hostname + verify_hostnames: true + hosts: + - localhost:8389 + bind_dn: null + password: null + userbase: 'ou=people,dc=example,dc=com' + # Filter to search for users (currently in the whole subtree beneath userbase) + # {0} is substituted with the username + usersearch: '(sAMAccountName={0})' + # Use this attribute from the user as username (if not set then DN is used) + username_attribute: null + authz: + roles_from_myldap: + description: "Authorize via LDAP or Active Directory" + http_enabled: false + transport_enabled: false + authorization_backend: + # LDAP authorization backend (gather roles from a LDAP or Active Directory, you have to configure the above LDAP authentication backend settings too) + type: ldap + config: + # enable ldaps + enable_ssl: false + # enable start tls, enable_ssl should be false + enable_start_tls: false + # send client certificate + enable_ssl_client_auth: false + # verify ldap hostname + verify_hostnames: true + hosts: + - localhost:8389 + bind_dn: null + password: null + rolebase: 'ou=groups,dc=example,dc=com' + # Filter to search for roles (currently in the whole subtree beneath rolebase) + # {0} is substituted with the DN of the user + # {1} is substituted with the username + # {2} is substituted with an attribute value from user's directory entry, of the authenticated user. Use userroleattribute to specify the name of the attribute + rolesearch: '(member={0})' + # Specify the name of the attribute which value should be substituted with {2} above + userroleattribute: null + # Roles as an attribute of the user entry + userrolename: disabled + #userrolename: memberOf + # The attribute in a role entry containing the name of that role, Default is "name". + # Can also be "dn" to use the full DN as rolename. + rolename: cn + # Resolve nested roles transitive (roles which are members of other roles and so on ...) + resolve_nested_roles: true + userbase: 'ou=people,dc=example,dc=com' + # Filter to search for users (currently in the whole subtree beneath userbase) + # {0} is substituted with the username + usersearch: '(uid={0})' + # Skip users matching a user name, a wildcard or a regex pattern + #skip_users: + # - 'cn=Michael Jackson,ou*people,o=TEST' + # - '/\S*/' + roles_from_another_ldap: + description: "Authorize via another Active Directory" + http_enabled: false + transport_enabled: false + authorization_backend: + type: ldap + #config goes here ... + # auth_failure_listeners: + # ip_rate_limiting: + # type: ip + # allowed_tries: 10 + # time_window_seconds: 3600 + # block_expiry_seconds: 600 + # max_blocked_clients: 100000 + # max_tracked_clients: 100000 + # internal_authentication_backend_limiting: + # type: username + # authentication_backend: intern + # allowed_tries: 10 + # time_window_seconds: 3600 + # block_expiry_seconds: 600 + # max_blocked_clients: 100000 +{% endif %} From 4b4bf51a01ac2e6db9a42c78e18914dab695063d Mon Sep 17 00:00:00 2001 From: Sergey Shubin Date: Wed, 18 May 2022 11:07:37 +0300 Subject: [PATCH 5/7] readme. description for OpenID, IaC, custom configuration files Signed-off-by: Sergey Shubin --- README.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/README.md b/README.md index 2bb6373..9d724bc 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,8 @@ This ansible playbook supports the following, - Configure TLS/SSL for OpenSearch transport layer(Nodes to Nodes communication) and REST API layer - Generate self-signed certificates to configure TLS/SSL for opensearch - Configure the Internal Users Database with limited users and user-defined passwords +- Configuration of authentication and authorization via OpenID +- Overriding default settings with your own - Install and configure the Apache2.0 opensource OpenSearch Dashboards ### Prerequisite @@ -87,6 +89,9 @@ cluster_type: single-node You should set the reserved users(`admin` and `kibanaserver`) password using `admin_password` and `kibanaserver_password` variables. +If you define your own internal users (in addition to the reserved `admin` and `kibanaserver`) in custom configuration +files, then passwords to them should be set via variables on the principle of `_password` + It will install and configure the opensearch. Once the deployment completed, you can access the opensearch Dashboards with user `admin` and password which you provided for variable `admin_password`. # Deploy with ansible playbook - run the playbook as non-root user which have sudo privileges, @@ -94,6 +99,43 @@ It will install and configure the opensearch. Once the deployment completed, you **Note**: Change the user details in `ansible_user` parameter in `inventories/opensearch/hosts` inventory file. +### OpenID authentification +To enable authentication via OpenID, you need to change the `auth_type` variable in the inventory file +`inventories/opensearch/group_vars/all/all.yml` by setting the value `oidc` and prescribe the necessary settings +in the `oidc:` block. + +### Custom configuration files + +To override the default settings files, you need to put your settings in the `files` directory. The files should be +named exactly the same as the original ones (internal_users.yml, roles.yml, tenants.yml, etc.) + +Especially note the file `files/internal_users.yml`. If it exists and the `copy_custom_security_configs: true` setting is enabled, +then only in this case the task of setting passwords for internal users from variables is started. If the file `internal_users.yml` +is not located in the `files` directory, but, for example, in one of its subdirectories, then playbook will not work correctly + +### IaC (Infrastructure-as-Code) + +If you want to use the role not only for the initial deployment of the cluster, but also for further management of it, +then set the `iac_enable` parameter to `true`. + +By default, if the /tmp/opensearch-nodecerts directory with certificates exists on the server from which the playbook +is launched, it is assumed that the configuration has not changed and some settings are not copied to the target servers. + +Conversely, if the /tmp/opensearch-nodecerts directory does not exist on the server from which the playbook is launched, +then new certificates and settings are generated and they are copied to the target servers. + +If you use this repository not only for the initial deployment of the cluster, but also for its automatic configuration +via CI/CD, then new certificates will be generated every time the pipeline is launched, overwriting existing ones, which +is not always necessary if the cluster is already in production. + +When iac_enable enabling, and all the cluster servers have all the necessary certificates, they will not be copied again. +If at least on one server (for example, when adding a new server to the cluster) if there is not at least one certificate +from the list, then all certificates on all cluster servers will be updated + +Also, if the option is enabled, the settings files will be updated with each execution (previously, the settings were +updated only if the /tmp/opensearch-nodecerts directory was missing on the server from which the playbook was launched +and new certificates were generated) + ## Contributing See [developer guide](DEVELOPER_GUIDE.md) and [how to contribute to this project](CONTRIBUTING.md). From 1ab0815c08f01d712f35aff80fcc87adc0bcf3ad Mon Sep 17 00:00:00 2001 From: Sergey Shubin Date: Tue, 24 May 2022 16:34:19 +0300 Subject: [PATCH 6/7] Added the ability to automatically create ISM policies. The "ISM Policies" section have been added to the readme. Signed-off-by: Sergey Shubin --- README.md | 9 ++ files/ism/policy/delete_after_30d.json | 42 ++++++ files/ism/policy/hot7_warm30_delete.json | 60 +++++++++ inventories/opensearch/group_vars/all/all.yml | 23 +++- roles/linux/opensearch/tasks/ism.yml | 122 ++++++++++++++++++ roles/linux/opensearch/tasks/main.yml | 4 + 6 files changed, 259 insertions(+), 1 deletion(-) create mode 100644 files/ism/policy/delete_after_30d.json create mode 100644 files/ism/policy/hot7_warm30_delete.json create mode 100644 roles/linux/opensearch/tasks/ism.yml diff --git a/README.md b/README.md index 9d724bc..9a33656 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ This ansible playbook supports the following, - Configure the Internal Users Database with limited users and user-defined passwords - Configuration of authentication and authorization via OpenID - Overriding default settings with your own +- Creation/Updating ISM Policies - Install and configure the Apache2.0 opensource OpenSearch Dashboards ### Prerequisite @@ -136,6 +137,14 @@ Also, if the option is enabled, the settings files will be updated with each exe updated only if the /tmp/opensearch-nodecerts directory was missing on the server from which the playbook was launched and new certificates were generated) +### ISM Policies + +OpenSearch uses the ISM (Index State Management) plugin to manage the lifecycle of indexes. With the help of policies, +you can, for example, change the number of replicas for indexes, when certain conditions occur, or delete them. + +If you want to manage policies using the opensearch role, set the `apply_custom_ism: yes` parameter, and create json +files with policies in the `files/ism/policy` directory. Examples of policies can be found in the same directory. + ## Contributing See [developer guide](DEVELOPER_GUIDE.md) and [how to contribute to this project](CONTRIBUTING.md). diff --git a/files/ism/policy/delete_after_30d.json b/files/ism/policy/delete_after_30d.json new file mode 100644 index 0000000..314f954 --- /dev/null +++ b/files/ism/policy/delete_after_30d.json @@ -0,0 +1,42 @@ +{ + "policy": { + "description": "delete after 30d workflow", + "default_state": "hot", + "schema_version": 1, + "ism_template": { + "index_patterns": [ + "ingress-nginx-*", + "mywebapp-*" + ], + "priority": 100 + }, + "states": [ + { + "name": "hot", + "actions": [ + { + "replica_count": { + "number_of_replicas": 1 + } + } + ], + "transitions": [ + { + "state_name": "delete", + "conditions": { + "min_index_age": "30d" + } + } + ] + }, + { + "name": "delete", + "actions": [ + { + "delete": {} + } + ] + } + ] + } +} \ No newline at end of file diff --git a/files/ism/policy/hot7_warm30_delete.json b/files/ism/policy/hot7_warm30_delete.json new file mode 100644 index 0000000..6eeae66 --- /dev/null +++ b/files/ism/policy/hot7_warm30_delete.json @@ -0,0 +1,60 @@ +{ + "policy": { + "description": "hot (7d) warm (30d) delete (after 30d) workflow", + "default_state": "hot", + "schema_version": 1, + "ism_template": { + "index_patterns": [ + "kube-apiserver-audit-", + "syslog-*" + ], + "priority": 100 + }, + "states": [ + { + "name": "hot", + "actions": [ + { + "replica_count": { + "number_of_replicas": 1 + } + } + ], + "transitions": [ + { + "state_name": "warm", + "conditions": { + "min_index_age": "7d" + } + } + ] + }, + { + "name": "warm", + "actions": [ + { + "replica_count": { + "number_of_replicas": 0 + } + } + ], + "transitions": [ + { + "state_name": "delete", + "conditions": { + "min_index_age": "30d" + } + } + ] + }, + { + "name": "delete", + "actions": [ + { + "delete": {} + } + ] + } + ] + } +} \ No newline at end of file diff --git a/inventories/opensearch/group_vars/all/all.yml b/inventories/opensearch/group_vars/all/all.yml index 34c79fd..1262b16 100644 --- a/inventories/opensearch/group_vars/all/all.yml +++ b/inventories/opensearch/group_vars/all/all.yml @@ -95,4 +95,25 @@ custom_security_plugin_configs: # execution (previously, the settings were updated only if the # /tmp/opensearch-nodecerts directory was missing on the server from which the # playbook was launched and new certificates were generated) -iac_enable: false \ No newline at end of file +iac_enable: false + +# Whether to apply the settings to the OpenSearch ISM instance. +# The policy files are located in the files/ism/policy directory +# and must have the extension ".json". +# The policies specify the index templates to which they should be applied. +# In this way, you can configure, for example, index rotation +apply_custom_ism: no + +# The index patterns for which the policy is applied are configured in the +# policy file in the section "policy.ism_template.index_patterns" +custom_ism: + # Settings for Policies + policy: + # Apply all policies from the {{ dir }} directory or only the specified ones + apply_all: yes + # Directory with policy files + dir: "files/ism/policy" + # Policy files to be applied to the OpenSearch instance if "apply_all: no" + files: + - files/ism/policy/hot7_warm30_delete.json + - files/ism/policy/delete_after_30d.json diff --git a/roles/linux/opensearch/tasks/ism.yml b/roles/linux/opensearch/tasks/ism.yml new file mode 100644 index 0000000..03bc8d0 --- /dev/null +++ b/roles/linux/opensearch/tasks/ism.yml @@ -0,0 +1,122 @@ +--- +# Tasks are written based on +# https://github.com/pashtet04/ansible-opensearch/blob/main/tasks/init.yml + +# register: 2 variables are used "ism_all" and "ism_custom", due to the fact +# that if "custom_ism.policy.apply_all: yes) is set, then when using one variable +# (for example, "register: ism") in the tasks "Get policies for all files" and +# "Get policies for selected files", then the task "Get policies for selected files" +# (which follows the second one) overwrites the state of the variable with values +# of the form: +# {'results': [ +# {'changed': False, 'skipped': True, 'skip_reason': 'Conditional result was False', 'item': 'files/ism/policy/test1.json', 'ansible_loop_var': 'item'}, +# {'changed': False, 'skipped': True, 'skip_reason': 'Conditional result was False', 'item': 'files/ism/policy/test2.json', 'ansible_loop_var': 'item'} +# ], 'msg': 'All items completed', 'changed': False} +# and the processing logic breaks down. + +- name: ISM Plugin configuration | Get policies for all files + run_once: yes + become: no + uri: + url: "https://{{ inventory_hostname }}:9200/_plugins/_ism/policies/{{ item | basename | splitext | first }}" + url_username: "admin" + url_password: "{{ admin_password }}" + method: GET + force_basic_auth: yes + validate_certs: false + status_code: [ 200, 404 ] + retries: 3 + delay: 5 + until: ism_all.status == 200 or ism_all.status == 404 + with_fileglob: + - "{{ custom_ism.policy.dir }}/*.json" + register: ism_all + when: custom_ism.policy.apply_all + +- name: ISM Plugin configuration | DEBUG. Show ism_all + debug: + msg: "{{ ism_all }}" + run_once: yes + become: no + when: custom_ism.policy.apply_all + +- name: ISM Plugin configuration | Set ism_all to ism + set_fact: + ism: "{{ ism_all }}" + run_once: yes + become: no + when: custom_ism.policy.apply_all + +- name: ISM Plugin configuration | Get policies for selected files + run_once: yes + become: no + uri: + url: "https://{{ inventory_hostname }}:9200/_plugins/_ism/policies/{{ item | basename | splitext | first }}" + url_username: "admin" + url_password: "{{ admin_password }}" + method: GET + force_basic_auth: yes + validate_certs: false + status_code: [ 200, 404 ] + retries: 3 + delay: 5 + until: ism_custom.status == 200 or ism_custom.status == 404 + with_items: "{{ custom_ism.policy.files }}" + register: ism_custom + when: not custom_ism.policy.apply_all + +- name: ISM Plugin configuration | DEBUG. Show ism_custom + debug: + msg: "{{ ism_custom }}" + run_once: yes + become: no + when: not custom_ism.policy.apply_all + +- name: ISM Plugin configuration | Set ism_custom to ism + set_fact: + ism: "{{ ism_custom }}" + run_once: yes + become: no + when: not custom_ism.policy.apply_all + +- name: ISM Plugin configuration | Create policies + run_once: yes + become: no + uri: + url: "{{ item.url }}" + url_username: "admin" + url_password: "{{ admin_password }}" + method: PUT + force_basic_auth: yes + body_format: json + body: "{{ lookup('file',item.item) }}" + validate_certs: no + status_code: [ 201 ] + retries: 3 + delay: 5 + until: _result.status == 201 + with_items: "{{ ism.results }}" + when: '404 == item.status' + register: _result + + +- name: ISM Plugin configuration | Update policies + run_once: yes + become: no + uri: + url: "{{ item.url }}?if_seq_no={{ item.json._seq_no }}&if_primary_term={{ item.json._primary_term }}" + url_username: "admin" + url_password: "{{ admin_password }}" + method: PUT + force_basic_auth: yes + body_format: json + body: "{{ lookup('file',item.item) }}" + validate_certs: no + status_code: [ 200 ] + retries: 3 + delay: 5 + until: _result.status == 200 + with_items: "{{ ism.results }}" + when: '200 == item.status' + register: _result + diff --git a/roles/linux/opensearch/tasks/main.yml b/roles/linux/opensearch/tasks/main.yml index 3fe556e..6cc1fe7 100644 --- a/roles/linux/opensearch/tasks/main.yml +++ b/roles/linux/opensearch/tasks/main.yml @@ -61,3 +61,7 @@ debug: msg: "{{ os_roles.stdout }}" run_once: true + +- name: Apply custom ism settings + include: ism.yml + when: apply_custom_ism \ No newline at end of file From db246f4b65a65e127c7de11cdd8aa0674be6abfd Mon Sep 17 00:00:00 2001 From: Sergey Shubin Date: Tue, 24 May 2022 16:46:01 +0300 Subject: [PATCH 7/7] Added the ability to automatically create index patterns. The "Index patterns" section have been added to the readme. Signed-off-by: Sergey Shubin --- README.md | 13 +++++++ inventories/opensearch/group_vars/all/all.yml | 24 +++++++++++++ .../linux/dashboards/tasks/index-patterns.yml | 35 +++++++++++++++++++ roles/linux/dashboards/tasks/main.yml | 3 ++ 4 files changed, 75 insertions(+) create mode 100644 roles/linux/dashboards/tasks/index-patterns.yml diff --git a/README.md b/README.md index 9a33656..3c67a34 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ This ansible playbook supports the following, - Configuration of authentication and authorization via OpenID - Overriding default settings with your own - Creation/Updating ISM Policies +- Creating Index patterns - Install and configure the Apache2.0 opensource OpenSearch Dashboards ### Prerequisite @@ -145,6 +146,18 @@ you can, for example, change the number of replicas for indexes, when certain co If you want to manage policies using the opensearch role, set the `apply_custom_ism: yes` parameter, and create json files with policies in the `files/ism/policy` directory. Examples of policies can be found in the same directory. +### Index patterns + +To search for indexes in Dashboards, you need to create an index pattern (`Stack Management` -> `Index patterns`). +If there are a lot of indexes and they are in different tenants, then manually creating them can be quite time-consuming, +especially if one pattern needs to be created in several tenants at the same time. + +If you want to create a large number of index patterns using the `dashboards` role, then set the `iac_enable: yes` +parameter and fill in the `create_index_patterns` list. + +When performing the role, index patterns will be created. When creating, the "overwrite=true" parameter is used, +which prevents the creation of identical objects when running multiple times. + ## Contributing See [developer guide](DEVELOPER_GUIDE.md) and [how to contribute to this project](CONTRIBUTING.md). diff --git a/inventories/opensearch/group_vars/all/all.yml b/inventories/opensearch/group_vars/all/all.yml index 1262b16..2f4bec1 100644 --- a/inventories/opensearch/group_vars/all/all.yml +++ b/inventories/opensearch/group_vars/all/all.yml @@ -117,3 +117,27 @@ custom_ism: files: - files/ism/policy/hot7_warm30_delete.json - files/ism/policy/delete_after_30d.json + +# Automatic creation of an "index pattern" in the specified tenants. +# "index pattern" objects are stored in OpenSearch Dashboards (Kubana) and they +# are managed via the Dashboards API, so you need to work with these objects at +# the very end, after turning and launching. +# When creating, the "overwrite=true" parameter is used so that the existing index +# pattern is not duplicated. +# +# Index templates are created only when "iac_enable: yes" + +create_index_patterns: [] +#create_index_patterns: +# - tenant: "SECURITY" +# title: "kube-apiserver-audit-*" +# timeFieldName: "@timestamp" +# - tenant: "SECURITY" +# title: "syslog-*" +# timeFieldName: "@timestamp" +# - tenant: "WEB" +# title: "mywebapp-*" +# timeFieldName: "@timestamp" +# - tenant: "WEB" +# title: "ingress-nginx-*" +# timeFieldName: "@timestamp" \ No newline at end of file diff --git a/roles/linux/dashboards/tasks/index-patterns.yml b/roles/linux/dashboards/tasks/index-patterns.yml new file mode 100644 index 0000000..3508024 --- /dev/null +++ b/roles/linux/dashboards/tasks/index-patterns.yml @@ -0,0 +1,35 @@ +--- + +- name: Index Pattern | Create index patterns with overwrite + run_once: yes + become: no + uri: + url: "http://{{ inventory_hostname }}:5601/api/saved_objects/index-pattern/{{ item.title }}?overwrite=true" + url_username: "admin" + url_password: "{{ admin_password }}" + method: POST + force_basic_auth: yes + body_format: json + body: "{\"attributes\":{\"title\":\"{{ item.title }}\",\"timeFieldName\":\"{{ item.timeFieldName }}\"}}" + validate_certs: no + status_code: [ 200 ] + headers: + Content-Type: "application/json" + securitytenant: "{{ item.tenant }}" + osd-xsrf: "true" + retries: 3 + delay: 5 + until: idxp.status == 200 or idxp.status == 404 + with_items: "{{ create_index_patterns }}" + register: idxp + when: iac_enable + + +- name: Index Pattern | DEBUG. Show idxp + debug: + msg: "{{ idxp }}" + run_once: yes + become: no + when: iac_enable + + diff --git a/roles/linux/dashboards/tasks/main.yml b/roles/linux/dashboards/tasks/main.yml index 57de979..5910cd3 100644 --- a/roles/linux/dashboards/tasks/main.yml +++ b/roles/linux/dashboards/tasks/main.yml @@ -18,6 +18,9 @@ - name: include dashboards installation include: dashboards.yml +- name: Create index patterns + include: index-patterns.yml + - name: Make sure opensearch dashboards is started service: name: dashboards