Bloodhound / Opensearch

Makefile

@@ -7,14 +7,14 @@ DOCKER_TAG            ?= $(USER)
 # default helm chart version must be 0.0.42 for local development (because 42 is the answer to the universe and everything)
 HELM_SEMVER           ?= 0.0.42
 # The list of helm charts needed on internal kubernetes testing environments
-CHARTS_INTEGRATION    := wire-server databases-ephemeral redis-cluster rabbitmq fake-aws ingress-nginx-controller nginx-ingress-controller nginx-ingress-services fluent-bit kibana restund k8ssandra-test-cluster
+CHARTS_INTEGRATION    := wire-server databases-ephemeral redis-cluster rabbitmq fake-aws ingress-nginx-controller nginx-ingress-controller nginx-ingress-services fluent-bit restund k8ssandra-test-cluster
 # The list of helm charts to publish on S3
 # FUTUREWORK: after we "inline local subcharts",
 # (e.g. move charts/brig to charts/wire-server/brig)
 # this list could be generated from the folder names under ./charts/ like so:
 # CHARTS_RELEASE := $(shell find charts/ -maxdepth 1 -type d | xargs -n 1 basename | grep -v charts)
 CHARTS_RELEASE := wire-server redis-ephemeral redis-cluster rabbitmq rabbitmq-external databases-ephemeral	\
-fake-aws fake-aws-s3 fake-aws-sqs aws-ingress fluent-bit kibana backoffice		\
+fake-aws fake-aws-s3 fake-aws-sqs aws-ingress fluent-bit backoffice		\
 calling-test demo-smtp elasticsearch-curator elasticsearch-external				\
 elasticsearch-ephemeral minio-external cassandra-external						\
 nginx-ingress-controller ingress-nginx-controller nginx-ingress-services reaper restund \

changelog.d/0-release-notes/elasticsearch-to-opensearch

@@ -0,0 +1,10 @@
+Switch from ElasticSearch 6.8 to OpenSearch 1.3 (we're testing with OpenSearch
+1.3.19.) Please note that the APIs of ElasticSearch 6.8 and OpenSearch 1.3 are
+not compatible. I.e., either a migration is required
+(https://opensearch.org/docs/latest/upgrade-to/) or the indexes need to be
+rebuilt from scratch. The latter is done automatically when the
+`elasticsearch-index-create` job of the `elasticsearch-index` Helm chart faces
+an OpenSearch instance where Wire's indexes are missing. Alternatively, this can
+be started manually; see
+https://docs.wire.com/developer/reference/elastic-search.html#refill-es-documents-from-cassandra
+. Depending on the amount of users in the database, this may take long.

charts/integration/values.yaml

@@ -113,7 +113,7 @@ config:
     replicationFactor: 1
 
   elasticsearch:
-    host: elasticsearch-ephemeral
+    host: opensearch-cluster-master
 
   sqsEndpointUrl: http://fake-aws-sqs:4568
   sesEndpointUrl: http://fake-aws-ses:4569

deploy/dockerephemeral/docker-compose.yaml

@@ -206,15 +206,9 @@ services:
       redis:
         ipv4_address: 172.20.0.36
 
-  elasticsearch:
-    container_name: demo_wire_elasticsearch
-    build:
-      context: .
-      dockerfile_inline: |
-        FROM quay.io/wire/elasticsearch:0.0.9-amd64
-        RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install x-pack -b
-        # this seems to be necessary to run X-Pack on Alpine (https://discuss.elastic.co/t/elasticsearch-failing-to-start-due-to-x-pack/85125/7)
-        RUN rm -rf /usr/share/elasticsearch/plugins/x-pack/platform/linux-x86_64
+  opensearch:
+    container_name: opensearch
+    image: opensearchproject/opensearch:1.3.19
     ulimits:
       nofile:
         soft: 65536
@@ -223,17 +217,37 @@ services:
       - "127.0.0.1:9200:9200"
       - "127.0.0.1:9300:9300"
     environment:
-      - "xpack.ml.enabled=false"
-      - "xpack.security.enabled=true"
-      - "xpack.security.http.ssl.enabled=true"
-      - "xpack.ssl.certificate=certs/elasticsearch-cert.pem"
-      - "xpack.ssl.key=certs/elasticsearch-key.pem"
       - "bootstrap.system_call_filter=false"
       - "JVM_OPTIONS_ES=-Xmx512m -Xms512m"
       - "discovery.type=single-node"
+
+      - "DISABLE_INSTALL_DEMO_CONFIG=true"
+      - "OPENSEARCH_INITIAL_ADMIN_PASSWORD=Ch4ng3m3Secr3t!"
+    volumes:
+      - ./docker/elasticsearch-cert.pem:/usr/share/opensearch/config/certs/tls.crt
+      - ./docker/elasticsearch-key.pem:/usr/share/opensearch/config/certs/tls.key
+      - ./docker/elasticsearch-ca.pem:/usr/share/opensearch/config/certs/ca.crt
+      - ./docker/opensearch/opensearch.yml:/usr/share/opensearch/config/opensearch.yml
+      - ./docker/opensearch/opensearch-security/config.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/config.yml
+      - ./docker/opensearch/opensearch-security/internal_users.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/internal_users.yml
+      - ./docker/opensearch/opensearch-security/roles_mapping.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/roles_mapping.yml
+      - ./docker/opensearch/opensearch-security/allowlist.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/allowlist.yml
+      - ./docker/opensearch/opensearch-security/roles.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/roles.yml
+      - ./docker/opensearch/opensearch-security/nodes_dn.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/nodes_dn.yml
+      - ./docker/opensearch/opensearch-security/action_groups.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/action_groups.yml
+      - ./docker/opensearch/opensearch-security/tenants.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/tenants.yml
+    networks:
+      - demo_wire
+
+  opensearch-dashboard:
+    image: opensearchproject/opensearch-dashboards:1
+    container_name: opensearch-dashboards
+    ports:
+      - 5601:5601
+    expose:
+      - "5601"
     volumes:
-      - ./docker/elasticsearch-cert.pem:/usr/share/elasticsearch/config/certs/elasticsearch-cert.pem
-      - ./docker/elasticsearch-key.pem:/usr/share/elasticsearch/config/certs/elasticsearch-key.pem
+      - ./docker/opensearch/opensearch_dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml
     networks:
       - demo_wire
 

deploy/dockerephemeral/docker/opensearch/opensearch-security/action_groups.yml

@@ -0,0 +1,3 @@
+_meta:
+  type: "actiongroups"
+  config_version: 2

deploy/dockerephemeral/docker/opensearch/opensearch-security/allowlist.yml

@@ -0,0 +1,6 @@
+_meta:
+  type: "allowlist"
+  config_version: 2
+
+config:
+  enabled: false

deploy/dockerephemeral/docker/opensearch/opensearch-security/config.yml

@@ -0,0 +1,17 @@
+_meta:
+  type: "config"
+  config_version: 2
+
+config:
+  dynamic:
+    authc:
+      basic_internal_auth_domain:
+        description: "Authenticate using HTTP basic against the internal users database"
+        http_enabled: true
+        transport_enabled: true
+        order: 1
+        http_authenticator:
+          type: basic
+          challenge: true
+        authentication_backend:
+          type: internal

deploy/dockerephemeral/docker/opensearch/opensearch-security/internal_users.yml

@@ -0,0 +1,12 @@
+_meta:
+  type: "internalusers"
+  config_version: 2
+
+# User: elastic
+# Password: changeme
+elastic:
+  hash: "$2y$12$GRc68jkEX1m4uQpTVbwURu79xHxZ7vsbyEctOAADQwPjlhYS4LJVa"
+  reserved: true
+  description: "Wire User"
+  backend_roles:
+    - index_manager

deploy/dockerephemeral/docker/opensearch/opensearch-security/nodes_dn.yml

@@ -0,0 +1,3 @@
+_meta:
+  type: "nodesdn"
+  config_version: 2

deploy/dockerephemeral/docker/opensearch/opensearch-security/roles.yml

@@ -0,0 +1,3 @@
+_meta:
+  type: "roles"
+  config_version: 2

deploy/dockerephemeral/docker/opensearch/opensearch-security/roles_mapping.yml

@@ -0,0 +1,9 @@
+_meta:
+  type: "rolesmapping"
+  config_version: 2
+
+all_access:
+  reserved: false
+  backend_roles:
+    - index_manager
+  description: "Map index_manager to full_access"

deploy/dockerephemeral/docker/opensearch/opensearch-security/tenants.yml

@@ -0,0 +1,3 @@
+_meta:
+  type: "tenants"
+  config_version: 2

deploy/dockerephemeral/docker/opensearch/opensearch.yml

@@ -0,0 +1,45 @@
+cluster.name: opensearch-cluster
+
+# Bind to all interfaces because we don't know what IP address Docker will assign to us.
+network.host: 0.0.0.0
+
+# Setting network.host to a non-loopback address enables the annoying bootstrap checks. "Single-node" mode disables them again.
+discovery.type: single-node
+
+path.data: /usr/share/opensearch/data
+
+# WARNING: This is not a production-ready config! (Good enough for testing,
+# though.)
+plugins:
+  security:
+    ssl:
+      transport:
+          pemcert_filepath: certs/tls.crt
+          pemkey_filepath: certs/tls.key
+          pemtrustedcas_filepath: certs/ca.crt
+          enforce_hostname_verification: false
+      http:
+        enabled: true
+        pemcert_filepath: certs/tls.crt
+        pemkey_filepath: certs/tls.key
+        pemtrustedcas_filepath: certs/ca.crt
+    allow_unsafe_democertificates: true
+    allow_default_init_securityindex: true
+    audit.type: internal_opensearch
+    restapi:
+      roles_enabled: ["all_access", "security_rest_api_access"]
+    system_indices:
+      enabled: true
+      indices:
+        [
+          ".opendistro-alerting-config",
+          ".opendistro-alerting-alert*",
+          ".opendistro-anomaly-results*",
+          ".opendistro-anomaly-detector*",
+          ".opendistro-anomaly-checkpoints",
+          ".opendistro-anomaly-detection-state",
+          ".opendistro-reports-*",
+          ".opendistro-notifications-*",
+          ".opendistro-notebooks",
+          ".opendistro-asynchronous-search-response*",
+        ]

deploy/dockerephemeral/docker/opensearch/opensearch_dashboards.yml

@@ -0,0 +1,8 @@
+opensearch.hosts: [https://opensearch:9200]
+opensearch.ssl.verificationMode: none
+opensearch.username: elastic
+opensearch.password: changeme
+
+# Use this setting if you are running opensearch-dashboards without https
+opensearch_security.cookie.secure: false
+server.host: '0.0.0.0'

docs/src/developer/developer/how-to.md

@@ -211,3 +211,20 @@ Note: Simply deleting the namespaces is insufficient, because it leaves some res
 We support two different ways of managing the docker-compose instance of rabbitmq:
 * A web console interface is available [here](http://localhost:15672)
 * `rabbitmqadmin` CLI is made available in the dev environment
+
+## How to use opensearch-dashboards
+
+Our docker-compose setup contains a configured
+[`opensearch-dashboards`](https://opensearch.org/docs/latest/dashboards/)
+instance:
+
+- URL: http://localhost:5601
+- Username: `elastic`
+- Password: `changeme`
+
+To search in indices, *Index patterns* need to be created; identifying which
+indices you care for. This is IMHO a bit hidden in the UI, so I'm stating it
+here for future reference:
+
+- `Management` -> `Stack Management` -> `Index patterns`
+- http://localhost:5601/app/management/opensearch-dashboards/indexPatterns

hack/bin/integration-setup-federation.sh

@@ -53,6 +53,7 @@ echo "Installing charts..."
 set +e
 # This exists because we need to run `helmfile` with `--skip-deps`, without that it doesn't work.
 helm repo add bedag https://bedag.github.io/helm-charts/
+helm repo add opensearch https://opensearch-project.github.io/helm-charts/
 
 helmfile --environment "$HELMFILE_ENV" --file "${TOP_LEVEL}/hack/helmfile.yaml" sync --skip-deps --concurrency 0
 EXIT_CODE=$?