Fw 2.9.0 migration

.gitignore

@@ -19,3 +19,8 @@ my_rules
 *~
 /rules/
 mod/
+comparisonFile
+comparisonFile2
+examples/ex/*
+examples/ex1/*
+comparisonFile*

.python-version

@@ -0,0 +1 @@
+3.11

CHANGELOG.md

@@ -1,3 +1,41 @@
+# fw_2.9.0.8
+- Percentage match rule type modified to work with haystack usecase [HAYS-4850]
+
+# fw_2.9.0.5
+- New Term Rule type modified to work with haystack usecase
+- Updated terms aggregation query to get existing terms
+- Updated Query used to get current terms to make use of terms aggregation instead of a search query
+- Added ability to refresh existing terms in a configureable refresh interval
+- Added Upper and Lower bounds for configureable query parameters like window_size, step_size and terms_size 
+- Modified Default values for new-term rule configurations
+
+# 2.9.0.1 FW updates
+- Downgraded elasticsearch library from version 8 to version 6 for the engine support the existing es clusters of haystack.
+- Downgraded various other libraries for the same.
+- es scrolls disabled as per requirement
+- replaced search queries with msearch for all es querying
+
+## Breaking Changes
+- Downgraded elasticsearch library from version 8 to version 6 for the engine support the existing es clusters of haystack.
+- Downgraded various other libraries for the same.
+- es scrolls disabled as per requirement
+- replaced search queries with msearch for all es querying
+
+## New Features
+- [Alertmanager] Added tenant specific config to alertmanager
+- [Prometheus] Added tenant config and modified prometheus route & port as per haystack requirements
+- [Haystack] Added Kibana adapter support for querying from router 
+- [Haystack] Added url_prefix for kibana adapter
+- [Haystack] Term-query optimzations
+- [Engine changes] Common index can be configured directly in config.yaml
+- [Trace Alerts] Added Error Rate rule that hits router aggregagte endpoint 
+- [Trace Alerts] Error rate rule enhancements
+	- error_calculation_method config for users to decide between two different error_rate calculation methods. 
+	- Default values for unique_column and error_condition
+	- Added Test cases for Error Rate Alert type
+- [Dockerfile] Distroless Docker setup for python 3 - elastalert
+- [Dockerfile] Docker optimization to fix create_index not running bug
+
 # 2.TBD.TBD
 
 ## Breaking changes

Dockerfile

@@ -1,44 +1,23 @@
-FROM python:3-slim-buster as builder
-
-LABEL description="ElastAlert 2 Official Image"
-LABEL maintainer="Jason Ertel"
-
-COPY . /tmp/elastalert
-
-RUN mkdir -p /opt/elastalert && \
-    cd /tmp/elastalert && \
-    pip install setuptools wheel && \
-    python setup.py sdist bdist_wheel
-
-FROM python:3-slim-buster
-
-ARG GID=1000
-ARG UID=1000
-ARG USERNAME=elastalert
-
-COPY --from=builder /tmp/elastalert/dist/*.tar.gz /tmp/
-
-RUN apt update && apt -y upgrade && \
-    apt -y install jq curl gcc libffi-dev && \
-    rm -rf /var/lib/apt/lists/* && \
-    pip install /tmp/*.tar.gz && \
-    rm -rf /tmp/* && \
-    apt -y remove gcc libffi-dev && \
-    apt -y autoremove && \
-    mkdir -p /opt/elastalert && \
-    echo "#!/bin/sh" >> /opt/elastalert/run.sh && \
-    echo "set -e" >> /opt/elastalert/run.sh && \
-    echo "elastalert-create-index --config /opt/elastalert/config.yaml" \
-        >> /opt/elastalert/run.sh && \
-    echo "elastalert --config /opt/elastalert/config.yaml \"\$@\"" \
-        >> /opt/elastalert/run.sh && \
-    chmod +x /opt/elastalert/run.sh && \
-    groupadd -g ${GID} ${USERNAME} && \
-    useradd -u ${UID} -g ${GID} -M -b /opt -s /sbin/nologin \
-        -c "ElastAlert 2 User" ${USERNAME}
-
-USER ${USERNAME}
-ENV TZ "UTC"
+FROM public.ecr.aws/i1i0w6p5/python:3.9.2 as build
+
+ENV ELASTALERT_HOME /opt/elastalert
+ADD . /opt/elastalert/
+
+WORKDIR /opt
+
+RUN pip install "setuptools==65.5.0" "elasticsearch==6.3.1"
+
+WORKDIR "${ELASTALERT_HOME}"
+
+RUN pip install -r requirements.txt
+RUN python setup.py install
+
+RUN pip show elastalert2
+
+RUN python --version
 
 WORKDIR /opt/elastalert
-ENTRYPOINT ["/opt/elastalert/run.sh"]
+
+COPY commands.sh /opt/elastalert/commands.sh
+RUN ["chmod", "+x", "/opt/elastalert/commands.sh"]
+ENTRYPOINT ["sh","/opt/elastalert/commands.sh"]

commands.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+echo "creating elastalert indices"
+python -m elastalert.create_index --config /data/elastalert/config.yaml
+echo "Starting elastalert"
+python -m elastalert.elastalert --config /data/elastalert/config.yaml --verbose

elastalert/alerters/alertmanager.py

@@ -30,7 +30,7 @@ def __init__(self, rule):
         self.timeout = self.rule.get('alertmanager_timeout', 10)
         self.alertmanager_basic_auth_login = self.rule.get('alertmanager_basic_auth_login', None)
         self.alertmanager_basic_auth_password = self.rule.get('alertmanager_basic_auth_password', None)
-
+        self.tenant = self.rule.get('tenant', "haystack")
 
     @staticmethod
     def _json_or_string(obj):
@@ -41,6 +41,7 @@ def _json_or_string(obj):
 
     def alert(self, matches):
         headers = {'content-type': 'application/json'}
+        headers.update({"X-Scope-OrgID": self.tenant})
         proxies = {'https': self.proxies} if self.proxies else None
         auth = HTTPBasicAuth(self.alertmanager_basic_auth_login, self.alertmanager_basic_auth_password) if self.alertmanager_basic_auth_login else None
 
@@ -50,6 +51,16 @@ def alert(self, matches):
         self.labels.update(
             alertname=self.alertname,
             elastalert_rule=self.rule.get('name'))
+        if 'json_payload' in self.rule and self.rule['json_payload'] == True:
+            self.labels.update(query_key_fields=self.rule.get('query_key'))
+            if self.rule.get('query_key') in matches[0].keys():
+                self.labels.update(query_key=matches[0][self.rule.get('query_key')])
+            if self.rule.get('alert_field'):
+                if 'value' in matches[0]:
+                    self.labels.update(query_key_fields=matches[0]['key'])
+                    self.labels.update(query_key=matches[0]['value'])
+                else:
+                    self.labels.update(query_key_fields=self.rule.get('alert_field'))
         self.annotations.update({
             self.title_labelname: self.create_title(matches),
             self.body_labelname: self.create_alert_body(matches)})
@@ -58,9 +69,12 @@ def alert(self, matches):
             'labels': self.labels
         }
 
+        if self.rule.get('timestamp_field') in matches[0]:
+            payload['labels']['alert_match_time']=matches[0][self.rule.get('timestamp_field')]
+
         for host in self.hosts:
             try:
-                url = '{}/api/{}/alerts'.format(host, self.api_version)
+                url = host
 
                 if self.ca_certs:
                     verify = self.ca_certs

elastalert/alerts.py

@@ -103,7 +103,10 @@ def _add_match_items(self):
                 except TypeError:
                     # Non serializable object, fallback to str
                     pass
-            self.text += '%s: %s\n' % (key, value_str)
+            if (isinstance(self.text,dict)):
+                self.text[key] = value_str
+            else:
+                self.text += '%s: %s\n' % (key, value_str)
 
     def _pretty_print_as_json(self, blob):
         try:
@@ -113,20 +116,31 @@ def _pretty_print_as_json(self, blob):
             return json.dumps(blob, cls=DateTimeEncoder, sort_keys=True, indent=4, encoding='Latin-1', ensure_ascii=False)
 
     def __str__(self):
-        self.text = ''
-        if 'alert_text' not in self.rule:
-            self.text += self.rule['name'] + '\n\n'
-
-        self._add_custom_alert_text()
-        self._ensure_new_line()
-        if self.rule.get('alert_text_type') != 'alert_text_only' and self.rule.get('alert_text_type') != 'alert_text_jinja':
-            self._add_rule_text()
+        if 'json_payload' in self.rule and self.rule['json_payload'] == True:
+            self.text= {}
+            if 'alert_text' not in self.rule:
+                self.text['elastalert_rule'] = self.rule['name']
+            if self.rule.get('alert_text_type') != 'alert_text_only' and self.rule.get('alert_text_type') != 'alert_text_jinja':
+                self.text['alert_criteria'] = self.rule['type'].get_match_str(self.match)
+                if self.rule.get('top_count_keys'):
+                    self._add_top_counts()
+                if self.rule.get('alert_text_type') != 'exclude_fields':
+                    self._add_match_items()
+            return str(self.text)
+        else:
+            self.text = ''
+            if 'alert_text' not in self.rule:
+                self.text += self.rule['name'] + '\n\n'
+            self._add_custom_alert_text()
             self._ensure_new_line()
-            if self.rule.get('top_count_keys'):
-                self._add_top_counts()
-            if self.rule.get('alert_text_type') != 'exclude_fields':
-                self._add_match_items()
-        return self.text
+            if self.rule.get('alert_text_type') != 'alert_text_only' and self.rule.get('alert_text_type') != 'alert_text_jinja':
+                self._add_rule_text()
+                self._ensure_new_line()
+                if self.rule.get('top_count_keys'):
+                    self._add_top_counts()
+                if self.rule.get('alert_text_type') != 'exclude_fields':
+                    self._add_match_items()
+            return self.text
 
 
 class Alerter(object):

elastalert/config.py

@@ -25,7 +25,8 @@
                 'ES_PORT': 'es_port',
                 'ES_URL_PREFIX': 'es_url_prefix',
                 'STATSD_INSTANCE_TAG': 'statsd_instance_tag',
-                'STATSD_HOST': 'statsd_host'}
+                'STATSD_HOST': 'statsd_host',
+                'X_ENV':'X_ENV'}
 
 env = Env(ES_USE_SSL=bool)
 
@@ -92,6 +93,8 @@ def load_conf(args, defaults=None, overrides=None):
             conf['old_query_limit'] = datetime.timedelta(**conf['old_query_limit'])
         else:
             conf['old_query_limit'] = datetime.timedelta(weeks=1)
+        if 'query_delay' in conf:
+            conf['query_delay'] = datetime.timedelta(**conf['query_delay'])
     except (KeyError, TypeError) as e:
         raise EAException('Invalid time format used: %s' % e)
 

elastalert/create_index.py

@@ -24,9 +24,10 @@ def create_index_mappings(es_client, ea_index, recreate=False, old_ea_index=None
     esversion = get_version_from_cluster_info(es_client)
 
     es_index_mappings = {}
+
     if is_atleasteight(esversion):
         es_index_mappings = read_es_index_mappings()
-    elif is_atleastseven(esversion):
+    elif is_atleastseven(esversion) or is_atleastsix(esversion):
         es_index_mappings = read_es_index_mappings(7)
     else:                                      
         print('FATAL - Unsupported Elasticsearch version: ' + esversion + '. Aborting.')
@@ -39,7 +40,7 @@ def create_index_mappings(es_client, ea_index, recreate=False, old_ea_index=None
             return None
 
     # (Re-)Create indices.
-    if is_atleastseven(esversion):
+    if is_atleastseven(esversion) or is_atleastsix(esversion):
         index_names = (
             ea_index,
             ea_index + '_status',
@@ -74,7 +75,7 @@ def create_index_mappings(es_client, ea_index, recreate=False, old_ea_index=None
                                       body=es_index_mappings['elastalert_error'])
         es_client.indices.put_mapping(index=ea_index + '_past',
                                       body=es_index_mappings['past_elastalert'])
-    elif is_atleastseven(esversion):
+    elif is_atleastseven(esversion) :
         es_client.indices.put_mapping(index=ea_index, doc_type='_doc',
                                       body=es_index_mappings['elastalert'], include_type_name=True)
         es_client.indices.put_mapping(index=ea_index + '_status', doc_type='_doc',
@@ -85,7 +86,17 @@ def create_index_mappings(es_client, ea_index, recreate=False, old_ea_index=None
                                       body=es_index_mappings['elastalert_error'], include_type_name=True)
         es_client.indices.put_mapping(index=ea_index + '_past', doc_type='_doc',
                                       body=es_index_mappings['past_elastalert'], include_type_name=True)
-
+    elif is_atleastsix(esversion):
+        es_client.indices.put_mapping(index=ea_index, doc_type='elastalert',
+                                      body=es_index_mappings['elastalert'])
+        es_client.indices.put_mapping(index=ea_index + '_status', doc_type='elastalert_status',
+                                      body=es_index_mappings['elastalert_status'])
+        es_client.indices.put_mapping(index=ea_index + '_silence', doc_type='silence',
+                                      body=es_index_mappings['silence'])
+        es_client.indices.put_mapping(index=ea_index + '_error', doc_type='elastalert_error',
+                                      body=es_index_mappings['elastalert_error'])
+        es_client.indices.put_mapping(index=ea_index + '_past', doc_type='past_elastalert',
+                                      body=es_index_mappings['past_elastalert'])
     print('New index %s created' % ea_index)
     if old_ea_index:
         print("Copying all data from old index '{0}' to new index '{1}'".format(old_ea_index, ea_index))
@@ -114,6 +125,9 @@ def read_es_index_mapping(mapping, es_version=7):
         print("Reading index mapping '{0}'".format(mapping_path))
         return json.load(f)
 
+def is_atleastsix(es_version):
+    return int(es_version.split(".")[0]) >= 6
+
 def is_atleastseven(es_version):
     return int(es_version.split(".")[0]) >= 7