diff --git a/app.py b/app.py index d74ab84..8934e2d 100644 --- a/app.py +++ b/app.py @@ -21,9 +21,11 @@ from serverless_stacks.custom_lambda_as_cron import CustomLambdaAsCronStack from serverless_stacks.custom_dynamodb import CustomDynamoDBStack from serverless_stacks.custom_privileges_to_lambda import CustomPrivilegesToLambdaStack +from serverless_stacks.custom_apigw import CustomApiGatewayStack # Import Monitoring Stacks from monitoring_stacks.custom_ec2_with_alarms import CustomEc2WithAlarmsStack +from monitoring_stacks.custom_cloudwatch_metrics import CustomMetricsStack # EC2 & VPC with Application LoadBalancer from app_stacks.vpc_stack import VpcStack @@ -163,10 +165,17 @@ # ) # Add CloudWatch Alarms to watch metrics and send notifications -custom_ec2_with_alarms = CustomEc2WithAlarmsStack( +# custom_ec2_with_alarms = CustomEc2WithAlarmsStack( +# app, +# "custom-ec2-with-alarms", +# description="Add CloudWatch Alarms to watch metrics and send notifications" +# ) + +# Create Custom Metrics & Alarms +custom_metric_and_alarms = CustomMetricsStack( app, - "custom-ec2-with-alarms", - description="Add CloudWatch Alarms to watch metrics and send notifications" + "custom-metric-and-alarms", + description="Create Custom Metrics & Alarms" ) diff --git a/monitoring_stacks/custom_cloudwatch_metrics.py b/monitoring_stacks/custom_cloudwatch_metrics.py new file mode 100644 index 0000000..8909b03 --- /dev/null +++ b/monitoring_stacks/custom_cloudwatch_metrics.py @@ -0,0 +1,78 @@ +from aws_cdk import aws_cloudwatch as _cloudwatch +from aws_cdk import aws_lambda as _lambda +from aws_cdk import aws_logs as _logs +from aws_cdk import core + + +class CustomMetricsStack(core.Stack): + + def __init__(self, scope: core.Construct, id: str, ** kwargs) -> None: + super().__init__(scope, id, **kwargs) + + # Read Lambda Code): + try: + with open("serverless_stacks/lambda_src/konstone_custom_metric_log_generator.py", mode="r") as f: + konstone_custom_metric_fn_code = f.read() + except OSError: + print("Unable to read Lambda Function Code") + + konstone_custom_metric_fn = _lambda.Function(self, + "konstoneFunction", + function_name="konstone_custom_metric_fn", + runtime=_lambda.Runtime.PYTHON_3_7, + handler="index.lambda_handler", + code=_lambda.InlineCode( + konstone_custom_metric_fn_code), + timeout=core.Duration.seconds( + 3), + reserved_concurrent_executions=1, + environment={ + "LOG_LEVEL": "INFO", + "PERCENTAGE_ERRORS": "75" + } + ) + + # Create Custom Loggroup + # /aws/lambda/function-name + konstone_custom_metric_lg = _logs.LogGroup(self, + "konstoneLoggroup", + log_group_name=f"/aws/lambda/{konstone_custom_metric_fn.function_name}", + removal_policy=core.RemovalPolicy.DESTROY, + retention=_logs.RetentionDays.ONE_DAY, + ) + + # Create Custom Metric Namespace + third_party_error_metric = _cloudwatch.Metric( + namespace=f"third-party-error-metric", + metric_name="third_party_error_metric", + label="Total No. of Third Party API Errors", + period=core.Duration.minutes(1), + statistic="Sum" + ) + + # Create Custom Metric Log Filter + third_party_error_metric_filter = _logs.MetricFilter(self, + "thirdPartyApiErrorMetricFilter", + filter_pattern=_logs.FilterPattern.boolean_value( + "$.third_party_api_error", True), + log_group=konstone_custom_metric_lg, + metric_namespace=third_party_error_metric.namespace, + metric_name=third_party_error_metric.metric_name, + default_value=0, + metric_value="1" + ) + + # Create Third Party Error Alarm + third_party_error_alarm = _cloudwatch.Alarm( + self, + "thirdPartyApiErrorAlarm", + alarm_description="Alert if 3rd party API has more than 2 errors in the last two minutes", + alarm_name="third-party-api-alarm", + metric=third_party_error_metric, + comparison_operator=_cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + threshold=2, + evaluation_periods=2, + datapoints_to_alarm=1, + period=core.Duration.minutes(1), + treat_missing_data=_cloudwatch.TreatMissingData.NOT_BREACHING + ) diff --git a/serverless_stacks/lambda_src/konstone_custom_metric_log_generator.py b/serverless_stacks/lambda_src/konstone_custom_metric_log_generator.py new file mode 100644 index 0000000..5e568cc --- /dev/null +++ b/serverless_stacks/lambda_src/konstone_custom_metric_log_generator.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +import json +import logging +import os +import random +from time import sleep + + +def call_third_party_api(): + return _random_error_generator(n=os.getenv("PERCENTAGE_ERRORS", 75)) + + +def _random_error_generator(n=75): + """ Generate error for given n % """ + r = False + if random.randint(1, 100) < int(n): + sleep(1) + r = True + return r + + +def lambda_handler(event, context): + global LOGGER + LOGGER = logging.getLogger() + LOGGER.setLevel(level=os.getenv('LOG_LEVEL', 'DEBUG').upper()) + + LOGGER.info(f"received_event:{event}") + + fmt_log_msg = { + "third_party_api_error": False, + } + # Add error logs, if + r = call_third_party_api() + if r: + fmt_log_msg["third_party_api_error"] = True + fmt_log_msg["remaining_time_in_millis"] = context.get_remaining_time_in_millis() + + LOGGER.info(json.dumps(fmt_log_msg)) + + return { + "statusCode": 200, + "body": json.dumps({ + "message": fmt_log_msg + }) + }