diff --git a/cli/config.py b/cli/config.py index 6d305fb..a857a33 100644 --- a/cli/config.py +++ b/cli/config.py @@ -58,6 +58,7 @@ class BinaryAlertConfig: """Wrapper around reading, validating, and updating the terraform.tfvars config file.""" # Expected configuration value formats. VALID_AWS_ACCOUNT_ID_FORMAT = r'\d{12}' + VALID_AWS_ACCOUNT_NAME_FORMAT = r'[a-z]+\/?[a-z]+' VALID_AWS_REGION_FORMAT = r'[a-z]{2}-[a-z]{2,15}-\d' VALID_NAME_PREFIX_FORMAT = r'[a-z][a-z0-9_]{3,50}' VALID_CB_API_TOKEN_FORMAT = r'[a-f0-9]{40}' # CarbonBlack API token. @@ -96,6 +97,19 @@ def aws_account_id(self, value: str) -> None: ) self._config['aws_account_id'] = value + @property + def aws_account_name(self) -> str: + return self._config['aws_account_name'] + + @aws_account_name.setter + def aws_account_name(self, value: str) -> None: + if not re.fullmatch(self.VALID_AWS_ACCOUNT_NAME_FORMAT, value, re.ASCII): + raise InvalidConfigError( + 'aws_account_name "{}" does not match format {}'.format( + value, self.VALID_AWS_ACCOUNT_NAME_FORMAT) + ) + self._config['aws_account_name'] = value + @property def aws_region(self) -> str: return self._config['aws_region'] @@ -260,6 +274,7 @@ def configure(self) -> None: Each request will be retried until the answer is in the correct format. """ get_input('AWS Account ID', self.aws_account_id, self, 'aws_account_id') + get_input('AWS Account Name', self.aws_account_name, self, 'aws_account_name') get_input('AWS Region', self.aws_region, self, 'aws_region') get_input('Unique name prefix, e.g. "company_team"', self.name_prefix, self, 'name_prefix') enable_downloader = get_input('Enable the CarbonBlack downloader?', @@ -285,6 +300,7 @@ def validate(self) -> None: """ # Go through the internal setters which have the validation logic. self.aws_account_id = self.aws_account_id + self.aws_account_name = self.aws_account_name self.aws_region = self.aws_region self.name_prefix = self.name_prefix self.enable_carbon_black_downloader = self.enable_carbon_black_downloader diff --git a/docs/source/conf.py b/docs/source/conf.py index 9c1fcaa..9402d9f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # -*- coding: utf-8 -*- # # BinaryAlert documentation build configuration file, created by diff --git a/lambda_functions/analyzer/Dockerfile b/lambda_functions/analyzer/Dockerfile new file mode 100644 index 0000000..7f356c6 --- /dev/null +++ b/lambda_functions/analyzer/Dockerfile @@ -0,0 +1,117 @@ +FROM amazonlinux:latest + +ENV APP_DIR /var/task +ENV LAMBDA_DIR ${APP_DIR}/lambda +ENV PIP_DIR $APP_DIR/pip + +ENV CRYPTO_VER 3.3.1 +ENV ASN1CRYPTO_VER 1.4.0 +ENV YARA_VER 4.0.0 +ENV UPX_VER 3.94 +ENV YEXTEND_VER 1.6 +ENV ARCH amd64 + +VOLUME /var/out +WORKDIR $APP_DIR + +RUN yum -y update \ + && yum -y install \ + git \ + zip \ + xz \ + wget \ + tar \ + install \ + autoconf \ + automake \ + pkgconfig \ + bzip2-devel \ + gcc \ + gcc-c++ \ + libarchive-devel \ + libffi-devel \ + libtool \ + libuuid-devel \ + make \ + openssl-devel \ + pcre-devel \ + poppler-utils \ + poppler-cpp-devel \ + zlib-devel \ + python3 \ + python3-pip \ + python3-devel \ + && yum clean all \ + && pip3 install nose + +# install Yara +RUN wget -O yara.tar.gz https://github.com/VirusTotal/yara/archive/v${YARA_VER}.tar.gz \ + && tar -xzf yara.tar.gz \ + && cd yara-${YARA_VER} \ + && ./bootstrap.sh \ + && ./configure \ + && make \ + && make check \ + && make install \ + && echo "/usr/local/lib" > /etc/ld.so.conf.d/yara.conf \ + && ldconfig \ + && yara -v + +# Install Python pacakges +RUN pip3 install \ + "cryptography==${CRYPTO_VER}" \ + "yara-python==${YARA_VER}" \ + "asn1crypto==${ASN1CRYPTO_VER}" \ + -t $PIP_DIR + +# Copy crypto libs +RUN cd $PIP_DIR \ + && rm -r *.dist-info *.egg-info \ + && find . -name __pycache__ | xargs rm -rf \ + && mv _cffi_backend.cpython-*m-x86_64-linux-gnu.so _cffi_backend.so \ + && cd cryptography/hazmat/bindings \ + && mv _openssl.abi3.so _openssl.so \ + && mv _padding.abi3.so _padding.so + +# Gather pip files +RUN mkdir ${LAMBDA_DIR} \ + && cp -r ${PIP_DIR}/* ${LAMBDA_DIR} + +# Compile yextend +RUN mkdir ${LAMBDA_DIR}/libs \ + && git clone https://github.com/BayshoreNetworks/yextend.git \ + && cd yextend \ + && git checkout yara-${YARA_VER} \ + && mv configure.ac configure.ac.orig \ + && sed 's/python \-c/python3 -c/g' configure.ac.orig > configure.ac \ + && ./build.sh \ + && make unittests \ + && cp yextend ${LAMBDA_DIR} \ + && cp libs/*.o ${LAMBDA_DIR}/libs \ + && cp libs/*.yara ${LAMBDA_DIR}/libs + +# UPX +RUN wget https://github.com/upx/upx/releases/download/v${UPX_VER}/upx-${UPX_VER}-${ARCH}_linux.tar.xz \ + && tar -xf upx-${UPX_VER}-${ARCH}_linux.tar.xz \ + && cp upx-${UPX_VER}-${ARCH}_linux/upx ${LAMBDA_DIR} + +## Gather compiled libraries +RUN cp /usr/bin/pdftotext ${LAMBDA_DIR} \ + && mkdir tmplib \ + && find \ + /usr/lib64/lib* \ + -depth \ + -type f \ + -o \ + -type d \ + -name 'lib*.so.*' | \ + grep -E 'lib(archive|fontconfig|jbig|jpeg|lcms|lzma|lzo2|openjpeg|pcrecpp|png|poppler|stdc|tiff|xml)' | \ + cpio -pamVd tmplib \ + && mv tmplib/usr/lib64/* ${LAMBDA_DIR} \ + && cp /usr/local/lib/libyara.so.* ${LAMBDA_DIR} + +RUN cd ${LAMBDA_DIR} \ + && zip -r dependencies.zip * \ + && mv ${LAMBDA_DIR}/dependencies.zip ${APP_DIR}/dependencies.zip + +CMD cp ${APP_DIR}/dependencies.zip /var/out/ diff --git a/lambda_functions/analyzer/Makefile b/lambda_functions/analyzer/Makefile new file mode 100644 index 0000000..b719c83 --- /dev/null +++ b/lambda_functions/analyzer/Makefile @@ -0,0 +1,3 @@ +all: + docker build -t binaryalert-lambda . + docker run --rm -it -v ${PWD}:/var/out binaryalert-lambda diff --git a/lambda_functions/analyzer/README.rst b/lambda_functions/analyzer/README.rst index 26ed475..798b10d 100644 --- a/lambda_functions/analyzer/README.rst +++ b/lambda_functions/analyzer/README.rst @@ -1,111 +1,20 @@ YARA Analyzer ============= -This Lambda function is the core of BinaryAlert. Each invocation downloads one or more binaries from -S3, scans them against all available YARA rules, and forwards any matches to Dynamo and SNS. +This Lambda function is the core of BinaryAlert. Each invocation downloads one +or more binaries from S3, scans them against all available YARA rules, and +forwards any matches to Dynamo and SNS. Updating YARA Binaries ---------------------- -Many libraries used by BinaryAlert are natively compiled, and must therefore be pre-built on an -Amazon Linux AMI in order to run in Lambda. This has already been done for you: -``dependencies.zip`` contains the following pre-built libraries: +Many libraries used by BinaryAlert are natively compiled, and must therefore be +pre-built on an Amazon Linux AMI in order to run in Lambda. This has already +been done for you in the `dependencies.zip` file that ships with the repo, but you +can rebuild it yourself via Docker. -- `cryptography `_ (v2.3) -- `UPX `_ (v3.94) -- `yara-python `_ (v3.8.0) - - `yara `_ (v3.8.0) -- `yextend `_ (v1.6) - - `pdftotext `_ (v0.26.5) - -If, however, you need to update or re-create the zipfile, SSH to an EC2 instance running the -`AWS Lambda AMI `_ -and install the dependencies as follows: +If you need to update or re-create the ZIP file, do it before deployment. .. code-block:: bash + $ make - # Install requirements - sudo yum update - sudo yum install autoconf automake bzip2-devel gcc64 gcc64-c++ libarchive-devel libffi-devel \ - libtool libuuid-devel openssl-devel pcre-devel poppler-utils python36 python36-devel zlib-devel - sudo pip install nose - - # Compile YARA - wget https://github.com/VirusTotal/yara/archive/v3.8.0.tar.gz - tar -xzf v3.8.0.tar.gz - cd yara-3.8.0 - ./bootstrap.sh - ./configure - make - make check # Run unit tests - sudo make install - - # Install cryptography and yara-python - cd ~ - mkdir pip - pip-3.6 install cryptography yara-python -t pip - - # Compile yextend - wget https://github.com/BayshoreNetworks/yextend/archive/1.6.tar.gz - tar -xvzf 1.6.tar.gz - cd yextend-1.6 - # Manually: modify main.cpp, line 473 to hardcode the yara version to 3.8 - ./build.sh - make unittests # Run unit tests - - # Clean cryptography files - cd ~/pip - rm -r *.dist-info *.egg-info - find . -name __pycache__ | xargs rm -r - mv _cffi_backend.cpython-36m-x86_64-linux-gnu.so _cffi_backend.so - cd cryptography/hazmat/bindings - mv _constant_time.abi3.so _constant_time.so - mv _openssl.abi3.so _openssl.so - mv _padding.abi3.so _padding.so - - # Gather pip files - cd ~ - mkdir lambda - cp pip/.libs_cffi_backend/* lambda - cp -r pip/* lambda - mv lambda/yara.cpython-36m-x86_64-linux-gnu.so lambda/yara.so - wget https://raw.githubusercontent.com/VirusTotal/yara/master/COPYING -O lambda/YARA_LICENSE - wget https://raw.githubusercontent.com/VirusTotal/yara-python/master/LICENSE -O lambda/YARA_PYTHON_LICENSE - - # Gather Yextend files - cp yextend-1.6/yextend lambda - cp yextend-1.6/LICENSE lambda/YEXTEND_LICENSE - mkdir lambda/libs - cp yextend-1.6/libs/*.o lambda/libs - cp yextend-1.6/libs/*.yara lambda/libs - - # Download UPX - wget https://github.com/upx/upx/releases/download/v3.94/upx-3.94-amd64_linux.tar.xz - tar -xf upx-3.94-amd64_linux.tar.xz - cp upx-3.94-amd64_linux/upx lambda - cp upx-3.94-amd64_linux/COPYING lambda/UPX_LICENSE - - # Gather compiled libraries - cp /usr/bin/pdftotext lambda - cp /usr/lib64/libarchive.so.13 lambda - cp /usr/lib64/libfontconfig.so.1 lambda - cp /usr/lib64/libfreetype.so.6 lambda - cp /usr/lib64/libjbig.so.2.0 lambda - cp /usr/lib64/libjpeg.so.62 lambda - cp /usr/lib64/liblcms2.so.2 lambda - cp /usr/lib64/liblzma.so.5 lambda - cp /usr/lib64/liblzo2.so.2 lambda - cp /usr/lib64/libopenjpeg.so.2 lambda - cp /usr/lib64/libpcrecpp.so.0 lambda - cp /usr/lib64/libpng12.so.0 lambda - cp /usr/lib64/libpoppler.so.46 lambda - cp /usr/lib64/libstdc++.so.6 lambda - cp /usr/lib64/libtiff.so.5 lambda - cp /usr/lib64/libxml2.so.2 lambda - cp /usr/local/lib/libyara.so.3 lambda - - # Build Zipfile - cd lambda - zip -r dependencies.zip * - - -Then ``scp`` the ``dependencies.zip`` package to replace the one in the repo. \ No newline at end of file +And you'll find the new `dependencies.zip` file in this folder. diff --git a/lambda_functions/analyzer/dependencies.zip b/lambda_functions/analyzer/dependencies.zip index 04104b1..ba0453e 100644 Binary files a/lambda_functions/analyzer/dependencies.zip and b/lambda_functions/analyzer/dependencies.zip differ diff --git a/manage.py b/manage.py index c674b21..d7102a3 100755 --- a/manage.py +++ b/manage.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python """Command-line tool for easily managing BinaryAlert.""" import argparse import os diff --git a/requirements.txt b/requirements.txt index 6d733cc..857fc45 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,61 +1,76 @@ alabaster==0.7.12 -asn1crypto==0.24.0 -astroid==2.1.0 +asn1crypto==1.4.0 +astroid==2.4.2 attrdict==2.0.1 -Babel==2.6.0 -bandit==1.5.1 -boto3==1.9.99 -botocore==1.12.99 -cachetools==3.1.0 -cbapi==1.3.6 -certifi==2018.11.29 -cffi==1.12.1 -chardet==3.0.4 -coverage==4.5.2 -coveralls==1.6.0 -cryptography==2.5 +Babel==2.9.0 +bandit==1.7.0 +boto3==1.16.59 +botocore==1.19.59 +cachetools==4.2.1 +cbapi==1.7.3 +certifi==2020.12.5 +cffi==1.14.4 +chardet==4.0.0 +coverage==5.4 +coveralls==3.0.0 +cryptography==3.3.1 +decorator==4.4.2 docopt==0.6.2 -docutils==0.14 +docutils==0.16 futures==3.1.1 -gitdb2==2.0.5 -GitPython==2.1.11 -idna==2.8 -imagesize==1.1.0 -isort==4.3.4 -Jinja2==2.10 -jmespath==0.9.3 -lazy-object-proxy==1.3.1 -MarkupSafe==1.1.0 +gitdb==4.0.5 +gitdb2==4.0.2 +GitPython==3.1.12 +idna==2.5 +imagesize==1.2.0 +importlib-metadata==3.4.0 +isort==5.7.0 +Jinja2==2.11.2 +jmespath==0.10.0 +lazy-object-proxy==1.4.3 +MarkupSafe==1.1.1 mccabe==0.6.1 -mypy==0.670 -mypy-extensions==0.4.1 -packaging==19.0 -pbr==5.1.2 -pika==0.13.0 -ply==3.10 -prompt-toolkit==2.0.9 -protobuf==3.6.1 -pycparser==2.19 -pyfakefs==3.5.7 -Pygments==2.3.1 -pyhcl==0.4.0 -pylint==2.2.2 -pyOpenSSL==19.0.0 -pyparsing==2.3.1 -python-dateutil==2.6.1 -pytz==2018.9 -PyYAML==3.13 -requests==2.21.0 -s3transfer==0.2.0 -six==1.12.0 -smmap2==2.0.5 -snowballstemmer==1.2.1 -Sphinx==1.8.4 -sphinx-rtd-theme==0.4.3 -sphinxcontrib-websupport==1.1.0 -stevedore==1.30.0 -typed-ast==1.3.1 -urllib3==1.24.1 -wcwidth==0.1.7 -wrapt==1.11.1 -yara-python==3.8.0 +mypy==0.800 +mypy-extensions==0.4.3 +packaging==20.8 +pbr==5.5.1 +pika==1.1.0 +ply==3.11 +prompt-toolkit==3.0.14 +protobuf==3.14.0 +pycparser==2.20 +pyfakefs==4.3.3 +Pygments==2.7.4 +pyhcl==0.4.4 +pylint==2.6.0 +pyOpenSSL==20.0.1 +pyparsing==2.4.7 +python-dateutil==2.8.1 +pytz==2020.5 +PyYAML==5.4.1 +requests==2.25.1 +s3transfer==0.3.4 +six==1.15.0 +smmap==3.0.1 +smmap2==3.0.1 +snowballstemmer==2.1.0 +solrq==1.1.1 +Sphinx==3.4.3 +sphinx-rtd-theme==0.5.1 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.4 +sphinxcontrib-websupport==1.2.4 +stevedore==3.3.0 +toml==0.10.2 +typed-ast==1.4.2 +typing-extensions==3.7.4.3 +urllib3==1.25.4 +validators==0.18.2 +wcwidth==0.2.5 +wrapt==1.12.1 +yara-python==4.0.2 +zipp==3.4.0 diff --git a/terraform/kms.tf b/terraform/kms.tf index cf45f74..ab5e60c 100644 --- a/terraform/kms.tf +++ b/terraform/kms.tf @@ -5,7 +5,7 @@ data "aws_iam_policy_document" "kms_allow_s3" { principals { type = "AWS" - identifiers = ["arn:aws:iam::${var.aws_account_id}:root"] + identifiers = ["arn:aws:iam::${var.aws_account_id}:${var.aws_account_name}"] } actions = ["kms:*"] diff --git a/terraform/modules/lambda/main.tf b/terraform/modules/lambda/main.tf index 138b06e..ea0ca8a 100644 --- a/terraform/modules/lambda/main.tf +++ b/terraform/modules/lambda/main.tf @@ -47,7 +47,7 @@ resource "aws_lambda_function" "function" { description = var.description handler = var.handler role = aws_iam_role.role[0].arn - runtime = "python3.6" + runtime = "python3.7" memory_size = var.memory_size_mb timeout = var.timeout_sec diff --git a/terraform/modules/lambda/versions.tf b/terraform/modules/lambda/versions.tf index c8f2804..1e8aa8f 100644 --- a/terraform/modules/lambda/versions.tf +++ b/terraform/modules/lambda/versions.tf @@ -1,4 +1,9 @@ terraform { - required_version = "~> 0.12.9" + required_version = ">= 0.14" + required_providers { + aws = { + source = "hashicorp/aws" + } + } } diff --git a/terraform/variables.tf b/terraform/variables.tf index e96487b..0b5f73f 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -1,100 +1,167 @@ /* See terraform.tfvars for descriptions of each of the variables. */ variable "aws_account_id" { + type = string + description = "12-digit AWS account ID" +} + +variable "aws_account_name" { + type = string + description = "AWS account name, last part of the ARN, right after the ':' (colon), for instance 'root', or 'user/thename'" } variable "aws_region" { + type = string + description = "AWS region in which to deploy the BinaryAlert components" } variable "name_prefix" { + type = string + description = "Prefix used in all resource names (required for uniqueness) E.g. 'company_team'" } variable "enable_carbon_black_downloader" { + type = bool + description = "Whether to enable CarbonBlack Downloader resources" } variable "carbon_black_url" { + type = string + description = "URL of the CarbonBlack server" } variable "carbon_black_timeout" { + type = number + description = "Timeout to use for Carbon Black API client. The client default is 60, so set to something lower if desired" } variable "encrypted_carbon_black_api_token" { + type = string + description = "Encrypted API token used to interface with CarbonBlack" } variable "s3_log_bucket" { + type = string + description = "Pre-existing bucket in which to store S3 access logs. If not specified, one will be created" } variable "s3_log_prefix" { + type = string + description = "Log files will be stored in S3 with this prefix" } variable "s3_log_expiration_days" { + type = number + description = "Access logs expire after this many days. Has no effect if using pre-existing bucket for logs" } variable "lambda_log_retention_days" { + type = number + description = "How long to retain Lambda function logs for in days" } variable "tagged_name" { + type = string + description = "Assigns this as the value for tag key 'Name' for all supported resources (CloudWatch logs, Dynamo, KMS, Lambda, S3, SQS)" } variable "metric_alarm_sns_topic_arn" { + type = string + description = "Use an existing SNS topic for metric alarms (instead of creating one automatically)" } variable "expected_analysis_frequency_minutes" { + type = number + description = "Alarm if no binaries are analyzed for this amount of time" } variable "dynamo_read_capacity" { + type = number + description = "Provisioned read capacity for the Dynamo table which stores match results" } variable "dynamo_write_capacity" { + type = number + description = "Provisioned write capacity for the Dynamo table which stores match results" } variable "lambda_analyze_memory_mb" { + type = number + description = "Memory limit for the analyzer function" } variable "lambda_analyze_timeout_sec" { + type = number + description = "Time limit for the analyzer function" } variable "lambda_analyze_concurrency_limit" { + type = number + description = "Concurrency limit for the analyzer function" } variable "lambda_download_memory_mb" { + type = number + description = "Memory limit for the downloader function" } variable "lambda_download_timeout_sec" { + type = number + description = "Time limit for the downloader function" } variable "lambda_download_concurrency_limit" { + type = number + description = "Concurrency limit for the downloader function" } variable "force_destroy" { + type = bool + description = "WARNING: If force destroy is enabled, all objects in the S3 bucket(s) will be deleted during" } variable "external_s3_bucket_resources" { - type = list(string) + type = list(string) + description = "Grants appropriate S3 bucket permissions to the analyzer function if you are using BinaryAlert to scan existing S3 buckets" } variable "external_kms_key_resources" { - type = list(string) + type = list(string) + description = "Grants appropriate KMS permissions to the analyzer function if you are using BinaryAlert to scan existing S3 buckets" } variable "enable_negative_match_alerts" { + type = bool + description = "Create a separate SNS topic which reports files that do NOT match any YARA rules" } variable "analyze_queue_batch_size" { + type = number + description = "Maximum number of messages that will be received by each invocation of the analyzer function" } variable "download_queue_batch_size" { + type = number + description = "Maximum number of messages that will be received by each invocation of the downloader function" } variable "analyze_queue_retention_secs" { + type = number + description = "Messages in the analyzer queue will be retained and retried for the specified duration until expiring" } variable "download_queue_retention_secs" { + type = number + description = "Messages in the downloader queue will be retained and retried for the specified duration until expiring" } variable "objects_per_retro_message" { + type = number + description = "During a retroactive scan, number of S3 objects to pack into a single SQS message" } variable "download_queue_max_receives" { + type = number + description = "Number of times a download SQS message is attempted to be delivered successfully before being moved to the DLQ" } diff --git a/terraform/versions.tf b/terraform/versions.tf index cab229b..55dad25 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -1,3 +1,8 @@ terraform { - required_version = "~> 0.12.9" -} \ No newline at end of file + required_version = ">= 0.13" + required_providers { + aws = { + source = "hashicorp/aws" + } + } +}