Enhance the Compose development set-up

Changes are:
- Allows Docker image builds in a restricted company context (limited
  access to remote Ubuntu, Python or Node repositories) using variables,
- Centralizes variables in a .env file (not versioned),
- Adds a .env.template file as .env template with predefined variables,
- Use a distinct directory for every service dependencies,
- Use a per-service environment file,
- Simplifies how development configuration files are transfered to
  Timesketch,
- Simplifies manipulation of containers using Compose CLI instead of
  the Docker one,
- Updates documentation.

.dockerignore

@@ -45,3 +45,6 @@
 
 # Exclude JetBrains IDE files
 /.idea/
+
+# Exclude Compose environment file
+/docker/dev/.env

.gitignore

@@ -44,3 +44,6 @@ vagrant/*.log
 
 # Exclude JetBrains IDE files
 .idea/
+
+# Exclude Compose environment file
+docker/dev/.env

docker/dev/.env.template

@@ -0,0 +1,15 @@
+GIFT_PPA_TRACK="stable"
+GIFT_PPA_URL="https://ppa.launchpadcontent.net/gift/${GIFT_PPA_TRACK}/ubuntu"
+NODE_VERSION="18.x"
+NODE_PPA_URL="https://deb.nodesource.com/node_${NODE_VERSION}"
+NODE_NPMRC=""
+PYTHON_PIP_CONF=""
+
+TIMESKETCH_BASE_IMAGE="ubuntu:22.04"
+TIMESKETCH_CONF_DIR="/etc/timesketch"
+TIMESKETCH_SECRET_KEY="L4np0jV3yAdAFdbVzWRMaBqiFMV8FKYd+Je1WKE40o8="
+TIMESKETCH_USER="dev"
+TIMESKETCH_PASSWORD="dev"
+
+POSTGRES_USER="timesketch"
+POSTGRES_PASSWORD="password"

docker/dev/README.md

@@ -5,6 +5,19 @@ Make sure to follow the docker [post-install](https://docs.docker.com/engine/ins
 
 NOTE: It is not recommended to try to run on a system with less than 8 GB of RAM.
 
+### Prepare a .env file
+
+Compose requires a `.env` file to be set.
+Copy the `.env.template` file to a `.env` one, not versioned.
+
+```bash
+cp .env.template .env
+```
+
+You can optionally edit the `.env` file.
+This is useful if you need to build images with some company restrictions (accessing
+remote Ubuntu, Python or Node repositories).
+
 ### Start a developer version of docker containers in this directory
 
 ```bash
@@ -18,36 +31,32 @@ If you see the following message you can continue
 ```text
 Timesketch development server is ready!
 ```
-### Find out container ID for the timesketch container
-
-```bash
-CONTAINER_ID="$(docker container list -f name=timesketch-dev -q)"
-```
-
-In the output look for CONTAINER ID for the timesketch container
-
-To write the ID to a variable, use:
-
-```bash
-export CONTAINER_ID="$(docker container list -f name=timesketch-dev -q)"
-```
-
-and test with
-
-```bash
-echo $CONTAINER_ID
-```
 
 ### Start a celery container shell
 
+Start the container in foreground (add `-d` to run in background):
+
 ```bash
-docker exec -it $CONTAINER_ID celery -A timesketch.lib.tasks worker --loglevel info
+docker compose exec timesketch \
+  celery \
+  -A timesketch.lib.tasks \
+  worker \
+  --loglevel info
 ```
 
 ### Start development webserver (and metrics server)
 
+Start the container in foreground (add `-d` to run in background):
+
 ```bash
-docker exec -it $CONTAINER_ID gunicorn --reload -b 0.0.0.0:5000 --log-file - --timeout 600 -c /usr/local/src/timesketch/data/gunicorn_config.py timesketch.wsgi:application
+docker compose exec timesketch \
+  gunicorn \
+  --reload \
+  -b 0.0.0.0:5000 \
+  --log-file - \
+  --timeout 600 \
+  -c /usr/local/src/timesketch/data/gunicorn_config.py \
+  timesketch.wsgi:application
 ```
 
 You now can access your development version at http://127.0.0.1:5000/
@@ -58,18 +67,17 @@ You can also access a metrics dashboard at http://127.0.0.1:3000/
 
 ### Non-interactive
 
-Running the following as a script after `docker compose up -d` will bring up the development environment in the background for you.
+The `restart.sh` script applies the 2 previous command in background for you.
 
 ```bash
-export CONTAINER_ID="$(docker container list -f name=timesketch-dev -q)"
-docker exec $CONTAINER_ID celery -A timesketch.lib.tasks worker --loglevel info
-docker exec $CONTAINER_ID gunicorn --reload -b 0.0.0.0:5000 --log-file - --timeout 120 timesketch.wsgi:application
+docker compose up -d
+./restart.sh
 ```
 
 ### Run tests
 
 ```bash
-docker exec -w /usr/local/src/timesketch -it $CONTAINER_ID python3 run_tests.py --coverage
+docker compose exec -w /usr/local/src/timesketch -it python3 run_tests.py --coverage
 ```
 
 That will run all tests in your docker container. It is recommended to run all tests at least before creating a pull request.

docker/dev/compose.yaml

@@ -6,26 +6,42 @@ networks:
 services:
   timesketch:
     image: us-docker.pkg.dev/osdfir-registry/timesketch/dev:latest
+    build:
+      context: ../..
+      dockerfile: docker/dev/timesketch/Dockerfile
+      args:
+        BASE_IMAGE: "${TIMESKETCH_BASE_IMAGE}"
+        GIFT_PPA_TRACK: "${GIFT_PPA_TRACK}"
+        GIFT_PPA_URL: "${GIFT_PPA_URL}"
+        NODE_VERSION: "${NODE_VERSION}"
+        NODE_PPA_URL: "${NODE_PPA_URL}"
+        NODE_NPMRC: "${NODE_NPMRC}"
+        PYTHON_PIP_CONF: "${PYTHON_PIP_CONF}"
     command: timesketch
     ports:
       - "5000:5000"
       - "5001:5001"
       - "8080:8080"
-    environment:
-      POSTGRES_USER: "timesketch"
-      POSTGRES_PASSWORD: "password"
-      POSTGRES_ADDRESS: "postgres"
-      POSTGRES_PORT: "5432"
-      OPENSEARCH_HOST: "opensearch"
-      OPENSEARCH_PORT: "9200"
-      REDIS_ADDRESS: "redis"
-      REDIS_PORT: "6379"
-      TIMESKETCH_USER: "dev"
-      TIMESKETCH_PASSWORD: "dev"
-      CHOKIDAR_USEPOLLING: "true"
-      prometheus_multiproc_dir: "/tmp/"
+    env_file:
+      - timesketch/timesketch.env
     volumes:
       - "../../:/usr/local/src/timesketch/"
+      - "./timesketch/timesketch.conf:${TIMESKETCH_CONF_DIR}/timesketch.conf:ro"
+      - "./timesketch/sigma_rules.txt:${TIMESKETCH_CONF_DIR}/sigma_rules.txt:ro"
+      - "../../data/regex_features.yaml:${TIMESKETCH_CONF_DIR}/regex_features.yaml:ro"
+      - "../../data/winevt_features.yaml:${TIMESKETCH_CONF_DIR}/winevt_features.yaml:ro"
+      - "../../data/tags.yaml:${TIMESKETCH_CONF_DIR}/tags.yaml:ro"
+      - "../../data/intelligence_tag_metadata.yaml:${TIMESKETCH_CONF_DIR}/intelligence_tag_metadata.yaml:ro"
+      - "../../data/plaso.mappings:${TIMESKETCH_CONF_DIR}/plaso.mappings:ro"
+      - "../../data/generic.mappings:${TIMESKETCH_CONF_DIR}/generic.mappings:ro"
+      - "../../data/ontology.yaml:${TIMESKETCH_CONF_DIR}/ontology.yaml:ro"
+      - "../../data/data_finder.yaml:${TIMESKETCH_CONF_DIR}/data_finder.yaml:ro"
+      - "../../data/bigquery_matcher.yaml:${TIMESKETCH_CONF_DIR}/bigquery_matcher.yaml:ro"
+      - "../../data/sigma_config.yaml:${TIMESKETCH_CONF_DIR}/sigma_config.yaml:ro"
+      - "../../data/sigma:${TIMESKETCH_CONF_DIR}/sigma:ro"
+      - "../../data/dfiq:${TIMESKETCH_CONF_DIR}/dfiq:ro"
+      - "../../data/context_links.yaml:${TIMESKETCH_CONF_DIR}/context_links.yaml:ro"
+      - "../../data/plaso_formatters.yaml:${TIMESKETCH_CONF_DIR}/plaso_formatters.yaml:ro"
     depends_on:
       - opensearch
       - postgres
@@ -35,13 +51,8 @@ services:
 
   opensearch:
     image: opensearchproject/opensearch:2.15.0
-    environment:
-      discovery.type: "single-node"
-      bootstrap.memory_lock: "true"
-      network.host: "0.0.0.0"
-      OPENSEARCH_JAVA_OPTS: "-Xms2g -Xmx2g"
-      DISABLE_INSTALL_DEMO_CONFIG: "true"
-      DISABLE_SECURITY_PLUGIN: "true" # TODO: Enable when we have migrated the python client to Opensearch as well.
+    env_file:
+      - opensearch/opensearch.env
     ports:
       - "9200:9200"
     networks:
@@ -56,9 +67,8 @@ services:
 
   postgres:
     image: postgres:13.1-alpine
-    environment:
-      POSTGRES_USER: "timesketch"
-      POSTGRES_PASSWORD: "password"
+    env_file:
+      - postgresql/postgresql.env
     ports:
       - "5432:5432"
     networks:
@@ -73,6 +83,11 @@ services:
 
   notebook:
     image: us-docker.pkg.dev/osdfir-registry/timesketch/notebook:latest
+    build:
+      context: ../..
+      dockerfile: docker/dev/notebook/Dockerfile
+      args:
+        PYTHON_PIP_CONF: "${PYTHON_PIP_CONF}"
     ports:
       - "8844:8844"
     volumes:

docker/dev/notebook/Dockerfile

@@ -18,8 +18,14 @@ COPY --chown=1000:1000 docker/dev/notebook/logo.png /home/picatrix/.jupyter/cust
 COPY --chown=1000:1000 docker/dev/notebook/custom.css /home/picatrix/.jupyter/custom/custom.css
 COPY --chown=1000:1000 docker/dev/notebook/timesketch /home/picatrix/picenv/share/jupyter/nbextensions/timesketch
 
+ARG PYTHON_PIP_CONF=""
+RUN if [ -n "${PYTHON_PIP_CONF}" ]; then \
+    mkdir -p ~/.config/pip; \
+    env echo -e "${PYTHON_PIP_CONF}" > ~/.config/pip/pip.conf; \
+  fi
+
 RUN sed -i -e "s/c.NotebookApp.token = 'picatrix'/c.NotebookApp.token = 'timesketch'/g" /home/picatrix/.jupyter/jupyter_notebook_config.py && \
-    sed -i -e "s/c.NotebookApp.port = 8899/c.NotebookApp.port = 8844/g" /home/picatrix/.jupyter/jupyter_notebook_config.py && \
+    sed -i -e "s/c.NotebookApp.port = 8899/c.NotebookApp.port = ${JUPYTER_PORT}/g" /home/picatrix/.jupyter/jupyter_notebook_config.py && \
     pip install -e /home/picatrix/code/api_client/python && \
     pip install -e /home/picatrix/code/importer_client/python/ && \
     jupyter nbextension enable snippets/main && \

docker/dev/opensearch.env

@@ -0,0 +1,6 @@
+discovery.type="single-node"
+bootstrap.memory_lock="true"
+network.host="0.0.0.0"
+OPENSEARCH_JAVA_OPTS="-Xms2g -Xmx2g"
+DISABLE_INSTALL_DEMO_CONFIG="true"
+DISABLE_SECURITY_PLUGIN="true" # TODO: Enable when we have migrated the python client to Opensearch as well.