From 28586c754462e452101347246be768b044992e90 Mon Sep 17 00:00:00 2001 From: JasonPowr Date: Mon, 4 Nov 2024 10:46:20 +0000 Subject: [PATCH] add-backfill-redis-job --- roles/tas_single_node/defaults/main.yml | 11 ++++ roles/tas_single_node/meta/argument_specs.yml | 55 +++++++++++++++++++ roles/tas_single_node/tasks/podman.yml | 2 + roles/tas_single_node/tasks/podman/rekor.yml | 30 ++++++++++ .../manifests/rekor/backfill_redis.sh.j2 | 38 +++++++++++++ vm-testing/dev-images.sed | 2 +- 6 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 roles/tas_single_node/templates/manifests/rekor/backfill_redis.sh.j2 diff --git a/roles/tas_single_node/defaults/main.yml b/roles/tas_single_node/defaults/main.yml index f970dae0..07bc3e6e 100644 --- a/roles/tas_single_node/defaults/main.yml +++ b/roles/tas_single_node/defaults/main.yml @@ -7,10 +7,19 @@ tas_single_node_podman_network: rhtas tas_single_node_rekor_redis: database_deploy: true + backfill_redis: true redis: host: rekor-redis-pod port: 6379 password: password + backfill_redis: + enabled: true + schedule: + minute: "0" + hour: "0" + day: "*" + month: "*" + weekday: "*" tas_single_node_trillian: database_deploy: true @@ -65,6 +74,8 @@ tas_single_node_ctlog_image: "registry.redhat.io/rhtas/certificate-transparency-rhel9@sha256:31e7318a9b19ed04ef0f25949f1f1709d293b532316b27a06f83fa5174547b17" tas_single_node_rekor_redis_image: "registry.redhat.io/rhtas/trillian-redis-rhel9@sha256:18820b1fbdbc2cc3e917822974910332d937b03cfe781628bd986fd6a5ee318e" +tas_single_node_backfill_redis_image: + "registry.redhat.io/rhtas/rekor-backfill-redis-rhel9@sha256:c5995c88063bd9875ae61c299bcf549002fcde724aab09807c70934e73daf356" tas_single_node_trillian_db_image: "registry.redhat.io/rhtas/trillian-database-rhel9@sha256:501612745e63e5504017079388bec191ffacf00ffdebde7be6ca5b8e4fd9d323" tas_single_node_tuf_image: diff --git a/roles/tas_single_node/meta/argument_specs.yml b/roles/tas_single_node/meta/argument_specs.yml index f11ea8fa..4a48621e 100644 --- a/roles/tas_single_node/meta/argument_specs.yml +++ b/roles/tas_single_node/meta/argument_specs.yml @@ -35,6 +35,14 @@ argument_specs: host: rekor-redis-pod port: 6379 password: password + backfill_redis: + enabled: true + schedule: + minute: "0" + hour: "0" + day: "*" + month: "*" + weekday: "*" options: database_deploy: description: "Whether or not to deploy Redis." @@ -62,6 +70,53 @@ argument_specs: type: "str" required: false version_added: "1.1.0" + backfill_redis: + description: "Backfill Redis configuration" + type: "dict" + required: false + version_added: "1.1.1" + options: + enabled: + description: "Whether or not to deploy the backfill Redis cron job." + type: "bool" + required: false + version_added: "1.1.1" + schedule: + description: "Cron schedule for the backfill Redis job." + type: "dict" + required: false + version_added: "1.1.1" + options: + minute: + description: "Specifies the minute(s) of each hour when the cron job should run (0-59)." + type: "str" + required: false + default: "*" + version_added: "1.1.1" + hour: + description: "Specifies the hour(s) of each day when the cron job should run (0-23)." + type: "str" + required: false + default: "*" + version_added: "1.1.1" + day: + description: "Specifies the day(s) of the month when the cron job should run (1-31)." + type: "str" + required: false + default: "*" + version_added: "1.1.1" + month: + description: "Specifies the month(s) when the cron job should run (1-12 or month names)." + type: "str" + required: false + default: "*" + version_added: "1.1.1" + weekday: + description: "Specifies the day(s) of the week when the cron job should run (0-6, where 0=Sunday)." + type: "str" + required: false + default: "*" + version_added: "1.1.1" tas_single_node_trillian: description: "Details on the database connection for Trillian. You can set this to a custom MySQL or MariaDB instance." type: "dict" diff --git a/roles/tas_single_node/tasks/podman.yml b/roles/tas_single_node/tasks/podman.yml index 45be3565..5e11271e 100644 --- a/roles/tas_single_node/tasks/podman.yml +++ b/roles/tas_single_node/tasks/podman.yml @@ -45,6 +45,7 @@ "{{ tas_single_node_rekor_enabled }}", "{{ tas_single_node_ctlog_enabled }}", "{{ tas_single_node_rekor_enabled and tas_single_node_rekor_redis.database_deploy }}", + "{{ tas_single_node_rekor_enabled and tas_single_node_rekor_redis.database_deploy and tas_single_node_rekor_redis.backfill_redis.enabled }}", "{{ tas_single_node_trillian_enabled and tas_single_node_trillian.database_deploy }}", "{{ tas_single_node_tuf_enabled }}", "{{ tas_single_node_trillian_enabled }}", @@ -61,6 +62,7 @@ - "{{ tas_single_node_rekor_server_image }}" - "{{ tas_single_node_ctlog_image }}" - "{{ tas_single_node_rekor_redis_image }}" + - "{{ tas_single_node_backfill_redis_image }}" - "{{ tas_single_node_trillian_db_image }}" - "{{ tas_single_node_tuf_image }}" - "{{ tas_single_node_trillian_netcat_image }}" diff --git a/roles/tas_single_node/tasks/podman/rekor.yml b/roles/tas_single_node/tasks/podman/rekor.yml index dacd7553..cff2b583 100644 --- a/roles/tas_single_node/tasks/podman/rekor.yml +++ b/roles/tas_single_node/tasks/podman/rekor.yml @@ -67,3 +67,33 @@ configmap_changed: "{{ configmap_result.changed }}" secret: "{{ tas_single_node_rekor_secret }}" secret_changed: "{{ secret_result.changed }}" + +- name: Get RHTAS network details + ansible.builtin.command: + cmd: "podman network inspect {{ tas_single_node_podman_network }}" + register: tas_podman_network_results + changed_when: false + when: tas_single_node_rekor_redis.backfill_redis.enabled + +- name: Set DNS Resolver + ansible.builtin.set_fact: + dns_resolver: "{{ (tas_podman_network_results.stdout | from_json)[0].subnets[0].gateway }}" + when: tas_single_node_rekor_redis.backfill_redis.enabled + +- name: Deploy backfill_redis script + ansible.builtin.template: + src: manifests/rekor/backfill_redis.sh.j2 + dest: /usr/local/bin/backfill_redis.sh + mode: '0755' + when: tas_single_node_rekor_redis.backfill_redis.enabled + +- name: Create Backfill Redis Cronjob + ansible.builtin.cron: + name: "backfill_redis" + minute: "{{ tas_single_node_rekor_redis.backfill_redis.schedule.minute }}" + hour: "{{ tas_single_node_rekor_redis.backfill_redis.schedule.hour }}" + day: "{{ tas_single_node_rekor_redis.backfill_redis.schedule.day }}" + month: "{{ tas_single_node_rekor_redis.backfill_redis.schedule.month }}" + weekday: "{{ tas_single_node_rekor_redis.backfill_redis.schedule.weekday }}" + job: "/usr/local/bin/backfill_redis.sh" + when: tas_single_node_rekor_redis.backfill_redis.enabled diff --git a/roles/tas_single_node/templates/manifests/rekor/backfill_redis.sh.j2 b/roles/tas_single_node/templates/manifests/rekor/backfill_redis.sh.j2 new file mode 100644 index 00000000..5ed5bcca --- /dev/null +++ b/roles/tas_single_node/templates/manifests/rekor/backfill_redis.sh.j2 @@ -0,0 +1,38 @@ +#!/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +LAST_INDEX_FILE="/tmp/rekor_last_index" + +endIndex=$(curl -sS http://localhost:3001/api/v1/log | jq -r '.treeSize - 1') +if [ "$endIndex" -lt 0 ]; then + echo "info: no rekor entries found" + exit 0 +fi + +if [ -f "$LAST_INDEX_FILE" ]; then + startIndex=$(cat "$LAST_INDEX_FILE") +else + startIndex=0 +fi + +if [ "$startIndex" -gt "$endIndex" ]; then + startIndex=0 +fi + +if [ "$startIndex" -le "$endIndex" ]; then + podman run --rm --name backfill_redis \ + -v "{{ tas_single_node_certs_dir }}/ingress-rekor.pem:/usr/local/share/ca-certificates/ingress-rekor.pem:ro" \ + -e SSL_CERT_FILE=/usr/local/share/ca-certificates/ingress-rekor.pem \ + --add-host="rekor.{{ tas_single_node_base_hostname }}:{{ dns_resolver }}" \ + --add-host="{{ tas_single_node_rekor_redis.redis.host }}:{{ dns_resolver }}" \ + "{{ tas_single_node_backfill_redis_image }}" \ + --hostname="{{ tas_single_node_rekor_redis.redis.host }}" \ + --password="{{ tas_single_node_rekor_redis.redis.password }}" \ + --port="{{ tas_single_node_rekor_redis.redis.port }}" \ + --rekor-address="http://rekor.{{ tas_single_node_base_hostname }}" \ + --start="$startIndex" --end="$endIndex" + + echo "$endIndex" > "$LAST_INDEX_FILE" +else + echo "info: no new entries to process" +fi diff --git a/vm-testing/dev-images.sed b/vm-testing/dev-images.sed index 03a7fcb5..9f2647fc 100644 --- a/vm-testing/dev-images.sed +++ b/vm-testing/dev-images.sed @@ -5,7 +5,6 @@ s#registry.redhat.io/rhtas/fulcio-rhel9#quay.io/securesign/fulcio-server# s#registry.redhat.io/rhtas/trillian-redis-rhel9#quay.io/securesign/trillian-redis# s#registry.redhat.io/rhtas/rekor-server-rhel9#quay.io/securesign/rekor-server# s#registry.redhat.io/rhtas/rekor-search-ui-rhel9#quay.io/securesign/rekor-search-ui# -s#registry.redhat.io/rhtas/rekor-backfill-redis-rhel9#quay.io/securesign/trillian-redis# s#registry.redhat.io/rhtas/tuf-server-rhel9#quay.io/securesign/scaffold-tuf-server# s#registry.redhat.io/rhtas/certificate-transparency-rhel9#quay.io/securesign/certificate-transparency-go# s#registry.redhat.io/rhtas/client-server-cg-rhel9#quay.io/securesign/cli-client-server-cg# @@ -14,3 +13,4 @@ s#registry.redhat.io/rhtas/segment-reporting-rhel9#quay.io/securesign/segment-ba s#registry.redhat.io/rhtas/timestamp-authority-rhel9#quay.io/securesign/timestamp-authority# s#registry.redhat.io/rhtas/trillian-createtree-rhel9#quay.io/securesign/trillian-createtree# s#registry.redhat.io/rhtas/client-server-rhel9#quay.io/securesign/client-server# +s#registry.redhat.io/rhtas/rekor-backfill-redis-rhel9#quay.io/securesign/rekor-backfill-redis#