Merge branch 'master' into merge-unprotect

sillsdev · Dec 13, 2024 · cfdf244 · cfdf244
2 parents b4d139b + 5251099
commit cfdf244
Show file tree

Hide file tree

Showing 45 changed files with 363 additions and 286 deletions.
diff --git a/.github/workflows/combine_deploy_image.yml b/.github/workflows/combine_deploy_image.yml
@@ -22,10 +22,10 @@ jobs:
           egress-policy: block
           allowed-endpoints: >
             api.ecr-public.us-east-1.amazonaws.com:443
-            api.github.com:443
             archive.ubuntu.com:80
             auth.docker.io:443
             cdn.dl.k8s.io:443
+            deb.debian.org:80
             dl.k8s.io:443
             files.pythonhosted.org:443
             get.helm.sh:443
@@ -40,7 +40,7 @@ jobs:
       - name: Set up QEMU
         uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
+        uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
         with:
@@ -54,7 +54,7 @@ jobs:
           username: ${{ secrets.AWS_ACCESS_KEY_ID }}
           password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       - name: Build combine_deploy
-        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
+        uses: docker/build-push-action@v6.10.0
         with:
           context: "{{defaultContext}}:deploy"
           push: true

diff --git a/.github/workflows/commit_message_check.yml b/.github/workflows/commit_message_check.yml
@@ -10,4 +10,4 @@ permissions: # added using https://github.com/step-security/secure-workflows
 
 jobs:
   commit-message-lint:
-    uses: sillsdev/FieldWorks/.github/workflows/CommitMessage.yml@ba50e637df9593a2a972b29bf670226e89c0a21b
+    uses: sillsdev/FieldWorks/.github/workflows/CommitMessage.yml@22859ef68af99ffbd016eca4e503278db8007913
diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml
@@ -24,4 +24,4 @@ jobs:
       - name: "Checkout Repository"
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: "Dependency Review"
-        uses: actions/dependency-review-action@5a2ce3f5b92ee19cbb1541a4984c76d921601d7c # v4.3.4
+        uses: actions/dependency-review-action@3b139cfc5fae8b618d3eae3675e383bb1769c019 # v4.5.0
diff --git a/.github/workflows/deploy_qa.yml b/.github/workflows/deploy_qa.yml
@@ -26,6 +26,7 @@ jobs:
           egress-policy: block
           allowed-endpoints: >
             *.actions.githubusercontent.com:443
+            *.cloudfront.net:443
             *.data.mcr.microsoft.com:443
             ${{ secrets.AWS_ACCOUNT }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com
             api.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com:443
@@ -42,6 +43,7 @@ jobs:
             github.com:443
             mcr.microsoft.com:443
             production.cloudflare.docker.com:443
+            public.ecr.aws:443
             pypi.org:443
             registry-1.docker.io:443
             registry.npmjs.org:443

diff --git a/.github/workflows/deploy_release.yml b/.github/workflows/deploy_release.yml
@@ -25,6 +25,7 @@ jobs:
           egress-policy: block
           allowed-endpoints: >
             *.actions.githubusercontent.com:443
+            *.cloudfront.net:443
             *.data.mcr.microsoft.com:443
             api.ecr-public.us-east-1.amazonaws.com:443
             api.github.com:443

diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml
@@ -31,7 +31,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4
+        uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0
         with:
           node-version: ${{ matrix.node-version }}
       - run: npm ci
@@ -60,7 +60,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4
+        uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0
         with:
           node-version: ${{ matrix.node-version }}
       - run: npm ci

diff --git a/.github/workflows/maintenance.yml b/.github/workflows/maintenance.yml
@@ -20,11 +20,13 @@ jobs:
           disable-sudo: true
           egress-policy: block
           allowed-endpoints: >
+            *.cloudfront.net:443
             archive.ubuntu.com:80
             auth.docker.io:443
             files.pythonhosted.org:443
             github.com:443
             production.cloudflare.docker.com:443
+            public.ecr.aws:443
             pypi.org:443
             registry-1.docker.io:443
             security.ubuntu.com:80

diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml
@@ -26,7 +26,7 @@ jobs:
             github.com:443
             pypi.org:443
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
-      - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
+      - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
         with:
           python-version: 3.12
       - name: Install dependencies

diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
@@ -30,7 +30,7 @@ jobs:
             pypi.org:443
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
+        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies

diff --git a/Backend/BackendFramework.csproj b/Backend/BackendFramework.csproj
@@ -11,10 +11,10 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
-    <PackageReference Include="OpenTelemetry.Exporter.Console" Version="1.8.1" />
-    <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.8.1" />
-    <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.8.1" />
-    <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.8.1" />
+    <PackageReference Include="OpenTelemetry.Exporter.Console" Version="1.10.0" />
+    <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.10.0" />
+    <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.10.0" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.9.0" />
     <PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.8.1" />
     <PackageReference Include="RelaxNG" Version="3.2.3">
       <NoWarn>NU1701</NoWarn>
@@ -24,7 +24,7 @@
     <PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="7.5.1" />
     <PackageReference Include="MailKit" Version="4.8.0" />
     <PackageReference Include="MongoDB.Driver" Version="2.29.0" />
-    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.8.1" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.9.0" />
     <PackageReference Include="Xabe.FFmpeg" Version="5.2.6"/>
 
     <!-- SIL Maintained Dependencies. -->

diff --git a/Backend/Dockerfile b/Backend/Dockerfile
@@ -7,7 +7,7 @@
 ############################################################
 
 # Docker multi-stage build
-FROM mcr.microsoft.com/dotnet/sdk:8.0.402-jammy AS builder
+FROM mcr.microsoft.com/dotnet/sdk:8.0.404-jammy AS builder
 WORKDIR /app
 
 # Copy csproj and restore (fetch dependencies) as distinct layers.
@@ -19,7 +19,7 @@ COPY . ./
 RUN dotnet publish -c Release -o build
 
 # Build runtime image.
-FROM mcr.microsoft.com/dotnet/aspnet:8.0.8-jammy
+FROM mcr.microsoft.com/dotnet/aspnet:8.0.11-jammy
 
 ENV ASPNETCORE_URLS=http://+:5000
 ENV COMBINE_IS_IN_CONTAINER=1
@@ -43,9 +43,9 @@ RUN mkdir -p $HOME
 # Setup app user and group to known UID/GID; no login.
 RUN groupmod --gid 999 app
 RUN usermod  --uid 999 --gid app \
-      --shell /sbin/nologin \
-      --comment "Docker image user" \
-      app
+  --shell /sbin/nologin \
+  --comment "Docker image user" \
+  app
 
 ## Set up application install directory.
 RUN mkdir $APP_HOME && \

diff --git a/Dockerfile b/Dockerfile
@@ -7,7 +7,7 @@
 ############################################################
 
 # User guide build environment
-FROM python:3.12.5-slim-bookworm AS user_guide_builder
+FROM python:3.12.8-slim-bookworm AS user_guide_builder
 
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1
@@ -24,7 +24,7 @@ COPY docs/user_guide docs/user_guide
 RUN tox -e user-guide
 
 # Frontend build environment.
-FROM node:20.17.0-bookworm-slim AS frontend_builder
+FROM node:20.18.1-bookworm-slim AS frontend_builder
 WORKDIR /app
 
 # Install app dependencies.

diff --git a/README.md b/README.md
@@ -550,7 +550,7 @@ the corresponding videos and any transcript translations downloaded from Crowdin
 optionally attach them to a video file), run from within a Python virtual environment:
 
 ```bash
-python scripts/subtitle_tutorial_video.py -s <subtitles_subfolder_name> [-i <input_video_path> -o <output_video_path] [-v]
+python scripts/subtitle_tutorial_video.py -s <subtitles_subfolder_name> [-i <input_video_path> -o <output_video_path>] [-v]
 ```
 
 ## Setup Local Kubernetes Cluster
@@ -619,7 +619,7 @@ Notes for installing _Docker Desktop_ in Linux:
 Once _Docker Desktop_ has been installed, start it, and set it up as follows:
 
 1. Click the gear icon in the upper right to open the settings dialog;
-2. Click on the _Resources_ link on the left-hand side and set the Memory to at least 4 GB (see Note);
+2. Click on the _Resources_ link on the left-hand side and set the Memory to at least 6 GB (see Note);
 3. Click on the _Kubernetes_ link on the left-hand side;
 4. Select _Enable Kubernetes_ and click _Apply & Restart_;
 5. Click _Install_ on the dialog that is displayed.
@@ -731,16 +731,20 @@ Install the Kubernetes resources to run _The Combine_ by running:
 python deploy/scripts/setup_combine.py [--target <target_name>] [--tag <image_tag>]
 ```
 
-The default target is `localhost`; the default tag is `latest`. For development testing the script will usually be run
-with no arguments.
+Notes:
 
-If an invalid target is entered, the script will list available targets and prompt the user his/her selection.
-`deploy/scripts/setup_combine.py` assumes that the `kubectl` configuration file is setup to manage the desired
-Kubernetes cluster. For most development users, there will only be the _Rancher Desktop/Docker Desktop_ cluster to
-manage and the installation process will set that up correctly. If there are multiple clusters to manage, the
-`--kubeconfig` and `--context` options will let you specify a different cluster.
+- The default target is `localhost`; the default tag is `latest`. For development testing the script will usually be run
+  with no arguments.
 
-Run the script with the `--help` option to see possible options for the script.
+- If an invalid target is entered, the script will list available targets and prompt the user his/her selection.
+  `deploy/scripts/setup_combine.py` assumes that the `kubectl` configuration file is setup to manage the desired
+  Kubernetes cluster. For most development users, there will only be the _Rancher Desktop/Docker Desktop_ cluster to
+  manage and the installation process will set that up correctly. If there are multiple clusters to manage, the
+  `--kubeconfig` and `--context` options will let you specify a different cluster.
+
+- Run the script with the `--help` option to see possible options for the script.
+
+- The setup assumes `amd64` architecture. If the target architecture is `arm64`, add `--set global.cpuArch=arm64`.
 
 When the script completes, the resources will be installed on the specified cluster. It may take a few moments before
 all the containers are up and running. If you are using _Rancher Desktop_, you can use the
@@ -749,22 +753,25 @@ all the containers are up and running. If you are using _Rancher Desktop_, you c
 
 ```console
 $ kubectl -n thecombine get deployments
-NAME          READY   UP-TO-DATE   AVAILABLE   AGE
-backend       1/1     1            1           10m
-database      1/1     1            1           10m
-frontend      1/1     1            1           10m
-maintenance   1/1     1            1           10m
+NAME                           READY   UP-TO-DATE   AVAILABLE   AGE
+backend                        1/1     1            1           10m
+database                       1/1     1            1           10m
+frontend                       1/1     1            1           10m
+maintenance                    1/1     1            1           10m
+otel-opentelemetry-collector   1/1     1            1           19m
 ```
 
 or
 
 ```console
 $ kubectl -n thecombine get pods
-NAME                           READY   STATUS    RESTARTS   AGE
-backend-5657559949-z2flp       1/1     Running   0          10m
-database-794b4d956f-zjszm      1/1     Running   0          10m
-frontend-7d6d79f8c5-lkhhz      1/1     Running   0          10m
-maintenance-7f4b5b89b8-rhgk9   1/1     Running   0          10m
+NAME                                            READY   STATUS      RESTARTS   AGE
+backend-5657559949-z2flp                        1/1     Running     0          10m
+database-794b4d956f-zjszm                       1/1     Running     0          10m
+frontend-7d6d79f8c5-lkhhz                       1/1     Running     0          10m
+install-fonts-4jcsl                             0/1     Completed   0          8m
+maintenance-7f4b5b89b8-rhgk9                    1/1     Running     0          10m
+otel-opentelemetry-collector-5b5b69557b-zqk5d   1/1     Running     0          19m
 ```
 
 ### Connecting to Your Cluster

diff --git a/database/Dockerfile b/database/Dockerfile
@@ -5,7 +5,7 @@
 #   - Intel/AMD 64-bit
 #   - ARM 64-bit
 ############################################################
-FROM mongo:7.0.14-jammy
+FROM mongo:7.0.15-jammy
 
 WORKDIR /
 

diff --git a/deploy/Dockerfile b/deploy/Dockerfile
@@ -5,12 +5,12 @@
 #   - Intel/AMD 64-bit
 ############################################################
 
-FROM ubuntu:22.04
+FROM python:3.12.8-slim-bookworm
 
 USER root
 
 RUN apt-get update && \
-  apt-get install -y python3 python3-pip nano curl openssh-client iputils-ping && \
+  apt-get install -y python3-pip nano curl openssh-client iputils-ping && \
   apt-get autoremove && \
   apt-get clean && \
   rm -rf /var/lib/apt/lists/*

diff --git a/deploy/helm/aws-login/templates/aws-ecr-login-cronjob.yaml b/deploy/helm/aws-login/templates/aws-ecr-login-cronjob.yaml
@@ -18,7 +18,7 @@ spec:
         spec:
           serviceAccountName: {{ .Values.awsEcr.serviceAccount }}
           containers:
-          - image: {{ .Values.awsEcr.image }}:{{ .Values.awsEcr.imageTag }}
+          - image: {{ .Values.awsEcr.image }}:{{ .Values.awsEcr.imageVersion }}-{{ .Values.global.cpuArch }}
             imagePullPolicy: IfNotPresent
             name: {{ .Values.awsEcr.cronJobName }}
             command:

diff --git a/deploy/helm/aws-login/templates/aws-ecr-login-oneshot.yaml b/deploy/helm/aws-login/templates/aws-ecr-login-oneshot.yaml
@@ -18,7 +18,7 @@ spec:
     spec:
       serviceAccountName: {{ .Values.awsEcr.serviceAccount }}
       containers:
-      - image: {{ .Values.awsEcr.image }}:{{ .Values.awsEcr.imageTag }}
+      - image: {{ .Values.awsEcr.image }}:{{ .Values.awsEcr.imageVersion }}-{{ .Values.global.cpuArch }}
         imagePullPolicy: IfNotPresent
         name: "{{ .Values.awsEcr.jobName }}"
         command:

diff --git a/deploy/helm/aws-login/values.yaml b/deploy/helm/aws-login/values.yaml
@@ -15,14 +15,15 @@ global:
   awsAccessKeyId: "Override"
   awsSecretAccessKey: "Override"
   pullSecretName: aws-login-credentials
+  cpuArch: "amd64"
 
 awsEcr:
   configName: aws-ecr-config
   cron: yes
   cronJobName: ecr-cred-helper-cron
   dockerEmail: [email protected]
-  image: sillsdev/aws-kubectl
-  imageTag: "0.3.0"
+  image: "public.ecr.aws/thecombine/aws-kubectl"
+  imageVersion: "0.4.0"
   jobName: ecr-cred-helper
   schedule: "0 */8 * * *"
   secretsName: aws-ecr-credentials

diff --git a/deploy/helm/thecombine/charts/maintenance/templates/cronjob-daily-backup.yaml b/deploy/helm/thecombine/charts/maintenance/templates/cronjob-daily-backup.yaml
@@ -18,7 +18,7 @@ spec:
         spec:
           serviceAccountName: {{ .Values.serviceAccount.name }}
           containers:
-          - image: sillsdev/aws-kubectl:0.3.0
+          - image: {{ .Values.awsEcr.image }}:{{ .Values.awsEcr.imageVersion }}-{{ .Values.global.cpuArch }}
             imagePullPolicy: Always
             name: daily-backup
             command:

diff --git a/deploy/helm/thecombine/charts/maintenance/templates/cronjob-update-fonts.yaml b/deploy/helm/thecombine/charts/maintenance/templates/cronjob-update-fonts.yaml
@@ -18,7 +18,7 @@ spec:
         spec:
           serviceAccountName: {{ .Values.serviceAccount.name }}
           containers:
-          - image: sillsdev/aws-kubectl:0.3.0
+          - image: {{ .Values.awsEcr.image }}:{{ .Values.awsEcr.imageVersion }}-{{ .Values.global.cpuArch }}
             imagePullPolicy: Always
             name: update-fonts
             command:

diff --git a/deploy/helm/thecombine/charts/maintenance/templates/get-fonts-hook.yaml b/deploy/helm/thecombine/charts/maintenance/templates/get-fonts-hook.yaml
@@ -26,7 +26,7 @@ spec:
     spec:
       serviceAccountName: {{ .Values.serviceAccount.name }}
       containers:
-      - image: sillsdev/aws-kubectl:0.3.0
+      - image: {{ .Values.awsEcr.image }}:{{ .Values.awsEcr.imageVersion }}-{{ .Values.global.cpuArch }}
         imagePullPolicy: Always
         name: "install-fonts"
         command:

diff --git a/deploy/helm/thecombine/charts/maintenance/values.yaml b/deploy/helm/thecombine/charts/maintenance/values.yaml
@@ -26,6 +26,7 @@ global:
   imageRegistry: ""
   # Default AWS S3 location
   awsS3Location: "thecombine.app"
+  cpuArch: "amd64"
 
 imageName: combine_maint
 
@@ -34,7 +35,10 @@ serviceAccount:
   role: role-maintenance
   roleBinding: role-maintenance-binding
 
-serviceAccount.name: account-maintenance
+awsEcr:
+  image: "public.ecr.aws/thecombine/aws-kubectl"
+  imageVersion: "0.4.0"
+
 #######################################
 # Variables controlling backups
 #######################################