add s3 upload func

.github/workflows/image-push.yml

@@ -42,7 +42,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: .
-          file: ./build/Dockerfile-sbi
+          file: ./build/sbi/Dockerfile
           push: true
           tags: ${{ steps.meta.outputs.tags }}
 
@@ -82,6 +82,6 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: .
-          file: ./build/Dockerfile-money-forward
+          file: ./build/money-forward/Dockerfile
           push: true
           tags: ${{ steps.meta.outputs.tags }}

Makefile

@@ -3,8 +3,8 @@ CONTAINER_NAME_MONEY_FORWARD=bill-fetcher-money-forward
 
 .PHONY: build start stop clean
 build:
-	docker build -t $(CONTAINER_NAME_SBI) -f build/Dockerfile-sbi .
-	docker build -t $(CONTAINER_NAME_MONEY_FORWARD) -f build/Dockerfile-money-forward .
+	docker build -t $(CONTAINER_NAME_SBI) -f build/sbi/Dockerfile .
+	docker build -t $(CONTAINER_NAME_MONEY_FORWARD) -f build/money-forward/Dockerfile .
 
 start:
 	docker compose -f deployment/compose.yml up -d

build/money-forward/Dockerfile

@@ -0,0 +1,19 @@
+FROM ghcr.io/azuki774/selenium-chrome:0.2.0
+
+# Required Packages
+RUN apt-get update && \
+    apt-get install -y curl unzip && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# AWS Setup
+RUN curl -o /var/tmp/awscli.zip https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip && \
+    unzip -d /usr/local/bin/ /var/tmp/awscli.zip
+
+COPY requirements/ /tmp/
+RUN pip install --upgrade pip && pip install -r /tmp/moneyforward_requirements.txt
+COPY --chmod=755 build/money-forward/main.sh /src/main.sh
+COPY src/moneyforward/ /src/
+
+ENTRYPOINT ["/src/main.sh"]
+

build/money-forward/main.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+set -e
+YYYYMM=`date '+%Y%m'`
+YYYYMMDD=`date '+%Y%m%d'`
+
+# BUCKET_URL # from env (ex: "https://s3.ap-northeast-1.wasabisys.com")
+# BUCKET_NAME # from env (ex: hoge-system-stg-bucket)
+# BUCKET_DIR # from env (ex: fetcher/money-forward)
+# AWS_REGION # from env (ex: ap-northeast-1)
+# AWS_ACCESS_KEY_ID # from env
+# AWS_SECRET_ACCESS_KEY # from env
+
+AWS_BIN="/usr/local/bin/aws/dist/aws"
+DATA_DIR="/data"
+
+REMOTE_DIR_DAILY="${BUCKET_DIR}/${YYYYMMDD}"
+REMOTE_DIR_LATEST="${BUCKET_DIR}/latest"
+
+function fetch () {
+    echo "fetcher start"
+    python3 -u /src/main.py
+    echo "fetcher complete"
+}
+
+function create_s3_credentials {
+    echo "s3 credentials create start"
+    mkdir -p ~/.aws/
+
+    echo "[default]" >> ~/.aws/config
+    echo "region = ${AWS_REGION}" >> ~/.aws/config
+
+    echo "[default]" >> ~/.aws/credentials
+    echo "aws_access_key_id = ${AWS_ACCESS_KEY_ID}" >> ~/.aws/credentials
+    echo "aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY}" >> ~/.aws/credentials
+
+    chmod 400 ~/.aws/config
+    chmod 400 ~/.aws/credentials
+    ls -la ~/.aws/
+    echo "s3 credentials create complete"
+}
+
+function s3_upload () {
+    echo "s3 upload start"
+    ${AWS_BIN} s3 cp ${DATA_DIR}/${YYYYMMDD}/ "s3://${BUCKET_NAME}/${REMOTE_DIR_DAILY}" --recursive --endpoint-url="${BUCKET_URL}"
+    mv ${DATA_DIR}/${YYYYMMDD} ${DATA_DIR}/latest
+    ${AWS_BIN} s3 cp ${DATA_DIR}/latest/ "s3://${BUCKET_NAME}/${REMOTE_DIR_LATEST}" --recursive --endpoint-url="${BUCKET_URL}"
+    echo "s3 upload complete"
+}
+
+fetch
+create_s3_credentials
+s3_upload

build/sbi/Dockerfile

@@ -0,0 +1,18 @@
+FROM ghcr.io/azuki774/selenium-chrome:0.2.0
+
+# Required Packages
+RUN apt-get update && \
+    apt-get install -y curl unzip && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# AWS Setup
+RUN curl -o /var/tmp/awscli.zip https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip && \
+    unzip -d /usr/local/bin/ /var/tmp/awscli.zip
+
+COPY requirements/ /tmp/
+RUN pip install --upgrade pip && pip install -r /tmp/sbi_requirements.txt
+COPY --chmod=755 build/sbi/main.sh /src/main.sh
+COPY src/sbi/ /src/
+
+ENTRYPOINT ["/src/main.sh"]

build/sbi/main.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+set -e
+YYYYMM=`date '+%Y%m'`
+YYYYMMDD=`date '+%Y%m%d'`
+
+# BUCKET_URL # from env (ex: "https://s3.ap-northeast-1.wasabisys.com")
+# BUCKET_NAME # from env (ex: hoge-system-stg-bucket)
+# BUCKET_DIR # from env (ex: fetcher/sbi)
+# AWS_REGION # from env (ex: ap-northeast-1)
+# AWS_ACCESS_KEY_ID # from env
+# AWS_SECRET_ACCESS_KEY # from env
+
+AWS_BIN="/usr/local/bin/aws/dist/aws"
+DATA_DIR="/data"
+
+REMOTE_DIR="${BUCKET_DIR}/${YYYYMM}"
+
+function fetch () {
+    echo "fetcher start"
+    python3 -u /src/main.py
+    echo "fetcher complete"
+}
+
+function create_s3_credentials {
+    echo "s3 credentials create start"
+    mkdir -p ~/.aws/
+
+    echo "[default]" >> ~/.aws/config
+    echo "region = ${AWS_REGION}" >> ~/.aws/config
+
+    echo "[default]" >> ~/.aws/credentials
+    echo "aws_access_key_id = ${AWS_ACCESS_KEY_ID}" >> ~/.aws/credentials
+    echo "aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY}" >> ~/.aws/credentials
+
+    chmod 400 ~/.aws/config
+    chmod 400 ~/.aws/credentials
+    ls -la ~/.aws/
+    echo "s3 credentials create complete"
+}
+
+function s3_upload () {
+    echo "s3 upload start"
+    ${AWS_BIN} s3 cp ${DATA_DIR}/${YYYYMM}/ "s3://${BUCKET_NAME}/${REMOTE_DIR}" --recursive --endpoint-url="${BUCKET_URL}"
+    echo "s3 upload complete"
+}
+
+fetch
+create_s3_credentials
+
+s3_upload

deployment/compose.yml

@@ -9,10 +9,26 @@ services:
   #   volumes:
   #     - /dev/shm:/dev/shm
 
-  money-forward:
+  # money-forward:
+  #   image: bill-fetcher-money-forward
+  #   container_name: bill-fetcher-money-forward
+  #   env_file:
+  #   - money-forward.env
+  #   volumes:
+  #   - ./:/data/
+
+  # fetcher-sbi:
+  #   image: bill-fetcher-sbi
+  #   container_name: bill-fetcher-sbi
+  #   env_file:
+  #   - sbi-token.env
+  #   volumes:
+  #   - ./:/data/
+
+  fetcher-money-forward:
     image: bill-fetcher-money-forward
     container_name: bill-fetcher-money-forward
     env_file:
-    - money-forward.env
+      - money-forward.env
     volumes:
-    - ./:/data/
+      - ./:/data/

deployment/sbi-token.env.sample

@@ -0,0 +1,8 @@
+sbi_user=xxxxxxxxxx
+sbi_pass="yyyyyyyyy"
+BUCKET_URL="https://s3.ap-northeast-1.wasabisys.com"
+BUCKET_NAME="hoge-system-stg-bucket"
+BUCKET_DIR="fetcher/sbi"
+AWS_REGION="ap-northeast-1"
+AWS_ACCESS_KEY_ID="xxxxxxxxxxxxx"
+AWS_SECRET_ACCESS_KEY="yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy"