Merge pull request #27 from azuki774/add-s3-upload

s3 upload機能を追加

.github/workflows/image-push.yml

@@ -6,57 +6,17 @@ on:
       - v*
 
 jobs:
-  build_and_push_remix:
-    runs-on: ubuntu-latest
-    env:
-      IMAGE_NAME: bill-fetcher-remix
-    steps:
-      - name: checkout
-        uses: actions/checkout@v2
-
-      - name: Set meta
-        id: meta
-        uses: docker/metadata-action@v3
-        with:
-          # list of Docker images to use as base name for tags
-          images: |
-            ghcr.io/azuki774/bill-fetcher-remix
-          # generate Docker tags based on the following events/attributes
-          tags: |
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}}
-            type=semver,pattern=latest
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GH_ACCESS_TOKEN }}
-
-      - name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: ./build/Dockerfile-remix
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-
   build_and_push_sbi:
     runs-on: ubuntu-latest
     env:
       IMAGE_NAME: bill-fetcher-sbi
     steps:
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Set meta
         id: meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v4
         with:
           # list of Docker images to use as base name for tags
           images: |
@@ -69,100 +29,20 @@ jobs:
             type=semver,pattern=latest
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GH_ACCESS_TOKEN }}
-
-      - name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: ./build/Dockerfile-sbi
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-
-  build_and_push_tokyowater:
-    runs-on: ubuntu-latest
-    env:
-      IMAGE_NAME: bill-fetcher-tokyowater
-    steps:
-      - name: checkout
-        uses: actions/checkout@v2
-
-      - name: Set meta
-        id: meta
-        uses: docker/metadata-action@v3
-        with:
-          # list of Docker images to use as base name for tags
-          images: |
-            ghcr.io/azuki774/bill-fetcher-tokyowater
-          # generate Docker tags based on the following events/attributes
-          tags: |
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}}
-            type=semver,pattern=latest
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GH_ACCESS_TOKEN }}
-
-      - name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: ./build/Dockerfile-tokyowater
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-
-  build_and_push_nicigas:
-    runs-on: ubuntu-latest
-    env:
-      IMAGE_NAME: bill-fetcher-nicigas
-    steps:
-      - name: checkout
-        uses: actions/checkout@v2
-
-      - name: Set meta
-        id: meta
-        uses: docker/metadata-action@v3
-        with:
-          # list of Docker images to use as base name for tags
-          images: |
-            ghcr.io/azuki774/bill-fetcher-nicigas
-          # generate Docker tags based on the following events/attributes
-          tags: |
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}}
-            type=semver,pattern=latest
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GH_ACCESS_TOKEN }}
 
       - name: Build and push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./build/Dockerfile-nicigas
+          file: ./build/sbi/Dockerfile
           push: true
           tags: ${{ steps.meta.outputs.tags }}
 
@@ -172,11 +52,11 @@ jobs:
       IMAGE_NAME: bill-fetcher-money-forward
     steps:
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Set meta
         id: meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v4
         with:
           # list of Docker images to use as base name for tags
           images: |
@@ -189,7 +69,7 @@ jobs:
             type=semver,pattern=latest
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v1
@@ -199,9 +79,9 @@ jobs:
           password: ${{ secrets.GH_ACCESS_TOKEN }}
 
       - name: Build and push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./build/Dockerfile-money-forward
+          file: ./build/money-forward/Dockerfile
           push: true
           tags: ${{ steps.meta.outputs.tags }}

.github/workflows/test.yml

@@ -0,0 +1,13 @@
+name: Build test
+
+on:
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+      - name: Build images
+        run: make build

Makefile

@@ -1,18 +1,10 @@
-CONTAINER_NAME_AUELECT=bill-fetcher-auelect
-CONTAINER_NAME_REMIX=bill-fetcher-remix
 CONTAINER_NAME_SBI=bill-fetcher-sbi
-CONTAINER_NAME_TOKYOWATER=bill-fetcher-tokyowater
-CONTAINER_NAME_NICIGAS=bill-fetcher-nicigas
 CONTAINER_NAME_MONEY_FORWARD=bill-fetcher-money-forward
 
 .PHONY: build start stop clean
 build:
-	docker build -t $(CONTAINER_NAME_AUELECT) -f build/Dockerfile-auelect .
-	docker build -t $(CONTAINER_NAME_REMIX) -f build/Dockerfile-remix .
-	docker build -t $(CONTAINER_NAME_SBI) -f build/Dockerfile-sbi .
-	docker build -t $(CONTAINER_NAME_TOKYOWATER) -f build/Dockerfile-tokyowater .
-	docker build -t $(CONTAINER_NAME_NICIGAS) -f build/Dockerfile-nicigas .
-	docker build -t $(CONTAINER_NAME_MONEY_FORWARD) -f build/Dockerfile-money-forward .
+	docker build -t $(CONTAINER_NAME_SBI) -f build/sbi/Dockerfile .
+	docker build -t $(CONTAINER_NAME_MONEY_FORWARD) -f build/money-forward/Dockerfile .
 
 start:
 	docker compose -f deployment/compose.yml up -d
@@ -24,9 +16,5 @@ debug:
 	docker compose -f deployment/compose.yml up
 
 clean:
-	docker image rm $(CONTAINER_NAME_AUELECT)
-	docker image rm $(CONTAINER_NAME_REMIX)
 	docker image rm $(CONTAINER_NAME_SBI)
-	docker image rm $(CONTAINER_NAME_TOKYOWATER)
-	docker image rm $(CONTAINER_NAME_NICIGAS)
 	docker image rm $(CONTAINER_NAME_MONEY_FORWARD)

README.md

@@ -2,10 +2,14 @@
 
 ## money-forward
 - https://moneyforward.com に自動的にログインして、指定したURLのHTMLソースを取得して保存する。
-### Usage
-- `deploymments/compose.yml` と `money-forward.yml` の要領で、コンテナ `bill-fetcher-money-forward` を起動させる。
+- 出力先は、コンテナ内の `/data/latest/＜各種ページ＞
 - 取得したいリンクと、必要に応じて金融機関連携の「更新」ボタンの XPATH を環境変数で渡す。
 
+## sbi
+- https://site1.sbisec.co.jp/ETGate/ に自動的にログインして、ポートフォリオの表ごとに保存する。
+- 出力先は、コンテナ内の `/data/YYYYMMDD/YYYYMMDD_x.csv`
+    - `x`: 連番
+
 ## データ取り込み例
 - [mf-importer](https://github.com/azuki774/mf-importer)
 

build/money-forward/Dockerfile

@@ -0,0 +1,18 @@
+FROM ghcr.io/azuki774/selenium-chrome:0.2.0
+
+# Required Packages
+RUN apt-get update && \
+    apt-get install -y curl unzip && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# AWS Setup
+RUN curl -o /var/tmp/awscli.zip https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip && \
+    unzip -d /usr/local/bin/ /var/tmp/awscli.zip
+
+COPY requirements/ /tmp/
+RUN pip install --upgrade pip && pip install -r /tmp/moneyforward_requirements.txt
+COPY --chmod=755 build/money-forward/main.sh /src/main.sh
+COPY src/moneyforward/ /src/
+
+ENTRYPOINT ["/src/main.sh"]

build/money-forward/main.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+set -e
+YYYYMM=`date '+%Y%m'`
+YYYYMMDD=`date '+%Y%m%d'`
+
+# BUCKET_URL # from env (ex: "https://s3.ap-northeast-1.wasabisys.com")
+# BUCKET_NAME # from env (ex: hoge-system-stg-bucket)
+# BUCKET_DIR # from env (ex: fetcher/money-forward)
+# AWS_REGION # from env (ex: ap-northeast-1)
+# AWS_ACCESS_KEY_ID # from env
+# AWS_SECRET_ACCESS_KEY # from env
+
+AWS_BIN="/usr/local/bin/aws/dist/aws"
+DATA_DIR="/data"
+
+REMOTE_DIR_DAILY="${BUCKET_DIR}/${YYYYMMDD}"
+REMOTE_DIR_LATEST="${BUCKET_DIR}/latest"
+
+function fetch () {
+    echo "fetcher start"
+    python3 -u /src/main.py
+    echo "fetcher complete"
+}
+
+function create_s3_credentials {
+    echo "s3 credentials create start"
+    mkdir -p ~/.aws/
+
+    echo "[default]" >> ~/.aws/config
+    echo "region = ${AWS_REGION}" >> ~/.aws/config
+
+    echo "[default]" >> ~/.aws/credentials
+    echo "aws_access_key_id = ${AWS_ACCESS_KEY_ID}" >> ~/.aws/credentials
+    echo "aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY}" >> ~/.aws/credentials
+
+    chmod 400 ~/.aws/config
+    chmod 400 ~/.aws/credentials
+    ls -la ~/.aws/
+    echo "s3 credentials create complete"
+}
+
+function s3_upload () {
+    echo "s3 upload start"
+    ${AWS_BIN} s3 cp ${DATA_DIR}/${YYYYMMDD}/ "s3://${BUCKET_NAME}/${REMOTE_DIR_DAILY}" --recursive --endpoint-url="${BUCKET_URL}"
+    mv ${DATA_DIR}/${YYYYMMDD} ${DATA_DIR}/latest
+    ${AWS_BIN} s3 cp ${DATA_DIR}/latest/ "s3://${BUCKET_NAME}/${REMOTE_DIR_LATEST}" --recursive --endpoint-url="${BUCKET_URL}"
+    echo "s3 upload complete"
+}
+
+fetch
+
+if [ -z $BUCKET_NAME ]; then
+    exit 0
+fi
+
+create_s3_credentials
+s3_upload

build/sbi/Dockerfile

@@ -0,0 +1,18 @@
+FROM ghcr.io/azuki774/selenium-chrome:0.2.0
+
+# Required Packages
+RUN apt-get update && \
+    apt-get install -y curl unzip && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# AWS Setup
+RUN curl -o /var/tmp/awscli.zip https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip && \
+    unzip -d /usr/local/bin/ /var/tmp/awscli.zip
+
+COPY requirements/ /tmp/
+RUN pip install --upgrade pip && pip install -r /tmp/sbi_requirements.txt
+COPY --chmod=755 build/sbi/main.sh /src/main.sh
+COPY src/sbi/ /src/
+
+ENTRYPOINT ["/src/main.sh"]