Merge pull request #16 from azuki774/add-api

Add API Server

.github/workflows/image-push.yml

@@ -205,3 +205,43 @@ jobs:
           file: ./build/Dockerfile-money-forward
           push: true
           tags: ${{ steps.meta.outputs.tags }}
+
+  build_and_push_api:
+    runs-on: ubuntu-latest
+    env:
+      IMAGE_NAME: bill-fetcher-api
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+
+      - name: Set meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          # list of Docker images to use as base name for tags
+          images: |
+            ghcr.io/azuki774/bill-fetcher-api
+          # generate Docker tags based on the following events/attributes
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=semver,pattern=latest
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GH_ACCESS_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./build/Dockerfile-api
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}

Makefile

@@ -1,12 +1,14 @@
+CONTAINER_NAME_API=bill-fetcher-api
 CONTAINER_NAME_AUELECT=bill-fetcher-auelect
 CONTAINER_NAME_REMIX=bill-fetcher-remix
 CONTAINER_NAME_SBI=bill-fetcher-sbi
 CONTAINER_NAME_TOKYOWATER=bill-fetcher-tokyowater
 CONTAINER_NAME_NICIGAS=bill-fetcher-nicigas
 CONTAINER_NAME_MONEY_FORWARD=bill-fetcher-money-forward
 
-.PHONY: build start
+.PHONY: build start stop clean
 build:
+	docker build -t $(CONTAINER_NAME_API) -f build/Dockerfile-api .
 	docker build -t $(CONTAINER_NAME_AUELECT) -f build/Dockerfile-auelect .
 	docker build -t $(CONTAINER_NAME_REMIX) -f build/Dockerfile-remix .
 	docker build -t $(CONTAINER_NAME_SBI) -f build/Dockerfile-sbi .
@@ -17,5 +19,17 @@ build:
 start:
 	docker compose -f deployment/compose.yml up -d
 
+stop:
+	docker compose -f deployment/compose.yml down
+
 debug:
 	docker compose -f deployment/compose.yml up
+
+clean:
+	docker image rm $(CONTAINER_NAME_API)
+	docker image rm $(CONTAINER_NAME_AUELECT)
+	docker image rm $(CONTAINER_NAME_REMIX)
+	docker image rm $(CONTAINER_NAME_SBI)
+	docker image rm $(CONTAINER_NAME_TOKYOWATER)
+	docker image rm $(CONTAINER_NAME_NICIGAS)
+	docker image rm $(CONTAINER_NAME_MONEY_FORWARD)

build/Dockerfile-api

@@ -0,0 +1,10 @@
+FROM ghcr.io/azuki774/selenium-chrome:0.2.0
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+COPY requirements/ /tmp/
+RUN pip install --upgrade pip && pip install -r /tmp/api_requirements.txt
+RUN pip install -r /tmp/moneyforward_requirements.txt
+COPY src/ /src
+WORKDIR /src/
+ENTRYPOINT ["gunicorn", "flaskapp:app", "--config", "gunicorn.py"]

build/Dockerfile-auelect

@@ -1,5 +1,5 @@
 FROM python:3.11-bullseye
-RUN python -m pip install --upgrade pip
+COPY requirements/ /tmp/
+RUN python -m pip install --upgrade pip && pip install -r /tmp/auelect_requirements.txt
 COPY src/auelect/ /src/
-RUN pip install -r src/requirement.txt
 ENTRYPOINT ["python3", "-u", "/src/main.py"]

build/Dockerfile-money-forward

@@ -1,4 +1,6 @@
-FROM ghcr.io/azuki774/selenium-chrome:latest
-COPY src/money-forward/ /src/
-RUN pip install -r /src/requirements.txt
+FROM ghcr.io/azuki774/selenium-chrome:0.2.0
+COPY requirements/ /tmp/
+RUN pip install --upgrade pip && pip install -r /tmp/moneyforward_requirements.txt
+
+COPY src/moneyforward/ /src/
 ENTRYPOINT ["python3", "-u", "/src/main.py"]

build/Dockerfile-sbi

@@ -1,4 +1,6 @@
 FROM ghcr.io/azuki774/selenium-chrome:latest
+
+COPY requirements/ /tmp/
+RUN pip install --upgrade pip && pip install -r /tmp/sbi_requirements.txt
 COPY src/sbi/ /src/
-RUN pip install -r /src/requirement.txt
 ENTRYPOINT ["python3", "-u", "/src/main.py"]

deployment/api.env.sample

@@ -0,0 +1,3 @@
+[email protected]
+pass=xxxxxx
+refresh_xpaths="xxxxxxxx,xxxxxxxx" # https://moneyforward.com に表示される金融機関等の[更新]ボタンのXPATHを , 区切りで記載

deployment/compose.yml

@@ -1,12 +1,20 @@
 version: '3'
 services:
-  money-forward:
-    image: bill-fetcher-money-forward
-    container_name: bill-fetcher-money-forward
+  fetcher-api:
+    image: bill-fetcher-api
+    container_name: bill-fetcher-api
     env_file:
-    - money-forward.env
-    volumes:
-    - ./:/data/
+    - api.env
+    ports:
+    - "8080:9876"
+
+  # money-forward:
+  #   image: bill-fetcher-money-forward
+  #   container_name: bill-fetcher-money-forward
+  #   env_file:
+  #   - money-forward.env
+  #   volumes:
+  #   - ./:/data/
 
   # remix:
   #   image: bill-fetcher-remix

requirements/api_requirements.txt

@@ -0,0 +1,4 @@
+Flask==2.3.3
+gunicorn==21.2.0
+requests==2.31.0
+python-json-logger==2.0.7

src/flaskapp.py

@@ -0,0 +1,104 @@
+from flask import Flask, Response
+from pythonjsonlogger import jsonlogger
+import logging
+import os
+import sys
+import asyncio
+import time
+import driver
+import moneyforward.money as money
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+h = logging.StreamHandler()
+h.setLevel(logging.DEBUG)
+json_fmt = jsonlogger.JsonFormatter(
+    fmt="%(asctime)s %(levelname)s %(filename)s %(lineno)s %(message)s",
+    json_ensure_ascii=False,
+)
+h.setFormatter(json_fmt)
+logger.addHandler(h)
+drv = None
+drv_locked = False # driver を同時に操作しないためのロック関数
+
+app = Flask("flaskapp")
+@app.route("/", methods=["GET"])
+def index_get():
+    return "OK"
+
+@app.route("/moneyforward/cf", methods=["GET"])
+def moneyforward_get():
+    global drv_locked
+    if drv_locked:
+        return Response(status=503)
+    drv_locked = True
+    html = money.get_from_url(drv, "https://moneyforward.com/cf")
+
+    drv_locked = False
+    return str(html.decode("utf-8"))
+
+@app.route("/moneyforward/cf/lastmonth", methods=["GET"])
+def moneyforward_lastmonth_get():
+    global drv_locked
+    if drv_locked:
+        return Response(status=503)
+    drv_locked = True
+    html = money.get_from_url_cf_lastmonth(drv)
+
+    drv_locked = False
+    return str(html.decode("utf-8"))
+
+@app.route("/moneyforward/status", methods=["GET"])
+def moneyforward_status():
+    global drv_locked
+    if drv_locked:
+        return Response(status=503)
+    drv_locked = True
+    xpaths = os.getenv("refresh_xpaths").split(",")
+    ret_f_json = money.get_status(drv, xpaths)
+
+    drv_locked = False
+    return ret_f_json
+
+@app.route("/moneyforward/status", methods=["PUT"])
+def moneyforward_status_update():
+    global drv_locked
+    if drv_locked:
+        return Response(status=503)
+    drv_locked = True
+
+    money.move_page(drv, "https://moneyforward.com")
+    # Async update button
+    asyncio.new_event_loop().run_in_executor(None, _async_update_button, drv)
+
+    # async で driver を使っているので drv_locked を解除しない
+    return "Received"
+
+def main():
+    ## Get Initial Browser setup & login
+    global drv
+    logger.info("api setup start")
+    logger.info("get driver")
+    drv = driver.get_driver()
+    logger.info("money forward login")
+    try:
+        money.login(drv)
+    except Exception as e:
+        logger.error("failed to login. maybe changing xpath: %s", e)
+        sys.exit(1)
+    logger.info("money forward login successful")
+
+def _async_update_button(driver):
+    global drv_locked
+    refresh_xpaths = os.getenv("refresh_xpaths").split(",")
+    for xpath in refresh_xpaths:
+        try:
+            money.press_from_xpath(driver, xpath)
+            logger.info("press update button: %s", xpath)
+            time.sleep(5) # 同時押し負荷対策
+        except Exception as e:
+            logger.warn('failed to press update button: %s', e)
+        finally:
+            drv_locked = False
+
+main()

src/gunicorn.py

@@ -0,0 +1,39 @@
+import os
+import json
+
+# TCPソケット
+socket_path = "0.0.0.0:" + str(os.getenv("PORT", 9876))
+bind = socket_path
+
+# Debugging
+reload = True
+
+# Logging
+accesslog = "-"
+access_log_format = json.dumps(
+    {
+        "remote_address": r"%(h)s",
+        "user_name": r"%(u)s",
+        "date": r"%(t)s",
+        "status": r"%(s)s",
+        "method": r"%(m)s",
+        "url_path": r"%(U)s",
+        "query_string": r"%(q)s",
+        "protocol": r"%(H)s",
+        "response_length": r"%(B)s",
+        "referer": r"%(f)s",
+        "user_agent": r"%(a)s",
+        "request_time_seconds": r"%(L)s",
+    }
+)
+# loglevel = 'info'
+loglevel = "debug"
+logfile = "./log/app.log"
+logconfig = None
+
+# Proc Name
+proc_name = "Infrastructure-Practice-Flask"
+
+# Worker Processes
+workers = 1
+worker_class = "sync"

src/money-forward/main.py → src/moneyforward/main.py

@@ -39,6 +39,7 @@ def main():
     lg.info("fetcher start")
     lg.info("Get driver")
     driver = get_driver()
+    driver.implicitly_wait(10)
 
     # login
     try:
@@ -67,7 +68,8 @@ def main():
             html = money.get_from_url(driver, url)
             money.write_html(html, url)
             if url == "https://moneyforward.com/cf":  # このページは先月分のデータも取っておく
-                money.get_from_url_cf_lastmonth(driver)
+                html = money.get_from_url_cf_lastmonth(driver)
+                money.write_html(html, url + "_lastmonth")
         except Exception as e:
             lg.error("failed to get HTML: %s", e)
             sys.exit(1)

src/money-forward/money.py → src/moneyforward/money.py

@@ -10,6 +10,7 @@
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 import logging
+import json
 from pythonjsonlogger import jsonlogger
 
 lg = logging.getLogger(__name__)
@@ -92,10 +93,9 @@ def get_from_url_cf_lastmonth(driver):
         value="/html/body/div[1]/div[3]/div/div/section/div[2]/button[1]",
     )
     lastmonth_button.click()
-    time.sleep(15)
+    time.sleep(2)
     html = driver.page_source.encode("utf-8")
-    write_html(html, url + "_lastmonth")
-    return
+    return html
 
 
 def move_page(driver, url):
@@ -116,6 +116,44 @@ def press_from_xpath(driver, xpath):
     xpath_link.click()
     return
 
+def get_status(driver, xpaths):
+    """
+    /html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div/section[3]/ul/li[3]/ul[2]/li[3]/a[2] <- key: 「更新」リンクのxpath
+    たちから
+    /html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div/section[3]/ul/li[3]/div/a[1] : 名前
+    /html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div/section[3]/ul/li[3]/div/div : 取得日
+    /html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div/section[3]/ul/li[3]/ul[2]/li[1] : 同期ステータス
+    を取得して、リストで返す
+    """
+    move_page(driver, "https://moneyforward.com")
+    ret_f = {}
+
+    for xpath in xpaths:
+        base_xpath_list = xpath.split("/")[0:-3]
+        base_xpath = "/".join(base_xpath_list) # /html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div/section[3]/ul/li[3]
+        name_xpath = base_xpath + "/div/a[1]"
+        syncday_xpath = base_xpath + "/div/div"
+        sync_status_xpath = base_xpath + "/ul[2]/li[1]"
+
+        name = driver.find_element(
+            by=By.XPATH,
+            value=name_xpath
+        ).get_attribute("textContent")
+
+        syncday = driver.find_element(
+            by=By.XPATH,
+            value=syncday_xpath
+        ).get_attribute("textContent")
+
+        sync_status = driver.find_element(
+            by=By.XPATH,
+            value=sync_status_xpath
+        ).get_attribute("textContent")
+
+        ret_f[name] = {"sync_day" : syncday, "sync_status": sync_status}
+
+    ret_f_json = json.dumps(ret_f, ensure_ascii=False)
+    return ret_f_json
 
 def write_html(html, url):
     today = dt.date.today()  # 出力：datetime.date(2020, 3, 22)