diff --git a/.github/workflows/image-push.yml b/.github/workflows/image-push.yml index 07efce4..f367e0a 100644 --- a/.github/workflows/image-push.yml +++ b/.github/workflows/image-push.yml @@ -205,3 +205,43 @@ jobs: file: ./build/Dockerfile-money-forward push: true tags: ${{ steps.meta.outputs.tags }} + + build_and_push_api: + runs-on: ubuntu-latest + env: + IMAGE_NAME: bill-fetcher-api + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Set meta + id: meta + uses: docker/metadata-action@v3 + with: + # list of Docker images to use as base name for tags + images: | + ghcr.io/azuki774/bill-fetcher-api + # generate Docker tags based on the following events/attributes + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=semver,pattern=latest + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GH_ACCESS_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: . + file: ./build/Dockerfile-api + push: true + tags: ${{ steps.meta.outputs.tags }} diff --git a/Makefile b/Makefile index a785185..fd44bd0 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,4 @@ +CONTAINER_NAME_API=bill-fetcher-api CONTAINER_NAME_AUELECT=bill-fetcher-auelect CONTAINER_NAME_REMIX=bill-fetcher-remix CONTAINER_NAME_SBI=bill-fetcher-sbi @@ -5,8 +6,9 @@ CONTAINER_NAME_TOKYOWATER=bill-fetcher-tokyowater CONTAINER_NAME_NICIGAS=bill-fetcher-nicigas CONTAINER_NAME_MONEY_FORWARD=bill-fetcher-money-forward -.PHONY: build start +.PHONY: build start stop clean build: + docker build -t $(CONTAINER_NAME_API) -f build/Dockerfile-api . docker build -t $(CONTAINER_NAME_AUELECT) -f build/Dockerfile-auelect . docker build -t $(CONTAINER_NAME_REMIX) -f build/Dockerfile-remix . docker build -t $(CONTAINER_NAME_SBI) -f build/Dockerfile-sbi . @@ -17,5 +19,17 @@ build: start: docker compose -f deployment/compose.yml up -d +stop: + docker compose -f deployment/compose.yml down + debug: docker compose -f deployment/compose.yml up + +clean: + docker image rm $(CONTAINER_NAME_API) + docker image rm $(CONTAINER_NAME_AUELECT) + docker image rm $(CONTAINER_NAME_REMIX) + docker image rm $(CONTAINER_NAME_SBI) + docker image rm $(CONTAINER_NAME_TOKYOWATER) + docker image rm $(CONTAINER_NAME_NICIGAS) + docker image rm $(CONTAINER_NAME_MONEY_FORWARD) diff --git a/build/Dockerfile-api b/build/Dockerfile-api new file mode 100644 index 0000000..342baac --- /dev/null +++ b/build/Dockerfile-api @@ -0,0 +1,10 @@ +FROM ghcr.io/azuki774/selenium-chrome:0.2.0 +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + +COPY requirements/ /tmp/ +RUN pip install --upgrade pip && pip install -r /tmp/api_requirements.txt +RUN pip install -r /tmp/moneyforward_requirements.txt +COPY src/ /src +WORKDIR /src/ +ENTRYPOINT ["gunicorn", "flaskapp:app", "--config", "gunicorn.py"] diff --git a/build/Dockerfile-auelect b/build/Dockerfile-auelect index 8798e29..f9c3649 100644 --- a/build/Dockerfile-auelect +++ b/build/Dockerfile-auelect @@ -1,5 +1,5 @@ FROM python:3.11-bullseye -RUN python -m pip install --upgrade pip +COPY requirements/ /tmp/ +RUN python -m pip install --upgrade pip && pip install -r /tmp/auelect_requirements.txt COPY src/auelect/ /src/ -RUN pip install -r src/requirement.txt ENTRYPOINT ["python3", "-u", "/src/main.py"] diff --git a/build/Dockerfile-money-forward b/build/Dockerfile-money-forward index 5ad326b..40aeccd 100644 --- a/build/Dockerfile-money-forward +++ b/build/Dockerfile-money-forward @@ -1,4 +1,6 @@ -FROM ghcr.io/azuki774/selenium-chrome:latest -COPY src/money-forward/ /src/ -RUN pip install -r /src/requirements.txt +FROM ghcr.io/azuki774/selenium-chrome:0.2.0 +COPY requirements/ /tmp/ +RUN pip install --upgrade pip && pip install -r /tmp/moneyforward_requirements.txt + +COPY src/moneyforward/ /src/ ENTRYPOINT ["python3", "-u", "/src/main.py"] diff --git a/build/Dockerfile-sbi b/build/Dockerfile-sbi index 1f384e4..e30c5de 100644 --- a/build/Dockerfile-sbi +++ b/build/Dockerfile-sbi @@ -1,4 +1,6 @@ FROM ghcr.io/azuki774/selenium-chrome:latest + +COPY requirements/ /tmp/ +RUN pip install --upgrade pip && pip install -r /tmp/sbi_requirements.txt COPY src/sbi/ /src/ -RUN pip install -r /src/requirement.txt ENTRYPOINT ["python3", "-u", "/src/main.py"] diff --git a/deployment/api.env.sample b/deployment/api.env.sample new file mode 100644 index 0000000..5bf8697 --- /dev/null +++ b/deployment/api.env.sample @@ -0,0 +1,3 @@ +id=xxxxx@gmail.com +pass=xxxxxx +refresh_xpaths="xxxxxxxx,xxxxxxxx" # https://moneyforward.com に表示される金融機関等の[更新]ボタンのXPATHを , 区切りで記載 diff --git a/deployment/compose.yml b/deployment/compose.yml index 4ecb595..7236986 100644 --- a/deployment/compose.yml +++ b/deployment/compose.yml @@ -1,12 +1,20 @@ version: '3' services: - money-forward: - image: bill-fetcher-money-forward - container_name: bill-fetcher-money-forward + fetcher-api: + image: bill-fetcher-api + container_name: bill-fetcher-api env_file: - - money-forward.env - volumes: - - ./:/data/ + - api.env + ports: + - "8080:9876" + + # money-forward: + # image: bill-fetcher-money-forward + # container_name: bill-fetcher-money-forward + # env_file: + # - money-forward.env + # volumes: + # - ./:/data/ # remix: # image: bill-fetcher-remix diff --git a/requirements/api_requirements.txt b/requirements/api_requirements.txt new file mode 100644 index 0000000..169f270 --- /dev/null +++ b/requirements/api_requirements.txt @@ -0,0 +1,4 @@ +Flask==2.3.3 +gunicorn==21.2.0 +requests==2.31.0 +python-json-logger==2.0.7 diff --git a/src/auelect/requirement.txt b/requirements/auelect_requirements.txt similarity index 100% rename from src/auelect/requirement.txt rename to requirements/auelect_requirements.txt diff --git a/src/money-forward/requirements.txt b/requirements/moneyforward_requirements.txt similarity index 100% rename from src/money-forward/requirements.txt rename to requirements/moneyforward_requirements.txt diff --git a/src/sbi/requirement.txt b/requirements/sbi_requirements.txt similarity index 100% rename from src/sbi/requirement.txt rename to requirements/sbi_requirements.txt diff --git a/src/flaskapp.py b/src/flaskapp.py new file mode 100644 index 0000000..a62f41c --- /dev/null +++ b/src/flaskapp.py @@ -0,0 +1,104 @@ +from flask import Flask, Response +from pythonjsonlogger import jsonlogger +import logging +import os +import sys +import asyncio +import time +import driver +import moneyforward.money as money + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +h = logging.StreamHandler() +h.setLevel(logging.DEBUG) +json_fmt = jsonlogger.JsonFormatter( + fmt="%(asctime)s %(levelname)s %(filename)s %(lineno)s %(message)s", + json_ensure_ascii=False, +) +h.setFormatter(json_fmt) +logger.addHandler(h) +drv = None +drv_locked = False # driver を同時に操作しないためのロック関数 + +app = Flask("flaskapp") +@app.route("/", methods=["GET"]) +def index_get(): + return "OK" + +@app.route("/moneyforward/cf", methods=["GET"]) +def moneyforward_get(): + global drv_locked + if drv_locked: + return Response(status=503) + drv_locked = True + html = money.get_from_url(drv, "https://moneyforward.com/cf") + + drv_locked = False + return str(html.decode("utf-8")) + +@app.route("/moneyforward/cf/lastmonth", methods=["GET"]) +def moneyforward_lastmonth_get(): + global drv_locked + if drv_locked: + return Response(status=503) + drv_locked = True + html = money.get_from_url_cf_lastmonth(drv) + + drv_locked = False + return str(html.decode("utf-8")) + +@app.route("/moneyforward/status", methods=["GET"]) +def moneyforward_status(): + global drv_locked + if drv_locked: + return Response(status=503) + drv_locked = True + xpaths = os.getenv("refresh_xpaths").split(",") + ret_f_json = money.get_status(drv, xpaths) + + drv_locked = False + return ret_f_json + +@app.route("/moneyforward/status", methods=["PUT"]) +def moneyforward_status_update(): + global drv_locked + if drv_locked: + return Response(status=503) + drv_locked = True + + money.move_page(drv, "https://moneyforward.com") + # Async update button + asyncio.new_event_loop().run_in_executor(None, _async_update_button, drv) + + # async で driver を使っているので drv_locked を解除しない + return "Received" + +def main(): + ## Get Initial Browser setup & login + global drv + logger.info("api setup start") + logger.info("get driver") + drv = driver.get_driver() + logger.info("money forward login") + try: + money.login(drv) + except Exception as e: + logger.error("failed to login. maybe changing xpath: %s", e) + sys.exit(1) + logger.info("money forward login successful") + +def _async_update_button(driver): + global drv_locked + refresh_xpaths = os.getenv("refresh_xpaths").split(",") + for xpath in refresh_xpaths: + try: + money.press_from_xpath(driver, xpath) + logger.info("press update button: %s", xpath) + time.sleep(5) # 同時押し負荷対策 + except Exception as e: + logger.warn('failed to press update button: %s', e) + finally: + drv_locked = False + +main() diff --git a/src/gunicorn.py b/src/gunicorn.py new file mode 100644 index 0000000..333d51b --- /dev/null +++ b/src/gunicorn.py @@ -0,0 +1,39 @@ +import os +import json + +# TCPソケット +socket_path = "0.0.0.0:" + str(os.getenv("PORT", 9876)) +bind = socket_path + +# Debugging +reload = True + +# Logging +accesslog = "-" +access_log_format = json.dumps( + { + "remote_address": r"%(h)s", + "user_name": r"%(u)s", + "date": r"%(t)s", + "status": r"%(s)s", + "method": r"%(m)s", + "url_path": r"%(U)s", + "query_string": r"%(q)s", + "protocol": r"%(H)s", + "response_length": r"%(B)s", + "referer": r"%(f)s", + "user_agent": r"%(a)s", + "request_time_seconds": r"%(L)s", + } +) +# loglevel = 'info' +loglevel = "debug" +logfile = "./log/app.log" +logconfig = None + +# Proc Name +proc_name = "Infrastructure-Practice-Flask" + +# Worker Processes +workers = 1 +worker_class = "sync" diff --git a/src/money-forward/main.py b/src/moneyforward/main.py similarity index 94% rename from src/money-forward/main.py rename to src/moneyforward/main.py index 89f6ea9..ffafbfd 100644 --- a/src/money-forward/main.py +++ b/src/moneyforward/main.py @@ -39,6 +39,7 @@ def main(): lg.info("fetcher start") lg.info("Get driver") driver = get_driver() + driver.implicitly_wait(10) # login try: @@ -67,7 +68,8 @@ def main(): html = money.get_from_url(driver, url) money.write_html(html, url) if url == "https://moneyforward.com/cf": # このページは先月分のデータも取っておく - money.get_from_url_cf_lastmonth(driver) + html = money.get_from_url_cf_lastmonth(driver) + money.write_html(html, url + "_lastmonth") except Exception as e: lg.error("failed to get HTML: %s", e) sys.exit(1) diff --git a/src/money-forward/money.py b/src/moneyforward/money.py similarity index 69% rename from src/money-forward/money.py rename to src/moneyforward/money.py index bfcdf40..98cc123 100644 --- a/src/money-forward/money.py +++ b/src/moneyforward/money.py @@ -10,6 +10,7 @@ from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC import logging +import json from pythonjsonlogger import jsonlogger lg = logging.getLogger(__name__) @@ -92,10 +93,9 @@ def get_from_url_cf_lastmonth(driver): value="/html/body/div[1]/div[3]/div/div/section/div[2]/button[1]", ) lastmonth_button.click() - time.sleep(15) + time.sleep(2) html = driver.page_source.encode("utf-8") - write_html(html, url + "_lastmonth") - return + return html def move_page(driver, url): @@ -116,6 +116,44 @@ def press_from_xpath(driver, xpath): xpath_link.click() return +def get_status(driver, xpaths): + """ + /html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div/section[3]/ul/li[3]/ul[2]/li[3]/a[2] <- key: 「更新」リンクのxpath + たちから + /html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div/section[3]/ul/li[3]/div/a[1] : 名前 + /html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div/section[3]/ul/li[3]/div/div : 取得日 + /html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div/section[3]/ul/li[3]/ul[2]/li[1] : 同期ステータス + を取得して、リストで返す + """ + move_page(driver, "https://moneyforward.com") + ret_f = {} + + for xpath in xpaths: + base_xpath_list = xpath.split("/")[0:-3] + base_xpath = "/".join(base_xpath_list) # /html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div/section[3]/ul/li[3] + name_xpath = base_xpath + "/div/a[1]" + syncday_xpath = base_xpath + "/div/div" + sync_status_xpath = base_xpath + "/ul[2]/li[1]" + + name = driver.find_element( + by=By.XPATH, + value=name_xpath + ).get_attribute("textContent") + + syncday = driver.find_element( + by=By.XPATH, + value=syncday_xpath + ).get_attribute("textContent") + + sync_status = driver.find_element( + by=By.XPATH, + value=sync_status_xpath + ).get_attribute("textContent") + + ret_f[name] = {"sync_day" : syncday, "sync_status": sync_status} + + ret_f_json = json.dumps(ret_f, ensure_ascii=False) + return ret_f_json def write_html(html, url): today = dt.date.today() # 出力：datetime.date(2020, 3, 22)