From 0a22ccc51ebf9bffc49d13734c58b07ccb82e9ed Mon Sep 17 00:00:00 2001 From: azuki774s Date: Thu, 7 Sep 2023 22:11:44 +0900 Subject: [PATCH 01/11] add commands down clean --- Makefile | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a785185..fd44bd0 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,4 @@ +CONTAINER_NAME_API=bill-fetcher-api CONTAINER_NAME_AUELECT=bill-fetcher-auelect CONTAINER_NAME_REMIX=bill-fetcher-remix CONTAINER_NAME_SBI=bill-fetcher-sbi @@ -5,8 +6,9 @@ CONTAINER_NAME_TOKYOWATER=bill-fetcher-tokyowater CONTAINER_NAME_NICIGAS=bill-fetcher-nicigas CONTAINER_NAME_MONEY_FORWARD=bill-fetcher-money-forward -.PHONY: build start +.PHONY: build start stop clean build: + docker build -t $(CONTAINER_NAME_API) -f build/Dockerfile-api . docker build -t $(CONTAINER_NAME_AUELECT) -f build/Dockerfile-auelect . docker build -t $(CONTAINER_NAME_REMIX) -f build/Dockerfile-remix . docker build -t $(CONTAINER_NAME_SBI) -f build/Dockerfile-sbi . @@ -17,5 +19,17 @@ build: start: docker compose -f deployment/compose.yml up -d +stop: + docker compose -f deployment/compose.yml down + debug: docker compose -f deployment/compose.yml up + +clean: + docker image rm $(CONTAINER_NAME_API) + docker image rm $(CONTAINER_NAME_AUELECT) + docker image rm $(CONTAINER_NAME_REMIX) + docker image rm $(CONTAINER_NAME_SBI) + docker image rm $(CONTAINER_NAME_TOKYOWATER) + docker image rm $(CONTAINER_NAME_NICIGAS) + docker image rm $(CONTAINER_NAME_MONEY_FORWARD) From 643400a4290595e4cbc59f94a33dc9e050067e36 Mon Sep 17 00:00:00 2001 From: azuki774s Date: Thu, 7 Sep 2023 22:15:08 +0900 Subject: [PATCH 02/11] start http server --- build/Dockerfile-api | 8 ++++++++ deployment/compose.yml | 18 +++++++++++++----- src/api/flaskapp.py | 21 +++++++++++++++++++++ src/api/gunicorn.py | 39 +++++++++++++++++++++++++++++++++++++++ src/api/requirements.txt | 4 ++++ 5 files changed, 85 insertions(+), 5 deletions(-) create mode 100644 build/Dockerfile-api create mode 100644 src/api/flaskapp.py create mode 100644 src/api/gunicorn.py create mode 100644 src/api/requirements.txt diff --git a/build/Dockerfile-api b/build/Dockerfile-api new file mode 100644 index 0000000..5ef87e9 --- /dev/null +++ b/build/Dockerfile-api @@ -0,0 +1,8 @@ +FROM python:3.11-bullseye +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + +COPY src/api /src +WORKDIR /src +RUN pip install --upgrade pip && pip install -r requirements.txt +CMD ["gunicorn", "flaskapp:app", "--config", "gunicorn.py"] diff --git a/deployment/compose.yml b/deployment/compose.yml index 4ecb595..ff7a79b 100644 --- a/deployment/compose.yml +++ b/deployment/compose.yml @@ -1,12 +1,20 @@ version: '3' services: - money-forward: - image: bill-fetcher-money-forward - container_name: bill-fetcher-money-forward + fetcher-api: + image: bill-fetcher-api + container_name: bill-fetcher-api env_file: - money-forward.env - volumes: - - ./:/data/ + ports: + - "8080:9876" + + # money-forward: + # image: bill-fetcher-money-forward + # container_name: bill-fetcher-money-forward + # env_file: + # - money-forward.env + # volumes: + # - ./:/data/ # remix: # image: bill-fetcher-remix diff --git a/src/api/flaskapp.py b/src/api/flaskapp.py new file mode 100644 index 0000000..6e8a09e --- /dev/null +++ b/src/api/flaskapp.py @@ -0,0 +1,21 @@ +from flask import Flask +from pythonjsonlogger import jsonlogger +import logging + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +h = logging.StreamHandler() +h.setLevel(logging.DEBUG) +json_fmt = jsonlogger.JsonFormatter( + fmt="%(asctime)s %(levelname)s %(filename)s %(lineno)s %(message)s", + json_ensure_ascii=False, +) +h.setFormatter(json_fmt) +logger.addHandler(h) + + +app = Flask(__name__) + +@app.route("/", methods=["GET"]) +def index_get(): + return "OK" diff --git a/src/api/gunicorn.py b/src/api/gunicorn.py new file mode 100644 index 0000000..333d51b --- /dev/null +++ b/src/api/gunicorn.py @@ -0,0 +1,39 @@ +import os +import json + +# TCPソケット +socket_path = "0.0.0.0:" + str(os.getenv("PORT", 9876)) +bind = socket_path + +# Debugging +reload = True + +# Logging +accesslog = "-" +access_log_format = json.dumps( + { + "remote_address": r"%(h)s", + "user_name": r"%(u)s", + "date": r"%(t)s", + "status": r"%(s)s", + "method": r"%(m)s", + "url_path": r"%(U)s", + "query_string": r"%(q)s", + "protocol": r"%(H)s", + "response_length": r"%(B)s", + "referer": r"%(f)s", + "user_agent": r"%(a)s", + "request_time_seconds": r"%(L)s", + } +) +# loglevel = 'info' +loglevel = "debug" +logfile = "./log/app.log" +logconfig = None + +# Proc Name +proc_name = "Infrastructure-Practice-Flask" + +# Worker Processes +workers = 1 +worker_class = "sync" diff --git a/src/api/requirements.txt b/src/api/requirements.txt new file mode 100644 index 0000000..169f270 --- /dev/null +++ b/src/api/requirements.txt @@ -0,0 +1,4 @@ +Flask==2.3.3 +gunicorn==21.2.0 +requests==2.31.0 +python-json-logger==2.0.7 From f91df3d0c3756d87d534402107cbe17c0b0a7f6d Mon Sep 17 00:00:00 2001 From: azuki774s Date: Thu, 7 Sep 2023 23:02:44 +0900 Subject: [PATCH 03/11] rename moneyforward module --- build/Dockerfile-money-forward | 2 +- src/{money-forward => moneyforward}/main.py | 0 src/{money-forward => moneyforward}/money.py | 0 src/{money-forward => moneyforward}/requirements.txt | 0 4 files changed, 1 insertion(+), 1 deletion(-) rename src/{money-forward => moneyforward}/main.py (100%) rename src/{money-forward => moneyforward}/money.py (100%) rename src/{money-forward => moneyforward}/requirements.txt (100%) diff --git a/build/Dockerfile-money-forward b/build/Dockerfile-money-forward index 5ad326b..db15323 100644 --- a/build/Dockerfile-money-forward +++ b/build/Dockerfile-money-forward @@ -1,4 +1,4 @@ FROM ghcr.io/azuki774/selenium-chrome:latest -COPY src/money-forward/ /src/ +COPY src/moneyforward/ /src/ RUN pip install -r /src/requirements.txt ENTRYPOINT ["python3", "-u", "/src/main.py"] diff --git a/src/money-forward/main.py b/src/moneyforward/main.py similarity index 100% rename from src/money-forward/main.py rename to src/moneyforward/main.py diff --git a/src/money-forward/money.py b/src/moneyforward/money.py similarity index 100% rename from src/money-forward/money.py rename to src/moneyforward/money.py diff --git a/src/money-forward/requirements.txt b/src/moneyforward/requirements.txt similarity index 100% rename from src/money-forward/requirements.txt rename to src/moneyforward/requirements.txt From ded23a64979182f49d2819ed6568f76a73bf6620 Mon Sep 17 00:00:00 2001 From: azuki774s Date: Thu, 7 Sep 2023 23:03:14 +0900 Subject: [PATCH 04/11] reorder module construction for API --- build/Dockerfile-api | 7 ++++--- src/{api => }/flaskapp.py | 14 +++++++++++--- src/{api => }/gunicorn.py | 0 src/{api => }/requirements.txt | 0 4 files changed, 15 insertions(+), 6 deletions(-) rename src/{api => }/flaskapp.py (69%) rename src/{api => }/gunicorn.py (100%) rename src/{api => }/requirements.txt (100%) diff --git a/build/Dockerfile-api b/build/Dockerfile-api index 5ef87e9..ac4d1e9 100644 --- a/build/Dockerfile-api +++ b/build/Dockerfile-api @@ -2,7 +2,8 @@ FROM python:3.11-bullseye ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 -COPY src/api /src -WORKDIR /src -RUN pip install --upgrade pip && pip install -r requirements.txt +COPY src/ /src +RUN pip install --upgrade pip && pip install -r /src/requirements.txt +RUN pip install --upgrade pip && pip install -r /src/moneyforward/requirements.txt +WORKDIR /src/ CMD ["gunicorn", "flaskapp:app", "--config", "gunicorn.py"] diff --git a/src/api/flaskapp.py b/src/flaskapp.py similarity index 69% rename from src/api/flaskapp.py rename to src/flaskapp.py index 6e8a09e..30a7624 100644 --- a/src/api/flaskapp.py +++ b/src/flaskapp.py @@ -1,6 +1,7 @@ from flask import Flask from pythonjsonlogger import jsonlogger import logging +import moneyforward.money logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -13,9 +14,16 @@ h.setFormatter(json_fmt) logger.addHandler(h) - -app = Flask(__name__) - +app = Flask("flaskapp") @app.route("/", methods=["GET"]) def index_get(): return "OK" + + +def main(): + ## Get Initial Browser setup + logger.info("api setup start") + logger.info("get driver") + driver = driver.get_driver() + +main() diff --git a/src/api/gunicorn.py b/src/gunicorn.py similarity index 100% rename from src/api/gunicorn.py rename to src/gunicorn.py diff --git a/src/api/requirements.txt b/src/requirements.txt similarity index 100% rename from src/api/requirements.txt rename to src/requirements.txt From 3ed145da03de4896428d51d5be248225dfd18365 Mon Sep 17 00:00:00 2001 From: azuki774s Date: Thu, 7 Sep 2023 23:49:33 +0900 Subject: [PATCH 05/11] success login --- build/Dockerfile-api | 6 +++--- src/flaskapp.py | 15 +++++++++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/build/Dockerfile-api b/build/Dockerfile-api index ac4d1e9..c7bbcae 100644 --- a/build/Dockerfile-api +++ b/build/Dockerfile-api @@ -1,9 +1,9 @@ -FROM python:3.11-bullseye +FROM ghcr.io/azuki774/selenium-chrome:0.2.0 ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 COPY src/ /src RUN pip install --upgrade pip && pip install -r /src/requirements.txt -RUN pip install --upgrade pip && pip install -r /src/moneyforward/requirements.txt +RUN pip install -r /src/moneyforward/requirements.txt WORKDIR /src/ -CMD ["gunicorn", "flaskapp:app", "--config", "gunicorn.py"] +ENTRYPOINT ["gunicorn", "flaskapp:app", "--config", "gunicorn.py"] diff --git a/src/flaskapp.py b/src/flaskapp.py index 30a7624..6f2ef38 100644 --- a/src/flaskapp.py +++ b/src/flaskapp.py @@ -1,7 +1,9 @@ from flask import Flask from pythonjsonlogger import jsonlogger import logging -import moneyforward.money +import sys +import driver +import moneyforward.money as money logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -19,11 +21,20 @@ def index_get(): return "OK" +drv = None def main(): ## Get Initial Browser setup + global drv logger.info("api setup start") logger.info("get driver") - driver = driver.get_driver() + drv = driver.get_driver() + logger.info("money forward login") + try: + money.login(drv) + except Exception as e: + logger.error("failed to login. maybe changing xpath: %s", e) + sys.exit(1) + logger.info("money forward login sucessful") main() From 5cbfdac56afc8e40a43da810e76bd2d7344aef40 Mon Sep 17 00:00:00 2001 From: azuki774s Date: Thu, 7 Sep 2023 23:55:24 +0900 Subject: [PATCH 06/11] separate requirements directory for fast build --- build/Dockerfile-api | 5 +++-- build/Dockerfile-auelect | 4 ++-- build/Dockerfile-money-forward | 6 ++++-- build/Dockerfile-sbi | 4 +++- src/requirements.txt => requirements/api_requirements.txt | 0 .../auelect_requirements.txt | 0 .../moneyforward_requirements.txt | 0 .../requirement.txt => requirements/sbi_requirements.txt | 0 8 files changed, 12 insertions(+), 7 deletions(-) rename src/requirements.txt => requirements/api_requirements.txt (100%) rename src/auelect/requirement.txt => requirements/auelect_requirements.txt (100%) rename src/moneyforward/requirements.txt => requirements/moneyforward_requirements.txt (100%) rename src/sbi/requirement.txt => requirements/sbi_requirements.txt (100%) diff --git a/build/Dockerfile-api b/build/Dockerfile-api index c7bbcae..342baac 100644 --- a/build/Dockerfile-api +++ b/build/Dockerfile-api @@ -2,8 +2,9 @@ FROM ghcr.io/azuki774/selenium-chrome:0.2.0 ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 +COPY requirements/ /tmp/ +RUN pip install --upgrade pip && pip install -r /tmp/api_requirements.txt +RUN pip install -r /tmp/moneyforward_requirements.txt COPY src/ /src -RUN pip install --upgrade pip && pip install -r /src/requirements.txt -RUN pip install -r /src/moneyforward/requirements.txt WORKDIR /src/ ENTRYPOINT ["gunicorn", "flaskapp:app", "--config", "gunicorn.py"] diff --git a/build/Dockerfile-auelect b/build/Dockerfile-auelect index 8798e29..f9c3649 100644 --- a/build/Dockerfile-auelect +++ b/build/Dockerfile-auelect @@ -1,5 +1,5 @@ FROM python:3.11-bullseye -RUN python -m pip install --upgrade pip +COPY requirements/ /tmp/ +RUN python -m pip install --upgrade pip && pip install -r /tmp/auelect_requirements.txt COPY src/auelect/ /src/ -RUN pip install -r src/requirement.txt ENTRYPOINT ["python3", "-u", "/src/main.py"] diff --git a/build/Dockerfile-money-forward b/build/Dockerfile-money-forward index db15323..40aeccd 100644 --- a/build/Dockerfile-money-forward +++ b/build/Dockerfile-money-forward @@ -1,4 +1,6 @@ -FROM ghcr.io/azuki774/selenium-chrome:latest +FROM ghcr.io/azuki774/selenium-chrome:0.2.0 +COPY requirements/ /tmp/ +RUN pip install --upgrade pip && pip install -r /tmp/moneyforward_requirements.txt + COPY src/moneyforward/ /src/ -RUN pip install -r /src/requirements.txt ENTRYPOINT ["python3", "-u", "/src/main.py"] diff --git a/build/Dockerfile-sbi b/build/Dockerfile-sbi index 1f384e4..e30c5de 100644 --- a/build/Dockerfile-sbi +++ b/build/Dockerfile-sbi @@ -1,4 +1,6 @@ FROM ghcr.io/azuki774/selenium-chrome:latest + +COPY requirements/ /tmp/ +RUN pip install --upgrade pip && pip install -r /tmp/sbi_requirements.txt COPY src/sbi/ /src/ -RUN pip install -r /src/requirement.txt ENTRYPOINT ["python3", "-u", "/src/main.py"] diff --git a/src/requirements.txt b/requirements/api_requirements.txt similarity index 100% rename from src/requirements.txt rename to requirements/api_requirements.txt diff --git a/src/auelect/requirement.txt b/requirements/auelect_requirements.txt similarity index 100% rename from src/auelect/requirement.txt rename to requirements/auelect_requirements.txt diff --git a/src/moneyforward/requirements.txt b/requirements/moneyforward_requirements.txt similarity index 100% rename from src/moneyforward/requirements.txt rename to requirements/moneyforward_requirements.txt diff --git a/src/sbi/requirement.txt b/requirements/sbi_requirements.txt similarity index 100% rename from src/sbi/requirement.txt rename to requirements/sbi_requirements.txt From b7452fb5724392a0712163feee9c150131cb0772 Mon Sep 17 00:00:00 2001 From: azuki774s Date: Sat, 9 Sep 2023 00:28:09 +0900 Subject: [PATCH 07/11] decrease sleep time --- src/moneyforward/money.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/moneyforward/money.py b/src/moneyforward/money.py index bfcdf40..971a4b4 100644 --- a/src/moneyforward/money.py +++ b/src/moneyforward/money.py @@ -92,7 +92,7 @@ def get_from_url_cf_lastmonth(driver): value="/html/body/div[1]/div[3]/div/div/section/div[2]/button[1]", ) lastmonth_button.click() - time.sleep(15) + time.sleep(2) html = driver.page_source.encode("utf-8") write_html(html, url + "_lastmonth") return From ffe93597cc8bfe36e89e8509f6156575b3dfafa8 Mon Sep 17 00:00:00 2001 From: azuki774s Date: Sat, 9 Sep 2023 00:33:25 +0900 Subject: [PATCH 08/11] refactor for consistency --- src/moneyforward/main.py | 3 ++- src/moneyforward/money.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/moneyforward/main.py b/src/moneyforward/main.py index 89f6ea9..2ba76bc 100644 --- a/src/moneyforward/main.py +++ b/src/moneyforward/main.py @@ -67,7 +67,8 @@ def main(): html = money.get_from_url(driver, url) money.write_html(html, url) if url == "https://moneyforward.com/cf": # このページは先月分のデータも取っておく - money.get_from_url_cf_lastmonth(driver) + html = money.get_from_url_cf_lastmonth(driver) + money.write_html(html, url + "_lastmonth") except Exception as e: lg.error("failed to get HTML: %s", e) sys.exit(1) diff --git a/src/moneyforward/money.py b/src/moneyforward/money.py index 971a4b4..86e85ef 100644 --- a/src/moneyforward/money.py +++ b/src/moneyforward/money.py @@ -94,8 +94,7 @@ def get_from_url_cf_lastmonth(driver): lastmonth_button.click() time.sleep(2) html = driver.page_source.encode("utf-8") - write_html(html, url + "_lastmonth") - return + return html def move_page(driver, url): From dab6b24cd6329b92c99236e0bbbc2928911098df Mon Sep 17 00:00:00 2001 From: azuki774s Date: Sat, 9 Sep 2023 15:09:42 +0900 Subject: [PATCH 09/11] get status API --- deployment/compose.yml | 2 +- src/flaskapp.py | 22 +++++++++++++++++++--- src/moneyforward/main.py | 1 + src/moneyforward/money.py | 39 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 4 deletions(-) diff --git a/deployment/compose.yml b/deployment/compose.yml index ff7a79b..7236986 100644 --- a/deployment/compose.yml +++ b/deployment/compose.yml @@ -4,7 +4,7 @@ services: image: bill-fetcher-api container_name: bill-fetcher-api env_file: - - money-forward.env + - api.env ports: - "8080:9876" diff --git a/src/flaskapp.py b/src/flaskapp.py index 6f2ef38..b0dc8f3 100644 --- a/src/flaskapp.py +++ b/src/flaskapp.py @@ -1,6 +1,7 @@ from flask import Flask from pythonjsonlogger import jsonlogger import logging +import os import sys import driver import moneyforward.money as money @@ -15,16 +16,31 @@ ) h.setFormatter(json_fmt) logger.addHandler(h) +drv = None app = Flask("flaskapp") @app.route("/", methods=["GET"]) def index_get(): return "OK" -drv = None +@app.route("/moneyforward/cf", methods=["GET"]) +def moneyforward_get(): + html = money.get_from_url(drv, "https://moneyforward.com/cf") + return str(html.decode("utf-8")) + +@app.route("/moneyforward/cf/lastmonth", methods=["GET"]) +def moneyforward_lastmonth_get(): + html = money.get_from_url_cf_lastmonth(drv) + return str(html.decode("utf-8")) + +@app.route("/moneyforward/status", methods=["GET"]) +def moneyforward_status(): + xpaths = os.getenv("refresh_xpaths").split(",") + ret_f_json = money.get_status(drv, xpaths) + return ret_f_json def main(): - ## Get Initial Browser setup + ## Get Initial Browser setup & login global drv logger.info("api setup start") logger.info("get driver") @@ -35,6 +51,6 @@ def main(): except Exception as e: logger.error("failed to login. maybe changing xpath: %s", e) sys.exit(1) - logger.info("money forward login sucessful") + logger.info("money forward login successful") main() diff --git a/src/moneyforward/main.py b/src/moneyforward/main.py index 2ba76bc..ffafbfd 100644 --- a/src/moneyforward/main.py +++ b/src/moneyforward/main.py @@ -39,6 +39,7 @@ def main(): lg.info("fetcher start") lg.info("Get driver") driver = get_driver() + driver.implicitly_wait(10) # login try: diff --git a/src/moneyforward/money.py b/src/moneyforward/money.py index 86e85ef..98cc123 100644 --- a/src/moneyforward/money.py +++ b/src/moneyforward/money.py @@ -10,6 +10,7 @@ from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC import logging +import json from pythonjsonlogger import jsonlogger lg = logging.getLogger(__name__) @@ -115,6 +116,44 @@ def press_from_xpath(driver, xpath): xpath_link.click() return +def get_status(driver, xpaths): + """ + /html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div/section[3]/ul/li[3]/ul[2]/li[3]/a[2] <- key: 「更新」リンクのxpath + たちから + /html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div/section[3]/ul/li[3]/div/a[1] : 名前 + /html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div/section[3]/ul/li[3]/div/div : 取得日 + /html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div/section[3]/ul/li[3]/ul[2]/li[1] : 同期ステータス + を取得して、リストで返す + """ + move_page(driver, "https://moneyforward.com") + ret_f = {} + + for xpath in xpaths: + base_xpath_list = xpath.split("/")[0:-3] + base_xpath = "/".join(base_xpath_list) # /html/body/div[1]/div[3]/div[1]/div[1]/div[2]/div[1]/div/section[3]/ul/li[3] + name_xpath = base_xpath + "/div/a[1]" + syncday_xpath = base_xpath + "/div/div" + sync_status_xpath = base_xpath + "/ul[2]/li[1]" + + name = driver.find_element( + by=By.XPATH, + value=name_xpath + ).get_attribute("textContent") + + syncday = driver.find_element( + by=By.XPATH, + value=syncday_xpath + ).get_attribute("textContent") + + sync_status = driver.find_element( + by=By.XPATH, + value=sync_status_xpath + ).get_attribute("textContent") + + ret_f[name] = {"sync_day" : syncday, "sync_status": sync_status} + + ret_f_json = json.dumps(ret_f, ensure_ascii=False) + return ret_f_json def write_html(html, url): today = dt.date.today() # 出力：datetime.date(2020, 3, 22) From e5d7bfb202aca5357051243bc4e4e167a4c4329e Mon Sep 17 00:00:00 2001 From: azuki774s Date: Sat, 9 Sep 2023 21:22:08 +0900 Subject: [PATCH 10/11] add bill-fetch-api publish --- .github/workflows/image-push.yml | 40 ++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/.github/workflows/image-push.yml b/.github/workflows/image-push.yml index 07efce4..f367e0a 100644 --- a/.github/workflows/image-push.yml +++ b/.github/workflows/image-push.yml @@ -205,3 +205,43 @@ jobs: file: ./build/Dockerfile-money-forward push: true tags: ${{ steps.meta.outputs.tags }} + + build_and_push_api: + runs-on: ubuntu-latest + env: + IMAGE_NAME: bill-fetcher-api + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Set meta + id: meta + uses: docker/metadata-action@v3 + with: + # list of Docker images to use as base name for tags + images: | + ghcr.io/azuki774/bill-fetcher-api + # generate Docker tags based on the following events/attributes + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=semver,pattern=latest + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GH_ACCESS_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: . + file: ./build/Dockerfile-api + push: true + tags: ${{ steps.meta.outputs.tags }} From cdeaa93b56305e6193ad18b364cfdcf990c2e98b Mon Sep 17 00:00:00 2001 From: azuki774s Date: Sat, 9 Sep 2023 21:22:38 +0900 Subject: [PATCH 11/11] add update status button --- deployment/api.env.sample | 3 +++ src/flaskapp.py | 50 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 deployment/api.env.sample diff --git a/deployment/api.env.sample b/deployment/api.env.sample new file mode 100644 index 0000000..5bf8697 --- /dev/null +++ b/deployment/api.env.sample @@ -0,0 +1,3 @@ +id=xxxxx@gmail.com +pass=xxxxxx +refresh_xpaths="xxxxxxxx,xxxxxxxx" # https://moneyforward.com に表示される金融機関等の[更新]ボタンのXPATHを , 区切りで記載 diff --git a/src/flaskapp.py b/src/flaskapp.py index b0dc8f3..a62f41c 100644 --- a/src/flaskapp.py +++ b/src/flaskapp.py @@ -1,8 +1,10 @@ -from flask import Flask +from flask import Flask, Response from pythonjsonlogger import jsonlogger import logging import os import sys +import asyncio +import time import driver import moneyforward.money as money @@ -17,6 +19,7 @@ h.setFormatter(json_fmt) logger.addHandler(h) drv = None +drv_locked = False # driver を同時に操作しないためのロック関数 app = Flask("flaskapp") @app.route("/", methods=["GET"]) @@ -25,20 +28,52 @@ def index_get(): @app.route("/moneyforward/cf", methods=["GET"]) def moneyforward_get(): + global drv_locked + if drv_locked: + return Response(status=503) + drv_locked = True html = money.get_from_url(drv, "https://moneyforward.com/cf") + + drv_locked = False return str(html.decode("utf-8")) @app.route("/moneyforward/cf/lastmonth", methods=["GET"]) def moneyforward_lastmonth_get(): + global drv_locked + if drv_locked: + return Response(status=503) + drv_locked = True html = money.get_from_url_cf_lastmonth(drv) + + drv_locked = False return str(html.decode("utf-8")) @app.route("/moneyforward/status", methods=["GET"]) def moneyforward_status(): + global drv_locked + if drv_locked: + return Response(status=503) + drv_locked = True xpaths = os.getenv("refresh_xpaths").split(",") ret_f_json = money.get_status(drv, xpaths) + + drv_locked = False return ret_f_json +@app.route("/moneyforward/status", methods=["PUT"]) +def moneyforward_status_update(): + global drv_locked + if drv_locked: + return Response(status=503) + drv_locked = True + + money.move_page(drv, "https://moneyforward.com") + # Async update button + asyncio.new_event_loop().run_in_executor(None, _async_update_button, drv) + + # async で driver を使っているので drv_locked を解除しない + return "Received" + def main(): ## Get Initial Browser setup & login global drv @@ -53,4 +88,17 @@ def main(): sys.exit(1) logger.info("money forward login successful") +def _async_update_button(driver): + global drv_locked + refresh_xpaths = os.getenv("refresh_xpaths").split(",") + for xpath in refresh_xpaths: + try: + money.press_from_xpath(driver, xpath) + logger.info("press update button: %s", xpath) + time.sleep(5) # 同時押し負荷対策 + except Exception as e: + logger.warn('failed to press update button: %s', e) + finally: + drv_locked = False + main()