From b241af4636f45b529ffc24fb1c9df1cf45678173 Mon Sep 17 00:00:00 2001 From: azuki774s Date: Sat, 2 Sep 2023 18:03:24 +0900 Subject: [PATCH 1/4] elect-remix no maintenance --- .github/workflows/image-push.yml | 40 -------------------------------- 1 file changed, 40 deletions(-) diff --git a/.github/workflows/image-push.yml b/.github/workflows/image-push.yml index 07efce4..a78d7ba 100644 --- a/.github/workflows/image-push.yml +++ b/.github/workflows/image-push.yml @@ -6,46 +6,6 @@ on: - v* jobs: - build_and_push_remix: - runs-on: ubuntu-latest - env: - IMAGE_NAME: bill-fetcher-remix - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Set meta - id: meta - uses: docker/metadata-action@v3 - with: - # list of Docker images to use as base name for tags - images: | - ghcr.io/azuki774/bill-fetcher-remix - # generate Docker tags based on the following events/attributes - tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=semver,pattern=latest - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GH_ACCESS_TOKEN }} - - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: . - file: ./build/Dockerfile-remix - push: true - tags: ${{ steps.meta.outputs.tags }} - build_and_push_sbi: runs-on: ubuntu-latest env: From b05a6a556735a8afa847dc064c1d22e57667b0dc Mon Sep 17 00:00:00 2001 From: azuki774s Date: Sat, 2 Sep 2023 23:30:43 +0900 Subject: [PATCH 2/4] wip add au fetcher --- Makefile | 2 + build/Dockerfile-au | 4 ++ deployment/compose.yml | 20 ++++++-- src/au/au.py | 111 ++++++++++++++++++++++++++++++++++++++++ src/au/main.py | 58 +++++++++++++++++++++ src/au/requirements.txt | 1 + 6 files changed, 191 insertions(+), 5 deletions(-) create mode 100644 build/Dockerfile-au create mode 100644 src/au/au.py create mode 100644 src/au/main.py create mode 100644 src/au/requirements.txt diff --git a/Makefile b/Makefile index a785185..c8ff4c0 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,4 @@ +CONTAINER_NAME_AU=bill-fetcher-au CONTAINER_NAME_AUELECT=bill-fetcher-auelect CONTAINER_NAME_REMIX=bill-fetcher-remix CONTAINER_NAME_SBI=bill-fetcher-sbi @@ -7,6 +8,7 @@ CONTAINER_NAME_MONEY_FORWARD=bill-fetcher-money-forward .PHONY: build start build: + docker build -t $(CONTAINER_NAME_AU) -f build/Dockerfile-au . docker build -t $(CONTAINER_NAME_AUELECT) -f build/Dockerfile-auelect . docker build -t $(CONTAINER_NAME_REMIX) -f build/Dockerfile-remix . docker build -t $(CONTAINER_NAME_SBI) -f build/Dockerfile-sbi . diff --git a/build/Dockerfile-au b/build/Dockerfile-au new file mode 100644 index 0000000..ff6a9e1 --- /dev/null +++ b/build/Dockerfile-au @@ -0,0 +1,4 @@ +FROM ghcr.io/azuki774/selenium-chrome:0.2.0-rc.2 +COPY src/au/ /src/ +RUN pip install -r /src/requirements.txt +ENTRYPOINT ["python3", "-u", "/src/main.py"] diff --git a/deployment/compose.yml b/deployment/compose.yml index 4ecb595..732a67a 100644 --- a/deployment/compose.yml +++ b/deployment/compose.yml @@ -1,12 +1,22 @@ version: '3' services: - money-forward: - image: bill-fetcher-money-forward - container_name: bill-fetcher-money-forward + # money-forward: + # image: bill-fetcher-money-forward + # container_name: bill-fetcher-money-forward + # env_file: + # - money-forward.env + # volumes: + # - ./:/data/ + + au: + image: bill-fetcher-au + container_name: bill-fetcher-au env_file: - - money-forward.env + - au-token.env volumes: - - ./:/data/ + - ./.cookies:/.cookies + command: + - "--login-2fa" # remix: # image: bill-fetcher-remix diff --git a/src/au/au.py b/src/au/au.py new file mode 100644 index 0000000..e96e872 --- /dev/null +++ b/src/au/au.py @@ -0,0 +1,111 @@ +import os +import time +from selenium import webdriver +from selenium.webdriver.common.keys import Keys +from selenium.webdriver.common.by import By +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.chrome.service import Service +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC + +import logging +from pythonjsonlogger import jsonlogger +import argparse +import pickle + +lg = logging.getLogger(__name__) +lg.setLevel(logging.DEBUG) +h = logging.StreamHandler() +h.setLevel(logging.DEBUG) +json_fmt = jsonlogger.JsonFormatter(fmt='%(asctime)s %(levelname)s %(name)s %(message)s', json_ensure_ascii=False) +h.setFormatter(json_fmt) +lg.addHandler(h) + +SECRET_2FA_FILE="/.2fa_secret" +COOKIES='/.cookies' + +def load_cookies(driver): + cookies = pickle.load(open(COOKIES, 'rb')) + for cookie in cookies: + driver.add_cookie(cookie) + return + +def save_cookies(driver): + pickle.dump(driver.get_cookies(), open(COOKIES, 'wb')) + return + +def login(driver): + lg.info("au login start") + url = "https://connect.auone.jp/net/vwc/cca_lg_eu_nets/login?targeturl=https%3A%2F%2Fid.auone.jp%2Findex.html%3Fstate%3Dlogin" + driver.get(url) + + telno_field = driver.find_element( + by=By.XPATH, + value="/html/body/div[2]/div/div[1]/div[2]/div/div/form/input[24]", + ) + telno_field.send_keys(os.getenv("telno")) + + next_button = driver.find_element( + by=By.XPATH, + value="/html/body/div[2]/div/div[1]/div[2]/div/div/form/button[1]", + ) + next_button.click() + + pass_field = driver.find_element( + by=By.XPATH, + value="/html/body/div[2]/div/div[1]/div[2]/div/div/form/input[25]", + ) + pass_field.send_keys(os.getenv("pass")) + + login_button = driver.find_element( + by=By.XPATH, + value="/html/body/div[2]/div/div[1]/div[2]/div/div/form/button[4]", + ) + login_button.click() + + lg.info("user/pass login ok") + +def login_2fa(driver): + login(driver) + _login_2fa_proc(driver) + return + +def _login_2fa_proc(driver): + # 前提: https://connect.auone.jp/net/vwc/cca_lg_eu_nets/cca で止まっている状態 + # 2段階認証のためのログイン操作をする + # /.2fa_secret に 2段階認証コードを書くことで認証を進める + lg.info("try 2FA login") + lg.info("please set {0} written 2FA code".format(SECRET_2FA_FILE)) + lg.info("docker exec -it /bin/sh") + lg.info("echo '' > {0}".format(SECRET_2FA_FILE)) + + while True: + # ファイルが存在するようになるまで待つ + if os.path.isfile(SECRET_2FA_FILE): + break + time.sleep(10) + + with open(SECRET_2FA_FILE, encoding='utf-8') as f: + lines = f.read() + secret_2fa_code = str(lines) + + pass_field = driver.find_element( + by=By.XPATH, + value="/html/body/div[2]/div/form/input[7]", + ) + pass_field.send_keys(os.getenv("pass")) + + login_button = driver.find_element( + by=By.XPATH, + value="/html/body/div[2]/div/form/button", + ) + login_button.click() + lg.info("2FA login information send") + +def get_from_url(driver, url): + wait = WebDriverWait(driver=driver, timeout=30) + driver.get(url) + wait.until(EC.presence_of_all_elements_located) + html = driver.page_source.encode("utf-8") + print(html) + return html diff --git a/src/au/main.py b/src/au/main.py new file mode 100644 index 0000000..8d801ba --- /dev/null +++ b/src/au/main.py @@ -0,0 +1,58 @@ +from selenium import webdriver +from selenium.webdriver.common.keys import Keys +from selenium.webdriver.common.by import By +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.chrome.service import Service +import logging +import driver +from pythonjsonlogger import jsonlogger +import argparse +import sys +import au +import os + +lg = logging.getLogger(__name__) +lg.setLevel(logging.DEBUG) +h = logging.StreamHandler() +h.setLevel(logging.DEBUG) +json_fmt = jsonlogger.JsonFormatter(fmt='%(asctime)s %(levelname)s %(name)s %(message)s', json_ensure_ascii=False) +h.setFormatter(json_fmt) +lg.addHandler(h) + +COOKIES='/.cookies' + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Process some integers.') + parser.add_argument('--login-2fa', action='store_true') # default false + args = parser.parse_args() + lg.info("au start") + + lg.info("get driver") + driver = driver.get_driver() + + if os.path.isfile(COOKIES): + lg.info("load cookies") + try: + au.load_cookies(driver) + lg.info("load cookies success") + except Exception as e: + lg.warning('failed to load cookies. skipping. {0}'.format(e)) + else: + lg.warning("cookie file not found") + + + if args.login_2fa: + # login mode + lg.info("au page login mode") + au.login_2fa(driver) + sys.exit(0) + + au.login(driver) + + lg.info("save cookies") + au.save_cookies(driver) + + lg.info("fetch url: " + "https://mieru.auone.jp/#/results/daily") + au.get_from_url(driver, "https://mieru.auone.jp/#/results/daily") + + lg.info("auelect end") diff --git a/src/au/requirements.txt b/src/au/requirements.txt new file mode 100644 index 0000000..fd0d82a --- /dev/null +++ b/src/au/requirements.txt @@ -0,0 +1 @@ +python-json-logger>=2.0.7 From f7c07ce8fdff5fbad59ccb8412671026069f38a2 Mon Sep 17 00:00:00 2001 From: azuki774s Date: Sun, 3 Sep 2023 00:38:29 +0900 Subject: [PATCH 3/4] wip not working --- build/Dockerfile-au | 2 +- deployment/compose.yml | 4 ++-- src/au/au.py | 30 +++++++++++++++++++++++++----- src/au/main.py | 13 ++----------- 4 files changed, 30 insertions(+), 19 deletions(-) diff --git a/build/Dockerfile-au b/build/Dockerfile-au index ff6a9e1..feede6b 100644 --- a/build/Dockerfile-au +++ b/build/Dockerfile-au @@ -1,4 +1,4 @@ -FROM ghcr.io/azuki774/selenium-chrome:0.2.0-rc.2 +FROM ghcr.io/azuki774/selenium-chrome:0.2.0-rc.3 COPY src/au/ /src/ RUN pip install -r /src/requirements.txt ENTRYPOINT ["python3", "-u", "/src/main.py"] diff --git a/deployment/compose.yml b/deployment/compose.yml index 732a67a..7702d9f 100644 --- a/deployment/compose.yml +++ b/deployment/compose.yml @@ -15,8 +15,8 @@ services: - au-token.env volumes: - ./.cookies:/.cookies - command: - - "--login-2fa" + # command: + # - "--login-2fa" # remix: # image: bill-fetcher-remix diff --git a/src/au/au.py b/src/au/au.py index e96e872..ba6f143 100644 --- a/src/au/au.py +++ b/src/au/au.py @@ -36,7 +36,20 @@ def save_cookies(driver): def login(driver): lg.info("au login start") - url = "https://connect.auone.jp/net/vwc/cca_lg_eu_nets/login?targeturl=https%3A%2F%2Fid.auone.jp%2Findex.html%3Fstate%3Dlogin" + url = "https://connect.auone.jp/net/vwc/cca_lg_eu_nets/login?targeturl=https%3A%2F%2Fwww.au.com%2Fenergy%2Fdenki%2Flogin" + driver.get(url) + if os.path.isfile(COOKIES): + lg.info("load cookies") + try: + load_cookies(driver) + lg.info("load cookies success") + except Exception as e: + lg.warning('failed to load cookies. skipping. {0}'.format(e)) + else: + lg.warning("cookie file not found") + + # After loading cookies + url = "https://connect.auone.jp/net/vwc/cca_lg_eu_nets/login?targeturl=https%3A%2F%2Fwww.au.com%2Fenergy%2Fdenki%2Flogin" driver.get(url) telno_field = driver.find_element( @@ -50,20 +63,26 @@ def login(driver): value="/html/body/div[2]/div/div[1]/div[2]/div/div/form/button[1]", ) next_button.click() + time.sleep(10) + html = driver.page_source.encode("utf-8").decode("utf-8") + print(html) pass_field = driver.find_element( by=By.XPATH, - value="/html/body/div[2]/div/div[1]/div[2]/div/div/form/input[25]", + value="/html/body/div[3]/div/div[1]/div[2]/div/div/form/input[25]", ) pass_field.send_keys(os.getenv("pass")) login_button = driver.find_element( by=By.XPATH, - value="/html/body/div[2]/div/div[1]/div[2]/div/div/form/button[4]", + value="/html/body/div[3]/div/div[1]/div[2]/div/div/form/button[4]", ) login_button.click() lg.info("user/pass login ok") + time.sleep(10) + html = driver.page_source.encode("utf-8").decode("utf-8") + print(html) def login_2fa(driver): login(driver) @@ -76,7 +95,7 @@ def _login_2fa_proc(driver): # /.2fa_secret に 2段階認証コードを書くことで認証を進める lg.info("try 2FA login") lg.info("please set {0} written 2FA code".format(SECRET_2FA_FILE)) - lg.info("docker exec -it /bin/sh") + lg.info("docker exec -it /bin/bash") lg.info("echo '' > {0}".format(SECRET_2FA_FILE)) while True: @@ -105,7 +124,8 @@ def _login_2fa_proc(driver): def get_from_url(driver, url): wait = WebDriverWait(driver=driver, timeout=30) driver.get(url) + time.sleep(10) wait.until(EC.presence_of_all_elements_located) - html = driver.page_source.encode("utf-8") + html = driver.page_source.encode("utf-8").decode("utf-8") print(html) return html diff --git a/src/au/main.py b/src/au/main.py index 8d801ba..30e750f 100644 --- a/src/au/main.py +++ b/src/au/main.py @@ -30,21 +30,12 @@ lg.info("get driver") driver = driver.get_driver() - if os.path.isfile(COOKIES): - lg.info("load cookies") - try: - au.load_cookies(driver) - lg.info("load cookies success") - except Exception as e: - lg.warning('failed to load cookies. skipping. {0}'.format(e)) - else: - lg.warning("cookie file not found") - - if args.login_2fa: # login mode lg.info("au page login mode") au.login_2fa(driver) + lg.info("save cookies") + au.save_cookies(driver) sys.exit(0) au.login(driver) From cbe2beb9c7e6272b35c7c61d211c4c622eee4f9a Mon Sep 17 00:00:00 2001 From: azuki774s Date: Sun, 3 Sep 2023 20:02:13 +0900 Subject: [PATCH 4/4] fixup! wip not working --- deployment/compose.yml | 4 ++-- src/au/au.py | 9 ++++++--- src/au/main.py | 2 ++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/deployment/compose.yml b/deployment/compose.yml index 7702d9f..732a67a 100644 --- a/deployment/compose.yml +++ b/deployment/compose.yml @@ -15,8 +15,8 @@ services: - au-token.env volumes: - ./.cookies:/.cookies - # command: - # - "--login-2fa" + command: + - "--login-2fa" # remix: # image: bill-fetcher-remix diff --git a/src/au/au.py b/src/au/au.py index ba6f143..4113fbd 100644 --- a/src/au/au.py +++ b/src/au/au.py @@ -69,13 +69,13 @@ def login(driver): pass_field = driver.find_element( by=By.XPATH, - value="/html/body/div[3]/div/div[1]/div[2]/div/div/form/input[25]", + value="/html/body/div[2]/div/div[1]/div[2]/div/div/form/input[25]", ) pass_field.send_keys(os.getenv("pass")) login_button = driver.find_element( by=By.XPATH, - value="/html/body/div[3]/div/div[1]/div[2]/div/div/form/button[4]", + value="/html/body/div[2]/div/div[1]/div[2]/div/div/form/button[4]", ) login_button.click() @@ -112,7 +112,7 @@ def _login_2fa_proc(driver): by=By.XPATH, value="/html/body/div[2]/div/form/input[7]", ) - pass_field.send_keys(os.getenv("pass")) + pass_field.send_keys(secret_2fa_code) login_button = driver.find_element( by=By.XPATH, @@ -120,6 +120,9 @@ def _login_2fa_proc(driver): ) login_button.click() lg.info("2FA login information send") + time.sleep(5) + html = driver.page_source.encode("utf-8").decode("utf-8") + print(html) def get_from_url(driver, url): wait = WebDriverWait(driver=driver, timeout=30) diff --git a/src/au/main.py b/src/au/main.py index 30e750f..6da2b52 100644 --- a/src/au/main.py +++ b/src/au/main.py @@ -10,6 +10,7 @@ import sys import au import os +import time lg = logging.getLogger(__name__) lg.setLevel(logging.DEBUG) @@ -36,6 +37,7 @@ au.login_2fa(driver) lg.info("save cookies") au.save_cookies(driver) + time.sleep(3600) sys.exit(0) au.login(driver)