diff --git a/.github/workflows/image-push.yml b/.github/workflows/image-push.yml index 07efce4..e119ebf 100644 --- a/.github/workflows/image-push.yml +++ b/.github/workflows/image-push.yml @@ -6,57 +6,17 @@ on: - v* jobs: - build_and_push_remix: - runs-on: ubuntu-latest - env: - IMAGE_NAME: bill-fetcher-remix - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Set meta - id: meta - uses: docker/metadata-action@v3 - with: - # list of Docker images to use as base name for tags - images: | - ghcr.io/azuki774/bill-fetcher-remix - # generate Docker tags based on the following events/attributes - tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=semver,pattern=latest - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GH_ACCESS_TOKEN }} - - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: . - file: ./build/Dockerfile-remix - push: true - tags: ${{ steps.meta.outputs.tags }} - build_and_push_sbi: runs-on: ubuntu-latest env: IMAGE_NAME: bill-fetcher-sbi steps: - name: checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set meta id: meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v4 with: # list of Docker images to use as base name for tags images: | @@ -69,100 +29,20 @@ jobs: type=semver,pattern=latest - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GH_ACCESS_TOKEN }} - - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: . - file: ./build/Dockerfile-sbi - push: true - tags: ${{ steps.meta.outputs.tags }} - - build_and_push_tokyowater: - runs-on: ubuntu-latest - env: - IMAGE_NAME: bill-fetcher-tokyowater - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Set meta - id: meta - uses: docker/metadata-action@v3 - with: - # list of Docker images to use as base name for tags - images: | - ghcr.io/azuki774/bill-fetcher-tokyowater - # generate Docker tags based on the following events/attributes - tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=semver,pattern=latest - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GH_ACCESS_TOKEN }} - - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: . - file: ./build/Dockerfile-tokyowater - push: true - tags: ${{ steps.meta.outputs.tags }} - - build_and_push_nicigas: - runs-on: ubuntu-latest - env: - IMAGE_NAME: bill-fetcher-nicigas - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Set meta - id: meta - uses: docker/metadata-action@v3 - with: - # list of Docker images to use as base name for tags - images: | - ghcr.io/azuki774/bill-fetcher-nicigas - # generate Docker tags based on the following events/attributes - tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=semver,pattern=latest - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GH_ACCESS_TOKEN }} - name: Build and push - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: context: . - file: ./build/Dockerfile-nicigas + file: ./build/sbi/Dockerfile push: true tags: ${{ steps.meta.outputs.tags }} @@ -172,11 +52,11 @@ jobs: IMAGE_NAME: bill-fetcher-money-forward steps: - name: checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set meta id: meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v4 with: # list of Docker images to use as base name for tags images: | @@ -189,7 +69,7 @@ jobs: type=semver,pattern=latest - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to GitHub Container Registry uses: docker/login-action@v1 @@ -199,9 +79,9 @@ jobs: password: ${{ secrets.GH_ACCESS_TOKEN }} - name: Build and push - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: context: . - file: ./build/Dockerfile-money-forward + file: ./build/money-forward/Dockerfile push: true tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..5877267 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,13 @@ +name: Build test + +on: + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v4 + - name: Build images + run: make build diff --git a/Makefile b/Makefile index ae4c0aa..51c85ca 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,10 @@ -CONTAINER_NAME_AUELECT=bill-fetcher-auelect -CONTAINER_NAME_REMIX=bill-fetcher-remix CONTAINER_NAME_SBI=bill-fetcher-sbi -CONTAINER_NAME_TOKYOWATER=bill-fetcher-tokyowater -CONTAINER_NAME_NICIGAS=bill-fetcher-nicigas CONTAINER_NAME_MONEY_FORWARD=bill-fetcher-money-forward .PHONY: build start stop clean build: - docker build -t $(CONTAINER_NAME_AUELECT) -f build/Dockerfile-auelect . - docker build -t $(CONTAINER_NAME_REMIX) -f build/Dockerfile-remix . - docker build -t $(CONTAINER_NAME_SBI) -f build/Dockerfile-sbi . - docker build -t $(CONTAINER_NAME_TOKYOWATER) -f build/Dockerfile-tokyowater . - docker build -t $(CONTAINER_NAME_NICIGAS) -f build/Dockerfile-nicigas . - docker build -t $(CONTAINER_NAME_MONEY_FORWARD) -f build/Dockerfile-money-forward . + docker build -t $(CONTAINER_NAME_SBI) -f build/sbi/Dockerfile . + docker build -t $(CONTAINER_NAME_MONEY_FORWARD) -f build/money-forward/Dockerfile . start: docker compose -f deployment/compose.yml up -d @@ -24,9 +16,5 @@ debug: docker compose -f deployment/compose.yml up clean: - docker image rm $(CONTAINER_NAME_AUELECT) - docker image rm $(CONTAINER_NAME_REMIX) docker image rm $(CONTAINER_NAME_SBI) - docker image rm $(CONTAINER_NAME_TOKYOWATER) - docker image rm $(CONTAINER_NAME_NICIGAS) docker image rm $(CONTAINER_NAME_MONEY_FORWARD) diff --git a/README.md b/README.md index f1bbd78..b734abf 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,14 @@ ## money-forward - https://moneyforward.com に自動的にログインして、指定したURLのHTMLソースを取得して保存する。 -### Usage -- `deploymments/compose.yml` と `money-forward.yml` の要領で、コンテナ `bill-fetcher-money-forward` を起動させる。 +- 出力先は、コンテナ内の `/data/latest/＜各種ページ＞ - 取得したいリンクと、必要に応じて金融機関連携の「更新」ボタンの XPATH を環境変数で渡す。 +## sbi +- https://site1.sbisec.co.jp/ETGate/ に自動的にログインして、ポートフォリオの表ごとに保存する。 +- 出力先は、コンテナ内の `/data/YYYYMMDD/YYYYMMDD_x.csv` + - `x`: 連番 + ## データ取り込み例 - [mf-importer](https://github.com/azuki774/mf-importer) diff --git a/build/Dockerfile-auelect b/build/Dockerfile-auelect deleted file mode 100644 index f9c3649..0000000 --- a/build/Dockerfile-auelect +++ /dev/null @@ -1,5 +0,0 @@ -FROM python:3.11-bullseye -COPY requirements/ /tmp/ -RUN python -m pip install --upgrade pip && pip install -r /tmp/auelect_requirements.txt -COPY src/auelect/ /src/ -ENTRYPOINT ["python3", "-u", "/src/main.py"] diff --git a/build/Dockerfile-money-forward b/build/Dockerfile-money-forward deleted file mode 100644 index 40aeccd..0000000 --- a/build/Dockerfile-money-forward +++ /dev/null @@ -1,6 +0,0 @@ -FROM ghcr.io/azuki774/selenium-chrome:0.2.0 -COPY requirements/ /tmp/ -RUN pip install --upgrade pip && pip install -r /tmp/moneyforward_requirements.txt - -COPY src/moneyforward/ /src/ -ENTRYPOINT ["python3", "-u", "/src/main.py"] diff --git a/build/Dockerfile-nicigas b/build/Dockerfile-nicigas deleted file mode 100644 index b9444d2..0000000 --- a/build/Dockerfile-nicigas +++ /dev/null @@ -1,5 +0,0 @@ -FROM ghcr.io/azuki774/selenium-chrome:0.1.1 -LABEL description="bill-fetcher-nicigas" -RUN python -m pip install --upgrade pip -ADD src/nicigas/ /src/ -ENTRYPOINT ["python3", "-u", "/src/main.py"] diff --git a/build/Dockerfile-remix b/build/Dockerfile-remix deleted file mode 100644 index 32b5279..0000000 --- a/build/Dockerfile-remix +++ /dev/null @@ -1,5 +0,0 @@ -FROM ghcr.io/azuki774/selenium-chrome:latest -LABEL description="bill-fetcher-remix" -RUN python -m pip install --upgrade pip -ADD src/remix/ /src/ -ENTRYPOINT ["python3", "-u", "/src/main.py"] diff --git a/build/Dockerfile-sbi b/build/Dockerfile-sbi deleted file mode 100644 index e30c5de..0000000 --- a/build/Dockerfile-sbi +++ /dev/null @@ -1,6 +0,0 @@ -FROM ghcr.io/azuki774/selenium-chrome:latest - -COPY requirements/ /tmp/ -RUN pip install --upgrade pip && pip install -r /tmp/sbi_requirements.txt -COPY src/sbi/ /src/ -ENTRYPOINT ["python3", "-u", "/src/main.py"] diff --git a/build/Dockerfile-tokyowater b/build/Dockerfile-tokyowater deleted file mode 100644 index 183f968..0000000 --- a/build/Dockerfile-tokyowater +++ /dev/null @@ -1,3 +0,0 @@ -FROM ghcr.io/azuki774/selenium-chrome:latest -COPY src/tokyowater/ /src/ -ENTRYPOINT ["python3", "-u", "/src/main.py"] diff --git a/build/money-forward/Dockerfile b/build/money-forward/Dockerfile new file mode 100644 index 0000000..c32bf3a --- /dev/null +++ b/build/money-forward/Dockerfile @@ -0,0 +1,18 @@ +FROM ghcr.io/azuki774/selenium-chrome:0.2.0 + +# Required Packages +RUN apt-get update && \ + apt-get install -y curl unzip && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# AWS Setup +RUN curl -o /var/tmp/awscli.zip https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip && \ + unzip -d /usr/local/bin/ /var/tmp/awscli.zip + +COPY requirements/ /tmp/ +RUN pip install --upgrade pip && pip install -r /tmp/moneyforward_requirements.txt +COPY --chmod=755 build/money-forward/main.sh /src/main.sh +COPY src/moneyforward/ /src/ + +ENTRYPOINT ["/src/main.sh"] diff --git a/build/money-forward/main.sh b/build/money-forward/main.sh new file mode 100644 index 0000000..0b3958a --- /dev/null +++ b/build/money-forward/main.sh @@ -0,0 +1,57 @@ +#!/bin/bash +set -e +YYYYMM=`date '+%Y%m'` +YYYYMMDD=`date '+%Y%m%d'` + +# BUCKET_URL # from env (ex: "https://s3.ap-northeast-1.wasabisys.com") +# BUCKET_NAME # from env (ex: hoge-system-stg-bucket) +# BUCKET_DIR # from env (ex: fetcher/money-forward) +# AWS_REGION # from env (ex: ap-northeast-1) +# AWS_ACCESS_KEY_ID # from env +# AWS_SECRET_ACCESS_KEY # from env + +AWS_BIN="/usr/local/bin/aws/dist/aws" +DATA_DIR="/data" + +REMOTE_DIR_DAILY="${BUCKET_DIR}/${YYYYMMDD}" +REMOTE_DIR_LATEST="${BUCKET_DIR}/latest" + +function fetch () { + echo "fetcher start" + python3 -u /src/main.py + echo "fetcher complete" +} + +function create_s3_credentials { + echo "s3 credentials create start" + mkdir -p ~/.aws/ + + echo "[default]" >> ~/.aws/config + echo "region = ${AWS_REGION}" >> ~/.aws/config + + echo "[default]" >> ~/.aws/credentials + echo "aws_access_key_id = ${AWS_ACCESS_KEY_ID}" >> ~/.aws/credentials + echo "aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY}" >> ~/.aws/credentials + + chmod 400 ~/.aws/config + chmod 400 ~/.aws/credentials + ls -la ~/.aws/ + echo "s3 credentials create complete" +} + +function s3_upload () { + echo "s3 upload start" + ${AWS_BIN} s3 cp ${DATA_DIR}/${YYYYMMDD}/ "s3://${BUCKET_NAME}/${REMOTE_DIR_DAILY}" --recursive --endpoint-url="${BUCKET_URL}" + mv ${DATA_DIR}/${YYYYMMDD} ${DATA_DIR}/latest + ${AWS_BIN} s3 cp ${DATA_DIR}/latest/ "s3://${BUCKET_NAME}/${REMOTE_DIR_LATEST}" --recursive --endpoint-url="${BUCKET_URL}" + echo "s3 upload complete" +} + +fetch + +if [ -z $BUCKET_NAME ]; then + exit 0 +fi + +create_s3_credentials +s3_upload diff --git a/build/sbi/Dockerfile b/build/sbi/Dockerfile new file mode 100644 index 0000000..ded5532 --- /dev/null +++ b/build/sbi/Dockerfile @@ -0,0 +1,18 @@ +FROM ghcr.io/azuki774/selenium-chrome:0.2.0 + +# Required Packages +RUN apt-get update && \ + apt-get install -y curl unzip && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# AWS Setup +RUN curl -o /var/tmp/awscli.zip https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip && \ + unzip -d /usr/local/bin/ /var/tmp/awscli.zip + +COPY requirements/ /tmp/ +RUN pip install --upgrade pip && pip install -r /tmp/sbi_requirements.txt +COPY --chmod=755 build/sbi/main.sh /src/main.sh +COPY src/sbi/ /src/ + +ENTRYPOINT ["/src/main.sh"] diff --git a/build/sbi/main.sh b/build/sbi/main.sh new file mode 100644 index 0000000..c623186 --- /dev/null +++ b/build/sbi/main.sh @@ -0,0 +1,54 @@ +#!/bin/bash +set -e +YYYYMM=`date '+%Y%m'` +YYYYMMDD=`date '+%Y%m%d'` + +# BUCKET_URL # from env (ex: "https://s3.ap-northeast-1.wasabisys.com") +# BUCKET_NAME # from env (ex: hoge-system-stg-bucket) +# BUCKET_DIR # from env (ex: fetcher/sbi) +# AWS_REGION # from env (ex: ap-northeast-1) +# AWS_ACCESS_KEY_ID # from env +# AWS_SECRET_ACCESS_KEY # from env + +AWS_BIN="/usr/local/bin/aws/dist/aws" +DATA_DIR="/data" + +REMOTE_DIR="${BUCKET_DIR}/${YYYYMM}" + +function fetch () { + echo "fetcher start" + python3 -u /src/main.py + echo "fetcher complete" +} + +function create_s3_credentials { + echo "s3 credentials create start" + mkdir -p ~/.aws/ + + echo "[default]" >> ~/.aws/config + echo "region = ${AWS_REGION}" >> ~/.aws/config + + echo "[default]" >> ~/.aws/credentials + echo "aws_access_key_id = ${AWS_ACCESS_KEY_ID}" >> ~/.aws/credentials + echo "aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY}" >> ~/.aws/credentials + + chmod 400 ~/.aws/config + chmod 400 ~/.aws/credentials + ls -la ~/.aws/ + echo "s3 credentials create complete" +} + +function s3_upload () { + echo "s3 upload start" + ${AWS_BIN} s3 cp ${DATA_DIR}/${YYYYMM}/ "s3://${BUCKET_NAME}/${REMOTE_DIR}" --recursive --endpoint-url="${BUCKET_URL}" + echo "s3 upload complete" +} + +fetch + +if [ -z $BUCKET_NAME ]; then + exit 0 +fi + +create_s3_credentials +s3_upload diff --git a/deployment/compose.yml b/deployment/compose.yml index 0e72bf5..0644539 100644 --- a/deployment/compose.yml +++ b/deployment/compose.yml @@ -9,10 +9,18 @@ services: # volumes: # - /dev/shm:/dev/shm - money-forward: - image: bill-fetcher-money-forward - container_name: bill-fetcher-money-forward + # money-forward: + # image: bill-fetcher-money-forward + # container_name: bill-fetcher-money-forward + # env_file: + # - money-forward.env + # volumes: + # - ./:/data/ + + fetcher-sbi: + image: bill-fetcher-sbi + container_name: bill-fetcher-sbi env_file: - - money-forward.env + - sbi-token.env volumes: - ./:/data/ diff --git a/deployment/sbi-token.env.sample b/deployment/sbi-token.env.sample new file mode 100644 index 0000000..dd4e362 --- /dev/null +++ b/deployment/sbi-token.env.sample @@ -0,0 +1,8 @@ +sbi_user=xxxxxxxxxx +sbi_pass="yyyyyyyyy" +BUCKET_URL="https://s3.ap-northeast-1.wasabisys.com" +BUCKET_NAME="hoge-system-stg-bucket" +BUCKET_DIR="fetcher/sbi" +AWS_REGION="ap-northeast-1" +AWS_ACCESS_KEY_ID="xxxxxxxxxxxxx" +AWS_SECRET_ACCESS_KEY="yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy" diff --git a/docs/auelect.png b/docs/auelect.png deleted file mode 100644 index 2075c7c..0000000 Binary files a/docs/auelect.png and /dev/null differ diff --git a/requirements/auelect_requirements.txt b/requirements/auelect_requirements.txt deleted file mode 100644 index 7ffeec0..0000000 --- a/requirements/auelect_requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -beautifulsoup4==4.12.0 -python-json-logger==2.0.7 diff --git a/src/auelect/load.py b/src/auelect/load.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/auelect/main.py b/src/auelect/main.py deleted file mode 100644 index fee369d..0000000 --- a/src/auelect/main.py +++ /dev/null @@ -1,20 +0,0 @@ -from pythonjsonlogger import jsonlogger -import logging -import os -import sys - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) -h = logging.StreamHandler() -h.setLevel(logging.DEBUG) -json_fmt = jsonlogger.JsonFormatter(fmt='%(asctime)s %(levelname)s %(filename)s %(lineno)s %(message)s', json_ensure_ascii=False) -h.setFormatter(json_fmt) -logger.addHandler(h) - -if __name__ == "__main__": - logger.info("auelect start") - - logger.info("writer start") - - logger.info("writer end") - logger.info("auelect end") diff --git a/src/auelect/write.py b/src/auelect/write.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/nicigas/main.py b/src/nicigas/main.py deleted file mode 100644 index a002b96..0000000 --- a/src/nicigas/main.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- -import csv -import time -import os -import sys -import driver -from venv import create -from selenium import webdriver -from selenium.webdriver.common.keys import Keys -from selenium.webdriver.common.by import By -from selenium.webdriver.chrome.options import Options -from selenium.webdriver.chrome.service import Service -from bs4 import BeautifulSoup -import nicigas -import writer - -def get_driver(): - options = webdriver.ChromeOptions() - options.add_argument("--headless") - options.add_argument("--no-sandbox") - options.add_argument("--disable-gpu") - options.add_argument("--disable-dev-shm-usage") - UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36' - options.add_argument('--user-agent=' + UA) - chrome_service = Service(executable_path="/usr/bin/chromedriver") - driver = webdriver.Chrome(service=chrome_service, options=options) - return driver - -if __name__ == "__main__": - print("fetcher start") - args = sys.argv - - options = webdriver.ChromeOptions() - - driver = get_driver() - driver.implicitly_wait(10) - - print("Get driver") - - nicigas.login(driver) - data = nicigas.fetch_invoice(driver) - print("fetcher complete") - - print("writer start") - writer.csvwrite_invoice(data) - print("writer end") - print("the program end") - diff --git a/src/nicigas/nicigas.py b/src/nicigas/nicigas.py deleted file mode 100644 index a7f3102..0000000 --- a/src/nicigas/nicigas.py +++ /dev/null @@ -1,157 +0,0 @@ -from curses import raw -# -*- coding: utf-8 -*- -import csv -import re -import time -import json -import datetime -import os -import driver -from venv import create -from selenium import webdriver -from selenium.webdriver.common.keys import Keys -from selenium.webdriver.common.by import By -from selenium.webdriver.chrome.options import Options -from selenium.webdriver.chrome.service import Service -from selenium.webdriver.support.ui import Select -from bs4 import BeautifulSoup -from datetime import datetime, timedelta - -def login(driver): - driver.get("https://my.nichigas.co.jp/entry") - html = driver.page_source.encode('utf-8') - - # 事業者選択 - selector = driver.find_element(by=By.XPATH, value='//*[@id="root"]/div/div/div/div/div/div/div[4]/div/select') - select = Select(selector) - select.select_by_value('00005') - - login_button = driver.find_element(by=By.XPATH, value='//*[@id="root"]/div/div/div/div/div/div/div[5]/div') - login_button.click() - - html = driver.page_source.encode('utf-8') - soup = BeautifulSoup(html, 'html.parser') - - # ログインページ操作 - login_id = driver.find_element(by=By.XPATH, value='//*[@id="root"]/div/div/div/div/div/div/div/div[2]/input[1]') - login_id.send_keys(os.getenv("nicigas_id")) - - password_form = driver.find_element(by=By.XPATH, value='//*[@id="root"]/div/div/div/div/div/div/div/div[2]/input[2]') - password_form.send_keys(os.getenv("nicigas_pass")) - - login_button = driver.find_element(by=By.XPATH, value='//*[@id="root"]/div/div/div/div/div/div/div/div[2]/div[4]') - login_button.click() - time.sleep(5) - - html = driver.page_source.encode('utf-8') - soup = BeautifulSoup(html, 'html.parser') - return - -def fetch_invoice(driver): - driver.get("https://my.nichigas.co.jp/dashboard/gas") - time.sleep(5) - html = driver.page_source.encode('utf-8') - soup = BeautifulSoup(html, 'html.parser') - - # soup = BeautifulSoup(open('/src/testtext.htm'), 'html.parser') - label_fields = soup.find_all(class_="card-title") - labels = [] - - price_fields = soup.find_all(class_="list-item--text idx-i0") - prices = [] - - use_amount_fields = soup.find_all(class_="list-item--text idx-i1") - amounts = [] - - billing_done_field = soup.find_all(class_="list-item--text idx-i2") - billing_done = [] - - # 20xx年yy月分総合料金詳細 取得 - i = 0 - for l in label_fields: - labels.append(trim_text(l.get_text())) - i = i + 1 - if i == 12: # 13レコード目以降は重複しているので skip - break - - # 料金 取得 - i = 0 - for l in price_fields: - prices.append(trim_text(l.get_text())) - i = i + 1 - if i == 12: # 13レコード目以降は重複しているので skip - break - - # 使用量 取得 - i = 0 - for l in use_amount_fields: - amounts.append(trim_text(l.get_text())) - i = i + 1 - if i == 12: # 13レコード目以降は重複しているので skip - break - - - # 支払い状態 取得 - i = 0 - for l in billing_done_field: - billing_done.append(trim_text(l.get_text())) - i = i + 1 - if i == 12: # 13レコード目以降は重複しているので skip - break - - records = [] - for i in range(12): - records.append([labels[i], prices[i], amounts[i], billing_done[i]]) - - print(records) - return records - - -def trim_text(text): - text = text.replace("\n", "") - text = text.replace(" ", "") - text = text.replace('\u3000', '') - return text - - -def testfetch_invoice(): - soup = BeautifulSoup(open('/src/testtext.htm'), 'html.parser') - label_fields = soup.find_all(class_="card-title") - labels = [] - - price_fields = soup.find_all(class_="list-item--text idx-i0") - prices = [] - - billing_done_field = soup.find_all(class_="list-item--text idx-i2") - billing_done = [] - - # 20xx年yy月分総合料金詳細 取得 - i = 0 - for l in label_fields: - labels.append(trim_text(l.get_text())) - i = i + 1 - if i == 12: # 13レコード目以降は重複しているので skip - break - - # 料金 取得 - i = 0 - for l in price_fields: - prices.append(trim_text(l.get_text())) - i = i + 1 - if i == 12: # 13レコード目以降は重複しているので skip - break - - # 支払い状態 取得 - i = 0 - for l in billing_done_field: - billing_done.append(trim_text(l.get_text())) - i = i + 1 - if i == 12: # 13レコード目以降は重複しているので skip - break - - records = [] - for i in range(12): - records.append([labels[i], prices[i], billing_done[i]]) - - print(records) - return records diff --git a/src/nicigas/writer.py b/src/nicigas/writer.py deleted file mode 100644 index fcc821c..0000000 --- a/src/nicigas/writer.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- -import datetime as dt -import os -SAVE_DIR = '/data/' - -import csv -def csvwrite_invoice(data): - today = dt.date.today() #出力：datetime.date(2020, 3, 22) - yyyymm = '{0:%Y%m}'.format(today) # 202003 - yyyymmdd = '{0:%Y%m%d}'.format(today) # 20200322 - os.makedirs(SAVE_DIR + yyyymm, exist_ok=True) - - csv_file = open(SAVE_DIR + yyyymm + '/' + yyyymmdd + '.csv', 'wt', newline = '', encoding = 'utf-8') - csv_write = csv.writer(csv_file) - for d in data: - csv_write.writerow(d) diff --git a/src/remix/main.py b/src/remix/main.py deleted file mode 100644 index 25e84b0..0000000 --- a/src/remix/main.py +++ /dev/null @@ -1,40 +0,0 @@ -# -*- coding: utf-8 -*- -import csv -import time -import os -import sys -import driver -from venv import create -from selenium import webdriver -from selenium.webdriver.common.keys import Keys -from selenium.webdriver.common.by import By -from selenium.webdriver.chrome.options import Options -from selenium.webdriver.chrome.service import Service -from bs4 import BeautifulSoup -import remix -import writer - -if __name__ == "__main__": - print("fetcher start") - args = sys.argv - - remix_fetch_data = [] - driver = driver.get_driver() - driver.implicitly_wait(10) - - print("Get driver") - - remix.login(driver) - if len(args) >= 2 and args[1] == 'invoice': - data = remix.fetch_invoice(driver) - print("fetcher complete") - print("writer start") - writer.csvwrite_invoice(data) - else: - data = remix.fetch_consume_month(driver) - print("fetcher complete") - print("writer start") - writer.csvwrite(data) - - print("writer end") - print("the program end") diff --git a/src/remix/remix.py b/src/remix/remix.py deleted file mode 100644 index ade089e..0000000 --- a/src/remix/remix.py +++ /dev/null @@ -1,95 +0,0 @@ -from curses import raw -# -*- coding: utf-8 -*- -import csv -import re -import time -import json -import datetime -import os -import driver -from venv import create -from selenium import webdriver -from selenium.webdriver.common.keys import Keys -from selenium.webdriver.common.by import By -from selenium.webdriver.chrome.options import Options -from selenium.webdriver.chrome.service import Service -from bs4 import BeautifulSoup -from datetime import datetime, timedelta -import datetime - -def login(driver): - driver.get("https://portal.remix-denki.com/index.php") - - login_id = driver.find_element(by=By.XPATH, value='/html/body/form/div/div/div/div/div[2]/div[1]/input') - - login_id.send_keys(os.getenv("remix_id")) - - password_form = driver.find_element(by=By.XPATH, value='/html/body/form/div/div/div/div/div[2]/div[2]/input') - password_form.send_keys(os.getenv("remix_pass")) - - login_button = driver.find_element(by=By.XPATH, value='/html/body/form/div/div/div/div/div[2]/a') - login_button.click() - driver.implicitly_wait(10) - - html = driver.page_source.encode('utf-8') - soup = BeautifulSoup(html, 'html.parser') - return - -def remove_return(text): - return text.replace("\n", "") - -def trim_text(text): - text = text.replace("\n", "") - text = text.replace('\u3000', '') - return text - -def fetch_consume_month(driver): - # [取得年月日, 使用量合計(kWh), 昼時間使用量(kWh), 夜時間使用量(kWh)] - driver.get("https://portal.remix-denki.com/consumption.php") - detail_button = driver.find_element( - by=By.XPATH, value='//*[@id="page-wrapper"]/div[2]/div/div/div[2]/div/table/tbody/tr/td[5]/a' - ) - detail_button.click() - - day_button = driver.find_element( - by=By.XPATH, value='//*[@id="page-wrapper"]/div[3]/div/ul/li[2]/a' - ) - day_button.click() - html = driver.page_source.encode('utf-8') - soup = BeautifulSoup(html, 'html.parser') - - result_data = [] - # セレクタ(タグ：table、クラス：test) - table = soup.find(id="graph_list") - - trs = table.findAll("tr") - - result_data = [] - for tr in trs: - row_data = [] - for cell in tr.findAll(['td', 'th']): - row_data.append(remove_return(cell.get_text())) - result_data.append(row_data) - - print(result_data) - return result_data - -def fetch_invoice(driver): - driver.get("https://portal.remix-denki.com/invoice.php") - - html = driver.page_source.encode('utf-8') - soup = BeautifulSoup(html, 'html.parser') - - # table = soup.find(class="table responsibveTbl") - - trs = soup.findAll("tr") - - result_data = [] - for tr in trs: - row_data = [] - for cell in tr.findAll(['td', 'th']): - row_data.append(remove_return(cell.get_text())) - result_data.append(row_data) - - print(result_data) - return result_data diff --git a/src/remix/writer.py b/src/remix/writer.py deleted file mode 100644 index 92835a4..0000000 --- a/src/remix/writer.py +++ /dev/null @@ -1,31 +0,0 @@ -# -*- coding: utf-8 -*- -import datetime as dt -import os -SAVE_DIR = '/data/' - -import csv -def csvwrite(data): - today = dt.date.today() #出力：datetime.date(2020, 3, 22) - yyyymm = '{0:%Y%m}'.format(today) # 202003 - yyyymmdd = '{0:%Y%m%d}'.format(today) # 20200322 - os.makedirs(SAVE_DIR + yyyymm, exist_ok=True) - - csv_file = open(SAVE_DIR + yyyymm + '/' + yyyymmdd + '.csv', 'wt', newline = '', encoding = 'utf-8') - csv_write = csv.writer(csv_file) - for d in data: - csv_write.writerow(d) - -def csvwrite_invoice(data): - today = dt.date.today() #出力：datetime.date(2020, 3, 22) - yyyymm = '{0:%Y%m}'.format(today) # 202003 - yyyymmdd = '{0:%Y%m%d}'.format(today) # 20200322 - os.makedirs(SAVE_DIR + yyyymm, exist_ok=True) - - csv_file = open(SAVE_DIR + yyyymm + '/' + yyyymmdd + '_inv.csv', 'wt', newline = '', encoding = 'utf-8') - csv_write = csv.writer(csv_file) - for d in data: - csv_write.writerow(d) - -# if __name__ == "__main__": -# data = [['取得年月日', '使用量合計(kWh)', '昼時間使用量(kWh)', '夜時間使用量(kWh)'], ['2022/12/31', '-', '-', '-'], ['2022/12/30', '-', '-', '-'], ['2022/12/29', '-', '-', '-'], ['2022/12/28', '-', '-', '-'], ['2022/12/27', '-', '-', '-'], ['2022/12/26', '-', '-', '-'], ['2022/12/25', '-', '-', '-'], ['2022/12/24', '-', '-', '-'], ['2022/12/23', '-', '-', '-'], ['2022/12/22', '-', '-', '-'], ['2022/12/21', '-', '-', '-'], ['2022/12/20', '-', '-', '-'], ['2022/12/19', '-', '-', '-'], ['2022/12/18', '-', '-', '-'], ['2022/12/17', '-', '-', '-'], ['2022/12/16', '-', '-', '-'], ['2022/12/15', '-', '-', '-'], ['2022/12/14', '-', '-', '-'], ['2022/12/13', '-', '-', '-'], ['2022/12/12', '-', '-', '-'], ['2022/12/11', '-', '-', '-'], ['2022/12/10', '-', '-', '-'], ['2022/12/09', '-', '-', '-'], ['2022/12/08', '-', '-', '-'], ['2022/12/07', '-', '-', '-'], ['2022/12/06', '-', '-', '-'], ['2022/12/05', '6', '3', '2'], ['2022/12/04', '8', '6', '2'], ['2022/12/03', '6', '3', '3'], ['2022/12/02', '6', '2', '4'], ['2022/12/01', '6', '4', '2']] -# csvwrite(data) diff --git a/src/tokyowater/main.py b/src/tokyowater/main.py deleted file mode 100644 index 3f1a8e6..0000000 --- a/src/tokyowater/main.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- -import csv -import time -import os -import sys -import driver -from venv import create -from selenium import webdriver -from selenium.webdriver.common.keys import Keys -from selenium.webdriver.common.by import By -from selenium.webdriver.chrome.options import Options -from selenium.webdriver.chrome.service import Service -from bs4 import BeautifulSoup -import water -import writer - -if __name__ == "__main__": - print("fetcher start") - args = sys.argv - - water_fetch_data = [] - driver = driver.get_driver() - driver.implicitly_wait(10) - - print("Get driver") - - water.login(driver) - data = water.extract_bill(driver) - print("fetcher complete") - print("writer start") - writer.csvwrite(data) - - print("writer end") - print("the program end") diff --git a/src/tokyowater/water.py b/src/tokyowater/water.py deleted file mode 100644 index d6d296f..0000000 --- a/src/tokyowater/water.py +++ /dev/null @@ -1,69 +0,0 @@ -from curses import raw -# -*- coding: utf-8 -*- -import time -import os -from venv import create -from selenium import webdriver -from selenium.webdriver.common.keys import Keys -from selenium.webdriver.common.by import By -from selenium.webdriver.chrome.options import Options -from selenium.webdriver.chrome.service import Service -from bs4 import BeautifulSoup -from datetime import datetime, timedelta -import datetime - -def login(driver): - driver.get("https://www.suidoapp.waterworks.metro.tokyo.lg.jp/#/login") - - login_id = driver.find_element(by=By.XPATH, value='//*[@id="login-mail"]') - - login_id.send_keys(os.getenv("water_id")) - print(os.getenv("water_id")) - password_form = driver.find_element(by=By.XPATH, value='//*[@id="login-pass"]') - password_form.send_keys(os.getenv("water_pass")) - - login_button = driver.find_element(by=By.XPATH, value='//*[@id="Content"]/div/div/div/div[1]/div/div[2]/form/button') - login_button.click() - - time.sleep(5) - html = driver.page_source.encode('utf-8') - soup = BeautifulSoup(html, 'html.parser') - - return - -def extract_bill(driver): - driver.get("https://www.suidoapp.waterworks.metro.tokyo.lg.jp/#/portal") - html = driver.page_source.encode('utf-8') - soup = BeautifulSoup(html, 'html.parser') - time.sleep(5) - - # 請求期間 - billing_date_field = driver.find_element(by=By.XPATH, value='//*[@id="Content"]/div/div/div/div[2]/div[1]/div[1]/div/p') - print(billing_date_field.get_attribute("textContent")) - # 使用量 - value_field = driver.find_element(by=By.XPATH, value='//*[@id="Content"]/div/div/div/div[2]/div[1]/div[1]/div/div[1]/p[2]/span[1]') - print(value_field.get_attribute("textContent")) - # 利用期間 - usage_term_field = driver.find_element(by=By.XPATH, value='//*[@id="Content"]/div/div/div/div[2]/div[1]/div[1]/div/div[2]/div[1]/p[2]/span') - print(usage_term_field.get_attribute("textContent")) - # 利用量 - usage_amount_field = driver.find_element(by=By.XPATH, value='//*[@id="Content"]/div/div/div/div[2]/div[1]/div[1]/div/div[2]/div[2]/p[2]/span[1]') - print(usage_amount_field.get_attribute("textContent")) - # 内訳（水道） - detail_waterprice_field = driver.find_element(by=By.XPATH, value='//*[@id="Content"]/div/div/div/div[2]/div[1]/div[1]/div/div[2]/div[3]/p[2]/span[1]') - print(detail_waterprice_field.get_attribute("textContent")) - # 内訳（下水道） - detail_sewerprice_field = driver.find_element(by=By.XPATH, value='//*[@id="Content"]/div/div/div/div[2]/div[1]/div[1]/div/div[2]/div[4]/p[2]/span[1]') - print(detail_sewerprice_field.get_attribute("textContent")) - # 現状では1ヶ月分 - ret = [[ - billing_date_field.get_attribute("textContent"), - value_field.get_attribute("textContent"), - usage_term_field.get_attribute("textContent"), - usage_amount_field.get_attribute("textContent"), - detail_waterprice_field.get_attribute("textContent"), - detail_sewerprice_field.get_attribute("textContent") - ]] - print(ret) - - return ret diff --git a/src/tokyowater/writer.py b/src/tokyowater/writer.py deleted file mode 100644 index aff234a..0000000 --- a/src/tokyowater/writer.py +++ /dev/null @@ -1,16 +0,0 @@ -# -*- coding: utf-8 -*- -import datetime as dt -import os -SAVE_DIR = '/data/' - -import csv -def csvwrite(data): - today = dt.date.today() #出力：datetime.date(2020, 3, 22) - yyyymm = '{0:%Y%m}'.format(today) # 202003 - yyyymmdd = '{0:%Y%m%d}'.format(today) # 20200322 - os.makedirs(SAVE_DIR + yyyymm, exist_ok=True) - - csv_file = open(SAVE_DIR + yyyymm + '/' + yyyymmdd + '.csv', 'wt', newline = '', encoding = 'utf-8') - csv_write = csv.writer(csv_file) - for d in data: - csv_write.writerow(d) diff --git a/test/auremix.html b/test/auremix.html deleted file mode 100644 index dceb99b..0000000 --- a/test/auremix.html +++ /dev/null @@ -1,23 +0,0 @@ -

- 1か月の電気料金の推移 -

日付 使用量 - 電気料金 -
- 11 (土) 8.50 kWh - 111 円
- 12 (日) 8.20 kWh - 111 円
- 13 (月) 5.80 kWh - 111 円
- 14 (火) 4.80 kWh - 111 円
- 15 (水) 5.80 kWh - 111 円
- 16 (木) 6.40 kWh - 222 円
- 17 (金) 6.20 kWh - 222 円
- 18 (土) 5.80 kWh - 222 円
- 19 (日) 8.10 kWh - 222 円
- 20 (月) 5.00 kWh - 222 円