-
Notifications
You must be signed in to change notification settings - Fork 0
/
functions.py
58 lines (48 loc) · 1.56 KB
/
functions.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
"""LFS Patches Downloader functions"""
import os
import os.path
import urllib.request
import shutil
try:
import httpx as requests
except ImportError:
import requests
from bs4 import BeautifulSoup
files = []
def remove_directory(directory: str):
"""Remove directory if exists"""
if os.path.exists(directory):
shutil.rmtree(directory)
def download_recursive(url: str, out_dir: str, download_after=True):
"""Download recursive"""
if not out_dir.endswith("/"):
out_dir += "/"
if not url.endswith("/"):
url += "/"
data = requests.get(url, timeout=25)
if data.status_code != 200:
print(f"error when requesting {url}, skip and continue")
return
soup = BeautifulSoup(data.text, "html.parser")
for link in soup.find_all("a")[1:]:
href = link.get("href")
next_url: str = url+href
new_out_dir = out_dir+href
if next_url.endswith("/"):
download_recursive(next_url, new_out_dir, False)
else:
print(f"New file: {next_url}")
files.append([next_url, new_out_dir])
if download_after:
total_files = len(files)
print(f"Count of files: {total_files}")
for index, data in enumerate(files):
folder = os.path.dirname(data[1])
try:
os.listdir(folder)
except FileNotFoundError:
os.makedirs(folder)
print(
f"{index+1}/{total_files} | Downloading {data[0]} to {data[1]}"
)
urllib.request.urlretrieve(*data)