Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add term as a folder. #56

Open
wants to merge 60 commits into
base: master
Choose a base branch
from
Open
Changes from all commits
Commits
Show all changes
60 commits
Select commit Hold shift + click to select a range
86163dd
New line for failure message
wznmickey Apr 25, 2021
c00576c
allow retry in clientGetJson
wznmickey Aug 30, 2022
610e3cb
JI changes canvas link.
wznmickey Aug 30, 2022
5235712
fix: format & remove unused var
wznmickey Aug 31, 2022
1a4c065
follow 2.0.6
wznmickey Aug 31, 2022
92ab973
follow 2.0.9
wznmickey Sep 1, 2022
5c9da3e
change from umjicanvas.com to jicanvas.com
wznmickey Sep 1, 2022
bd45233
allow retry in function json
wznmickey Sep 1, 2022
fd3fc53
Merge branch 'changeDefaultCanvasLink' into allowRetry
wznmickey Sep 1, 2022
41ab286
reformat
wznmickey Sep 1, 2022
afe8f7e
change from umjicanvas.com to jicanvas.com
wznmickey Sep 1, 2022
20e2516
reformat
wznmickey Sep 1, 2022
d597d6f
give detail in debug mode
wznmickey Sep 8, 2022
38cc139
Merge remote-tracking branch 'wznmickey_self/master' into HEAD
wznmickey Sep 8, 2022
a66eb0c
remove useless ()
wznmickey Sep 8, 2022
30420f7
remove useless distutils.log.debug
wznmickey Sep 8, 2022
61e7c79
format
wznmickey Sep 8, 2022
f820fde
Merge branch 'BoYanZh:master' into master
wznmickey Sep 9, 2022
033ed79
allow remove old config
wznmickey Sep 9, 2022
b9e41f2
add hint message
wznmickey Sep 15, 2022
7330c96
rewrite logic
wznmickey Sep 15, 2022
4295b07
rewrite logic
wznmickey Sep 15, 2022
e3e13f5
check file size in downloading
wznmickey Sep 15, 2022
7e236e0
check file size in downloading
wznmickey Sep 15, 2022
ff5a2b0
change hint
wznmickey Sep 17, 2022
1806d72
use temp file in downloading
wznmickey Sep 17, 2022
15d3cd8
use Black to format
wznmickey Sep 17, 2022
9364a8d
feat: allow filter files by type
wznmickey May 31, 2023
4a1598d
Merge branch 'BoYanZh:master' into master
wznmickey May 31, 2023
2b765e9
Merge remote-tracking branch 'my/master'
wznmickey May 31, 2023
71be38a
feat: allow filter files by type
wznmickey Jun 1, 2023
a829992
fix: remove unused var
wznmickey Jun 1, 2023
fa3575e
refactor: checkAllowDownload
wznmickey Jun 6, 2023
b465279
fix: allow Old config
wznmickey Jun 6, 2023
ceb91d9
fix: avoid Redefining built-in 'type'
wznmickey Jun 6, 2023
6dad3ae
change back version
wznmickey Jun 30, 2023
30c4350
Merge branch 'BoYanZh:master' into master
wznmickey Jul 2, 2023
28cd9d4
Update .canvassyncer_.json
wznmickey Jul 2, 2023
b7d9b25
Delete .canvassyncer_.json
wznmickey Jul 2, 2023
bc2e4ec
fix:allow retry when failed to get folder
wznmickey Aug 1, 2023
ec168b7
change back version
wznmickey Jun 30, 2023
15a0c01
fix:allow retry when failed to get folder
wznmickey Aug 1, 2023
af3f52f
fix:update version
wznmickey Aug 1, 2023
c01e569
Merge remote-tracking branch 'my/master' into majarfeatFilterTypes
wznmickey Aug 1, 2023
e526880
Update README.md
wznmickey Sep 7, 2023
bd53c4a
Merge pull request #2 from wznmickey/patch-2
wznmickey Sep 7, 2023
a117fb3
feat. use term folder
wznmickey Sep 8, 2023
4818315
Merge remote-tracking branch 'my/featFilterTypes' into dev
wznmickey Sep 8, 2023
e48aaac
Merge remote-tracking branch 'my/master' into dev
wznmickey Sep 8, 2023
ed1194b
Merge remote-tracking branch 'my/updateNotcompleteFile' into dev
wznmickey Sep 8, 2023
99f5a91
Merge branch 'my/allowRetry' into dev
wznmickey Sep 8, 2023
2c0f7d4
Merge remote-tracking branch 'my/patch-1' into dev
wznmickey Sep 8, 2023
aaae319
merge
wznmickey Sep 8, 2023
76dbb75
Revert "Merge branch 'my/allowRetry' into dev"
wznmickey Sep 8, 2023
2446896
Merge remote-tracking branch 'my/allowReDownload' into dev
wznmickey Sep 8, 2023
a7b2745
Revert "Update README.md"
wznmickey Sep 8, 2023
ad2e208
Revert "fix:allow retry when failed to get folder"
wznmickey Sep 8, 2023
3b73aac
revert commit
wznmickey Sep 8, 2023
bde0fac
Revert "check file size in downloading"
wznmickey Sep 8, 2023
7b00a2d
allow paging
wznmickey Sep 8, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 45 additions & 8 deletions canvassyncer/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,23 @@ async def head(self, *args, **kwargs):
async def aclose(self):
await self.client.aclose()

async def getCourseList(self, *args, **kwargs):
async with self.sem:
resp = await self.client.get(*args, **kwargs)
res = resp.json()
courseMapTerm = {}
termMapDisplayname = {}
for i in res:
try:
courseId = i["id"]
termId = i["enrollment_term_id"]
termName = i["term"]["name"]
courseMapTerm[courseId] = termId
termMapDisplayname[termId] = termName
except Exception:
print("get data error in" + str(i))
return (courseMapTerm, termMapDisplayname)


class CanvasSyncer:
def __init__(self, config):
Expand All @@ -109,9 +126,25 @@ def __init__(self, config):
self.laterInfo = []
self.skipfiles = []
self.totalFileCount = 0
self.__courseMapTerm = {}
self.__termMapDisplayname = {}
if not os.path.exists(self.downloadDir):
os.mkdir(self.downloadDir)

async def getCourseList(self):
courseMapTerm = {}
termMapDisplayname = {}
i = 0
while True:
i += 1
res = await self.client.getCourseList(
f"{self.baseUrl}/courses?include[]=term&page={i}"
)
courseMapTerm.update(res[0])
termMapDisplayname.update(res[1])
if res == ({}, {}):
return (courseMapTerm, termMapDisplayname)

async def aclose(self):
await self.client.aclose()

Expand All @@ -133,12 +166,14 @@ async def dictFromPages(self, helperFunc, *args, **kwargs):
def prepareLocalFiles(self, courseID, folders):
localFiles = []
for folder in folders.values():
path = os.path.join(
self.downloadDir,
self.__termMapDisplayname[self.__courseMapTerm[courseID]],
)
if self.config["no_subfolder"]:
path = os.path.join(self.downloadDir, folder[1:])
path = os.path.join(path, folder[1:])
else:
path = os.path.join(
self.downloadDir, f"{self.courseCode[courseID]}{folder}"
)
path = os.path.join(path, f"{self.courseCode[courseID]}{folder}")
if not os.path.exists(path):
os.makedirs(path)
localFiles += [
Expand Down Expand Up @@ -231,12 +266,13 @@ async def getCourseTaskInfoHelper(
):
if not fileUrl:
return
path = os.path.join(
self.downloadDir, self.__termMapDisplayname[self.__courseMapTerm[courseID]]
)
if self.config["no_subfolder"]:
path = os.path.join(self.downloadDir, fileName[1:])
path = os.path.join(path, fileName[1:])
else:
path = os.path.join(
self.downloadDir, f"{self.courseCode[courseID]}{fileName}"
)
path = os.path.join(path, f"{self.courseCode[courseID]}{fileName}")
path = path.replace("\\", "/").replace("//", "/")
if fileName in localFiles and fileModifiedTimeStamp <= os.path.getctime(path):
return
Expand Down Expand Up @@ -358,6 +394,7 @@ def checkFilesType(self):
async def sync(self):
print("Getting course IDs...")
await self.getCourseID()
(self.__courseMapTerm, self.__termMapDisplayname) = await self.getCourseList()
print(f"Get {len(self.courseCode)} available courses!")
print("Finding files on canvas...")
await asyncio.gather(
Expand Down