From 4efb05684b767026344ac6f2edb5118eba79e87c Mon Sep 17 00:00:00 2001 From: BoYanZh <32470225+BoYanZh@users.noreply.github.com> Date: Sun, 4 Oct 2020 23:19:11 +0800 Subject: update: rubric detail --- VG101GradeHelper.py | 10 +- worker/CanvasWorker.py | 21 ++-- worker/GitWorker.py | 278 +++++++++++++++++++++++++++++++++++++++++++++++++ worker/GiteaWorker.py | 262 ---------------------------------------------- 4 files changed, 296 insertions(+), 275 deletions(-) create mode 100644 worker/GitWorker.py delete mode 100644 worker/GiteaWorker.py diff --git a/VG101GradeHelper.py b/VG101GradeHelper.py index 5530d57..677a25e 100644 --- a/VG101GradeHelper.py +++ b/VG101GradeHelper.py @@ -4,7 +4,7 @@ import argparse import json import os -from worker import CanvasWorker, GiteaWorker, JOJWorker +from worker import CanvasWorker, GitWorker, JOJWorker from settings import * @@ -62,12 +62,12 @@ if __name__ == "__main__": pwd = os.getcwd() args = parse() indvScores, groupScores, jojScores = {}, {}, {} - giteaWorker = GiteaWorker(args, hgroups, + gitWorker = GitWorker(args, hgroups, [item[0] for item in JOJ_INFO["problemInfo"]]) if args.indv: - indvScores = giteaWorker.checkIndv() + indvScores = gitWorker.checkIndv() if args.group: - groupScores = giteaWorker.checkGroup() + groupScores = gitWorker.checkGroup() if args.joj: jojWorker = JOJWorker(args, JOJ_COURSE_ID, SID, hgroups) jojScores = jojWorker.checkGroupJOJ(JOJ_INFO) @@ -78,4 +78,4 @@ if __name__ == "__main__": if args.upload: canvasWorker.grade2Canvas() if args.proj: - giteaWorker.checkProj(args.proj, args.ms) + gitWorker.checkProj(args.proj, args.ms) diff --git a/worker/CanvasWorker.py b/worker/CanvasWorker.py index 99ad61c..e5457a2 100644 --- a/worker/CanvasWorker.py +++ b/worker/CanvasWorker.py @@ -37,11 +37,16 @@ class CanvasWorker(): for _ in range(scoreInfo.get(key, 0)): score += value[0] comment.append(f"{value[1]}, {value[0]}") - comment.extend( - scoreInfo.get("indvComment", []) + - scoreInfo.get("groupComment", []) + - scoreInfo.get("jojComment", [])) - if not comment: comment = ['good job'] + if not comment: + comment = ['good job'] + else: + comment.insert(0, "Genral Info:") + comment.append("") + comment.append("Detail:") + comment.extend( + scoreInfo.get("indvComment", []) + + scoreInfo.get("groupComment", []) + + scoreInfo.get("jojComment", [])) return { 'submission': { 'posted_grade': score @@ -62,12 +67,12 @@ class CanvasWorker(): name = currentUser.name.strip() if name not in self.names: continue data = self.generateHomeworkData(self.scores[name]) - self.logger.debug(data.__repr__()) - # submission.edit(**data) + self.logger.debug(f"{name} {data.__repr__()}") + submission.edit(**data) def exportScores(self, fileName): json.dump(self.scores, open(fileName, "w"), ensure_ascii=False, indent=4) - self.logger.debug("score dump to score.json succeed") \ No newline at end of file + self.logger.debug(f"score dump to {fileName} succeed") \ No newline at end of file diff --git a/worker/GitWorker.py b/worker/GitWorker.py new file mode 100644 index 0000000..1213e98 --- /dev/null +++ b/worker/GitWorker.py @@ -0,0 +1,278 @@ +from shutil import ignore_patterns, copytree, rmtree +from util import Logger +import multiprocessing +import traceback +import git +import os +import re + + +class GiteaWorker(): + def __init__(self, + args, + hgroups, + mandatoryFiles, + logger=Logger(), + processCount=16): + self.args = args + self.hgroups = hgroups + self.logger = logger + self.processCount = processCount + self.mandatoryFiles = mandatoryFiles + + @classmethod + def isREADME(cls, fn): + fn = fn.lower() + if len(fn) < 6: return False + if len(fn) == 6: return fn == "readme" + return fn[:7] == "readme." + + def checkProjRepoName(self, arg): + id_, name, projNum, *_ = arg + eng = re.sub('[\u4e00-\u9fa5]', '', name) + eng = ''.join( + [word[0].capitalize() + word[1:] for word in eng.split()]) + return f"{eng}{id_}-p{projNum}" + + def checkIndvProcess(self, groupNum, hwNum): + tidy = self.args.tidy + repoName = f"hgroup-{groupNum:02}" + repoDir = os.path.join('hwrepos', repoName) + hwDir = os.path.join(repoDir, f"h{hwNum}") + if not os.path.exists(repoDir): + repo = git.Repo.clone_from( + f"https://focs.ji.sjtu.edu.cn/git/vg101/{repoName}", + repoDir, + branch='master') + else: + repo = git.Repo(repoDir) + repo.git.fetch() + remoteBranches = [ref.name for ref in repo.remote().refs] + scores = { + stuName: { + "indvFailSubmit": 0, + "indvUntidy": 0, + "indvComment": [], + } + for _, stuName in self.hgroups[repoName] + } + for stuID, stuName in self.hgroups[repoName]: + try: + if f"origin/{stuID}" not in remoteBranches: + self.logger.warning( + f"{repoName} {stuID} {stuName} branch missing") + scores[stuName]['indvFailSubmit'] = 1 + scores[stuName]['indvComment'].append( + "individual branch individual branch missing") + continue + repo.git.checkout(f"{stuID}", "-f") + repo.git.reset('--hard') + repo.git.pull("origin", f"{stuID}", "-f") + repo.git.reset(f"origin/{stuID}", "--hard") + repo.git.clean("-d", "-f", "-x") + self.logger.debug(f"{repoName} {stuID} {stuName} pull succeed") + if self.args.dir: + copytree(repoDir, + os.path.join('indv', + f"{repoName} {stuID} {stuName}"), + ignore=ignore_patterns('.git')) + if not os.path.exists(hwDir): + self.logger.warning( + f"{repoName} {stuID} {stuName} h{hwNum} dir missing") + scores[stuName]['indvFailSubmit'] = 1 + scores[stuName]['indvComment'].append( + f"individual branch h{hwNum} dir missing") + else: + for fn, path in [(fn, os.path.join(hwDir, fn)) + for fn in self.mandatoryFiles]: + if os.path.exists(path): continue + self.logger.warning( + f"{repoName} {stuID} {stuName} h{hwNum}/{fn} file missing" + ) + scores[stuName]['indvFailSubmit'] = 1 + scores[stuName]['indvComment'].append( + f"individual branch h{hwNum}/{fn} file missing") + if not list(filter(GiteaWorker.isREADME, + os.listdir(hwDir))): + self.logger.warning( + f"{repoName} {stuID} {stuName} h{hwNum}/README file missing" + ) + scores[stuName]['indvFailSubmit'] = 1 + scores[stuName]['indvComment'].append( + f"individual branch h{hwNum}/README file missing") + if not tidy: continue + dirList = list( + filter( + lambda x: x not in [ + ".git", *[f"h{n}" for n in range(20)] + ] and not GiteaWorker.isREADME(x), + os.listdir(repoDir))) + if dirList: + self.logger.warning( + f"{repoName} {stuID} {stuName} untidy {dirList.__repr__()}" + ) + scores[stuName]['indvUntidy'] = 1 + scores[stuName]['indvComment'].append( + f"individual branch redundant files: {dirList.__repr__()}" + ) + if os.path.exists(hwDir): + dirList = os.listdir(hwDir) + dirList = list( + filter( + lambda x: not x.startswith("ex") and + not GiteaWorker.isREADME(x), dirList)) + if dirList: + self.logger.warning( + f"{repoName} {stuID} {stuName} h{hwNum}/ untidy {dirList.__repr__()}" + ) + scores[stuName]['indvUntidy'] = 1 + scores[stuName]['indvComment'].append( + f"individual branch redundant files: {dirList.__repr__()}" + ) + except Exception: + self.logger.error(f"{repoName} {stuID} {stuName} error") + self.logger.error(traceback.format_exc()) + return scores + + def checkGroupProcess(self, groupNum, hwNum): + tidy = self.args.tidy + repoName = f"hgroup-{groupNum:02}" + repoDir = os.path.join('hwrepos', repoName) + hwDir = os.path.join(repoDir, f"h{hwNum}") + if not os.path.exists(repoDir): + repo = git.Repo.clone_from( + f"https://focs.ji.sjtu.edu.cn/git/vg101/{repoName}", + repoDir, + branch='master') + else: + repo = git.Repo(repoDir) + repo.git.checkout("master", "-f") + repo.git.fetch("--tags", "-f") + tagNames = [tag.name for tag in repo.tags] + scores = { + stuName: { + "groupFailSubmit": 0, + "groupUntidy": 0, + "groupComment": [], + } + for _, stuName in self.hgroups[repoName] + } + if f"h{hwNum}" not in tagNames: + self.logger.warning(f"{repoName} tags/h{hwNum} missing") + for _, stuName in self.hgroups[repoName]: + scores[stuName]['groupFailSubmit'] = 1 + scores[stuName]['groupComment'].append( + f"tags/h{hwNum} missing") + return scores + repo.git.checkout(f"tags/h{hwNum}", "-f") + if not os.path.exists(hwDir): + self.logger.warning(f"{repoName} h{hwNum} dir missing") + for _, stuName in self.hgroups[repoName]: + scores[stuName]['groupFailSubmit'] = 1 + scores[stuName]['groupComment'].append( + f"master branch h{hwNum} dir missing") + else: + for fn, path in [(fn, os.path.join(hwDir, fn)) + for fn in self.mandatoryFiles]: + if os.path.exists(path): continue + self.logger.warning(f"{repoName} h{hwNum}/{fn} file missing") + for _, stuName in self.hgroups[repoName]: + scores[stuName]['groupFailSubmit'] = 1 + scores[stuName]['groupComment'].append( + f"master branch h{hwNum}/{fn} missing") + if not list(filter(GiteaWorker.isREADME, os.listdir(hwDir))): + self.logger.warning(f"{repoName} h{hwNum}/README file missing") + for _, stuName in self.hgroups[repoName]: + scores[stuName]['groupFailSubmit'] = 1 + scores[stuName]['groupComment'].append( + f"master branch h{hwNum}/README file missing") + self.logger.debug(f"{repoName} checkout to tags/h{hwNum} succeed") + if not tidy: return scores + dirList = os.listdir(repoDir) + dirList = list( + filter( + lambda x: x not in [".git", *[f"h{n}" for n in range(20)]] and + not GiteaWorker.isREADME(x), dirList)) + if dirList: + self.logger.warning(f"{repoName} untidy {dirList.__repr__()}") + for _, stuName in self.hgroups[repoName]: + scores[stuName]['groupUntidy'] = 1 + scores[stuName]['groupComment'].append( + f"master branch redundant files: {dirList.__repr__()}") + if os.path.exists(hwDir): + dirList = os.listdir(hwDir) + dirList = list( + filter( + lambda x: not x.startswith("ex") and not GiteaWorker. + isREADME(x), dirList)) + if dirList: + self.logger.warning( + f"{repoName} h{hwNum} untidy {dirList.__repr__()}") + for _, stuName in self.hgroups[repoName]: + scores[stuName]['groupUntidy'] = 1 + scores[stuName]['groupComment'].append( + f"master branch redundant files: {dirList.__repr__()}") + return scores + + def checkProjProcess(self, id_, name, projNum, milestoneNum): + repoName = self.checkProjRepoName([id_, name, projNum, milestoneNum]) + repoDir = os.path.join('projrepos', f'p{projNum}', repoName) + if not os.path.exists(repoDir): + repo = git.Repo.clone_from( + f"https://focs.ji.sjtu.edu.cn/git/vg101/{repoName}", repoDir) + else: + repo = git.Repo(os.path.join('projrepos', f'p{projNum}', repoName)) + repo.git.fetch() + remoteBranches = [ref.name for ref in repo.remote().refs] + if 'origin/master' not in remoteBranches: + self.logger.warning(f"{repoName} branch master missing") + return + repo.git.checkout(f"master", "-f") + repo.git.reset('--hard') + repo.git.pull("origin", "master", "-f") + repo.git.reset('--hard') + repo.git.clean("-d", "-f", "-x") + if not list(filter(GiteaWorker.isREADME, os.listdir(repoDir))): + self.logger.warning(f"{repoName} README missing") + if milestoneNum: + tagNames = [tag.name for tag in repo.tags] + if f"m{milestoneNum}" not in tagNames: + self.logger.warning(f"{repoName} tags/m{milestoneNum} missing") + return + repo.git.checkout(f"tags/m{milestoneNum}", "-f") + self.logger.debug( + f"{repoName} checkout to tags/m{milestoneNum} succeed") + else: + self.logger.debug(f"{repoName} pull succeed") + + def checkIndv(self): + if self.args.dir: + if os.path.exists(os.path.join('indv')): + rmtree(os.path.join('indv')) + hwNum = self.args.hw + with multiprocessing.Pool(self.processCount) as p: + res = p.starmap(self.checkIndvProcess, + [(i, hwNum) + for i in range(len(self.hgroups.keys()))]) + return {k: v for d in res for k, v in d.items()} + + def checkGroup(self): + hwNum = self.args.hw + with multiprocessing.Pool(self.processCount) as p: + res = p.starmap(self.checkGroupProcess, + [(i, hwNum) + for i in range(len(self.hgroups.keys()))]) + return {k: v for d in res for k, v in d.items()} + + def checkProj(self, projNum, milestoneNum): + milestoneNum = 0 if milestoneNum is None else milestoneNum + if projNum in [1, 2]: + infos = [[*info, projNum, milestoneNum] + for hgroup in self.hgroups.values() for info in hgroup] + elif projNum in [3]: + infos = [] + return + else: + return + with multiprocessing.Pool(self.processCount) as p: + p.starmap(self.checkProjProcess, infos) \ No newline at end of file diff --git a/worker/GiteaWorker.py b/worker/GiteaWorker.py deleted file mode 100644 index 8bbbe60..0000000 --- a/worker/GiteaWorker.py +++ /dev/null @@ -1,262 +0,0 @@ -from shutil import ignore_patterns, copytree, rmtree -from util import Logger -import multiprocessing -import traceback -import git -import os -import re - - -class GiteaWorker(): - def __init__(self, - args, - hgroups, - mandatoryFiles, - logger=Logger(), - processCount=16): - self.args = args - self.hgroups = hgroups - self.logger = logger - self.processCount = processCount - self.mandatoryFiles = mandatoryFiles - - def checkProjRepoName(self, arg): - id_, name, projNum, *_ = arg - eng = re.sub('[\u4e00-\u9fa5]', '', name) - eng = ''.join( - [word[0].capitalize() + word[1:] for word in eng.split()]) - return f"{eng}{id_}-p{projNum}" - - def checkIndvProcess(self, groupNum, hwNum): - tidy = self.args.tidy - repoName = f"hgroup-{groupNum:02}" - if not os.path.exists(os.path.join('hwrepos', repoName)): - repo = git.Repo.clone_from( - f"https://focs.ji.sjtu.edu.cn/git/vg101/{repoName}", - os.path.join('hwrepos', repoName), - branch='master') - else: - repo = git.Repo(os.path.join('hwrepos', repoName)) - repo.git.fetch() - remoteBranches = [ref.name for ref in repo.remote().refs] - scores = { - stuName: { - "indvFailSubmit": 0, - "indvUntidy": 0, - "indvComment": [], - } - for _, stuName in self.hgroups[repoName] - } - for stuID, stuName in self.hgroups[repoName]: - try: - if f"origin/{stuID}" not in remoteBranches: - self.logger.warning( - f"{repoName} {stuID} {stuName} branch missing") - scores[stuName]['indvFailSubmit'] = 1 - scores[stuName]['indvComment'].append( - "individual branch missing") - continue - repo.git.checkout(f"{stuID}", "-f") - repo.git.reset('--hard') - repo.git.pull("origin", f"{stuID}", "-f") - repo.git.reset(f"origin/{stuID}", "--hard") - repo.git.clean("-d", "-f", "-x") - if self.args.dir: - copytree(os.path.join('hwrepos', repoName), - os.path.join('indv', - f"{repoName} {stuID} {stuName}"), - ignore=ignore_patterns('.git')) - hwDir = os.path.join('hwrepos', repoName, f"h{hwNum}") - if not os.path.exists(hwDir): - self.logger.warning( - f"{repoName} {stuID} {stuName} h{hwNum} dir missing") - scores[stuName]['indvFailSubmit'] = 1 - scores[stuName]['indvComment'].append( - f"h{hwNum} dir missing") - else: - for fn, path in [(fn, - os.path.join('hwrepos', repoName, - f"h{hwNum}", fn)) - for fn in self.mandatoryFiles]: - if not os.path.exists(path): - self.logger.warning( - f"{repoName} {stuID} {stuName} h{hwNum}/{fn} file missing" - ) - scores[stuName]['indvFailSubmit'] = 1 - scores[stuName]['indvComment'].append( - f"h{hwNum}/{fn} file missing") - self.logger.debug(f"{repoName} {stuID} {stuName} succeed") - if tidy: - dirList = os.listdir(os.path.join('hwrepos', repoName)) - dirList = list( - filter( - lambda x: x not in [ - "README.md", "README.txt", "README.rst", - ".git", *[f"h{n}" for n in range(20)] - ], dirList)) - if dirList: - self.logger.warning( - f"{repoName} {stuID} {stuName} untidy {dirList.__repr__()}" - ) - scores[stuName]['indvUntidy'] = 1 - scores[stuName]['indvComment'].append( - f"individual branch redundant files: {dirList.__repr__()}" - ) - if os.path.exists(hwDir): - dirList = os.listdir(hwDir) - dirList = list( - filter( - lambda x: not x.startswith("ex") and not x. - startswith("README."), dirList)) - if dirList: - self.logger.warning( - f"{repoName} {stuID} {stuName} h{hwNum}/ untidy {dirList.__repr__()}" - ) - scores[stuName]['indvUntidy'] = 1 - scores[stuName]['indvComment'].append( - f"individual branch redundant files: {dirList.__repr__()}" - ) - except Exception: - self.logger.error(f"{repoName} {stuID} {stuName} error") - self.logger.error(traceback.format_exc()) - return scores - - def checkGroupProcess(self, groupNum, hwNum): - tidy = self.args.tidy - repoName = f"hgroup-{groupNum:02}" - repoDir = os.path.join('hwrepos', repoName) - hwDir = os.path.join(repoDir, f"h{hwNum}") - if not os.path.exists(repoDir): - repo = git.Repo.clone_from( - f"https://focs.ji.sjtu.edu.cn/git/vg101/{repoName}", - repoDir, - branch='master') - else: - repo = git.Repo(repoDir) - repo.git.checkout("master", "-f") - repo.git.fetch("--tags", "-f") - tagNames = [tag.name for tag in repo.tags] - scores = { - stuName: { - "groupFailSubmit": 0, - "groupUntidy": 0, - "groupComment": [], - } - for _, stuName in self.hgroups[repoName] - } - if f"h{hwNum}" not in tagNames: - self.logger.warning(f"{repoName} tags/h{hwNum} missing") - for _, stuName in self.hgroups[repoName]: - scores[stuName]['groupFailSubmit'] = 1 - scores[stuName]['groupComment'].append( - f"tags/h{hwNum} missing") - return - repo.git.checkout(f"tags/h{hwNum}", "-f") - if not os.path.exists(hwDir): - self.logger.warning(f"{repoName} h{hwNum} dir missing") - for _, stuName in self.hgroups[repoName]: - scores[stuName]['groupFailSubmit'] = 1 - scores[stuName]['groupComment'].append(f"h{hwNum} dir missing") - else: - for fn, path in [(fn, os.path.join(hwDir, fn)) - for fn in self.mandatoryFiles]: - if not os.path.exists(path): - self.logger.warning(f"{repoName} h{hwNum}/{fn} file missing") - for _, stuName in self.hgroups[repoName]: - scores[stuName]['groupFailSubmit'] = 1 - scores[stuName]['groupComment'].append( - f"h{hwNum}/{fn} missing") - self.logger.debug(f"{repoName} checkout to tags/h{hwNum} succeed") - if tidy: - dirList = os.listdir(repoDir) - dirList = list( - filter( - lambda x: x not in - ["README.md", ".git", *[f"h{n}" for n in range(20)]], - dirList)) - if dirList: - self.logger.warning(f"{repoName} untidy {dirList.__repr__()}") - for _, stuName in self.hgroups[repoName]: - scores[stuName]['groupUntidy'] = 1 - scores[stuName]['groupComment'].append( - f"master branch redundant files: {dirList.__repr__()}") - if os.path.exists(hwDir): - dirList = os.listdir(hwDir) - dirList = list( - filter( - lambda x: not x.startswith("ex") and not x.startswith( - "README."), dirList)) - if dirList: - self.logger.warning( - f"{repoName} h{hwNum} untidy {dirList.__repr__()}") - for _, stuName in self.hgroups[repoName]: - scores[stuName]['groupUntidy'] = 1 - scores[stuName]['groupComment'].append( - f"master branch redundant files: {dirList.__repr__()}" - ) - return scores - - def checkProjProcess(self, id_, name, projNum, milestoneNum): - repoName = self.checkProjRepoName([id_, name, projNum, milestoneNum]) - repoDir = os.path.join('projrepos', f'p{projNum}', repoName) - if not os.path.exists(repoDir): - repo = git.Repo.clone_from( - f"https://focs.ji.sjtu.edu.cn/git/vg101/{repoName}", repoDir) - else: - repo = git.Repo(os.path.join('projrepos', f'p{projNum}', repoName)) - repo.git.fetch() - remoteBranches = [ref.name for ref in repo.remote().refs] - if 'origin/master' not in remoteBranches: - self.logger.warning(f"{repoName} branch master missing") - return - repo.git.checkout(f"master", "-f") - repo.git.reset('--hard') - repo.git.pull("origin", "master", "-f") - repo.git.reset('--hard') - repo.git.clean("-d", "-f", "-x") - if not list( - filter(lambda x: x.lower().startswith('readme'), - os.listdir(repoDir))): - self.logger.warning(f"{repoName} README missing") - if milestoneNum: - tagNames = [tag.name for tag in repo.tags] - if f"m{milestoneNum}" not in tagNames: - self.logger.warning(f"{repoName} tags/m{milestoneNum} missing") - return - repo.git.checkout(f"tags/m{milestoneNum}", "-f") - self.logger.debug( - f"{repoName} checkout to tags/m{milestoneNum} succeed") - else: - self.logger.debug(f"{repoName} pull succeed") - - def checkIndv(self): - if self.args.dir: - if os.path.exists(os.path.join('indv')): - rmtree(os.path.join('indv')) - hwNum = self.args.hw - with multiprocessing.Pool(self.processCount) as p: - res = p.starmap(self.checkIndvProcess, - [(i, hwNum) - for i in range(len(self.hgroups.keys()))]) - return {k: v for d in res for k, v in d.items()} - - def checkGroup(self): - hwNum, tidy = self.args.hw, self.args.tidy - with multiprocessing.Pool(self.processCount) as p: - res = p.starmap(self.checkGroupProcess, - [(i, hwNum) - for i in range(len(self.hgroups.keys()))]) - return {k: v for d in res for k, v in d.items()} - - def checkProj(self, projNum, milestoneNum): - milestoneNum = 0 if milestoneNum is None else milestoneNum - if projNum in [1, 2]: - infos = [[*info, projNum, milestoneNum] - for hgroup in self.hgroups.values() for info in hgroup] - elif projNum in [3]: - infos = [] - return - else: - return - with multiprocessing.Pool(self.processCount) as p: - p.starmap(self.checkProjProcess, infos) \ No newline at end of file -- cgit v1.2.3