From be29e385d39c1dce1f9516225ffb9e251895d751 Mon Sep 17 00:00:00 2001 From: BoYanZh Date: Sat, 31 Oct 2020 02:20:04 +0800 Subject: update for C & multiple files uploading & code review checking --- worker/CanvasWorker.py | 20 +++++++++------ worker/GitWorker.py | 70 ++++++++++++++++++++++++++++++++------------------ worker/GiteaWorker.py | 22 ++++++++++++++++ worker/JOJWorker.py | 36 ++++++++++++++------------ 4 files changed, 99 insertions(+), 49 deletions(-) (limited to 'worker') diff --git a/worker/CanvasWorker.py b/worker/CanvasWorker.py index 954d1da..3ff60e3 100644 --- a/worker/CanvasWorker.py +++ b/worker/CanvasWorker.py @@ -13,6 +13,7 @@ class CanvasWorker(): indvScores, groupScores, jojScores, + totalScores=None, logger=Logger()): self.args = args self.rubric = rubric @@ -22,13 +23,16 @@ class CanvasWorker(): self.assignments = self.course.get_assignments() self.logger = logger self.scores = {} - self.names = names - for key in names: - self.scores[key] = { - **indvScores.get(key, {}), - **groupScores.get(key, {}), - **jojScores.get(key, {}) - } + if totalScores is None: + self.names = names + for key in names: + self.scores[key] = { + **indvScores.get(key, {}), + **groupScores.get(key, {}), + **jojScores.get(key, {}) + } + else: + self.scores = totalScores def generateHomeworkData(self, scoreInfo): score = 0 @@ -49,7 +53,7 @@ class CanvasWorker(): scoreInfo.get("jojComment", [])) return { 'submission': { - 'posted_grade': score + 'posted_grade': max(score, -2.5) }, 'comment': { 'text_comment': '\n'.join(comment) diff --git a/worker/GitWorker.py b/worker/GitWorker.py index f5715ba..e124b1f 100644 --- a/worker/GitWorker.py +++ b/worker/GitWorker.py @@ -1,5 +1,5 @@ +from util import Logger, getProjRepoName, passCodeQuality, getAllFiles from shutil import ignore_patterns, copytree, rmtree -from util import Logger, getProjRepoName, passCodeQuality import multiprocessing import traceback import git @@ -12,6 +12,7 @@ class GitWorker(): hgroups, language, mandatoryFiles, + optionalFiles, logger=Logger(), processCount=16): self.args = args @@ -20,6 +21,8 @@ class GitWorker(): self.logger = logger self.processCount = processCount self.mandatoryFiles = mandatoryFiles + self.optionalFiles = optionalFiles + self.moss = None @classmethod def isREADME(cls, fn): @@ -40,7 +43,7 @@ class GitWorker(): branch="master") else: repo = git.Repo(repoDir) - repo.git.fetch("--tags", "--all") + repo.git.fetch("--all", "-f") remoteBranches = [ref.name for ref in repo.remote().refs] scores = { stuName: { @@ -75,8 +78,10 @@ class GitWorker(): scores[stuName]["indvComment"].append( f"individual branch h{hwNum} dir missing") else: - for fn, path in [(fn, os.path.join(hwDir, fn)) - for fn in self.mandatoryFiles]: + for fn, path in [ + (fn, os.path.join(hwDir, fn)) for fn in + [*self.mandatoryFiles, *self.optionalFiles] + ]: if os.path.exists(path): if not passCodeQuality(path, self.language): scores[stuName]["indvLowCodeQuality"] = 1 @@ -86,12 +91,13 @@ class GitWorker(): f"{repoName} {stuID} {stuName} {fn} low quality" ) continue - self.logger.warning( - f"{repoName} {stuID} {stuName} h{hwNum}/{fn} file missing" - ) - scores[stuName]["indvFailSubmit"] = 1 - scores[stuName]["indvComment"].append( - f"individual branch h{hwNum}/{fn} file missing") + if fn in self.mandatoryFiles: + self.logger.warning( + f"{repoName} {stuID} {stuName} h{hwNum}/{fn} file missing" + ) + scores[stuName]["indvFailSubmit"] = 1 + scores[stuName]["indvComment"].append( + f"individual branch h{hwNum}/{fn} file missing") if not list(filter(GitWorker.isREADME, os.listdir(hwDir))): self.logger.warning( f"{repoName} {stuID} {stuName} h{hwNum}/README file missing" @@ -117,8 +123,9 @@ class GitWorker(): dirList = os.listdir(hwDir) dirList = list( filter( - lambda x: not x.startswith("ex") and not GitWorker. - isREADME(x), dirList)) + lambda x: x not in self.mandatoryFiles and x not in + self.optionalFiles and not GitWorker.isREADME(x), + dirList)) if dirList: self.logger.warning( f"{repoName} {stuID} {stuName} h{hwNum}/ untidy {', '.join(dirList)}" @@ -144,7 +151,7 @@ class GitWorker(): branch="master") else: repo = git.Repo(repoDir) - repo.git.fetch("--tags", "--all") + repo.git.fetch("--tags", "--all", "-f") tagNames = [tag.name for tag in repo.tags] scores = { stuName: { @@ -173,8 +180,10 @@ class GitWorker(): scores[stuName]["groupComment"].append( f"tags/h{hwNum} h{hwNum} dir missing") else: - for fn, path in [(fn, os.path.join(hwDir, fn)) - for fn in self.mandatoryFiles]: + for fn, path in [ + (fn, os.path.join(hwDir, fn)) + for fn in [*self.mandatoryFiles, *self.optionalFiles] + ]: if os.path.exists(path): if not passCodeQuality(path, self.language): for _, stuName in self.hgroups[repoName]: @@ -183,11 +192,12 @@ class GitWorker(): f"group {fn} low quality") self.logger.warning(f"{repoName} {fn} low quality") continue - self.logger.warning(f"{repoName} h{hwNum}/{fn} file missing") - for _, stuName in self.hgroups[repoName]: - scores[stuName]["groupFailSubmit"] = 1 - scores[stuName]["groupComment"].append( - f"tags/h{hwNum} h{hwNum}/{fn} missing") + if fn in self.mandatoryFiles: + self.logger.warning(f"{repoName} h{hwNum}/{fn} file missing") + for _, stuName in self.hgroups[repoName]: + scores[stuName]["groupFailSubmit"] = 1 + scores[stuName]["groupComment"].append( + f"tags/h{hwNum} h{hwNum}/{fn} missing") if not list(filter(GitWorker.isREADME, os.listdir(hwDir))): self.logger.warning(f"{repoName} h{hwNum}/README file missing") for _, stuName in self.hgroups[repoName]: @@ -212,8 +222,8 @@ class GitWorker(): dirList = os.listdir(hwDir) dirList = list( filter( - lambda x: not x.startswith("ex") and not GitWorker. - isREADME(x), dirList)) + lambda x: x not in self.mandatoryFiles and x not in self. + optionalFiles and not GitWorker.isREADME(x), dirList)) if dirList: self.logger.warning( f"{repoName} h{hwNum} untidy {', '.join(dirList)}") @@ -237,7 +247,7 @@ class GitWorker(): f"https://focs.ji.sjtu.edu.cn/git/vg101/{repoName}", repoDir) else: repo = git.Repo(os.path.join("projrepos", f"p{projNum}", repoName)) - repo.git.fetch("--tags", "--all") + repo.git.fetch("--tags", "--all", "-f") remoteBranches = [ref.name for ref in repo.remote().refs] if "origin/master" not in remoteBranches: self.logger.warning(f"{repoName} master branch missing") @@ -246,7 +256,7 @@ class GitWorker(): repo.git.reset("--hard", "origin/master") repo.git.clean("-d", "-f", "-x") if milestoneNum: - repo.git.fetch("--tags", "--all") + repo.git.fetch("--tags", "--all", "-f") tagNames = [tag.name for tag in repo.tags] if f"m{milestoneNum}" not in tagNames: self.logger.warning(f"{repoName} tags/m{milestoneNum} missing") @@ -259,12 +269,22 @@ class GitWorker(): if not list(filter(GitWorker.isREADME, os.listdir(repoDir))): self.logger.warning(f"{repoName} README file missing") scores[stuName]["projComment"].append(f"README file missing") + language = ["matlab", "c", "cpp"] if projNum == 1: for fn in list( filter(lambda x: x.endswith(".m"), os.listdir(repoDir))): path = os.path.join(repoDir, fn) - if not passCodeQuality(path, "matlab"): + if not passCodeQuality(path, language[projNum - 1]): + self.logger.warning(f"{repoName} {fn} low quality") + scores[stuName]["projComment"].append( + f"{fn} low quality") + elif projNum == 2: + for fn in getAllFiles(repoDir): + if (fn.endswith(".c") + or fn.endswith(".h")) and not passCodeQuality( + os.path.join(repoDir, fn), + language[projNum - 1]): self.logger.warning(f"{repoName} {fn} low quality") scores[stuName]["projComment"].append( f"{fn} low quality") diff --git a/worker/GiteaWorker.py b/worker/GiteaWorker.py index 55dd41c..1b85ea9 100644 --- a/worker/GiteaWorker.py +++ b/worker/GiteaWorker.py @@ -11,6 +11,11 @@ class GiteaWorker(): item[1]: item[0] for items in hgroups.values() for item in items } + self.ids = { + item[0]: item[1] + for items in hgroups.values() for item in items + } + self.hgroups = hgroups self.baseUrl = baseUrl self.orgName = orgName self.sess = requests.Session() @@ -29,3 +34,20 @@ class GiteaWorker(): } req = self.sess.post(url, data) self.logger.debug(f"{repoName} issue {req.status_code} {req.text}") + + def checkReview(self): + hwNum = self.args.hw + res = {key: {"noReview": 1} for key in self.names.keys()} + for repoName, users in self.hgroups.items(): + url = f"{self.baseUrl}/repos/{self.orgName}/{repoName}/pulls" + pulls = self.sess.get(url).json() + for pull in pulls: + if not pull["title"].startswith(f"h{hwNum}"): continue + url = f"{self.baseUrl}/repos/{self.orgName}/{repoName}/pulls/{pull['number']}/reviews" + self.logger.info(f"{repoName} h{hwNum} get pr: {url}") + for item in self.sess.get(url).json(): + stuID = ''.join( + [s for s in item['user']['full_name'] if s.isdigit()]) + name = self.ids[stuID] + res[name]["noReview"] = 0 + return res \ No newline at end of file diff --git a/worker/JOJWorker.py b/worker/JOJWorker.py index aed12da..b0f316d 100644 --- a/worker/JOJWorker.py +++ b/worker/JOJWorker.py @@ -70,7 +70,6 @@ class JOJWorker(): zipPath, lang, groupName='', - fn='', hwNum=0): tryTime = 0 while True: @@ -79,27 +78,32 @@ class JOJWorker(): if response.status_code == 200: break self.logger.error( - f"{groupName} h{hwNum} {fn} upload error, code {response.status_code}, url {response.url}" + f"{groupName} h{hwNum} {problemID} upload error, code {response.status_code}, url {response.url}" ) time.sleep(1) self.logger.debug( - f"{groupName} h{hwNum} {fn} upload succeed, url {response.url}") + f"{groupName} h{hwNum} {problemID} upload succeed, url {response.url}") return self.getProblemStatus(response.url) - def checkGroupJOJProcess(self, groupNum, hwNum, jojInfo, fn, problemID): + def checkGroupJOJProcess(self, groupNum, hwNum, jojInfo, fns, problemID): groupName = f"hgroup-{groupNum:02}" hwDir = os.path.join('hwrepos', groupName, f"h{hwNum}") - filePath = os.path.join(hwDir, fn) - if not os.path.exists(filePath): - self.logger.warning(f"{groupName} h{hwNum} {fn} not exist") - return 0 - if os.path.exists(filePath + ".zip"): os.remove(filePath + ".zip") - with zipfile.ZipFile(filePath + ".zip", mode='w') as zf: - zf.write(filePath, fn) + if not os.path.exists(hwDir): return 0 + zipPath = os.path.join(hwDir, problemID) + ".zip" + if os.path.exists(zipPath): os.remove(zipPath) + with zipfile.ZipFile(zipPath, mode='w') as zf: + for fn in fns: + filePath = os.path.join(hwDir, fn) + if not os.path.exists(filePath): + if not fn.endswith(".h"): + self.logger.warning(f"{groupName} h{hwNum} {fn} not exist") + return 0 + else: + zf.write(filePath, fn) res = self.getProblemResult(jojInfo["homeworkID"], problemID, - filePath + ".zip", jojInfo["lang"], - groupName, fn, hwNum) - os.remove(filePath + ".zip") + zipPath, jojInfo["lang"], + groupName, hwNum) + # os.remove(zipPath) return res def checkGroupJOJ(self, jojInfo): @@ -109,8 +113,8 @@ class JOJWorker(): with multiprocessing.Pool(len(jojInfo["problemInfo"])) as p: scores = p.starmap( self.checkGroupJOJProcess, - [[i, hwNum, jojInfo, fn, problemID] - for fn, problemID, _ in jojInfo["problemInfo"]]) + [[i, hwNum, jojInfo, fns, problemID] + for fns, problemID, _ in jojInfo["problemInfo"]]) scores = [(scores[i], jojInfo["problemInfo"][i][2]) for i in range(len(scores))] self.logger.info(f"{key} h{hwNum} score {scores.__repr__()}") -- cgit v1.2.3