diff options
Diffstat (limited to 'worker')
| -rw-r--r-- | worker/CanvasWorker.py | 67 | ||||
| -rw-r--r-- | worker/GiteaWorker.py | 200 | ||||
| -rw-r--r-- | worker/JOJWorker.py | 135 | ||||
| -rw-r--r-- | worker/__init__.py | 3 |
4 files changed, 405 insertions, 0 deletions
diff --git a/worker/CanvasWorker.py b/worker/CanvasWorker.py new file mode 100644 index 0000000..1d86189 --- /dev/null +++ b/worker/CanvasWorker.py @@ -0,0 +1,67 @@ +from canvasapi import Canvas +from util import first, Logger +import json + + +class CanvasWorker(): + def __init__(self, + args, + rubric, + canvasToken, + courseID, + indvScores, + groupScores, + jojScores, + logger=Logger()): + self.args = args + self.rubric = rubric + self.canvas = Canvas("https://umjicanvas.com/", canvasToken) + self.course = self.canvas.get_course(courseID) + self.users = self.course.get_users() + self.assignments = self.course.get_assignments() + self.logger = logger + if not indvScores or not groupScores or not jojScores: + raise Exception("Not enough scores") + self.scores = indvScores + for key, value in self.scores.items(): + self.scores[key] = { + **value, + **groupScores[key], + **jojScores[key] + } + + def generateHomeworkData(self, scoreInfo): + score = 0 + comment = [] + for key, value in self.rubric: + for _ in range(scoreInfo[key]): + score -= value[0] + comment.append(value[1]) + if not comment: comment = ['good job'] + return { + 'submission': { + 'posted_grade': score + }, + 'comment': { + 'text_comment': '\n'.join(comment) + }, + } + + def grade2Canvas(self): + hwNum = self.args.hw + assignment = first(self.assignments, + lambda x: x.name.startswith(f"h{hwNum}")) + for submission in assignment.get_submissions(): + currentUser = first(self.users, + lambda user: user.id == submission.user_id) + if currentUser is None: continue + name = currentUser.name.strip() + data = self.generateHomeworkData(self.scores[name]) + submission.edit(**data) + + def exportScores(self, fileName): + json.dump(self.scores, + open(fileName, "w"), + ensure_ascii=False, + indent=4) + self.logger.debug("score dump to score.json succeed")
\ No newline at end of file diff --git a/worker/GiteaWorker.py b/worker/GiteaWorker.py new file mode 100644 index 0000000..6dac863 --- /dev/null +++ b/worker/GiteaWorker.py @@ -0,0 +1,200 @@ +from logging import FATAL +from shutil import ignore_patterns, copytree, rmtree +from util import Logger +import multiprocessing +import git +import os +import re + + +class GiteaWorker(): + def __init__(self, + args, + hgroups, + mandatoryFiles, + logger=Logger(), + processCount=16): + self.args = args + self.hgroups = hgroups + self.logger = logger + self.processCount = processCount + self.mandatoryFiles = mandatoryFiles + + def checkProjRepoName(self, arg): + id_, name, projNum, *_ = arg + eng = re.sub('[\u4e00-\u9fa5]', '', name) + eng = ''.join( + [word[0].capitalize() + word[1:] for word in eng.split()]) + return f"{eng}{id_}-p{projNum}" + + def checkIndvProcess(self, groupNum, hwNum, tidy): + repoName = f"hgroup-{groupNum:02}" + if not os.path.exists(os.path.join('hwrepos', repoName)): + repo = git.Repo.clone_from( + f"https://focs.ji.sjtu.edu.cn/git/vg101/{repoName}", + os.path.join('hwrepos', repoName), + branch='master') + else: + repo = git.Repo(os.path.join('hwrepos', repoName)) + repo.git.fetch() + remoteBranches = [ref.name for ref in repo.remote().refs] + scores = { + stuName: { + "indvFailSubmit": 0, + "indvUntidy": 0, + } + for _, stuName in self.hgroups[repoName] + } + for stuID, stuName in self.hgroups[repoName]: + try: + if f"origin/{stuID}" not in remoteBranches: + self.logger.warning( + f"{repoName} {stuID} {stuName} branch missing") + scores[stuName]['indvFailSubmit'] = 1 + continue + repo.git.checkout(f"{stuID}", "-f") + repo.git.pull("origin", f"{stuID}", "--rebase", "-f") + repo.git.reset(f"origin/{stuID}", "--hard") + # copytree(os.path.join( 'hwrepos', repoName), + # os.path.join( 'indv', + # f"{repoName} {stuID} {stuName}"), + # ignore=ignore_patterns('.git')) + if not os.path.exists( + os.path.join('hwrepos', repoName, f"h{hwNum}")): + self.logger.warning( + f"{repoName} {stuID} {stuName} h{hwNum} dir missing") + scores[stuName]['indvFailSubmit'] = 1 + for path in [ + os.path.join('hwrepos', repoName, f"h{hwNum}", fn) + for fn in self.mandatoryFiles + ]: + if not os.path.exists(path): + self.logger.warning( + f"{repoName} {stuID} {stuName} h{hwNum} file missing" + ) + scores[stuName]['indvFailSubmit'] = 1 + self.logger.debug(f"{repoName} {stuID} {stuName} succeed") + if tidy: + dirList = os.listdir(os.path.join('hwrepos', repoName)) + dirList = list( + filter( + lambda x: x not in [ + "README.md", ".git", + *[f"h{n}" for n in range(20)] + ], dirList)) + if dirList: + self.logger.warning( + f"{repoName} {stuID} {stuName} untidy") + scores[stuName]['indvUntidy'] = 1 + except: + self.logger.error(f"{repoName} {stuID} {stuName} error") + return scores + + def checkGroupProcess(self, groupNum, hwNum, tidy): + repoName = f"hgroup-{groupNum:02}" + if not os.path.exists(os.path.join('hwrepos', repoName)): + repo = git.Repo.clone_from( + f"https://focs.ji.sjtu.edu.cn/git/vg101/{repoName}", + os.path.join('hwrepos', repoName), + branch='master') + else: + repo = git.Repo(os.path.join('hwrepos', repoName)) + repo.git.checkout("master", "-f") + repo.git.fetch("--tags", "-f") + tagNames = [tag.name for tag in repo.tags] + scores = { + stuName: { + "groupFailSubmit": 0, + "groupUntidy": 0, + } + for _, stuName in self.hgroups[repoName] + } + if f"h{hwNum}" not in tagNames: + self.logger.warning(f"{repoName} tags/h{hwNum} missing") + for _, stuName in self.hgroups[repoName]: + scores[stuName]['groupFailSubmit'] = 1 + return + repo.git.checkout(f"tags/h{hwNum}", "-f") + if not os.path.exists(os.path.join('hwrepos', repoName, f"h{hwNum}")): + self.logger.warning(f"{repoName} h{hwNum} dir missing") + for _, stuName in self.hgroups[repoName]: + scores[stuName]['groupFailSubmit'] = 1 + for path in [ + os.path.join('hwrepos', repoName, f"h{hwNum}", fn) + for fn in self.mandatoryFiles + ]: + if not os.path.exists(path): + self.logger.warning(f"{repoName} h{hwNum} file missing") + for _, stuName in self.hgroups[repoName]: + scores[stuName]['groupFailSubmit'] = 1 + self.logger.debug(f"{repoName} checkout to tags/h{hwNum} succeed") + if tidy: + dirList = os.listdir(os.path.join('hwrepos', repoName)) + dirList = list( + filter( + lambda x: x not in + ["README.md", ".git", *[f"h{n}" for n in range(20)]], + dirList)) + if dirList: + self.logger.warning(f"{repoName} untidy") + for _, stuName in self.hgroups[repoName]: + scores[stuName]['groupUntidy'] = 1 + return scores + + def checkProjProcess(self, id_, name, projNum, milestoneNum): + repoName = self.checkProjRepoName([id_, name, projNum, milestoneNum]) + repoDir = os.path.join('projrepos', f'p{projNum}', repoName) + if not os.path.exists(repoDir): + repo = git.Repo.clone_from( + f"https://focs.ji.sjtu.edu.cn/git/vg101/{repoName}", repoDir) + else: + repo = git.Repo(os.path.join('projrepos', f'p{projNum}', repoName)) + repo.git.fetch() + if 'master' not in [branch.name for branch in repo.branches]: + self.logger.warning(f"{repoName} branch master missing") + return + repo.git.reset('--hard') + repo.git.pull("origin", "master", "--rebase", "-f") + if not list( + filter(lambda x: x.lower().startswith('readme'), + os.listdir(repoDir))): + self.logger.warning(f"{repoName} README missing") + if milestoneNum: + tagNames = [tag.name for tag in repo.tags] + if f"m{milestoneNum}" not in tagNames: + self.logger.warning(f"{repoName} tags/m{milestoneNum} missing") + return + repo.git.checkout(f"tags/m{milestoneNum}", "-f") + self.logger.debug( + f"{repoName} checkout to tags/m{milestoneNum} succeed") + else: + self.logger.debug(f"{repoName} pull succeed") + + def checkIndv(self): + # if os.path.exists(os.path.join( 'indv')): + # rmtree(os.path.join( 'indv')) + hwNum, tidy = self.args.hw, self.args.tidy + with multiprocessing.Pool(self.processCount) as p: + res = p.starmap(self.checkIndvProcess, + [(i, hwNum, tidy) for i in range(26)]) + return {k: v for d in res for k, v in d.items()} + + def checkGroup(self): + hwNum, tidy = self.args.hw, self.args.tidy + with multiprocessing.Pool(self.processCount) as p: + res = p.starmap(self.checkGroupProcess, + [(i, hwNum, tidy) for i in range(26)]) + return {k: v for d in res for k, v in d.items()} + + def checkProj(self, projNum, milestoneNum): + milestoneNum = 0 if milestoneNum is None else milestoneNum + if projNum in [1, 2]: + infos = [[*info, projNum, milestoneNum] + for hgroup in self.hgroups.values() for info in hgroup] + elif projNum in [3]: + infos = [] + return + else: + return + with multiprocessing.Pool(self.processCount) as p: + p.starmap(self.checkProjProcess, infos)
\ No newline at end of file diff --git a/worker/JOJWorker.py b/worker/JOJWorker.py new file mode 100644 index 0000000..9082a47 --- /dev/null +++ b/worker/JOJWorker.py @@ -0,0 +1,135 @@ +from bs4 import BeautifulSoup +from util import Logger +import multiprocessing +import requests +import zipfile +import time +import os + + +class JOJWorker(): + def __init__(self, args, courseID, sid, hgroups, logger=Logger()): + def createSess(cookies): + s = requests.Session() + s.cookies.update(cookies) + return s + + cookies = { + 'JSESSIONID': 'dummy', + 'save': '1', + 'sid': sid, + } + self.args = args + self.sess = createSess(cookies=cookies) + self.courseID = courseID + self.hgroups = hgroups + self.logger = logger + + def uploadZip(self, homeworkID, problemID, zipPath, lang): + files = { + 'code': ('code.zip', open(zipPath, 'rb'), 'application/zip'), + } + postUrl = f'https://joj.sjtu.edu.cn/d/{self.courseID}/homework/{homeworkID}/{problemID}/submit' + html = self.sess.get(postUrl).text + soup = BeautifulSoup(html, features="lxml") + csrfToken = soup.select( + "#panel > div.main > div > div.medium-9.columns > div:nth-child(2) > div.section__body > form > div:nth-child(3) > div > input[type=hidden]:nth-child(1)" + )[0].get('value') + response = self.sess.post( + postUrl, + files=files, + data={ + 'csrf_token': csrfToken, + 'lang': lang + }, + ) + return response + + def getProblemStatus(self, url): + soup = None + while True: + html = self.sess.get(url).text + soup = BeautifulSoup(html, features="lxml") + status = soup.select( + "#status > div.section__header > h1 > span:nth-child(2)" + )[0].get_text().strip() + if status not in ["Waiting", "Compiling", "Pending", "Running"]: + break + else: + time.sleep(1) + resultSet = soup.findAll("td", class_="col--status typo") + acCount = 0 + for result in resultSet: + if "Accepted" == result.find_all('span')[1].get_text().strip(): + acCount += 1 + return acCount + + def getProblemResult(self, + homeworkID, + problemID, + zipPath, + lang, + groupName='', + fn='', + hwNum=0): + tryTime = 0 + while True: + tryTime += 1 + response = self.uploadZip(homeworkID, problemID, zipPath, lang) + if response.status_code != 200: + self.logger.error( + f"{groupName} h{hwNum} {fn} upload error, code {response.status_code}" + ) + else: + break + self.logger.debug( + f"{groupName} h{hwNum} {fn} upload succeed {response.url}") + return self.getProblemStatus(response.url) + + def checkGroupJOJProcess(self, groupNum, hwNum, jojInfo, fn, problemID): + groupName = f"hgroup-{groupNum:02}" + hwDir = os.path.join('hwrepos', groupName, f"h{hwNum}") + filePath = os.path.join(hwDir, fn) + if not os.path.exists(filePath): return 0 + with zipfile.ZipFile(filePath + ".zip", mode='w') as zf: + zf.write(filePath, fn) + res = self.getProblemResult(jojInfo["homeworkID"], problemID, + filePath + ".zip", jojInfo["lang"], + groupName, fn, hwNum) + return res + + def checkGroupJOJ(self, jojInfo): + res = {} + hwNum = self.args.hw + for i, (key, value) in enumerate(self.hgroups.items()): + with multiprocessing.Pool(len(jojInfo["problemInfo"])) as p: + scores = p.starmap( + self.checkGroupJOJProcess, + [[i, hwNum, jojInfo, fn, problemID] + for fn, problemID, _ in jojInfo["problemInfo"]]) + scores = [(scores[i], jojInfo["problemInfo"][i][2]) + for i in range(len(scores))] + jojFailExercise = min( + sum([ + int(acCount < 0.25 * totalCount) + for acCount, totalCount in scores + ]), 2) + self.logger.info(f"{key} h{hwNum} score {scores.__repr__()}") + jojFailHomework = int( + sum([item[0] for item in scores]) < 0.5 * + sum([item[1] for item in scores])) + for _, stuName in value: + res[stuName] = { + "jojFailHomework": jojFailHomework, + "jojFailExercise": jojFailExercise + } + return res + + +if __name__ == "__main__": + from settings import * + res = JOJWorker(JOJ_COURSE_ID, + SID).getProblemResult("5f66161a91df0600062ff7aa", + "5f6614eb91df0600062ff7a7", + "ex2.zip", "matlab") + print(res)
\ No newline at end of file diff --git a/worker/__init__.py b/worker/__init__.py new file mode 100644 index 0000000..78bd11f --- /dev/null +++ b/worker/__init__.py @@ -0,0 +1,3 @@ +from .CanvasWorker import CanvasWorker +from .GiteaWorker import GiteaWorker +from .JOJWorker import JOJWorker
\ No newline at end of file |