update: rubric detail

1 files changed, 278 insertions, 0 deletions

diff --git a/worker/GitWorker.py b/worker/GitWorker.py
new file mode 100644
index 0000000..1213e98
--- /dev/null
+++ b/worker/GitWorker.py
@@ -0,0 +1,278 @@
+from shutil import ignore_patterns, copytree, rmtree
+from util import Logger
+import multiprocessing
+import traceback
+import git
+import os
+import re
+
+
+class GiteaWorker():
+    def __init__(self,
+                 args,
+                 hgroups,
+                 mandatoryFiles,
+                 logger=Logger(),
+                 processCount=16):
+        self.args = args
+        self.hgroups = hgroups
+        self.logger = logger
+        self.processCount = processCount
+        self.mandatoryFiles = mandatoryFiles
+
+    @classmethod
+    def isREADME(cls, fn):
+        fn = fn.lower()
+        if len(fn) < 6: return False
+        if len(fn) == 6: return fn == "readme"
+        return fn[:7] == "readme."
+
+    def checkProjRepoName(self, arg):
+        id_, name, projNum, *_ = arg
+        eng = re.sub('[\u4e00-\u9fa5]', '', name)
+        eng = ''.join(
+            [word[0].capitalize() + word[1:] for word in eng.split()])
+        return f"{eng}{id_}-p{projNum}"
+
+    def checkIndvProcess(self, groupNum, hwNum):
+        tidy = self.args.tidy
+        repoName = f"hgroup-{groupNum:02}"
+        repoDir = os.path.join('hwrepos', repoName)
+        hwDir = os.path.join(repoDir, f"h{hwNum}")
+        if not os.path.exists(repoDir):
+            repo = git.Repo.clone_from(
+                f"https://focs.ji.sjtu.edu.cn/git/vg101/{repoName}",
+                repoDir,
+                branch='master')
+        else:
+            repo = git.Repo(repoDir)
+        repo.git.fetch()
+        remoteBranches = [ref.name for ref in repo.remote().refs]
+        scores = {
+            stuName: {
+                "indvFailSubmit": 0,
+                "indvUntidy": 0,
+                "indvComment": [],
+            }
+            for _, stuName in self.hgroups[repoName]
+        }
+        for stuID, stuName in self.hgroups[repoName]:
+            try:
+                if f"origin/{stuID}" not in remoteBranches:
+                    self.logger.warning(
+                        f"{repoName} {stuID} {stuName} branch missing")
+                    scores[stuName]['indvFailSubmit'] = 1
+                    scores[stuName]['indvComment'].append(
+                        "individual branch individual branch missing")
+                    continue
+                repo.git.checkout(f"{stuID}", "-f")
+                repo.git.reset('--hard')
+                repo.git.pull("origin", f"{stuID}", "-f")
+                repo.git.reset(f"origin/{stuID}", "--hard")
+                repo.git.clean("-d", "-f", "-x")
+                self.logger.debug(f"{repoName} {stuID} {stuName} pull succeed")
+                if self.args.dir:
+                    copytree(repoDir,
+                             os.path.join('indv',
+                                          f"{repoName} {stuID} {stuName}"),
+                             ignore=ignore_patterns('.git'))
+                if not os.path.exists(hwDir):
+                    self.logger.warning(
+                        f"{repoName} {stuID} {stuName} h{hwNum} dir missing")
+                    scores[stuName]['indvFailSubmit'] = 1
+                    scores[stuName]['indvComment'].append(
+                        f"individual branch h{hwNum} dir missing")
+                else:
+                    for fn, path in [(fn, os.path.join(hwDir, fn))
+                                     for fn in self.mandatoryFiles]:
+                        if os.path.exists(path): continue
+                        self.logger.warning(
+                            f"{repoName} {stuID} {stuName} h{hwNum}/{fn} file missing"
+                        )
+                        scores[stuName]['indvFailSubmit'] = 1
+                        scores[stuName]['indvComment'].append(
+                            f"individual branch h{hwNum}/{fn} file missing")
+                    if not list(filter(GiteaWorker.isREADME,
+                                       os.listdir(hwDir))):
+                        self.logger.warning(
+                            f"{repoName} {stuID} {stuName} h{hwNum}/README file missing"
+                        )
+                        scores[stuName]['indvFailSubmit'] = 1
+                        scores[stuName]['indvComment'].append(
+                            f"individual branch h{hwNum}/README file missing")
+                if not tidy: continue
+                dirList = list(
+                    filter(
+                        lambda x: x not in [
+                            ".git", *[f"h{n}" for n in range(20)]
+                        ] and not GiteaWorker.isREADME(x),
+                        os.listdir(repoDir)))
+                if dirList:
+                    self.logger.warning(
+                        f"{repoName} {stuID} {stuName} untidy {dirList.__repr__()}"
+                    )
+                    scores[stuName]['indvUntidy'] = 1
+                    scores[stuName]['indvComment'].append(
+                        f"individual branch redundant files: {dirList.__repr__()}"
+                    )
+                if os.path.exists(hwDir):
+                    dirList = os.listdir(hwDir)
+                    dirList = list(
+                        filter(
+                            lambda x: not x.startswith("ex") and
+                            not GiteaWorker.isREADME(x), dirList))
+                    if dirList:
+                        self.logger.warning(
+                            f"{repoName} {stuID} {stuName} h{hwNum}/ untidy {dirList.__repr__()}"
+                        )
+                        scores[stuName]['indvUntidy'] = 1
+                        scores[stuName]['indvComment'].append(
+                            f"individual branch redundant files: {dirList.__repr__()}"
+                        )
+            except Exception:
+                self.logger.error(f"{repoName} {stuID} {stuName} error")
+                self.logger.error(traceback.format_exc())
+        return scores
+
+    def checkGroupProcess(self, groupNum, hwNum):
+        tidy = self.args.tidy
+        repoName = f"hgroup-{groupNum:02}"
+        repoDir = os.path.join('hwrepos', repoName)
+        hwDir = os.path.join(repoDir, f"h{hwNum}")
+        if not os.path.exists(repoDir):
+            repo = git.Repo.clone_from(
+                f"https://focs.ji.sjtu.edu.cn/git/vg101/{repoName}",
+                repoDir,
+                branch='master')
+        else:
+            repo = git.Repo(repoDir)
+        repo.git.checkout("master", "-f")
+        repo.git.fetch("--tags", "-f")
+        tagNames = [tag.name for tag in repo.tags]
+        scores = {
+            stuName: {
+                "groupFailSubmit": 0,
+                "groupUntidy": 0,
+                "groupComment": [],
+            }
+            for _, stuName in self.hgroups[repoName]
+        }
+        if f"h{hwNum}" not in tagNames:
+            self.logger.warning(f"{repoName} tags/h{hwNum} missing")
+            for _, stuName in self.hgroups[repoName]:
+                scores[stuName]['groupFailSubmit'] = 1
+                scores[stuName]['groupComment'].append(
+                    f"tags/h{hwNum} missing")
+            return scores
+        repo.git.checkout(f"tags/h{hwNum}", "-f")
+        if not os.path.exists(hwDir):
+            self.logger.warning(f"{repoName} h{hwNum} dir missing")
+            for _, stuName in self.hgroups[repoName]:
+                scores[stuName]['groupFailSubmit'] = 1
+                scores[stuName]['groupComment'].append(
+                    f"master branch h{hwNum} dir missing")
+        else:
+            for fn, path in [(fn, os.path.join(hwDir, fn))
+                             for fn in self.mandatoryFiles]:
+                if os.path.exists(path): continue
+                self.logger.warning(f"{repoName} h{hwNum}/{fn} file missing")
+                for _, stuName in self.hgroups[repoName]:
+                    scores[stuName]['groupFailSubmit'] = 1
+                    scores[stuName]['groupComment'].append(
+                        f"master branch h{hwNum}/{fn} missing")
+            if not list(filter(GiteaWorker.isREADME, os.listdir(hwDir))):
+                self.logger.warning(f"{repoName} h{hwNum}/README file missing")
+                for _, stuName in self.hgroups[repoName]:
+                    scores[stuName]['groupFailSubmit'] = 1
+                    scores[stuName]['groupComment'].append(
+                        f"master branch h{hwNum}/README file missing")
+        self.logger.debug(f"{repoName} checkout to tags/h{hwNum} succeed")
+        if not tidy: return scores
+        dirList = os.listdir(repoDir)
+        dirList = list(
+            filter(
+                lambda x: x not in [".git", *[f"h{n}" for n in range(20)]] and
+                not GiteaWorker.isREADME(x), dirList))
+        if dirList:
+            self.logger.warning(f"{repoName} untidy {dirList.__repr__()}")
+            for _, stuName in self.hgroups[repoName]:
+                scores[stuName]['groupUntidy'] = 1
+                scores[stuName]['groupComment'].append(
+                    f"master branch redundant files: {dirList.__repr__()}")
+        if os.path.exists(hwDir):
+            dirList = os.listdir(hwDir)
+            dirList = list(
+                filter(
+                    lambda x: not x.startswith("ex") and not GiteaWorker.
+                    isREADME(x), dirList))
+            if dirList:
+                self.logger.warning(
+                    f"{repoName} h{hwNum} untidy {dirList.__repr__()}")
+                for _, stuName in self.hgroups[repoName]:
+                    scores[stuName]['groupUntidy'] = 1
+                    scores[stuName]['groupComment'].append(
+                        f"master branch redundant files: {dirList.__repr__()}")
+        return scores
+
+    def checkProjProcess(self, id_, name, projNum, milestoneNum):
+        repoName = self.checkProjRepoName([id_, name, projNum, milestoneNum])
+        repoDir = os.path.join('projrepos', f'p{projNum}', repoName)
+        if not os.path.exists(repoDir):
+            repo = git.Repo.clone_from(
+                f"https://focs.ji.sjtu.edu.cn/git/vg101/{repoName}", repoDir)
+        else:
+            repo = git.Repo(os.path.join('projrepos', f'p{projNum}', repoName))
+            repo.git.fetch()
+            remoteBranches = [ref.name for ref in repo.remote().refs]
+            if 'origin/master' not in remoteBranches:
+                self.logger.warning(f"{repoName} branch master missing")
+                return
+            repo.git.checkout(f"master", "-f")
+            repo.git.reset('--hard')
+            repo.git.pull("origin", "master", "-f")
+            repo.git.reset('--hard')
+            repo.git.clean("-d", "-f", "-x")
+        if not list(filter(GiteaWorker.isREADME, os.listdir(repoDir))):
+            self.logger.warning(f"{repoName} README missing")
+        if milestoneNum:
+            tagNames = [tag.name for tag in repo.tags]
+            if f"m{milestoneNum}" not in tagNames:
+                self.logger.warning(f"{repoName} tags/m{milestoneNum} missing")
+                return
+            repo.git.checkout(f"tags/m{milestoneNum}", "-f")
+            self.logger.debug(
+                f"{repoName} checkout to tags/m{milestoneNum} succeed")
+        else:
+            self.logger.debug(f"{repoName} pull succeed")
+
+    def checkIndv(self):
+        if self.args.dir:
+            if os.path.exists(os.path.join('indv')):
+                rmtree(os.path.join('indv'))
+        hwNum = self.args.hw
+        with multiprocessing.Pool(self.processCount) as p:
+            res = p.starmap(self.checkIndvProcess,
+                            [(i, hwNum)
+                             for i in range(len(self.hgroups.keys()))])
+        return {k: v for d in res for k, v in d.items()}
+
+    def checkGroup(self):
+        hwNum = self.args.hw
+        with multiprocessing.Pool(self.processCount) as p:
+            res = p.starmap(self.checkGroupProcess,
+                            [(i, hwNum)
+                             for i in range(len(self.hgroups.keys()))])
+        return {k: v for d in res for k, v in d.items()}
+
+    def checkProj(self, projNum, milestoneNum):
+        milestoneNum = 0 if milestoneNum is None else milestoneNum
+        if projNum in [1, 2]:
+            infos = [[*info, projNum, milestoneNum]
+                     for hgroup in self.hgroups.values() for info in hgroup]
+        elif projNum in [3]:
+            infos = []
+            return
+        else:
+            return
+        with multiprocessing.Pool(self.processCount) as p:
+            p.starmap(self.checkProjProcess, infos)
\ No newline at end of file