update for C & multiple files uploading & code review checking

4 files changed, 99 insertions, 49 deletions

diff --git a/worker/CanvasWorker.py b/worker/CanvasWorker.py
index 954d1da..3ff60e3 100644
--- a/worker/CanvasWorker.py
+++ b/worker/CanvasWorker.py
@@ -13,6 +13,7 @@ class CanvasWorker():
                  indvScores,
                  groupScores,
                  jojScores,
+                 totalScores=None,
                  logger=Logger()):
         self.args = args
         self.rubric = rubric
@@ -22,13 +23,16 @@ class CanvasWorker():
         self.assignments = self.course.get_assignments()
         self.logger = logger
         self.scores = {}
-        self.names = names
-        for key in names:
-            self.scores[key] = {
-                **indvScores.get(key, {}),
-                **groupScores.get(key, {}),
-                **jojScores.get(key, {})
-            }
+        if totalScores is None:
+            self.names = names
+            for key in names:
+                self.scores[key] = {
+                    **indvScores.get(key, {}),
+                    **groupScores.get(key, {}),
+                    **jojScores.get(key, {})
+                }
+        else:
+            self.scores = totalScores
 
     def generateHomeworkData(self, scoreInfo):
         score = 0
@@ -49,7 +53,7 @@ class CanvasWorker():
                 scoreInfo.get("jojComment", []))
         return {
             'submission': {
-                'posted_grade': score
+                'posted_grade': max(score, -2.5)
             },
             'comment': {
                 'text_comment': '\n'.join(comment)
diff --git a/worker/GitWorker.py b/worker/GitWorker.py
index f5715ba..e124b1f 100644
--- a/worker/GitWorker.py
+++ b/worker/GitWorker.py
@@ -1,5 +1,5 @@
+from util import Logger, getProjRepoName, passCodeQuality, getAllFiles
 from shutil import ignore_patterns, copytree, rmtree
-from util import Logger, getProjRepoName, passCodeQuality
 import multiprocessing
 import traceback
 import git
@@ -12,6 +12,7 @@ class GitWorker():
                  hgroups,
                  language,
                  mandatoryFiles,
+                 optionalFiles,
                  logger=Logger(),
                  processCount=16):
         self.args = args
@@ -20,6 +21,8 @@ class GitWorker():
         self.logger = logger
         self.processCount = processCount
         self.mandatoryFiles = mandatoryFiles
+        self.optionalFiles = optionalFiles
+        self.moss = None
 
     @classmethod
     def isREADME(cls, fn):
@@ -40,7 +43,7 @@ class GitWorker():
                 branch="master")
         else:
             repo = git.Repo(repoDir)
-        repo.git.fetch("--tags", "--all")
+        repo.git.fetch("--all", "-f")
         remoteBranches = [ref.name for ref in repo.remote().refs]
         scores = {
             stuName: {
@@ -75,8 +78,10 @@ class GitWorker():
                     scores[stuName]["indvComment"].append(
                         f"individual branch h{hwNum} dir missing")
                 else:
-                    for fn, path in [(fn, os.path.join(hwDir, fn))
-                                     for fn in self.mandatoryFiles]:
+                    for fn, path in [
+                        (fn, os.path.join(hwDir, fn)) for fn in
+                        [*self.mandatoryFiles, *self.optionalFiles]
+                    ]:
                         if os.path.exists(path):
                             if not passCodeQuality(path, self.language):
                                 scores[stuName]["indvLowCodeQuality"] = 1
@@ -86,12 +91,13 @@ class GitWorker():
                                     f"{repoName} {stuID} {stuName} {fn} low quality"
                                 )
                             continue
-                        self.logger.warning(
-                            f"{repoName} {stuID} {stuName} h{hwNum}/{fn} file missing"
-                        )
-                        scores[stuName]["indvFailSubmit"] = 1
-                        scores[stuName]["indvComment"].append(
-                            f"individual branch h{hwNum}/{fn} file missing")
+                        if fn in self.mandatoryFiles:
+                            self.logger.warning(
+                                f"{repoName} {stuID} {stuName} h{hwNum}/{fn} file missing"
+                            )
+                            scores[stuName]["indvFailSubmit"] = 1
+                            scores[stuName]["indvComment"].append(
+                                f"individual branch h{hwNum}/{fn} file missing")
                     if not list(filter(GitWorker.isREADME, os.listdir(hwDir))):
                         self.logger.warning(
                             f"{repoName} {stuID} {stuName} h{hwNum}/README file missing"
@@ -117,8 +123,9 @@ class GitWorker():
                     dirList = os.listdir(hwDir)
                     dirList = list(
                         filter(
-                            lambda x: not x.startswith("ex") and not GitWorker.
-                            isREADME(x), dirList))
+                            lambda x: x not in self.mandatoryFiles and x not in
+                            self.optionalFiles and not GitWorker.isREADME(x),
+                            dirList))
                     if dirList:
                         self.logger.warning(
                             f"{repoName} {stuID} {stuName} h{hwNum}/ untidy {', '.join(dirList)}"
@@ -144,7 +151,7 @@ class GitWorker():
                 branch="master")
         else:
             repo = git.Repo(repoDir)
-        repo.git.fetch("--tags", "--all")
+        repo.git.fetch("--tags", "--all", "-f")
         tagNames = [tag.name for tag in repo.tags]
         scores = {
             stuName: {
@@ -173,8 +180,10 @@ class GitWorker():
                 scores[stuName]["groupComment"].append(
                     f"tags/h{hwNum} h{hwNum} dir missing")
         else:
-            for fn, path in [(fn, os.path.join(hwDir, fn))
-                             for fn in self.mandatoryFiles]:
+            for fn, path in [
+                (fn, os.path.join(hwDir, fn))
+                    for fn in [*self.mandatoryFiles, *self.optionalFiles]
+            ]:
                 if os.path.exists(path):
                     if not passCodeQuality(path, self.language):
                         for _, stuName in self.hgroups[repoName]:
@@ -183,11 +192,12 @@ class GitWorker():
                                 f"group {fn} low quality")
                         self.logger.warning(f"{repoName} {fn} low quality")
                     continue
-                self.logger.warning(f"{repoName} h{hwNum}/{fn} file missing")
-                for _, stuName in self.hgroups[repoName]:
-                    scores[stuName]["groupFailSubmit"] = 1
-                    scores[stuName]["groupComment"].append(
-                        f"tags/h{hwNum} h{hwNum}/{fn} missing")
+                if fn in self.mandatoryFiles:
+                    self.logger.warning(f"{repoName} h{hwNum}/{fn} file missing")
+                    for _, stuName in self.hgroups[repoName]:
+                        scores[stuName]["groupFailSubmit"] = 1
+                        scores[stuName]["groupComment"].append(
+                            f"tags/h{hwNum} h{hwNum}/{fn} missing")
             if not list(filter(GitWorker.isREADME, os.listdir(hwDir))):
                 self.logger.warning(f"{repoName} h{hwNum}/README file missing")
                 for _, stuName in self.hgroups[repoName]:
@@ -212,8 +222,8 @@ class GitWorker():
             dirList = os.listdir(hwDir)
             dirList = list(
                 filter(
-                    lambda x: not x.startswith("ex") and not GitWorker.
-                    isREADME(x), dirList))
+                    lambda x: x not in self.mandatoryFiles and x not in self.
+                    optionalFiles and not GitWorker.isREADME(x), dirList))
             if dirList:
                 self.logger.warning(
                     f"{repoName} h{hwNum} untidy {', '.join(dirList)}")
@@ -237,7 +247,7 @@ class GitWorker():
                 f"https://focs.ji.sjtu.edu.cn/git/vg101/{repoName}", repoDir)
         else:
             repo = git.Repo(os.path.join("projrepos", f"p{projNum}", repoName))
-            repo.git.fetch("--tags", "--all")
+            repo.git.fetch("--tags", "--all", "-f")
             remoteBranches = [ref.name for ref in repo.remote().refs]
             if "origin/master" not in remoteBranches:
                 self.logger.warning(f"{repoName} master branch missing")
@@ -246,7 +256,7 @@ class GitWorker():
             repo.git.reset("--hard", "origin/master")
             repo.git.clean("-d", "-f", "-x")
         if milestoneNum:
-            repo.git.fetch("--tags", "--all")
+            repo.git.fetch("--tags", "--all", "-f")
             tagNames = [tag.name for tag in repo.tags]
             if f"m{milestoneNum}" not in tagNames:
                 self.logger.warning(f"{repoName} tags/m{milestoneNum} missing")
@@ -259,12 +269,22 @@ class GitWorker():
             if not list(filter(GitWorker.isREADME, os.listdir(repoDir))):
                 self.logger.warning(f"{repoName} README file missing")
                 scores[stuName]["projComment"].append(f"README file missing")
+            language = ["matlab", "c", "cpp"]
             if projNum == 1:
                 for fn in list(
                         filter(lambda x: x.endswith(".m"),
                                os.listdir(repoDir))):
                     path = os.path.join(repoDir, fn)
-                    if not passCodeQuality(path, "matlab"):
+                    if not passCodeQuality(path, language[projNum - 1]):
+                        self.logger.warning(f"{repoName} {fn} low quality")
+                        scores[stuName]["projComment"].append(
+                            f"{fn} low quality")
+            elif projNum == 2:
+                for fn in getAllFiles(repoDir):
+                    if (fn.endswith(".c")
+                            or fn.endswith(".h")) and not passCodeQuality(
+                                os.path.join(repoDir, fn),
+                                language[projNum - 1]):
                         self.logger.warning(f"{repoName} {fn} low quality")
                         scores[stuName]["projComment"].append(
                             f"{fn} low quality")
diff --git a/worker/GiteaWorker.py b/worker/GiteaWorker.py
index 55dd41c..1b85ea9 100644
--- a/worker/GiteaWorker.py
+++ b/worker/GiteaWorker.py
@@ -11,6 +11,11 @@ class GiteaWorker():
             item[1]: item[0]
             for items in hgroups.values() for item in items
         }
+        self.ids = {
+            item[0]: item[1]
+            for items in hgroups.values() for item in items
+        }
+        self.hgroups = hgroups
         self.baseUrl = baseUrl
         self.orgName = orgName
         self.sess = requests.Session()
@@ -29,3 +34,20 @@ class GiteaWorker():
             }
             req = self.sess.post(url, data)
             self.logger.debug(f"{repoName} issue {req.status_code} {req.text}")
+
+    def checkReview(self):
+        hwNum = self.args.hw
+        res = {key: {"noReview": 1} for key in self.names.keys()}
+        for repoName, users in self.hgroups.items():
+            url = f"{self.baseUrl}/repos/{self.orgName}/{repoName}/pulls"
+            pulls = self.sess.get(url).json()
+            for pull in pulls:
+                if not pull["title"].startswith(f"h{hwNum}"): continue
+                url = f"{self.baseUrl}/repos/{self.orgName}/{repoName}/pulls/{pull['number']}/reviews"
+                self.logger.info(f"{repoName} h{hwNum} get pr: {url}")
+                for item in self.sess.get(url).json():
+                    stuID = ''.join(
+                        [s for s in item['user']['full_name'] if s.isdigit()])
+                    name = self.ids[stuID]
+                    res[name]["noReview"] = 0
+        return res
\ No newline at end of file
diff --git a/worker/JOJWorker.py b/worker/JOJWorker.py
index aed12da..b0f316d 100644
--- a/worker/JOJWorker.py
+++ b/worker/JOJWorker.py
@@ -70,7 +70,6 @@ class JOJWorker():
                          zipPath,
                          lang,
                          groupName='',
-                         fn='',
                          hwNum=0):
         tryTime = 0
         while True:
@@ -79,27 +78,32 @@ class JOJWorker():
             if response.status_code == 200:
                 break
             self.logger.error(
-                f"{groupName} h{hwNum} {fn} upload error, code {response.status_code}, url {response.url}"
+                f"{groupName} h{hwNum} {problemID} upload error, code {response.status_code}, url {response.url}"
             )
             time.sleep(1)
         self.logger.debug(
-            f"{groupName} h{hwNum} {fn} upload succeed, url {response.url}")
+            f"{groupName} h{hwNum} {problemID} upload succeed, url {response.url}")
         return self.getProblemStatus(response.url)
 
-    def checkGroupJOJProcess(self, groupNum, hwNum, jojInfo, fn, problemID):
+    def checkGroupJOJProcess(self, groupNum, hwNum, jojInfo, fns, problemID):
         groupName = f"hgroup-{groupNum:02}"
         hwDir = os.path.join('hwrepos', groupName, f"h{hwNum}")
-        filePath = os.path.join(hwDir, fn)
-        if not os.path.exists(filePath):
-            self.logger.warning(f"{groupName} h{hwNum} {fn} not exist")
-            return 0
-        if os.path.exists(filePath + ".zip"): os.remove(filePath + ".zip")
-        with zipfile.ZipFile(filePath + ".zip", mode='w') as zf:
-            zf.write(filePath, fn)
+        if not os.path.exists(hwDir): return 0
+        zipPath = os.path.join(hwDir, problemID) + ".zip"
+        if os.path.exists(zipPath): os.remove(zipPath)
+        with zipfile.ZipFile(zipPath, mode='w') as zf:
+            for fn in fns:
+                filePath = os.path.join(hwDir, fn)
+                if not os.path.exists(filePath):
+                    if not fn.endswith(".h"):
+                        self.logger.warning(f"{groupName} h{hwNum} {fn} not exist")
+                        return 0
+                else:
+                    zf.write(filePath, fn)
         res = self.getProblemResult(jojInfo["homeworkID"], problemID,
-                                    filePath + ".zip", jojInfo["lang"],
-                                    groupName, fn, hwNum)
-        os.remove(filePath + ".zip")
+                                    zipPath, jojInfo["lang"],
+                                    groupName, hwNum)
+        # os.remove(zipPath)
         return res
 
     def checkGroupJOJ(self, jojInfo):
@@ -109,8 +113,8 @@ class JOJWorker():
             with multiprocessing.Pool(len(jojInfo["problemInfo"])) as p:
                 scores = p.starmap(
                     self.checkGroupJOJProcess,
-                    [[i, hwNum, jojInfo, fn, problemID]
-                     for fn, problemID, _ in jojInfo["problemInfo"]])
+                    [[i, hwNum, jojInfo, fns, problemID]
+                     for fns, problemID, _ in jojInfo["problemInfo"]])
             scores = [(scores[i], jojInfo["problemInfo"][i][2])
                       for i in range(len(scores))]
             self.logger.info(f"{key} h{hwNum} score {scores.__repr__()}")