update for C & multiple files uploading & code review checking

7 files changed, 141 insertions, 58 deletions

diff --git a/README.md b/README.md
index c896928..4699c12 100644
--- a/README.md
+++ b/README.md
@@ -53,7 +53,7 @@ Please modify `JOJ_INFO` for different homework.
 ## Features
 
 - [x] At least two days before the group deadline, all students should individually complete all the mandatory tasks and push their work to their personal branch of their group git repository and issue a pull request. Students late for the individual submission must open an issue featuring: (i) apologies to the reviewer, (ii) clear explanations on why the work is late, and (iii) a request for a new deadline fitting the reviewer. The reviewer is allowed to reject the request and should set the deadline based on his/her own schedule. **(-0.5 mark)**
-- [ ] ~~A student should provide feedbacks to at least one teammate for each mandatory exercise. Low quality reviews will be ignored. Each student should receive feedbacks on his individual submission. e.g. for a three students group: student1 → student2 → student3 → student1. **(-1 mark)**~~
+- [x] A student should provide feedbacks to at least one teammate for each mandatory exercise. Low quality reviews will be ignored. Each student should receive feedbacks on his individual submission. e.g. for a three students group: student1 → student2 → student3 → student1. **(-1 mark)**
 - [x] The final group submission, done on the master branch of the group repository, should successfully compile or be interpreted. **(-1 mark)**
 - [x] Any group submission that is more than 24 hours late will be rejected. **(-2.5 marks)**
 - [x] For each exercise, the final submission must pass at least 25% of the test cases. **(-0.25 mark per exercise, up to -0.5)**
diff --git a/VG101GradeHelper.py b/VG101GradeHelper.py
index 6f03a06..6890e1e 100644
--- a/VG101GradeHelper.py
+++ b/VG101GradeHelper.py
@@ -31,6 +31,8 @@ def parse():
                         action='store_true',
                         help='generate score')
     parser.add_argument('-t', '--tidy', action='store_true', help='check tidy')
+    # TODO: automatically check moss
+    parser.add_argument('-o', '--moss', action='store_true', help='check moss')
     parser.add_argument('-d',
                         '--dir',
                         action='store_true',
@@ -71,13 +73,25 @@ if __name__ == "__main__":
     pwd = os.getcwd()
     args = parse()
     indvScores, groupScores, jojScores = {}, {}, {}
-    gitWorker = GitWorker(args, hgroups, JOJ_INFO["lang"],
-                          [item[0] for item in JOJ_INFO["problemInfo"]
-                           ]) if args.indv or args.group or args.proj else None
+    mandatoryFiles = MANDATORY_FILES
+    mandatoryFiles.extend(
+        [fn for item in JOJ_INFO["problemInfo"] for fn in item[0]])
+    mandatoryFiles = list(set(mandatoryFiles))
+    gitWorker = GitWorker(
+        args, hgroups, JOJ_INFO["lang"], mandatoryFiles,
+        OPTIONAL_FILES) if args.indv or args.group or args.proj else None
+    giteaWorker = GiteaWorker(args, GITEA_BASE_URL, ORG_NAME,
+                                GITEA_TOKEN, hgroups)
     if args.indv:
         indvScores = gitWorker.checkIndv()
     if args.group:
         groupScores = gitWorker.checkGroup()
+        tmpScores = giteaWorker.checkReview()
+        for key in groupScores.keys():
+            groupScores[key] = {
+                **groupScores.get(key, {}),
+                **tmpScores.get(key, {})
+            }
     if args.joj:
         jojWorker = JOJWorker(args, JOJ_COURSE_ID, SID, hgroups)
         jojScores = jojWorker.checkGroupJOJ(JOJ_INFO)
@@ -90,6 +104,4 @@ if __name__ == "__main__":
     if args.proj:
         projScores = gitWorker.checkProj(args.proj, args.ms)
         if args.feedback:
-            giteaWorker = GiteaWorker(args, GITEA_BASE_URL, ORG_NAME,
-                                      GITEA_TOKEN, hgroups)
             giteaWorker.raiseIssues(projScores)
diff --git a/util.py b/util.py
index 76cb182..e1dd9af 100644
--- a/util.py
+++ b/util.py
@@ -1,4 +1,6 @@
+import subprocess
 import logging
+import os
 import re
 
 
@@ -39,7 +41,26 @@ def getProjRepoName(arg):
 
 
 def passCodeQuality(path, language):
-    with open(path, encoding='utf-8', errors='replace') as f:
-        res = f.read()
     if language == "matlab":
-        return "global " not in res
\ No newline at end of file
+        with open(path, encoding='utf-8', errors='replace') as f:
+            res = f.read()
+        return "global " not in res
+    if language == "c":
+        res = subprocess.check_output(
+            ["ctags", "-R", "-x", "--sort=yes", "--c-kinds=v", path])
+        lines = res.splitlines()
+        return len([line for line in lines if b"const" not in line]) == 0
+
+
+def getAllFiles(root):
+    for f in os.listdir(root):
+        if os.path.isfile(os.path.join(root, f)):
+            yield os.path.join(f)
+    dirs = [
+        d for d in os.listdir(root)
+        if os.path.isdir(os.path.join(root, d)) and d != ".git"
+    ]
+    for d in dirs:
+        dirfiles = getAllFiles(os.path.join(root, d))
+        for f in dirfiles:
+            yield os.path.join(d, f)
diff --git a/worker/CanvasWorker.py b/worker/CanvasWorker.py
index 954d1da..3ff60e3 100644
--- a/worker/CanvasWorker.py
+++ b/worker/CanvasWorker.py
@@ -13,6 +13,7 @@ class CanvasWorker():
                  indvScores,
                  groupScores,
                  jojScores,
+                 totalScores=None,
                  logger=Logger()):
         self.args = args
         self.rubric = rubric
@@ -22,13 +23,16 @@ class CanvasWorker():
         self.assignments = self.course.get_assignments()
         self.logger = logger
         self.scores = {}
-        self.names = names
-        for key in names:
-            self.scores[key] = {
-                **indvScores.get(key, {}),
-                **groupScores.get(key, {}),
-                **jojScores.get(key, {})
-            }
+        if totalScores is None:
+            self.names = names
+            for key in names:
+                self.scores[key] = {
+                    **indvScores.get(key, {}),
+                    **groupScores.get(key, {}),
+                    **jojScores.get(key, {})
+                }
+        else:
+            self.scores = totalScores
 
     def generateHomeworkData(self, scoreInfo):
         score = 0
@@ -49,7 +53,7 @@ class CanvasWorker():
                 scoreInfo.get("jojComment", []))
         return {
             'submission': {
-                'posted_grade': score
+                'posted_grade': max(score, -2.5)
             },
             'comment': {
                 'text_comment': '\n'.join(comment)
diff --git a/worker/GitWorker.py b/worker/GitWorker.py
index f5715ba..e124b1f 100644
--- a/worker/GitWorker.py
+++ b/worker/GitWorker.py
@@ -1,5 +1,5 @@
+from util import Logger, getProjRepoName, passCodeQuality, getAllFiles
 from shutil import ignore_patterns, copytree, rmtree
-from util import Logger, getProjRepoName, passCodeQuality
 import multiprocessing
 import traceback
 import git
@@ -12,6 +12,7 @@ class GitWorker():
                  hgroups,
                  language,
                  mandatoryFiles,
+                 optionalFiles,
                  logger=Logger(),
                  processCount=16):
         self.args = args
@@ -20,6 +21,8 @@ class GitWorker():
         self.logger = logger
         self.processCount = processCount
         self.mandatoryFiles = mandatoryFiles
+        self.optionalFiles = optionalFiles
+        self.moss = None
 
     @classmethod
     def isREADME(cls, fn):
@@ -40,7 +43,7 @@ class GitWorker():
                 branch="master")
         else:
             repo = git.Repo(repoDir)
-        repo.git.fetch("--tags", "--all")
+        repo.git.fetch("--all", "-f")
         remoteBranches = [ref.name for ref in repo.remote().refs]
         scores = {
             stuName: {
@@ -75,8 +78,10 @@ class GitWorker():
                     scores[stuName]["indvComment"].append(
                         f"individual branch h{hwNum} dir missing")
                 else:
-                    for fn, path in [(fn, os.path.join(hwDir, fn))
-                                     for fn in self.mandatoryFiles]:
+                    for fn, path in [
+                        (fn, os.path.join(hwDir, fn)) for fn in
+                        [*self.mandatoryFiles, *self.optionalFiles]
+                    ]:
                         if os.path.exists(path):
                             if not passCodeQuality(path, self.language):
                                 scores[stuName]["indvLowCodeQuality"] = 1
@@ -86,12 +91,13 @@ class GitWorker():
                                     f"{repoName} {stuID} {stuName} {fn} low quality"
                                 )
                             continue
-                        self.logger.warning(
-                            f"{repoName} {stuID} {stuName} h{hwNum}/{fn} file missing"
-                        )
-                        scores[stuName]["indvFailSubmit"] = 1
-                        scores[stuName]["indvComment"].append(
-                            f"individual branch h{hwNum}/{fn} file missing")
+                        if fn in self.mandatoryFiles:
+                            self.logger.warning(
+                                f"{repoName} {stuID} {stuName} h{hwNum}/{fn} file missing"
+                            )
+                            scores[stuName]["indvFailSubmit"] = 1
+                            scores[stuName]["indvComment"].append(
+                                f"individual branch h{hwNum}/{fn} file missing")
                     if not list(filter(GitWorker.isREADME, os.listdir(hwDir))):
                         self.logger.warning(
                             f"{repoName} {stuID} {stuName} h{hwNum}/README file missing"
@@ -117,8 +123,9 @@ class GitWorker():
                     dirList = os.listdir(hwDir)
                     dirList = list(
                         filter(
-                            lambda x: not x.startswith("ex") and not GitWorker.
-                            isREADME(x), dirList))
+                            lambda x: x not in self.mandatoryFiles and x not in
+                            self.optionalFiles and not GitWorker.isREADME(x),
+                            dirList))
                     if dirList:
                         self.logger.warning(
                             f"{repoName} {stuID} {stuName} h{hwNum}/ untidy {', '.join(dirList)}"
@@ -144,7 +151,7 @@ class GitWorker():
                 branch="master")
         else:
             repo = git.Repo(repoDir)
-        repo.git.fetch("--tags", "--all")
+        repo.git.fetch("--tags", "--all", "-f")
         tagNames = [tag.name for tag in repo.tags]
         scores = {
             stuName: {
@@ -173,8 +180,10 @@ class GitWorker():
                 scores[stuName]["groupComment"].append(
                     f"tags/h{hwNum} h{hwNum} dir missing")
         else:
-            for fn, path in [(fn, os.path.join(hwDir, fn))
-                             for fn in self.mandatoryFiles]:
+            for fn, path in [
+                (fn, os.path.join(hwDir, fn))
+                    for fn in [*self.mandatoryFiles, *self.optionalFiles]
+            ]:
                 if os.path.exists(path):
                     if not passCodeQuality(path, self.language):
                         for _, stuName in self.hgroups[repoName]:
@@ -183,11 +192,12 @@ class GitWorker():
                                 f"group {fn} low quality")
                         self.logger.warning(f"{repoName} {fn} low quality")
                     continue
-                self.logger.warning(f"{repoName} h{hwNum}/{fn} file missing")
-                for _, stuName in self.hgroups[repoName]:
-                    scores[stuName]["groupFailSubmit"] = 1
-                    scores[stuName]["groupComment"].append(
-                        f"tags/h{hwNum} h{hwNum}/{fn} missing")
+                if fn in self.mandatoryFiles:
+                    self.logger.warning(f"{repoName} h{hwNum}/{fn} file missing")
+                    for _, stuName in self.hgroups[repoName]:
+                        scores[stuName]["groupFailSubmit"] = 1
+                        scores[stuName]["groupComment"].append(
+                            f"tags/h{hwNum} h{hwNum}/{fn} missing")
             if not list(filter(GitWorker.isREADME, os.listdir(hwDir))):
                 self.logger.warning(f"{repoName} h{hwNum}/README file missing")
                 for _, stuName in self.hgroups[repoName]:
@@ -212,8 +222,8 @@ class GitWorker():
             dirList = os.listdir(hwDir)
             dirList = list(
                 filter(
-                    lambda x: not x.startswith("ex") and not GitWorker.
-                    isREADME(x), dirList))
+                    lambda x: x not in self.mandatoryFiles and x not in self.
+                    optionalFiles and not GitWorker.isREADME(x), dirList))
             if dirList:
                 self.logger.warning(
                     f"{repoName} h{hwNum} untidy {', '.join(dirList)}")
@@ -237,7 +247,7 @@ class GitWorker():
                 f"https://focs.ji.sjtu.edu.cn/git/vg101/{repoName}", repoDir)
         else:
             repo = git.Repo(os.path.join("projrepos", f"p{projNum}", repoName))
-            repo.git.fetch("--tags", "--all")
+            repo.git.fetch("--tags", "--all", "-f")
             remoteBranches = [ref.name for ref in repo.remote().refs]
             if "origin/master" not in remoteBranches:
                 self.logger.warning(f"{repoName} master branch missing")
@@ -246,7 +256,7 @@ class GitWorker():
             repo.git.reset("--hard", "origin/master")
             repo.git.clean("-d", "-f", "-x")
         if milestoneNum:
-            repo.git.fetch("--tags", "--all")
+            repo.git.fetch("--tags", "--all", "-f")
             tagNames = [tag.name for tag in repo.tags]
             if f"m{milestoneNum}" not in tagNames:
                 self.logger.warning(f"{repoName} tags/m{milestoneNum} missing")
@@ -259,12 +269,22 @@ class GitWorker():
             if not list(filter(GitWorker.isREADME, os.listdir(repoDir))):
                 self.logger.warning(f"{repoName} README file missing")
                 scores[stuName]["projComment"].append(f"README file missing")
+            language = ["matlab", "c", "cpp"]
             if projNum == 1:
                 for fn in list(
                         filter(lambda x: x.endswith(".m"),
                                os.listdir(repoDir))):
                     path = os.path.join(repoDir, fn)
-                    if not passCodeQuality(path, "matlab"):
+                    if not passCodeQuality(path, language[projNum - 1]):
+                        self.logger.warning(f"{repoName} {fn} low quality")
+                        scores[stuName]["projComment"].append(
+                            f"{fn} low quality")
+            elif projNum == 2:
+                for fn in getAllFiles(repoDir):
+                    if (fn.endswith(".c")
+                            or fn.endswith(".h")) and not passCodeQuality(
+                                os.path.join(repoDir, fn),
+                                language[projNum - 1]):
                         self.logger.warning(f"{repoName} {fn} low quality")
                         scores[stuName]["projComment"].append(
                             f"{fn} low quality")
diff --git a/worker/GiteaWorker.py b/worker/GiteaWorker.py
index 55dd41c..1b85ea9 100644
--- a/worker/GiteaWorker.py
+++ b/worker/GiteaWorker.py
@@ -11,6 +11,11 @@ class GiteaWorker():
             item[1]: item[0]
             for items in hgroups.values() for item in items
         }
+        self.ids = {
+            item[0]: item[1]
+            for items in hgroups.values() for item in items
+        }
+        self.hgroups = hgroups
         self.baseUrl = baseUrl
         self.orgName = orgName
         self.sess = requests.Session()
@@ -29,3 +34,20 @@ class GiteaWorker():
             }
             req = self.sess.post(url, data)
             self.logger.debug(f"{repoName} issue {req.status_code} {req.text}")
+
+    def checkReview(self):
+        hwNum = self.args.hw
+        res = {key: {"noReview": 1} for key in self.names.keys()}
+        for repoName, users in self.hgroups.items():
+            url = f"{self.baseUrl}/repos/{self.orgName}/{repoName}/pulls"
+            pulls = self.sess.get(url).json()
+            for pull in pulls:
+                if not pull["title"].startswith(f"h{hwNum}"): continue
+                url = f"{self.baseUrl}/repos/{self.orgName}/{repoName}/pulls/{pull['number']}/reviews"
+                self.logger.info(f"{repoName} h{hwNum} get pr: {url}")
+                for item in self.sess.get(url).json():
+                    stuID = ''.join(
+                        [s for s in item['user']['full_name'] if s.isdigit()])
+                    name = self.ids[stuID]
+                    res[name]["noReview"] = 0
+        return res
\ No newline at end of file
diff --git a/worker/JOJWorker.py b/worker/JOJWorker.py
index aed12da..b0f316d 100644
--- a/worker/JOJWorker.py
+++ b/worker/JOJWorker.py
@@ -70,7 +70,6 @@ class JOJWorker():
                          zipPath,
                          lang,
                          groupName='',
-                         fn='',
                          hwNum=0):
         tryTime = 0
         while True:
@@ -79,27 +78,32 @@ class JOJWorker():
             if response.status_code == 200:
                 break
             self.logger.error(
-                f"{groupName} h{hwNum} {fn} upload error, code {response.status_code}, url {response.url}"
+                f"{groupName} h{hwNum} {problemID} upload error, code {response.status_code}, url {response.url}"
             )
             time.sleep(1)
         self.logger.debug(
-            f"{groupName} h{hwNum} {fn} upload succeed, url {response.url}")
+            f"{groupName} h{hwNum} {problemID} upload succeed, url {response.url}")
         return self.getProblemStatus(response.url)
 
-    def checkGroupJOJProcess(self, groupNum, hwNum, jojInfo, fn, problemID):
+    def checkGroupJOJProcess(self, groupNum, hwNum, jojInfo, fns, problemID):
         groupName = f"hgroup-{groupNum:02}"
         hwDir = os.path.join('hwrepos', groupName, f"h{hwNum}")
-        filePath = os.path.join(hwDir, fn)
-        if not os.path.exists(filePath):
-            self.logger.warning(f"{groupName} h{hwNum} {fn} not exist")
-            return 0
-        if os.path.exists(filePath + ".zip"): os.remove(filePath + ".zip")
-        with zipfile.ZipFile(filePath + ".zip", mode='w') as zf:
-            zf.write(filePath, fn)
+        if not os.path.exists(hwDir): return 0
+        zipPath = os.path.join(hwDir, problemID) + ".zip"
+        if os.path.exists(zipPath): os.remove(zipPath)
+        with zipfile.ZipFile(zipPath, mode='w') as zf:
+            for fn in fns:
+                filePath = os.path.join(hwDir, fn)
+                if not os.path.exists(filePath):
+                    if not fn.endswith(".h"):
+                        self.logger.warning(f"{groupName} h{hwNum} {fn} not exist")
+                        return 0
+                else:
+                    zf.write(filePath, fn)
         res = self.getProblemResult(jojInfo["homeworkID"], problemID,
-                                    filePath + ".zip", jojInfo["lang"],
-                                    groupName, fn, hwNum)
-        os.remove(filePath + ".zip")
+                                    zipPath, jojInfo["lang"],
+                                    groupName, hwNum)
+        # os.remove(zipPath)
         return res
 
     def checkGroupJOJ(self, jojInfo):
@@ -109,8 +113,8 @@ class JOJWorker():
             with multiprocessing.Pool(len(jojInfo["problemInfo"])) as p:
                 scores = p.starmap(
                     self.checkGroupJOJProcess,
-                    [[i, hwNum, jojInfo, fn, problemID]
-                     for fn, problemID, _ in jojInfo["problemInfo"]])
+                    [[i, hwNum, jojInfo, fns, problemID]
+                     for fns, problemID, _ in jojInfo["problemInfo"]])
             scores = [(scores[i], jojInfo["problemInfo"][i][2])
                       for i in range(len(scores))]
             self.logger.info(f"{key} h{hwNum} score {scores.__repr__()}")