Adding exit code ok status as a flag in the test results.

Author: krulis-martin

app/helpers/Evaluation/EvaluationResults/TestResult.php

@@ -253,17 +253,51 @@ public function isMemoryOK(): bool
     }
 
     /**
-     * Get the return code
-     * @return int If all tasks are successful, return 0. If not, return first nonzero code returned.
+     * Get the exit code. If multiple processes are executed, the exit code is determined by priorities:
+     * - regular non-success exit codes
+     * - irregular (timeout, signal) non-success exit codes
+     * - success exit codes
+     * EXIT_CODE_UNKNOWN is returned if no process was executed
+     * @return int exit code
      */
     public function getExitCode(): int
     {
+        // first, try to find regular but non-success exit code
         foreach ($this->sandboxResultsList as $results) {
-            if ($results->getExitCode() !== 0) {
+            if ($results->getStatus() === ISandboxResults::STATUS_RE) {
                 return $results->getExitCode();
             }
         }
-        return 0;
+
+        // then, try to find any non-success code
+        foreach ($this->sandboxResultsList as $results) {
+            if ($results->getStatus() !== ISandboxResults::STATUS_OK) {
+                return $results->getExitCode();
+            }
+        }
+
+        // finally, try to find the first successful exit code
+        foreach ($this->sandboxResultsList as $results) {
+            if ($results->getStatus() === ISandboxResults::STATUS_OK) {
+                return $results->getExitCode();
+            }
+        }
+
+        return ISandboxResults::EXIT_CODE_UNKNOWN; // default fallback
+    }
+
+    /**
+     * Check if the exit code indicates successful termination of the process.
+     * @return bool True if all tasks ended in OK state.
+     */
+    public function isExitCodeOk(): bool
+    {
+        foreach ($this->sandboxResultsList as $results) {
+            if ($results->getStatus() !== ISandboxResults::STATUS_OK) {
+                return false;
+            }
+        }
+        return true;
     }
 
     /**

app/model/entity/TestResult.php

@@ -20,6 +20,7 @@ public function __construct(
         $this->status = $result->getStatus();
         $this->score = $result->getScore();
         $this->exitCode = $result->getExitCode();
+        $this->exitCodeOk = $result->isExitCodeOk();
         $this->exitCodeNative = $result->isExitCodeNative();
         $this->exitSignal = $result->getExitSignal();
         $this->usedMemory = $result->getUsedMemory();
@@ -115,6 +116,12 @@ public function __construct(
      */
     protected $exitCode;
 
+    /**
+     * @ORM\Column(type="boolean")
+     * Whether the exit code corresponds to configured success exit codes.
+     */
+    protected $exitCodeOk;
+
     /**
      * @ORM\Column(type="boolean")
      * Whether the exit code was produced by regular termination (not by signal, timeout, or sandbox error).
@@ -193,6 +200,7 @@ public function getDataForView(SubmissionViewOptions $options)
             "wallTimeExceeded" => $this->wallTimeExceeded,
             "cpuTimeExceeded" => $this->cpuTimeExceeded,
             "exitCode" => $this->exitCode,
+            "exitCodeOk" => $this->exitCodeOk,
             "exitCodeNative" => $this->exitCodeNative,
             "exitSignal" => $this->exitSignal,
             "message" => $this->message,
@@ -274,6 +282,11 @@ public function getExitCode(): int
         return $this->exitCode;
     }
 
+    public function isExitCodeOk(): bool
+    {
+        return $this->exitCodeOk;
+    }
+
     public function isExitCodeNative(): bool
     {
         return $this->exitCodeNative;

migrations/Version20240716161149.php

@@ -42,6 +42,7 @@ public function up(Schema $schema): void
         // this up() migration is auto-generated, please modify it to your needs
         $this->addSql('ALTER TABLE assignment ADD can_view_measured_values TINYINT(1) NOT NULL');
         $this->addSql('ALTER TABLE test_result ADD exit_code_native TINYINT(1) NOT NULL');
+        $this->addSql('ALTER TABLE test_result ADD exit_code_ok TINYINT(1) NOT NULL');
 
         $this->addSql('ALTER TABLE assignment_localized_exercise DROP FOREIGN KEY FK_9C8F78CD19302F8');
         $this->addSql('ALTER TABLE assignment_localized_exercise ADD CONSTRAINT FK_9DF069D6D19302F8 FOREIGN KEY (assignment_id) REFERENCES assignment (id) ON DELETE CASCADE');
@@ -75,7 +76,14 @@ public function postUp(Schema $schema): void
         $this->connection->executeQuery(
             "UPDATE test_result SET exit_code_native = 1 WHERE `status` = 'OK' OR
             (`status` = 'FAILED' AND memory_exceeded = 0 AND wall_time_exceeded = 0 AND
-            exit_signal IS NULL AND exit_code >= 0 AND message NOT LIKE '%Caught%')"
+            (exit_signal IS NULL OR exit_signal = 0) AND exit_code >= 0 AND message NOT LIKE '%Caught%')"
+        );
+
+        /*
+         * fill newly created exit_code_ok column
+         */
+        $this->connection->executeQuery(
+            "UPDATE test_result SET exit_code_ok = 1 WHERE exit_code_native = 1 AND exit_code = 0"
         );
 
         /*
@@ -184,6 +192,7 @@ public function down(Schema $schema): void
         // this down() migration is auto-generated, please modify it to your needs
         $this->addSql('ALTER TABLE assignment DROP can_view_measured_values');
         $this->addSql('ALTER TABLE test_result DROP exit_code_native');
+        $this->addSql('ALTER TABLE test_result DROP exit_code_ok');
 
         $this->addSql('ALTER TABLE assignment_localized_exercise DROP FOREIGN KEY FK_9DF069D6D19302F8');
         $this->addSql('ALTER TABLE assignment_localized_exercise ADD CONSTRAINT FK_9C8F78CD19302F8 FOREIGN KEY (assignment_id) REFERENCES assignment (id) ON DELETE CASCADE');