Refine Gymnasium task

Author: smelfungus

src/SharpNeat.Tasks.Windows/Gymnasium/GymnasiumControl.cs

@@ -86,7 +86,7 @@ private void InitializeComponent()
     /// </summary>
     private void SimulationThread()
     {
-        // Wait to be signalled to start the next trial run.
+        // Wait to be signaled to start the next trial run.
         _simStartEvent.WaitOne();
 
         IBlackBox<double> agent = _agent;
@@ -96,7 +96,7 @@ private void SimulationThread()
 
         while (true)
         {
-            // Check if we have been signalled to terminate before starting a simulation run.
+            // Check if we have been signaled to terminate before starting a simulation run.
             if(_terminateSimThread)
                 break;
 

src/SharpNeat.Tasks/Gymnasium/GymnasiumEpisode.cs

@@ -7,16 +7,16 @@ namespace SharpNeat.Tasks.Gymnasium;
 
 public sealed class GymnasiumEpisode
 {
-    readonly int _inputCount;
-    readonly int _outputCount;
-    readonly bool _isContinious;
-    readonly bool _test;
+    private readonly int _inputCount;
+    private readonly int _outputCount;
+    private readonly bool _isContinuous;
+    private readonly bool _test;
 
-    public GymnasiumEpisode(int inputCount, int outputCount, bool isContinious, bool test)
+    public GymnasiumEpisode(int inputCount, int outputCount, bool isContinuous, bool test)
     {
         _inputCount = inputCount;
         _outputCount = outputCount;
-        _isContinious = isContinious;
+        _isContinuous = isContinuous;
         _test = test;
     }
 
@@ -28,12 +28,12 @@ public FitnessInfo Evaluate(IBlackBox<double> phenome)
         {
             FileName = @"pythonw.exe",
             WorkingDirectory = @"./",
-            Arguments = string.Format(CultureInfo.InvariantCulture, @"gymnasium/main.py -uuid {0} -render {1}", uuid.ToString(), _test),
+            Arguments = string.Format(CultureInfo.InvariantCulture, @"gymnasium/main.py -uuid {0} -render {1} -test False", uuid.ToString(), _test),
             UseShellExecute = false,
             RedirectStandardOutput = false
         };
 
-        var process = Process.Start(start) ?? throw new InvalidOperationException("No proccess resource is started");
+        var process = Process.Start(start) ?? throw new InvalidOperationException("No process resource is started");
         var totalReward = 0.0;
 
         try
@@ -52,10 +52,9 @@ public FitnessInfo Evaluate(IBlackBox<double> phenome)
                 var inputs = phenome.Inputs.Span;
                 inputs.Clear();
 
-                var observationTuple = ReadObservation(namedPipeClientStream, _inputCount);
-                var observation = observationTuple.observation;
-                totalReward = observationTuple.reward[0];
-                var done = observationTuple.done[0];
+                var (observation, rewardArray, doneArray) = ReadObservation(namedPipeClientStream, _inputCount);
+                totalReward = rewardArray[0];
+                var done = doneArray[0];
 
                 if (done == 1)
                 {
@@ -66,7 +65,7 @@ public FitnessInfo Evaluate(IBlackBox<double> phenome)
                 phenome.Activate();
 
                 // var clampedOutputs = outputs.Select(output => Math.Clamp(output, -1.0, 1.0)).ToArray();
-                if (_isContinious)
+                if (_isContinuous)
                 {
                     var outputBuffer = new byte[_outputCount * sizeof(float)];
                     var outputs = new double[_outputCount];
@@ -76,7 +75,7 @@ public FitnessInfo Evaluate(IBlackBox<double> phenome)
                 }
                 else
                 {
-                    int maxSigIndex = ReadMaxSigIndex(phenome);
+                    var maxSigIndex = ReadMaxSigIndex(phenome);
                     var outputBuffer = new byte[sizeof(int)];
                     Buffer.BlockCopy(new int[] { maxSigIndex }, 0, outputBuffer, 0, outputBuffer.Length);
                     namedPipeClientStream.Write(outputBuffer, 0, outputBuffer.Length);
@@ -101,16 +100,16 @@ public FitnessInfo Evaluate(IBlackBox<double> phenome)
         return new FitnessInfo(maskedReward);
     }
 
-    static (double[] observation, double[] reward, int[] done) ReadObservation(NamedPipeClientStream namedPipeClientStream, int count)
+    private static (double[] observation, double[] reward, int[] done) ReadObservation(Stream namedPipeClientStream, int count)
     {
         var count0 = count * sizeof(double);
-        var count1 = sizeof(double);
-        var count2 = sizeof(int);
+        const int count1 = sizeof(double);
+        const int count2 = sizeof(int);
         var inputBuffer = new byte[count0 + count1 + count2];
         namedPipeClientStream.Read(inputBuffer, 0, inputBuffer.Length);
-        double[] observation = new double[count];
-        double[] reward = new double[1];
-        int[] done = new int[1];
+        var observation = new double[count];
+        var reward = new double[1];
+        var done = new int[1];
         var offset1 = count0;
         var offset2 = count0 + count1;
         Buffer.BlockCopy(inputBuffer, 0, observation, 0, count0);
@@ -119,46 +118,44 @@ public FitnessInfo Evaluate(IBlackBox<double> phenome)
         return (observation, reward, done);
     }
 
-    static double[] ReadDoubleArray(NamedPipeClientStream namedPipeClientStream, int count)
+    private static double[] ReadDoubleArray(Stream namedPipeClientStream, int count)
     {
         var inputBuffer = new byte[count * sizeof(double)];
         namedPipeClientStream.Read(inputBuffer, 0, inputBuffer.Length);
-        double[] values = new double[inputBuffer.Length / sizeof(double)];
+        var values = new double[inputBuffer.Length / sizeof(double)];
         Buffer.BlockCopy(inputBuffer, 0, values, 0, values.Length * sizeof(double));
         return values;
     }
 
-    static float[] ReadFloatArray(NamedPipeClientStream namedPipeClientStream, int count)
+    private static float[] ReadFloatArray(Stream namedPipeClientStream, int count)
     {
         var inputBuffer = new byte[count * sizeof(float)];
         namedPipeClientStream.Read(inputBuffer, 0, inputBuffer.Length);
-        float[] values = new float[inputBuffer.Length / sizeof(float)];
+        var values = new float[inputBuffer.Length / sizeof(float)];
         Buffer.BlockCopy(inputBuffer, 0, values, 0, values.Length * sizeof(float));
         return values;
     }
 
-    static int[] ReadIntArray(NamedPipeClientStream namedPipeClientStream, int count)
+    private static int[] ReadIntArray(Stream namedPipeClientStream, int count)
     {
         var inputBuffer = new byte[count * sizeof(int)];
         namedPipeClientStream.Read(inputBuffer, 0, inputBuffer.Length);
-        int[] values = new int[inputBuffer.Length / sizeof(int)];
+        var values = new int[inputBuffer.Length / sizeof(int)];
         Buffer.BlockCopy(inputBuffer, 0, values, 0, values.Length * sizeof(int));
         return values;
     }
 
-    int ReadMaxSigIndex(IBlackBox<double> phenome)
+    private int ReadMaxSigIndex(IBlackBox<double> phenome)
     {
-        double maxSig = phenome.Outputs.Span[0];
-        int maxSigIdx = 0;
+        var maxSig = phenome.Outputs.Span[0];
+        var maxSigIdx = 0;
 
-        for (int i = 1; i < _outputCount; i++)
+        for (var i = 1; i < _outputCount; i++)
         {
-            double v = phenome.Outputs.Span[i];
-            if (v > maxSig)
-            {
-                maxSig = v;
-                maxSigIdx = i;
-            }
+            var v = phenome.Outputs.Span[i];
+            if (!(v > maxSig)) continue;
+            maxSig = v;
+            maxSigIdx = i;
         }
 
         return maxSigIdx;

src/SharpNeat.Tasks/Gymnasium/GymnasiumEvaluationScheme.cs

@@ -11,7 +11,7 @@ internal class GymnasiumEvaluationScheme : IBlackBoxEvaluationScheme<double>
     public int OutputCount => 4;
 
     /// <inheritdoc/>
-    public bool IsDeterministic => true;
+    public bool IsDeterministic => false;
 
     /// <inheritdoc/>
     public IComparer<FitnessInfo> FitnessComparer => PrimaryFitnessInfoComparer.Singleton;

src/SharpNeat.Tasks/Gymnasium/GymnasiumEvaluator.cs

@@ -4,29 +4,29 @@ namespace SharpNeat.Tasks.Gymnasium;
 
 public sealed class GymnasiumEvaluator : IPhenomeEvaluator<IBlackBox<double>>
 {
-    readonly int _inputCount;
-    readonly int _outputCount;
-    readonly bool _isContinious;
-    readonly bool _test;
-    readonly int _trialsPerEvaluation = 1;
+    private const int TrialsPerEvaluation = 1;
+    private readonly int _inputCount;
+    private readonly int _outputCount;
+    private readonly bool _isContinuous;
+    private readonly bool _test;
 
-    public GymnasiumEvaluator(int inputCount, int outputCount, bool isContinious, bool test)
+    public GymnasiumEvaluator(int inputCount, int outputCount, bool isContinuous, bool test)
     {
         _inputCount = inputCount;
         _outputCount = outputCount;
-        _isContinious = isContinious;
+        _isContinuous = isContinuous;
         _test = test;
     }
 
     public FitnessInfo Evaluate(IBlackBox<double> phenome)
     {
         var finesses = new List<FitnessInfo>();
-        for (int i = 0; i < _trialsPerEvaluation; i++)
+        for (var i = 0; i < TrialsPerEvaluation; i++)
         {
-            var episode = new GymnasiumEpisode(_inputCount, _outputCount, _isContinious, _test);
+            var episode = new GymnasiumEpisode(_inputCount, _outputCount, _isContinuous, _test);
             finesses.Add(episode.Evaluate(phenome));
         }
 
-        return new FitnessInfo(finesses.Average(finesses => finesses.PrimaryFitness));
+        return new FitnessInfo(finesses.Average(fitness => fitness.PrimaryFitness));
     }
 }

src/SharpNeat.Tasks/Gymnasium/GymnasiumExperimentFactory.cs

@@ -8,7 +8,7 @@ namespace SharpNeat.Tasks.Gymnasium;
 internal class GymnasiumExperimentFactory : INeatExperimentFactory
 {
     /// <inheritdoc/>
-    public string Id => "gmnasium";
+    public string Id => "gymnasium";
 
     /// <inheritdoc/>
     public INeatExperiment<double> CreateExperiment(Stream jsonConfigStream)

src/SharpNeat.Windows.App/config/experiments-config/gymnasium.config.json

@@ -1,6 +1,6 @@
 {
   "name": "Gymnasium",
-  "isAcyclic": true,
+  "isAcyclic": false,
   "cyclesPerActivation": 1,
   "activationFnName": "LeakyReLU",
   "evolutionAlgorithm": {

src/SharpNeat.Windows.App/gymnasium/main.py

@@ -1,3 +1,4 @@
+import collections
 import struct
 import time
 import traceback
@@ -11,29 +12,31 @@
 import numpy as np
 import logging
 
-
 logging.basicConfig(filename='debug.log', encoding='utf-8', level=logging.FATAL)
 logging.debug("start")
 
 parser = ArgumentParser()
-parser.add_argument("-uuid", dest="uuid")
-parser.add_argument("-render", dest="render")
+parser.add_argument("-uuid", dest="uuid", default="test")
+parser.add_argument("-render", dest="render", default="False")
+parser.add_argument("-test", dest="test", default="True")
 args = parser.parse_args()
 render = args.render == "True"
+test = args.test == "True"
 
-pipe = win32pipe.CreateNamedPipe("\\\\.\\pipe\\gymnasium_pipe_" + args.uuid,
-                                 win32pipe.PIPE_ACCESS_DUPLEX,
-                                 win32pipe.PIPE_TYPE_MESSAGE | win32pipe.PIPE_READMODE_MESSAGE | win32pipe.PIPE_WAIT,
-                                 1, 1024, 1024, 0, None)
-logging.debug("Connecting pipe...")
-win32pipe.ConnectNamedPipe(pipe, None)
-logging.debug("Pipe connected")
+if not test:
+    pipe = win32pipe.CreateNamedPipe("\\\\.\\pipe\\gymnasium_pipe_" + args.uuid,
+                                     win32pipe.PIPE_ACCESS_DUPLEX,
+                                     win32pipe.PIPE_TYPE_MESSAGE | win32pipe.PIPE_READMODE_MESSAGE | win32pipe.PIPE_WAIT,
+                                     1, 1024, 1024, 0, None)
+    logging.debug("Connecting pipe...")
+    win32pipe.ConnectNamedPipe(pipe, None)
+    logging.debug("Pipe connected")
 
 # env = gym.make("BipedalWalker-v3", hardcore=False, render_mode="human" if render else None)
 # env = gym.make("LunarLander-v2", render_mode="human" if render else None)
 try:
     # env = gym.make("LunarLander-v2", enable_wind=True, render_mode="human" if render else None)
-    env = gym.make("BipedalWalker-v3", hardcore=True, render_mode="human" if render else None)
+    env = gym.make("BipedalWalker-v3", hardcore=False, render_mode="human" if render else None)
     env = ClipAction(env)
 
     logging.debug("Environment created")
@@ -48,17 +51,23 @@
 def run_episode():
     observation, info = env.reset()
 
-    logging.debug("Initial observation:", observation)
-    send_observation(observation, 0, False)
-    logging.debug("Initial observation sent")
+    if not test:
+        logging.debug("Initial observation:", observation)
+        send_observation(observation, 0, False)
+        logging.debug("Initial observation sent")
 
+    max_reward_history_len = 100
     total_reward = 0
     total_timesteps = 0
+    latest_rewards = collections.deque(maxlen=max_reward_history_len)
 
     while 1:
         logging.debug("Starting step")
-        a = read_action(env.action_space)
-        logging.debug("Action read:", a)
+
+        if not test:
+            a = read_action(env.action_space)
+        else:
+            a = env.action_space.sample()
 
         total_timesteps += 1
 
@@ -71,20 +80,28 @@ def run_episode():
         #     print("reward %0.3f" % reward)
 
         total_reward += reward
+        latest_rewards.append(float(reward))
 
         masked_done = done
 
-        # if render:
-        #     masked_done = False
+        if total_timesteps >= max_reward_history_len:
+            low_performing = True
+            for historical_reward in latest_rewards:
+                if historical_reward > 0:
+                    low_performing = False
+                    break
+            if low_performing:
+                masked_done = True
 
-        send_observation(observation, float(total_reward), masked_done)
-        logging.debug("Observation sent")
+        if not test:
+            send_observation(observation, float(total_reward), masked_done)
+            logging.debug("Observation sent")
 
         if render:
             env.render()
-            time.sleep(0.02)
+            time.sleep(0.01)
 
-        if done:
+        if masked_done:
             logging.debug("Terminated")
             if not render:
                 # pipe.close()
@@ -99,7 +116,8 @@ def run_episode():
 
 
 def send_observation(observation: np.array, reward: float, done: bool):
-    win32file.WriteFile(pipe, bytes(observation.astype(float)) + bytes(np.array([reward]).astype(float)) + bytes(np.array([int(done)])))
+    win32file.WriteFile(pipe, bytes(observation.astype(float)) + bytes(np.array([reward]).astype(float)) + bytes(
+        np.array([int(done)])))
 
 
 def read_action(space: spaces.Space):