feat: make number of chunk processing threads configurable (#5237)

BenjaminAmos · web-flow · commit 33dabf53cc67 · 2024-04-28T23:10:42.000+02:00
* feat: make number of chunk processing threads configurable
diff --git a/engine-tests/src/test/java/org/terasology/engine/world/chunks/localChunkProvider/LocalChunkProviderTest.java b/engine-tests/src/test/java/org/terasology/engine/world/chunks/localChunkProvider/LocalChunkProviderTest.java
@@ -10,6 +10,8 @@
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.mockito.ArgumentCaptor;
+import org.terasology.engine.config.Config;
+import org.terasology.engine.config.RenderingConfig;
 import org.terasology.engine.entitySystem.entity.EntityManager;
 import org.terasology.engine.entitySystem.entity.EntityRef;
 import org.terasology.engine.world.BlockEntityRegistry;
@@ -43,6 +45,7 @@
 import static org.mockito.Mockito.atLeast;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 class LocalChunkProviderTest {
 
@@ -54,6 +57,7 @@ class LocalChunkProviderTest {
     private ExtraBlockDataManager extraDataManager;
     private BlockEntityRegistry blockEntityRegistry;
     private EntityRef worldEntity;
+    private Config config;
     private Map<Vector3ic, Chunk> chunkCache;
     private Block blockAtBlockManager;
     private TestStorageManager storageManager;
@@ -71,13 +75,18 @@ public void setUp() {
         blockEntityRegistry = mock(BlockEntityRegistry.class);
         worldEntity = mock(EntityRef.class);
         chunkCache = Maps.newConcurrentMap();
+        config = mock(Config.class);
+        RenderingConfig renderConfig = mock(RenderingConfig.class);
+        when(renderConfig.getChunkThreads()).thenReturn(0);
+        when(config.getRendering()).thenReturn(renderConfig);
         storageManager = new TestStorageManager();
         generator = new TestWorldGenerator(blockManager);
         chunkProvider = new LocalChunkProvider(storageManager,
                 entityManager,
                 generator,
                 blockManager,
                 extraDataManager,
+                config,
                 chunkCache);
         chunkProvider.setBlockEntityRegistry(blockEntityRegistry);
         chunkProvider.setWorldEntity(worldEntity);
diff --git a/engine-tests/src/test/java/org/terasology/engine/world/chunks/pipeline/ChunkProcessingPipelineTest.java b/engine-tests/src/test/java/org/terasology/engine/world/chunks/pipeline/ChunkProcessingPipelineTest.java
@@ -51,7 +51,7 @@ class ChunkProcessingPipelineTest extends TerasologyTestingEnvironment {
 
     @Test
     void simpleProcessingSuccess() throws ExecutionException, InterruptedException, TimeoutException {
-        pipeline = new ChunkProcessingPipeline((p) -> null, (o1, o2) -> 0);
+        pipeline = new ChunkProcessingPipeline(0, (p) -> null, (o1, o2) -> 0);
 
         Vector3i chunkPos = new Vector3i(0, 0, 0);
         Chunk chunk = createChunkAt(chunkPos);
@@ -67,7 +67,7 @@ void simpleProcessingSuccess() throws ExecutionException, InterruptedException,
 
     @Test
     void simpleStopProcessingSuccess() {
-        pipeline = new ChunkProcessingPipeline((p) -> null, (o1, o2) -> 0);
+        pipeline = new ChunkProcessingPipeline(0, (p) -> null, (o1, o2) -> 0);
 
         Vector3i position = new Vector3i(0, 0, 0);
         Chunk chunk = createChunkAt(position);
@@ -106,7 +106,7 @@ void multiRequirementsChunksExistsSuccess() throws ExecutionException, Interrupt
                                 Function.identity()
                         ));
 
-        pipeline = new ChunkProcessingPipeline(chunkCache::get, (o1, o2) -> 0);
+        pipeline = new ChunkProcessingPipeline(0, chunkCache::get, (o1, o2) -> 0);
         pipeline.addStage(ChunkTaskProvider.createMulti(
                 "flat merging task",
                 (chunks) -> chunks.stream()
@@ -140,7 +140,7 @@ void multiRequirementsChunksWillGeneratedSuccess() throws ExecutionException, In
                                 Function.identity()
                         ));
 
-        pipeline = new ChunkProcessingPipeline((p) -> null, (o1, o2) -> 0);
+        pipeline = new ChunkProcessingPipeline(0, (p) -> null, (o1, o2) -> 0);
         pipeline.addStage(ChunkTaskProvider.createMulti(
                 "flat merging task",
                 (chunks) -> chunks.stream()
@@ -169,7 +169,7 @@ void emulateEntityMoving() throws InterruptedException {
         final AtomicReference<Vector3ic> position = new AtomicReference<>();
         Map<Vector3ic, Future<Chunk>> futures = Maps.newHashMap();
         Map<Vector3ic, Chunk> chunkCache = Maps.newConcurrentMap();
-        pipeline = new ChunkProcessingPipeline(chunkCache::get, (o1, o2) -> {
+        pipeline = new ChunkProcessingPipeline(0, chunkCache::get, (o1, o2) -> {
             if (position.get() != null) {
                 Vector3ic entityPos = position.get();
                 return (int) (entityPos.distance(((PositionFuture<?>) o1).getPosition())
diff --git a/engine/src/main/java/org/terasology/engine/config/RenderingConfig.java b/engine/src/main/java/org/terasology/engine/config/RenderingConfig.java
@@ -26,6 +26,7 @@ public class RenderingConfig extends AbstractSubscribable {
     public static final String ANIMATED_MENU = "AnimatedMenu";
     public static final String VIEW_DISTANCE = "viewDistance";
     public static final String CHUNK_LODS = "chunkLods";
+    public static final String CHUNK_THREADS = "chunkThreads";
     public static final String BILLBOARD_LIMIT = "billboardLimit";
     public static final String FLICKERING_LIGHT = "FlickeringLight";
     public static final String ANIMATE_GRASS = "AnimateGrass";
@@ -76,6 +77,7 @@ public class RenderingConfig extends AbstractSubscribable {
     private boolean animatedMenu;
     private ViewDistance viewDistance;
     private float chunkLods;
+    private int chunkThreads;
     private float billboardLimit;
     private boolean flickeringLight;
     private boolean animateGrass;
@@ -270,6 +272,16 @@ public void setChunkLods(float chunkLods) {
         propertyChangeSupport.firePropertyChange(CHUNK_LODS, oldLods, chunkLods);
     }
 
+    public int getChunkThreads() {
+        return chunkThreads;
+    }
+
+    public void setChunkThreads(int chunkThreads) {
+        float oldChunkThreads = this.chunkThreads;
+        this.chunkThreads = chunkThreads;
+        propertyChangeSupport.firePropertyChange(CHUNK_THREADS, oldChunkThreads, chunkThreads);
+    }
+
     public float getBillboardLimit() {
         return billboardLimit;
     }
diff --git a/engine/src/main/java/org/terasology/engine/core/modes/loadProcesses/InitialiseRemoteWorld.java b/engine/src/main/java/org/terasology/engine/core/modes/loadProcesses/InitialiseRemoteWorld.java
@@ -3,6 +3,7 @@
 
 package org.terasology.engine.core.modes.loadProcesses;
 
+import org.terasology.engine.config.Config;
 import org.terasology.engine.context.Context;
 import org.terasology.engine.core.ComponentSystemManager;
 import org.terasology.engine.core.TerasologyConstants;
@@ -54,7 +55,7 @@ public boolean step() {
         BlockManager blockManager = context.get(BlockManager.class);
         ExtraBlockDataManager extraDataManager = context.get(ExtraBlockDataManager.class);
 
-        RemoteChunkProvider chunkProvider = new RemoteChunkProvider(blockManager, localPlayer);
+        RemoteChunkProvider chunkProvider = new RemoteChunkProvider(blockManager, localPlayer, context.get(Config.class));
 
         WorldProviderCoreImpl worldProviderCore = new WorldProviderCoreImpl(gameManifest.getWorldInfo(TerasologyConstants.MAIN_WORLD), chunkProvider,
                 blockManager.getBlock(BlockManager.UNLOADED_ID), context);
diff --git a/engine/src/main/java/org/terasology/engine/core/modes/loadProcesses/InitialiseWorld.java b/engine/src/main/java/org/terasology/engine/core/modes/loadProcesses/InitialiseWorld.java
@@ -6,6 +6,7 @@
 import com.google.common.collect.Maps;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.terasology.engine.config.Config;
 import org.terasology.engine.config.SystemConfig;
 import org.terasology.engine.context.Context;
 import org.terasology.engine.core.ComponentSystemManager;
@@ -136,6 +137,7 @@ public boolean step() {
                 worldGenerator,
                 blockManager,
                 extraDataManager,
+                context.get(Config.class),
                 Maps.newConcurrentMap());
         RelevanceSystem relevanceSystem = new RelevanceSystem(chunkProvider);
         context.put(RelevanceSystem.class, relevanceSystem);
diff --git a/engine/src/main/java/org/terasology/engine/rendering/nui/layers/mainMenu/videoSettings/VideoSettingsScreen.java b/engine/src/main/java/org/terasology/engine/rendering/nui/layers/mainMenu/videoSettings/VideoSettingsScreen.java
@@ -75,6 +75,33 @@ public void initialise() {
             viewDistance.bindSelection(BindHelper.bindBeanProperty("viewDistance", config.getRendering(), ViewDistance.class));
         }
 
+        UISlider chunkThreads = find("chunkThreads", UISlider.class);
+        if (chunkThreads != null) {
+            chunkThreads.setIncrement(1.0f);
+            chunkThreads.setPrecision(0);
+            chunkThreads.setMinimum(0);
+            chunkThreads.setRange(Runtime.getRuntime().availableProcessors());
+            chunkThreads.setLabelFunction(input -> {
+                if (input == 0) {
+                    return "Auto";
+                } else {
+                    return String.valueOf(input.intValue());
+                }
+            });
+            chunkThreads.bindValue(new Binding<Float>() {
+                @Override
+                public Float get() {
+                    return (float) config.getRendering().getChunkThreads();
+                }
+
+                @Override
+                public void set(Float value) {
+                    int chunkThreads = value.intValue();
+                    config.getRendering().setChunkThreads(chunkThreads);
+                }
+            });
+        }
+
         UIDropdown<WaterReflection> waterReflection = find("reflections", UIDropdown.class);
         if (waterReflection != null) {
             waterReflection.setOptionRenderer(new ToStringTextRenderer<>(translationSystem));
diff --git a/engine/src/main/java/org/terasology/engine/world/chunks/localChunkProvider/LocalChunkProvider.java b/engine/src/main/java/org/terasology/engine/world/chunks/localChunkProvider/LocalChunkProvider.java
@@ -14,6 +14,7 @@
 import org.joml.Vector3ic;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.terasology.engine.config.Config;
 import org.terasology.engine.entitySystem.entity.EntityManager;
 import org.terasology.engine.entitySystem.entity.EntityRef;
 import org.terasology.engine.entitySystem.entity.EntityStore;
@@ -94,6 +95,7 @@ public class LocalChunkProvider implements ChunkProvider {
     private final WorldGenerator generator;
     private final BlockManager blockManager;
     private final ExtraBlockDataManager extraDataManager;
+    private final Config config;
     private ChunkProcessingPipeline loadingPipeline;
     private TaskMaster<ChunkUnloadRequest> unloadRequestTaskMaster;
     private EntityRef worldEntity = EntityRef.NULL;
@@ -102,13 +104,14 @@ public class LocalChunkProvider implements ChunkProvider {
     private RelevanceSystem relevanceSystem;
 
     public LocalChunkProvider(StorageManager storageManager, EntityManager entityManager, WorldGenerator generator,
-                              BlockManager blockManager, ExtraBlockDataManager extraDataManager,
+                              BlockManager blockManager, ExtraBlockDataManager extraDataManager, Config config,
                               Map<Vector3ic, Chunk> chunkCache) {
         this.storageManager = storageManager;
         this.entityManager = entityManager;
         this.generator = generator;
         this.blockManager = blockManager;
         this.extraDataManager = extraDataManager;
+        this.config = config;
         this.unloadRequestTaskMaster = TaskMaster.createFIFOTaskMaster("Chunk-Unloader", 4);
         this.chunkCache = chunkCache;
         ChunkMonitor.fireChunkProviderInitialized(this);
@@ -406,7 +409,8 @@ public void purgeWorld() {
         storageManager.deleteWorld();
         worldEntity.send(new PurgeWorldEvent());
 
-        loadingPipeline = new ChunkProcessingPipeline(this::getChunk, relevanceSystem.createChunkTaskComporator());
+        loadingPipeline = new ChunkProcessingPipeline(config.getRendering().getChunkThreads(), this::getChunk,
+                relevanceSystem.createChunkTaskComporator());
         loadingPipeline.addStage(
             ChunkTaskProvider.create("Chunk generate internal lightning",
                 (Consumer<Chunk>) InternalLightProcessor::generateInternalLighting))
@@ -441,7 +445,8 @@ private boolean isChunkReady(Chunk chunk) {
     // TODO: move loadingPipeline initialization into constructor.
     public void setRelevanceSystem(RelevanceSystem relevanceSystem) {
         this.relevanceSystem = relevanceSystem;
-        loadingPipeline = new ChunkProcessingPipeline(this::getChunk, relevanceSystem.createChunkTaskComporator());
+        loadingPipeline = new ChunkProcessingPipeline(config.getRendering().getChunkThreads(), this::getChunk,
+                relevanceSystem.createChunkTaskComporator());
         loadingPipeline.addStage(
                         ChunkTaskProvider.create("Chunk generate internal lightning",
                                 (Consumer<Chunk>) InternalLightProcessor::generateInternalLighting))
diff --git a/engine/src/main/java/org/terasology/engine/world/chunks/pipeline/ChunkProcessingPipeline.java b/engine/src/main/java/org/terasology/engine/world/chunks/pipeline/ChunkProcessingPipeline.java
@@ -39,8 +39,8 @@
 public class ChunkProcessingPipeline {
 
     @SuppressWarnings("UnstableApiUsage")
-    private static final int NUM_TASK_THREADS = constrainToRange(
-            Runtime.getRuntime().availableProcessors() - 1, 1, 8);
+    private static final int DEFAULT_TASK_THREADS = constrainToRange(
+            Runtime.getRuntime().availableProcessors() - 2, 1, 4);
     private static final Logger logger = LoggerFactory.getLogger(ChunkProcessingPipeline.class);
 
     private final List<ChunkTaskProvider> stages = Lists.newArrayList();
@@ -54,17 +54,18 @@ public class ChunkProcessingPipeline {
     /**
      * Create ChunkProcessingPipeline.
      */
-    public ChunkProcessingPipeline(Function<Vector3ic, Chunk> chunkProvider, Comparator<Future<Chunk>> comparable) {
+    public ChunkProcessingPipeline(int chunkThreads, Function<Vector3ic, Chunk> chunkProvider, Comparator<Future<Chunk>> comparable) {
         this.chunkProvider = chunkProvider;
 
+        int taskThreads = (chunkThreads == 0) ? DEFAULT_TASK_THREADS : chunkThreads;
         executor = new ThreadPoolExecutor(
-                NUM_TASK_THREADS,
-                NUM_TASK_THREADS, 0L,
+                taskThreads,
+                taskThreads, 0L,
                 TimeUnit.MILLISECONDS,
                 new PriorityBlockingQueue(800, comparable),
                 this::threadFactory,
                 this::rejectQueueHandler);
-        logger.debug("allocated {} threads", NUM_TASK_THREADS);
+        logger.debug("allocated {} threads", taskThreads);
         chunkProcessor = new ChunkExecutorCompletionService(executor,
                 new PriorityBlockingQueue<>(800, comparable));
         reactor = new Thread(this::chunkTaskHandler);
diff --git a/engine/src/main/java/org/terasology/engine/world/chunks/remoteChunkProvider/RemoteChunkProvider.java b/engine/src/main/java/org/terasology/engine/world/chunks/remoteChunkProvider/RemoteChunkProvider.java
@@ -9,6 +9,7 @@
 import org.joml.Vector3f;
 import org.joml.Vector3i;
 import org.joml.Vector3ic;
+import org.terasology.engine.config.Config;
 import org.terasology.engine.entitySystem.entity.EntityRef;
 import org.terasology.engine.logic.players.LocalPlayer;
 import org.terasology.engine.monitoring.chunk.ChunkMonitor;
@@ -56,9 +57,9 @@ public class RemoteChunkProvider implements ChunkProvider {
     private EntityRef worldEntity = EntityRef.NULL;
     private ChunkReadyListener listener;
 
-    public RemoteChunkProvider(BlockManager blockManager, LocalPlayer localPlayer) {
+    public RemoteChunkProvider(BlockManager blockManager, LocalPlayer localPlayer, Config config) {
         this.blockManager = blockManager;
-        loadingPipeline = new ChunkProcessingPipeline(this::getChunk,
+        loadingPipeline = new ChunkProcessingPipeline(config.getRendering().getChunkThreads(), this::getChunk,
                 new LocalPlayerRelativeChunkComparator(localPlayer));
 
         loadingPipeline.addStage(
diff --git a/engine/src/main/resources/org/terasology/engine/assets/i18n/menu.lang b/engine/src/main/resources/org/terasology/engine/assets/i18n/menu.lang
@@ -426,6 +426,7 @@
     "view-distance-moderate": "view-distance-moderate",
     "view-distance-near": "view-distance-near",
     "view-distance-ultra": "view-distance-ultra",
+    "chunk-threads": "chunk-threads",
     "vignette": "vignette",
     "volumetric-fog": "volumetric-fog",
     "warn-modules": "warn-modules",
diff --git a/engine/src/main/resources/org/terasology/engine/assets/i18n/menu_en.lang b/engine/src/main/resources/org/terasology/engine/assets/i18n/menu_en.lang
@@ -443,6 +443,7 @@
     "view-distance-moderate": "Moderate",
     "view-distance-near": "Near",
     "view-distance-ultra": "Ultra",
+    "chunk-threads": "Chunk Threads",
     "vignette": "Vignette",
     "volumetric-fog": "Volumetric Fog",
     "warn-modules": "WARNING: Enabling extra modules, especially all or those not in the stable lineup can cause game crashes or broken worlds.\nPlease handle with care.",
diff --git a/engine/src/main/resources/org/terasology/engine/assets/ui/menu/videoMenuScreen.ui b/engine/src/main/resources/org/terasology/engine/assets/ui/menu/videoMenuScreen.ui
@@ -213,6 +213,14 @@
                                     "type": "UIDropdown",
                                     "id": "viewDistance"
                                 },
+                                {
+                                    "type": "UILabel",
+                                    "text": "${engine:menu#chunk-threads}: "
+                                },
+                                {
+                                    "type": "UISlider",
+                                    "id": "chunkThreads"
+                                },
                                 {
                                     "type": "UISpace",
                                     "size": [1, 13]
