diff --git a/.github/workflows/generate-definitions.yml b/.github/workflows/generate-definitions.yml index 28c71bf..89614ee 100644 --- a/.github/workflows/generate-definitions.yml +++ b/.github/workflows/generate-definitions.yml @@ -32,7 +32,23 @@ jobs: run: | git clone https://github.com/commandblock2/LiquidBounce.git lbng cd lbng - git checkout 9888c42aa793bdac38767839443de934e509e103 + git checkout 9888c42aa793bdac38767839443de934e509e103 + - name: Create Mods Directory + run: mkdir -p lbng/run/mods + + - name: Download and Install Baritone + uses: dawidd6/action-download-artifact@v11 + with: + github_token: ${{secrets.BARITONE_DOWNLOAD_TOKEN}} + name: Artifacts + repo: cabaletta/baritone + workflow: gradle_build.yml + branch: 1.21.4 + path: lbng/run/mods + workflow_conclusion: success + - name: Extract Baritone Fabric API JAR + run: | + find lbng/run/mods -name "baritone-api-fabric-*.jar" -exec mv {} lbng/run/mods/ \; - name: Download Latest ts-generator run: | diff --git a/.roomodes b/.roomodes index 758d739..dc772ba 100644 --- a/.roomodes +++ b/.roomodes @@ -85,3 +85,60 @@ customModes: - command - mcp source: project + - slug: lbng-debugger + name: LBNG Debugger + description: Specialized for LBNG script debugging. + roleDefinition: |- + You are Roo, a highly skilled software engineer with extensive knowledge in many programming languages, frameworks, design patterns, and best practices. + + You are in a paused breakpoint in Minecraft. Your primary goal is to assist the user in debugging LiquidBounce NextGen scripts by evaluating expressions and inspecting variables. + + Roo will reference `@src/complete.ts` first to understand what Roo is dealing with, many are not required to import, if Roo need to know which are already there, see `node_modules/jvm-types/ambient/ambient.d.ts`. + + Roo **must assume that classes use the yarn mapping on the latest Minecraft protocol and prioritize using native Minecraft classes**. **Avoid using ViaVersion classes unless there is no native Minecraft equivalent or it is explicitly required for a specific task.** + + When Roo are not confident about if an API exists in the current API, Roo may look up the type information at `node_modules/jvm-types/`. For example, `node_modules/jvm-types/types/net/minecraft/client/MinecraftClient.d.ts` contains the definition for `export class MinecraftClient extends ReentrantThreadExecutor<() => void> implements WindowEventHandler, MinecraftClientAccessor {` and `node_modules/jvm-types/types/net/ccbluex/liquidbounce/utils/movement/MovementUtilsKt.d.ts` contains `export class MovementUtilsKt extends Object {` + + Roo will use `Outter$Inner` syntax to access inner classes. For example, if `PlayerMoveC2SPacket` has an inner class `PositionAndOnGround`, Roo would refer to it as `PlayerMoveC2SPacket$PositionAndOnGround`. When importing such classes, Roo will import them directly from their respective `.d.ts` files, e.g., `import { PlayerMoveC2SPacket$PositionAndOnGround } from "jvm-types/net/minecraft/network/packet/c2s/play/PlayerMoveC2SPacket$PositionAndOnGround";`. + + Roo will use `import` instead of `Java.type` for imports from JVM, as the custom `require`/`import` implementation will be redirected to the graaljs's `Java.type`, direct usage of `Java.type` will lose all type information. + + --- + **API Usage and Event Handling in LiquidBounce NextGen (Debugging Context):** + + Roo's primary source of truth for LiquidBounce NextGen API definitions is the `node_modules/jvm-types/` directory. Always consult these type definitions, especially when: + * Roo is not confident about the existence or exact signature of an API. + * Roo is working with events or Kotlin extension functions. + * Roo encounter type errors. + + * **Kotlin Extension Functions:** When migrating Kotlin code to TypeScript, pay special attention to Kotlin extension functions (e.g., `object.extensionFunction(...)` or `object.property`). These often compile to static methods in a utility class (frequently ending in `Kt`, like `RenderShortcutsKt`) where the extended object is passed as the first argument. Before assuming a direct method on the extended object, consult the `node_modules/jvm-types/` definitions for such utility classes or static methods that mimic the extension function's behavior. If a direct translation is not apparent or causes type errors, prioritize finding the corresponding static utility method. + * **`FunctionN` Types and Type Errors (General Context):** For Kotlin `FunctionN` types (e.g., `Function0`, `Function1`, `Function2`) in `jvm-types` definitions, understand that direct TypeScript lambda expressions might not always satisfy these types due to strict type checking. If a TypeScript lambda provides the correct number and types of arguments and the expected return type, but the type system still reports an incompatibility with a `FunctionN` type *in contexts other than `mod.on()`*, use `// @ts-expect-error` to suppress the error. This indicates a potential limitation in the generated TypeScript definitions for these functional interfaces, and Roo should proceed assuming the runtime behavior is correct. + + --- + + **Special Handling for GraalJS Globals**: + Some of the node.js standard are not implemented, for example `setTimeout` and `clearTimeout`. Roo will look around the `src` folder to understand how other script accomplish similar functionality. + + Seeing errors like the following may suggest that the generated TypeScript definition has some problems instead of Roo's work does. When seeing those errors, Roo will consider if they are errors made by the definition generator and ignore them if appropriate. + + Conversion of type 'LivingEntity' to type 'Entity' may be a mistake because neither type sufficiently overlaps with the other. If this was intentional, convert the expression to 'unknown' first. + + Types of property 'getBoundingBox' are incompatible. + Type '(pose: EntityPose) => Box' is not comparable to type '() => Box'. + Target signature provides too few arguments. Expected 1 or more, but got 0. + + If Roo see `@ts-expect-error` around float-related operations, do not change them, as graaljs uses `double` for the type and cannot tolerate implicit precision loss. + --- + + When evaluating expressions or assigning variables, you MUST use string substitution with backticks (` `) and the `${}` syntax. For variable assignment, use the format `${globalThis.yourVariable = value}` to ensure the variable is accessible globally and its assignment is reflected in the output. + + Example evaluation: `${mc.player}` + Example assignment: `${globalThis.myServer = mc.getServer()}` + + You have full access to the debugger and can evaluate any valid JavaScript/GraalJS expression. + whenToUse: >- + Use this mode when actively debugging LiquidBounce NextGen scripts, evaluating expressions, + or inspecting runtime variables within a paused debugger session. + groups: + - read + - mcp diff --git a/.vscode/launch.json b/.vscode/launch.json index 6559fbe..6622a44 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,9 +5,19 @@ "name": "Attach", "type": "node", "request": "attach", - "debugServer": 4242, + "debugServer": 4243, "restart": true, "sourceMaps": true + }, + { + "name": "Attach by cdp (not recommended)", + "type": "node", + "request": "attach", + "port": 4242, // Default inspect port + "address": "localhost", + "restart": true, + "sourceMaps": true } + ] } \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 1ec2dd4..191d8c7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -47,9 +47,9 @@ } }, "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", - "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.4.tgz", + "integrity": "sha512-VT2+G1VQs/9oz078bLrYbecdZKs912zQlkelYpuf+SXF+QvZDYJlbx/LSx+meSAwdDFnF8FVXW92AVjjkVmgFw==", "dev": true, "license": "MIT" }, @@ -93,9 +93,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "22.15.32", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.15.32.tgz", - "integrity": "sha512-3jigKqgSjsH6gYZv2nEsqdXfZqIFGAV36XYYjf9KGZ3PSG+IhLecqPnI310RvjutyMwifE2hhhNEklOUrvx/wA==", + "version": "22.17.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.17.0.tgz", + "integrity": "sha512-bbAKTCqX5aNVryi7qXVMi+OkB3w/OyblodicMbvE38blyAz7GxXf6XYhklokijuPwwVg9sDLKRxt0ZHXQwZVfQ==", "dev": true, "license": "MIT", "dependencies": { @@ -158,6 +158,10 @@ "dev": true, "license": "MIT" }, + "node_modules/deep-learning-bot-utils": { + "resolved": "packages/deep-learning-bot-utils", + "link": true + }, "node_modules/diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -169,9 +173,9 @@ } }, "node_modules/jvm-types": { - "version": "2025.7.26", - "resolved": "https://registry.npmjs.org/jvm-types/-/jvm-types-2025.7.26.tgz", - "integrity": "sha512-ROfzV1Pm8wena8wfpvHBXqEafxtRL73FAFgolhRJfhow1yOacvrLA/SlYvMKY0FEOWFTNPGOH+6Cgks0mD2XkA==" + "version": "2025.7.30", + "resolved": "https://registry.npmjs.org/jvm-types/-/jvm-types-2025.7.30.tgz", + "integrity": "sha512-FXkNeXIC9ZOg2eDmZvd5TJL673m6c7vQ0BYVfmh2jxUxRsBvY/XYsvNduxe3jiOCgJBvXo3lJnd4A91m8lo3Hw==" }, "node_modules/lbng-utils-typed": { "resolved": "packages/lbng-utils-typed", @@ -280,9 +284,13 @@ "node": ">=6" } }, + "packages/deep-learning-bot-utils": { + "version": "1.0.0", + "license": "GPL-3.0-only" + }, "packages/lbng-utils-typed": { "version": "1.0.0", - "license": "GPL-3.0", + "license": "GPL-3.0-only", "dependencies": { "jvm-types": "^2025.7.8" } diff --git a/packages/deep-learning-bot-utils/README.md b/packages/deep-learning-bot-utils/README.md new file mode 100644 index 0000000..e69de29 diff --git a/packages/deep-learning-bot-utils/package.json b/packages/deep-learning-bot-utils/package.json new file mode 100644 index 0000000..e397952 --- /dev/null +++ b/packages/deep-learning-bot-utils/package.json @@ -0,0 +1,7 @@ +{ + "name": "deep-learning-bot-utils", + "version": "1.0.0", + "description": "Utils for making a deep learning bot, prioritizing pathfinding and traversability, then expanding to inventory and game context.", + "author": "commandblock2", + "license": "GPL-3.0-only" +} diff --git a/packages/deep-learning-bot-utils/spec-v1/SPEC-DATA.md b/packages/deep-learning-bot-utils/spec-v1/SPEC-DATA.md new file mode 100644 index 0000000..119bb30 --- /dev/null +++ b/packages/deep-learning-bot-utils/spec-v1/SPEC-DATA.md @@ -0,0 +1,128 @@ +### Bot Training Data Specification (Information Types) + +**Objective:** Define the data types required for iterative training of a Minecraft bot, prioritizing pathfinding and traversability, then expanding to inventory and game context. + +**Core Principles:** + +1. **State-Action Pairing:** Data is structured as `(State, Action)` pairs. +2. **Modularity:** State components can be included or excluded based on training phase. +3. **Actionability:** Inventory and game item data must be represented to enable specific in-game actions. +4. **Contextual Relevance:** Data collected should accurately reflect the environment relevant to the bot's current skill focus. +5. **Temporal Awareness:** Incorporate historical context for improved decision-making and learning from past experience. + +--- + +#### **Phase 1: Core Pathfinding & Traversability (Information Types)** + +**Player State:** + +* **Position:** Absolute 3D world coordinates (`X`, `Y`, `Z`). +* **Velocity:** 3D velocity vector (`VX`, `VY`, `VZ`). +* **Look Direction:** Spherical coordinates (`Yaw`, `Pitch`). +* **Player Pose:** Categorical state (e.g., `STANDING`, `SNEAKING`, `SPRINTING`). +* **Ground Proximity:** Boolean indicating if the player is on the ground. +* **Predicted Passive Next Tick State:** The player's simulated position and velocity at the next tick, assuming no active input from the bot, but accounting for current game physics (gravity, potion effects, block friction). + * `Predicted_Pos_X`, `Predicted_Pos_Y`, `Predicted_Pos_Z` + * `Predicted_Vel_X`, `Predicted_Vel_Y`, `Predicted_Vel_Z` +* **Historical Player States:** A fixed-size list (e.g., last 3-5 ticks) of: + * **Previous Position:** (`X`, `Y`, `Z`) at past tick. + * **Previous Velocity:** (`VX`, `VY`, `VZ`) at past tick. + * **Previous Look Direction:** (`Yaw`, `Pitch`) at past tick. + * **Previous Player Pose:** Categorical state at past tick. + * **Previous Fall Distance:**. + +**Local Environment Scan (Collision Box List):** + +* A collection of collision boxes from nearby environmental elements within a defined spatial radius, plus dynamic areas of interest. Each collision box contains: + * **Bounding Box Coordinates:** The precise geometric bounds (`minX`, `minY`, `minZ`, `maxX`, `maxY`, `maxZ`) of the collision box, normalized relative to the player position. + * **Relative Position:** Center point of the collision box relative to the player (`centerX`, `centerY`, `centerZ`). + * **Box Dimensions:** Width, height, and depth of the collision box (`width`, `height`, `depth`). + * **Element Identifier:** A unique string label for the element type that owns this collision box (e.g., `minecraft:stone`, `minecraft:water`, `minecraft:boat`). + * **Traversability Data:** A categorization indicating how the player can interact with this collision box: + * `SOLID_WALKABLE` + * `FLUID` + * `OBSTRUCTION` + * `AIR` + * `LIQUID_PLACEABLE` + * `PLACEABLE_BLOCK` + * `OTHER` + * **Element State Properties:** Specific configuration data for the element (e.g., directionality of stairs, state of a farmland block, door open/closed). + * **Area Source Type:** Categorical indicator of how this collision box was included in the scan: + * `FIXED_RADIUS` - Collision box within fixed scanning radius around player + * `DYNAMIC_INTEREST` - Collision box included due to area of interest output + * **Box Validity:** Boolean indicating if this collision box slot contains valid data (used for padding in fixed-size arrays). + +**Simplified Inventory Snapshot:** + +* **Hotbar Slots:** A fixed-size array (e.g., 9 elements) representing the player's hotbar. For each slot: + * **Item Identifier:** String label for the item (e.g., `minecraft:water_bucket`, `null` for empty). + * **Item Quantity:** The count of that item in the slot. +* **Key Utility Indicators:** Boolean flags for critical items relevant to traversal: + * `hasWaterBucket` + * `hasPlaceableBlocks` + +**Baritone Reference Data (Training Phase Only):** + +* **Current Target Path:** A sequence of waypoints representing Baritone's computed optimal path to the current goal. + * **Path Waypoints:** Array of 3D coordinates (`X`, `Y`, `Z`) representing the sequence of blocks to traverse. + * **Path Length:** Total number of waypoints in the current path. + * **Next Waypoint Index:** Index of the next waypoint the bot should move toward. + * **Estimated Completion Time:** Baritone's estimate of ticks required to complete the path. +* **Path Metadata:** + * **Path Computation Tick:** The game tick when this path was computed by Baritone. + * **Goal Coordinates:** The target destination (`GoalX`, `GoalY`, `GoalZ`) for this path. + * **Path Validity:** Boolean indicating if the path is still valid (no environmental changes detected). + +--- + +#### **Phase 2: PvP & Game Objective Awareness (Information Types)** + +**Player State (Additions):** + +* **Equipment:** Identifiers and states of items held in main and off-hand. + +**Targeting Information:** + +* A collection of nearby entities within a defined radius. For each entity relevant to objectives or combat: + * **Entity Identifier:** Unique ID. + * **Entity Type:** Categorical label (e.g., `PLAYER`, `ITEM`, `MOB`). + * **Entity Position:** Relative 3D coordinates. + * **Entity State:** Key attributes like health, distance. + * **Combat Relevance:** A flag indicating if the entity is considered an "enemy" or target for the current game mode. + +--- + +**Action Types:** + +* **Movement:** `MOVE` (with directional components), `JUMP`, `SNEAK`, `SPRINT`. +* **Interaction:** `LOOK` (with direction changes), `USE_ITEM` (specifying item/slot and target), `PLACE_BLOCK` (specifying item/slot, target position, and face). +* **Combat:** `ATTACK_ENTITY` (specifying target entity and arm swing), `SWAP_OFFHAND` (to switch items between hands). +* **Path of Interest Generation:** `GENERATE_AREA_OF_INTEREST` (specifying a sequence of 3D coordinates that define additional areas for detailed environmental scanning beyond the fixed radius). + +--- + +#### **Phase 3: Network Manipulation & Latency Awareness (Information Types)** + +* **Network State:** + * **Current Client Latency:** Numerical value representing the round-trip time (ping) to the server in milliseconds. + * **Server Tick Delta:** Numerical value indicating the difference between client and server tick counts, useful for gauging desync. + * **Outgoing Packet Queue Size:** Numerical count of packets currently queued for transmission from the client. + * **Incoming Packet Queue Size:** Numerical count of packets currently buffered for processing on the client. + +--- + + +#### **Phase 4: Inventory Management & Contextual Actions (Information Types)** + + +**Local Environment Scan (Enhancements):** + +* For each environmental element: + * **Element State Properties:** Specific configuration data for the element (e.g., directionality of stairs, state of a farmland block). + +**Detailed Inventory Snapshot:** + +* **Hotbar Slots:** (As in Phase 1) +* **Main Inventory Counts:** A mapping of `Item Identifier` to `Quantity` for key stackable items NOT on the hotbar. +* **Selected Hotbar Slot:** The index of the currently active hotbar slot. + diff --git a/packages/deep-learning-bot-utils/spec-v1/SPEC-MODEL.md b/packages/deep-learning-bot-utils/spec-v1/SPEC-MODEL.md new file mode 100644 index 0000000..09eff13 --- /dev/null +++ b/packages/deep-learning-bot-utils/spec-v1/SPEC-MODEL.md @@ -0,0 +1,119 @@ + +### Neural Network Input and Output Definitions + +The neural network will be designed to process current observations and an internal recurrent state to produce concurrent actions and specific outputs like "areas of interest." + +#### Neural Network Inputs (Observation Space) + +The input to the neural network for each game tick will consist of the current environmental and player state, which will then be combined with the network's recurrent hidden state to form the *effective* input for the final decision-making layers. + +1. **Current Player State (Numerical Vector + Categorical/Embeddings):** + * **Numerical:** + * `Position`: `[X, Y, Z]` (normalized, possibly relative to goal or spawn) + * `Velocity`: `[VX, VY, VZ]` (normalized) + * `Look Direction`: `[sin(Yaw), cos(Yaw), sin(Pitch), cos(Pitch)]` + * `Fall Distance`: `[FallDistance_Normalized]` + * `Predicted Passive Next Tick Position`: `[Predicted_X, Predicted_Y, Predicted_Z]` (normalized) + * `Predicted Passive Next Tick Velocity`: `[Predicted_VX, Predicted_VY, Predicted_VZ]` (normalized) + * **Categorical/Binary (One-Hot or Embeddings):** + * `Player Pose`: e.g., `[OneHot(STANDING), OneHot(SNEAKING), ...]` + * `Ground Proximity`: `[0]` or `[1]` + * `Key Utility Indicators`: `[hasWaterBucket_Binary, hasPlaceableBlocks_Binary]` + * **Goal Encoding:** + * `Target Coordinates`: `[GoalX, GoalY, GoalZ]` (normalized) + * `Target Entity Info`: `[TargetEntity_Type_OneHot, TargetEntity_Health_Normalized, TargetEntity_Distance_Normalized]` (if the goal is an enemy entity) + +2. **Local Environment Scan (Collision Box List):** + * **Shape:** `(MAX_COLLISION_BOXES, BOX_FEATURE_SIZE)` (e.g., up to 512 boxes, 16-20 features per box). + * **Composition:** Combined list of collision boxes from fixed radius scan + dynamic area of interest + * **Per-Box Features (Concatenated Vector):** + * `Bounding Box Coordinates`: `[minX, minY, minZ, maxX, maxY, maxZ]` (normalized relative to player) + * `Relative Position`: `[relativeX, relativeY, relativeZ]` (box center relative to player) + * `Box Dimensions`: `[width, height, depth]` (derived from min/max coordinates) + * `Element Identifier`: Learned embedding of block/entity type + * `Traversability Data`: One-hot encoding (e.g., `SOLID_WALKABLE`, `FLUID`, `OBSTRUCTION`) + * `Element State Properties`: Numerical features for block states (e.g., `IsDoorOpen`, `StairDirection`) + * `Area Source Type`: One-hot encoding (`FIXED_RADIUS`, `DYNAMIC_INTEREST`) + * `Box Validity Mask`: Binary flag indicating if this slot contains a valid collision box (for padding) + * **Neural Network Processing Notes:** + * Use attention mechanisms or set-based networks (e.g., Deep Sets, Set Transformer) to handle variable-length collision box lists + * The `Box Validity Mask` enables proper masking for padded entries in fixed-size tensors + * Consider spatial attention based on `Relative Position` to focus on nearby collision boxes + * Learned embeddings for `Element Identifier` can capture semantic relationships between block types + +3. **Inventory Snapshot (Numerical Vector + Categorical/Embeddings):** + * **Hotbar Slots (Fixed-Size List of Vectors):** For each of 9 slots: + * `[ItemIdentifier_OneHot/Embedding, ItemQuantity_Normalized]` + * `Selected Hotbar Slot`: `[OneHot(Index_of_Active_Slot)]` + * **Main Inventory Counts (Fixed-Size List for N Pre-Defined Items):** For each important + * `[ItemIdentifier_OneHot/Embedding, ItemQuantity_Normalized]` + +4. **Targeting Information (Concatenated Vectors for Fixed N Entities):** + * For `N` closest/most relevant entities: + * `[EntityType_OneHot/Embedding, RelativePosition_dX,dY,dZ, EntityHealth_Normalized, EntityDistance_Normalized, CombatRelevance_Binary]` + * (Padding with zeros or "null" entity embeddings if fewer than `N` entities are present). + +5. **Game State (Numerical/Binary Vector):** + * `Team Status`: `[IsBedBroken_Binary]` + * `Objective Locations`: `[Objective1_X, Objective1_Y, Objective1_Z, ...]` + +6. **Recurrent Hidden State (Managed by Inference Loop):** + * This is the compressed "memory" of the sequence processed so far. It's the output of the previous timestep's recurrent layer and is fed back as an input to the current timestep's recurrent layer. This is not part of the raw `Observation Space` from the game but is an essential component of the network's input processing. + * **Size:** Fixed, e.g., `(num_layers, batch_size, hidden_size)` for GRU/RNN or tuple for LSTM. + +7. **Network State (Numerical and Categorical):** + * `Current Client Latency`: `[Latency_ms_Normalized]` + * `Server Tick Delta`: `[Server_Tick_Delta_Normalized]` + * `Outgoing Packet Queue Size`: `[Outgoing_Queue_Normalized]` + * `Incoming Packet Queue Size`: `[Incoming_Queue_Normalized]` + * `Current Incoming Policy Applied`: `[OneHot(NORMAL), OneHot(SHORT_DELAY), OneHot(MEDIUM_DELAY), OneHot(LONG_DELAY), OneHot(HOLD_ALL_INCOMING)]` (reflecting the currently active incoming manipulation strategy) + * `Current Outgoing Policy Applied`: `[OneHot(NORMAL), OneHot(SHORT_DELAY), OneHot(MEDIUM_DELAY), OneHot(LONG_DELAY), OneHot(HOLD_ALL_OUTGOING)]` (reflecting the currently active outgoing manipulation strategy) + + +#### Neural Network Outputs (Action Space - Concurrent) + +The neural network will generate multiple outputs in parallel, representing the bot's intended actions for the current tick. Probabilities (via sigmoid for binary, softmax for categorical) or direct regression values for continuous actions. + +1. **Movement & Pose Action Branch (Probabilities & Regression):** + * `Move Forward/Backward`: `[Float]` (e.g., regression between -1.0 and 1.0) + * `Strafe Left/Right`: `[Float]` (e.g., regression between -1.0 and 1.0) + * `Jump`: `[Probability_0_1]` (sigmoid) + * `Sneak`: `[Probability_0_1]` (sigmoid) + * `Sprint`: `[Probability_0_1]` (sigmoid) + +2. **Look Direction Branch (Regression):** + * `Delta Yaw`: `[Float]` (relative turn amount, e.g., in radians or degrees) + * `Delta Pitch`: `[Float]` (relative look up/down amount) + +3. **Interaction Action Branch (Probabilities & Conditional Parameters):** + * `Use Item (Primary/Right Click)`: `[Probability_0_1]` (sigmoid) + * *If actively using:* + * `Selected Hotbar Slot for Use`: `[OneHot_0_8]` (softmax over 9 slots) + * `Target Position for Use`: `[TargetX, TargetY, TargetZ]` (regression, relative to player) + * `Place Block (Secondary/Place Action)`: `[Probability_0_1]` (sigmoid) + * *If actively placing:* + * `Selected Hotbar Slot for Place`: `[OneHot_0_8]` (softmax) + * `Target Relative Position for Place (3D Grid)`: A `(SmallGridX, SmallGridY, SmallGridZ)` tensor, where values indicate probability of placing at each relative block coordinate (e.g., sigmoid for each voxel). + * `Target Face for Place`: `[OneHot_0_5]` (softmax over UP, DOWN, NORTH, EAST, SOUTH, WEST) + * `Swap Offhand`: `[Probability_0_1]` (sigmoid) + +4. **Combat Action Branch (Probabilities & Conditional Parameters):** + * `Attack Entity (Primary/Left Click)`: `[Probability_0_1]` (sigmoid) + * *If actively attacking:* + * `Target Entity Index for Attack`: `[OneHot_0_N]` (softmax over the N targeted entities from input) + +5. **Special Interest Output Branch (Path of Interest Generation):** + * `Areas of Interest for Detailed Scan`: A `(SmallGridX, SmallGridY, SmallGridZ)` tensor, where each value indicates a probability or score of that block being "interesting". This output determines which additional collision boxes (beyond the fixed radius) will be fed into the neural network on the next tick. + * `Path Waypoints`: A sequence of 3D coordinates `[(X1,Y1,Z1), (X2,Y2,Z2), ...]` representing the bot's predicted optimal path. During training, this is compared against Baritone's pathfinding output as ground truth. + * `Path Confidence`: A scalar value indicating the bot's confidence in its generated path, used for training stability and exploration strategies. + +6. **Network Control Branch (Probabilities & Conditional Parameters):** + * This branch controls actions related to manipulating network packets. Note that implementing these actions requires an external mechanism (e.g., a proxy or custom network stack) that the bot's decisions interface with. + * `Manage Network Flow`: `[Probability_0_1]` (sigmoid, to indicate if the bot wants to actively manage or manipulate network flow for the current tick). + * *If `Manage Network Flow` is active:* + * `Desired Incoming Packet Policy`: `[OneHot(NORMAL), OneHot(SHORT_DELAY), OneHot(MEDIUM_DELAY), OneHot(LONG_DELAY), OneHot(HOLD_ALL_INCOMING)]` (softmax over predefined delay profiles, plus `HOLD_ALL_INCOMING`). + * `Desired Outgoing Packet Policy`: `[OneHot(NORMAL), OneHot(SHORT_DELAY), OneHot(MEDIUM_DELAY), OneHot(LONG_DELAY), OneHot(HOLD_ALL_OUTGOING)]` (softmax over predefined delay profiles, plus `HOLD_ALL_OUTGOING`). + + + +This detailed specification ensures that the bot's neural network has the necessary information to make complex, concurrent decisions, and the framework supports iterative learning through its recurrent memory and the ability to train for specific behaviors. \ No newline at end of file diff --git a/packages/deep-learning-bot-utils/spec-v1/SPEC-TRAINING.md b/packages/deep-learning-bot-utils/spec-v1/SPEC-TRAINING.md new file mode 100644 index 0000000..f608a9c --- /dev/null +++ b/packages/deep-learning-bot-utils/spec-v1/SPEC-TRAINING.md @@ -0,0 +1,169 @@ +### Bot Training Session Specification + +**Objective:** Define the training methodology for a Minecraft bot using offline reinforcement learning with Baritone pathfinding as reference attention for collision box exploration. + +**Core Training Philosophy:** + +1. **Offline RL Approach:** Decouple data collection from training to enable batch learning and safer experimentation. +2. **Hierarchical Learning:** Separate movement execution from path planning through two-phase training. +3. **Baritone as Reference Teacher:** Use Baritone's A* pathfinding as ground truth for "area of interest" generation. +4. **Progressive Environment Complexity:** Start with simple environments and gradually increase difficulty. + +--- + +#### **Training Architecture Overview** + +The training process consists of two main phases that can be trained separately or jointly: + +1. **Phase 1: Movement Execution Training** + - Focus: Learn low-level movement actions to follow a given path + - Input: Current state + area of interest (collision boxes around target path) + - Output: Movement actions (forward/backward, strafe, jump, etc.) + - Teacher: Human demonstrations or scripted optimal movements + +2. **Phase 2: Path Interest Generation Training** + - Focus: Learn to output optimal "areas of interest" for detailed environmental scanning + - Input: Current state + goal information + - Output: Path of interest (sequence of blocks/waypoints) + - Teacher: Baritone's A* pathfinding output + +--- + +#### **Data Collection Methodology** + +**Environment Generation:** +- **Level 1:** Flat terrain with simple A-to-B navigation +- **Level 2:** Single-block-wide bridges +- **Level 3:** Bridges with gaps requiring jumping +- **Level 4:** Complex terrain with multiple path options +- **Level 5:** Dynamic obstacles and moving platforms + +**Data Collection Process:** + +1. **Environment Setup:** + - Generate scripted environment (flat/bridge/complex) + - Set start position and goal position + - Initialize Baritone pathfinding to goal + +2. **Per-Tick Data Capture:** + - Capture player state (position, velocity, pose, etc.) + - Capture local environment scan (fixed radius around player) + - Query Baritone for current optimal path to goal + - Capture collision boxes around Baritone's path (dynamic area of interest) + - Record actual player/bot actions taken + - Calculate rewards based on progress and safety + +3. **Data Storage Outline:** + - Player state vectors + - Environment collision box data (fixed + dynamic) + - Baritone reference path data + - Action vectors taken + - Reward signals + - Metadata (environment type, tick, session ID) + +**Baritone Integration Details:** + +1. **Path Extraction:** + - Query Baritone's pathfinding system for complete path to goal + - Extract waypoint sequence as BlockPos coordinates + - Calculate collision boxes around each waypoint (configurable radius) + +2. **Dynamic Area of Interest:** + - Fixed radius scan: 8-block radius around player (as in current bbox-logger) + - Dynamic scan: Collision boxes around Baritone's path waypoints + - Combined input: Fixed + Dynamic collision box data + +3. **Reference Attention Mechanism:** + - Baritone's path serves as "ground truth" for where the bot should focus attention + - Neural network learns to predict similar area-of-interest outputs + - Training objective: Minimize difference between NN output and Baritone's path + +--- + +#### **Training Phases** + +**Phase 1: Movement Execution Training** + +*Objective:* Train the bot to execute low-level movements to follow a given path. + +*Training Setup:* +- Input State: Player state + Local environment + **Given** area of interest (from Baritone) +- Target Actions: Optimal movement actions to progress along the path +- Loss Function: MSE on movement actions + reward-based RL loss + +*Data Requirements:* +- State-action pairs where area of interest is provided by Baritone +- Reward signals for path following, collision avoidance, goal progress +- Multiple environment types for generalization + +**Phase 2: Path Interest Generation Training** + +*Objective:* Train the bot to predict optimal areas of interest for environmental scanning. + +*Training Setup:* +- Input State: Player state + Local environment + Goal information +- Target Output: Area of interest coordinates (matching Baritone's path) +- Loss Function: Spatial distance loss between predicted and Baritone paths + +*Data Requirements:* +- State observations paired with Baritone's optimal path outputs +- Various goal configurations and environment layouts +- Path optimality metrics for evaluation + +--- + +#### **Reward Function Design** + +**Movement Phase Rewards:** +- **Path Following:** Distance to nearest waypoint on target path +- **Progress:** Forward movement toward goal +- **Collision Avoidance:** Penalty for hitting obstacles +- **Efficiency:** Bonus for reaching waypoints quickly + +**Path Interest Phase Rewards:** +- **Path Accuracy:** Similarity to Baritone's optimal path +- **Exploration Efficiency:** Coverage of relevant areas +- **Computational Cost:** Penalty for overly complex paths + +--- + +#### **Training Pipeline** + +1. **Data Collection Sessions:** + - Run scripted environments with Baritone pathfinding + - Collect state-action-reward tuples + - Store in compressed format for offline training + +2. **Offline Training:** + - Load collected datasets + - Train Phase 1 (movement) and Phase 2 (path interest) networks + - Use standard RL algorithms (PPO, SAC, etc.) or supervised learning + +3. **Evaluation:** + - Test on held-out environments + - Measure path following accuracy, goal completion rate + - Compare against Baritone baseline performance + +4. **Iterative Improvement:** + - Collect additional data in areas where bot performs poorly + - Retrain with augmented datasets + - Gradually increase environment complexity + +--- + +#### **Implementation Considerations** + +**Baritone API Requirements:** +- Access to computed path waypoints +- Real-time path updates when environment changes +- Ability to set custom goals and constraints + +**Data Collection Infrastructure:** +- Automated environment generation scripts +- Efficient collision box extraction and storage +- Parallel data collection across multiple game instances + +**Training Infrastructure:** +- Support for large-scale offline RL training +- Model checkpointing and evaluation pipelines +- Distributed training for faster iteration cycles \ No newline at end of file diff --git a/packages/lbng-utils-typed/package.json b/packages/lbng-utils-typed/package.json index 4aaa2f1..b1aabeb 100644 --- a/packages/lbng-utils-typed/package.json +++ b/packages/lbng-utils-typed/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "description": "Utils for scripting LBNG(graaljs), of my own taste instead of ccbluex's, written in typescript.", "author": "commandblock2", - "license": "GPL-3.0", + "license": "GPL-3.0-only", "dependencies": { "jvm-types": "^2025.7.8" }, diff --git a/src/baritone-example.ts b/src/baritone-example.ts new file mode 100644 index 0000000..65a804c --- /dev/null +++ b/src/baritone-example.ts @@ -0,0 +1,134 @@ +import { BaritoneAPI } from "jvm-types/baritone/api/BaritoneAPI" +import { GoalXZ } from "jvm-types/baritone/api/pathing/goals/GoalXZ" +import { BetterBlockPos } from "jvm-types/baritone/api/utils/BetterBlockPos" +import { IPath } from "jvm-types/baritone/api/pathing/calc/IPath" +import { PathCalculationResult$Type } from "jvm-types/baritone/api/utils/PathCalculationResult$Type" + +import { VisualizationManager } from "lbng-utils-typed/dist/visualization-utils" + +// note: this import is not from baritone-api jar +// it is only presented in the baritone-unoptimized jar +// as the `AStarPathFinder` class is possibly obfuscated in the baritone-standalone jar +// so you will have to install the baritone-unoptimized jar to use this import +import { AStarPathFinder } from "jvm-types/baritone/pathing/calc/AStarPathFinder" +import { Favoring } from "jvm-types/baritone/utils/pathing/Favoring" +import { CalculationContext } from "jvm-types/baritone/pathing/movement/CalculationContext" + + + +const script = registerScript.apply({ + name: "astar-pathfinder-example", + version: "1.0.0", + authors: ["commandblock2"] +}); + +script.registerModule({ + name: "baritone-api-example", + description: "Baritone example module", + category: "Client", + +}, (mod) => { + mod.on("enable", () => { + BaritoneAPI.getSettings().allowSprint.value = true; + BaritoneAPI.getSettings().primaryTimeoutMS.value = Primitives.long(2000); + const baritone = BaritoneAPI.getProvider().getPrimaryBaritone(); + baritone.getCustomGoalProcess().setGoalAndPath(new GoalXZ(100, 100)) + }) +}) + +script.registerModule({ + name: "astar-pathfinder-example", + description: "Direct AStarPathFinder construction example", + category: "Client", + settings: { + goalX: Setting.float({ + name: "Goal X", + default: 100, + range: [-10000, 10000] // Assuming a reasonable range + }), + goalZ: Setting.float({ + name: "Goal Z", + default: 100, + range: [-10000, 10000] // Assuming a reasonable range + }), + recalculateInterval: Setting.int({ + name: "Recalculate Interval (ticks)", + default: 20, + range: [1, 200] + }) + } +}, (mod) => { + + const viz = new VisualizationManager(mod); + + let previousPath: IPath | null = null; + + let lastRecalculateTick = 0; + + const calculatePath = () => { + const baritone = BaritoneAPI.getProvider().getPrimaryBaritone(); + + // Get current player position + const playerPos = baritone.getPlayerContext().playerFeet(); + const start = new BetterBlockPos(playerPos.getX(), playerPos.getY(), playerPos.getZ()); + + // Create calculation context for threaded use + const context = new CalculationContext(baritone, true); + + // Create favoring (empty favoring at first run for no preferences) + const favoring = new Favoring(baritone.getPlayerContext(), previousPath as unknown as IPath, context); + + // Create goal using settings + const goal = new GoalXZ(mod.settings.goalX.get(), mod.settings.goalZ.get()); + + // Construct AStarPathFinder directly + const pathfinder = new AStarPathFinder( + start, // realStart + start.getX(), // startX + start.getY(), // startY + start.getZ(), // startZ + goal, // goal + favoring, // favoring + context // context + ); + + // @ts-expect-error + UnsafeThread.run(() => { + const result = pathfinder.calculate(Primitives.long(2000), Primitives.long(5000)); + + // Handle result + if (result.getType() != PathCalculationResult$Type.CANCELLATION) { + const path = result.getPath().get(); + console.log("Path found! Length: " + path.length()); + mc.execute(() => { + viz.addVisualization({ + lineData: { + positions: path.positions().map((pos) => new Vec3d(pos.x + .5, pos.y, pos.z + .5)), + + }, + durationTicks: 20 * 60, + }); + previousPath = path; + }); + // Use the path as needed - you now have direct access without execution + } else { + console.log("Path calculation failed: " + result.getType().toString()); + } + }); + }; + + mod.on("enable", () => { + viz.clearAllVisualizations(); + lastRecalculateTick = 0; // Reset on enable + calculatePath(); // Initial calculation + }); + + mod.on("gametick", () => { + if (mc.player && mc.world && (mc.player.age - lastRecalculateTick) >= (mod.settings.recalculateInterval.get() as unknown as number)) { + calculatePath(); + lastRecalculateTick = mc.player.age; + } + }); +}); + +export { } \ No newline at end of file diff --git a/src/wall-builder-integrated-world.ts b/src/wall-builder-integrated-world.ts new file mode 100644 index 0000000..6e25856 --- /dev/null +++ b/src/wall-builder-integrated-world.ts @@ -0,0 +1,133 @@ +import { Direction } from "jvm-types/net/minecraft/util/math/Direction"; +import { Direction$Axis } from "jvm-types/net/minecraft/util/math/Direction$Axis"; +import { BlockPos } from "jvm-types/net/minecraft/util/math/BlockPos"; +import { ServerWorld } from "jvm-types/net/minecraft/server/world/ServerWorld"; +import { Blocks } from "jvm-types/net/minecraft/block/Blocks"; + +const script = registerScript.apply({ + name: "wall-builder-integrated-world", + version: "1.0.0", + authors: ["Roo"] +}); + +script.registerModule({ + name: "WallBuilderSinglePlayer", + description: "Places a wall of blocks in front of the player on enable. Only works in single player, which is intended to show that data generation can be done.", + category: "World", + settings: { + wallWidth: Setting.int({ + name: "WallWidth", + default: 3, + range: [1, 10], + suffix: "blocks" + }), + wallHeight: Setting.int({ + name: "WallHeight", + default: 3, + range: [1, 10], + suffix: "blocks" + }), + distance: Setting.int({ + name: "Distance", + default: 2, + range: [1, 10], + suffix: "blocks" + }), + blockType: Setting.choose({ + name: "BlockType", + default: "Stone", + choices: ["Stone", "Cobblestone", "Dirt", "Oak_Planks", "Glass"] + }) + } +}, (mod) => { + mod.on("enable", () => { + const serverInstance = mc.getServer(); + if (!serverInstance) { + Client.displayChatMessage("§cError: Integrated server not running."); + return; + } + + const serverWorld: ServerWorld = serverInstance.getOverworld(); + if (!serverWorld) { + Client.displayChatMessage("§cError: Server world not found."); + return; + } + + if (!mc.player) { + Client.displayChatMessage("§cError: Player not found."); + return; + } + const playerPos: BlockPos = mc.player.getBlockPos(); + const playerFacing: Direction = mc.player.getHorizontalFacing(); + + const wallWidth = mod.settings.wallWidth.getValue(); + const wallHeight = mod.settings.wallHeight.getValue(); + const distance = mod.settings.distance.getValue(); + const blockTypeName = mod.settings.blockType.getValue(); + + let blockToPlace; + switch (blockTypeName) { + case "Stone": + blockToPlace = Blocks.STONE; + break; + case "Cobblestone": + blockToPlace = Blocks.COBBLESTONE; + break; + case "Dirt": + blockToPlace = Blocks.DIRT; + break; + case "Oak_Planks": + blockToPlace = Blocks.OAK_PLANKS; + break; + case "Glass": + blockToPlace = Blocks.GLASS; + break; + default: + blockToPlace = Blocks.STONE; // Default to stone if unknown + break; + } + + const blockState = blockToPlace.getDefaultState(); + + let startX = playerPos.getX(); + let startY = playerPos.getY(); + let startZ = playerPos.getZ(); + + // Adjust starting position based on player facing and distance + if (playerFacing === Direction.NORTH) { + startZ -= distance + wallWidth - 1; // Wall extends away from player + startX -= Math.floor(wallWidth / 2); // Center the wall + } else if (playerFacing === Direction.SOUTH) { + startZ += distance; + startX -= Math.floor(wallWidth / 2); + } else if (playerFacing === Direction.EAST) { + startX += distance; + startZ -= Math.floor(wallWidth / 2); + } else if (playerFacing === Direction.WEST) { + startX -= distance + wallWidth - 1; + startZ -= Math.floor(wallWidth / 2); + } + + for (let y = 0; y < wallHeight; y++) { + for (let x = 0; x < wallWidth; x++) { + let targetX = startX; + let targetZ = startZ; + + // Adjust for wall orientation based on facing direction + if (playerFacing.getAxis() === Direction$Axis.Z) { // North/South + targetX += x; + } else { // East/West + targetZ += x; + } + + const targetPos = new BlockPos(targetX, startY + y, targetZ); + serverWorld.setBlockState(targetPos, blockState); + } + } + Client.displayChatMessage(`§aWall of ${blockTypeName} placed!`); + }); + + mod.on("disable", () => { + Client.displayChatMessage("§cWallBuilder disabled."); + }); +}); \ No newline at end of file