From 591190cfeebd7590cdde7fb5f26449a8b666b5aa Mon Sep 17 00:00:00 2001 From: "kabir.kbr" Date: Fri, 17 Oct 2025 20:19:20 +0300 Subject: [PATCH 1/4] sorting out freeze crate requirements and declarations; also updadting BytesTrieMap renaming to PathMap to correspond to latest changes in PathMap repo --- .dockerignore | 37 +++++++++++++++++++++++++++++++++++++ expr/Cargo.toml | 1 + frontend/src/main.rs | 8 ++++---- 3 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..78d5e27f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,37 @@ +# Build artifacts - exclude to reduce context size +target/ +!target/docker/Dockerfile +!target/docker/build.sh +!target/docker/README.md + +# Git files +.git +.gitignore + +# IDE and editor files +.vscode +.idea +*.swp +*.swo +*~ +.DS_Store + +# Documentation (README at root is kept) +docs/ + +# Cache and temporary files +.cache +tmp/ +*.tmp + +# Claude and user-specific files +.claude +.channels_cache_v2.json +.users_cache.json + +# Logs +*.log + +# Environment files +.env +.env.local diff --git a/expr/Cargo.toml b/expr/Cargo.toml index d86a7c44..a1ea5ab7 100644 --- a/expr/Cargo.toml +++ b/expr/Cargo.toml @@ -10,6 +10,7 @@ gxhash = ["dep:gxhash"] [dependencies] +freeze = { workspace = true } smallvec = "1.13.2" gxhash = { optional = true, version="3.4.1", git="https://github.com/luketpeterson/gxhash/" } xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3", "const_xxh3"] } # Replacement for gxhash running under miri diff --git a/frontend/src/main.rs b/frontend/src/main.rs index b2d2fb52..9e8500b4 100644 --- a/frontend/src/main.rs +++ b/frontend/src/main.rs @@ -184,18 +184,18 @@ use pathmap::zipper::ZipperWriting; use mork_expr::{Expr, ExprZipper}; use mork_frontend::bytestring_parser::{Parser, ParserError, Context}; -use pathmap::trie_map::BytesTrieMap; +use pathmap::PathMap; struct DataParser { count: u64, - symbols: BytesTrieMap, + symbols: PathMap, } impl DataParser { fn new() -> Self { Self { count: 3, - symbols: BytesTrieMap::new(), + symbols: PathMap::new(), } } @@ -241,7 +241,7 @@ fn main() { let mut parser = DataParser::new(); let t0 = Instant::now(); - // let mut btm = BytesTrieMap::new(); + // let mut btm = PathMap::new(); let mut i = 0; let mut stack = [0; 2 << 19]; loop { From 2b76ceb4665c33049173e3e9665641fd1648aab7 Mon Sep 17 00:00:00 2001 From: "kabir.kbr" Date: Fri, 17 Oct 2025 20:20:35 +0300 Subject: [PATCH 2/4] files to build docker; unfortunately multiarch does not work due to rust nightly inconsistencies with qemu; need to build on different machines; --- docker/Dockerfile | 67 ++++++++++++++ docker/README.md | 216 ++++++++++++++++++++++++++++++++++++++++++++ docker/build.sh | 221 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 504 insertions(+) create mode 100644 docker/Dockerfile create mode 100644 docker/README.md create mode 100755 docker/build.sh diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..a2250cc4 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,67 @@ +# Dockerfile for MORK Environment +# Multi-architecture support: linux/amd64, linux/arm64 +# Base: Ubuntu with Rust, CMake, and all required dependencies + +# Declare platform arguments (automatically provided by buildx) +ARG TARGETPLATFORM +ARG BUILDPLATFORM + +FROM --platform=$TARGETPLATFORM ubuntu:22.04 + +# Re-declare for use in this stage +ARG TARGETPLATFORM +ARG BUILDPLATFORM +ARG TARGETOS +ARG TARGETARCH + +# Prevent interactive prompts during package installation +ENV DEBIAN_FRONTEND=noninteractive + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + build-essential \ + cmake \ + git \ + curl \ + ca-certificates \ + pkg-config \ + libssl-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install Rust (nightly toolchain with pinned version for stability) +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH \ + RUST_MIN_STACK=16777216 \ + CARGO_NET_GIT_FETCH_WITH_CLI=true + +# Use latest nightly for full edition2024 support +ARG RUST_VERSION=nightly + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ + sh -s -- -y --default-toolchain ${RUST_VERSION} --profile minimal && \ + chmod -R a+w $RUSTUP_HOME $CARGO_HOME + +# Set working directory +WORKDIR /build + +# Clone PathMap dependency +ARG PATHMAP_REPO=https://github.com/Adam-Vandervorst/PathMap.git +ARG PATHMAP_BRANCH=master +RUN git clone --depth 1 --branch ${PATHMAP_BRANCH} ${PATHMAP_REPO} PathMap + +# Copy MORK source code +COPY . /build/MORK + +# Build MORK in release mode with increased stack size and optimization for smaller code +WORKDIR /build/MORK +RUN cargo build --release --verbose + +# Set the entrypoint to the built binary +CMD ["/build/MORK/target/release/mork", "--help"] + +# Metadata +LABEL maintainer="MORK Team" +LABEL description="MeTTa Optimal Reduction Kernel - Multi-architecture support" +LABEL version="0.1.0" +LABEL org.opencontainers.image.source="https://github.com/trueagi-io/mork" \ No newline at end of file diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..02257c50 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,216 @@ +# MORK Docker Environment + +This directory contains Docker configuration for running MORK (MeTTa Optimal Reduction Kernel) in a containerized environment. + +**Key Feature**: This Docker setup uses your **local MORK repository** for building, so any local changes will be included in the image. PathMap is cloned from git during the build. + +## Contents + +- `Dockerfile` - Multi-stage Dockerfile for building MORK +- `build.sh` - Build script with multi-platform support +- `../.dockerignore` - Files to exclude from Docker build context (at MORK root) + +## Quick Start + +### Building the Image + +Build for all platforms (AMD64 and ARM64): +```bash +./target/docker/build.sh all +``` + +Build for specific platform: +```bash +# For AMD64/Intel systems +./target/docker/build.sh amd64 --load + +# For ARM64/Apple Silicon +./target/docker/build.sh arm64 --load +``` + +The `--load` flag loads the image into your local Docker daemon (required for single-platform builds if you want to run locally). + +### Running MORK in Docker + +Once built and loaded, run MORK: +```bash +docker run --rm mork:latest +``` + +Run with specific command: +```bash +docker run --rm mork:latest mork [your-args] +``` + +Mount a volume for data persistence: +```bash +docker run --rm -v $(pwd)/data:/mork/data mork:latest +``` + +## Build Script Options + +```bash +./target/docker/build.sh [PLATFORM] [OPTIONS] +``` + +### Platform Options +- `amd64` - Build for AMD64/Intel 64-bit systems (linux/amd64) +- `arm64` - Build for ARM64/Apple Silicon (linux/arm64) +- `all` - Build for all platforms (default) + +### Build Options +- `--no-cache` - Build without using Docker cache +- `--push` - Push image to container registry (requires `docker login`) +- `--load` - Load image into local Docker (single platform only) +- `-h, --help` - Display help message + +### Environment Variables +- `MORK_IMAGE_NAME` - Docker image name (default: `mork`) +- `MORK_IMAGE_TAG` - Docker image tag (default: `latest`) +- `PATHMAP_BRANCH` - Git branch/tag for PathMap repository (default: `master`) + +## Examples + +### Build for local development (Apple Silicon) +```bash +./target/docker/build.sh arm64 --load +``` + +### Build for local development (Intel Mac or Linux) +```bash +./target/docker/build.sh amd64 --load +``` + +### Build and push to registry +```bash +export MORK_IMAGE_NAME=myregistry/mork +export MORK_IMAGE_TAG=v0.1.0 +docker login myregistry +./target/docker/build.sh all --push +``` + +### Build without cache (clean build) +```bash +./target/docker/build.sh arm64 --no-cache --load +``` + +### Build with specific PathMap branch +```bash +export PATHMAP_BRANCH=experimental +./target/docker/build.sh arm64 --load +``` + +## Architecture + +### Multi-Stage Build + +The Dockerfile uses a multi-stage build process: + +1. **Builder Stage**: + - Based on `debian:bookworm-slim` + - Installs Rust nightly toolchain via rustup + - Installs build dependencies (gcc, cmake, libssl, etc.) + - **Clones PathMap from git** (https://github.com/Adam-Vandervorst/PathMap.git) + - **Copies local MORK repository** (including any uncommitted changes) + - Builds MORK with `--release` flag + +2. **Runtime Stage**: + - Based on minimal `debian:bookworm-slim` + - Includes only runtime dependencies + - Runs as non-root user (`mork`) + - Contains only the compiled binaries + +This approach: +- Minimizes the final image size (builder artifacts are discarded) +- Uses your local MORK code (perfect for development and testing) +- Ensures PathMap dependency is always up-to-date from git +- Includes all local changes (even uncommitted ones) + +### Multi-Platform Support + +The build script uses Docker Buildx to support multiple platforms: +- **linux/amd64** - For Intel/AMD 64-bit systems +- **linux/arm64** - For ARM64 systems (Apple Silicon, ARM servers) + +## Requirements + +- Docker with buildx support (Docker Desktop includes this) +- Internet connection (to clone PathMap) +- Sufficient disk space for build artifacts (~2GB during build, ~200MB final image) + +**Important**: The build uses your local MORK files, so make sure you're in the MORK directory when running the build script. + +## Troubleshooting + +### Git clone fails for PathMap +If you see errors cloning PathMap: +- Check your internet connection +- Verify https://github.com/Adam-Vandervorst/PathMap.git is accessible +- Try building with a specific branch: + ```bash + export PATHMAP_BRANCH=master + ./target/docker/build.sh arm64 --load + ``` + +### Buildx not available +Install or enable Docker Buildx: +```bash +docker buildx version +``` + +If not available, update Docker Desktop or install the buildx plugin. + +### Build fails with memory error +Increase Docker's memory allocation in Docker Desktop settings (Preferences → Resources → Memory). + +### Multi-platform build not loading +Multi-platform builds cannot be loaded directly into Docker. Either: +- Build for a single platform with `--load`: + ```bash + ./target/docker/build.sh arm64 --load + ``` +- Or push to a registry and pull back: + ```bash + ./target/docker/build.sh all --push + docker pull mork:latest + ``` + +### Build context is too large +If the build is slow due to large context: +- Check that `target/` directory is excluded (via `.dockerignore`) +- Remove any large files from the MORK directory +- The `.dockerignore` file should already exclude build artifacts + +## Image Details + +- **Base Image**: debian:bookworm-slim (both builder and runtime stages) +- **Rust Version**: Nightly (installed via rustup in builder stage) +- **User**: Non-root user `mork` (UID 1000) +- **Working Directory**: `/mork` +- **Data Directory**: `/mork/data` (for mounting volumes) +- **Binaries**: Installed in `/usr/local/bin/` + +## Notes + +- The Dockerfile installs Rust nightly due to MORK's use of edition 2024 features +- PathMap dependency is cloned from git during build +- **Local MORK files** (including uncommitted changes) are included in the build +- Build artifacts from local builds are excluded via `.dockerignore` +- The image runs as a non-root user for security + +## Development Workflow + +This Docker setup is ideal for development: + +1. **Make changes** to MORK code locally +2. **Build the image** with your changes: + ```bash + ./target/docker/build.sh arm64 --load + ``` +3. **Test** the changes in the container: + ```bash + docker run --rm mork:latest mork [test-args] + ``` +4. **Iterate** - repeat steps 1-3 as needed + +Since the build uses local files, you can test changes before committing them to git. diff --git a/docker/build.sh b/docker/build.sh new file mode 100755 index 00000000..e16b6e1f --- /dev/null +++ b/docker/build.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash + +# MORK Docker Build Script +# Supports multi-platform builds for amd64, Apple Silicon (arm64), and Intel Mac +# Usage: ./build.sh [PLATFORM] [OPTIONS] +# +# PLATFORM options: +# amd64 - Build for AMD64/Intel 64-bit (linux/amd64) +# arm64 - Build for ARM64/Apple Silicon (linux/arm64) +# all - Build for all platforms (default) +# +# OPTIONS: +# --no-cache - Build without using cache +# --push - Push image to registry (requires login) +# --load - Load image into local Docker (single platform only) + +set -e + +# Configuration +IMAGE_NAME="${MORK_IMAGE_NAME:-mork}" +IMAGE_TAG="${MORK_IMAGE_TAG:-latest}" +DOCKERFILE="./target/docker/Dockerfile" +PATHMAP_BRANCH="${PATHMAP_BRANCH:-master}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Function to print colored messages +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Function to display usage +usage() { + cat << EOF +MORK Docker Build Script + +Usage: $0 [PLATFORM] [OPTIONS] + +PLATFORM: + amd64 Build for AMD64/Intel 64-bit (linux/amd64) + arm64 Build for ARM64/Apple Silicon (linux/arm64) + all Build for all platforms (default) + +OPTIONS: + --no-cache Build without using cache + --push Push image to registry (requires docker login) + --load Load image into local Docker (single platform only) + -h, --help Display this help message + +ENVIRONMENT VARIABLES: + MORK_IMAGE_NAME Docker image name (default: mork) + MORK_IMAGE_TAG Docker image tag (default: latest) + PATHMAP_BRANCH Git branch/tag for PathMap (default: master) + +EXAMPLES: + # Build for all platforms + $0 all + + # Build for Apple Silicon and load into Docker + $0 arm64 --load + + # Build for amd64 without cache + $0 amd64 --no-cache + + # Build and push to registry + $0 all --push + +EOF + exit 0 +} + +# Parse arguments +PLATFORM="all" +EXTRA_ARGS="" +USE_LOAD=false +USE_PUSH=false + +while [[ $# -gt 0 ]]; do + case $1 in + amd64|arm64|all) + PLATFORM="$1" + shift + ;; + --no-cache) + EXTRA_ARGS="$EXTRA_ARGS --no-cache" + shift + ;; + --push) + USE_PUSH=true + shift + ;; + --load) + USE_LOAD=true + shift + ;; + -h|--help) + usage + ;; + *) + log_error "Unknown option: $1" + usage + ;; + esac +done + +# Validate platform and load combination +if [ "$USE_LOAD" = true ] && [ "$PLATFORM" = "all" ]; then + log_error "--load can only be used with a single platform (amd64 or arm64)" + exit 1 +fi + +if [ "$USE_PUSH" = true ] && [ "$USE_LOAD" = true ]; then + log_error "--push and --load cannot be used together" + exit 1 +fi + +# Set platform string +case $PLATFORM in + amd64) + PLATFORMS="linux/amd64" + PLATFORM_DESC="AMD64/Intel 64-bit" + ;; + arm64) + PLATFORMS="linux/arm64" + PLATFORM_DESC="ARM64/Apple Silicon" + ;; + all) + PLATFORMS="linux/amd64,linux/arm64" + PLATFORM_DESC="All platforms (AMD64 and ARM64)" + ;; +esac + +# Add push or load flag +if [ "$USE_PUSH" = true ]; then + EXTRA_ARGS="$EXTRA_ARGS --push" + ACTION="build and push" +elif [ "$USE_LOAD" = true ]; then + EXTRA_ARGS="$EXTRA_ARGS --load" + ACTION="build and load" +else + ACTION="build" +fi + +# Check if buildx is available +if ! docker buildx version > /dev/null 2>&1; then + log_error "Docker buildx is not available. Please install or enable Docker buildx." + exit 1 +fi + +# Create builder if it doesn't exist +BUILDER_NAME="mork-builder" +if ! docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then + log_info "Creating buildx builder: $BUILDER_NAME" + docker buildx create --name "$BUILDER_NAME" --use +else + log_info "Using existing buildx builder: $BUILDER_NAME" + docker buildx use "$BUILDER_NAME" +fi + +# Display build information +log_info "=========================================" +log_info "MORK Docker Build Configuration" +log_info "=========================================" +log_info "Image: $IMAGE_NAME:$IMAGE_TAG" +log_info "Platform(s): $PLATFORM_DESC" +log_info "Action: $ACTION" +log_info "Dockerfile: $DOCKERFILE" +log_info "PathMap Branch: $PATHMAP_BRANCH" +log_info "=========================================" + +# Build the image +log_info "Starting Docker build..." +log_info "This may take several minutes (building from local source)..." + +# Build from MORK root directory (need local files in context) +cd "$(dirname "$0")/../.." + +if docker buildx build \ + --platform "$PLATFORMS" \ + --tag "$IMAGE_NAME:$IMAGE_TAG" \ + --file "$DOCKERFILE" \ + --build-arg PATHMAP_BRANCH="$PATHMAP_BRANCH" \ + $EXTRA_ARGS \ + .; then + log_success "Docker image built successfully!" + log_success "Image: $IMAGE_NAME:$IMAGE_TAG" + log_success "Platform(s): $PLATFORMS" + + if [ "$USE_LOAD" = true ]; then + log_info "Image loaded into local Docker" + log_info "Run with: docker run --rm $IMAGE_NAME:$IMAGE_TAG" + elif [ "$USE_PUSH" = true ]; then + log_info "Image pushed to registry" + else + log_warning "Image built but not loaded or pushed" + log_info "To load into Docker, use: --load (single platform only)" + log_info "To push to registry, use: --push" + fi +else + log_error "Docker build failed!" + exit 1 +fi + +log_info "Build completed successfully!" From 4262b18449730b9a1a612dc2f8ac82ef3329297c Mon Sep 17 00:00:00 2001 From: "kabir.kbr" Date: Fri, 17 Oct 2025 20:58:18 +0300 Subject: [PATCH 3/4] corrected script to account for directory changes --- docker/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/build.sh b/docker/build.sh index e16b6e1f..dbaa5ea4 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -19,7 +19,7 @@ set -e # Configuration IMAGE_NAME="${MORK_IMAGE_NAME:-mork}" IMAGE_TAG="${MORK_IMAGE_TAG:-latest}" -DOCKERFILE="./target/docker/Dockerfile" +DOCKERFILE="./docker/Dockerfile" PATHMAP_BRANCH="${PATHMAP_BRANCH:-master}" # Colors for output @@ -190,7 +190,7 @@ log_info "Starting Docker build..." log_info "This may take several minutes (building from local source)..." # Build from MORK root directory (need local files in context) -cd "$(dirname "$0")/../.." +cd "$(dirname "$0")/.." if docker buildx build \ --platform "$PLATFORMS" \ From 200da6aaf827d0a7e14953a401dffefb7e1763e1 Mon Sep 17 00:00:00 2001 From: "kabir.kbr" Date: Fri, 17 Oct 2025 21:04:50 +0300 Subject: [PATCH 4/4] actually, we need freeze crate; adding it back to main Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 5075bfd8..d6bf5e89 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ mork-frontend = {path = "./frontend"} mork = {path = "./kernel"} # External to MORK -#freeze = { version="0.1.1", git="https://github.com/luketpeterson/Freeze.git" } #Fixes build on MacOS +freeze = {version="0.1.1", git="https://github.com/luketpeterson/Freeze.git"} #Fixes build on MacOS gxhash = {version="3.4.1", git="https://github.com/luketpeterson/gxhash/"} # for dag_serialization [workspace.dependencies.pathmap]