Plasmod/Makefile at main · CodeSoul-co/Plasmod · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
.PHONY: dev dev-with-s3 minio-start minio-stop demo-warm-batch build build-benchmark test integration-test integration-test-s3 cpp sdk-python fmt proto docker-up docker-down member-a-capture member-a-verify member-a-gpu-check member-a-task4-strict member-a-all prod-safety-check setup bench-layer1 bench-retrieval

# Default MinIO settings for local S3/MinIO integration tests.
# Override these when invoking make if your MinIO differs.
S3_ENDPOINT ?= 127.0.0.1:9000
S3_ACCESS_KEY ?= minioadmin
S3_SECRET_KEY ?= minioadmin
S3_BUCKET ?= plasmod-integration
S3_SECURE ?= false
S3_REGION ?= us-east-1
S3_PREFIX ?= plasmod/integration_tests

# RETRIEVAL_TAG enables the CGO Knowhere/HNSW retriever.
# It is only safe to set when cpp/build/libplasmod_retrieval.so/dylib exists.
# Use `make cpp` to build the C++ library and set this automatically.
RETRIEVAL_TAG :=
CPP_LIB := cpp/build/libplasmod_retrieval.dylib
CPP_LIB_SO := cpp/build/libplasmod_retrieval.so

# Cross-platform CPU count: nproc on Linux, sysctl on macOS.
NPROC := $(shell nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)
ifeq ($(shell [ -f $(CPP_LIB) ] && echo yes),yes)
  RETRIEVAL_TAG := -tags retrieval
  CGO_LDFLAGS := -L$(shell pwd)/cpp/build -lplasmod_retrieval -Wl,-rpath,$(shell pwd)/cpp/build
else ifeq ($(shell [ -f $(CPP_LIB_SO) ] && echo yes),yes)
  RETRIEVAL_TAG := -tags retrieval
  CGO_LDFLAGS := -L$(shell pwd)/cpp/build -lplasmod_retrieval -Wl,-rpath,$(shell pwd)/cpp/build
endif

# ── MinIO (local S3) ──────────────────────────────────────────────────────────
# Starts a local MinIO instance in the background.
# Requires: brew install minio minio-mc
# Bucket plasmod-experiments is auto-created on first run.
MINIO_DATA_DIR ?= /tmp/minio-data
MINIO_PID_FILE ?= /tmp/minio.pid

minio-start:
	@mkdir -p $(MINIO_DATA_DIR)
	@if curl -s http://127.0.0.1:9000/minio/health/live > /dev/null 2>&1; then \
		echo "MinIO already running at http://127.0.0.1:9000 (API) / http://127.0.0.1:9001 (Console)"; \
	else \
		echo "Starting MinIO..."; \
		minio server $(MINIO_DATA_DIR) --address ":9000" --console-address ":9001" >> /tmp/minio.log 2>&1 & \
		echo $$! > $(MINIO_PID_FILE); \
		for i in 1 2 3 4 5 6 7 8 9 10; do \
			sleep 1; \
			curl -s http://127.0.0.1:9000/minio/health/live > /dev/null 2>&1 && break; \
		done; \
		mc alias set myminio http://127.0.0.1:9000 minioadmin minioadmin 2>/dev/null || true; \
		mc mb myminio/plasmod-experiments 2>/dev/null || true; \
		echo "MinIO started. API: http://127.0.0.1:9000  Console: http://127.0.0.1:9001"; \
		echo "Log: /tmp/minio.log"; \
	fi

minio-stop:
	@if [ -f $(MINIO_PID_FILE) ]; then \
		kill $$(cat $(MINIO_PID_FILE)) 2>/dev/null && echo "MinIO stopped." || echo "MinIO not running."; \
		rm -f $(MINIO_PID_FILE); \
	else \
		pkill -f "minio server" 2>/dev/null && echo "MinIO stopped." || echo "MinIO not running."; \
	fi

# ── Dev ───────────────────────────────────────────────────────────────────────
dev-with-s3:
	$(MAKE) minio-start
	$(MAKE) dev

setup:
	go mod download
	bash scripts/setup_env.sh
	cd sdk/nodejs && npm install

dev:
	PLASMOD_BATCH_PLUGIN=1 CGO_LDFLAGS="$(CGO_LDFLAGS)" RETRIEVAL_TAG="$(RETRIEVAL_TAG)" bash scripts/dev_up.sh

# Second terminal only: assumes `make dev` is already running (default http://127.0.0.1:8080).
# Uses existing HTTP APIs: POST /v1/ingest/vectors then POST /v1/query/batch (single + multi agent).
# Override base URL: make demo-warm-batch PLASMOD_BASE_URL=http://127.0.0.1:19530
PLASMOD_BASE_URL ?= http://127.0.0.1:8080
demo-warm-batch:
	PLASMOD_BASE_URL="$(PLASMOD_BASE_URL)" bash scripts/demo/run_demo_warm_batch.sh

build:
	bash -c 'set -a; [ -f .env ] && source .env; set +a; CGO_LDFLAGS="$(CGO_LDFLAGS)" go build $(RETRIEVAL_TAG) -o bin/plasmod ./src/cmd/server'

build-benchmark:
	bash -c 'set -a; [ -f .env ] && source .env; set +a; CGO_LDFLAGS="$(CGO_LDFLAGS)" go build $(RETRIEVAL_TAG) -o plasmod_test_env/bin/benchmark ./src/cmd/benchmark'

cpp:
	cmake -S cpp -B cpp/build -DANDB_KNOWHERE_FAISS=ON && cmake --build cpp/build --parallel $(NPROC)

cpp-gpu:
	cmake -S cpp -B cpp/build -DANDB_WITH_GPU=ON && cmake --build cpp/build --parallel $(NPROC)

tensorrt:
	cmake -S cpp -B cpp/build_trt -DANDB_WITH_TENSORRT=ON -DANDB_WITH_GPU=OFF
	cmake --build cpp/build_trt --target plasmod_tensorrt --parallel $(NPROC)
	CGO_CFLAGS="-I/usr/local/cuda-12.9/include -I/usr/include/x86_64-linux-gnu" \
	CGO_LDFLAGS="-L$(shell pwd)/cpp/build_trt -lplasmod_tensorrt -lcudart -lnvinfer -Wl,-rpath,$(shell pwd)/cpp/build_trt" \
	go build -tags cuda,tensorrt,linux ./src/...

tensorrt-dev:
	bash -c 'set -a; [ -f .env ] && source .env; set +a; \
	CGO_CFLAGS="-I/usr/local/cuda-12.9/include -I/usr/include/x86_64-linux-gnu" \
	CGO_LDFLAGS="-L$(shell pwd)/cpp/build_trt -lplasmod_tensorrt -lcudart -lnvinfer -Wl,-rpath,$(shell pwd)/cpp/build_trt" \
	go run -tags cuda,tensorrt,linux ./src/cmd/server'

gguf:
	$(MAKE) -C libs/go-llama.cpp -j$(NPROC)
	go build -tags gguf ./src/...

gguf-dev:
	bash -c 'set -a; [ -f .env ] && source .env; set +a; CGO_LDFLAGS="$(CGO_LDFLAGS)" go run -tags gguf ./src/cmd/server'

# cpp-with-knowhere builds the full C++ stack including Knowhere HNSW.
# Requires: libomp, folly, prometheus-cpp, opentelemetry-cpp installed via Homebrew.
# Set CMAKE_PREFIX_PATH=/opt/homebrew when using Homebrew-installed deps.
# Run `make cpp` (without knowhere) first to build the stub, then upgrade.
cpp-with-knowhere:
	cmake -S cpp -B cpp/build \
	  -DANDB_WITH_KNOWHERE=ON \
	  -DOpenMP_C_FLAGS="-Xclang -fopenmp -I/opt/homebrew/Cellar/libomp/22.1.1/include" \
	  -DOpenMP_omp_LIBRARY="/opt/homebrew/Cellar/libomp/22.1.1/lib/libomp.dylib" \
	  -DCMAKE_PREFIX_PATH="/opt/homebrew" && cmake --build cpp/build

sdk-python:
	pip install -e ./sdk/python

test:
	go test ./src/...
	pytest -q

# Run full integration test suite against a running server (default: http://127.0.0.1:8080).
# Go tests cover all HTTP API routes; Python tests validate the Python SDK (AndbClient).
#
# Optional S3/MinIO dataflow test:
#   ANDB_RUN_S3_TESTS=true \
#   S3_ENDPOINT=127.0.0.1:9000 S3_ACCESS_KEY=minioadmin \
#   S3_SECRET_KEY=minioadmin S3_BUCKET=andb-integration \
#   make integration-test
integration-test:
	go test ./integration_tests/... -v -timeout 120s
	cd integration_tests/python && python run_all.py

integration-test-s3:
	ANDB_RUN_S3_TESTS=true \
	S3_ENDPOINT=$(S3_ENDPOINT) S3_ACCESS_KEY=$(S3_ACCESS_KEY) \
	S3_SECRET_KEY=$(S3_SECRET_KEY) S3_BUCKET=$(S3_BUCKET) \
	S3_SECURE=$(S3_SECURE) S3_REGION=$(S3_REGION) S3_PREFIX=$(S3_PREFIX) \
	$(MAKE) integration-test

fmt:
	gofmt -w $(shell find src -name '*.go')

# Regenerate Plasmod gRPC stubs from plasmod.proto (requires protoc + plugins on PATH).
PROTO_DIR := src/internal/api/grpc/proto
PROTO_OUT := src/internal/api/grpc/pb
proto:
	protoc -I $(PROTO_DIR) \
	  --go_out=$(PROTO_OUT) --go_opt=paths=source_relative \
	  --go-grpc_out=$(PROTO_OUT) --go-grpc_opt=paths=source_relative \
	  $(PROTO_DIR)/plasmod/v1/plasmod.proto


# Docker Compose: MinIO + ANDB with disk storage and S3 cold tier (see README Integration Tests).
docker-up:
	docker compose up -d

docker-down:
	docker compose down

# Requires a running server (e.g. `make dev` or `make docker-up`). Writes JSON per scenario under ./out/member_a/
member-a-capture:
	python scripts/e2e/member_a_capture.py --out-dir ./out/member_a

# One-command Member A verification: docker up + healthz + fixture capture.
member-a-verify:
	bash scripts/e2e/member_a_verify.sh

# Member A GPU visibility check (compose GPU overlay).
member-a-gpu-check:
	bash scripts/e2e/member_a_gpu_check.sh

# Strict Task 4: API-level E2E + S3 cold roundtrip unit tests in builder container.
member-a-task4-strict:
	bash scripts/e2e/member_a_task4_strict.sh

# Unified Member A entrypoint: verify + optional GPU check + strict task4.
member-a-all:
	bash scripts/e2e/member_a_all.sh

# Production visibility/sanitization guard.
prod-safety-check:
	bash scripts/check_prod_visibility.sh

# ── Layer-1 Benchmarks ─────────────────────────────────────────────────────────
# Group 2: FAISS HNSW (pure ANN baseline — no pipeline overhead)
# Group 3: Plasmod retrievalplane direct (Go CGO → Knowhere HNSW)
# Group 4: Plasmod full system  (HTTP → full agent-native pipeline)
#
# Requires:
#   - make cpp-with-knowhere  (builds cpp/build/libplasmod_retrieval.dylib)
#   - make build              (builds bin/plasmod with -tags retrieval)
#   - pip install numpy faiss-cpu requests
#   - Plasmod server running: bash start_server.sh
#
# Python script (Groups 2, 3, 4 via HTTP):
bench-layer1:
	python3 scripts/benchmark_layer1.py