diff --git a/README.md b/README.md index cb371b4..1a680d0 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,19 @@ # Apache Pulsar Java Apache Pulsar user-defined source and sink implemented using Java. -## Getting Started -#### Run Pulsar and Pulsar Manager UI: -``` docker-compose up``` +**Note for Production:** For StreamNative Cloud or secured clusters, see [Pulsar on StreamNative](docs/get-started/pulsar-on-streamnative.md) -- locally, go to http://localhost:9527/#/login?redirect=%2F +## Getting Started: Local Development Setup -#### Numaflow: -* mvn clean install -* kubectl apply -f pipeline.yaml -* kubectl apply -f pulsar-config-map.yaml -* kubectl -n numaflow-system port-forward deployment/numaflow-server 8443:8443 +#### Run local Pulsar cluster and Pulsar Manager UI: +This starts a local Pulsar broker and the Pulsar Manager web interface for monitoring. -This builds the UDsink image, and the pipeline file uses it. Go to https://localhost:8443/ to use Numaflow UI +```bash +docker-compose up +``` -#### Pulsar Manager set up account: -```` +#### Set Pulsar Manager account by running this in your terminal: +``` CSRF_TOKEN=$(curl http://localhost:7750/pulsar-manager/csrf-token) curl \ -H "X-XSRF-TOKEN: $CSRF_TOKEN" \ @@ -24,6 +21,27 @@ curl \ -H "Content-Type: application/json" \ -X PUT http://localhost:7750/pulsar-manager/users/superuser \ -d '{"name": "admin", "password": "apachepulsar", "description": "test", "email": "username@test.org"}' -```` +``` + +#### Access Pulsar Manager UI: + +Go to http://localhost:9527/#/login?redirect=%2F and login with the name and password fields specified in the command above. In Pulsar Manager, once logged in, create an environment with the service URL http://pulsar:8080/ and Bookie url http://pulsar:8080/.image + +#### Deploy to Numaflow: +Note: navigate to the ```docs``` folder, and go to either the sink/source subfolder (```docs/source/byte-array/``` or ```docs/sink/byte-array/```) for example producer and consumer pipeline and configmap configurations. For local Pulsar cluster, you can omit the ```authPluginClassName``` and ```authParams``` fields for the config map implementation. + +```bash +# Build the project and create the Docker image +mvn clean install + +# Deploy the pipeline and configuration to Kubernetes +kubectl apply -f +kubectl apply -f + +# Port-forward to access Numaflow UI +kubectl -n numaflow-system port-forward deployment/numaflow-server 8443:8443 +``` + +Then access the Numaflow UI at https://localhost:8443/ \ No newline at end of file diff --git a/docs/sink/byte-array/manifests/api-key/byte-arr-producer-pipeline.yaml b/docs/sink/byte-array/manifests/api-key/byte-arr-producer-pipeline.yaml index 03f723d..efd1a96 100644 --- a/docs/sink/byte-array/manifests/api-key/byte-arr-producer-pipeline.yaml +++ b/docs/sink/byte-array/manifests/api-key/byte-arr-producer-pipeline.yaml @@ -10,10 +10,6 @@ spec: rpu: 1 duration: 10s msgSize: 10 - - name: p1 - udf: - builtin: - name: cat - name: out volumes: # Shared between containers that are part of the same pod, useful for sharing configurations - name: pulsar-config-volume @@ -33,6 +29,4 @@ spec: mountPath: /conf edges: - from: in - to: p1 - - from: p1 to: out \ No newline at end of file diff --git a/docs/source/byte-array/manifests/api-key/byte-arr-consumer-config.yaml b/docs/source/byte-array/manifests/api-key/byte-arr-consumer-config.yaml index 2e5747b..104fc1f 100644 --- a/docs/source/byte-array/manifests/api-key/byte-arr-consumer-config.yaml +++ b/docs/source/byte-array/manifests/api-key/byte-arr-consumer-config.yaml @@ -15,7 +15,7 @@ data: consumer: # see here for all configurations: https://pulsar.apache.org/reference/#/4.0.x/client/client-configuration-consumer enabled: true consumerConfig: - topicName: "persistent://public/default/test-topic" + topicNames: "persistent://public/default/test-topic" subscriptionName: "test-subscription" admin: adminConfig: # Accepts the same key-value pair configurations as pulsar client: https://pulsar.apache.org/reference/#/4.0.x/client/client-configuration-client diff --git a/docs/source/byte-array/manifests/api-key/byte-arr-consumer-pipeline.yaml b/docs/source/byte-array/manifests/api-key/byte-arr-consumer-pipeline.yaml index ea7b27c..d25d316 100644 --- a/docs/source/byte-array/manifests/api-key/byte-arr-consumer-pipeline.yaml +++ b/docs/source/byte-array/manifests/api-key/byte-arr-consumer-pipeline.yaml @@ -25,12 +25,6 @@ spec: volumeMounts: - name: pulsar-config-volume mountPath: /conf - - name: p1 - scale: - min: 1 - udf: - builtin: - name: cat - name: out scale: min: 1 @@ -38,6 +32,4 @@ spec: log: {} edges: - from: in - to: p1 - - from: p1 to: out \ No newline at end of file