From cf60682c8290f1191c5d3e4609a8ad3b8d1b162a Mon Sep 17 00:00:00 2001 From: Chingis Yundunov Date: Thu, 13 Feb 2025 10:02:03 +0700 Subject: [PATCH 001/226] DocSum - add files for deploy app with ROCm vLLM Signed-off-by: Chingis Yundunov --- DocSum/Dockerfile-vllm-rocm | 18 ++ .../amd/gpu/rocm-vllm/README.md | 175 ++++++++++++ .../amd/gpu/rocm-vllm/compose.yaml | 107 ++++++++ .../amd/gpu/rocm-vllm/set_env.sh | 16 ++ DocSum/docker_image_build/build.yaml | 9 + DocSum/tests/test_compose_on_rocm_vllm.sh | 249 ++++++++++++++++++ 6 files changed, 574 insertions(+) create mode 100644 DocSum/Dockerfile-vllm-rocm create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh create mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm new file mode 100644 index 0000000000..f0e8a8743a --- /dev/null +++ b/DocSum/Dockerfile-vllm-rocm @@ -0,0 +1,18 @@ +FROM rocm/vllm-dev:main + +# Set the working directory +WORKDIR /workspace + +# Copy the api_server.py into the image +ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py + +# Expose the port used by the API server +EXPOSE 8011 + +# Set environment variables +ENV HUGGINGFACE_HUB_CACHE=/workspace +ENV WILM_USE_TRITON_FLASH_ATTENTION=0 +ENV PYTORCH_JIT=0 + +# Set the entrypoint to the api_server.py script +ENTRYPOINT ["python3", "/workspace/api_server.py"] diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md new file mode 100644 index 0000000000..4d41a5cd31 --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md @@ -0,0 +1,175 @@ +# Build and deploy DocSum Application on AMD GPU (ROCm) + +## Build images + +## 🚀 Build Docker Images + +First of all, you need to build Docker Images locally and install the python package of it. + +### 1. Build LLM Image + +```bash +git clone https://github.com/opea-project/GenAIComps.git +cd GenAIComps +docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile . +``` + +Then run the command `docker images`, you will have the following four Docker Images: + +### 2. Build MegaService Docker Image + +To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command: + +```bash +git clone https://github.com/opea-project/GenAIExamples +cd GenAIExamples/DocSum/ +docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . +``` + +### 3. Build UI Docker Image + +Build the frontend Docker image via below command: + +```bash +cd GenAIExamples/DocSum/ui +docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . +``` + +Then run the command `docker images`, you will have the following Docker Images: + +1. `opea/llm-docsum-tgi:latest` +2. `opea/docsum:latest` +3. `opea/docsum-ui:latest` + +### 4. Build React UI Docker Image + +Build the frontend Docker image via below command: + +```bash +cd GenAIExamples/DocSum/ui +export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum" +docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react . + +docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . +``` + +Then run the command `docker images`, you will have the following Docker Images: + +1. `opea/llm-docsum-tgi:latest` +2. `opea/docsum:latest` +3. `opea/docsum-ui:latest` +4. `opea/docsum-react-ui:latest` + +## 🚀 Start Microservices and MegaService + +### Required Models + +Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model. +For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable. + +### Setup Environment Variables + +Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below. + +```bash +export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm" +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export HOST_IP=${host_ip} +export DOCSUM_TGI_SERVICE_PORT="18882" +export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} +export DOCSUM_LLM_SERVER_PORT="8008" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export DocSum_COMPONENT_NAME="OpeaDocSumTgi" +``` + +Note: Please replace with `host_ip` with your external IP address, do not use localhost. + +Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered, where is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) + +Example for set isolation for 1 GPU + +``` + - /dev/dri/card0:/dev/dri/card0 + - /dev/dri/renderD128:/dev/dri/renderD128 +``` + +Example for set isolation for 2 GPUs + +``` + - /dev/dri/card0:/dev/dri/card0 + - /dev/dri/renderD128:/dev/dri/renderD128 + - /dev/dri/card1:/dev/dri/card1 + - /dev/dri/renderD129:/dev/dri/renderD129 +``` + +Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) + +### Start Microservice Docker Containers + +```bash +cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm +docker compose up -d +``` + +### Validate Microservices + +1. TGI Service + + ```bash + curl http://${host_ip}:8008/generate \ + -X POST \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \ + -H 'Content-Type: application/json' + ``` + +2. LLM Microservice + + ```bash + curl http://${host_ip}:9000/v1/docsum \ + -X POST \ + -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \ + -H 'Content-Type: application/json' + ``` + +3. MegaService + + ```bash + curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{ + "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false + }' + ``` + +## 🚀 Launch the Svelte UI + +Open this URL `http://{host_ip}:5173` in your browser to access the frontend. + +![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b) + +Here is an example for summarizing a article. + +![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55) + +## 🚀 Launch the React UI (Optional) + +To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below: + +```yaml +docsum-rocm-react-ui-server: + image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest} + container_name: docsum-rocm-react-ui-server + depends_on: + - docsum-rocm-backend-server + ports: + - "5174:80" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT} +``` + +Open this URL `http://{host_ip}:5175` in your browser to access the frontend. + +![project-screenshot](../../../../assets/img/docsum-ui-react.png) diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml new file mode 100644 index 0000000000..037aa06395 --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml @@ -0,0 +1,107 @@ +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +services: + docsum-vllm-service: + image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} + container_name: docsum-vllm-service + ports: + - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011" + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + WILM_USE_TRITON_FLASH_ATTENTION: 0 + PYTORCH_JIT: 0 + volumes: + - "./data:/data" + shm_size: 20G + devices: + - /dev/kfd:/dev/kfd + - /dev/dri/:/dev/dri/ + cap_add: + - SYS_PTRACE + group_add: + - video + security_opt: + - seccomp:unconfined + - apparmor=unconfined + command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\"" + ipc: host + + docsum-llm-server: + image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} + container_name: docsum-llm-server + depends_on: + - docsum-vllm-service + ports: + - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000" + ipc: host + cap_add: + - SYS_PTRACE + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}" + HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID} + LOGFLAG: ${DOCSUM_LOGFLAG:-False} + MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS} + MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS} + restart: unless-stopped + + whisper-service: + image: ${REGISTRY:-opea}/whisper:${TAG:-latest} + container_name: whisper-service + ports: + - "${DOCSUM_WHISPER_PORT:-7066}:7066" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + restart: unless-stopped + + docsum-backend-server: + image: ${REGISTRY:-opea}/docsum:${TAG:-latest} + container_name: docsum-backend-server + depends_on: + - docsum-tgi-service + - docsum-llm-server + ports: + - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888" + environment: + no_proxy: ${no_proxy} + https_proxy: ${https_proxy} + http_proxy: ${http_proxy} + MEGA_SERVICE_HOST_IP: ${HOST_IP} + LLM_SERVICE_HOST_IP: ${HOST_IP} + ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP} + ipc: host + restart: always + + docsum-gradio-ui: + image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest} + container_name: docsum-ui-server + depends_on: + - docsum-backend-server + ports: + - "${DOCSUM_FRONTEND_PORT:-5173}:5173" + environment: + no_proxy: ${no_proxy} + https_proxy: ${https_proxy} + http_proxy: ${http_proxy} + BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} + DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} + ipc: host + restart: always + +networks: + default: + driver: bridge diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh new file mode 100644 index 0000000000..43e71e0fbf --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +export HOST_IP="" +export DOCSUM_MAX_INPUT_TOKENS=2048 +export DOCSUM_MAX_TOTAL_TOKENS=4096 +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export DOCSUM_VLLM_SERVICE_PORT="8008" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN="" +export DOCSUM_LLM_SERVER_PORT="9000" +export DOCSUM_WHISPER_PORT="7066" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml index 095fd28c93..dc0d546189 100644 --- a/DocSum/docker_image_build/build.yaml +++ b/DocSum/docker_image_build/build.yaml @@ -47,3 +47,12 @@ services: dockerfile: comps/llms/src/doc-summarization/Dockerfile extends: docsum image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} + vllm_rocm: + build: + args: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + no_proxy: ${no_proxy} + context: ../ + dockerfile: ./Dockerfile-vllm-rocm + image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh new file mode 100644 index 0000000000..d0919a019a --- /dev/null +++ b/DocSum/tests/test_compose_on_rocm_vllm.sh @@ -0,0 +1,249 @@ +#!/bin/bash +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +set -xe +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') +export MAX_INPUT_TOKENS=1024 +export MAX_TOTAL_TOKENS=2048 +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export HOST_IP=${ip_address} +export DOCSUM_VLLM_SERVICE_PORT="8008" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} +export DOCSUM_LLM_SERVER_PORT="9000" +export DOCSUM_WHISPER_PORT="7066" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export MEGA_SERVICE_HOST_IP=${HOST_IP} +export LLM_SERVICE_HOST_IP=${HOST_IP} +export ASR_SERVICE_HOST_IP=${HOST_IP} +export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" + +function build_docker_images() { + opea_branch=${opea_branch:-"main"} + # If the opea_branch isn't main, replace the git clone branch in Dockerfile. + if [[ "${opea_branch}" != "main" ]]; then + cd $WORKPATH + OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" + NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" + find . -type f -name "Dockerfile*" | while read -r file; do + echo "Processing file: $file" + sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" + done + fi + + cd $WORKPATH/docker_image_build + git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + + echo "Build all the images with --no-cache, check docker_image_build.log for details..." + service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper" + docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log + + docker images && sleep 1s +} + +function start_services() { + cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm + sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env + # Start Docker Containers + docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log + sleep 1m +} + +function validate_services() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + + echo "===========================================" + + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "EXPECTED_RESULT==> $EXPECTED_RESULT" + echo "CONTENT==> $CONTENT" + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +get_base64_str() { + local file_name=$1 + base64 -w 0 "$file_name" +} + +# Function to generate input data for testing based on the document type +input_data_for_test() { + local document_type=$1 + case $document_type in + ("text") + echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are." + ;; + ("audio") + get_base64_str "$WORKPATH/tests/data/test.wav" + ;; + ("video") + get_base64_str "$WORKPATH/tests/data/test.mp4" + ;; + (*) + echo "Invalid document type" >&2 + exit 1 + ;; + esac +} + +function validate_microservices() { + # Check if the microservices are running correctly. + + # whisper microservice + ulimit -s 65536 + validate_services \ + "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \ + '{"asr_result":"well"}' \ + "whisper-service" \ + "whisper-service" \ + "{\"audio\": \"$(input_data_for_test "audio")\"}" + + # vLLM service + validate_services \ + "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \ + "generated_text" \ + "docsum-vllm-service" \ + "docsum-vllm-service" \ + '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}' + + # llm microservice + validate_services \ + "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \ + "text" \ + "docsum-llm-server" \ + "docsum-llm-server" \ + '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' + +} + +function validate_megaservice() { + local SERVICE_NAME="docsum-backend-server" + local DOCKER_NAME="docsum-backend-server" + local EXPECTED_RESULT="[DONE]" + local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." + local URL="${host_ip}:8888/v1/docsum" + local DATA_TYPE="type=text" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL") + + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +function validate_megaservice_json() { + # Curl the Mega Service + echo "" + echo ">>> Checking text data with Content-Type: application/json" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' + + echo ">>> Checking audio data" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + "{\"type\": \"audio\", \"messages\": \"$(input_data_for_test "audio")\"}" + + echo ">>> Checking video data" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + "{\"type\": \"video\", \"messages\": \"$(input_data_for_test "video")\"}" + +} + +function stop_docker() { + cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/ + docker compose stop && docker compose rm -f +} + +function main() { + echo "===========================================" + echo ">>>> Stopping any running Docker containers..." + stop_docker + + echo "===========================================" + if [[ "$IMAGE_REPO" == "opea" ]]; then + echo ">>>> Building Docker images..." + build_docker_images + fi + + echo "===========================================" + echo ">>>> Starting Docker services..." + start_services + + echo "===========================================" + echo ">>>> Validating microservices..." + validate_microservices + + echo "===========================================" + echo ">>>> Validating megaservice..." + validate_megaservice + echo ">>>> Validating validate_megaservice_json..." + validate_megaservice_json + + echo "===========================================" + echo ">>>> Stopping Docker containers..." + stop_docker + + echo "===========================================" + echo ">>>> Pruning Docker system..." + echo y | docker system prune + echo ">>>> Docker system pruned successfully." + echo "===========================================" +} + +main From 1fd1de1530328321d28aa6d9db85fffeb876574c Mon Sep 17 00:00:00 2001 From: Chingis Yundunov Date: Thu, 13 Feb 2025 10:07:05 +0700 Subject: [PATCH 002/226] DocSum - fix main Signed-off-by: Chingis Yundunov --- DocSum/Dockerfile-vllm-rocm | 18 -- .../amd/gpu/rocm-vllm/README.md | 175 ------------ .../amd/gpu/rocm-vllm/compose.yaml | 107 -------- .../amd/gpu/rocm-vllm/set_env.sh | 16 -- DocSum/docker_image_build/build.yaml | 9 - DocSum/tests/test_compose_on_rocm_vllm.sh | 249 ------------------ 6 files changed, 574 deletions(-) delete mode 100644 DocSum/Dockerfile-vllm-rocm delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh delete mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm deleted file mode 100644 index f0e8a8743a..0000000000 --- a/DocSum/Dockerfile-vllm-rocm +++ /dev/null @@ -1,18 +0,0 @@ -FROM rocm/vllm-dev:main - -# Set the working directory -WORKDIR /workspace - -# Copy the api_server.py into the image -ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py - -# Expose the port used by the API server -EXPOSE 8011 - -# Set environment variables -ENV HUGGINGFACE_HUB_CACHE=/workspace -ENV WILM_USE_TRITON_FLASH_ATTENTION=0 -ENV PYTORCH_JIT=0 - -# Set the entrypoint to the api_server.py script -ENTRYPOINT ["python3", "/workspace/api_server.py"] diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md deleted file mode 100644 index 4d41a5cd31..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md +++ /dev/null @@ -1,175 +0,0 @@ -# Build and deploy DocSum Application on AMD GPU (ROCm) - -## Build images - -## 🚀 Build Docker Images - -First of all, you need to build Docker Images locally and install the python package of it. - -### 1. Build LLM Image - -```bash -git clone https://github.com/opea-project/GenAIComps.git -cd GenAIComps -docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile . -``` - -Then run the command `docker images`, you will have the following four Docker Images: - -### 2. Build MegaService Docker Image - -To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command: - -```bash -git clone https://github.com/opea-project/GenAIExamples -cd GenAIExamples/DocSum/ -docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . -``` - -### 3. Build UI Docker Image - -Build the frontend Docker image via below command: - -```bash -cd GenAIExamples/DocSum/ui -docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . -``` - -Then run the command `docker images`, you will have the following Docker Images: - -1. `opea/llm-docsum-tgi:latest` -2. `opea/docsum:latest` -3. `opea/docsum-ui:latest` - -### 4. Build React UI Docker Image - -Build the frontend Docker image via below command: - -```bash -cd GenAIExamples/DocSum/ui -export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum" -docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react . - -docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . -``` - -Then run the command `docker images`, you will have the following Docker Images: - -1. `opea/llm-docsum-tgi:latest` -2. `opea/docsum:latest` -3. `opea/docsum-ui:latest` -4. `opea/docsum-react-ui:latest` - -## 🚀 Start Microservices and MegaService - -### Required Models - -Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model. -For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable. - -### Setup Environment Variables - -Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below. - -```bash -export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm" -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export HOST_IP=${host_ip} -export DOCSUM_TGI_SERVICE_PORT="18882" -export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} -export DOCSUM_LLM_SERVER_PORT="8008" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export DocSum_COMPONENT_NAME="OpeaDocSumTgi" -``` - -Note: Please replace with `host_ip` with your external IP address, do not use localhost. - -Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered, where is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) - -Example for set isolation for 1 GPU - -``` - - /dev/dri/card0:/dev/dri/card0 - - /dev/dri/renderD128:/dev/dri/renderD128 -``` - -Example for set isolation for 2 GPUs - -``` - - /dev/dri/card0:/dev/dri/card0 - - /dev/dri/renderD128:/dev/dri/renderD128 - - /dev/dri/card1:/dev/dri/card1 - - /dev/dri/renderD129:/dev/dri/renderD129 -``` - -Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) - -### Start Microservice Docker Containers - -```bash -cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm -docker compose up -d -``` - -### Validate Microservices - -1. TGI Service - - ```bash - curl http://${host_ip}:8008/generate \ - -X POST \ - -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \ - -H 'Content-Type: application/json' - ``` - -2. LLM Microservice - - ```bash - curl http://${host_ip}:9000/v1/docsum \ - -X POST \ - -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \ - -H 'Content-Type: application/json' - ``` - -3. MegaService - - ```bash - curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{ - "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false - }' - ``` - -## 🚀 Launch the Svelte UI - -Open this URL `http://{host_ip}:5173` in your browser to access the frontend. - -![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b) - -Here is an example for summarizing a article. - -![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55) - -## 🚀 Launch the React UI (Optional) - -To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below: - -```yaml -docsum-rocm-react-ui-server: - image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest} - container_name: docsum-rocm-react-ui-server - depends_on: - - docsum-rocm-backend-server - ports: - - "5174:80" - environment: - - no_proxy=${no_proxy} - - https_proxy=${https_proxy} - - http_proxy=${http_proxy} - - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT} -``` - -Open this URL `http://{host_ip}:5175` in your browser to access the frontend. - -![project-screenshot](../../../../assets/img/docsum-ui-react.png) diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml deleted file mode 100644 index 037aa06395..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -services: - docsum-vllm-service: - image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} - container_name: docsum-vllm-service - ports: - - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011" - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_HUB_DISABLE_PROGRESS_BARS: 1 - HF_HUB_ENABLE_HF_TRANSFER: 0 - WILM_USE_TRITON_FLASH_ATTENTION: 0 - PYTORCH_JIT: 0 - volumes: - - "./data:/data" - shm_size: 20G - devices: - - /dev/kfd:/dev/kfd - - /dev/dri/:/dev/dri/ - cap_add: - - SYS_PTRACE - group_add: - - video - security_opt: - - seccomp:unconfined - - apparmor=unconfined - command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\"" - ipc: host - - docsum-llm-server: - image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} - container_name: docsum-llm-server - depends_on: - - docsum-vllm-service - ports: - - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000" - ipc: host - cap_add: - - SYS_PTRACE - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}" - HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID} - LOGFLAG: ${DOCSUM_LOGFLAG:-False} - MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS} - MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS} - restart: unless-stopped - - whisper-service: - image: ${REGISTRY:-opea}/whisper:${TAG:-latest} - container_name: whisper-service - ports: - - "${DOCSUM_WHISPER_PORT:-7066}:7066" - ipc: host - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - restart: unless-stopped - - docsum-backend-server: - image: ${REGISTRY:-opea}/docsum:${TAG:-latest} - container_name: docsum-backend-server - depends_on: - - docsum-tgi-service - - docsum-llm-server - ports: - - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888" - environment: - no_proxy: ${no_proxy} - https_proxy: ${https_proxy} - http_proxy: ${http_proxy} - MEGA_SERVICE_HOST_IP: ${HOST_IP} - LLM_SERVICE_HOST_IP: ${HOST_IP} - ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP} - ipc: host - restart: always - - docsum-gradio-ui: - image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest} - container_name: docsum-ui-server - depends_on: - - docsum-backend-server - ports: - - "${DOCSUM_FRONTEND_PORT:-5173}:5173" - environment: - no_proxy: ${no_proxy} - https_proxy: ${https_proxy} - http_proxy: ${http_proxy} - BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} - DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} - ipc: host - restart: always - -networks: - default: - driver: bridge diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh deleted file mode 100644 index 43e71e0fbf..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -export HOST_IP="" -export DOCSUM_MAX_INPUT_TOKENS=2048 -export DOCSUM_MAX_TOTAL_TOKENS=4096 -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export DOCSUM_VLLM_SERVICE_PORT="8008" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN="" -export DOCSUM_LLM_SERVER_PORT="9000" -export DOCSUM_WHISPER_PORT="7066" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml index dc0d546189..095fd28c93 100644 --- a/DocSum/docker_image_build/build.yaml +++ b/DocSum/docker_image_build/build.yaml @@ -47,12 +47,3 @@ services: dockerfile: comps/llms/src/doc-summarization/Dockerfile extends: docsum image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} - vllm_rocm: - build: - args: - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - no_proxy: ${no_proxy} - context: ../ - dockerfile: ./Dockerfile-vllm-rocm - image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh deleted file mode 100644 index d0919a019a..0000000000 --- a/DocSum/tests/test_compose_on_rocm_vllm.sh +++ /dev/null @@ -1,249 +0,0 @@ -#!/bin/bash -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -set -xe -IMAGE_REPO=${IMAGE_REPO:-"opea"} -IMAGE_TAG=${IMAGE_TAG:-"latest"} -echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" -echo "TAG=IMAGE_TAG=${IMAGE_TAG}" - -WORKPATH=$(dirname "$PWD") -LOG_PATH="$WORKPATH/tests" -ip_address=$(hostname -I | awk '{print $1}') -export MAX_INPUT_TOKENS=1024 -export MAX_TOTAL_TOKENS=2048 -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export HOST_IP=${ip_address} -export DOCSUM_VLLM_SERVICE_PORT="8008" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} -export DOCSUM_LLM_SERVER_PORT="9000" -export DOCSUM_WHISPER_PORT="7066" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export MEGA_SERVICE_HOST_IP=${HOST_IP} -export LLM_SERVICE_HOST_IP=${HOST_IP} -export ASR_SERVICE_HOST_IP=${HOST_IP} -export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" - -function build_docker_images() { - opea_branch=${opea_branch:-"main"} - # If the opea_branch isn't main, replace the git clone branch in Dockerfile. - if [[ "${opea_branch}" != "main" ]]; then - cd $WORKPATH - OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" - NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" - find . -type f -name "Dockerfile*" | while read -r file; do - echo "Processing file: $file" - sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" - done - fi - - cd $WORKPATH/docker_image_build - git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git - - echo "Build all the images with --no-cache, check docker_image_build.log for details..." - service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper" - docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log - - docker images && sleep 1s -} - -function start_services() { - cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm - sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env - # Start Docker Containers - docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log - sleep 1m -} - -function validate_services() { - local URL="$1" - local EXPECTED_RESULT="$2" - local SERVICE_NAME="$3" - local DOCKER_NAME="$4" - local INPUT_DATA="$5" - - local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") - - echo "===========================================" - - if [ "$HTTP_STATUS" -eq 200 ]; then - echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." - - local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) - - if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then - echo "[ $SERVICE_NAME ] Content is as expected." - else - echo "EXPECTED_RESULT==> $EXPECTED_RESULT" - echo "CONTENT==> $CONTENT" - echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - - fi - else - echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - sleep 1s -} - -get_base64_str() { - local file_name=$1 - base64 -w 0 "$file_name" -} - -# Function to generate input data for testing based on the document type -input_data_for_test() { - local document_type=$1 - case $document_type in - ("text") - echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are." - ;; - ("audio") - get_base64_str "$WORKPATH/tests/data/test.wav" - ;; - ("video") - get_base64_str "$WORKPATH/tests/data/test.mp4" - ;; - (*) - echo "Invalid document type" >&2 - exit 1 - ;; - esac -} - -function validate_microservices() { - # Check if the microservices are running correctly. - - # whisper microservice - ulimit -s 65536 - validate_services \ - "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \ - '{"asr_result":"well"}' \ - "whisper-service" \ - "whisper-service" \ - "{\"audio\": \"$(input_data_for_test "audio")\"}" - - # vLLM service - validate_services \ - "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \ - "generated_text" \ - "docsum-vllm-service" \ - "docsum-vllm-service" \ - '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}' - - # llm microservice - validate_services \ - "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \ - "text" \ - "docsum-llm-server" \ - "docsum-llm-server" \ - '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' - -} - -function validate_megaservice() { - local SERVICE_NAME="docsum-backend-server" - local DOCKER_NAME="docsum-backend-server" - local EXPECTED_RESULT="[DONE]" - local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." - local URL="${host_ip}:8888/v1/docsum" - local DATA_TYPE="type=text" - - local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL") - - if [ "$HTTP_STATUS" -eq 200 ]; then - echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." - - local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) - - if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then - echo "[ $SERVICE_NAME ] Content is as expected." - else - echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - else - echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - sleep 1s -} - -function validate_megaservice_json() { - # Curl the Mega Service - echo "" - echo ">>> Checking text data with Content-Type: application/json" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' - - echo ">>> Checking audio data" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - "{\"type\": \"audio\", \"messages\": \"$(input_data_for_test "audio")\"}" - - echo ">>> Checking video data" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - "{\"type\": \"video\", \"messages\": \"$(input_data_for_test "video")\"}" - -} - -function stop_docker() { - cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/ - docker compose stop && docker compose rm -f -} - -function main() { - echo "===========================================" - echo ">>>> Stopping any running Docker containers..." - stop_docker - - echo "===========================================" - if [[ "$IMAGE_REPO" == "opea" ]]; then - echo ">>>> Building Docker images..." - build_docker_images - fi - - echo "===========================================" - echo ">>>> Starting Docker services..." - start_services - - echo "===========================================" - echo ">>>> Validating microservices..." - validate_microservices - - echo "===========================================" - echo ">>>> Validating megaservice..." - validate_megaservice - echo ">>>> Validating validate_megaservice_json..." - validate_megaservice_json - - echo "===========================================" - echo ">>>> Stopping Docker containers..." - stop_docker - - echo "===========================================" - echo ">>>> Pruning Docker system..." - echo y | docker system prune - echo ">>>> Docker system pruned successfully." - echo "===========================================" -} - -main From bd2d47e7e53e1241c27aed0f823fa680d8ecf4e2 Mon Sep 17 00:00:00 2001 From: Chingis Yundunov Date: Thu, 13 Feb 2025 10:02:03 +0700 Subject: [PATCH 003/226] DocSum - add files for deploy app with ROCm vLLM Signed-off-by: Chingis Yundunov --- DocSum/Dockerfile-vllm-rocm | 18 ++ .../amd/gpu/rocm-vllm/README.md | 175 ++++++++++++ .../amd/gpu/rocm-vllm/compose.yaml | 107 ++++++++ .../amd/gpu/rocm-vllm/set_env.sh | 16 ++ DocSum/docker_image_build/build.yaml | 9 + DocSum/tests/test_compose_on_rocm_vllm.sh | 249 ++++++++++++++++++ 6 files changed, 574 insertions(+) create mode 100644 DocSum/Dockerfile-vllm-rocm create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh create mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm new file mode 100644 index 0000000000..f0e8a8743a --- /dev/null +++ b/DocSum/Dockerfile-vllm-rocm @@ -0,0 +1,18 @@ +FROM rocm/vllm-dev:main + +# Set the working directory +WORKDIR /workspace + +# Copy the api_server.py into the image +ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py + +# Expose the port used by the API server +EXPOSE 8011 + +# Set environment variables +ENV HUGGINGFACE_HUB_CACHE=/workspace +ENV WILM_USE_TRITON_FLASH_ATTENTION=0 +ENV PYTORCH_JIT=0 + +# Set the entrypoint to the api_server.py script +ENTRYPOINT ["python3", "/workspace/api_server.py"] diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md new file mode 100644 index 0000000000..4d41a5cd31 --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md @@ -0,0 +1,175 @@ +# Build and deploy DocSum Application on AMD GPU (ROCm) + +## Build images + +## 🚀 Build Docker Images + +First of all, you need to build Docker Images locally and install the python package of it. + +### 1. Build LLM Image + +```bash +git clone https://github.com/opea-project/GenAIComps.git +cd GenAIComps +docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile . +``` + +Then run the command `docker images`, you will have the following four Docker Images: + +### 2. Build MegaService Docker Image + +To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command: + +```bash +git clone https://github.com/opea-project/GenAIExamples +cd GenAIExamples/DocSum/ +docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . +``` + +### 3. Build UI Docker Image + +Build the frontend Docker image via below command: + +```bash +cd GenAIExamples/DocSum/ui +docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . +``` + +Then run the command `docker images`, you will have the following Docker Images: + +1. `opea/llm-docsum-tgi:latest` +2. `opea/docsum:latest` +3. `opea/docsum-ui:latest` + +### 4. Build React UI Docker Image + +Build the frontend Docker image via below command: + +```bash +cd GenAIExamples/DocSum/ui +export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum" +docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react . + +docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . +``` + +Then run the command `docker images`, you will have the following Docker Images: + +1. `opea/llm-docsum-tgi:latest` +2. `opea/docsum:latest` +3. `opea/docsum-ui:latest` +4. `opea/docsum-react-ui:latest` + +## 🚀 Start Microservices and MegaService + +### Required Models + +Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model. +For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable. + +### Setup Environment Variables + +Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below. + +```bash +export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm" +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export HOST_IP=${host_ip} +export DOCSUM_TGI_SERVICE_PORT="18882" +export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} +export DOCSUM_LLM_SERVER_PORT="8008" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export DocSum_COMPONENT_NAME="OpeaDocSumTgi" +``` + +Note: Please replace with `host_ip` with your external IP address, do not use localhost. + +Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered, where is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) + +Example for set isolation for 1 GPU + +``` + - /dev/dri/card0:/dev/dri/card0 + - /dev/dri/renderD128:/dev/dri/renderD128 +``` + +Example for set isolation for 2 GPUs + +``` + - /dev/dri/card0:/dev/dri/card0 + - /dev/dri/renderD128:/dev/dri/renderD128 + - /dev/dri/card1:/dev/dri/card1 + - /dev/dri/renderD129:/dev/dri/renderD129 +``` + +Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) + +### Start Microservice Docker Containers + +```bash +cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm +docker compose up -d +``` + +### Validate Microservices + +1. TGI Service + + ```bash + curl http://${host_ip}:8008/generate \ + -X POST \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \ + -H 'Content-Type: application/json' + ``` + +2. LLM Microservice + + ```bash + curl http://${host_ip}:9000/v1/docsum \ + -X POST \ + -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \ + -H 'Content-Type: application/json' + ``` + +3. MegaService + + ```bash + curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{ + "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false + }' + ``` + +## 🚀 Launch the Svelte UI + +Open this URL `http://{host_ip}:5173` in your browser to access the frontend. + +![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b) + +Here is an example for summarizing a article. + +![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55) + +## 🚀 Launch the React UI (Optional) + +To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below: + +```yaml +docsum-rocm-react-ui-server: + image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest} + container_name: docsum-rocm-react-ui-server + depends_on: + - docsum-rocm-backend-server + ports: + - "5174:80" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT} +``` + +Open this URL `http://{host_ip}:5175` in your browser to access the frontend. + +![project-screenshot](../../../../assets/img/docsum-ui-react.png) diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml new file mode 100644 index 0000000000..037aa06395 --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml @@ -0,0 +1,107 @@ +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +services: + docsum-vllm-service: + image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} + container_name: docsum-vllm-service + ports: + - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011" + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + WILM_USE_TRITON_FLASH_ATTENTION: 0 + PYTORCH_JIT: 0 + volumes: + - "./data:/data" + shm_size: 20G + devices: + - /dev/kfd:/dev/kfd + - /dev/dri/:/dev/dri/ + cap_add: + - SYS_PTRACE + group_add: + - video + security_opt: + - seccomp:unconfined + - apparmor=unconfined + command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\"" + ipc: host + + docsum-llm-server: + image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} + container_name: docsum-llm-server + depends_on: + - docsum-vllm-service + ports: + - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000" + ipc: host + cap_add: + - SYS_PTRACE + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}" + HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID} + LOGFLAG: ${DOCSUM_LOGFLAG:-False} + MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS} + MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS} + restart: unless-stopped + + whisper-service: + image: ${REGISTRY:-opea}/whisper:${TAG:-latest} + container_name: whisper-service + ports: + - "${DOCSUM_WHISPER_PORT:-7066}:7066" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + restart: unless-stopped + + docsum-backend-server: + image: ${REGISTRY:-opea}/docsum:${TAG:-latest} + container_name: docsum-backend-server + depends_on: + - docsum-tgi-service + - docsum-llm-server + ports: + - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888" + environment: + no_proxy: ${no_proxy} + https_proxy: ${https_proxy} + http_proxy: ${http_proxy} + MEGA_SERVICE_HOST_IP: ${HOST_IP} + LLM_SERVICE_HOST_IP: ${HOST_IP} + ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP} + ipc: host + restart: always + + docsum-gradio-ui: + image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest} + container_name: docsum-ui-server + depends_on: + - docsum-backend-server + ports: + - "${DOCSUM_FRONTEND_PORT:-5173}:5173" + environment: + no_proxy: ${no_proxy} + https_proxy: ${https_proxy} + http_proxy: ${http_proxy} + BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} + DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} + ipc: host + restart: always + +networks: + default: + driver: bridge diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh new file mode 100644 index 0000000000..43e71e0fbf --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +export HOST_IP="" +export DOCSUM_MAX_INPUT_TOKENS=2048 +export DOCSUM_MAX_TOTAL_TOKENS=4096 +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export DOCSUM_VLLM_SERVICE_PORT="8008" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN="" +export DOCSUM_LLM_SERVER_PORT="9000" +export DOCSUM_WHISPER_PORT="7066" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml index 095fd28c93..dc0d546189 100644 --- a/DocSum/docker_image_build/build.yaml +++ b/DocSum/docker_image_build/build.yaml @@ -47,3 +47,12 @@ services: dockerfile: comps/llms/src/doc-summarization/Dockerfile extends: docsum image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} + vllm_rocm: + build: + args: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + no_proxy: ${no_proxy} + context: ../ + dockerfile: ./Dockerfile-vllm-rocm + image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh new file mode 100644 index 0000000000..d0919a019a --- /dev/null +++ b/DocSum/tests/test_compose_on_rocm_vllm.sh @@ -0,0 +1,249 @@ +#!/bin/bash +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +set -xe +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') +export MAX_INPUT_TOKENS=1024 +export MAX_TOTAL_TOKENS=2048 +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export HOST_IP=${ip_address} +export DOCSUM_VLLM_SERVICE_PORT="8008" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} +export DOCSUM_LLM_SERVER_PORT="9000" +export DOCSUM_WHISPER_PORT="7066" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export MEGA_SERVICE_HOST_IP=${HOST_IP} +export LLM_SERVICE_HOST_IP=${HOST_IP} +export ASR_SERVICE_HOST_IP=${HOST_IP} +export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" + +function build_docker_images() { + opea_branch=${opea_branch:-"main"} + # If the opea_branch isn't main, replace the git clone branch in Dockerfile. + if [[ "${opea_branch}" != "main" ]]; then + cd $WORKPATH + OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" + NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" + find . -type f -name "Dockerfile*" | while read -r file; do + echo "Processing file: $file" + sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" + done + fi + + cd $WORKPATH/docker_image_build + git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + + echo "Build all the images with --no-cache, check docker_image_build.log for details..." + service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper" + docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log + + docker images && sleep 1s +} + +function start_services() { + cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm + sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env + # Start Docker Containers + docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log + sleep 1m +} + +function validate_services() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + + echo "===========================================" + + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "EXPECTED_RESULT==> $EXPECTED_RESULT" + echo "CONTENT==> $CONTENT" + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +get_base64_str() { + local file_name=$1 + base64 -w 0 "$file_name" +} + +# Function to generate input data for testing based on the document type +input_data_for_test() { + local document_type=$1 + case $document_type in + ("text") + echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are." + ;; + ("audio") + get_base64_str "$WORKPATH/tests/data/test.wav" + ;; + ("video") + get_base64_str "$WORKPATH/tests/data/test.mp4" + ;; + (*) + echo "Invalid document type" >&2 + exit 1 + ;; + esac +} + +function validate_microservices() { + # Check if the microservices are running correctly. + + # whisper microservice + ulimit -s 65536 + validate_services \ + "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \ + '{"asr_result":"well"}' \ + "whisper-service" \ + "whisper-service" \ + "{\"audio\": \"$(input_data_for_test "audio")\"}" + + # vLLM service + validate_services \ + "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \ + "generated_text" \ + "docsum-vllm-service" \ + "docsum-vllm-service" \ + '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}' + + # llm microservice + validate_services \ + "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \ + "text" \ + "docsum-llm-server" \ + "docsum-llm-server" \ + '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' + +} + +function validate_megaservice() { + local SERVICE_NAME="docsum-backend-server" + local DOCKER_NAME="docsum-backend-server" + local EXPECTED_RESULT="[DONE]" + local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." + local URL="${host_ip}:8888/v1/docsum" + local DATA_TYPE="type=text" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL") + + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +function validate_megaservice_json() { + # Curl the Mega Service + echo "" + echo ">>> Checking text data with Content-Type: application/json" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' + + echo ">>> Checking audio data" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + "{\"type\": \"audio\", \"messages\": \"$(input_data_for_test "audio")\"}" + + echo ">>> Checking video data" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + "{\"type\": \"video\", \"messages\": \"$(input_data_for_test "video")\"}" + +} + +function stop_docker() { + cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/ + docker compose stop && docker compose rm -f +} + +function main() { + echo "===========================================" + echo ">>>> Stopping any running Docker containers..." + stop_docker + + echo "===========================================" + if [[ "$IMAGE_REPO" == "opea" ]]; then + echo ">>>> Building Docker images..." + build_docker_images + fi + + echo "===========================================" + echo ">>>> Starting Docker services..." + start_services + + echo "===========================================" + echo ">>>> Validating microservices..." + validate_microservices + + echo "===========================================" + echo ">>>> Validating megaservice..." + validate_megaservice + echo ">>>> Validating validate_megaservice_json..." + validate_megaservice_json + + echo "===========================================" + echo ">>>> Stopping Docker containers..." + stop_docker + + echo "===========================================" + echo ">>>> Pruning Docker system..." + echo y | docker system prune + echo ">>>> Docker system pruned successfully." + echo "===========================================" +} + +main From 2459ecbc53fdb7c9c449930700cff290de15c152 Mon Sep 17 00:00:00 2001 From: Chingis Yundunov Date: Thu, 13 Feb 2025 10:07:05 +0700 Subject: [PATCH 004/226] DocSum - fix main Signed-off-by: Chingis Yundunov --- DocSum/Dockerfile-vllm-rocm | 18 -- .../amd/gpu/rocm-vllm/README.md | 175 ------------ .../amd/gpu/rocm-vllm/compose.yaml | 107 -------- .../amd/gpu/rocm-vllm/set_env.sh | 16 -- DocSum/docker_image_build/build.yaml | 9 - DocSum/tests/test_compose_on_rocm_vllm.sh | 249 ------------------ 6 files changed, 574 deletions(-) delete mode 100644 DocSum/Dockerfile-vllm-rocm delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh delete mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm deleted file mode 100644 index f0e8a8743a..0000000000 --- a/DocSum/Dockerfile-vllm-rocm +++ /dev/null @@ -1,18 +0,0 @@ -FROM rocm/vllm-dev:main - -# Set the working directory -WORKDIR /workspace - -# Copy the api_server.py into the image -ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py - -# Expose the port used by the API server -EXPOSE 8011 - -# Set environment variables -ENV HUGGINGFACE_HUB_CACHE=/workspace -ENV WILM_USE_TRITON_FLASH_ATTENTION=0 -ENV PYTORCH_JIT=0 - -# Set the entrypoint to the api_server.py script -ENTRYPOINT ["python3", "/workspace/api_server.py"] diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md deleted file mode 100644 index 4d41a5cd31..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md +++ /dev/null @@ -1,175 +0,0 @@ -# Build and deploy DocSum Application on AMD GPU (ROCm) - -## Build images - -## 🚀 Build Docker Images - -First of all, you need to build Docker Images locally and install the python package of it. - -### 1. Build LLM Image - -```bash -git clone https://github.com/opea-project/GenAIComps.git -cd GenAIComps -docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile . -``` - -Then run the command `docker images`, you will have the following four Docker Images: - -### 2. Build MegaService Docker Image - -To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command: - -```bash -git clone https://github.com/opea-project/GenAIExamples -cd GenAIExamples/DocSum/ -docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . -``` - -### 3. Build UI Docker Image - -Build the frontend Docker image via below command: - -```bash -cd GenAIExamples/DocSum/ui -docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . -``` - -Then run the command `docker images`, you will have the following Docker Images: - -1. `opea/llm-docsum-tgi:latest` -2. `opea/docsum:latest` -3. `opea/docsum-ui:latest` - -### 4. Build React UI Docker Image - -Build the frontend Docker image via below command: - -```bash -cd GenAIExamples/DocSum/ui -export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum" -docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react . - -docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . -``` - -Then run the command `docker images`, you will have the following Docker Images: - -1. `opea/llm-docsum-tgi:latest` -2. `opea/docsum:latest` -3. `opea/docsum-ui:latest` -4. `opea/docsum-react-ui:latest` - -## 🚀 Start Microservices and MegaService - -### Required Models - -Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model. -For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable. - -### Setup Environment Variables - -Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below. - -```bash -export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm" -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export HOST_IP=${host_ip} -export DOCSUM_TGI_SERVICE_PORT="18882" -export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} -export DOCSUM_LLM_SERVER_PORT="8008" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export DocSum_COMPONENT_NAME="OpeaDocSumTgi" -``` - -Note: Please replace with `host_ip` with your external IP address, do not use localhost. - -Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered, where is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) - -Example for set isolation for 1 GPU - -``` - - /dev/dri/card0:/dev/dri/card0 - - /dev/dri/renderD128:/dev/dri/renderD128 -``` - -Example for set isolation for 2 GPUs - -``` - - /dev/dri/card0:/dev/dri/card0 - - /dev/dri/renderD128:/dev/dri/renderD128 - - /dev/dri/card1:/dev/dri/card1 - - /dev/dri/renderD129:/dev/dri/renderD129 -``` - -Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) - -### Start Microservice Docker Containers - -```bash -cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm -docker compose up -d -``` - -### Validate Microservices - -1. TGI Service - - ```bash - curl http://${host_ip}:8008/generate \ - -X POST \ - -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \ - -H 'Content-Type: application/json' - ``` - -2. LLM Microservice - - ```bash - curl http://${host_ip}:9000/v1/docsum \ - -X POST \ - -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \ - -H 'Content-Type: application/json' - ``` - -3. MegaService - - ```bash - curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{ - "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false - }' - ``` - -## 🚀 Launch the Svelte UI - -Open this URL `http://{host_ip}:5173` in your browser to access the frontend. - -![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b) - -Here is an example for summarizing a article. - -![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55) - -## 🚀 Launch the React UI (Optional) - -To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below: - -```yaml -docsum-rocm-react-ui-server: - image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest} - container_name: docsum-rocm-react-ui-server - depends_on: - - docsum-rocm-backend-server - ports: - - "5174:80" - environment: - - no_proxy=${no_proxy} - - https_proxy=${https_proxy} - - http_proxy=${http_proxy} - - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT} -``` - -Open this URL `http://{host_ip}:5175` in your browser to access the frontend. - -![project-screenshot](../../../../assets/img/docsum-ui-react.png) diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml deleted file mode 100644 index 037aa06395..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -services: - docsum-vllm-service: - image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} - container_name: docsum-vllm-service - ports: - - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011" - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_HUB_DISABLE_PROGRESS_BARS: 1 - HF_HUB_ENABLE_HF_TRANSFER: 0 - WILM_USE_TRITON_FLASH_ATTENTION: 0 - PYTORCH_JIT: 0 - volumes: - - "./data:/data" - shm_size: 20G - devices: - - /dev/kfd:/dev/kfd - - /dev/dri/:/dev/dri/ - cap_add: - - SYS_PTRACE - group_add: - - video - security_opt: - - seccomp:unconfined - - apparmor=unconfined - command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\"" - ipc: host - - docsum-llm-server: - image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} - container_name: docsum-llm-server - depends_on: - - docsum-vllm-service - ports: - - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000" - ipc: host - cap_add: - - SYS_PTRACE - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}" - HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID} - LOGFLAG: ${DOCSUM_LOGFLAG:-False} - MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS} - MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS} - restart: unless-stopped - - whisper-service: - image: ${REGISTRY:-opea}/whisper:${TAG:-latest} - container_name: whisper-service - ports: - - "${DOCSUM_WHISPER_PORT:-7066}:7066" - ipc: host - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - restart: unless-stopped - - docsum-backend-server: - image: ${REGISTRY:-opea}/docsum:${TAG:-latest} - container_name: docsum-backend-server - depends_on: - - docsum-tgi-service - - docsum-llm-server - ports: - - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888" - environment: - no_proxy: ${no_proxy} - https_proxy: ${https_proxy} - http_proxy: ${http_proxy} - MEGA_SERVICE_HOST_IP: ${HOST_IP} - LLM_SERVICE_HOST_IP: ${HOST_IP} - ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP} - ipc: host - restart: always - - docsum-gradio-ui: - image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest} - container_name: docsum-ui-server - depends_on: - - docsum-backend-server - ports: - - "${DOCSUM_FRONTEND_PORT:-5173}:5173" - environment: - no_proxy: ${no_proxy} - https_proxy: ${https_proxy} - http_proxy: ${http_proxy} - BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} - DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} - ipc: host - restart: always - -networks: - default: - driver: bridge diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh deleted file mode 100644 index 43e71e0fbf..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -export HOST_IP="" -export DOCSUM_MAX_INPUT_TOKENS=2048 -export DOCSUM_MAX_TOTAL_TOKENS=4096 -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export DOCSUM_VLLM_SERVICE_PORT="8008" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN="" -export DOCSUM_LLM_SERVER_PORT="9000" -export DOCSUM_WHISPER_PORT="7066" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml index dc0d546189..095fd28c93 100644 --- a/DocSum/docker_image_build/build.yaml +++ b/DocSum/docker_image_build/build.yaml @@ -47,12 +47,3 @@ services: dockerfile: comps/llms/src/doc-summarization/Dockerfile extends: docsum image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} - vllm_rocm: - build: - args: - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - no_proxy: ${no_proxy} - context: ../ - dockerfile: ./Dockerfile-vllm-rocm - image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh deleted file mode 100644 index d0919a019a..0000000000 --- a/DocSum/tests/test_compose_on_rocm_vllm.sh +++ /dev/null @@ -1,249 +0,0 @@ -#!/bin/bash -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -set -xe -IMAGE_REPO=${IMAGE_REPO:-"opea"} -IMAGE_TAG=${IMAGE_TAG:-"latest"} -echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" -echo "TAG=IMAGE_TAG=${IMAGE_TAG}" - -WORKPATH=$(dirname "$PWD") -LOG_PATH="$WORKPATH/tests" -ip_address=$(hostname -I | awk '{print $1}') -export MAX_INPUT_TOKENS=1024 -export MAX_TOTAL_TOKENS=2048 -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export HOST_IP=${ip_address} -export DOCSUM_VLLM_SERVICE_PORT="8008" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} -export DOCSUM_LLM_SERVER_PORT="9000" -export DOCSUM_WHISPER_PORT="7066" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export MEGA_SERVICE_HOST_IP=${HOST_IP} -export LLM_SERVICE_HOST_IP=${HOST_IP} -export ASR_SERVICE_HOST_IP=${HOST_IP} -export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" - -function build_docker_images() { - opea_branch=${opea_branch:-"main"} - # If the opea_branch isn't main, replace the git clone branch in Dockerfile. - if [[ "${opea_branch}" != "main" ]]; then - cd $WORKPATH - OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" - NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" - find . -type f -name "Dockerfile*" | while read -r file; do - echo "Processing file: $file" - sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" - done - fi - - cd $WORKPATH/docker_image_build - git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git - - echo "Build all the images with --no-cache, check docker_image_build.log for details..." - service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper" - docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log - - docker images && sleep 1s -} - -function start_services() { - cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm - sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env - # Start Docker Containers - docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log - sleep 1m -} - -function validate_services() { - local URL="$1" - local EXPECTED_RESULT="$2" - local SERVICE_NAME="$3" - local DOCKER_NAME="$4" - local INPUT_DATA="$5" - - local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") - - echo "===========================================" - - if [ "$HTTP_STATUS" -eq 200 ]; then - echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." - - local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) - - if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then - echo "[ $SERVICE_NAME ] Content is as expected." - else - echo "EXPECTED_RESULT==> $EXPECTED_RESULT" - echo "CONTENT==> $CONTENT" - echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - - fi - else - echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - sleep 1s -} - -get_base64_str() { - local file_name=$1 - base64 -w 0 "$file_name" -} - -# Function to generate input data for testing based on the document type -input_data_for_test() { - local document_type=$1 - case $document_type in - ("text") - echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are." - ;; - ("audio") - get_base64_str "$WORKPATH/tests/data/test.wav" - ;; - ("video") - get_base64_str "$WORKPATH/tests/data/test.mp4" - ;; - (*) - echo "Invalid document type" >&2 - exit 1 - ;; - esac -} - -function validate_microservices() { - # Check if the microservices are running correctly. - - # whisper microservice - ulimit -s 65536 - validate_services \ - "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \ - '{"asr_result":"well"}' \ - "whisper-service" \ - "whisper-service" \ - "{\"audio\": \"$(input_data_for_test "audio")\"}" - - # vLLM service - validate_services \ - "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \ - "generated_text" \ - "docsum-vllm-service" \ - "docsum-vllm-service" \ - '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}' - - # llm microservice - validate_services \ - "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \ - "text" \ - "docsum-llm-server" \ - "docsum-llm-server" \ - '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' - -} - -function validate_megaservice() { - local SERVICE_NAME="docsum-backend-server" - local DOCKER_NAME="docsum-backend-server" - local EXPECTED_RESULT="[DONE]" - local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." - local URL="${host_ip}:8888/v1/docsum" - local DATA_TYPE="type=text" - - local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL") - - if [ "$HTTP_STATUS" -eq 200 ]; then - echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." - - local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) - - if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then - echo "[ $SERVICE_NAME ] Content is as expected." - else - echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - else - echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - sleep 1s -} - -function validate_megaservice_json() { - # Curl the Mega Service - echo "" - echo ">>> Checking text data with Content-Type: application/json" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' - - echo ">>> Checking audio data" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - "{\"type\": \"audio\", \"messages\": \"$(input_data_for_test "audio")\"}" - - echo ">>> Checking video data" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - "{\"type\": \"video\", \"messages\": \"$(input_data_for_test "video")\"}" - -} - -function stop_docker() { - cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/ - docker compose stop && docker compose rm -f -} - -function main() { - echo "===========================================" - echo ">>>> Stopping any running Docker containers..." - stop_docker - - echo "===========================================" - if [[ "$IMAGE_REPO" == "opea" ]]; then - echo ">>>> Building Docker images..." - build_docker_images - fi - - echo "===========================================" - echo ">>>> Starting Docker services..." - start_services - - echo "===========================================" - echo ">>>> Validating microservices..." - validate_microservices - - echo "===========================================" - echo ">>>> Validating megaservice..." - validate_megaservice - echo ">>>> Validating validate_megaservice_json..." - validate_megaservice_json - - echo "===========================================" - echo ">>>> Stopping Docker containers..." - stop_docker - - echo "===========================================" - echo ">>>> Pruning Docker system..." - echo y | docker system prune - echo ">>>> Docker system pruned successfully." - echo "===========================================" -} - -main From 6d5049dd1c6bb3e201c4ca807da6950e0ab4b9d2 Mon Sep 17 00:00:00 2001 From: Chingis Yundunov Date: Thu, 13 Feb 2025 10:02:03 +0700 Subject: [PATCH 005/226] DocSum - add files for deploy app with ROCm vLLM Signed-off-by: Chingis Yundunov --- DocSum/Dockerfile-vllm-rocm | 18 ++ .../amd/gpu/rocm-vllm/README.md | 175 ++++++++++++ .../amd/gpu/rocm-vllm/compose.yaml | 107 ++++++++ .../amd/gpu/rocm-vllm/set_env.sh | 16 ++ DocSum/docker_image_build/build.yaml | 9 + DocSum/tests/test_compose_on_rocm_vllm.sh | 249 ++++++++++++++++++ 6 files changed, 574 insertions(+) create mode 100644 DocSum/Dockerfile-vllm-rocm create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh create mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm new file mode 100644 index 0000000000..f0e8a8743a --- /dev/null +++ b/DocSum/Dockerfile-vllm-rocm @@ -0,0 +1,18 @@ +FROM rocm/vllm-dev:main + +# Set the working directory +WORKDIR /workspace + +# Copy the api_server.py into the image +ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py + +# Expose the port used by the API server +EXPOSE 8011 + +# Set environment variables +ENV HUGGINGFACE_HUB_CACHE=/workspace +ENV WILM_USE_TRITON_FLASH_ATTENTION=0 +ENV PYTORCH_JIT=0 + +# Set the entrypoint to the api_server.py script +ENTRYPOINT ["python3", "/workspace/api_server.py"] diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md new file mode 100644 index 0000000000..4d41a5cd31 --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md @@ -0,0 +1,175 @@ +# Build and deploy DocSum Application on AMD GPU (ROCm) + +## Build images + +## 🚀 Build Docker Images + +First of all, you need to build Docker Images locally and install the python package of it. + +### 1. Build LLM Image + +```bash +git clone https://github.com/opea-project/GenAIComps.git +cd GenAIComps +docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile . +``` + +Then run the command `docker images`, you will have the following four Docker Images: + +### 2. Build MegaService Docker Image + +To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command: + +```bash +git clone https://github.com/opea-project/GenAIExamples +cd GenAIExamples/DocSum/ +docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . +``` + +### 3. Build UI Docker Image + +Build the frontend Docker image via below command: + +```bash +cd GenAIExamples/DocSum/ui +docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . +``` + +Then run the command `docker images`, you will have the following Docker Images: + +1. `opea/llm-docsum-tgi:latest` +2. `opea/docsum:latest` +3. `opea/docsum-ui:latest` + +### 4. Build React UI Docker Image + +Build the frontend Docker image via below command: + +```bash +cd GenAIExamples/DocSum/ui +export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum" +docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react . + +docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . +``` + +Then run the command `docker images`, you will have the following Docker Images: + +1. `opea/llm-docsum-tgi:latest` +2. `opea/docsum:latest` +3. `opea/docsum-ui:latest` +4. `opea/docsum-react-ui:latest` + +## 🚀 Start Microservices and MegaService + +### Required Models + +Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model. +For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable. + +### Setup Environment Variables + +Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below. + +```bash +export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm" +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export HOST_IP=${host_ip} +export DOCSUM_TGI_SERVICE_PORT="18882" +export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} +export DOCSUM_LLM_SERVER_PORT="8008" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export DocSum_COMPONENT_NAME="OpeaDocSumTgi" +``` + +Note: Please replace with `host_ip` with your external IP address, do not use localhost. + +Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered, where is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) + +Example for set isolation for 1 GPU + +``` + - /dev/dri/card0:/dev/dri/card0 + - /dev/dri/renderD128:/dev/dri/renderD128 +``` + +Example for set isolation for 2 GPUs + +``` + - /dev/dri/card0:/dev/dri/card0 + - /dev/dri/renderD128:/dev/dri/renderD128 + - /dev/dri/card1:/dev/dri/card1 + - /dev/dri/renderD129:/dev/dri/renderD129 +``` + +Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) + +### Start Microservice Docker Containers + +```bash +cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm +docker compose up -d +``` + +### Validate Microservices + +1. TGI Service + + ```bash + curl http://${host_ip}:8008/generate \ + -X POST \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \ + -H 'Content-Type: application/json' + ``` + +2. LLM Microservice + + ```bash + curl http://${host_ip}:9000/v1/docsum \ + -X POST \ + -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \ + -H 'Content-Type: application/json' + ``` + +3. MegaService + + ```bash + curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{ + "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false + }' + ``` + +## 🚀 Launch the Svelte UI + +Open this URL `http://{host_ip}:5173` in your browser to access the frontend. + +![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b) + +Here is an example for summarizing a article. + +![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55) + +## 🚀 Launch the React UI (Optional) + +To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below: + +```yaml +docsum-rocm-react-ui-server: + image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest} + container_name: docsum-rocm-react-ui-server + depends_on: + - docsum-rocm-backend-server + ports: + - "5174:80" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT} +``` + +Open this URL `http://{host_ip}:5175` in your browser to access the frontend. + +![project-screenshot](../../../../assets/img/docsum-ui-react.png) diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml new file mode 100644 index 0000000000..037aa06395 --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml @@ -0,0 +1,107 @@ +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +services: + docsum-vllm-service: + image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} + container_name: docsum-vllm-service + ports: + - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011" + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + WILM_USE_TRITON_FLASH_ATTENTION: 0 + PYTORCH_JIT: 0 + volumes: + - "./data:/data" + shm_size: 20G + devices: + - /dev/kfd:/dev/kfd + - /dev/dri/:/dev/dri/ + cap_add: + - SYS_PTRACE + group_add: + - video + security_opt: + - seccomp:unconfined + - apparmor=unconfined + command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\"" + ipc: host + + docsum-llm-server: + image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} + container_name: docsum-llm-server + depends_on: + - docsum-vllm-service + ports: + - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000" + ipc: host + cap_add: + - SYS_PTRACE + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}" + HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID} + LOGFLAG: ${DOCSUM_LOGFLAG:-False} + MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS} + MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS} + restart: unless-stopped + + whisper-service: + image: ${REGISTRY:-opea}/whisper:${TAG:-latest} + container_name: whisper-service + ports: + - "${DOCSUM_WHISPER_PORT:-7066}:7066" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + restart: unless-stopped + + docsum-backend-server: + image: ${REGISTRY:-opea}/docsum:${TAG:-latest} + container_name: docsum-backend-server + depends_on: + - docsum-tgi-service + - docsum-llm-server + ports: + - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888" + environment: + no_proxy: ${no_proxy} + https_proxy: ${https_proxy} + http_proxy: ${http_proxy} + MEGA_SERVICE_HOST_IP: ${HOST_IP} + LLM_SERVICE_HOST_IP: ${HOST_IP} + ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP} + ipc: host + restart: always + + docsum-gradio-ui: + image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest} + container_name: docsum-ui-server + depends_on: + - docsum-backend-server + ports: + - "${DOCSUM_FRONTEND_PORT:-5173}:5173" + environment: + no_proxy: ${no_proxy} + https_proxy: ${https_proxy} + http_proxy: ${http_proxy} + BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} + DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} + ipc: host + restart: always + +networks: + default: + driver: bridge diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh new file mode 100644 index 0000000000..43e71e0fbf --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +export HOST_IP="" +export DOCSUM_MAX_INPUT_TOKENS=2048 +export DOCSUM_MAX_TOTAL_TOKENS=4096 +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export DOCSUM_VLLM_SERVICE_PORT="8008" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN="" +export DOCSUM_LLM_SERVER_PORT="9000" +export DOCSUM_WHISPER_PORT="7066" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml index 095fd28c93..dc0d546189 100644 --- a/DocSum/docker_image_build/build.yaml +++ b/DocSum/docker_image_build/build.yaml @@ -47,3 +47,12 @@ services: dockerfile: comps/llms/src/doc-summarization/Dockerfile extends: docsum image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} + vllm_rocm: + build: + args: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + no_proxy: ${no_proxy} + context: ../ + dockerfile: ./Dockerfile-vllm-rocm + image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh new file mode 100644 index 0000000000..d0919a019a --- /dev/null +++ b/DocSum/tests/test_compose_on_rocm_vllm.sh @@ -0,0 +1,249 @@ +#!/bin/bash +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +set -xe +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') +export MAX_INPUT_TOKENS=1024 +export MAX_TOTAL_TOKENS=2048 +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export HOST_IP=${ip_address} +export DOCSUM_VLLM_SERVICE_PORT="8008" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} +export DOCSUM_LLM_SERVER_PORT="9000" +export DOCSUM_WHISPER_PORT="7066" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export MEGA_SERVICE_HOST_IP=${HOST_IP} +export LLM_SERVICE_HOST_IP=${HOST_IP} +export ASR_SERVICE_HOST_IP=${HOST_IP} +export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" + +function build_docker_images() { + opea_branch=${opea_branch:-"main"} + # If the opea_branch isn't main, replace the git clone branch in Dockerfile. + if [[ "${opea_branch}" != "main" ]]; then + cd $WORKPATH + OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" + NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" + find . -type f -name "Dockerfile*" | while read -r file; do + echo "Processing file: $file" + sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" + done + fi + + cd $WORKPATH/docker_image_build + git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + + echo "Build all the images with --no-cache, check docker_image_build.log for details..." + service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper" + docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log + + docker images && sleep 1s +} + +function start_services() { + cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm + sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env + # Start Docker Containers + docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log + sleep 1m +} + +function validate_services() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + + echo "===========================================" + + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "EXPECTED_RESULT==> $EXPECTED_RESULT" + echo "CONTENT==> $CONTENT" + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +get_base64_str() { + local file_name=$1 + base64 -w 0 "$file_name" +} + +# Function to generate input data for testing based on the document type +input_data_for_test() { + local document_type=$1 + case $document_type in + ("text") + echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are." + ;; + ("audio") + get_base64_str "$WORKPATH/tests/data/test.wav" + ;; + ("video") + get_base64_str "$WORKPATH/tests/data/test.mp4" + ;; + (*) + echo "Invalid document type" >&2 + exit 1 + ;; + esac +} + +function validate_microservices() { + # Check if the microservices are running correctly. + + # whisper microservice + ulimit -s 65536 + validate_services \ + "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \ + '{"asr_result":"well"}' \ + "whisper-service" \ + "whisper-service" \ + "{\"audio\": \"$(input_data_for_test "audio")\"}" + + # vLLM service + validate_services \ + "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \ + "generated_text" \ + "docsum-vllm-service" \ + "docsum-vllm-service" \ + '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}' + + # llm microservice + validate_services \ + "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \ + "text" \ + "docsum-llm-server" \ + "docsum-llm-server" \ + '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' + +} + +function validate_megaservice() { + local SERVICE_NAME="docsum-backend-server" + local DOCKER_NAME="docsum-backend-server" + local EXPECTED_RESULT="[DONE]" + local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." + local URL="${host_ip}:8888/v1/docsum" + local DATA_TYPE="type=text" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL") + + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +function validate_megaservice_json() { + # Curl the Mega Service + echo "" + echo ">>> Checking text data with Content-Type: application/json" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' + + echo ">>> Checking audio data" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + "{\"type\": \"audio\", \"messages\": \"$(input_data_for_test "audio")\"}" + + echo ">>> Checking video data" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + "{\"type\": \"video\", \"messages\": \"$(input_data_for_test "video")\"}" + +} + +function stop_docker() { + cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/ + docker compose stop && docker compose rm -f +} + +function main() { + echo "===========================================" + echo ">>>> Stopping any running Docker containers..." + stop_docker + + echo "===========================================" + if [[ "$IMAGE_REPO" == "opea" ]]; then + echo ">>>> Building Docker images..." + build_docker_images + fi + + echo "===========================================" + echo ">>>> Starting Docker services..." + start_services + + echo "===========================================" + echo ">>>> Validating microservices..." + validate_microservices + + echo "===========================================" + echo ">>>> Validating megaservice..." + validate_megaservice + echo ">>>> Validating validate_megaservice_json..." + validate_megaservice_json + + echo "===========================================" + echo ">>>> Stopping Docker containers..." + stop_docker + + echo "===========================================" + echo ">>>> Pruning Docker system..." + echo y | docker system prune + echo ">>>> Docker system pruned successfully." + echo "===========================================" +} + +main From 9dfbdc5cffe708b084e7367d6df2910908f5e76a Mon Sep 17 00:00:00 2001 From: Chingis Yundunov Date: Thu, 13 Feb 2025 10:07:05 +0700 Subject: [PATCH 006/226] DocSum - fix main Signed-off-by: Chingis Yundunov --- DocSum/Dockerfile-vllm-rocm | 18 -- .../amd/gpu/rocm-vllm/README.md | 175 ------------ .../amd/gpu/rocm-vllm/compose.yaml | 107 -------- .../amd/gpu/rocm-vllm/set_env.sh | 16 -- DocSum/docker_image_build/build.yaml | 9 - DocSum/tests/test_compose_on_rocm_vllm.sh | 249 ------------------ 6 files changed, 574 deletions(-) delete mode 100644 DocSum/Dockerfile-vllm-rocm delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh delete mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm deleted file mode 100644 index f0e8a8743a..0000000000 --- a/DocSum/Dockerfile-vllm-rocm +++ /dev/null @@ -1,18 +0,0 @@ -FROM rocm/vllm-dev:main - -# Set the working directory -WORKDIR /workspace - -# Copy the api_server.py into the image -ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py - -# Expose the port used by the API server -EXPOSE 8011 - -# Set environment variables -ENV HUGGINGFACE_HUB_CACHE=/workspace -ENV WILM_USE_TRITON_FLASH_ATTENTION=0 -ENV PYTORCH_JIT=0 - -# Set the entrypoint to the api_server.py script -ENTRYPOINT ["python3", "/workspace/api_server.py"] diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md deleted file mode 100644 index 4d41a5cd31..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md +++ /dev/null @@ -1,175 +0,0 @@ -# Build and deploy DocSum Application on AMD GPU (ROCm) - -## Build images - -## 🚀 Build Docker Images - -First of all, you need to build Docker Images locally and install the python package of it. - -### 1. Build LLM Image - -```bash -git clone https://github.com/opea-project/GenAIComps.git -cd GenAIComps -docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile . -``` - -Then run the command `docker images`, you will have the following four Docker Images: - -### 2. Build MegaService Docker Image - -To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command: - -```bash -git clone https://github.com/opea-project/GenAIExamples -cd GenAIExamples/DocSum/ -docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . -``` - -### 3. Build UI Docker Image - -Build the frontend Docker image via below command: - -```bash -cd GenAIExamples/DocSum/ui -docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . -``` - -Then run the command `docker images`, you will have the following Docker Images: - -1. `opea/llm-docsum-tgi:latest` -2. `opea/docsum:latest` -3. `opea/docsum-ui:latest` - -### 4. Build React UI Docker Image - -Build the frontend Docker image via below command: - -```bash -cd GenAIExamples/DocSum/ui -export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum" -docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react . - -docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . -``` - -Then run the command `docker images`, you will have the following Docker Images: - -1. `opea/llm-docsum-tgi:latest` -2. `opea/docsum:latest` -3. `opea/docsum-ui:latest` -4. `opea/docsum-react-ui:latest` - -## 🚀 Start Microservices and MegaService - -### Required Models - -Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model. -For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable. - -### Setup Environment Variables - -Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below. - -```bash -export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm" -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export HOST_IP=${host_ip} -export DOCSUM_TGI_SERVICE_PORT="18882" -export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} -export DOCSUM_LLM_SERVER_PORT="8008" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export DocSum_COMPONENT_NAME="OpeaDocSumTgi" -``` - -Note: Please replace with `host_ip` with your external IP address, do not use localhost. - -Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered, where is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) - -Example for set isolation for 1 GPU - -``` - - /dev/dri/card0:/dev/dri/card0 - - /dev/dri/renderD128:/dev/dri/renderD128 -``` - -Example for set isolation for 2 GPUs - -``` - - /dev/dri/card0:/dev/dri/card0 - - /dev/dri/renderD128:/dev/dri/renderD128 - - /dev/dri/card1:/dev/dri/card1 - - /dev/dri/renderD129:/dev/dri/renderD129 -``` - -Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) - -### Start Microservice Docker Containers - -```bash -cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm -docker compose up -d -``` - -### Validate Microservices - -1. TGI Service - - ```bash - curl http://${host_ip}:8008/generate \ - -X POST \ - -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \ - -H 'Content-Type: application/json' - ``` - -2. LLM Microservice - - ```bash - curl http://${host_ip}:9000/v1/docsum \ - -X POST \ - -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \ - -H 'Content-Type: application/json' - ``` - -3. MegaService - - ```bash - curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{ - "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false - }' - ``` - -## 🚀 Launch the Svelte UI - -Open this URL `http://{host_ip}:5173` in your browser to access the frontend. - -![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b) - -Here is an example for summarizing a article. - -![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55) - -## 🚀 Launch the React UI (Optional) - -To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below: - -```yaml -docsum-rocm-react-ui-server: - image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest} - container_name: docsum-rocm-react-ui-server - depends_on: - - docsum-rocm-backend-server - ports: - - "5174:80" - environment: - - no_proxy=${no_proxy} - - https_proxy=${https_proxy} - - http_proxy=${http_proxy} - - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT} -``` - -Open this URL `http://{host_ip}:5175` in your browser to access the frontend. - -![project-screenshot](../../../../assets/img/docsum-ui-react.png) diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml deleted file mode 100644 index 037aa06395..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -services: - docsum-vllm-service: - image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} - container_name: docsum-vllm-service - ports: - - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011" - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_HUB_DISABLE_PROGRESS_BARS: 1 - HF_HUB_ENABLE_HF_TRANSFER: 0 - WILM_USE_TRITON_FLASH_ATTENTION: 0 - PYTORCH_JIT: 0 - volumes: - - "./data:/data" - shm_size: 20G - devices: - - /dev/kfd:/dev/kfd - - /dev/dri/:/dev/dri/ - cap_add: - - SYS_PTRACE - group_add: - - video - security_opt: - - seccomp:unconfined - - apparmor=unconfined - command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\"" - ipc: host - - docsum-llm-server: - image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} - container_name: docsum-llm-server - depends_on: - - docsum-vllm-service - ports: - - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000" - ipc: host - cap_add: - - SYS_PTRACE - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}" - HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID} - LOGFLAG: ${DOCSUM_LOGFLAG:-False} - MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS} - MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS} - restart: unless-stopped - - whisper-service: - image: ${REGISTRY:-opea}/whisper:${TAG:-latest} - container_name: whisper-service - ports: - - "${DOCSUM_WHISPER_PORT:-7066}:7066" - ipc: host - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - restart: unless-stopped - - docsum-backend-server: - image: ${REGISTRY:-opea}/docsum:${TAG:-latest} - container_name: docsum-backend-server - depends_on: - - docsum-tgi-service - - docsum-llm-server - ports: - - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888" - environment: - no_proxy: ${no_proxy} - https_proxy: ${https_proxy} - http_proxy: ${http_proxy} - MEGA_SERVICE_HOST_IP: ${HOST_IP} - LLM_SERVICE_HOST_IP: ${HOST_IP} - ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP} - ipc: host - restart: always - - docsum-gradio-ui: - image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest} - container_name: docsum-ui-server - depends_on: - - docsum-backend-server - ports: - - "${DOCSUM_FRONTEND_PORT:-5173}:5173" - environment: - no_proxy: ${no_proxy} - https_proxy: ${https_proxy} - http_proxy: ${http_proxy} - BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} - DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} - ipc: host - restart: always - -networks: - default: - driver: bridge diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh deleted file mode 100644 index 43e71e0fbf..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -export HOST_IP="" -export DOCSUM_MAX_INPUT_TOKENS=2048 -export DOCSUM_MAX_TOTAL_TOKENS=4096 -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export DOCSUM_VLLM_SERVICE_PORT="8008" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN="" -export DOCSUM_LLM_SERVER_PORT="9000" -export DOCSUM_WHISPER_PORT="7066" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml index dc0d546189..095fd28c93 100644 --- a/DocSum/docker_image_build/build.yaml +++ b/DocSum/docker_image_build/build.yaml @@ -47,12 +47,3 @@ services: dockerfile: comps/llms/src/doc-summarization/Dockerfile extends: docsum image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} - vllm_rocm: - build: - args: - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - no_proxy: ${no_proxy} - context: ../ - dockerfile: ./Dockerfile-vllm-rocm - image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh deleted file mode 100644 index d0919a019a..0000000000 --- a/DocSum/tests/test_compose_on_rocm_vllm.sh +++ /dev/null @@ -1,249 +0,0 @@ -#!/bin/bash -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -set -xe -IMAGE_REPO=${IMAGE_REPO:-"opea"} -IMAGE_TAG=${IMAGE_TAG:-"latest"} -echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" -echo "TAG=IMAGE_TAG=${IMAGE_TAG}" - -WORKPATH=$(dirname "$PWD") -LOG_PATH="$WORKPATH/tests" -ip_address=$(hostname -I | awk '{print $1}') -export MAX_INPUT_TOKENS=1024 -export MAX_TOTAL_TOKENS=2048 -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export HOST_IP=${ip_address} -export DOCSUM_VLLM_SERVICE_PORT="8008" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} -export DOCSUM_LLM_SERVER_PORT="9000" -export DOCSUM_WHISPER_PORT="7066" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export MEGA_SERVICE_HOST_IP=${HOST_IP} -export LLM_SERVICE_HOST_IP=${HOST_IP} -export ASR_SERVICE_HOST_IP=${HOST_IP} -export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" - -function build_docker_images() { - opea_branch=${opea_branch:-"main"} - # If the opea_branch isn't main, replace the git clone branch in Dockerfile. - if [[ "${opea_branch}" != "main" ]]; then - cd $WORKPATH - OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" - NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" - find . -type f -name "Dockerfile*" | while read -r file; do - echo "Processing file: $file" - sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" - done - fi - - cd $WORKPATH/docker_image_build - git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git - - echo "Build all the images with --no-cache, check docker_image_build.log for details..." - service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper" - docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log - - docker images && sleep 1s -} - -function start_services() { - cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm - sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env - # Start Docker Containers - docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log - sleep 1m -} - -function validate_services() { - local URL="$1" - local EXPECTED_RESULT="$2" - local SERVICE_NAME="$3" - local DOCKER_NAME="$4" - local INPUT_DATA="$5" - - local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") - - echo "===========================================" - - if [ "$HTTP_STATUS" -eq 200 ]; then - echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." - - local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) - - if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then - echo "[ $SERVICE_NAME ] Content is as expected." - else - echo "EXPECTED_RESULT==> $EXPECTED_RESULT" - echo "CONTENT==> $CONTENT" - echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - - fi - else - echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - sleep 1s -} - -get_base64_str() { - local file_name=$1 - base64 -w 0 "$file_name" -} - -# Function to generate input data for testing based on the document type -input_data_for_test() { - local document_type=$1 - case $document_type in - ("text") - echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are." - ;; - ("audio") - get_base64_str "$WORKPATH/tests/data/test.wav" - ;; - ("video") - get_base64_str "$WORKPATH/tests/data/test.mp4" - ;; - (*) - echo "Invalid document type" >&2 - exit 1 - ;; - esac -} - -function validate_microservices() { - # Check if the microservices are running correctly. - - # whisper microservice - ulimit -s 65536 - validate_services \ - "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \ - '{"asr_result":"well"}' \ - "whisper-service" \ - "whisper-service" \ - "{\"audio\": \"$(input_data_for_test "audio")\"}" - - # vLLM service - validate_services \ - "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \ - "generated_text" \ - "docsum-vllm-service" \ - "docsum-vllm-service" \ - '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}' - - # llm microservice - validate_services \ - "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \ - "text" \ - "docsum-llm-server" \ - "docsum-llm-server" \ - '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' - -} - -function validate_megaservice() { - local SERVICE_NAME="docsum-backend-server" - local DOCKER_NAME="docsum-backend-server" - local EXPECTED_RESULT="[DONE]" - local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." - local URL="${host_ip}:8888/v1/docsum" - local DATA_TYPE="type=text" - - local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL") - - if [ "$HTTP_STATUS" -eq 200 ]; then - echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." - - local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) - - if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then - echo "[ $SERVICE_NAME ] Content is as expected." - else - echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - else - echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - sleep 1s -} - -function validate_megaservice_json() { - # Curl the Mega Service - echo "" - echo ">>> Checking text data with Content-Type: application/json" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' - - echo ">>> Checking audio data" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - "{\"type\": \"audio\", \"messages\": \"$(input_data_for_test "audio")\"}" - - echo ">>> Checking video data" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - "{\"type\": \"video\", \"messages\": \"$(input_data_for_test "video")\"}" - -} - -function stop_docker() { - cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/ - docker compose stop && docker compose rm -f -} - -function main() { - echo "===========================================" - echo ">>>> Stopping any running Docker containers..." - stop_docker - - echo "===========================================" - if [[ "$IMAGE_REPO" == "opea" ]]; then - echo ">>>> Building Docker images..." - build_docker_images - fi - - echo "===========================================" - echo ">>>> Starting Docker services..." - start_services - - echo "===========================================" - echo ">>>> Validating microservices..." - validate_microservices - - echo "===========================================" - echo ">>>> Validating megaservice..." - validate_megaservice - echo ">>>> Validating validate_megaservice_json..." - validate_megaservice_json - - echo "===========================================" - echo ">>>> Stopping Docker containers..." - stop_docker - - echo "===========================================" - echo ">>>> Pruning Docker system..." - echo y | docker system prune - echo ">>>> Docker system pruned successfully." - echo "===========================================" -} - -main From a8857ae326b2d71ca66bc6f86715ac9ab467ac85 Mon Sep 17 00:00:00 2001 From: Chingis Yundunov Date: Thu, 13 Feb 2025 10:02:03 +0700 Subject: [PATCH 007/226] DocSum - add files for deploy app with ROCm vLLM Signed-off-by: Chingis Yundunov --- DocSum/Dockerfile-vllm-rocm | 18 ++ .../amd/gpu/rocm-vllm/README.md | 175 ++++++++++++ .../amd/gpu/rocm-vllm/compose.yaml | 107 ++++++++ .../amd/gpu/rocm-vllm/set_env.sh | 16 ++ DocSum/docker_image_build/build.yaml | 9 + DocSum/tests/test_compose_on_rocm_vllm.sh | 249 ++++++++++++++++++ 6 files changed, 574 insertions(+) create mode 100644 DocSum/Dockerfile-vllm-rocm create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh create mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm new file mode 100644 index 0000000000..f0e8a8743a --- /dev/null +++ b/DocSum/Dockerfile-vllm-rocm @@ -0,0 +1,18 @@ +FROM rocm/vllm-dev:main + +# Set the working directory +WORKDIR /workspace + +# Copy the api_server.py into the image +ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py + +# Expose the port used by the API server +EXPOSE 8011 + +# Set environment variables +ENV HUGGINGFACE_HUB_CACHE=/workspace +ENV WILM_USE_TRITON_FLASH_ATTENTION=0 +ENV PYTORCH_JIT=0 + +# Set the entrypoint to the api_server.py script +ENTRYPOINT ["python3", "/workspace/api_server.py"] diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md new file mode 100644 index 0000000000..4d41a5cd31 --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md @@ -0,0 +1,175 @@ +# Build and deploy DocSum Application on AMD GPU (ROCm) + +## Build images + +## 🚀 Build Docker Images + +First of all, you need to build Docker Images locally and install the python package of it. + +### 1. Build LLM Image + +```bash +git clone https://github.com/opea-project/GenAIComps.git +cd GenAIComps +docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile . +``` + +Then run the command `docker images`, you will have the following four Docker Images: + +### 2. Build MegaService Docker Image + +To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command: + +```bash +git clone https://github.com/opea-project/GenAIExamples +cd GenAIExamples/DocSum/ +docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . +``` + +### 3. Build UI Docker Image + +Build the frontend Docker image via below command: + +```bash +cd GenAIExamples/DocSum/ui +docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . +``` + +Then run the command `docker images`, you will have the following Docker Images: + +1. `opea/llm-docsum-tgi:latest` +2. `opea/docsum:latest` +3. `opea/docsum-ui:latest` + +### 4. Build React UI Docker Image + +Build the frontend Docker image via below command: + +```bash +cd GenAIExamples/DocSum/ui +export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum" +docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react . + +docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . +``` + +Then run the command `docker images`, you will have the following Docker Images: + +1. `opea/llm-docsum-tgi:latest` +2. `opea/docsum:latest` +3. `opea/docsum-ui:latest` +4. `opea/docsum-react-ui:latest` + +## 🚀 Start Microservices and MegaService + +### Required Models + +Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model. +For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable. + +### Setup Environment Variables + +Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below. + +```bash +export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm" +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export HOST_IP=${host_ip} +export DOCSUM_TGI_SERVICE_PORT="18882" +export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} +export DOCSUM_LLM_SERVER_PORT="8008" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export DocSum_COMPONENT_NAME="OpeaDocSumTgi" +``` + +Note: Please replace with `host_ip` with your external IP address, do not use localhost. + +Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered, where is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) + +Example for set isolation for 1 GPU + +``` + - /dev/dri/card0:/dev/dri/card0 + - /dev/dri/renderD128:/dev/dri/renderD128 +``` + +Example for set isolation for 2 GPUs + +``` + - /dev/dri/card0:/dev/dri/card0 + - /dev/dri/renderD128:/dev/dri/renderD128 + - /dev/dri/card1:/dev/dri/card1 + - /dev/dri/renderD129:/dev/dri/renderD129 +``` + +Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) + +### Start Microservice Docker Containers + +```bash +cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm +docker compose up -d +``` + +### Validate Microservices + +1. TGI Service + + ```bash + curl http://${host_ip}:8008/generate \ + -X POST \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \ + -H 'Content-Type: application/json' + ``` + +2. LLM Microservice + + ```bash + curl http://${host_ip}:9000/v1/docsum \ + -X POST \ + -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \ + -H 'Content-Type: application/json' + ``` + +3. MegaService + + ```bash + curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{ + "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false + }' + ``` + +## 🚀 Launch the Svelte UI + +Open this URL `http://{host_ip}:5173` in your browser to access the frontend. + +![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b) + +Here is an example for summarizing a article. + +![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55) + +## 🚀 Launch the React UI (Optional) + +To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below: + +```yaml +docsum-rocm-react-ui-server: + image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest} + container_name: docsum-rocm-react-ui-server + depends_on: + - docsum-rocm-backend-server + ports: + - "5174:80" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT} +``` + +Open this URL `http://{host_ip}:5175` in your browser to access the frontend. + +![project-screenshot](../../../../assets/img/docsum-ui-react.png) diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml new file mode 100644 index 0000000000..037aa06395 --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml @@ -0,0 +1,107 @@ +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +services: + docsum-vllm-service: + image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} + container_name: docsum-vllm-service + ports: + - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011" + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + WILM_USE_TRITON_FLASH_ATTENTION: 0 + PYTORCH_JIT: 0 + volumes: + - "./data:/data" + shm_size: 20G + devices: + - /dev/kfd:/dev/kfd + - /dev/dri/:/dev/dri/ + cap_add: + - SYS_PTRACE + group_add: + - video + security_opt: + - seccomp:unconfined + - apparmor=unconfined + command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\"" + ipc: host + + docsum-llm-server: + image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} + container_name: docsum-llm-server + depends_on: + - docsum-vllm-service + ports: + - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000" + ipc: host + cap_add: + - SYS_PTRACE + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}" + HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID} + LOGFLAG: ${DOCSUM_LOGFLAG:-False} + MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS} + MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS} + restart: unless-stopped + + whisper-service: + image: ${REGISTRY:-opea}/whisper:${TAG:-latest} + container_name: whisper-service + ports: + - "${DOCSUM_WHISPER_PORT:-7066}:7066" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + restart: unless-stopped + + docsum-backend-server: + image: ${REGISTRY:-opea}/docsum:${TAG:-latest} + container_name: docsum-backend-server + depends_on: + - docsum-tgi-service + - docsum-llm-server + ports: + - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888" + environment: + no_proxy: ${no_proxy} + https_proxy: ${https_proxy} + http_proxy: ${http_proxy} + MEGA_SERVICE_HOST_IP: ${HOST_IP} + LLM_SERVICE_HOST_IP: ${HOST_IP} + ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP} + ipc: host + restart: always + + docsum-gradio-ui: + image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest} + container_name: docsum-ui-server + depends_on: + - docsum-backend-server + ports: + - "${DOCSUM_FRONTEND_PORT:-5173}:5173" + environment: + no_proxy: ${no_proxy} + https_proxy: ${https_proxy} + http_proxy: ${http_proxy} + BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} + DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} + ipc: host + restart: always + +networks: + default: + driver: bridge diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh new file mode 100644 index 0000000000..43e71e0fbf --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +export HOST_IP="" +export DOCSUM_MAX_INPUT_TOKENS=2048 +export DOCSUM_MAX_TOTAL_TOKENS=4096 +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export DOCSUM_VLLM_SERVICE_PORT="8008" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN="" +export DOCSUM_LLM_SERVER_PORT="9000" +export DOCSUM_WHISPER_PORT="7066" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml index 095fd28c93..dc0d546189 100644 --- a/DocSum/docker_image_build/build.yaml +++ b/DocSum/docker_image_build/build.yaml @@ -47,3 +47,12 @@ services: dockerfile: comps/llms/src/doc-summarization/Dockerfile extends: docsum image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} + vllm_rocm: + build: + args: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + no_proxy: ${no_proxy} + context: ../ + dockerfile: ./Dockerfile-vllm-rocm + image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh new file mode 100644 index 0000000000..d0919a019a --- /dev/null +++ b/DocSum/tests/test_compose_on_rocm_vllm.sh @@ -0,0 +1,249 @@ +#!/bin/bash +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +set -xe +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') +export MAX_INPUT_TOKENS=1024 +export MAX_TOTAL_TOKENS=2048 +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export HOST_IP=${ip_address} +export DOCSUM_VLLM_SERVICE_PORT="8008" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} +export DOCSUM_LLM_SERVER_PORT="9000" +export DOCSUM_WHISPER_PORT="7066" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export MEGA_SERVICE_HOST_IP=${HOST_IP} +export LLM_SERVICE_HOST_IP=${HOST_IP} +export ASR_SERVICE_HOST_IP=${HOST_IP} +export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" + +function build_docker_images() { + opea_branch=${opea_branch:-"main"} + # If the opea_branch isn't main, replace the git clone branch in Dockerfile. + if [[ "${opea_branch}" != "main" ]]; then + cd $WORKPATH + OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" + NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" + find . -type f -name "Dockerfile*" | while read -r file; do + echo "Processing file: $file" + sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" + done + fi + + cd $WORKPATH/docker_image_build + git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + + echo "Build all the images with --no-cache, check docker_image_build.log for details..." + service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper" + docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log + + docker images && sleep 1s +} + +function start_services() { + cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm + sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env + # Start Docker Containers + docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log + sleep 1m +} + +function validate_services() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + + echo "===========================================" + + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "EXPECTED_RESULT==> $EXPECTED_RESULT" + echo "CONTENT==> $CONTENT" + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +get_base64_str() { + local file_name=$1 + base64 -w 0 "$file_name" +} + +# Function to generate input data for testing based on the document type +input_data_for_test() { + local document_type=$1 + case $document_type in + ("text") + echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are." + ;; + ("audio") + get_base64_str "$WORKPATH/tests/data/test.wav" + ;; + ("video") + get_base64_str "$WORKPATH/tests/data/test.mp4" + ;; + (*) + echo "Invalid document type" >&2 + exit 1 + ;; + esac +} + +function validate_microservices() { + # Check if the microservices are running correctly. + + # whisper microservice + ulimit -s 65536 + validate_services \ + "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \ + '{"asr_result":"well"}' \ + "whisper-service" \ + "whisper-service" \ + "{\"audio\": \"$(input_data_for_test "audio")\"}" + + # vLLM service + validate_services \ + "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \ + "generated_text" \ + "docsum-vllm-service" \ + "docsum-vllm-service" \ + '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}' + + # llm microservice + validate_services \ + "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \ + "text" \ + "docsum-llm-server" \ + "docsum-llm-server" \ + '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' + +} + +function validate_megaservice() { + local SERVICE_NAME="docsum-backend-server" + local DOCKER_NAME="docsum-backend-server" + local EXPECTED_RESULT="[DONE]" + local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." + local URL="${host_ip}:8888/v1/docsum" + local DATA_TYPE="type=text" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL") + + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +function validate_megaservice_json() { + # Curl the Mega Service + echo "" + echo ">>> Checking text data with Content-Type: application/json" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' + + echo ">>> Checking audio data" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + "{\"type\": \"audio\", \"messages\": \"$(input_data_for_test "audio")\"}" + + echo ">>> Checking video data" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + "{\"type\": \"video\", \"messages\": \"$(input_data_for_test "video")\"}" + +} + +function stop_docker() { + cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/ + docker compose stop && docker compose rm -f +} + +function main() { + echo "===========================================" + echo ">>>> Stopping any running Docker containers..." + stop_docker + + echo "===========================================" + if [[ "$IMAGE_REPO" == "opea" ]]; then + echo ">>>> Building Docker images..." + build_docker_images + fi + + echo "===========================================" + echo ">>>> Starting Docker services..." + start_services + + echo "===========================================" + echo ">>>> Validating microservices..." + validate_microservices + + echo "===========================================" + echo ">>>> Validating megaservice..." + validate_megaservice + echo ">>>> Validating validate_megaservice_json..." + validate_megaservice_json + + echo "===========================================" + echo ">>>> Stopping Docker containers..." + stop_docker + + echo "===========================================" + echo ">>>> Pruning Docker system..." + echo y | docker system prune + echo ">>>> Docker system pruned successfully." + echo "===========================================" +} + +main From 5a38b266ac77a2bf0766cefab14ec62f28633a8d Mon Sep 17 00:00:00 2001 From: Chingis Yundunov Date: Thu, 13 Feb 2025 10:07:05 +0700 Subject: [PATCH 008/226] DocSum - fix main Signed-off-by: Chingis Yundunov --- DocSum/Dockerfile-vllm-rocm | 18 -- .../amd/gpu/rocm-vllm/README.md | 175 ------------ .../amd/gpu/rocm-vllm/compose.yaml | 107 -------- .../amd/gpu/rocm-vllm/set_env.sh | 16 -- DocSum/docker_image_build/build.yaml | 9 - DocSum/tests/test_compose_on_rocm_vllm.sh | 249 ------------------ 6 files changed, 574 deletions(-) delete mode 100644 DocSum/Dockerfile-vllm-rocm delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh delete mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm deleted file mode 100644 index f0e8a8743a..0000000000 --- a/DocSum/Dockerfile-vllm-rocm +++ /dev/null @@ -1,18 +0,0 @@ -FROM rocm/vllm-dev:main - -# Set the working directory -WORKDIR /workspace - -# Copy the api_server.py into the image -ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py - -# Expose the port used by the API server -EXPOSE 8011 - -# Set environment variables -ENV HUGGINGFACE_HUB_CACHE=/workspace -ENV WILM_USE_TRITON_FLASH_ATTENTION=0 -ENV PYTORCH_JIT=0 - -# Set the entrypoint to the api_server.py script -ENTRYPOINT ["python3", "/workspace/api_server.py"] diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md deleted file mode 100644 index 4d41a5cd31..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md +++ /dev/null @@ -1,175 +0,0 @@ -# Build and deploy DocSum Application on AMD GPU (ROCm) - -## Build images - -## 🚀 Build Docker Images - -First of all, you need to build Docker Images locally and install the python package of it. - -### 1. Build LLM Image - -```bash -git clone https://github.com/opea-project/GenAIComps.git -cd GenAIComps -docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile . -``` - -Then run the command `docker images`, you will have the following four Docker Images: - -### 2. Build MegaService Docker Image - -To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command: - -```bash -git clone https://github.com/opea-project/GenAIExamples -cd GenAIExamples/DocSum/ -docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . -``` - -### 3. Build UI Docker Image - -Build the frontend Docker image via below command: - -```bash -cd GenAIExamples/DocSum/ui -docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . -``` - -Then run the command `docker images`, you will have the following Docker Images: - -1. `opea/llm-docsum-tgi:latest` -2. `opea/docsum:latest` -3. `opea/docsum-ui:latest` - -### 4. Build React UI Docker Image - -Build the frontend Docker image via below command: - -```bash -cd GenAIExamples/DocSum/ui -export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum" -docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react . - -docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . -``` - -Then run the command `docker images`, you will have the following Docker Images: - -1. `opea/llm-docsum-tgi:latest` -2. `opea/docsum:latest` -3. `opea/docsum-ui:latest` -4. `opea/docsum-react-ui:latest` - -## 🚀 Start Microservices and MegaService - -### Required Models - -Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model. -For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable. - -### Setup Environment Variables - -Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below. - -```bash -export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm" -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export HOST_IP=${host_ip} -export DOCSUM_TGI_SERVICE_PORT="18882" -export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} -export DOCSUM_LLM_SERVER_PORT="8008" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export DocSum_COMPONENT_NAME="OpeaDocSumTgi" -``` - -Note: Please replace with `host_ip` with your external IP address, do not use localhost. - -Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered, where is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) - -Example for set isolation for 1 GPU - -``` - - /dev/dri/card0:/dev/dri/card0 - - /dev/dri/renderD128:/dev/dri/renderD128 -``` - -Example for set isolation for 2 GPUs - -``` - - /dev/dri/card0:/dev/dri/card0 - - /dev/dri/renderD128:/dev/dri/renderD128 - - /dev/dri/card1:/dev/dri/card1 - - /dev/dri/renderD129:/dev/dri/renderD129 -``` - -Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) - -### Start Microservice Docker Containers - -```bash -cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm -docker compose up -d -``` - -### Validate Microservices - -1. TGI Service - - ```bash - curl http://${host_ip}:8008/generate \ - -X POST \ - -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \ - -H 'Content-Type: application/json' - ``` - -2. LLM Microservice - - ```bash - curl http://${host_ip}:9000/v1/docsum \ - -X POST \ - -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \ - -H 'Content-Type: application/json' - ``` - -3. MegaService - - ```bash - curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{ - "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false - }' - ``` - -## 🚀 Launch the Svelte UI - -Open this URL `http://{host_ip}:5173` in your browser to access the frontend. - -![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b) - -Here is an example for summarizing a article. - -![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55) - -## 🚀 Launch the React UI (Optional) - -To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below: - -```yaml -docsum-rocm-react-ui-server: - image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest} - container_name: docsum-rocm-react-ui-server - depends_on: - - docsum-rocm-backend-server - ports: - - "5174:80" - environment: - - no_proxy=${no_proxy} - - https_proxy=${https_proxy} - - http_proxy=${http_proxy} - - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT} -``` - -Open this URL `http://{host_ip}:5175` in your browser to access the frontend. - -![project-screenshot](../../../../assets/img/docsum-ui-react.png) diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml deleted file mode 100644 index 037aa06395..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -services: - docsum-vllm-service: - image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} - container_name: docsum-vllm-service - ports: - - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011" - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_HUB_DISABLE_PROGRESS_BARS: 1 - HF_HUB_ENABLE_HF_TRANSFER: 0 - WILM_USE_TRITON_FLASH_ATTENTION: 0 - PYTORCH_JIT: 0 - volumes: - - "./data:/data" - shm_size: 20G - devices: - - /dev/kfd:/dev/kfd - - /dev/dri/:/dev/dri/ - cap_add: - - SYS_PTRACE - group_add: - - video - security_opt: - - seccomp:unconfined - - apparmor=unconfined - command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\"" - ipc: host - - docsum-llm-server: - image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} - container_name: docsum-llm-server - depends_on: - - docsum-vllm-service - ports: - - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000" - ipc: host - cap_add: - - SYS_PTRACE - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}" - HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID} - LOGFLAG: ${DOCSUM_LOGFLAG:-False} - MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS} - MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS} - restart: unless-stopped - - whisper-service: - image: ${REGISTRY:-opea}/whisper:${TAG:-latest} - container_name: whisper-service - ports: - - "${DOCSUM_WHISPER_PORT:-7066}:7066" - ipc: host - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - restart: unless-stopped - - docsum-backend-server: - image: ${REGISTRY:-opea}/docsum:${TAG:-latest} - container_name: docsum-backend-server - depends_on: - - docsum-tgi-service - - docsum-llm-server - ports: - - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888" - environment: - no_proxy: ${no_proxy} - https_proxy: ${https_proxy} - http_proxy: ${http_proxy} - MEGA_SERVICE_HOST_IP: ${HOST_IP} - LLM_SERVICE_HOST_IP: ${HOST_IP} - ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP} - ipc: host - restart: always - - docsum-gradio-ui: - image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest} - container_name: docsum-ui-server - depends_on: - - docsum-backend-server - ports: - - "${DOCSUM_FRONTEND_PORT:-5173}:5173" - environment: - no_proxy: ${no_proxy} - https_proxy: ${https_proxy} - http_proxy: ${http_proxy} - BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} - DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} - ipc: host - restart: always - -networks: - default: - driver: bridge diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh deleted file mode 100644 index 43e71e0fbf..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -export HOST_IP="" -export DOCSUM_MAX_INPUT_TOKENS=2048 -export DOCSUM_MAX_TOTAL_TOKENS=4096 -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export DOCSUM_VLLM_SERVICE_PORT="8008" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN="" -export DOCSUM_LLM_SERVER_PORT="9000" -export DOCSUM_WHISPER_PORT="7066" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml index dc0d546189..095fd28c93 100644 --- a/DocSum/docker_image_build/build.yaml +++ b/DocSum/docker_image_build/build.yaml @@ -47,12 +47,3 @@ services: dockerfile: comps/llms/src/doc-summarization/Dockerfile extends: docsum image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} - vllm_rocm: - build: - args: - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - no_proxy: ${no_proxy} - context: ../ - dockerfile: ./Dockerfile-vllm-rocm - image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh deleted file mode 100644 index d0919a019a..0000000000 --- a/DocSum/tests/test_compose_on_rocm_vllm.sh +++ /dev/null @@ -1,249 +0,0 @@ -#!/bin/bash -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -set -xe -IMAGE_REPO=${IMAGE_REPO:-"opea"} -IMAGE_TAG=${IMAGE_TAG:-"latest"} -echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" -echo "TAG=IMAGE_TAG=${IMAGE_TAG}" - -WORKPATH=$(dirname "$PWD") -LOG_PATH="$WORKPATH/tests" -ip_address=$(hostname -I | awk '{print $1}') -export MAX_INPUT_TOKENS=1024 -export MAX_TOTAL_TOKENS=2048 -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export HOST_IP=${ip_address} -export DOCSUM_VLLM_SERVICE_PORT="8008" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} -export DOCSUM_LLM_SERVER_PORT="9000" -export DOCSUM_WHISPER_PORT="7066" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export MEGA_SERVICE_HOST_IP=${HOST_IP} -export LLM_SERVICE_HOST_IP=${HOST_IP} -export ASR_SERVICE_HOST_IP=${HOST_IP} -export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" - -function build_docker_images() { - opea_branch=${opea_branch:-"main"} - # If the opea_branch isn't main, replace the git clone branch in Dockerfile. - if [[ "${opea_branch}" != "main" ]]; then - cd $WORKPATH - OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" - NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" - find . -type f -name "Dockerfile*" | while read -r file; do - echo "Processing file: $file" - sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" - done - fi - - cd $WORKPATH/docker_image_build - git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git - - echo "Build all the images with --no-cache, check docker_image_build.log for details..." - service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper" - docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log - - docker images && sleep 1s -} - -function start_services() { - cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm - sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env - # Start Docker Containers - docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log - sleep 1m -} - -function validate_services() { - local URL="$1" - local EXPECTED_RESULT="$2" - local SERVICE_NAME="$3" - local DOCKER_NAME="$4" - local INPUT_DATA="$5" - - local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") - - echo "===========================================" - - if [ "$HTTP_STATUS" -eq 200 ]; then - echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." - - local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) - - if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then - echo "[ $SERVICE_NAME ] Content is as expected." - else - echo "EXPECTED_RESULT==> $EXPECTED_RESULT" - echo "CONTENT==> $CONTENT" - echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - - fi - else - echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - sleep 1s -} - -get_base64_str() { - local file_name=$1 - base64 -w 0 "$file_name" -} - -# Function to generate input data for testing based on the document type -input_data_for_test() { - local document_type=$1 - case $document_type in - ("text") - echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are." - ;; - ("audio") - get_base64_str "$WORKPATH/tests/data/test.wav" - ;; - ("video") - get_base64_str "$WORKPATH/tests/data/test.mp4" - ;; - (*) - echo "Invalid document type" >&2 - exit 1 - ;; - esac -} - -function validate_microservices() { - # Check if the microservices are running correctly. - - # whisper microservice - ulimit -s 65536 - validate_services \ - "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \ - '{"asr_result":"well"}' \ - "whisper-service" \ - "whisper-service" \ - "{\"audio\": \"$(input_data_for_test "audio")\"}" - - # vLLM service - validate_services \ - "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \ - "generated_text" \ - "docsum-vllm-service" \ - "docsum-vllm-service" \ - '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}' - - # llm microservice - validate_services \ - "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \ - "text" \ - "docsum-llm-server" \ - "docsum-llm-server" \ - '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' - -} - -function validate_megaservice() { - local SERVICE_NAME="docsum-backend-server" - local DOCKER_NAME="docsum-backend-server" - local EXPECTED_RESULT="[DONE]" - local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." - local URL="${host_ip}:8888/v1/docsum" - local DATA_TYPE="type=text" - - local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL") - - if [ "$HTTP_STATUS" -eq 200 ]; then - echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." - - local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) - - if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then - echo "[ $SERVICE_NAME ] Content is as expected." - else - echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - else - echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - sleep 1s -} - -function validate_megaservice_json() { - # Curl the Mega Service - echo "" - echo ">>> Checking text data with Content-Type: application/json" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' - - echo ">>> Checking audio data" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - "{\"type\": \"audio\", \"messages\": \"$(input_data_for_test "audio")\"}" - - echo ">>> Checking video data" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - "{\"type\": \"video\", \"messages\": \"$(input_data_for_test "video")\"}" - -} - -function stop_docker() { - cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/ - docker compose stop && docker compose rm -f -} - -function main() { - echo "===========================================" - echo ">>>> Stopping any running Docker containers..." - stop_docker - - echo "===========================================" - if [[ "$IMAGE_REPO" == "opea" ]]; then - echo ">>>> Building Docker images..." - build_docker_images - fi - - echo "===========================================" - echo ">>>> Starting Docker services..." - start_services - - echo "===========================================" - echo ">>>> Validating microservices..." - validate_microservices - - echo "===========================================" - echo ">>>> Validating megaservice..." - validate_megaservice - echo ">>>> Validating validate_megaservice_json..." - validate_megaservice_json - - echo "===========================================" - echo ">>>> Stopping Docker containers..." - stop_docker - - echo "===========================================" - echo ">>>> Pruning Docker system..." - echo y | docker system prune - echo ">>>> Docker system pruned successfully." - echo "===========================================" -} - -main From 9ccf540b892c0ae3a58a004afcb01d3647a92c90 Mon Sep 17 00:00:00 2001 From: Chingis Yundunov Date: Thu, 24 Apr 2025 20:01:07 +0700 Subject: [PATCH 009/226] DocSum - refactoring README.md Signed-off-by: Chingis Yundunov --- DocSum/docker_compose/amd/gpu/rocm/README.md | 138 +++++++++++++++---- 1 file changed, 108 insertions(+), 30 deletions(-) diff --git a/DocSum/docker_compose/amd/gpu/rocm/README.md b/DocSum/docker_compose/amd/gpu/rocm/README.md index 2c4a196149..92922f4b65 100644 --- a/DocSum/docker_compose/amd/gpu/rocm/README.md +++ b/DocSum/docker_compose/amd/gpu/rocm/README.md @@ -25,15 +25,15 @@ This section describes how to quickly deploy and test the DocSum service manuall Clone the GenAIExample repository and access the ChatQnA AMD GPU platform Docker Compose files and supporting scripts: -``` +```bash git clone https://github.com/opea-project/GenAIExamples.git cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm ``` -Checkout a released version, such as v1.2: +Checkout a released version, such as v1.3: ``` -git checkout v1.2 +git checkout v1.3 ``` ### Generate a HuggingFace Access Token @@ -42,33 +42,96 @@ Some HuggingFace resources, such as some models, are only accessible if you have ### Configure the Deployment Environment -To set up environment variables for deploying DocSum services, source the _set_env.sh_ script in this directory: +To set up environment variables for deploying ChatQnA services, set up some parameters specific to the deployment environment and source the `set_env_*.sh` script in this directory: -``` -source ./set_env.sh +- if used vLLM - set_env_vllm.sh +- if used TGI - set_env.sh + +Set the values of the variables: + +- **HOST_IP, HOST_IP_EXTERNAL** - These variables are used to configure the name/address of the service in the operating system environment for the application services to interact with each other and with the outside world. + + If your server uses only an internal address and is not accessible from the Internet, then the values for these two variables will be the same and the value will be equal to the server's internal name/address. + + If your server uses only an external, Internet-accessible address, then the values for these two variables will be the same and the value will be equal to the server's external name/address. + + If your server is located on an internal network, has an internal address, but is accessible from the Internet via a proxy/firewall/load balancer, then the HOST_IP variable will have a value equal to the internal name/address of the server, and the EXTERNAL_HOST_IP variable will have a value equal to the external name/address of the proxy/firewall/load balancer behind which the server is located. + + We set these values in the file set_env\*\*\*\*.sh + +- **Variables with names like "**\*\*\*\*\*\*\_PORT"\*\* - These variables set the IP port numbers for establishing network connections to the application services. + The values shown in the file set_env.sh or set_env_vllm.sh they are the values used for the development and testing of the application, as well as configured for the environment in which the development is performed. These values must be configured in accordance with the rules of network access to your environment's server, and must not overlap with the IP ports of other applications that are already in use. + +Setting variables in the operating system environment: + +```bash +export HUGGINGFACEHUB_API_TOKEN="Your_HuggingFace_API_Token" +source ./set_env_*.sh # replace the script name with the appropriate one ``` -The _set_env.sh_ script will prompt for required and optional environment variables used to configure the DocSum services. If a value is not entered, the script will use a default value for the same. It will also generate a _.env_ file defining the desired configuration. Consult the section on [DocSum Service configuration](#docsum-service-configuration) for information on how service specific configuration parameters affect deployments. +Consult the section on [DocSum Service configuration](#docsum-configuration) for information on how service specific configuration parameters affect deployments. ### Deploy the Services Using Docker Compose -To deploy the DocSum services, execute the `docker compose up` command with the appropriate arguments. For a default deployment, execute: +To deploy the DocSum services, execute the `docker compose up` command with the appropriate arguments. For a default deployment with TGI, execute the command below. It uses the 'compose.yaml' file. ```bash -docker compose up -d +cd docker_compose/amd/gpu/rocm +# if used TGI +docker compose -f compose.yaml up -d +# if used vLLM +# docker compose -f compose_vllm.yaml up -d +``` + +To enable GPU support for AMD GPUs, the following configuration is added to the Docker Compose file: + +- compose_vllm.yaml - for vLLM-based application +- compose.yaml - for TGI-based + +```yaml +shm_size: 1g +devices: + - /dev/kfd:/dev/kfd + - /dev/dri:/dev/dri +cap_add: + - SYS_PTRACE +group_add: + - video +security_opt: + - seccomp:unconfined +``` + +This configuration forwards all available GPUs to the container. To use a specific GPU, specify its `cardN` and `renderN` device IDs. For example: + +```yaml +shm_size: 1g +devices: + - /dev/kfd:/dev/kfd + - /dev/dri/card0:/dev/dri/card0 + - /dev/dri/render128:/dev/dri/render128 +cap_add: + - SYS_PTRACE +group_add: + - video +security_opt: + - seccomp:unconfined ``` -**Note**: developers should build docker image from source when: +**How to Identify GPU Device IDs:** +Use AMD GPU driver utilities to determine the correct `cardN` and `renderN` IDs for your GPU. -- Developing off the git main branch (as the container's ports in the repo may be different from the published docker image). -- Unable to download the docker image. -- Use a specific version of Docker image. +> **Note**: developers should build docker image from source when: +> +> - Developing off the git main branch (as the container's ports in the repo may be different > from the published docker image). +> - Unable to download the docker image. +> - Use a specific version of Docker image. Please refer to the table below to build different microservices from source: | Microservice | Deployment Guide | -| ------------ | ------------------------------------------------------------------------------------------------------------------------------------- | +|--------------| ------------------------------------------------------------------------------------------------------------------------------------- | | whisper | [whisper build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/third_parties/whisper/src) | +| TGI | [TGI project](https://github.com/huggingface/text-generation-inference.git) | | vLLM | [vLLM build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/third_parties/vllm#build-docker) | | llm-docsum | [LLM-DocSum build guide](https://github.com/opea-project/GenAIComps/tree/main/comps/llms/src/doc-summarization#12-build-docker-image) | | MegaService | [MegaService build guide](../../../../README_miscellaneous.md#build-megaservice-docker-image) | @@ -84,6 +147,7 @@ docker ps -a For the default deployment, the following 5 containers should have started: +If used TGI: ``` CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 748f577b3c78 opea/whisper:latest "python whisper_s…" 5 minutes ago Up About a minute 0.0.0.0:7066->7066/tcp, :::7066->7066/tcp whisper-service @@ -93,24 +157,38 @@ fds3dd5b9fd8 opea/docsum:latest "py 78964d0c1hg5 ghcr.io/huggingface/text-generation-inference:2.4.1-rocm "/tgi-entrypoint.sh" 5 minutes ago Up 5 minutes (healthy) 0.0.0.0:8008->80/tcp, [::]:8008->80/tcp docsum-tgi-service ``` +If used vLLM: +``` +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +748f577b3c78 opea/whisper:latest "python whisper_s…" 5 minutes ago Up About a minute 0.0.0.0:7066->7066/tcp, :::7066->7066/tcp whisper-service +4eq8b7034fd9 opea/docsum-gradio-ui:latest "docker-entrypoint.s…" 5 minutes ago Up About a minute 0.0.0.0:5173->5173/tcp, :::5173->5173/tcp docsum-ui-server +fds3dd5b9fd8 opea/docsum:latest "python docsum.py" 5 minutes ago Up About a minute 0.0.0.0:8888->8888/tcp, :::8888->8888/tcp docsum-backend-server +78fsd6fabfs7 opea/llm-docsum:latest "bash entrypoint.sh" 5 minutes ago Up About a minute 0.0.0.0:9000->9000/tcp, :::9000->9000/tcp docsum-llm-server +78964d0c1hg5 opea/vllm-rocm:latest "python3 /workspace/…" 5 minutes ago Up 5 minutes (healthy) 0.0.0.0:8008->80/tcp, [::]:8008->80/tcp docsum-vllm-service +``` + ### Test the Pipeline Once the DocSum services are running, test the pipeline using the following command: ```bash -curl -X POST http://${host_ip}:8888/v1/docsum \ +curl -X POST http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum \ -H "Content-Type: application/json" \ -d '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' ``` -**Note** The value of _host_ip_ was set using the _set_env.sh_ script and can be found in the _.env_ file. +**Note** The value of _HOST_IP_ was set using the _set_env.sh_ script and can be found in the _.env_ file. ### Cleanup the Deployment To stop the containers associated with the deployment, execute the following command: -``` +```bash +# if used TGI docker compose -f compose.yaml down +# if used vLLM +# docker compose -f compose_vllm.yaml down + ``` All the DocSum containers will be stopped and then removed on completion of the "down" command. @@ -132,7 +210,7 @@ There are also some customized usage. ```bash # form input. Use English mode (default). -curl http://${host_ip}:8888/v1/docsum \ +curl http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum \ -H "Content-Type: multipart/form-data" \ -F "type=text" \ -F "messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." \ @@ -141,7 +219,7 @@ curl http://${host_ip}:8888/v1/docsum \ -F "stream=True" # Use Chinese mode. -curl http://${host_ip}:8888/v1/docsum \ +curl http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum \ -H "Content-Type: multipart/form-data" \ -F "type=text" \ -F "messages=2024年9月26日，北京——今日，英特尔正式发布英特尔® 至强® 6性能核处理器（代号Granite Rapids），为AI、数据分析、科学计算等计算密集型业务提供卓越性能。" \ @@ -150,7 +228,7 @@ curl http://${host_ip}:8888/v1/docsum \ -F "stream=True" # Upload file -curl http://${host_ip}:8888/v1/docsum \ +curl http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum \ -H "Content-Type: multipart/form-data" \ -F "type=text" \ -F "messages=" \ @@ -166,11 +244,11 @@ curl http://${host_ip}:8888/v1/docsum \ Audio: ```bash -curl -X POST http://${host_ip}:8888/v1/docsum \ +curl -X POST http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum \ -H "Content-Type: application/json" \ -d '{"type": "audio", "messages": "UklGRigAAABXQVZFZm10IBIAAAABAAEARKwAAIhYAQACABAAAABkYXRhAgAAAAEA"}' -curl http://${host_ip}:8888/v1/docsum \ +curl http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum \ -H "Content-Type: multipart/form-data" \ -F "type=audio" \ -F "messages=UklGRigAAABXQVZFZm10IBIAAAABAAEARKwAAIhYAQACABAAAABkYXRhAgAAAAEA" \ @@ -182,11 +260,11 @@ curl http://${host_ip}:8888/v1/docsum \ Video: ```bash -curl -X POST http://${host_ip}:8888/v1/docsum \ +curl -X POST http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum \ -H "Content-Type: application/json" \ -d '{"type": "video", "messages": "convert your video to base64 data type"}' -curl http://${host_ip}:8888/v1/docsum \ +curl http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum \ -H "Content-Type: multipart/form-data" \ -F "type=video" \ -F "messages=convert your video to base64 data type" \ @@ -208,7 +286,7 @@ If you want to deal with long context, can set following parameters and select s "summary_type" is set to be "auto" by default, in this mode we will check input token length, if it exceed `MAX_INPUT_TOKENS`, `summary_type` will automatically be set to `refine` mode, otherwise will be set to `stuff` mode. ```bash -curl http://${host_ip}:8888/v1/docsum \ +curl http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum \ -H "Content-Type: multipart/form-data" \ -F "type=text" \ -F "messages=" \ @@ -223,7 +301,7 @@ curl http://${host_ip}:8888/v1/docsum \ In this mode LLM generate summary based on complete input text. In this case please carefully set `MAX_INPUT_TOKENS` and `MAX_TOTAL_TOKENS` according to your model and device memory, otherwise it may exceed LLM context limit and raise error when meet long context. ```bash -curl http://${host_ip}:8888/v1/docsum \ +curl http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum \ -H "Content-Type: multipart/form-data" \ -F "type=text" \ -F "messages=" \ @@ -238,7 +316,7 @@ curl http://${host_ip}:8888/v1/docsum \ Truncate mode will truncate the input text and keep only the first chunk, whose length is equal to `min(MAX_TOTAL_TOKENS - input.max_tokens - 50, MAX_INPUT_TOKENS)` ```bash -curl http://${host_ip}:8888/v1/docsum \ +curl http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum \ -H "Content-Type: multipart/form-data" \ -F "type=text" \ -F "messages=" \ @@ -255,7 +333,7 @@ Map_reduce mode will split the inputs into multiple chunks, map each document to In this mode, default `chunk_size` is set to be `min(MAX_TOTAL_TOKENS - input.max_tokens - 50, MAX_INPUT_TOKENS)` ```bash -curl http://${host_ip}:8888/v1/docsum \ +curl http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum \ -H "Content-Type: multipart/form-data" \ -F "type=text" \ -F "messages=" \ @@ -272,7 +350,7 @@ Refin mode will split the inputs into multiple chunks, generate summary for the In this mode, default `chunk_size` is set to be `min(MAX_TOTAL_TOKENS - 2 * input.max_tokens - 128, MAX_INPUT_TOKENS)`. ```bash -curl http://${host_ip}:8888/v1/docsum \ +curl http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum \ -H "Content-Type: multipart/form-data" \ -F "type=text" \ -F "messages=" \ @@ -288,7 +366,7 @@ Several UI options are provided. If you need to work with multimedia documents, ### Gradio UI -To access the UI, use the URL - http://${EXTERNAL_HOST_IP}:${FAGGEN_UI_PORT} +To access the UI, use the URL - http://${HOST_IP}:${DOCSUM_FRONTEND_PORT} A page should open when you click through to this address: ![UI start page](../../../../assets/img/ui-starting-page.png) From b5df3482351bc2a00f262f8449ea410ef0e817ab Mon Sep 17 00:00:00 2001 From: xiguiw <111278656+xiguiw@users.noreply.github.com> Date: Wed, 19 Feb 2025 19:24:10 +0800 Subject: [PATCH 010/226] Fix mismatched environment variable (#1575) Signed-off-by: Wang, Xigui Signed-off-by: Chingis Yundunov --- ChatQnA/docker_compose/intel/cpu/aipc/set_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChatQnA/docker_compose/intel/cpu/aipc/set_env.sh b/ChatQnA/docker_compose/intel/cpu/aipc/set_env.sh index 4eda65f97a..3ee4cd6d6c 100644 --- a/ChatQnA/docker_compose/intel/cpu/aipc/set_env.sh +++ b/ChatQnA/docker_compose/intel/cpu/aipc/set_env.sh @@ -17,7 +17,7 @@ if [ -z "${host_ip}" ]; then echo "Error: host_ip is not set. Please set host_ip first." fi -export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} +export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5" export RERANK_MODEL_ID="BAAI/bge-reranker-base" export INDEX_NAME="rag-redis" From 60dd862d5ac44e6e7b1b41595e02b727ff8b8244 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Thu, 20 Feb 2025 14:41:52 +0800 Subject: [PATCH 011/226] Fix trivy issue (#1569) Fix docker image security issue Signed-off-by: ZePan110 Signed-off-by: Chingis Yundunov --- AvatarChatbot/Dockerfile | 2 +- ChatQnA/tests/test_compose_on_gaudi.sh | 3 +-- EdgeCraftRAG/ui/docker/Dockerfile.ui | 3 ++- VideoQnA/ui/docker/Dockerfile | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/AvatarChatbot/Dockerfile b/AvatarChatbot/Dockerfile index 3266bc296a..f0fa5744e7 100644 --- a/AvatarChatbot/Dockerfile +++ b/AvatarChatbot/Dockerfile @@ -32,7 +32,7 @@ COPY --from=git $HOME/GenAIComps/comps $HOME/GenAIComps/comps COPY --from=git $HOME/GenAIComps/*.* $HOME/GenAIComps/LICENSE $HOME/GenAIComps/ WORKDIR $HOME/GenAIComps -RUN pip install --no-cache-dir --upgrade pip && \ +RUN pip install --no-cache-dir --upgrade pip setuptools && \ pip install --no-cache-dir -r $HOME/GenAIComps/requirements.txt WORKDIR $HOME diff --git a/ChatQnA/tests/test_compose_on_gaudi.sh b/ChatQnA/tests/test_compose_on_gaudi.sh index 59ffbb3ded..2785995bbb 100644 --- a/ChatQnA/tests/test_compose_on_gaudi.sh +++ b/ChatQnA/tests/test_compose_on_gaudi.sh @@ -2,7 +2,7 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -set -xe +set -e IMAGE_REPO=${IMAGE_REPO:-"opea"} IMAGE_TAG=${IMAGE_TAG:-"latest"} echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" @@ -47,7 +47,6 @@ function start_services() { export NUM_CARDS=1 export INDEX_NAME="rag-redis" export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} - export HF_TOKEN=${HUGGINGFACEHUB_API_TOKEN} export host_ip=${ip_address} export JAEGER_IP=$(ip route get 8.8.8.8 | grep -oP 'src \K[^ ]+') export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=grpc://$JAEGER_IP:4317 diff --git a/EdgeCraftRAG/ui/docker/Dockerfile.ui b/EdgeCraftRAG/ui/docker/Dockerfile.ui index 3dacb35d8d..8abffc5557 100644 --- a/EdgeCraftRAG/ui/docker/Dockerfile.ui +++ b/EdgeCraftRAG/ui/docker/Dockerfile.ui @@ -15,7 +15,8 @@ RUN mkdir -p /home/user/gradio_cache ENV GRADIO_TEMP_DIR=/home/user/gradio_cache WORKDIR /home/user/ui -RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir --upgrade pip setuptools && \ + pip install --no-cache-dir -r requirements.txt USER user diff --git a/VideoQnA/ui/docker/Dockerfile b/VideoQnA/ui/docker/Dockerfile index dcd029a0b8..019999de8a 100644 --- a/VideoQnA/ui/docker/Dockerfile +++ b/VideoQnA/ui/docker/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y curl && \ rm -rf /var/lib/apt/lists/* -RUN pip install --no-cache-dir --upgrade pip && \ +RUN pip install --no-cache-dir --upgrade pip setuptools && \ pip install --no-cache-dir streamlit COPY ui.py /app/ui.py From 06d31cc67426fe4501cc743ebc9b687b21af257d Mon Sep 17 00:00:00 2001 From: minmin-intel Date: Fri, 21 Feb 2025 17:51:26 -0800 Subject: [PATCH 012/226] Update AgentQnA and DocIndexRetriever (#1564) Signed-off-by: minmin-intel Signed-off-by: Chingis Yundunov --- AgentQnA/README.md | 43 ++++------ .../intel/cpu/xeon/compose_openai.yaml | 11 ++- .../cpu/xeon/launch_agent_service_openai.sh | 4 +- .../intel/hpu/gaudi/compose.yaml | 5 +- .../hpu/gaudi/launch_agent_service_gaudi.sh | 2 +- ... step4_launch_and_validate_agent_gaudi.sh} | 42 ++-------- AgentQnA/tests/test.py | 79 ++++++++++++------- AgentQnA/tests/test_compose_on_gaudi.sh | 2 +- .../intel/cpu/xeon/compose.yaml | 6 ++ DocIndexRetriever/retrieval_tool.py | 56 ++++++++----- DocIndexRetriever/tests/test.py | 38 +++++++++ 11 files changed, 170 insertions(+), 118 deletions(-) rename AgentQnA/tests/{step4_launch_and_validate_agent_tgi.sh => step4_launch_and_validate_agent_gaudi.sh} (87%) create mode 100644 DocIndexRetriever/tests/test.py diff --git a/AgentQnA/README.md b/AgentQnA/README.md index d45b14ef55..397bd0c775 100644 --- a/AgentQnA/README.md +++ b/AgentQnA/README.md @@ -84,7 +84,7 @@ flowchart LR 3. Hierarchical multi-agents can improve performance. Expert worker agents, such as RAG agent and SQL agent, can provide high-quality output for different aspects of a complex query, and the supervisor agent can aggregate the information together to provide a comprehensive answer. If we only use one agent and provide all the tools to this single agent, it may get overwhelmed and not able to provide accurate answers. -## Deployment with docker +## Deploy with docker 1. Build agent docker image [Optional] @@ -217,13 +217,19 @@ docker build -t opea/agent:latest --build-arg https_proxy=$https_proxy --build-a ::: :::: +## Deploy AgentQnA UI + +The AgentQnA UI can be deployed locally or using Docker. + +For detailed instructions on deploying AgentQnA UI, refer to the [AgentQnA UI Guide](./ui/svelte/README.md). + ## Deploy using Helm Chart Refer to the [AgentQnA helm chart](./kubernetes/helm/README.md) for instructions on deploying AgentQnA on Kubernetes. ## Validate services -First look at logs of the agent docker containers: +1. First look at logs of the agent docker containers: ``` # worker RAG agent @@ -240,35 +246,18 @@ docker logs react-agent-endpoint You should see something like "HTTP server setup successful" if the docker containers are started successfully.

-Second, validate worker RAG agent: +2. You can use python to validate the agent system -``` -curl http://${host_ip}:9095/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{ - "messages": "Michael Jackson song Thriller" - }' -``` +```bash +# RAG worker agent +python tests/test.py --prompt "Tell me about Michael Jackson song Thriller" --agent_role "worker" --ext_port 9095 -Third, validate worker SQL agent: +# SQL agent +python tests/test.py --prompt "How many employees in company" --agent_role "worker" --ext_port 9096 +# supervisor agent: this will test a two-turn conversation +python tests/test.py --agent_role "supervisor" --ext_port 9090 ``` -curl http://${host_ip}:9096/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{ - "messages": "How many employees are in the company" - }' -``` - -Finally, validate supervisor agent: - -``` -curl http://${host_ip}:9090/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{ - "messages": "How many albums does Iron Maiden have?" - }' -``` - -## Deploy AgentQnA UI - -The AgentQnA UI can be deployed locally or using Docker. - -For detailed instructions on deploying AgentQnA UI, refer to the [AgentQnA UI Guide](./ui/svelte/README.md). ## How to register your own tools with agent diff --git a/AgentQnA/docker_compose/intel/cpu/xeon/compose_openai.yaml b/AgentQnA/docker_compose/intel/cpu/xeon/compose_openai.yaml index 09bde26bde..bbd64ceb30 100644 --- a/AgentQnA/docker_compose/intel/cpu/xeon/compose_openai.yaml +++ b/AgentQnA/docker_compose/intel/cpu/xeon/compose_openai.yaml @@ -13,6 +13,7 @@ services: environment: ip_address: ${ip_address} strategy: rag_agent + with_memory: false recursion_limit: ${recursion_limit_worker} llm_engine: openai OPENAI_API_KEY: ${OPENAI_API_KEY} @@ -35,17 +36,17 @@ services: image: opea/agent:latest container_name: sql-agent-endpoint volumes: - - ${WORKDIR}/TAG-Bench/:/home/user/TAG-Bench # SQL database + - ${WORKDIR}/GenAIExamples/AgentQnA/tests:/home/user/chinook-db # SQL database ports: - "9096:9096" ipc: host environment: ip_address: ${ip_address} strategy: sql_agent + with_memory: false db_name: ${db_name} db_path: ${db_path} use_hints: false - hints_file: /home/user/TAG-Bench/${db_name}_hints.csv recursion_limit: ${recursion_limit_worker} llm_engine: openai OPENAI_API_KEY: ${OPENAI_API_KEY} @@ -64,6 +65,7 @@ services: container_name: react-agent-endpoint depends_on: - worker-rag-agent + - worker-sql-agent volumes: - ${TOOLSET_PATH}:/home/user/tools/ ports: @@ -71,14 +73,15 @@ services: ipc: host environment: ip_address: ${ip_address} - strategy: react_langgraph + strategy: react_llama + with_memory: true recursion_limit: ${recursion_limit_supervisor} llm_engine: openai OPENAI_API_KEY: ${OPENAI_API_KEY} model: ${model} temperature: ${temperature} max_new_tokens: ${max_new_tokens} - stream: false + stream: true tools: /home/user/tools/supervisor_agent_tools.yaml require_human_feedback: false no_proxy: ${no_proxy} diff --git a/AgentQnA/docker_compose/intel/cpu/xeon/launch_agent_service_openai.sh b/AgentQnA/docker_compose/intel/cpu/xeon/launch_agent_service_openai.sh index 7b4e86a781..2455865f27 100644 --- a/AgentQnA/docker_compose/intel/cpu/xeon/launch_agent_service_openai.sh +++ b/AgentQnA/docker_compose/intel/cpu/xeon/launch_agent_service_openai.sh @@ -16,7 +16,7 @@ export WORKER_AGENT_URL="http://${ip_address}:9095/v1/chat/completions" export SQL_AGENT_URL="http://${ip_address}:9096/v1/chat/completions" export RETRIEVAL_TOOL_URL="http://${ip_address}:8889/v1/retrievaltool" export CRAG_SERVER=http://${ip_address}:8080 -export db_name=california_schools -export db_path="sqlite:////home/user/TAG-Bench/dev_folder/dev_databases/${db_name}/${db_name}.sqlite" +export db_name=Chinook +export db_path="sqlite:////home/user/chinook-db/Chinook_Sqlite.sqlite" docker compose -f compose_openai.yaml up -d diff --git a/AgentQnA/docker_compose/intel/hpu/gaudi/compose.yaml b/AgentQnA/docker_compose/intel/hpu/gaudi/compose.yaml index 4895722c93..c14d58c10b 100644 --- a/AgentQnA/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/AgentQnA/docker_compose/intel/hpu/gaudi/compose.yaml @@ -13,6 +13,7 @@ services: environment: ip_address: ${ip_address} strategy: rag_agent_llama + with_memory: false recursion_limit: ${recursion_limit_worker} llm_engine: vllm HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} @@ -43,6 +44,7 @@ services: environment: ip_address: ${ip_address} strategy: sql_agent_llama + with_memory: false db_name: ${db_name} db_path: ${db_path} use_hints: false @@ -74,6 +76,7 @@ services: environment: ip_address: ${ip_address} strategy: react_llama + with_memory: true recursion_limit: ${recursion_limit_supervisor} llm_engine: vllm HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} @@ -81,7 +84,7 @@ services: model: ${LLM_MODEL_ID} temperature: ${temperature} max_new_tokens: ${max_new_tokens} - stream: false + stream: true tools: /home/user/tools/supervisor_agent_tools.yaml require_human_feedback: false no_proxy: ${no_proxy} diff --git a/AgentQnA/docker_compose/intel/hpu/gaudi/launch_agent_service_gaudi.sh b/AgentQnA/docker_compose/intel/hpu/gaudi/launch_agent_service_gaudi.sh index fff5d53f8d..298feee3fd 100644 --- a/AgentQnA/docker_compose/intel/hpu/gaudi/launch_agent_service_gaudi.sh +++ b/AgentQnA/docker_compose/intel/hpu/gaudi/launch_agent_service_gaudi.sh @@ -14,7 +14,7 @@ export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} export HF_CACHE_DIR=${HF_CACHE_DIR} ls $HF_CACHE_DIR export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} -export LLM_MODEL_ID="meta-llama/Meta-Llama-3.1-70B-Instruct" +export LLM_MODEL_ID="meta-llama/Llama-3.3-70B-Instruct" #"meta-llama/Meta-Llama-3.1-70B-Instruct" export NUM_SHARDS=4 export LLM_ENDPOINT_URL="http://${ip_address}:8086" export temperature=0 diff --git a/AgentQnA/tests/step4_launch_and_validate_agent_tgi.sh b/AgentQnA/tests/step4_launch_and_validate_agent_gaudi.sh similarity index 87% rename from AgentQnA/tests/step4_launch_and_validate_agent_tgi.sh rename to AgentQnA/tests/step4_launch_and_validate_agent_gaudi.sh index 824f7aa855..56f017239b 100644 --- a/AgentQnA/tests/step4_launch_and_validate_agent_tgi.sh +++ b/AgentQnA/tests/step4_launch_and_validate_agent_gaudi.sh @@ -11,7 +11,7 @@ export ip_address=$(hostname -I | awk '{print $1}') export TOOLSET_PATH=$WORKPATH/tools/ export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} HF_TOKEN=${HUGGINGFACEHUB_API_TOKEN} -model="meta-llama/Meta-Llama-3.1-70B-Instruct" +model="meta-llama/Llama-3.3-70B-Instruct" #"meta-llama/Meta-Llama-3.1-70B-Instruct" export HF_CACHE_DIR=/data2/huggingface if [ ! -d "$HF_CACHE_DIR" ]; then @@ -60,23 +60,6 @@ function start_vllm_service_70B() { echo "Service started successfully" } - -function prepare_data() { - cd $WORKDIR - - echo "Downloading data..." - git clone https://github.com/TAG-Research/TAG-Bench.git - cd TAG-Bench/setup - chmod +x get_dbs.sh - ./get_dbs.sh - - echo "Split data..." - cd $WORKPATH/tests/sql_agent_test - bash run_data_split.sh - - echo "Data preparation done!" -} - function download_chinook_data(){ echo "Downloading chinook data..." cd $WORKDIR @@ -113,7 +96,7 @@ function validate_agent_service() { echo "======================Testing worker rag agent======================" export agent_port="9095" prompt="Tell me about Michael Jackson song Thriller" - local CONTENT=$(python3 $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --prompt "$prompt") + local CONTENT=$(python3 $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --prompt "$prompt" --agent_role "worker" --ext_port $agent_port) # echo $CONTENT local EXIT_CODE=$(validate "$CONTENT" "Thriller" "rag-agent-endpoint") echo $EXIT_CODE @@ -127,7 +110,7 @@ function validate_agent_service() { echo "======================Testing worker sql agent======================" export agent_port="9096" prompt="How many employees are there in the company?" - local CONTENT=$(python3 $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --prompt "$prompt") + local CONTENT=$(python3 $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --prompt "$prompt" --agent_role "worker" --ext_port $agent_port) local EXIT_CODE=$(validate "$CONTENT" "8" "sql-agent-endpoint") echo $CONTENT # echo $EXIT_CODE @@ -140,9 +123,8 @@ function validate_agent_service() { # test supervisor react agent echo "======================Testing supervisor react agent======================" export agent_port="9090" - prompt="How many albums does Iron Maiden have?" - local CONTENT=$(python3 $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --prompt "$prompt") - local EXIT_CODE=$(validate "$CONTENT" "21" "react-agent-endpoint") + local CONTENT=$(python3 $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --agent_role "supervisor" --ext_port $agent_port --stream) + local EXIT_CODE=$(validate "$CONTENT" "Iron" "react-agent-endpoint") # echo $CONTENT echo $EXIT_CODE local EXIT_CODE="${EXIT_CODE:0-1}" @@ -153,15 +135,6 @@ function validate_agent_service() { } -function remove_data() { - echo "Removing data..." - cd $WORKDIR - if [ -d "TAG-Bench" ]; then - rm -rf TAG-Bench - fi - echo "Data removed!" -} - function remove_chinook_data(){ echo "Removing chinook data..." cd $WORKDIR @@ -189,8 +162,9 @@ function main() { echo "==================== Agent service validated ====================" } -remove_data + remove_chinook_data + main -remove_data + remove_chinook_data diff --git a/AgentQnA/tests/test.py b/AgentQnA/tests/test.py index 400684ffd6..18254f16c5 100644 --- a/AgentQnA/tests/test.py +++ b/AgentQnA/tests/test.py @@ -1,34 +1,20 @@ -# Copyright (C) 2024 Intel Corporation +# Copyright (C) 2025 Intel Corporation # SPDX-License-Identifier: Apache-2.0 import argparse -import os +import json +import uuid import requests -def generate_answer_agent_api(url, prompt): - proxies = {"http": ""} - payload = { - "messages": prompt, - } - response = requests.post(url, json=payload, proxies=proxies) - answer = response.json()["text"] - return answer - - def process_request(url, query, is_stream=False): proxies = {"http": ""} - - payload = { - "messages": query, - } - + content = json.dumps(query) if query is not None else None try: - resp = requests.post(url=url, json=payload, proxies=proxies, stream=is_stream) + resp = requests.post(url=url, data=content, proxies=proxies, stream=is_stream) if not is_stream: ret = resp.json()["text"] - print(ret) else: for line in resp.iter_lines(decode_unicode=True): print(line) @@ -38,19 +24,54 @@ def process_request(url, query, is_stream=False): return ret except requests.exceptions.RequestException as e: ret = f"An error occurred:{e}" - print(ret) - return False + return None + + +def test_worker_agent(args): + url = f"http://{args.ip_addr}:{args.ext_port}/v1/chat/completions" + query = {"role": "user", "messages": args.prompt, "stream": "false"} + ret = process_request(url, query) + print("Response: ", ret) + + +def add_message_and_run(url, user_message, thread_id, stream=False): + print("User message: ", user_message) + query = {"role": "user", "messages": user_message, "thread_id": thread_id, "stream": stream} + ret = process_request(url, query, is_stream=stream) + print("Response: ", ret) + + +def test_chat_completion_multi_turn(args): + url = f"http://{args.ip_addr}:{args.ext_port}/v1/chat/completions" + thread_id = f"{uuid.uuid4()}" + + # first turn + print("===============First turn==================") + user_message = "Which artist has the most albums in the database?" + add_message_and_run(url, user_message, thread_id, stream=args.stream) + print("===============End of first turn==================") + + # second turn + print("===============Second turn==================") + user_message = "Give me a few examples of the artist's albums?" + add_message_and_run(url, user_message, thread_id, stream=args.stream) + print("===============End of second turn==================") if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("--prompt", type=str) - parser.add_argument("--stream", action="store_true") - args = parser.parse_args() + parser.add_argument("--ip_addr", type=str, default="127.0.0.1", help="endpoint ip address") + parser.add_argument("--ext_port", type=str, default="9090", help="endpoint port") + parser.add_argument("--stream", action="store_true", help="streaming mode") + parser.add_argument("--prompt", type=str, help="prompt message") + parser.add_argument("--agent_role", type=str, default="supervisor", help="supervisor or worker") + args, _ = parser.parse_known_args() - ip_address = os.getenv("ip_address", "localhost") - agent_port = os.getenv("agent_port", "9090") - url = f"http://{ip_address}:{agent_port}/v1/chat/completions" - prompt = args.prompt + print(args) - process_request(url, prompt, args.stream) + if args.agent_role == "supervisor": + test_chat_completion_multi_turn(args) + elif args.agent_role == "worker": + test_worker_agent(args) + else: + raise ValueError("Invalid agent role") diff --git a/AgentQnA/tests/test_compose_on_gaudi.sh b/AgentQnA/tests/test_compose_on_gaudi.sh index de70514ba6..ab0ce295cb 100644 --- a/AgentQnA/tests/test_compose_on_gaudi.sh +++ b/AgentQnA/tests/test_compose_on_gaudi.sh @@ -78,7 +78,7 @@ bash step3_ingest_data_and_validate_retrieval.sh echo "=================== #3 Data ingestion and validation completed====================" echo "=================== #4 Start agent and API server====================" -bash step4_launch_and_validate_agent_tgi.sh +bash step4_launch_and_validate_agent_gaudi.sh echo "=================== #4 Agent test passed ====================" echo "=================== #5 Stop agent and API server====================" diff --git a/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml b/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml index d4bfe0446f..9624df7300 100644 --- a/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml +++ b/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml @@ -13,6 +13,8 @@ services: dataprep-redis-service: image: ${REGISTRY:-opea}/dataprep:${TAG:-latest} container_name: dataprep-redis-server + # volumes: + # - $WORKDIR/GenAIExamples/DocIndexRetriever/docker_image_build/GenAIComps/comps:/home/user/comps depends_on: - redis-vector-db ports: @@ -52,6 +54,8 @@ services: embedding: image: ${REGISTRY:-opea}/embedding:${TAG:-latest} container_name: embedding-server + # volumes: + # - $WORKDIR/GenAIExamples/DocIndexRetriever/docker_image_build/GenAIComps/comps:/home/comps ports: - "6000:6000" ipc: host @@ -110,6 +114,8 @@ services: reranking: image: ${REGISTRY:-opea}/reranking:${TAG:-latest} container_name: reranking-tei-xeon-server + # volumes: + # - $WORKDIR/GenAIExamples/DocIndexRetriever/docker_image_build/GenAIComps/comps:/home/user/comps depends_on: tei-reranking-service: condition: service_healthy diff --git a/DocIndexRetriever/retrieval_tool.py b/DocIndexRetriever/retrieval_tool.py index b627f45537..99fab7b1b5 100644 --- a/DocIndexRetriever/retrieval_tool.py +++ b/DocIndexRetriever/retrieval_tool.py @@ -22,16 +22,38 @@ def align_inputs(self, inputs, cur_node, runtime_graph, llm_parameters_dict, **kwargs): - if self.services[cur_node].service_type == ServiceType.EMBEDDING: - inputs["input"] = inputs["text"] - del inputs["text"] + print(f"Inputs to {cur_node}: {inputs}") + for key, value in kwargs.items(): + print(f"{key}: {value}") return inputs def align_outputs(self, data, cur_node, inputs, runtime_graph, llm_parameters_dict, **kwargs): next_data = {} if self.services[cur_node].service_type == ServiceType.EMBEDDING: - next_data = {"text": inputs["input"], "embedding": [item["embedding"] for item in data["data"]]} + # turn into chat completion request + # next_data = {"text": inputs["input"], "embedding": [item["embedding"] for item in data["data"]]} + print("Assembing output from Embedding for next node...") + print("Inputs to Embedding: ", inputs) + print("Keyword arguments: ") + for key, value in kwargs.items(): + print(f"{key}: {value}") + + next_data = { + "input": inputs["input"], + "messages": inputs["input"], + "embedding": data, # [item["embedding"] for item in data["data"]], + "k": kwargs["k"] if "k" in kwargs else 4, + "search_type": kwargs["search_type"] if "search_type" in kwargs else "similarity", + "distance_threshold": kwargs["distance_threshold"] if "distance_threshold" in kwargs else None, + "fetch_k": kwargs["fetch_k"] if "fetch_k" in kwargs else 20, + "lambda_mult": kwargs["lambda_mult"] if "lambda_mult" in kwargs else 0.5, + "score_threshold": kwargs["score_threshold"] if "score_threshold" in kwargs else 0.2, + "top_n": kwargs["top_n"] if "top_n" in kwargs else 1, + } + + print("Output from Embedding for next node:\n", next_data) + else: next_data = data @@ -99,18 +121,6 @@ def parser_input(data, TypeClass, key): raise ValueError(f"Unknown request type: {data}") if isinstance(chat_request, ChatCompletionRequest): - retriever_parameters = RetrieverParms( - search_type=chat_request.search_type if chat_request.search_type else "similarity", - k=chat_request.k if chat_request.k else 4, - distance_threshold=chat_request.distance_threshold if chat_request.distance_threshold else None, - fetch_k=chat_request.fetch_k if chat_request.fetch_k else 20, - lambda_mult=chat_request.lambda_mult if chat_request.lambda_mult else 0.5, - score_threshold=chat_request.score_threshold if chat_request.score_threshold else 0.2, - ) - reranker_parameters = RerankerParms( - top_n=chat_request.top_n if chat_request.top_n else 1, - ) - initial_inputs = { "messages": query, "input": query, # has to be input due to embedding expects either input or text @@ -123,13 +133,21 @@ def parser_input(data, TypeClass, key): "top_n": chat_request.top_n if chat_request.top_n else 1, } + kwargs = { + "search_type": chat_request.search_type if chat_request.search_type else "similarity", + "k": chat_request.k if chat_request.k else 4, + "distance_threshold": chat_request.distance_threshold if chat_request.distance_threshold else None, + "fetch_k": chat_request.fetch_k if chat_request.fetch_k else 20, + "lambda_mult": chat_request.lambda_mult if chat_request.lambda_mult else 0.5, + "score_threshold": chat_request.score_threshold if chat_request.score_threshold else 0.2, + "top_n": chat_request.top_n if chat_request.top_n else 1, + } result_dict, runtime_graph = await self.megaservice.schedule( initial_inputs=initial_inputs, - retriever_parameters=retriever_parameters, - reranker_parameters=reranker_parameters, + **kwargs, ) else: - result_dict, runtime_graph = await self.megaservice.schedule(initial_inputs={"text": query}) + result_dict, runtime_graph = await self.megaservice.schedule(initial_inputs={"input": query}) last_node = runtime_graph.all_leaves()[-1] response = result_dict[last_node] diff --git a/DocIndexRetriever/tests/test.py b/DocIndexRetriever/tests/test.py new file mode 100644 index 0000000000..ba74827fa6 --- /dev/null +++ b/DocIndexRetriever/tests/test.py @@ -0,0 +1,38 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import os + +import requests + + +def search_knowledge_base(query: str) -> str: + """Search the knowledge base for a specific query.""" + url = os.environ.get("RETRIEVAL_TOOL_URL") + print(url) + proxies = {"http": ""} + payload = {"messages": query, "k": 5, "top_n": 2} + response = requests.post(url, json=payload, proxies=proxies) + print(response) + if "documents" in response.json(): + docs = response.json()["documents"] + context = "" + for i, doc in enumerate(docs): + context += f"Doc[{i+1}]:\n{doc}\n" + return context + elif "text" in response.json(): + return response.json()["text"] + elif "reranked_docs" in response.json(): + docs = response.json()["reranked_docs"] + context = "" + for i, doc in enumerate(docs): + context += f"Doc[{i+1}]:\n{doc}\n" + return context + else: + return "Error parsing response from the knowledge base." + + +if __name__ == "__main__": + resp = search_knowledge_base("What is OPEA?") + # resp = search_knowledge_base("Thriller") + print(resp) From 59ffc84c246f1264ee5f63498eadce7e20ca57ed Mon Sep 17 00:00:00 2001 From: Ying Hu Date: Sun, 23 Feb 2025 17:38:27 +0800 Subject: [PATCH 013/226] Update README.md of AIPC quick start (#1578) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: Chingis Yundunov --- .../docker_compose/intel/cpu/aipc/README.md | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/ChatQnA/docker_compose/intel/cpu/aipc/README.md b/ChatQnA/docker_compose/intel/cpu/aipc/README.md index 5fd253c623..5a217b1f3b 100644 --- a/ChatQnA/docker_compose/intel/cpu/aipc/README.md +++ b/ChatQnA/docker_compose/intel/cpu/aipc/README.md @@ -2,6 +2,84 @@ This document outlines the deployment process for a ChatQnA application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on AIPC. The steps include Docker image creation, container deployment via Docker Compose, and service execution to integrate microservices such as `embedding`, `retriever`, `rerank`, and `llm`. +## Quick Start: + +1. Set up the environment variables. +2. Run Docker Compose. +3. Consume the ChatQnA Service. + +### Quick Start: 1. Set up Environment Variable + +To set up environment variables for deploying ChatQnA services, follow these steps: + +```bash +mkdir ~/OPEA -p +cd ~/OPEA +git clone https://github.com/opea-project/GenAIExamples.git +cd GenAIExamples/ChatQnA/docker_compose/intel/cpu/aipc +``` + +1. Set the required environment variables: + + ```bash + export HUGGINGFACEHUB_API_TOKEN="Your_Huggingface_API_Token" + ``` + +2. If you are in a proxy environment, also set the proxy-related environment variables: + + ```bash + export https_proxy="Your_HTTPs_Proxy" + # Example: no_proxy="localhost, 127.0.0.1, 192.168.1.1" + export no_proxy=$no_proxy,chatqna-aipc-backend-server,tei-embedding-service,retriever,tei-reranking-service,redis-vector-db,dataprep-redis-service,ollama-service + ``` + +3. Set up other environment variables + + By default, llama3.2 is used for LLM serving, the default model can be changed to other LLM models. Please pick a [validated llm models](https://github.com/opea-project/GenAIComps/tree/main/comps/llms/src/text-generation#validated-llm-models) from the table. + To change the default model defined in set_env.sh, overwrite it by exporting OLLAMA_MODEL to the new model or by modifying set_env.sh. + For example, change to using the following model. + + ```bash + export OLLAMA_MODEL="deepseek-r1:8b" + ``` + + to use the [DeepSeek-R1-Distill-Llama-8B model](https://ollama.com/library/deepseek-r1:8b) + + ```bash + source ./set_env.sh + ``` + +### Quick Start: 2. Run Docker Compose + +```bash + docker compose up -d +``` + +It will take several minutes to automatically download the docker images + +NB: You should build docker image from source by yourself if: + +- You are developing off the git main branch (as the container's ports in the repo may be different from the published docker image). +- You can't download the docker image. +- You want to use a specific version of Docker image. + +Please refer to ['Build Docker Images'](#🚀-build-docker-images) in below. + +### Quick Start:3. Consume the ChatQnA Service + +Once the services are up, open the following URL from your browser: http://{host_ip}:80. +Enter Prompt like What is deep learning? + +Or if you prefer to try only on the localhost machine, then try + +```bash +curl http://${host_ip}:8888/v1/chatqna \ + -H "Content-Type: application/json" \ + -d '{ + "messages": "What is deep learning?" + }' +``` + ## 🚀 Build Docker Images First of all, you need to build Docker Images locally and install the python package of it. From 4bd9c1a256ef86a96f19c7a96d892165930ba9e2 Mon Sep 17 00:00:00 2001 From: Eero Tamminen Date: Tue, 25 Feb 2025 06:45:21 +0200 Subject: [PATCH 014/226] Fix "OpenAI" & "response" spelling (#1561) Signed-off-by: Chingis Yundunov --- AgentQnA/docker_compose/intel/cpu/xeon/README.md | 2 +- ChatQnA/chatqna.py | 2 +- GraphRAG/graphrag.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/AgentQnA/docker_compose/intel/cpu/xeon/README.md b/AgentQnA/docker_compose/intel/cpu/xeon/README.md index dde535f2ae..a2abfc7ce9 100644 --- a/AgentQnA/docker_compose/intel/cpu/xeon/README.md +++ b/AgentQnA/docker_compose/intel/cpu/xeon/README.md @@ -60,7 +60,7 @@ This example showcases a hierarchical multi-agent system for question-answering ``` 6. Launch multi-agent system - The configurations of the supervisor agent and the worker agents are defined in the docker-compose yaml file. We currently use openAI GPT-4o-mini as LLM. + The configurations of the supervisor agent and the worker agents are defined in the docker-compose yaml file. We currently use OpenAI GPT-4o-mini as LLM. ``` cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/cpu/xeon diff --git a/ChatQnA/chatqna.py b/ChatQnA/chatqna.py index 104c6fdb13..afb9706cb2 100644 --- a/ChatQnA/chatqna.py +++ b/ChatQnA/chatqna.py @@ -167,7 +167,7 @@ def align_outputs(self, data, cur_node, inputs, runtime_graph, llm_parameters_di def align_generator(self, gen, **kwargs): - # openai reaponse format + # OpenAI response format # b'data:{"id":"","object":"text_completion","created":1725530204,"model":"meta-llama/Meta-Llama-3-8B-Instruct","system_fingerprint":"2.0.1-native","choices":[{"index":0,"delta":{"role":"assistant","content":"?"},"logprobs":null,"finish_reason":null}]}\n\n' for line in gen: line = line.decode("utf-8") diff --git a/GraphRAG/graphrag.py b/GraphRAG/graphrag.py index 4eafaab244..6433e410ad 100644 --- a/GraphRAG/graphrag.py +++ b/GraphRAG/graphrag.py @@ -110,7 +110,7 @@ def align_outputs(self, data, cur_node, inputs, runtime_graph, llm_parameters_di def align_generator(self, gen, **kwargs): - # openai reaponse format + # OpenAI response format # b'data:{"id":"","object":"text_completion","created":1725530204,"model":"meta-llama/Meta-Llama-3-8B-Instruct","system_fingerprint":"2.0.1-native","choices":[{"index":0,"delta":{"role":"assistant","content":"?"},"logprobs":null,"finish_reason":null}]}\n\n' print("generator in align generator:\n", gen) for line in gen: From 2abf73842ab5e983ccb265c028612553b85cabc3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Feb 2025 14:32:03 +0800 Subject: [PATCH 015/226] Bump gradio from 5.5.0 to 5.11.0 in /DocSum/ui/gradio (#1576) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Liang Lv Signed-off-by: Chingis Yundunov --- DocSum/ui/gradio/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DocSum/ui/gradio/requirements.txt b/DocSum/ui/gradio/requirements.txt index 9086603d04..5824f07218 100644 --- a/DocSum/ui/gradio/requirements.txt +++ b/DocSum/ui/gradio/requirements.txt @@ -1,5 +1,5 @@ docx2txt -gradio==5.5.0 +gradio==5.11.0 langchain_community moviepy==1.0.3 numpy==1.26.4 From 8e8d296965f3aac88ea948d93e2bc4b3d4e51089 Mon Sep 17 00:00:00 2001 From: Chingis Yundunov Date: Thu, 13 Feb 2025 10:02:03 +0700 Subject: [PATCH 016/226] DocSum - add files for deploy app with ROCm vLLM Signed-off-by: Chingis Yundunov Signed-off-by: Chingis Yundunov --- DocSum/Dockerfile-vllm-rocm | 18 ++ .../amd/gpu/rocm-vllm/README.md | 175 ++++++++++++ .../amd/gpu/rocm-vllm/compose.yaml | 107 ++++++++ .../amd/gpu/rocm-vllm/set_env.sh | 16 ++ DocSum/docker_image_build/build.yaml | 9 + DocSum/tests/test_compose_on_rocm_vllm.sh | 249 ++++++++++++++++++ 6 files changed, 574 insertions(+) create mode 100644 DocSum/Dockerfile-vllm-rocm create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh create mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm new file mode 100644 index 0000000000..f0e8a8743a --- /dev/null +++ b/DocSum/Dockerfile-vllm-rocm @@ -0,0 +1,18 @@ +FROM rocm/vllm-dev:main + +# Set the working directory +WORKDIR /workspace + +# Copy the api_server.py into the image +ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py + +# Expose the port used by the API server +EXPOSE 8011 + +# Set environment variables +ENV HUGGINGFACE_HUB_CACHE=/workspace +ENV WILM_USE_TRITON_FLASH_ATTENTION=0 +ENV PYTORCH_JIT=0 + +# Set the entrypoint to the api_server.py script +ENTRYPOINT ["python3", "/workspace/api_server.py"] diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md new file mode 100644 index 0000000000..4d41a5cd31 --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md @@ -0,0 +1,175 @@ +# Build and deploy DocSum Application on AMD GPU (ROCm) + +## Build images + +## 🚀 Build Docker Images + +First of all, you need to build Docker Images locally and install the python package of it. + +### 1. Build LLM Image + +```bash +git clone https://github.com/opea-project/GenAIComps.git +cd GenAIComps +docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile . +``` + +Then run the command `docker images`, you will have the following four Docker Images: + +### 2. Build MegaService Docker Image + +To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command: + +```bash +git clone https://github.com/opea-project/GenAIExamples +cd GenAIExamples/DocSum/ +docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . +``` + +### 3. Build UI Docker Image + +Build the frontend Docker image via below command: + +```bash +cd GenAIExamples/DocSum/ui +docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . +``` + +Then run the command `docker images`, you will have the following Docker Images: + +1. `opea/llm-docsum-tgi:latest` +2. `opea/docsum:latest` +3. `opea/docsum-ui:latest` + +### 4. Build React UI Docker Image + +Build the frontend Docker image via below command: + +```bash +cd GenAIExamples/DocSum/ui +export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum" +docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react . + +docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . +``` + +Then run the command `docker images`, you will have the following Docker Images: + +1. `opea/llm-docsum-tgi:latest` +2. `opea/docsum:latest` +3. `opea/docsum-ui:latest` +4. `opea/docsum-react-ui:latest` + +## 🚀 Start Microservices and MegaService + +### Required Models + +Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model. +For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable. + +### Setup Environment Variables + +Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below. + +```bash +export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm" +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export HOST_IP=${host_ip} +export DOCSUM_TGI_SERVICE_PORT="18882" +export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} +export DOCSUM_LLM_SERVER_PORT="8008" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export DocSum_COMPONENT_NAME="OpeaDocSumTgi" +``` + +Note: Please replace with `host_ip` with your external IP address, do not use localhost. + +Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered, where is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) + +Example for set isolation for 1 GPU + +``` + - /dev/dri/card0:/dev/dri/card0 + - /dev/dri/renderD128:/dev/dri/renderD128 +``` + +Example for set isolation for 2 GPUs + +``` + - /dev/dri/card0:/dev/dri/card0 + - /dev/dri/renderD128:/dev/dri/renderD128 + - /dev/dri/card1:/dev/dri/card1 + - /dev/dri/renderD129:/dev/dri/renderD129 +``` + +Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) + +### Start Microservice Docker Containers + +```bash +cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm +docker compose up -d +``` + +### Validate Microservices + +1. TGI Service + + ```bash + curl http://${host_ip}:8008/generate \ + -X POST \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \ + -H 'Content-Type: application/json' + ``` + +2. LLM Microservice + + ```bash + curl http://${host_ip}:9000/v1/docsum \ + -X POST \ + -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \ + -H 'Content-Type: application/json' + ``` + +3. MegaService + + ```bash + curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{ + "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false + }' + ``` + +## 🚀 Launch the Svelte UI + +Open this URL `http://{host_ip}:5173` in your browser to access the frontend. + +![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b) + +Here is an example for summarizing a article. + +![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55) + +## 🚀 Launch the React UI (Optional) + +To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below: + +```yaml +docsum-rocm-react-ui-server: + image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest} + container_name: docsum-rocm-react-ui-server + depends_on: + - docsum-rocm-backend-server + ports: + - "5174:80" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT} +``` + +Open this URL `http://{host_ip}:5175` in your browser to access the frontend. + +![project-screenshot](../../../../assets/img/docsum-ui-react.png) diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml new file mode 100644 index 0000000000..037aa06395 --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml @@ -0,0 +1,107 @@ +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +services: + docsum-vllm-service: + image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} + container_name: docsum-vllm-service + ports: + - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011" + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + WILM_USE_TRITON_FLASH_ATTENTION: 0 + PYTORCH_JIT: 0 + volumes: + - "./data:/data" + shm_size: 20G + devices: + - /dev/kfd:/dev/kfd + - /dev/dri/:/dev/dri/ + cap_add: + - SYS_PTRACE + group_add: + - video + security_opt: + - seccomp:unconfined + - apparmor=unconfined + command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\"" + ipc: host + + docsum-llm-server: + image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} + container_name: docsum-llm-server + depends_on: + - docsum-vllm-service + ports: + - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000" + ipc: host + cap_add: + - SYS_PTRACE + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}" + HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID} + LOGFLAG: ${DOCSUM_LOGFLAG:-False} + MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS} + MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS} + restart: unless-stopped + + whisper-service: + image: ${REGISTRY:-opea}/whisper:${TAG:-latest} + container_name: whisper-service + ports: + - "${DOCSUM_WHISPER_PORT:-7066}:7066" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + restart: unless-stopped + + docsum-backend-server: + image: ${REGISTRY:-opea}/docsum:${TAG:-latest} + container_name: docsum-backend-server + depends_on: + - docsum-tgi-service + - docsum-llm-server + ports: + - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888" + environment: + no_proxy: ${no_proxy} + https_proxy: ${https_proxy} + http_proxy: ${http_proxy} + MEGA_SERVICE_HOST_IP: ${HOST_IP} + LLM_SERVICE_HOST_IP: ${HOST_IP} + ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP} + ipc: host + restart: always + + docsum-gradio-ui: + image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest} + container_name: docsum-ui-server + depends_on: + - docsum-backend-server + ports: + - "${DOCSUM_FRONTEND_PORT:-5173}:5173" + environment: + no_proxy: ${no_proxy} + https_proxy: ${https_proxy} + http_proxy: ${http_proxy} + BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} + DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} + ipc: host + restart: always + +networks: + default: + driver: bridge diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh new file mode 100644 index 0000000000..43e71e0fbf --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +export HOST_IP="" +export DOCSUM_MAX_INPUT_TOKENS=2048 +export DOCSUM_MAX_TOTAL_TOKENS=4096 +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export DOCSUM_VLLM_SERVICE_PORT="8008" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN="" +export DOCSUM_LLM_SERVER_PORT="9000" +export DOCSUM_WHISPER_PORT="7066" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml index 095fd28c93..dc0d546189 100644 --- a/DocSum/docker_image_build/build.yaml +++ b/DocSum/docker_image_build/build.yaml @@ -47,3 +47,12 @@ services: dockerfile: comps/llms/src/doc-summarization/Dockerfile extends: docsum image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} + vllm_rocm: + build: + args: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + no_proxy: ${no_proxy} + context: ../ + dockerfile: ./Dockerfile-vllm-rocm + image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh new file mode 100644 index 0000000000..d0919a019a --- /dev/null +++ b/DocSum/tests/test_compose_on_rocm_vllm.sh @@ -0,0 +1,249 @@ +#!/bin/bash +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +set -xe +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') +export MAX_INPUT_TOKENS=1024 +export MAX_TOTAL_TOKENS=2048 +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export HOST_IP=${ip_address} +export DOCSUM_VLLM_SERVICE_PORT="8008" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} +export DOCSUM_LLM_SERVER_PORT="9000" +export DOCSUM_WHISPER_PORT="7066" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export MEGA_SERVICE_HOST_IP=${HOST_IP} +export LLM_SERVICE_HOST_IP=${HOST_IP} +export ASR_SERVICE_HOST_IP=${HOST_IP} +export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" + +function build_docker_images() { + opea_branch=${opea_branch:-"main"} + # If the opea_branch isn't main, replace the git clone branch in Dockerfile. + if [[ "${opea_branch}" != "main" ]]; then + cd $WORKPATH + OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" + NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" + find . -type f -name "Dockerfile*" | while read -r file; do + echo "Processing file: $file" + sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" + done + fi + + cd $WORKPATH/docker_image_build + git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + + echo "Build all the images with --no-cache, check docker_image_build.log for details..." + service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper" + docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log + + docker images && sleep 1s +} + +function start_services() { + cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm + sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env + # Start Docker Containers + docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log + sleep 1m +} + +function validate_services() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + + echo "===========================================" + + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "EXPECTED_RESULT==> $EXPECTED_RESULT" + echo "CONTENT==> $CONTENT" + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +get_base64_str() { + local file_name=$1 + base64 -w 0 "$file_name" +} + +# Function to generate input data for testing based on the document type +input_data_for_test() { + local document_type=$1 + case $document_type in + ("text") + echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are." + ;; + ("audio") + get_base64_str "$WORKPATH/tests/data/test.wav" + ;; + ("video") + get_base64_str "$WORKPATH/tests/data/test.mp4" + ;; + (*) + echo "Invalid document type" >&2 + exit 1 + ;; + esac +} + +function validate_microservices() { + # Check if the microservices are running correctly. + + # whisper microservice + ulimit -s 65536 + validate_services \ + "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \ + '{"asr_result":"well"}' \ + "whisper-service" \ + "whisper-service" \ + "{\"audio\": \"$(input_data_for_test "audio")\"}" + + # vLLM service + validate_services \ + "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \ + "generated_text" \ + "docsum-vllm-service" \ + "docsum-vllm-service" \ + '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}' + + # llm microservice + validate_services \ + "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \ + "text" \ + "docsum-llm-server" \ + "docsum-llm-server" \ + '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' + +} + +function validate_megaservice() { + local SERVICE_NAME="docsum-backend-server" + local DOCKER_NAME="docsum-backend-server" + local EXPECTED_RESULT="[DONE]" + local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." + local URL="${host_ip}:8888/v1/docsum" + local DATA_TYPE="type=text" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL") + + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +function validate_megaservice_json() { + # Curl the Mega Service + echo "" + echo ">>> Checking text data with Content-Type: application/json" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' + + echo ">>> Checking audio data" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + "{\"type\": \"audio\", \"messages\": \"$(input_data_for_test "audio")\"}" + + echo ">>> Checking video data" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + "{\"type\": \"video\", \"messages\": \"$(input_data_for_test "video")\"}" + +} + +function stop_docker() { + cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/ + docker compose stop && docker compose rm -f +} + +function main() { + echo "===========================================" + echo ">>>> Stopping any running Docker containers..." + stop_docker + + echo "===========================================" + if [[ "$IMAGE_REPO" == "opea" ]]; then + echo ">>>> Building Docker images..." + build_docker_images + fi + + echo "===========================================" + echo ">>>> Starting Docker services..." + start_services + + echo "===========================================" + echo ">>>> Validating microservices..." + validate_microservices + + echo "===========================================" + echo ">>>> Validating megaservice..." + validate_megaservice + echo ">>>> Validating validate_megaservice_json..." + validate_megaservice_json + + echo "===========================================" + echo ">>>> Stopping Docker containers..." + stop_docker + + echo "===========================================" + echo ">>>> Pruning Docker system..." + echo y | docker system prune + echo ">>>> Docker system pruned successfully." + echo "===========================================" +} + +main From 9aba6d05c9b1a6f1bd7f332167171d79373e39ea Mon Sep 17 00:00:00 2001 From: Chingis Yundunov Date: Thu, 13 Feb 2025 10:07:05 +0700 Subject: [PATCH 017/226] DocSum - fix main Signed-off-by: Chingis Yundunov Signed-off-by: Chingis Yundunov --- DocSum/Dockerfile-vllm-rocm | 18 -- .../amd/gpu/rocm-vllm/README.md | 175 ------------ .../amd/gpu/rocm-vllm/compose.yaml | 107 -------- .../amd/gpu/rocm-vllm/set_env.sh | 16 -- DocSum/docker_image_build/build.yaml | 9 - DocSum/tests/test_compose_on_rocm_vllm.sh | 249 ------------------ 6 files changed, 574 deletions(-) delete mode 100644 DocSum/Dockerfile-vllm-rocm delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh delete mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm deleted file mode 100644 index f0e8a8743a..0000000000 --- a/DocSum/Dockerfile-vllm-rocm +++ /dev/null @@ -1,18 +0,0 @@ -FROM rocm/vllm-dev:main - -# Set the working directory -WORKDIR /workspace - -# Copy the api_server.py into the image -ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py - -# Expose the port used by the API server -EXPOSE 8011 - -# Set environment variables -ENV HUGGINGFACE_HUB_CACHE=/workspace -ENV WILM_USE_TRITON_FLASH_ATTENTION=0 -ENV PYTORCH_JIT=0 - -# Set the entrypoint to the api_server.py script -ENTRYPOINT ["python3", "/workspace/api_server.py"] diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md deleted file mode 100644 index 4d41a5cd31..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md +++ /dev/null @@ -1,175 +0,0 @@ -# Build and deploy DocSum Application on AMD GPU (ROCm) - -## Build images - -## 🚀 Build Docker Images - -First of all, you need to build Docker Images locally and install the python package of it. - -### 1. Build LLM Image - -```bash -git clone https://github.com/opea-project/GenAIComps.git -cd GenAIComps -docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile . -``` - -Then run the command `docker images`, you will have the following four Docker Images: - -### 2. Build MegaService Docker Image - -To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command: - -```bash -git clone https://github.com/opea-project/GenAIExamples -cd GenAIExamples/DocSum/ -docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . -``` - -### 3. Build UI Docker Image - -Build the frontend Docker image via below command: - -```bash -cd GenAIExamples/DocSum/ui -docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . -``` - -Then run the command `docker images`, you will have the following Docker Images: - -1. `opea/llm-docsum-tgi:latest` -2. `opea/docsum:latest` -3. `opea/docsum-ui:latest` - -### 4. Build React UI Docker Image - -Build the frontend Docker image via below command: - -```bash -cd GenAIExamples/DocSum/ui -export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum" -docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react . - -docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . -``` - -Then run the command `docker images`, you will have the following Docker Images: - -1. `opea/llm-docsum-tgi:latest` -2. `opea/docsum:latest` -3. `opea/docsum-ui:latest` -4. `opea/docsum-react-ui:latest` - -## 🚀 Start Microservices and MegaService - -### Required Models - -Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model. -For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable. - -### Setup Environment Variables - -Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below. - -```bash -export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm" -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export HOST_IP=${host_ip} -export DOCSUM_TGI_SERVICE_PORT="18882" -export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} -export DOCSUM_LLM_SERVER_PORT="8008" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export DocSum_COMPONENT_NAME="OpeaDocSumTgi" -``` - -Note: Please replace with `host_ip` with your external IP address, do not use localhost. - -Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered, where is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) - -Example for set isolation for 1 GPU - -``` - - /dev/dri/card0:/dev/dri/card0 - - /dev/dri/renderD128:/dev/dri/renderD128 -``` - -Example for set isolation for 2 GPUs - -``` - - /dev/dri/card0:/dev/dri/card0 - - /dev/dri/renderD128:/dev/dri/renderD128 - - /dev/dri/card1:/dev/dri/card1 - - /dev/dri/renderD129:/dev/dri/renderD129 -``` - -Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) - -### Start Microservice Docker Containers - -```bash -cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm -docker compose up -d -``` - -### Validate Microservices - -1. TGI Service - - ```bash - curl http://${host_ip}:8008/generate \ - -X POST \ - -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \ - -H 'Content-Type: application/json' - ``` - -2. LLM Microservice - - ```bash - curl http://${host_ip}:9000/v1/docsum \ - -X POST \ - -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \ - -H 'Content-Type: application/json' - ``` - -3. MegaService - - ```bash - curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{ - "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false - }' - ``` - -## 🚀 Launch the Svelte UI - -Open this URL `http://{host_ip}:5173` in your browser to access the frontend. - -![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b) - -Here is an example for summarizing a article. - -![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55) - -## 🚀 Launch the React UI (Optional) - -To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below: - -```yaml -docsum-rocm-react-ui-server: - image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest} - container_name: docsum-rocm-react-ui-server - depends_on: - - docsum-rocm-backend-server - ports: - - "5174:80" - environment: - - no_proxy=${no_proxy} - - https_proxy=${https_proxy} - - http_proxy=${http_proxy} - - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT} -``` - -Open this URL `http://{host_ip}:5175` in your browser to access the frontend. - -![project-screenshot](../../../../assets/img/docsum-ui-react.png) diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml deleted file mode 100644 index 037aa06395..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -services: - docsum-vllm-service: - image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} - container_name: docsum-vllm-service - ports: - - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011" - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_HUB_DISABLE_PROGRESS_BARS: 1 - HF_HUB_ENABLE_HF_TRANSFER: 0 - WILM_USE_TRITON_FLASH_ATTENTION: 0 - PYTORCH_JIT: 0 - volumes: - - "./data:/data" - shm_size: 20G - devices: - - /dev/kfd:/dev/kfd - - /dev/dri/:/dev/dri/ - cap_add: - - SYS_PTRACE - group_add: - - video - security_opt: - - seccomp:unconfined - - apparmor=unconfined - command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\"" - ipc: host - - docsum-llm-server: - image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} - container_name: docsum-llm-server - depends_on: - - docsum-vllm-service - ports: - - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000" - ipc: host - cap_add: - - SYS_PTRACE - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}" - HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID} - LOGFLAG: ${DOCSUM_LOGFLAG:-False} - MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS} - MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS} - restart: unless-stopped - - whisper-service: - image: ${REGISTRY:-opea}/whisper:${TAG:-latest} - container_name: whisper-service - ports: - - "${DOCSUM_WHISPER_PORT:-7066}:7066" - ipc: host - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - restart: unless-stopped - - docsum-backend-server: - image: ${REGISTRY:-opea}/docsum:${TAG:-latest} - container_name: docsum-backend-server - depends_on: - - docsum-tgi-service - - docsum-llm-server - ports: - - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888" - environment: - no_proxy: ${no_proxy} - https_proxy: ${https_proxy} - http_proxy: ${http_proxy} - MEGA_SERVICE_HOST_IP: ${HOST_IP} - LLM_SERVICE_HOST_IP: ${HOST_IP} - ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP} - ipc: host - restart: always - - docsum-gradio-ui: - image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest} - container_name: docsum-ui-server - depends_on: - - docsum-backend-server - ports: - - "${DOCSUM_FRONTEND_PORT:-5173}:5173" - environment: - no_proxy: ${no_proxy} - https_proxy: ${https_proxy} - http_proxy: ${http_proxy} - BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} - DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} - ipc: host - restart: always - -networks: - default: - driver: bridge diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh deleted file mode 100644 index 43e71e0fbf..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -export HOST_IP="" -export DOCSUM_MAX_INPUT_TOKENS=2048 -export DOCSUM_MAX_TOTAL_TOKENS=4096 -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export DOCSUM_VLLM_SERVICE_PORT="8008" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN="" -export DOCSUM_LLM_SERVER_PORT="9000" -export DOCSUM_WHISPER_PORT="7066" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml index dc0d546189..095fd28c93 100644 --- a/DocSum/docker_image_build/build.yaml +++ b/DocSum/docker_image_build/build.yaml @@ -47,12 +47,3 @@ services: dockerfile: comps/llms/src/doc-summarization/Dockerfile extends: docsum image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} - vllm_rocm: - build: - args: - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - no_proxy: ${no_proxy} - context: ../ - dockerfile: ./Dockerfile-vllm-rocm - image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh deleted file mode 100644 index d0919a019a..0000000000 --- a/DocSum/tests/test_compose_on_rocm_vllm.sh +++ /dev/null @@ -1,249 +0,0 @@ -#!/bin/bash -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -set -xe -IMAGE_REPO=${IMAGE_REPO:-"opea"} -IMAGE_TAG=${IMAGE_TAG:-"latest"} -echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" -echo "TAG=IMAGE_TAG=${IMAGE_TAG}" - -WORKPATH=$(dirname "$PWD") -LOG_PATH="$WORKPATH/tests" -ip_address=$(hostname -I | awk '{print $1}') -export MAX_INPUT_TOKENS=1024 -export MAX_TOTAL_TOKENS=2048 -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export HOST_IP=${ip_address} -export DOCSUM_VLLM_SERVICE_PORT="8008" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} -export DOCSUM_LLM_SERVER_PORT="9000" -export DOCSUM_WHISPER_PORT="7066" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export MEGA_SERVICE_HOST_IP=${HOST_IP} -export LLM_SERVICE_HOST_IP=${HOST_IP} -export ASR_SERVICE_HOST_IP=${HOST_IP} -export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" - -function build_docker_images() { - opea_branch=${opea_branch:-"main"} - # If the opea_branch isn't main, replace the git clone branch in Dockerfile. - if [[ "${opea_branch}" != "main" ]]; then - cd $WORKPATH - OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" - NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" - find . -type f -name "Dockerfile*" | while read -r file; do - echo "Processing file: $file" - sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" - done - fi - - cd $WORKPATH/docker_image_build - git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git - - echo "Build all the images with --no-cache, check docker_image_build.log for details..." - service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper" - docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log - - docker images && sleep 1s -} - -function start_services() { - cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm - sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env - # Start Docker Containers - docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log - sleep 1m -} - -function validate_services() { - local URL="$1" - local EXPECTED_RESULT="$2" - local SERVICE_NAME="$3" - local DOCKER_NAME="$4" - local INPUT_DATA="$5" - - local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") - - echo "===========================================" - - if [ "$HTTP_STATUS" -eq 200 ]; then - echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." - - local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) - - if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then - echo "[ $SERVICE_NAME ] Content is as expected." - else - echo "EXPECTED_RESULT==> $EXPECTED_RESULT" - echo "CONTENT==> $CONTENT" - echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - - fi - else - echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - sleep 1s -} - -get_base64_str() { - local file_name=$1 - base64 -w 0 "$file_name" -} - -# Function to generate input data for testing based on the document type -input_data_for_test() { - local document_type=$1 - case $document_type in - ("text") - echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are." - ;; - ("audio") - get_base64_str "$WORKPATH/tests/data/test.wav" - ;; - ("video") - get_base64_str "$WORKPATH/tests/data/test.mp4" - ;; - (*) - echo "Invalid document type" >&2 - exit 1 - ;; - esac -} - -function validate_microservices() { - # Check if the microservices are running correctly. - - # whisper microservice - ulimit -s 65536 - validate_services \ - "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \ - '{"asr_result":"well"}' \ - "whisper-service" \ - "whisper-service" \ - "{\"audio\": \"$(input_data_for_test "audio")\"}" - - # vLLM service - validate_services \ - "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \ - "generated_text" \ - "docsum-vllm-service" \ - "docsum-vllm-service" \ - '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}' - - # llm microservice - validate_services \ - "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \ - "text" \ - "docsum-llm-server" \ - "docsum-llm-server" \ - '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' - -} - -function validate_megaservice() { - local SERVICE_NAME="docsum-backend-server" - local DOCKER_NAME="docsum-backend-server" - local EXPECTED_RESULT="[DONE]" - local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." - local URL="${host_ip}:8888/v1/docsum" - local DATA_TYPE="type=text" - - local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL") - - if [ "$HTTP_STATUS" -eq 200 ]; then - echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." - - local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) - - if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then - echo "[ $SERVICE_NAME ] Content is as expected." - else - echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - else - echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - sleep 1s -} - -function validate_megaservice_json() { - # Curl the Mega Service - echo "" - echo ">>> Checking text data with Content-Type: application/json" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' - - echo ">>> Checking audio data" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - "{\"type\": \"audio\", \"messages\": \"$(input_data_for_test "audio")\"}" - - echo ">>> Checking video data" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - "{\"type\": \"video\", \"messages\": \"$(input_data_for_test "video")\"}" - -} - -function stop_docker() { - cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/ - docker compose stop && docker compose rm -f -} - -function main() { - echo "===========================================" - echo ">>>> Stopping any running Docker containers..." - stop_docker - - echo "===========================================" - if [[ "$IMAGE_REPO" == "opea" ]]; then - echo ">>>> Building Docker images..." - build_docker_images - fi - - echo "===========================================" - echo ">>>> Starting Docker services..." - start_services - - echo "===========================================" - echo ">>>> Validating microservices..." - validate_microservices - - echo "===========================================" - echo ">>>> Validating megaservice..." - validate_megaservice - echo ">>>> Validating validate_megaservice_json..." - validate_megaservice_json - - echo "===========================================" - echo ">>>> Stopping Docker containers..." - stop_docker - - echo "===========================================" - echo ">>>> Pruning Docker system..." - echo y | docker system prune - echo ">>>> Docker system pruned successfully." - echo "===========================================" -} - -main From 24f886f4057c9739c4bef3d655a159608420d8cd Mon Sep 17 00:00:00 2001 From: Chingis Yundunov Date: Thu, 13 Feb 2025 10:02:03 +0700 Subject: [PATCH 018/226] DocSum - add files for deploy app with ROCm vLLM Signed-off-by: Chingis Yundunov Signed-off-by: Chingis Yundunov --- DocSum/Dockerfile-vllm-rocm | 18 ++ .../amd/gpu/rocm-vllm/README.md | 175 ++++++++++++ .../amd/gpu/rocm-vllm/compose.yaml | 107 ++++++++ .../amd/gpu/rocm-vllm/set_env.sh | 16 ++ DocSum/docker_image_build/build.yaml | 9 + DocSum/tests/test_compose_on_rocm_vllm.sh | 249 ++++++++++++++++++ 6 files changed, 574 insertions(+) create mode 100644 DocSum/Dockerfile-vllm-rocm create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh create mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm new file mode 100644 index 0000000000..f0e8a8743a --- /dev/null +++ b/DocSum/Dockerfile-vllm-rocm @@ -0,0 +1,18 @@ +FROM rocm/vllm-dev:main + +# Set the working directory +WORKDIR /workspace + +# Copy the api_server.py into the image +ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py + +# Expose the port used by the API server +EXPOSE 8011 + +# Set environment variables +ENV HUGGINGFACE_HUB_CACHE=/workspace +ENV WILM_USE_TRITON_FLASH_ATTENTION=0 +ENV PYTORCH_JIT=0 + +# Set the entrypoint to the api_server.py script +ENTRYPOINT ["python3", "/workspace/api_server.py"] diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md new file mode 100644 index 0000000000..4d41a5cd31 --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md @@ -0,0 +1,175 @@ +# Build and deploy DocSum Application on AMD GPU (ROCm) + +## Build images + +## 🚀 Build Docker Images + +First of all, you need to build Docker Images locally and install the python package of it. + +### 1. Build LLM Image + +```bash +git clone https://github.com/opea-project/GenAIComps.git +cd GenAIComps +docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile . +``` + +Then run the command `docker images`, you will have the following four Docker Images: + +### 2. Build MegaService Docker Image + +To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command: + +```bash +git clone https://github.com/opea-project/GenAIExamples +cd GenAIExamples/DocSum/ +docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . +``` + +### 3. Build UI Docker Image + +Build the frontend Docker image via below command: + +```bash +cd GenAIExamples/DocSum/ui +docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . +``` + +Then run the command `docker images`, you will have the following Docker Images: + +1. `opea/llm-docsum-tgi:latest` +2. `opea/docsum:latest` +3. `opea/docsum-ui:latest` + +### 4. Build React UI Docker Image + +Build the frontend Docker image via below command: + +```bash +cd GenAIExamples/DocSum/ui +export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum" +docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react . + +docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . +``` + +Then run the command `docker images`, you will have the following Docker Images: + +1. `opea/llm-docsum-tgi:latest` +2. `opea/docsum:latest` +3. `opea/docsum-ui:latest` +4. `opea/docsum-react-ui:latest` + +## 🚀 Start Microservices and MegaService + +### Required Models + +Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model. +For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable. + +### Setup Environment Variables + +Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below. + +```bash +export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm" +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export HOST_IP=${host_ip} +export DOCSUM_TGI_SERVICE_PORT="18882" +export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} +export DOCSUM_LLM_SERVER_PORT="8008" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export DocSum_COMPONENT_NAME="OpeaDocSumTgi" +``` + +Note: Please replace with `host_ip` with your external IP address, do not use localhost. + +Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered, where is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) + +Example for set isolation for 1 GPU + +``` + - /dev/dri/card0:/dev/dri/card0 + - /dev/dri/renderD128:/dev/dri/renderD128 +``` + +Example for set isolation for 2 GPUs + +``` + - /dev/dri/card0:/dev/dri/card0 + - /dev/dri/renderD128:/dev/dri/renderD128 + - /dev/dri/card1:/dev/dri/card1 + - /dev/dri/renderD129:/dev/dri/renderD129 +``` + +Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) + +### Start Microservice Docker Containers + +```bash +cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm +docker compose up -d +``` + +### Validate Microservices + +1. TGI Service + + ```bash + curl http://${host_ip}:8008/generate \ + -X POST \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \ + -H 'Content-Type: application/json' + ``` + +2. LLM Microservice + + ```bash + curl http://${host_ip}:9000/v1/docsum \ + -X POST \ + -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \ + -H 'Content-Type: application/json' + ``` + +3. MegaService + + ```bash + curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{ + "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false + }' + ``` + +## 🚀 Launch the Svelte UI + +Open this URL `http://{host_ip}:5173` in your browser to access the frontend. + +![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b) + +Here is an example for summarizing a article. + +![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55) + +## 🚀 Launch the React UI (Optional) + +To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below: + +```yaml +docsum-rocm-react-ui-server: + image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest} + container_name: docsum-rocm-react-ui-server + depends_on: + - docsum-rocm-backend-server + ports: + - "5174:80" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT} +``` + +Open this URL `http://{host_ip}:5175` in your browser to access the frontend. + +![project-screenshot](../../../../assets/img/docsum-ui-react.png) diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml new file mode 100644 index 0000000000..037aa06395 --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml @@ -0,0 +1,107 @@ +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +services: + docsum-vllm-service: + image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} + container_name: docsum-vllm-service + ports: + - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011" + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + WILM_USE_TRITON_FLASH_ATTENTION: 0 + PYTORCH_JIT: 0 + volumes: + - "./data:/data" + shm_size: 20G + devices: + - /dev/kfd:/dev/kfd + - /dev/dri/:/dev/dri/ + cap_add: + - SYS_PTRACE + group_add: + - video + security_opt: + - seccomp:unconfined + - apparmor=unconfined + command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\"" + ipc: host + + docsum-llm-server: + image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} + container_name: docsum-llm-server + depends_on: + - docsum-vllm-service + ports: + - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000" + ipc: host + cap_add: + - SYS_PTRACE + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}" + HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID} + LOGFLAG: ${DOCSUM_LOGFLAG:-False} + MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS} + MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS} + restart: unless-stopped + + whisper-service: + image: ${REGISTRY:-opea}/whisper:${TAG:-latest} + container_name: whisper-service + ports: + - "${DOCSUM_WHISPER_PORT:-7066}:7066" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + restart: unless-stopped + + docsum-backend-server: + image: ${REGISTRY:-opea}/docsum:${TAG:-latest} + container_name: docsum-backend-server + depends_on: + - docsum-tgi-service + - docsum-llm-server + ports: + - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888" + environment: + no_proxy: ${no_proxy} + https_proxy: ${https_proxy} + http_proxy: ${http_proxy} + MEGA_SERVICE_HOST_IP: ${HOST_IP} + LLM_SERVICE_HOST_IP: ${HOST_IP} + ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP} + ipc: host + restart: always + + docsum-gradio-ui: + image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest} + container_name: docsum-ui-server + depends_on: + - docsum-backend-server + ports: + - "${DOCSUM_FRONTEND_PORT:-5173}:5173" + environment: + no_proxy: ${no_proxy} + https_proxy: ${https_proxy} + http_proxy: ${http_proxy} + BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} + DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} + ipc: host + restart: always + +networks: + default: + driver: bridge diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh new file mode 100644 index 0000000000..43e71e0fbf --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +export HOST_IP="" +export DOCSUM_MAX_INPUT_TOKENS=2048 +export DOCSUM_MAX_TOTAL_TOKENS=4096 +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export DOCSUM_VLLM_SERVICE_PORT="8008" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN="" +export DOCSUM_LLM_SERVER_PORT="9000" +export DOCSUM_WHISPER_PORT="7066" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml index 095fd28c93..dc0d546189 100644 --- a/DocSum/docker_image_build/build.yaml +++ b/DocSum/docker_image_build/build.yaml @@ -47,3 +47,12 @@ services: dockerfile: comps/llms/src/doc-summarization/Dockerfile extends: docsum image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} + vllm_rocm: + build: + args: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + no_proxy: ${no_proxy} + context: ../ + dockerfile: ./Dockerfile-vllm-rocm + image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh new file mode 100644 index 0000000000..d0919a019a --- /dev/null +++ b/DocSum/tests/test_compose_on_rocm_vllm.sh @@ -0,0 +1,249 @@ +#!/bin/bash +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +set -xe +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') +export MAX_INPUT_TOKENS=1024 +export MAX_TOTAL_TOKENS=2048 +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export HOST_IP=${ip_address} +export DOCSUM_VLLM_SERVICE_PORT="8008" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} +export DOCSUM_LLM_SERVER_PORT="9000" +export DOCSUM_WHISPER_PORT="7066" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export MEGA_SERVICE_HOST_IP=${HOST_IP} +export LLM_SERVICE_HOST_IP=${HOST_IP} +export ASR_SERVICE_HOST_IP=${HOST_IP} +export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" + +function build_docker_images() { + opea_branch=${opea_branch:-"main"} + # If the opea_branch isn't main, replace the git clone branch in Dockerfile. + if [[ "${opea_branch}" != "main" ]]; then + cd $WORKPATH + OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" + NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" + find . -type f -name "Dockerfile*" | while read -r file; do + echo "Processing file: $file" + sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" + done + fi + + cd $WORKPATH/docker_image_build + git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + + echo "Build all the images with --no-cache, check docker_image_build.log for details..." + service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper" + docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log + + docker images && sleep 1s +} + +function start_services() { + cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm + sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env + # Start Docker Containers + docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log + sleep 1m +} + +function validate_services() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + + echo "===========================================" + + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "EXPECTED_RESULT==> $EXPECTED_RESULT" + echo "CONTENT==> $CONTENT" + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +get_base64_str() { + local file_name=$1 + base64 -w 0 "$file_name" +} + +# Function to generate input data for testing based on the document type +input_data_for_test() { + local document_type=$1 + case $document_type in + ("text") + echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are." + ;; + ("audio") + get_base64_str "$WORKPATH/tests/data/test.wav" + ;; + ("video") + get_base64_str "$WORKPATH/tests/data/test.mp4" + ;; + (*) + echo "Invalid document type" >&2 + exit 1 + ;; + esac +} + +function validate_microservices() { + # Check if the microservices are running correctly. + + # whisper microservice + ulimit -s 65536 + validate_services \ + "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \ + '{"asr_result":"well"}' \ + "whisper-service" \ + "whisper-service" \ + "{\"audio\": \"$(input_data_for_test "audio")\"}" + + # vLLM service + validate_services \ + "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \ + "generated_text" \ + "docsum-vllm-service" \ + "docsum-vllm-service" \ + '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}' + + # llm microservice + validate_services \ + "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \ + "text" \ + "docsum-llm-server" \ + "docsum-llm-server" \ + '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' + +} + +function validate_megaservice() { + local SERVICE_NAME="docsum-backend-server" + local DOCKER_NAME="docsum-backend-server" + local EXPECTED_RESULT="[DONE]" + local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." + local URL="${host_ip}:8888/v1/docsum" + local DATA_TYPE="type=text" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL") + + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +function validate_megaservice_json() { + # Curl the Mega Service + echo "" + echo ">>> Checking text data with Content-Type: application/json" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' + + echo ">>> Checking audio data" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + "{\"type\": \"audio\", \"messages\": \"$(input_data_for_test "audio")\"}" + + echo ">>> Checking video data" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + "{\"type\": \"video\", \"messages\": \"$(input_data_for_test "video")\"}" + +} + +function stop_docker() { + cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/ + docker compose stop && docker compose rm -f +} + +function main() { + echo "===========================================" + echo ">>>> Stopping any running Docker containers..." + stop_docker + + echo "===========================================" + if [[ "$IMAGE_REPO" == "opea" ]]; then + echo ">>>> Building Docker images..." + build_docker_images + fi + + echo "===========================================" + echo ">>>> Starting Docker services..." + start_services + + echo "===========================================" + echo ">>>> Validating microservices..." + validate_microservices + + echo "===========================================" + echo ">>>> Validating megaservice..." + validate_megaservice + echo ">>>> Validating validate_megaservice_json..." + validate_megaservice_json + + echo "===========================================" + echo ">>>> Stopping Docker containers..." + stop_docker + + echo "===========================================" + echo ">>>> Pruning Docker system..." + echo y | docker system prune + echo ">>>> Docker system pruned successfully." + echo "===========================================" +} + +main From 2e1b401ad5edb84b769ef8a9ac52062b2213c720 Mon Sep 17 00:00:00 2001 From: Chingis Yundunov Date: Thu, 13 Feb 2025 10:07:05 +0700 Subject: [PATCH 019/226] DocSum - fix main Signed-off-by: Chingis Yundunov Signed-off-by: Chingis Yundunov --- DocSum/Dockerfile-vllm-rocm | 18 -- .../amd/gpu/rocm-vllm/README.md | 175 ------------ .../amd/gpu/rocm-vllm/compose.yaml | 107 -------- .../amd/gpu/rocm-vllm/set_env.sh | 16 -- DocSum/docker_image_build/build.yaml | 9 - DocSum/tests/test_compose_on_rocm_vllm.sh | 249 ------------------ 6 files changed, 574 deletions(-) delete mode 100644 DocSum/Dockerfile-vllm-rocm delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh delete mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm deleted file mode 100644 index f0e8a8743a..0000000000 --- a/DocSum/Dockerfile-vllm-rocm +++ /dev/null @@ -1,18 +0,0 @@ -FROM rocm/vllm-dev:main - -# Set the working directory -WORKDIR /workspace - -# Copy the api_server.py into the image -ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py - -# Expose the port used by the API server -EXPOSE 8011 - -# Set environment variables -ENV HUGGINGFACE_HUB_CACHE=/workspace -ENV WILM_USE_TRITON_FLASH_ATTENTION=0 -ENV PYTORCH_JIT=0 - -# Set the entrypoint to the api_server.py script -ENTRYPOINT ["python3", "/workspace/api_server.py"] diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md deleted file mode 100644 index 4d41a5cd31..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md +++ /dev/null @@ -1,175 +0,0 @@ -# Build and deploy DocSum Application on AMD GPU (ROCm) - -## Build images - -## 🚀 Build Docker Images - -First of all, you need to build Docker Images locally and install the python package of it. - -### 1. Build LLM Image - -```bash -git clone https://github.com/opea-project/GenAIComps.git -cd GenAIComps -docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile . -``` - -Then run the command `docker images`, you will have the following four Docker Images: - -### 2. Build MegaService Docker Image - -To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command: - -```bash -git clone https://github.com/opea-project/GenAIExamples -cd GenAIExamples/DocSum/ -docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . -``` - -### 3. Build UI Docker Image - -Build the frontend Docker image via below command: - -```bash -cd GenAIExamples/DocSum/ui -docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . -``` - -Then run the command `docker images`, you will have the following Docker Images: - -1. `opea/llm-docsum-tgi:latest` -2. `opea/docsum:latest` -3. `opea/docsum-ui:latest` - -### 4. Build React UI Docker Image - -Build the frontend Docker image via below command: - -```bash -cd GenAIExamples/DocSum/ui -export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum" -docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react . - -docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . -``` - -Then run the command `docker images`, you will have the following Docker Images: - -1. `opea/llm-docsum-tgi:latest` -2. `opea/docsum:latest` -3. `opea/docsum-ui:latest` -4. `opea/docsum-react-ui:latest` - -## 🚀 Start Microservices and MegaService - -### Required Models - -Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model. -For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable. - -### Setup Environment Variables - -Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below. - -```bash -export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm" -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export HOST_IP=${host_ip} -export DOCSUM_TGI_SERVICE_PORT="18882" -export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} -export DOCSUM_LLM_SERVER_PORT="8008" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export DocSum_COMPONENT_NAME="OpeaDocSumTgi" -``` - -Note: Please replace with `host_ip` with your external IP address, do not use localhost. - -Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered, where is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) - -Example for set isolation for 1 GPU - -``` - - /dev/dri/card0:/dev/dri/card0 - - /dev/dri/renderD128:/dev/dri/renderD128 -``` - -Example for set isolation for 2 GPUs - -``` - - /dev/dri/card0:/dev/dri/card0 - - /dev/dri/renderD128:/dev/dri/renderD128 - - /dev/dri/card1:/dev/dri/card1 - - /dev/dri/renderD129:/dev/dri/renderD129 -``` - -Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) - -### Start Microservice Docker Containers - -```bash -cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm -docker compose up -d -``` - -### Validate Microservices - -1. TGI Service - - ```bash - curl http://${host_ip}:8008/generate \ - -X POST \ - -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \ - -H 'Content-Type: application/json' - ``` - -2. LLM Microservice - - ```bash - curl http://${host_ip}:9000/v1/docsum \ - -X POST \ - -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \ - -H 'Content-Type: application/json' - ``` - -3. MegaService - - ```bash - curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{ - "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false - }' - ``` - -## 🚀 Launch the Svelte UI - -Open this URL `http://{host_ip}:5173` in your browser to access the frontend. - -![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b) - -Here is an example for summarizing a article. - -![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55) - -## 🚀 Launch the React UI (Optional) - -To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below: - -```yaml -docsum-rocm-react-ui-server: - image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest} - container_name: docsum-rocm-react-ui-server - depends_on: - - docsum-rocm-backend-server - ports: - - "5174:80" - environment: - - no_proxy=${no_proxy} - - https_proxy=${https_proxy} - - http_proxy=${http_proxy} - - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT} -``` - -Open this URL `http://{host_ip}:5175` in your browser to access the frontend. - -![project-screenshot](../../../../assets/img/docsum-ui-react.png) diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml deleted file mode 100644 index 037aa06395..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -services: - docsum-vllm-service: - image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} - container_name: docsum-vllm-service - ports: - - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011" - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_HUB_DISABLE_PROGRESS_BARS: 1 - HF_HUB_ENABLE_HF_TRANSFER: 0 - WILM_USE_TRITON_FLASH_ATTENTION: 0 - PYTORCH_JIT: 0 - volumes: - - "./data:/data" - shm_size: 20G - devices: - - /dev/kfd:/dev/kfd - - /dev/dri/:/dev/dri/ - cap_add: - - SYS_PTRACE - group_add: - - video - security_opt: - - seccomp:unconfined - - apparmor=unconfined - command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\"" - ipc: host - - docsum-llm-server: - image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} - container_name: docsum-llm-server - depends_on: - - docsum-vllm-service - ports: - - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000" - ipc: host - cap_add: - - SYS_PTRACE - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}" - HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID} - LOGFLAG: ${DOCSUM_LOGFLAG:-False} - MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS} - MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS} - restart: unless-stopped - - whisper-service: - image: ${REGISTRY:-opea}/whisper:${TAG:-latest} - container_name: whisper-service - ports: - - "${DOCSUM_WHISPER_PORT:-7066}:7066" - ipc: host - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - restart: unless-stopped - - docsum-backend-server: - image: ${REGISTRY:-opea}/docsum:${TAG:-latest} - container_name: docsum-backend-server - depends_on: - - docsum-tgi-service - - docsum-llm-server - ports: - - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888" - environment: - no_proxy: ${no_proxy} - https_proxy: ${https_proxy} - http_proxy: ${http_proxy} - MEGA_SERVICE_HOST_IP: ${HOST_IP} - LLM_SERVICE_HOST_IP: ${HOST_IP} - ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP} - ipc: host - restart: always - - docsum-gradio-ui: - image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest} - container_name: docsum-ui-server - depends_on: - - docsum-backend-server - ports: - - "${DOCSUM_FRONTEND_PORT:-5173}:5173" - environment: - no_proxy: ${no_proxy} - https_proxy: ${https_proxy} - http_proxy: ${http_proxy} - BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} - DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} - ipc: host - restart: always - -networks: - default: - driver: bridge diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh deleted file mode 100644 index 43e71e0fbf..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -export HOST_IP="" -export DOCSUM_MAX_INPUT_TOKENS=2048 -export DOCSUM_MAX_TOTAL_TOKENS=4096 -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export DOCSUM_VLLM_SERVICE_PORT="8008" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN="" -export DOCSUM_LLM_SERVER_PORT="9000" -export DOCSUM_WHISPER_PORT="7066" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml index dc0d546189..095fd28c93 100644 --- a/DocSum/docker_image_build/build.yaml +++ b/DocSum/docker_image_build/build.yaml @@ -47,12 +47,3 @@ services: dockerfile: comps/llms/src/doc-summarization/Dockerfile extends: docsum image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} - vllm_rocm: - build: - args: - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - no_proxy: ${no_proxy} - context: ../ - dockerfile: ./Dockerfile-vllm-rocm - image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh deleted file mode 100644 index d0919a019a..0000000000 --- a/DocSum/tests/test_compose_on_rocm_vllm.sh +++ /dev/null @@ -1,249 +0,0 @@ -#!/bin/bash -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -set -xe -IMAGE_REPO=${IMAGE_REPO:-"opea"} -IMAGE_TAG=${IMAGE_TAG:-"latest"} -echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" -echo "TAG=IMAGE_TAG=${IMAGE_TAG}" - -WORKPATH=$(dirname "$PWD") -LOG_PATH="$WORKPATH/tests" -ip_address=$(hostname -I | awk '{print $1}') -export MAX_INPUT_TOKENS=1024 -export MAX_TOTAL_TOKENS=2048 -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export HOST_IP=${ip_address} -export DOCSUM_VLLM_SERVICE_PORT="8008" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} -export DOCSUM_LLM_SERVER_PORT="9000" -export DOCSUM_WHISPER_PORT="7066" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export MEGA_SERVICE_HOST_IP=${HOST_IP} -export LLM_SERVICE_HOST_IP=${HOST_IP} -export ASR_SERVICE_HOST_IP=${HOST_IP} -export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" - -function build_docker_images() { - opea_branch=${opea_branch:-"main"} - # If the opea_branch isn't main, replace the git clone branch in Dockerfile. - if [[ "${opea_branch}" != "main" ]]; then - cd $WORKPATH - OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" - NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" - find . -type f -name "Dockerfile*" | while read -r file; do - echo "Processing file: $file" - sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" - done - fi - - cd $WORKPATH/docker_image_build - git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git - - echo "Build all the images with --no-cache, check docker_image_build.log for details..." - service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper" - docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log - - docker images && sleep 1s -} - -function start_services() { - cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm - sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env - # Start Docker Containers - docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log - sleep 1m -} - -function validate_services() { - local URL="$1" - local EXPECTED_RESULT="$2" - local SERVICE_NAME="$3" - local DOCKER_NAME="$4" - local INPUT_DATA="$5" - - local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") - - echo "===========================================" - - if [ "$HTTP_STATUS" -eq 200 ]; then - echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." - - local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) - - if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then - echo "[ $SERVICE_NAME ] Content is as expected." - else - echo "EXPECTED_RESULT==> $EXPECTED_RESULT" - echo "CONTENT==> $CONTENT" - echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - - fi - else - echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - sleep 1s -} - -get_base64_str() { - local file_name=$1 - base64 -w 0 "$file_name" -} - -# Function to generate input data for testing based on the document type -input_data_for_test() { - local document_type=$1 - case $document_type in - ("text") - echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are." - ;; - ("audio") - get_base64_str "$WORKPATH/tests/data/test.wav" - ;; - ("video") - get_base64_str "$WORKPATH/tests/data/test.mp4" - ;; - (*) - echo "Invalid document type" >&2 - exit 1 - ;; - esac -} - -function validate_microservices() { - # Check if the microservices are running correctly. - - # whisper microservice - ulimit -s 65536 - validate_services \ - "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \ - '{"asr_result":"well"}' \ - "whisper-service" \ - "whisper-service" \ - "{\"audio\": \"$(input_data_for_test "audio")\"}" - - # vLLM service - validate_services \ - "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \ - "generated_text" \ - "docsum-vllm-service" \ - "docsum-vllm-service" \ - '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}' - - # llm microservice - validate_services \ - "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \ - "text" \ - "docsum-llm-server" \ - "docsum-llm-server" \ - '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' - -} - -function validate_megaservice() { - local SERVICE_NAME="docsum-backend-server" - local DOCKER_NAME="docsum-backend-server" - local EXPECTED_RESULT="[DONE]" - local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." - local URL="${host_ip}:8888/v1/docsum" - local DATA_TYPE="type=text" - - local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL") - - if [ "$HTTP_STATUS" -eq 200 ]; then - echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." - - local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) - - if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then - echo "[ $SERVICE_NAME ] Content is as expected." - else - echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - else - echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - sleep 1s -} - -function validate_megaservice_json() { - # Curl the Mega Service - echo "" - echo ">>> Checking text data with Content-Type: application/json" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' - - echo ">>> Checking audio data" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - "{\"type\": \"audio\", \"messages\": \"$(input_data_for_test "audio")\"}" - - echo ">>> Checking video data" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - "{\"type\": \"video\", \"messages\": \"$(input_data_for_test "video")\"}" - -} - -function stop_docker() { - cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/ - docker compose stop && docker compose rm -f -} - -function main() { - echo "===========================================" - echo ">>>> Stopping any running Docker containers..." - stop_docker - - echo "===========================================" - if [[ "$IMAGE_REPO" == "opea" ]]; then - echo ">>>> Building Docker images..." - build_docker_images - fi - - echo "===========================================" - echo ">>>> Starting Docker services..." - start_services - - echo "===========================================" - echo ">>>> Validating microservices..." - validate_microservices - - echo "===========================================" - echo ">>>> Validating megaservice..." - validate_megaservice - echo ">>>> Validating validate_megaservice_json..." - validate_megaservice_json - - echo "===========================================" - echo ">>>> Stopping Docker containers..." - stop_docker - - echo "===========================================" - echo ">>>> Pruning Docker system..." - echo y | docker system prune - echo ">>>> Docker system pruned successfully." - echo "===========================================" -} - -main From c9a78079957c580116093d234e1ee481ec196951 Mon Sep 17 00:00:00 2001 From: Chingis Yundunov Date: Thu, 13 Feb 2025 10:02:03 +0700 Subject: [PATCH 020/226] DocSum - add files for deploy app with ROCm vLLM Signed-off-by: Chingis Yundunov Signed-off-by: Chingis Yundunov --- DocSum/Dockerfile-vllm-rocm | 18 ++ .../amd/gpu/rocm-vllm/README.md | 175 ++++++++++++ .../amd/gpu/rocm-vllm/compose.yaml | 107 ++++++++ .../amd/gpu/rocm-vllm/set_env.sh | 16 ++ DocSum/docker_image_build/build.yaml | 9 + DocSum/tests/test_compose_on_rocm_vllm.sh | 249 ++++++++++++++++++ 6 files changed, 574 insertions(+) create mode 100644 DocSum/Dockerfile-vllm-rocm create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh create mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm new file mode 100644 index 0000000000..f0e8a8743a --- /dev/null +++ b/DocSum/Dockerfile-vllm-rocm @@ -0,0 +1,18 @@ +FROM rocm/vllm-dev:main + +# Set the working directory +WORKDIR /workspace + +# Copy the api_server.py into the image +ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py + +# Expose the port used by the API server +EXPOSE 8011 + +# Set environment variables +ENV HUGGINGFACE_HUB_CACHE=/workspace +ENV WILM_USE_TRITON_FLASH_ATTENTION=0 +ENV PYTORCH_JIT=0 + +# Set the entrypoint to the api_server.py script +ENTRYPOINT ["python3", "/workspace/api_server.py"] diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md new file mode 100644 index 0000000000..4d41a5cd31 --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md @@ -0,0 +1,175 @@ +# Build and deploy DocSum Application on AMD GPU (ROCm) + +## Build images + +## 🚀 Build Docker Images + +First of all, you need to build Docker Images locally and install the python package of it. + +### 1. Build LLM Image + +```bash +git clone https://github.com/opea-project/GenAIComps.git +cd GenAIComps +docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile . +``` + +Then run the command `docker images`, you will have the following four Docker Images: + +### 2. Build MegaService Docker Image + +To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command: + +```bash +git clone https://github.com/opea-project/GenAIExamples +cd GenAIExamples/DocSum/ +docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . +``` + +### 3. Build UI Docker Image + +Build the frontend Docker image via below command: + +```bash +cd GenAIExamples/DocSum/ui +docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . +``` + +Then run the command `docker images`, you will have the following Docker Images: + +1. `opea/llm-docsum-tgi:latest` +2. `opea/docsum:latest` +3. `opea/docsum-ui:latest` + +### 4. Build React UI Docker Image + +Build the frontend Docker image via below command: + +```bash +cd GenAIExamples/DocSum/ui +export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum" +docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react . + +docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . +``` + +Then run the command `docker images`, you will have the following Docker Images: + +1. `opea/llm-docsum-tgi:latest` +2. `opea/docsum:latest` +3. `opea/docsum-ui:latest` +4. `opea/docsum-react-ui:latest` + +## 🚀 Start Microservices and MegaService + +### Required Models + +Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model. +For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable. + +### Setup Environment Variables + +Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below. + +```bash +export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm" +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export HOST_IP=${host_ip} +export DOCSUM_TGI_SERVICE_PORT="18882" +export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} +export DOCSUM_LLM_SERVER_PORT="8008" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export DocSum_COMPONENT_NAME="OpeaDocSumTgi" +``` + +Note: Please replace with `host_ip` with your external IP address, do not use localhost. + +Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered, where is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) + +Example for set isolation for 1 GPU + +``` + - /dev/dri/card0:/dev/dri/card0 + - /dev/dri/renderD128:/dev/dri/renderD128 +``` + +Example for set isolation for 2 GPUs + +``` + - /dev/dri/card0:/dev/dri/card0 + - /dev/dri/renderD128:/dev/dri/renderD128 + - /dev/dri/card1:/dev/dri/card1 + - /dev/dri/renderD129:/dev/dri/renderD129 +``` + +Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) + +### Start Microservice Docker Containers + +```bash +cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm +docker compose up -d +``` + +### Validate Microservices + +1. TGI Service + + ```bash + curl http://${host_ip}:8008/generate \ + -X POST \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \ + -H 'Content-Type: application/json' + ``` + +2. LLM Microservice + + ```bash + curl http://${host_ip}:9000/v1/docsum \ + -X POST \ + -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \ + -H 'Content-Type: application/json' + ``` + +3. MegaService + + ```bash + curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{ + "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false + }' + ``` + +## 🚀 Launch the Svelte UI + +Open this URL `http://{host_ip}:5173` in your browser to access the frontend. + +![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b) + +Here is an example for summarizing a article. + +![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55) + +## 🚀 Launch the React UI (Optional) + +To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below: + +```yaml +docsum-rocm-react-ui-server: + image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest} + container_name: docsum-rocm-react-ui-server + depends_on: + - docsum-rocm-backend-server + ports: + - "5174:80" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT} +``` + +Open this URL `http://{host_ip}:5175` in your browser to access the frontend. + +![project-screenshot](../../../../assets/img/docsum-ui-react.png) diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml new file mode 100644 index 0000000000..037aa06395 --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml @@ -0,0 +1,107 @@ +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +services: + docsum-vllm-service: + image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} + container_name: docsum-vllm-service + ports: + - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011" + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + WILM_USE_TRITON_FLASH_ATTENTION: 0 + PYTORCH_JIT: 0 + volumes: + - "./data:/data" + shm_size: 20G + devices: + - /dev/kfd:/dev/kfd + - /dev/dri/:/dev/dri/ + cap_add: + - SYS_PTRACE + group_add: + - video + security_opt: + - seccomp:unconfined + - apparmor=unconfined + command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\"" + ipc: host + + docsum-llm-server: + image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} + container_name: docsum-llm-server + depends_on: + - docsum-vllm-service + ports: + - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000" + ipc: host + cap_add: + - SYS_PTRACE + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}" + HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID} + LOGFLAG: ${DOCSUM_LOGFLAG:-False} + MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS} + MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS} + restart: unless-stopped + + whisper-service: + image: ${REGISTRY:-opea}/whisper:${TAG:-latest} + container_name: whisper-service + ports: + - "${DOCSUM_WHISPER_PORT:-7066}:7066" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + restart: unless-stopped + + docsum-backend-server: + image: ${REGISTRY:-opea}/docsum:${TAG:-latest} + container_name: docsum-backend-server + depends_on: + - docsum-tgi-service + - docsum-llm-server + ports: + - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888" + environment: + no_proxy: ${no_proxy} + https_proxy: ${https_proxy} + http_proxy: ${http_proxy} + MEGA_SERVICE_HOST_IP: ${HOST_IP} + LLM_SERVICE_HOST_IP: ${HOST_IP} + ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP} + ipc: host + restart: always + + docsum-gradio-ui: + image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest} + container_name: docsum-ui-server + depends_on: + - docsum-backend-server + ports: + - "${DOCSUM_FRONTEND_PORT:-5173}:5173" + environment: + no_proxy: ${no_proxy} + https_proxy: ${https_proxy} + http_proxy: ${http_proxy} + BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} + DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} + ipc: host + restart: always + +networks: + default: + driver: bridge diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh new file mode 100644 index 0000000000..43e71e0fbf --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +export HOST_IP="" +export DOCSUM_MAX_INPUT_TOKENS=2048 +export DOCSUM_MAX_TOTAL_TOKENS=4096 +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export DOCSUM_VLLM_SERVICE_PORT="8008" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN="" +export DOCSUM_LLM_SERVER_PORT="9000" +export DOCSUM_WHISPER_PORT="7066" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml index 095fd28c93..dc0d546189 100644 --- a/DocSum/docker_image_build/build.yaml +++ b/DocSum/docker_image_build/build.yaml @@ -47,3 +47,12 @@ services: dockerfile: comps/llms/src/doc-summarization/Dockerfile extends: docsum image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} + vllm_rocm: + build: + args: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + no_proxy: ${no_proxy} + context: ../ + dockerfile: ./Dockerfile-vllm-rocm + image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh new file mode 100644 index 0000000000..d0919a019a --- /dev/null +++ b/DocSum/tests/test_compose_on_rocm_vllm.sh @@ -0,0 +1,249 @@ +#!/bin/bash +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +set -xe +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') +export MAX_INPUT_TOKENS=1024 +export MAX_TOTAL_TOKENS=2048 +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export HOST_IP=${ip_address} +export DOCSUM_VLLM_SERVICE_PORT="8008" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} +export DOCSUM_LLM_SERVER_PORT="9000" +export DOCSUM_WHISPER_PORT="7066" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export MEGA_SERVICE_HOST_IP=${HOST_IP} +export LLM_SERVICE_HOST_IP=${HOST_IP} +export ASR_SERVICE_HOST_IP=${HOST_IP} +export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" + +function build_docker_images() { + opea_branch=${opea_branch:-"main"} + # If the opea_branch isn't main, replace the git clone branch in Dockerfile. + if [[ "${opea_branch}" != "main" ]]; then + cd $WORKPATH + OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" + NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" + find . -type f -name "Dockerfile*" | while read -r file; do + echo "Processing file: $file" + sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" + done + fi + + cd $WORKPATH/docker_image_build + git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + + echo "Build all the images with --no-cache, check docker_image_build.log for details..." + service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper" + docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log + + docker images && sleep 1s +} + +function start_services() { + cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm + sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env + # Start Docker Containers + docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log + sleep 1m +} + +function validate_services() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + + echo "===========================================" + + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "EXPECTED_RESULT==> $EXPECTED_RESULT" + echo "CONTENT==> $CONTENT" + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +get_base64_str() { + local file_name=$1 + base64 -w 0 "$file_name" +} + +# Function to generate input data for testing based on the document type +input_data_for_test() { + local document_type=$1 + case $document_type in + ("text") + echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are." + ;; + ("audio") + get_base64_str "$WORKPATH/tests/data/test.wav" + ;; + ("video") + get_base64_str "$WORKPATH/tests/data/test.mp4" + ;; + (*) + echo "Invalid document type" >&2 + exit 1 + ;; + esac +} + +function validate_microservices() { + # Check if the microservices are running correctly. + + # whisper microservice + ulimit -s 65536 + validate_services \ + "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \ + '{"asr_result":"well"}' \ + "whisper-service" \ + "whisper-service" \ + "{\"audio\": \"$(input_data_for_test "audio")\"}" + + # vLLM service + validate_services \ + "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \ + "generated_text" \ + "docsum-vllm-service" \ + "docsum-vllm-service" \ + '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}' + + # llm microservice + validate_services \ + "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \ + "text" \ + "docsum-llm-server" \ + "docsum-llm-server" \ + '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' + +} + +function validate_megaservice() { + local SERVICE_NAME="docsum-backend-server" + local DOCKER_NAME="docsum-backend-server" + local EXPECTED_RESULT="[DONE]" + local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." + local URL="${host_ip}:8888/v1/docsum" + local DATA_TYPE="type=text" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL") + + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +function validate_megaservice_json() { + # Curl the Mega Service + echo "" + echo ">>> Checking text data with Content-Type: application/json" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' + + echo ">>> Checking audio data" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + "{\"type\": \"audio\", \"messages\": \"$(input_data_for_test "audio")\"}" + + echo ">>> Checking video data" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + "{\"type\": \"video\", \"messages\": \"$(input_data_for_test "video")\"}" + +} + +function stop_docker() { + cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/ + docker compose stop && docker compose rm -f +} + +function main() { + echo "===========================================" + echo ">>>> Stopping any running Docker containers..." + stop_docker + + echo "===========================================" + if [[ "$IMAGE_REPO" == "opea" ]]; then + echo ">>>> Building Docker images..." + build_docker_images + fi + + echo "===========================================" + echo ">>>> Starting Docker services..." + start_services + + echo "===========================================" + echo ">>>> Validating microservices..." + validate_microservices + + echo "===========================================" + echo ">>>> Validating megaservice..." + validate_megaservice + echo ">>>> Validating validate_megaservice_json..." + validate_megaservice_json + + echo "===========================================" + echo ">>>> Stopping Docker containers..." + stop_docker + + echo "===========================================" + echo ">>>> Pruning Docker system..." + echo y | docker system prune + echo ">>>> Docker system pruned successfully." + echo "===========================================" +} + +main From b2e1523b4b8975582f40594feeb476cf192efe75 Mon Sep 17 00:00:00 2001 From: Chingis Yundunov Date: Thu, 13 Feb 2025 10:07:05 +0700 Subject: [PATCH 021/226] DocSum - fix main Signed-off-by: Chingis Yundunov Signed-off-by: Chingis Yundunov --- DocSum/Dockerfile-vllm-rocm | 18 -- .../amd/gpu/rocm-vllm/README.md | 175 ------------ .../amd/gpu/rocm-vllm/compose.yaml | 107 -------- .../amd/gpu/rocm-vllm/set_env.sh | 16 -- DocSum/docker_image_build/build.yaml | 9 - DocSum/tests/test_compose_on_rocm_vllm.sh | 249 ------------------ 6 files changed, 574 deletions(-) delete mode 100644 DocSum/Dockerfile-vllm-rocm delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh delete mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm deleted file mode 100644 index f0e8a8743a..0000000000 --- a/DocSum/Dockerfile-vllm-rocm +++ /dev/null @@ -1,18 +0,0 @@ -FROM rocm/vllm-dev:main - -# Set the working directory -WORKDIR /workspace - -# Copy the api_server.py into the image -ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py - -# Expose the port used by the API server -EXPOSE 8011 - -# Set environment variables -ENV HUGGINGFACE_HUB_CACHE=/workspace -ENV WILM_USE_TRITON_FLASH_ATTENTION=0 -ENV PYTORCH_JIT=0 - -# Set the entrypoint to the api_server.py script -ENTRYPOINT ["python3", "/workspace/api_server.py"] diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md deleted file mode 100644 index 4d41a5cd31..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md +++ /dev/null @@ -1,175 +0,0 @@ -# Build and deploy DocSum Application on AMD GPU (ROCm) - -## Build images - -## 🚀 Build Docker Images - -First of all, you need to build Docker Images locally and install the python package of it. - -### 1. Build LLM Image - -```bash -git clone https://github.com/opea-project/GenAIComps.git -cd GenAIComps -docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile . -``` - -Then run the command `docker images`, you will have the following four Docker Images: - -### 2. Build MegaService Docker Image - -To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command: - -```bash -git clone https://github.com/opea-project/GenAIExamples -cd GenAIExamples/DocSum/ -docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . -``` - -### 3. Build UI Docker Image - -Build the frontend Docker image via below command: - -```bash -cd GenAIExamples/DocSum/ui -docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . -``` - -Then run the command `docker images`, you will have the following Docker Images: - -1. `opea/llm-docsum-tgi:latest` -2. `opea/docsum:latest` -3. `opea/docsum-ui:latest` - -### 4. Build React UI Docker Image - -Build the frontend Docker image via below command: - -```bash -cd GenAIExamples/DocSum/ui -export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum" -docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react . - -docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . -``` - -Then run the command `docker images`, you will have the following Docker Images: - -1. `opea/llm-docsum-tgi:latest` -2. `opea/docsum:latest` -3. `opea/docsum-ui:latest` -4. `opea/docsum-react-ui:latest` - -## 🚀 Start Microservices and MegaService - -### Required Models - -Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model. -For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable. - -### Setup Environment Variables - -Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below. - -```bash -export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm" -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export HOST_IP=${host_ip} -export DOCSUM_TGI_SERVICE_PORT="18882" -export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} -export DOCSUM_LLM_SERVER_PORT="8008" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export DocSum_COMPONENT_NAME="OpeaDocSumTgi" -``` - -Note: Please replace with `host_ip` with your external IP address, do not use localhost. - -Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered, where is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) - -Example for set isolation for 1 GPU - -``` - - /dev/dri/card0:/dev/dri/card0 - - /dev/dri/renderD128:/dev/dri/renderD128 -``` - -Example for set isolation for 2 GPUs - -``` - - /dev/dri/card0:/dev/dri/card0 - - /dev/dri/renderD128:/dev/dri/renderD128 - - /dev/dri/card1:/dev/dri/card1 - - /dev/dri/renderD129:/dev/dri/renderD129 -``` - -Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) - -### Start Microservice Docker Containers - -```bash -cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm -docker compose up -d -``` - -### Validate Microservices - -1. TGI Service - - ```bash - curl http://${host_ip}:8008/generate \ - -X POST \ - -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \ - -H 'Content-Type: application/json' - ``` - -2. LLM Microservice - - ```bash - curl http://${host_ip}:9000/v1/docsum \ - -X POST \ - -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \ - -H 'Content-Type: application/json' - ``` - -3. MegaService - - ```bash - curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{ - "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false - }' - ``` - -## 🚀 Launch the Svelte UI - -Open this URL `http://{host_ip}:5173` in your browser to access the frontend. - -![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b) - -Here is an example for summarizing a article. - -![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55) - -## 🚀 Launch the React UI (Optional) - -To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below: - -```yaml -docsum-rocm-react-ui-server: - image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest} - container_name: docsum-rocm-react-ui-server - depends_on: - - docsum-rocm-backend-server - ports: - - "5174:80" - environment: - - no_proxy=${no_proxy} - - https_proxy=${https_proxy} - - http_proxy=${http_proxy} - - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT} -``` - -Open this URL `http://{host_ip}:5175` in your browser to access the frontend. - -![project-screenshot](../../../../assets/img/docsum-ui-react.png) diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml deleted file mode 100644 index 037aa06395..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -services: - docsum-vllm-service: - image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} - container_name: docsum-vllm-service - ports: - - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011" - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_HUB_DISABLE_PROGRESS_BARS: 1 - HF_HUB_ENABLE_HF_TRANSFER: 0 - WILM_USE_TRITON_FLASH_ATTENTION: 0 - PYTORCH_JIT: 0 - volumes: - - "./data:/data" - shm_size: 20G - devices: - - /dev/kfd:/dev/kfd - - /dev/dri/:/dev/dri/ - cap_add: - - SYS_PTRACE - group_add: - - video - security_opt: - - seccomp:unconfined - - apparmor=unconfined - command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\"" - ipc: host - - docsum-llm-server: - image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} - container_name: docsum-llm-server - depends_on: - - docsum-vllm-service - ports: - - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000" - ipc: host - cap_add: - - SYS_PTRACE - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}" - HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID} - LOGFLAG: ${DOCSUM_LOGFLAG:-False} - MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS} - MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS} - restart: unless-stopped - - whisper-service: - image: ${REGISTRY:-opea}/whisper:${TAG:-latest} - container_name: whisper-service - ports: - - "${DOCSUM_WHISPER_PORT:-7066}:7066" - ipc: host - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - restart: unless-stopped - - docsum-backend-server: - image: ${REGISTRY:-opea}/docsum:${TAG:-latest} - container_name: docsum-backend-server - depends_on: - - docsum-tgi-service - - docsum-llm-server - ports: - - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888" - environment: - no_proxy: ${no_proxy} - https_proxy: ${https_proxy} - http_proxy: ${http_proxy} - MEGA_SERVICE_HOST_IP: ${HOST_IP} - LLM_SERVICE_HOST_IP: ${HOST_IP} - ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP} - ipc: host - restart: always - - docsum-gradio-ui: - image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest} - container_name: docsum-ui-server - depends_on: - - docsum-backend-server - ports: - - "${DOCSUM_FRONTEND_PORT:-5173}:5173" - environment: - no_proxy: ${no_proxy} - https_proxy: ${https_proxy} - http_proxy: ${http_proxy} - BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} - DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} - ipc: host - restart: always - -networks: - default: - driver: bridge diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh deleted file mode 100644 index 43e71e0fbf..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -export HOST_IP="" -export DOCSUM_MAX_INPUT_TOKENS=2048 -export DOCSUM_MAX_TOTAL_TOKENS=4096 -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export DOCSUM_VLLM_SERVICE_PORT="8008" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN="" -export DOCSUM_LLM_SERVER_PORT="9000" -export DOCSUM_WHISPER_PORT="7066" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml index dc0d546189..095fd28c93 100644 --- a/DocSum/docker_image_build/build.yaml +++ b/DocSum/docker_image_build/build.yaml @@ -47,12 +47,3 @@ services: dockerfile: comps/llms/src/doc-summarization/Dockerfile extends: docsum image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} - vllm_rocm: - build: - args: - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - no_proxy: ${no_proxy} - context: ../ - dockerfile: ./Dockerfile-vllm-rocm - image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh deleted file mode 100644 index d0919a019a..0000000000 --- a/DocSum/tests/test_compose_on_rocm_vllm.sh +++ /dev/null @@ -1,249 +0,0 @@ -#!/bin/bash -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -set -xe -IMAGE_REPO=${IMAGE_REPO:-"opea"} -IMAGE_TAG=${IMAGE_TAG:-"latest"} -echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" -echo "TAG=IMAGE_TAG=${IMAGE_TAG}" - -WORKPATH=$(dirname "$PWD") -LOG_PATH="$WORKPATH/tests" -ip_address=$(hostname -I | awk '{print $1}') -export MAX_INPUT_TOKENS=1024 -export MAX_TOTAL_TOKENS=2048 -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export HOST_IP=${ip_address} -export DOCSUM_VLLM_SERVICE_PORT="8008" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} -export DOCSUM_LLM_SERVER_PORT="9000" -export DOCSUM_WHISPER_PORT="7066" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export MEGA_SERVICE_HOST_IP=${HOST_IP} -export LLM_SERVICE_HOST_IP=${HOST_IP} -export ASR_SERVICE_HOST_IP=${HOST_IP} -export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" - -function build_docker_images() { - opea_branch=${opea_branch:-"main"} - # If the opea_branch isn't main, replace the git clone branch in Dockerfile. - if [[ "${opea_branch}" != "main" ]]; then - cd $WORKPATH - OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" - NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" - find . -type f -name "Dockerfile*" | while read -r file; do - echo "Processing file: $file" - sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" - done - fi - - cd $WORKPATH/docker_image_build - git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git - - echo "Build all the images with --no-cache, check docker_image_build.log for details..." - service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper" - docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log - - docker images && sleep 1s -} - -function start_services() { - cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm - sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env - # Start Docker Containers - docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log - sleep 1m -} - -function validate_services() { - local URL="$1" - local EXPECTED_RESULT="$2" - local SERVICE_NAME="$3" - local DOCKER_NAME="$4" - local INPUT_DATA="$5" - - local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") - - echo "===========================================" - - if [ "$HTTP_STATUS" -eq 200 ]; then - echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." - - local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) - - if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then - echo "[ $SERVICE_NAME ] Content is as expected." - else - echo "EXPECTED_RESULT==> $EXPECTED_RESULT" - echo "CONTENT==> $CONTENT" - echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - - fi - else - echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - sleep 1s -} - -get_base64_str() { - local file_name=$1 - base64 -w 0 "$file_name" -} - -# Function to generate input data for testing based on the document type -input_data_for_test() { - local document_type=$1 - case $document_type in - ("text") - echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are." - ;; - ("audio") - get_base64_str "$WORKPATH/tests/data/test.wav" - ;; - ("video") - get_base64_str "$WORKPATH/tests/data/test.mp4" - ;; - (*) - echo "Invalid document type" >&2 - exit 1 - ;; - esac -} - -function validate_microservices() { - # Check if the microservices are running correctly. - - # whisper microservice - ulimit -s 65536 - validate_services \ - "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \ - '{"asr_result":"well"}' \ - "whisper-service" \ - "whisper-service" \ - "{\"audio\": \"$(input_data_for_test "audio")\"}" - - # vLLM service - validate_services \ - "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \ - "generated_text" \ - "docsum-vllm-service" \ - "docsum-vllm-service" \ - '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}' - - # llm microservice - validate_services \ - "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \ - "text" \ - "docsum-llm-server" \ - "docsum-llm-server" \ - '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' - -} - -function validate_megaservice() { - local SERVICE_NAME="docsum-backend-server" - local DOCKER_NAME="docsum-backend-server" - local EXPECTED_RESULT="[DONE]" - local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." - local URL="${host_ip}:8888/v1/docsum" - local DATA_TYPE="type=text" - - local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL") - - if [ "$HTTP_STATUS" -eq 200 ]; then - echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." - - local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) - - if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then - echo "[ $SERVICE_NAME ] Content is as expected." - else - echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - else - echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - sleep 1s -} - -function validate_megaservice_json() { - # Curl the Mega Service - echo "" - echo ">>> Checking text data with Content-Type: application/json" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' - - echo ">>> Checking audio data" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - "{\"type\": \"audio\", \"messages\": \"$(input_data_for_test "audio")\"}" - - echo ">>> Checking video data" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - "{\"type\": \"video\", \"messages\": \"$(input_data_for_test "video")\"}" - -} - -function stop_docker() { - cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/ - docker compose stop && docker compose rm -f -} - -function main() { - echo "===========================================" - echo ">>>> Stopping any running Docker containers..." - stop_docker - - echo "===========================================" - if [[ "$IMAGE_REPO" == "opea" ]]; then - echo ">>>> Building Docker images..." - build_docker_images - fi - - echo "===========================================" - echo ">>>> Starting Docker services..." - start_services - - echo "===========================================" - echo ">>>> Validating microservices..." - validate_microservices - - echo "===========================================" - echo ">>>> Validating megaservice..." - validate_megaservice - echo ">>>> Validating validate_megaservice_json..." - validate_megaservice_json - - echo "===========================================" - echo ">>>> Stopping Docker containers..." - stop_docker - - echo "===========================================" - echo ">>>> Pruning Docker system..." - echo y | docker system prune - echo ">>>> Docker system pruned successfully." - echo "===========================================" -} - -main From 947aa8129a2ed40e7873f926d74c341a765f6ce6 Mon Sep 17 00:00:00 2001 From: Chingis Yundunov Date: Thu, 13 Feb 2025 10:02:03 +0700 Subject: [PATCH 022/226] DocSum - add files for deploy app with ROCm vLLM Signed-off-by: Chingis Yundunov Signed-off-by: Chingis Yundunov --- DocSum/Dockerfile-vllm-rocm | 18 ++ .../amd/gpu/rocm-vllm/README.md | 175 ++++++++++++ .../amd/gpu/rocm-vllm/compose.yaml | 107 ++++++++ .../amd/gpu/rocm-vllm/set_env.sh | 16 ++ DocSum/docker_image_build/build.yaml | 9 + DocSum/tests/test_compose_on_rocm_vllm.sh | 249 ++++++++++++++++++ 6 files changed, 574 insertions(+) create mode 100644 DocSum/Dockerfile-vllm-rocm create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml create mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh create mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm new file mode 100644 index 0000000000..f0e8a8743a --- /dev/null +++ b/DocSum/Dockerfile-vllm-rocm @@ -0,0 +1,18 @@ +FROM rocm/vllm-dev:main + +# Set the working directory +WORKDIR /workspace + +# Copy the api_server.py into the image +ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py + +# Expose the port used by the API server +EXPOSE 8011 + +# Set environment variables +ENV HUGGINGFACE_HUB_CACHE=/workspace +ENV WILM_USE_TRITON_FLASH_ATTENTION=0 +ENV PYTORCH_JIT=0 + +# Set the entrypoint to the api_server.py script +ENTRYPOINT ["python3", "/workspace/api_server.py"] diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md new file mode 100644 index 0000000000..4d41a5cd31 --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md @@ -0,0 +1,175 @@ +# Build and deploy DocSum Application on AMD GPU (ROCm) + +## Build images + +## 🚀 Build Docker Images + +First of all, you need to build Docker Images locally and install the python package of it. + +### 1. Build LLM Image + +```bash +git clone https://github.com/opea-project/GenAIComps.git +cd GenAIComps +docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile . +``` + +Then run the command `docker images`, you will have the following four Docker Images: + +### 2. Build MegaService Docker Image + +To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command: + +```bash +git clone https://github.com/opea-project/GenAIExamples +cd GenAIExamples/DocSum/ +docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . +``` + +### 3. Build UI Docker Image + +Build the frontend Docker image via below command: + +```bash +cd GenAIExamples/DocSum/ui +docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . +``` + +Then run the command `docker images`, you will have the following Docker Images: + +1. `opea/llm-docsum-tgi:latest` +2. `opea/docsum:latest` +3. `opea/docsum-ui:latest` + +### 4. Build React UI Docker Image + +Build the frontend Docker image via below command: + +```bash +cd GenAIExamples/DocSum/ui +export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum" +docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react . + +docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . +``` + +Then run the command `docker images`, you will have the following Docker Images: + +1. `opea/llm-docsum-tgi:latest` +2. `opea/docsum:latest` +3. `opea/docsum-ui:latest` +4. `opea/docsum-react-ui:latest` + +## 🚀 Start Microservices and MegaService + +### Required Models + +Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model. +For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable. + +### Setup Environment Variables + +Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below. + +```bash +export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm" +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export HOST_IP=${host_ip} +export DOCSUM_TGI_SERVICE_PORT="18882" +export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} +export DOCSUM_LLM_SERVER_PORT="8008" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export DocSum_COMPONENT_NAME="OpeaDocSumTgi" +``` + +Note: Please replace with `host_ip` with your external IP address, do not use localhost. + +Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered, where is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) + +Example for set isolation for 1 GPU + +``` + - /dev/dri/card0:/dev/dri/card0 + - /dev/dri/renderD128:/dev/dri/renderD128 +``` + +Example for set isolation for 2 GPUs + +``` + - /dev/dri/card0:/dev/dri/card0 + - /dev/dri/renderD128:/dev/dri/renderD128 + - /dev/dri/card1:/dev/dri/card1 + - /dev/dri/renderD129:/dev/dri/renderD129 +``` + +Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) + +### Start Microservice Docker Containers + +```bash +cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm +docker compose up -d +``` + +### Validate Microservices + +1. TGI Service + + ```bash + curl http://${host_ip}:8008/generate \ + -X POST \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \ + -H 'Content-Type: application/json' + ``` + +2. LLM Microservice + + ```bash + curl http://${host_ip}:9000/v1/docsum \ + -X POST \ + -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \ + -H 'Content-Type: application/json' + ``` + +3. MegaService + + ```bash + curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{ + "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false + }' + ``` + +## 🚀 Launch the Svelte UI + +Open this URL `http://{host_ip}:5173` in your browser to access the frontend. + +![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b) + +Here is an example for summarizing a article. + +![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55) + +## 🚀 Launch the React UI (Optional) + +To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below: + +```yaml +docsum-rocm-react-ui-server: + image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest} + container_name: docsum-rocm-react-ui-server + depends_on: + - docsum-rocm-backend-server + ports: + - "5174:80" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT} +``` + +Open this URL `http://{host_ip}:5175` in your browser to access the frontend. + +![project-screenshot](../../../../assets/img/docsum-ui-react.png) diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml new file mode 100644 index 0000000000..037aa06395 --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml @@ -0,0 +1,107 @@ +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +services: + docsum-vllm-service: + image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} + container_name: docsum-vllm-service + ports: + - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011" + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + WILM_USE_TRITON_FLASH_ATTENTION: 0 + PYTORCH_JIT: 0 + volumes: + - "./data:/data" + shm_size: 20G + devices: + - /dev/kfd:/dev/kfd + - /dev/dri/:/dev/dri/ + cap_add: + - SYS_PTRACE + group_add: + - video + security_opt: + - seccomp:unconfined + - apparmor=unconfined + command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\"" + ipc: host + + docsum-llm-server: + image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} + container_name: docsum-llm-server + depends_on: + - docsum-vllm-service + ports: + - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000" + ipc: host + cap_add: + - SYS_PTRACE + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}" + HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} + LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID} + LOGFLAG: ${DOCSUM_LOGFLAG:-False} + MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS} + MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS} + restart: unless-stopped + + whisper-service: + image: ${REGISTRY:-opea}/whisper:${TAG:-latest} + container_name: whisper-service + ports: + - "${DOCSUM_WHISPER_PORT:-7066}:7066" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + restart: unless-stopped + + docsum-backend-server: + image: ${REGISTRY:-opea}/docsum:${TAG:-latest} + container_name: docsum-backend-server + depends_on: + - docsum-tgi-service + - docsum-llm-server + ports: + - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888" + environment: + no_proxy: ${no_proxy} + https_proxy: ${https_proxy} + http_proxy: ${http_proxy} + MEGA_SERVICE_HOST_IP: ${HOST_IP} + LLM_SERVICE_HOST_IP: ${HOST_IP} + ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP} + ipc: host + restart: always + + docsum-gradio-ui: + image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest} + container_name: docsum-ui-server + depends_on: + - docsum-backend-server + ports: + - "${DOCSUM_FRONTEND_PORT:-5173}:5173" + environment: + no_proxy: ${no_proxy} + https_proxy: ${https_proxy} + http_proxy: ${http_proxy} + BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} + DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} + ipc: host + restart: always + +networks: + default: + driver: bridge diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh new file mode 100644 index 0000000000..43e71e0fbf --- /dev/null +++ b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +export HOST_IP="" +export DOCSUM_MAX_INPUT_TOKENS=2048 +export DOCSUM_MAX_TOTAL_TOKENS=4096 +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export DOCSUM_VLLM_SERVICE_PORT="8008" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN="" +export DOCSUM_LLM_SERVER_PORT="9000" +export DOCSUM_WHISPER_PORT="7066" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml index 095fd28c93..dc0d546189 100644 --- a/DocSum/docker_image_build/build.yaml +++ b/DocSum/docker_image_build/build.yaml @@ -47,3 +47,12 @@ services: dockerfile: comps/llms/src/doc-summarization/Dockerfile extends: docsum image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} + vllm_rocm: + build: + args: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + no_proxy: ${no_proxy} + context: ../ + dockerfile: ./Dockerfile-vllm-rocm + image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh new file mode 100644 index 0000000000..d0919a019a --- /dev/null +++ b/DocSum/tests/test_compose_on_rocm_vllm.sh @@ -0,0 +1,249 @@ +#!/bin/bash +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +set -xe +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') +export MAX_INPUT_TOKENS=1024 +export MAX_TOTAL_TOKENS=2048 +export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" +export HOST_IP=${ip_address} +export DOCSUM_VLLM_SERVICE_PORT="8008" +export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} +export DOCSUM_LLM_SERVER_PORT="9000" +export DOCSUM_WHISPER_PORT="7066" +export DOCSUM_BACKEND_SERVER_PORT="8888" +export DOCSUM_FRONTEND_PORT="5173" +export MEGA_SERVICE_HOST_IP=${HOST_IP} +export LLM_SERVICE_HOST_IP=${HOST_IP} +export ASR_SERVICE_HOST_IP=${HOST_IP} +export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" + +function build_docker_images() { + opea_branch=${opea_branch:-"main"} + # If the opea_branch isn't main, replace the git clone branch in Dockerfile. + if [[ "${opea_branch}" != "main" ]]; then + cd $WORKPATH + OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" + NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" + find . -type f -name "Dockerfile*" | while read -r file; do + echo "Processing file: $file" + sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" + done + fi + + cd $WORKPATH/docker_image_build + git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + + echo "Build all the images with --no-cache, check docker_image_build.log for details..." + service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper" + docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log + + docker images && sleep 1s +} + +function start_services() { + cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm + sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env + # Start Docker Containers + docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log + sleep 1m +} + +function validate_services() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + + echo "===========================================" + + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "EXPECTED_RESULT==> $EXPECTED_RESULT" + echo "CONTENT==> $CONTENT" + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +get_base64_str() { + local file_name=$1 + base64 -w 0 "$file_name" +} + +# Function to generate input data for testing based on the document type +input_data_for_test() { + local document_type=$1 + case $document_type in + ("text") + echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are." + ;; + ("audio") + get_base64_str "$WORKPATH/tests/data/test.wav" + ;; + ("video") + get_base64_str "$WORKPATH/tests/data/test.mp4" + ;; + (*) + echo "Invalid document type" >&2 + exit 1 + ;; + esac +} + +function validate_microservices() { + # Check if the microservices are running correctly. + + # whisper microservice + ulimit -s 65536 + validate_services \ + "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \ + '{"asr_result":"well"}' \ + "whisper-service" \ + "whisper-service" \ + "{\"audio\": \"$(input_data_for_test "audio")\"}" + + # vLLM service + validate_services \ + "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \ + "generated_text" \ + "docsum-vllm-service" \ + "docsum-vllm-service" \ + '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}' + + # llm microservice + validate_services \ + "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \ + "text" \ + "docsum-llm-server" \ + "docsum-llm-server" \ + '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' + +} + +function validate_megaservice() { + local SERVICE_NAME="docsum-backend-server" + local DOCKER_NAME="docsum-backend-server" + local EXPECTED_RESULT="[DONE]" + local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." + local URL="${host_ip}:8888/v1/docsum" + local DATA_TYPE="type=text" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL") + + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +function validate_megaservice_json() { + # Curl the Mega Service + echo "" + echo ">>> Checking text data with Content-Type: application/json" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' + + echo ">>> Checking audio data" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + "{\"type\": \"audio\", \"messages\": \"$(input_data_for_test "audio")\"}" + + echo ">>> Checking video data" + validate_services \ + "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ + "[DONE]" \ + "docsum-backend-server" \ + "docsum-backend-server" \ + "{\"type\": \"video\", \"messages\": \"$(input_data_for_test "video")\"}" + +} + +function stop_docker() { + cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/ + docker compose stop && docker compose rm -f +} + +function main() { + echo "===========================================" + echo ">>>> Stopping any running Docker containers..." + stop_docker + + echo "===========================================" + if [[ "$IMAGE_REPO" == "opea" ]]; then + echo ">>>> Building Docker images..." + build_docker_images + fi + + echo "===========================================" + echo ">>>> Starting Docker services..." + start_services + + echo "===========================================" + echo ">>>> Validating microservices..." + validate_microservices + + echo "===========================================" + echo ">>>> Validating megaservice..." + validate_megaservice + echo ">>>> Validating validate_megaservice_json..." + validate_megaservice_json + + echo "===========================================" + echo ">>>> Stopping Docker containers..." + stop_docker + + echo "===========================================" + echo ">>>> Pruning Docker system..." + echo y | docker system prune + echo ">>>> Docker system pruned successfully." + echo "===========================================" +} + +main From 6b2b29703e30add06fac3e955f4410b86ed35a69 Mon Sep 17 00:00:00 2001 From: Chingis Yundunov Date: Thu, 13 Feb 2025 10:07:05 +0700 Subject: [PATCH 023/226] DocSum - fix main Signed-off-by: Chingis Yundunov Signed-off-by: Chingis Yundunov --- DocSum/Dockerfile-vllm-rocm | 18 -- .../amd/gpu/rocm-vllm/README.md | 175 ------------ .../amd/gpu/rocm-vllm/compose.yaml | 107 -------- .../amd/gpu/rocm-vllm/set_env.sh | 16 -- DocSum/docker_image_build/build.yaml | 9 - DocSum/tests/test_compose_on_rocm_vllm.sh | 249 ------------------ 6 files changed, 574 deletions(-) delete mode 100644 DocSum/Dockerfile-vllm-rocm delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/README.md delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml delete mode 100644 DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh delete mode 100644 DocSum/tests/test_compose_on_rocm_vllm.sh diff --git a/DocSum/Dockerfile-vllm-rocm b/DocSum/Dockerfile-vllm-rocm deleted file mode 100644 index f0e8a8743a..0000000000 --- a/DocSum/Dockerfile-vllm-rocm +++ /dev/null @@ -1,18 +0,0 @@ -FROM rocm/vllm-dev:main - -# Set the working directory -WORKDIR /workspace - -# Copy the api_server.py into the image -ADD https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.7.0/vllm/entrypoints/openai/api_server.py /workspace/api_server.py - -# Expose the port used by the API server -EXPOSE 8011 - -# Set environment variables -ENV HUGGINGFACE_HUB_CACHE=/workspace -ENV WILM_USE_TRITON_FLASH_ATTENTION=0 -ENV PYTORCH_JIT=0 - -# Set the entrypoint to the api_server.py script -ENTRYPOINT ["python3", "/workspace/api_server.py"] diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md b/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md deleted file mode 100644 index 4d41a5cd31..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/README.md +++ /dev/null @@ -1,175 +0,0 @@ -# Build and deploy DocSum Application on AMD GPU (ROCm) - -## Build images - -## 🚀 Build Docker Images - -First of all, you need to build Docker Images locally and install the python package of it. - -### 1. Build LLM Image - -```bash -git clone https://github.com/opea-project/GenAIComps.git -cd GenAIComps -docker build -t opea/llm-docsum-tgi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/doc-summarization/Dockerfile . -``` - -Then run the command `docker images`, you will have the following four Docker Images: - -### 2. Build MegaService Docker Image - -To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `docsum.py` Python script. Build the MegaService Docker image via below command: - -```bash -git clone https://github.com/opea-project/GenAIExamples -cd GenAIExamples/DocSum/ -docker build -t opea/docsum:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . -``` - -### 3. Build UI Docker Image - -Build the frontend Docker image via below command: - -```bash -cd GenAIExamples/DocSum/ui -docker build -t opea/docsum-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . -``` - -Then run the command `docker images`, you will have the following Docker Images: - -1. `opea/llm-docsum-tgi:latest` -2. `opea/docsum:latest` -3. `opea/docsum-ui:latest` - -### 4. Build React UI Docker Image - -Build the frontend Docker image via below command: - -```bash -cd GenAIExamples/DocSum/ui -export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/docsum" -docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f ./docker/Dockerfile.react . - -docker build -t opea/docsum-react-ui:latest --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . -``` - -Then run the command `docker images`, you will have the following Docker Images: - -1. `opea/llm-docsum-tgi:latest` -2. `opea/docsum:latest` -3. `opea/docsum-ui:latest` -4. `opea/docsum-react-ui:latest` - -## 🚀 Start Microservices and MegaService - -### Required Models - -Default model is "Intel/neural-chat-7b-v3-3". Change "LLM_MODEL_ID" in environment variables below if you want to use another model. -For gated models, you also need to provide [HuggingFace token](https://huggingface.co/docs/hub/security-tokens) in "HUGGINGFACEHUB_API_TOKEN" environment variable. - -### Setup Environment Variables - -Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below. - -```bash -export DOCSUM_TGI_IMAGE="ghcr.io/huggingface/text-generation-inference:2.3.1-rocm" -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export HOST_IP=${host_ip} -export DOCSUM_TGI_SERVICE_PORT="18882" -export DOCSUM_TGI_LLM_ENDPOINT="http://${HOST_IP}:${DOCSUM_TGI_SERVICE_PORT}" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} -export DOCSUM_LLM_SERVER_PORT="8008" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export DocSum_COMPONENT_NAME="OpeaDocSumTgi" -``` - -Note: Please replace with `host_ip` with your external IP address, do not use localhost. - -Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered, where is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) - -Example for set isolation for 1 GPU - -``` - - /dev/dri/card0:/dev/dri/card0 - - /dev/dri/renderD128:/dev/dri/renderD128 -``` - -Example for set isolation for 2 GPUs - -``` - - /dev/dri/card0:/dev/dri/card0 - - /dev/dri/renderD128:/dev/dri/renderD128 - - /dev/dri/card1:/dev/dri/card1 - - /dev/dri/renderD129:/dev/dri/renderD129 -``` - -Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) - -### Start Microservice Docker Containers - -```bash -cd GenAIExamples/DocSum/docker_compose/amd/gpu/rocm -docker compose up -d -``` - -### Validate Microservices - -1. TGI Service - - ```bash - curl http://${host_ip}:8008/generate \ - -X POST \ - -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \ - -H 'Content-Type: application/json' - ``` - -2. LLM Microservice - - ```bash - curl http://${host_ip}:9000/v1/docsum \ - -X POST \ - -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \ - -H 'Content-Type: application/json' - ``` - -3. MegaService - - ```bash - curl http://${host_ip}:8888/v1/docsum -H "Content-Type: application/json" -d '{ - "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5.","max_tokens":32, "language":"en", "stream":false - }' - ``` - -## 🚀 Launch the Svelte UI - -Open this URL `http://{host_ip}:5173` in your browser to access the frontend. - -![project-screenshot](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/93b1ed4b-4b76-4875-927e-cc7818b4825b) - -Here is an example for summarizing a article. - -![image](https://github.com/intel-ai-tce/GenAIExamples/assets/21761437/67ecb2ec-408d-4e81-b124-6ded6b833f55) - -## 🚀 Launch the React UI (Optional) - -To access the React-based frontend, modify the UI service in the `compose.yaml` file. Replace `docsum-rocm-ui-server` service with the `docsum-rocm-react-ui-server` service as per the config below: - -```yaml -docsum-rocm-react-ui-server: - image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest} - container_name: docsum-rocm-react-ui-server - depends_on: - - docsum-rocm-backend-server - ports: - - "5174:80" - environment: - - no_proxy=${no_proxy} - - https_proxy=${https_proxy} - - http_proxy=${http_proxy} - - DOC_BASE_URL=${BACKEND_SERVICE_ENDPOINT} -``` - -Open this URL `http://{host_ip}:5175` in your browser to access the frontend. - -![project-screenshot](../../../../assets/img/docsum-ui-react.png) diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml b/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml deleted file mode 100644 index 037aa06395..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/compose.yaml +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -services: - docsum-vllm-service: - image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} - container_name: docsum-vllm-service - ports: - - "${DOCSUM_VLLM_SERVICE_PORT:-8081}:8011" - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_HUB_DISABLE_PROGRESS_BARS: 1 - HF_HUB_ENABLE_HF_TRANSFER: 0 - WILM_USE_TRITON_FLASH_ATTENTION: 0 - PYTORCH_JIT: 0 - volumes: - - "./data:/data" - shm_size: 20G - devices: - - /dev/kfd:/dev/kfd - - /dev/dri/:/dev/dri/ - cap_add: - - SYS_PTRACE - group_add: - - video - security_opt: - - seccomp:unconfined - - apparmor=unconfined - command: "--model ${DOCSUM_LLM_MODEL_ID} --swap-space 16 --disable-log-requests --dtype float16 --tensor-parallel-size 4 --host 0.0.0.0 --port 8011 --num-scheduler-steps 1 --distributed-executor-backend \"mp\"" - ipc: host - - docsum-llm-server: - image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} - container_name: docsum-llm-server - depends_on: - - docsum-vllm-service - ports: - - "${DOCSUM_LLM_SERVER_PORT:-9000}:9000" - ipc: host - cap_add: - - SYS_PTRACE - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - LLM_ENDPOINT: "http://${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}" - HUGGINGFACEHUB_API_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - HF_TOKEN: ${DOCSUM_HUGGINGFACEHUB_API_TOKEN} - LLM_MODEL_ID: ${DOCSUM_LLM_MODEL_ID} - LOGFLAG: ${DOCSUM_LOGFLAG:-False} - MAX_INPUT_TOKENS: ${DOCSUM_MAX_INPUT_TOKENS} - MAX_TOTAL_TOKENS: ${DOCSUM_MAX_TOTAL_TOKENS} - restart: unless-stopped - - whisper-service: - image: ${REGISTRY:-opea}/whisper:${TAG:-latest} - container_name: whisper-service - ports: - - "${DOCSUM_WHISPER_PORT:-7066}:7066" - ipc: host - environment: - no_proxy: ${no_proxy} - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - restart: unless-stopped - - docsum-backend-server: - image: ${REGISTRY:-opea}/docsum:${TAG:-latest} - container_name: docsum-backend-server - depends_on: - - docsum-tgi-service - - docsum-llm-server - ports: - - "${DOCSUM_BACKEND_SERVER_PORT:-8888}:8888" - environment: - no_proxy: ${no_proxy} - https_proxy: ${https_proxy} - http_proxy: ${http_proxy} - MEGA_SERVICE_HOST_IP: ${HOST_IP} - LLM_SERVICE_HOST_IP: ${HOST_IP} - ASR_SERVICE_HOST_IP: ${ASR_SERVICE_HOST_IP} - ipc: host - restart: always - - docsum-gradio-ui: - image: ${REGISTRY:-opea}/docsum-gradio-ui:${TAG:-latest} - container_name: docsum-ui-server - depends_on: - - docsum-backend-server - ports: - - "${DOCSUM_FRONTEND_PORT:-5173}:5173" - environment: - no_proxy: ${no_proxy} - https_proxy: ${https_proxy} - http_proxy: ${http_proxy} - BACKEND_SERVICE_ENDPOINT: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} - DOC_BASE_URL: ${DOCSUM_BACKEND_SERVICE_ENDPOINT} - ipc: host - restart: always - -networks: - default: - driver: bridge diff --git a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh b/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh deleted file mode 100644 index 43e71e0fbf..0000000000 --- a/DocSum/docker_compose/amd/gpu/rocm-vllm/set_env.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -export HOST_IP="" -export DOCSUM_MAX_INPUT_TOKENS=2048 -export DOCSUM_MAX_TOTAL_TOKENS=4096 -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export DOCSUM_VLLM_SERVICE_PORT="8008" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN="" -export DOCSUM_LLM_SERVER_PORT="9000" -export DOCSUM_WHISPER_PORT="7066" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export DOCSUM_BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml index dc0d546189..095fd28c93 100644 --- a/DocSum/docker_image_build/build.yaml +++ b/DocSum/docker_image_build/build.yaml @@ -47,12 +47,3 @@ services: dockerfile: comps/llms/src/doc-summarization/Dockerfile extends: docsum image: ${REGISTRY:-opea}/llm-docsum:${TAG:-latest} - vllm_rocm: - build: - args: - http_proxy: ${http_proxy} - https_proxy: ${https_proxy} - no_proxy: ${no_proxy} - context: ../ - dockerfile: ./Dockerfile-vllm-rocm - image: ${REGISTRY:-opea}/llm-vllm-rocm:${TAG:-latest} diff --git a/DocSum/tests/test_compose_on_rocm_vllm.sh b/DocSum/tests/test_compose_on_rocm_vllm.sh deleted file mode 100644 index d0919a019a..0000000000 --- a/DocSum/tests/test_compose_on_rocm_vllm.sh +++ /dev/null @@ -1,249 +0,0 @@ -#!/bin/bash -# Copyright (C) 2024 Advanced Micro Devices, Inc. -# SPDX-License-Identifier: Apache-2.0 - -set -xe -IMAGE_REPO=${IMAGE_REPO:-"opea"} -IMAGE_TAG=${IMAGE_TAG:-"latest"} -echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" -echo "TAG=IMAGE_TAG=${IMAGE_TAG}" - -WORKPATH=$(dirname "$PWD") -LOG_PATH="$WORKPATH/tests" -ip_address=$(hostname -I | awk '{print $1}') -export MAX_INPUT_TOKENS=1024 -export MAX_TOTAL_TOKENS=2048 -export DOCSUM_LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" -export HOST_IP=${ip_address} -export DOCSUM_VLLM_SERVICE_PORT="8008" -export DOCSUM_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} -export DOCSUM_LLM_SERVER_PORT="9000" -export DOCSUM_WHISPER_PORT="7066" -export DOCSUM_BACKEND_SERVER_PORT="8888" -export DOCSUM_FRONTEND_PORT="5173" -export MEGA_SERVICE_HOST_IP=${HOST_IP} -export LLM_SERVICE_HOST_IP=${HOST_IP} -export ASR_SERVICE_HOST_IP=${HOST_IP} -export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" - -function build_docker_images() { - opea_branch=${opea_branch:-"main"} - # If the opea_branch isn't main, replace the git clone branch in Dockerfile. - if [[ "${opea_branch}" != "main" ]]; then - cd $WORKPATH - OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" - NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" - find . -type f -name "Dockerfile*" | while read -r file; do - echo "Processing file: $file" - sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" - done - fi - - cd $WORKPATH/docker_image_build - git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git - - echo "Build all the images with --no-cache, check docker_image_build.log for details..." - service_list="vllm_rocm llm-docsum docsum docsum-gradio-ui whisper" - docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log - - docker images && sleep 1s -} - -function start_services() { - cd "$WORKPATH"/docker_compose/amd/gpu/rocm-vllm - sed -i "s/backend_address/$ip_address/g" "$WORKPATH"/ui/svelte/.env - # Start Docker Containers - docker compose up -d > "${LOG_PATH}"/start_services_with_compose.log - sleep 1m -} - -function validate_services() { - local URL="$1" - local EXPECTED_RESULT="$2" - local SERVICE_NAME="$3" - local DOCKER_NAME="$4" - local INPUT_DATA="$5" - - local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") - - echo "===========================================" - - if [ "$HTTP_STATUS" -eq 200 ]; then - echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." - - local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) - - if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then - echo "[ $SERVICE_NAME ] Content is as expected." - else - echo "EXPECTED_RESULT==> $EXPECTED_RESULT" - echo "CONTENT==> $CONTENT" - echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - - fi - else - echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - sleep 1s -} - -get_base64_str() { - local file_name=$1 - base64 -w 0 "$file_name" -} - -# Function to generate input data for testing based on the document type -input_data_for_test() { - local document_type=$1 - case $document_type in - ("text") - echo "THIS IS A TEST >>>> and a number of states are starting to adopt them voluntarily special correspondent john delenco of education week reports it takes just 10 minutes to cross through gillette wyoming this small city sits in the northeast corner of the state surrounded by 100s of miles of prairie but schools here in campbell county are on the edge of something big the next generation science standards you are going to build a strand of dna and you are going to decode it and figure out what that dna actually says for christy mathis at sage valley junior high school the new standards are about learning to think like a scientist there is a lot of really good stuff in them every standard is a performance task it is not you know the child needs to memorize these things it is the student needs to be able to do some pretty intense stuff we are analyzing we are critiquing we are." - ;; - ("audio") - get_base64_str "$WORKPATH/tests/data/test.wav" - ;; - ("video") - get_base64_str "$WORKPATH/tests/data/test.mp4" - ;; - (*) - echo "Invalid document type" >&2 - exit 1 - ;; - esac -} - -function validate_microservices() { - # Check if the microservices are running correctly. - - # whisper microservice - ulimit -s 65536 - validate_services \ - "${HOST_IP}:${DOCSUM_WHISPER_PORT}/v1/asr" \ - '{"asr_result":"well"}' \ - "whisper-service" \ - "whisper-service" \ - "{\"audio\": \"$(input_data_for_test "audio")\"}" - - # vLLM service - validate_services \ - "${HOST_IP}:${DOCSUM_VLLM_SERVICE_PORT}/v1/chat/completions" \ - "generated_text" \ - "docsum-vllm-service" \ - "docsum-vllm-service" \ - '{"model": "Intel/neural-chat-7b-v3-3", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}' - - # llm microservice - validate_services \ - "${HOST_IP}:${DOCSUM_LLM_SERVER_PORT}/v1/docsum" \ - "text" \ - "docsum-llm-server" \ - "docsum-llm-server" \ - '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' - -} - -function validate_megaservice() { - local SERVICE_NAME="docsum-backend-server" - local DOCKER_NAME="docsum-backend-server" - local EXPECTED_RESULT="[DONE]" - local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." - local URL="${host_ip}:8888/v1/docsum" - local DATA_TYPE="type=text" - - local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL") - - if [ "$HTTP_STATUS" -eq 200 ]; then - echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." - - local CONTENT=$(curl -s -X POST -F "$DATA_TYPE" -F "$INPUT_DATA" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) - - if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then - echo "[ $SERVICE_NAME ] Content is as expected." - else - echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - else - echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" - docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log - exit 1 - fi - sleep 1s -} - -function validate_megaservice_json() { - # Curl the Mega Service - echo "" - echo ">>> Checking text data with Content-Type: application/json" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - '{"type": "text", "messages": "Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' - - echo ">>> Checking audio data" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - "{\"type\": \"audio\", \"messages\": \"$(input_data_for_test "audio")\"}" - - echo ">>> Checking video data" - validate_services \ - "${HOST_IP}:${DOCSUM_BACKEND_SERVER_PORT}/v1/docsum" \ - "[DONE]" \ - "docsum-backend-server" \ - "docsum-backend-server" \ - "{\"type\": \"video\", \"messages\": \"$(input_data_for_test "video")\"}" - -} - -function stop_docker() { - cd $WORKPATH/docker_compose/amd/gpu/rocm-vllm/ - docker compose stop && docker compose rm -f -} - -function main() { - echo "===========================================" - echo ">>>> Stopping any running Docker containers..." - stop_docker - - echo "===========================================" - if [[ "$IMAGE_REPO" == "opea" ]]; then - echo ">>>> Building Docker images..." - build_docker_images - fi - - echo "===========================================" - echo ">>>> Starting Docker services..." - start_services - - echo "===========================================" - echo ">>>> Validating microservices..." - validate_microservices - - echo "===========================================" - echo ">>>> Validating megaservice..." - validate_megaservice - echo ">>>> Validating validate_megaservice_json..." - validate_megaservice_json - - echo "===========================================" - echo ">>>> Stopping Docker containers..." - stop_docker - - echo "===========================================" - echo ">>>> Pruning Docker system..." - echo y | docker system prune - echo ">>>> Docker system pruned successfully." - echo "===========================================" -} - -main From f7b3be60ecf572b50a4c0bf3a5eee70513bee904 Mon Sep 17 00:00:00 2001 From: Spycsh <39623753+Spycsh@users.noreply.github.com> Date: Thu, 27 Feb 2025 09:25:49 +0800 Subject: [PATCH 024/226] Align mongo related image names with comps (#1543) - chathistory-mongo-server -> chathistory-mongo (except container names) - feedbackmanagement -> feedbackmanagement-mongo - promptregistry-server/promptregistry-mongo-server -> promptregistry-mongo (except container names) Signed-off-by: Spycsh Signed-off-by: Chingis Yundunov --- ProductivitySuite/docker_compose/intel/cpu/xeon/README.md | 4 ++-- .../docker_compose/intel/cpu/xeon/compose.yaml | 4 ++-- ProductivitySuite/docker_image_build/build.yaml | 8 ++++---- docker_images_list.md | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ProductivitySuite/docker_compose/intel/cpu/xeon/README.md b/ProductivitySuite/docker_compose/intel/cpu/xeon/README.md index 5ab4816096..ce7a874b38 100644 --- a/ProductivitySuite/docker_compose/intel/cpu/xeon/README.md +++ b/ProductivitySuite/docker_compose/intel/cpu/xeon/README.md @@ -45,13 +45,13 @@ docker build --no-cache -t opea/dataprep:latest --build-arg https_proxy=$https_p ### 6. Build Prompt Registry Image ```bash -docker build -t opea/promptregistry-mongo-server:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/prompt_registry/src/Dockerfile . +docker build -t opea/promptregistry-mongo:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/prompt_registry/src/Dockerfile . ``` ### 7. Build Chat History Image ```bash -docker build -t opea/chathistory-mongo-server:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/chathistory/src/Dockerfile . +docker build -t opea/chathistory-mongo:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/chathistory/src/Dockerfile . cd .. ``` diff --git a/ProductivitySuite/docker_compose/intel/cpu/xeon/compose.yaml b/ProductivitySuite/docker_compose/intel/cpu/xeon/compose.yaml index ee7d23a640..149109e4b7 100644 --- a/ProductivitySuite/docker_compose/intel/cpu/xeon/compose.yaml +++ b/ProductivitySuite/docker_compose/intel/cpu/xeon/compose.yaml @@ -310,7 +310,7 @@ services: command: mongod --quiet --logpath /dev/null chathistory-mongo: - image: ${REGISTRY:-opea}/chathistory-mongo-server:${TAG:-latest} + image: ${REGISTRY:-opea}/chathistory-mongo:${TAG:-latest} container_name: chathistory-mongo-server ports: - "6012:6012" @@ -326,7 +326,7 @@ services: restart: unless-stopped promptregistry-mongo: - image: ${REGISTRY:-opea}/promptregistry-mongo-server:${TAG:-latest} + image: ${REGISTRY:-opea}/promptregistry-mongo:${TAG:-latest} container_name: promptregistry-mongo-server ports: - "6018:6018" diff --git a/ProductivitySuite/docker_image_build/build.yaml b/ProductivitySuite/docker_image_build/build.yaml index 807aa1242c..9bfc65e362 100644 --- a/ProductivitySuite/docker_image_build/build.yaml +++ b/ProductivitySuite/docker_image_build/build.yaml @@ -41,18 +41,18 @@ services: dockerfile: comps/dataprep/src/Dockerfile extends: chatqna image: ${REGISTRY:-opea}/dataprep:${TAG:-latest} - promptregistry-mongo-server: + promptregistry-mongo: build: context: GenAIComps dockerfile: comps/prompt_registry/src/Dockerfile extends: chatqna - image: ${REGISTRY:-opea}/promptregistry-mongo-server:${TAG:-latest} - chathistory-mongo-server: + image: ${REGISTRY:-opea}/promptregistry-mongo:${TAG:-latest} + chathistory-mongo: build: context: GenAIComps dockerfile: comps/chathistory/src/Dockerfile extends: chatqna - image: ${REGISTRY:-opea}/chathistory-mongo-server:${TAG:-latest} + image: ${REGISTRY:-opea}/chathistory-mongo:${TAG:-latest} productivity-suite-react-ui-server: build: context: ../ui diff --git a/docker_images_list.md b/docker_images_list.md index 242aad57c0..ab6349fd97 100644 --- a/docker_images_list.md +++ b/docker_images_list.md @@ -56,7 +56,7 @@ Take ChatQnA for example. ChatQnA is a chatbot application service based on the | [opea/agent-ui](https://hub.docker.com/r/opea/agent-ui) | [Link](https://github.com/opea-project/GenAIExamples/blob/main/AgentQnA/ui/docker/Dockerfile) | The docker image exposed the OPEA agent microservice UI entry for GenAI application use | | [opea/asr](https://hub.docker.com/r/opea/asr) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/asr/src/Dockerfile) | The docker image exposed the OPEA Audio-Speech-Recognition microservice for GenAI application use | | [opea/animation](https://hub.docker.com/r/opea/animation) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/animation/src/Dockerfile) | The purpose of the Docker image is to expose the OPEA Avatar Animation microservice for GenAI application use. | -| [opea/chathistory-mongo-server](https://hub.docker.com/r/opea/chathistory-mongo-server) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/chathistory/src/Dockerfile) | The docker image exposes OPEA Chat History microservice which based on MongoDB database, designed to allow user to store, retrieve and manage chat conversations | +| [opea/chathistory-mongo](https://hub.docker.com/r/opea/chathistory-mongo) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/chathistory/src/Dockerfile) | The docker image exposes OPEA Chat History microservice which based on MongoDB database, designed to allow user to store, retrieve and manage chat conversations | | [opea/dataprep](https://hub.docker.com/r/opea/dataprep) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/dataprep/src/Dockerfile) | The docker image exposed the OPEA dataprep microservice for GenAI application use | | [opea/embedding](https://hub.docker.com/r/opea/embedding) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/embeddings/src/Dockerfile) | The docker image exposed the OPEA mosec embedding microservice  for GenAI application use | | [opea/embedding-multimodal-clip](https://hub.docker.com/r/opea/embedding-multimodal-clip) | [Link](https://github.com/opea-project/GenAIComps/blob/main/comps/third_parties/clip/src/Dockerfile) | The docker image exposed the OPEA mosec embedding microservice base on Langchain framework for GenAI application use | From 23fbd2fb44ddf511fdbbcdfea828d975b9ed3eea Mon Sep 17 00:00:00 2001 From: Artem Astafev Date: Thu, 27 Feb 2025 14:26:45 +0700 Subject: [PATCH 025/226] Fix ChatQnA ROCm compose Readme file and absolute path for ROCM CI test (#1159) Signed-off-by: Artem Astafev Signed-off-by: Chingis Yundunov --- ChatQnA/docker_compose/amd/gpu/rocm/README.md | 2 +- ChatQnA/tests/test_compose_on_rocm.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ChatQnA/docker_compose/amd/gpu/rocm/README.md b/ChatQnA/docker_compose/amd/gpu/rocm/README.md index cfd9245541..1bb82838c0 100644 --- a/ChatQnA/docker_compose/amd/gpu/rocm/README.md +++ b/ChatQnA/docker_compose/amd/gpu/rocm/README.md @@ -1,4 +1,4 @@ -# Build and deploy CodeGen Application on AMD GPU (ROCm) +# Build and deploy ChatQnA Application on AMD GPU (ROCm) ## Build MegaService of ChatQnA on AMD ROCm GPU diff --git a/ChatQnA/tests/test_compose_on_rocm.sh b/ChatQnA/tests/test_compose_on_rocm.sh index 9a25392997..d6dc5dfae1 100644 --- a/ChatQnA/tests/test_compose_on_rocm.sh +++ b/ChatQnA/tests/test_compose_on_rocm.sh @@ -45,7 +45,7 @@ export CHATQNA_RERANK_SERVICE_HOST_IP=${HOST_IP} export CHATQNA_LLM_SERVICE_HOST_IP=${HOST_IP} export CHATQNA_NGINX_PORT=80 export CHATQNA_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} -export PATH="/home/huggingface/miniconda3/bin:$PATH" +export PATH="~/miniconda3/bin:$PATH" function build_docker_images() { opea_branch=${opea_branch:-"main"} From 4b47c3ec25543246a9ae015271927b5a05950273 Mon Sep 17 00:00:00 2001 From: XinyaoWa Date: Thu, 27 Feb 2025 23:32:29 +0800 Subject: [PATCH 026/226] Fix async in chatqna bug (#1589) Algin async with comps: related PR: opea-project/GenAIComps#1300 Signed-off-by: Xinyao Wang Signed-off-by: Chingis Yundunov --- ChatQnA/chatqna.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ChatQnA/chatqna.py b/ChatQnA/chatqna.py index afb9706cb2..e25ab4d39a 100644 --- a/ChatQnA/chatqna.py +++ b/ChatQnA/chatqna.py @@ -166,10 +166,10 @@ def align_outputs(self, data, cur_node, inputs, runtime_graph, llm_parameters_di return next_data -def align_generator(self, gen, **kwargs): +async def align_generator(self, gen, **kwargs): # OpenAI response format # b'data:{"id":"","object":"text_completion","created":1725530204,"model":"meta-llama/Meta-Llama-3-8B-Instruct","system_fingerprint":"2.0.1-native","choices":[{"index":0,"delta":{"role":"assistant","content":"?"},"logprobs":null,"finish_reason":null}]}\n\n' - for line in gen: + async for line in gen: line = line.decode("utf-8") start = line.find("{") end = line.rfind("}") + 1 From ed01594f26a9a75f149df6f5e2c7498beff1dbe1 Mon Sep 17 00:00:00 2001 From: "chen, suyue" Date: Fri, 28 Feb 2025 10:30:54 +0800 Subject: [PATCH 027/226] Fix benchmark scripts (#1517) - Align benchmark default config: 1. Update default helm charts version. 2. Add `# mandatory` comment. 3. Update default model ID for LLM. - Fix deploy issue: 1. Support different `replicaCount` for w/ w/o rerank test. 2. Add `max_num_seqs` for vllm. 3. Add resource setting for tune mode. - Fix Benchmark issue: 1. Update `user_queries` and `concurrency` setting. 2. Remove invalid parameters. 3. Fix `dataset` and `prompt` setting. And dataset ingest into db. 5. Fix the benchmark hang issue with large user queries. Update `"processes": 16` will fix this issue. 6. Update the eval_path setting logical. - Optimize benchmark readme. - Optimize the log path to make the logs more readable. Signed-off-by: chensuyue Signed-off-by: Cathy Zhang Signed-off-by: letonghan Signed-off-by: Chingis Yundunov --- ChatQnA/benchmark_chatqna.yaml | 101 +++++++++----- README-deploy-benchmark.md | 143 ++++++++++++++++++-- benchmark.py | 233 ++++++++++++++++++++++++--------- deploy.py | 150 +++++++++++++-------- deploy_and_benchmark.py | 220 ++++++++++++++++++++++++------- requirements.txt | 1 + 6 files changed, 641 insertions(+), 207 deletions(-) diff --git a/ChatQnA/benchmark_chatqna.yaml b/ChatQnA/benchmark_chatqna.yaml index c608b8afbf..407d555ceb 100644 --- a/ChatQnA/benchmark_chatqna.yaml +++ b/ChatQnA/benchmark_chatqna.yaml @@ -3,55 +3,89 @@ deploy: device: gaudi - version: 1.1.0 + version: 1.2.0 modelUseHostPath: /mnt/models - HUGGINGFACEHUB_API_TOKEN: "" + HUGGINGFACEHUB_API_TOKEN: "" # mandatory node: [1, 2, 4, 8] namespace: "" + timeout: 1000 # timeout in seconds for services to be ready, default 30 minutes + interval: 5 # interval in seconds between service ready checks, default 5 seconds services: backend: - instance_num: [2, 2, 4, 8] - cores_per_instance: "" - memory_capacity: "" + resources: + enabled: False + cores_per_instance: "16" + memory_capacity: "8000Mi" + replicaCount: [1, 2, 4, 8] teirerank: enabled: True model_id: "" + resources: + enabled: False + cards_per_instance: 1 replicaCount: [1, 1, 1, 1] - cards_per_instance: 1 tei: model_id: "" + resources: + enabled: False + cores_per_instance: "80" + memory_capacity: "20000Mi" replicaCount: [1, 2, 4, 8] - cores_per_instance: "" - memory_capacity: "" llm: - engine: tgi - model_id: "" - replicaCount: [7, 15, 31, 63] - max_batch_size: [1, 2, 4, 8] - max_input_length: "" - max_total_tokens: "" - max_batch_total_tokens: "" - max_batch_prefill_tokens: "" - cards_per_instance: 1 + engine: vllm # or tgi + model_id: "meta-llama/Meta-Llama-3-8B-Instruct" # mandatory + replicaCount: + with_teirerank: [7, 15, 31, 63] # When teirerank.enabled is True + without_teirerank: [8, 16, 32, 64] # When teirerank.enabled is False + resources: + enabled: False + cards_per_instance: 1 + model_params: + vllm: # VLLM specific parameters + batch_params: + enabled: True + max_num_seqs: [1, 2, 4, 8] # Each value triggers an LLM service upgrade + token_params: + enabled: False + max_input_length: "" + max_total_tokens: "" + max_batch_total_tokens: "" + max_batch_prefill_tokens: "" + tgi: # TGI specific parameters + batch_params: + enabled: True + max_batch_size: [1, 2, 4, 8] # Each value triggers an LLM service upgrade + token_params: + enabled: False + max_input_length: "1280" + max_total_tokens: "2048" + max_batch_total_tokens: "65536" + max_batch_prefill_tokens: "4096" data-prep: + resources: + enabled: False + cores_per_instance: "" + memory_capacity: "" replicaCount: [1, 1, 1, 1] - cores_per_instance: "" - memory_capacity: "" retriever-usvc: - replicaCount: [2, 2, 4, 8] - cores_per_instance: "" - memory_capacity: "" + resources: + enabled: False + cores_per_instance: "8" + memory_capacity: "8000Mi" + replicaCount: [1, 2, 4, 8] redis-vector-db: + resources: + enabled: False + cores_per_instance: "" + memory_capacity: "" replicaCount: [1, 1, 1, 1] - cores_per_instance: "" - memory_capacity: "" chatqna-ui: replicaCount: [1, 1, 1, 1] @@ -61,22 +95,17 @@ deploy: benchmark: # http request behavior related fields - concurrency: [1, 2, 4] - totoal_query_num: [2048, 4096] - duration: [5, 10] # unit minutes - query_num_per_concurrency: [4, 8, 16] - possion: True - possion_arrival_rate: 1.0 + user_queries: [640] + concurrency: [128] + load_shape_type: "constant" # "constant" or "poisson" + poisson_arrival_rate: 1.0 # only used when load_shape_type is "poisson" warmup_iterations: 10 seed: 1024 # workload, all of the test cases will run for benchmark - test_cases: - - chatqnafixed - - chatqna_qlist_pubmed: - dataset: pub_med10 # pub_med10, pub_med100, pub_med1000 - user_queries: [1, 2, 4] - query_token_size: 128 # if specified, means fixed query token size will be sent out + bench_target: [chatqnafixed, chatqna_qlist_pubmed] # specify the bench_target for benchmark + dataset: ["/home/sdp/upload_file.txt", "/home/sdp/pubmed_10000.txt"] # specify the absolute path to the dataset file + prompt: [10, 1000] # set the prompt length for the chatqna_qlist_pubmed workload, set to 10 for chatqnafixed workload llm: # specify the llm output token size diff --git a/README-deploy-benchmark.md b/README-deploy-benchmark.md index 4b813cccca..9f1a13f8ff 100644 --- a/README-deploy-benchmark.md +++ b/README-deploy-benchmark.md @@ -11,10 +11,9 @@ We aim to run these benchmarks and share them with the OPEA community for three ## Table of Contents - [Prerequisites](#prerequisites) -- [Overview](#overview) - - [Using deploy_and_benchmark.py](#using-deploy_and_benchmark.py-recommended) - [Data Preparation](#data-preparation) -- [Configuration](#configuration) +- [Running Deploy and Benchmark Tests](#running-deploy-and-benchmark-tests) +- [Troubleshooting](#troubleshooting) ## Prerequisites @@ -25,8 +24,50 @@ Before running the benchmarks, ensure you have: - Kubernetes installation: Use [kubespray](https://github.com/opea-project/docs/blob/main/guide/installation/k8s_install/k8s_install_kubespray.md) or other official Kubernetes installation guides - (Optional) [Kubernetes set up guide on Intel Gaudi product](https://github.com/opea-project/GenAIInfra/blob/main/README.md#setup-kubernetes-cluster) -2. **Configuration YAML** - The configuration file (e.g., `./ChatQnA/benchmark_chatqna.yaml`) consists of two main sections: deployment and benchmarking. Required fields must be filled with valid values (like the Hugging Face token). For all other fields, you can either customize them according to your needs or leave them empty ("") to use the default values from the [helm charts](https://github.com/opea-project/GenAIInfra/tree/main/helm-charts). +2. **Configuration YAML** + The configuration file (e.g., `./ChatQnA/benchmark_chatqna.yaml`) consists of two main sections: deployment and benchmarking. Required fields with `# mandatory` comment must be filled with valid values, such as `HUGGINGFACEHUB_API_TOKEN`. For all other fields, you can either customize them according to our needs or leave them empty ("") to use the default values from the [helm charts](https://github.com/opea-project/GenAIInfra/tree/main/helm-charts). + + **Default Models**: + + - LLM: `meta-llama/Meta-Llama-3-8B-Instruct` (Required: must be specified as it's shared between deployment and benchmarking phases) + - Embedding: `BAAI/bge-base-en-v1.5` + - Reranking: `BAAI/bge-reranker-base` + + You can customize which models to use by setting the `model_id` field in the corresponding service section. Note that the LLM model must be specified in the configuration as it is used by both deployment and benchmarking processes. + + **Important Notes**: + + - For Gaudi deployments: + - LLM service runs on Gaudi devices + - If enabled, the reranking service (teirerank) also runs on Gaudi devices + - **Llama Model Access**: + - Downloading Llama models requires both: + 1. HuggingFace API token + 2. Special authorization from Meta + - Please visit [meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) to request access + - Deployment will fail if model download is unsuccessful due to missing authorization + + **Node and Replica Configuration**: + + ```yaml + node: [1, 2, 4, 8] # Number of nodes to deploy + replicaCount: [1, 2, 4, 8] # Must align with node configuration + ``` + + The `replicaCount` values must align with the `node` configuration by index: + + - When deploying on 1 node → uses replicaCount[0] = 1 + - When deploying on 2 nodes → uses replicaCount[1] = 2 + - When deploying on 4 nodes → uses replicaCount[2] = 4 + - When deploying on 8 nodes → uses replicaCount[3] = 8 + + Note: Model parameters that accept lists (e.g., `max_batch_size`, `max_num_seqs`) are deployment parameters that affect model service behavior but not the number of service instances. When these parameters are lists, each value will trigger a service upgrade followed by a new round of testing, while maintaining the same number of service instances. + +3. **Install required Python packages** + Run the following command to install all necessary dependencies: + ```bash + pip install -r requirements.txt + ``` ## Data Preparation @@ -34,36 +75,114 @@ Before running benchmarks, you need to: 1. **Prepare Test Data** - - Download the retrieval file: + - Testing for general benchmark target: + + Download the retrieval file using the command below for data ingestion in RAG: + ```bash wget https://github.com/opea-project/GenAIEval/tree/main/evals/benchmark/data/upload_file.txt ``` - - For the `chatqna_qlist_pubmed` test case, prepare `pubmed_${max_lines}.txt` by following this [README](https://github.com/opea-project/GenAIEval/blob/main/evals/benchmark/stresscli/README_Pubmed_qlist.md) + + - Testing for pubmed benchmark target: + + For the `chatqna_qlist_pubmed` test case, prepare `pubmed_${max_lines}.txt` by following this [README](https://github.com/opea-project/GenAIEval/blob/main/evals/benchmark/stresscli/README_Pubmed_qlist.md) + + After the data is prepared, please update the `absolute path` of this file in the benchmark.yaml file. For example, in the `ChatQnA/benchmark_chatqna.yaml` file, `/home/sdp/upload_file.txt` should be replaced by your file path. 2. **Prepare Model Files (Recommended)** ```bash pip install -U "huggingface_hub[cli]" sudo mkdir -p /mnt/models sudo chmod 777 /mnt/models - huggingface-cli download --cache-dir /mnt/models Intel/neural-chat-7b-v3-3 + huggingface-cli download --cache-dir /mnt/models meta-llama/Meta-Llama-3-8B-Instruct ``` -## Overview +## Running Deploy and Benchmark Tests The benchmarking process consists of two main components: deployment and benchmarking. We provide `deploy_and_benchmark.py` as a unified entry point that combines both steps. -### Using deploy_and_benchmark.py (Recommended) +### Running the Tests -The script `deploy_and_benchmark.py` serves as the main entry point. Here's an example using ChatQnA configuration (you can replace it with any other example's configuration YAML file): +The script `deploy_and_benchmark.py` serves as the main entry point. You can use any example's configuration YAML file. Here are examples using ChatQnA configuration: 1. For a specific number of nodes: ```bash + # Default OOB (Out of Box) mode python deploy_and_benchmark.py ./ChatQnA/benchmark_chatqna.yaml --target-node 1 + + # Or specify test mode explicitly + python deploy_and_benchmark.py ./ChatQnA/benchmark_chatqna.yaml --target-node 1 --test-mode [oob|tune] ``` 2. For all node configurations: + ```bash + # Default OOB (Out of Box) mode python deploy_and_benchmark.py ./ChatQnA/benchmark_chatqna.yaml + + # Or specify test mode explicitly + python deploy_and_benchmark.py ./ChatQnA/benchmark_chatqna.yaml --test-mode [oob|tune] + ``` + + This will process all node configurations defined in your YAML file. + +### Test Modes + +The script provides two test modes controlled by the `--test-mode` parameter: + +1. **OOB (Out of Box) Mode** - Default + + ```bash + --test-mode oob # or omit the parameter + ``` + + - Uses enabled configurations only: + - Resources: Only uses resources when `resources.enabled` is True + - Model parameters: + - Uses batch parameters when `batch_params.enabled` is True + - Uses token parameters when `token_params.enabled` is True + - Suitable for basic functionality testing with selected optimizations + +2. **Tune Mode** + ```bash + --test-mode tune ``` - This will iterate through the node list in your configuration YAML file, performing deployment and benchmarking for each node count. + - Applies all configurations regardless of enabled status: + - Resource-related parameters: + - `resources.cores_per_instance`: CPU cores allocation + - `resources.memory_capacity`: Memory allocation + - `resources.cards_per_instance`: GPU/Accelerator cards allocation + - Model parameters: + - Batch parameters: + - `max_batch_size`: Maximum batch size (TGI engine) + - `max_num_seqs`: Maximum number of sequences (vLLM engine) + - Token parameters: + - `max_input_length`: Maximum input sequence length + - `max_total_tokens`: Maximum total tokens per request + - `max_batch_total_tokens`: Maximum total tokens in a batch + - `max_batch_prefill_tokens`: Maximum tokens in prefill phase + +Choose "oob" mode when you want to selectively enable optimizations, or "tune" mode when you want to apply all available optimizations regardless of their enabled status. + +### Troubleshooting + +**Helm Chart Directory Issues** + +- During execution, the script downloads and extracts the Helm chart to a directory named after your example +- The directory name is derived from your input YAML file path + - For example: if your input is `./ChatQnA/benchmark_chatqna.yaml`, the extracted directory will be `chatqna/` +- In some error cases, this directory might not be properly cleaned up +- If you encounter deployment issues, check if there's a leftover Helm chart directory: + + ```bash + # Example: for ./ChatQnA/benchmark_chatqna.yaml + ls -la chatqna/ + + # Clean up if needed + rm -rf chatqna/ + ``` + +- After cleaning up the directory, try running the deployment again + +Note: Always ensure there are no leftover Helm chart directories from previous failed runs before starting a new deployment. diff --git a/benchmark.py b/benchmark.py index fb20367c08..202a2cb012 100644 --- a/benchmark.py +++ b/benchmark.py @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 import os -import sys from datetime import datetime +import requests import yaml from evals.benchmark.stresscli.commands.load_test import locust_runtests from kubernetes import client, config @@ -25,17 +25,15 @@ def construct_benchmark_config(test_suite_config): """Extract relevant data from the YAML based on the specified test cases.""" return { - "concurrency": test_suite_config.get("concurrency", []), - "totoal_query_num": test_suite_config.get("user_queries", []), - "duration:": test_suite_config.get("duration:", []), - "query_num_per_concurrency": test_suite_config.get("query_num_per_concurrency", []), - "possion": test_suite_config.get("possion", False), - "possion_arrival_rate": test_suite_config.get("possion_arrival_rate", 1.0), + "user_queries": test_suite_config.get("user_queries", [1]), + "concurrency": test_suite_config.get("concurrency", [1]), + "load_shape_type": test_suite_config.get("load_shape_type", "constant"), + "poisson_arrival_rate": test_suite_config.get("poisson_arrival_rate", 1.0), "warmup_iterations": test_suite_config.get("warmup_iterations", 10), "seed": test_suite_config.get("seed", None), - "test_cases": test_suite_config.get("test_cases", ["chatqnafixed"]), - "user_queries": test_suite_config.get("user_queries", [1]), - "query_token_size": test_suite_config.get("query_token_size", 128), + "bench_target": test_suite_config.get("bench_target", ["chatqnafixed"]), + "dataset": test_suite_config.get("dataset", ""), + "prompt": test_suite_config.get("prompt", [10]), "llm_max_token_size": test_suite_config.get("llm", {}).get("max_token_size", [128]), } @@ -97,17 +95,11 @@ def _get_service_ip(service_name, deployment_type="k8s", service_ip=None, servic return svc_ip, port -def _create_yaml_content(service, base_url, bench_target, test_phase, num_queries, test_params): +def _create_yaml_content(service, base_url, bench_target, test_phase, num_queries, test_params, concurrency=1): """Create content for the run.yaml file.""" - # If a load shape includes the parameter concurrent_level, - # the parameter will be passed to Locust to launch fixed - # number of simulated users. - concurrency = 1 - if num_queries >= 0: - concurrency = max(1, num_queries // test_params["concurrent_level"]) - else: - concurrency = test_params["concurrent_level"] + # calculate the number of concurrent users + concurrent_level = int(num_queries // concurrency) import importlib.util @@ -116,16 +108,21 @@ def _create_yaml_content(service, base_url, bench_target, test_phase, num_querie print(spec) # get folder path of opea-eval - eval_path = None - import pkg_resources + eval_path = os.getenv("EVAL_PATH", "") + if not eval_path: + import pkg_resources - for dist in pkg_resources.working_set: - if "opea-eval" in dist.project_name: - eval_path = dist.location + for dist in pkg_resources.working_set: + if "opea-eval" in dist.project_name: + eval_path = dist.location + break if not eval_path: - print("Fail to load opea-eval package. Please install it first.") + print("Fail to find the opea-eval package. Please set/install it first.") exit(1) + load_shape = test_params["load_shape"] + load_shape["params"]["constant"] = {"concurrent_level": concurrent_level} + yaml_content = { "profile": { "storage": {"hostpath": test_params["test_output_dir"]}, @@ -133,8 +130,9 @@ def _create_yaml_content(service, base_url, bench_target, test_phase, num_querie "tool": "locust", "locustfile": os.path.join(eval_path, "evals/benchmark/stresscli/locust/aistress.py"), "host": base_url, + "run-time": test_params["run_time"], "stop-timeout": test_params["query_timeout"], - "processes": 2, + "processes": 16, # set to 2 by default "namespace": test_params["namespace"], "bench-target": bench_target, "service-metric-collect": test_params["collect_service_metric"], @@ -145,42 +143,38 @@ def _create_yaml_content(service, base_url, bench_target, test_phase, num_querie "seed": test_params.get("seed", None), "llm-model": test_params["llm_model"], "deployment-type": test_params["deployment_type"], - "load-shape": test_params["load_shape"], + "load-shape": load_shape, }, "runs": [{"name": test_phase, "users": concurrency, "max-request": num_queries}], } } - # For the following scenarios, test will stop after the specified run-time - if test_params["run_time"] is not None and test_phase != "warmup": - yaml_content["profile"]["global-settings"]["run-time"] = test_params["run_time"] - return yaml_content -def _create_stresscli_confs(case_params, test_params, test_phase, num_queries, base_url, ts) -> str: +def _create_stresscli_confs(case_params, test_params, test_phase, num_queries, base_url, ts, concurrency=1) -> str: """Create a stresscli configuration file and persist it on disk.""" stresscli_confs = [] # Get the workload - test_cases = test_params["test_cases"] - for test_case in test_cases: + bench_target = test_params["bench_target"] + for i, b_target in enumerate(bench_target): stresscli_conf = {} - print(test_case) - if isinstance(test_case, str): - bench_target = test_case - elif isinstance(test_case, dict): - bench_target = list(test_case.keys())[0] - dataset_conf = test_case[bench_target] - if bench_target == "chatqna_qlist_pubmed": - max_lines = dataset_conf["dataset"].split("pub_med")[-1] - stresscli_conf["envs"] = {"DATASET": f"pubmed_{max_lines}.txt", "MAX_LINES": max_lines} + print(f"[OPEA BENCHMARK] 🚀 Running test for {b_target} in phase {test_phase} for {num_queries} queries") + if len(test_params["dataset"]) > i: + stresscli_conf["envs"] = {"DATASET": test_params["dataset"][i], "MAX_LINES": str(test_params["prompt"][i])} + else: + stresscli_conf["envs"] = {"MAX_LINES": str(test_params["prompt"][i])} # Generate the content of stresscli configuration file - stresscli_yaml = _create_yaml_content(case_params, base_url, bench_target, test_phase, num_queries, test_params) + stresscli_yaml = _create_yaml_content( + case_params, base_url, b_target, test_phase, num_queries, test_params, concurrency + ) # Dump the stresscli configuration file service_name = case_params.get("service_name") + max_output = case_params.get("max_output") run_yaml_path = os.path.join( - test_params["test_output_dir"], f"run_{service_name}_{ts}_{test_phase}_{num_queries}_{bench_target}.yaml" + test_params["test_output_dir"], + f"run_{test_phase}_{service_name}_{num_queries}_{b_target}_{max_output}_{ts}.yaml", ) with open(run_yaml_path, "w") as yaml_file: yaml.dump(stresscli_yaml, yaml_file) @@ -207,15 +201,79 @@ def create_stresscli_confs(service, base_url, test_suite_config, index): stresscli_confs.extend(_create_stresscli_confs(service, test_suite_config, "benchmark", -1, base_url, index)) else: # Test stop is controlled by request count - for user_queries in user_queries_lst: + for i, user_query in enumerate(user_queries_lst): + concurrency_list = test_suite_config["concurrency"] + user_query *= test_suite_config["node_num"] stresscli_confs.extend( - _create_stresscli_confs(service, test_suite_config, "benchmark", user_queries, base_url, index) + _create_stresscli_confs( + service, + test_suite_config, + "benchmark", + user_query, + base_url, + index, + concurrency=concurrency_list[i], + ) ) return stresscli_confs -def _run_service_test(example, service, test_suite_config): +def ingest_data_to_db(service, dataset, namespace): + """Ingest data into the database.""" + for service_name in service.get("service_list"): + if "data" in service_name: + # Ingest data into the database + print(f"[OPEA BENCHMARK] 🚀 Ingesting data into the database for {service_name}...") + try: + svc_ip, port = _get_service_ip(service_name, "k8s", None, None, namespace) + url = f"http://{svc_ip}:{port}/v1/dataprep/ingest" + + files = {"files": open(dataset, "rb")} + + response = requests.post(url, files=files) + if response.status_code != 200: + print(f"Error ingesting data: {response.text}. Status code: {response.status_code}") + return False + if "Data preparation succeeded" not in response.text: + print(f"Error ingesting data: {response.text}. Response: {response}") + return False + + except Exception as e: + print(f"Error ingesting data: {e}") + return False + print(f"[OPEA BENCHMARK] 🚀 Data ingestion completed for {service_name}.") + break + return True + + +def clear_db(service, namespace): + """Delete all files from the database.""" + for service_name in service.get("service_list"): + if "data" in service_name: + # Delete data from the database + try: + svc_ip, port = _get_service_ip(service_name, "k8s", None, None, namespace) + url = f"http://{svc_ip}:{port}/v1/dataprep/delete" + data = {"file_path": "all"} + print(f"[OPEA BENCHMARK] 🚀 Deleting data from the database for {service_name} with {url}") + + response = requests.post(url, json=data, headers={"Content-Type": "application/json"}) + if response.status_code != 200: + print(f"Error deleting data: {response.text}. Status code: {response.status_code}") + return False + if "true" not in response.text: + print(f"Error deleting data: {response.text}. Response: {response}") + return False + except Exception as e: + print(f"Error deleting data: {e}") + return False + print(f"[OPEA BENCHMARK] 🚀 Data deletion completed for {service_name}.") + break + return True + + +def _run_service_test(example, service, test_suite_config, namespace): """Run the test for a specific service and example.""" print(f"[OPEA BENCHMARK] 🚀 Example: [ {example} ] Service: [ {service.get('service_name')} ], Running test...") @@ -251,44 +309,94 @@ def _run_service_test(example, service, test_suite_config): run_yaml_path = stresscli_conf["run_yaml_path"] print(f"[OPEA BENCHMARK] 🚀 The {index} time test is running, run yaml: {run_yaml_path}...") os.environ["MAX_TOKENS"] = str(service.get("max_output")) + + dataset = None if stresscli_conf.get("envs") is not None: for key, value in stresscli_conf.get("envs").items(): os.environ[key] = value + if key == "DATASET": + dataset = value + + if dataset: + # Ingest data into the database for single run of benchmark + result = ingest_data_to_db(service, dataset, namespace) + if not result: + print(f"[OPEA BENCHMARK] 🚀 Data ingestion failed for {service_name}.") + exit(1) + else: + print(f"[OPEA BENCHMARK] 🚀 Dataset is not specified for {service_name}. Check the benchmark.yaml again.") + + # Run the benchmark test and append the output folder to the list + print("[OPEA BENCHMARK] 🚀 Start locust_runtests at", datetime.now().strftime("%Y%m%d_%H%M%S")) + locust_output = locust_runtests(None, run_yaml_path) + print(f"[OPEA BENCHMARK] 🚀 locust_output origin name is {locust_output}") + # Rename the output folder to include the index + new_output_path = os.path.join( + os.path.dirname(run_yaml_path), f"{os.path.splitext(os.path.basename(run_yaml_path))[0]}_output" + ) + os.rename(locust_output, new_output_path) + print(f"[OPEA BENCHMARK] 🚀 locust new_output_path is {new_output_path}") + + output_folders.append(new_output_path) + print("[OPEA BENCHMARK] 🚀 End locust_runtests at", datetime.now().strftime("%Y%m%d_%H%M%S")) - output_folders.append(locust_runtests(None, run_yaml_path)) + # Delete all files from the database after the test + result = clear_db(service, namespace) + print("[OPEA BENCHMARK] 🚀 End of clean up db", datetime.now().strftime("%Y%m%d_%H%M%S")) + if not result: + print(f"[OPEA BENCHMARK] 🚀 Data deletion failed for {service_name}.") + exit(1) print(f"[OPEA BENCHMARK] 🚀 Test completed for {service_name} at {url}") return output_folders -def run_benchmark(benchmark_config, chart_name, namespace, llm_model=None, report=False): +def run_benchmark(benchmark_config, chart_name, namespace, node_num=1, llm_model=None, report=False, output_dir=None): + """Run the benchmark test for the specified helm chart and configuration. + + Args: + benchmark_config (dict): The benchmark configuration. + chart_name (str): The name of the helm chart. + namespace (str): The namespace to deploy the chart. + node_num (int): The number of nodes of current deployment. + llm_model (str): The LLM model to use for the test. + report (bool): Whether to generate a report after the test. + output_dir (str): Directory to store the test output. If None, uses default directory. + """ # If llm_model is None or an empty string, set to default value if not llm_model: - llm_model = "Qwen/Qwen2.5-Coder-7B-Instruct" + llm_model = "meta-llama/Meta-Llama-3-8B-Instruct" # Extract data parsed_data = construct_benchmark_config(benchmark_config) test_suite_config = { "user_queries": parsed_data["user_queries"], # num of user queries "random_prompt": False, # whether to use random prompt, set to False by default - "run_time": "60m", # The max total run time for the test suite, set to 60m by default + "run_time": "30m", # The max total run time for the test suite, set to 60m by default "collect_service_metric": False, # whether to collect service metrics, set to False by default "llm_model": llm_model, # The LLM model used for the test "deployment_type": "k8s", # Default is "k8s", can also be "docker" "service_ip": None, # Leave as None for k8s, specify for Docker "service_port": None, # Leave as None for k8s, specify for Docker - "test_output_dir": os.getcwd() + "/benchmark_output", # The directory to store the test output + "test_output_dir": ( + output_dir if output_dir else os.getcwd() + "/benchmark_output" + ), # Use output_dir if provided + "node_num": node_num, "load_shape": { - "name": "constant", - "params": {"constant": {"concurrent_level": 4}, "poisson": {"arrival_rate": 1.0}}, + "name": parsed_data["load_shape_type"], + "params": { + "poisson": {"arrival_rate": parsed_data["poisson_arrival_rate"]}, + }, }, - "concurrent_level": 4, - "arrival_rate": 1.0, + "concurrency": parsed_data["concurrency"], + "arrival_rate": parsed_data["poisson_arrival_rate"], "query_timeout": 120, "warm_ups": parsed_data["warmup_iterations"], "seed": parsed_data["seed"], "namespace": namespace, - "test_cases": parsed_data["test_cases"], + "bench_target": parsed_data["bench_target"], + "dataset": parsed_data["dataset"], + "prompt": parsed_data["prompt"], "llm_max_token_size": parsed_data["llm_max_token_size"], } @@ -313,15 +421,14 @@ def run_benchmark(benchmark_config, chart_name, namespace, llm_model=None, repor "chatqna-retriever-usvc", "chatqna-tei", "chatqna-teirerank", - "chatqna-tgi", + "chatqna-vllm", ], - "test_cases": parsed_data["test_cases"], # Activate if random_prompt=true: leave blank = default dataset(WebQuestions) or sharegpt "prompts": query_data, "max_output": llm_max_token, # max number of output tokens "k": 1, # number of retrieved documents } - output_folder = _run_service_test(chart_name, case_data, test_suite_config) + output_folder = _run_service_test(chart_name, case_data, test_suite_config, namespace) print(f"[OPEA BENCHMARK] 🚀 Test Finished. Output saved in {output_folder}.") @@ -339,5 +446,5 @@ def run_benchmark(benchmark_config, chart_name, namespace, llm_model=None, repor if __name__ == "__main__": - benchmark_config = load_yaml("./benchmark.yaml") - run_benchmark(benchmark_config=benchmark_config, chart_name="chatqna", namespace="deploy-benchmark") + benchmark_config = load_yaml("./ChatQnA/benchmark_chatqna.yaml") + run_benchmark(benchmark_config=benchmark_config, chart_name="chatqna", namespace="benchmark") diff --git a/deploy.py b/deploy.py index 21dd278cc2..bd3a8a87d5 100644 --- a/deploy.py +++ b/deploy.py @@ -49,12 +49,14 @@ def configure_replica(values, deploy_config): return values -def get_output_filename(num_nodes, with_rerank, example_type, device, action_type): +def get_output_filename(num_nodes, with_rerank, example_type, device, action_type, batch_size=None): """Generate output filename based on configuration.""" rerank_suffix = "with-rerank-" if with_rerank else "" action_suffix = "deploy-" if action_type == 0 else "update-" if action_type == 1 else "" + # Only include batch_suffix if batch_size is not None + batch_suffix = f"batch{batch_size}-" if batch_size else "" - return f"{example_type}-{num_nodes}-{device}-{action_suffix}{rerank_suffix}values.yaml" + return f"{example_type}-{rerank_suffix}{device}-{action_suffix}node{num_nodes}-{batch_suffix}values.yaml" def configure_resources(values, deploy_config): @@ -62,30 +64,31 @@ def configure_resources(values, deploy_config): resource_configs = [] for service_name, config in deploy_config["services"].items(): + # Skip if resources configuration doesn't exist or is not enabled + resources_config = config.get("resources", {}) + if not resources_config: + continue + resources = {} - if deploy_config["device"] == "gaudi" and config.get("cards_per_instance", 0) > 1: + if deploy_config["device"] == "gaudi" and resources_config.get("cards_per_instance", 0) > 1: resources = { - "limits": {"habana.ai/gaudi": config["cards_per_instance"]}, - "requests": {"habana.ai/gaudi": config["cards_per_instance"]}, + "limits": {"habana.ai/gaudi": resources_config["cards_per_instance"]}, + "requests": {"habana.ai/gaudi": resources_config["cards_per_instance"]}, } else: - limits = {} - requests = {} - - # Only add CPU if cores_per_instance has a value - if config.get("cores_per_instance"): - limits["cpu"] = config["cores_per_instance"] - requests["cpu"] = config["cores_per_instance"] - - # Only add memory if memory_capacity has a value - if config.get("memory_capacity"): - limits["memory"] = config["memory_capacity"] - requests["memory"] = config["memory_capacity"] - - # Only create resources if we have any limits/requests - if limits and requests: - resources["limits"] = limits - resources["requests"] = requests + # Only add CPU if cores_per_instance has a valid value + cores = resources_config.get("cores_per_instance") + if cores is not None and cores != "": + resources = {"limits": {"cpu": cores}, "requests": {"cpu": cores}} + + # Only add memory if memory_capacity has a valid value + memory = resources_config.get("memory_capacity") + if memory is not None and memory != "": + if not resources: + resources = {"limits": {"memory": memory}, "requests": {"memory": memory}} + else: + resources["limits"]["memory"] = memory + resources["requests"]["memory"] = memory if resources: if service_name == "llm": @@ -116,48 +119,64 @@ def configure_resources(values, deploy_config): def configure_extra_cmd_args(values, deploy_config): """Configure extra command line arguments for services.""" + batch_size = None for service_name, config in deploy_config["services"].items(): - extra_cmd_args = [] - - for param in [ - "max_batch_size", - "max_input_length", - "max_total_tokens", - "max_batch_total_tokens", - "max_batch_prefill_tokens", - ]: - if config.get(param): - extra_cmd_args.extend([f"--{param.replace('_', '-')}", str(config[param])]) - - if extra_cmd_args: - if service_name == "llm": - engine = config.get("engine", "tgi") + if service_name == "llm": + extra_cmd_args = [] + engine = config.get("engine", "tgi") + model_params = config.get("model_params", {}) + + # Get engine-specific parameters + engine_params = model_params.get(engine, {}) + + # Get batch parameters and token parameters configuration + batch_params = engine_params.get("batch_params", {}) + token_params = engine_params.get("token_params", {}) + + # Get batch size based on engine type + if engine == "tgi": + batch_size = batch_params.get("max_batch_size") + elif engine == "vllm": + batch_size = batch_params.get("max_num_seqs") + batch_size = batch_size if batch_size and batch_size != "" else None + + # Add all parameters that exist in batch_params + for param, value in batch_params.items(): + if value is not None and value != "": + extra_cmd_args.extend([f"--{param.replace('_', '-')}", str(value)]) + + # Add all parameters that exist in token_params + for param, value in token_params.items(): + if value is not None and value != "": + extra_cmd_args.extend([f"--{param.replace('_', '-')}", str(value)]) + + if extra_cmd_args: if engine not in values: values[engine] = {} values[engine]["extraCmdArgs"] = extra_cmd_args - else: - if service_name not in values: - values[service_name] = {} - values[service_name]["extraCmdArgs"] = extra_cmd_args + print(f"extraCmdArgs: {extra_cmd_args}") - return values + return values, batch_size def configure_models(values, deploy_config): """Configure model settings for services.""" for service_name, config in deploy_config["services"].items(): - # Skip if no model_id defined or service is disabled - if not config.get("model_id") or config.get("enabled") is False: + # Get model_id and check if it's valid (not None or empty string) + model_id = config.get("model_id") + if not model_id or model_id == "" or config.get("enabled") is False: continue if service_name == "llm": # For LLM service, use its engine as the key + # Check if engine is valid (not None or empty string) engine = config.get("engine", "tgi") - values[engine]["LLM_MODEL_ID"] = config.get("model_id") + if engine and engine != "": + values[engine]["LLM_MODEL_ID"] = model_id elif service_name == "tei": - values[service_name]["EMBEDDING_MODEL_ID"] = config.get("model_id") + values[service_name]["EMBEDDING_MODEL_ID"] = model_id elif service_name == "teirerank": - values[service_name]["RERANK_MODEL_ID"] = config.get("model_id") + values[service_name]["RERANK_MODEL_ID"] = model_id return values @@ -209,13 +228,13 @@ def generate_helm_values(example_type, deploy_config, chart_dir, action_type, no values = configure_rerank(values, with_rerank, deploy_config, example_type, node_selector or {}) values = configure_replica(values, deploy_config) values = configure_resources(values, deploy_config) - values = configure_extra_cmd_args(values, deploy_config) + values, batch_size = configure_extra_cmd_args(values, deploy_config) values = configure_models(values, deploy_config) device = deploy_config.get("device", "unknown") # Generate and write YAML file - filename = get_output_filename(num_nodes, with_rerank, example_type, device, action_type) + filename = get_output_filename(num_nodes, with_rerank, example_type, device, action_type, batch_size) yaml_string = yaml.dump(values, default_flow_style=False) filepath = os.path.join(chart_dir, filename) @@ -376,12 +395,24 @@ def install_helm_release(release_name, chart_name, namespace, hw_values_file, de def uninstall_helm_release(release_name, namespace=None): - """Uninstall a Helm release and clean up resources, optionally delete the namespace if not 'default'.""" + """Uninstall a Helm release and clean up resources, optionally delete the namespace if not 'default'. + + First checks if the release exists before attempting to uninstall. + """ # Default to 'default' namespace if none is specified if not namespace: namespace = "default" try: + # Check if the release exists + check_command = ["helm", "list", "--namespace", namespace, "--filter", release_name, "--output", "json"] + output = run_kubectl_command(check_command) + releases = json.loads(output) + + if not releases: + print(f"Helm release {release_name} not found in namespace {namespace}. Nothing to uninstall.") + return + # Uninstall the Helm release command = ["helm", "uninstall", release_name, "--namespace", namespace] print(f"Uninstalling Helm release {release_name} in namespace {namespace}...") @@ -399,6 +430,8 @@ def uninstall_helm_release(release_name, namespace=None): except subprocess.CalledProcessError as e: print(f"Error occurred while uninstalling Helm release or deleting namespace: {e}") + except json.JSONDecodeError as e: + print(f"Error parsing helm list output: {e}") def update_service(release_name, chart_name, namespace, hw_values_file, deploy_values_file, update_values_file): @@ -449,7 +482,7 @@ def read_deploy_config(config_path): return None -def check_deployment_ready(release_name, namespace, timeout=300, interval=5, logfile="deployment.log"): +def check_deployment_ready(release_name, namespace, timeout=1000, interval=5, logfile="deployment.log"): """Wait until all pods in the deployment are running and ready. Args: @@ -586,6 +619,18 @@ def main(): parser.add_argument("--update-service", action="store_true", help="Update the deployment with new configuration.") parser.add_argument("--check-ready", action="store_true", help="Check if all services in the deployment are ready.") parser.add_argument("--chart-dir", default=".", help="Path to the untarred Helm chart directory.") + parser.add_argument( + "--timeout", + type=int, + default=1000, + help="Maximum time to wait for deployment readiness in seconds (default: 1000)", + ) + parser.add_argument( + "--interval", + type=int, + default=5, + help="Interval between readiness checks in seconds (default: 5)", + ) args = parser.parse_args() @@ -597,7 +642,7 @@ def main(): clear_labels_from_nodes(args.label, args.node_names) return elif args.check_ready: - is_ready = check_deployment_ready(args.chart_name, args.namespace) + is_ready = check_deployment_ready(args.chart_name, args.namespace, args.timeout, args.interval) return is_ready elif args.uninstall: uninstall_helm_release(args.chart_name, args.namespace) @@ -659,6 +704,7 @@ def main(): update_service( args.chart_name, args.chart_name, args.namespace, hw_values_file, args.user_values, values_file_path ) + print(f"values_file_path: {values_file_path}") return except Exception as e: parser.error(f"Failed to update deployment: {str(e)}") diff --git a/deploy_and_benchmark.py b/deploy_and_benchmark.py index 1dc4c4308d..f210f215dc 100644 --- a/deploy_and_benchmark.py +++ b/deploy_and_benchmark.py @@ -23,13 +23,14 @@ def read_yaml(file_path): return None -def construct_deploy_config(deploy_config, target_node, max_batch_size=None): - """Construct a new deploy config based on the target node number and optional max_batch_size. +def construct_deploy_config(deploy_config, target_node, batch_param_value=None, test_mode="oob"): + """Construct a new deploy config based on the target node number and optional batch parameter value. Args: deploy_config: Original deploy config dictionary target_node: Target node number to match in the node array - max_batch_size: Optional specific max_batch_size value to use + batch_param_value: Optional specific batch parameter value to use + test_mode: Test mode, either 'oob' or 'tune' Returns: A new deploy config with single values for node and instance_num @@ -51,21 +52,79 @@ def construct_deploy_config(deploy_config, target_node, max_batch_size=None): # Set the single node value new_config["node"] = target_node - # Update instance_num for each service based on the same index - for service_name, service_config in new_config.get("services", {}).items(): + # First determine which llm replicaCount to use based on teirerank.enabled + services = new_config.get("services", {}) + teirerank_enabled = services.get("teirerank", {}).get("enabled", True) + + # Process each service's configuration + for service_name, service_config in services.items(): + # Handle replicaCount if "replicaCount" in service_config: - instance_nums = service_config["replicaCount"] - if isinstance(instance_nums, list): - if len(instance_nums) != len(nodes): + if service_name == "llm" and isinstance(service_config["replicaCount"], dict): + replica_counts = service_config["replicaCount"] + service_config["replicaCount"] = ( + replica_counts["with_teirerank"] if teirerank_enabled else replica_counts["without_teirerank"] + ) + + if isinstance(service_config["replicaCount"], list): + if len(service_config["replicaCount"]) < len(nodes): raise ValueError( - f"instance_num array length ({len(instance_nums)}) for service {service_name} " - f"doesn't match node array length ({len(nodes)})" + f"replicaCount array length ({len(service_config['replicaCount'])}) for service {service_name} " + f"smaller than node array length ({len(nodes)})" ) - service_config["replicaCount"] = instance_nums[node_index] - - # Update max_batch_size if specified - if max_batch_size is not None and "llm" in new_config["services"]: - new_config["services"]["llm"]["max_batch_size"] = max_batch_size + service_config["replicaCount"] = service_config["replicaCount"][node_index] + + # Handle resources based on test_mode + if "resources" in service_config: + resources = service_config["resources"] + if test_mode == "tune" or resources.get("enabled", False): + # Keep resource configuration but remove enabled field + resources.pop("enabled", None) + else: + # Remove resource configuration in OOB mode when disabled + service_config.pop("resources") + + # Handle model parameters for LLM service + if service_name == "llm" and "model_params" in service_config: + model_params = service_config["model_params"] + engine = service_config.get("engine", "tgi") + + # Get engine-specific parameters + engine_params = model_params.get(engine, {}) + + # Handle batch parameters + if "batch_params" in engine_params: + batch_params = engine_params["batch_params"] + if test_mode == "tune" or batch_params.get("enabled", False): + # Keep batch parameters configuration but remove enabled field + batch_params.pop("enabled", None) + + # Update batch parameter value if specified + if batch_param_value is not None: + if engine == "tgi": + batch_params["max_batch_size"] = str(batch_param_value) + elif engine == "vllm": + batch_params["max_num_seqs"] = str(batch_param_value) + else: + engine_params.pop("batch_params") + + # Handle token parameters + if "token_params" in engine_params: + token_params = engine_params["token_params"] + if test_mode == "tune" or token_params.get("enabled", False): + # Keep token parameters configuration but remove enabled field + token_params.pop("enabled", None) + else: + # Remove token parameters in OOB mode when disabled + engine_params.pop("token_params") + + # Update model_params with engine-specific parameters only + model_params.clear() + model_params[engine] = engine_params + + # Remove model_params if empty or if engine_params is empty + if not model_params or not engine_params: + service_config.pop("model_params") return new_config @@ -84,13 +143,18 @@ def pull_helm_chart(chart_pull_url, version, chart_name): return untar_dir -def main(yaml_file, target_node=None): +def main(yaml_file, target_node=None, test_mode="oob"): """Main function to process deployment configuration. Args: yaml_file: Path to the YAML configuration file target_node: Optional target number of nodes to deploy. If not specified, will process all nodes. + test_mode: Test mode, either "oob" (out of box) or "tune". Defaults to "oob". """ + if test_mode not in ["oob", "tune"]: + print("Error: test_mode must be either 'oob' or 'tune'") + return None + config = read_yaml(yaml_file) if config is None: print("Failed to read YAML file.") @@ -116,7 +180,7 @@ def main(yaml_file, target_node=None): # Pull the Helm chart chart_pull_url = f"oci://ghcr.io/opea-project/charts/{chart_name}" - version = deploy_config.get("version", "1.1.0") + version = deploy_config.get("version", "0-latest") chart_dir = pull_helm_chart(chart_pull_url, version, chart_name) if not chart_dir: return @@ -140,20 +204,61 @@ def main(yaml_file, target_node=None): continue try: - # Process max_batch_sizes - max_batch_sizes = deploy_config.get("services", {}).get("llm", {}).get("max_batch_size", []) - if not isinstance(max_batch_sizes, list): - max_batch_sizes = [max_batch_sizes] + # Process batch parameters based on engine type + services = deploy_config.get("services", {}) + llm_config = services.get("llm", {}) + + if "model_params" in llm_config: + model_params = llm_config["model_params"] + engine = llm_config.get("engine", "tgi") + + # Get engine-specific parameters + engine_params = model_params.get(engine, {}) + + # Handle batch parameters + batch_params = [] + if "batch_params" in engine_params: + key = "max_batch_size" if engine == "tgi" else "max_num_seqs" + batch_params = engine_params["batch_params"].get(key, []) + param_name = key + + if not isinstance(batch_params, list): + batch_params = [batch_params] + + # Skip multiple iterations if batch parameter is empty + if batch_params == [""] or not batch_params: + batch_params = [None] + else: + batch_params = [None] + param_name = "batch_param" + + # Get timeout and interval from deploy config for check-ready + timeout = deploy_config.get("timeout", 1000) # default 1000s + interval = deploy_config.get("interval", 5) # default 5s values_file_path = None - for i, max_batch_size in enumerate(max_batch_sizes): - print(f"\nProcessing max_batch_size: {max_batch_size}") + # Create benchmark output directory + benchmark_dir = os.path.join(os.getcwd(), "benchmark_output") + os.makedirs(benchmark_dir, exist_ok=True) + + for i, batch_param in enumerate(batch_params): + print(f"\nProcessing {test_mode} mode {param_name}: {batch_param}") + # Create subdirectory for this iteration with test mode in the name + iteration_dir = os.path.join( + benchmark_dir, + f"benchmark_{test_mode}_node{node}_batch{batch_param if batch_param is not None else 'default'}", + ) + os.makedirs(iteration_dir, exist_ok=True) # Construct new deploy config - new_deploy_config = construct_deploy_config(deploy_config, node, max_batch_size) + new_deploy_config = construct_deploy_config(deploy_config, node, batch_param, test_mode) # Write the new deploy config to a temporary file - temp_config_file = f"temp_deploy_config_{node}_{max_batch_size}.yaml" + temp_config_file = ( + f"temp_deploy_config_{node}.yaml" + if batch_param is None + else f"temp_deploy_config_{node}_{batch_param}.yaml" + ) try: with open(temp_config_file, "w") as f: yaml.dump(new_deploy_config, f) @@ -178,6 +283,8 @@ def main(yaml_file, target_node=None): if match: values_file_path = match.group(1) print(f"Captured values_file_path: {values_file_path}") + # Copy values file to iteration directory + shutil.copy2(values_file_path, iteration_dir) else: print("values_file_path not found in the output") @@ -198,12 +305,20 @@ def main(yaml_file, target_node=None): values_file_path, "--update-service", ] - result = subprocess.run(cmd, check=True) + result = subprocess.run(cmd, check=True, capture_output=True, text=True) if result.returncode != 0: - print( - f"Update failed for {node} nodes configuration with max_batch_size {max_batch_size}" - ) - break # Skip remaining max_batch_sizes for this node + print(f"Update failed for {node} nodes configuration with {param_name} {batch_param}") + break # Skip remaining {param_name} for this node + + # Update values_file_path from the output + match = re.search(r"values_file_path: (\S+)", result.stdout) + if match: + values_file_path = match.group(1) + print(f"Updated values_file_path: {values_file_path}") + # Copy values file to iteration directory + shutil.copy2(values_file_path, iteration_dir) + else: + print("values_file_path not found in the output") # Wait for deployment to be ready print("\nWaiting for deployment to be ready...") @@ -215,26 +330,42 @@ def main(yaml_file, target_node=None): "--namespace", namespace, "--check-ready", + "--timeout", + str(timeout), + "--interval", + str(interval), ] try: - result = subprocess.run(cmd, check=True) - print("Deployments are ready!") + result = subprocess.run( + cmd, check=False + ) # Changed to check=False to handle return code manually + if result.returncode == 0: + print("Deployments are ready!") + # Run benchmark only if deployment is ready + run_benchmark( + benchmark_config=benchmark_config, + chart_name=chart_name, + namespace=namespace, + node_num=node, + llm_model=deploy_config.get("services", {}).get("llm", {}).get("model_id", ""), + output_dir=iteration_dir, + ) + else: + print( + f"Deployments are not ready after timeout period during " + f"{'deployment' if i == 0 else 'update'} for {node} nodes. " + f"Skipping remaining iterations." + ) + break # Exit the batch parameter loop for current node except subprocess.CalledProcessError as e: - print(f"Deployments status failed with returncode: {e.returncode}") - - # Run benchmark - run_benchmark( - benchmark_config=benchmark_config, - chart_name=chart_name, - namespace=namespace, - llm_model=deploy_config.get("services", {}).get("llm", {}).get("model_id", ""), - ) + print(f"Error while checking deployment status: {str(e)}") + break # Exit the batch parameter loop for current node except Exception as e: print( - f"Error during {'deployment' if i == 0 else 'update'} for {node} nodes with max_batch_size {max_batch_size}: {str(e)}" + f"Error during {'deployment' if i == 0 else 'update'} for {node} nodes with {param_name} {batch_param}: {str(e)}" ) - break # Skip remaining max_batch_sizes for this node + break # Skip remaining {param_name} for this node finally: # Clean up the temporary file if os.path.exists(temp_config_file): @@ -287,6 +418,7 @@ def main(yaml_file, target_node=None): parser = argparse.ArgumentParser(description="Deploy and benchmark with specific node configuration.") parser.add_argument("yaml_file", help="Path to the YAML configuration file") parser.add_argument("--target-node", type=int, help="Optional: Target number of nodes to deploy.", default=None) + parser.add_argument("--test-mode", type=str, help="Test mode, either 'oob' (out of box) or 'tune'.", default="oob") args = parser.parse_args() - main(args.yaml_file, args.target_node) + main(args.yaml_file, args.target_node, args.test_mode) diff --git a/requirements.txt b/requirements.txt index 44f6445aa0..637668c3d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ kubernetes locust numpy opea-eval>=1.2 +prometheus_client pytest pyyaml requests From d1861f9a45599e83bc8f8804c9c37db087db124f Mon Sep 17 00:00:00 2001 From: alexsin368 <109180236+alexsin368@users.noreply.github.com> Date: Thu, 27 Feb 2025 21:43:43 -0800 Subject: [PATCH 028/226] Top level README: add link to github.io documentation (#1584) Signed-off-by: alexsin368 Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: Chingis Yundunov --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6738cef202..2db55575bd 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,10 @@ GenAIExamples are designed to give developers an easy entry into generative AI, [GenAIEval](https://github.com/opea-project/GenAIEval) measures service performance metrics such as throughput, latency, and accuracy for GenAIExamples. This feature helps users compare performance across various hardware configurations easily. +## Documentation + +The GenAIExamples [documentation](https://opea-project.github.io/latest/examples/index.html) contains a comprehensive guide on all available examples including architecture, deployment guides, and more. Information on GenAIComps, GenAIInfra, and GenAIEval can also be found there. + ## Getting Started GenAIExamples offers flexible deployment options that cater to different user needs, enabling efficient use and deployment in various environments. Here’s a brief overview of the three primary methods: Python startup, Docker Compose, and Kubernetes. @@ -20,7 +24,7 @@ Users can choose the most suitable approach based on ease of setup, scalability ### Deployment Guide -Deployment are based on released docker images by default, check [docker image list](./docker_images_list.md) for detailed information. You can also build your own images following instructions. +Deployment is based on released docker images by default, check [docker image list](./docker_images_list.md) for detailed information. You can also build your own images following instructions. #### Prerequisite @@ -43,6 +47,8 @@ Deployment are based on released docker images by default, check [docker image l #### Deploy Examples +> **Note**: Check for [sample guides](https://opea-project.github.io/latest/examples/index.html) first for your use case. If it is not available, then refer to the table below. + | Use Case | Docker Compose
Deployment on Xeon | Docker Compose
Deployment on Gaudi | Docker Compose
Deployment on ROCm | Kubernetes with Helm Charts | Kubernetes with GMC | | ----------------- | ------------------------------------------------------------------------------ | ---------------------------------------------------------------------------- | ------------------------------------------------------------------------ | ------------------------------------------------------------------- | ------------------------------------------------------------ | | ChatQnA | [Xeon Instructions](ChatQnA/docker_compose/intel/cpu/xeon/README.md) | [Gaudi Instructions](ChatQnA/docker_compose/intel/hpu/gaudi/README.md) | [ROCm Instructions](ChatQnA/docker_compose/amd/gpu/rocm/README.md) | [ChatQnA with Helm Charts](ChatQnA/kubernetes/helm/README.md) | [ChatQnA with GMC](ChatQnA/kubernetes/gmc/README.md) | From a30a6e3c52a639194730c3826ff3b2d0edb80540 Mon Sep 17 00:00:00 2001 From: WenjiaoYue Date: Fri, 28 Feb 2025 16:10:58 +0800 Subject: [PATCH 029/226] fix click example button issue (#1586) Signed-off-by: WenjiaoYue Signed-off-by: Chingis Yundunov --- AgentQnA/ui/svelte/src/lib/components/home.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AgentQnA/ui/svelte/src/lib/components/home.svelte b/AgentQnA/ui/svelte/src/lib/components/home.svelte index f35ee44575..ba37f1672d 100644 --- a/AgentQnA/ui/svelte/src/lib/components/home.svelte +++ b/AgentQnA/ui/svelte/src/lib/components/home.svelte @@ -108,7 +108,7 @@
handleCreate(feature)} + on:click={() => handleCreate(feature.description)} >
Date: Fri, 28 Feb 2025 23:40:31 +0900 Subject: [PATCH 030/226] ChatQnA Docker compose file for Milvus as vdb (#1548) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ezequiel Lanza Signed-off-by: Kendall González León Signed-off-by: chensuyue Signed-off-by: Spycsh Signed-off-by: Wang, Xigui Signed-off-by: ZePan110 Signed-off-by: dependabot[bot] Signed-off-by: minmin-intel Signed-off-by: Artem Astafev Signed-off-by: Xinyao Wang Signed-off-by: Cathy Zhang Signed-off-by: letonghan Signed-off-by: alexsin368 Signed-off-by: WenjiaoYue Co-authored-by: Ezequiel Lanza Co-authored-by: Kendall González León Co-authored-by: chen, suyue Co-authored-by: Spycsh <39623753+Spycsh@users.noreply.github.com> Co-authored-by: xiguiw <111278656+xiguiw@users.noreply.github.com> Co-authored-by: jotpalch <49465120+jotpalch@users.noreply.github.com> Co-authored-by: ZePan110 Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: minmin-intel Co-authored-by: Ying Hu Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Eero Tamminen Co-authored-by: Liang Lv Co-authored-by: Artem Astafev Co-authored-by: XinyaoWa Co-authored-by: alexsin368 <109180236+alexsin368@users.noreply.github.com> Co-authored-by: WenjiaoYue Signed-off-by: Chingis Yundunov --- .../intel/cpu/xeon/compose_milvus.yaml | 227 +++++ .../docker_compose/intel/cpu/xeon/milvus.yaml | 811 ++++++++++++++++++ ChatQnA/tests/test_compose_milvus_on_xeon.sh | 249 ++++++ 3 files changed, 1287 insertions(+) create mode 100644 ChatQnA/docker_compose/intel/cpu/xeon/compose_milvus.yaml create mode 100644 ChatQnA/docker_compose/intel/cpu/xeon/milvus.yaml create mode 100644 ChatQnA/tests/test_compose_milvus_on_xeon.sh diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/compose_milvus.yaml b/ChatQnA/docker_compose/intel/cpu/xeon/compose_milvus.yaml new file mode 100644 index 0000000000..740f5eba42 --- /dev/null +++ b/ChatQnA/docker_compose/intel/cpu/xeon/compose_milvus.yaml @@ -0,0 +1,227 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +services: + etcd: + container_name: milvus-etcd + image: quay.io/coreos/etcd:v3.5.5 + environment: + - ETCD_AUTO_COMPACTION_MODE=revision + - ETCD_AUTO_COMPACTION_RETENTION=1000 + - ETCD_QUOTA_BACKEND_BYTES=4294967296 + - ETCD_SNAPSHOT_COUNT=50000 + volumes: + - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd + command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd + healthcheck: + test: ["CMD", "etcdctl", "endpoint", "health"] + interval: 30s + timeout: 20s + retries: 3 + + minio: + container_name: milvus-minio + image: minio/minio:RELEASE.2023-03-20T20-16-18Z + environment: + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin + ports: + - "${MINIO_PORT1:-5044}:9001" + - "${MINIO_PORT2:-5043}:9000" + volumes: + - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data + command: minio server /minio_data --console-address ":9001" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + + milvus-standalone: + container_name: milvus-standalone + image: milvusdb/milvus:v2.4.6 + command: ["milvus", "run", "standalone"] + security_opt: + - seccomp:unconfined + environment: + ETCD_ENDPOINTS: etcd:2379 + MINIO_ADDRESS: minio:9000 + volumes: + - ${DOCKER_VOLUME_DIRECTORY:-.}/milvus.yaml:/milvus/configs/milvus.yaml + - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"] + interval: 30s + start_period: 90s + timeout: 20s + retries: 3 + ports: + - "19530:19530" + - "${MILVUS_STANDALONE_PORT:-9091}:9091" + depends_on: + - "etcd" + - "minio" + + dataprep-milvus-service: + image: ${REGISTRY:-opea}/dataprep:${TAG:-latest} + container_name: dataprep-milvus-server + ports: + - "${DATAPREP_PORT:-11101}:5000" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + DATAPREP_COMPONENT_NAME: "OPEA_DATAPREP_MILVUS" + MILVUS_HOST: ${host_ip} + MILVUS_PORT: 19530 + TEI_EMBEDDING_ENDPOINT: http://tei-embedding-service:80 + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + EMBEDDING_MODEL_ID: ${EMBEDDING_MODEL_ID} + LOGFLAG: ${LOGFLAG} + restart: unless-stopped + depends_on: + milvus-standalone: + condition: service_healthy + etcd: + condition: service_healthy + minio: + condition: service_healthy + + retriever: + image: ${REGISTRY:-opea}/retriever:${TAG:-latest} + container_name: retriever-milvus-server + depends_on: + - milvus-standalone + ports: + - "7000:7000" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + MILVUS_HOST: ${host_ip} + MILVUS_PORT: 19530 + TEI_EMBEDDING_ENDPOINT: http://tei-embedding-service:80 + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + LOGFLAG: ${LOGFLAG} + RETRIEVER_COMPONENT_NAME: "OPEA_RETRIEVER_MILVUS" + restart: unless-stopped + + tei-embedding-service: + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 + container_name: tei-embedding-server + ports: + - "6006:80" + volumes: + - "./data:/data" + shm_size: 1g + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate + + tei-reranking-service: + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 + container_name: tei-reranking-server + ports: + - "8808:80" + volumes: + - "./data:/data" + shm_size: 1g + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + command: --model-id ${RERANK_MODEL_ID} --auto-truncate + + vllm-service: + image: ${REGISTRY:-opea}/vllm:${TAG:-latest} + container_name: vllm-service + ports: + - "9009:80" + volumes: + - "./data:/data" + shm_size: 128g + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + LLM_MODEL_ID: ${LLM_MODEL_ID} + VLLM_TORCH_PROFILER_DIR: "/mnt" + healthcheck: + test: ["CMD-SHELL", "curl -f http://$host_ip:9009/health || exit 1"] + interval: 10s + timeout: 10s + retries: 100 + command: --model $LLM_MODEL_ID --host 0.0.0.0 --port 80 + + chatqna-xeon-backend-server: + image: ${REGISTRY:-opea}/chatqna:${TAG:-latest} + container_name: chatqna-xeon-backend-server + depends_on: + - milvus-standalone + - tei-embedding-service + - dataprep-milvus-service + - retriever + - tei-reranking-service + - vllm-service + ports: + - "8888:8888" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - MEGA_SERVICE_HOST_IP=chatqna-xeon-backend-server + - EMBEDDING_SERVER_HOST_IP=tei-embedding-service + - RETRIEVER_SERVICE_HOST_IP=retriever + - EMBEDDING_SERVER_PORT=${EMBEDDING_SERVER_PORT:-80} + - RERANK_SERVER_HOST_IP=tei-reranking-service + - RERANK_SERVER_PORT=${RERANK_SERVER_PORT:-80} + - LLM_SERVER_HOST_IP=vllm-service + - LLM_SERVER_PORT=${LLM_SERVER_PORT:-80} + - LLM_MODEL=${LLM_MODEL_ID} + - LOGFLAG=${LOGFLAG} + ipc: host + restart: always + + chatqna-xeon-ui-server: + image: ${REGISTRY:-opea}/chatqna-ui:${TAG:-latest} + container_name: chatqna-xeon-ui-server + depends_on: + - chatqna-xeon-backend-server + ports: + - "5173:5173" + ipc: host + restart: always + + chatqna-xeon-nginx-server: + image: ${REGISTRY:-opea}/nginx:${TAG:-latest} + container_name: chatqna-xeon-nginx-server + depends_on: + - chatqna-xeon-backend-server + - chatqna-xeon-ui-server + ports: + - "${NGINX_PORT:-80}:80" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - FRONTEND_SERVICE_IP=chatqna-xeon-ui-server + - FRONTEND_SERVICE_PORT=5173 + - BACKEND_SERVICE_NAME=chatqna + - BACKEND_SERVICE_IP=chatqna-xeon-backend-server + - BACKEND_SERVICE_PORT=8888 + - DATAPREP_SERVICE_IP=dataprep-milvus-service + - DATAPREP_SERVICE_PORT=5000 + ipc: host + restart: always + + +networks: + default: + driver: bridge diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/milvus.yaml b/ChatQnA/docker_compose/intel/cpu/xeon/milvus.yaml new file mode 100644 index 0000000000..b9f22cb3d1 --- /dev/null +++ b/ChatQnA/docker_compose/intel/cpu/xeon/milvus.yaml @@ -0,0 +1,811 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# Licensed to the LF AI & Data foundation under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Related configuration of etcd, used to store Milvus metadata & service discovery. +etcd: + endpoints: localhost:2379 + rootPath: by-dev # The root path where data is stored in etcd + metaSubPath: meta # metaRootPath = rootPath + '/' + metaSubPath + kvSubPath: kv # kvRootPath = rootPath + '/' + kvSubPath + log: + level: info # Only supports debug, info, warn, error, panic, or fatal. Default 'info'. + # path is one of: + # - "default" as os.Stderr, + # - "stderr" as os.Stderr, + # - "stdout" as os.Stdout, + # - file path to append server logs to. + # please adjust in embedded Milvus: /tmp/milvus/logs/etcd.log + path: stdout + ssl: + enabled: false # Whether to support ETCD secure connection mode + tlsCert: /path/to/etcd-client.pem # path to your cert file + tlsKey: /path/to/etcd-client-key.pem # path to your key file + tlsCACert: /path/to/ca.pem # path to your CACert file + # TLS min version + # Optional values: 1.0, 1.1, 1.2, 1.3。 + # We recommend using version 1.2 and above. + tlsMinVersion: 1.3 + requestTimeout: 10000 # Etcd operation timeout in milliseconds + use: + embed: false # Whether to enable embedded Etcd (an in-process EtcdServer). + data: + dir: default.etcd # Embedded Etcd only. please adjust in embedded Milvus: /tmp/milvus/etcdData/ + auth: + enabled: false # Whether to enable authentication + userName: # username for etcd authentication + password: # password for etcd authentication + +metastore: + type: etcd # Default value: etcd, Valid values: [etcd, tikv] + +# Related configuration of tikv, used to store Milvus metadata. +# Notice that when TiKV is enabled for metastore, you still need to have etcd for service discovery. +# TiKV is a good option when the metadata size requires better horizontal scalability. +tikv: + endpoints: 127.0.0.1:2389 # Note that the default pd port of tikv is 2379, which conflicts with etcd. + rootPath: by-dev # The root path where data is stored in tikv + metaSubPath: meta # metaRootPath = rootPath + '/' + metaSubPath + kvSubPath: kv # kvRootPath = rootPath + '/' + kvSubPath + requestTimeout: 10000 # ms, tikv request timeout + snapshotScanSize: 256 # batch size of tikv snapshot scan + ssl: + enabled: false # Whether to support TiKV secure connection mode + tlsCert: # path to your cert file + tlsKey: # path to your key file + tlsCACert: # path to your CACert file + +localStorage: + path: /var/lib/milvus/data/ # please adjust in embedded Milvus: /tmp/milvus/data/ + +# Related configuration of MinIO/S3/GCS or any other service supports S3 API, which is responsible for data persistence for Milvus. +# We refer to the storage service as MinIO/S3 in the following description for simplicity. +minio: + address: localhost # Address of MinIO/S3 + port: 9000 # Port of MinIO/S3 + accessKeyID: minioadmin # accessKeyID of MinIO/S3 + secretAccessKey: minioadmin # MinIO/S3 encryption string + useSSL: false # Access to MinIO/S3 with SSL + ssl: + tlsCACert: /path/to/public.crt # path to your CACert file + bucketName: a-bucket # Bucket name in MinIO/S3 + rootPath: files # The root path where the message is stored in MinIO/S3 + # Whether to useIAM role to access S3/GCS instead of access/secret keys + # For more information, refer to + # aws: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html + # gcp: https://cloud.google.com/storage/docs/access-control/iam + # aliyun (ack): https://www.alibabacloud.com/help/en/container-service-for-kubernetes/latest/use-rrsa-to-enforce-access-control + # aliyun (ecs): https://www.alibabacloud.com/help/en/elastic-compute-service/latest/attach-an-instance-ram-role + useIAM: false + # Cloud Provider of S3. Supports: "aws", "gcp", "aliyun". + # You can use "aws" for other cloud provider supports S3 API with signature v4, e.g.: minio + # You can use "gcp" for other cloud provider supports S3 API with signature v2 + # You can use "aliyun" for other cloud provider uses virtual host style bucket + # When useIAM enabled, only "aws", "gcp", "aliyun" is supported for now + cloudProvider: aws + # Custom endpoint for fetch IAM role credentials. when useIAM is true & cloudProvider is "aws". + # Leave it empty if you want to use AWS default endpoint + iamEndpoint: + logLevel: fatal # Log level for aws sdk log. Supported level: off, fatal, error, warn, info, debug, trace + region: # Specify minio storage system location region + useVirtualHost: false # Whether use virtual host mode for bucket + requestTimeoutMs: 10000 # minio timeout for request time in milliseconds + # The maximum number of objects requested per batch in minio ListObjects rpc, + # 0 means using oss client by default, decrease these configuration if ListObjects timeout + listObjectsMaxKeys: 0 + +# Milvus supports four MQ: rocksmq(based on RockDB), natsmq(embedded nats-server), Pulsar and Kafka. +# You can change your mq by setting mq.type field. +# If you don't set mq.type field as default, there is a note about enabling priority if we config multiple mq in this file. +# 1. standalone(local) mode: rocksmq(default) > natsmq > Pulsar > Kafka +# 2. cluster mode: Pulsar(default) > Kafka (rocksmq and natsmq is unsupported in cluster mode) +mq: + # Default value: "default" + # Valid values: [default, pulsar, kafka, rocksmq, natsmq] + type: default + enablePursuitMode: true # Default value: "true" + pursuitLag: 10 # time tick lag threshold to enter pursuit mode, in seconds + pursuitBufferSize: 8388608 # pursuit mode buffer size in bytes + mqBufSize: 16 # MQ client consumer buffer length + dispatcher: + mergeCheckInterval: 1 # the interval time(in seconds) for dispatcher to check whether to merge + targetBufSize: 16 # the length of channel buffer for targe + maxTolerantLag: 3 # Default value: "3", the timeout(in seconds) that target sends msgPack + +# Related configuration of pulsar, used to manage Milvus logs of recent mutation operations, output streaming log, and provide log publish-subscribe services. +pulsar: + address: localhost # Address of pulsar + port: 6650 # Port of Pulsar + webport: 80 # Web port of pulsar, if you connect directly without proxy, should use 8080 + maxMessageSize: 5242880 # 5 * 1024 * 1024 Bytes, Maximum size of each message in pulsar. + tenant: public + namespace: default + requestTimeout: 60 # pulsar client global request timeout in seconds + enableClientMetrics: false # Whether to register pulsar client metrics into milvus metrics path. + +# If you want to enable kafka, needs to comment the pulsar configs +# kafka: +# brokerList: +# saslUsername: +# saslPassword: +# saslMechanisms: +# securityProtocol: +# ssl: +# enabled: false # whether to enable ssl mode +# tlsCert: # path to client's public key (PEM) used for authentication +# tlsKey: # path to client's private key (PEM) used for authentication +# tlsCaCert: # file or directory path to CA certificate(s) for verifying the broker's key +# tlsKeyPassword: # private key passphrase for use with ssl.key.location and set_ssl_cert(), if any +# readTimeout: 10 + +rocksmq: + # The path where the message is stored in rocksmq + # please adjust in embedded Milvus: /tmp/milvus/rdb_data + path: /var/lib/milvus/rdb_data + lrucacheratio: 0.06 # rocksdb cache memory ratio + rocksmqPageSize: 67108864 # 64 MB, 64 * 1024 * 1024 bytes, The size of each page of messages in rocksmq + retentionTimeInMinutes: 4320 # 3 days, 3 * 24 * 60 minutes, The retention time of the message in rocksmq. + retentionSizeInMB: 8192 # 8 GB, 8 * 1024 MB, The retention size of the message in rocksmq. + compactionInterval: 86400 # 1 day, trigger rocksdb compaction every day to remove deleted data + compressionTypes: 0,0,7,7,7 # compaction compression type, only support use 0,7. 0 means not compress, 7 will use zstd. Length of types means num of rocksdb level. + +# natsmq configuration. +# more detail: https://docs.nats.io/running-a-nats-service/configuration +natsmq: + server: + port: 4222 # Port for nats server listening + storeDir: /var/lib/milvus/nats # Directory to use for JetStream storage of nats + maxFileStore: 17179869184 # Maximum size of the 'file' storage + maxPayload: 8388608 # Maximum number of bytes in a message payload + maxPending: 67108864 # Maximum number of bytes buffered for a connection Applies to client connections + initializeTimeout: 4000 # waiting for initialization of natsmq finished + monitor: + trace: false # If true enable protocol trace log messages + debug: false # If true enable debug log messages + logTime: true # If set to false, log without timestamps. + logFile: /tmp/milvus/logs/nats.log # Log file path relative to .. of milvus binary if use relative path + logSizeLimit: 536870912 # Size in bytes after the log file rolls over to a new one + retention: + maxAge: 4320 # Maximum age of any message in the P-channel + maxBytes: # How many bytes the single P-channel may contain. Removing oldest messages if the P-channel exceeds this size + maxMsgs: # How many message the single P-channel may contain. Removing oldest messages if the P-channel exceeds this limit + +# Related configuration of rootCoord, used to handle data definition language (DDL) and data control language (DCL) requests +rootCoord: + dmlChannelNum: 16 # The number of dml channels created at system startup + maxPartitionNum: 1024 # Maximum number of partitions in a collection + minSegmentSizeToEnableIndex: 1024 # It's a threshold. When the segment size is less than this value, the segment will not be indexed + enableActiveStandby: false + maxDatabaseNum: 64 # Maximum number of database + maxGeneralCapacity: 65536 # upper limit for the sum of of product of partitionNumber and shardNumber + gracefulStopTimeout: 5 # seconds. force stop node without graceful stop + ip: # if not specified, use the first unicastable address + port: 53100 + grpc: + serverMaxSendSize: 536870912 + serverMaxRecvSize: 268435456 + clientMaxSendSize: 268435456 + clientMaxRecvSize: 536870912 + +# Related configuration of proxy, used to validate client requests and reduce the returned results. +proxy: + timeTickInterval: 200 # ms, the interval that proxy synchronize the time tick + healthCheckTimeout: 3000 # ms, the interval that to do component healthy check + msgStream: + timeTick: + bufSize: 512 + maxNameLength: 255 # Maximum length of name for a collection or alias + # Maximum number of fields in a collection. + # As of today (2.2.0 and after) it is strongly DISCOURAGED to set maxFieldNum >= 64. + # So adjust at your risk! + maxFieldNum: 64 + maxVectorFieldNum: 4 # Maximum number of vector fields in a collection. + maxShardNum: 16 # Maximum number of shards in a collection + maxDimension: 32768 # Maximum dimension of a vector + # Whether to produce gin logs.\n + # please adjust in embedded Milvus: false + ginLogging: true + ginLogSkipPaths: / # skip url path for gin log + maxTaskNum: 1024 # max task number of proxy task queue + mustUsePartitionKey: false # switch for whether proxy must use partition key for the collection + accessLog: + enable: false # if use access log + minioEnable: false # if upload sealed access log file to minio + localPath: /tmp/milvus_access + filename: # Log filename, leave empty to use stdout. + maxSize: 64 # Max size for a single file, in MB. + cacheSize: 10240 # Size of log of memory cache, in B + rotatedTime: 0 # Max time for single access log file in seconds + remotePath: access_log/ # File path in minIO + remoteMaxTime: 0 # Max time for log file in minIO, in hours + formatters: + base: + format: "[$time_now] [ACCESS] <$user_name: $user_addr> $method_name [status: $method_status] [code: $error_code] [sdk: $sdk_version] [msg: $error_msg] [traceID: $trace_id] [timeCost: $time_cost]" + query: + format: "[$time_now] [ACCESS] <$user_name: $user_addr> $method_name [status: $method_status] [code: $error_code] [sdk: $sdk_version] [msg: $error_msg] [traceID: $trace_id] [timeCost: $time_cost] [database: $database_name] [collection: $collection_name] [partitions: $partition_name] [expr: $method_expr]" + methods: "Query,Search,Delete" + connectionCheckIntervalSeconds: 120 # the interval time(in seconds) for connection manager to scan inactive client info + connectionClientInfoTTLSeconds: 86400 # inactive client info TTL duration, in seconds + maxConnectionNum: 10000 # the max client info numbers that proxy should manage, avoid too many client infos + gracefulStopTimeout: 30 # seconds. force stop node without graceful stop + slowQuerySpanInSeconds: 5 # query whose executed time exceeds the `slowQuerySpanInSeconds` can be considered slow, in seconds. + http: + enabled: true # Whether to enable the http server + debug_mode: false # Whether to enable http server debug mode + port: # high-level restful api + acceptTypeAllowInt64: true # high-level restful api, whether http client can deal with int64 + enablePprof: true # Whether to enable pprof middleware on the metrics port + ip: # if not specified, use the first unicastable address + port: 19530 + internalPort: 19529 + grpc: + serverMaxSendSize: 268435456 + serverMaxRecvSize: 67108864 + clientMaxSendSize: 268435456 + clientMaxRecvSize: 67108864 + +# Related configuration of queryCoord, used to manage topology and load balancing for the query nodes, and handoff from growing segments to sealed segments. +queryCoord: + taskMergeCap: 1 + taskExecutionCap: 256 + autoHandoff: true # Enable auto handoff + autoBalance: true # Enable auto balance + autoBalanceChannel: true # Enable auto balance channel + balancer: ScoreBasedBalancer # auto balancer used for segments on queryNodes + globalRowCountFactor: 0.1 # the weight used when balancing segments among queryNodes + scoreUnbalanceTolerationFactor: 0.05 # the least value for unbalanced extent between from and to nodes when doing balance + reverseUnBalanceTolerationFactor: 1.3 # the largest value for unbalanced extent between from and to nodes after doing balance + overloadedMemoryThresholdPercentage: 90 # The threshold percentage that memory overload + balanceIntervalSeconds: 60 + memoryUsageMaxDifferencePercentage: 30 + rowCountFactor: 0.4 # the row count weight used when balancing segments among queryNodes + segmentCountFactor: 0.4 # the segment count weight used when balancing segments among queryNodes + globalSegmentCountFactor: 0.1 # the segment count weight used when balancing segments among queryNodes + segmentCountMaxSteps: 50 # segment count based plan generator max steps + rowCountMaxSteps: 50 # segment count based plan generator max steps + randomMaxSteps: 10 # segment count based plan generator max steps + growingRowCountWeight: 4 # the memory weight of growing segment row count + balanceCostThreshold: 0.001 # the threshold of balance cost, if the difference of cluster's cost after executing the balance plan is less than this value, the plan will not be executed + checkSegmentInterval: 1000 + checkChannelInterval: 1000 + checkBalanceInterval: 10000 + checkIndexInterval: 10000 + channelTaskTimeout: 60000 # 1 minute + segmentTaskTimeout: 120000 # 2 minute + distPullInterval: 500 + collectionObserverInterval: 200 + checkExecutedFlagInterval: 100 + heartbeatAvailableInterval: 10000 # 10s, Only QueryNodes which fetched heartbeats within the duration are available + loadTimeoutSeconds: 600 + distRequestTimeout: 5000 # the request timeout for querycoord fetching data distribution from querynodes, in milliseconds + heatbeatWarningLag: 5000 # the lag value for querycoord report warning when last heartbeat is too old, in milliseconds + checkHandoffInterval: 5000 + enableActiveStandby: false + checkInterval: 1000 + checkHealthInterval: 3000 # 3s, the interval when query coord try to check health of query node + checkHealthRPCTimeout: 2000 # 100ms, the timeout of check health rpc to query node + brokerTimeout: 5000 # 5000ms, querycoord broker rpc timeout + collectionRecoverTimes: 3 # if collection recover times reach the limit during loading state, release it + observerTaskParallel: 16 # the parallel observer dispatcher task number + checkAutoBalanceConfigInterval: 10 # the interval of check auto balance config + checkNodeSessionInterval: 60 # the interval(in seconds) of check querynode cluster session + gracefulStopTimeout: 5 # seconds. force stop node without graceful stop + enableStoppingBalance: true # whether enable stopping balance + channelExclusiveNodeFactor: 4 # the least node number for enable channel's exclusive mode + cleanExcludeSegmentInterval: 60 # the time duration of clean pipeline exclude segment which used for filter invalid data, in seconds + ip: # if not specified, use the first unicastable address + port: 19531 + grpc: + serverMaxSendSize: 536870912 + serverMaxRecvSize: 268435456 + clientMaxSendSize: 268435456 + clientMaxRecvSize: 536870912 + +# Related configuration of queryNode, used to run hybrid search between vector and scalar data. +queryNode: + stats: + publishInterval: 1000 # Interval for querynode to report node information (milliseconds) + segcore: + knowhereThreadPoolNumRatio: 4 # The number of threads in knowhere's thread pool. If disk is enabled, the pool size will multiply with knowhereThreadPoolNumRatio([1, 32]). + chunkRows: 128 # The number of vectors in a chunk. + interimIndex: + enableIndex: true # Enable segment build with index to accelerate vector search when segment is in growing or binlog. + nlist: 128 # temp index nlist, recommend to set sqrt(chunkRows), must smaller than chunkRows/8 + nprobe: 16 # nprobe to search small index, based on your accuracy requirement, must smaller than nlist + memExpansionRate: 1.15 # extra memory needed by building interim index + buildParallelRate: 0.5 # the ratio of building interim index parallel matched with cpu num + knowhereScoreConsistency: false # Enable knowhere strong consistency score computation logic + loadMemoryUsageFactor: 1 # The multiply factor of calculating the memory usage while loading segments + enableDisk: false # enable querynode load disk index, and search on disk index + maxDiskUsagePercentage: 95 + cache: + enabled: true + memoryLimit: 2147483648 # 2 GB, 2 * 1024 *1024 *1024 + readAheadPolicy: willneed # The read ahead policy of chunk cache, options: `normal, random, sequential, willneed, dontneed` + # options: async, sync, disable. + # Specifies the necessity for warming up the chunk cache. + # 1. If set to "sync" or "async" the original vector data will be synchronously/asynchronously loaded into the + # chunk cache during the load process. This approach has the potential to substantially reduce query/search latency + # for a specific duration post-load, albeit accompanied by a concurrent increase in disk usage; + # 2. If set to "disable" original vector data will only be loaded into the chunk cache during search/query. + warmup: disable + mmap: + mmapEnabled: false # Enable mmap for loading data + lazyload: + enabled: false # Enable lazyload for loading data + waitTimeout: 30000 # max wait timeout duration in milliseconds before start to do lazyload search and retrieve + requestResourceTimeout: 5000 # max timeout in milliseconds for waiting request resource for lazy load, 5s by default + requestResourceRetryInterval: 2000 # retry interval in milliseconds for waiting request resource for lazy load, 2s by default + maxRetryTimes: 1 # max retry times for lazy load, 1 by default + maxEvictPerRetry: 1 # max evict count for lazy load, 1 by default + grouping: + enabled: true + maxNQ: 1000 + topKMergeRatio: 20 + scheduler: + receiveChanSize: 10240 + unsolvedQueueSize: 10240 + # maxReadConcurrentRatio is the concurrency ratio of read task (search task and query task). + # Max read concurrency would be the value of hardware.GetCPUNum * maxReadConcurrentRatio. + # It defaults to 2.0, which means max read concurrency would be the value of hardware.GetCPUNum * 2. + # Max read concurrency must greater than or equal to 1, and less than or equal to hardware.GetCPUNum * 100. + # (0, 100] + maxReadConcurrentRatio: 1 + cpuRatio: 10 # ratio used to estimate read task cpu usage. + maxTimestampLag: 86400 + scheduleReadPolicy: + # fifo: A FIFO queue support the schedule. + # user-task-polling: + # The user's tasks will be polled one by one and scheduled. + # Scheduling is fair on task granularity. + # The policy is based on the username for authentication. + # And an empty username is considered the same user. + # When there are no multi-users, the policy decay into FIFO" + name: fifo + taskQueueExpire: 60 # Control how long (many seconds) that queue retains since queue is empty + enableCrossUserGrouping: false # Enable Cross user grouping when using user-task-polling policy. (Disable it if user's task can not merge each other) + maxPendingTaskPerUser: 1024 # Max pending task per user in scheduler + dataSync: + flowGraph: + maxQueueLength: 16 # Maximum length of task queue in flowgraph + maxParallelism: 1024 # Maximum number of tasks executed in parallel in the flowgraph + enableSegmentPrune: false # use partition prune function on shard delegator + ip: # if not specified, use the first unicastable address + port: 21123 + grpc: + serverMaxSendSize: 536870912 + serverMaxRecvSize: 268435456 + clientMaxSendSize: 268435456 + clientMaxRecvSize: 536870912 + +indexCoord: + bindIndexNodeMode: + enable: false + address: localhost:22930 + withCred: false + nodeID: 0 + segment: + minSegmentNumRowsToEnableIndex: 1024 # It's a threshold. When the segment num rows is less than this value, the segment will not be indexed + +indexNode: + scheduler: + buildParallel: 1 + enableDisk: true # enable index node build disk vector index + maxDiskUsagePercentage: 95 + ip: # if not specified, use the first unicastable address + port: 21121 + grpc: + serverMaxSendSize: 536870912 + serverMaxRecvSize: 268435456 + clientMaxSendSize: 268435456 + clientMaxRecvSize: 536870912 + +dataCoord: + channel: + watchTimeoutInterval: 300 # Timeout on watching channels (in seconds). Datanode tickler update watch progress will reset timeout timer. + balanceWithRpc: true # Whether to enable balance with RPC, default to use etcd watch + legacyVersionWithoutRPCWatch: 2.4.1 # Datanodes <= this version are considered as legacy nodes, which doesn't have rpc based watch(). This is only used during rolling upgrade where legacy nodes won't get new channels + balanceSilentDuration: 300 # The duration after which the channel manager start background channel balancing + balanceInterval: 360 # The interval with which the channel manager check dml channel balance status + checkInterval: 1 # The interval in seconds with which the channel manager advances channel states + notifyChannelOperationTimeout: 5 # Timeout notifing channel operations (in seconds). + segment: + maxSize: 1024 # Maximum size of a segment in MB + diskSegmentMaxSize: 2048 # Maximum size of a segment in MB for collection which has Disk index + sealProportion: 0.12 + assignmentExpiration: 2000 # The time of the assignment expiration in ms + allocLatestExpireAttempt: 200 # The time attempting to alloc latest lastExpire from rootCoord after restart + maxLife: 86400 # The max lifetime of segment in seconds, 24*60*60 + # If a segment didn't accept dml records in maxIdleTime and the size of segment is greater than + # minSizeFromIdleToSealed, Milvus will automatically seal it. + # The max idle time of segment in seconds, 10*60. + maxIdleTime: 600 + minSizeFromIdleToSealed: 16 # The min size in MB of segment which can be idle from sealed. + # The max number of binlog file for one segment, the segment will be sealed if + # the number of binlog file reaches to max value. + maxBinlogFileNumber: 32 + smallProportion: 0.5 # The segment is considered as "small segment" when its # of rows is smaller than + # (smallProportion * segment max # of rows). + # A compaction will happen on small segments if the segment after compaction will have + compactableProportion: 0.85 + # over (compactableProportion * segment max # of rows) rows. + # MUST BE GREATER THAN OR EQUAL TO !!! + # During compaction, the size of segment # of rows is able to exceed segment max # of rows by (expansionRate-1) * 100%. + expansionRate: 1.25 + autoUpgradeSegmentIndex: false # whether auto upgrade segment index to index engine's version + enableCompaction: true # Enable data segment compaction + compaction: + enableAutoCompaction: true + indexBasedCompaction: true + rpcTimeout: 10 + maxParallelTaskNum: 10 + workerMaxParallelTaskNum: 2 + levelzero: + forceTrigger: + minSize: 8388608 # The minimum size in bytes to force trigger a LevelZero Compaction, default as 8MB + maxSize: 67108864 # The maxmum size in bytes to force trigger a LevelZero Compaction, default as 64MB + deltalogMinNum: 10 # The minimum number of deltalog files to force trigger a LevelZero Compaction + deltalogMaxNum: 30 # The maxmum number of deltalog files to force trigger a LevelZero Compaction, default as 30 + enableGarbageCollection: true + gc: + interval: 3600 # gc interval in seconds + missingTolerance: 86400 # file meta missing tolerance duration in seconds, default to 24hr(1d) + dropTolerance: 10800 # file belongs to dropped entity tolerance duration in seconds. 3600 + removeConcurrent: 32 # number of concurrent goroutines to remove dropped s3 objects + scanInterval: 168 # garbage collection scan residue interval in hours + enableActiveStandby: false + brokerTimeout: 5000 # 5000ms, dataCoord broker rpc timeout + autoBalance: true # Enable auto balance + checkAutoBalanceConfigInterval: 10 # the interval of check auto balance config + import: + filesPerPreImportTask: 2 # The maximum number of files allowed per pre-import task. + taskRetention: 10800 # The retention period in seconds for tasks in the Completed or Failed state. + maxSizeInMBPerImportTask: 6144 # To prevent generating of small segments, we will re-group imported files. This parameter represents the sum of file sizes in each group (each ImportTask). + scheduleInterval: 2 # The interval for scheduling import, measured in seconds. + checkIntervalHigh: 2 # The interval for checking import, measured in seconds, is set to a high frequency for the import checker. + checkIntervalLow: 120 # The interval for checking import, measured in seconds, is set to a low frequency for the import checker. + maxImportFileNumPerReq: 1024 # The maximum number of files allowed per single import request. + waitForIndex: true # Indicates whether the import operation waits for the completion of index building. + gracefulStopTimeout: 5 # seconds. force stop node without graceful stop + ip: # if not specified, use the first unicastable address + port: 13333 + grpc: + serverMaxSendSize: 536870912 + serverMaxRecvSize: 268435456 + clientMaxSendSize: 268435456 + clientMaxRecvSize: 536870912 + +dataNode: + dataSync: + flowGraph: + maxQueueLength: 16 # Maximum length of task queue in flowgraph + maxParallelism: 1024 # Maximum number of tasks executed in parallel in the flowgraph + maxParallelSyncMgrTasks: 256 # The max concurrent sync task number of datanode sync mgr globally + skipMode: + enable: true # Support skip some timetick message to reduce CPU usage + skipNum: 4 # Consume one for every n records skipped + coldTime: 60 # Turn on skip mode after there are only timetick msg for x seconds + segment: + insertBufSize: 16777216 # Max buffer size to flush for a single segment. + deleteBufBytes: 16777216 # Max buffer size in bytes to flush del for a single channel, default as 16MB + syncPeriod: 600 # The period to sync segments if buffer is not empty. + memory: + forceSyncEnable: true # Set true to force sync if memory usage is too high + forceSyncSegmentNum: 1 # number of segments to sync, segments with top largest buffer will be synced. + checkInterval: 3000 # the interval to check datanode memory usage, in milliseconds + forceSyncWatermark: 0.5 # memory watermark for standalone, upon reaching this watermark, segments will be synced. + timetick: + byRPC: true + interval: 500 + channel: + # specify the size of global work pool of all channels + # if this parameter <= 0, will set it as the maximum number of CPUs that can be executing + # suggest to set it bigger on large collection numbers to avoid blocking + workPoolSize: -1 + # specify the size of global work pool for channel checkpoint updating + # if this parameter <= 0, will set it as 10 + updateChannelCheckpointMaxParallel: 10 + updateChannelCheckpointInterval: 60 # the interval duration(in seconds) for datanode to update channel checkpoint of each channel + updateChannelCheckpointRPCTimeout: 20 # timeout in seconds for UpdateChannelCheckpoint RPC call + maxChannelCheckpointsPerPRC: 128 # The maximum number of channel checkpoints per UpdateChannelCheckpoint RPC. + channelCheckpointUpdateTickInSeconds: 10 # The frequency, in seconds, at which the channel checkpoint updater executes updates. + import: + maxConcurrentTaskNum: 16 # The maximum number of import/pre-import tasks allowed to run concurrently on a datanode. + maxImportFileSizeInGB: 16 # The maximum file size (in GB) for an import file, where an import file refers to either a Row-Based file or a set of Column-Based files. + readBufferSizeInMB: 16 # The data block size (in MB) read from chunk manager by the datanode during import. + compaction: + levelZeroBatchMemoryRatio: 0.05 # The minimal memory ratio of free memory for level zero compaction executing in batch mode + gracefulStopTimeout: 1800 # seconds. force stop node without graceful stop + ip: # if not specified, use the first unicastable address + port: 21124 + grpc: + serverMaxSendSize: 536870912 + serverMaxRecvSize: 268435456 + clientMaxSendSize: 268435456 + clientMaxRecvSize: 536870912 + +# Configures the system log output. +log: + level: info # Only supports debug, info, warn, error, panic, or fatal. Default 'info'. + file: + rootPath: # root dir path to put logs, default "" means no log file will print. please adjust in embedded Milvus: /tmp/milvus/logs + maxSize: 300 # MB + maxAge: 10 # Maximum time for log retention in day. + maxBackups: 20 + format: text # text or json + stdout: true # Stdout enable or not + +grpc: + log: + level: WARNING + gracefulStopTimeout: 10 # second, time to wait graceful stop finish + client: + compressionEnabled: false + dialTimeout: 200 + keepAliveTime: 10000 + keepAliveTimeout: 20000 + maxMaxAttempts: 10 + initialBackoff: 0.2 + maxBackoff: 10 + minResetInterval: 1000 + maxCancelError: 32 + minSessionCheckInterval: 200 + +# Configure the proxy tls enable. +tls: + serverPemPath: configs/cert/server.pem + serverKeyPath: configs/cert/server.key + caPemPath: configs/cert/ca.pem + +common: + defaultPartitionName: _default # default partition name for a collection + defaultIndexName: _default_idx # default index name + entityExpiration: -1 # Entity expiration in seconds, CAUTION -1 means never expire + indexSliceSize: 16 # MB + threadCoreCoefficient: + highPriority: 10 # This parameter specify how many times the number of threads is the number of cores in high priority pool + middlePriority: 5 # This parameter specify how many times the number of threads is the number of cores in middle priority pool + lowPriority: 1 # This parameter specify how many times the number of threads is the number of cores in low priority pool + buildIndexThreadPoolRatio: 0.75 + DiskIndex: + MaxDegree: 56 + SearchListSize: 100 + PQCodeBudgetGBRatio: 0.125 + BuildNumThreadsRatio: 1 + SearchCacheBudgetGBRatio: 0.1 + LoadNumThreadRatio: 8 + BeamWidthRatio: 4 + gracefulTime: 5000 # milliseconds. it represents the interval (in ms) by which the request arrival time needs to be subtracted in the case of Bounded Consistency. + gracefulStopTimeout: 1800 # seconds. it will force quit the server if the graceful stop process is not completed during this time. + storageType: remote # please adjust in embedded Milvus: local, available values are [local, remote, opendal], value minio is deprecated, use remote instead + # Default value: auto + # Valid values: [auto, avx512, avx2, avx, sse4_2] + # This configuration is only used by querynode and indexnode, it selects CPU instruction set for Searching and Index-building. + simdType: auto + security: + authorizationEnabled: false + # The superusers will ignore some system check processes, + # like the old password verification when updating the credential + superUsers: + tlsMode: 0 + session: + ttl: 30 # ttl value when session granting a lease to register service + retryTimes: 30 # retry times when session sending etcd requests + locks: + metrics: + enable: false # whether gather statistics for metrics locks + threshold: + info: 500 # minimum milliseconds for printing durations in info level + warn: 1000 # minimum milliseconds for printing durations in warn level + storage: + scheme: s3 + enablev2: false + ttMsgEnabled: true # Whether the instance disable sending ts messages + traceLogMode: 0 # trace request info + bloomFilterSize: 100000 # bloom filter initial size + maxBloomFalsePositive: 0.001 # max false positive rate for bloom filter + +# QuotaConfig, configurations of Milvus quota and limits. +# By default, we enable: +# 1. TT protection; +# 2. Memory protection. +# 3. Disk quota protection. +# You can enable: +# 1. DML throughput limitation; +# 2. DDL, DQL qps/rps limitation; +# 3. DQL Queue length/latency protection; +# 4. DQL result rate protection; +# If necessary, you can also manually force to deny RW requests. +quotaAndLimits: + enabled: true # `true` to enable quota and limits, `false` to disable. + # quotaCenterCollectInterval is the time interval that quotaCenter + # collects metrics from Proxies, Query cluster and Data cluster. + # seconds, (0 ~ 65536) + quotaCenterCollectInterval: 3 + ddl: + enabled: false + collectionRate: -1 # qps, default no limit, rate for CreateCollection, DropCollection, LoadCollection, ReleaseCollection + partitionRate: -1 # qps, default no limit, rate for CreatePartition, DropPartition, LoadPartition, ReleasePartition + db: + collectionRate: -1 # qps of db level , default no limit, rate for CreateCollection, DropCollection, LoadCollection, ReleaseCollection + partitionRate: -1 # qps of db level, default no limit, rate for CreatePartition, DropPartition, LoadPartition, ReleasePartition + indexRate: + enabled: false + max: -1 # qps, default no limit, rate for CreateIndex, DropIndex + db: + max: -1 # qps of db level, default no limit, rate for CreateIndex, DropIndex + flushRate: + enabled: true + max: -1 # qps, default no limit, rate for flush + collection: + max: 0.1 # qps, default no limit, rate for flush at collection level. + db: + max: -1 # qps of db level, default no limit, rate for flush + compactionRate: + enabled: false + max: -1 # qps, default no limit, rate for manualCompaction + db: + max: -1 # qps of db level, default no limit, rate for manualCompaction + dml: + # dml limit rates, default no limit. + # The maximum rate will not be greater than max. + enabled: false + insertRate: + max: -1 # MB/s, default no limit + db: + max: -1 # MB/s, default no limit + collection: + max: -1 # MB/s, default no limit + partition: + max: -1 # MB/s, default no limit + upsertRate: + max: -1 # MB/s, default no limit + db: + max: -1 # MB/s, default no limit + collection: + max: -1 # MB/s, default no limit + partition: + max: -1 # MB/s, default no limit + deleteRate: + max: -1 # MB/s, default no limit + db: + max: -1 # MB/s, default no limit + collection: + max: -1 # MB/s, default no limit + partition: + max: -1 # MB/s, default no limit + bulkLoadRate: + max: -1 # MB/s, default no limit, not support yet. TODO: limit bulkLoad rate + db: + max: -1 # MB/s, default no limit, not support yet. TODO: limit db bulkLoad rate + collection: + max: -1 # MB/s, default no limit, not support yet. TODO: limit collection bulkLoad rate + partition: + max: -1 # MB/s, default no limit, not support yet. TODO: limit partition bulkLoad rate + dql: + # dql limit rates, default no limit. + # The maximum rate will not be greater than max. + enabled: false + searchRate: + max: -1 # vps (vectors per second), default no limit + db: + max: -1 # vps (vectors per second), default no limit + collection: + max: -1 # vps (vectors per second), default no limit + partition: + max: -1 # vps (vectors per second), default no limit + queryRate: + max: -1 # qps, default no limit + db: + max: -1 # qps, default no limit + collection: + max: -1 # qps, default no limit + partition: + max: -1 # qps, default no limit + limits: + maxCollectionNum: 65536 + maxCollectionNumPerDB: 65536 + maxInsertSize: -1 # maximum size of a single insert request, in bytes, -1 means no limit + maxResourceGroupNumOfQueryNode: 1024 # maximum number of resource groups of query nodes + limitWriting: + # forceDeny false means dml requests are allowed (except for some + # specific conditions, such as memory of nodes to water marker), true means always reject all dml requests. + forceDeny: false + ttProtection: + enabled: false + # maxTimeTickDelay indicates the backpressure for DML Operations. + # DML rates would be reduced according to the ratio of time tick delay to maxTimeTickDelay, + # if time tick delay is greater than maxTimeTickDelay, all DML requests would be rejected. + # seconds + maxTimeTickDelay: 300 + memProtection: + # When memory usage > memoryHighWaterLevel, all dml requests would be rejected; + # When memoryLowWaterLevel < memory usage < memoryHighWaterLevel, reduce the dml rate; + # When memory usage < memoryLowWaterLevel, no action. + enabled: true + dataNodeMemoryLowWaterLevel: 0.85 # (0, 1], memoryLowWaterLevel in DataNodes + dataNodeMemoryHighWaterLevel: 0.95 # (0, 1], memoryHighWaterLevel in DataNodes + queryNodeMemoryLowWaterLevel: 0.85 # (0, 1], memoryLowWaterLevel in QueryNodes + queryNodeMemoryHighWaterLevel: 0.95 # (0, 1], memoryHighWaterLevel in QueryNodes + growingSegmentsSizeProtection: + # No action will be taken if the growing segments size is less than the low watermark. + # When the growing segments size exceeds the low watermark, the dml rate will be reduced, + # but the rate will not be lower than minRateRatio * dmlRate. + enabled: false + minRateRatio: 0.5 + lowWaterLevel: 0.2 + highWaterLevel: 0.4 + diskProtection: + enabled: true # When the total file size of object storage is greater than `diskQuota`, all dml requests would be rejected; + diskQuota: -1 # MB, (0, +inf), default no limit + diskQuotaPerDB: -1 # MB, (0, +inf), default no limit + diskQuotaPerCollection: -1 # MB, (0, +inf), default no limit + diskQuotaPerPartition: -1 # MB, (0, +inf), default no limit + limitReading: + # forceDeny false means dql requests are allowed (except for some + # specific conditions, such as collection has been dropped), true means always reject all dql requests. + forceDeny: false + queueProtection: + enabled: false + # nqInQueueThreshold indicated that the system was under backpressure for Search/Query path. + # If NQ in any QueryNode's queue is greater than nqInQueueThreshold, search&query rates would gradually cool off + # until the NQ in queue no longer exceeds nqInQueueThreshold. We think of the NQ of query request as 1. + # int, default no limit + nqInQueueThreshold: -1 + # queueLatencyThreshold indicated that the system was under backpressure for Search/Query path. + # If dql latency of queuing is greater than queueLatencyThreshold, search&query rates would gradually cool off + # until the latency of queuing no longer exceeds queueLatencyThreshold. + # The latency here refers to the averaged latency over a period of time. + # milliseconds, default no limit + queueLatencyThreshold: -1 + resultProtection: + enabled: false + # maxReadResultRate indicated that the system was under backpressure for Search/Query path. + # If dql result rate is greater than maxReadResultRate, search&query rates would gradually cool off + # until the read result rate no longer exceeds maxReadResultRate. + # MB/s, default no limit + maxReadResultRate: -1 + maxReadResultRatePerDB: -1 + maxReadResultRatePerCollection: -1 + # colOffSpeed is the speed of search&query rates cool off. + # (0, 1] + coolOffSpeed: 0.9 + +trace: + # trace exporter type, default is stdout, + # optional values: ['noop','stdout', 'jaeger', 'otlp'] + exporter: noop + # fraction of traceID based sampler, + # optional values: [0, 1] + # Fractions >= 1 will always sample. Fractions < 0 are treated as zero. + sampleFraction: 0 + jaeger: + url: # when exporter is jaeger should set the jaeger's URL + otlp: + endpoint: # example: "127.0.0.1:4318" + secure: true + +#when using GPU indexing, Milvus will utilize a memory pool to avoid frequent memory allocation and deallocation. +#here, you can set the size of the memory occupied by the memory pool, with the unit being MB. +#note that there is a possibility of Milvus crashing when the actual memory demand exceeds the value set by maxMemSize. +#if initMemSize and MaxMemSize both set zero, +#milvus will automatically initialize half of the available GPU memory, +#maxMemSize will the whole available GPU memory. +gpu: + initMemSize: # Gpu Memory Pool init size + maxMemSize: # Gpu Memory Pool Max size diff --git a/ChatQnA/tests/test_compose_milvus_on_xeon.sh b/ChatQnA/tests/test_compose_milvus_on_xeon.sh new file mode 100644 index 0000000000..d2953a9992 --- /dev/null +++ b/ChatQnA/tests/test_compose_milvus_on_xeon.sh @@ -0,0 +1,249 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -e +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" +export REGISTRY=${IMAGE_REPO} +export TAG=${IMAGE_TAG} + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') +export host_ip=$(hostname -I | awk '{print $1}') + +function build_docker_images() { + opea_branch=${opea_branch:-"main"} + # If the opea_branch isn't main, replace the git clone branch in Dockerfile. + if [[ "${opea_branch}" != "main" ]]; then + cd $WORKPATH + OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" + NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" + find . -type f -name "Dockerfile*" | while read -r file; do + echo "Processing file: $file" + sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" + done + fi + + cd $WORKPATH/docker_image_build + git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + git clone https://github.com/vllm-project/vllm.git && cd vllm + VLLM_VER="$(git describe --tags "$(git rev-list --tags --max-count=1)" )" + echo "Check out vLLM tag ${VLLM_VER}" + git checkout ${VLLM_VER} &> /dev/null + # make sure NOT change the pwd + cd ../ + + echo "Build all the images with --no-cache, check docker_image_build.log for details..." + service_list="chatqna chatqna-ui dataprep retriever vllm nginx" + docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log + + docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 + + docker images && sleep 1s +} +function start_services() { + cd $WORKPATH/docker_compose/intel/cpu/xeon/ + export no_proxy=${no_proxy},${ip_address} + export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5" + export RERANK_MODEL_ID="BAAI/bge-reranker-base" + export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct" + export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} + export LOGFLAG=true + + # Start Docker Containers + docker compose -f compose_milvus.yaml up -d > ${LOG_PATH}/start_services_with_compose.log + + n=0 + until [[ "$n" -ge 100 ]]; do + docker logs vllm-service > ${LOG_PATH}/vllm_service_start.log 2>&1 + if grep -q complete ${LOG_PATH}/vllm_service_start.log; then + break + fi + sleep 5s + n=$((n+1)) + done +} + +function validate_service() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + if [[ $SERVICE_NAME == *"dataprep_upload_file"* ]]; then + cd $LOG_PATH + HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" -X POST -F 'files=@./dataprep_file.txt' -H 'Content-Type: multipart/form-data' "$URL") + elif [[ $SERVICE_NAME == *"dataprep_del"* ]]; then + HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" -X POST -d '{"file_path": "all"}' -H 'Content-Type: application/json' "$URL") + else + HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + fi + HTTP_STATUS=$(echo $HTTP_RESPONSE | tr -d '\n' | sed -e 's/.*HTTPSTATUS://') + RESPONSE_BODY=$(echo $HTTP_RESPONSE | sed -e 's/HTTPSTATUS\:.*//g') + + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + + + # check response status + if [ "$HTTP_STATUS" -ne "200" ]; then + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + exit 1 + else + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + fi + echo "Response" + echo $RESPONSE_BODY + echo "Expected Result" + echo $EXPECTED_RESULT + # check response body + if [[ "$RESPONSE_BODY" != *"$EXPECTED_RESULT"* ]]; then + echo "[ $SERVICE_NAME ] Content does not match the expected result: $RESPONSE_BODY" + exit 1 + else + echo "[ $SERVICE_NAME ] Content is as expected." + fi + + sleep 1s +} + +function validate_microservices() { + # Check if the microservices are running correctly. + + # tei for embedding service + validate_service \ + "${ip_address}:6006/embed" \ + "[[" \ + "tei-embedding" \ + "tei-embedding-server" \ + '{"inputs":"What is Deep Learning?"}' + + sleep 1m # retrieval can't curl as expected, try to wait for more time + + # test /v1/dataprep/ingest upload file + echo "Deep learning is a subset of machine learning that utilizes neural networks with multiple layers to analyze various levels of abstract data representations. It enables computers to identify patterns and make decisions with minimal human intervention by learning from large amounts of data." > $LOG_PATH/dataprep_file.txt + validate_service \ + "http://${ip_address}:11101/v1/dataprep/ingest" \ + "Data preparation succeeded" \ + "dataprep_upload_file" \ + "dataprep-milvus-server" + + # test /v1/dataprep/delete + validate_service \ + "http://${ip_address}:11101/v1/dataprep/delete" \ + '{"status":true}' \ + "dataprep_del" \ + "dataprep-milvus-server" + + # test /v1/dataprep/delete + validate_service \ + "http://${ip_address}:11101/v1/dataprep/delete" \ + '{"status":true}' \ + "dataprep_del" \ + "dataprep-milvus-server" + + + # retrieval microservice + test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") + validate_service \ + "${ip_address}:7000/v1/retrieval" \ + " " \ + "retrieval" \ + "retriever-milvus-server" \ + "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}" + + # tei for rerank microservice + echo "Validating reranking service" + validate_service \ + "${ip_address}:8808/rerank" \ + '{"index":1,"score":' \ + "tei-rerank" \ + "tei-reranking-server" \ + '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}' + + + # tgi for llm service + echo "Validating llm service" + validate_service \ + "${ip_address}:9009/v1/chat/completions" \ + "content" \ + "vllm-llm" \ + "vllm-service" \ + '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 17}' +} + +function validate_megaservice() { + # Curl the Mega Service + validate_service \ + "${ip_address}:8888/v1/chatqna" \ + "data: " \ + "chatqna-megaservice" \ + "chatqna-xeon-backend-server" \ + '{"messages": "What is the revenue of Nike in 2023?"}' + +} + +function validate_frontend() { + echo "[ TEST INFO ]: --------- frontend test started ---------" + cd $WORKPATH/ui/svelte + local conda_env_name="OPEA_e2e" + export PATH=${HOME}/miniforge3/bin/:$PATH + if conda info --envs | grep -q "$conda_env_name"; then + echo "$conda_env_name exist!" + else + conda create -n ${conda_env_name} python=3.12 -y + fi + source activate ${conda_env_name} + echo "[ TEST INFO ]: --------- conda env activated ---------" + + sed -i "s/localhost/$ip_address/g" playwright.config.ts + + conda install -c conda-forge nodejs=22.6.0 -y + npm install && npm ci && npx playwright install --with-deps + node -v && npm -v && pip list + + exit_status=0 + npx playwright test || exit_status=$? + + if [ $exit_status -ne 0 ]; then + echo "[TEST INFO]: ---------frontend test failed---------" + exit $exit_status + else + echo "[TEST INFO]: ---------frontend test passed---------" + fi +} + +function stop_docker() { + echo "In stop docker" + echo $WORKPATH + cd $WORKPATH/docker_compose/intel/cpu/xeon/ + docker compose -f compose_milvus.yaml down +} + +function main() { + + stop_docker + + if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi + + start_time=$(date +%s) + start_services + end_time=$(date +%s) + duration=$((end_time-start_time)) + echo "Mega service start duration is $duration s" && sleep 1s + + validate_microservices + echo "==== microservices validated ====" + validate_megaservice + echo "==== megaservice validated ====" + + stop_docker + echo y | docker system prune + +} + +main From 3460a380b62485303d2f4e46e44127d7305553f4 Mon Sep 17 00:00:00 2001 From: "chen, suyue" Date: Mon, 3 Mar 2025 08:45:10 +0800 Subject: [PATCH 031/226] Fix cd workflow condition (#1588) Fix cd workflow condition Signed-off-by: chensuyue Co-authored-by: ZePan110 Signed-off-by: Chingis Yundunov --- .github/workflows/manual-example-workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual-example-workflow.yml b/.github/workflows/manual-example-workflow.yml index 9e119dcf7a..6fde18ddce 100644 --- a/.github/workflows/manual-example-workflow.yml +++ b/.github/workflows/manual-example-workflow.yml @@ -76,7 +76,7 @@ jobs: build-deploy-gmc: needs: [get-test-matrix] - if: ${{ fromJSON(inputs.deploy_gmc) }} && ${{ fromJSON(needs.get-test-matrix.outputs.nodes).length != 0 }} + if: ${{ fromJSON(inputs.deploy_gmc) }} strategy: matrix: node: ${{ fromJson(needs.get-test-matrix.outputs.nodes) }} @@ -90,7 +90,7 @@ jobs: run-examples: needs: [get-test-matrix, build-deploy-gmc] - if: always() && ${{ fromJSON(needs.get-test-matrix.outputs.examples).length != 0 }} + if: always() strategy: matrix: example: ${{ fromJson(needs.get-test-matrix.outputs.examples) }} From fc75a8c7a18062706dec7e907bdb22fa461fdffa Mon Sep 17 00:00:00 2001 From: Ying Hu Date: Mon, 3 Mar 2025 16:17:19 +0800 Subject: [PATCH 032/226] Update DBQnA tgi docker image to latest tgi 2.4.0 (#1593) Signed-off-by: Chingis Yundunov --- DBQnA/docker_compose/intel/cpu/xeon/README.md | 37 ++++++++++++------- .../intel/cpu/xeon/compose.yaml | 4 +- .../docker_compose/intel/cpu/xeon/set_env.sh | 27 ++++++++++++++ DBQnA/tests/test_compose_on_xeon.sh | 3 ++ 4 files changed, 55 insertions(+), 16 deletions(-) create mode 100644 DBQnA/docker_compose/intel/cpu/xeon/set_env.sh diff --git a/DBQnA/docker_compose/intel/cpu/xeon/README.md b/DBQnA/docker_compose/intel/cpu/xeon/README.md index 78d5b60419..26b46ec4b9 100644 --- a/DBQnA/docker_compose/intel/cpu/xeon/README.md +++ b/DBQnA/docker_compose/intel/cpu/xeon/README.md @@ -38,25 +38,29 @@ We set default model as "mistralai/Mistral-7B-Instruct-v0.3", change "LLM_MODEL_ If use gated models, you also need to provide [huggingface token](https://huggingface.co/docs/hub/security-tokens) to "HUGGINGFACEHUB_API_TOKEN" environment variable. +```bash +export HUGGINGFACEHUB_API_TOKEN="xxx" +``` + ### 2.1 Setup Environment Variables Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below. ```bash -# your_ip should be your external IP address, do not use localhost. -export your_ip=$(hostname -I | awk '{print $1}') +# host_ip should be your external IP address, do not use localhost. +export host_ip=$(hostname -I | awk '{print $1}') # Example: no_proxy="localhost,127.0.0.1,192.168.1.1" -export no_proxy=${your_no_proxy},${your_ip} +export no_proxy=${no_proxy},${host_ip} # If you are in a proxy environment, also set the proxy-related environment variables: -export http_proxy=${your_http_proxy} -export https_proxy=${your_http_proxy} +export http_proxy=${http_proxy} +export https_proxy=${https_proxy} # Set other required variables export TGI_PORT=8008 -export TGI_LLM_ENDPOINT=http://${your_ip}:${TGI_PORT} +export TGI_LLM_ENDPOINT=http://${host_ip}:${TGI_PORT} export HF_TOKEN=${HUGGINGFACEHUB_API_TOKEN} export LLM_MODEL_ID="mistralai/Mistral-7B-Instruct-v0.3" export POSTGRES_USER=postgres @@ -65,7 +69,14 @@ export POSTGRES_DB=chinook export text2sql_port=9090 ``` -Note: Please replace with `your_ip` with your external IP address, do not use localhost. +or +edit the file set_env.sh to set those environment variables, + +```bash +source set_env.sh +``` + +Note: Please replace with `host_ip` with your external IP address, do not use localhost. ### 2.2 Start Microservice Docker Containers @@ -120,7 +131,7 @@ docker run -d --name="test-dbqna-react-ui-server" --ipc=host -p 5174:80 -e no_pr ```bash -curl http://${your_ip}:$TGI_PORT/generate \ +curl http://${host_ip}:$TGI_PORT/generate \ -X POST \ -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \ -H 'Content-Type: application/json' @@ -133,17 +144,17 @@ Once Text-to-SQL microservice is started, user can use below command #### 3.2.1 Test the Database connection ```bash -curl --location http://${your_ip}:9090/v1/postgres/health \ +curl --location http://${host_ip}:9090/v1/postgres/health \ --header 'Content-Type: application/json' \ - --data '{"user": "'${POSTGRES_USER}'","password": "'${POSTGRES_PASSWORD}'","host": "'${your_ip}'", "port": "5442", "database": "'${POSTGRES_DB}'"}' + --data '{"user": "'${POSTGRES_USER}'","password": "'${POSTGRES_PASSWORD}'","host": "'${host_ip}'", "port": "5442", "database": "'${POSTGRES_DB}'"}' ``` #### 3.2.2 Invoke the microservice. ```bash -curl http://${your_ip}:9090/v1/text2sql\ +curl http://${host_ip}:9090/v1/text2sql\ -X POST \ - -d '{"input_text": "Find the total number of Albums.","conn_str": {"user": "'${POSTGRES_USER}'","password": "'${POSTGRES_PASSWORD}'","host": "'${your_ip}'", "port": "5442", "database": "'${POSTGRES_DB}'"}}' \ + -d '{"input_text": "Find the total number of Albums.","conn_str": {"user": "'${POSTGRES_USER}'","password": "'${POSTGRES_PASSWORD}'","host": "'${host_ip}'", "port": "5442", "database": "'${POSTGRES_DB}'"}}' \ -H 'Content-Type: application/json' ``` @@ -161,7 +172,7 @@ npm run test ## 🚀 Launch the React UI -Open this URL `http://{your_ip}:5174` in your browser to access the frontend. +Open this URL `http://{host_ip}:5174` in your browser to access the frontend. ![project-screenshot](../../../../assets/img/dbQnA_ui_init.png) diff --git a/DBQnA/docker_compose/intel/cpu/xeon/compose.yaml b/DBQnA/docker_compose/intel/cpu/xeon/compose.yaml index 9b2bcbfbaa..6654bc535d 100644 --- a/DBQnA/docker_compose/intel/cpu/xeon/compose.yaml +++ b/DBQnA/docker_compose/intel/cpu/xeon/compose.yaml @@ -1,11 +1,9 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -version: "3.8" - services: tgi-service: - image: ghcr.io/huggingface/text-generation-inference:2.1.0 + image: ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu container_name: tgi-service ports: - "8008:80" diff --git a/DBQnA/docker_compose/intel/cpu/xeon/set_env.sh b/DBQnA/docker_compose/intel/cpu/xeon/set_env.sh new file mode 100644 index 0000000000..beae6d5bc9 --- /dev/null +++ b/DBQnA/docker_compose/intel/cpu/xeon/set_env.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +pushd "../../../../../" > /dev/null +source .set_env.sh +popd > /dev/null + +#export host_ip=$(hostname -I | awk '{print $1}') + +if [ -z "${HUGGINGFACEHUB_API_TOKEN}" ]; then + echo "Error: HUGGINGFACEHUB_API_TOKEN is not set. Please set HUGGINGFACEHUB_API_TOKEN." +fi + +if [ -z "${host_ip}" ]; then + echo "Error: host_ip is not set. Please set host_ip first." +fi +export no_proxy=$no_proxy,$host_ip,dbqna-xeon-react-ui-server,text2sql-service,tgi-service,postgres-container +export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} +export TGI_PORT=8008 +export TGI_LLM_ENDPOINT="http://${host_ip}:${TGI_PORT}" +export LLM_MODEL_ID="mistralai/Mistral-7B-Instruct-v0.3" +export POSTGRES_USER=postgres +export POSTGRES_PASSWORD=testpwd +export POSTGRES_DB=chinook +export TEXT2SQL_PORT=9090 +"set_env.sh" 27L, 974B diff --git a/DBQnA/tests/test_compose_on_xeon.sh b/DBQnA/tests/test_compose_on_xeon.sh index e9a50cf0e7..8775fc79dc 100755 --- a/DBQnA/tests/test_compose_on_xeon.sh +++ b/DBQnA/tests/test_compose_on_xeon.sh @@ -22,6 +22,9 @@ function build_docker_images() { echo "Build all the images with --no-cache, check docker_image_build.log for details..." docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log + + docker pull ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu + docker images && sleep 1s } function start_service() { From 2b701cad4f0e96904bc4dc57a807109a9a7b197b Mon Sep 17 00:00:00 2001 From: Spycsh <39623753+Spycsh@users.noreply.github.com> Date: Mon, 3 Mar 2025 23:03:44 +0800 Subject: [PATCH 033/226] Revert chatqna async and enhance tests (#1598) align with opea-project/GenAIComps#1354 Signed-off-by: Chingis Yundunov --- ChatQnA/chatqna.py | 4 ++-- ChatQnA/tests/test_compose_on_gaudi.sh | 2 +- ChatQnA/tests/test_compose_on_rocm.sh | 6 +++--- ChatQnA/tests/test_compose_on_xeon.sh | 3 ++- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ChatQnA/chatqna.py b/ChatQnA/chatqna.py index e25ab4d39a..afb9706cb2 100644 --- a/ChatQnA/chatqna.py +++ b/ChatQnA/chatqna.py @@ -166,10 +166,10 @@ def align_outputs(self, data, cur_node, inputs, runtime_graph, llm_parameters_di return next_data -async def align_generator(self, gen, **kwargs): +def align_generator(self, gen, **kwargs): # OpenAI response format # b'data:{"id":"","object":"text_completion","created":1725530204,"model":"meta-llama/Meta-Llama-3-8B-Instruct","system_fingerprint":"2.0.1-native","choices":[{"index":0,"delta":{"role":"assistant","content":"?"},"logprobs":null,"finish_reason":null}]}\n\n' - async for line in gen: + for line in gen: line = line.decode("utf-8") start = line.find("{") end = line.rfind("}") + 1 diff --git a/ChatQnA/tests/test_compose_on_gaudi.sh b/ChatQnA/tests/test_compose_on_gaudi.sh index 2785995bbb..e1a37707e1 100644 --- a/ChatQnA/tests/test_compose_on_gaudi.sh +++ b/ChatQnA/tests/test_compose_on_gaudi.sh @@ -137,7 +137,7 @@ function validate_megaservice() { # Curl the Mega Service validate_service \ "${ip_address}:8888/v1/chatqna" \ - "data:" \ + "Nike" \ "mega-chatqna" \ "chatqna-gaudi-backend-server" \ '{"messages": "What is the revenue of Nike in 2023?"}' diff --git a/ChatQnA/tests/test_compose_on_rocm.sh b/ChatQnA/tests/test_compose_on_rocm.sh index d6dc5dfae1..ebfd9562a4 100644 --- a/ChatQnA/tests/test_compose_on_rocm.sh +++ b/ChatQnA/tests/test_compose_on_rocm.sh @@ -207,7 +207,7 @@ function validate_megaservice() { # Curl the Mega Service validate_service \ "${ip_address}:8888/v1/chatqna" \ - "data: " \ + "Nike" \ "chatqna-megaservice" \ "chatqna-backend-server" \ '{"messages": "What is the revenue of Nike in 2023?"}' @@ -263,8 +263,8 @@ function main() { echo "==== microservices validated ====" validate_megaservice echo "==== megaservice validated ====" - validate_frontend - echo "==== frontend validated ====" + # validate_frontend + # echo "==== frontend validated ====" stop_docker echo y | docker system prune diff --git a/ChatQnA/tests/test_compose_on_xeon.sh b/ChatQnA/tests/test_compose_on_xeon.sh index 69df81a0e8..a61fbf11bb 100644 --- a/ChatQnA/tests/test_compose_on_xeon.sh +++ b/ChatQnA/tests/test_compose_on_xeon.sh @@ -101,6 +101,7 @@ function validate_service() { function validate_microservices() { # Check if the microservices are running correctly. + sleep 10m # tei for embedding service validate_service \ @@ -142,7 +143,7 @@ function validate_megaservice() { # Curl the Mega Service validate_service \ "${ip_address}:8888/v1/chatqna" \ - "data" \ + "Nike" \ "mega-chatqna" \ "chatqna-xeon-backend-server" \ '{"messages": "What is the revenue of Nike in 2023?"}' From 12845f1f751e6b1b82daee90e5097d870b8a03c5 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Tue, 4 Mar 2025 09:48:27 +0800 Subject: [PATCH 034/226] Use model cache for docker compose test (#1582) Signed-off-by: ZePan110 Signed-off-by: Chingis Yundunov --- .github/workflows/_example-workflow.yml | 9 +++-- .github/workflows/_run-docker-compose.yml | 6 ++++ .github/workflows/manual-example-workflow.yml | 35 +++++++++++-------- .github/workflows/pr-docker-compose-e2e.yml | 1 + .../intel/cpu/xeon/compose.yaml | 2 +- .../intel/cpu/xeon/compose_multilang.yaml | 2 +- .../intel/hpu/gaudi/compose.yaml | 2 +- AudioQnA/tests/test_compose_on_gaudi.sh | 1 + AudioQnA/tests/test_compose_on_xeon.sh | 1 + .../intel/cpu/xeon/compose.yaml | 2 +- DBQnA/tests/test_compose_on_xeon.sh | 1 + .../intel/cpu/xeon/compose.yaml | 4 +-- .../cpu/xeon/compose_without_rerank.yaml | 2 +- .../intel/hpu/gaudi/compose.yaml | 4 +-- .../tests/test_compose_on_gaudi.sh | 1 + .../tests/test_compose_on_xeon.sh | 1 + .../test_compose_without_rerank_on_xeon.sh | 1 + .../intel/cpu/xeon/compose.yaml | 2 +- FaqGen/tests/test_compose_on_gaudi.sh | 2 +- FaqGen/tests/test_compose_on_xeon.sh | 1 + .../intel/cpu/xeon/compose.yaml | 2 +- .../intel/hpu/gaudi/compose.yaml | 2 +- Translation/tests/test_compose_on_gaudi.sh | 1 + Translation/tests/test_compose_on_xeon.sh | 1 + .../intel/cpu/xeon/compose.yaml | 2 +- .../intel/hpu/gaudi/compose.yaml | 2 +- VisualQnA/tests/test_compose_on_gaudi.sh | 1 + VisualQnA/tests/test_compose_on_xeon.sh | 1 + 28 files changed, 61 insertions(+), 31 deletions(-) diff --git a/.github/workflows/_example-workflow.yml b/.github/workflows/_example-workflow.yml index d79c4132e1..010eece64a 100644 --- a/.github/workflows/_example-workflow.yml +++ b/.github/workflows/_example-workflow.yml @@ -43,7 +43,11 @@ on: inject_commit: default: false required: false - type: string + type: boolean + use_model_cache: + default: false + required: false + type: boolean jobs: #################################################################################################### @@ -110,6 +114,7 @@ jobs: tag: ${{ inputs.tag }} example: ${{ inputs.example }} hardware: ${{ inputs.node }} + use_model_cache: ${{ inputs.use_model_cache }} secrets: inherit @@ -131,7 +136,7 @@ jobs: #################################################################################################### test-gmc-pipeline: needs: [build-images] - if: ${{ fromJSON(inputs.test_gmc) }} + if: false # ${{ fromJSON(inputs.test_gmc) }} uses: ./.github/workflows/_gmc-e2e.yml with: example: ${{ inputs.example }} diff --git a/.github/workflows/_run-docker-compose.yml b/.github/workflows/_run-docker-compose.yml index 54ec72eea3..3d02b7b4ae 100644 --- a/.github/workflows/_run-docker-compose.yml +++ b/.github/workflows/_run-docker-compose.yml @@ -28,6 +28,10 @@ on: required: false type: string default: "" + use_model_cache: + required: false + type: boolean + default: false jobs: get-test-case: runs-on: ubuntu-latest @@ -144,9 +148,11 @@ jobs: example: ${{ inputs.example }} hardware: ${{ inputs.hardware }} test_case: ${{ matrix.test_case }} + use_model_cache: ${{ inputs.use_model_cache }} run: | cd ${{ github.workspace }}/$example/tests if [[ "$IMAGE_REPO" == "" ]]; then export IMAGE_REPO="${OPEA_IMAGE_REPO}opea"; fi + if [[ "$use_model_cache" == "true" ]]; then export model_cache="/data2/hf_model"; fi if [ -f ${test_case} ]; then timeout 30m bash ${test_case}; else echo "Test script {${test_case}} not found, skip test!"; fi - name: Clean up container after test diff --git a/.github/workflows/manual-example-workflow.yml b/.github/workflows/manual-example-workflow.yml index 6fde18ddce..3a98b3d40e 100644 --- a/.github/workflows/manual-example-workflow.yml +++ b/.github/workflows/manual-example-workflow.yml @@ -20,11 +20,11 @@ on: description: "Tag to apply to images" required: true type: string - deploy_gmc: - default: false - description: 'Whether to deploy gmc' - required: true - type: boolean + # deploy_gmc: + # default: false + # description: 'Whether to deploy gmc' + # required: true + # type: boolean build: default: true description: 'Build test required images for Examples' @@ -40,11 +40,11 @@ on: description: 'Test examples with helm charts' required: false type: boolean - test_gmc: - default: false - description: 'Test examples with gmc' - required: false - type: boolean + # test_gmc: + # default: false + # description: 'Test examples with gmc' + # required: false + # type: boolean opea_branch: default: "main" description: 'OPEA branch for image build' @@ -54,7 +54,12 @@ on: default: false description: "inject commit to docker images true or false" required: false - type: string + type: boolean + use_model_cache: + default: false + description: "use model cache true or false" + required: false + type: boolean permissions: read-all jobs: @@ -76,7 +81,8 @@ jobs: build-deploy-gmc: needs: [get-test-matrix] - if: ${{ fromJSON(inputs.deploy_gmc) }} + if: false + #${{ fromJSON(inputs.deploy_gmc) }} strategy: matrix: node: ${{ fromJson(needs.get-test-matrix.outputs.nodes) }} @@ -89,7 +95,7 @@ jobs: secrets: inherit run-examples: - needs: [get-test-matrix, build-deploy-gmc] + needs: [get-test-matrix] #[get-test-matrix, build-deploy-gmc] if: always() strategy: matrix: @@ -104,7 +110,8 @@ jobs: build: ${{ fromJSON(inputs.build) }} test_compose: ${{ fromJSON(inputs.test_compose) }} test_helmchart: ${{ fromJSON(inputs.test_helmchart) }} - test_gmc: ${{ fromJSON(inputs.test_gmc) }} + # test_gmc: ${{ fromJSON(inputs.test_gmc) }} opea_branch: ${{ inputs.opea_branch }} inject_commit: ${{ inputs.inject_commit }} + use_model_cache: ${{ inputs.use_model_cache }} secrets: inherit diff --git a/.github/workflows/pr-docker-compose-e2e.yml b/.github/workflows/pr-docker-compose-e2e.yml index c924f0e26a..a7604f29af 100644 --- a/.github/workflows/pr-docker-compose-e2e.yml +++ b/.github/workflows/pr-docker-compose-e2e.yml @@ -42,5 +42,6 @@ jobs: tag: "ci" example: ${{ matrix.example }} hardware: ${{ matrix.hardware }} + use_model_cache: true diff_excluded_files: '\.github|\.md|\.txt|kubernetes|gmc|assets|benchmark' secrets: inherit diff --git a/AudioQnA/docker_compose/intel/cpu/xeon/compose.yaml b/AudioQnA/docker_compose/intel/cpu/xeon/compose.yaml index 78a17dda04..48756c00b6 100644 --- a/AudioQnA/docker_compose/intel/cpu/xeon/compose.yaml +++ b/AudioQnA/docker_compose/intel/cpu/xeon/compose.yaml @@ -30,7 +30,7 @@ services: ports: - "3006:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/AudioQnA/docker_compose/intel/cpu/xeon/compose_multilang.yaml b/AudioQnA/docker_compose/intel/cpu/xeon/compose_multilang.yaml index d83e1002c0..c6ad650943 100644 --- a/AudioQnA/docker_compose/intel/cpu/xeon/compose_multilang.yaml +++ b/AudioQnA/docker_compose/intel/cpu/xeon/compose_multilang.yaml @@ -31,7 +31,7 @@ services: ports: - "3006:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/AudioQnA/docker_compose/intel/hpu/gaudi/compose.yaml b/AudioQnA/docker_compose/intel/hpu/gaudi/compose.yaml index 2624dbf531..45691f478b 100644 --- a/AudioQnA/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/AudioQnA/docker_compose/intel/hpu/gaudi/compose.yaml @@ -40,7 +40,7 @@ services: ports: - "3006:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} diff --git a/AudioQnA/tests/test_compose_on_gaudi.sh b/AudioQnA/tests/test_compose_on_gaudi.sh index 2eb0bf3408..fe5cff379a 100644 --- a/AudioQnA/tests/test_compose_on_gaudi.sh +++ b/AudioQnA/tests/test_compose_on_gaudi.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/AudioQnA/tests/test_compose_on_xeon.sh b/AudioQnA/tests/test_compose_on_xeon.sh index 48047948cc..11a86ba5c8 100644 --- a/AudioQnA/tests/test_compose_on_xeon.sh +++ b/AudioQnA/tests/test_compose_on_xeon.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/DBQnA/docker_compose/intel/cpu/xeon/compose.yaml b/DBQnA/docker_compose/intel/cpu/xeon/compose.yaml index 6654bc535d..8e4c15bd6b 100644 --- a/DBQnA/docker_compose/intel/cpu/xeon/compose.yaml +++ b/DBQnA/docker_compose/intel/cpu/xeon/compose.yaml @@ -8,7 +8,7 @@ services: ports: - "8008:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} diff --git a/DBQnA/tests/test_compose_on_xeon.sh b/DBQnA/tests/test_compose_on_xeon.sh index 8775fc79dc..da9fa1b71a 100755 --- a/DBQnA/tests/test_compose_on_xeon.sh +++ b/DBQnA/tests/test_compose_on_xeon.sh @@ -10,6 +10,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml b/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml index 9624df7300..6ecebfdc23 100644 --- a/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml +++ b/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml @@ -38,7 +38,7 @@ services: ports: - "6006:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -96,7 +96,7 @@ services: ports: - "8808:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose_without_rerank.yaml b/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose_without_rerank.yaml index 68afbf18e7..edc563cdbe 100644 --- a/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose_without_rerank.yaml +++ b/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose_without_rerank.yaml @@ -36,7 +36,7 @@ services: ports: - "6006:80" volumes: - - "/mnt/models:/data" + - "${MODEL_CACHE:-/mnt/models}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/DocIndexRetriever/docker_compose/intel/hpu/gaudi/compose.yaml b/DocIndexRetriever/docker_compose/intel/hpu/gaudi/compose.yaml index eedbe66719..f47d01a7cf 100644 --- a/DocIndexRetriever/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/DocIndexRetriever/docker_compose/intel/hpu/gaudi/compose.yaml @@ -34,7 +34,7 @@ services: ports: - "8090:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" runtime: habana cap_add: - SYS_NICE @@ -95,7 +95,7 @@ services: ports: - "8808:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/DocIndexRetriever/tests/test_compose_on_gaudi.sh b/DocIndexRetriever/tests/test_compose_on_gaudi.sh index 2176caf638..d6dd8a7138 100644 --- a/DocIndexRetriever/tests/test_compose_on_gaudi.sh +++ b/DocIndexRetriever/tests/test_compose_on_gaudi.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/DocIndexRetriever/tests/test_compose_on_xeon.sh b/DocIndexRetriever/tests/test_compose_on_xeon.sh index 1e490a517d..0027ebe9a3 100644 --- a/DocIndexRetriever/tests/test_compose_on_xeon.sh +++ b/DocIndexRetriever/tests/test_compose_on_xeon.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/DocIndexRetriever/tests/test_compose_without_rerank_on_xeon.sh b/DocIndexRetriever/tests/test_compose_without_rerank_on_xeon.sh index ddd62ebd8a..16aed41242 100644 --- a/DocIndexRetriever/tests/test_compose_without_rerank_on_xeon.sh +++ b/DocIndexRetriever/tests/test_compose_without_rerank_on_xeon.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/FaqGen/docker_compose/intel/cpu/xeon/compose.yaml b/FaqGen/docker_compose/intel/cpu/xeon/compose.yaml index a20c784786..ea24486cda 100644 --- a/FaqGen/docker_compose/intel/cpu/xeon/compose.yaml +++ b/FaqGen/docker_compose/intel/cpu/xeon/compose.yaml @@ -8,7 +8,7 @@ services: ports: - ${LLM_ENDPOINT_PORT:-8008}:80 volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/FaqGen/tests/test_compose_on_gaudi.sh b/FaqGen/tests/test_compose_on_gaudi.sh index 8726c0f027..eeba304279 100644 --- a/FaqGen/tests/test_compose_on_gaudi.sh +++ b/FaqGen/tests/test_compose_on_gaudi.sh @@ -13,7 +13,7 @@ export TAG=${IMAGE_TAG} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" ip_address=$(hostname -I | awk '{print $1}') -export DATA_PATH="/data/cache" +export DATA_PATH=${model_cache:-"/data/cache"} function build_docker_images() { opea_branch=${opea_branch:-"main"} diff --git a/FaqGen/tests/test_compose_on_xeon.sh b/FaqGen/tests/test_compose_on_xeon.sh index 9d494234a0..cc527b7e9d 100755 --- a/FaqGen/tests/test_compose_on_xeon.sh +++ b/FaqGen/tests/test_compose_on_xeon.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/Translation/docker_compose/intel/cpu/xeon/compose.yaml b/Translation/docker_compose/intel/cpu/xeon/compose.yaml index d876f99f2a..d1a6ee337d 100644 --- a/Translation/docker_compose/intel/cpu/xeon/compose.yaml +++ b/Translation/docker_compose/intel/cpu/xeon/compose.yaml @@ -21,7 +21,7 @@ services: timeout: 10s retries: 100 volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g command: --model-id ${LLM_MODEL_ID} --cuda-graphs 0 llm: diff --git a/Translation/docker_compose/intel/hpu/gaudi/compose.yaml b/Translation/docker_compose/intel/hpu/gaudi/compose.yaml index be983b7b13..7e49db9c39 100644 --- a/Translation/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/Translation/docker_compose/intel/hpu/gaudi/compose.yaml @@ -30,7 +30,7 @@ services: - SYS_NICE ipc: host volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" command: --model-id ${LLM_MODEL_ID} --max-input-length 1024 --max-total-tokens 2048 llm: image: ${REGISTRY:-opea}/llm-textgen:${TAG:-latest} diff --git a/Translation/tests/test_compose_on_gaudi.sh b/Translation/tests/test_compose_on_gaudi.sh index a4a201a762..63167b6e74 100644 --- a/Translation/tests/test_compose_on_gaudi.sh +++ b/Translation/tests/test_compose_on_gaudi.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/Translation/tests/test_compose_on_xeon.sh b/Translation/tests/test_compose_on_xeon.sh index ed085b842a..9e2ac58cb7 100644 --- a/Translation/tests/test_compose_on_xeon.sh +++ b/Translation/tests/test_compose_on_xeon.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/VisualQnA/docker_compose/intel/cpu/xeon/compose.yaml b/VisualQnA/docker_compose/intel/cpu/xeon/compose.yaml index 89525bd65d..4a81704be4 100644 --- a/VisualQnA/docker_compose/intel/cpu/xeon/compose.yaml +++ b/VisualQnA/docker_compose/intel/cpu/xeon/compose.yaml @@ -8,7 +8,7 @@ services: ports: - "8399:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/VisualQnA/docker_compose/intel/hpu/gaudi/compose.yaml b/VisualQnA/docker_compose/intel/hpu/gaudi/compose.yaml index fa17cf36d1..73e2747085 100644 --- a/VisualQnA/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/VisualQnA/docker_compose/intel/hpu/gaudi/compose.yaml @@ -8,7 +8,7 @@ services: ports: - "8399:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} diff --git a/VisualQnA/tests/test_compose_on_gaudi.sh b/VisualQnA/tests/test_compose_on_gaudi.sh index 19dc07fcfb..3515be94e4 100644 --- a/VisualQnA/tests/test_compose_on_gaudi.sh +++ b/VisualQnA/tests/test_compose_on_gaudi.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/VisualQnA/tests/test_compose_on_xeon.sh b/VisualQnA/tests/test_compose_on_xeon.sh index 9ab2e281f5..4e345b3f91 100644 --- a/VisualQnA/tests/test_compose_on_xeon.sh +++ b/VisualQnA/tests/test_compose_on_xeon.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" From aef57f6664461fed510a4aaf5333a172b1d83d97 Mon Sep 17 00:00:00 2001 From: "chen, suyue" Date: Tue, 4 Mar 2025 10:41:22 +0800 Subject: [PATCH 035/226] open chatqna frontend test (#1594) Signed-off-by: chensuyue Signed-off-by: Chingis Yundunov --- ChatQnA/tests/test_compose_on_gaudi.sh | 2 +- ChatQnA/tests/test_compose_on_rocm.sh | 4 ++-- ChatQnA/tests/test_compose_on_xeon.sh | 6 ++---- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/ChatQnA/tests/test_compose_on_gaudi.sh b/ChatQnA/tests/test_compose_on_gaudi.sh index e1a37707e1..d9b40529e8 100644 --- a/ChatQnA/tests/test_compose_on_gaudi.sh +++ b/ChatQnA/tests/test_compose_on_gaudi.sh @@ -189,7 +189,7 @@ function main() { validate_microservices validate_megaservice - # validate_frontend + validate_frontend stop_docker echo y | docker system prune diff --git a/ChatQnA/tests/test_compose_on_rocm.sh b/ChatQnA/tests/test_compose_on_rocm.sh index ebfd9562a4..732e2684aa 100644 --- a/ChatQnA/tests/test_compose_on_rocm.sh +++ b/ChatQnA/tests/test_compose_on_rocm.sh @@ -263,8 +263,8 @@ function main() { echo "==== microservices validated ====" validate_megaservice echo "==== megaservice validated ====" - # validate_frontend - # echo "==== frontend validated ====" + validate_frontend + echo "==== frontend validated ====" stop_docker echo y | docker system prune diff --git a/ChatQnA/tests/test_compose_on_xeon.sh b/ChatQnA/tests/test_compose_on_xeon.sh index a61fbf11bb..e3cd8db1d1 100644 --- a/ChatQnA/tests/test_compose_on_xeon.sh +++ b/ChatQnA/tests/test_compose_on_xeon.sh @@ -101,7 +101,7 @@ function validate_service() { function validate_microservices() { # Check if the microservices are running correctly. - sleep 10m + sleep 3m # tei for embedding service validate_service \ @@ -111,8 +111,6 @@ function validate_microservices() { "tei-embedding-server" \ '{"inputs":"What is Deep Learning?"}' - sleep 1m # retrieval can't curl as expected, try to wait for more time - # retrieval microservice test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") validate_service \ @@ -196,7 +194,7 @@ function main() { validate_microservices validate_megaservice - # validate_frontend + validate_frontend stop_docker echo y | docker system prune From bd0996c936075d44cba88253ec2ceb03815534cd Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Tue, 4 Mar 2025 16:10:20 +0800 Subject: [PATCH 036/226] Enable CodeGen,CodeTrans and DocSum model cache for docker compose test. (#1599) 1.Add cache path check 2.Enable CodeGen,CodeTrans and DocSum model cache for docker compose test. Signed-off-by: ZePan110 Signed-off-by: Chingis Yundunov --- .github/workflows/_run-docker-compose.yml | 9 ++++++++- CodeGen/docker_compose/intel/cpu/xeon/compose.yaml | 2 +- CodeGen/docker_compose/intel/hpu/gaudi/compose.yaml | 2 +- CodeGen/tests/test_compose_on_gaudi.sh | 1 + CodeGen/tests/test_compose_on_xeon.sh | 1 + CodeTrans/docker_compose/intel/cpu/xeon/compose.yaml | 2 +- CodeTrans/docker_compose/intel/hpu/gaudi/compose.yaml | 2 +- CodeTrans/tests/test_compose_on_gaudi.sh | 1 + CodeTrans/tests/test_compose_on_xeon.sh | 1 + DocSum/docker_compose/intel/cpu/xeon/compose.yaml | 2 +- DocSum/tests/test_compose_on_gaudi.sh | 2 +- DocSum/tests/test_compose_on_xeon.sh | 1 + 12 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.github/workflows/_run-docker-compose.yml b/.github/workflows/_run-docker-compose.yml index 3d02b7b4ae..f21c3202f9 100644 --- a/.github/workflows/_run-docker-compose.yml +++ b/.github/workflows/_run-docker-compose.yml @@ -152,7 +152,14 @@ jobs: run: | cd ${{ github.workspace }}/$example/tests if [[ "$IMAGE_REPO" == "" ]]; then export IMAGE_REPO="${OPEA_IMAGE_REPO}opea"; fi - if [[ "$use_model_cache" == "true" ]]; then export model_cache="/data2/hf_model"; fi + if [[ "$use_model_cache" == "true" ]]; then + if [ -d "/data2/hf_model" ]; then + export model_cache="/data2/hf_model" + else + echo "Model cache directory /data2/hf_model does not exist" + export model_cache="~/.cache/huggingface/hub" + fi + fi if [ -f ${test_case} ]; then timeout 30m bash ${test_case}; else echo "Test script {${test_case}} not found, skip test!"; fi - name: Clean up container after test diff --git a/CodeGen/docker_compose/intel/cpu/xeon/compose.yaml b/CodeGen/docker_compose/intel/cpu/xeon/compose.yaml index 28940c9ba4..f9e7e26280 100644 --- a/CodeGen/docker_compose/intel/cpu/xeon/compose.yaml +++ b/CodeGen/docker_compose/intel/cpu/xeon/compose.yaml @@ -8,7 +8,7 @@ services: ports: - "8028:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/CodeGen/docker_compose/intel/hpu/gaudi/compose.yaml b/CodeGen/docker_compose/intel/hpu/gaudi/compose.yaml index 4d5ed95683..62ec96e626 100644 --- a/CodeGen/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/CodeGen/docker_compose/intel/hpu/gaudi/compose.yaml @@ -8,7 +8,7 @@ services: ports: - "8028:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} diff --git a/CodeGen/tests/test_compose_on_gaudi.sh b/CodeGen/tests/test_compose_on_gaudi.sh index 9ffbc41147..e6e6d1f033 100644 --- a/CodeGen/tests/test_compose_on_gaudi.sh +++ b/CodeGen/tests/test_compose_on_gaudi.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/CodeGen/tests/test_compose_on_xeon.sh b/CodeGen/tests/test_compose_on_xeon.sh index f323e72070..70e5ba9c4f 100644 --- a/CodeGen/tests/test_compose_on_xeon.sh +++ b/CodeGen/tests/test_compose_on_xeon.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/CodeTrans/docker_compose/intel/cpu/xeon/compose.yaml b/CodeTrans/docker_compose/intel/cpu/xeon/compose.yaml index b818956fa5..0ece6dff1d 100644 --- a/CodeTrans/docker_compose/intel/cpu/xeon/compose.yaml +++ b/CodeTrans/docker_compose/intel/cpu/xeon/compose.yaml @@ -8,7 +8,7 @@ services: ports: - "8008:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/CodeTrans/docker_compose/intel/hpu/gaudi/compose.yaml b/CodeTrans/docker_compose/intel/hpu/gaudi/compose.yaml index cbccde0605..3e25dee894 100644 --- a/CodeTrans/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/CodeTrans/docker_compose/intel/hpu/gaudi/compose.yaml @@ -8,7 +8,7 @@ services: ports: - "8008:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} diff --git a/CodeTrans/tests/test_compose_on_gaudi.sh b/CodeTrans/tests/test_compose_on_gaudi.sh index 377937435f..e2aedcd6e9 100644 --- a/CodeTrans/tests/test_compose_on_gaudi.sh +++ b/CodeTrans/tests/test_compose_on_gaudi.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/CodeTrans/tests/test_compose_on_xeon.sh b/CodeTrans/tests/test_compose_on_xeon.sh index 9060eb2833..efa09fe0a5 100644 --- a/CodeTrans/tests/test_compose_on_xeon.sh +++ b/CodeTrans/tests/test_compose_on_xeon.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/DocSum/docker_compose/intel/cpu/xeon/compose.yaml b/DocSum/docker_compose/intel/cpu/xeon/compose.yaml index 2c4344cc23..0d87eaeb2b 100644 --- a/DocSum/docker_compose/intel/cpu/xeon/compose.yaml +++ b/DocSum/docker_compose/intel/cpu/xeon/compose.yaml @@ -21,7 +21,7 @@ services: timeout: 10s retries: 100 volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g command: --model-id ${LLM_MODEL_ID} --cuda-graphs 0 --max-input-length ${MAX_INPUT_TOKENS} --max-total-tokens ${MAX_TOTAL_TOKENS} diff --git a/DocSum/tests/test_compose_on_gaudi.sh b/DocSum/tests/test_compose_on_gaudi.sh index e129608aa1..66dd5b3180 100644 --- a/DocSum/tests/test_compose_on_gaudi.sh +++ b/DocSum/tests/test_compose_on_gaudi.sh @@ -28,7 +28,7 @@ export DOCSUM_PORT=9000 export LLM_ENDPOINT="http://${host_ip}:${LLM_ENDPOINT_PORT}" export DocSum_COMPONENT_NAME="OpeaDocSumTgi" export LOGFLAG=True -export DATA_PATH="/data/cache" +export DATA_PATH=${model_cache:-"/data/cache"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/DocSum/tests/test_compose_on_xeon.sh b/DocSum/tests/test_compose_on_xeon.sh index de208292a5..7dc194ff68 100644 --- a/DocSum/tests/test_compose_on_xeon.sh +++ b/DocSum/tests/test_compose_on_xeon.sh @@ -12,6 +12,7 @@ export host_ip=$(hostname -I | awk '{print $1}') echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" +export MODEL_CACHE=${model_cache:-"./data"} export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} export MAX_INPUT_TOKENS=2048 From fe3132e13a82fb8d09a7c852bb7bf07117ea2f7d Mon Sep 17 00:00:00 2001 From: rbrugaro Date: Tue, 4 Mar 2025 09:44:13 -0800 Subject: [PATCH 037/226] bugfix GraphRAG updated docker compose and env settings to fix issues post refactor (#1567) Signed-off-by: rbrugaro Signed-off-by: Rita Brugarolas Brufau Co-authored-by: chen, suyue Co-authored-by: WenjiaoYue Signed-off-by: Chingis Yundunov --- GraphRAG/README.md | 5 +- .../intel/hpu/gaudi/compose.yaml | 140 +++++++++++------- .../docker_compose/intel/hpu/gaudi/set_env.sh | 23 ++- GraphRAG/tests/test_compose_on_gaudi.sh | 72 +++++---- 4 files changed, 144 insertions(+), 96 deletions(-) diff --git a/GraphRAG/README.md b/GraphRAG/README.md index d654357d44..3c9de58d69 100644 --- a/GraphRAG/README.md +++ b/GraphRAG/README.md @@ -72,7 +72,7 @@ Here is an example of `Nike 2023` pdf. # download pdf file wget https://raw.githubusercontent.com/opea-project/GenAIComps/v1.1/comps/retrievers/redis/data/nke-10k-2023.pdf # upload pdf file with dataprep -curl -X POST "http://${host_ip}:6004/v1/dataprep/ingest" \ +curl -X POST "http://${host_ip}:11103/v1/dataprep/ingest" \ -H "Content-Type: multipart/form-data" \ -F "files=@./nke-10k-2023.pdf" ``` @@ -80,8 +80,7 @@ curl -X POST "http://${host_ip}:6004/v1/dataprep/ingest" \ ```bash curl http://${host_ip}:8888/v1/graphrag \ -H "Content-Type: application/json" \ - -d '{ - "model": "gpt-4o-mini","messages": [{"role": "user","content": "What is the revenue of Nike in 2023? + -d '{"messages": [{"role": "user","content": "where do Nike subsidiaries operate? "}]}' ``` diff --git a/GraphRAG/docker_compose/intel/hpu/gaudi/compose.yaml b/GraphRAG/docker_compose/intel/hpu/gaudi/compose.yaml index 29171a20f2..76f1ab9f63 100644 --- a/GraphRAG/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/GraphRAG/docker_compose/intel/hpu/gaudi/compose.yaml @@ -5,52 +5,65 @@ services: neo4j-apoc: image: neo4j:latest container_name: neo4j-apoc + ports: + - "${NEO4J_PORT1:-7474}:7474" + - "${NEO4J_PORT2:-7687}:7687" volumes: - - /$HOME/neo4j/logs:/logs - - /$HOME/neo4j/config:/config - - /$HOME/neo4j/data:/data - - /$HOME/neo4j/plugins:/plugins + - ./data/neo4j/logs:/logs + - ./data/neo4j/config:/config + - ./data/neo4j/data:/data + - ./data/neo4j/plugins:/plugins ipc: host environment: + - no_proxy=${no_proxy} + - http_proxy=${http_proxy} + - https_proxy=${https_proxy} - NEO4J_AUTH=${NEO4J_USERNAME}/${NEO4J_PASSWORD} - NEO4J_PLUGINS=["apoc"] - NEO4J_apoc_export_file_enabled=true - NEO4J_apoc_import_file_enabled=true - NEO4J_apoc_import_file_use__neo4j__config=true - NEO4J_dbms_security_procedures_unrestricted=apoc.\* - ports: - - "7474:7474" - - "7687:7687" + - NEO4J_server_bolt_advertised__address=localhost:${NEO4J_PORT2} restart: always - tei-embedding-service: + healthcheck: + test: wget http://localhost:7474 || exit 1 + interval: 5s + timeout: 10s + retries: 20 + start_period: 3s + tei-embedding-serving: image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 - container_name: tei-embedding-server + container_name: tei-embedding-serving + entrypoint: /bin/sh -c "apt-get update && apt-get install -y curl && text-embeddings-router --json-output --model-id ${EMBEDDING_MODEL_ID} --auto-truncate" ports: - - "6006:80" + - "${TEI_EMBEDDER_PORT:-12000}:80" volumes: - "./data:/data" shm_size: 1g environment: no_proxy: ${no_proxy} - NO_PROXY: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} - HUGGING_FACE_HUB_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} - ipc: host - command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate - tgi-gaudi-service: - image: ghcr.io/huggingface/tgi-gaudi:2.0.6 + host_ip: ${host_ip} + HF_TOKEN: ${HF_TOKEN} + healthcheck: + test: ["CMD", "curl", "-f", "http://${host_ip}:${TEI_EMBEDDER_PORT}/health"] + interval: 10s + timeout: 6s + retries: 48 + tgi-gaudi-server: + image: ghcr.io/huggingface/tgi-gaudi:2.3.1 container_name: tgi-gaudi-server ports: - - "6005:80" + - ${LLM_ENDPOINT_PORT:-8008}:80 volumes: - - "./data:/data" + - "${DATA_PATH:-./data}:/data" environment: no_proxy: ${no_proxy} - NO_PROXY: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} - HUGGING_FACE_HUB_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + HUGGING_FACE_HUB_TOKEN: ${HF_TOKEN} HF_TOKEN: ${HF_TOKEN} HF_HUB_DISABLE_PROGRESS_BARS: 1 HF_HUB_ENABLE_HF_TRANSFER: 0 @@ -60,33 +73,44 @@ services: LIMIT_HPU_GRAPH: true USE_FLASH_ATTENTION: true FLASH_ATTENTION_RECOMPUTE: true + host_ip: ${host_ip} + LLM_ENDPOINT_PORT: ${LLM_ENDPOINT_PORT} + MAX_INPUT_TOKENS: ${MAX_INPUT_TOKENS:-2048} + MAX_TOTAL_TOKENS: ${MAX_TOTAL_TOKENS:-4096} TEXT_GENERATION_SERVER_IGNORE_EOS_TOKEN: false runtime: habana cap_add: - SYS_NICE ipc: host - command: --model-id ${LLM_MODEL_ID} --max-input-length 6000 --max-total-tokens 8192 + healthcheck: + test: ["CMD-SHELL", "curl -f http://${host_ip}:${LLM_ENDPOINT_PORT}/health || exit 1"] + interval: 10s + timeout: 10s + retries: 100 + command: --model-id ${LLM_MODEL_ID} + dataprep-neo4j-llamaindex: image: ${REGISTRY:-opea}/dataprep:${TAG:-latest} - container_name: dataprep-neo4j-server + container_name: dataprep-neo4j-llamaindex depends_on: - - neo4j-apoc - - tgi-gaudi-service - - tei-embedding-service + neo4j-apoc: + condition: service_healthy + tgi-gaudi-server: + condition: service_healthy + tei-embedding-serving: + condition: service_healthy ports: - - "6004:5000" + - "${DATAPREP_PORT:-11103}:5000" ipc: host environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} host_ip: ${host_ip} - HUGGING_FACE_HUB_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} - HF_TOKEN: ${HF_TOKEN} + DATAPREP_COMPONENT_NAME: "OPEA_DATAPREP_NEO4J_LLAMAINDEX" NEO4J_URL: ${NEO4J_URL} NEO4J_USERNAME: ${NEO4J_USERNAME} NEO4J_PASSWORD: ${NEO4J_PASSWORD} - DATAPREP_COMPONENT_NAME: "OPEA_DATAPREP_NEO4J_LLAMAINDEX" TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT} TEI_EMBEDDING_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT} OPENAI_API_KEY: ${OPENAI_API_KEY} @@ -94,59 +118,61 @@ services: OPENAI_LLM_MODEL: ${OPENAI_LLM_MODEL} EMBEDDING_MODEL_ID: ${EMBEDDING_MODEL_ID} LLM_MODEL_ID: ${LLM_MODEL_ID} - MAX_OUTPUT_TOKENS: ${MAX_OUTPUT_TOKENS} LOGFLAG: ${LOGFLAG} + HUGGINGFACEHUB_API_TOKEN: ${HF_TOKEN} + HF_TOKEN: ${HF_TOKEN} + MAX_INPUT_TOKENS: ${MAX_INPUT_TOKENS:-4096} restart: unless-stopped - retriever-neo4j-llamaindex: + retriever-neo4j: image: ${REGISTRY:-opea}/retriever:${TAG:-latest} - container_name: retriever-neo4j-server - depends_on: - - neo4j-apoc - - tgi-gaudi-service - - tei-embedding-service + container_name: retriever-neo4j ports: - - "7000:7000" + - "${RETRIEVER_PORT:-7000}:7000" ipc: host environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} - host_ip: ${host_ip} - HUGGING_FACE_HUB_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} - HF_TOKEN: ${HF_TOKEN} - NEO4J_URI: ${NEO4J_URL} - NEO4J_USERNAME: ${NEO4J_USERNAME} - NEO4J_PASSWORD: ${NEO4J_PASSWORD} - TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT} + HUGGINGFACEHUB_API_TOKEN: ${HF_TOKEN} + LOGFLAG: ${LOGFLAG:-False} + RETRIEVER_COMPONENT_NAME: ${RETRIEVER_COMPONENT_NAME:-OPEA_RETRIEVER_NEO4J} TEI_EMBEDDING_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT} - OPENAI_API_KEY: ${OPENAI_API_KEY} - OPENAI_EMBEDDING_MODEL: ${OPENAI_EMBEDDING_MODEL} - OPENAI_LLM_MODEL: ${OPENAI_LLM_MODEL} + TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT} EMBEDDING_MODEL_ID: ${EMBEDDING_MODEL_ID} LLM_MODEL_ID: ${LLM_MODEL_ID} - MAX_OUTPUT_TOKENS: ${MAX_OUTPUT_TOKENS} - LOGFLAG: ${LOGFLAG} - RETRIEVER_COMPONENT_NAME: "OPEA_RETRIEVER_NEO4J" - restart: unless-stopped + NEO4J_URI: ${NEO4J_URI} + NEO4J_URL: ${NEO4J_URI} + NEO4J_USERNAME: ${NEO4J_USERNAME} + NEO4J_PASSWORD: ${NEO4J_PASSWORD} + VDMS_USE_CLIP: 0 + host_ip: ${host_ip} + depends_on: + neo4j-apoc: + condition: service_healthy + tei-embedding-serving: + condition: service_healthy + tgi-gaudi-server: + condition: service_healthy graphrag-gaudi-backend-server: image: ${REGISTRY:-opea}/graphrag:${TAG:-latest} container_name: graphrag-gaudi-backend-server depends_on: - neo4j-apoc - - tei-embedding-service - - retriever-neo4j-llamaindex - - tgi-gaudi-service + - tei-embedding-serving + - retriever-neo4j + - tgi-gaudi-server ports: - "8888:8888" + - "${MEGA_SERVICE_PORT:-8888}:8888" environment: - no_proxy=${no_proxy} - https_proxy=${https_proxy} - http_proxy=${http_proxy} - MEGA_SERVICE_HOST_IP=graphrag-gaudi-backend-server - - RETRIEVER_SERVICE_HOST_IP=retriever-neo4j-llamaindex + - RETRIEVER_SERVICE_HOST_IP=retriever-neo4j - RETRIEVER_SERVICE_PORT=7000 - - LLM_SERVER_HOST_IP=tgi-gaudi-service - - LLM_SERVER_PORT=${LLM_SERVER_PORT:-80} + - LLM_SERVER_HOST_IP=tgi-gaudi-server + - LLM_SERVER_PORT=80 - LLM_MODEL_ID=${LLM_MODEL_ID} - LOGFLAG=${LOGFLAG} ipc: host diff --git a/GraphRAG/docker_compose/intel/hpu/gaudi/set_env.sh b/GraphRAG/docker_compose/intel/hpu/gaudi/set_env.sh index 97c462c581..a4fd8049b0 100644 --- a/GraphRAG/docker_compose/intel/hpu/gaudi/set_env.sh +++ b/GraphRAG/docker_compose/intel/hpu/gaudi/set_env.sh @@ -10,16 +10,25 @@ pushd "../../../../../" > /dev/null source .set_env.sh popd > /dev/null +export TEI_EMBEDDER_PORT=11633 +export LLM_ENDPOINT_PORT=11634 export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5" export OPENAI_EMBEDDING_MODEL="text-embedding-3-small" export LLM_MODEL_ID="meta-llama/Meta-Llama-3.1-8B-Instruct" export OPENAI_LLM_MODEL="gpt-4o" -export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:6006" -export TGI_LLM_ENDPOINT="http://${host_ip}:6005" -export NEO4J_URL="bolt://${host_ip}:7687" -export NEO4J_USERNAME=neo4j +export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:${TEI_EMBEDDER_PORT}" +export LLM_MODEL_ID="meta-llama/Meta-Llama-3.1-8B-Instruct" +export TGI_LLM_ENDPOINT="http://${host_ip}:${LLM_ENDPOINT_PORT}" +export NEO4J_PORT1=11631 +export NEO4J_PORT2=11632 +export NEO4J_URI="bolt://${host_ip}:${NEO4J_PORT2}" +export NEO4J_URL="bolt://${host_ip}:${NEO4J_PORT2}" +export NEO4J_USERNAME="neo4j" +export NEO4J_PASSWORD="neo4jtest" export DATAPREP_SERVICE_ENDPOINT="http://${host_ip}:5000/v1/dataprep/ingest" export LOGFLAG=True -export RETRIEVER_SERVICE_PORT=80 -export LLM_SERVER_PORT=80 -export MAX_OUTPUT_TOKENS=1024 +export MAX_INPUT_TOKENS=4096 +export MAX_TOTAL_TOKENS=8192 +export DATA_PATH="/mnt/nvme2n1/hf_cache" +export DATAPREP_PORT=11103 +export RETRIEVER_PORT=11635 diff --git a/GraphRAG/tests/test_compose_on_gaudi.sh b/GraphRAG/tests/test_compose_on_gaudi.sh index 17f03ce61e..bec978ad51 100755 --- a/GraphRAG/tests/test_compose_on_gaudi.sh +++ b/GraphRAG/tests/test_compose_on_gaudi.sh @@ -12,7 +12,7 @@ export TAG=${IMAGE_TAG} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" -ip_address=$(hostname -I | awk '{print $1}') +export host_ip=$(hostname -I | awk '{print $1}') function build_docker_images() { opea_branch=${opea_branch:-"main"} @@ -33,25 +33,38 @@ function build_docker_images() { echo "Build all the images with --no-cache, check docker_image_build.log for details..." docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log - docker pull ghcr.io/huggingface/tgi-gaudi:2.0.6 + docker pull ghcr.io/huggingface/tgi-gaudi:2.3.1 docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 docker images && sleep 1s } function start_services() { cd $WORKPATH/docker_compose/intel/hpu/gaudi - export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5" - export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct" export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} export HF_TOKEN=${HUGGINGFACEHUB_API_TOKEN} + + export TEI_EMBEDDER_PORT=11633 + export LLM_ENDPOINT_PORT=11634 + export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5" + export OPENAI_EMBEDDING_MODEL="text-embedding-3-small" + export LLM_MODEL_ID="meta-llama/Meta-Llama-3.1-8B-Instruct" + export OPENAI_LLM_MODEL="gpt-4o" + export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:${TEI_EMBEDDER_PORT}" + export LLM_MODEL_ID="meta-llama/Meta-Llama-3.1-8B-Instruct" + export TGI_LLM_ENDPOINT="http://${host_ip}:${LLM_ENDPOINT_PORT}" + export NEO4J_PORT1=11631 + export NEO4J_PORT2=11632 + export NEO4J_URI="bolt://${host_ip}:${NEO4J_PORT2}" + export NEO4J_URL="bolt://${host_ip}:${NEO4J_PORT2}" export NEO4J_USERNAME="neo4j" export NEO4J_PASSWORD="neo4jtest" - export NEO4J_URL="bolt://${ip_address}:7687" - export TEI_EMBEDDING_ENDPOINT="http://${ip_address}:6006" - export TGI_LLM_ENDPOINT="http://${ip_address}:6005" - export host_ip=${ip_address} - export LOGFLAG=true - export MAX_OUTPUT_TOKENS="1024" + export DATAPREP_SERVICE_ENDPOINT="http://${host_ip}:5000/v1/dataprep/ingest" + export LOGFLAG=True + export MAX_INPUT_TOKENS=4096 + export MAX_TOTAL_TOKENS=8192 + export DATAPREP_PORT=11103 + export RETRIEVER_PORT=11635 + export MEGA_SERVICE_PORT=8888 unset OPENAI_API_KEY # Start Docker Containers @@ -116,7 +129,7 @@ function validate_microservices() { # validate neo4j-apoc validate_service \ - "${ip_address}:7474" \ + "${host_ip}:${NEO4J_PORT1}" \ "200 OK" \ "neo4j-apoc" \ "neo4j-apoc" \ @@ -124,45 +137,46 @@ function validate_microservices() { # tei for embedding service validate_service \ - "${ip_address}:6006/embed" \ + "${host_ip}:${TEI_EMBEDDER_PORT}/embed" \ "[[" \ "tei-embedding-service" \ - "tei-embedding-server" \ + "tei-embedding-serving" \ '{"inputs":"What is Deep Learning?"}' sleep 1m # retrieval can't curl as expected, try to wait for more time + # tgi for llm service + validate_service \ + "${host_ip}:${LLM_ENDPOINT_PORT}/generate" \ + "generated_text" \ + "tgi-gaudi-service" \ + "tgi-gaudi-server" \ + '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' + # test /v1/dataprep/ingest graph extraction echo "Like many companies in the O&G sector, the stock of Chevron (NYSE:CVX) has declined about 10% over the past 90-days despite the fact that Q2 consensus earnings estimates have risen sharply (~25%) during that same time frame. Over the years, Chevron has kept a very strong balance sheet. FirstEnergy (NYSE:FE – Get Rating) posted its earnings results on Tuesday. The utilities provider reported $0.53 earnings per share for the quarter, topping the consensus estimate of $0.52 by $0.01, RTT News reports. FirstEnergy had a net margin of 10.85% and a return on equity of 17.17%. The Dáil was almost suspended on Thursday afternoon after Sinn Féin TD John Brady walked across the chamber and placed an on-call pager in front of the Minister for Housing Darragh O’Brien during a debate on retained firefighters. Mr O’Brien said Mr Brady had taken part in an act of theatre that was obviously choreographed.Around 2,000 retained firefighters around the country staged a second day of industrial action on Tuesday and are due to start all out-strike action from next Tuesday. The mostly part-time workers, who keep the services going outside of Ireland’s larger urban centres, are taking industrial action in a dispute over pay and working conditions. Speaking in the Dáil, Sinn Féin deputy leader Pearse Doherty said firefighters had marched on Leinster House today and were very angry at the fact the Government will not intervene. Reintroduction of tax relief on mortgages needs to be considered, O’Brien says. Martin withdraws comment after saying People Before Profit would ‘put the jackboot on people’ Taoiseach ‘propagated fears’ farmers forced to rewet land due to nature restoration law – Cairns An intervention is required now. I’m asking you to make an improved offer in relation to pay for retained firefighters, Mr Doherty told the housing minister.I’m also asking you, and challenging you, to go outside after this Order of Business and meet with the firefighters because they are just fed up to the hilt in relation to what you said.Some of them have handed in their pagers to members of the Opposition and have challenged you to wear the pager for the next number of weeks, put up with an €8,600 retainer and not leave your community for the two and a half kilometres and see how you can stand over those type of pay and conditions. At this point, Mr Brady got up from his seat, walked across the chamber and placed the pager on the desk in front of Mr O’Brien. Ceann Comhairle Seán Ó Fearghaíl said the Sinn Féin TD was completely out of order and told him not to carry out a charade in this House, adding it was absolutely outrageous behaviour and not to be encouraged.Mr O’Brien said Mr Brady had engaged in an act of theatre here today which was obviously choreographed and was then interrupted with shouts from the Opposition benches. Mr Ó Fearghaíl said he would suspend the House if this racket continues.Mr O’Brien later said he said he was confident the dispute could be resolved and he had immense regard for firefighters. The minister said he would encourage the unions to re-engage with the State’s industrial relations process while also accusing Sinn Féin of using the issue for their own political gain." > $LOG_PATH/dataprep_file.txt validate_service \ - "http://${ip_address}:6004/v1/dataprep/ingest" \ + "http://${host_ip}:${DATAPREP_PORT}/v1/dataprep/ingest" \ "Data preparation succeeded" \ "extract_graph_neo4j" \ - "dataprep-neo4j-server" + "dataprep-neo4j-llamaindex" sleep 2m # retrieval microservice validate_service \ - "${ip_address}:7000/v1/retrieval" \ - "retrieved_docs" \ + "${host_ip}:${RETRIEVER_PORT}/v1/retrieval" \ + "documents" \ "retriever_community_answers_neo4j" \ - "retriever-neo4j-server" \ - "{\"model\": \"gpt-4o-mini\",\"messages\": [{\"role\": \"user\",\"content\": \"Who is John Brady and has he had any confrontations?\"}]}" + "retriever-neo4j" \ + "{\"messages\": [{\"role\": \"user\",\"content\": \"Who is John Brady and has he had any confrontations?\"}]}" - # tgi for llm service - validate_service \ - "${ip_address}:6005/generate" \ - "generated_text" \ - "tgi-gaudi-service" \ - "tgi-gaudi-server" \ - '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' -} + } function validate_megaservice() { # Curl the Mega Service validate_service \ - "${ip_address}:8888/v1/graphrag" \ + "${host_ip}:${MEGA_SERVICE_PORT}/v1/graphrag" \ "data: " \ "graphrag-megaservice" \ "graphrag-gaudi-backend-server" \ @@ -181,7 +195,7 @@ function validate_frontend() { fi source activate ${conda_env_name} - sed -i "s/localhost/$ip_address/g" playwright.config.ts + sed -i "s/localhost/$host_ip/g" playwright.config.ts conda install -c conda-forge nodejs=22.6.0 -y npm install && npm ci && npx playwright install --with-deps From a9154e8e29ca298a53c2ff04b7e87f52bb7584fb Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 5 Mar 2025 11:30:04 +0800 Subject: [PATCH 038/226] Enable ChatQnA model cache for docker compose test. (#1605) Enable ChatQnA model cache for docker compose test. Signed-off-by: ZePan110 Signed-off-by: Chingis Yundunov --- ChatQnA/docker_compose/amd/gpu/rocm/compose.yaml | 6 +++--- ChatQnA/docker_compose/intel/cpu/xeon/compose.yaml | 6 +++--- .../docker_compose/intel/cpu/xeon/compose_pinecone.yaml | 6 +++--- ChatQnA/docker_compose/intel/cpu/xeon/compose_qdrant.yaml | 6 +++--- ChatQnA/docker_compose/intel/cpu/xeon/compose_tgi.yaml | 6 +++--- .../intel/cpu/xeon/compose_without_rerank.yaml | 4 ++-- ChatQnA/docker_compose/intel/hpu/gaudi/compose.yaml | 6 +++--- .../intel/hpu/gaudi/compose_guardrails.yaml | 8 ++++---- ChatQnA/docker_compose/intel/hpu/gaudi/compose_tgi.yaml | 6 +++--- .../intel/hpu/gaudi/compose_without_rerank.yaml | 4 ++-- ChatQnA/tests/test_compose_guardrails_on_gaudi.sh | 1 + ChatQnA/tests/test_compose_on_gaudi.sh | 1 + ChatQnA/tests/test_compose_on_rocm.sh | 1 + ChatQnA/tests/test_compose_on_xeon.sh | 1 + ChatQnA/tests/test_compose_pinecone_on_xeon.sh | 1 + ChatQnA/tests/test_compose_qdrant_on_xeon.sh | 1 + ChatQnA/tests/test_compose_tgi_on_gaudi.sh | 1 + ChatQnA/tests/test_compose_tgi_on_xeon.sh | 1 + ChatQnA/tests/test_compose_without_rerank_on_gaudi.sh | 1 + ChatQnA/tests/test_compose_without_rerank_on_xeon.sh | 1 + 20 files changed, 39 insertions(+), 29 deletions(-) diff --git a/ChatQnA/docker_compose/amd/gpu/rocm/compose.yaml b/ChatQnA/docker_compose/amd/gpu/rocm/compose.yaml index 11c0b78cae..193f4346e7 100644 --- a/ChatQnA/docker_compose/amd/gpu/rocm/compose.yaml +++ b/ChatQnA/docker_compose/amd/gpu/rocm/compose.yaml @@ -30,7 +30,7 @@ services: ports: - "${CHATQNA_TEI_EMBEDDING_PORT}:80" volumes: - - "/var/opea/chatqna-service/data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g ipc: host environment: @@ -72,7 +72,7 @@ services: ports: - "${CHATQNA_TEI_RERANKING_PORT}:80" volumes: - - "/var/opea/chatqna-service/data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -104,7 +104,7 @@ services: HF_HUB_DISABLE_PROGRESS_BARS: 1 HF_HUB_ENABLE_HF_TRANSFER: 0 volumes: - - "/var/opea/chatqna-service/data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g devices: - /dev/kfd:/dev/kfd diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/compose.yaml b/ChatQnA/docker_compose/intel/cpu/xeon/compose.yaml index 3c3e6f49a7..00c6a2aec2 100644 --- a/ChatQnA/docker_compose/intel/cpu/xeon/compose.yaml +++ b/ChatQnA/docker_compose/intel/cpu/xeon/compose.yaml @@ -31,7 +31,7 @@ services: ports: - "6006:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -64,7 +64,7 @@ services: ports: - "8808:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -80,7 +80,7 @@ services: ports: - "9009:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 128g environment: no_proxy: ${no_proxy} diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/compose_pinecone.yaml b/ChatQnA/docker_compose/intel/cpu/xeon/compose_pinecone.yaml index de784dfabd..a2d2318945 100644 --- a/ChatQnA/docker_compose/intel/cpu/xeon/compose_pinecone.yaml +++ b/ChatQnA/docker_compose/intel/cpu/xeon/compose_pinecone.yaml @@ -28,7 +28,7 @@ services: ports: - "6006:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -59,7 +59,7 @@ services: ports: - "8808:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -75,7 +75,7 @@ services: ports: - "9009:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 128g environment: no_proxy: ${no_proxy} diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/compose_qdrant.yaml b/ChatQnA/docker_compose/intel/cpu/xeon/compose_qdrant.yaml index 46123d3e90..8a7fabdfad 100644 --- a/ChatQnA/docker_compose/intel/cpu/xeon/compose_qdrant.yaml +++ b/ChatQnA/docker_compose/intel/cpu/xeon/compose_qdrant.yaml @@ -32,7 +32,7 @@ services: ports: - "6040:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -64,7 +64,7 @@ services: ports: - "6041:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -80,7 +80,7 @@ services: ports: - "6042:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 128g environment: no_proxy: ${no_proxy} diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/compose_tgi.yaml b/ChatQnA/docker_compose/intel/cpu/xeon/compose_tgi.yaml index 5831181370..4a7c4f4627 100644 --- a/ChatQnA/docker_compose/intel/cpu/xeon/compose_tgi.yaml +++ b/ChatQnA/docker_compose/intel/cpu/xeon/compose_tgi.yaml @@ -31,7 +31,7 @@ services: ports: - "6006:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -64,7 +64,7 @@ services: ports: - "8808:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -80,7 +80,7 @@ services: ports: - "9009:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/compose_without_rerank.yaml b/ChatQnA/docker_compose/intel/cpu/xeon/compose_without_rerank.yaml index 917c6ee078..72fbdead0a 100644 --- a/ChatQnA/docker_compose/intel/cpu/xeon/compose_without_rerank.yaml +++ b/ChatQnA/docker_compose/intel/cpu/xeon/compose_without_rerank.yaml @@ -31,7 +31,7 @@ services: ports: - "6006:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -64,7 +64,7 @@ services: ports: - "9009:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 128g environment: no_proxy: ${no_proxy} diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/compose.yaml b/ChatQnA/docker_compose/intel/hpu/gaudi/compose.yaml index b75312824e..855613fbc2 100644 --- a/ChatQnA/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/compose.yaml @@ -31,7 +31,7 @@ services: ports: - "8090:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -62,7 +62,7 @@ services: ports: - "8808:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" runtime: habana cap_add: - SYS_NICE @@ -83,7 +83,7 @@ services: ports: - "8007:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/compose_guardrails.yaml b/ChatQnA/docker_compose/intel/hpu/gaudi/compose_guardrails.yaml index d5b56e424c..bd1b3cc0ff 100644 --- a/ChatQnA/docker_compose/intel/hpu/gaudi/compose_guardrails.yaml +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/compose_guardrails.yaml @@ -31,7 +31,7 @@ services: ports: - "8088:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} @@ -70,7 +70,7 @@ services: ports: - "8090:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -103,7 +103,7 @@ services: ports: - "8808:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" runtime: habana cap_add: - SYS_NICE @@ -124,7 +124,7 @@ services: ports: - "8008:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/compose_tgi.yaml b/ChatQnA/docker_compose/intel/hpu/gaudi/compose_tgi.yaml index 7fb743e814..fd27be4dfd 100644 --- a/ChatQnA/docker_compose/intel/hpu/gaudi/compose_tgi.yaml +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/compose_tgi.yaml @@ -31,7 +31,7 @@ services: ports: - "8090:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -64,7 +64,7 @@ services: ports: - "8808:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" runtime: habana cap_add: - SYS_NICE @@ -85,7 +85,7 @@ services: ports: - "8005:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/compose_without_rerank.yaml b/ChatQnA/docker_compose/intel/hpu/gaudi/compose_without_rerank.yaml index 2d872c4d30..6f8c7fe0dd 100644 --- a/ChatQnA/docker_compose/intel/hpu/gaudi/compose_without_rerank.yaml +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/compose_without_rerank.yaml @@ -31,7 +31,7 @@ services: ports: - "8090:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -64,7 +64,7 @@ services: ports: - "8007:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} diff --git a/ChatQnA/tests/test_compose_guardrails_on_gaudi.sh b/ChatQnA/tests/test_compose_guardrails_on_gaudi.sh index 9a70a4aff0..d667a89f3c 100644 --- a/ChatQnA/tests/test_compose_guardrails_on_gaudi.sh +++ b/ChatQnA/tests/test_compose_guardrails_on_gaudi.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/ChatQnA/tests/test_compose_on_gaudi.sh b/ChatQnA/tests/test_compose_on_gaudi.sh index d9b40529e8..8858900148 100644 --- a/ChatQnA/tests/test_compose_on_gaudi.sh +++ b/ChatQnA/tests/test_compose_on_gaudi.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/ChatQnA/tests/test_compose_on_rocm.sh b/ChatQnA/tests/test_compose_on_rocm.sh index 732e2684aa..f9623f1691 100644 --- a/ChatQnA/tests/test_compose_on_rocm.sh +++ b/ChatQnA/tests/test_compose_on_rocm.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"/var/opea/chatqna-service/data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/ChatQnA/tests/test_compose_on_xeon.sh b/ChatQnA/tests/test_compose_on_xeon.sh index e3cd8db1d1..bdab2637bf 100644 --- a/ChatQnA/tests/test_compose_on_xeon.sh +++ b/ChatQnA/tests/test_compose_on_xeon.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/ChatQnA/tests/test_compose_pinecone_on_xeon.sh b/ChatQnA/tests/test_compose_pinecone_on_xeon.sh index b45d53871c..17f32ed6cc 100755 --- a/ChatQnA/tests/test_compose_pinecone_on_xeon.sh +++ b/ChatQnA/tests/test_compose_pinecone_on_xeon.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/ChatQnA/tests/test_compose_qdrant_on_xeon.sh b/ChatQnA/tests/test_compose_qdrant_on_xeon.sh index a8539d617c..8c84a9a9ff 100644 --- a/ChatQnA/tests/test_compose_qdrant_on_xeon.sh +++ b/ChatQnA/tests/test_compose_qdrant_on_xeon.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/ChatQnA/tests/test_compose_tgi_on_gaudi.sh b/ChatQnA/tests/test_compose_tgi_on_gaudi.sh index 6b93618932..25bfe8cdee 100644 --- a/ChatQnA/tests/test_compose_tgi_on_gaudi.sh +++ b/ChatQnA/tests/test_compose_tgi_on_gaudi.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/ChatQnA/tests/test_compose_tgi_on_xeon.sh b/ChatQnA/tests/test_compose_tgi_on_xeon.sh index 0a9687a5ff..f00d8c6436 100644 --- a/ChatQnA/tests/test_compose_tgi_on_xeon.sh +++ b/ChatQnA/tests/test_compose_tgi_on_xeon.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/ChatQnA/tests/test_compose_without_rerank_on_gaudi.sh b/ChatQnA/tests/test_compose_without_rerank_on_gaudi.sh index 5975358f29..9e9d7df735 100644 --- a/ChatQnA/tests/test_compose_without_rerank_on_gaudi.sh +++ b/ChatQnA/tests/test_compose_without_rerank_on_gaudi.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/ChatQnA/tests/test_compose_without_rerank_on_xeon.sh b/ChatQnA/tests/test_compose_without_rerank_on_xeon.sh index 50e3feb243..279bc780d0 100644 --- a/ChatQnA/tests/test_compose_without_rerank_on_xeon.sh +++ b/ChatQnA/tests/test_compose_without_rerank_on_xeon.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" From 10fb9288100b3e5c59949f5bef05b3353d1fac30 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 5 Mar 2025 17:13:24 +0800 Subject: [PATCH 039/226] Enable SearchQnA model cache for docker compose test. (#1606) Enable SearchQnA model cache for docker compose test. Signed-off-by: ZePan110 Signed-off-by: Chingis Yundunov --- SearchQnA/docker_compose/amd/gpu/rocm/compose.yaml | 6 +++--- SearchQnA/docker_compose/intel/cpu/xeon/compose.yaml | 6 +++--- SearchQnA/docker_compose/intel/hpu/gaudi/compose.yaml | 6 +++--- SearchQnA/tests/test_compose_on_gaudi.sh | 1 + SearchQnA/tests/test_compose_on_rocm.sh | 1 + SearchQnA/tests/test_compose_on_xeon.sh | 1 + 6 files changed, 12 insertions(+), 9 deletions(-) diff --git a/SearchQnA/docker_compose/amd/gpu/rocm/compose.yaml b/SearchQnA/docker_compose/amd/gpu/rocm/compose.yaml index f531281271..f8318de8fd 100644 --- a/SearchQnA/docker_compose/amd/gpu/rocm/compose.yaml +++ b/SearchQnA/docker_compose/amd/gpu/rocm/compose.yaml @@ -10,7 +10,7 @@ services: ports: - "3001:80" volumes: - - "./data:/data" + - "${MODEL_PATH}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -56,7 +56,7 @@ services: ports: - "3004:80" volumes: - - "./data:/data" + - "${MODEL_PATH}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -86,7 +86,7 @@ services: ports: - "3006:80" volumes: - - "./data:/data" + - "${MODEL_PATH}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} diff --git a/SearchQnA/docker_compose/intel/cpu/xeon/compose.yaml b/SearchQnA/docker_compose/intel/cpu/xeon/compose.yaml index 61f5f2a2fc..7ce41a4205 100644 --- a/SearchQnA/docker_compose/intel/cpu/xeon/compose.yaml +++ b/SearchQnA/docker_compose/intel/cpu/xeon/compose.yaml @@ -9,7 +9,7 @@ services: ports: - "3001:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -60,7 +60,7 @@ services: ports: - "3004:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -96,7 +96,7 @@ services: ports: - "3006:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/SearchQnA/docker_compose/intel/hpu/gaudi/compose.yaml b/SearchQnA/docker_compose/intel/hpu/gaudi/compose.yaml index f79bb9758c..7ad5990b3d 100644 --- a/SearchQnA/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/SearchQnA/docker_compose/intel/hpu/gaudi/compose.yaml @@ -9,7 +9,7 @@ services: ports: - "3001:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" runtime: habana cap_add: - SYS_NICE @@ -67,7 +67,7 @@ services: ports: - "3004:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -103,7 +103,7 @@ services: ports: - "3006:80" volumes: - - "./data:/data" + - "${MODEL_CACHE}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} diff --git a/SearchQnA/tests/test_compose_on_gaudi.sh b/SearchQnA/tests/test_compose_on_gaudi.sh index bf357eebfb..e73d921b7a 100644 --- a/SearchQnA/tests/test_compose_on_gaudi.sh +++ b/SearchQnA/tests/test_compose_on_gaudi.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/SearchQnA/tests/test_compose_on_rocm.sh b/SearchQnA/tests/test_compose_on_rocm.sh index cebe86133f..27de2b9bb0 100644 --- a/SearchQnA/tests/test_compose_on_rocm.sh +++ b/SearchQnA/tests/test_compose_on_rocm.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_PATH=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/SearchQnA/tests/test_compose_on_xeon.sh b/SearchQnA/tests/test_compose_on_xeon.sh index 121d4db9d4..aa8c3aa6e7 100644 --- a/SearchQnA/tests/test_compose_on_xeon.sh +++ b/SearchQnA/tests/test_compose_on_xeon.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" From f746c78000050deea6541be8f12268dfba22ce93 Mon Sep 17 00:00:00 2001 From: Zhu Yongbo Date: Wed, 5 Mar 2025 22:13:53 +0800 Subject: [PATCH 040/226] Fix docker image opea/edgecraftrag security issue #1577 (#1617) Signed-off-by: Zhu, Yongbo Signed-off-by: Chingis Yundunov --- EdgeCraftRAG/Dockerfile.server | 3 ++- EdgeCraftRAG/edgecraftrag/requirements.txt | 4 ++-- EdgeCraftRAG/ui/docker/Dockerfile.ui | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/EdgeCraftRAG/Dockerfile.server b/EdgeCraftRAG/Dockerfile.server index 3bb572f116..ab4060de80 100644 --- a/EdgeCraftRAG/Dockerfile.server +++ b/EdgeCraftRAG/Dockerfile.server @@ -37,7 +37,8 @@ RUN mkdir -p /home/user/gradio_cache ENV GRADIO_TEMP_DIR=/home/user/gradio_cache WORKDIR /home/user/edgecraftrag -RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir --upgrade pip setuptools==70.0.0 && \ + pip install --no-cache-dir -r requirements.txt WORKDIR /home/user/ diff --git a/EdgeCraftRAG/edgecraftrag/requirements.txt b/EdgeCraftRAG/edgecraftrag/requirements.txt index 5ac0137fd1..becbfa767a 100644 --- a/EdgeCraftRAG/edgecraftrag/requirements.txt +++ b/EdgeCraftRAG/edgecraftrag/requirements.txt @@ -1,6 +1,6 @@ docx2txt faiss-cpu>=1.8.0.post1 -langchain-core==0.2.29 +langchain-core>=0.2.29 llama-index>=0.11.0 llama-index-embeddings-openvino>=0.4.0 llama-index-llms-openai-like>=0.2.0 @@ -9,7 +9,7 @@ llama-index-postprocessor-openvino-rerank>=0.3.0 llama-index-readers-file>=0.4.0 llama-index-retrievers-bm25>=0.3.0 llama-index-vector-stores-faiss>=0.2.1 -opea-comps>=0.9 +opea-comps>=1.2 pillow>=10.4.0 python-docx==1.1.2 unstructured==0.16.11 diff --git a/EdgeCraftRAG/ui/docker/Dockerfile.ui b/EdgeCraftRAG/ui/docker/Dockerfile.ui index 8abffc5557..8f8b9b0fb6 100644 --- a/EdgeCraftRAG/ui/docker/Dockerfile.ui +++ b/EdgeCraftRAG/ui/docker/Dockerfile.ui @@ -15,7 +15,7 @@ RUN mkdir -p /home/user/gradio_cache ENV GRADIO_TEMP_DIR=/home/user/gradio_cache WORKDIR /home/user/ui -RUN pip install --no-cache-dir --upgrade pip setuptools && \ +RUN pip install --no-cache-dir --upgrade pip setuptools==70.0.0 && \ pip install --no-cache-dir -r requirements.txt USER user From a7c83e31559c6171b2061da8786302168d04402d Mon Sep 17 00:00:00 2001 From: "Wang, Kai Lawrence" <109344418+wangkl2@users.noreply.github.com> Date: Wed, 5 Mar 2025 22:15:07 +0800 Subject: [PATCH 041/226] [AudioQnA] Fix the LLM model field for inputs alignment (#1611) Signed-off-by: Wang, Kai Lawrence Signed-off-by: Chingis Yundunov --- AudioQnA/audioqna.py | 3 ++- AudioQnA/audioqna_multilang.py | 3 ++- AudioQnA/docker_compose/amd/gpu/rocm/compose.yaml | 1 + AudioQnA/docker_compose/intel/cpu/xeon/compose.yaml | 1 + AudioQnA/docker_compose/intel/hpu/gaudi/compose.yaml | 1 + 5 files changed, 7 insertions(+), 2 deletions(-) diff --git a/AudioQnA/audioqna.py b/AudioQnA/audioqna.py index 79abcccb96..f74e58053f 100644 --- a/AudioQnA/audioqna.py +++ b/AudioQnA/audioqna.py @@ -16,13 +16,14 @@ SPEECHT5_SERVER_PORT = int(os.getenv("SPEECHT5_SERVER_PORT", 7055)) LLM_SERVER_HOST_IP = os.getenv("LLM_SERVER_HOST_IP", "0.0.0.0") LLM_SERVER_PORT = int(os.getenv("LLM_SERVER_PORT", 3006)) +LLM_MODEL_ID = os.getenv("LLM_MODEL_ID", "Intel/neural-chat-7b-v3-3") def align_inputs(self, inputs, cur_node, runtime_graph, llm_parameters_dict, **kwargs): if self.services[cur_node].service_type == ServiceType.LLM: # convert TGI/vLLM to unified OpenAI /v1/chat/completions format next_inputs = {} - next_inputs["model"] = "tgi" # specifically clarify the fake model to make the format unified + next_inputs["model"] = LLM_MODEL_ID next_inputs["messages"] = [{"role": "user", "content": inputs["asr_result"]}] next_inputs["max_tokens"] = llm_parameters_dict["max_tokens"] next_inputs["top_p"] = llm_parameters_dict["top_p"] diff --git a/AudioQnA/audioqna_multilang.py b/AudioQnA/audioqna_multilang.py index 66c2ad1a37..edc14cc93c 100644 --- a/AudioQnA/audioqna_multilang.py +++ b/AudioQnA/audioqna_multilang.py @@ -17,6 +17,7 @@ GPT_SOVITS_SERVER_PORT = int(os.getenv("GPT_SOVITS_SERVER_PORT", 9088)) LLM_SERVER_HOST_IP = os.getenv("LLM_SERVER_HOST_IP", "0.0.0.0") LLM_SERVER_PORT = int(os.getenv("LLM_SERVER_PORT", 8888)) +LLM_MODEL_ID = os.getenv("LLM_MODEL_ID", "Intel/neural-chat-7b-v3-3") def align_inputs(self, inputs, cur_node, runtime_graph, llm_parameters_dict, **kwargs): @@ -24,7 +25,7 @@ def align_inputs(self, inputs, cur_node, runtime_graph, llm_parameters_dict, **k if self.services[cur_node].service_type == ServiceType.LLM: # convert TGI/vLLM to unified OpenAI /v1/chat/completions format next_inputs = {} - next_inputs["model"] = "tgi" # specifically clarify the fake model to make the format unified + next_inputs["model"] = LLM_MODEL_ID next_inputs["messages"] = [{"role": "user", "content": inputs["asr_result"]}] next_inputs["max_tokens"] = llm_parameters_dict["max_tokens"] next_inputs["top_p"] = llm_parameters_dict["top_p"] diff --git a/AudioQnA/docker_compose/amd/gpu/rocm/compose.yaml b/AudioQnA/docker_compose/amd/gpu/rocm/compose.yaml index 4cef1598c2..646b079a29 100644 --- a/AudioQnA/docker_compose/amd/gpu/rocm/compose.yaml +++ b/AudioQnA/docker_compose/amd/gpu/rocm/compose.yaml @@ -69,6 +69,7 @@ services: - WHISPER_SERVER_PORT=${WHISPER_SERVER_PORT} - LLM_SERVER_HOST_IP=${LLM_SERVER_HOST_IP} - LLM_SERVER_PORT=${LLM_SERVER_PORT} + - LLM_MODEL_ID=${LLM_MODEL_ID} - SPEECHT5_SERVER_HOST_IP=${SPEECHT5_SERVER_HOST_IP} - SPEECHT5_SERVER_PORT=${SPEECHT5_SERVER_PORT} ipc: host diff --git a/AudioQnA/docker_compose/intel/cpu/xeon/compose.yaml b/AudioQnA/docker_compose/intel/cpu/xeon/compose.yaml index 48756c00b6..cf9579960b 100644 --- a/AudioQnA/docker_compose/intel/cpu/xeon/compose.yaml +++ b/AudioQnA/docker_compose/intel/cpu/xeon/compose.yaml @@ -61,6 +61,7 @@ services: - WHISPER_SERVER_PORT=${WHISPER_SERVER_PORT} - LLM_SERVER_HOST_IP=${LLM_SERVER_HOST_IP} - LLM_SERVER_PORT=${LLM_SERVER_PORT} + - LLM_MODEL_ID=${LLM_MODEL_ID} - SPEECHT5_SERVER_HOST_IP=${SPEECHT5_SERVER_HOST_IP} - SPEECHT5_SERVER_PORT=${SPEECHT5_SERVER_PORT} ipc: host diff --git a/AudioQnA/docker_compose/intel/hpu/gaudi/compose.yaml b/AudioQnA/docker_compose/intel/hpu/gaudi/compose.yaml index 45691f478b..bcbbac0070 100644 --- a/AudioQnA/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/AudioQnA/docker_compose/intel/hpu/gaudi/compose.yaml @@ -82,6 +82,7 @@ services: - WHISPER_SERVER_PORT=${WHISPER_SERVER_PORT} - LLM_SERVER_HOST_IP=${LLM_SERVER_HOST_IP} - LLM_SERVER_PORT=${LLM_SERVER_PORT} + - LLM_MODEL_ID=${LLM_MODEL_ID} - SPEECHT5_SERVER_HOST_IP=${SPEECHT5_SERVER_HOST_IP} - SPEECHT5_SERVER_PORT=${SPEECHT5_SERVER_PORT} ipc: host From db31e553a6f114fd67a0eb6a2405682986fb6b6d Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Fri, 7 Mar 2025 08:38:59 +0800 Subject: [PATCH 042/226] Update compose.yaml for SearchQnA (#1622) Signed-off-by: ZePan110 Signed-off-by: Chingis Yundunov --- SearchQnA/docker_compose/amd/gpu/rocm/compose.yaml | 6 +++--- SearchQnA/docker_compose/intel/cpu/xeon/compose.yaml | 6 +++--- SearchQnA/docker_compose/intel/hpu/gaudi/compose.yaml | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/SearchQnA/docker_compose/amd/gpu/rocm/compose.yaml b/SearchQnA/docker_compose/amd/gpu/rocm/compose.yaml index f8318de8fd..fef008250d 100644 --- a/SearchQnA/docker_compose/amd/gpu/rocm/compose.yaml +++ b/SearchQnA/docker_compose/amd/gpu/rocm/compose.yaml @@ -10,7 +10,7 @@ services: ports: - "3001:80" volumes: - - "${MODEL_PATH}:/data" + - "${MODEL_PATH:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -56,7 +56,7 @@ services: ports: - "3004:80" volumes: - - "${MODEL_PATH}:/data" + - "${MODEL_PATH:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -86,7 +86,7 @@ services: ports: - "3006:80" volumes: - - "${MODEL_PATH}:/data" + - "${MODEL_PATH:-./data}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} diff --git a/SearchQnA/docker_compose/intel/cpu/xeon/compose.yaml b/SearchQnA/docker_compose/intel/cpu/xeon/compose.yaml index 7ce41a4205..29b5229b83 100644 --- a/SearchQnA/docker_compose/intel/cpu/xeon/compose.yaml +++ b/SearchQnA/docker_compose/intel/cpu/xeon/compose.yaml @@ -9,7 +9,7 @@ services: ports: - "3001:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -60,7 +60,7 @@ services: ports: - "3004:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -96,7 +96,7 @@ services: ports: - "3006:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/SearchQnA/docker_compose/intel/hpu/gaudi/compose.yaml b/SearchQnA/docker_compose/intel/hpu/gaudi/compose.yaml index 7ad5990b3d..d1df099437 100644 --- a/SearchQnA/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/SearchQnA/docker_compose/intel/hpu/gaudi/compose.yaml @@ -9,7 +9,7 @@ services: ports: - "3001:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" runtime: habana cap_add: - SYS_NICE @@ -67,7 +67,7 @@ services: ports: - "3004:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -103,7 +103,7 @@ services: ports: - "3006:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} From 3b33f30ec8ddb7670be5852da228e79282c23477 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Fri, 7 Mar 2025 09:19:39 +0800 Subject: [PATCH 043/226] Update compose.yaml for ChatQnA (#1621) Update compose.yaml for ChatQnA Signed-off-by: ZePan110 Signed-off-by: Chingis Yundunov --- ChatQnA/docker_compose/amd/gpu/rocm/compose.yaml | 6 +++--- ChatQnA/docker_compose/intel/cpu/xeon/compose.yaml | 6 +++--- .../docker_compose/intel/cpu/xeon/compose_pinecone.yaml | 6 +++--- ChatQnA/docker_compose/intel/cpu/xeon/compose_qdrant.yaml | 6 +++--- ChatQnA/docker_compose/intel/cpu/xeon/compose_tgi.yaml | 6 +++--- .../intel/cpu/xeon/compose_without_rerank.yaml | 4 ++-- ChatQnA/docker_compose/intel/hpu/gaudi/compose.yaml | 6 +++--- .../intel/hpu/gaudi/compose_guardrails.yaml | 8 ++++---- ChatQnA/docker_compose/intel/hpu/gaudi/compose_tgi.yaml | 6 +++--- .../intel/hpu/gaudi/compose_without_rerank.yaml | 4 ++-- 10 files changed, 29 insertions(+), 29 deletions(-) diff --git a/ChatQnA/docker_compose/amd/gpu/rocm/compose.yaml b/ChatQnA/docker_compose/amd/gpu/rocm/compose.yaml index 193f4346e7..da1f4ddda4 100644 --- a/ChatQnA/docker_compose/amd/gpu/rocm/compose.yaml +++ b/ChatQnA/docker_compose/amd/gpu/rocm/compose.yaml @@ -30,7 +30,7 @@ services: ports: - "${CHATQNA_TEI_EMBEDDING_PORT}:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-/var/opea/chatqna-service/data}:/data" shm_size: 1g ipc: host environment: @@ -72,7 +72,7 @@ services: ports: - "${CHATQNA_TEI_RERANKING_PORT}:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-/var/opea/chatqna-service/data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -104,7 +104,7 @@ services: HF_HUB_DISABLE_PROGRESS_BARS: 1 HF_HUB_ENABLE_HF_TRANSFER: 0 volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-/var/opea/chatqna-service/data}:/data" shm_size: 1g devices: - /dev/kfd:/dev/kfd diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/compose.yaml b/ChatQnA/docker_compose/intel/cpu/xeon/compose.yaml index 00c6a2aec2..1ec229115e 100644 --- a/ChatQnA/docker_compose/intel/cpu/xeon/compose.yaml +++ b/ChatQnA/docker_compose/intel/cpu/xeon/compose.yaml @@ -31,7 +31,7 @@ services: ports: - "6006:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -64,7 +64,7 @@ services: ports: - "8808:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -80,7 +80,7 @@ services: ports: - "9009:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 128g environment: no_proxy: ${no_proxy} diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/compose_pinecone.yaml b/ChatQnA/docker_compose/intel/cpu/xeon/compose_pinecone.yaml index a2d2318945..a398e9d983 100644 --- a/ChatQnA/docker_compose/intel/cpu/xeon/compose_pinecone.yaml +++ b/ChatQnA/docker_compose/intel/cpu/xeon/compose_pinecone.yaml @@ -28,7 +28,7 @@ services: ports: - "6006:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -59,7 +59,7 @@ services: ports: - "8808:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -75,7 +75,7 @@ services: ports: - "9009:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 128g environment: no_proxy: ${no_proxy} diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/compose_qdrant.yaml b/ChatQnA/docker_compose/intel/cpu/xeon/compose_qdrant.yaml index 8a7fabdfad..0504ff07a1 100644 --- a/ChatQnA/docker_compose/intel/cpu/xeon/compose_qdrant.yaml +++ b/ChatQnA/docker_compose/intel/cpu/xeon/compose_qdrant.yaml @@ -32,7 +32,7 @@ services: ports: - "6040:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -64,7 +64,7 @@ services: ports: - "6041:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -80,7 +80,7 @@ services: ports: - "6042:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 128g environment: no_proxy: ${no_proxy} diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/compose_tgi.yaml b/ChatQnA/docker_compose/intel/cpu/xeon/compose_tgi.yaml index 4a7c4f4627..34d95ffc68 100644 --- a/ChatQnA/docker_compose/intel/cpu/xeon/compose_tgi.yaml +++ b/ChatQnA/docker_compose/intel/cpu/xeon/compose_tgi.yaml @@ -31,7 +31,7 @@ services: ports: - "6006:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -64,7 +64,7 @@ services: ports: - "8808:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -80,7 +80,7 @@ services: ports: - "9009:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/compose_without_rerank.yaml b/ChatQnA/docker_compose/intel/cpu/xeon/compose_without_rerank.yaml index 72fbdead0a..70ea084408 100644 --- a/ChatQnA/docker_compose/intel/cpu/xeon/compose_without_rerank.yaml +++ b/ChatQnA/docker_compose/intel/cpu/xeon/compose_without_rerank.yaml @@ -31,7 +31,7 @@ services: ports: - "6006:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -64,7 +64,7 @@ services: ports: - "9009:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 128g environment: no_proxy: ${no_proxy} diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/compose.yaml b/ChatQnA/docker_compose/intel/hpu/gaudi/compose.yaml index 855613fbc2..8ff06ecc35 100644 --- a/ChatQnA/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/compose.yaml @@ -31,7 +31,7 @@ services: ports: - "8090:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -62,7 +62,7 @@ services: ports: - "8808:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" runtime: habana cap_add: - SYS_NICE @@ -83,7 +83,7 @@ services: ports: - "8007:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/compose_guardrails.yaml b/ChatQnA/docker_compose/intel/hpu/gaudi/compose_guardrails.yaml index bd1b3cc0ff..b3388e0b5f 100644 --- a/ChatQnA/docker_compose/intel/hpu/gaudi/compose_guardrails.yaml +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/compose_guardrails.yaml @@ -31,7 +31,7 @@ services: ports: - "8088:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} @@ -70,7 +70,7 @@ services: ports: - "8090:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -103,7 +103,7 @@ services: ports: - "8808:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" runtime: habana cap_add: - SYS_NICE @@ -124,7 +124,7 @@ services: ports: - "8008:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/compose_tgi.yaml b/ChatQnA/docker_compose/intel/hpu/gaudi/compose_tgi.yaml index fd27be4dfd..a14e3fca67 100644 --- a/ChatQnA/docker_compose/intel/hpu/gaudi/compose_tgi.yaml +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/compose_tgi.yaml @@ -31,7 +31,7 @@ services: ports: - "8090:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -64,7 +64,7 @@ services: ports: - "8808:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" runtime: habana cap_add: - SYS_NICE @@ -85,7 +85,7 @@ services: ports: - "8005:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/compose_without_rerank.yaml b/ChatQnA/docker_compose/intel/hpu/gaudi/compose_without_rerank.yaml index 6f8c7fe0dd..167ce9c1e1 100644 --- a/ChatQnA/docker_compose/intel/hpu/gaudi/compose_without_rerank.yaml +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/compose_without_rerank.yaml @@ -31,7 +31,7 @@ services: ports: - "8090:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -64,7 +64,7 @@ services: ports: - "8007:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} From e6d0c279d0619af1894c350c9abceca17153f678 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Fri, 7 Mar 2025 09:20:08 +0800 Subject: [PATCH 044/226] Update compose.yaml (#1620) Update compose.yaml for AudioQnA, DBQnA, DocIndexRetriever, FaqGen, Translation and VisualQnA. Signed-off-by: ZePan110 Signed-off-by: Chingis Yundunov --- AudioQnA/docker_compose/intel/cpu/xeon/compose.yaml | 2 +- AudioQnA/docker_compose/intel/cpu/xeon/compose_multilang.yaml | 2 +- AudioQnA/docker_compose/intel/hpu/gaudi/compose.yaml | 2 +- DBQnA/docker_compose/intel/cpu/xeon/compose.yaml | 2 +- DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml | 4 ++-- DocIndexRetriever/docker_compose/intel/hpu/gaudi/compose.yaml | 4 ++-- FaqGen/docker_compose/intel/cpu/xeon/compose.yaml | 2 +- Translation/docker_compose/intel/cpu/xeon/compose.yaml | 2 +- Translation/docker_compose/intel/hpu/gaudi/compose.yaml | 2 +- VisualQnA/docker_compose/intel/cpu/xeon/compose.yaml | 2 +- VisualQnA/docker_compose/intel/hpu/gaudi/compose.yaml | 2 +- 11 files changed, 13 insertions(+), 13 deletions(-) diff --git a/AudioQnA/docker_compose/intel/cpu/xeon/compose.yaml b/AudioQnA/docker_compose/intel/cpu/xeon/compose.yaml index cf9579960b..3b47780d80 100644 --- a/AudioQnA/docker_compose/intel/cpu/xeon/compose.yaml +++ b/AudioQnA/docker_compose/intel/cpu/xeon/compose.yaml @@ -30,7 +30,7 @@ services: ports: - "3006:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/AudioQnA/docker_compose/intel/cpu/xeon/compose_multilang.yaml b/AudioQnA/docker_compose/intel/cpu/xeon/compose_multilang.yaml index c6ad650943..fde5a56902 100644 --- a/AudioQnA/docker_compose/intel/cpu/xeon/compose_multilang.yaml +++ b/AudioQnA/docker_compose/intel/cpu/xeon/compose_multilang.yaml @@ -31,7 +31,7 @@ services: ports: - "3006:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/AudioQnA/docker_compose/intel/hpu/gaudi/compose.yaml b/AudioQnA/docker_compose/intel/hpu/gaudi/compose.yaml index bcbbac0070..9e43a355b5 100644 --- a/AudioQnA/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/AudioQnA/docker_compose/intel/hpu/gaudi/compose.yaml @@ -40,7 +40,7 @@ services: ports: - "3006:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} diff --git a/DBQnA/docker_compose/intel/cpu/xeon/compose.yaml b/DBQnA/docker_compose/intel/cpu/xeon/compose.yaml index 8e4c15bd6b..b96a71d01d 100644 --- a/DBQnA/docker_compose/intel/cpu/xeon/compose.yaml +++ b/DBQnA/docker_compose/intel/cpu/xeon/compose.yaml @@ -8,7 +8,7 @@ services: ports: - "8008:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} diff --git a/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml b/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml index 6ecebfdc23..119b460d92 100644 --- a/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml +++ b/DocIndexRetriever/docker_compose/intel/cpu/xeon/compose.yaml @@ -38,7 +38,7 @@ services: ports: - "6006:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -96,7 +96,7 @@ services: ports: - "8808:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/DocIndexRetriever/docker_compose/intel/hpu/gaudi/compose.yaml b/DocIndexRetriever/docker_compose/intel/hpu/gaudi/compose.yaml index f47d01a7cf..a2bfd878fc 100644 --- a/DocIndexRetriever/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/DocIndexRetriever/docker_compose/intel/hpu/gaudi/compose.yaml @@ -34,7 +34,7 @@ services: ports: - "8090:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" runtime: habana cap_add: - SYS_NICE @@ -95,7 +95,7 @@ services: ports: - "8808:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/FaqGen/docker_compose/intel/cpu/xeon/compose.yaml b/FaqGen/docker_compose/intel/cpu/xeon/compose.yaml index ea24486cda..ca86a18f2d 100644 --- a/FaqGen/docker_compose/intel/cpu/xeon/compose.yaml +++ b/FaqGen/docker_compose/intel/cpu/xeon/compose.yaml @@ -8,7 +8,7 @@ services: ports: - ${LLM_ENDPOINT_PORT:-8008}:80 volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/Translation/docker_compose/intel/cpu/xeon/compose.yaml b/Translation/docker_compose/intel/cpu/xeon/compose.yaml index d1a6ee337d..4b77d84484 100644 --- a/Translation/docker_compose/intel/cpu/xeon/compose.yaml +++ b/Translation/docker_compose/intel/cpu/xeon/compose.yaml @@ -21,7 +21,7 @@ services: timeout: 10s retries: 100 volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g command: --model-id ${LLM_MODEL_ID} --cuda-graphs 0 llm: diff --git a/Translation/docker_compose/intel/hpu/gaudi/compose.yaml b/Translation/docker_compose/intel/hpu/gaudi/compose.yaml index 7e49db9c39..9516e60ce6 100644 --- a/Translation/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/Translation/docker_compose/intel/hpu/gaudi/compose.yaml @@ -30,7 +30,7 @@ services: - SYS_NICE ipc: host volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" command: --model-id ${LLM_MODEL_ID} --max-input-length 1024 --max-total-tokens 2048 llm: image: ${REGISTRY:-opea}/llm-textgen:${TAG:-latest} diff --git a/VisualQnA/docker_compose/intel/cpu/xeon/compose.yaml b/VisualQnA/docker_compose/intel/cpu/xeon/compose.yaml index 4a81704be4..b595bdcba7 100644 --- a/VisualQnA/docker_compose/intel/cpu/xeon/compose.yaml +++ b/VisualQnA/docker_compose/intel/cpu/xeon/compose.yaml @@ -8,7 +8,7 @@ services: ports: - "8399:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/VisualQnA/docker_compose/intel/hpu/gaudi/compose.yaml b/VisualQnA/docker_compose/intel/hpu/gaudi/compose.yaml index 73e2747085..bd4004e399 100644 --- a/VisualQnA/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/VisualQnA/docker_compose/intel/hpu/gaudi/compose.yaml @@ -8,7 +8,7 @@ services: ports: - "8399:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} From caefabe22045f4ef34f897c93a2bf612bf790f70 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Fri, 7 Mar 2025 09:20:28 +0800 Subject: [PATCH 045/226] Update compose.yaml (#1619) Update compose.yaml for CodeGen, CodeTrans and DocSum Signed-off-by: ZePan110 Signed-off-by: Chingis Yundunov --- CodeGen/docker_compose/intel/cpu/xeon/compose.yaml | 2 +- CodeGen/docker_compose/intel/hpu/gaudi/compose.yaml | 2 +- CodeTrans/docker_compose/intel/cpu/xeon/compose.yaml | 2 +- CodeTrans/docker_compose/intel/hpu/gaudi/compose.yaml | 2 +- DocSum/docker_compose/intel/cpu/xeon/compose.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CodeGen/docker_compose/intel/cpu/xeon/compose.yaml b/CodeGen/docker_compose/intel/cpu/xeon/compose.yaml index f9e7e26280..7973951000 100644 --- a/CodeGen/docker_compose/intel/cpu/xeon/compose.yaml +++ b/CodeGen/docker_compose/intel/cpu/xeon/compose.yaml @@ -8,7 +8,7 @@ services: ports: - "8028:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/CodeGen/docker_compose/intel/hpu/gaudi/compose.yaml b/CodeGen/docker_compose/intel/hpu/gaudi/compose.yaml index 62ec96e626..19a77bef54 100644 --- a/CodeGen/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/CodeGen/docker_compose/intel/hpu/gaudi/compose.yaml @@ -8,7 +8,7 @@ services: ports: - "8028:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} diff --git a/CodeTrans/docker_compose/intel/cpu/xeon/compose.yaml b/CodeTrans/docker_compose/intel/cpu/xeon/compose.yaml index 0ece6dff1d..2028760c48 100644 --- a/CodeTrans/docker_compose/intel/cpu/xeon/compose.yaml +++ b/CodeTrans/docker_compose/intel/cpu/xeon/compose.yaml @@ -8,7 +8,7 @@ services: ports: - "8008:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/CodeTrans/docker_compose/intel/hpu/gaudi/compose.yaml b/CodeTrans/docker_compose/intel/hpu/gaudi/compose.yaml index 3e25dee894..e697a0927b 100644 --- a/CodeTrans/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/CodeTrans/docker_compose/intel/hpu/gaudi/compose.yaml @@ -8,7 +8,7 @@ services: ports: - "8008:80" volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} diff --git a/DocSum/docker_compose/intel/cpu/xeon/compose.yaml b/DocSum/docker_compose/intel/cpu/xeon/compose.yaml index 0d87eaeb2b..8d91db5e73 100644 --- a/DocSum/docker_compose/intel/cpu/xeon/compose.yaml +++ b/DocSum/docker_compose/intel/cpu/xeon/compose.yaml @@ -21,7 +21,7 @@ services: timeout: 10s retries: 100 volumes: - - "${MODEL_CACHE}:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g command: --model-id ${LLM_MODEL_ID} --cuda-graphs 0 --max-input-length ${MAX_INPUT_TOKENS} --max-total-tokens ${MAX_TOTAL_TOKENS} From a74dc1e988bf1723c9fe351eee2fc42c7ac84269 Mon Sep 17 00:00:00 2001 From: Letong Han <106566639+letonghan@users.noreply.github.com> Date: Fri, 7 Mar 2025 10:56:21 +0800 Subject: [PATCH 046/226] Enable vllm for CodeTrans (#1626) Set vllm as default llm serving, and add related docker compose files, readmes, and test scripts. Issue: https://github.com/opea-project/GenAIExamples/issues/1436 Signed-off-by: letonghan Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: Chingis Yundunov --- .../docker_compose/intel/cpu/xeon/README.md | 71 ++++++- .../intel/cpu/xeon/compose.yaml | 26 +-- .../intel/cpu/xeon/compose_tgi.yaml | 95 +++++++++ .../docker_compose/intel/hpu/gaudi/README.md | 68 +++++- .../intel/hpu/gaudi/compose.yaml | 40 ++-- .../intel/hpu/gaudi/compose_tgi.yaml | 99 +++++++++ CodeTrans/docker_compose/set_env.sh | 7 +- CodeTrans/docker_image_build/build.yaml | 12 ++ CodeTrans/tests/test_compose_on_gaudi.sh | 33 ++- CodeTrans/tests/test_compose_on_xeon.sh | 35 ++-- CodeTrans/tests/test_compose_tgi_on_gaudi.sh | 194 ++++++++++++++++++ CodeTrans/tests/test_compose_tgi_on_xeon.sh | 194 ++++++++++++++++++ 12 files changed, 801 insertions(+), 73 deletions(-) create mode 100644 CodeTrans/docker_compose/intel/cpu/xeon/compose_tgi.yaml create mode 100644 CodeTrans/docker_compose/intel/hpu/gaudi/compose_tgi.yaml create mode 100644 CodeTrans/tests/test_compose_tgi_on_gaudi.sh create mode 100644 CodeTrans/tests/test_compose_tgi_on_xeon.sh diff --git a/CodeTrans/docker_compose/intel/cpu/xeon/README.md b/CodeTrans/docker_compose/intel/cpu/xeon/README.md index b5aebe8690..a7a8066202 100755 --- a/CodeTrans/docker_compose/intel/cpu/xeon/README.md +++ b/CodeTrans/docker_compose/intel/cpu/xeon/README.md @@ -2,6 +2,8 @@ This document outlines the deployment process for a CodeTrans application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on Intel Xeon server. The steps include Docker image creation, container deployment via Docker Compose, and service execution using microservices `llm`. We will publish the Docker images to Docker Hub soon, it will simplify the deployment process for this service. +The default pipeline deploys with vLLM as the LLM serving component. It also provides options of using TGI backend for LLM microservice, please refer to [start-microservice-docker-containers](#start-microservice-docker-containers) section in this page. + ## 🚀 Create an AWS Xeon Instance To run the example on a AWS Xeon instance, start by creating an AWS account if you don't have one already. Then, get started with the [EC2 Console](https://console.aws.amazon.com/ec2/v2/home). AWS EC2 M7i, C7i, C7i-flex and M7i-flex are Intel Xeon Scalable processor instances suitable for the task. (code named Sapphire Rapids). @@ -63,6 +65,37 @@ By default, the LLM model is set to a default value as listed below: Change the `LLM_MODEL_ID` below for your needs. +For users in China who are unable to download models directly from Huggingface, you can use [ModelScope](https://www.modelscope.cn/models) or a Huggingface mirror to download models. The vLLM/TGI can load the models either online or offline as described below: + +1. Online + + ```bash + export HF_TOKEN=${your_hf_token} + export HF_ENDPOINT="https://hf-mirror.com" + model_name="mistralai/Mistral-7B-Instruct-v0.3" + # Start vLLM LLM Service + docker run -p 8008:80 -v ./data:/data --name vllm-service -e HF_ENDPOINT=$HF_ENDPOINT -e http_proxy=$http_proxy -e https_proxy=$https_proxy --shm-size 128g opea/vllm:latest --model $model_name --host 0.0.0.0 --port 80 + # Start TGI LLM Service + docker run -p 8008:80 -v ./data:/data --name tgi-service -e HF_ENDPOINT=$HF_ENDPOINT -e http_proxy=$http_proxy -e https_proxy=$https_proxy --shm-size 1g ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu --model-id $model_name + ``` + +2. Offline + + - Search your model name in ModelScope. For example, check [this page](https://www.modelscope.cn/models/rubraAI/Mistral-7B-Instruct-v0.3/files) for model `mistralai/Mistral-7B-Instruct-v0.3`. + + - Click on `Download this model` button, and choose one way to download the model to your local path `/path/to/model`. + + - Run the following command to start the LLM service. + + ```bash + export HF_TOKEN=${your_hf_token} + export model_path="/path/to/model" + # Start vLLM LLM Service + docker run -p 8008:80 -v $model_path:/data --name vllm-service --shm-size 128g opea/vllm:latest --model /data --host 0.0.0.0 --port 80 + # Start TGI LLM Service + docker run -p 8008:80 -v $model_path:/data --name tgi-service --shm-size 1g ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu --model-id /data + ``` + ### Setup Environment Variables 1. Set the required environment variables: @@ -95,15 +128,47 @@ Change the `LLM_MODEL_ID` below for your needs. ```bash cd GenAIExamples/CodeTrans/docker_compose/intel/cpu/xeon -docker compose up -d +``` + +If use vLLM as the LLM serving backend. + +```bash +docker compose -f compose.yaml up -d +``` + +If use TGI as the LLM serving backend. + +```bash +docker compose -f compose_tgi.yaml up -d ``` ### Validate Microservices -1. TGI Service +1. LLM backend Service + + In the first startup, this service will take more time to download, load and warm up the model. After it's finished, the service will be ready. + + Try the command below to check whether the LLM serving is ready. + + ```bash + # vLLM service + docker logs codetrans-xeon-vllm-service 2>&1 | grep complete + # If the service is ready, you will get the response like below. + INFO: Application startup complete. + ``` + + ```bash + # TGI service + docker logs codetrans-xeon-tgi-service | grep Connected + # If the service is ready, you will get the response like below. + 2024-09-03T02:47:53.402023Z INFO text_generation_router::server: router/src/server.rs:2311: Connected + ``` + + Then try the `cURL` command below to validate services. ```bash - curl http://${host_ip}:8008/generate \ + # either vLLM or TGI service + curl http://${host_ip}:8008/v1/chat/completions \ -X POST \ -d '{"inputs":" ### System: Please translate the following Golang codes into Python codes. ### Original codes: '\'''\'''\''Golang \npackage main\n\nimport \"fmt\"\nfunc main() {\n fmt.Println(\"Hello, World!\");\n '\'''\'''\'' ### Translated codes:","parameters":{"max_new_tokens":17, "do_sample": true}}' \ -H 'Content-Type: application/json' diff --git a/CodeTrans/docker_compose/intel/cpu/xeon/compose.yaml b/CodeTrans/docker_compose/intel/cpu/xeon/compose.yaml index 2028760c48..24c8bfdd39 100644 --- a/CodeTrans/docker_compose/intel/cpu/xeon/compose.yaml +++ b/CodeTrans/docker_compose/intel/cpu/xeon/compose.yaml @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 services: - tgi-service: - image: ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu - container_name: codetrans-tgi-service + vllm-service: + image: ${REGISTRY:-opea}/vllm:${TAG:-latest} + container_name: codetrans-xeon-vllm-service ports: - "8008:80" volumes: @@ -15,18 +15,19 @@ services: http_proxy: ${http_proxy} https_proxy: ${https_proxy} HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} - host_ip: ${host_ip} + LLM_MODEL_ID: ${LLM_MODEL_ID} + VLLM_TORCH_PROFILER_DIR: "/mnt" healthcheck: test: ["CMD-SHELL", "curl -f http://$host_ip:8008/health || exit 1"] interval: 10s timeout: 10s retries: 100 - command: --model-id ${LLM_MODEL_ID} --cuda-graphs 0 + command: --model $LLM_MODEL_ID --host 0.0.0.0 --port 80 llm: image: ${REGISTRY:-opea}/llm-textgen:${TAG:-latest} - container_name: llm-textgen-server + container_name: codetrans-xeon-llm-server depends_on: - tgi-service: + vllm-service: condition: service_healthy ports: - "9000:9000" @@ -35,18 +36,19 @@ services: no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} - LLM_ENDPOINT: ${TGI_LLM_ENDPOINT} + LLM_ENDPOINT: ${LLM_ENDPOINT} LLM_MODEL_ID: ${LLM_MODEL_ID} - HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + LLM_COMPONENT_NAME: ${LLM_COMPONENT_NAME} + HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} restart: unless-stopped codetrans-xeon-backend-server: image: ${REGISTRY:-opea}/codetrans:${TAG:-latest} container_name: codetrans-xeon-backend-server depends_on: - - tgi-service + - vllm-service - llm ports: - - "7777:7777" + - "${BACKEND_SERVICE_PORT:-7777}:7777" environment: - no_proxy=${no_proxy} - https_proxy=${https_proxy} @@ -61,7 +63,7 @@ services: depends_on: - codetrans-xeon-backend-server ports: - - "5173:5173" + - "${FRONTEND_SERVICE_PORT:-5173}:5173" environment: - no_proxy=${no_proxy} - https_proxy=${https_proxy} diff --git a/CodeTrans/docker_compose/intel/cpu/xeon/compose_tgi.yaml b/CodeTrans/docker_compose/intel/cpu/xeon/compose_tgi.yaml new file mode 100644 index 0000000000..77c668241c --- /dev/null +++ b/CodeTrans/docker_compose/intel/cpu/xeon/compose_tgi.yaml @@ -0,0 +1,95 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +services: + tgi-service: + image: ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu + container_name: codetrans-xeon-tgi-service + ports: + - "8008:80" + volumes: + - "${MODEL_CACHE}:/data" + shm_size: 1g + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + host_ip: ${host_ip} + healthcheck: + test: ["CMD-SHELL", "curl -f http://$host_ip:8008/health || exit 1"] + interval: 10s + timeout: 10s + retries: 100 + command: --model-id ${LLM_MODEL_ID} --cuda-graphs 0 + llm: + image: ${REGISTRY:-opea}/llm-textgen:${TAG:-latest} + container_name: codetrans-xeon-llm-server + depends_on: + tgi-service: + condition: service_healthy + ports: + - "9000:9000" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + LLM_ENDPOINT: ${LLM_ENDPOINT} + LLM_MODEL_ID: ${LLM_MODEL_ID} + LLM_COMPONENT_NAME: ${LLM_COMPONENT_NAME} + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + restart: unless-stopped + codetrans-xeon-backend-server: + image: ${REGISTRY:-opea}/codetrans:${TAG:-latest} + container_name: codetrans-xeon-backend-server + depends_on: + - tgi-service + - llm + ports: + - "${BACKEND_SERVICE_PORT:-7777}:7777" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP} + - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP} + ipc: host + restart: always + codetrans-xeon-ui-server: + image: ${REGISTRY:-opea}/codetrans-ui:${TAG:-latest} + container_name: codetrans-xeon-ui-server + depends_on: + - codetrans-xeon-backend-server + ports: + - "${FRONTEND_SERVICE_PORT:-5173}:5173" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - BASE_URL=${BACKEND_SERVICE_ENDPOINT} + ipc: host + restart: always + codetrans-xeon-nginx-server: + image: ${REGISTRY:-opea}/nginx:${TAG:-latest} + container_name: codetrans-xeon-nginx-server + depends_on: + - codetrans-xeon-backend-server + - codetrans-xeon-ui-server + ports: + - "${NGINX_PORT:-80}:80" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - FRONTEND_SERVICE_IP=${FRONTEND_SERVICE_IP} + - FRONTEND_SERVICE_PORT=${FRONTEND_SERVICE_PORT} + - BACKEND_SERVICE_NAME=${BACKEND_SERVICE_NAME} + - BACKEND_SERVICE_IP=${BACKEND_SERVICE_IP} + - BACKEND_SERVICE_PORT=${BACKEND_SERVICE_PORT} + ipc: host + restart: always + +networks: + default: + driver: bridge diff --git a/CodeTrans/docker_compose/intel/hpu/gaudi/README.md b/CodeTrans/docker_compose/intel/hpu/gaudi/README.md index 00241d6acf..cf5f2d3c11 100755 --- a/CodeTrans/docker_compose/intel/hpu/gaudi/README.md +++ b/CodeTrans/docker_compose/intel/hpu/gaudi/README.md @@ -2,6 +2,8 @@ This document outlines the deployment process for a CodeTrans application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on Intel Gaudi server. The steps include Docker image creation, container deployment via Docker Compose, and service execution using microservices `llm`. We will publish the Docker images to Docker Hub soon, it will simplify the deployment process for this service. +The default pipeline deploys with vLLM as the LLM serving component. It also provides options of using TGI backend for LLM microservice, please refer to [start-microservice-docker-containers](#start-microservice-docker-containers) section in this page. + ## 🚀 Build Docker Images First of all, you need to build Docker Images locally and install the python package of it. This step can be ignored after the Docker images published to Docker hub. @@ -55,6 +57,37 @@ By default, the LLM model is set to a default value as listed below: Change the `LLM_MODEL_ID` below for your needs. +For users in China who are unable to download models directly from Huggingface, you can use [ModelScope](https://www.modelscope.cn/models) or a Huggingface mirror to download models. The vLLM/TGI can load the models either online or offline as described below: + +1. Online + + ```bash + export HF_TOKEN=${your_hf_token} + export HF_ENDPOINT="https://hf-mirror.com" + model_name="mistralai/Mistral-7B-Instruct-v0.3" + # Start vLLM LLM Service + docker run -p 8008:80 -v ./data:/data --name vllm-service -e HF_ENDPOINT=$HF_ENDPOINT -e http_proxy=$http_proxy -e https_proxy=$https_proxy --shm-size 128g opea/vllm:latest --model $model_name --host 0.0.0.0 --port 80 + # Start TGI LLM Service + docker run -p 8008:80 -v ./data:/data --name tgi-service -e HF_ENDPOINT=$HF_ENDPOINT -e http_proxy=$http_proxy -e https_proxy=$https_proxy --shm-size 1g ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu --model-id $model_name + ``` + +2. Offline + + - Search your model name in ModelScope. For example, check [this page](https://www.modelscope.cn/models/rubraAI/Mistral-7B-Instruct-v0.3/files) for model `mistralai/Mistral-7B-Instruct-v0.3`. + + - Click on `Download this model` button, and choose one way to download the model to your local path `/path/to/model`. + + - Run the following command to start the LLM service. + + ```bash + export HF_TOKEN=${your_hf_token} + export model_path="/path/to/model" + # Start vLLM LLM Service + docker run -p 8008:80 -v $model_path:/data --name vllm-service --shm-size 128g opea/vllm:latest --model /data --host 0.0.0.0 --port 80 + # Start TGI LLM Service + docker run -p 8008:80 -v $model_path:/data --name tgi-service --shm-size 1g ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu --model-id /data + ``` + ### Setup Environment Variables 1. Set the required environment variables: @@ -87,12 +120,43 @@ Change the `LLM_MODEL_ID` below for your needs. ```bash cd GenAIExamples/CodeTrans/docker_compose/intel/hpu/gaudi -docker compose up -d +``` + +If use vLLM as the LLM serving backend. + +```bash +docker compose -f compose.yaml up -d +``` + +If use TGI as the LLM serving backend. + +```bash +docker compose -f compose_tgi.yaml up -d ``` ### Validate Microservices -1. TGI Service +1. LLM backend Service + + In the first startup, this service will take more time to download, load and warm up the model. After it's finished, the service will be ready. + + Try the command below to check whether the LLM serving is ready. + + ```bash + # vLLM service + docker logs codetrans-gaudi-vllm-service 2>&1 | grep complete + # If the service is ready, you will get the response like below. + INFO: Application startup complete. + ``` + + ```bash + # TGI service + docker logs codetrans-gaudi-tgi-service | grep Connected + # If the service is ready, you will get the response like below. + 2024-09-03T02:47:53.402023Z INFO text_generation_router::server: router/src/server.rs:2311: Connected + ``` + + Then try the `cURL` command below to validate services. ```bash curl http://${host_ip}:8008/generate \ diff --git a/CodeTrans/docker_compose/intel/hpu/gaudi/compose.yaml b/CodeTrans/docker_compose/intel/hpu/gaudi/compose.yaml index e697a0927b..2caeaf0ec3 100644 --- a/CodeTrans/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/CodeTrans/docker_compose/intel/hpu/gaudi/compose.yaml @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 services: - tgi-service: - image: ghcr.io/huggingface/tgi-gaudi:2.0.6 - container_name: codetrans-tgi-service + vllm-service: + image: ${REGISTRY:-opea}/vllm-gaudi:${TAG:-latest} + container_name: codetrans-gaudi-vllm-service ports: - "8008:80" volumes: @@ -13,28 +13,27 @@ services: no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} + HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} HABANA_VISIBLE_DEVICES: all OMPI_MCA_btl_vader_single_copy_mechanism: none - HUGGING_FACE_HUB_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} - ENABLE_HPU_GRAPH: true - LIMIT_HPU_GRAPH: true - USE_FLASH_ATTENTION: true - FLASH_ATTENTION_RECOMPUTE: true + LLM_MODEL_ID: ${LLM_MODEL_ID} + NUM_CARDS: ${NUM_CARDS} + VLLM_TORCH_PROFILER_DIR: "/mnt" healthcheck: - test: ["CMD-SHELL", "sleep 500 && exit 0"] - interval: 1s - timeout: 505s - retries: 1 + test: ["CMD-SHELL", "curl -f http://$host_ip:8008/health || exit 1"] + interval: 10s + timeout: 10s + retries: 100 runtime: habana cap_add: - SYS_NICE ipc: host - command: --model-id ${LLM_MODEL_ID} --max-input-length 1024 --max-total-tokens 2048 + command: --model $LLM_MODEL_ID --tensor-parallel-size ${NUM_CARDS} --host 0.0.0.0 --port 80 --block-size ${BLOCK_SIZE} --max-num-seqs ${MAX_NUM_SEQS} --max-seq_len-to-capture ${MAX_SEQ_LEN_TO_CAPTURE} llm: image: ${REGISTRY:-opea}/llm-textgen:${TAG:-latest} - container_name: llm-textgen-gaudi-server + container_name: codetrans-xeon-llm-server depends_on: - tgi-service: + vllm-service: condition: service_healthy ports: - "9000:9000" @@ -43,18 +42,19 @@ services: no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} - LLM_ENDPOINT: ${TGI_LLM_ENDPOINT} + LLM_ENDPOINT: ${LLM_ENDPOINT} LLM_MODEL_ID: ${LLM_MODEL_ID} - HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + LLM_COMPONENT_NAME: ${LLM_COMPONENT_NAME} + HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} restart: unless-stopped codetrans-gaudi-backend-server: image: ${REGISTRY:-opea}/codetrans:${TAG:-latest} container_name: codetrans-gaudi-backend-server depends_on: - - tgi-service + - vllm-service - llm ports: - - "7777:7777" + - "${BACKEND_SERVICE_PORT:-7777}:7777" environment: - no_proxy=${no_proxy} - https_proxy=${https_proxy} @@ -69,7 +69,7 @@ services: depends_on: - codetrans-gaudi-backend-server ports: - - "5173:5173" + - "${FRONTEND_SERVICE_PORT:-5173}:5173" environment: - no_proxy=${no_proxy} - https_proxy=${https_proxy} diff --git a/CodeTrans/docker_compose/intel/hpu/gaudi/compose_tgi.yaml b/CodeTrans/docker_compose/intel/hpu/gaudi/compose_tgi.yaml new file mode 100644 index 0000000000..9bcc01f318 --- /dev/null +++ b/CodeTrans/docker_compose/intel/hpu/gaudi/compose_tgi.yaml @@ -0,0 +1,99 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +services: + tgi-service: + image: ghcr.io/huggingface/tgi-gaudi:2.0.6 + container_name: codetrans-gaudi-tgi-service + ports: + - "8008:80" + volumes: + - "${MODEL_CACHE}:/data" + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HUGGING_FACE_HUB_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + HABANA_VISIBLE_DEVICES: all + OMPI_MCA_btl_vader_single_copy_mechanism: none + ENABLE_HPU_GRAPH: true + LIMIT_HPU_GRAPH: true + USE_FLASH_ATTENTION: true + FLASH_ATTENTION_RECOMPUTE: true + runtime: habana + cap_add: + - SYS_NICE + ipc: host + command: --model-id ${LLM_MODEL_ID} --max-input-length 2048 --max-total-tokens 4096 + llm: + image: ${REGISTRY:-opea}/llm-textgen:${TAG:-latest} + container_name: codetrans-gaudi-llm-server + depends_on: + - tgi-service + ports: + - "9000:9000" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + LLM_ENDPOINT: ${LLM_ENDPOINT} + LLM_MODEL_ID: ${LLM_MODEL_ID} + LLM_COMPONENT_NAME: ${LLM_COMPONENT_NAME} + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + restart: unless-stopped + codetrans-gaudi-backend-server: + image: ${REGISTRY:-opea}/codetrans:${TAG:-latest} + container_name: codetrans-gaudi-backend-server + depends_on: + - tgi-service + - llm + ports: + - "${BACKEND_SERVICE_PORT:-7777}:7777" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP} + - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP} + ipc: host + restart: always + codetrans-gaudi-ui-server: + image: ${REGISTRY:-opea}/codetrans-ui:${TAG:-latest} + container_name: codetrans-gaudi-ui-server + depends_on: + - codetrans-gaudi-backend-server + ports: + - "${FRONTEND_SERVICE_PORT:-5173}:5173" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - BASE_URL=${BACKEND_SERVICE_ENDPOINT} + ipc: host + restart: always + codetrans-gaudi-nginx-server: + image: ${REGISTRY:-opea}/nginx:${TAG:-latest} + container_name: codetrans-gaudi-nginx-server + depends_on: + - codetrans-gaudi-backend-server + - codetrans-gaudi-ui-server + ports: + - "${NGINX_PORT:-80}:80" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - FRONTEND_SERVICE_IP=${FRONTEND_SERVICE_IP} + - FRONTEND_SERVICE_PORT=${FRONTEND_SERVICE_PORT} + - BACKEND_SERVICE_NAME=${BACKEND_SERVICE_NAME} + - BACKEND_SERVICE_IP=${BACKEND_SERVICE_IP} + - BACKEND_SERVICE_PORT=${BACKEND_SERVICE_PORT} + ipc: host + restart: always + +networks: + default: + driver: bridge diff --git a/CodeTrans/docker_compose/set_env.sh b/CodeTrans/docker_compose/set_env.sh index b44c763a2e..d24bc1c20b 100644 --- a/CodeTrans/docker_compose/set_env.sh +++ b/CodeTrans/docker_compose/set_env.sh @@ -8,7 +8,12 @@ popd > /dev/null export LLM_MODEL_ID="mistralai/Mistral-7B-Instruct-v0.3" -export TGI_LLM_ENDPOINT="http://${host_ip}:8008" +export LLM_ENDPOINT="http://${host_ip}:8008" +export LLM_COMPONENT_NAME="OpeaTextGenService" +export NUM_CARDS=1 +export BLOCK_SIZE=128 +export MAX_NUM_SEQS=256 +export MAX_SEQ_LEN_TO_CAPTURE=2048 export MEGA_SERVICE_HOST_IP=${host_ip} export LLM_SERVICE_HOST_IP=${host_ip} export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:7777/v1/codetrans" diff --git a/CodeTrans/docker_image_build/build.yaml b/CodeTrans/docker_image_build/build.yaml index bfc0070619..e42102170f 100644 --- a/CodeTrans/docker_image_build/build.yaml +++ b/CodeTrans/docker_image_build/build.yaml @@ -23,6 +23,18 @@ services: dockerfile: comps/llms/src/text-generation/Dockerfile extends: codetrans image: ${REGISTRY:-opea}/llm-textgen:${TAG:-latest} + vllm: + build: + context: vllm + dockerfile: Dockerfile.cpu + extends: codetrans + image: ${REGISTRY:-opea}/vllm:${TAG:-latest} + vllm-gaudi: + build: + context: vllm-fork + dockerfile: Dockerfile.hpu + extends: codetrans + image: ${REGISTRY:-opea}/vllm-gaudi:${TAG:-latest} nginx: build: context: GenAIComps diff --git a/CodeTrans/tests/test_compose_on_gaudi.sh b/CodeTrans/tests/test_compose_on_gaudi.sh index e2aedcd6e9..9c78ea5972 100644 --- a/CodeTrans/tests/test_compose_on_gaudi.sh +++ b/CodeTrans/tests/test_compose_on_gaudi.sh @@ -30,12 +30,12 @@ function build_docker_images() { cd $WORKPATH/docker_image_build git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + git clone --depth 1 --branch v0.6.4.post2+Gaudi-1.19.0 https://github.com/HabanaAI/vllm-fork.git echo "Build all the images with --no-cache, check docker_image_build.log for details..." - service_list="codetrans codetrans-ui llm-textgen nginx" + service_list="codetrans codetrans-ui llm-textgen vllm-gaudi nginx" docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log - docker pull ghcr.io/huggingface/tgi-gaudi:2.0.6 docker images && sleep 1s } @@ -45,7 +45,12 @@ function start_services() { export http_proxy=${http_proxy} export https_proxy=${http_proxy} export LLM_MODEL_ID="mistralai/Mistral-7B-Instruct-v0.3" - export TGI_LLM_ENDPOINT="http://${ip_address}:8008" + export LLM_ENDPOINT="http://${ip_address}:8008" + export LLM_COMPONENT_NAME="OpeaTextGenService" + export NUM_CARDS=1 + export BLOCK_SIZE=128 + export MAX_NUM_SEQS=256 + export MAX_SEQ_LEN_TO_CAPTURE=2048 export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} export MEGA_SERVICE_HOST_IP=${ip_address} export LLM_SERVICE_HOST_IP=${ip_address} @@ -65,13 +70,15 @@ function start_services() { n=0 until [[ "$n" -ge 100 ]]; do - docker logs codetrans-tgi-service > ${LOG_PATH}/tgi_service_start.log - if grep -q Connected ${LOG_PATH}/tgi_service_start.log; then + docker logs codetrans-gaudi-vllm-service > ${LOG_PATH}/vllm_service_start.log 2>&1 + if grep -q complete ${LOG_PATH}/vllm_service_start.log; then break fi sleep 5s n=$((n+1)) done + + sleep 1m } function validate_services() { @@ -103,27 +110,19 @@ function validate_services() { } function validate_microservices() { - # tgi for embedding service - validate_services \ - "${ip_address}:8008/generate" \ - "generated_text" \ - "tgi" \ - "codetrans-tgi-service" \ - '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' - # llm microservice validate_services \ "${ip_address}:9000/v1/chat/completions" \ "data: " \ "llm" \ - "llm-textgen-gaudi-server" \ + "codetrans-xeon-llm-server" \ '{"query":" ### System: Please translate the following Golang codes into Python codes. ### Original codes: '\'''\'''\''Golang \npackage main\n\nimport \"fmt\"\nfunc main() {\n fmt.Println(\"Hello, World!\");\n '\'''\'''\'' ### Translated codes:"}' } function validate_megaservice() { # Curl the Mega Service validate_services \ - "${ip_address}:7777/v1/codetrans" \ + "${ip_address}:${BACKEND_SERVICE_PORT}/v1/codetrans" \ "print" \ "mega-codetrans" \ "codetrans-gaudi-backend-server" \ @@ -131,7 +130,7 @@ function validate_megaservice() { # test the megeservice via nginx validate_services \ - "${ip_address}:80/v1/codetrans" \ + "${ip_address}:${NGINX_PORT}/v1/codetrans" \ "print" \ "mega-codetrans-nginx" \ "codetrans-gaudi-nginx-server" \ @@ -170,7 +169,7 @@ function validate_frontend() { function stop_docker() { cd $WORKPATH/docker_compose/intel/hpu/gaudi - docker compose stop && docker compose rm -f + docker compose -f compose.yaml stop && docker compose rm -f } function main() { diff --git a/CodeTrans/tests/test_compose_on_xeon.sh b/CodeTrans/tests/test_compose_on_xeon.sh index efa09fe0a5..23660848d5 100644 --- a/CodeTrans/tests/test_compose_on_xeon.sh +++ b/CodeTrans/tests/test_compose_on_xeon.sh @@ -30,12 +30,16 @@ function build_docker_images() { cd $WORKPATH/docker_image_build git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + git clone https://github.com/vllm-project/vllm.git && cd vllm + VLLM_VER="$(git describe --tags "$(git rev-list --tags --max-count=1)" )" + echo "Check out vLLM tag ${VLLM_VER}" + git checkout ${VLLM_VER} &> /dev/null + cd ../ echo "Build all the images with --no-cache, check docker_image_build.log for details..." - service_list="codetrans codetrans-ui llm-textgen nginx" + service_list="codetrans codetrans-ui llm-textgen vllm nginx" docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log - docker pull ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu docker images && sleep 1s } @@ -44,7 +48,8 @@ function start_services() { export http_proxy=${http_proxy} export https_proxy=${http_proxy} export LLM_MODEL_ID="mistralai/Mistral-7B-Instruct-v0.3" - export TGI_LLM_ENDPOINT="http://${ip_address}:8008" + export LLM_ENDPOINT="http://${ip_address}:8008" + export LLM_COMPONENT_NAME="OpeaTextGenService" export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} export MEGA_SERVICE_HOST_IP=${ip_address} export LLM_SERVICE_HOST_IP=${ip_address} @@ -60,17 +65,19 @@ function start_services() { sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env # Start Docker Containers - docker compose up -d > ${LOG_PATH}/start_services_with_compose.log + docker compose -f compose.yaml up -d > ${LOG_PATH}/start_services_with_compose.log n=0 until [[ "$n" -ge 100 ]]; do - docker logs codetrans-tgi-service > ${LOG_PATH}/tgi_service_start.log - if grep -q Connected ${LOG_PATH}/tgi_service_start.log; then + docker logs codetrans-xeon-vllm-service > ${LOG_PATH}/vllm_service_start.log 2>&1 + if grep -q complete ${LOG_PATH}/vllm_service_start.log; then break fi sleep 5s n=$((n+1)) done + + sleep 1m } function validate_services() { @@ -102,20 +109,12 @@ function validate_services() { } function validate_microservices() { - # tgi for embedding service - validate_services \ - "${ip_address}:8008/generate" \ - "generated_text" \ - "tgi" \ - "codetrans-tgi-service" \ - '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' - # llm microservice validate_services \ "${ip_address}:9000/v1/chat/completions" \ "data: " \ "llm" \ - "llm-textgen-server" \ + "codetrans-xeon-llm-server" \ '{"query":" ### System: Please translate the following Golang codes into Python codes. ### Original codes: '\'''\'''\''Golang \npackage main\n\nimport \"fmt\"\nfunc main() {\n fmt.Println(\"Hello, World!\");\n '\'''\'''\'' ### Translated codes:"}' } @@ -123,7 +122,7 @@ function validate_microservices() { function validate_megaservice() { # Curl the Mega Service validate_services \ - "${ip_address}:7777/v1/codetrans" \ + "${ip_address}:${BACKEND_SERVICE_PORT}/v1/codetrans" \ "print" \ "mega-codetrans" \ "codetrans-xeon-backend-server" \ @@ -131,7 +130,7 @@ function validate_megaservice() { # test the megeservice via nginx validate_services \ - "${ip_address}:80/v1/codetrans" \ + "${ip_address}:${NGINX_PORT}/v1/codetrans" \ "print" \ "mega-codetrans-nginx" \ "codetrans-xeon-nginx-server" \ @@ -169,7 +168,7 @@ function validate_frontend() { function stop_docker() { cd $WORKPATH/docker_compose/intel/cpu/xeon/ - docker compose stop && docker compose rm -f + docker compose -f compose.yaml stop && docker compose rm -f } function main() { diff --git a/CodeTrans/tests/test_compose_tgi_on_gaudi.sh b/CodeTrans/tests/test_compose_tgi_on_gaudi.sh new file mode 100644 index 0000000000..1c0404d397 --- /dev/null +++ b/CodeTrans/tests/test_compose_tgi_on_gaudi.sh @@ -0,0 +1,194 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" +export REGISTRY=${IMAGE_REPO} +export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') + +function build_docker_images() { + opea_branch=${opea_branch:-"main"} + # If the opea_branch isn't main, replace the git clone branch in Dockerfile. + if [[ "${opea_branch}" != "main" ]]; then + cd $WORKPATH + OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" + NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" + find . -type f -name "Dockerfile*" | while read -r file; do + echo "Processing file: $file" + sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" + done + fi + + cd $WORKPATH/docker_image_build + git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + + echo "Build all the images with --no-cache, check docker_image_build.log for details..." + service_list="codetrans codetrans-ui llm-textgen nginx" + docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log + + docker pull ghcr.io/huggingface/tgi-gaudi:2.0.6 + docker images && sleep 1s +} + +function start_services() { + cd $WORKPATH/docker_compose/intel/hpu/gaudi/ + export http_proxy=${http_proxy} + export https_proxy=${http_proxy} + export LLM_MODEL_ID="mistralai/Mistral-7B-Instruct-v0.3" + export LLM_ENDPOINT="http://${ip_address}:8008" + export LLM_COMPONENT_NAME="OpeaTextGenService" + export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} + export MEGA_SERVICE_HOST_IP=${ip_address} + export LLM_SERVICE_HOST_IP=${ip_address} + export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:7777/v1/codetrans" + export FRONTEND_SERVICE_IP=${ip_address} + export FRONTEND_SERVICE_PORT=5173 + export BACKEND_SERVICE_NAME=codetrans + export BACKEND_SERVICE_IP=${ip_address} + export BACKEND_SERVICE_PORT=7777 + export NGINX_PORT=80 + export host_ip=${ip_address} + + sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env + + # Start Docker Containers + docker compose -f compose_tgi.yaml up -d > ${LOG_PATH}/start_services_with_compose.log + + n=0 + until [[ "$n" -ge 100 ]]; do + docker logs codetrans-gaudi-tgi-service > ${LOG_PATH}/tgi_service_start.log + if grep -q Connected ${LOG_PATH}/tgi_service_start.log; then + break + fi + sleep 5s + n=$((n+1)) + done + + sleep 1m +} + +function validate_services() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 5s +} + +function validate_microservices() { + # tgi for embedding service + validate_services \ + "${ip_address}:8008/generate" \ + "generated_text" \ + "tgi" \ + "codetrans-gaudi-tgi-service" \ + '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' + + # llm microservice + validate_services \ + "${ip_address}:9000/v1/chat/completions" \ + "data: " \ + "llm" \ + "codetrans-gaudi-llm-server" \ + '{"query":" ### System: Please translate the following Golang codes into Python codes. ### Original codes: '\'''\'''\''Golang \npackage main\n\nimport \"fmt\"\nfunc main() {\n fmt.Println(\"Hello, World!\");\n '\'''\'''\'' ### Translated codes:"}' + +} + +function validate_megaservice() { + # Curl the Mega Service + validate_services \ + "${ip_address}:${BACKEND_SERVICE_PORT}/v1/codetrans" \ + "print" \ + "mega-codetrans" \ + "codetrans-gaudi-backend-server" \ + '{"language_from": "Golang","language_to": "Python","source_code": "package main\n\nimport \"fmt\"\nfunc main() {\n fmt.Println(\"Hello, World!\");\n}"}' + + # test the megeservice via nginx + validate_services \ + "${ip_address}:${NGINX_PORT}/v1/codetrans" \ + "print" \ + "mega-codetrans-nginx" \ + "codetrans-gaudi-nginx-server" \ + '{"language_from": "Golang","language_to": "Python","source_code": "package main\n\nimport \"fmt\"\nfunc main() {\n fmt.Println(\"Hello, World!\");\n}"}' + +} + +function validate_frontend() { + cd $WORKPATH/ui/svelte + local conda_env_name="OPEA_e2e" + export PATH=${HOME}/miniforge3/bin/:$PATH + if conda info --envs | grep -q "$conda_env_name"; then + echo "$conda_env_name exist!" + else + conda create -n ${conda_env_name} python=3.12 -y + fi + source activate ${conda_env_name} + + sed -i "s/localhost/$ip_address/g" playwright.config.ts + + conda install -c conda-forge nodejs=22.6.0 -y + npm install && npm ci && npx playwright install --with-deps + node -v && npm -v && pip list + + exit_status=0 + npx playwright test || exit_status=$? + + if [ $exit_status -ne 0 ]; then + echo "[TEST INFO]: ---------frontend test failed---------" + exit $exit_status + else + echo "[TEST INFO]: ---------frontend test passed---------" + fi +} + +function stop_docker() { + cd $WORKPATH/docker_compose/intel/hpu/gaudi/ + docker compose -f compose_tgi.yaml stop && docker compose rm -f +} + +function main() { + + stop_docker + + if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi + start_services + + validate_microservices + validate_megaservice + validate_frontend + + stop_docker + echo y | docker system prune + +} + +main diff --git a/CodeTrans/tests/test_compose_tgi_on_xeon.sh b/CodeTrans/tests/test_compose_tgi_on_xeon.sh new file mode 100644 index 0000000000..95154c7c9d --- /dev/null +++ b/CodeTrans/tests/test_compose_tgi_on_xeon.sh @@ -0,0 +1,194 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" +export REGISTRY=${IMAGE_REPO} +export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') + +function build_docker_images() { + opea_branch=${opea_branch:-"main"} + # If the opea_branch isn't main, replace the git clone branch in Dockerfile. + if [[ "${opea_branch}" != "main" ]]; then + cd $WORKPATH + OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" + NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" + find . -type f -name "Dockerfile*" | while read -r file; do + echo "Processing file: $file" + sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" + done + fi + + cd $WORKPATH/docker_image_build + git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + + echo "Build all the images with --no-cache, check docker_image_build.log for details..." + service_list="codetrans codetrans-ui llm-textgen nginx" + docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log + + docker pull ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu + docker images && sleep 1s +} + +function start_services() { + cd $WORKPATH/docker_compose/intel/cpu/xeon/ + export http_proxy=${http_proxy} + export https_proxy=${http_proxy} + export LLM_MODEL_ID="mistralai/Mistral-7B-Instruct-v0.3" + export LLM_ENDPOINT="http://${ip_address}:8008" + export LLM_COMPONENT_NAME="OpeaTextGenService" + export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} + export MEGA_SERVICE_HOST_IP=${ip_address} + export LLM_SERVICE_HOST_IP=${ip_address} + export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:7777/v1/codetrans" + export FRONTEND_SERVICE_IP=${ip_address} + export FRONTEND_SERVICE_PORT=5173 + export BACKEND_SERVICE_NAME=codetrans + export BACKEND_SERVICE_IP=${ip_address} + export BACKEND_SERVICE_PORT=7777 + export NGINX_PORT=80 + export host_ip=${ip_address} + + sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env + + # Start Docker Containers + docker compose -f compose_tgi.yaml up -d > ${LOG_PATH}/start_services_with_compose.log + + n=0 + until [[ "$n" -ge 100 ]]; do + docker logs codetrans-xeon-tgi-service > ${LOG_PATH}/tgi_service_start.log + if grep -q Connected ${LOG_PATH}/tgi_service_start.log; then + break + fi + sleep 5s + n=$((n+1)) + done + + sleep 1m +} + +function validate_services() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 5s +} + +function validate_microservices() { + # tgi for embedding service + validate_services \ + "${ip_address}:8008/generate" \ + "generated_text" \ + "tgi" \ + "codetrans-xeon-tgi-service" \ + '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' + + # llm microservice + validate_services \ + "${ip_address}:9000/v1/chat/completions" \ + "data: " \ + "llm" \ + "codetrans-xeon-llm-server" \ + '{"query":" ### System: Please translate the following Golang codes into Python codes. ### Original codes: '\'''\'''\''Golang \npackage main\n\nimport \"fmt\"\nfunc main() {\n fmt.Println(\"Hello, World!\");\n '\'''\'''\'' ### Translated codes:"}' + +} + +function validate_megaservice() { + # Curl the Mega Service + validate_services \ + "${ip_address}:${BACKEND_SERVICE_PORT}/v1/codetrans" \ + "print" \ + "mega-codetrans" \ + "codetrans-xeon-backend-server" \ + '{"language_from": "Golang","language_to": "Python","source_code": "package main\n\nimport \"fmt\"\nfunc main() {\n fmt.Println(\"Hello, World!\");\n}"}' + + # test the megeservice via nginx + validate_services \ + "${ip_address}:${NGINX_PORT}/v1/codetrans" \ + "print" \ + "mega-codetrans-nginx" \ + "codetrans-xeon-nginx-server" \ + '{"language_from": "Golang","language_to": "Python","source_code": "package main\n\nimport \"fmt\"\nfunc main() {\n fmt.Println(\"Hello, World!\");\n}"}' + +} + +function validate_frontend() { + cd $WORKPATH/ui/svelte + local conda_env_name="OPEA_e2e" + export PATH=${HOME}/miniforge3/bin/:$PATH + if conda info --envs | grep -q "$conda_env_name"; then + echo "$conda_env_name exist!" + else + conda create -n ${conda_env_name} python=3.12 -y + fi + source activate ${conda_env_name} + + sed -i "s/localhost/$ip_address/g" playwright.config.ts + + conda install -c conda-forge nodejs=22.6.0 -y + npm install && npm ci && npx playwright install --with-deps + node -v && npm -v && pip list + + exit_status=0 + npx playwright test || exit_status=$? + + if [ $exit_status -ne 0 ]; then + echo "[TEST INFO]: ---------frontend test failed---------" + exit $exit_status + else + echo "[TEST INFO]: ---------frontend test passed---------" + fi +} + +function stop_docker() { + cd $WORKPATH/docker_compose/intel/cpu/xeon/ + docker compose -f compose_tgi.yaml stop && docker compose rm -f +} + +function main() { + + stop_docker + + if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi + start_services + + validate_microservices + validate_megaservice + validate_frontend + + stop_docker + echo y | docker system prune + +} + +main From f6b63d1f26fde85e422a8c8d1b068ae1879d1d97 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Fri, 7 Mar 2025 11:00:48 +0800 Subject: [PATCH 047/226] Update model cache for AgentQnA (#1627) Signed-off-by: ZePan110 Signed-off-by: Chingis Yundunov --- AgentQnA/docker_compose/amd/gpu/rocm/compose.yaml | 2 +- AgentQnA/tests/step2_start_retrieval_tool.sh | 2 +- AgentQnA/tests/step4_launch_and_validate_agent_gaudi.sh | 2 +- AgentQnA/tests/step4a_launch_and_validate_agent_tgi_on_rocm.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/AgentQnA/docker_compose/amd/gpu/rocm/compose.yaml b/AgentQnA/docker_compose/amd/gpu/rocm/compose.yaml index c1864ff374..e264411aef 100644 --- a/AgentQnA/docker_compose/amd/gpu/rocm/compose.yaml +++ b/AgentQnA/docker_compose/amd/gpu/rocm/compose.yaml @@ -8,7 +8,7 @@ services: ports: - "${AGENTQNA_TGI_SERVICE_PORT-8085}:80" volumes: - - /var/opea/agent-service/:/data + - ${HF_CACHE_DIR:-/var/opea/agent-service/}:/data environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} diff --git a/AgentQnA/tests/step2_start_retrieval_tool.sh b/AgentQnA/tests/step2_start_retrieval_tool.sh index 91fb1ea0ae..c4b9a31fc7 100644 --- a/AgentQnA/tests/step2_start_retrieval_tool.sh +++ b/AgentQnA/tests/step2_start_retrieval_tool.sh @@ -9,7 +9,7 @@ echo "WORKDIR=${WORKDIR}" export ip_address=$(hostname -I | awk '{print $1}') export host_ip=${ip_address} -export HF_CACHE_DIR=$WORKDIR/hf_cache +export HF_CACHE_DIR=${model_cache:-"$WORKDIR/hf_cache"} if [ ! -d "$HF_CACHE_DIR" ]; then echo "Creating HF_CACHE directory" mkdir -p "$HF_CACHE_DIR" diff --git a/AgentQnA/tests/step4_launch_and_validate_agent_gaudi.sh b/AgentQnA/tests/step4_launch_and_validate_agent_gaudi.sh index 56f017239b..798f38526a 100644 --- a/AgentQnA/tests/step4_launch_and_validate_agent_gaudi.sh +++ b/AgentQnA/tests/step4_launch_and_validate_agent_gaudi.sh @@ -13,7 +13,7 @@ export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} HF_TOKEN=${HUGGINGFACEHUB_API_TOKEN} model="meta-llama/Llama-3.3-70B-Instruct" #"meta-llama/Meta-Llama-3.1-70B-Instruct" -export HF_CACHE_DIR=/data2/huggingface +export HF_CACHE_DIR=${model_cache:-"/data2/huggingface"} if [ ! -d "$HF_CACHE_DIR" ]; then HF_CACHE_DIR=$WORKDIR/hf_cache mkdir -p "$HF_CACHE_DIR" diff --git a/AgentQnA/tests/step4a_launch_and_validate_agent_tgi_on_rocm.sh b/AgentQnA/tests/step4a_launch_and_validate_agent_tgi_on_rocm.sh index 5b90aa41f2..0e3e8d1697 100644 --- a/AgentQnA/tests/step4a_launch_and_validate_agent_tgi_on_rocm.sh +++ b/AgentQnA/tests/step4a_launch_and_validate_agent_tgi_on_rocm.sh @@ -11,7 +11,7 @@ export ip_address=$(hostname -I | awk '{print $1}') export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/ export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} -export HF_CACHE_DIR=$WORKDIR/hf_cache +export HF_CACHE_DIR=${model_cache:-"$WORKDIR/hf_cache"} if [ ! -d "$HF_CACHE_DIR" ]; then mkdir -p "$HF_CACHE_DIR" fi From e836b36750839cd8a21cfcd5381a2815acab59d1 Mon Sep 17 00:00:00 2001 From: Eero Tamminen Date: Fri, 7 Mar 2025 07:13:29 +0200 Subject: [PATCH 048/226] Use GenAIComp base image to simplify Dockerfiles (#1612) Signed-off-by: Eero Tamminen Signed-off-by: Chingis Yundunov --- AudioQnA/Dockerfile | 44 ++--------------------------------- AudioQnA/Dockerfile.multilang | 44 ++--------------------------------- DocIndexRetriever/Dockerfile | 44 ++--------------------------------- EdgeCraftRAG/Dockerfile | 44 ++--------------------------------- FaqGen/Dockerfile | 44 ++--------------------------------- VideoQnA/Dockerfile | 44 ++--------------------------------- 6 files changed, 12 insertions(+), 252 deletions(-) diff --git a/AudioQnA/Dockerfile b/AudioQnA/Dockerfile index 07245de371..1294c218ca 100644 --- a/AudioQnA/Dockerfile +++ b/AudioQnA/Dockerfile @@ -1,48 +1,8 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -# Stage 1: base setup used by other stages -FROM python:3.11-slim AS base - -# get security updates -RUN apt-get update && apt-get upgrade -y && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -ENV HOME=/home/user - -RUN useradd -m -s /bin/bash user && \ - mkdir -p $HOME && \ - chown -R user $HOME - -WORKDIR $HOME - - -# Stage 2: latest GenAIComps sources -FROM base AS git - -RUN apt-get update && apt-get install -y --no-install-recommends git -RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git - - -# Stage 3: common layer shared by services using GenAIComps -FROM base AS comps-base - -# copy just relevant parts -COPY --from=git $HOME/GenAIComps/comps $HOME/GenAIComps/comps -COPY --from=git $HOME/GenAIComps/*.* $HOME/GenAIComps/LICENSE $HOME/GenAIComps/ - -WORKDIR $HOME/GenAIComps -RUN pip install --no-cache-dir --upgrade pip setuptools && \ - pip install --no-cache-dir -r $HOME/GenAIComps/requirements.txt -WORKDIR $HOME - -ENV PYTHONPATH=$PYTHONPATH:$HOME/GenAIComps - -USER user - - -# Stage 4: unique part -FROM comps-base +ARG BASE_TAG=latest +FROM opea/comps-base:$BASE_TAG COPY ./audioqna.py $HOME/audioqna.py diff --git a/AudioQnA/Dockerfile.multilang b/AudioQnA/Dockerfile.multilang index 1d0573d217..997e4bed37 100644 --- a/AudioQnA/Dockerfile.multilang +++ b/AudioQnA/Dockerfile.multilang @@ -1,48 +1,8 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -# Stage 1: base setup used by other stages -FROM python:3.11-slim AS base - -# get security updates -RUN apt-get update && apt-get upgrade -y && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -ENV HOME=/home/user - -RUN useradd -m -s /bin/bash user && \ - mkdir -p $HOME && \ - chown -R user $HOME - -WORKDIR $HOME - - -# Stage 2: latest GenAIComps sources -FROM base AS git - -RUN apt-get update && apt-get install -y --no-install-recommends git -RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git - - -# Stage 3: common layer shared by services using GenAIComps -FROM base AS comps-base - -# copy just relevant parts -COPY --from=git $HOME/GenAIComps/comps $HOME/GenAIComps/comps -COPY --from=git $HOME/GenAIComps/*.* $HOME/GenAIComps/LICENSE $HOME/GenAIComps/ - -WORKDIR $HOME/GenAIComps -RUN pip install --no-cache-dir --upgrade pip setuptools && \ - pip install --no-cache-dir -r $HOME/GenAIComps/requirements.txt -WORKDIR $HOME - -ENV PYTHONPATH=$PYTHONPATH:$HOME/GenAIComps - -USER user - - -# Stage 4: unique part -FROM comps-base +ARG BASE_TAG=latest +FROM opea/comps-base:$BASE_TAG COPY ./audioqna_multilang.py $HOME/audioqna_multilang.py diff --git a/DocIndexRetriever/Dockerfile b/DocIndexRetriever/Dockerfile index dcfd665f74..06fb1dc016 100644 --- a/DocIndexRetriever/Dockerfile +++ b/DocIndexRetriever/Dockerfile @@ -1,48 +1,8 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -# Stage 1: base setup used by other stages -FROM python:3.11-slim AS base - -# get security updates -RUN apt-get update && apt-get upgrade -y && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -ENV HOME=/home/user - -RUN useradd -m -s /bin/bash user && \ - mkdir -p $HOME && \ - chown -R user $HOME - -WORKDIR $HOME - - -# Stage 2: latest GenAIComps sources -FROM base AS git - -RUN apt-get update && apt-get install -y --no-install-recommends git -RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git - - -# Stage 3: common layer shared by services using GenAIComps -FROM base AS comps-base - -# copy just relevant parts -COPY --from=git $HOME/GenAIComps/comps $HOME/GenAIComps/comps -COPY --from=git $HOME/GenAIComps/*.* $HOME/GenAIComps/LICENSE $HOME/GenAIComps/ - -WORKDIR $HOME/GenAIComps -RUN pip install --no-cache-dir --upgrade pip setuptools && \ - pip install --no-cache-dir -r $HOME/GenAIComps/requirements.txt -WORKDIR $HOME - -ENV PYTHONPATH=$PYTHONPATH:$HOME/GenAIComps - -USER user - - -# Stage 4: unique part -FROM comps-base +ARG BASE_TAG=latest +FROM opea/comps-base:$BASE_TAG COPY ./retrieval_tool.py $HOME/retrieval_tool.py diff --git a/EdgeCraftRAG/Dockerfile b/EdgeCraftRAG/Dockerfile index fb7f5e14ec..fffb8d8970 100644 --- a/EdgeCraftRAG/Dockerfile +++ b/EdgeCraftRAG/Dockerfile @@ -1,48 +1,8 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -# Stage 1: base setup used by other stages -FROM python:3.11-slim AS base - -# get security updates -RUN apt-get update && apt-get upgrade -y && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -ENV HOME=/home/user - -RUN useradd -m -s /bin/bash user && \ - mkdir -p $HOME && \ - chown -R user $HOME - -WORKDIR $HOME - - -# Stage 2: latest GenAIComps sources -FROM base AS git - -RUN apt-get update && apt-get install -y --no-install-recommends git -RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git - - -# Stage 3: common layer shared by services using GenAIComps -FROM base AS comps-base - -# copy just relevant parts -COPY --from=git $HOME/GenAIComps/comps $HOME/GenAIComps/comps -COPY --from=git $HOME/GenAIComps/*.* $HOME/GenAIComps/LICENSE $HOME/GenAIComps/ - -WORKDIR $HOME/GenAIComps -RUN pip install --no-cache-dir --upgrade pip setuptools && \ - pip install --no-cache-dir -r $HOME/GenAIComps/requirements.txt -WORKDIR $HOME - -ENV PYTHONPATH=$PYTHONPATH:$HOME/GenAIComps - -USER user - - -# Stage 4: unique part -FROM comps-base +ARG BASE_TAG=latest +FROM opea/comps-base:$BASE_TAG COPY ./chatqna.py $HOME/chatqna.py diff --git a/FaqGen/Dockerfile b/FaqGen/Dockerfile index 2d1afd002a..d315bbb61b 100644 --- a/FaqGen/Dockerfile +++ b/FaqGen/Dockerfile @@ -1,48 +1,8 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -# Stage 1: base setup used by other stages -FROM python:3.11-slim AS base - -# get security updates -RUN apt-get update && apt-get upgrade -y && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -ENV HOME=/home/user - -RUN useradd -m -s /bin/bash user && \ - mkdir -p $HOME && \ - chown -R user $HOME - -WORKDIR $HOME - - -# Stage 2: latest GenAIComps sources -FROM base AS git - -RUN apt-get update && apt-get install -y --no-install-recommends git -RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git - - -# Stage 3: common layer shared by services using GenAIComps -FROM base AS comps-base - -# copy just relevant parts -COPY --from=git $HOME/GenAIComps/comps $HOME/GenAIComps/comps -COPY --from=git $HOME/GenAIComps/*.* $HOME/GenAIComps/LICENSE $HOME/GenAIComps/ - -WORKDIR $HOME/GenAIComps -RUN pip install --no-cache-dir --upgrade pip setuptools && \ - pip install --no-cache-dir -r $HOME/GenAIComps/requirements.txt -WORKDIR $HOME - -ENV PYTHONPATH=$PYTHONPATH:$HOME/GenAIComps - -USER user - - -# Stage 4: unique part -FROM comps-base +ARG BASE_TAG=latest +FROM opea/comps-base:$BASE_TAG COPY ./faqgen.py $HOME/faqgen.py diff --git a/VideoQnA/Dockerfile b/VideoQnA/Dockerfile index 0504a71881..2aade6088f 100644 --- a/VideoQnA/Dockerfile +++ b/VideoQnA/Dockerfile @@ -1,48 +1,8 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -# Stage 1: base setup used by other stages -FROM python:3.11-slim AS base - -# get security updates -RUN apt-get update && apt-get upgrade -y && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -ENV HOME=/home/user - -RUN useradd -m -s /bin/bash user && \ - mkdir -p $HOME && \ - chown -R user $HOME - -WORKDIR $HOME - - -# Stage 2: latest GenAIComps sources -FROM base AS git - -RUN apt-get update && apt-get install -y --no-install-recommends git -RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git - - -# Stage 3: common layer shared by services using GenAIComps -FROM base AS comps-base - -# copy just relevant parts -COPY --from=git $HOME/GenAIComps/comps $HOME/GenAIComps/comps -COPY --from=git $HOME/GenAIComps/*.* $HOME/GenAIComps/LICENSE $HOME/GenAIComps/ - -WORKDIR $HOME/GenAIComps -RUN pip install --no-cache-dir --upgrade pip setuptools && \ - pip install --no-cache-dir -r $HOME/GenAIComps/requirements.txt -WORKDIR $HOME - -ENV PYTHONPATH=$PYTHONPATH:$HOME/GenAIComps - -USER user - - -# Stage 4: unique part -FROM comps-base +ARG BASE_TAG=latest +FROM opea/comps-base:$BASE_TAG COPY ./videoqna.py $HOME/videoqna.py From 48a6a0a3459f041ac359ecef7fa7b192fb61cb72 Mon Sep 17 00:00:00 2001 From: Shifani Rajabose Date: Fri, 7 Mar 2025 01:31:34 -0500 Subject: [PATCH 049/226] [Bug: 112] Fix introduction in GenAIExamples main README (#1631) Signed-off-by: Chingis Yundunov --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 2db55575bd..369e504200 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,13 @@ ## Introduction -GenAIExamples are designed to give developers an easy entry into generative AI, featuring microservice-based samples that simplify the processes of deploying, testing, and scaling GenAI applications. All examples are fully compatible with Docker and Kubernetes, supporting a wide range of hardware platforms such as Gaudi, Xeon, and NVIDIA GPU, and other hardwares, ensuring flexibility and efficiency for your GenAI adoption. +GenAIExamples are designed to give developers an easy entry into generative AI, featuring microservice-based samples that simplify the processes of deploying, testing, and scaling GenAI applications. All examples are fully compatible with both Docker and Kubernetes, supporting a wide range of hardware platforms such as Gaudi, Xeon, NVIDIA GPUs, and other hardwares including AMD GPUs, ensuring flexibility and efficiency for your GenAI adoption. ## Architecture -[GenAIComps](https://github.com/opea-project/GenAIComps) is a service-based tool that includes microservice components such as llm, embedding, reranking, and so on. Using these components, various examples in GenAIExample can be constructed, including ChatQnA, DocSum, etc. +[GenAIComps](https://github.com/opea-project/GenAIComps) is a service-based tool that includes microservice components such as llm, embedding, reranking, and so on. Using these components, various examples in GenAIExample can be constructed including ChatQnA, DocSum, etc. -[GenAIInfra](https://github.com/opea-project/GenAIInfra), part of the OPEA containerization and cloud-native suite, enables quick and efficient deployment of GenAIExamples in the cloud. +[GenAIInfra](https://github.com/opea-project/GenAIInfra) is part of the OPEA containerization and cloud-native suite and enables quick and efficient deployment of GenAIExamples in the cloud. [GenAIEval](https://github.com/opea-project/GenAIEval) measures service performance metrics such as throughput, latency, and accuracy for GenAIExamples. This feature helps users compare performance across various hardware configurations easily. @@ -18,18 +18,18 @@ The GenAIExamples [documentation](https://opea-project.github.io/latest/examples ## Getting Started -GenAIExamples offers flexible deployment options that cater to different user needs, enabling efficient use and deployment in various environments. Here’s a brief overview of the three primary methods: Python startup, Docker Compose, and Kubernetes. +GenAIExamples offers flexible deployment options that cater to different user needs, enabling efficient use and deployment in various environments. Three primary methods are presently used to do this: Python startup, Docker Compose, and Kubernetes. Users can choose the most suitable approach based on ease of setup, scalability needs, and the environment in which they are operating. ### Deployment Guide -Deployment is based on released docker images by default, check [docker image list](./docker_images_list.md) for detailed information. You can also build your own images following instructions. +Deployment is based on released docker images by default - check [docker image list](./docker_images_list.md) for detailed information. You can also build your own images following instructions. #### Prerequisite -- For Docker Compose based deployment, you should have docker compose installed. Refer to [docker compose install](https://docs.docker.com/compose/install/). -- For Kubernetes based deployment, you can use [Helm](https://helm.sh) or [GMC](https://github.com/opea-project/GenAIInfra/tree/main/microservices-connector/README.md) based deployment. +- For Docker Compose-based deployment, you should have docker compose installed. Refer to [docker compose install](https://docs.docker.com/compose/install/) for more information. +- For Kubernetes-based deployment, you can use [Helm](https://helm.sh) or [GMC](https://github.com/opea-project/GenAIInfra/tree/main/microservices-connector/README.md)-based deployment. - You should have a kubernetes cluster ready for use. If not, you can refer to [k8s install](https://github.com/opea-project/docs/tree/main/guide/installation/k8s_install/README.md) to deploy one. - (Optional) You should have Helm (version >= 3.15) installed if you want to deploy with Helm Charts. Refer to the [Helm Installation Guide](https://helm.sh/docs/intro/install/) for more information. @@ -37,7 +37,7 @@ Deployment is based on released docker images by default, check [docker image li - Recommended Hardware Reference - Based on different deployment model size and performance requirement, you may choose different hardware platforms or cloud instances. Here are some reference platforms + Based on different deployment model sizes and performance requirements, you may choose different hardware platforms or cloud instances. Here are some of the reference platforms: | Use Case | Deployment model | Reference Configuration | Hardware access/instances | | -------- | ------------------------- | -------------------------------------------------------------------- | ---------------------------------------------------------------------------- | @@ -47,7 +47,7 @@ Deployment is based on released docker images by default, check [docker image li #### Deploy Examples -> **Note**: Check for [sample guides](https://opea-project.github.io/latest/examples/index.html) first for your use case. If it is not available, then refer to the table below. +> **Note**: Check for [sample guides](https://opea-project.github.io/latest/examples/index.html) first for your use case. If it is not available, then refer to the table below: | Use Case | Docker Compose
Deployment on Xeon | Docker Compose
Deployment on Gaudi | Docker Compose
Deployment on ROCm | Kubernetes with Helm Charts | Kubernetes with GMC | | ----------------- | ------------------------------------------------------------------------------ | ---------------------------------------------------------------------------- | ------------------------------------------------------------------------ | ------------------------------------------------------------------- | ------------------------------------------------------------ | From 48ee4c4b3a3fc57e76cb2c1c4bf73cb2120443fb Mon Sep 17 00:00:00 2001 From: "chen, suyue" Date: Fri, 7 Mar 2025 15:05:08 +0800 Subject: [PATCH 050/226] Fix corner CI issue when the example path deleted (#1634) Signed-off-by: chensuyue Signed-off-by: Chingis Yundunov --- .github/workflows/scripts/get_test_matrix.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/scripts/get_test_matrix.sh b/.github/workflows/scripts/get_test_matrix.sh index 2d6efddd24..5ad6992104 100644 --- a/.github/workflows/scripts/get_test_matrix.sh +++ b/.github/workflows/scripts/get_test_matrix.sh @@ -12,6 +12,7 @@ run_matrix="{\"include\":[" examples=$(printf '%s\n' "${changed_files[@]}" | grep '/' | cut -d'/' -f1 | sort -u) for example in ${examples}; do + if [[ ! -d $WORKSPACE/$example ]]; then continue; fi cd $WORKSPACE/$example if [[ ! $(find . -type f | grep ${test_mode}) ]]; then continue; fi cd tests From 7b7824770c793e237fda7b34f3e13f6e3ba784de Mon Sep 17 00:00:00 2001 From: wangleflex <106506636+wangleflex@users.noreply.github.com> Date: Fri, 7 Mar 2025 17:08:53 +0800 Subject: [PATCH 051/226] [ChatQnA] Show spinner after query to improve user experience (#1003) (#1628) Signed-off-by: Wang,Le3 Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: Chingis Yundunov --- .../shared/components/loading/Spinner.svelte | 68 +++++++++++++++++++ ChatQnA/ui/svelte/src/routes/+page.svelte | 11 ++- 2 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 ChatQnA/ui/svelte/src/lib/shared/components/loading/Spinner.svelte diff --git a/ChatQnA/ui/svelte/src/lib/shared/components/loading/Spinner.svelte b/ChatQnA/ui/svelte/src/lib/shared/components/loading/Spinner.svelte new file mode 100644 index 0000000000..1b0a086ad1 --- /dev/null +++ b/ChatQnA/ui/svelte/src/lib/shared/components/loading/Spinner.svelte @@ -0,0 +1,68 @@ + + + + + + + diff --git a/ChatQnA/ui/svelte/src/routes/+page.svelte b/ChatQnA/ui/svelte/src/routes/+page.svelte index b6f6d9c334..bcd0b8b708 100644 --- a/ChatQnA/ui/svelte/src/routes/+page.svelte +++ b/ChatQnA/ui/svelte/src/routes/+page.svelte @@ -39,6 +39,8 @@ import ChatMessage from "$lib/modules/chat/ChatMessage.svelte"; import { fetchAllFile } from "$lib/network/upload/Network.js"; import { getNotificationsContext } from "svelte-notifications"; + import Spinner from "$lib/shared/components/loading/Spinner.svelte"; + let query: string = ""; let loading: boolean = false; @@ -241,8 +243,13 @@ type="submit" id="send" class="absolute bottom-2.5 end-2.5 px-4 py-2 text-sm font-medium text-white dark:bg-blue-600 dark:hover:bg-blue-700 dark:focus:ring-blue-800" - > + > + {#if loading} + + {:else} + + {/if} +
From 6fbe02dc242f213cf123bbb04f213c3a1805d607 Mon Sep 17 00:00:00 2001 From: "chen, suyue" Date: Fri, 7 Mar 2025 20:40:32 +0800 Subject: [PATCH 052/226] Use the latest HabanaAI/vllm-fork release tag to build vllm-gaudi image (#1635) Signed-off-by: chensuyue Co-authored-by: Liang Lv Signed-off-by: Chingis Yundunov --- .github/workflows/_example-workflow.yml | 14 ++++++++------ AgentQnA/tests/step1_build_images.sh | 3 ++- ChatQnA/tests/test_compose_guardrails_on_gaudi.sh | 4 +++- ChatQnA/tests/test_compose_on_gaudi.sh | 4 +++- .../tests/test_compose_without_rerank_on_gaudi.sh | 4 +++- CodeTrans/tests/test_compose_on_gaudi.sh | 4 +++- 6 files changed, 22 insertions(+), 11 deletions(-) diff --git a/.github/workflows/_example-workflow.yml b/.github/workflows/_example-workflow.yml index 010eece64a..f3b717a284 100644 --- a/.github/workflows/_example-workflow.yml +++ b/.github/workflows/_example-workflow.yml @@ -78,16 +78,18 @@ jobs: cd ${{ github.workspace }}/${{ inputs.example }}/docker_image_build docker_compose_path=${{ github.workspace }}/${{ inputs.example }}/docker_image_build/build.yaml if [[ $(grep -c "vllm:" ${docker_compose_path}) != 0 ]]; then - git clone https://github.com/vllm-project/vllm.git && cd vllm + git clone https://github.com/vllm-project/vllm.git && cd vllm # Get the latest tag - VLLM_VER="$(git describe --tags "$(git rev-list --tags --max-count=1)" )" + VLLM_VER=$(git describe --tags "$(git rev-list --tags --max-count=1)") echo "Check out vLLM tag ${VLLM_VER}" - git checkout ${VLLM_VER} &> /dev/null - # make sure do not change the pwd - git rev-parse HEAD && cd ../ + git checkout ${VLLM_VER} &> /dev/null && cd ../ fi if [[ $(grep -c "vllm-gaudi:" ${docker_compose_path}) != 0 ]]; then - git clone --depth 1 --branch v0.6.4.post2+Gaudi-1.19.0 https://github.com/HabanaAI/vllm-fork.git + git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork + # Get the latest tag + VLLM_VER=$(git describe --tags "$(git rev-list --tags --max-count=1)") + echo "Check out vLLM tag ${VLLM_VER}" + git checkout ${VLLM_VER} &> /dev/null && cd ../ fi git clone --depth 1 --branch ${{ inputs.opea_branch }} https://github.com/opea-project/GenAIComps.git cd GenAIComps && git rev-parse HEAD && cd ../ diff --git a/AgentQnA/tests/step1_build_images.sh b/AgentQnA/tests/step1_build_images.sh index 4cb8a2e4d1..aa83521448 100644 --- a/AgentQnA/tests/step1_build_images.sh +++ b/AgentQnA/tests/step1_build_images.sh @@ -42,7 +42,8 @@ function build_vllm_docker_image() { git clone https://github.com/HabanaAI/vllm-fork.git fi cd ./vllm-fork - git checkout v0.6.4.post2+Gaudi-1.19.0 + VLLM_VER=$(git describe --tags "$(git rev-list --tags --max-count=1)") + git checkout ${VLLM_VER} &> /dev/null docker build --no-cache -f Dockerfile.hpu -t opea/vllm-gaudi:ci --shm-size=128g . --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy if [ $? -ne 0 ]; then echo "opea/vllm-gaudi:ci failed" diff --git a/ChatQnA/tests/test_compose_guardrails_on_gaudi.sh b/ChatQnA/tests/test_compose_guardrails_on_gaudi.sh index d667a89f3c..c882a7ef77 100644 --- a/ChatQnA/tests/test_compose_guardrails_on_gaudi.sh +++ b/ChatQnA/tests/test_compose_guardrails_on_gaudi.sh @@ -29,7 +29,9 @@ function build_docker_images() { fi cd $WORKPATH/docker_image_build git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git - git clone --depth 1 --branch v0.6.4.post2+Gaudi-1.19.0 https://github.com/HabanaAI/vllm-fork.git + git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork + VLLM_VER=$(git describe --tags "$(git rev-list --tags --max-count=1)") + git checkout ${VLLM_VER} &> /dev/null && cd ../ echo "Build all the images with --no-cache, check docker_image_build.log for details..." service_list="chatqna-guardrails chatqna-ui dataprep retriever vllm-gaudi guardrails nginx" diff --git a/ChatQnA/tests/test_compose_on_gaudi.sh b/ChatQnA/tests/test_compose_on_gaudi.sh index 8858900148..7f64e3b0d6 100644 --- a/ChatQnA/tests/test_compose_on_gaudi.sh +++ b/ChatQnA/tests/test_compose_on_gaudi.sh @@ -29,7 +29,9 @@ function build_docker_images() { fi cd $WORKPATH/docker_image_build git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git - git clone --depth 1 --branch v0.6.4.post2+Gaudi-1.19.0 https://github.com/HabanaAI/vllm-fork.git + git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork + VLLM_VER=$(git describe --tags "$(git rev-list --tags --max-count=1)") + git checkout ${VLLM_VER} &> /dev/null && cd ../ echo "Build all the images with --no-cache, check docker_image_build.log for details..." service_list="chatqna chatqna-ui dataprep retriever vllm-gaudi nginx" diff --git a/ChatQnA/tests/test_compose_without_rerank_on_gaudi.sh b/ChatQnA/tests/test_compose_without_rerank_on_gaudi.sh index 9e9d7df735..c9dc86a0bd 100644 --- a/ChatQnA/tests/test_compose_without_rerank_on_gaudi.sh +++ b/ChatQnA/tests/test_compose_without_rerank_on_gaudi.sh @@ -29,7 +29,9 @@ function build_docker_images() { fi cd $WORKPATH/docker_image_build git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git - git clone --depth 1 --branch v0.6.4.post2+Gaudi-1.19.0 https://github.com/HabanaAI/vllm-fork.git + git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork + VLLM_VER=$(git describe --tags "$(git rev-list --tags --max-count=1)") + git checkout ${VLLM_VER} &> /dev/null && cd ../ echo "Build all the images with --no-cache, check docker_image_build.log for details..." service_list="chatqna-without-rerank chatqna-ui dataprep retriever vllm-gaudi nginx" diff --git a/CodeTrans/tests/test_compose_on_gaudi.sh b/CodeTrans/tests/test_compose_on_gaudi.sh index 9c78ea5972..39bf472521 100644 --- a/CodeTrans/tests/test_compose_on_gaudi.sh +++ b/CodeTrans/tests/test_compose_on_gaudi.sh @@ -30,7 +30,9 @@ function build_docker_images() { cd $WORKPATH/docker_image_build git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git - git clone --depth 1 --branch v0.6.4.post2+Gaudi-1.19.0 https://github.com/HabanaAI/vllm-fork.git + git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork + VLLM_VER=$(git describe --tags "$(git rev-list --tags --max-count=1)") + git checkout ${VLLM_VER} &> /dev/null && cd ../ echo "Build all the images with --no-cache, check docker_image_build.log for details..." service_list="codetrans codetrans-ui llm-textgen vllm-gaudi nginx" From 06dab2106ce1cae6f041561c1e55ef15dabf5b85 Mon Sep 17 00:00:00 2001 From: XinyaoWa Date: Mon, 10 Mar 2025 09:39:35 +0800 Subject: [PATCH 053/226] Set vLLM as default model for FaqGen (#1580) Signed-off-by: Xinyao Wang Signed-off-by: Chingis Yundunov --- .../docker_compose/intel/cpu/xeon/README.md | 92 +++++---- .../intel/cpu/xeon/compose.yaml | 21 +- .../intel/cpu/xeon/compose_tgi.yaml | 78 +++++++ .../docker_compose/intel/hpu/gaudi/README.md | 66 +++--- .../intel/hpu/gaudi/compose.yaml | 34 ++-- .../intel/hpu/gaudi/compose_tgi.yaml | 94 +++++++++ FaqGen/tests/test_compose_on_gaudi.sh | 44 ++-- FaqGen/tests/test_compose_on_xeon.sh | 43 ++-- FaqGen/tests/test_compose_tgi_on_gaudi.sh | 192 ++++++++++++++++++ FaqGen/tests/test_compose_tgi_on_xeon.sh | 190 +++++++++++++++++ 10 files changed, 726 insertions(+), 128 deletions(-) create mode 100644 FaqGen/docker_compose/intel/cpu/xeon/compose_tgi.yaml create mode 100644 FaqGen/docker_compose/intel/hpu/gaudi/compose_tgi.yaml create mode 100644 FaqGen/tests/test_compose_tgi_on_gaudi.sh create mode 100755 FaqGen/tests/test_compose_tgi_on_xeon.sh diff --git a/FaqGen/docker_compose/intel/cpu/xeon/README.md b/FaqGen/docker_compose/intel/cpu/xeon/README.md index a961a6aa98..576c5724ec 100644 --- a/FaqGen/docker_compose/intel/cpu/xeon/README.md +++ b/FaqGen/docker_compose/intel/cpu/xeon/README.md @@ -14,7 +14,17 @@ After launching your instance, you can connect to it using SSH (for Linux instan First of all, you need to build Docker Images locally. This step can be ignored once the Docker images are published to Docker hub. -### 1. Build LLM Image +### 1. Build vLLM Image + +```bash +git clone https://github.com/vllm-project/vllm.git +cd ./vllm/ +VLLM_VER="$(git describe --tags "$(git rev-list --tags --max-count=1)" )" +git checkout ${VLLM_VER} +docker build --no-cache --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile.cpu -t opea/vllm:latest --shm-size=128g . +``` + +### 2. Build LLM Image ```bash git clone https://github.com/opea-project/GenAIComps.git @@ -22,7 +32,7 @@ cd GenAIComps docker build -t opea/llm-faqgen:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/faq-generation/Dockerfile . ``` -### 2. Build MegaService Docker Image +### 3. Build MegaService Docker Image To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `faqgen.py` Python script. Build the MegaService Docker image via below command: @@ -32,7 +42,7 @@ cd GenAIExamples/FaqGen/ docker build --no-cache -t opea/faqgen:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f GenAIExamples/FaqGen/Dockerfile . ``` -### 3. Build UI Docker Image +### 4. Build UI Docker Image Build the frontend Docker image via below command: @@ -41,7 +51,7 @@ cd GenAIExamples/FaqGen/ui docker build -t opea/faqgen-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . ``` -### 4. Build react UI Docker Image (Optional) +### 5. Build react UI Docker Image (Optional) Build the frontend Docker image based on react framework via below command: @@ -53,10 +63,11 @@ docker build -t opea/faqgen-react-ui:latest --build-arg https_proxy=$https_proxy Then run the command `docker images`, you will have the following Docker Images: -1. `opea/llm-faqgen:latest` -2. `opea/faqgen:latest` -3. `opea/faqgen-ui:latest` -4. `opea/faqgen-react-ui:latest` +1. `opea/vllm:latest` +2. `opea/llm-faqgen:latest` +3. `opea/faqgen:latest` +4. `opea/faqgen-ui:latest` +5. `opea/faqgen-react-ui:latest` ## 🚀 Start Microservices and MegaService @@ -77,7 +88,8 @@ export https_proxy=${your_http_proxy} export host_ip=${your_host_ip} export LLM_ENDPOINT_PORT=8008 export LLM_SERVICE_PORT=9000 -export FAQGen_COMPONENT_NAME="OpeaFaqGenTgi" +export FAQGEN_BACKEND_PORT=8888 +export FAQGen_COMPONENT_NAME="OpeaFaqGenvLLM" export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct" export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} export MEGA_SERVICE_HOST_IP=${host_ip} @@ -97,44 +109,44 @@ docker compose up -d ### Validate Microservices -1. TGI Service +1. vLLM Service - ```bash - curl http://${host_ip}:8008/generate \ - -X POST \ - -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \ - -H 'Content-Type: application/json' - ``` +```bash +curl http://${host_ip}:${LLM_ENDPOINT_PORT}/v1/chat/completions \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "user", "content": "What is Deep Learning?"}]}' +``` 2. LLM Microservice - ```bash - curl http://${host_ip}:9000/v1/faqgen \ - -X POST \ - -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \ - -H 'Content-Type: application/json' - ``` +```bash +curl http://${host_ip}:${LLM_SERVICE_PORT}/v1/faqgen \ + -X POST \ + -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \ + -H 'Content-Type: application/json' +``` 3. MegaService - ```bash - curl http://${host_ip}:8888/v1/faqgen \ - -H "Content-Type: multipart/form-data" \ - -F "messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." \ - -F "max_tokens=32" \ - -F "stream=False" - ``` - - ```bash - ## enable stream - curl http://${host_ip}:8888/v1/faqgen \ - -H "Content-Type: multipart/form-data" \ - -F "messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." \ - -F "max_tokens=32" \ - -F "stream=True" - ``` - - Following the validation of all aforementioned microservices, we are now prepared to construct a mega-service. +```bash +curl http://${host_ip}:${FAQGEN_BACKEND_PORT}/v1/faqgen \ + -H "Content-Type: multipart/form-data" \ + -F "messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." \ + -F "max_tokens=32" \ + -F "stream=False" +``` + +```bash +## enable stream +curl http://${host_ip}:${FAQGEN_BACKEND_PORT}/v1/faqgen \ + -H "Content-Type: multipart/form-data" \ + -F "messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." \ + -F "max_tokens=32" \ + -F "stream=True" +``` + +Following the validation of all aforementioned microservices, we are now prepared to construct a mega-service. ## 🚀 Launch the UI diff --git a/FaqGen/docker_compose/intel/cpu/xeon/compose.yaml b/FaqGen/docker_compose/intel/cpu/xeon/compose.yaml index ca86a18f2d..7da122f9ab 100644 --- a/FaqGen/docker_compose/intel/cpu/xeon/compose.yaml +++ b/FaqGen/docker_compose/intel/cpu/xeon/compose.yaml @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 services: - tgi-service: - image: ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu - container_name: tgi-xeon-server + vllm-service: + image: ${REGISTRY:-opea}/vllm:${TAG:-latest} + container_name: vllm-service ports: - ${LLM_ENDPOINT_PORT:-8008}:80 volumes: @@ -14,20 +14,23 @@ services: no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} - HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + HF_TOKEN: ${HF_TOKEN} + LLM_MODEL_ID: ${LLM_MODEL_ID} + VLLM_TORCH_PROFILER_DIR: "${VLLM_TORCH_PROFILER_DIR:-/mnt}" host_ip: ${host_ip} LLM_ENDPOINT_PORT: ${LLM_ENDPOINT_PORT} + VLLM_SKIP_WARMUP: ${VLLM_SKIP_WARMUP:-false} healthcheck: test: ["CMD-SHELL", "curl -f http://${host_ip}:${LLM_ENDPOINT_PORT}/health || exit 1"] interval: 10s timeout: 10s retries: 100 - command: --model-id ${LLM_MODEL_ID} --cuda-graphs 0 + command: --model $LLM_MODEL_ID --host 0.0.0.0 --port 80 llm_faqgen: image: ${REGISTRY:-opea}/llm-faqgen:${TAG:-latest} container_name: llm-faqgen-server depends_on: - tgi-service: + vllm-service: condition: service_healthy ports: - ${LLM_SERVICE_PORT:-9000}:9000 @@ -39,17 +42,17 @@ services: LLM_ENDPOINT: ${LLM_ENDPOINT} LLM_MODEL_ID: ${LLM_MODEL_ID} HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} - FAQGen_COMPONENT_NAME: ${FAQGen_COMPONENT_NAME} + FAQGen_COMPONENT_NAME: ${FAQGen_COMPONENT_NAME:-OpeaFaqGenvLLM} LOGFLAG: ${LOGFLAG:-False} restart: unless-stopped faqgen-xeon-backend-server: image: ${REGISTRY:-opea}/faqgen:${TAG:-latest} container_name: faqgen-xeon-backend-server depends_on: - - tgi-service + - vllm-service - llm_faqgen ports: - - "8888:8888" + - ${FAQGEN_BACKEND_PORT:-8888}:8888 environment: - no_proxy=${no_proxy} - https_proxy=${https_proxy} diff --git a/FaqGen/docker_compose/intel/cpu/xeon/compose_tgi.yaml b/FaqGen/docker_compose/intel/cpu/xeon/compose_tgi.yaml new file mode 100644 index 0000000000..b900331ad8 --- /dev/null +++ b/FaqGen/docker_compose/intel/cpu/xeon/compose_tgi.yaml @@ -0,0 +1,78 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +services: + tgi-service: + image: ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu + container_name: tgi-xeon-server + ports: + - ${LLM_ENDPOINT_PORT:-8008}:80 + volumes: + - "${MODEL_CACHE:-./data}:/data" + shm_size: 1g + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + host_ip: ${host_ip} + LLM_ENDPOINT_PORT: ${LLM_ENDPOINT_PORT} + healthcheck: + test: ["CMD-SHELL", "curl -f http://${host_ip}:${LLM_ENDPOINT_PORT}/health || exit 1"] + interval: 10s + timeout: 10s + retries: 100 + command: --model-id ${LLM_MODEL_ID} --cuda-graphs 0 + llm_faqgen: + image: ${REGISTRY:-opea}/llm-faqgen:${TAG:-latest} + container_name: llm-faqgen-server + depends_on: + tgi-service: + condition: service_healthy + ports: + - ${LLM_SERVICE_PORT:-9000}:9000 + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + LLM_ENDPOINT: ${LLM_ENDPOINT} + LLM_MODEL_ID: ${LLM_MODEL_ID} + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + FAQGen_COMPONENT_NAME: ${FAQGen_COMPONENT_NAME:-OpeaFaqGenTgi} + LOGFLAG: ${LOGFLAG:-False} + restart: unless-stopped + faqgen-xeon-backend-server: + image: ${REGISTRY:-opea}/faqgen:${TAG:-latest} + container_name: faqgen-xeon-backend-server + depends_on: + - tgi-service + - llm_faqgen + ports: + - ${FAQGEN_BACKEND_PORT:-8888}:8888 + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP} + - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP} + - LLM_SERVICE_PORT=${LLM_SERVICE_PORT} + ipc: host + restart: always + faqgen-xeon-ui-server: + image: ${REGISTRY:-opea}/faqgen-ui:${TAG:-latest} + container_name: faqgen-xeon-ui-server + depends_on: + - faqgen-xeon-backend-server + ports: + - "5173:5173" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - FAQ_BASE_URL=${BACKEND_SERVICE_ENDPOINT} + ipc: host + restart: always +networks: + default: + driver: bridge diff --git a/FaqGen/docker_compose/intel/hpu/gaudi/README.md b/FaqGen/docker_compose/intel/hpu/gaudi/README.md index 7364e92387..4f8793d4ac 100644 --- a/FaqGen/docker_compose/intel/hpu/gaudi/README.md +++ b/FaqGen/docker_compose/intel/hpu/gaudi/README.md @@ -6,7 +6,7 @@ This document outlines the deployment process for a FAQ Generation application u 1. Set up the environment variables. 2. Run Docker Compose. -3. Consume the ChatQnA Service. +3. Consume the FaqGen Service. ### Quick Start: 1.Setup Environment Variable @@ -32,12 +32,14 @@ To set up environment variables for deploying ChatQnA services, follow these ste 3. Set up other environment variables: ```bash - export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct" - export TGI_LLM_ENDPOINT="http://${host_ip}:8008" export MEGA_SERVICE_HOST_IP=${host_ip} export LLM_SERVICE_HOST_IP=${host_ip} + export LLM_ENDPOINT_PORT=8008 export LLM_SERVICE_PORT=9000 - export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/faqgen" + export FAQGEN_BACKEND_PORT=8888 + export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct" + export vLLM_LLM_ENDPOINT="http://${host_ip}:${LLM_ENDPOINT_PORT}" + export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:${FAQGEN_BACKEND_PORT}/v1/faqgen" ``` ### Quick Start: 2.Run Docker Compose @@ -50,7 +52,7 @@ It will automatically download the docker image on `docker hub`, please check th ```bash docker ps -a - docker logs tgi-gaudi-server -t + docker logs vllm-gaudi-service -t ``` it may take some time to download the model. @@ -65,32 +67,33 @@ Please refer to 'Build Docker Images' in below. ### QuickStart: 3.Consume the Service ```bash -curl localhost:8008/generate \ - -X POST \ - -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \ - -H 'Content-Type: application/json' +curl http://localhost:${LLM_ENDPOINT_PORT}/v1/chat/completions \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "user", "content": "What is Deep Learning?"}]}' ``` here we just test the service on the host machine for a quick start. If all networks work fine, please try ```bash - curl http://${host_ip}:8008/generate \ - -X POST \ - -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \ - -H 'Content-Type: application/json' +curl http://${host_ip}:${LLM_ENDPOINT_PORT}/v1/chat/completions \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "user", "content": "What is Deep Learning?"}]}' ``` ## 🚀 Build Docker Images First of all, you need to build Docker Images locally. This step can be ignored once the Docker images are published to Docker hub. -### 1. Pull TGI Gaudi Image - -As TGI Gaudi has been officially published as a Docker image, we simply need to pull it: +### 1. Build vLLM Image ```bash -docker pull ghcr.io/huggingface/tgi-gaudi:2.0.6 +git clone https://github.com/HabanaAI/vllm-fork.git +cd vllm-fork/ +git checkout v0.6.4.post2+Gaudi-1.19.0 +docker build --no-cache --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile.hpu -t opea/vllm-gaudi:latest --shm-size=128g . ``` ### 2. Build LLM Image @@ -126,13 +129,13 @@ Build the frontend Docker image based on react framework via below command: ```bash cd GenAIExamples/FaqGen/ui -export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/faqgen" +export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:${FAQGEN_BACKEND_PORT}/v1/faqgen" docker build -t opea/faqgen-react-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy --build-arg BACKEND_SERVICE_ENDPOINT=$BACKEND_SERVICE_ENDPOINT -f docker/Dockerfile.react . ``` Then run the command `docker images`, you will have the following Docker Images: -1. `ghcr.io/huggingface/tgi-gaudi:2.0.6` +1. `opea/vllm-gaudi:latest` 2. `opea/llm-faqgen:latest` 3. `opea/faqgen:latest` 4. `opea/faqgen-ui:latest` @@ -157,13 +160,14 @@ export https_proxy=${your_http_proxy} export host_ip=${your_host_ip} export LLM_ENDPOINT_PORT=8008 export LLM_SERVICE_PORT=9000 -export FAQGen_COMPONENT_NAME="OpeaFaqGenTgi" +export FAQGEN_BACKEND_PORT=8888 +export FAQGen_COMPONENT_NAME="OpeaFaqGenvLLM" export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct" export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} export MEGA_SERVICE_HOST_IP=${host_ip} export LLM_SERVICE_HOST_IP=${host_ip} export LLM_ENDPOINT="http://${host_ip}:${LLM_ENDPOINT_PORT}" -export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/faqgen" +export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:${FAQGEN_BACKEND_PORT}/v1/faqgen" ``` Note: Please replace with `host_ip` with your external IP address, do not use localhost. @@ -177,19 +181,19 @@ docker compose up -d ### Validate Microservices -1. TGI Service +1.vLLM Service - ```bash - curl http://${host_ip}:8008/generate \ - -X POST \ - -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":64, "do_sample": true}}' \ - -H 'Content-Type: application/json' - ``` + ```bash + curl http://${host_ip}:${LLM_ENDPOINT_PORT}/v1/chat/completions \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "user", "content": "What is Deep Learning?"}]}' + ``` 2. LLM Microservice ```bash - curl http://${host_ip}:9000/v1/faqgen \ + curl http://${host_ip}:${LLM_SERVICE_PORT}/v1/faqgen \ -X POST \ -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \ -H 'Content-Type: application/json' @@ -198,7 +202,7 @@ docker compose up -d 3. MegaService ```bash - curl http://${host_ip}:8888/v1/faqgen \ + curl http://${host_ip}:${FAQGEN_BACKEND_PORT}/v1/faqgen \ -H "Content-Type: multipart/form-data" \ -F "messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." \ -F "max_tokens=32" \ @@ -207,7 +211,7 @@ docker compose up -d ```bash ##enable stream - curl http://${host_ip}:8888/v1/faqgen \ + curl http://${host_ip}:${FAQGEN_BACKEND_PORT}/v1/faqgen \ -H "Content-Type: multipart/form-data" \ -F "messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." \ -F "max_tokens=32" \ diff --git a/FaqGen/docker_compose/intel/hpu/gaudi/compose.yaml b/FaqGen/docker_compose/intel/hpu/gaudi/compose.yaml index 90503069c1..fbc8812d58 100644 --- a/FaqGen/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/FaqGen/docker_compose/intel/hpu/gaudi/compose.yaml @@ -2,30 +2,26 @@ # SPDX-License-Identifier: Apache-2.0 services: - tgi-service: - image: ghcr.io/huggingface/tgi-gaudi:2.3.1 - container_name: tgi-gaudi-server + vllm-gaudi-service: + image: ${REGISTRY:-opea}/vllm-gaudi:${TAG:-latest} + container_name: vllm-gaudi-service ports: - ${LLM_ENDPOINT_PORT:-8008}:80 volumes: - - "${DATA_PATH:-./data}:/data" + - "${MODEL_CACHE:-./data}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} - HUGGING_FACE_HUB_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} - HF_HUB_DISABLE_PROGRESS_BARS: 1 - HF_HUB_ENABLE_HF_TRANSFER: 0 + HF_TOKEN: ${HF_TOKEN} HABANA_VISIBLE_DEVICES: all OMPI_MCA_btl_vader_single_copy_mechanism: none - PREFILL_BATCH_BUCKET_SIZE: 1 - BATCH_BUCKET_SIZE: 8 - ENABLE_HPU_GRAPH: true - LIMIT_HPU_GRAPH: true - USE_FLASH_ATTENTION: true - FLASH_ATTENTION_RECOMPUTE: true + LLM_MODEL_ID: ${LLM_MODEL_ID} + VLLM_TORCH_PROFILER_DIR: "/mnt" host_ip: ${host_ip} LLM_ENDPOINT_PORT: ${LLM_ENDPOINT_PORT} + VLLM_SKIP_WARMUP: ${VLLM_SKIP_WARMUP:-false} + NUM_CARDS: ${NUM_CARDS:-1} runtime: habana cap_add: - SYS_NICE @@ -34,13 +30,13 @@ services: test: ["CMD-SHELL", "curl -f http://${host_ip}:${LLM_ENDPOINT_PORT}/health || exit 1"] interval: 10s timeout: 10s - retries: 100 - command: --model-id ${LLM_MODEL_ID} --max-input-length 1024 --max-total-tokens 2048 --max-batch-total-tokens 65536 --max-batch-prefill-tokens 4096 + retries: 150 + command: --model $LLM_MODEL_ID --tensor-parallel-size ${NUM_CARDS} --host 0.0.0.0 --port 80 --block-size 128 --max-num-seqs 256 llm_faqgen: image: ${REGISTRY:-opea}/llm-faqgen:${TAG:-latest} container_name: llm-faqgen-server depends_on: - tgi-service: + vllm-gaudi-service: condition: service_healthy ports: - ${LLM_SERVICE_PORT:-9000}:9000 @@ -52,17 +48,17 @@ services: LLM_ENDPOINT: ${LLM_ENDPOINT} LLM_MODEL_ID: ${LLM_MODEL_ID} HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} - FAQGen_COMPONENT_NAME: ${FAQGen_COMPONENT_NAME} + FAQGen_COMPONENT_NAME: ${FAQGen_COMPONENT_NAME:-OpeaFaqGenvLLM} LOGFLAG: ${LOGFLAG:-False} restart: unless-stopped faqgen-gaudi-backend-server: image: ${REGISTRY:-opea}/faqgen:${TAG:-latest} container_name: faqgen-gaudi-backend-server depends_on: - - tgi-service + - vllm-gaudi-service - llm_faqgen ports: - - "8888:8888" + - ${FAQGEN_BACKEND_PORT:-8888}:8888 environment: - no_proxy=${no_proxy} - https_proxy=${https_proxy} diff --git a/FaqGen/docker_compose/intel/hpu/gaudi/compose_tgi.yaml b/FaqGen/docker_compose/intel/hpu/gaudi/compose_tgi.yaml new file mode 100644 index 0000000000..082321583b --- /dev/null +++ b/FaqGen/docker_compose/intel/hpu/gaudi/compose_tgi.yaml @@ -0,0 +1,94 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +services: + tgi-service: + image: ghcr.io/huggingface/tgi-gaudi:2.3.1 + container_name: tgi-gaudi-server + ports: + - ${LLM_ENDPOINT_PORT:-8008}:80 + volumes: + - "${MODEL_CACHE:-./data}:/data" + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HUGGING_FACE_HUB_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + HABANA_VISIBLE_DEVICES: all + OMPI_MCA_btl_vader_single_copy_mechanism: none + PREFILL_BATCH_BUCKET_SIZE: 1 + BATCH_BUCKET_SIZE: 8 + ENABLE_HPU_GRAPH: true + LIMIT_HPU_GRAPH: true + USE_FLASH_ATTENTION: true + FLASH_ATTENTION_RECOMPUTE: true + host_ip: ${host_ip} + LLM_ENDPOINT_PORT: ${LLM_ENDPOINT_PORT} + MAX_INPUT_TOKENS: ${MAX_INPUT_TOKENS:-4096} + MAX_TOTAL_TOKENS: ${MAX_TOTAL_TOKENS:-8192} + runtime: habana + cap_add: + - SYS_NICE + ipc: host + healthcheck: + test: ["CMD-SHELL", "curl -f http://${host_ip}:${LLM_ENDPOINT_PORT}/health || exit 1"] + interval: 10s + timeout: 10s + retries: 100 + command: --model-id ${LLM_MODEL_ID} --max-input-tokens ${MAX_INPUT_TOKENS} --max-total-tokens ${MAX_TOTAL_TOKENS} --max-batch-total-tokens 65536 --max-batch-prefill-tokens 4096 + llm_faqgen: + image: ${REGISTRY:-opea}/llm-faqgen:${TAG:-latest} + container_name: llm-faqgen-server + depends_on: + tgi-service: + condition: service_healthy + ports: + - ${LLM_SERVICE_PORT:-9000}:9000 + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + LLM_ENDPOINT: ${LLM_ENDPOINT} + LLM_MODEL_ID: ${LLM_MODEL_ID} + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + FAQGen_COMPONENT_NAME: ${FAQGen_COMPONENT_NAME:-OpeaFaqGenTgi} + LOGFLAG: ${LOGFLAG:-False} + restart: unless-stopped + faqgen-gaudi-backend-server: + image: ${REGISTRY:-opea}/faqgen:${TAG:-latest} + container_name: faqgen-gaudi-backend-server + depends_on: + - tgi-service + - llm_faqgen + ports: + - ${FAQGEN_BACKEND_PORT:-8888}:8888 + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP} + - LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP} + - LLM_SERVICE_PORT=${LLM_SERVICE_PORT} + ipc: host + restart: always + faqgen-gaudi-ui-server: + image: ${REGISTRY:-opea}/faqgen-ui:${TAG:-latest} + container_name: faqgen-gaudi-ui-server + depends_on: + - faqgen-gaudi-backend-server + ports: + - "5173:5173" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - FAQ_BASE_URL=${BACKEND_SERVICE_ENDPOINT} + ipc: host + restart: always + +networks: + default: + driver: bridge diff --git a/FaqGen/tests/test_compose_on_gaudi.sh b/FaqGen/tests/test_compose_on_gaudi.sh index eeba304279..125e71a8e4 100644 --- a/FaqGen/tests/test_compose_on_gaudi.sh +++ b/FaqGen/tests/test_compose_on_gaudi.sh @@ -13,9 +13,21 @@ export TAG=${IMAGE_TAG} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" ip_address=$(hostname -I | awk '{print $1}') -export DATA_PATH=${model_cache:-"/data/cache"} +export MODEL_CACHE=${model_cache:-"/data/cache"} function build_docker_images() { + cd $WORKPATH + git clone https://github.com/HabanaAI/vllm-fork.git + cd vllm-fork/ + git checkout v0.6.4.post2+Gaudi-1.19.0 + docker build --no-cache -f Dockerfile.hpu -t ${REGISTRY:-opea}/vllm-gaudi:${TAG:-latest} --shm-size=128g . + if [ $? -ne 0 ]; then + echo "opea/vllm-gaudi built fail" + exit 1 + else + echo "opea/vllm-gaudi built successful" + fi + opea_branch=${opea_branch:-"main"} # If the opea_branch isn't main, replace the git clone branch in Dockerfile. if [[ "${opea_branch}" != "main" ]]; then @@ -35,7 +47,6 @@ function build_docker_images() { service_list="faqgen faqgen-ui llm-faqgen" docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log - docker pull ghcr.io/huggingface/tgi-gaudi:2.0.6 docker images && sleep 1s } @@ -43,15 +54,17 @@ function start_services() { cd $WORKPATH/docker_compose/intel/hpu/gaudi export host_ip=${ip_address} - export LLM_ENDPOINT_PORT=8008 - export FAQGen_COMPONENT_NAME="OpeaFaqGenTgi" - export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" + export LLM_ENDPOINT_PORT=8010 + export FAQGen_COMPONENT_NAME="OpeaFaqGenvLLM" + export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct" export LLM_ENDPOINT="http://${host_ip}:${LLM_ENDPOINT_PORT}" export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} export MEGA_SERVICE_HOST_IP=${ip_address} export LLM_SERVICE_HOST_IP=${ip_address} - export LLM_SERVICE_PORT=9000 - export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/faqgen" + export LLM_SERVICE_PORT=9001 + export FAQGEN_BACKEND_PORT=8888 + export NUM_CARDS=1 + export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:${FAQGEN_BACKEND_PORT}/v1/faqgen" export LOGFLAG=True sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env @@ -93,17 +106,18 @@ function validate_services() { function validate_microservices() { # Check if the microservices are running correctly. - # tgi for llm service + # vllm + echo "Validate vllm..." validate_services \ - "${ip_address}:8008/generate" \ - "generated_text" \ - "tgi-service" \ - "tgi-gaudi-server" \ - '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' + "http://${host_ip}:${LLM_ENDPOINT_PORT}/v1/chat/completions" \ + "text" \ + "vllm-gaudi-service" \ + "vllm-gaudi-service" \ + '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "user", "content": "What is Deep Learning?"}]}' # llm microservice validate_services \ - "${ip_address}:9000/v1/faqgen" \ + "${ip_address}:${LLM_SERVICE_PORT}/v1/faqgen" \ "text" \ "llm" \ "llm-faqgen-server" \ @@ -115,7 +129,7 @@ function validate_megaservice() { local DOCKER_NAME="faqgen-gaudi-backend-server" local EXPECTED_RESULT="Embeddings" local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." - local URL="${ip_address}:8888/v1/faqgen" + local URL="${ip_address}:${FAQGEN_BACKEND_PORT}/v1/faqgen" local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$INPUT_DATA" -F "max_tokens=32" -F "stream=False" -H 'Content-Type: multipart/form-data' "$URL") if [ "$HTTP_STATUS" -eq 200 ]; then echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." diff --git a/FaqGen/tests/test_compose_on_xeon.sh b/FaqGen/tests/test_compose_on_xeon.sh index cc527b7e9d..fb859ebe04 100755 --- a/FaqGen/tests/test_compose_on_xeon.sh +++ b/FaqGen/tests/test_compose_on_xeon.sh @@ -16,6 +16,20 @@ LOG_PATH="$WORKPATH/tests" ip_address=$(hostname -I | awk '{print $1}') function build_docker_images() { + cd $WORKPATH + git clone https://github.com/vllm-project/vllm.git + cd ./vllm/ + VLLM_VER="$(git describe --tags "$(git rev-list --tags --max-count=1)" )" + echo "Check out vLLM tag ${VLLM_VER}" + git checkout ${VLLM_VER} &> /dev/null + docker build --no-cache -f Dockerfile.cpu -t ${REGISTRY:-opea}/vllm:${TAG:-latest} --shm-size=128g . + if [ $? -ne 0 ]; then + echo "opea/vllm built fail" + exit 1 + else + echo "opea/vllm built successful" + fi + opea_branch=${opea_branch:-"main"} # If the opea_branch isn't main, replace the git clone branch in Dockerfile. if [[ "${opea_branch}" != "main" ]]; then @@ -35,7 +49,6 @@ function build_docker_images() { service_list="faqgen faqgen-ui llm-faqgen" docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log - docker pull ghcr.io/huggingface/text-generation-inference:1.4 docker images && sleep 1s } @@ -43,15 +56,16 @@ function start_services() { cd $WORKPATH/docker_compose/intel/cpu/xeon/ export host_ip=${ip_address} - export LLM_ENDPOINT_PORT=8008 - export FAQGen_COMPONENT_NAME="OpeaFaqGenTgi" - export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" + export LLM_ENDPOINT_PORT=8011 + export FAQGen_COMPONENT_NAME="OpeaFaqGenvLLM" + export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct" export LLM_ENDPOINT="http://${host_ip}:${LLM_ENDPOINT_PORT}" export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} export MEGA_SERVICE_HOST_IP=${ip_address} export LLM_SERVICE_HOST_IP=${ip_address} - export LLM_SERVICE_PORT=9000 - export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/faqgen" + export LLM_SERVICE_PORT=9002 + export FAQGEN_BACKEND_PORT=8888 + export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:${FAQGEN_BACKEND_PORT}/v1/faqgen" export LOGFLAG=True sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env @@ -93,17 +107,18 @@ function validate_services() { function validate_microservices() { # Check if the microservices are running correctly. - # tgi for llm service + # vllm for llm service + echo "Validate vllm..." validate_services \ - "${ip_address}:8008/generate" \ - "generated_text" \ - "tgi-service" \ - "tgi-xeon-server" \ - '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' + "http://${host_ip}:${LLM_ENDPOINT_PORT}/v1/chat/completions" \ + "text" \ + "vllm-service" \ + "vllm-service" \ + '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "user", "content": "What is Deep Learning?"}]}' # llm microservice validate_services \ - "${ip_address}:9000/v1/faqgen" \ + "${ip_address}:${LLM_SERVICE_PORT}/v1/faqgen" \ "text" \ "llm" \ "llm-faqgen-server" \ @@ -115,7 +130,7 @@ function validate_megaservice() { local DOCKER_NAME="faqgen-xeon-backend-server" local EXPECTED_RESULT="Embeddings" local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." - local URL="${ip_address}:8888/v1/faqgen" + local URL="${ip_address}:${FAQGEN_BACKEND_PORT}/v1/faqgen" local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$INPUT_DATA" -F "max_tokens=32" -F "stream=False" -H 'Content-Type: multipart/form-data' "$URL") if [ "$HTTP_STATUS" -eq 200 ]; then echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." diff --git a/FaqGen/tests/test_compose_tgi_on_gaudi.sh b/FaqGen/tests/test_compose_tgi_on_gaudi.sh new file mode 100644 index 0000000000..1c596322b4 --- /dev/null +++ b/FaqGen/tests/test_compose_tgi_on_gaudi.sh @@ -0,0 +1,192 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" +export REGISTRY=${IMAGE_REPO} +export TAG=${IMAGE_TAG} + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') +export MODEL_CACHE=${model_cache:-"/data/cache"} + +function build_docker_images() { + opea_branch=${opea_branch:-"main"} + # If the opea_branch isn't main, replace the git clone branch in Dockerfile. + if [[ "${opea_branch}" != "main" ]]; then + cd $WORKPATH + OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" + NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" + find . -type f -name "Dockerfile*" | while read -r file; do + echo "Processing file: $file" + sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" + done + fi + + cd $WORKPATH/docker_image_build + git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + + echo "Build all the images with --no-cache, check docker_image_build.log for details..." + service_list="faqgen faqgen-ui llm-faqgen" + docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log + + docker pull ghcr.io/huggingface/tgi-gaudi:2.0.6 + docker images && sleep 1s +} + +function start_services() { + cd $WORKPATH/docker_compose/intel/hpu/gaudi + + export host_ip=${ip_address} + export LLM_ENDPOINT_PORT=8009 + export FAQGen_COMPONENT_NAME="OpeaFaqGenTgi" + export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" + export LLM_ENDPOINT="http://${host_ip}:${LLM_ENDPOINT_PORT}" + export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} + export MEGA_SERVICE_HOST_IP=${ip_address} + export LLM_SERVICE_HOST_IP=${ip_address} + export LLM_SERVICE_PORT=9001 + export MAX_INPUT_TOKENS=4096 + export MAX_TOTAL_TOKENS=8192 + export FAQGEN_BACKEND_PORT=8889 + export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:${FAQGEN_BACKEND_PORT}/v1/faqgen" + export LOGFLAG=True + + sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env + + # Start Docker Containers + docker compose -f compose_tgi.yaml up -d > ${LOG_PATH}/start_services_with_compose.log + + sleep 30s +} + +function validate_services() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +function validate_microservices() { + # Check if the microservices are running correctly. + + # tgi for llm service + validate_services \ + "${ip_address}:${LLM_ENDPOINT_PORT}/generate" \ + "generated_text" \ + "tgi-service" \ + "tgi-gaudi-server" \ + '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' + + # llm microservice + validate_services \ + "${ip_address}:${LLM_SERVICE_PORT}/v1/faqgen" \ + "text" \ + "llm" \ + "llm-faqgen-server" \ + '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' +} + +function validate_megaservice() { + local SERVICE_NAME="mega-faqgen" + local DOCKER_NAME="faqgen-gaudi-backend-server" + local EXPECTED_RESULT="Embeddings" + local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." + local URL="${ip_address}:${FAQGEN_BACKEND_PORT}/v1/faqgen" + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$INPUT_DATA" -F "max_tokens=32" -F "stream=False" -H 'Content-Type: multipart/form-data' "$URL") + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -F "$INPUT_DATA" -F "max_tokens=32" -F "stream=False" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +function validate_frontend() { + cd $WORKPATH/ui/svelte + local conda_env_name="OPEA_e2e" + export PATH=${HOME}/miniforge3/bin/:$PATH + if conda info --envs | grep -q "$conda_env_name"; then + echo "$conda_env_name exist!" + else + conda create -n ${conda_env_name} python=3.12 -y + fi + source activate ${conda_env_name} + + sed -i "s/localhost/$ip_address/g" playwright.config.ts + + conda install -c conda-forge nodejs=22.6.0 -y + npm install && npm ci && npx playwright install --with-deps + node -v && npm -v && pip list + + exit_status=0 + npx playwright test || exit_status=$? + + if [ $exit_status -ne 0 ]; then + echo "[TEST INFO]: ---------frontend test failed---------" + exit $exit_status + else + echo "[TEST INFO]: ---------frontend test passed---------" + fi +} + +function stop_docker() { + cd $WORKPATH/docker_compose/intel/hpu/gaudi + docker compose -f compose_tgi.yaml stop && docker compose rm -f +} + +function main() { + + stop_docker + + if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi + start_services + + validate_microservices + validate_megaservice + # validate_frontend + + stop_docker + echo y | docker system prune + +} + +main diff --git a/FaqGen/tests/test_compose_tgi_on_xeon.sh b/FaqGen/tests/test_compose_tgi_on_xeon.sh new file mode 100755 index 0000000000..9676288a63 --- /dev/null +++ b/FaqGen/tests/test_compose_tgi_on_xeon.sh @@ -0,0 +1,190 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" +export REGISTRY=${IMAGE_REPO} +export TAG=${IMAGE_TAG} + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') +export MODEL_CACHE=${model_cache:-"./data"} + +function build_docker_images() { + opea_branch=${opea_branch:-"main"} + # If the opea_branch isn't main, replace the git clone branch in Dockerfile. + if [[ "${opea_branch}" != "main" ]]; then + cd $WORKPATH + OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" + NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" + find . -type f -name "Dockerfile*" | while read -r file; do + echo "Processing file: $file" + sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" + done + fi + + cd $WORKPATH/docker_image_build + git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + + echo "Build all the images with --no-cache, check docker_image_build.log for details..." + service_list="faqgen faqgen-ui llm-faqgen" + docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log + + docker pull ghcr.io/huggingface/text-generation-inference:1.4 + docker images && sleep 1s +} + +function start_services() { + cd $WORKPATH/docker_compose/intel/cpu/xeon/ + + export host_ip=${ip_address} + export LLM_ENDPOINT_PORT=8009 + export FAQGen_COMPONENT_NAME="OpeaFaqGenTgi" + export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" + export LLM_ENDPOINT="http://${host_ip}:${LLM_ENDPOINT_PORT}" + export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} + export MEGA_SERVICE_HOST_IP=${ip_address} + export LLM_SERVICE_HOST_IP=${ip_address} + export LLM_SERVICE_PORT=9001 + export FAQGEN_BACKEND_PORT=8889 + export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:${FAQGEN_BACKEND_PORT}/v1/faqgen" + export LOGFLAG=True + + sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env + + # Start Docker Containers + docker compose -f compose_tgi.yaml up -d > ${LOG_PATH}/start_services_with_compose.log + + sleep 30s +} + +function validate_services() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +function validate_microservices() { + # Check if the microservices are running correctly. + + # tgi for llm service + validate_services \ + "${ip_address}:${LLM_ENDPOINT_PORT}/generate" \ + "generated_text" \ + "tgi-service" \ + "tgi-xeon-server" \ + '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' + + # llm microservice + validate_services \ + "${ip_address}:${LLM_SERVICE_PORT}/v1/faqgen" \ + "text" \ + "llm" \ + "llm-faqgen-server" \ + '{"messages":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' +} + +function validate_megaservice() { + local SERVICE_NAME="mega-faqgen" + local DOCKER_NAME="faqgen-xeon-backend-server" + local EXPECTED_RESULT="Embeddings" + local INPUT_DATA="messages=Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5." + local URL="${ip_address}:${FAQGEN_BACKEND_PORT}/v1/faqgen" + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F "$INPUT_DATA" -F "max_tokens=32" -F "stream=False" -H 'Content-Type: multipart/form-data' "$URL") + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -F "$INPUT_DATA" -F "max_tokens=32" -F "stream=False" -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +function validate_frontend() { + cd $WORKPATH/ui/svelte + local conda_env_name="OPEA_e2e" + export PATH=${HOME}/miniforge3/bin/:$PATH + if conda info --envs | grep -q "$conda_env_name"; then + echo "$conda_env_name exist!" + else + conda create -n ${conda_env_name} python=3.12 -y + fi + source activate ${conda_env_name} + + sed -i "s/localhost/$ip_address/g" playwright.config.ts + + conda install -c conda-forge nodejs=22.6.0 -y + npm install && npm ci && npx playwright install --with-deps + node -v && npm -v && pip list + + exit_status=0 + npx playwright test || exit_status=$? + + if [ $exit_status -ne 0 ]; then + echo "[TEST INFO]: ---------frontend test failed---------" + exit $exit_status + else + echo "[TEST INFO]: ---------frontend test passed---------" + fi +} + +function stop_docker() { + cd $WORKPATH/docker_compose/intel/cpu/xeon/ + docker compose -f compose_tgi.yaml stop && docker compose rm -f +} + +function main() { + + stop_docker + + if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi + start_services + + validate_microservices + validate_megaservice + # validate_frontend + + stop_docker + echo y | docker system prune + +} + +main From a6d6f1f6c02005578e544fd76f0c480459f79bcd Mon Sep 17 00:00:00 2001 From: "Wang, Kai Lawrence" <109344418+wangkl2@users.noreply.github.com> Date: Mon, 10 Mar 2025 13:40:42 +0800 Subject: [PATCH 054/226] Fix vllm model cache directory (#1642) Signed-off-by: Wang, Kai Lawrence Signed-off-by: Chingis Yundunov --- ChatQnA/docker_compose/intel/cpu/xeon/README.md | 4 ++-- ChatQnA/docker_compose/intel/cpu/xeon/README_pinecone.md | 4 ++-- ChatQnA/docker_compose/intel/cpu/xeon/compose.yaml | 2 +- ChatQnA/docker_compose/intel/cpu/xeon/compose_milvus.yaml | 2 +- ChatQnA/docker_compose/intel/cpu/xeon/compose_qdrant.yaml | 2 +- .../docker_compose/intel/cpu/xeon/compose_without_rerank.yaml | 2 +- CodeTrans/docker_compose/intel/cpu/xeon/README.md | 4 ++-- CodeTrans/docker_compose/intel/cpu/xeon/compose.yaml | 2 +- CodeTrans/docker_compose/intel/hpu/gaudi/README.md | 4 ++-- CodeTrans/docker_compose/intel/hpu/gaudi/compose.yaml | 2 +- FaqGen/docker_compose/intel/cpu/xeon/compose.yaml | 2 +- FaqGen/docker_compose/intel/hpu/gaudi/compose.yaml | 2 +- 12 files changed, 16 insertions(+), 16 deletions(-) diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/README.md b/ChatQnA/docker_compose/intel/cpu/xeon/README.md index c71a866cf5..f8475e94d0 100644 --- a/ChatQnA/docker_compose/intel/cpu/xeon/README.md +++ b/ChatQnA/docker_compose/intel/cpu/xeon/README.md @@ -219,7 +219,7 @@ For users in China who are unable to download models directly from Huggingface, export HF_ENDPOINT="https://hf-mirror.com" model_name="meta-llama/Meta-Llama-3-8B-Instruct" # Start vLLM LLM Service - docker run -p 8008:80 -v ./data:/data --name vllm-service -e HF_ENDPOINT=$HF_ENDPOINT -e http_proxy=$http_proxy -e https_proxy=$https_proxy --shm-size 128g opea/vllm:latest --model $model_name --host 0.0.0.0 --port 80 + docker run -p 8008:80 -v ./data:/root/.cache/huggingface/hub --name vllm-service -e HF_ENDPOINT=$HF_ENDPOINT -e http_proxy=$http_proxy -e https_proxy=$https_proxy --shm-size 128g opea/vllm:latest --model $model_name --host 0.0.0.0 --port 80 # Start TGI LLM Service docker run -p 8008:80 -v ./data:/data --name tgi-service -e HF_ENDPOINT=$HF_ENDPOINT -e http_proxy=$http_proxy -e https_proxy=$https_proxy --shm-size 1g ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu --model-id $model_name ``` @@ -236,7 +236,7 @@ For users in China who are unable to download models directly from Huggingface, export HF_TOKEN=${your_hf_token} export model_path="/path/to/model" # Start vLLM LLM Service - docker run -p 8008:80 -v $model_path:/data --name vllm-service --shm-size 128g opea/vllm:latest --model /data --host 0.0.0.0 --port 80 + docker run -p 8008:80 -v $model_path:/root/.cache/huggingface/hub --name vllm-service --shm-size 128g opea/vllm:latest --model /root/.cache/huggingface/hub --host 0.0.0.0 --port 80 # Start TGI LLM Service docker run -p 8008:80 -v $model_path:/data --name tgi-service --shm-size 1g ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu --model-id /data ``` diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/README_pinecone.md b/ChatQnA/docker_compose/intel/cpu/xeon/README_pinecone.md index 8e8a9cd441..4d127be19a 100644 --- a/ChatQnA/docker_compose/intel/cpu/xeon/README_pinecone.md +++ b/ChatQnA/docker_compose/intel/cpu/xeon/README_pinecone.md @@ -201,7 +201,7 @@ For users in China who are unable to download models directly from Huggingface, export HF_TOKEN=${your_hf_token} export HF_ENDPOINT="https://hf-mirror.com" model_name="meta-llama/Meta-Llama-3-8B-Instruct" - docker run -p 8008:80 -v ./data:/data --name vllm-service -e HF_ENDPOINT=$HF_ENDPOINT -e http_proxy=$http_proxy -e https_proxy=$https_proxy --shm-size 128g opea/vllm:latest --model $model_name --host 0.0.0.0 --port 80 + docker run -p 8008:80 -v ./data:/root/.cache/huggingface/hub --name vllm-service -e HF_ENDPOINT=$HF_ENDPOINT -e http_proxy=$http_proxy -e https_proxy=$https_proxy --shm-size 128g opea/vllm:latest --model $model_name --host 0.0.0.0 --port 80 ``` 2. Offline @@ -215,7 +215,7 @@ For users in China who are unable to download models directly from Huggingface, ```bash export HF_TOKEN=${your_hf_token} export model_path="/path/to/model" - docker run -p 8008:80 -v $model_path:/data --name vllm-service --shm-size 128g opea/vllm:latest --model /data --host 0.0.0.0 --port 80 + docker run -p 8008:80 -v $model_path:/root/.cache/huggingface/hub --name vllm-service --shm-size 128g opea/vllm:latest --model /root/.cache/huggingface/hub --host 0.0.0.0 --port 80 ``` ### Setup Environment Variables diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/compose.yaml b/ChatQnA/docker_compose/intel/cpu/xeon/compose.yaml index 1ec229115e..2427e3e1c3 100644 --- a/ChatQnA/docker_compose/intel/cpu/xeon/compose.yaml +++ b/ChatQnA/docker_compose/intel/cpu/xeon/compose.yaml @@ -80,7 +80,7 @@ services: ports: - "9009:80" volumes: - - "${MODEL_CACHE:-./data}:/data" + - "${MODEL_CACHE:-./data}:/root/.cache/huggingface/hub" shm_size: 128g environment: no_proxy: ${no_proxy} diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/compose_milvus.yaml b/ChatQnA/docker_compose/intel/cpu/xeon/compose_milvus.yaml index 740f5eba42..7025c9018a 100644 --- a/ChatQnA/docker_compose/intel/cpu/xeon/compose_milvus.yaml +++ b/ChatQnA/docker_compose/intel/cpu/xeon/compose_milvus.yaml @@ -144,7 +144,7 @@ services: ports: - "9009:80" volumes: - - "./data:/data" + - "${MODEL_CACHE:-./data}:/root/.cache/huggingface/hub" shm_size: 128g environment: no_proxy: ${no_proxy} diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/compose_qdrant.yaml b/ChatQnA/docker_compose/intel/cpu/xeon/compose_qdrant.yaml index 0504ff07a1..40e1992d75 100644 --- a/ChatQnA/docker_compose/intel/cpu/xeon/compose_qdrant.yaml +++ b/ChatQnA/docker_compose/intel/cpu/xeon/compose_qdrant.yaml @@ -80,7 +80,7 @@ services: ports: - "6042:80" volumes: - - "${MODEL_CACHE:-./data}:/data" + - "${MODEL_CACHE:-./data}:/root/.cache/huggingface/hub" shm_size: 128g environment: no_proxy: ${no_proxy} diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/compose_without_rerank.yaml b/ChatQnA/docker_compose/intel/cpu/xeon/compose_without_rerank.yaml index 70ea084408..e121e77b6b 100644 --- a/ChatQnA/docker_compose/intel/cpu/xeon/compose_without_rerank.yaml +++ b/ChatQnA/docker_compose/intel/cpu/xeon/compose_without_rerank.yaml @@ -64,7 +64,7 @@ services: ports: - "9009:80" volumes: - - "${MODEL_CACHE:-./data}:/data" + - "${MODEL_CACHE:-./data}:/root/.cache/huggingface/hub" shm_size: 128g environment: no_proxy: ${no_proxy} diff --git a/CodeTrans/docker_compose/intel/cpu/xeon/README.md b/CodeTrans/docker_compose/intel/cpu/xeon/README.md index a7a8066202..3d250c7036 100755 --- a/CodeTrans/docker_compose/intel/cpu/xeon/README.md +++ b/CodeTrans/docker_compose/intel/cpu/xeon/README.md @@ -74,7 +74,7 @@ For users in China who are unable to download models directly from Huggingface, export HF_ENDPOINT="https://hf-mirror.com" model_name="mistralai/Mistral-7B-Instruct-v0.3" # Start vLLM LLM Service - docker run -p 8008:80 -v ./data:/data --name vllm-service -e HF_ENDPOINT=$HF_ENDPOINT -e http_proxy=$http_proxy -e https_proxy=$https_proxy --shm-size 128g opea/vllm:latest --model $model_name --host 0.0.0.0 --port 80 + docker run -p 8008:80 -v ./data:/root/.cache/huggingface/hub --name vllm-service -e HF_ENDPOINT=$HF_ENDPOINT -e http_proxy=$http_proxy -e https_proxy=$https_proxy --shm-size 128g opea/vllm:latest --model $model_name --host 0.0.0.0 --port 80 # Start TGI LLM Service docker run -p 8008:80 -v ./data:/data --name tgi-service -e HF_ENDPOINT=$HF_ENDPOINT -e http_proxy=$http_proxy -e https_proxy=$https_proxy --shm-size 1g ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu --model-id $model_name ``` @@ -91,7 +91,7 @@ For users in China who are unable to download models directly from Huggingface, export HF_TOKEN=${your_hf_token} export model_path="/path/to/model" # Start vLLM LLM Service - docker run -p 8008:80 -v $model_path:/data --name vllm-service --shm-size 128g opea/vllm:latest --model /data --host 0.0.0.0 --port 80 + docker run -p 8008:80 -v $model_path:/root/.cache/huggingface/hub --name vllm-service --shm-size 128g opea/vllm:latest --model /root/.cache/huggingface/hub --host 0.0.0.0 --port 80 # Start TGI LLM Service docker run -p 8008:80 -v $model_path:/data --name tgi-service --shm-size 1g ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu --model-id /data ``` diff --git a/CodeTrans/docker_compose/intel/cpu/xeon/compose.yaml b/CodeTrans/docker_compose/intel/cpu/xeon/compose.yaml index 24c8bfdd39..f4aa9f2b95 100644 --- a/CodeTrans/docker_compose/intel/cpu/xeon/compose.yaml +++ b/CodeTrans/docker_compose/intel/cpu/xeon/compose.yaml @@ -8,7 +8,7 @@ services: ports: - "8008:80" volumes: - - "${MODEL_CACHE:-./data}:/data" + - "${MODEL_CACHE:-./data}:/root/.cache/huggingface/hub" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/CodeTrans/docker_compose/intel/hpu/gaudi/README.md b/CodeTrans/docker_compose/intel/hpu/gaudi/README.md index cf5f2d3c11..d07326598f 100755 --- a/CodeTrans/docker_compose/intel/hpu/gaudi/README.md +++ b/CodeTrans/docker_compose/intel/hpu/gaudi/README.md @@ -66,7 +66,7 @@ For users in China who are unable to download models directly from Huggingface, export HF_ENDPOINT="https://hf-mirror.com" model_name="mistralai/Mistral-7B-Instruct-v0.3" # Start vLLM LLM Service - docker run -p 8008:80 -v ./data:/data --name vllm-service -e HF_ENDPOINT=$HF_ENDPOINT -e http_proxy=$http_proxy -e https_proxy=$https_proxy --shm-size 128g opea/vllm:latest --model $model_name --host 0.0.0.0 --port 80 + docker run -p 8008:80 -v ./data:/root/.cache/huggingface/hub --name vllm-service -e HF_ENDPOINT=$HF_ENDPOINT -e http_proxy=$http_proxy -e https_proxy=$https_proxy --shm-size 128g opea/vllm:latest --model $model_name --host 0.0.0.0 --port 80 # Start TGI LLM Service docker run -p 8008:80 -v ./data:/data --name tgi-service -e HF_ENDPOINT=$HF_ENDPOINT -e http_proxy=$http_proxy -e https_proxy=$https_proxy --shm-size 1g ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu --model-id $model_name ``` @@ -83,7 +83,7 @@ For users in China who are unable to download models directly from Huggingface, export HF_TOKEN=${your_hf_token} export model_path="/path/to/model" # Start vLLM LLM Service - docker run -p 8008:80 -v $model_path:/data --name vllm-service --shm-size 128g opea/vllm:latest --model /data --host 0.0.0.0 --port 80 + docker run -p 8008:80 -v $model_path:/root/.cache/huggingface/hub --name vllm-service --shm-size 128g opea/vllm:latest --model /root/.cache/huggingface/hub --host 0.0.0.0 --port 80 # Start TGI LLM Service docker run -p 8008:80 -v $model_path:/data --name tgi-service --shm-size 1g ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu --model-id /data ``` diff --git a/CodeTrans/docker_compose/intel/hpu/gaudi/compose.yaml b/CodeTrans/docker_compose/intel/hpu/gaudi/compose.yaml index 2caeaf0ec3..7fe0538f60 100644 --- a/CodeTrans/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/CodeTrans/docker_compose/intel/hpu/gaudi/compose.yaml @@ -8,7 +8,7 @@ services: ports: - "8008:80" volumes: - - "${MODEL_CACHE:-./data}:/data" + - "${MODEL_CACHE:-./data}:/root/.cache/huggingface/hub" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} diff --git a/FaqGen/docker_compose/intel/cpu/xeon/compose.yaml b/FaqGen/docker_compose/intel/cpu/xeon/compose.yaml index 7da122f9ab..b122f43157 100644 --- a/FaqGen/docker_compose/intel/cpu/xeon/compose.yaml +++ b/FaqGen/docker_compose/intel/cpu/xeon/compose.yaml @@ -8,7 +8,7 @@ services: ports: - ${LLM_ENDPOINT_PORT:-8008}:80 volumes: - - "${MODEL_CACHE:-./data}:/data" + - "${MODEL_CACHE:-./data}:/root/.cache/huggingface/hub" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/FaqGen/docker_compose/intel/hpu/gaudi/compose.yaml b/FaqGen/docker_compose/intel/hpu/gaudi/compose.yaml index fbc8812d58..80d4cd8438 100644 --- a/FaqGen/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/FaqGen/docker_compose/intel/hpu/gaudi/compose.yaml @@ -8,7 +8,7 @@ services: ports: - ${LLM_ENDPOINT_PORT:-8008}:80 volumes: - - "${MODEL_CACHE:-./data}:/data" + - "${MODEL_CACHE:-./data}:/root/.cache/huggingface/hub" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} From cb831b0cc22a9ebfb248ec46fb309ecfd2e9571f Mon Sep 17 00:00:00 2001 From: "chen, suyue" Date: Mon, 10 Mar 2025 17:36:26 +0800 Subject: [PATCH 055/226] Enhance ChatQnA test scripts (#1643) Signed-off-by: chensuyue Signed-off-by: Chingis Yundunov --- ChatQnA/docker_compose/intel/cpu/xeon/compose_milvus.yaml | 4 ++-- ChatQnA/tests/test_compose_guardrails_on_gaudi.sh | 2 +- ChatQnA/tests/test_compose_milvus_on_xeon.sh | 5 ++++- ChatQnA/tests/test_compose_qdrant_on_xeon.sh | 2 +- ChatQnA/tests/test_compose_tgi_on_gaudi.sh | 2 +- ChatQnA/tests/test_compose_tgi_on_xeon.sh | 2 +- ChatQnA/tests/test_compose_without_rerank_on_gaudi.sh | 2 +- ChatQnA/tests/test_compose_without_rerank_on_xeon.sh | 2 +- 8 files changed, 12 insertions(+), 9 deletions(-) diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/compose_milvus.yaml b/ChatQnA/docker_compose/intel/cpu/xeon/compose_milvus.yaml index 7025c9018a..13306b1bf2 100644 --- a/ChatQnA/docker_compose/intel/cpu/xeon/compose_milvus.yaml +++ b/ChatQnA/docker_compose/intel/cpu/xeon/compose_milvus.yaml @@ -113,7 +113,7 @@ services: ports: - "6006:80" volumes: - - "./data:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -127,7 +127,7 @@ services: ports: - "8808:80" volumes: - - "./data:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/ChatQnA/tests/test_compose_guardrails_on_gaudi.sh b/ChatQnA/tests/test_compose_guardrails_on_gaudi.sh index c882a7ef77..855b986af7 100644 --- a/ChatQnA/tests/test_compose_guardrails_on_gaudi.sh +++ b/ChatQnA/tests/test_compose_guardrails_on_gaudi.sh @@ -162,7 +162,7 @@ function validate_megaservice() { # Curl the Mega Service validate_service \ "${ip_address}:8888/v1/chatqna" \ - "data: " \ + "Nike" \ "mega-chatqna" \ "chatqna-gaudi-guardrails-server" \ '{"messages": "What is the revenue of Nike in 2023?"}' diff --git a/ChatQnA/tests/test_compose_milvus_on_xeon.sh b/ChatQnA/tests/test_compose_milvus_on_xeon.sh index d2953a9992..0a8814954a 100644 --- a/ChatQnA/tests/test_compose_milvus_on_xeon.sh +++ b/ChatQnA/tests/test_compose_milvus_on_xeon.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" @@ -180,7 +181,7 @@ function validate_megaservice() { # Curl the Mega Service validate_service \ "${ip_address}:8888/v1/chatqna" \ - "data: " \ + "Nike" \ "chatqna-megaservice" \ "chatqna-xeon-backend-server" \ '{"messages": "What is the revenue of Nike in 2023?"}' @@ -240,6 +241,8 @@ function main() { echo "==== microservices validated ====" validate_megaservice echo "==== megaservice validated ====" + validate_frontend + echo "==== frontend validated ====" stop_docker echo y | docker system prune diff --git a/ChatQnA/tests/test_compose_qdrant_on_xeon.sh b/ChatQnA/tests/test_compose_qdrant_on_xeon.sh index 8c84a9a9ff..fe66abaf12 100644 --- a/ChatQnA/tests/test_compose_qdrant_on_xeon.sh +++ b/ChatQnA/tests/test_compose_qdrant_on_xeon.sh @@ -162,7 +162,7 @@ function validate_megaservice() { # Curl the Mega Service validate_service \ "${ip_address}:8912/v1/chatqna" \ - "data: " \ + "Nike" \ "mega-chatqna" \ "chatqna-xeon-backend-server" \ '{"messages": "What is the revenue of Nike in 2023?"}' diff --git a/ChatQnA/tests/test_compose_tgi_on_gaudi.sh b/ChatQnA/tests/test_compose_tgi_on_gaudi.sh index 25bfe8cdee..483df8ef97 100644 --- a/ChatQnA/tests/test_compose_tgi_on_gaudi.sh +++ b/ChatQnA/tests/test_compose_tgi_on_gaudi.sh @@ -182,7 +182,7 @@ function validate_megaservice() { # Curl the Mega Service validate_service \ "${ip_address}:8888/v1/chatqna" \ - "data: " \ + "Nike" \ "chatqna-megaservice" \ "chatqna-gaudi-backend-server" \ '{"messages": "What is the revenue of Nike in 2023?"}' diff --git a/ChatQnA/tests/test_compose_tgi_on_xeon.sh b/ChatQnA/tests/test_compose_tgi_on_xeon.sh index f00d8c6436..1f871a38f6 100644 --- a/ChatQnA/tests/test_compose_tgi_on_xeon.sh +++ b/ChatQnA/tests/test_compose_tgi_on_xeon.sh @@ -181,7 +181,7 @@ function validate_megaservice() { # Curl the Mega Service validate_service \ "${ip_address}:8888/v1/chatqna" \ - "data: " \ + "Nike" \ "chatqna-megaservice" \ "chatqna-xeon-backend-server" \ '{"messages": "What is the revenue of Nike in 2023?"}' diff --git a/ChatQnA/tests/test_compose_without_rerank_on_gaudi.sh b/ChatQnA/tests/test_compose_without_rerank_on_gaudi.sh index c9dc86a0bd..bc60054291 100644 --- a/ChatQnA/tests/test_compose_without_rerank_on_gaudi.sh +++ b/ChatQnA/tests/test_compose_without_rerank_on_gaudi.sh @@ -171,7 +171,7 @@ function validate_megaservice() { # Curl the Mega Service validate_service \ "${ip_address}:8888/v1/chatqna" \ - "data: " \ + "Nike" \ "chatqna-megaservice" \ "chatqna-gaudi-backend-server" \ '{"messages": "What is the revenue of Nike in 2023?"}' diff --git a/ChatQnA/tests/test_compose_without_rerank_on_xeon.sh b/ChatQnA/tests/test_compose_without_rerank_on_xeon.sh index 279bc780d0..66c6fe420e 100644 --- a/ChatQnA/tests/test_compose_without_rerank_on_xeon.sh +++ b/ChatQnA/tests/test_compose_without_rerank_on_xeon.sh @@ -174,7 +174,7 @@ function validate_megaservice() { # Curl the Mega Service validate_service \ "${ip_address}:8888/v1/chatqna" \ - "data: " \ + "Nike" \ "chatqna-megaservice" \ "chatqna-xeon-backend-server" \ '{"messages": "What is the revenue of Nike in 2023?"}' From ffa0eadb7e76d56c8705d4dd198e4fc1f89f9f6c Mon Sep 17 00:00:00 2001 From: "Sun, Xuehao" Date: Wed, 12 Mar 2025 10:56:07 +0800 Subject: [PATCH 056/226] Add GitHub Action to check and close stale issues and PRs (#1646) Signed-off-by: Sun, Xuehao Signed-off-by: Chingis Yundunov --- .../workflows/daily_check_issue_and_pr.yml | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/daily_check_issue_and_pr.yml diff --git a/.github/workflows/daily_check_issue_and_pr.yml b/.github/workflows/daily_check_issue_and_pr.yml new file mode 100644 index 0000000000..b578580602 --- /dev/null +++ b/.github/workflows/daily_check_issue_and_pr.yml @@ -0,0 +1,29 @@ +# Copyright (C) 2025 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +name: Check stale issue and pr + +on: + schedule: + - cron: "30 22 * * *" + +jobs: + close-issues: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/stale@v9 + with: + days-before-issue-stale: 60 + days-before-pr-stale: 60 + days-before-issue-close: 7 + days-before-pr-close: 7 + stale-issue-message: "This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 7 days." + stale-pr-message: "This PR is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 7 days." + close-issue-message: "This issue was closed because it has been stalled for 7 days with no activity." + close-pr-message: "This PR was closed because it has been stalled for 7 days with no activity." + repo-token: ${{ secrets.ACTION_TOKEN }} + start-date: "2025-01-01T00:00:00Z" + debug-only: true # will remove this line when ready to merge From b725c267380aab8c8bcd99baf8298d31f7dbd59e Mon Sep 17 00:00:00 2001 From: Eero Tamminen Date: Thu, 13 Mar 2025 02:23:07 +0200 Subject: [PATCH 057/226] Use GenAIComp base image to simplify Dockerfiles & reduce image sizes - part 2 (#1638) Signed-off-by: Eero Tamminen Signed-off-by: Chingis Yundunov --- ChatQnA/Dockerfile | 44 ++----------------------------- ChatQnA/Dockerfile.guardrails | 44 ++----------------------------- ChatQnA/Dockerfile.without_rerank | 44 ++----------------------------- DocSum/Dockerfile | 44 ++----------------------------- GraphRAG/Dockerfile | 44 ++----------------------------- SearchQnA/Dockerfile | 44 ++----------------------------- Translation/Dockerfile | 44 ++----------------------------- VisualQnA/Dockerfile | 44 ++----------------------------- 8 files changed, 16 insertions(+), 336 deletions(-) diff --git a/ChatQnA/Dockerfile b/ChatQnA/Dockerfile index fb7f5e14ec..fffb8d8970 100644 --- a/ChatQnA/Dockerfile +++ b/ChatQnA/Dockerfile @@ -1,48 +1,8 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -# Stage 1: base setup used by other stages -FROM python:3.11-slim AS base - -# get security updates -RUN apt-get update && apt-get upgrade -y && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -ENV HOME=/home/user - -RUN useradd -m -s /bin/bash user && \ - mkdir -p $HOME && \ - chown -R user $HOME - -WORKDIR $HOME - - -# Stage 2: latest GenAIComps sources -FROM base AS git - -RUN apt-get update && apt-get install -y --no-install-recommends git -RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git - - -# Stage 3: common layer shared by services using GenAIComps -FROM base AS comps-base - -# copy just relevant parts -COPY --from=git $HOME/GenAIComps/comps $HOME/GenAIComps/comps -COPY --from=git $HOME/GenAIComps/*.* $HOME/GenAIComps/LICENSE $HOME/GenAIComps/ - -WORKDIR $HOME/GenAIComps -RUN pip install --no-cache-dir --upgrade pip setuptools && \ - pip install --no-cache-dir -r $HOME/GenAIComps/requirements.txt -WORKDIR $HOME - -ENV PYTHONPATH=$PYTHONPATH:$HOME/GenAIComps - -USER user - - -# Stage 4: unique part -FROM comps-base +ARG BASE_TAG=latest +FROM opea/comps-base:$BASE_TAG COPY ./chatqna.py $HOME/chatqna.py diff --git a/ChatQnA/Dockerfile.guardrails b/ChatQnA/Dockerfile.guardrails index 4fe5fd2087..07a358d922 100644 --- a/ChatQnA/Dockerfile.guardrails +++ b/ChatQnA/Dockerfile.guardrails @@ -1,48 +1,8 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -# Stage 1: base setup used by other stages -FROM python:3.11-slim AS base - -# get security updates -RUN apt-get update && apt-get upgrade -y && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -ENV HOME=/home/user - -RUN useradd -m -s /bin/bash user && \ - mkdir -p $HOME && \ - chown -R user $HOME - -WORKDIR $HOME - - -# Stage 2: latest GenAIComps sources -FROM base AS git - -RUN apt-get update && apt-get install -y --no-install-recommends git -RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git - - -# Stage 3: common layer shared by services using GenAIComps -FROM base AS comps-base - -# copy just relevant parts -COPY --from=git $HOME/GenAIComps/comps $HOME/GenAIComps/comps -COPY --from=git $HOME/GenAIComps/*.* $HOME/GenAIComps/LICENSE $HOME/GenAIComps/ - -WORKDIR $HOME/GenAIComps -RUN pip install --no-cache-dir --upgrade pip setuptools && \ - pip install --no-cache-dir -r $HOME/GenAIComps/requirements.txt -WORKDIR $HOME - -ENV PYTHONPATH=$PYTHONPATH:$HOME/GenAIComps - -USER user - - -# Stage 4: unique part -FROM comps-base +ARG BASE_TAG=latest +FROM opea/comps-base:$BASE_TAG COPY ./chatqna.py $HOME/chatqna.py diff --git a/ChatQnA/Dockerfile.without_rerank b/ChatQnA/Dockerfile.without_rerank index 9e6740e9b8..ad1611110a 100644 --- a/ChatQnA/Dockerfile.without_rerank +++ b/ChatQnA/Dockerfile.without_rerank @@ -1,48 +1,8 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -# Stage 1: base setup used by other stages -FROM python:3.11-slim AS base - -# get security updates -RUN apt-get update && apt-get upgrade -y && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -ENV HOME=/home/user - -RUN useradd -m -s /bin/bash user && \ - mkdir -p $HOME && \ - chown -R user $HOME - -WORKDIR $HOME - - -# Stage 2: latest GenAIComps sources -FROM base AS git - -RUN apt-get update && apt-get install -y --no-install-recommends git -RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git - - -# Stage 3: common layer shared by services using GenAIComps -FROM base AS comps-base - -# copy just relevant parts -COPY --from=git $HOME/GenAIComps/comps $HOME/GenAIComps/comps -COPY --from=git $HOME/GenAIComps/*.* $HOME/GenAIComps/LICENSE $HOME/GenAIComps/ - -WORKDIR $HOME/GenAIComps -RUN pip install --no-cache-dir --upgrade pip setuptools && \ - pip install --no-cache-dir -r $HOME/GenAIComps/requirements.txt -WORKDIR $HOME - -ENV PYTHONPATH=$PYTHONPATH:$HOME/GenAIComps - -USER user - - -# Stage 4: unique part -FROM comps-base +ARG BASE_TAG=latest +FROM opea/comps-base:$BASE_TAG COPY ./chatqna.py $HOME/chatqna.py diff --git a/DocSum/Dockerfile b/DocSum/Dockerfile index fd01f3bca0..2cc8c3d5a5 100644 --- a/DocSum/Dockerfile +++ b/DocSum/Dockerfile @@ -1,48 +1,8 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -# Stage 1: base setup used by other stages -FROM python:3.11-slim AS base - -# get security updates -RUN apt-get update && apt-get upgrade -y && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -ENV HOME=/home/user - -RUN useradd -m -s /bin/bash user && \ - mkdir -p $HOME && \ - chown -R user $HOME - -WORKDIR $HOME - - -# Stage 2: latest GenAIComps sources -FROM base AS git - -RUN apt-get update && apt-get install -y --no-install-recommends git -RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git - - -# Stage 3: common layer shared by services using GenAIComps -FROM base AS comps-base - -# copy just relevant parts -COPY --from=git $HOME/GenAIComps/comps $HOME/GenAIComps/comps -COPY --from=git $HOME/GenAIComps/*.* $HOME/GenAIComps/LICENSE $HOME/GenAIComps/ - -WORKDIR $HOME/GenAIComps -RUN pip install --no-cache-dir --upgrade pip setuptools && \ - pip install --no-cache-dir -r $HOME/GenAIComps/requirements.txt -WORKDIR $HOME - -ENV PYTHONPATH=$PYTHONPATH:$HOME/GenAIComps - -USER user - - -# Stage 4: unique part -FROM comps-base +ARG BASE_TAG=latest +FROM opea/comps-base:$BASE_TAG USER root # FFmpeg needed for media processing diff --git a/GraphRAG/Dockerfile b/GraphRAG/Dockerfile index 1e50649dd5..0c2c91d85f 100644 --- a/GraphRAG/Dockerfile +++ b/GraphRAG/Dockerfile @@ -1,48 +1,8 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -# Stage 1: base setup used by other stages -FROM python:3.11-slim AS base - -# get security updates -RUN apt-get update && apt-get upgrade -y && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -ENV HOME=/home/user - -RUN useradd -m -s /bin/bash user && \ - mkdir -p $HOME && \ - chown -R user $HOME - -WORKDIR $HOME - - -# Stage 2: latest GenAIComps sources -FROM base AS git - -RUN apt-get update && apt-get install -y --no-install-recommends git -RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git - - -# Stage 3: common layer shared by services using GenAIComps -FROM base AS comps-base - -# copy just relevant parts -COPY --from=git $HOME/GenAIComps/comps $HOME/GenAIComps/comps -COPY --from=git $HOME/GenAIComps/*.* $HOME/GenAIComps/LICENSE $HOME/GenAIComps/ - -WORKDIR $HOME/GenAIComps -RUN pip install --no-cache-dir --upgrade pip setuptools && \ - pip install --no-cache-dir -r $HOME/GenAIComps/requirements.txt -WORKDIR $HOME - -ENV PYTHONPATH=$PYTHONPATH:$HOME/GenAIComps - -USER user - - -# Stage 4: unique part -FROM comps-base +ARG BASE_TAG=latest +FROM opea/comps-base:$BASE_TAG COPY ./graphrag.py $HOME/graphrag.py diff --git a/SearchQnA/Dockerfile b/SearchQnA/Dockerfile index df8d536b08..a93afd6093 100644 --- a/SearchQnA/Dockerfile +++ b/SearchQnA/Dockerfile @@ -1,48 +1,8 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -# Stage 1: base setup used by other stages -FROM python:3.11-slim AS base - -# get security updates -RUN apt-get update && apt-get upgrade -y && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -ENV HOME=/home/user - -RUN useradd -m -s /bin/bash user && \ - mkdir -p $HOME && \ - chown -R user $HOME - -WORKDIR $HOME - - -# Stage 2: latest GenAIComps sources -FROM base AS git - -RUN apt-get update && apt-get install -y --no-install-recommends git -RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git - - -# Stage 3: common layer shared by services using GenAIComps -FROM base AS comps-base - -# copy just relevant parts -COPY --from=git $HOME/GenAIComps/comps $HOME/GenAIComps/comps -COPY --from=git $HOME/GenAIComps/*.* $HOME/GenAIComps/LICENSE $HOME/GenAIComps/ - -WORKDIR $HOME/GenAIComps -RUN pip install --no-cache-dir --upgrade pip setuptools && \ - pip install --no-cache-dir -r $HOME/GenAIComps/requirements.txt -WORKDIR $HOME - -ENV PYTHONPATH=$PYTHONPATH:$HOME/GenAIComps - -USER user - - -# Stage 4: unique part -FROM comps-base +ARG BASE_TAG=latest +FROM opea/comps-base:$BASE_TAG COPY ./searchqna.py $HOME/searchqna.py diff --git a/Translation/Dockerfile b/Translation/Dockerfile index 70266c9b87..853935af84 100644 --- a/Translation/Dockerfile +++ b/Translation/Dockerfile @@ -1,48 +1,8 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -# Stage 1: base setup used by other stages -FROM python:3.11-slim AS base - -# get security updates -RUN apt-get update && apt-get upgrade -y && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -ENV HOME=/home/user - -RUN useradd -m -s /bin/bash user && \ - mkdir -p $HOME && \ - chown -R user $HOME - -WORKDIR $HOME - - -# Stage 2: latest GenAIComps sources -FROM base AS git - -RUN apt-get update && apt-get install -y --no-install-recommends git -RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git - - -# Stage 3: common layer shared by services using GenAIComps -FROM base AS comps-base - -# copy just relevant parts -COPY --from=git $HOME/GenAIComps/comps $HOME/GenAIComps/comps -COPY --from=git $HOME/GenAIComps/*.* $HOME/GenAIComps/LICENSE $HOME/GenAIComps/ - -WORKDIR $HOME/GenAIComps -RUN pip install --no-cache-dir --upgrade pip setuptools && \ - pip install --no-cache-dir -r $HOME/GenAIComps/requirements.txt -WORKDIR $HOME - -ENV PYTHONPATH=$PYTHONPATH:$HOME/GenAIComps - -USER user - - -# Stage 4: unique part -FROM comps-base +ARG BASE_TAG=latest +FROM opea/comps-base:$BASE_TAG COPY ./translation.py $HOME/translation.py diff --git a/VisualQnA/Dockerfile b/VisualQnA/Dockerfile index 257b39df89..95936d9c03 100644 --- a/VisualQnA/Dockerfile +++ b/VisualQnA/Dockerfile @@ -1,48 +1,8 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -# Stage 1: base setup used by other stages -FROM python:3.11-slim AS base - -# get security updates -RUN apt-get update && apt-get upgrade -y && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -ENV HOME=/home/user - -RUN useradd -m -s /bin/bash user && \ - mkdir -p $HOME && \ - chown -R user $HOME - -WORKDIR $HOME - - -# Stage 2: latest GenAIComps sources -FROM base AS git - -RUN apt-get update && apt-get install -y --no-install-recommends git -RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git - - -# Stage 3: common layer shared by services using GenAIComps -FROM base AS comps-base - -# copy just relevant parts -COPY --from=git $HOME/GenAIComps/comps $HOME/GenAIComps/comps -COPY --from=git $HOME/GenAIComps/*.* $HOME/GenAIComps/LICENSE $HOME/GenAIComps/ - -WORKDIR $HOME/GenAIComps -RUN pip install --no-cache-dir --upgrade pip setuptools && \ - pip install --no-cache-dir -r $HOME/GenAIComps/requirements.txt -WORKDIR $HOME - -ENV PYTHONPATH=$PYTHONPATH:$HOME/GenAIComps - -USER user - - -# Stage 4: unique part -FROM comps-base +ARG BASE_TAG=latest +FROM opea/comps-base:$BASE_TAG COPY ./visualqna.py $HOME/visualqna.py From faf8f09e87bf9a6b064c689de3e2a0c08568cf1d Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Thu, 13 Mar 2025 09:39:42 +0800 Subject: [PATCH 058/226] Enable inject_commit to docker image feature. (#1653) Signed-off-by: ZePan110 Signed-off-by: Chingis Yundunov --- .github/workflows/nightly-docker-build-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nightly-docker-build-publish.yml b/.github/workflows/nightly-docker-build-publish.yml index 84e1fe88bc..3a05fae1df 100644 --- a/.github/workflows/nightly-docker-build-publish.yml +++ b/.github/workflows/nightly-docker-build-publish.yml @@ -44,6 +44,7 @@ jobs: node: gaudi example: ${{ matrix.example }} test_compose: true + inject_commit: true secrets: inherit get-image-list: From 6e262af6ce165684e9a1baf540c50ec487e729e3 Mon Sep 17 00:00:00 2001 From: xiguiw <111278656+xiguiw@users.noreply.github.com> Date: Thu, 13 Mar 2025 10:38:47 +0800 Subject: [PATCH 059/226] Enable CodeGen vLLM (#1636) Signed-off-by: Wang, Xigui Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: Chingis Yundunov --- .../docker_compose/intel/cpu/xeon/README.md | 152 ++++++++++-------- .../intel/cpu/xeon/compose.yaml | 60 +++++-- .../docker_compose/intel/hpu/gaudi/README.md | 152 ++++++++++-------- .../intel/hpu/gaudi/compose.yaml | 66 ++++++-- CodeGen/docker_compose/set_env.sh | 14 +- CodeGen/docker_image_build/build.yaml | 12 ++ CodeGen/tests/test_compose_on_gaudi.sh | 78 ++++++--- CodeGen/tests/test_compose_on_xeon.sh | 79 ++++++--- 8 files changed, 419 insertions(+), 194 deletions(-) diff --git a/CodeGen/docker_compose/intel/cpu/xeon/README.md b/CodeGen/docker_compose/intel/cpu/xeon/README.md index 01ee5d1fa4..3cc7a19b3c 100644 --- a/CodeGen/docker_compose/intel/cpu/xeon/README.md +++ b/CodeGen/docker_compose/intel/cpu/xeon/README.md @@ -1,6 +1,7 @@ # Build MegaService of CodeGen on Xeon This document outlines the deployment process for a CodeGen application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on Intel Xeon server. The steps include Docker images creation, container deployment via Docker Compose, and service execution to integrate microservices such as `llm`. We will publish the Docker images to Docker Hub soon, further simplifying the deployment process for this service. +The default pipeline deploys with vLLM as the LLM serving component. It also provides options of using TGI backend for LLM microservice. ## 🚀 Create an AWS Xeon Instance @@ -10,55 +11,6 @@ For detailed information about these instance types, you can refer to [m7i](http After launching your instance, you can connect to it using SSH (for Linux instances) or Remote Desktop Protocol (RDP) (for Windows instances). From there, you'll have full access to your Xeon server, allowing you to install, configure, and manage your applications as needed. -## 🚀 Download or Build Docker Images - -Should the Docker image you seek not yet be available on Docker Hub, you can build the Docker image locally. - -### 1. Build the LLM Docker Image - -```bash -git clone https://github.com/opea-project/GenAIComps.git -cd GenAIComps -docker build -t opea/llm-textgen:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/text-generation/Dockerfile . -``` - -### 2. Build the MegaService Docker Image - -To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `codegen.py` Python script. Build MegaService Docker image via the command below: - -```bash -git clone https://github.com/opea-project/GenAIExamples -cd GenAIExamples/CodeGen -docker build -t opea/codegen:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . -``` - -### 3. Build the UI Docker Image - -Build the frontend Docker image via the command below: - -```bash -cd GenAIExamples/CodeGen/ui -docker build -t opea/codegen-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile . -``` - -### 4. Build CodeGen React UI Docker Image (Optional) - -Build react frontend Docker image via below command: - -**Export the value of the public IP address of your Xeon server to the `host_ip` environment variable** - -```bash -cd GenAIExamples/CodeGen/ui -docker build --no-cache -t opea/codegen-react-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . -``` - -Then run the command `docker images`, you will have the following Docker Images: - -- `opea/llm-textgen:latest` -- `opea/codegen:latest` -- `opea/codegen-ui:latest` -- `opea/codegen-react-ui:latest` (optional) - ## 🚀 Start Microservices and MegaService The CodeGen megaservice manages a single microservice called LLM within a Directed Acyclic Graph (DAG). In the diagram above, the LLM microservice is a language model microservice that generates code snippets based on the user's input query. The TGI service serves as a text generation interface, providing a RESTful API for the LLM microservice. The CodeGen Gateway acts as the entry point for the CodeGen application, invoking the Megaservice to generate code snippets in response to the user's input query. @@ -89,42 +41,57 @@ flowchart LR Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below. -**Append the value of the public IP address to the no_proxy list** +1. set the host_ip and huggingface token +> Note: +> Please replace the `your_ip_address` with you external IP address, do not use `localhost`. + +```bash +export host_ip=${your_ip_address} +export HUGGINGFACEHUB_API_TOKEN=you_huggingface_token ``` -export your_no_proxy=${your_no_proxy},"External_Public_IP" -``` + +2. Set Netowork Proxy + +**If you access public network through proxy, set the network proxy, otherwise, skip this step** ```bash export no_proxy=${your_no_proxy} export http_proxy=${your_http_proxy} -export https_proxy=${your_http_proxy} -export LLM_MODEL_ID="Qwen/Qwen2.5-Coder-7B-Instruct" -export TGI_LLM_ENDPOINT="http://${host_ip}:8028" -export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} -export MEGA_SERVICE_HOST_IP=${host_ip} -export LLM_SERVICE_HOST_IP=${host_ip} -export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:7778/v1/codegen" +export https_proxy=${your_https_proxy} ``` -Note: Please replace the `host_ip` with you external IP address, do not use `localhost`. - ### Start the Docker Containers for All Services +CodeGen support TGI service and vLLM service, you can choose start either one of them. + +Start CodeGen based on TGI service: + ```bash -cd GenAIExamples/CodeGen/docker_compose/intel/cpu/xeon -docker compose up -d +cd GenAIExamples/CodeGen/docker_compose +source set_env.sh +cd intel/cpu/xeon +docker compose --profile codegen-xeon-tgi up -d +``` + +Start CodeGen based on vLLM service: + +```bash +cd GenAIExamples/CodeGen/docker_compose +source set_env.sh +cd intel/cpu/xeon +docker compose --profile codegen-xeon-vllm up -d ``` ### Validate the MicroServices and MegaService -1. TGI Service +1. LLM Service (for TGI, vLLM) ```bash - curl http://${host_ip}:8028/generate \ - -X POST \ - -d '{"inputs":"Implement a high-level API for a TODO list application. The API takes as input an operation request and updates the TODO list in place. If the request is invalid, raise an exception.","parameters":{"max_new_tokens":256, "do_sample": true}}' \ - -H 'Content-Type: application/json' + curl http://${host_ip}:8028/v1/chat/completions \ + -X POST \ + -d '{"model": "Qwen/Qwen2.5-Coder-7B-Instruct", "messages": [{"role": "user", "content": "Implement a high-level API for a TODO list application. The API takes as input an operation request and updates the TODO list in place. If the request is invalid, raise an exception."}], "max_tokens":32}' \ + -H 'Content-Type: application/json' ``` 2. LLM Microservices @@ -257,3 +224,52 @@ For example: - Ask question and get answer ![qna](../../../../assets/img/codegen_qna.png) + +## 🚀 Download or Build Docker Images + +Should the Docker image you seek not yet be available on Docker Hub, you can build the Docker image locally. + +### 1. Build the LLM Docker Image + +```bash +git clone https://github.com/opea-project/GenAIComps.git +cd GenAIComps +docker build -t opea/llm-textgen:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/text-generation/Dockerfile . +``` + +### 2. Build the MegaService Docker Image + +To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `codegen.py` Python script. Build MegaService Docker image via the command below: + +```bash +git clone https://github.com/opea-project/GenAIExamples +cd GenAIExamples/CodeGen +docker build -t opea/codegen:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . +``` + +### 3. Build the UI Docker Image + +Build the frontend Docker image via the command below: + +```bash +cd GenAIExamples/CodeGen/ui +docker build -t opea/codegen-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile . +``` + +### 4. Build CodeGen React UI Docker Image (Optional) + +Build react frontend Docker image via below command: + +**Export the value of the public IP address of your Xeon server to the `host_ip` environment variable** + +```bash +cd GenAIExamples/CodeGen/ui +docker build --no-cache -t opea/codegen-react-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . +``` + +Then run the command `docker images`, you will have the following Docker Images: + +- `opea/llm-textgen:latest` +- `opea/codegen:latest` +- `opea/codegen-ui:latest` +- `opea/codegen-react-ui:latest` (optional) diff --git a/CodeGen/docker_compose/intel/cpu/xeon/compose.yaml b/CodeGen/docker_compose/intel/cpu/xeon/compose.yaml index 7973951000..5567d9e368 100644 --- a/CodeGen/docker_compose/intel/cpu/xeon/compose.yaml +++ b/CodeGen/docker_compose/intel/cpu/xeon/compose.yaml @@ -4,7 +4,9 @@ services: tgi-service: image: ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu - container_name: tgi-service + container_name: tgi-server + profiles: + - codegen-xeon-tgi ports: - "8028:80" volumes: @@ -22,28 +24,66 @@ services: timeout: 10s retries: 100 command: --model-id ${LLM_MODEL_ID} --cuda-graphs 0 - llm: + vllm-service: + image: ${REGISTRY:-opea}/vllm:${TAG:-latest} + container_name: vllm-server + profiles: + - codegen-xeon-vllm + ports: + - "8028:80" + volumes: + - "${MODEL_CACHE:-./data}:/root/.cache/huggingface/hub" + shm_size: 1g + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + host_ip: ${host_ip} + healthcheck: + test: ["CMD-SHELL", "curl -f http://$host_ip:8028/health || exit 1"] + interval: 10s + timeout: 10s + retries: 100 + command: --model ${LLM_MODEL_ID} --host 0.0.0.0 --port 80 + llm-base: image: ${REGISTRY:-opea}/llm-textgen:${TAG:-latest} container_name: llm-textgen-server - depends_on: - tgi-service: - condition: service_healthy - ports: - - "9000:9000" - ipc: host environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} - LLM_ENDPOINT: ${TGI_LLM_ENDPOINT} + LLM_ENDPOINT: ${LLM_ENDPOINT} LLM_MODEL_ID: ${LLM_MODEL_ID} HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} restart: unless-stopped + llm-tgi-service: + extends: llm-base + container_name: llm-codegen-tgi-server + profiles: + - codegen-xeon-tgi + ports: + - "9000:9000" + ipc: host + depends_on: + tgi-service: + condition: service_healthy + llm-vllm-service: + extends: llm-base + container_name: llm-codegen-vllm-server + profiles: + - codegen-xeon-vllm + ports: + - "9000:9000" + ipc: host + depends_on: + vllm-service: + condition: service_healthy codegen-xeon-backend-server: image: ${REGISTRY:-opea}/codegen:${TAG:-latest} container_name: codegen-xeon-backend-server depends_on: - - llm + - llm-base ports: - "7778:7778" environment: diff --git a/CodeGen/docker_compose/intel/hpu/gaudi/README.md b/CodeGen/docker_compose/intel/hpu/gaudi/README.md index 106f7d1ffc..133b32f09f 100644 --- a/CodeGen/docker_compose/intel/hpu/gaudi/README.md +++ b/CodeGen/docker_compose/intel/hpu/gaudi/README.md @@ -2,54 +2,7 @@ This document outlines the deployment process for a CodeGen application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on Intel Gaudi2 server. The steps include Docker images creation, container deployment via Docker Compose, and service execution to integrate microservices such as `llm`. We will publish the Docker images to the Docker Hub soon, further simplifying the deployment process for this service. -## 🚀 Build Docker Images - -First of all, you need to build the Docker images locally. This step can be ignored after the Docker images published to the Docker Hub. - -### 1. Build the LLM Docker Image - -```bash -git clone https://github.com/opea-project/GenAIComps.git -cd GenAIComps -docker build -t opea/llm-textgen:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/text-generation/Dockerfile . -``` - -### 2. Build the MegaService Docker Image - -To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `codegen.py` Python script. Build the MegaService Docker image via the command below: - -```bash -git clone https://github.com/opea-project/GenAIExamples -cd GenAIExamples/CodeGen -docker build -t opea/codegen:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . -``` - -### 3. Build the UI Docker Image - -Construct the frontend Docker image via the command below: - -```bash -cd GenAIExamples/CodeGen/ui -docker build -t opea/codegen-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile . -``` - -### 4. Build CodeGen React UI Docker Image (Optional) - -Build react frontend Docker image via below command: - -**Export the value of the public IP address of your Xeon server to the `host_ip` environment variable** - -```bash -cd GenAIExamples/CodeGen/ui -docker build --no-cache -t opea/codegen-react-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . -``` - -Then run the command `docker images`, you will have the following Docker images: - -- `opea/llm-textgen:latest` -- `opea/codegen:latest` -- `opea/codegen-ui:latest` -- `opea/codegen-react-ui:latest` +The default pipeline deploys with vLLM as the LLM serving component. It also provides options of using TGI backend for LLM microservice. ## 🚀 Start MicroServices and MegaService @@ -81,37 +34,57 @@ flowchart LR Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below. +1. set the host_ip and huggingface token + +> [!NOTE] +> Please replace the `your_ip_address` with you external IP address, do not use `localhost`. + +```bash +export host_ip=${your_ip_address} +export HUGGINGFACEHUB_API_TOKEN=you_huggingface_token +``` + +2. Set Netowork Proxy + +**If you access public network through proxy, set the network proxy, otherwise, skip this step** + ```bash export no_proxy=${your_no_proxy} export http_proxy=${your_http_proxy} -export https_proxy=${your_http_proxy} -export LLM_MODEL_ID="Qwen/Qwen2.5-Coder-7B-Instruct" -export TGI_LLM_ENDPOINT="http://${host_ip}:8028" -export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} -export MEGA_SERVICE_HOST_IP=${host_ip} -export LLM_SERVICE_HOST_IP=${host_ip} -export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:7778/v1/codegen" +export https_proxy=${your_https_proxy} ``` -> [!NOTE] -> Please replace the `host_ip` with you external IP address, do not use `localhost`. - ### Start the Docker Containers for All Services +CodeGen support TGI service and vLLM service, you can choose start either one of them. + +Start CodeGen based on TGI service: + +```bash +cd GenAIExamples/CodeGen/docker_compose +source set_env.sh +cd intel/hpu/gaudi +docker compose --profile codegen-gaudi-tgi up -d +``` + +Start CodeGen based on vLLM service: + ```bash -cd GenAIExamples/CodeGen/docker_compose/intel/hpu/gaudi -docker compose up -d +cd GenAIExamples/CodeGen/docker_compose +source set_env.sh +cd intel/hpu/gaudi +docker compose --profile codegen-gaudi-vllm up -d ``` ### Validate the MicroServices and MegaService -1. TGI Service +1. LLM Service (for TGI, vLLM) ```bash - curl http://${host_ip}:8028/generate \ - -X POST \ - -d '{"inputs":"Implement a high-level API for a TODO list application. The API takes as input an operation request and updates the TODO list in place. If the request is invalid, raise an exception.","parameters":{"max_new_tokens":256, "do_sample": true}}' \ - -H 'Content-Type: application/json' + curl http://${host_ip}:8028/v1/chat/completions \ + -X POST \ + -d '{"model": "Qwen/Qwen2.5-Coder-7B-Instruct", "messages": [{"role": "user", "content": "Implement a high-level API for a TODO list application. The API takes as input an operation request and updates the TODO list in place. If the request is invalid, raise an exception."}], "max_tokens":32}' \ + -H 'Content-Type: application/json' ``` 2. LLM Microservices @@ -240,3 +213,52 @@ For example: - Ask question and get answer ![qna](../../../../assets/img/codegen_qna.png) + +## 🚀 Build Docker Images + +First of all, you need to build the Docker images locally. This step can be ignored after the Docker images published to the Docker Hub. + +### 1. Build the LLM Docker Image + +```bash +git clone https://github.com/opea-project/GenAIComps.git +cd GenAIComps +docker build -t opea/llm-textgen:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/llms/src/text-generation/Dockerfile . +``` + +### 2. Build the MegaService Docker Image + +To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `codegen.py` Python script. Build the MegaService Docker image via the command below: + +```bash +git clone https://github.com/opea-project/GenAIExamples +cd GenAIExamples/CodeGen +docker build -t opea/codegen:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . +``` + +### 3. Build the UI Docker Image + +Construct the frontend Docker image via the command below: + +```bash +cd GenAIExamples/CodeGen/ui +docker build -t opea/codegen-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile . +``` + +### 4. Build CodeGen React UI Docker Image (Optional) + +Build react frontend Docker image via below command: + +**Export the value of the public IP address of your Xeon server to the `host_ip` environment variable** + +```bash +cd GenAIExamples/CodeGen/ui +docker build --no-cache -t opea/codegen-react-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . +``` + +Then run the command `docker images`, you will have the following Docker images: + +- `opea/llm-textgen:latest` +- `opea/codegen:latest` +- `opea/codegen-ui:latest` +- `opea/codegen-react-ui:latest` diff --git a/CodeGen/docker_compose/intel/hpu/gaudi/compose.yaml b/CodeGen/docker_compose/intel/hpu/gaudi/compose.yaml index 19a77bef54..2f669e9465 100644 --- a/CodeGen/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/CodeGen/docker_compose/intel/hpu/gaudi/compose.yaml @@ -5,6 +5,8 @@ services: tgi-service: image: ghcr.io/huggingface/tgi-gaudi:2.3.1 container_name: tgi-gaudi-server + profiles: + - codegen-gaudi-tgi ports: - "8028:80" volumes: @@ -30,28 +32,74 @@ services: - SYS_NICE ipc: host command: --model-id ${LLM_MODEL_ID} --max-input-length 1024 --max-total-tokens 2048 - llm: - image: ${REGISTRY:-opea}/llm-textgen:${TAG:-latest} - container_name: llm-textgen-gaudi-server - depends_on: - tgi-service: - condition: service_healthy + vllm-service: + image: ${REGISTRY:-opea}/vllm-gaudi:${TAG:-latest} + container_name: vllm-gaudi-server + profiles: + - codegen-gaudi-vllm ports: - - "9000:9000" + - "8028:80" + volumes: + - "${MODEL_CACHE:-./data}:/root/.cache/huggingface/hub" + shm_size: 1g + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + HABANA_VISIBLE_DEVICES: all + OMPI_MCA_btl_vader_single_copy_mechanism: none + VLLM_SKIP_WARMUP: ${VLLM_SKIP_WARMUP:-false} + NUM_CARDS: ${NUM_CARDS:-1} + VLLM_TORCH_PROFILER_DIR: "/mnt" + healthcheck: + test: ["CMD-SHELL", "curl -f http://$host_ip:8028/health || exit 1"] + interval: 10s + timeout: 10s + retries: 100 + runtime: habana + cap_add: + - SYS_NICE ipc: host + command: --model ${LLM_MODEL_ID} --tensor-parallel-size ${NUM_CARDS} --host 0.0.0.0 --port 80 --block-size 128 --max-num-seqs 256 + llm-base: + image: ${REGISTRY:-opea}/llm-textgen:${TAG:-latest} + container_name: llm-textgen-gaudi-server environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} - LLM_ENDPOINT: ${TGI_LLM_ENDPOINT} + LLM_ENDPOINT: ${LLM_ENDPOINT} LLM_MODEL_ID: ${LLM_MODEL_ID} HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} restart: unless-stopped + llm-tgi-service: + extends: llm-base + container_name: llm-codegen-tgi-gaudi-server + profiles: + - codegen-gaudi-tgi + ports: + - "9000:9000" + ipc: host + depends_on: + tgi-service: + condition: service_healthy + llm-vllm-service: + extends: llm-base + container_name: llm-codegen-gaudi-vllm-server + profiles: + - codegen-gaudi-vllm + ports: + - "9000:9000" + ipc: host + depends_on: + vllm-service: + condition: service_healthy codegen-gaudi-backend-server: image: ${REGISTRY:-opea}/codegen:${TAG:-latest} container_name: codegen-gaudi-backend-server depends_on: - - llm + - llm-base ports: - "7778:7778" environment: diff --git a/CodeGen/docker_compose/set_env.sh b/CodeGen/docker_compose/set_env.sh index 3144ef9589..cb9e742847 100644 --- a/CodeGen/docker_compose/set_env.sh +++ b/CodeGen/docker_compose/set_env.sh @@ -6,9 +6,21 @@ pushd "../../" > /dev/null source .set_env.sh popd > /dev/null +export host_ip=$(hostname -I | awk '{print $1}') + +if [ -z "${HUGGINGFACEHUB_API_TOKEN}" ]; then + echo "Error: HUGGINGFACEHUB_API_TOKEN is not set. Please set HUGGINGFACEHUB_API_TOKEN" +fi + +if [ -z "${host_ip}" ]; then + echo "Error: host_ip is not set. Please set host_ip first." +fi + +export no_proxy=${no_proxy},${host_ip} export LLM_MODEL_ID="Qwen/Qwen2.5-Coder-7B-Instruct" -export TGI_LLM_ENDPOINT="http://${host_ip}:8028" +export LLM_ENDPOINT="http://${host_ip}:8028" export MEGA_SERVICE_HOST_IP=${host_ip} export LLM_SERVICE_HOST_IP=${host_ip} export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:7778/v1/codegen" +export MODEL_CACHE="./data" diff --git a/CodeGen/docker_image_build/build.yaml b/CodeGen/docker_image_build/build.yaml index aaee45977a..529984e35c 100644 --- a/CodeGen/docker_image_build/build.yaml +++ b/CodeGen/docker_image_build/build.yaml @@ -29,3 +29,15 @@ services: dockerfile: comps/llms/src/text-generation/Dockerfile extends: codegen image: ${REGISTRY:-opea}/llm-textgen:${TAG:-latest} + vllm: + build: + context: vllm + dockerfile: Dockerfile.cpu + extends: codegen + image: ${REGISTRY:-opea}/vllm:${TAG:-latest} + vllm-gaudi: + build: + context: vllm-fork + dockerfile: Dockerfile.hpu + extends: codegen + image: ${REGISTRY:-opea}/vllm-gaudi:${TAG:-latest} diff --git a/CodeGen/tests/test_compose_on_gaudi.sh b/CodeGen/tests/test_compose_on_gaudi.sh index e6e6d1f033..c7b6b83f7e 100644 --- a/CodeGen/tests/test_compose_on_gaudi.sh +++ b/CodeGen/tests/test_compose_on_gaudi.sh @@ -30,34 +30,44 @@ function build_docker_images() { cd $WORKPATH/docker_image_build git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + # Download Gaudi vllm of latest tag + git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork + VLLM_VER=$(git describe --tags "$(git rev-list --tags --max-count=1)") + echo "Check out vLLM tag ${VLLM_VER}" + git checkout ${VLLM_VER} &> /dev/null && cd ../ echo "Build all the images with --no-cache, check docker_image_build.log for details..." - service_list="codegen codegen-ui llm-textgen" + service_list="codegen codegen-ui llm-textgen vllm-gaudi" docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log - docker pull ghcr.io/huggingface/tgi-gaudi:2.0.6 docker images && sleep 1s } function start_services() { + local compose_profile="$1" + local llm_container_name="$2" + cd $WORKPATH/docker_compose/intel/hpu/gaudi - export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" - export TGI_LLM_ENDPOINT="http://${ip_address}:8028" + export http_proxy=${http_proxy} + export https_proxy=${https_proxy} + export LLM_MODEL_ID="Qwen/Qwen2.5-Coder-7B-Instruct" + export LLM_ENDPOINT="http://${ip_address}:8028" export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} export MEGA_SERVICE_HOST_IP=${ip_address} export LLM_SERVICE_HOST_IP=${ip_address} export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:7778/v1/codegen" + export NUM_CARDS=1 export host_ip=${ip_address} sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env # Start Docker Containers - docker compose up -d > ${LOG_PATH}/start_services_with_compose.log + docker compose --profile ${compose_profile} up -d | tee ${LOG_PATH}/start_services_with_compose.log n=0 until [[ "$n" -ge 100 ]]; do - docker logs tgi-gaudi-server > ${LOG_PATH}/tgi_service_start.log - if grep -q Connected ${LOG_PATH}/tgi_service_start.log; then + docker logs ${llm_container_name} > ${LOG_PATH}/llm_service_start.log 2>&1 + if grep -E "Connected|complete" ${LOG_PATH}/llm_service_start.log; then break fi sleep 5s @@ -94,13 +104,15 @@ function validate_services() { } function validate_microservices() { + local llm_container_name="$1" + # tgi for llm service validate_services \ - "${ip_address}:8028/generate" \ - "generated_text" \ - "tgi-llm" \ - "tgi-gaudi-server" \ - '{"inputs":"def print_hello_world():","parameters":{"max_new_tokens":256, "do_sample": true}}' + "${ip_address}:8028/v1/chat/completions" \ + "completion_tokens" \ + "llm-service" \ + "${llm_container_name}" \ + '{"model": "Qwen/Qwen2.5-Coder-7B-Instruct", "messages": [{"role": "user", "content": "def print_hello_world():"}], "max_tokens": 256}' # llm microservice validate_services \ @@ -152,24 +164,50 @@ function validate_frontend() { } function stop_docker() { + local docker_profile="$1" + cd $WORKPATH/docker_compose/intel/hpu/gaudi - docker compose stop && docker compose rm -f + docker compose --profile ${docker_profile} down } function main() { + # all docker docker compose profiles for XEON Platform + docker_compose_profiles=("codegen-gaudi-vllm" "codegen-gaudi-tgi") + docker_llm_container_names=("vllm-gaudi-server" "tgi-gaudi-server") - stop_docker + # get number of profiels and container + len_profiles=${#docker_compose_profiles[@]} + len_containers=${#docker_llm_container_names[@]} + # number of profiels and docker container names must be matched + if [ ${len_profiles} -ne ${len_containers} ]; then + echo "Error: number of profiles ${len_profiles} and container names ${len_containers} mismatched" + exit 1 + fi + + # stop_docker, stop all profiles + for ((i = 0; i < len_profiles; i++)); do + stop_docker "${docker_compose_profiles[${i}]}" + done + + # build docker images if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi - start_services - validate_microservices - validate_megaservice - validate_frontend + # loop all profiles + for ((i = 0; i < len_profiles; i++)); do + echo "Process [${i}]: ${docker_compose_profiles[$i]}, ${docker_llm_container_names[${i}]}" + start_services "${docker_compose_profiles[${i}]}" "${docker_llm_container_names[${i}]}" + docker ps -a - stop_docker - echo y | docker system prune + validate_microservices "${docker_llm_container_names[${i}]}" + validate_megaservice + validate_frontend + + stop_docker "${docker_compose_profiles[${i}]}" + sleep 5s + done + echo y | docker system prune } main diff --git a/CodeGen/tests/test_compose_on_xeon.sh b/CodeGen/tests/test_compose_on_xeon.sh index 70e5ba9c4f..6fc25963ac 100644 --- a/CodeGen/tests/test_compose_on_xeon.sh +++ b/CodeGen/tests/test_compose_on_xeon.sh @@ -31,8 +31,14 @@ function build_docker_images() { cd $WORKPATH/docker_image_build git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + git clone https://github.com/vllm-project/vllm.git && cd vllm + VLLM_VER="$(git describe --tags "$(git rev-list --tags --max-count=1)" )" + echo "Check out vLLM tag ${VLLM_VER}" + git checkout ${VLLM_VER} &> /dev/null + cd ../ + echo "Build all the images with --no-cache, check docker_image_build.log for details..." - service_list="codegen codegen-ui llm-textgen" + service_list="codegen codegen-ui llm-textgen vllm" docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log docker pull ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu @@ -40,10 +46,13 @@ function build_docker_images() { } function start_services() { + local compose_profile="$1" + local llm_container_name="$2" + cd $WORKPATH/docker_compose/intel/cpu/xeon/ - export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" - export TGI_LLM_ENDPOINT="http://${ip_address}:8028" + export LLM_MODEL_ID="Qwen/Qwen2.5-Coder-7B-Instruct" + export LLM_ENDPOINT="http://${ip_address}:8028" export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} export MEGA_SERVICE_HOST_IP=${ip_address} export LLM_SERVICE_HOST_IP=${ip_address} @@ -53,12 +62,12 @@ function start_services() { sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env # Start Docker Containers - docker compose up -d > ${LOG_PATH}/start_services_with_compose.log + docker compose --profile ${compose_profile} up -d > ${LOG_PATH}/start_services_with_compose.log n=0 until [[ "$n" -ge 100 ]]; do - docker logs tgi-service > ${LOG_PATH}/tgi_service_start.log - if grep -q Connected ${LOG_PATH}/tgi_service_start.log; then + docker logs ${llm_container_name} > ${LOG_PATH}/llm_service_start.log 2>&1 + if grep -E "Connected|complete" ${LOG_PATH}/llm_service_start.log; then break fi sleep 5s @@ -95,13 +104,15 @@ function validate_services() { } function validate_microservices() { + local llm_container_name="$1" + # tgi for llm service validate_services \ - "${ip_address}:8028/generate" \ - "generated_text" \ - "tgi-llm" \ - "tgi-service" \ - '{"inputs":"def print_hello_world():","parameters":{"max_new_tokens":256, "do_sample": true}}' + "${ip_address}:8028/v1/chat/completions" \ + "completion_tokens" \ + "llm-service" \ + "${llm_container_name}" \ + '{"model": "Qwen/Qwen2.5-Coder-7B-Instruct", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens": 256}' # llm microservice validate_services \ @@ -109,7 +120,7 @@ function validate_microservices() { "data: " \ "llm" \ "llm-textgen-server" \ - '{"query":"def print_hello_world():"}' + '{"query":"def print_hello_world():", "max_tokens": 256}' } @@ -120,7 +131,7 @@ function validate_megaservice() { "print" \ "mega-codegen" \ "codegen-xeon-backend-server" \ - '{"messages": "def print_hello_world():"}' + '{"messages": "def print_hello_world():", "max_tokens": 256}' } @@ -154,24 +165,50 @@ function validate_frontend() { function stop_docker() { + local docker_profile="$1" + cd $WORKPATH/docker_compose/intel/cpu/xeon/ - docker compose stop && docker compose rm -f + docker compose --profile ${docker_profile} down } function main() { + # all docker docker compose profiles for Xeon Platform + docker_compose_profiles=("codegen-xeon-tgi" "codegen-xeon-vllm") + docker_llm_container_names=("tgi-server" "vllm-server") + + # get number of profiels and LLM docker container names + len_profiles=${#docker_compose_profiles[@]} + len_containers=${#docker_llm_container_names[@]} + + # number of profiels and docker container names must be matched + if [ ${len_profiles} -ne ${len_containers} ]; then + echo "Error: number of profiles ${len_profiles} and container names ${len_containers} mismatched" + exit 1 + fi - stop_docker + # stop_docker, stop all profiles + for ((i = 0; i < len_profiles; i++)); do + stop_docker "${docker_compose_profiles[${i}]}" + done + # build docker images if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi - start_services - validate_microservices - validate_megaservice - validate_frontend + # loop all profiles + for ((i = 0; i < len_profiles; i++)); do + echo "Process [${i}]: ${docker_compose_profiles[$i]}, ${docker_llm_container_names[${i}]}" + docker ps -a + start_services "${docker_compose_profiles[${i}]}" "${docker_llm_container_names[${i}]}" - stop_docker - echo y | docker system prune + validate_microservices "${docker_llm_container_names[${i}]}" + validate_megaservice + validate_frontend + stop_docker "${docker_compose_profiles[${i}]}" + sleep 5s + done + + echo y | docker system prune } main From ceffcffa083e2e99ceb6b36aed91c666b52c9d6f Mon Sep 17 00:00:00 2001 From: Li Gang Date: Thu, 13 Mar 2025 10:52:33 +0800 Subject: [PATCH 060/226] [ChatQnA][docker]Check healthy of redis to avoid dataprep failure (#1591) Signed-off-by: Li Gang Signed-off-by: Chingis Yundunov --- ChatQnA/docker_compose/intel/cpu/xeon/compose.yaml | 11 +++++++++-- ChatQnA/docker_compose/intel/hpu/gaudi/compose.yaml | 11 +++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/compose.yaml b/ChatQnA/docker_compose/intel/cpu/xeon/compose.yaml index 2427e3e1c3..47e3b73494 100644 --- a/ChatQnA/docker_compose/intel/cpu/xeon/compose.yaml +++ b/ChatQnA/docker_compose/intel/cpu/xeon/compose.yaml @@ -8,12 +8,19 @@ services: ports: - "6379:6379" - "8001:8001" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 dataprep-redis-service: image: ${REGISTRY:-opea}/dataprep:${TAG:-latest} container_name: dataprep-redis-server depends_on: - - redis-vector-db - - tei-embedding-service + redis-vector-db: + condition: service_healthy + tei-embedding-service: + condition: service_started ports: - "6007:5000" environment: diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/compose.yaml b/ChatQnA/docker_compose/intel/hpu/gaudi/compose.yaml index 8ff06ecc35..6e501cb7bf 100644 --- a/ChatQnA/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/compose.yaml @@ -8,12 +8,19 @@ services: ports: - "6379:6379" - "8001:8001" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 dataprep-redis-service: image: ${REGISTRY:-opea}/dataprep:${TAG:-latest} container_name: dataprep-redis-server depends_on: - - redis-vector-db - - tei-embedding-service + redis-vector-db: + condition: service_healthy + tei-embedding-service: + condition: service_started ports: - "6007:5000" environment: From 7a4e2a7d1c9852902f0d1a78649b5f83979d4969 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Thu, 13 Mar 2025 11:23:03 +0800 Subject: [PATCH 061/226] Enable GraphRAG and ProductivitySuite model cache for docker compose test. (#1608) Signed-off-by: ZePan110 Signed-off-by: Chingis Yundunov --- GraphRAG/docker_compose/intel/hpu/gaudi/compose.yaml | 4 ++-- GraphRAG/tests/test_compose_on_gaudi.sh | 1 + .../docker_compose/intel/cpu/xeon/compose.yaml | 8 ++++---- ProductivitySuite/tests/test_compose_on_xeon.sh | 1 + 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/GraphRAG/docker_compose/intel/hpu/gaudi/compose.yaml b/GraphRAG/docker_compose/intel/hpu/gaudi/compose.yaml index 76f1ab9f63..0b1b9d78ad 100644 --- a/GraphRAG/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/GraphRAG/docker_compose/intel/hpu/gaudi/compose.yaml @@ -39,7 +39,7 @@ services: ports: - "${TEI_EMBEDDER_PORT:-12000}:80" volumes: - - "./data:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -58,7 +58,7 @@ services: ports: - ${LLM_ENDPOINT_PORT:-8008}:80 volumes: - - "${DATA_PATH:-./data}:/data" + - "${MODEL_CACHE:-./data}:/data" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} diff --git a/GraphRAG/tests/test_compose_on_gaudi.sh b/GraphRAG/tests/test_compose_on_gaudi.sh index bec978ad51..4d9a4128d4 100755 --- a/GraphRAG/tests/test_compose_on_gaudi.sh +++ b/GraphRAG/tests/test_compose_on_gaudi.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" diff --git a/ProductivitySuite/docker_compose/intel/cpu/xeon/compose.yaml b/ProductivitySuite/docker_compose/intel/cpu/xeon/compose.yaml index 149109e4b7..807ef90a7f 100644 --- a/ProductivitySuite/docker_compose/intel/cpu/xeon/compose.yaml +++ b/ProductivitySuite/docker_compose/intel/cpu/xeon/compose.yaml @@ -39,7 +39,7 @@ services: ports: - "6006:80" volumes: - - "./data_embedding:/data" + - "${MODEL_CACHE:-./data_embedding}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -95,7 +95,7 @@ services: ports: - "8808:80" volumes: - - "./data_tei:/data" + - "${MODEL_CACHE:-./data_tei}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -136,7 +136,7 @@ services: ports: - "9009:80" volumes: - - "./data:/data" + - "${MODEL_CACHE:-./data}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} @@ -209,7 +209,7 @@ services: ports: - "8028:80" volumes: - - "./data_codegen:/data" + - "${MODEL_CACHE:-./data_codegen}:/data" shm_size: 1g environment: no_proxy: ${no_proxy} diff --git a/ProductivitySuite/tests/test_compose_on_xeon.sh b/ProductivitySuite/tests/test_compose_on_xeon.sh index 333253feb2..b2717d8efc 100755 --- a/ProductivitySuite/tests/test_compose_on_xeon.sh +++ b/ProductivitySuite/tests/test_compose_on_xeon.sh @@ -9,6 +9,7 @@ echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" From 0e6eacbc2a3b0ee8022001be0413cd764b931aa9 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Thu, 13 Mar 2025 13:38:53 +0800 Subject: [PATCH 062/226] Enable Gaudi3, Rocm and Arc on manually release test. (#1615) 1. Enable Gaudi3, Rocm and Arc on manually release test. 2. Fix the issue that manual workflow can't be canceled. Signed-off-by: ZePan110 Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: Chingis Yundunov --- .github/workflows/_example-workflow.yml | 41 ++++++++++++++++++- .github/workflows/_run-docker-compose.yml | 9 +++- .github/workflows/manual-example-workflow.yml | 3 +- 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/.github/workflows/_example-workflow.yml b/.github/workflows/_example-workflow.yml index f3b717a284..d56099a476 100644 --- a/.github/workflows/_example-workflow.yml +++ b/.github/workflows/_example-workflow.yml @@ -50,10 +50,26 @@ on: type: boolean jobs: + pre-build-image-check: + runs-on: ubuntu-latest + outputs: + should_skip: ${{ steps.check-skip.outputs.should_skip }} + steps: + - name: Check if job should be skipped + id: check-skip + run: | + if [[ "${{ inputs.node }}" == "gaudi3" || "${{ inputs.node }}" == "rocm" || "${{ inputs.node }}" == "arc" ]]; then + echo "should_skip=true" >> $GITHUB_OUTPUT + else + echo "should_skip=false" >> $GITHUB_OUTPUT + fi + #################################################################################################### # Image Build #################################################################################################### build-images: + needs: [pre-build-image-check] + if: ${{ needs.pre-build-image-check.outputs.should_skip == 'false' }} runs-on: "docker-build-${{ inputs.node }}" steps: - name: Clean Up Working Directory @@ -105,12 +121,33 @@ jobs: inject_commit: ${{ inputs.inject_commit }} tag: ${{ inputs.tag }} + pre-compose-test-check: + needs: [pre-build-image-check, build-images] + if: always() + runs-on: ubuntu-latest + outputs: + run_compose: ${{ steps.check-compose.outputs.run_compose }} + steps: + - name: Check if job should be skipped + id: check-compose + run: | + set -x + run_compose="false" + if [[ ${{ inputs.test_compose }} ]]; then + if [[ "${{ needs.pre-build-image-check.outputs.should_skip }}" == "false" && "${{ needs.build-images.result}}" == "success" || "${{ needs.pre-build-image-check.outputs.should_skip }}" == "true" ]]; then + run_compose="true" + fi + fi + echo "run_compose=$run_compose" + echo "run_compose=$run_compose" >> $GITHUB_OUTPUT + + #################################################################################################### # Docker Compose Test #################################################################################################### test-example-compose: - needs: [build-images] - if: ${{ fromJSON(inputs.test_compose) }} + needs: [pre-compose-test-check] + if: ${{ always() && needs.pre-compose-test-check.outputs.run_compose == 'true' }} uses: ./.github/workflows/_run-docker-compose.yml with: tag: ${{ inputs.tag }} diff --git a/.github/workflows/_run-docker-compose.yml b/.github/workflows/_run-docker-compose.yml index f21c3202f9..a84912ed36 100644 --- a/.github/workflows/_run-docker-compose.yml +++ b/.github/workflows/_run-docker-compose.yml @@ -64,9 +64,14 @@ jobs: cd ${{ github.workspace }}/${{ inputs.example }}/tests run_test_cases="" - default_test_case=$(find . -type f -name "test_compose_on_${{ inputs.hardware }}.sh" | cut -d/ -f2) + if [ "${{ inputs.hardware }}" == "gaudi2" ] || [ "${{ inputs.hardware }}" == "gaudi3" ]; then + hardware="gaudi" + else + hardware="${{ inputs.hardware }}" + fi + default_test_case=$(find . -type f -name "test_compose_on_$hardware.sh" | cut -d/ -f2) if [ "$default_test_case" ]; then run_test_cases="$default_test_case"; fi - other_test_cases=$(find . -type f -name "test_compose_*_on_${{ inputs.hardware }}.sh" | cut -d/ -f2) + other_test_cases=$(find . -type f -name "test_compose_*_on_$hardware.sh" | cut -d/ -f2) echo "default_test_case=$default_test_case" echo "other_test_cases=$other_test_cases" diff --git a/.github/workflows/manual-example-workflow.yml b/.github/workflows/manual-example-workflow.yml index 3a98b3d40e..9616f87032 100644 --- a/.github/workflows/manual-example-workflow.yml +++ b/.github/workflows/manual-example-workflow.yml @@ -7,7 +7,7 @@ on: inputs: nodes: default: "gaudi,xeon" - description: "Hardware to run test" + description: "Hardware to run test gaudi,gaudi3,xeon,rocm,arc" required: true type: string examples: @@ -96,7 +96,6 @@ jobs: run-examples: needs: [get-test-matrix] #[get-test-matrix, build-deploy-gmc] - if: always() strategy: matrix: example: ${{ fromJson(needs.get-test-matrix.outputs.examples) }} From a658f80883c9986aada38789811599e91efbaa78 Mon Sep 17 00:00:00 2001 From: CharleneHu-42 <37971369+CharleneHu-42@users.noreply.github.com> Date: Thu, 13 Mar 2025 13:50:28 +0800 Subject: [PATCH 063/226] Refine README with highlighted examples and updated support info (#1006) Signed-off-by: CharleneHu-42 Co-authored-by: Yi Yao Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ying Hu Signed-off-by: Chingis Yundunov --- README.md | 16 ++++++++++++++ supported_examples.md | 51 ++++++++++++++++++++++++++++++------------- 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 369e504200..283ffb12e2 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,22 @@ GenAIExamples are designed to give developers an easy entry into generative AI, [GenAIEval](https://github.com/opea-project/GenAIEval) measures service performance metrics such as throughput, latency, and accuracy for GenAIExamples. This feature helps users compare performance across various hardware configurations easily. +## Use Cases + +Below are some highlighted GenAI use cases across various application scenarios: + +| Scenario | Use Case | +| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| Question Answering | [ChatQnA](ChatQnA) ✨: Chatbot with Retrieval Augmented Generation (RAG).
[VisualQnA](VisualQnA) ✨: Visual Question-answering. | +| Image Generation | [Text2Image](Text2Image) ✨: Text-to-image generation. | +| Content Summarization | [DocSum](DocSum): Document Summarization Application. | +| FAQ Generation | [FaqGen](FaqGen): Frequently asked questions (FAQs) generation from your documents, legal texts, customer queries etc. | +| Code Generation | [CodeGen](CodeGen): Gen-AI Powered Code Generator. | +| Information Retrieval | [DocIndexRetriever](DocIndexRetriever): Document Retrieval with Retrieval Augmented Generation (RAG). | +| Fine-tuning | [InstructionTuning](InstructionTuning): Application of Instruction Tuning. | + +For the full list of the available use cases and their supported deployment type, please refer [here](#deploy-examples). + ## Documentation The GenAIExamples [documentation](https://opea-project.github.io/latest/examples/index.html) contains a comprehensive guide on all available examples including architecture, deployment guides, and more. Information on GenAIComps, GenAIInfra, and GenAIEval can also be found there. diff --git a/supported_examples.md b/supported_examples.md index 0754be3eee..a0562a7145 100644 --- a/supported_examples.md +++ b/supported_examples.md @@ -105,9 +105,9 @@ This document introduces the supported examples of GenAIExamples. The supported [VisualQnA](./VisualQnA/README.md) is an example of chatbot for question and answering based on the images. -| LVM | HW | Description | -| --------------------------------------------------------------------------------------------- | ------ | ----------- | -| [llava-hf/llava-v1.6-mistral-7b-hf](https://huggingface.co/llava-hf/llava-v1.6-mistral-7b-hf) | Gaudi2 | Chatbot | +| LVM | HW | Description | +| --------------------------------------------------------------------------------------------- | ----------- | ----------- | +| [llava-hf/llava-v1.6-mistral-7b-hf](https://huggingface.co/llava-hf/llava-v1.6-mistral-7b-hf) | Xeon/Gaudi2 | Chatbot | ### VideoQnA @@ -122,27 +122,27 @@ By default, the embedding and LVM models are set to a default value as listed be ### RerankFinetuning -Rerank model finetuning example is for training rerank model on a dataset for improving its capability on specific field. +[Rerank model finetuning](./RerankFinetuning/README.md) example is for training rerank model on a dataset for improving its capability on specific field. By default, the base model is set to a default value as listed below: -| Service | Base Model | HW | Description | -| ----------------- | ------------------------------------------------------------------------- | ---- | ------------------------------- | -| Rerank Finetuning | [BAAI/bge-reranker-large](https://huggingface.co/BAAI/bge-reranker-large) | Xeon | Rerank model finetuning service | +| Service | Base Model | HW | Description | +| ----------------- | ------------------------------------------------------------------------- | ----------- | ------------------------------- | +| Rerank Finetuning | [BAAI/bge-reranker-large](https://huggingface.co/BAAI/bge-reranker-large) | Xeon/Gaudi2 | Rerank model finetuning service | ### InstructionTuning -The Instruction Tuning example is designed to further train large language models (LLMs) on a dataset consisting of (instruction, output) pairs using supervised learning. This process bridges the gap between the LLM's original objective of next-word prediction and the user’s objective of having the model follow human instructions accurately. By leveraging Instruction Tuning, this example enhances the LLM's ability to better understand and execute specific tasks, improving the model's alignment with user instructions and its overall performance. +The [Instruction Tuning](./InstructionTuning/README.md) example is designed to further train large language models (LLMs) on a dataset consisting of (instruction, output) pairs using supervised learning. This process bridges the gap between the LLM's original objective of next-word prediction and the user’s objective of having the model follow human instructions accurately. By leveraging Instruction Tuning, this example enhances the LLM's ability to better understand and execute specific tasks, improving the model's alignment with user instructions and its overall performance. By default, the base model is set to a default value as listed below: -| Service | Base Model | HW | Description | -| ----------------- | ------------------------------------------------------------------------------------- | ---------- | ------------------------------------ | -| InstructionTuning | [meta-llama/Llama-2-7b-chat-hf](https://huggingface.co/meta-llama/Llama-2-7b-chat-hf) | Xeon/Gaudi | LLM model Instruction Tuning service | +| Service | Base Model | HW | Description | +| ----------------- | ------------------------------------------------------------------------------------- | ----------- | ------------------------------------ | +| InstructionTuning | [meta-llama/Llama-2-7b-chat-hf](https://huggingface.co/meta-llama/Llama-2-7b-chat-hf) | Xeon/Gaudi2 | LLM model Instruction Tuning service | ### DocIndexRetriever -The DocRetriever example demonstrates how to match user queries with free-text records using various retrieval methods. It plays a key role in Retrieval-Augmented Generation (RAG) systems by dynamically fetching relevant information from external sources, ensuring responses are factual and up-to-date. Powered by vector databases, DocRetriever enables efficient, semantic retrieval by storing data as vectors and quickly identifying the most relevant documents based on similarity. +The [DocRetriever](./DocIndexRetriever/README.md) example demonstrates how to match user queries with free-text records using various retrieval methods. It plays a key role in Retrieval-Augmented Generation (RAG) systems by dynamically fetching relevant information from external sources, ensuring responses are factual and up-to-date. Powered by vector databases, DocRetriever enables efficient, semantic retrieval by storing data as vectors and quickly identifying the most relevant documents based on similarity. | Framework | Embedding | Vector Database | Serving | HW | Description | | ------------------------------------------------------------------------------ | --------------------------------------------------- | -------------------------- | --------------------------------------------------------------- | ----------- | -------------------------- | @@ -150,7 +150,7 @@ The DocRetriever example demonstrates how to match user queries with free-text r ### AgentQnA -The AgentQnA example demonstrates a hierarchical, multi-agent system designed for question-answering tasks. A supervisor agent interacts directly with the user, delegating tasks to a worker agent and utilizing various tools to gather information and generate answers. The worker agent primarily uses a retrieval tool to respond to the supervisor's queries. Additionally, the supervisor can access other tools, such as APIs to query knowledge graphs, SQL databases, or external knowledge bases, to enhance the accuracy and relevance of its responses. +The [AgentQnA](./AgentQnA/README.md) example demonstrates a hierarchical, multi-agent system designed for question-answering tasks. A supervisor agent interacts directly with the user, delegating tasks to a worker agent and utilizing various tools to gather information and generate answers. The worker agent primarily uses a retrieval tool to respond to the supervisor's queries. Additionally, the supervisor can access other tools, such as APIs to query knowledge graphs, SQL databases, or external knowledge bases, to enhance the accuracy and relevance of its responses. Worker agent uses open-source websearch tool (duckduckgo), agents use OpenAI GPT-4o-mini as llm backend. @@ -158,7 +158,7 @@ Worker agent uses open-source websearch tool (duckduckgo), agents use OpenAI GPT ### AudioQnA -The AudioQnA example demonstrates the integration of Generative AI (GenAI) models for performing question-answering (QnA) on audio files, with the added functionality of Text-to-Speech (TTS) for generating spoken responses. The example showcases how to convert audio input to text using Automatic Speech Recognition (ASR), generate answers to user queries using a language model, and then convert those answers back to speech using Text-to-Speech (TTS). +The [AudioQnA](./AudioQnA/README.md) example demonstrates the integration of Generative AI (GenAI) models for performing question-answering (QnA) on audio files, with the added functionality of Text-to-Speech (TTS) for generating spoken responses. The example showcases how to convert audio input to text using Automatic Speech Recognition (ASR), generate answers to user queries using a language model, and then convert those answers back to speech using Text-to-Speech (TTS). @@ -179,7 +179,7 @@ The AudioQnA example demonstrates the integration of Generative AI (GenAI) model ### FaqGen -FAQ Generation Application leverages the power of large language models (LLMs) to revolutionize the way you interact with and comprehend complex textual data. By harnessing cutting-edge natural language processing techniques, our application can automatically generate comprehensive and natural-sounding frequently asked questions (FAQs) from your documents, legal texts, customer queries, and other sources. In this example use case, we utilize LangChain to implement FAQ Generation and facilitate LLM inference using Text Generation Inference on Intel Xeon and Gaudi2 processors. +[FAQ Generation](./FaqGen/README.md) application leverages the power of large language models (LLMs) to revolutionize the way you interact with and comprehend complex textual data. By harnessing cutting-edge natural language processing techniques, our application can automatically generate comprehensive and natural-sounding frequently asked questions (FAQs) from your documents, legal texts, customer queries, and other sources. In this example use case, we utilize LangChain to implement FAQ Generation and facilitate LLM inference using Text Generation Inference on Intel Xeon and Gaudi2 processors. | Framework | LLM | Serving | HW | Description | | ------------------------------------------------------------------------------ | ----------------------------------------------------------------- | --------------------------------------------------------------- | ----------- | ----------- | | [LangChain](https://www.langchain.com)/[LlamaIndex](https://www.llamaindex.ai) | [Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) | [TGI](https://github.com/huggingface/text-generation-inference) | Xeon/Gaudi2 | Chatbot | @@ -199,3 +199,24 @@ FAQ Generation Application leverages the power of large language models (LLMs) t ### ProductivitySuite [Productivity Suite](./ProductivitySuite/README.md) streamlines your workflow to boost productivity. It leverages the power of OPEA microservices to deliver a comprehensive suite of features tailored to meet the diverse needs of modern enterprises. + +### DBQnA + +[DBQnA](./DBQnA/README.md) converts your natural language query into an SQL query, automatically executes the generated query on the database and delivers real-time query results. +| Framework | LLM | Database | HW | Description | +|----------------------------------------|-------------------------------------------------------------------------------------------------|-------------------------------------------|------|----------------------------| +| [LangChain](https://www.langchain.com) | [mistralai/Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3) | [PostgresDB](https://www.postgresql.org/) | Xeon | Natural language SQL agent | + +### Text2Image + +[Text2Image](./Text2Image/README.md) generates image based on your provided text. +| Framework | LDM | HW | Description | +|----------------------------------------|--------------------------------------------------------------------------------------------------------|-------------|-------------| +| [LangChain](https://www.langchain.com) | [stabilityai/stable-diffusion](https://huggingface.co/stabilityai/stable-diffusion-3-medium-diffusers) | Xeon/Gaudi2 | Text2Image | + +### AvatarChatbot + +[AvatarChatbot](./AvatarChatbot/README.md) example is a chatbot with a visual character that provides users dynamic, engaging interactions, by leveraging multiple generative AI components including LLM, ASR (Audio-Speech-Recognition), and TTS (Text-To-Speech). +| LLM | ASR | TTS | Animation | HW | Description | +|-------------------------------------------------------------------------------|---------------------------------------------------------------------|---------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------|-------------|----------------------------| +| [Intel/neural-chat-7b-v3-3](https://huggingface.co/Intel/neural-chat-7b-v3-3) | [openai/whisper-small](https://huggingface.co/openai/whisper-small) | [microsoft/SpeechT5](https://huggingface.co/microsoft/speecht5_tts) | [Rudrabha/Wav2Lip](https://github.com/Rudrabha/Wav2Lip)
[TencentARC/GFPGAN](https://github.com/TencentARC/GFPGAN) | Xeon/Gaudi2 | Interactive chatbot Avatar | From d12765ba908826ceeff9f08952d2b06b837c150e Mon Sep 17 00:00:00 2001 From: "Wang, Kai Lawrence" <109344418+wangkl2@users.noreply.github.com> Date: Fri, 14 Mar 2025 09:56:33 +0800 Subject: [PATCH 064/226] [AudioQnA] Enable vLLM and set it as default LLM serving (#1657) Signed-off-by: Wang, Kai Lawrence Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: Chingis Yundunov --- AudioQnA/audioqna.py | 2 +- AudioQnA/audioqna_multilang.py | 2 +- .../docker_compose/intel/cpu/xeon/README.md | 111 ++++++++++--- .../intel/cpu/xeon/compose.yaml | 25 +-- .../intel/cpu/xeon/compose_multilang.yaml | 26 ++-- .../intel/cpu/xeon/compose_tgi.yaml | 87 +++++++++++ .../docker_compose/intel/cpu/xeon/set_env.sh | 2 +- .../docker_compose/intel/hpu/gaudi/README.md | 115 +++++++++++--- .../intel/hpu/gaudi/compose.yaml | 29 ++-- .../intel/hpu/gaudi/compose_tgi.yaml | 108 +++++++++++++ .../docker_compose/intel/hpu/gaudi/set_env.sh | 8 +- AudioQnA/docker_image_build/build.yaml | 12 ++ AudioQnA/tests/test_compose_on_gaudi.sh | 23 ++- AudioQnA/tests/test_compose_on_xeon.sh | 19 ++- AudioQnA/tests/test_compose_tgi_on_gaudi.sh | 146 ++++++++++++++++++ AudioQnA/tests/test_compose_tgi_on_xeon.sh | 137 ++++++++++++++++ 16 files changed, 750 insertions(+), 102 deletions(-) create mode 100644 AudioQnA/docker_compose/intel/cpu/xeon/compose_tgi.yaml create mode 100644 AudioQnA/docker_compose/intel/hpu/gaudi/compose_tgi.yaml create mode 100644 AudioQnA/tests/test_compose_tgi_on_gaudi.sh create mode 100644 AudioQnA/tests/test_compose_tgi_on_xeon.sh diff --git a/AudioQnA/audioqna.py b/AudioQnA/audioqna.py index f74e58053f..dcb59633c0 100644 --- a/AudioQnA/audioqna.py +++ b/AudioQnA/audioqna.py @@ -16,7 +16,7 @@ SPEECHT5_SERVER_PORT = int(os.getenv("SPEECHT5_SERVER_PORT", 7055)) LLM_SERVER_HOST_IP = os.getenv("LLM_SERVER_HOST_IP", "0.0.0.0") LLM_SERVER_PORT = int(os.getenv("LLM_SERVER_PORT", 3006)) -LLM_MODEL_ID = os.getenv("LLM_MODEL_ID", "Intel/neural-chat-7b-v3-3") +LLM_MODEL_ID = os.getenv("LLM_MODEL_ID", "meta-llama/Meta-Llama-3-8B-Instruct") def align_inputs(self, inputs, cur_node, runtime_graph, llm_parameters_dict, **kwargs): diff --git a/AudioQnA/audioqna_multilang.py b/AudioQnA/audioqna_multilang.py index edc14cc93c..8f4a65e748 100644 --- a/AudioQnA/audioqna_multilang.py +++ b/AudioQnA/audioqna_multilang.py @@ -17,7 +17,7 @@ GPT_SOVITS_SERVER_PORT = int(os.getenv("GPT_SOVITS_SERVER_PORT", 9088)) LLM_SERVER_HOST_IP = os.getenv("LLM_SERVER_HOST_IP", "0.0.0.0") LLM_SERVER_PORT = int(os.getenv("LLM_SERVER_PORT", 8888)) -LLM_MODEL_ID = os.getenv("LLM_MODEL_ID", "Intel/neural-chat-7b-v3-3") +LLM_MODEL_ID = os.getenv("LLM_MODEL_ID", "meta-llama/Meta-Llama-3-8B-Instruct") def align_inputs(self, inputs, cur_node, runtime_graph, llm_parameters_dict, **kwargs): diff --git a/AudioQnA/docker_compose/intel/cpu/xeon/README.md b/AudioQnA/docker_compose/intel/cpu/xeon/README.md index 3f91c02e02..aabaf36595 100644 --- a/AudioQnA/docker_compose/intel/cpu/xeon/README.md +++ b/AudioQnA/docker_compose/intel/cpu/xeon/README.md @@ -2,6 +2,10 @@ This document outlines the deployment process for a AudioQnA application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on Intel Xeon server. +The default pipeline deploys with vLLM as the LLM serving component. It also provides options of using TGI backend for LLM microservice, please refer to [Start the MegaService](#-start-the-megaservice) section in this page. + +Note: The default LLM is `meta-llama/Meta-Llama-3-8B-Instruct`. Before deploying the application, please make sure either you've requested and been granted the access to it on [Huggingface](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) or you've downloaded the model locally from [ModelScope](https://www.modelscope.cn/models). + ## 🚀 Build Docker images ### 1. Source Code install GenAIComps @@ -17,9 +21,15 @@ cd GenAIComps docker build -t opea/whisper:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/asr/src/integrations/dependency/whisper/Dockerfile . ``` -### 3. Build LLM Image +### 3. Build vLLM Image -Intel Xeon optimized image hosted in huggingface repo will be used for TGI service: ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu (https://github.com/huggingface/text-generation-inference) +```bash +git clone https://github.com/vllm-project/vllm.git +cd ./vllm/ +VLLM_VER="$(git describe --tags "$(git rev-list --tags --max-count=1)" )" +git checkout ${VLLM_VER} +docker build --no-cache --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile.cpu -t opea/vllm:latest --shm-size=128g . +``` ### 4. Build TTS Image @@ -43,9 +53,10 @@ docker build --no-cache -t opea/audioqna:latest --build-arg https_proxy=$https_p Then run the command `docker images`, you will have following images ready: 1. `opea/whisper:latest` -2. `opea/speecht5:latest` -3. `opea/audioqna:latest` -4. `opea/gpt-sovits:latest` (optional) +2. `opea/vllm:latest` +3. `opea/speecht5:latest` +4. `opea/audioqna:latest` +5. `opea/gpt-sovits:latest` (optional) ## 🚀 Set the environment variables @@ -55,7 +66,7 @@ Before starting the services with `docker compose`, you have to recheck the foll export host_ip= # export host_ip=$(hostname -I | awk '{print $1}') export HUGGINGFACEHUB_API_TOKEN= -export LLM_MODEL_ID=Intel/neural-chat-7b-v3-3 +export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct" export MEGA_SERVICE_HOST_IP=${host_ip} export WHISPER_SERVER_HOST_IP=${host_ip} @@ -73,40 +84,90 @@ export BACKEND_SERVICE_ENDPOINT=http://${host_ip}:3008/v1/audioqna or use set_env.sh file to setup environment variables. -Note: Please replace with host_ip with your external IP address, do not use localhost. +Note: + +- Please replace with host_ip with your external IP address, do not use localhost. +- If you are in a proxy environment, also set the proxy-related environment variables: + +``` +export http_proxy="Your_HTTP_Proxy" +export https_proxy="Your_HTTPs_Proxy" +# Example: no_proxy="localhost, 127.0.0.1, 192.168.1.1" +export no_proxy="Your_No_Proxy",${host_ip},whisper-service,speecht5-service,gpt-sovits-service,tgi-service,vllm-service,audioqna-xeon-backend-server,audioqna-xeon-ui-server +``` ## 🚀 Start the MegaService ```bash cd GenAIExamples/AudioQnA/docker_compose/intel/cpu/xeon/ +``` + +If use vLLM as the LLM serving backend: + +``` docker compose up -d # multilang tts (optional) docker compose -f compose_multilang.yaml up -d ``` +If use TGI as the LLM serving backend: + +``` +docker compose -f compose_tgi.yaml up -d +``` + ## 🚀 Test MicroServices -```bash -# whisper service -wget https://github.com/intel/intel-extension-for-transformers/raw/main/intel_extension_for_transformers/neural_chat/assets/audio/sample.wav -curl http://${host_ip}:7066/v1/audio/transcriptions \ - -H "Content-Type: multipart/form-data" \ - -F file="@./sample.wav" \ - -F model="openai/whisper-small" - -# tgi service -curl http://${host_ip}:3006/generate \ - -X POST \ - -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \ - -H 'Content-Type: application/json' +1. Whisper Service -# speecht5 service -curl http://${host_ip}:7055/v1/audio/speech -XPOST -d '{"input": "Who are you?"}' -H 'Content-Type: application/json' --output speech.mp3 + ```bash + wget https://github.com/intel/intel-extension-for-transformers/raw/main/intel_extension_for_transformers/neural_chat/assets/audio/sample.wav + curl http://${host_ip}:${WHISPER_SERVER_PORT}/v1/audio/transcriptions \ + -H "Content-Type: multipart/form-data" \ + -F file="@./sample.wav" \ + -F model="openai/whisper-small" + ``` -# gpt-sovits service (optional) -curl http://${host_ip}:9880/v1/audio/speech -XPOST -d '{"input": "Who are you?"}' -H 'Content-Type: application/json' --output speech.mp3 -``` +2. LLM backend Service + + In the first startup, this service will take more time to download, load and warm up the model. After it's finished, the service will be ready and the container (`vllm-service` or `tgi-service`) status shown via `docker ps` will be `healthy`. Before that, the status will be `health: starting`. + + Or try the command below to check whether the LLM serving is ready. + + ```bash + # vLLM service + docker logs vllm-service 2>&1 | grep complete + # If the service is ready, you will get the response like below. + INFO: Application startup complete. + ``` + + ```bash + # TGI service + docker logs tgi-service | grep Connected + # If the service is ready, you will get the response like below. + 2024-09-03T02:47:53.402023Z INFO text_generation_router::server: router/src/server.rs:2311: Connected + ``` + + Then try the `cURL` command below to validate services. + + ```bash + # either vLLM or TGI service + curl http://${host_ip}:${LLM_SERVER_PORT}/v1/chat/completions \ + -X POST \ + -d '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens":17}' \ + -H 'Content-Type: application/json' + ``` + +3. TTS Service + + ``` + # speecht5 service + curl http://${host_ip}:${SPEECHT5_SERVER_PORT}/v1/audio/speech -XPOST -d '{"input": "Who are you?"}' -H 'Content-Type: application/json' --output speech.mp3 + + # gpt-sovits service (optional) + curl http://${host_ip}:${GPT_SOVITS_SERVER_PORT}/v1/audio/speech -XPOST -d '{"input": "Who are you?"}' -H 'Content-Type: application/json' --output speech.mp3 + ``` ## 🚀 Test MegaService diff --git a/AudioQnA/docker_compose/intel/cpu/xeon/compose.yaml b/AudioQnA/docker_compose/intel/cpu/xeon/compose.yaml index 3b47780d80..1fe5e6b2a6 100644 --- a/AudioQnA/docker_compose/intel/cpu/xeon/compose.yaml +++ b/AudioQnA/docker_compose/intel/cpu/xeon/compose.yaml @@ -6,7 +6,7 @@ services: image: ${REGISTRY:-opea}/whisper:${TAG:-latest} container_name: whisper-service ports: - - "7066:7066" + - ${WHISPER_SERVER_PORT:-7066}:7066 ipc: host environment: no_proxy: ${no_proxy} @@ -17,38 +17,41 @@ services: image: ${REGISTRY:-opea}/speecht5:${TAG:-latest} container_name: speecht5-service ports: - - "7055:7055" + - ${SPEECHT5_SERVER_PORT:-7055}:7055 ipc: host environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} restart: unless-stopped - tgi-service: - image: ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu - container_name: tgi-service + vllm-service: + image: ${REGISTRY:-opea}/vllm:${TAG:-latest} + container_name: vllm-service ports: - - "3006:80" + - ${LLM_SERVER_PORT:-3006}:80 volumes: - - "${MODEL_CACHE:-./data}:/data" - shm_size: 1g + - "${MODEL_CACHE:-./data}:/root/.cache/huggingface/hub" + shm_size: 128g environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + LLM_MODEL_ID: ${LLM_MODEL_ID} + VLLM_TORCH_PROFILER_DIR: "/mnt" + LLM_SERVER_PORT: ${LLM_SERVER_PORT} healthcheck: - test: ["CMD-SHELL", "curl -f http://$host_ip:3006/health || exit 1"] + test: ["CMD-SHELL", "curl -f http://$host_ip:${LLM_SERVER_PORT}/health || exit 1"] interval: 10s timeout: 10s retries: 100 - command: --model-id ${LLM_MODEL_ID} --cuda-graphs 0 + command: --model ${LLM_MODEL_ID} --host 0.0.0.0 --port 80 audioqna-xeon-backend-server: image: ${REGISTRY:-opea}/audioqna:${TAG:-latest} container_name: audioqna-xeon-backend-server depends_on: - whisper-service - - tgi-service + - vllm-service - speecht5-service ports: - "3008:8888" diff --git a/AudioQnA/docker_compose/intel/cpu/xeon/compose_multilang.yaml b/AudioQnA/docker_compose/intel/cpu/xeon/compose_multilang.yaml index fde5a56902..3aecacf591 100644 --- a/AudioQnA/docker_compose/intel/cpu/xeon/compose_multilang.yaml +++ b/AudioQnA/docker_compose/intel/cpu/xeon/compose_multilang.yaml @@ -6,7 +6,7 @@ services: image: ${REGISTRY:-opea}/whisper:${TAG:-latest} container_name: whisper-service ports: - - "7066:7066" + - ${WHISPER_SERVER_PORT:-7066}:7066 ipc: host environment: no_proxy: ${no_proxy} @@ -18,27 +18,35 @@ services: image: ${REGISTRY:-opea}/gpt-sovits:${TAG:-latest} container_name: gpt-sovits-service ports: - - "9880:9880" + - ${GPT_SOVITS_SERVER_PORT:-9880}:9880 ipc: host environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} restart: unless-stopped - tgi-service: - image: ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu - container_name: tgi-service + vllm-service: + image: ${REGISTRY:-opea}/vllm:${TAG:-latest} + container_name: vllm-service ports: - - "3006:80" + - ${LLM_SERVER_PORT:-3006}:80 volumes: - - "${MODEL_CACHE:-./data}:/data" - shm_size: 1g + - "${MODEL_CACHE:-./data}:/root/.cache/huggingface/hub" + shm_size: 128g environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} - command: --model-id ${LLM_MODEL_ID} --cuda-graphs 0 + LLM_MODEL_ID: ${LLM_MODEL_ID} + VLLM_TORCH_PROFILER_DIR: "/mnt" + LLM_SERVER_PORT: ${LLM_SERVER_PORT} + healthcheck: + test: ["CMD-SHELL", "curl -f http://$host_ip:${LLM_SERVER_PORT}/health || exit 1"] + interval: 10s + timeout: 10s + retries: 100 + command: --model ${LLM_MODEL_ID} --host 0.0.0.0 --port 80 audioqna-xeon-backend-server: image: ${REGISTRY:-opea}/audioqna-multilang:${TAG:-latest} container_name: audioqna-xeon-backend-server diff --git a/AudioQnA/docker_compose/intel/cpu/xeon/compose_tgi.yaml b/AudioQnA/docker_compose/intel/cpu/xeon/compose_tgi.yaml new file mode 100644 index 0000000000..d421f488fd --- /dev/null +++ b/AudioQnA/docker_compose/intel/cpu/xeon/compose_tgi.yaml @@ -0,0 +1,87 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +services: + whisper-service: + image: ${REGISTRY:-opea}/whisper:${TAG:-latest} + container_name: whisper-service + ports: + - ${WHISPER_SERVER_PORT:-7066}:7066 + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + restart: unless-stopped + speecht5-service: + image: ${REGISTRY:-opea}/speecht5:${TAG:-latest} + container_name: speecht5-service + ports: + - ${SPEECHT5_SERVER_PORT:-7055}:7055 + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + restart: unless-stopped + tgi-service: + image: ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu + container_name: tgi-service + ports: + - ${LLM_SERVER_PORT:-3006}:80 + volumes: + - "${MODEL_CACHE:-./data}:/data" + shm_size: 1g + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + LLM_SERVER_PORT: ${LLM_SERVER_PORT} + healthcheck: + test: ["CMD-SHELL", "curl -f http://$host_ip:${LLM_SERVER_PORT}/health || exit 1"] + interval: 10s + timeout: 10s + retries: 100 + command: --model-id ${LLM_MODEL_ID} --cuda-graphs 0 + audioqna-xeon-backend-server: + image: ${REGISTRY:-opea}/audioqna:${TAG:-latest} + container_name: audioqna-xeon-backend-server + depends_on: + - whisper-service + - tgi-service + - speecht5-service + ports: + - "3008:8888" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP} + - WHISPER_SERVER_HOST_IP=${WHISPER_SERVER_HOST_IP} + - WHISPER_SERVER_PORT=${WHISPER_SERVER_PORT} + - LLM_SERVER_HOST_IP=${LLM_SERVER_HOST_IP} + - LLM_SERVER_PORT=${LLM_SERVER_PORT} + - LLM_MODEL_ID=${LLM_MODEL_ID} + - SPEECHT5_SERVER_HOST_IP=${SPEECHT5_SERVER_HOST_IP} + - SPEECHT5_SERVER_PORT=${SPEECHT5_SERVER_PORT} + ipc: host + restart: always + audioqna-xeon-ui-server: + image: ${REGISTRY:-opea}/audioqna-ui:${TAG:-latest} + container_name: audioqna-xeon-ui-server + depends_on: + - audioqna-xeon-backend-server + ports: + - "5173:5173" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - CHAT_URL=${BACKEND_SERVICE_ENDPOINT} + ipc: host + restart: always + +networks: + default: + driver: bridge diff --git a/AudioQnA/docker_compose/intel/cpu/xeon/set_env.sh b/AudioQnA/docker_compose/intel/cpu/xeon/set_env.sh index e98f6e04ec..adc652f169 100644 --- a/AudioQnA/docker_compose/intel/cpu/xeon/set_env.sh +++ b/AudioQnA/docker_compose/intel/cpu/xeon/set_env.sh @@ -8,7 +8,7 @@ export host_ip=$(hostname -I | awk '{print $1}') export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} # -export LLM_MODEL_ID=Intel/neural-chat-7b-v3-3 +export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct" export MEGA_SERVICE_HOST_IP=${host_ip} export WHISPER_SERVER_HOST_IP=${host_ip} diff --git a/AudioQnA/docker_compose/intel/hpu/gaudi/README.md b/AudioQnA/docker_compose/intel/hpu/gaudi/README.md index b60253a147..602b99ea22 100644 --- a/AudioQnA/docker_compose/intel/hpu/gaudi/README.md +++ b/AudioQnA/docker_compose/intel/hpu/gaudi/README.md @@ -2,6 +2,10 @@ This document outlines the deployment process for a AudioQnA application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on Intel Gaudi server. +The default pipeline deploys with vLLM as the LLM serving component. It also provides options of using TGI backend for LLM microservice, please refer to [Start the MegaService](#-start-the-megaservice) section in this page. + +Note: The default LLM is `meta-llama/Meta-Llama-3-8B-Instruct`. Before deploying the application, please make sure either you've requested and been granted the access to it on [Huggingface](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) or you've downloaded the model locally from [ModelScope](https://www.modelscope.cn/models). + ## 🚀 Build Docker images ### 1. Source Code install GenAIComps @@ -17,9 +21,13 @@ cd GenAIComps docker build -t opea/whisper-gaudi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/asr/src/integrations/dependency/whisper/Dockerfile.intel_hpu . ``` -### 3. Build LLM Image +### 3. Build vLLM Image -Intel Xeon optimized image hosted in huggingface repo will be used for TGI service: ghcr.io/huggingface/tgi-gaudi:2.0.6 (https://github.com/huggingface/tgi-gaudi) +git clone https://github.com/HabanaAI/vllm-fork.git +cd vllm-fork/ +VLLM_VER=$(git describe --tags "$(git rev-list --tags --max-count=1)") +git checkout ${VLLM_VER} +docker build --no-cache --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile.hpu -t opea/vllm-gaudi:latest --shm-size=128g . ### 4. Build TTS Image @@ -40,8 +48,9 @@ docker build --no-cache -t opea/audioqna:latest --build-arg https_proxy=$https_p Then run the command `docker images`, you will have following images ready: 1. `opea/whisper-gaudi:latest` -2. `opea/speecht5-gaudi:latest` -3. `opea/audioqna:latest` +2. `opea/vllm-gaudi:latest` +3. `opea/speecht5-gaudi:latest` +4. `opea/audioqna:latest` ## 🚀 Set the environment variables @@ -51,7 +60,12 @@ Before starting the services with `docker compose`, you have to recheck the foll export host_ip= # export host_ip=$(hostname -I | awk '{print $1}') export HUGGINGFACEHUB_API_TOKEN= -export LLM_MODEL_ID=Intel/neural-chat-7b-v3-3 +export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct" +# set vLLM parameters +export NUM_CARDS=1 +export BLOCK_SIZE=128 +export MAX_NUM_SEQS=256 +export MAX_SEQ_LEN_TO_CAPTURE=2048 export MEGA_SERVICE_HOST_IP=${host_ip} export WHISPER_SERVER_HOST_IP=${host_ip} @@ -65,37 +79,90 @@ export LLM_SERVER_PORT=3006 export BACKEND_SERVICE_ENDPOINT=http://${host_ip}:3008/v1/audioqna ``` +or use set_env.sh file to setup environment variables. + +Note: + +- Please replace with host_ip with your external IP address, do not use localhost. +- If you are in a proxy environment, also set the proxy-related environment variables: + +``` +export http_proxy="Your_HTTP_Proxy" +export https_proxy="Your_HTTPs_Proxy" +# Example: no_proxy="localhost, 127.0.0.1, 192.168.1.1" +export no_proxy="Your_No_Proxy",${host_ip},whisper-service,speecht5-service,tgi-service,vllm-service,audioqna-gaudi-backend-server,audioqna-gaudi-ui-server +``` + ## 🚀 Start the MegaService > **_NOTE:_** Users will need at least three Gaudi cards for AudioQnA. ```bash cd GenAIExamples/AudioQnA/docker_compose/intel/hpu/gaudi/ -docker compose up -d ``` -## 🚀 Test MicroServices - -```bash -# whisper service -curl http://${host_ip}:7066/v1/asr \ - -X POST \ - -d '{"audio": "UklGRigAAABXQVZFZm10IBIAAAABAAEARKwAAIhYAQACABAAAABkYXRhAgAAAAEA"}' \ - -H 'Content-Type: application/json' +If use vLLM as the LLM serving backend: -# tgi service -curl http://${host_ip}:3006/generate \ - -X POST \ - -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \ - -H 'Content-Type: application/json' +``` +docker compose up -d +``` -# speecht5 service -curl http://${host_ip}:7055/v1/tts \ - -X POST \ - -d '{"text": "Who are you?"}' \ - -H 'Content-Type: application/json' +If use TGI as the LLM serving backend: ``` +docker compose -f compose_tgi.yaml up -d +``` + +## 🚀 Test MicroServices + +1. Whisper Service + + ```bash + curl http://${host_ip}:${WHISPER_SERVER_PORT}/v1/asr \ + -X POST \ + -d '{"audio": "UklGRigAAABXQVZFZm10IBIAAAABAAEARKwAAIhYAQACABAAAABkYXRhAgAAAAEA"}' \ + -H 'Content-Type: application/json' + ``` + +2. LLM backend Service + + In the first startup, this service will take more time to download, load and warm up the model. After it's finished, the service will be ready and the container (`vllm-gaudi-service` or `tgi-gaudi-service`) status shown via `docker ps` will be `healthy`. Before that, the status will be `health: starting`. + + Or try the command below to check whether the LLM serving is ready. + + ```bash + # vLLM service + docker logs vllm-gaudi-service 2>&1 | grep complete + # If the service is ready, you will get the response like below. + INFO: Application startup complete. + ``` + + ```bash + # TGI service + docker logs tgi-gaudi-service | grep Connected + # If the service is ready, you will get the response like below. + 2024-09-03T02:47:53.402023Z INFO text_generation_router::server: router/src/server.rs:2311: Connected + ``` + + Then try the `cURL` command below to validate services. + + ```bash + # either vLLM or TGI service + curl http://${host_ip}:${LLM_SERVER_PORT}/v1/chat/completions \ + -X POST \ + -d '{"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens":17}' \ + -H 'Content-Type: application/json' + ``` + +3. TTS Service + + ``` + # speecht5 service + curl http://${host_ip}:${SPEECHT5_SERVER_PORT}/v1/tts + -X POST \ + -d '{"text": "Who are you?"}' \ + -H 'Content-Type: application/json' + ``` ## 🚀 Test MegaService diff --git a/AudioQnA/docker_compose/intel/hpu/gaudi/compose.yaml b/AudioQnA/docker_compose/intel/hpu/gaudi/compose.yaml index 9e43a355b5..db93cd8223 100644 --- a/AudioQnA/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/AudioQnA/docker_compose/intel/hpu/gaudi/compose.yaml @@ -6,7 +6,7 @@ services: image: ${REGISTRY:-opea}/whisper-gaudi:${TAG:-latest} container_name: whisper-service ports: - - "7066:7066" + - ${WHISPER_SERVER_PORT:-7066}:7066 ipc: host environment: no_proxy: ${no_proxy} @@ -22,7 +22,7 @@ services: image: ${REGISTRY:-opea}/speecht5-gaudi:${TAG:-latest} container_name: speecht5-service ports: - - "7055:7055" + - ${SPEECHT5_SERVER_PORT:-7055}:7055 ipc: host environment: no_proxy: ${no_proxy} @@ -34,28 +34,27 @@ services: cap_add: - SYS_NICE restart: unless-stopped - tgi-service: - image: ghcr.io/huggingface/tgi-gaudi:2.3.1 - container_name: tgi-gaudi-server + vllm-service: + image: ${REGISTRY:-opea}/vllm-gaudi:${TAG:-latest} + container_name: vllm-gaudi-service ports: - - "3006:80" + - ${LLM_SERVER_PORT:-3006}:80 volumes: - - "${MODEL_CACHE:-./data}:/data" + - "${MODEL_CACHE:-./data}:/root/.cache/huggingface/hub" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} - HUGGING_FACE_HUB_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} HF_HUB_DISABLE_PROGRESS_BARS: 1 HF_HUB_ENABLE_HF_TRANSFER: 0 HABANA_VISIBLE_DEVICES: all OMPI_MCA_btl_vader_single_copy_mechanism: none - ENABLE_HPU_GRAPH: true - LIMIT_HPU_GRAPH: true - USE_FLASH_ATTENTION: true - FLASH_ATTENTION_RECOMPUTE: true + LLM_MODEL_ID: ${LLM_MODEL_ID} + VLLM_TORCH_PROFILER_DIR: "/mnt" + LLM_SERVER_PORT: ${LLM_SERVER_PORT} healthcheck: - test: ["CMD-SHELL", "curl -f http://$host_ip:3006/health || exit 1"] + test: ["CMD-SHELL", "curl -f http://$host_ip:${LLM_SERVER_PORT}/health || exit 1"] interval: 10s timeout: 10s retries: 100 @@ -63,13 +62,13 @@ services: cap_add: - SYS_NICE ipc: host - command: --model-id ${LLM_MODEL_ID} --max-input-length 1024 --max-total-tokens 2048 + command: --model ${LLM_MODEL_ID} --tensor-parallel-size ${NUM_CARDS} --host 0.0.0.0 --port 80 --block-size ${BLOCK_SIZE} --max-num-seqs ${MAX_NUM_SEQS} --max-seq_len-to-capture ${MAX_SEQ_LEN_TO_CAPTURE} audioqna-gaudi-backend-server: image: ${REGISTRY:-opea}/audioqna:${TAG:-latest} container_name: audioqna-gaudi-backend-server depends_on: - whisper-service - - tgi-service + - vllm-service - speecht5-service ports: - "3008:8888" diff --git a/AudioQnA/docker_compose/intel/hpu/gaudi/compose_tgi.yaml b/AudioQnA/docker_compose/intel/hpu/gaudi/compose_tgi.yaml new file mode 100644 index 0000000000..f14bd8cb99 --- /dev/null +++ b/AudioQnA/docker_compose/intel/hpu/gaudi/compose_tgi.yaml @@ -0,0 +1,108 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +services: + whisper-service: + image: ${REGISTRY:-opea}/whisper-gaudi:${TAG:-latest} + container_name: whisper-service + ports: + - ${WHISPER_SERVER_PORT:-7066}:7066 + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HABANA_VISIBLE_DEVICES: all + OMPI_MCA_btl_vader_single_copy_mechanism: none + runtime: habana + cap_add: + - SYS_NICE + restart: unless-stopped + speecht5-service: + image: ${REGISTRY:-opea}/speecht5-gaudi:${TAG:-latest} + container_name: speecht5-service + ports: + - ${SPEECHT5_SERVER_PORT:-7055}:7055 + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HABANA_VISIBLE_DEVICES: all + OMPI_MCA_btl_vader_single_copy_mechanism: none + runtime: habana + cap_add: + - SYS_NICE + restart: unless-stopped + tgi-service: + image: ghcr.io/huggingface/tgi-gaudi:2.3.1 + container_name: tgi-gaudi-service + ports: + - ${LLM_SERVER_PORT:-3006}:80 + volumes: + - "${MODEL_CACHE:-./data}:/data" + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + HABANA_VISIBLE_DEVICES: all + OMPI_MCA_btl_vader_single_copy_mechanism: none + ENABLE_HPU_GRAPH: true + LIMIT_HPU_GRAPH: true + USE_FLASH_ATTENTION: true + FLASH_ATTENTION_RECOMPUTE: true + LLM_SERVER_PORT: ${LLM_SERVER_PORT} + healthcheck: + test: ["CMD-SHELL", "curl -f http://$host_ip:${LLM_SERVER_PORT}/health || exit 1"] + interval: 10s + timeout: 10s + retries: 100 + runtime: habana + cap_add: + - SYS_NICE + ipc: host + command: --model-id ${LLM_MODEL_ID} --max-input-length 1024 --max-total-tokens 2048 + audioqna-gaudi-backend-server: + image: ${REGISTRY:-opea}/audioqna:${TAG:-latest} + container_name: audioqna-gaudi-backend-server + depends_on: + - whisper-service + - tgi-service + - speecht5-service + ports: + - "3008:8888" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP} + - WHISPER_SERVER_HOST_IP=${WHISPER_SERVER_HOST_IP} + - WHISPER_SERVER_PORT=${WHISPER_SERVER_PORT} + - LLM_SERVER_HOST_IP=${LLM_SERVER_HOST_IP} + - LLM_SERVER_PORT=${LLM_SERVER_PORT} + - LLM_MODEL_ID=${LLM_MODEL_ID} + - SPEECHT5_SERVER_HOST_IP=${SPEECHT5_SERVER_HOST_IP} + - SPEECHT5_SERVER_PORT=${SPEECHT5_SERVER_PORT} + ipc: host + restart: always + audioqna-gaudi-ui-server: + image: ${REGISTRY:-opea}/audioqna-ui:${TAG:-latest} + container_name: audioqna-gaudi-ui-server + depends_on: + - audioqna-gaudi-backend-server + ports: + - "5173:5173" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - CHAT_URL=${BACKEND_SERVICE_ENDPOINT} + ipc: host + restart: always + +networks: + default: + driver: bridge diff --git a/AudioQnA/docker_compose/intel/hpu/gaudi/set_env.sh b/AudioQnA/docker_compose/intel/hpu/gaudi/set_env.sh index e98f6e04ec..179a8c2a24 100644 --- a/AudioQnA/docker_compose/intel/hpu/gaudi/set_env.sh +++ b/AudioQnA/docker_compose/intel/hpu/gaudi/set_env.sh @@ -8,7 +8,13 @@ export host_ip=$(hostname -I | awk '{print $1}') export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} # -export LLM_MODEL_ID=Intel/neural-chat-7b-v3-3 +export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct" + +# set vLLM parameters +export NUM_CARDS=1 +export BLOCK_SIZE=128 +export MAX_NUM_SEQS=256 +export MAX_SEQ_LEN_TO_CAPTURE=2048 export MEGA_SERVICE_HOST_IP=${host_ip} export WHISPER_SERVER_HOST_IP=${host_ip} diff --git a/AudioQnA/docker_image_build/build.yaml b/AudioQnA/docker_image_build/build.yaml index bc9f67d9c0..71bb44c810 100644 --- a/AudioQnA/docker_image_build/build.yaml +++ b/AudioQnA/docker_image_build/build.yaml @@ -71,3 +71,15 @@ services: dockerfile: comps/tts/src/integrations/dependency/gpt-sovits/Dockerfile extends: audioqna image: ${REGISTRY:-opea}/gpt-sovits:${TAG:-latest} + vllm: + build: + context: vllm + dockerfile: Dockerfile.cpu + extends: audioqna + image: ${REGISTRY:-opea}/vllm:${TAG:-latest} + vllm-gaudi: + build: + context: vllm-fork + dockerfile: Dockerfile.hpu + extends: audioqna + image: ${REGISTRY:-opea}/vllm-gaudi:${TAG:-latest} diff --git a/AudioQnA/tests/test_compose_on_gaudi.sh b/AudioQnA/tests/test_compose_on_gaudi.sh index fe5cff379a..1e356750e6 100644 --- a/AudioQnA/tests/test_compose_on_gaudi.sh +++ b/AudioQnA/tests/test_compose_on_gaudi.sh @@ -31,18 +31,27 @@ function build_docker_images() { cd $WORKPATH/docker_image_build git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + git clone https://github.com/HabanaAI/vllm-fork.git + cd vllm-fork/ + VLLM_VER=$(git describe --tags "$(git rev-list --tags --max-count=1)") + echo "Check out vLLM tag ${VLLM_VER}" + git checkout ${VLLM_VER} &> /dev/null && cd ../ + echo "Build all the images with --no-cache, check docker_image_build.log for details..." - service_list="audioqna audioqna-ui whisper-gaudi speecht5-gaudi" + service_list="audioqna audioqna-ui whisper-gaudi speecht5-gaudi vllm-gaudi" docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log - docker pull ghcr.io/huggingface/tgi-gaudi:2.0.6 docker images && sleep 1s } function start_services() { cd $WORKPATH/docker_compose/intel/hpu/gaudi export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} - export LLM_MODEL_ID=Intel/neural-chat-7b-v3-3 + export LLM_MODEL_ID=meta-llama/Meta-Llama-3-8B-Instruct + export NUM_CARDS=1 + export BLOCK_SIZE=128 + export MAX_NUM_SEQS=256 + export MAX_SEQ_LEN_TO_CAPTURE=2048 export MEGA_SERVICE_HOST_IP=${ip_address} export WHISPER_SERVER_HOST_IP=${ip_address} @@ -61,8 +70,8 @@ function start_services() { docker compose up -d > ${LOG_PATH}/start_services_with_compose.log n=0 until [[ "$n" -ge 200 ]]; do - docker logs tgi-gaudi-server > $LOG_PATH/tgi_service_start.log - if grep -q Connected $LOG_PATH/tgi_service_start.log; then + docker logs vllm-gaudi-service > $LOG_PATH/vllm_service_start.log 2>&1 + if grep -q complete $LOG_PATH/vllm_service_start.log; then break fi sleep 5s @@ -86,7 +95,7 @@ function validate_megaservice() { # always print the log docker logs whisper-service > $LOG_PATH/whisper-service.log docker logs speecht5-service > $LOG_PATH/tts-service.log - docker logs tgi-gaudi-server > $LOG_PATH/tgi-gaudi-server.log + docker logs vllm-gaudi-service > $LOG_PATH/vllm-gaudi-service.log docker logs audioqna-gaudi-backend-server > $LOG_PATH/audioqna-gaudi-backend-server.log echo "$response" | sed 's/^"//;s/"$//' | base64 -d > speech.mp3 @@ -126,7 +135,7 @@ function validate_megaservice() { function stop_docker() { cd $WORKPATH/docker_compose/intel/hpu/gaudi - docker compose stop && docker compose rm -f + docker compose -f compose.yaml stop && docker compose rm -f } function main() { diff --git a/AudioQnA/tests/test_compose_on_xeon.sh b/AudioQnA/tests/test_compose_on_xeon.sh index 11a86ba5c8..b1ff1164d2 100644 --- a/AudioQnA/tests/test_compose_on_xeon.sh +++ b/AudioQnA/tests/test_compose_on_xeon.sh @@ -31,18 +31,23 @@ function build_docker_images() { cd $WORKPATH/docker_image_build git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + git clone https://github.com/vllm-project/vllm.git + cd ./vllm/ + VLLM_VER="$(git describe --tags "$(git rev-list --tags --max-count=1)" )" + echo "Check out vLLM tag ${VLLM_VER}" + git checkout ${VLLM_VER} &> /dev/null && cd ../ + echo "Build all the images with --no-cache, check docker_image_build.log for details..." - service_list="audioqna audioqna-ui whisper speecht5" + service_list="audioqna audioqna-ui whisper speecht5 vllm" docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log - docker pull ghcr.io/huggingface/tgi-gaudi:2.0.6 docker images && sleep 1s } function start_services() { cd $WORKPATH/docker_compose/intel/cpu/xeon/ export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} - export LLM_MODEL_ID=Intel/neural-chat-7b-v3-3 + export LLM_MODEL_ID=meta-llama/Meta-Llama-3-8B-Instruct export MEGA_SERVICE_HOST_IP=${ip_address} export WHISPER_SERVER_HOST_IP=${ip_address} @@ -62,8 +67,8 @@ function start_services() { docker compose up -d > ${LOG_PATH}/start_services_with_compose.log n=0 until [[ "$n" -ge 200 ]]; do - docker logs tgi-service > $LOG_PATH/tgi_service_start.log - if grep -q Connected $LOG_PATH/tgi_service_start.log; then + docker logs vllm-service > $LOG_PATH/vllm_service_start.log 2>&1 + if grep -q complete $LOG_PATH/vllm_service_start.log; then break fi sleep 5s @@ -77,7 +82,7 @@ function validate_megaservice() { # always print the log docker logs whisper-service > $LOG_PATH/whisper-service.log docker logs speecht5-service > $LOG_PATH/tts-service.log - docker logs tgi-service > $LOG_PATH/tgi-service.log + docker logs vllm-service > $LOG_PATH/vllm-service.log docker logs audioqna-xeon-backend-server > $LOG_PATH/audioqna-xeon-backend-server.log echo "$response" | sed 's/^"//;s/"$//' | base64 -d > speech.mp3 @@ -117,7 +122,7 @@ function validate_megaservice() { function stop_docker() { cd $WORKPATH/docker_compose/intel/cpu/xeon/ - docker compose stop && docker compose rm -f + docker compose -f compose.yaml stop && docker compose rm -f } function main() { diff --git a/AudioQnA/tests/test_compose_tgi_on_gaudi.sh b/AudioQnA/tests/test_compose_tgi_on_gaudi.sh new file mode 100644 index 0000000000..5a046adfdb --- /dev/null +++ b/AudioQnA/tests/test_compose_tgi_on_gaudi.sh @@ -0,0 +1,146 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -e +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" +export REGISTRY=${IMAGE_REPO} +export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') + +function build_docker_images() { + opea_branch=${opea_branch:-"main"} + # If the opea_branch isn't main, replace the git clone branch in Dockerfile. + if [[ "${opea_branch}" != "main" ]]; then + cd $WORKPATH + OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" + NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" + find . -type f -name "Dockerfile*" | while read -r file; do + echo "Processing file: $file" + sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" + done + fi + + cd $WORKPATH/docker_image_build + git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + + echo "Build all the images with --no-cache, check docker_image_build.log for details..." + service_list="audioqna audioqna-ui whisper-gaudi speecht5-gaudi" + docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log + + docker pull ghcr.io/huggingface/tgi-gaudi:2.0.6 + docker images && sleep 1s +} + +function start_services() { + cd $WORKPATH/docker_compose/intel/hpu/gaudi + export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} + export LLM_MODEL_ID=meta-llama/Meta-Llama-3-8B-Instruct + + export MEGA_SERVICE_HOST_IP=${ip_address} + export WHISPER_SERVER_HOST_IP=${ip_address} + export SPEECHT5_SERVER_HOST_IP=${ip_address} + export LLM_SERVER_HOST_IP=${ip_address} + + export WHISPER_SERVER_PORT=7066 + export SPEECHT5_SERVER_PORT=7055 + export LLM_SERVER_PORT=3006 + + export BACKEND_SERVICE_ENDPOINT=http://${ip_address}:3008/v1/audioqna + export host_ip=${ip_address} + # sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env + + # Start Docker Containers + docker compose -f compose_tgi.yaml up -d > ${LOG_PATH}/start_services_with_compose.log + n=0 + until [[ "$n" -ge 200 ]]; do + docker logs tgi-gaudi-service > $LOG_PATH/tgi_service_start.log + if grep -q Connected $LOG_PATH/tgi_service_start.log; then + break + fi + sleep 5s + n=$((n+1)) + done + + n=0 + until [[ "$n" -ge 100 ]]; do + docker logs whisper-service > $LOG_PATH/whisper_service_start.log + if grep -q "Uvicorn server setup on port" $LOG_PATH/whisper_service_start.log; then + break + fi + sleep 5s + n=$((n+1)) + done +} + + +function validate_megaservice() { + response=$(http_proxy="" curl http://${ip_address}:3008/v1/audioqna -XPOST -d '{"audio": "UklGRigAAABXQVZFZm10IBIAAAABAAEARKwAAIhYAQACABAAAABkYXRhAgAAAAEA", "max_tokens":64}' -H 'Content-Type: application/json') + # always print the log + docker logs whisper-service > $LOG_PATH/whisper-service.log + docker logs speecht5-service > $LOG_PATH/tts-service.log + docker logs tgi-gaudi-service > $LOG_PATH/tgi-gaudi-service.log + docker logs audioqna-gaudi-backend-server > $LOG_PATH/audioqna-gaudi-backend-server.log + echo "$response" | sed 's/^"//;s/"$//' | base64 -d > speech.mp3 + + if [[ $(file speech.mp3) == *"RIFF"* ]]; then + echo "Result correct." + else + echo "Result wrong." + exit 1 + fi + +} + +#function validate_frontend() { +# cd $WORKPATH/ui/svelte +# local conda_env_name="OPEA_e2e" +# export PATH=${HOME}/miniforge3/bin/:$PATH +## conda remove -n ${conda_env_name} --all -y +## conda create -n ${conda_env_name} python=3.12 -y +# source activate ${conda_env_name} +# +# sed -i "s/localhost/$ip_address/g" playwright.config.ts +# +## conda install -c conda-forge nodejs=22.6.0 -y +# npm install && npm ci && npx playwright install --with-deps +# node -v && npm -v && pip list +# +# exit_status=0 +# npx playwright test || exit_status=$? +# +# if [ $exit_status -ne 0 ]; then +# echo "[TEST INFO]: ---------frontend test failed---------" +# exit $exit_status +# else +# echo "[TEST INFO]: ---------frontend test passed---------" +# fi +#} + +function stop_docker() { + cd $WORKPATH/docker_compose/intel/hpu/gaudi + docker compose -f compose_tgi.yaml stop && docker compose rm -f +} + +function main() { + + stop_docker + if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi + start_services + + validate_megaservice + # validate_frontend + + stop_docker + echo y | docker system prune + +} + +main diff --git a/AudioQnA/tests/test_compose_tgi_on_xeon.sh b/AudioQnA/tests/test_compose_tgi_on_xeon.sh new file mode 100644 index 0000000000..d735c87b94 --- /dev/null +++ b/AudioQnA/tests/test_compose_tgi_on_xeon.sh @@ -0,0 +1,137 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -e +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" +export REGISTRY=${IMAGE_REPO} +export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') + +function build_docker_images() { + opea_branch=${opea_branch:-"main"} + # If the opea_branch isn't main, replace the git clone branch in Dockerfile. + if [[ "${opea_branch}" != "main" ]]; then + cd $WORKPATH + OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" + NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" + find . -type f -name "Dockerfile*" | while read -r file; do + echo "Processing file: $file" + sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" + done + fi + + cd $WORKPATH/docker_image_build + git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + + echo "Build all the images with --no-cache, check docker_image_build.log for details..." + service_list="audioqna audioqna-ui whisper speecht5" + docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log + + docker pull ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu + docker images && sleep 1s +} + +function start_services() { + cd $WORKPATH/docker_compose/intel/cpu/xeon/ + export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} + export LLM_MODEL_ID=meta-llama/Meta-Llama-3-8B-Instruct + + export MEGA_SERVICE_HOST_IP=${ip_address} + export WHISPER_SERVER_HOST_IP=${ip_address} + export SPEECHT5_SERVER_HOST_IP=${ip_address} + export LLM_SERVER_HOST_IP=${ip_address} + + export WHISPER_SERVER_PORT=7066 + export SPEECHT5_SERVER_PORT=7055 + export LLM_SERVER_PORT=3006 + + export BACKEND_SERVICE_ENDPOINT=http://${ip_address}:3008/v1/audioqna + export host_ip=${ip_address} + + # sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env + + # Start Docker Containers + docker compose -f compose_tgi.yaml up -d > ${LOG_PATH}/start_services_with_compose.log + n=0 + until [[ "$n" -ge 200 ]]; do + docker logs tgi-service > $LOG_PATH/tgi_service_start.log + if grep -q Connected $LOG_PATH/tgi_service_start.log; then + break + fi + sleep 5s + n=$((n+1)) + done +} + + +function validate_megaservice() { + response=$(http_proxy="" curl http://${ip_address}:3008/v1/audioqna -XPOST -d '{"audio": "UklGRigAAABXQVZFZm10IBIAAAABAAEARKwAAIhYAQACABAAAABkYXRhAgAAAAEA", "max_tokens":64}' -H 'Content-Type: application/json') + # always print the log + docker logs whisper-service > $LOG_PATH/whisper-service.log + docker logs speecht5-service > $LOG_PATH/tts-service.log + docker logs tgi-service > $LOG_PATH/tgi-service.log + docker logs audioqna-xeon-backend-server > $LOG_PATH/audioqna-xeon-backend-server.log + echo "$response" | sed 's/^"//;s/"$//' | base64 -d > speech.mp3 + + if [[ $(file speech.mp3) == *"RIFF"* ]]; then + echo "Result correct." + else + echo "Result wrong." + exit 1 + fi + +} + +#function validate_frontend() { +# cd $WORKPATH/ui/svelte +# local conda_env_name="OPEA_e2e" +# export PATH=${HOME}/miniforge3/bin/:$PATH +## conda remove -n ${conda_env_name} --all -y +## conda create -n ${conda_env_name} python=3.12 -y +# source activate ${conda_env_name} +# +# sed -i "s/localhost/$ip_address/g" playwright.config.ts +# +## conda install -c conda-forge nodejs=22.6.0 -y +# npm install && npm ci && npx playwright install --with-deps +# node -v && npm -v && pip list +# +# exit_status=0 +# npx playwright test || exit_status=$? +# +# if [ $exit_status -ne 0 ]; then +# echo "[TEST INFO]: ---------frontend test failed---------" +# exit $exit_status +# else +# echo "[TEST INFO]: ---------frontend test passed---------" +# fi +#} + +function stop_docker() { + cd $WORKPATH/docker_compose/intel/cpu/xeon/ + docker compose -f compose_tgi.yaml stop && docker compose rm -f +} + +function main() { + + stop_docker + if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi + start_services + + validate_megaservice + # validate_frontend + + stop_docker + echo y | docker system prune + +} + +main From a355478ef05d465f2c560fab6db7305d3393aba6 Mon Sep 17 00:00:00 2001 From: Louie Tsai Date: Thu, 13 Mar 2025 23:18:29 -0700 Subject: [PATCH 065/226] [ChatQnA] Enable Prometheus and Grafana with telemetry docker compose file. (#1623) Signed-off-by: Tsai, Louie Signed-off-by: Chingis Yundunov --- ChatQnA/README.md | 4 +- .../docker_compose/intel/cpu/xeon/README.md | 4 +- .../intel/cpu/xeon/compose.telemetry.yaml | 61 ++++++++++++++++-- .../intel/cpu/xeon/compose_tgi.telemetry.yaml | 61 ++++++++++++++++-- .../dashboards/download_opea_dashboard.sh | 6 ++ .../provisioning/dashboards/local.yaml | 14 +++++ .../provisioning/datasources/datasource.yml | 54 ++++++++++++++++ .../intel/cpu/xeon/prometheus.yaml | 43 +++++++++++++ .../docker_compose/intel/cpu/xeon/set_env.sh | 2 + .../docker_compose/intel/hpu/gaudi/README.md | 3 + .../intel/hpu/gaudi/compose.telemetry.yaml | 63 ++++++++++++++++++- .../hpu/gaudi/compose_tgi.telemetry.yaml | 63 ++++++++++++++++++- .../dashboards/download_opea_dashboard.sh | 7 +++ .../provisioning/dashboards/local.yaml | 14 +++++ .../provisioning/datasources/datasource.yml | 54 ++++++++++++++++ .../intel/hpu/gaudi/prometheus.yaml | 47 ++++++++++++++ .../docker_compose/intel/hpu/gaudi/set_env.sh | 1 + 17 files changed, 488 insertions(+), 13 deletions(-) create mode 100644 ChatQnA/docker_compose/intel/cpu/xeon/grafana/dashboards/download_opea_dashboard.sh create mode 100644 ChatQnA/docker_compose/intel/cpu/xeon/grafana/provisioning/dashboards/local.yaml create mode 100644 ChatQnA/docker_compose/intel/cpu/xeon/grafana/provisioning/datasources/datasource.yml create mode 100644 ChatQnA/docker_compose/intel/cpu/xeon/prometheus.yaml create mode 100644 ChatQnA/docker_compose/intel/hpu/gaudi/grafana/dashboards/download_opea_dashboard.sh create mode 100644 ChatQnA/docker_compose/intel/hpu/gaudi/grafana/provisioning/dashboards/local.yaml create mode 100644 ChatQnA/docker_compose/intel/hpu/gaudi/grafana/provisioning/datasources/datasource.yml create mode 100644 ChatQnA/docker_compose/intel/hpu/gaudi/prometheus.yaml diff --git a/ChatQnA/README.md b/ChatQnA/README.md index 40fdac003a..50fd79d324 100644 --- a/ChatQnA/README.md +++ b/ChatQnA/README.md @@ -70,11 +70,11 @@ To set up environment variables for deploying ChatQnA services, follow these ste # on Gaudi cd GenAIExamples/ChatQnA/docker_compose/intel/hpu/gaudi/ source ./set_env.sh - export no_proxy="Your_No_Proxy",chatqna-gaudi-ui-server,chatqna-gaudi-backend-server,dataprep-redis-service,tei-embedding-service,retriever,tei-reranking-service,tgi-service,vllm-service,guardrails + export no_proxy="Your_No_Proxy",chatqna-gaudi-ui-server,chatqna-gaudi-backend-server,dataprep-redis-service,tei-embedding-service,retriever,tei-reranking-service,tgi-service,vllm-service,guardrails,jaeger,prometheus,grafana,gaudi-node-exporter-1 # on Xeon cd GenAIExamples/ChatQnA/docker_compose/intel/cpu/xeon/ source ./set_env.sh - export no_proxy="Your_No_Proxy",chatqna-xeon-ui-server,chatqna-xeon-backend-server,dataprep-redis-service,tei-embedding-service,retriever,tei-reranking-service,tgi-service,vllm-service + export no_proxy="Your_No_Proxy",chatqna-xeon-ui-server,chatqna-xeon-backend-server,dataprep-redis-service,tei-embedding-service,retriever,tei-reranking-service,tgi-service,vllm-service,jaeger,prometheus,grafana,xeon-node-exporter-1 # on Nvidia GPU cd GenAIExamples/ChatQnA/docker_compose/nvidia/gpu source ./set_env.sh diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/README.md b/ChatQnA/docker_compose/intel/cpu/xeon/README.md index f8475e94d0..6ba093216e 100644 --- a/ChatQnA/docker_compose/intel/cpu/xeon/README.md +++ b/ChatQnA/docker_compose/intel/cpu/xeon/README.md @@ -59,8 +59,10 @@ docker compose up -d To enable Open Telemetry Tracing, compose.telemetry.yaml file need to be merged along with default compose.yaml file. CPU example with Open Telemetry feature: +> NOTE : To get supported Grafana Dashboard, please run download_opea_dashboard.sh following below commands. + ```bash -cd GenAIExamples/ChatQnA/docker_compose/intel/cpu/xeon/ +./grafana/dashboards/download_opea_dashboard.sh docker compose -f compose.yaml -f compose.telemetry.yaml up -d ``` diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/compose.telemetry.yaml b/ChatQnA/docker_compose/intel/cpu/xeon/compose.telemetry.yaml index 4da33d6d50..4456fee747 100644 --- a/ChatQnA/docker_compose/intel/cpu/xeon/compose.telemetry.yaml +++ b/ChatQnA/docker_compose/intel/cpu/xeon/compose.telemetry.yaml @@ -4,10 +4,19 @@ services: tei-embedding-service: command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate --otlp-endpoint $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + environment: + - TELEMETRY_ENDPOINT=${TELEMETRY_ENDPOINT} tei-reranking-service: command: --model-id ${RERANK_MODEL_ID} --auto-truncate --otlp-endpoint $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + environment: + - TELEMETRY_ENDPOINT=${TELEMETRY_ENDPOINT} +# vllm-service: +# command: --model $LLM_MODEL_ID --host 0.0.0.0 --port 80 --otlp-traces-endpoint $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + chatqna-xeon-backend-server: + environment: + - TELEMETRY_ENDPOINT=${TELEMETRY_ENDPOINT} jaeger: - image: jaegertracing/all-in-one:latest + image: jaegertracing/all-in-one:1.67.0 container_name: jaeger ports: - "16686:16686" @@ -21,7 +30,51 @@ services: https_proxy: ${https_proxy} COLLECTOR_ZIPKIN_HOST_PORT: 9411 restart: unless-stopped - chatqna-xeon-backend-server: + prometheus: + image: prom/prometheus:v2.52.0 + container_name: prometheus + user: root + volumes: + - ./prometheus.yaml:/etc/prometheus/prometheus.yaml + - ./prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yaml' + ports: + - '9090:9090' + ipc: host + restart: unless-stopped + grafana: + image: grafana/grafana:11.0.0 + container_name: grafana + volumes: + - ./grafana_data:/var/lib/grafana + - ./grafana/dashboards:/var/lib/grafana/dashboards + - ./grafana/provisioning:/etc/grafana/provisioning + user: root environment: - - ENABLE_OPEA_TELEMETRY=true - - TELEMETRY_ENDPOINT=${TELEMETRY_ENDPOINT} + GF_SECURITY_ADMIN_PASSWORD: admin + GF_RENDERING_CALLBACK_URL: http://grafana:3000/ + GF_LOG_FILTERS: rendering:debug + depends_on: + - prometheus + ports: + - '3000:3000' + ipc: host + restart: unless-stopped + node-exporter: + image: prom/node-exporter + container_name: node-exporter + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - '--path.procfs=/host/proc' + - '--path.sysfs=/host/sys' + - --collector.filesystem.ignored-mount-points + - "^/(sys|proc|dev|host|etc|rootfs/var/lib/docker/containers|rootfs/var/lib/docker/overlay2|rootfs/run/docker/netns|rootfs/var/lib/docker/aufs)($$|/)" + ports: + - 9100:9100 + restart: always + deploy: + mode: global diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/compose_tgi.telemetry.yaml b/ChatQnA/docker_compose/intel/cpu/xeon/compose_tgi.telemetry.yaml index 2ba1375398..dfd263d305 100644 --- a/ChatQnA/docker_compose/intel/cpu/xeon/compose_tgi.telemetry.yaml +++ b/ChatQnA/docker_compose/intel/cpu/xeon/compose_tgi.telemetry.yaml @@ -4,12 +4,21 @@ services: tei-embedding-service: command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate --otlp-endpoint $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + environment: + - TELEMETRY_ENDPOINT=${TELEMETRY_ENDPOINT} tei-reranking-service: command: --model-id ${RERANK_MODEL_ID} --auto-truncate --otlp-endpoint $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + environment: + - TELEMETRY_ENDPOINT=${TELEMETRY_ENDPOINT} tgi-service: command: --model-id ${LLM_MODEL_ID} --cuda-graphs 0 --otlp-endpoint $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + environment: + - TELEMETRY_ENDPOINT=${TELEMETRY_ENDPOINT} + chatqna-xeon-backend-server: + environment: + - TELEMETRY_ENDPOINT=${TELEMETRY_ENDPOINT} jaeger: - image: jaegertracing/all-in-one:latest + image: jaegertracing/all-in-one:1.67.0 container_name: jaeger ports: - "16686:16686" @@ -23,7 +32,51 @@ services: https_proxy: ${https_proxy} COLLECTOR_ZIPKIN_HOST_PORT: 9411 restart: unless-stopped - chatqna-xeon-backend-server: + prometheus: + image: prom/prometheus:v2.52.0 + container_name: prometheus + user: root + volumes: + - ./prometheus.yaml:/etc/prometheus/prometheus.yaml + - ./prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yaml' + ports: + - '9090:9090' + ipc: host + restart: unless-stopped + grafana: + image: grafana/grafana:11.0.0 + container_name: grafana + volumes: + - ./grafana_data:/var/lib/grafana + - ./grafana/dashboards:/var/lib/grafana/dashboards + - ./grafana/provisioning:/etc/grafana/provisioning + user: root environment: - - ENABLE_OPEA_TELEMETRY=true - - TELEMETRY_ENDPOINT=${TELEMETRY_ENDPOINT} + GF_SECURITY_ADMIN_PASSWORD: admin + GF_RENDERING_CALLBACK_URL: http://grafana:3000/ + GF_LOG_FILTERS: rendering:debug + depends_on: + - prometheus + ports: + - '3000:3000' + ipc: host + restart: unless-stopped + node-exporter: + image: prom/node-exporter + container_name: node-exporter + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - '--path.procfs=/host/proc' + - '--path.sysfs=/host/sys' + - --collector.filesystem.ignored-mount-points + - "^/(sys|proc|dev|host|etc|rootfs/var/lib/docker/containers|rootfs/var/lib/docker/overlay2|rootfs/run/docker/netns|rootfs/var/lib/docker/aufs)($$|/)" + ports: + - 9100:9100 + restart: always + deploy: + mode: global diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/grafana/dashboards/download_opea_dashboard.sh b/ChatQnA/docker_compose/intel/cpu/xeon/grafana/dashboards/download_opea_dashboard.sh new file mode 100644 index 0000000000..9b603c0403 --- /dev/null +++ b/ChatQnA/docker_compose/intel/cpu/xeon/grafana/dashboards/download_opea_dashboard.sh @@ -0,0 +1,6 @@ +# Copyright (C) 2025 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +wget https://raw.githubusercontent.com/opea-project/GenAIEval/refs/heads/main/evals/benchmark/grafana/vllm_grafana.json +wget https://raw.githubusercontent.com/opea-project/GenAIEval/refs/heads/main/evals/benchmark/grafana/tgi_grafana.json +wget https://raw.githubusercontent.com/opea-project/GenAIEval/refs/heads/main/evals/benchmark/grafana/node_grafana.json diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/grafana/provisioning/dashboards/local.yaml b/ChatQnA/docker_compose/intel/cpu/xeon/grafana/provisioning/dashboards/local.yaml new file mode 100644 index 0000000000..13922a769b --- /dev/null +++ b/ChatQnA/docker_compose/intel/cpu/xeon/grafana/provisioning/dashboards/local.yaml @@ -0,0 +1,14 @@ +# Copyright (C) 2025 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: 1 + +providers: +- name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + updateIntervalSeconds: 10 #how often Grafana will scan for changed dashboards + options: + path: /var/lib/grafana/dashboards diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/grafana/provisioning/datasources/datasource.yml b/ChatQnA/docker_compose/intel/cpu/xeon/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 0000000000..109fc0978f --- /dev/null +++ b/ChatQnA/docker_compose/intel/cpu/xeon/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,54 @@ +# Copyright (C) 2025 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# config file version +apiVersion: 1 + +# list of datasources that should be deleted from the database +deleteDatasources: + - name: Prometheus + orgId: 1 + +# list of datasources to insert/update depending +# what's available in the database +datasources: + # name of the datasource. Required +- name: Prometheus + # datasource type. Required + type: prometheus + # access mode. direct or proxy. Required + access: proxy + # org id. will default to orgId 1 if not specified + orgId: 1 + # url + url: http://prometheus:9090 + # database password, if used + password: + # database user, if used + user: + # database name, if used + database: + # enable/disable basic auth + basicAuth: false + # basic auth username, if used + basicAuthUser: + # basic auth password, if used + basicAuthPassword: + # enable/disable with credentials headers + withCredentials: + # mark as default datasource. Max one per org + isDefault: true + # fields that will be converted to json and stored in json_data + jsonData: + httpMethod: GET + graphiteVersion: "1.1" + tlsAuth: false + tlsAuthWithCACert: false + # json object of data that will be encrypted. + secureJsonData: + tlsCACert: "..." + tlsClientCert: "..." + tlsClientKey: "..." + version: 1 + # allow users to edit datasources from the UI. + editable: true diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/prometheus.yaml b/ChatQnA/docker_compose/intel/cpu/xeon/prometheus.yaml new file mode 100644 index 0000000000..79746aea05 --- /dev/null +++ b/ChatQnA/docker_compose/intel/cpu/xeon/prometheus.yaml @@ -0,0 +1,43 @@ +# Copyright (C) 2025 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# [IP_ADDR]:{PORT_OUTSIDE_CONTAINER} -> {PORT_INSIDE_CONTAINER} / {PROTOCOL} +global: + scrape_interval: 5s + external_labels: + monitor: "my-monitor" +scrape_configs: + - job_name: "prometheus" + static_configs: + - targets: ["prometheus:9090"] + - job_name: "vllm" + metrics_path: /metrics + static_configs: + - targets: ["vllm-service:80"] + - job_name: "tgi" + metrics_path: /metrics + static_configs: + - targets: ["tgi-service:80"] + - job_name: "tei-embedding" + metrics_path: /metrics + static_configs: + - targets: ["tei-embedding-server:80"] + - job_name: "tei-reranking" + metrics_path: /metrics + static_configs: + - targets: ["tei-reranking-server:80"] + - job_name: "retriever" + metrics_path: /metrics + static_configs: + - targets: ["retriever-redis-server:7000"] + - job_name: "dataprep-redis-service" + metrics_path: /metrics + static_configs: + - targets: ["dataprep-redis-server:5000"] + - job_name: "chatqna-backend-server" + metrics_path: /metrics + static_configs: + - targets: ["chatqna-xeon-backend-server:8888"] + - job_name: "prometheus-node-exporter" + metrics_path: /metrics + static_configs: + - targets: ["node-exporter:9100"] diff --git a/ChatQnA/docker_compose/intel/cpu/xeon/set_env.sh b/ChatQnA/docker_compose/intel/cpu/xeon/set_env.sh index 1d287c8648..f59f2314a7 100755 --- a/ChatQnA/docker_compose/intel/cpu/xeon/set_env.sh +++ b/ChatQnA/docker_compose/intel/cpu/xeon/set_env.sh @@ -18,3 +18,5 @@ export LOGFLAG="" export JAEGER_IP=$(ip route get 8.8.8.8 | grep -oP 'src \K[^ ]+') export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=grpc://$JAEGER_IP:4317 export TELEMETRY_ENDPOINT=http://$JAEGER_IP:4318/v1/traces +# Set no proxy +export no_proxy="$no_proxy,chatqna-xeon-ui-server,chatqna-xeon-backend-server,dataprep-redis-service,tei-embedding-service,retriever,tei-reranking-service,tgi-service,vllm-service,jaeger,prometheus,grafana,node-exporter,$JAEGER_IP" diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/README.md b/ChatQnA/docker_compose/intel/hpu/gaudi/README.md index bd5c634903..119b1ac8c9 100644 --- a/ChatQnA/docker_compose/intel/hpu/gaudi/README.md +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/README.md @@ -66,7 +66,10 @@ docker compose up -d To enable Open Telemetry Tracing, compose.telemetry.yaml file need to be merged along with default compose.yaml file. +> NOTE : To get supported Grafana Dashboard, please run download_opea_dashboard.sh following below commands. + ```bash +./grafana/dashboards/download_opea_dashboard.sh docker compose -f compose.yaml -f compose.telemetry.yaml up -d ``` diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/compose.telemetry.yaml b/ChatQnA/docker_compose/intel/hpu/gaudi/compose.telemetry.yaml index 97c71720cc..62154bcb1d 100644 --- a/ChatQnA/docker_compose/intel/hpu/gaudi/compose.telemetry.yaml +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/compose.telemetry.yaml @@ -7,7 +7,7 @@ services: tei-reranking-service: command: --model-id ${RERANK_MODEL_ID} --auto-truncate --otlp-endpoint $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT jaeger: - image: jaegertracing/all-in-one:latest + image: jaegertracing/all-in-one:1.67.0 container_name: jaeger ports: - "16686:16686" @@ -21,6 +21,67 @@ services: https_proxy: ${https_proxy} COLLECTOR_ZIPKIN_HOST_PORT: 9411 restart: unless-stopped + prometheus: + image: prom/prometheus:v2.52.0 + container_name: prometheus + user: root + volumes: + - ./prometheus.yaml:/etc/prometheus/prometheus.yaml + - ./prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yaml' + ports: + - '9090:9090' + ipc: host + restart: unless-stopped + grafana: + image: grafana/grafana:11.0.0 + container_name: grafana + volumes: + - ./grafana_data:/var/lib/grafana + - ./grafana/dashboards:/var/lib/grafana/dashboards + - ./grafana/provisioning:/etc/grafana/provisioning + user: root + environment: + GF_SECURITY_ADMIN_PASSWORD: admin + GF_RENDERING_CALLBACK_URL: http://grafana:3000/ + GF_LOG_FILTERS: rendering:debug + depends_on: + - prometheus + ports: + - '3000:3000' + ipc: host + restart: unless-stopped + node-exporter: + image: prom/node-exporter + container_name: node-exporter + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - '--path.procfs=/host/proc' + - '--path.sysfs=/host/sys' + - --collector.filesystem.ignored-mount-points + - "^/(sys|proc|dev|host|etc|rootfs/var/lib/docker/containers|rootfs/var/lib/docker/overlay2|rootfs/run/docker/netns|rootfs/var/lib/docker/aufs)($$|/)" + ports: + - 9100:9100 + restart: always + deploy: + mode: global + gaudi-exporter: + image: vault.habana.ai/gaudi-metric-exporter/metric-exporter:1.19.2-32 + container_name: gaudi-exporter + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + - /dev:/dev + ports: + - 41611:41611 + restart: always + deploy: + mode: global chatqna-gaudi-backend-server: environment: - ENABLE_OPEA_TELEMETRY=true diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/compose_tgi.telemetry.yaml b/ChatQnA/docker_compose/intel/hpu/gaudi/compose_tgi.telemetry.yaml index 0da707ebc0..64edd63064 100644 --- a/ChatQnA/docker_compose/intel/hpu/gaudi/compose_tgi.telemetry.yaml +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/compose_tgi.telemetry.yaml @@ -9,7 +9,7 @@ services: tgi-service: command: --model-id ${LLM_MODEL_ID} --max-input-length 2048 --max-total-tokens 4096 --otlp-endpoint $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT jaeger: - image: jaegertracing/all-in-one:latest + image: jaegertracing/all-in-one:1.67.0 container_name: jaeger ports: - "16686:16686" @@ -23,6 +23,67 @@ services: https_proxy: ${https_proxy} COLLECTOR_ZIPKIN_HOST_PORT: 9411 restart: unless-stopped + prometheus: + image: prom/prometheus:v2.52.0 + container_name: prometheus + user: root + volumes: + - ./prometheus.yaml:/etc/prometheus/prometheus.yaml + - ./prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yaml' + ports: + - '9090:9090' + ipc: host + restart: unless-stopped + grafana: + image: grafana/grafana:11.0.0 + container_name: grafana + volumes: + - ./grafana_data:/var/lib/grafana + - ./grafana/dashboards:/var/lib/grafana/dashboards + - ./grafana/provisioning:/etc/grafana/provisioning + user: root + environment: + GF_SECURITY_ADMIN_PASSWORD: admin + GF_RENDERING_CALLBACK_URL: http://grafana:3000/ + GF_LOG_FILTERS: rendering:debug + depends_on: + - prometheus + ports: + - '3000:3000' + ipc: host + restart: unless-stopped + node-exporter: + image: prom/node-exporter + container_name: node-exporter + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - '--path.procfs=/host/proc' + - '--path.sysfs=/host/sys' + - --collector.filesystem.ignored-mount-points + - "^/(sys|proc|dev|host|etc|rootfs/var/lib/docker/containers|rootfs/var/lib/docker/overlay2|rootfs/run/docker/netns|rootfs/var/lib/docker/aufs)($$|/)" + ports: + - 9100:9100 + restart: always + deploy: + mode: global + gaudi-exporter: + image: vault.habana.ai/gaudi-metric-exporter/metric-exporter:1.19.2-32 + container_name: gaudi-exporter + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + - /dev:/dev + ports: + - 41611:41611 + restart: always + deploy: + mode: global chatqna-gaudi-backend-server: environment: - ENABLE_OPEA_TELEMETRY=true diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/grafana/dashboards/download_opea_dashboard.sh b/ChatQnA/docker_compose/intel/hpu/gaudi/grafana/dashboards/download_opea_dashboard.sh new file mode 100644 index 0000000000..93ba5d7454 --- /dev/null +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/grafana/dashboards/download_opea_dashboard.sh @@ -0,0 +1,7 @@ +# Copyright (C) 2025 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +wget https://raw.githubusercontent.com/opea-project/GenAIEval/refs/heads/main/evals/benchmark/grafana/vllm_grafana.json +wget https://raw.githubusercontent.com/opea-project/GenAIEval/refs/heads/main/evals/benchmark/grafana/tgi_grafana.json +wget https://raw.githubusercontent.com/opea-project/GenAIEval/refs/heads/main/evals/benchmark/grafana/node_grafana.json +wget https://raw.githubusercontent.com/opea-project/GenAIEval/refs/heads/main/evals/benchmark/grafana/gaudi_grafana.json diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/grafana/provisioning/dashboards/local.yaml b/ChatQnA/docker_compose/intel/hpu/gaudi/grafana/provisioning/dashboards/local.yaml new file mode 100644 index 0000000000..13922a769b --- /dev/null +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/grafana/provisioning/dashboards/local.yaml @@ -0,0 +1,14 @@ +# Copyright (C) 2025 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: 1 + +providers: +- name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + updateIntervalSeconds: 10 #how often Grafana will scan for changed dashboards + options: + path: /var/lib/grafana/dashboards diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/grafana/provisioning/datasources/datasource.yml b/ChatQnA/docker_compose/intel/hpu/gaudi/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 0000000000..109fc0978f --- /dev/null +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,54 @@ +# Copyright (C) 2025 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# config file version +apiVersion: 1 + +# list of datasources that should be deleted from the database +deleteDatasources: + - name: Prometheus + orgId: 1 + +# list of datasources to insert/update depending +# what's available in the database +datasources: + # name of the datasource. Required +- name: Prometheus + # datasource type. Required + type: prometheus + # access mode. direct or proxy. Required + access: proxy + # org id. will default to orgId 1 if not specified + orgId: 1 + # url + url: http://prometheus:9090 + # database password, if used + password: + # database user, if used + user: + # database name, if used + database: + # enable/disable basic auth + basicAuth: false + # basic auth username, if used + basicAuthUser: + # basic auth password, if used + basicAuthPassword: + # enable/disable with credentials headers + withCredentials: + # mark as default datasource. Max one per org + isDefault: true + # fields that will be converted to json and stored in json_data + jsonData: + httpMethod: GET + graphiteVersion: "1.1" + tlsAuth: false + tlsAuthWithCACert: false + # json object of data that will be encrypted. + secureJsonData: + tlsCACert: "..." + tlsClientCert: "..." + tlsClientKey: "..." + version: 1 + # allow users to edit datasources from the UI. + editable: true diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/prometheus.yaml b/ChatQnA/docker_compose/intel/hpu/gaudi/prometheus.yaml new file mode 100644 index 0000000000..8816f4ec68 --- /dev/null +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/prometheus.yaml @@ -0,0 +1,47 @@ +# Copyright (C) 2025 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# [IP_ADDR]:{PORT_OUTSIDE_CONTAINER} -> {PORT_INSIDE_CONTAINER} / {PROTOCOL} +global: + scrape_interval: 5s + external_labels: + monitor: "my-monitor" +scrape_configs: + - job_name: "prometheus" + static_configs: + - targets: ["prometheus:9090"] + - job_name: "vllm" + metrics_path: /metrics + static_configs: + - targets: ["vllm-gaudi-server:80"] + - job_name: "tgi" + metrics_path: /metrics + static_configs: + - targets: ["tgi-gaudi-server:80"] + - job_name: "tei-embedding" + metrics_path: /metrics + static_configs: + - targets: ["tei-embedding-gaudi-server:80"] + - job_name: "tei-reranking" + metrics_path: /metrics + static_configs: + - targets: ["tei-reranking-gaudi-server:80"] + - job_name: "retriever" + metrics_path: /metrics + static_configs: + - targets: ["retriever:7000"] + - job_name: "dataprep-redis-service" + metrics_path: /metrics + static_configs: + - targets: ["dataprep-redis-service:5000"] + - job_name: "chatqna-backend-server" + metrics_path: /metrics + static_configs: + - targets: ["chatqna-gaudi-backend-server:8888"] + - job_name: "prometheus-node-exporter" + metrics_path: /metrics + static_configs: + - targets: ["node-exporter:9100"] + - job_name: "prometheus-gaudi-exporter" + metrics_path: /metrics + static_configs: + - targets: ["gaudi-exporter:41611"] diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/set_env.sh b/ChatQnA/docker_compose/intel/hpu/gaudi/set_env.sh index 27339c478f..b0cba1834c 100644 --- a/ChatQnA/docker_compose/intel/hpu/gaudi/set_env.sh +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/set_env.sh @@ -19,3 +19,4 @@ export LOGFLAG="" export JAEGER_IP=$(ip route get 8.8.8.8 | grep -oP 'src \K[^ ]+') export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=grpc://$JAEGER_IP:4317 export TELEMETRY_ENDPOINT=http://$JAEGER_IP:4318/v1/traces +export no_proxy="$no_proxy,chatqna-gaudi-ui-server,chatqna-gaudi-backend-server,dataprep-redis-service,tei-embedding-service,retriever,tei-reranking-service,tgi-gaudi-server,vllm-gaudi-server,guardrails,jaeger,prometheus,grafana,node-exporter,gaudi-exporter,$JAEGER_IP" From 43df12401ed54a0a1f83fba78405b941ad05fa87 Mon Sep 17 00:00:00 2001 From: "Sun, Xuehao" Date: Fri, 14 Mar 2025 17:55:49 +0800 Subject: [PATCH 066/226] Update stale issue and PR settings to 30 days for inactivity (#1661) Signed-off-by: Sun, Xuehao Signed-off-by: Chingis Yundunov --- .github/workflows/daily_check_issue_and_pr.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/daily_check_issue_and_pr.yml b/.github/workflows/daily_check_issue_and_pr.yml index b578580602..4664b250c0 100644 --- a/.github/workflows/daily_check_issue_and_pr.yml +++ b/.github/workflows/daily_check_issue_and_pr.yml @@ -16,14 +16,13 @@ jobs: steps: - uses: actions/stale@v9 with: - days-before-issue-stale: 60 - days-before-pr-stale: 60 + days-before-issue-stale: 30 + days-before-pr-stale: 30 days-before-issue-close: 7 days-before-pr-close: 7 - stale-issue-message: "This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 7 days." - stale-pr-message: "This PR is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 7 days." + stale-issue-message: "This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 7 days." + stale-pr-message: "This PR is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 7 days." close-issue-message: "This issue was closed because it has been stalled for 7 days with no activity." close-pr-message: "This PR was closed because it has been stalled for 7 days with no activity." repo-token: ${{ secrets.ACTION_TOKEN }} - start-date: "2025-01-01T00:00:00Z" - debug-only: true # will remove this line when ready to merge + start-date: "2025-03-01T00:00:00Z" From 62b72dd750832689e3e586cd3be25185cb096044 Mon Sep 17 00:00:00 2001 From: James Edwards <65970741+jedwards-habana@users.noreply.github.com> Date: Fri, 14 Mar 2025 18:05:01 -0500 Subject: [PATCH 067/226] Add final README.md and set_env.sh script for quickstart review. Previous pull request was 1595. (#1662) Signed-off-by: Edwards, James A Co-authored-by: Edwards, James A Signed-off-by: Chingis Yundunov --- .../docker_compose/intel/hpu/gaudi/README.md | 687 +++++------------- .../docker_compose/intel/hpu/gaudi/set_env.sh | 100 ++- 2 files changed, 269 insertions(+), 518 deletions(-) mode change 100644 => 100755 ChatQnA/docker_compose/intel/hpu/gaudi/set_env.sh diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/README.md b/ChatQnA/docker_compose/intel/hpu/gaudi/README.md index 119b1ac8c9..29d5829d4f 100644 --- a/ChatQnA/docker_compose/intel/hpu/gaudi/README.md +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/README.md @@ -1,601 +1,280 @@ -# Build MegaService of ChatQnA on Gaudi +# Example ChatQnA deployments on an Intel® Gaudi® Platform -This document outlines the deployment process for a ChatQnA application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on Intel Gaudi server. The steps include Docker image creation, container deployment via Docker Compose, and service execution to integrate microservices such as `embedding`, `retriever`, `rerank`, and `llm`. +This example covers the single-node on-premises deployment of the ChatQnA example using OPEA components. There are various ways to enable ChatQnA, but this example will focus on four options available for deploying the ChatQnA pipeline to Intel® Gaudi® AI Accelerators. This example begins with a Quick Start section and then documents how to modify deployments, leverage new models and configure the number of allocated devices. -The default pipeline deploys with vLLM as the LLM serving component and leverages rerank component. It also provides options of not using rerank in the pipeline, leveraging guardrails, or using TGI backend for LLM microservice, please refer to [start-all-the-services-docker-containers](#start-all-the-services-docker-containers) section in this page. +This example includes the following sections: -Quick Start: +- [ChatQnA Quick Start Deployment](#chatqna-quick-start-deployment): Demonstrates how to quickly deploy a ChatQnA application/pipeline on a Intel® Gaudi® platform. +- [ChatQnA Docker Compose Files](#chatqna-docker-compose-files): Describes some example deployments and their docker compose files. +- [ChatQnA Service Configuration](#chatqna-service-configuration): Describes the services and possible configuration changes. -1. Set up the environment variables. -2. Run Docker Compose. -3. Consume the ChatQnA Service. +**Note** This example requires access to a properly installed Intel® Gaudi® platform with a functional Docker service configured to use the habanalabs-container-runtime. Please consult the [Intel® Gaudi® software Installation Guide](https://docs.habana.ai/en/v1.20.0/Installation_Guide/Driver_Installation.html) for more information. -Note: The default LLM is `meta-llama/Meta-Llama-3-8B-Instruct`. Before deploying the application, please make sure either you've requested and been granted the access to it on [Huggingface](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) or you've downloaded the model locally from [ModelScope](https://www.modelscope.cn/models). We now support running the latest DeepSeek models, including [deepseek-ai/DeepSeek-R1-Distill-Llama-70B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B) and [deepseek-ai/DeepSeek-R1-Distill-Qwen-32B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B) on Gaudi accelerators. To run `deepseek-ai/DeepSeek-R1-Distill-Llama-70B`, update the `LLM_MODEL_ID` and configure `NUM_CARDS` to 8 in the [set_env.sh](./set_env.sh) script. To run `deepseek-ai/DeepSeek-R1-Distill-Qwen-32B`, update the `LLM_MODEL_ID` and configure `NUM_CARDS` to 4 in the [set_env.sh](./set_env.sh) script. +## ChatQnA Quick Start Deployment -## Quick Start: 1.Setup Environment Variable +This section describes how to quickly deploy and test the ChatQnA service manually on an Intel® Gaudi® platform. The basic steps are: -To set up environment variables for deploying ChatQnA services, follow these steps: +1. [Access the Code](#access-the-code) +2. [Generate a HuggingFace Access Token](#generate-a-huggingface-access-token) +3. [Configure the Deployment Environment](#configure-the-deployment-environment) +4. [Deploy the Services Using Docker Compose](#deploy-the-services-using-docker-compose) +5. [Check the Deployment Status](#check-the-deployment-status) +6. [Test the Pipeline](#test-the-pipeline) +7. [Cleanup the Deployment](#cleanup-the-deployment) -1. Set the required environment variables: +### Access the Code - ```bash - # Example: host_ip="192.168.1.1" - export host_ip="External_Public_IP" - export HUGGINGFACEHUB_API_TOKEN="Your_Huggingface_API_Token" - ``` +Clone the GenAIExample repository and access the ChatQnA Intel® Gaudi® platform Docker Compose files and supporting scripts: -2. If you are in a proxy environment, also set the proxy-related environment variables: - - ```bash - export http_proxy="Your_HTTP_Proxy" - export https_proxy="Your_HTTPs_Proxy" - # Example: no_proxy="localhost, 127.0.0.1, 192.168.1.1" - export no_proxy="Your_No_Proxy",chatqna-gaudi-ui-server,chatqna-gaudi-backend-server,dataprep-redis-service,tei-embedding-service,retriever,tei-reranking-service,tgi-service,vllm-service,guardrails - ``` - -3. Set up other environment variables: - - ```bash - source ./set_env.sh - ``` - -4. Change Model for LLM serving - - By default, Meta-Llama-3-8B-Instruct is used for LLM serving, the default model can be changed to other validated LLM models. - Please pick a [validated llm models](https://github.com/opea-project/GenAIComps/tree/main/comps/llms/src/text-generation#validated-llm-models) from the table. - To change the default model defined in set_env.sh, overwrite it by exporting LLM_MODEL_ID to the new model or by modifying set_env.sh, and then repeat step 3. - For example, change to DeepSeek-R1-Distill-Qwen-32B using the following command. - - ```bash - export LLM_MODEL_ID="deepseek-ai/DeepSeek-R1-Distill-Qwen-32B" - ``` - - Please also check [required gaudi cards for different models](https://github.com/opea-project/GenAIComps/tree/main/comps/llms/src/text-generation#system-requirements-for-llm-models) for new models. - It might be necessary to increase the number of Gaudi cards for the model by exporting NUM_CARDS to the new model or by modifying set_env.sh, and then repeating step 3. For example, increase the number of Gaudi cards for DeepSeek-R1- - Distill-Qwen-32B using the following command: - - ```bash - export NUM_CARDS=4 - ``` - -## Quick Start: 2.Run Docker Compose - -```bash -docker compose up -d ``` - -To enable Open Telemetry Tracing, compose.telemetry.yaml file need to be merged along with default compose.yaml file. - -> NOTE : To get supported Grafana Dashboard, please run download_opea_dashboard.sh following below commands. - -```bash -./grafana/dashboards/download_opea_dashboard.sh -docker compose -f compose.yaml -f compose.telemetry.yaml up -d +git clone https://github.com/opea-project/GenAIExamples.git +cd GenAIExamples/ChatQnA/docker_compose/intel/hpu/gaudi/ ``` -It will automatically download the docker image on `docker hub`: +Checkout a released version, such as v1.2: -```bash -docker pull opea/chatqna:latest -docker pull opea/chatqna-ui:latest +``` +git checkout v1.2 ``` -In following cases, you could build docker image from source by yourself. - -- Failed to download the docker image. +### Generate a HuggingFace Access Token -- If you want to use a specific version of Docker image. +Some HuggingFace resources, such as some models, are only accessible if you have an access token. If you do not already have a HuggingFace access token, you can create one by first creating an account by following the steps provided at [HuggingFace](https://huggingface.co/) and then generating a [user access token](https://huggingface.co/docs/transformers.js/en/guides/private#step-1-generating-a-user-access-token). -Please refer to 'Build Docker Images' in below. +### Configure the Deployment Environment -## QuickStart: 3.Consume the ChatQnA Service +To set up environment variables for deploying ChatQnA services, source the _setup_env.sh_ script in this directory: -```bash -curl http://${host_ip}:8888/v1/chatqna \ - -H "Content-Type: application/json" \ - -d '{ - "messages": "What is the revenue of Nike in 2023?" - }' ``` - -## 🚀 Build Docker Images - -First of all, you need to build Docker Images locally. This step can be ignored after the Docker images published to Docker hub. - -```bash -git clone https://github.com/opea-project/GenAIComps.git -cd GenAIComps +source ./set_env.sh ``` -### 1. Build Retriever Image +The _set_env.sh_ script will prompt for required and optional environment variables used to configure the ChatQnA services. If a value is not entered, the script will use a default value for the same. It will also generate a _.env_ file defining the desired configuration. Consult the section on [ChatQnA Service configuration](#chatqna-service-configuration) for information on how service specific configuration parameters affect deployments. -```bash -docker build --no-cache -t opea/retriever:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/src/Dockerfile . -``` +### Deploy the Services Using Docker Compose -### 2. Build Dataprep Image +To deploy the ChatQnA services, execute the `docker compose up` command with the appropriate arguments. For a default deployment, execute: ```bash -docker build --no-cache -t opea/dataprep:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile . +docker compose up -d ``` -### 3. Build Guardrails Docker Image (Optional) +The ChatQnA docker images should automatically be downloaded from the `OPEA registry` and deployed on the Intel® Gaudi® Platform: -To fortify AI initiatives in production, Guardrails microservice can secure model inputs and outputs, building Trustworthy, Safe, and Secure LLM-based Applications. - -```bash -docker build -t opea/guardrails:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/guardrails/src/guardrails/Dockerfile . ``` - -### 4. Build MegaService Docker Image - -1. MegaService with Rerank - - To construct the Mega Service with Rerank, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `chatqna.py` Python script. Build the MegaService Docker image using the command below: - - ```bash - git clone https://github.com/opea-project/GenAIExamples.git - cd GenAIExamples/ChatQnA - docker build --no-cache -t opea/chatqna:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . - ``` - -2. MegaService with Guardrails - - If you want to enable guardrails microservice in the pipeline, please use the below command instead: - - ```bash - git clone https://github.com/opea-project/GenAIExamples.git - cd GenAIExamples/ChatQnA/ - docker build --no-cache -t opea/chatqna-guardrails:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile.guardrails . - ``` - -3. MegaService without Rerank - - To construct the Mega Service without Rerank, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `chatqna_without_rerank.py` Python script. Build MegaService Docker image via below command: - - ```bash - git clone https://github.com/opea-project/GenAIExamples.git - cd GenAIExamples/ChatQnA - docker build --no-cache -t opea/chatqna-without-rerank:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile.without_rerank . - ``` - -### 5. Build UI Docker Image - -Construct the frontend Docker image using the command below: - -```bash -cd GenAIExamples/ChatQnA/ui -docker build --no-cache -t opea/chatqna-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile . +[+] Running 10/10 + ✔ Network gaudi_default Created 0.1s + ✔ Container tei-reranking-gaudi-server Started 0.7s + ✔ Container vllm-gaudi-server Started 0.7s + ✔ Container tei-embedding-gaudi-server Started 0.3s + ✔ Container redis-vector-db Started 0.6s + ✔ Container retriever-redis-server Started 1.1s + ✔ Container dataprep-redis-server Started 1.1s + ✔ Container chatqna-gaudi-backend-server Started 1.3s + ✔ Container chatqna-gaudi-ui-server Started 1.7s + ✔ Container chatqna-gaudi-nginx-server Started 1.9s ``` -### 6. Build Conversational React UI Docker Image (Optional) - -Build frontend Docker image that enables Conversational experience with ChatQnA megaservice via below command: +### Check the Deployment Status -**Export the value of the public IP address of your Gaudi node to the `host_ip` environment variable** +After running docker compose, check if all the containers launched via docker compose have started: -```bash -cd GenAIExamples/ChatQnA/ui -docker build --no-cache -t opea/chatqna-conversation-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile.react . ``` - -### 7. Build Nginx Docker Image - -```bash -cd GenAIComps -docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/third_parties/nginx/src/Dockerfile . +docker ps -a ``` -Then run the command `docker images`, you will have the following 5 Docker Images: - -- `opea/retriever:latest` -- `opea/dataprep:latest` -- `opea/chatqna:latest` -- `opea/chatqna-ui:latest` -- `opea/nginx:latest` - -If Conversation React UI is built, you will find one more image: - -- `opea/chatqna-conversation-ui:latest` - -If Guardrails docker image is built, you will find one more image: - -- `opea/guardrails:latest` - -## 🚀 Start MicroServices and MegaService - -### Required Models - -By default, the embedding, reranking and LLM models are set to a default value as listed below: - -| Service | Model | -| --------- | ----------------------------------- | -| Embedding | BAAI/bge-base-en-v1.5 | -| Reranking | BAAI/bge-reranker-base | -| LLM | meta-llama/Meta-Llama-3-8B-Instruct | - -Change the `xxx_MODEL_ID` below for your needs. - -For users in China who are unable to download models directly from Huggingface, you can use [ModelScope](https://www.modelscope.cn/models) or a Huggingface mirror to download models. The vLLM/TGI can load the models either online or offline as described below: - -1. Online - - ```bash - export HF_TOKEN=${your_hf_token} - export HF_ENDPOINT="https://hf-mirror.com" - model_name="meta-llama/Meta-Llama-3-8B-Instruct" - # Start vLLM LLM Service - docker run -p 8007:80 -v ./data:/data --name vllm-gaudi-server -e HF_ENDPOINT=$HF_ENDPOINT -e http_proxy=$http_proxy -e https_proxy=$https_proxy --runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none -e HUGGING_FACE_HUB_TOKEN=$HF_TOKEN -e VLLM_TORCH_PROFILER_DIR="/mnt" --cap-add=sys_nice --ipc=host opea/vllm-gaudi:latest --model $model_name --tensor-parallel-size 1 --host 0.0.0.0 --port 80 --block-size 128 --max-num-seqs 256 --max-seq_len-to-capture 2048 - # Start TGI LLM Service - docker run -p 8005:80 -v ./data:/data --name tgi-gaudi-server -e HF_ENDPOINT=$HF_ENDPOINT -e http_proxy=$http_proxy -e https_proxy=$https_proxy --runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none -e HUGGING_FACE_HUB_TOKEN=$HF_TOKEN -e ENABLE_HPU_GRAPH=true -e LIMIT_HPU_GRAPH=true -e USE_FLASH_ATTENTION=true -e FLASH_ATTENTION_RECOMPUTE=true --cap-add=sys_nice --ipc=host ghcr.io/huggingface/tgi-gaudi:2.0.6 --model-id $model_name --max-input-tokens 1024 --max-total-tokens 2048 - ``` - -2. Offline - - - Search your model name in ModelScope. For example, check [this page](https://modelscope.cn/models/LLM-Research/Meta-Llama-3-8B-Instruct/files) for model `Meta-Llama-3-8B-Instruct`. - - - Click on `Download this model` button, and choose one way to download the model to your local path `/path/to/model`. - - - Run the following command to start the LLM service. - - ```bash - export HF_TOKEN=${your_hf_token} - export model_path="/path/to/model" - # Start vLLM LLM Service - docker run -p 8007:80 -v $model_path:/data --name vllm-gaudi-server --runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none -e HUGGING_FACE_HUB_TOKEN=$HF_TOKEN -e VLLM_TORCH_PROFILER_DIR="/mnt" --cap-add=sys_nice --ipc=host opea/vllm-gaudi:latest --model /data --tensor-parallel-size 1 --host 0.0.0.0 --port 80 --block-size 128 --max-num-seqs 256 --max-seq_len-to-capture 2048 - # Start TGI LLM Service - docker run -p 8005:80 -v $model_path:/data --name tgi-gaudi-server --runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none -e HUGGING_FACE_HUB_TOKEN=$HF_TOKEN -e ENABLE_HPU_GRAPH=true -e LIMIT_HPU_GRAPH=true -e USE_FLASH_ATTENTION=true -e FLASH_ATTENTION_RECOMPUTE=true --cap-add=sys_nice --ipc=host ghcr.io/huggingface/tgi-gaudi:2.0.6 --model-id /data --max-input-tokens 1024 --max-total-tokens 2048 - ``` - -### Setup Environment Variables - -1. Set the required environment variables: +For the default deployment, the following 10 containers should have started: - ```bash - # Example: host_ip="192.168.1.1" - export host_ip="External_Public_IP" - export HUGGINGFACEHUB_API_TOKEN="Your_Huggingface_API_Token" - # Example: NGINX_PORT=80 - export NGINX_PORT=${your_nginx_port} - ``` - -2. If you are in a proxy environment, also set the proxy-related environment variables: - - ```bash - export http_proxy="Your_HTTP_Proxy" - export https_proxy="Your_HTTPs_Proxy" - # Example: no_proxy="localhost, 127.0.0.1, 192.168.1.1" - export no_proxy="Your_No_Proxy",chatqna-gaudi-ui-server,chatqna-gaudi-backend-server,dataprep-redis-service,tei-embedding-service,retriever,tei-reranking-service,tgi-service,vllm-service,guardrails - ``` - -3. Set up other environment variables: - - ```bash - source ./set_env.sh - ``` - -### Start all the services Docker Containers - -```bash -cd GenAIExamples/ChatQnA/docker_compose/intel/hpu/gaudi/ ``` - -If use vLLM as the LLM serving backend. - -```bash -# Start ChatQnA with Rerank Pipeline -docker compose -f compose.yaml up -d -# Start ChatQnA without Rerank Pipeline -docker compose -f compose_without_rerank.yaml up -d -# Start ChatQnA with Rerank Pipeline and Open Telemetry Tracing -docker compose -f compose.yaml -f compose.telemetry.yaml up -d +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +8365b0a6024d opea/nginx:latest "/docker-entrypoint.…" 2 minutes ago Up 2 minutes 0.0.0.0:80->80/tcp, :::80->80/tcp chatqna-gaudi-nginx-server +f090fe262c74 opea/chatqna-ui:latest "docker-entrypoint.s…" 2 minutes ago Up 2 minutes 0.0.0.0:5173->5173/tcp, :::5173->5173/tcp chatqna-gaudi-ui-server +ec97d7651c96 opea/chatqna:latest "python chatqna.py" 2 minutes ago Up 2 minutes 0.0.0.0:8888->8888/tcp, :::8888->8888/tcp chatqna-gaudi-backend-server +a61fb7dc4fae opea/dataprep:latest "sh -c 'python $( [ …" 2 minutes ago Up 2 minutes 0.0.0.0:6007->5000/tcp, [::]:6007->5000/tcp dataprep-redis-server +d560c232b120 opea/retriever:latest "python opea_retriev…" 2 minutes ago Up 2 minutes 0.0.0.0:7000->7000/tcp, :::7000->7000/tcp retriever-redis-server +a1d7ca2d3787 ghcr.io/huggingface/tei-gaudi:1.5.0 "text-embeddings-rou…" 2 minutes ago Up 2 minutes 0.0.0.0:8808->80/tcp, [::]:8808->80/tcp tei-reranking-gaudi-server +9a9f3fd4fd4c opea/vllm-gaudi:latest "python3 -m vllm.ent…" 2 minutes ago Exited (1) 2 minutes ago vllm-gaudi-server +1ab9bbdf5182 redis/redis-stack:7.2.0-v9 "/entrypoint.sh" 2 minutes ago Up 2 minutes 0.0.0.0:6379->6379/tcp, :::6379->6379/tcp, 0.0.0.0:8001->8001/tcp, :::8001->8001/tcp redis-vector-db +9ee0789d819e ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 "text-embeddings-rou…" 2 minutes ago Up 2 minutes 0.0.0.0:8090->80/tcp, [::]:8090->80/tcp tei-embedding-gaudi-server ``` -If use TGI as the LLM serving backend. +### Test the Pipeline -```bash -docker compose -f compose_tgi.yaml up -d -# Start ChatQnA with Open Telemetry Tracing -docker compose -f compose_tgi.yaml -f compose_tgi.telemetry.yaml up -d -``` - -If you want to enable guardrails microservice in the pipeline, please follow the below command instead: +Once the ChatQnA services are running, test the pipeline using the following command: ```bash -cd GenAIExamples/ChatQnA/docker_compose/intel/hpu/gaudi/ -docker compose -f compose_guardrails.yaml up -d +curl http://${host_ip}:8888/v1/chatqna \ + -H "Content-Type: application/json" \ + -d '{ + "messages": "What is the revenue of Nike in 2023?" + }' ``` -> **_NOTE:_** Users need at least two Gaudi cards to run the ChatQnA successfully. - -### Validate MicroServices and MegaService - -Follow the instructions to validate MicroServices. -For validation details, please refer to [how-to-validate_service](./how_to_validate_service.md). - -1. TEI Embedding Service - - ```bash - curl ${host_ip}:8090/embed \ - -X POST \ - -d '{"inputs":"What is Deep Learning?"}' \ - -H 'Content-Type: application/json' - ``` - -2. Retriever Microservice - - To consume the retriever microservice, you need to generate a mock embedding vector by Python script. The length of embedding vector - is determined by the embedding model. - Here we use the model `EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5"`, which vector size is 768. - - Check the vecotor dimension of your embedding model, set `your_embedding` dimension equals to it. - - ```bash - export your_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") - curl http://${host_ip}:7000/v1/retrieval \ - -X POST \ - -d "{\"text\":\"test\",\"embedding\":${your_embedding}}" \ - -H 'Content-Type: application/json' - ``` - -3. TEI Reranking Service - - > Skip for ChatQnA without Rerank pipeline - - ```bash - curl http://${host_ip}:8808/rerank \ - -X POST \ - -d '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}' \ - -H 'Content-Type: application/json' - ``` - -4. LLM backend Service - - In the first startup, this service will take more time to download, load and warm up the model. After it's finished, the service will be ready. - - Try the command below to check whether the LLM serving is ready. - - ```bash - # vLLM service - docker logs vllm-gaudi-server 2>&1 | grep complete - # If the service is ready, you will get the response like below. - INFO: Application startup complete. - ``` - - ```bash - # TGI service - docker logs tgi-gaudi-server | grep Connected - If the service is ready, you will get the response like below. - 2024-09-03T02:47:53.402023Z INFO text_generation_router::server: router/src/server.rs:2311: Connected - ``` +**Note** The value of _host_ip_ was set using the _set_env.sh_ script and can be found in the _.env_ file. - Then try the `cURL` command below to validate services. +### Cleanup the Deployment - ```bash - # vLLM Service - curl http://${host_ip}:8007/v1/chat/completions \ - -X POST \ - -d '{"model": ${LLM_MODEL_ID}, "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens":17}' \ - -H 'Content-Type: application/json' - ``` +To stop the containers associated with the deployment, execute the following command: - ```bash - # TGI service - curl http://${host_ip}:8005/v1/chat/completions \ - -X POST \ - -d '{"model": ${LLM_MODEL_ID}, "messages": [{"role": "user", "content": "What is Deep Learning?"}], "max_tokens":17}' \ - -H 'Content-Type: application/json' - ``` - -5. MegaService - - ```bash - curl http://${host_ip}:8888/v1/chatqna -H "Content-Type: application/json" -d '{ - "messages": "What is the revenue of Nike in 2023?" - }' - ``` - -6. Nginx Service - - ```bash - curl http://${host_ip}:${NGINX_PORT}/v1/chatqna \ - -H "Content-Type: application/json" \ - -d '{"messages": "What is the revenue of Nike in 2023?"}' - ``` - -7. Dataprep Microservice（Optional） - -If you want to update the default knowledge base, you can use the following commands: - -Update Knowledge Base via Local File Upload: +``` +docker compose -f compose.yaml down +``` -```bash -curl -X POST "http://${host_ip}:6007/v1/dataprep/ingest" \ - -H "Content-Type: multipart/form-data" \ - -F "files=@./nke-10k-2023.pdf" +``` +[+] Running 10/10 + ✔ Container chatqna-gaudi-nginx-server Removed 10.5s + ✔ Container dataprep-redis-server Removed 10.5s + ✔ Container chatqna-gaudi-ui-server Removed 10.3s + ✔ Container chatqna-gaudi-backend-server Removed 10.3s + ✔ Container vllm-gaudi-server Removed 0.0s + ✔ Container retriever-redis-server Removed 10.4s + ✔ Container tei-reranking-gaudi-server Removed 2.0s + ✔ Container tei-embedding-gaudi-server Removed 1.2s + ✔ Container redis-vector-db Removed 0.4s + ✔ Network gaudi_default Removed 0.4s ``` -This command updates a knowledge base by uploading a local file for processing. Update the file path according to your environment. +All the ChatQnA containers will be stopped and then removed on completion of the "down" command. -Add Knowledge Base via HTTP Links: +## ChatQnA Docker Compose Files -```bash -curl -X POST "http://${host_ip}:6007/v1/dataprep/ingest" \ - -H "Content-Type: multipart/form-data" \ - -F 'link_list=["https://opea.dev"]' -``` +In the context of deploying a ChatQnA pipeline on an Intel® Gaudi® platform, the allocation and utilization of Gaudi devices across different services are important considerations for optimizing performance and resource efficiency. Each of the four example deployments, defined by the example Docker compose yaml files, demonstrates a unique approach to leveraging Gaudi hardware, reflecting different priorities and operational strategies. -This command updates a knowledge base by submitting a list of HTTP links for processing. +### compose.yaml - Default Deployment -Also, you are able to get the file/link list that you uploaded: +The default deployment utilizes Gaudi devices primarily for the `vllm-service`, which handles large language model (LLM) tasks. This service is configured to maximize the use of Gaudi's capabilities, potentially allocating multiple devices to enhance parallel processing and throughput. The `tei-reranking-service` also uses Gaudi hardware (1 card), however, indicating a balanced approach where both LLM processing and reranking tasks benefit from Gaudi's performance enhancements. -```bash -curl -X POST "http://${host_ip}:6007/v1/dataprep/get" \ - -H "Content-Type: application/json" -``` +| Service Name | Image Name | Gaudi Use | +| ---------------------------- | ----------------------------------------------------- | ------------ | +| redis-vector-db | redis/redis-stack:7.2.0-v9 | No | +| dataprep-redis-service | opea/dataprep:latest | No | +| tei-embedding-service | ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 | No | +| retriever | opea/retriever:latest | No | +| tei-reranking-service | ghcr.io/huggingface/tei-gaudi:1.5.0 | 1 card | +| vllm-service | opea/vllm-gaudi:latest | Configurable | +| chatqna-gaudi-backend-server | opea/chatqna:latest | No | +| chatqna-gaudi-ui-server | opea/chatqna-ui:latest | No | +| chatqna-gaudi-nginx-server | opea/nginx:latest | No | -Then you will get the response JSON like this. Notice that the returned `name`/`id` of the uploaded link is `https://xxx.txt`. - -```json -[ - { - "name": "nke-10k-2023.pdf", - "id": "nke-10k-2023.pdf", - "type": "File", - "parent": "" - }, - { - "name": "https://opea.dev.txt", - "id": "https://opea.dev.txt", - "type": "File", - "parent": "" - } -] -``` +### compose_tgi.yaml - TGI Deployment -To delete the file/link you uploaded: +The TGI (Text Generation Inference) deployment and the default deployment differ primarily in their service configurations and specific focus on handling large language models (LLMs). The TGI deployment includes a unique `tgi-service`, which utilizes the `ghcr.io/huggingface/tgi-gaudi:2.0.6` image and is specifically configured to run on Gaudi hardware. This service is designed to handle LLM tasks with optimizations such as `ENABLE_HPU_GRAPH` and `USE_FLASH_ATTENTION`. The `chatqna-gaudi-backend-server` in the TGI deployment depends on the `tgi-service`, whereas in the default deployment, it relies on the `vllm-service`. -```bash -# delete link -curl -X POST "http://${host_ip}:6007/v1/dataprep/delete" \ - -d '{"file_path": "https://opea.dev.txt"}' \ - -H "Content-Type: application/json" - -# delete file -curl -X POST "http://${host_ip}:6007/v1/dataprep/delete" \ - -d '{"file_path": "nke-10k-2023.pdf"}' \ - -H "Content-Type: application/json" - -# delete all uploaded files and links -curl -X POST "http://${host_ip}:6007/v1/dataprep/delete" \ - -d '{"file_path": "all"}' \ - -H "Content-Type: application/json" -``` +| Service Name | Image Name | Gaudi Specific | +| ---------------------------- | ----------------------------------------------------- | -------------- | +| redis-vector-db | redis/redis-stack:7.2.0-v9 | No | +| dataprep-redis-service | opea/dataprep:latest | No | +| tei-embedding-service | ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 | No | +| retriever | opea/retriever:latest | No | +| tei-reranking-service | ghcr.io/huggingface/tei-gaudi:1.5.0 | 1 card | +| **tgi-service** | ghcr.io/huggingface/tgi-gaudi:2.0.6 | Configurable | +| chatqna-gaudi-backend-server | opea/chatqna:latest | No | +| chatqna-gaudi-ui-server | opea/chatqna-ui:latest | No | +| chatqna-gaudi-nginx-server | opea/nginx:latest | No | -8. Guardrails (Optional) +This deployment may allocate more Gaudi resources to the tgi-service to optimize LLM tasks depending on the specific configuration and workload requirements. -```bash -curl http://${host_ip}:9090/v1/guardrails\ - -X POST \ - -d '{"text":"How do you buy a tiger in the US?","parameters":{"max_new_tokens":32}}' \ - -H 'Content-Type: application/json' -``` +### compose_without_rerank.yaml - No ReRank Deployment -### Profile Microservices +The _compose_without_rerank.yaml_ Docker Compose file is distinct from the default deployment primarily due to the exclusion of the reranking service. In this version, the `tei-reranking-service`, which is typically responsible for providing reranking capabilities for text embeddings and is configured to run on Gaudi hardware, is absent. This omission simplifies the service architecture by removing a layer of processing that would otherwise enhance the ranking of text embeddings. Consequently, the `chatqna-gaudi-backend-server` in this deployment uses a specialized image, `opea/chatqna-without-rerank:latest`, indicating that it is tailored to function without the reranking feature. As a result, the backend server's dependencies are adjusted, without the need for the reranking service. This streamlined setup may impact the application's functionality and performance by focusing on core operations without the additional processing layer provided by reranking, potentially making it more efficient for scenarios where reranking is not essential and freeing Intel® Gaudi® accelerators for other tasks. -To further analyze MicroService Performance, users could follow the instructions to profile MicroServices. +| Service Name | Image Name | Gaudi Specific | +| ---------------------------- | ----------------------------------------------------- | -------------- | +| redis-vector-db | redis/redis-stack:7.2.0-v9 | No | +| dataprep-redis-service | opea/dataprep:latest | No | +| tei-embedding-service | ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 | No | +| retriever | opea/retriever:latest | No | +| vllm-service | opea/vllm-gaudi:latest | Configurable | +| chatqna-gaudi-backend-server | **opea/chatqna-without-rerank:latest** | No | +| chatqna-gaudi-ui-server | opea/chatqna-ui:latest | No | +| chatqna-gaudi-nginx-server | opea/nginx:latest | No | -#### 1. vLLM backend Service +This setup might allow for more Gaudi devices to be dedicated to the `vllm-service`, enhancing LLM processing capabilities and accommodating larger models. However, it also means that the benefits of reranking are sacrificed, which could impact the overall quality of the pipeline's output. -Users could follow previous section to testing vLLM microservice or ChatQnA MegaService. - By default, vLLM profiling is not enabled. Users could start and stop profiling by following commands. +### compose_guardrails.yaml - Guardrails Deployment -##### Start vLLM profiling +The _compose_guardrails.yaml_ Docker Compose file introduces enhancements over the default deployment by incorporating additional services focused on safety and ChatQnA response control. Notably, it includes the `tgi-guardrails-service` and `guardrails` services. The `tgi-guardrails-service` uses the `ghcr.io/huggingface/tgi-gaudi:2.0.6` image and is configured to run on Gaudi hardware, providing functionality to manage input constraints and ensure safe operations within defined limits. The guardrails service, using the `opea/guardrails:latest` image, acts as a safety layer that interfaces with the `tgi-guardrails-service` to enforce safety protocols and manage interactions with the large language model (LLM). Additionally, the `chatqna-gaudi-backend-server` is updated to use the `opea/chatqna-guardrails:latest` image, indicating its design to integrate with these new guardrail services. This backend server now depends on the `tgi-guardrails-service` and `guardrails`, alongside existing dependencies like `redis-vector-db`, `tei-embedding-service`, `retriever`, `tei-reranking-service`, and `vllm-service`. The environment configurations for the backend are also updated to include settings for the guardrail services. -```bash -curl http://${host_ip}:9009/start_profile \ - -H "Content-Type: application/json" \ - -d '{"model": ${LLM_MODEL_ID}}' -``` +| Service Name | Image Name | Gaudi Specific | Uses LLM | +| ---------------------------- | ----------------------------------------------------- | -------------- | -------- | +| redis-vector-db | redis/redis-stack:7.2.0-v9 | No | No | +| dataprep-redis-service | opea/dataprep:latest | No | No | +| _tgi-guardrails-service_ | ghcr.io/huggingface/tgi-gaudi:2.0.6 | 1 card | Yes | +| _guardrails_ | opea/guardrails:latest | No | No | +| tei-embedding-service | ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 | No | No | +| retriever | opea/retriever:latest | No | No | +| tei-reranking-service | ghcr.io/huggingface/tei-gaudi:1.5.0 | 1 card | No | +| vllm-service | opea/vllm-gaudi:latest | Configurable | Yes | +| chatqna-gaudi-backend-server | opea/chatqna-guardrails:latest | No | No | +| chatqna-gaudi-ui-server | opea/chatqna-ui:latest | No | No | +| chatqna-gaudi-nginx-server | opea/nginx:latest | No | No | -Users would see below docker logs from vllm-service if profiling is started correctly. +The deployment with guardrails introduces additional Gaudi-specific services, such as the `tgi-guardrails-service`, which necessitates careful consideration of Gaudi allocation. This deployment aims to balance safety and performance, potentially requiring a strategic distribution of Gaudi devices between the guardrail services and the LLM tasks to maintain both operational safety and efficiency. -```bash -INFO api_server.py:361] Starting profiler... -INFO api_server.py:363] Profiler started. -INFO: x.x.x.x:35940 - "POST /start_profile HTTP/1.1" 200 OK -``` +### Telemetry Enablement - compose.telemetry.yaml and compose_tgi.telemetry.yaml -After vLLM profiling is started, users could start asking questions and get responses from vLLM MicroService - or ChatQnA MicroService. +The telemetry Docker Compose files are incremental configurations designed to enhance existing deployments by integrating telemetry metrics, thereby providing valuable insights into the performance and behavior of certain services. This setup modifies specific services, such as the `tgi-service`, `tei-embedding-service` and `tei-reranking-service`, by adding a command-line argument that specifies an OpenTelemetry Protocol (OTLP) endpoint. This enables these services to export telemetry data to a designated endpoint, facilitating detailed monitoring and analysis. The `chatqna-gaudi-backend-server` is configured with environment variables that enable telemetry and specify the telemetry endpoint, ensuring that the backend server's operations are also monitored. -##### Stop vLLM profiling +Additionally, the telemetry files introduce a new service, `jaeger`, which uses the `jaegertracing/all-in-one:latest` image. Jaeger is a powerful open-source tool for tracing and monitoring distributed systems, offering a user-friendly interface for visualizing traces and understanding the flow of requests through the system. -By following command, users could stop vLLM profliing and generate a \*.pt.trace.json.gz file as profiling result - under /mnt folder in vllm-service docker instance. +To enable Open Telemetry Tracing, compose.telemetry.yaml file needs to be merged along with default compose.yaml file on deployment: -```bash -# vLLM Service -curl http://${host_ip}:9009/stop_profile \ - -H "Content-Type: application/json" \ - -d '{"model": ${LLM_MODEL_ID}}' ``` - -Users would see below docker logs from vllm-service if profiling is stopped correctly. - -```bash -INFO api_server.py:368] Stopping profiler... -INFO api_server.py:370] Profiler stopped. -INFO: x.x.x.x:41614 - "POST /stop_profile HTTP/1.1" 200 OK +docker compose -f compose.yaml -f compose.telemetry.yaml up -d ``` -After vllm profiling is stopped, users could use below command to get the \*.pt.trace.json.gz file under /mnt folder. +For a TGI Deployment, this would become: -```bash -docker cp vllm-service:/mnt/ . +``` +docker compose -f compose_tgi.yaml -f compose_tgi.telemetry.yaml up -d ``` -##### Check profiling result - -Open a web browser and type "chrome://tracing" or "ui.perfetto.dev", and then load the json.gz file, you should be able - to see the vLLM profiling result as below diagram. -![image](https://github.com/user-attachments/assets/487c52c8-d187-46dc-ab3a-43f21d657d41) - -![image](https://github.com/user-attachments/assets/e3c51ce5-d704-4eb7-805e-0d88b0c158e3) +## ChatQnA Service Configuration -## 🚀 Launch the UI +The table provides a comprehensive overview of the ChatQnA services utilized across various deployments as illustrated in the example Docker Compose files. Each row in the table represents a distinct service, detailing its possible images used to enable it and a concise description of its function within the deployment architecture. These services collectively enable functionalities such as data storage and management, text embedding, retrieval, reranking, and large language model processing. Additionally, specialized services like `tgi-service` and `guardrails` are included to enhance text generation inference and ensure operational safety, respectively. The table also highlights the integration of telemetry through the `jaeger` service, which provides tracing and monitoring capabilities. -### Launch with origin port +| Service Name | Possible Image Names | Optional | Description | +| ---------------------------- | ----------------------------------------------------- | -------- | -------------------------------------------------------------------------------------------------- | +| redis-vector-db | redis/redis-stack:7.2.0-v9 | No | Acts as a Redis database for storing and managing data. | +| dataprep-redis-service | opea/dataprep:latest | No | Prepares data and interacts with the Redis database. | +| tei-embedding-service | ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 | No | Provides text embedding services, often using Hugging Face models. | +| retriever | opea/retriever:latest | No | Retrieves data from the Redis database and interacts with embedding services. | +| tei-reranking-service | ghcr.io/huggingface/tei-gaudi:1.5.0 | Yes | Reranks text embeddings, typically using Gaudi hardware for enhanced performance. | +| vllm-service | opea/vllm-gaudi:latest | No | Handles large language model (LLM) tasks, utilizing Gaudi hardware. | +| tgi-service | ghcr.io/huggingface/tgi-gaudi:2.0.6 | Yes | Specific to the TGI deployment, focuses on text generation inference using Gaudi hardware. | +| tgi-guardrails-service | ghcr.io/huggingface/tgi-gaudi:2.0.6 | Yes | Provides guardrails functionality, ensuring safe operations within defined limits. | +| guardrails | opea/guardrails:latest | Yes | Acts as a safety layer, interfacing with the `tgi-guardrails-service` to enforce safety protocols. | +| chatqna-gaudi-backend-server | opea/chatqna:latest | No | Serves as the backend for the ChatQnA application, with variations depending on the deployment. | +| | opea/chatqna-without-rerank:latest | | | +| | opea/chatqna-guardrails:latest | | | +| chatqna-gaudi-ui-server | opea/chatqna-ui:latest | No | Provides the user interface for the ChatQnA application. | +| chatqna-gaudi-nginx-server | opea/nginx:latest | No | Acts as a reverse proxy, managing traffic between the UI and backend services. | +| jaeger | jaegertracing/all-in-one:latest | Yes | Provides tracing and monitoring capabilities for distributed systems. | -To access the frontend, open the following URL in your browser: http://{host_ip}:5173. By default, the UI runs on port 5173 internally. If you prefer to use a different host port to access the frontend, you can modify the port mapping in the `compose.yaml` file as shown below: +Many of these services provide pipeline support required for all ChatQnA deployments, and are not specific to supporting the Intel® Gaudi® platform. Therefore, while the `redis-vector-db`, `dataprep-redis-service`, `retriever`, `chatqna-gaudi-backend-server`, `chatqna-gaudi-ui-server`, `chatqna-gaudi-nginx-server`, `jaeger` are configurable, they will not be covered by this example, which will focus on the configuration specifics of the services modified to support the Intel® Gaudi® platform. -```yaml - chatqna-gaudi-ui-server: - image: opea/chatqna-ui:latest - ... - ports: - - "80:5173" -``` +### vllm-service & tgi-service -### Launch with Nginx +In the configuration of the `vllm-service` and the `tgi-service`, two variables play a primary role in determining the service's performance and functionality: `LLM_MODEL_ID` and `NUM_CARDS`. Both can be set using the appropriate environment variables. The `LLM_MODEL_ID` parameter specifies the particular large language model (LLM) that the service will utilize, effectively determining the capabilities and characteristics of the language processing tasks it can perform. This model identifier ensures that the service is aligned with the specific requirements of the application, whether it involves text generation, comprehension, or other language-related tasks. The `NUM_CARDS` parameter dictates the number of Gaudi devices allocated to the service. A higher number of Gaudi devices can enhance parallel processing capabilities, reduce latency, and improve throughput. -If you want to launch the UI using Nginx, open this URL: `http://${host_ip}:${NGINX_PORT}` in your browser to access the frontend. +However, developers need to be aware of the models that have been tested with the respective service image supporting the `vllm-service` and `tgi-service`. For example, documentation for the OPEA GenAIComps v1.0 release specify the list of [validated LLM models](https://github.com/opea-project/GenAIComps/blob/v1.0/comps/llms/text-generation/README.md#validated-llm-models) for each Gaudi enabled service image. Specific models may have stringent requirements on the number of Intel® Gaudi® devices required to support them. -## 🚀 Launch the Conversational UI (Optional) +#### Deepseek Model Support for Intel® Gaudi® Platform ChatQnA pipeline -To access the Conversational UI (react based) frontend, modify the UI service in the `compose.yaml` file. Replace `chatqna-gaudi-ui-server` service with the `chatqna-gaudi-conversation-ui-server` service as per the config below: +ChatQnA now supports running the latest DeepSeek models, including [deepseek-ai/DeepSeek-R1-Distill-Llama-70B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B) and [deepseek-ai/DeepSeek-R1-Distill-Qwen-32B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B) on Gaudi accelerators. To run `deepseek-ai/DeepSeek-R1-Distill-Llama-70B`, set the `LLM_MODEL_ID` appropriately and the `NUM_CARDS` to 8. To run `deepseek-ai/DeepSeek-R1-Distill-Qwen-32B`, update the `LLM_MODEL_ID` appropriately and set the `NUM_CARDS` to 4. -```yaml -chatqna-gaudi-conversation-ui-server: - image: opea/chatqna-conversation-ui:latest - container_name: chatqna-gaudi-conversation-ui-server - environment: - - APP_BACKEND_SERVICE_ENDPOINT=${BACKEND_SERVICE_ENDPOINT} - - APP_DATA_PREP_SERVICE_URL=${DATAPREP_SERVICE_ENDPOINT} - ports: - - "5174:80" - depends_on: - - chatqna-gaudi-backend-server - ipc: host - restart: always -``` +### tei-embedding-service & tei-reranking-service -Once the services are up, open the following URL in your browser: http://{host_ip}:5174. By default, the UI runs on port 80 internally. If you prefer to use a different host port to access the frontend, you can modify the port mapping in the `compose.yaml` file as shown below: +The `ghcr.io/huggingface/text-embeddings-inference:cpu-1.5` image supporting `tei-embedding-service` and `tei-reranking-service` depends on the `EMBEDDING_MODEL_ID` or `RERANK_MODEL_ID` environment variables respectively to specify the embedding model and reranking model used for converting text into vector representations and rankings. This choice impacts the quality and relevance of the embeddings rerankings for various applications. Unlike the `vllm-service`, the `tei-embedding-service` and `tei-reranking-service` each typically acquires only one Gaudi device and does not use the `NUM_CARDS` parameter; embedding and reranking tasks generally do not require extensive parallel processing and one Gaudi per service is appropriate. The list of [supported embedding and reranking models](https://github.com/huggingface/tei-gaudi?tab=readme-ov-file#supported-models) can be found at the the [huggingface/tei-gaudi](https://github.com/huggingface/tei-gaudi?tab=readme-ov-file#supported-models) website. -```yaml - chatqna-gaudi-conversation-ui-server: - image: opea/chatqna-conversation-ui:latest - ... - ports: - - "80:80" -``` +### tgi-gaurdrails-service -![project-screenshot](../../../../assets/img/chat_ui_init.png) +The `tgi-guardrails-service` uses the `GUARDRAILS_MODEL_ID` parameter to select a [supported model](https://github.com/huggingface/tgi-gaudi?tab=readme-ov-file#tested-models-and-configurations) for the associated `ghcr.io/huggingface/tgi-gaudi:2.0.6` image. Like the `tei-embedding-service` and `tei-reranking-service` services, it doesn't use the `NUM_CARDS` parameter. -Here is an example of running ChatQnA: +## Conclusion -![project-screenshot](../../../../assets/img/chat_ui_response.png) +In examining the various services and configurations across different deployments, developers should gain a comprehensive understanding of how each component contributes to the overall functionality and performance of a ChatQnA pipeline on an Intel® Gaudi® platform. Key services such as the `vllm-service`, `tei-embedding-service`, `tei-reranking-service`, and `tgi-guardrails-service` each consume Gaudi accelerators, leveraging specific models and hardware resources to optimize their respective tasks. The `LLM_MODEL_ID`, `EMBEDDING_MODEL_ID`, `RERANK_MODEL_ID`, and `GUARDRAILS_MODEL_ID` parameters specify the models used, directly impacting the quality and effectiveness of language processing, embedding, reranking, and safety operations. -Here is an example of running ChatQnA with Conversational UI (React): +The allocation of Gaudi devices, affected by the Gaudi dependent services and the `NUM_CARDS` parameter supporting the `vllm-service` or `tgi-service`, determines where computational power is utilized to enhance performance. -![project-screenshot](../../../../assets/img/conversation_ui_response.png) +Overall, the strategic configuration of these services, through careful selection of models and resource allocation, enables a balanced and efficient deployment. This approach ensures that the ChatQnA pipeline can meet diverse operational needs, from high-performance language model processing to robust safety protocols, all while optimizing the use of available hardware resources. diff --git a/ChatQnA/docker_compose/intel/hpu/gaudi/set_env.sh b/ChatQnA/docker_compose/intel/hpu/gaudi/set_env.sh old mode 100644 new mode 100755 index b0cba1834c..824d1d8fb1 --- a/ChatQnA/docker_compose/intel/hpu/gaudi/set_env.sh +++ b/ChatQnA/docker_compose/intel/hpu/gaudi/set_env.sh @@ -1,22 +1,94 @@ -#!/usr/bin/env bash +#/usr/bin/env bash # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 + +# Function to prompt for input and set environment variables +prompt_for_env_var() { + local var_name="$1" + local prompt_message="$2" + local default_value="$3" + local mandatory="$4" + + if [[ "$mandatory" == "true" ]]; then + while [[ -z "$value" ]]; do + read -p "$prompt_message [default: \"${default_value}\"]: " value + if [[ -z "$value" ]]; then + echo "Input cannot be empty. Please try again." + fi + done + else + read -p "$prompt_message [default: \"${default_value}\"]: " value + fi + + if [[ "$value" == "" ]]; then + export "$var_name"="$default_value" + else + export "$var_name"="$value" + fi +} + pushd "../../../../../" > /dev/null source .set_env.sh popd > /dev/null +# Prompt the user for each required environment variable +prompt_for_env_var "EMBEDDING_MODEL_ID" "Enter the EMBEDDING_MODEL_ID" "BAAI/bge-base-en-v1.5" false +prompt_for_env_var "HUGGINGFACEHUB_API_TOKEN" "Enter the HUGGINGFACEHUB_API_TOKEN" "" true +prompt_for_env_var "RERANK_MODEL_ID" "Enter the RERANK_MODEL_ID" "BAAI/bge-reranker-base" false +prompt_for_env_var "LLM_MODEL_ID" "Enter the LLM_MODEL_ID" "meta-llama/Meta-Llama-3-8B-Instruct" false +prompt_for_env_var "INDEX_NAME" "Enter the INDEX_NAME" "rag-redis" false +prompt_for_env_var "NUM_CARDS" "Enter the number of Gaudi devices" "1" false +prompt_for_env_var "host_ip" "Enter the host_ip" "$(curl ifconfig.me)" false + +#Query for enabling http_proxy +prompt_for_env_var "http_proxy" "Enter the http_proxy." "" false + +#Query for enabling https_proxy +prompt_for_env_var "https_proxy" "Enter the https_proxy." "" false + +#Query for enabling no_proxy +prompt_for_env_var "no_proxy" "Enter the no_proxy." "" false + +# Query for enabling logging +read -p "Enable logging? (yes/no): " logging && logging=$(echo "$logging" | tr '[:upper:]' '[:lower:]') +if [[ "$logging" == "yes" || "$logging" == "y" ]]; then + export LOGFLAG=true +else + export LOGFLAG=false +fi + +# Query for enabling OpenTelemetry Tracing Endpoint +read -p "Enable OpenTelemetry Tracing Endpoint? (yes/no): " telemetry && telemetry=$(echo "$telemetry" | tr '[:upper:]' '[:lower:]') +if [[ "$telemetry" == "yes" || "$telemetry" == "y" ]]; then + export JAEGER_IP=$(ip route get 8.8.8.8 | grep -oP 'src \K[^ ]+') + export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=grpc://$JAEGER_IP:4317 + export TELEMETRY_ENDPOINT=http://$JAEGER_IP:4318/v1/traces + telemetry_flag=true +else + telemetry_flag=false +fi + +# Generate the .env file +cat < .env +#!/bin/bash +# Set all required ENV values +export TAG=${TAG} +export EMBEDDING_MODEL_ID=${EMBEDDING_MODEL_ID} +export HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN +export RERANK_MODEL_ID=${RERANK_MODEL_ID} +export LLM_MODEL_ID=${LLM_MODEL_ID} +export INDEX_NAME=${INDEX_NAME} +export NUM_CARDS=${NUM_CARDS} +export host_ip=${host_ip} +export http_proxy=${http_proxy} +export https_proxy=${https_proxy} +export no_proxy=${no_proxy} +export LOGFLAG=${LOGFLAG} +export JAEGER_IP=${JAEGER_IP} +export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT} +export TELEMETRY_ENDPOINT=${TELEMETRY_ENDPOINT} +EOF -export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5" -export RERANK_MODEL_ID="BAAI/bge-reranker-base" -export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct" -export INDEX_NAME="rag-redis" -export NUM_CARDS=1 -# Set it as a non-null string, such as true, if you want to enable logging facility, -# otherwise, keep it as "" to disable it. -export LOGFLAG="" -# Set OpenTelemetry Tracing Endpoint -export JAEGER_IP=$(ip route get 8.8.8.8 | grep -oP 'src \K[^ ]+') -export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=grpc://$JAEGER_IP:4317 -export TELEMETRY_ENDPOINT=http://$JAEGER_IP:4318/v1/traces -export no_proxy="$no_proxy,chatqna-gaudi-ui-server,chatqna-gaudi-backend-server,dataprep-redis-service,tei-embedding-service,retriever,tei-reranking-service,tgi-gaudi-server,vllm-gaudi-server,guardrails,jaeger,prometheus,grafana,node-exporter,gaudi-exporter,$JAEGER_IP" +echo ".env file has been created with the following content:" +cat .env From 67ab2462b169cf7518158bba1d8dc7247c0bdba5 Mon Sep 17 00:00:00 2001 From: "chen, suyue" Date: Mon, 17 Mar 2025 13:11:53 +0800 Subject: [PATCH 068/226] Fix input issue for manual-image-build.yml (#1666) Signed-off-by: chensuyue Signed-off-by: Chingis Yundunov --- .github/workflows/_example-workflow.yml | 8 ++++---- .github/workflows/manual-example-workflow.yml | 4 ++-- .github/workflows/manual-image-build.yml | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/_example-workflow.yml b/.github/workflows/_example-workflow.yml index d56099a476..897f98e914 100644 --- a/.github/workflows/_example-workflow.yml +++ b/.github/workflows/_example-workflow.yml @@ -50,6 +50,9 @@ on: type: boolean jobs: +#################################################################################################### +# Image Build +#################################################################################################### pre-build-image-check: runs-on: ubuntu-latest outputs: @@ -64,9 +67,6 @@ jobs: echo "should_skip=false" >> $GITHUB_OUTPUT fi -#################################################################################################### -# Image Build -#################################################################################################### build-images: needs: [pre-build-image-check] if: ${{ needs.pre-build-image-check.outputs.should_skip == 'false' }} @@ -133,7 +133,7 @@ jobs: run: | set -x run_compose="false" - if [[ ${{ inputs.test_compose }} ]]; then + if [[ "${{ inputs.test_compose }}" == "true" ]]; then if [[ "${{ needs.pre-build-image-check.outputs.should_skip }}" == "false" && "${{ needs.build-images.result}}" == "success" || "${{ needs.pre-build-image-check.outputs.should_skip }}" == "true" ]]; then run_compose="true" fi diff --git a/.github/workflows/manual-example-workflow.yml b/.github/workflows/manual-example-workflow.yml index 9616f87032..7159b8a8a0 100644 --- a/.github/workflows/manual-example-workflow.yml +++ b/.github/workflows/manual-example-workflow.yml @@ -52,12 +52,12 @@ on: type: string inject_commit: default: false - description: "inject commit to docker images true or false" + description: "inject commit to docker images" required: false type: boolean use_model_cache: default: false - description: "use model cache true or false" + description: "use model cache" required: false type: boolean diff --git a/.github/workflows/manual-image-build.yml b/.github/workflows/manual-image-build.yml index b2b41ea0d7..92da9c2231 100644 --- a/.github/workflows/manual-image-build.yml +++ b/.github/workflows/manual-image-build.yml @@ -32,9 +32,9 @@ on: type: string inject_commit: default: false - description: "inject commit to docker images true or false" + description: "inject commit to docker images" required: false - type: string + type: boolean jobs: get-test-matrix: From 422698dcee84b7500d8ba55d3cd6166e1ce34fee Mon Sep 17 00:00:00 2001 From: Spycsh <39623753+Spycsh@users.noreply.github.com> Date: Tue, 18 Mar 2025 15:29:49 +0800 Subject: [PATCH 069/226] Set vLLM as default model for VisualQnA (#1644) Signed-off-by: Chingis Yundunov --- .../docker_compose/intel/cpu/xeon/README.md | 66 ++---- .../intel/cpu/xeon/compose.yaml | 30 +-- .../intel/cpu/xeon/compose_tgi.yaml | 96 ++++++++ .../docker_compose/intel/cpu/xeon/set_env.sh | 3 +- .../docker_compose/intel/hpu/gaudi/README.md | 43 ++-- .../intel/hpu/gaudi/compose.yaml | 46 ++-- .../intel/hpu/gaudi/compose_tgi.yaml | 105 +++++++++ .../docker_compose/intel/hpu/gaudi/set_env.sh | 5 +- VisualQnA/tests/test_compose_on_gaudi.sh | 57 ++--- VisualQnA/tests/test_compose_on_xeon.sh | 48 +--- VisualQnA/tests/test_compose_tgi_on_gaudi.sh | 222 ++++++++++++++++++ VisualQnA/tests/test_compose_tgi_on_xeon.sh | 222 ++++++++++++++++++ 12 files changed, 762 insertions(+), 181 deletions(-) create mode 100644 VisualQnA/docker_compose/intel/cpu/xeon/compose_tgi.yaml create mode 100644 VisualQnA/docker_compose/intel/hpu/gaudi/compose_tgi.yaml create mode 100644 VisualQnA/tests/test_compose_tgi_on_gaudi.sh create mode 100644 VisualQnA/tests/test_compose_tgi_on_xeon.sh diff --git a/VisualQnA/docker_compose/intel/cpu/xeon/README.md b/VisualQnA/docker_compose/intel/cpu/xeon/README.md index a6799eb38b..cfbc3ab1c1 100644 --- a/VisualQnA/docker_compose/intel/cpu/xeon/README.md +++ b/VisualQnA/docker_compose/intel/cpu/xeon/README.md @@ -10,28 +10,6 @@ For detailed information about these instance types, you can refer to this [link After launching your instance, you can connect to it using SSH (for Linux instances) or Remote Desktop Protocol (RDP) (for Windows instances). From there, you'll have full access to your Xeon server, allowing you to install, configure, and manage your applications as needed. -**Certain ports in the EC2 instance need to opened up in the security group, for the microservices to work with the curl commands** - -> See one example below. Please open up these ports in the EC2 instance based on the IP addresses you want to allow - -``` -llava-tgi-service -=========== -Port 8399 - Open to 0.0.0.0/0 - -llm -=== -Port 9399 - Open to 0.0.0.0/0 - -visualqna-xeon-backend-server -========================== -Port 8888 - Open to 0.0.0.0/0 - -visualqna-xeon-ui-server -===================== -Port 5173 - Open to 0.0.0.0/0 -``` - ## 🚀 Build Docker Images First of all, you need to build Docker Images locally and install the python package of it. @@ -64,19 +42,23 @@ cd GenAIExamples/VisualQnA/ui docker build --no-cache -t opea/visualqna-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . ``` -### 4. Pull TGI Xeon Image +### 4. Pull vLLM/TGI Xeon Image ```bash +# vLLM +docker pull opea/vllm:latest +# TGI (Optional) docker pull ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu ``` -Then run the command `docker images`, you will have the following 5 Docker Images: +Then run the command `docker images`, you will have the following Docker Images: -1. `ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu` -2. `opea/lvm:latest` -3. `opea/visualqna:latest` -4. `opea/visualqna-ui:latest` -5. `opea/nginx` +1. `opea/vllm:latest` +2. `ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu` (Optional) +3. `opea/lvm:latest` +4. `opea/visualqna:latest` +5. `opea/visualqna-ui:latest` +6. `opea/nginx` ## 🚀 Start Microservices @@ -84,30 +66,8 @@ Then run the command `docker images`, you will have the following 5 Docker Image Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below. -**Export the value of the public IP address of your Xeon server to the `host_ip` environment variable** - -> Change the External_Public_IP below with the actual IPV4 value - -``` -export host_ip="External_Public_IP" -``` - -**Append the value of the public IP address to the no_proxy list** - -``` -export your_no_proxy="${your_no_proxy},${host_ip}" -``` - ```bash -export no_proxy=${your_no_proxy} -export http_proxy=${your_http_proxy} -export https_proxy=${your_http_proxy} -export LVM_MODEL_ID="llava-hf/llava-v1.6-mistral-7b-hf" -export LVM_ENDPOINT="http://${host_ip}:8399" -export LVM_SERVICE_PORT=9399 -export MEGA_SERVICE_HOST_IP=${host_ip} -export LVM_SERVICE_HOST_IP=${host_ip} -export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/visualqna" +source set_env.sh ``` Note: Please replace with `host_ip` with you external IP address, do not use localhost. @@ -122,6 +82,8 @@ cd GenAIExamples/VisualQnA/docker_compose/intel/cpu/xeon ```bash docker compose -f compose.yaml up -d +# if use TGI as the LLM serving backend +docker compose -f compose_tgi.yaml up -d ``` ### Validate Microservices diff --git a/VisualQnA/docker_compose/intel/cpu/xeon/compose.yaml b/VisualQnA/docker_compose/intel/cpu/xeon/compose.yaml index b595bdcba7..9c19695493 100644 --- a/VisualQnA/docker_compose/intel/cpu/xeon/compose.yaml +++ b/VisualQnA/docker_compose/intel/cpu/xeon/compose.yaml @@ -2,32 +2,31 @@ # SPDX-License-Identifier: Apache-2.0 services: - llava-tgi-service: - image: ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu - container_name: tgi-llava-xeon-server + vllm-service: + image: ${REGISTRY:-opea}/vllm:${TAG:-latest} + container_name: vllm-service ports: - - "8399:80" + - ${VLLM_PORT:-8399}:80 volumes: - - "${MODEL_CACHE:-./data}:/data" - shm_size: 1g + - "${MODEL_CACHE:-./data}:/root/.cache/huggingface/hub" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} - HF_HUB_DISABLE_PROGRESS_BARS: 1 - HF_HUB_ENABLE_HF_TRANSFER: 0 - host_ip: ${host_ip} + HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + VLLM_TORCH_PROFILER_DIR: "/mnt" healthcheck: - test: ["CMD-SHELL", "curl -f http://$host_ip:8399/health || exit 1"] + test: ["CMD-SHELL", "curl -f http://localhost:80/health || exit 1"] interval: 10s timeout: 10s - retries: 60 - command: --model-id ${LVM_MODEL_ID} --max-input-length 4096 --max-total-tokens 8192 --cuda-graphs 0 + retries: 100 + command: --model $LVM_MODEL_ID --host 0.0.0.0 --port 80 --chat-template examples/template_llava.jinja # https://docs.vllm.ai/en/v0.5.0/models/vlm.html + lvm: image: ${REGISTRY:-opea}/lvm:${TAG:-latest} container_name: lvm-xeon-server depends_on: - llava-tgi-service: + vllm-service: condition: service_healthy ports: - "9399:9399" @@ -37,7 +36,8 @@ services: http_proxy: ${http_proxy} https_proxy: ${https_proxy} LVM_ENDPOINT: ${LVM_ENDPOINT} - LVM_COMPONENT_NAME: "OPEA_TGI_LLAVA_LVM" + LVM_COMPONENT_NAME: "OPEA_VLLM_LVM" + LLM_MODEL_ID: ${LVM_MODEL_ID} HF_HUB_DISABLE_PROGRESS_BARS: 1 HF_HUB_ENABLE_HF_TRANSFER: 0 restart: unless-stopped @@ -45,7 +45,7 @@ services: image: ${REGISTRY:-opea}/visualqna:${TAG:-latest} container_name: visualqna-xeon-backend-server depends_on: - - llava-tgi-service + - vllm-service - lvm ports: - "8888:8888" diff --git a/VisualQnA/docker_compose/intel/cpu/xeon/compose_tgi.yaml b/VisualQnA/docker_compose/intel/cpu/xeon/compose_tgi.yaml new file mode 100644 index 0000000000..b595bdcba7 --- /dev/null +++ b/VisualQnA/docker_compose/intel/cpu/xeon/compose_tgi.yaml @@ -0,0 +1,96 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +services: + llava-tgi-service: + image: ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu + container_name: tgi-llava-xeon-server + ports: + - "8399:80" + volumes: + - "${MODEL_CACHE:-./data}:/data" + shm_size: 1g + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + host_ip: ${host_ip} + healthcheck: + test: ["CMD-SHELL", "curl -f http://$host_ip:8399/health || exit 1"] + interval: 10s + timeout: 10s + retries: 60 + command: --model-id ${LVM_MODEL_ID} --max-input-length 4096 --max-total-tokens 8192 --cuda-graphs 0 + lvm: + image: ${REGISTRY:-opea}/lvm:${TAG:-latest} + container_name: lvm-xeon-server + depends_on: + llava-tgi-service: + condition: service_healthy + ports: + - "9399:9399" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + LVM_ENDPOINT: ${LVM_ENDPOINT} + LVM_COMPONENT_NAME: "OPEA_TGI_LLAVA_LVM" + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + restart: unless-stopped + visualqna-xeon-backend-server: + image: ${REGISTRY:-opea}/visualqna:${TAG:-latest} + container_name: visualqna-xeon-backend-server + depends_on: + - llava-tgi-service + - lvm + ports: + - "8888:8888" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP} + - LVM_SERVICE_HOST_IP=${LVM_SERVICE_HOST_IP} + ipc: host + restart: always + visualqna-xeon-ui-server: + image: ${REGISTRY:-opea}/visualqna-ui:${TAG:-latest} + container_name: visualqna-xeon-ui-server + depends_on: + - visualqna-xeon-backend-server + ports: + - "5173:5173" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - BACKEND_BASE_URL=${BACKEND_SERVICE_ENDPOINT} + ipc: host + restart: always + visualqna-xeon-nginx-server: + image: ${REGISTRY:-opea}/nginx:${TAG:-latest} + container_name: visualqna-xeon-nginx-server + depends_on: + - visualqna-xeon-backend-server + - visualqna-xeon-ui-server + ports: + - "${NGINX_PORT:-80}:80" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - FRONTEND_SERVICE_IP=${FRONTEND_SERVICE_IP} + - FRONTEND_SERVICE_PORT=${FRONTEND_SERVICE_PORT} + - BACKEND_SERVICE_NAME=${BACKEND_SERVICE_NAME} + - BACKEND_SERVICE_IP=${BACKEND_SERVICE_IP} + - BACKEND_SERVICE_PORT=${BACKEND_SERVICE_PORT} + ipc: host + restart: always + +networks: + default: + driver: bridge diff --git a/VisualQnA/docker_compose/intel/cpu/xeon/set_env.sh b/VisualQnA/docker_compose/intel/cpu/xeon/set_env.sh index 58874c7d66..b47f12fe31 100644 --- a/VisualQnA/docker_compose/intel/cpu/xeon/set_env.sh +++ b/VisualQnA/docker_compose/intel/cpu/xeon/set_env.sh @@ -6,7 +6,8 @@ pushd "../../../../../" > /dev/null source .set_env.sh popd > /dev/null - +export host_ip=$(hostname -I | awk '{print $1}') +export no_proxy=$host_ip,$no_proxy export LVM_MODEL_ID="llava-hf/llava-v1.6-mistral-7b-hf" export LVM_ENDPOINT="http://${host_ip}:8399" export LVM_SERVICE_PORT=9399 diff --git a/VisualQnA/docker_compose/intel/hpu/gaudi/README.md b/VisualQnA/docker_compose/intel/hpu/gaudi/README.md index c7b026cd9f..9c3b0cd4e0 100644 --- a/VisualQnA/docker_compose/intel/hpu/gaudi/README.md +++ b/VisualQnA/docker_compose/intel/hpu/gaudi/README.md @@ -15,15 +15,29 @@ docker build --no-cache -t opea/lvm:latest --build-arg https_proxy=$https_proxy docker build --no-cache -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/third_parties/nginx/src/Dockerfile . ``` -### 2. Pull TGI Gaudi Image +### 2. Build vLLM/Pull TGI Gaudi Image ```bash +# vLLM + +# currently you have to build the opea/vllm-gaudi with the habana_main branch and the specific commit locally +# we will update it to stable release tag in the future +git clone https://github.com/HabanaAI/vllm-fork.git +cd ./vllm-fork/ +docker build -f Dockerfile.hpu -t opea/vllm-gaudi:latest --shm-size=128g . --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy +cd .. +rm -rf vllm-fork +``` + +```bash +# TGI (Optional) + docker pull ghcr.io/huggingface/tgi-gaudi:2.0.6 ``` ### 3. Build MegaService Docker Image -To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `visuralqna.py` Python script. Build the MegaService Docker image using the command below: +To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the `visualqna.py` Python script. Build the MegaService Docker image using the command below: ```bash git clone https://github.com/opea-project/GenAIExamples.git @@ -43,11 +57,12 @@ docker build --no-cache -t opea/visualqna-ui:latest --build-arg https_proxy=$htt Then run the command `docker images`, you will have the following 5 Docker Images: -1. `ghcr.io/huggingface/tgi-gaudi:2.0.6` -2. `opea/lvm:latest` -3. `opea/visualqna:latest` -4. `opea/visualqna-ui:latest` -5. `opea/nginx` +1. `opea/vllm-gaudi:latest` +2. `ghcr.io/huggingface/tgi-gaudi:2.0.6` (Optional) +3. `opea/lvm:latest` +4. `opea/visualqna:latest` +5. `opea/visualqna-ui:latest` +6. `opea/nginx` ## 🚀 Start MicroServices and MegaService @@ -56,18 +71,10 @@ Then run the command `docker images`, you will have the following 5 Docker Image Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below. ```bash -export no_proxy=${your_no_proxy} -export http_proxy=${your_http_proxy} -export https_proxy=${your_http_proxy} -export LVM_MODEL_ID="llava-hf/llava-v1.6-mistral-7b-hf" -export LVM_ENDPOINT="http://${host_ip}:8399" -export LVM_SERVICE_PORT=9399 -export MEGA_SERVICE_HOST_IP=${host_ip} -export LVM_SERVICE_HOST_IP=${host_ip} -export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/visualqna" +source set_env.sh ``` -Note: Please replace with `host_ip` with you external IP address, do **NOT** use localhost. +Note: Please replace with `host_ip` with you external IP address, do not use localhost. ### Start all the services Docker Containers @@ -77,6 +84,8 @@ cd GenAIExamples/VisualQnA/docker_compose/intel/hpu/gaudi/ ```bash docker compose -f compose.yaml up -d +# if use TGI as the LLM serving backend +docker compose -f compose_tgi.yaml up -d ``` > **_NOTE:_** Users need at least one Gaudi cards to run the VisualQnA successfully. diff --git a/VisualQnA/docker_compose/intel/hpu/gaudi/compose.yaml b/VisualQnA/docker_compose/intel/hpu/gaudi/compose.yaml index bd4004e399..c1950a14d4 100644 --- a/VisualQnA/docker_compose/intel/hpu/gaudi/compose.yaml +++ b/VisualQnA/docker_compose/intel/hpu/gaudi/compose.yaml @@ -2,41 +2,42 @@ # SPDX-License-Identifier: Apache-2.0 services: - llava-tgi-service: - image: ghcr.io/huggingface/tgi-gaudi:2.3.1 - container_name: tgi-llava-gaudi-server + vllm-gaudi-service: + image: ${REGISTRY:-opea}/vllm-gaudi:${TAG:-latest} + container_name: vllm-gaudi-service ports: - - "8399:80" + - ${VLLM_PORT:-8399}:80 volumes: - - "${MODEL_CACHE:-./data}:/data" + - "./data:/root/.cache/huggingface/hub" environment: no_proxy: ${no_proxy} http_proxy: ${http_proxy} https_proxy: ${https_proxy} - HF_HUB_DISABLE_PROGRESS_BARS: 1 - HF_HUB_ENABLE_HF_TRANSFER: 0 + HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} HABANA_VISIBLE_DEVICES: all OMPI_MCA_btl_vader_single_copy_mechanism: none - HUGGING_FACE_HUB_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} - ENABLE_HPU_GRAPH: true - LIMIT_HPU_GRAPH: true - USE_FLASH_ATTENTION: true - FLASH_ATTENTION_RECOMPUTE: true - healthcheck: - test: ["CMD-SHELL", "curl -f http://$host_ip:8399/health || exit 1"] - interval: 10s - timeout: 10s - retries: 60 + LLM_MODEL_ID: ${LVM_MODEL_ID} + VLLM_TORCH_PROFILER_DIR: "/mnt" + VLLM_SKIP_WARMUP: ${VLLM_SKIP_WARMUP:-false} + MAX_MODEL_LEN: ${MAX_TOTAL_TOKENS:-4096} + MAX_SEQ_LEN_TO_CAPTURE: ${MAX_TOTAL_TOKENS:-4096} + PT_HPUGRAPH_DISABLE_TENSOR_CACHE: false # https://github.com/HabanaAI/vllm-fork/issues/841#issuecomment-2700421704 runtime: habana cap_add: - SYS_NICE ipc: host - command: --model-id ${LVM_MODEL_ID} --max-input-length 4096 --max-total-tokens 8192 + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:80/health || exit 1"] + interval: 10s + timeout: 10s + retries: 150 + command: --model $LVM_MODEL_ID --tensor-parallel-size 1 --host 0.0.0.0 --port 80 --chat-template examples/template_llava.jinja # https://docs.vllm.ai/en/v0.5.0/models/vlm.html lvm: image: ${REGISTRY:-opea}/lvm:${TAG:-latest} - container_name: lvm-gaudi-server + container_name: lvm-vllm-gaudi-service depends_on: - - llava-tgi-service + vllm-gaudi-service: + condition: service_healthy ports: - "9399:9399" ipc: host @@ -45,7 +46,8 @@ services: http_proxy: ${http_proxy} https_proxy: ${https_proxy} LVM_ENDPOINT: ${LVM_ENDPOINT} - LVM_COMPONENT_NAME: "OPEA_TGI_LLAVA_LVM" + LVM_COMPONENT_NAME: "OPEA_VLLM_LVM" + LLM_MODEL_ID: ${LVM_MODEL_ID} HF_HUB_DISABLE_PROGRESS_BARS: 1 HF_HUB_ENABLE_HF_TRANSFER: 0 restart: unless-stopped @@ -53,7 +55,7 @@ services: image: ${REGISTRY:-opea}/visualqna:${TAG:-latest} container_name: visualqna-gaudi-backend-server depends_on: - - llava-tgi-service + - vllm-gaudi-service - lvm ports: - "8888:8888" diff --git a/VisualQnA/docker_compose/intel/hpu/gaudi/compose_tgi.yaml b/VisualQnA/docker_compose/intel/hpu/gaudi/compose_tgi.yaml new file mode 100644 index 0000000000..251b4fce70 --- /dev/null +++ b/VisualQnA/docker_compose/intel/hpu/gaudi/compose_tgi.yaml @@ -0,0 +1,105 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +services: + llava-tgi-service: + image: ghcr.io/huggingface/tgi-gaudi:2.3.1 + container_name: tgi-llava-gaudi-server + ports: + - "8399:80" + volumes: + - "${MODEL_CACHE:-./data}:/data" + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + HABANA_VISIBLE_DEVICES: all + OMPI_MCA_btl_vader_single_copy_mechanism: none + HUGGING_FACE_HUB_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + ENABLE_HPU_GRAPH: true + LIMIT_HPU_GRAPH: true + USE_FLASH_ATTENTION: true + FLASH_ATTENTION_RECOMPUTE: true + healthcheck: + test: ["CMD-SHELL", "curl -f http://$host_ip:8399/health || exit 1"] + interval: 10s + timeout: 10s + retries: 60 + runtime: habana + cap_add: + - SYS_NICE + ipc: host + command: --model-id ${LVM_MODEL_ID} --max-input-length 4096 --max-total-tokens 8192 + lvm: + image: ${REGISTRY:-opea}/lvm:${TAG:-latest} + container_name: lvm-gaudi-server + depends_on: + llava-tgi-service: + condition: service_healthy + ports: + - "9399:9399" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + LVM_ENDPOINT: ${LVM_ENDPOINT} + LVM_COMPONENT_NAME: "OPEA_TGI_LLAVA_LVM" + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + restart: unless-stopped + visualqna-gaudi-backend-server: + image: ${REGISTRY:-opea}/visualqna:${TAG:-latest} + container_name: visualqna-gaudi-backend-server + depends_on: + - llava-tgi-service + - lvm + ports: + - "8888:8888" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP} + - LVM_SERVICE_HOST_IP=${LVM_SERVICE_HOST_IP} + ipc: host + restart: always + visualqna-gaudi-ui-server: + image: ${REGISTRY:-opea}/visualqna-ui:${TAG:-latest} + container_name: visualqna-gaudi-ui-server + depends_on: + - visualqna-gaudi-backend-server + ports: + - "5173:5173" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - BACKEND_BASE_URL=${BACKEND_SERVICE_ENDPOINT} + ipc: host + restart: always + visualqna-gaudi-nginx-server: + image: ${REGISTRY:-opea}/nginx:${TAG:-latest} + container_name: visualqna-gaudi-nginx-server + depends_on: + - visualqna-gaudi-backend-server + - visualqna-gaudi-ui-server + ports: + - "${NGINX_PORT:-80}:80" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - FRONTEND_SERVICE_IP=${FRONTEND_SERVICE_IP} + - FRONTEND_SERVICE_PORT=${FRONTEND_SERVICE_PORT} + - BACKEND_SERVICE_NAME=${BACKEND_SERVICE_NAME} + - BACKEND_SERVICE_IP=${BACKEND_SERVICE_IP} + - BACKEND_SERVICE_PORT=${BACKEND_SERVICE_PORT} + ipc: host + restart: always + +networks: + default: + driver: bridge diff --git a/VisualQnA/docker_compose/intel/hpu/gaudi/set_env.sh b/VisualQnA/docker_compose/intel/hpu/gaudi/set_env.sh index 028966b77c..57032fdce5 100644 --- a/VisualQnA/docker_compose/intel/hpu/gaudi/set_env.sh +++ b/VisualQnA/docker_compose/intel/hpu/gaudi/set_env.sh @@ -6,7 +6,10 @@ pushd "../../../../../" > /dev/null source .set_env.sh popd > /dev/null -export LVM_MODEL_ID="llava-hf/llava-v1.6-mistral-7b-hf" +export host_ip=$(hostname -I | awk '{print $1}') +export no_proxy=$host_ip,$no_proxy +# export LVM_MODEL_ID="llava-hf/llava-v1.6-mistral-7b-hf" +export LVM_MODEL_ID="llava-hf/llava-1.5-7b-hf" export LVM_ENDPOINT="http://${host_ip}:8399" export LVM_SERVICE_PORT=9399 export MEGA_SERVICE_HOST_IP=${host_ip} diff --git a/VisualQnA/tests/test_compose_on_gaudi.sh b/VisualQnA/tests/test_compose_on_gaudi.sh index 3515be94e4..3fbc8e0adc 100644 --- a/VisualQnA/tests/test_compose_on_gaudi.sh +++ b/VisualQnA/tests/test_compose_on_gaudi.sh @@ -10,51 +10,32 @@ echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} export MODEL_CACHE=${model_cache:-"./data"} +export NGINX_PORT=81 +export VLLM_SKIP_WARMUP=true WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" ip_address=$(hostname -I | awk '{print $1}') function build_docker_images() { - opea_branch=${opea_branch:-"main"} - # If the opea_branch isn't main, replace the git clone branch in Dockerfile. - if [[ "${opea_branch}" != "main" ]]; then - cd $WORKPATH - OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" - NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" - find . -type f -name "Dockerfile*" | while read -r file; do - echo "Processing file: $file" - sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" - done - fi - cd $WORKPATH/docker_image_build - git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git - - echo "Build all the images with --no-cache, check docker_image_build.log for details..." + git clone --depth 1 --branch main https://github.com/opea-project/GenAIComps.git docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log - docker pull ghcr.io/huggingface/tgi-gaudi:2.0.6 + git clone https://github.com/HabanaAI/vllm-fork.git + cd ./vllm-fork/ + docker build -f Dockerfile.hpu -t opea/vllm-gaudi:${TAG} --shm-size=128g . --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy + cd .. + rm -rf vllm-fork + docker images && sleep 1s } function start_services() { cd $WORKPATH/docker_compose/intel/hpu/gaudi - export LVM_MODEL_ID="llava-hf/llava-v1.6-mistral-7b-hf" - export LVM_ENDPOINT="http://${ip_address}:8399" - export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} - export LVM_SERVICE_PORT=9399 - export MEGA_SERVICE_HOST_IP=${ip_address} - export LVM_SERVICE_HOST_IP=${ip_address} - export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/visualqna" - export FRONTEND_SERVICE_IP=${ip_address} - export FRONTEND_SERVICE_PORT=5173 - export BACKEND_SERVICE_NAME=visualqna - export BACKEND_SERVICE_IP=${ip_address} - export BACKEND_SERVICE_PORT=8888 - export NGINX_PORT=80 - export host_ip=${ip_address} + source ./set_env.sh + sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env @@ -63,8 +44,8 @@ function start_services() { n=0 until [[ "$n" -ge 100 ]]; do - docker logs lvm-gaudi-server > ${LOG_PATH}/lvm_tgi_service_start.log - if grep -q Connected ${LOG_PATH}/lvm_tgi_service_start.log; then + docker logs vllm-gaudi-service > ${LOG_PATH}/lvm_vllm_service_start.log + if grep -q Starting ${LOG_PATH}/lvm_vllm_service_start.log; then break fi sleep 5s @@ -101,22 +82,24 @@ function validate_services() { } function validate_microservices() { + sleep 15s # Check if the microservices are running correctly. # lvm microservice validate_services \ "${ip_address}:9399/v1/lvm" \ - "The image" \ + "yellow" \ "lvm" \ - "lvm-gaudi-server" \ + "lvm-vllm-gaudi-service" \ '{"image": "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8/5+hnoEIwDiqkL4KAcT9GO0U4BxoAAAAAElFTkSuQmCC", "prompt":"What is this?"}' } function validate_megaservice() { + sleep 15s # Curl the Mega Service validate_services \ "${ip_address}:8888/v1/visualqna" \ - "The image" \ + "sign" \ "visualqna-gaudi-backend-server" \ "visualqna-gaudi-backend-server" \ '{ @@ -142,8 +125,8 @@ function validate_megaservice() { # test the megeservice via nginx validate_services \ - "${ip_address}:80/v1/visualqna" \ - "The image" \ + "${ip_address}:${NGINX_PORT}/v1/visualqna" \ + "sign" \ "visualqna-gaudi-nginx-server" \ "visualqna-gaudi-nginx-server" \ '{ diff --git a/VisualQnA/tests/test_compose_on_xeon.sh b/VisualQnA/tests/test_compose_on_xeon.sh index 4e345b3f91..0e645c324b 100644 --- a/VisualQnA/tests/test_compose_on_xeon.sh +++ b/VisualQnA/tests/test_compose_on_xeon.sh @@ -10,51 +10,26 @@ echo "TAG=IMAGE_TAG=${IMAGE_TAG}" export REGISTRY=${IMAGE_REPO} export TAG=${IMAGE_TAG} export MODEL_CACHE=${model_cache:-"./data"} +export NGINX_PORT=81 WORKPATH=$(dirname "$PWD") LOG_PATH="$WORKPATH/tests" ip_address=$(hostname -I | awk '{print $1}') function build_docker_images() { - opea_branch=${opea_branch:-"main"} - # If the opea_branch isn't main, replace the git clone branch in Dockerfile. - if [[ "${opea_branch}" != "main" ]]; then - cd $WORKPATH - OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" - NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" - find . -type f -name "Dockerfile*" | while read -r file; do - echo "Processing file: $file" - sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" - done - fi - cd $WORKPATH/docker_image_build - git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git - - echo "Build all the images with --no-cache, check docker_image_build.log for details..." + git clone --depth 1 --branch main https://github.com/opea-project/GenAIComps.git docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log - docker pull ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu + docker pull opea/vllm:latest + docker tag opea/vllm:latest opea/vllm:${TAG} docker images && sleep 1s } function start_services() { cd $WORKPATH/docker_compose/intel/cpu/xeon/ - export LVM_MODEL_ID="llava-hf/llava-v1.6-mistral-7b-hf" - export LVM_ENDPOINT="http://${ip_address}:8399" - export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} - export LVM_SERVICE_PORT=9399 - export MEGA_SERVICE_HOST_IP=${ip_address} - export LVM_SERVICE_HOST_IP=${ip_address} - export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/visualqna" - export FRONTEND_SERVICE_IP=${ip_address} - export FRONTEND_SERVICE_PORT=5173 - export BACKEND_SERVICE_NAME=visualqna - export BACKEND_SERVICE_IP=${ip_address} - export BACKEND_SERVICE_PORT=8888 - export NGINX_PORT=80 - export host_ip=${ip_address} + source ./set_env.sh sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env @@ -63,8 +38,8 @@ function start_services() { n=0 until [[ "$n" -ge 200 ]]; do - docker logs lvm-xeon-server > ${LOG_PATH}/lvm_tgi_service_start.log - if grep -q Connected ${LOG_PATH}/lvm_tgi_service_start.log; then + docker logs vllm-service > ${LOG_PATH}/lvm_vllm_service_start.log + if grep -q Starting ${LOG_PATH}/lvm_vllm_service_start.log; then break fi sleep 5s @@ -101,12 +76,13 @@ function validate_services() { } function validate_microservices() { + sleep 15s # Check if the microservices are running correctly. # lvm microservice validate_services \ "${ip_address}:9399/v1/lvm" \ - "The image" \ + "yellow" \ "lvm" \ "lvm-xeon-server" \ '{"image": "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8/5+hnoEIwDiqkL4KAcT9GO0U4BxoAAAAAElFTkSuQmCC", "prompt":"What is this?"}' @@ -116,7 +92,7 @@ function validate_megaservice() { # Curl the Mega Service validate_services \ "${ip_address}:8888/v1/visualqna" \ - "The image" \ + "sign" \ "visualqna-xeon-backend-server" \ "visualqna-xeon-backend-server" \ '{ @@ -142,8 +118,8 @@ function validate_megaservice() { # test the megeservice via nginx validate_services \ - "${ip_address}:80/v1/visualqna" \ - "The image" \ + "${ip_address}:${NGINX_PORT}/v1/visualqna" \ + "sign" \ "visualqna-xeon-nginx-server" \ "visualqna-xeon-nginx-server" \ '{ diff --git a/VisualQnA/tests/test_compose_tgi_on_gaudi.sh b/VisualQnA/tests/test_compose_tgi_on_gaudi.sh new file mode 100644 index 0000000000..913d6ed527 --- /dev/null +++ b/VisualQnA/tests/test_compose_tgi_on_gaudi.sh @@ -0,0 +1,222 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -x +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" +export REGISTRY=${IMAGE_REPO} +export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') + +function build_docker_images() { + opea_branch=${opea_branch:-"main"} + # If the opea_branch isn't main, replace the git clone branch in Dockerfile. + if [[ "${opea_branch}" != "main" ]]; then + cd $WORKPATH + OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" + NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" + find . -type f -name "Dockerfile*" | while read -r file; do + echo "Processing file: $file" + sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" + done + fi + + cd $WORKPATH/docker_image_build + git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + + echo "Build all the images with --no-cache, check docker_image_build.log for details..." + docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log + + docker pull ghcr.io/huggingface/tgi-gaudi:2.0.6 + docker images && sleep 1s +} + +function start_services() { + cd $WORKPATH/docker_compose/intel/hpu/gaudi + + export LVM_MODEL_ID="llava-hf/llava-v1.6-mistral-7b-hf" + export LVM_ENDPOINT="http://${ip_address}:8399" + export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} + export LVM_SERVICE_PORT=9399 + export MEGA_SERVICE_HOST_IP=${ip_address} + export LVM_SERVICE_HOST_IP=${ip_address} + export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/visualqna" + export FRONTEND_SERVICE_IP=${ip_address} + export FRONTEND_SERVICE_PORT=5173 + export BACKEND_SERVICE_NAME=visualqna + export BACKEND_SERVICE_IP=${ip_address} + export BACKEND_SERVICE_PORT=8888 + export NGINX_PORT=80 + export host_ip=${ip_address} + + sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env + + # Start Docker Containers + docker compose -f compose_tgi.yaml up -d > ${LOG_PATH}/start_services_with_compose.log + + n=0 + until [[ "$n" -ge 100 ]]; do + docker logs tgi-llava-gaudi-server > ${LOG_PATH}/lvm_tgi_service_start.log + if grep -q Connected ${LOG_PATH}/lvm_tgi_service_start.log; then + break + fi + sleep 5s + n=$((n+1)) + done +} + +function validate_services() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +function validate_microservices() { + sleep 15s + # Check if the microservices are running correctly. + + # lvm microservice + validate_services \ + "${ip_address}:9399/v1/lvm" \ + "The image" \ + "lvm" \ + "lvm-gaudi-server" \ + '{"image": "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8/5+hnoEIwDiqkL4KAcT9GO0U4BxoAAAAAElFTkSuQmCC", "prompt":"What is this?"}' +} + +function validate_megaservice() { + sleep 15s + # Curl the Mega Service + validate_services \ + "${ip_address}:8888/v1/visualqna" \ + "The image" \ + "visualqna-gaudi-backend-server" \ + "visualqna-gaudi-backend-server" \ + '{ + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What'\''s in this image?" + }, + { + "type": "image_url", + "image_url": { + "url": "https://www.ilankelman.org/stopsigns/australia.jpg" + } + } + ] + } + ], + "max_tokens": 300 + }' + + # test the megeservice via nginx + validate_services \ + "${ip_address}:80/v1/visualqna" \ + "The image" \ + "visualqna-gaudi-nginx-server" \ + "visualqna-gaudi-nginx-server" \ + '{ + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What'\''s in this image?" + }, + { + "type": "image_url", + "image_url": { + "url": "https://www.ilankelman.org/stopsigns/australia.jpg" + } + } + ] + } + ], + "max_tokens": 300 + }' +} + +function validate_frontend() { + cd $WORKPATH/ui/svelte + local conda_env_name="OPEA_e2e" + export PATH=${HOME}/miniforge3/bin/:$PATH + if conda info --envs | grep -q "$conda_env_name"; then + echo "$conda_env_name exist!" + else + conda create -n ${conda_env_name} python=3.12 -y + fi + source activate ${conda_env_name} + + sed -i "s/localhost/$ip_address/g" playwright.config.ts + + conda install -c conda-forge nodejs=22.6.0 -y + npm install && npm ci && npx playwright install --with-deps + node -v && npm -v && pip list + + exit_status=0 + npx playwright test || exit_status=$? + + if [ $exit_status -ne 0 ]; then + echo "[TEST INFO]: ---------frontend test failed---------" + exit $exit_status + else + echo "[TEST INFO]: ---------frontend test passed---------" + fi +} + +function stop_docker() { + cd $WORKPATH/docker_compose/intel/hpu/gaudi + docker compose stop && docker compose rm -f +} + +function main() { + + stop_docker + + if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi + start_services + + validate_microservices + validate_megaservice + # validate_frontend + + stop_docker + echo y | docker system prune + +} + +main diff --git a/VisualQnA/tests/test_compose_tgi_on_xeon.sh b/VisualQnA/tests/test_compose_tgi_on_xeon.sh new file mode 100644 index 0000000000..d6311719d0 --- /dev/null +++ b/VisualQnA/tests/test_compose_tgi_on_xeon.sh @@ -0,0 +1,222 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -x +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" +export REGISTRY=${IMAGE_REPO} +export TAG=${IMAGE_TAG} +export MODEL_CACHE=${model_cache:-"./data"} + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') + +function build_docker_images() { + opea_branch=${opea_branch:-"main"} + # If the opea_branch isn't main, replace the git clone branch in Dockerfile. + if [[ "${opea_branch}" != "main" ]]; then + cd $WORKPATH + OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" + NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" + find . -type f -name "Dockerfile*" | while read -r file; do + echo "Processing file: $file" + sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" + done + fi + + cd $WORKPATH/docker_image_build + git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + + echo "Build all the images with --no-cache, check docker_image_build.log for details..." + docker compose -f build.yaml build --no-cache > ${LOG_PATH}/docker_image_build.log + + docker pull ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu + docker images && sleep 1s +} + +function start_services() { + cd $WORKPATH/docker_compose/intel/cpu/xeon/ + + export LVM_MODEL_ID="llava-hf/llava-v1.6-mistral-7b-hf" + export LVM_ENDPOINT="http://${ip_address}:8399" + export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} + export LVM_SERVICE_PORT=9399 + export MEGA_SERVICE_HOST_IP=${ip_address} + export LVM_SERVICE_HOST_IP=${ip_address} + export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/visualqna" + export FRONTEND_SERVICE_IP=${ip_address} + export FRONTEND_SERVICE_PORT=5173 + export BACKEND_SERVICE_NAME=visualqna + export BACKEND_SERVICE_IP=${ip_address} + export BACKEND_SERVICE_PORT=8888 + export NGINX_PORT=80 + export host_ip=${ip_address} + + sed -i "s/backend_address/$ip_address/g" $WORKPATH/ui/svelte/.env + + # Start Docker Containers + docker compose -f compose_tgi.yaml up -d > ${LOG_PATH}/start_services_with_compose.log + + n=0 + until [[ "$n" -ge 200 ]]; do + docker logs tgi-llava-xeon-server > ${LOG_PATH}/lvm_tgi_service_start.log + if grep -q Connected ${LOG_PATH}/lvm_tgi_service_start.log; then + break + fi + sleep 5s + n=$((n+1)) + done +} + +function validate_services() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +function validate_microservices() { + sleep 15s + # Check if the microservices are running correctly. + + # lvm microservice + validate_services \ + "${ip_address}:9399/v1/lvm" \ + "The image" \ + "lvm" \ + "lvm-xeon-server" \ + '{"image": "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8/5+hnoEIwDiqkL4KAcT9GO0U4BxoAAAAAElFTkSuQmCC", "prompt":"What is this?"}' +} + +function validate_megaservice() { + sleep 15s + # Curl the Mega Service + validate_services \ + "${ip_address}:8888/v1/visualqna" \ + "The image" \ + "visualqna-xeon-backend-server" \ + "visualqna-xeon-backend-server" \ + '{ + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What'\''s in this image?" + }, + { + "type": "image_url", + "image_url": { + "url": "https://www.ilankelman.org/stopsigns/australia.jpg" + } + } + ] + } + ], + "max_tokens": 300 + }' + + # test the megeservice via nginx + validate_services \ + "${ip_address}:80/v1/visualqna" \ + "The image" \ + "visualqna-xeon-nginx-server" \ + "visualqna-xeon-nginx-server" \ + '{ + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What'\''s in this image?" + }, + { + "type": "image_url", + "image_url": { + "url": "https://www.ilankelman.org/stopsigns/australia.jpg" + } + } + ] + } + ], + "max_tokens": 300 + }' +} + +function validate_frontend() { + cd $WORKPATH/ui/svelte + local conda_env_name="OPEA_e2e" + export PATH=${HOME}/miniforge3/bin/:$PATH + if conda info --envs | grep -q "$conda_env_name"; then + echo "$conda_env_name exist!" + else + conda create -n ${conda_env_name} python=3.12 -y + fi + source activate ${conda_env_name} + + sed -i "s/localhost/$ip_address/g" playwright.config.ts + + conda install -c conda-forge nodejs=22.6.0 -y + npm install && npm ci && npx playwright install --with-deps + node -v && npm -v && pip list + + exit_status=0 + npx playwright test || exit_status=$? + + if [ $exit_status -ne 0 ]; then + echo "[TEST INFO]: ---------frontend test failed---------" + exit $exit_status + else + echo "[TEST INFO]: ---------frontend test passed---------" + fi +} + +function stop_docker() { + cd $WORKPATH/docker_compose/intel/cpu/xeon/ + docker compose stop && docker compose rm -f +} + +function main() { + + stop_docker + + if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi + start_services + + validate_microservices + validate_megaservice + # validate_frontend + + stop_docker + echo y | docker system prune + +} + +main From 0516abd7dbc22bb6796ea16b61118355c5d36ac7 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 19 Mar 2025 09:21:27 +0800 Subject: [PATCH 070/226] Fix workflow issues. (#1691) Signed-off-by: ZePan110 Signed-off-by: Chingis Yundunov --- .github/workflows/_run-docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_run-docker-compose.yml b/.github/workflows/_run-docker-compose.yml index a84912ed36..eb5219b89c 100644 --- a/.github/workflows/_run-docker-compose.yml +++ b/.github/workflows/_run-docker-compose.yml @@ -104,7 +104,7 @@ jobs: compose-test: needs: [get-test-case] - if: ${{ needs.get-test-case.outputs.test_cases != '' }} + if: ${{ needs.get-test-case.outputs.test_cases != '[""]' }} strategy: matrix: test_case: ${{ fromJSON(needs.get-test-case.outputs.test_cases) }} @@ -165,7 +165,7 @@ jobs: export model_cache="~/.cache/huggingface/hub" fi fi - if [ -f ${test_case} ]; then timeout 30m bash ${test_case}; else echo "Test script {${test_case}} not found, skip test!"; fi + if [ -f "${test_case}" ]; then timeout 30m bash "${test_case}"; else echo "Test script {${test_case}} not found, skip test!"; fi - name: Clean up container after test shell: bash From d8191aff2dbec9cb51955b0538b0074eb0cadcd3 Mon Sep 17 00:00:00 2001 From: "chen, suyue" Date: Wed, 19 Mar 2025 09:21:51 +0800 Subject: [PATCH 071/226] Enable base image build in CI/CD (#1669) Signed-off-by: chensuyue Signed-off-by: Chingis Yundunov --- .github/workflows/_build_comps_base_image.yml | 65 +++++++++++++++++++ .github/workflows/manual-example-workflow.yml | 22 ++----- .../nightly-docker-build-publish.yml | 6 ++ .github/workflows/weekly-update-images.yml | 4 +- DocSum/Dockerfile | 3 +- DocSum/docker_image_build/build.yaml | 2 + DocSum/tests/test_compose_on_gaudi.sh | 14 +--- DocSum/tests/test_compose_on_rocm.sh | 14 +--- DocSum/tests/test_compose_on_xeon.sh | 14 +--- 9 files changed, 89 insertions(+), 55 deletions(-) create mode 100644 .github/workflows/_build_comps_base_image.yml diff --git a/.github/workflows/_build_comps_base_image.yml b/.github/workflows/_build_comps_base_image.yml new file mode 100644 index 0000000000..e83f6bd14c --- /dev/null +++ b/.github/workflows/_build_comps_base_image.yml @@ -0,0 +1,65 @@ +# Copyright (C) 2025 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +name: Build Comps Base Image +permissions: read-all +on: + workflow_call: + inputs: + node: + required: true + type: string + build: + default: true + required: false + type: boolean + tag: + default: "latest" + required: false + type: string + opea_branch: + default: "main" + required: false + type: string + inject_commit: + default: false + required: false + type: boolean + +jobs: + pre-build-image-check: + runs-on: ubuntu-latest + outputs: + should_skip: ${{ steps.check-skip.outputs.should_skip }} + steps: + - name: Check if job should be skipped + id: check-skip + run: | + should_skip=false + if [[ "${{ inputs.node }}" == "gaudi3" || "${{ inputs.node }}" == "rocm" || "${{ inputs.node }}" == "arc" ]]; then + should_skip=true + fi + echo "should_skip=$should_skip" + echo "should_skip=$should_skip" >> $GITHUB_OUTPUT + + build-images: + needs: [ pre-build-image-check ] + if: ${{ needs.pre-build-image-check.outputs.should_skip == 'false' && fromJSON(inputs.build) }} + runs-on: "docker-build-${{ inputs.node }}" + steps: + - name: Clean Up Working Directory + run: sudo rm -rf ${{github.workspace}}/* + + - name: Clone Required Repo + run: | + git clone --depth 1 --branch ${{ inputs.opea_branch }} https://github.com/opea-project/GenAIComps.git + cd GenAIComps && git rev-parse HEAD && cd ../ && ls -l + + - name: Build Image + uses: opea-project/validation/actions/image-build@main + with: + work_dir: ${{ github.workspace }}/GenAIComps + docker_compose_path: ${{ github.workspace }}/GenAIComps/.github/workflows/docker/compose/base-compose.yaml + registry: ${OPEA_IMAGE_REPO}opea + inject_commit: ${{ inputs.inject_commit }} + tag: ${{ inputs.tag }} diff --git a/.github/workflows/manual-example-workflow.yml b/.github/workflows/manual-example-workflow.yml index 7159b8a8a0..6f0c9d1dc2 100644 --- a/.github/workflows/manual-example-workflow.yml +++ b/.github/workflows/manual-example-workflow.yml @@ -20,11 +20,6 @@ on: description: "Tag to apply to images" required: true type: string - # deploy_gmc: - # default: false - # description: 'Whether to deploy gmc' - # required: true - # type: boolean build: default: true description: 'Build test required images for Examples' @@ -40,11 +35,6 @@ on: description: 'Test examples with helm charts' required: false type: boolean - # test_gmc: - # default: false - # description: 'Test examples with gmc' - # required: false - # type: boolean opea_branch: default: "main" description: 'OPEA branch for image build' @@ -79,23 +69,20 @@ jobs: nodes_json=$(printf '%s\n' "${nodes[@]}" | sort -u | jq -R '.' | jq -sc '.') echo "nodes=$nodes_json" >> $GITHUB_OUTPUT - build-deploy-gmc: + build-comps-base: needs: [get-test-matrix] - if: false - #${{ fromJSON(inputs.deploy_gmc) }} strategy: matrix: node: ${{ fromJson(needs.get-test-matrix.outputs.nodes) }} - fail-fast: false - uses: ./.github/workflows/_gmc-workflow.yml + uses: ./.github/workflows/_build_comps_base_image.yml with: node: ${{ matrix.node }} + build: ${{ fromJSON(inputs.build) }} tag: ${{ inputs.tag }} opea_branch: ${{ inputs.opea_branch }} - secrets: inherit run-examples: - needs: [get-test-matrix] #[get-test-matrix, build-deploy-gmc] + needs: [get-test-matrix, build-comps-base] strategy: matrix: example: ${{ fromJson(needs.get-test-matrix.outputs.examples) }} @@ -109,7 +96,6 @@ jobs: build: ${{ fromJSON(inputs.build) }} test_compose: ${{ fromJSON(inputs.test_compose) }} test_helmchart: ${{ fromJSON(inputs.test_helmchart) }} - # test_gmc: ${{ fromJSON(inputs.test_gmc) }} opea_branch: ${{ inputs.opea_branch }} inject_commit: ${{ inputs.inject_commit }} use_model_cache: ${{ inputs.use_model_cache }} diff --git a/.github/workflows/nightly-docker-build-publish.yml b/.github/workflows/nightly-docker-build-publish.yml index 3a05fae1df..c584dcd571 100644 --- a/.github/workflows/nightly-docker-build-publish.yml +++ b/.github/workflows/nightly-docker-build-publish.yml @@ -32,6 +32,12 @@ jobs: echo "TAG=$TAG" >> $GITHUB_OUTPUT echo "PUBLISH_TAGS=$PUBLISH_TAGS" >> $GITHUB_OUTPUT + build-comps-base: + needs: [get-build-matrix] + uses: ./.github/workflows/_build_comps_base_image.yml + with: + node: gaudi + build-and-test: needs: get-build-matrix if: ${{ needs.get-build-matrix.outputs.examples_json != '' }} diff --git a/.github/workflows/weekly-update-images.yml b/.github/workflows/weekly-update-images.yml index 0c970874a3..487fa4f609 100644 --- a/.github/workflows/weekly-update-images.yml +++ b/.github/workflows/weekly-update-images.yml @@ -1,11 +1,9 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -name: Weekly update base images and 3rd party images +name: Weekly update 3rd party images on: - schedule: - - cron: "0 0 * * 0" workflow_dispatch: permissions: diff --git a/DocSum/Dockerfile b/DocSum/Dockerfile index 2cc8c3d5a5..c7d9a76184 100644 --- a/DocSum/Dockerfile +++ b/DocSum/Dockerfile @@ -1,8 +1,9 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +ARG IMAGE_REPO=opea ARG BASE_TAG=latest -FROM opea/comps-base:$BASE_TAG +FROM $IMAGE_REPO/comps-base:$BASE_TAG USER root # FFmpeg needed for media processing diff --git a/DocSum/docker_image_build/build.yaml b/DocSum/docker_image_build/build.yaml index 095fd28c93..7eabbfc6aa 100644 --- a/DocSum/docker_image_build/build.yaml +++ b/DocSum/docker_image_build/build.yaml @@ -5,6 +5,8 @@ services: docsum: build: args: + IMAGE_REPO: ${REGISTRY} + BASE_TAG: ${TAG} http_proxy: ${http_proxy} https_proxy: ${https_proxy} no_proxy: ${no_proxy} diff --git a/DocSum/tests/test_compose_on_gaudi.sh b/DocSum/tests/test_compose_on_gaudi.sh index 66dd5b3180..1c8632b76d 100644 --- a/DocSum/tests/test_compose_on_gaudi.sh +++ b/DocSum/tests/test_compose_on_gaudi.sh @@ -39,19 +39,11 @@ ROOT_FOLDER=$(dirname "$(readlink -f "$0")") function build_docker_images() { opea_branch=${opea_branch:-"main"} - # If the opea_branch isn't main, replace the git clone branch in Dockerfile. - if [[ "${opea_branch}" != "main" ]]; then - cd $WORKPATH - OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" - NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" - find . -type f -name "Dockerfile*" | while read -r file; do - echo "Processing file: $file" - sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" - done - fi - cd $WORKPATH/docker_image_build git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + pushd GenAIComps + docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . + popd && sleep 1s echo "Build all the images with --no-cache, check docker_image_build.log for details..." service_list="docsum docsum-gradio-ui whisper llm-docsum" diff --git a/DocSum/tests/test_compose_on_rocm.sh b/DocSum/tests/test_compose_on_rocm.sh index 0226acb43d..cbb7fc90bb 100644 --- a/DocSum/tests/test_compose_on_rocm.sh +++ b/DocSum/tests/test_compose_on_rocm.sh @@ -35,19 +35,11 @@ export LOGFLAG=True function build_docker_images() { opea_branch=${opea_branch:-"main"} - # If the opea_branch isn't main, replace the git clone branch in Dockerfile. - if [[ "${opea_branch}" != "main" ]]; then - cd $WORKPATH - OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" - NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" - find . -type f -name "Dockerfile*" | while read -r file; do - echo "Processing file: $file" - sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" - done - fi - cd $WORKPATH/docker_image_build git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + pushd GenAIComps + docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . + popd && sleep 1s echo "Build all the images with --no-cache, check docker_image_build.log for details..." service_list="docsum docsum-gradio-ui whisper llm-docsum" diff --git a/DocSum/tests/test_compose_on_xeon.sh b/DocSum/tests/test_compose_on_xeon.sh index 7dc194ff68..c813da75d4 100644 --- a/DocSum/tests/test_compose_on_xeon.sh +++ b/DocSum/tests/test_compose_on_xeon.sh @@ -38,19 +38,11 @@ ROOT_FOLDER=$(dirname "$(readlink -f "$0")") function build_docker_images() { opea_branch=${opea_branch:-"main"} - # If the opea_branch isn't main, replace the git clone branch in Dockerfile. - if [[ "${opea_branch}" != "main" ]]; then - cd $WORKPATH - OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git" - NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git" - find . -type f -name "Dockerfile*" | while read -r file; do - echo "Processing file: $file" - sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file" - done - fi - cd $WORKPATH/docker_image_build git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git + pushd GenAIComps + docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . + popd && sleep 1s echo "Build all the images with --no-cache, check docker_image_build.log for details..." service_list="docsum docsum-gradio-ui whisper llm-docsum" From c1a8cdc733faa01f38700186d7512bbe00d52f1f Mon Sep 17 00:00:00 2001 From: minmin-intel Date: Wed, 19 Mar 2025 18:57:18 -0700 Subject: [PATCH 072/226] fix errors for running AgentQnA on xeon with openai and update readme (#1664) Signed-off-by: minmin-intel Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: Chingis Yundunov --- AgentQnA/README.md | 1 + .../docker_compose/intel/cpu/xeon/README.md | 8 +- .../intel/cpu/xeon/compose_openai.yaml | 5 ++ .../docker_compose/intel/hpu/gaudi/README.md | 2 + .../tests/_test_compose_openai_on_xeon.sh | 17 +++-- .../step4_launch_and_validate_agent_openai.sh | 73 +++++++++++++++++-- AgentQnA/ui/svelte/README.md | 18 ++++- 7 files changed, 104 insertions(+), 20 deletions(-) diff --git a/AgentQnA/README.md b/AgentQnA/README.md index 397bd0c775..fc2309f0ae 100644 --- a/AgentQnA/README.md +++ b/AgentQnA/README.md @@ -192,6 +192,7 @@ docker build -t opea/agent:latest --build-arg https_proxy=$https_proxy --build-a ```bash vllm_port=8086 model="meta-llama/Meta-Llama-3.1-70B-Instruct" + vllm_volume=$HF_CACHE_DIR # you should have set this env var in previous step docker run -d --runtime=habana --rm --name "vllm-gaudi-server" -e HABANA_VISIBLE_DEVICES=0,1,2,3 -p $vllm_port:8000 -v $vllm_volume:/data -e HF_TOKEN=$HF_TOKEN -e HUGGING_FACE_HUB_TOKEN=$HF_TOKEN -e HF_HOME=/data -e OMPI_MCA_btl_vader_single_copy_mechanism=none -e PT_HPU_ENABLE_LAZY_COLLECTIVES=true -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy -e VLLM_SKIP_WARMUP=true --cap-add=sys_nice --ipc=host opea/vllm-gaudi:latest --model ${model} --max-seq-len-to-capture 16384 --tensor-parallel-size 4 ``` diff --git a/AgentQnA/docker_compose/intel/cpu/xeon/README.md b/AgentQnA/docker_compose/intel/cpu/xeon/README.md index a2abfc7ce9..b5de3a81cb 100644 --- a/AgentQnA/docker_compose/intel/cpu/xeon/README.md +++ b/AgentQnA/docker_compose/intel/cpu/xeon/README.md @@ -42,15 +42,13 @@ This example showcases a hierarchical multi-agent system for question-answering ``` 4. Prepare SQL database - In this example, we will use the SQLite database provided in the [TAG-Bench](https://github.com/TAG-Research/TAG-Bench/tree/main). Run the commands below. + In this example, we will use the Chinook SQLite database. Run the commands below. ``` # Download data cd $WORKDIR - git clone https://github.com/TAG-Research/TAG-Bench.git - cd TAG-Bench/setup - chmod +x get_dbs.sh - ./get_dbs.sh + git clone https://github.com/lerocha/chinook-database.git + cp chinook-database/ChinookDatabase/DataSources/Chinook_Sqlite.sqlite $WORKDIR/GenAIExamples/AgentQnA/tests/ ``` 5. Launch Tool service diff --git a/AgentQnA/docker_compose/intel/cpu/xeon/compose_openai.yaml b/AgentQnA/docker_compose/intel/cpu/xeon/compose_openai.yaml index bbd64ceb30..c0aaaae606 100644 --- a/AgentQnA/docker_compose/intel/cpu/xeon/compose_openai.yaml +++ b/AgentQnA/docker_compose/intel/cpu/xeon/compose_openai.yaml @@ -92,4 +92,9 @@ services: LANGCHAIN_PROJECT: "opea-supervisor-agent-service" CRAG_SERVER: $CRAG_SERVER WORKER_AGENT_URL: $WORKER_AGENT_URL + SQL_AGENT_URL: $SQL_AGENT_URL port: 9090 + +networks: + default: + driver: bridge diff --git a/AgentQnA/docker_compose/intel/hpu/gaudi/README.md b/AgentQnA/docker_compose/intel/hpu/gaudi/README.md index b920dff804..ffd34204fd 100644 --- a/AgentQnA/docker_compose/intel/hpu/gaudi/README.md +++ b/AgentQnA/docker_compose/intel/hpu/gaudi/README.md @@ -23,6 +23,7 @@ For more details, please refer to the deployment guide [here](../../../../README export no_proxy="Your_No_Proxy" export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/ + # for using open-source llms export HUGGINGFACEHUB_API_TOKEN= # Example export HF_CACHE_DIR=$WORKDIR so that no need to redownload every time @@ -78,6 +79,7 @@ For more details, please refer to the deployment guide [here](../../../../README ```bash vllm_port=8086 + vllm_volume=$HF_CACHE_DIR # you should have set this env var in previous step model="meta-llama/Meta-Llama-3.1-70B-Instruct" docker run -d --runtime=habana --rm --name "vllm-gaudi-server" -e HABANA_VISIBLE_DEVICES=0,1,2,3 -p $vllm_port:8000 -v $vllm_volume:/data -e HF_TOKEN=$HF_TOKEN -e HUGGING_FACE_HUB_TOKEN=$HF_TOKEN -e HF_HOME=/data -e OMPI_MCA_btl_vader_single_copy_mechanism=none -e PT_HPU_ENABLE_LAZY_COLLECTIVES=true -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy -e VLLM_SKIP_WARMUP=true --cap-add=sys_nice --ipc=host opea/vllm-gaudi:latest --model ${model} --max-seq-len-to-capture 16384 --tensor-parallel-size 4 ``` diff --git a/AgentQnA/tests/_test_compose_openai_on_xeon.sh b/AgentQnA/tests/_test_compose_openai_on_xeon.sh index 5c8ef8913b..daf5c48b12 100644 --- a/AgentQnA/tests/_test_compose_openai_on_xeon.sh +++ b/AgentQnA/tests/_test_compose_openai_on_xeon.sh @@ -20,23 +20,30 @@ function stop_agent_and_api_server() { function stop_retrieval_tool() { echo "Stopping Retrieval tool" - docker compose -f $WORKDIR/GenAIExamples/AgentQnA/retrieval_tool/docker/docker-compose-retrieval-tool.yaml down + local RETRIEVAL_TOOL_PATH=$WORKPATH/../DocIndexRetriever + cd $RETRIEVAL_TOOL_PATH/docker_compose/intel/cpu/xeon/ + container_list=$(cat compose.yaml | grep container_name | cut -d':' -f2) + for container_name in $container_list; do + cid=$(docker ps -aq --filter "name=$container_name") + echo "Stopping container $container_name" + if [[ ! -z "$cid" ]]; then docker rm $cid -f && sleep 1s; fi + done } echo "=================== #1 Building docker images====================" -bash 1_build_images.sh +bash step1_build_images.sh echo "=================== #1 Building docker images completed====================" echo "=================== #2 Start retrieval tool====================" -bash 2_start_retrieval_tool.sh +bash step2_start_retrieval_tool.sh echo "=================== #2 Retrieval tool started====================" echo "=================== #3 Ingest data and validate retrieval====================" -bash 3_ingest_data_and_validate_retrieval.sh +bash step3_ingest_data_and_validate_retrieval.sh echo "=================== #3 Data ingestion and validation completed====================" echo "=================== #4 Start agent and API server====================" -bash 4_launch_and_validate_agent_openai.sh +bash step4_launch_and_validate_agent_openai.sh echo "=================== #4 Agent test passed ====================" echo "=================== #5 Stop agent and API server====================" diff --git a/AgentQnA/tests/step4_launch_and_validate_agent_openai.sh b/AgentQnA/tests/step4_launch_and_validate_agent_openai.sh index b3220c09de..1fd0e8954d 100644 --- a/AgentQnA/tests/step4_launch_and_validate_agent_openai.sh +++ b/AgentQnA/tests/step4_launch_and_validate_agent_openai.sh @@ -11,13 +11,22 @@ echo "WORKDIR=${WORKDIR}" export ip_address=$(hostname -I | awk '{print $1}') export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/ + +function download_chinook_data(){ + echo "Downloading chinook data..." + cd $WORKDIR + git clone https://github.com/lerocha/chinook-database.git + cp chinook-database/ChinookDatabase/DataSources/Chinook_Sqlite.sqlite $WORKDIR/GenAIExamples/AgentQnA/tests/ +} + function start_agent_and_api_server() { echo "Starting CRAG server" docker run -d --runtime=runc --name=kdd-cup-24-crag-service -p=8080:8000 docker.io/aicrowd/kdd-cup-24-crag-mock-api:v0 echo "Starting Agent services" - cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/cpu/xeon + cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/intel/cpu/xeon/ bash launch_agent_service_openai.sh + sleep 2m } function validate() { @@ -35,19 +44,64 @@ function validate() { } function validate_agent_service() { - echo "----------------Test agent ----------------" - local CONTENT=$(http_proxy="" curl http://${ip_address}:9090/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{ - "query": "Tell me about Michael Jackson song thriller" - }') - local EXIT_CODE=$(validate "$CONTENT" "Thriller" "react-agent-endpoint") - docker logs react-agent-endpoint + # # test worker rag agent + echo "======================Testing worker rag agent======================" + export agent_port="9095" + prompt="Tell me about Michael Jackson song Thriller" + local CONTENT=$(python3 $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --prompt "$prompt" --agent_role "worker" --ext_port $agent_port) + # echo $CONTENT + local EXIT_CODE=$(validate "$CONTENT" "Thriller" "rag-agent-endpoint") + echo $EXIT_CODE + local EXIT_CODE="${EXIT_CODE:0-1}" + if [ "$EXIT_CODE" == "1" ]; then + docker logs rag-agent-endpoint + exit 1 + fi + + # # test worker sql agent + echo "======================Testing worker sql agent======================" + export agent_port="9096" + prompt="How many employees are there in the company?" + local CONTENT=$(python3 $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --prompt "$prompt" --agent_role "worker" --ext_port $agent_port) + local EXIT_CODE=$(validate "$CONTENT" "8" "sql-agent-endpoint") + echo $CONTENT + # echo $EXIT_CODE + local EXIT_CODE="${EXIT_CODE:0-1}" if [ "$EXIT_CODE" == "1" ]; then + docker logs sql-agent-endpoint + exit 1 + fi + + # test supervisor react agent + echo "======================Testing supervisor react agent======================" + export agent_port="9090" + local CONTENT=$(python3 $WORKDIR/GenAIExamples/AgentQnA/tests/test.py --agent_role "supervisor" --ext_port $agent_port --stream) + local EXIT_CODE=$(validate "$CONTENT" "Iron" "react-agent-endpoint") + # echo $CONTENT + echo $EXIT_CODE + local EXIT_CODE="${EXIT_CODE:0-1}" + if [ "$EXIT_CODE" == "1" ]; then + docker logs react-agent-endpoint exit 1 fi } +function remove_chinook_data(){ + echo "Removing chinook data..." + cd $WORKDIR + if [ -d "chinook-database" ]; then + rm -rf chinook-database + fi + echo "Chinook data removed!" +} + + function main() { + echo "==================== Prepare data ====================" + download_chinook_data + echo "==================== Data prepare done ====================" + echo "==================== Start agent ====================" start_agent_and_api_server echo "==================== Agent started ====================" @@ -57,4 +111,9 @@ function main() { echo "==================== Agent service validated ====================" } + +remove_chinook_data + main + +remove_chinook_data diff --git a/AgentQnA/ui/svelte/README.md b/AgentQnA/ui/svelte/README.md index a31aabd1f2..12087b5fa7 100644 --- a/AgentQnA/ui/svelte/README.md +++ b/AgentQnA/ui/svelte/README.md @@ -21,10 +21,22 @@ Here're some of the project's features: cd AgentQnA/ui/svelte ``` -3. Modify the required .env variables. +3. Modify the required .env variables. The `AGENT_URL` should be in the form of the following: ``` - AGENT_URL = '' + AGENT_URL = "http://${ip_address}:${agent_port}/v1/chat/completions" + ``` + + For example: assume that the ip address of the host machine is 10.10.10.1, and the agent port is 9090,then + + ``` + AGENT_URL = "http://10.10.10.1:9090/v1/chat/completions" + ``` + + You can get the ip address of the host machine by running the command below: + + ```bash + export ip_address=$(hostname -I | awk '{print $1}') ``` 4. **For Local Development:** @@ -57,4 +69,4 @@ Here're some of the project's features: docker run -d -p 5173:5173 --name agent-ui opea:agent-ui ``` -- The application will be available at `http://localhost:5173`. +- The application will be available at `http://${ip_address}:5173`. You can access it with a web browser on your laptop. Note the `ip_address` should be the ip address of the host machine where the UI container runs. From bbfe10a83d48946216c003535893e01517e53212 Mon Sep 17 00:00:00 2001 From: Zhu Yongbo Date: Thu, 20 Mar 2025 10:46:01 +0800 Subject: [PATCH 073/226] Add new UI/new features for EC-RAG (#1665) Signed-off-by: Zhu, Yongbo Signed-off-by: Chingis Yundunov --- .github/code_spell_ignore.txt | 1 + EdgeCraftRAG/Dockerfile | 2 +- EdgeCraftRAG/Dockerfile.server | 10 +- EdgeCraftRAG/README.md | 46 ++- EdgeCraftRAG/assets/img/chat_with_rag.png | Bin 94641 -> 27304 bytes EdgeCraftRAG/assets/img/create_pipeline.png | Bin 171708 -> 86636 bytes EdgeCraftRAG/assets/img/upload_data.png | Bin 87811 -> 46701 bytes .../docker_compose/intel/gpu/arc/compose.yaml | 4 +- .../intel/gpu/arc/compose_gradio.yaml | 94 ++++++ .../intel/gpu/arc/compose_vllm.yaml | 4 +- EdgeCraftRAG/docker_image_build/build.yaml | 8 + EdgeCraftRAG/edgecraftrag/api/v1/chatqna.py | 82 +++-- EdgeCraftRAG/edgecraftrag/api/v1/data.py | 95 +++--- EdgeCraftRAG/edgecraftrag/api/v1/model.py | 69 +++- EdgeCraftRAG/edgecraftrag/api/v1/pipeline.py | 111 +++++-- EdgeCraftRAG/edgecraftrag/api/v1/system.py | 71 ++++ EdgeCraftRAG/edgecraftrag/api_schema.py | 2 +- EdgeCraftRAG/edgecraftrag/base.py | 1 - .../edgecraftrag/components/benchmark.py | 83 +++-- .../edgecraftrag/components/generator.py | 146 ++++++-- .../edgecraftrag/components/indexer.py | 3 +- EdgeCraftRAG/edgecraftrag/components/model.py | 3 +- .../edgecraftrag/components/node_parser.py | 2 +- .../edgecraftrag/components/pipeline.py | 131 +++++--- .../edgecraftrag/components/postprocessor.py | 4 +- .../edgecraftrag/components/retriever.py | 20 ++ .../edgecraftrag/controllers/modelmgr.py | 35 ++ .../edgecraftrag/controllers/nodemgr.py | 2 +- .../edgecraftrag/controllers/pipelinemgr.py | 33 +- EdgeCraftRAG/edgecraftrag/server.py | 15 +- EdgeCraftRAG/edgecraftrag/utils.py | 4 +- EdgeCraftRAG/tests/test_compose_on_arc.sh | 4 +- .../tests/test_compose_vllm_on_arc.sh | 4 +- EdgeCraftRAG/ui/docker/Dockerfile.gradio | 25 ++ EdgeCraftRAG/ui/docker/Dockerfile.ui | 40 ++- EdgeCraftRAG/ui/gradio/ecragui.py | 85 +---- EdgeCraftRAG/ui/vue/.env.development | 6 + EdgeCraftRAG/ui/vue/.env.production | 6 + EdgeCraftRAG/ui/vue/.gitignore | 24 ++ EdgeCraftRAG/ui/vue/README.md | 0 EdgeCraftRAG/ui/vue/auto-imports.d.ts | 106 ++++++ EdgeCraftRAG/ui/vue/components.d.ts | 52 +++ EdgeCraftRAG/ui/vue/index.html | 20 ++ EdgeCraftRAG/ui/vue/nginx.conf | 56 ++++ EdgeCraftRAG/ui/vue/package.json | 51 +++ EdgeCraftRAG/ui/vue/public/favicon.ico | Bin 0 -> 1059 bytes EdgeCraftRAG/ui/vue/public/favicon1.ico | Bin 0 -> 4286 bytes EdgeCraftRAG/ui/vue/src/App.vue | 28 ++ EdgeCraftRAG/ui/vue/src/api/chatbot/index.ts | 52 +++ EdgeCraftRAG/ui/vue/src/api/pipeline/index.ts | 92 +++++ EdgeCraftRAG/ui/vue/src/api/request.ts | 62 ++++ .../ui/vue/src/assets/iconFont/iconfont.css | 99 ++++++ .../ui/vue/src/assets/iconFont/iconfont.js | 68 ++++ .../ui/vue/src/assets/iconFont/iconfont.json | 156 +++++++++ .../ui/vue/src/assets/iconFont/iconfont.ttf | Bin 0 -> 6480 bytes .../ui/vue/src/assets/iconFont/iconfont.woff | Bin 0 -> 4332 bytes .../ui/vue/src/assets/iconFont/iconfont.woff2 | Bin 0 -> 3596 bytes .../ui/vue/src/assets/images/404-bg.png | Bin 0 -> 190281 bytes .../ui/vue/src/assets/images/noData.png | Bin 0 -> 118178 bytes .../ui/vue/src/assets/svgs/404-icon.svg | 1 + .../ui/vue/src/assets/svgs/dark-icon.svg | 1 + .../ui/vue/src/assets/svgs/header-log.svg | 1 + .../ui/vue/src/assets/svgs/light-icon.svg | 1 + .../ui/vue/src/assets/svgs/lightBulb.svg | 1 + .../ui/vue/src/assets/svgs/uploaded.svg | 1 + EdgeCraftRAG/ui/vue/src/auto-imports.d.ts | 106 ++++++ EdgeCraftRAG/ui/vue/src/components.d.ts | 20 ++ .../ui/vue/src/components/FormTooltip.vue | 26 ++ .../ui/vue/src/components/SvgIcon.vue | 56 ++++ EdgeCraftRAG/ui/vue/src/enums/antEnum.ts | 6 + EdgeCraftRAG/ui/vue/src/i18n/en.ts | 26 ++ EdgeCraftRAG/ui/vue/src/i18n/index.ts | 22 ++ EdgeCraftRAG/ui/vue/src/i18n/zh.ts | 26 ++ EdgeCraftRAG/ui/vue/src/layout/Header.vue | 84 +++++ EdgeCraftRAG/ui/vue/src/layout/Main.vue | 15 + EdgeCraftRAG/ui/vue/src/main.ts | 43 +++ EdgeCraftRAG/ui/vue/src/router/index.ts | 12 + EdgeCraftRAG/ui/vue/src/router/routes.ts | 38 +++ EdgeCraftRAG/ui/vue/src/store/chatbot.ts | 31 ++ EdgeCraftRAG/ui/vue/src/store/index.ts | 10 + EdgeCraftRAG/ui/vue/src/store/pipeline.ts | 19 ++ EdgeCraftRAG/ui/vue/src/store/theme.ts | 19 ++ EdgeCraftRAG/ui/vue/src/store/user.ts | 19 ++ EdgeCraftRAG/ui/vue/src/theme/ant.less | 156 +++++++++ EdgeCraftRAG/ui/vue/src/theme/common.less | 179 ++++++++++ EdgeCraftRAG/ui/vue/src/theme/index.less | 6 + EdgeCraftRAG/ui/vue/src/theme/layout.less | 104 ++++++ EdgeCraftRAG/ui/vue/src/theme/loading.less | 54 +++ EdgeCraftRAG/ui/vue/src/theme/markdown.less | 72 ++++ EdgeCraftRAG/ui/vue/src/theme/variables.less | 77 +++++ EdgeCraftRAG/ui/vue/src/types/axios.d.ts | 21 ++ EdgeCraftRAG/ui/vue/src/types/global.d.ts | 133 ++++++++ EdgeCraftRAG/ui/vue/src/utils/antTheme.ts | 62 ++++ EdgeCraftRAG/ui/vue/src/utils/common.ts | 21 ++ EdgeCraftRAG/ui/vue/src/utils/loading.ts | 46 +++ EdgeCraftRAG/ui/vue/src/utils/notification.ts | 40 +++ EdgeCraftRAG/ui/vue/src/utils/other.ts | 24 ++ .../ui/vue/src/utils/serviceManager.ts | 30 ++ EdgeCraftRAG/ui/vue/src/utils/storage.ts | 64 ++++ .../src/views/chatbot/components/Chatbot.vue | 275 +++++++++++++++ .../views/chatbot/components/ConfigDrawer.vue | 234 +++++++++++++ .../src/views/chatbot/components/Header.vue | 118 +++++++ .../views/chatbot/components/MessageItem.vue | 217 ++++++++++++ .../views/chatbot/components/SseService.ts | 47 +++ .../views/chatbot/components/UploadFile.vue | 276 +++++++++++++++ .../vue/src/views/chatbot/components/index.ts | 10 + .../ui/vue/src/views/chatbot/index.vue | 108 ++++++ EdgeCraftRAG/ui/vue/src/views/chatbot/type.ts | 22 ++ EdgeCraftRAG/ui/vue/src/views/error/404.vue | 43 +++ .../pipeline/components/Configuration.vue | 54 +++ .../pipeline/components/DetailDrawer.vue | 283 ++++++++++++++++ .../pipeline/components/ImportDialog.vue | 74 +++++ .../views/pipeline/components/QuickStart.vue | 153 +++++++++ .../src/views/pipeline/components/System.vue | 118 +++++++ .../views/pipeline/components/SystemChart.vue | 314 ++++++++++++++++++ .../src/views/pipeline/components/Table.vue | 283 ++++++++++++++++ .../components/UpdateDialog/Activated.vue | 64 ++++ .../components/UpdateDialog/Basic.vue | 81 +++++ .../components/UpdateDialog/CreateDialog.vue | 225 +++++++++++++ .../components/UpdateDialog/EditDialog.vue | 239 +++++++++++++ .../components/UpdateDialog/Generator.vue | 271 +++++++++++++++ .../components/UpdateDialog/Indexer.vue | 186 +++++++++++ .../components/UpdateDialog/NodeParser.vue | 238 +++++++++++++ .../components/UpdateDialog/PostProcessor.vue | 267 +++++++++++++++ .../components/UpdateDialog/Retriever.vue | 120 +++++++ .../pipeline/components/UpdateDialog/index.ts | 12 + .../src/views/pipeline/components/index.ts | 14 + .../ui/vue/src/views/pipeline/enum.ts | 76 +++++ .../ui/vue/src/views/pipeline/index.vue | 174 ++++++++++ .../ui/vue/src/views/pipeline/type.ts | 22 ++ EdgeCraftRAG/ui/vue/src/vite-env.d.ts | 4 + EdgeCraftRAG/ui/vue/tsconfig.app.json | 28 ++ EdgeCraftRAG/ui/vue/tsconfig.json | 4 + EdgeCraftRAG/ui/vue/tsconfig.node.json | 22 ++ EdgeCraftRAG/ui/vue/vite.config.ts | 76 +++++ 135 files changed, 8206 insertions(+), 336 deletions(-) mode change 100644 => 100755 EdgeCraftRAG/README.md mode change 100644 => 100755 EdgeCraftRAG/assets/img/chat_with_rag.png mode change 100644 => 100755 EdgeCraftRAG/assets/img/create_pipeline.png mode change 100644 => 100755 EdgeCraftRAG/assets/img/upload_data.png create mode 100644 EdgeCraftRAG/docker_compose/intel/gpu/arc/compose_gradio.yaml mode change 100644 => 100755 EdgeCraftRAG/edgecraftrag/api/v1/data.py create mode 100644 EdgeCraftRAG/edgecraftrag/api/v1/system.py create mode 100644 EdgeCraftRAG/ui/docker/Dockerfile.gradio create mode 100644 EdgeCraftRAG/ui/vue/.env.development create mode 100644 EdgeCraftRAG/ui/vue/.env.production create mode 100644 EdgeCraftRAG/ui/vue/.gitignore create mode 100644 EdgeCraftRAG/ui/vue/README.md create mode 100644 EdgeCraftRAG/ui/vue/auto-imports.d.ts create mode 100644 EdgeCraftRAG/ui/vue/components.d.ts create mode 100644 EdgeCraftRAG/ui/vue/index.html create mode 100644 EdgeCraftRAG/ui/vue/nginx.conf create mode 100644 EdgeCraftRAG/ui/vue/package.json create mode 100644 EdgeCraftRAG/ui/vue/public/favicon.ico create mode 100644 EdgeCraftRAG/ui/vue/public/favicon1.ico create mode 100644 EdgeCraftRAG/ui/vue/src/App.vue create mode 100644 EdgeCraftRAG/ui/vue/src/api/chatbot/index.ts create mode 100644 EdgeCraftRAG/ui/vue/src/api/pipeline/index.ts create mode 100644 EdgeCraftRAG/ui/vue/src/api/request.ts create mode 100644 EdgeCraftRAG/ui/vue/src/assets/iconFont/iconfont.css create mode 100644 EdgeCraftRAG/ui/vue/src/assets/iconFont/iconfont.js create mode 100644 EdgeCraftRAG/ui/vue/src/assets/iconFont/iconfont.json create mode 100644 EdgeCraftRAG/ui/vue/src/assets/iconFont/iconfont.ttf create mode 100644 EdgeCraftRAG/ui/vue/src/assets/iconFont/iconfont.woff create mode 100644 EdgeCraftRAG/ui/vue/src/assets/iconFont/iconfont.woff2 create mode 100644 EdgeCraftRAG/ui/vue/src/assets/images/404-bg.png create mode 100644 EdgeCraftRAG/ui/vue/src/assets/images/noData.png create mode 100644 EdgeCraftRAG/ui/vue/src/assets/svgs/404-icon.svg create mode 100644 EdgeCraftRAG/ui/vue/src/assets/svgs/dark-icon.svg create mode 100644 EdgeCraftRAG/ui/vue/src/assets/svgs/header-log.svg create mode 100644 EdgeCraftRAG/ui/vue/src/assets/svgs/light-icon.svg create mode 100644 EdgeCraftRAG/ui/vue/src/assets/svgs/lightBulb.svg create mode 100644 EdgeCraftRAG/ui/vue/src/assets/svgs/uploaded.svg create mode 100644 EdgeCraftRAG/ui/vue/src/auto-imports.d.ts create mode 100644 EdgeCraftRAG/ui/vue/src/components.d.ts create mode 100644 EdgeCraftRAG/ui/vue/src/components/FormTooltip.vue create mode 100644 EdgeCraftRAG/ui/vue/src/components/SvgIcon.vue create mode 100644 EdgeCraftRAG/ui/vue/src/enums/antEnum.ts create mode 100644 EdgeCraftRAG/ui/vue/src/i18n/en.ts create mode 100644 EdgeCraftRAG/ui/vue/src/i18n/index.ts create mode 100644 EdgeCraftRAG/ui/vue/src/i18n/zh.ts create mode 100644 EdgeCraftRAG/ui/vue/src/layout/Header.vue create mode 100644 EdgeCraftRAG/ui/vue/src/layout/Main.vue create mode 100644 EdgeCraftRAG/ui/vue/src/main.ts create mode 100644 EdgeCraftRAG/ui/vue/src/router/index.ts create mode 100644 EdgeCraftRAG/ui/vue/src/router/routes.ts create mode 100644 EdgeCraftRAG/ui/vue/src/store/chatbot.ts create mode 100644 EdgeCraftRAG/ui/vue/src/store/index.ts create mode 100644 EdgeCraftRAG/ui/vue/src/store/pipeline.ts create mode 100644 EdgeCraftRAG/ui/vue/src/store/theme.ts create mode 100644 EdgeCraftRAG/ui/vue/src/store/user.ts create mode 100644 EdgeCraftRAG/ui/vue/src/theme/ant.less create mode 100644 EdgeCraftRAG/ui/vue/src/theme/common.less create mode 100644 EdgeCraftRAG/ui/vue/src/theme/index.less create mode 100644 EdgeCraftRAG/ui/vue/src/theme/layout.less create mode 100644 EdgeCraftRAG/ui/vue/src/theme/loading.less create mode 100644 EdgeCraftRAG/ui/vue/src/theme/markdown.less create mode 100644 EdgeCraftRAG/ui/vue/src/theme/variables.less create mode 100644 EdgeCraftRAG/ui/vue/src/types/axios.d.ts create mode 100644 EdgeCraftRAG/ui/vue/src/types/global.d.ts create mode 100644 EdgeCraftRAG/ui/vue/src/utils/antTheme.ts create mode 100644 EdgeCraftRAG/ui/vue/src/utils/common.ts create mode 100644 EdgeCraftRAG/ui/vue/src/utils/loading.ts create mode 100644 EdgeCraftRAG/ui/vue/src/utils/notification.ts create mode 100644 EdgeCraftRAG/ui/vue/src/utils/other.ts create mode 100644 EdgeCraftRAG/ui/vue/src/utils/serviceManager.ts create mode 100644 EdgeCraftRAG/ui/vue/src/utils/storage.ts create mode 100644 EdgeCraftRAG/ui/vue/src/views/chatbot/components/Chatbot.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/chatbot/components/ConfigDrawer.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/chatbot/components/Header.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/chatbot/components/MessageItem.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/chatbot/components/SseService.ts create mode 100644 EdgeCraftRAG/ui/vue/src/views/chatbot/components/UploadFile.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/chatbot/components/index.ts create mode 100644 EdgeCraftRAG/ui/vue/src/views/chatbot/index.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/chatbot/type.ts create mode 100644 EdgeCraftRAG/ui/vue/src/views/error/404.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/pipeline/components/Configuration.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/pipeline/components/DetailDrawer.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/pipeline/components/ImportDialog.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/pipeline/components/QuickStart.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/pipeline/components/System.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/pipeline/components/SystemChart.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/pipeline/components/Table.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/pipeline/components/UpdateDialog/Activated.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/pipeline/components/UpdateDialog/Basic.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/pipeline/components/UpdateDialog/CreateDialog.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/pipeline/components/UpdateDialog/EditDialog.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/pipeline/components/UpdateDialog/Generator.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/pipeline/components/UpdateDialog/Indexer.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/pipeline/components/UpdateDialog/NodeParser.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/pipeline/components/UpdateDialog/PostProcessor.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/pipeline/components/UpdateDialog/Retriever.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/pipeline/components/UpdateDialog/index.ts create mode 100644 EdgeCraftRAG/ui/vue/src/views/pipeline/components/index.ts create mode 100644 EdgeCraftRAG/ui/vue/src/views/pipeline/enum.ts create mode 100644 EdgeCraftRAG/ui/vue/src/views/pipeline/index.vue create mode 100644 EdgeCraftRAG/ui/vue/src/views/pipeline/type.ts create mode 100644 EdgeCraftRAG/ui/vue/src/vite-env.d.ts create mode 100644 EdgeCraftRAG/ui/vue/tsconfig.app.json create mode 100644 EdgeCraftRAG/ui/vue/tsconfig.json create mode 100644 EdgeCraftRAG/ui/vue/tsconfig.node.json create mode 100644 EdgeCraftRAG/ui/vue/vite.config.ts diff --git a/.github/code_spell_ignore.txt b/.github/code_spell_ignore.txt index 4566d4f3a2..3c59d07a31 100644 --- a/.github/code_spell_ignore.txt +++ b/.github/code_spell_ignore.txt @@ -1,2 +1,3 @@ ModelIn modelin +pressEnter \ No newline at end of file diff --git a/EdgeCraftRAG/Dockerfile b/EdgeCraftRAG/Dockerfile index fffb8d8970..c7100d1600 100644 --- a/EdgeCraftRAG/Dockerfile +++ b/EdgeCraftRAG/Dockerfile @@ -6,4 +6,4 @@ FROM opea/comps-base:$BASE_TAG COPY ./chatqna.py $HOME/chatqna.py -ENTRYPOINT ["python", "chatqna.py"] +ENTRYPOINT ["python", "chatqna.py"] \ No newline at end of file diff --git a/EdgeCraftRAG/Dockerfile.server b/EdgeCraftRAG/Dockerfile.server index ab4060de80..6a701f72dd 100644 --- a/EdgeCraftRAG/Dockerfile.server +++ b/EdgeCraftRAG/Dockerfile.server @@ -10,7 +10,7 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missin poppler-utils \ tesseract-ocr -RUN apt-get update && apt-get install -y gnupg wget +RUN apt-get update && apt-get install -y gnupg wget git RUN wget -qO - https://repositories.intel.com/gpu/intel-graphics.key | \ gpg --yes --dearmor --output /usr/share/keyrings/intel-graphics.gpg RUN echo "deb [arch=amd64,i386 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/gpu/ubuntu jammy client" | \ @@ -33,15 +33,17 @@ RUN chown -R user /templates/default_prompt.txt COPY ./edgecraftrag /home/user/edgecraftrag -RUN mkdir -p /home/user/gradio_cache -ENV GRADIO_TEMP_DIR=/home/user/gradio_cache +RUN mkdir -p /home/user/ui_cache +ENV UI_UPLOAD_PATH=/home/user/ui_cache WORKDIR /home/user/edgecraftrag RUN pip install --no-cache-dir --upgrade pip setuptools==70.0.0 && \ pip install --no-cache-dir -r requirements.txt WORKDIR /home/user/ +RUN git clone https://github.com/openvinotoolkit/openvino.genai.git genai +ENV PYTHONPATH="$PYTHONPATH:/home/user/genai/tools/llm_bench" USER user -ENTRYPOINT ["python", "-m", "edgecraftrag.server"] +ENTRYPOINT ["python", "-m", "edgecraftrag.server"] \ No newline at end of file diff --git a/EdgeCraftRAG/README.md b/EdgeCraftRAG/README.md old mode 100644 new mode 100755 index e2479c07a8..34975d3747 --- a/EdgeCraftRAG/README.md +++ b/EdgeCraftRAG/README.md @@ -7,10 +7,10 @@ quality and performance. ## What's New in this release? -- Support image/url data retrieval and display in EC-RAG -- Support display of document source used by LLM in UI -- Support pipeline remove operation in RESTful API and UI -- Support RAG pipeline performance benchmark and display in UI +- A sleek new UI with enhanced user experience, built on Vue and Ant Design +- Support concurrent multi-requests handling on vLLM inference backend +- Support pipeline configuration through json file +- Support system prompt modification through API - Fixed known issues in EC-RAG UI and server ## Quick Start Guide @@ -36,7 +36,7 @@ You can select "local" type in generation field which is the default approach to #### vLLM with OpenVINO for Intel Arc GPU You can also select "vLLM" as generation type, to enable this type, you'll need to build the vLLM image for Intel Arc GPU before service bootstrap. -Please follow this link [vLLM with OpenVINO](https://github.com/opea-project/GenAIComps/tree/main/comps/llms/text-generation/vllm/langchain#build-docker-image) to build the vLLM image. +Please follow this link [vLLM with OpenVINO](https://github.com/opea-project/GenAIComps/tree/main/comps/third_parties/vllm#23-vllm-with-openvino-on-intel-gpu-and-cpu) to build the vLLM image. ### Start Edge Craft RAG Services with Docker Compose @@ -45,12 +45,12 @@ cd GenAIExamples/EdgeCraftRAG/docker_compose/intel/gpu/arc export MODEL_PATH="your model path for all your models" export DOC_PATH="your doc path for uploading a dir of files" -export GRADIO_PATH="your gradio cache path for transferring files" +export UI_TMPFILE_PATH="your UI cache path for transferring files" # If you have a specific prompt template, please uncomment the following line # export PROMPT_PATH="your prompt path for prompt templates" # Make sure all 3 folders have 1000:1000 permission, otherwise -# chown 1000:1000 ${MODEL_PATH} ${DOC_PATH} ${GRADIO_PATH} +# chown 1000:1000 ${MODEL_PATH} ${DOC_PATH} ${UI_TMPFILE_PATH} # In addition, also make sure the .cache folder has 1000:1000 permission, otherwise # chown 1000:1000 $HOME/.cache @@ -189,6 +189,12 @@ After the pipeline creation, you can upload your data in the `Chatbot` page. Then, you can submit messages in the chat box. ![chat_with_rag](assets/img/chat_with_rag.png) +If you want to try Gradio UI, please launch service through compose_gradio.yaml, then access http://${HOST_IP}:8082 on your browser: + +```bash +docker compose -f compose_gradio.yaml up -d +``` + ## Advanced User Guide ### Pipeline Management @@ -226,8 +232,26 @@ curl -X PATCH http://${HOST_IP}:16010/v1/settings/pipelines/rag_test_local_llm - curl -X DELETE http://${HOST_IP}:16010/v1/settings/pipelines/rag_test_local_llm -H "Content-Type: application/json" | jq '.' ``` +#### Get pipeline json + +```bash +curl -X GET http://${HOST_IP}:16010/v1/settings/pipelines/{name}/json -H "Content-Type: application/json" | jq '.' +``` + +#### Import pipeline from a json file + +```bash +curl -X POST http://${HOST_IP}:16010/v1/settings/pipelines/import -H "Content-Type: multipart/form-data" -F "file=@your_test_pipeline_json_file.txt"| jq '.' +``` + #### Enable and check benchmark for pipelines +##### ⚠️ NOTICE ⚠️ + +Benchmarking activities may significantly reduce system performance. + +**DO NOT** perform benchmarking in a production environment. + ```bash # Set ENABLE_BENCHMARK as true before launch services export ENABLE_BENCHMARK="true" @@ -308,3 +332,11 @@ curl -X DELETE http://${HOST_IP}:16010/v1/data/files/test2.docx -H "Content-Type ```bash curl -X PATCH http://${HOST_IP}:16010/v1/data/files/test.pdf -H "Content-Type: application/json" -d '{"local_path":"docs/#REPLACE WITH YOUR FILE WITHIN MOUNTED DOC PATH#"}' | jq '.' ``` + +### System Prompt Management + +#### Use custom system prompt + +```bash +curl -X POST http://${HOST_IP}:16010/v1/chatqna/prompt -H "Content-Type: multipart/form-data" -F "file=@your_prompt_file.txt" +``` diff --git a/EdgeCraftRAG/assets/img/chat_with_rag.png b/EdgeCraftRAG/assets/img/chat_with_rag.png old mode 100644 new mode 100755 index 04000ef374a9ef1eafab99fac34a1b51c87e0872..16262fdeac03f8b4374be9cb3a2953f130ba245b GIT binary patch literal 27304 zcmeFZc{G*n`!~ExrBXDAP*lbuLu8%`AtDKx%ba?Q5Ucc^<>(_#B_}@_8&LdG<8rX%q@| zR$5B@2?|B%fkGW!Ie8S`AyShsfqza}N~zkQP-OJTe@CRB(EWus&)Z5ovsJV(uyuH0 zt&dVPe`)!`##UdGyloUdBu73}veCD`CT3w~rf+VGQq;B9*EhE@varS2os~qPuA!vG z?>%*lUK~Zg|FHRDd({wQzEa#RP)mF5o)p`|>yJ1l5<6~QJfGRqdD802a98Kk2aj9= z8sn82V|}`wuNZXIa*!pTN@D2yok(YF#h=VoFIxUG=PZMm9dGCR`wF62!N*C8%PO$@ zt35dL<%TipqPSrKt?a5|L(Gm~+^8FI14h_2R$N|Ip zPoJ;6hbK3I82DV{$Dz6|MN@tA;S6S?`~{V?r>;T+B?a`g@uK=xkkM^CEf(QX7r(r zFjFv{;x`oL5x4!`DCt2R^fvQ*iBRb~E4t7|TRXeW&CLMoD;|mu_AqAh^70ZA5~`}I zX{m@UF;X0H9+_K44%WRLt1IwQF8N8?)A=_+QH^?mG$QWRWdzaHySuyaS_{5)55D#N z-7^B`D|1UrLiURT@8)f?;M=+lfiVMJ6mD*Anwpx7_8K1DZ_uadg)0$9oCOZJ@v2m>HCIqv$9wd4(yBR`jOJw$=!0hg}$nts4Az` zI&w*oUwxVCQBhGv=A)mkUV&dk#nT;|eKNni9LV;>+{8q{@$auHJhqBP*wvAXfLhpf zyFp}sRbZvOyRVN-Lczkq!ok78>JJhvQSA4R+BR-%Zt5@mjpnlfXLoXPx)FT2^&^;U zdAVSuY+EEJ<5<_|=xBdG&rk_60VsbB#8q~iDx`mfsc7v42f6BvmfEpZ?X4w8Xbp9D zuXo4oZ;0$qiHy0dRJ>^5oAT#mEuBr2cV7#go}PyDtR*ESb#*796e}p+qio+2yG1O9*@Q=^}>UN+R2yB1RS#gUvz7b~abIw(ZFtCVyXf%NZXPM++U=bgsP(x>&Zbq4#_umC_4fAC?Em#{ZYIQk zbar-j-)Yvix3{k+Ft`&b!&>q=E6cp2=F~w%`+oUiyo_!K>Z*i-n3$N03ziMN)6C>6 zypcfSwv`u`nU&SXDQ9RnQ&hQ`iQiwr2PV$$t+&Ssxoo{7K5*r8a)Uyc5yg2vf0I|r z@?eJ4#XsvW$_uXk4wa8}`V;9*d=-|ta^fu&60%p^F=gXBZ4N`j!!vC$RPrf#c{WQ! z1@c0hJ+BA|h4)v3;#`(Xs-X-`kj6<*8d4S`p|8LD=Bv*G8ayy1W7 z&2(*W#ifCqsqe1{?%ut7>Cz=n;t;$gvI?ksr#hydGK9{}&+}s!^HOCaFGwhunJsP1 zbx(Px$j51^t5ZpIez>XFCX8J?jGaw~=5?l?vSX?TQjeDV&a}MSydrv)9KHG{%A)Nf zM^Z`(gS%1Fhno%~Rt>R2E^T1dkhdbV-FJQl(}prUy0!G?2=eQ@R>wECHz)nrg!lf| zSXAM*@?Ul%YZsH0nrb+d{}OqthD2l!&Ior-N$MIKbMIABDJUqY6Dce#WYw-LPfAM4 z&*MLCYj2;WQIcCyQqpsgLZG#^RW7!3c6RnzrK^*PiHWp5vTSnFSsttbdUyC1OIu)$ zEGl>Y@Kmk_BwjJ!@Z9Jg931TFA(c?Dv9YnUvs)H3Gc$wBkO=<$zBriY=RV>NQ;7^G zB_+j+h&<3JgfS*CFi_p1!fs`(qJ)ZylJenx*CdYR_J51gr1U&7ds8u`r>{@$Bm4C-`e3Q2N=ji7B>Nnj8yi=Iob$7@+4ilf z-<*QES^d;WK%(;5M^Q&b!>Z0v#yh)wt*I?Wz{J?N&}^9dgf`!AgYRs=6 zo^!U&pIyv`3zr~FP~^Wdhk1uv8dP$)T+!Cn)*8hl0+z8_M4XwK=`ikAbOHenx}q?Z z_x%3$s%X1y_Nveb*k@E!2#wt_8_I8YTecC?93L7S-1%EWQU!Or8Wf>vKd7XlskA*I zM*$WX$|%pe$s(s6#bcpWZ27{?4F{nz;7zKVHyncVi@s;!{)< z3HZ`f0MD3ByU^;f$)uTm_5D3^g+Gz5_N(tl%Nz{HBRe%hDA>GmVBWSHAb>jjve|Vz z4Sq(&ZTi!#v4qGC9P&)rF>x6E*3z&=#b#e)D5J>E?@%$#F7;p0*vyBW7QMkEJd7B{ z=2*GanXs!bel6SlX=yYs`SZynFyFn zGh2&;-78ne@?R>FN+64X1gnuH&o;{r34;e!YqHA}!hehIVmbx0d0gH~u2k`P28J89 z5%B!k`FJ90FVwIiI7u>SdAth!19h@rd{nju%@^^`t4PWNXC;AhYL&9Uj-Ja>v2RlvGsx zS(?KmBjxV6-2f_{dUaTdJe3R#IOh%t{x1o5qo|;}j7;-Tfk|GGy1&r=_SE1hE^h8I zH_Rw5_HjITWR!X7P2WUEopuOVNkPJ&&X(o;`SXV{&ea;S-|9pnQ&auyGl=88EYq|%q68X9A$ zsHnL22I2>xgx;Q>{QP{=!QAN|gfuMQcC>6eCGrefcPB{DAIoweL4aLD1X;oibM%}^ zPfsUjdG-zdh7_b1S8muI8zbPL>)vVukHuIt4i~_=`aSh5_Wq>66$23rtWHae0ERPer{!^#n-da3wJP|1 z6X|8b{cW|uAfA6dHXCx2nVHJg zr$19saahPK+G2j>X>4JEUTb8eqN2*{uV{-EDg%R$VokRl(IU<7o7jbBojZpSxUK8#4KZkYjW8n!ez#?Oa!BY3cSTBnv8qa~hB~wk#nH$V9M?R=T-pa2A;j zE32zVJ$ge#&gw7k>26n$m9-4Xj+%GvaT$p*xN2t6Sm|m#RUYHtu4EY^zE|t%U%#q- zdw#FVfko>vcE}|Aw(Hwf?~g7Q>a*S2qWKuPb1-8rY%VwI7?&XUOocMbfZPd$!yxe}bO>DI08xy1g`?^aiXGUV9 zguyxl!Q?484fKjy07)(v?nF4u_Mgkc%Ebx0VU|a)lUW4xhX_0VeQvJ<8D{W!3fSb7 zpIKgZ^4G7%^F3)Wp||w@I${V2iEGKT>Z!n9-{Wiq)v6db^1v<^FC6Q{GE3DiJg>dZ zi+$`WZN1e14#va9)e^ysCEW{bx5Q>9a6|bGF8o!SyWM$h7pitfhs~MoqrVGG`prj*Iofq5#g<|v zCsKd>ZVa`l(&2ltY63G;6GAToMz?z$S8Z5q)R_pe(vfct;>_Z35nKDUny=Z}E0`(B zU{fM15Oo0r} zl=sci^0Id#ugsr#t7|ImaP>;CLdy5;6jy*I66FlmSN#0$7Hv}-&Drw1=;m+R+t*Q| zKc~n>hE$bZvhM$pQ9Qi_>GruR%bxNZD+LIy{@F6ExcS7=(p?Cg{^K=eE0qG6K%3R- ztE-ie!>{DC%?!H5$EV(NUzO;H;x<#Oblv%mW`6wZj7_%CI2b+OOf={0H3@}nO|tdb z{^Gp)KVHXnF&AOf-{UimXzBnuW}N$Vyh+nR=QFmizaP9y1f%7rKzdck*%rT#8}v|u z@M1n%dQtsULQ;}$&D*nJr>`a*T|FxqV;yxq5Zyh)tQDpMCR#!@#dB+a{c+s=Wv?%T zN>@({&Zazy>YB6N3V%CDb&9Og-_FIwMO|H8Mn=Y-V)oVK@QEz-BGwqWhIEu+UHU0- zj||m3gD+9PrU!+XH(tKi-pXB#!|%;Nn(-XtvmBr>Ae@_>9VoRksZwI|HR{lA+}n)_ z-^uQynfE&5{IICen?L3$4 zhTXa}U8MWibN}I={*`?Heziti2bN;Pt9C}I{EE|7Bu=?35YHjLoO-Ux!b|=V-0jI1 z8poQtHK1DVq_PD9qQiwGP^z5L0zf{@e0efd4<#i|108%|?+e*g?ogJ&+^&XiZEb-g zz&pY&kF38}C*AFP@wty6= z1OV(o9uXDShU$CS`4wA3CPbzmti1LROdzDZe6*p&yKU@R4)}US1&(Tb`$4&o-1}8c z?Cx3Np5;z#v!x-y#;m+2ujP!E+!%}&s&bOuzO$s`t}C1U+BEY`)AtQ{0aAlW3E}tF z$u%tR6*DhSrTy7|2!%3$hMAqp>llAu!&odek{&Zg43WKHLWGjvK`4wdrO^Qcg$l*AN5a<(NuVi2XD9Y^1kW=|g9|l3U=Ic0f@{ z$xgVsd84`M%a=Fx!aM+Xw?mM_v^gFhdz^I+Qf<0&W+`rKcuL71vNW~xYyyDXR3=YJ zg?!hYH7Jzy8OOW2?kozO@V=Do9F5~BGhTYr2;w0cDHArc(L&f3 z2y3v2fVUQ}{#H{{vxnIAqUPIOG5GJ|H3d?@6G##9i)+$Ya5k*aDj>i@;v>GG zeol_n&f0X6;Mqww z)FkHiepd2dH(fun+e45@Ah@1z%YT3IZslcHuq|fgjLUPKlAqJk(kd&3F%=NcT5L$Z zhlhtlGBj-e@&b+acD@di0jb3Z(4u|oDgYu$8aFOXc5oQ8ib4C|PoF-)9YHxF2inPY4EvlqWXX zbsS{lH*)}zeICCYP(*&`{Wv+`|Ly?WZ~&~>H9_N3fmU>o$e`8?dl)mYoc7chYRf6T zQN#Kw$FsARJQ=zybv(B;05=r}6dvc-*E$JfId_q{AqF^6s7xOwpO$R|wy;cQw!~@; zz#zUWqVfZ-l2wbq8Q?+<>xUoG6H()P2BZ>Vwe+E)TW27cu7c|7eSU5EzA#WZKb4z0hk-vWex#S1*!xp@XvNHFJ>Ni081I0Z&tUf=7 z`k-|ul7*Q$PwqUN6@B^0r}6jyOxdkXad->*0fa_R zg5)PR50x3uP3f9JrXl2=6N=T*KY*_u@gdOq?yDoABdHME_kC4h3R3ses zM|S(G_BRpM$`T9hggifMoNPOHaJ`&hCY^~#sogxIw%tT*w9 z@)dsDSwfNRd&~7uq()g(@&!wgnuI`q8@fb3n6ubDD6(~~9xN#DTy1~boe#j$OgrTp z9oZeo(JdOaA6AqX0vaGUr1AGukcd0<3dv1G51|HAG%7}WAElX3&s%6dy1UUWPi~T* z0a6}sYG)?sh`Q+^V@18brRIUO@ZvW=3GnlS9)w!w@Pq1`Y95upUK0UXq?FNVeOs&C z5t>xj9~{;@(TT@rN!n)GMI@LK8QobU&tYqk=*%*@8dw*2kj(r=XW4@Ceqv_XPT ze#YQVKh@U8|D+9q-UFG5=)pPlvK$1qAhPO~7tyQ@i+5NW+O)uLS=3P;XsA4Lp0*_4 z)`A`=@B-6lIiTDPMX-Y}yHlVqTMjEVj^D=wT@?tQ*%`-~rFO$t+KFj|_t@}jY*3_&3EhsA9Yf=BMI+WSP#W4#WutKmKmZ5>ZzHw@=4m(X zhKaFh!>5`k)`nHWq9XFK%5rjE@}N(3KD zzx>}EESb>I-FQT=K=veVrZvg|x7klZ=?blrmC}VQ=<~lIbXsra!S80{cf}1(A$R)y zBbAwlJ7&~@$Z2S@&JXFOB~asjN(!%a2D#_Ix7u16*Pd-Tm=mV7&J(joLHj@hgN=k% z5A-?%TV)SEyG*4^8jo2V44vr&ErKn-5I5&R4g1{GpIbB{EbP)+e!rH>?e#~frmGz>b8%V~_dGJFI zznTKQl{AQlW~)qU_e%M?9Pol5lK`p5#><8v8c^qekfynvcgo+L#B8j5$4`5265@w9 ze=X4KjE_EX+0FMrTNv|}%3O*;@=|LMjY#ZIj>UDwv>u~F5lM8(gV=PmjhXU#5)>hWDoJo)ru}ywFJSzas zg74emb~o!NZ>JX(VG_KlAEtIo&>Sq9N4zgtJPEWwHfB3@0>Vvzf_(LF9u8b4I9}$n zr0Gv0Xm`+ny$e2MHBuo0iP7wKfMegWnyhsnh75B0%JpC=5CytZ&9p??fCq&|M3BGm zJsd{@jDyLS?6%s*k_B3z=pK44G|m)wt2ueJ^UibxW2hLl3EVz-}%Y9ZE8Df6-uspJpir!6yZXd`~&FrzDgRom1$Z>DS zSBzsTQsdlr!O#5Bt`Hh6%9aWdw<%p~Vg}_HsBxZ*%w7F&UhOAvaDTt(y8hcbuSOJp zgf8Bn)+j=Anp@l9VPVte_*qGjQiw!}?FSk%4-Vy)e@LNgCW}WmptV58@{HXQ3k0VX zaMAvQhqBMhRCU?}{6Ww)6ai6AyWn~Yw8N5^_-CSRD*t-1X*f-XA<2=O@E=OeEAND@ z`%b?HbvH_g`cN_|I&o}vZ;jCCpELgN!cqTQqL|IiJ0$4s8nFFn-W=##Lh}&rr0!G4pkbV*mXj?X8kSZisOha{_V1i=dBksZ|@vY@m$MgdPqc z9B6?c;zCjhpmR&s+XTP?=F+oDjW_3ekyzYcxmA!F>umfv`(PzJ&Yf0!O^bAwKoGcx z8^^mbgdzwCy6Y!|XFyHHGv+vTmS+SA&@tzQOpyL;g36|m2ER%Z=e9K%B(ybfoLu8C zbOb6&OM$`nxya}af;hLivXkB zK6>2hzK+!QmnMZTAF|RN8FH1W9Oh!d6uz8$RD8Tpy)8ER)vyg_qj1`MV9EMlA>rF&A+E!n)U1_G@T2Lg3v!gJ^s8l*5m?e_LxwZqK<9)!mQ z<;`SBexj(D;S4d-TJLnJgz1tF*?^YD{-!p56HwRA4lFnltYIC3(0fzxmTJgnKxB|% zil7$SDD`Po{Eig`}!2CEnlWG>ocYir*mGQr#`Gm-A@ zW2&+>OA-=Zu^y;e;pZNo%sm|s0oEJ@CtY1#pkIfcjN4$Y{!OffRLIjOPjvDIYsUdm znsPVju=#_g^z`&0bYAzbSB+Z~4MW&3ww-%o zr(ZT&39V^U;)6Jb++I4|UePel5WQsx`jGA{3cos?oR8b;3LnzaZkCM#?anG4GHUL? zdj&e1d1{$Mj~+aDV4zi~mp7=vs=`HC0TL@=Oi6pI&Vhg&nKI4RZH70InibBQFv%Cu z=#5Uvn3xza+vW(iY&byg0RpG_`3vn2k~$$Na$Y{uw(O;+I#z6@OLyU=iHV|%IVf~v zAPp@`9yk+R^kW&t$QyPlgS;K^Oh>>;X57QzArlyE)lRF0va<5SkbHpe@Y$pb;6t;b z(L(T#2rcKvX=_b1ZZpq4bJ)CWX&RK1kRA~A7&vqi)(lR{B>F8L%7|gQD!^ z54^~Fwm7Uh3Q`8#^1Tq$;5Ro`jq>oSha@A*pc;OgjHs&bIO=)q4OrfI$J(h?OWq77 z-%c+WGQs3SV-_+$@f*1;>Y2(qu+V?5s=?1brc3NJV!ur1b^=98FQM@I$8#>qyQSwq ze!|&;T;*FW?S4lXI$(D^;J9;Z;N5UvgW*ZGNS5iyB46hoty~17)pbG973m#>%WEXj_jIth%%GOXO?x}hzRaP%jCSM?3#h% zUCf&p!qK#4T*BkoPDvU7lb8IUSVVfoOr%W{GA;QZx3=uXw1i|%(Z1#TG+O4YRjx#K zP5A1%1k`47au><9kDxxX9sa0dL~I0|G^>e*?P&X6)$kM=+YEzf7tx>>p575TrCv9+ zy12ss@s0`YLr#sX+>bknd3oy{31Y%=?X&m&dmtZ=uDJ8;l>0S+u<2}?@z}&ddi-yE zx~u{z18mk{XY1f5LV3JCcU9oFbV%lYPz`OiWVuDM>P$?nWTL_-b8`~Nm?$-mz9#`> zN9+xH@6_yU3n;oCL@3gkat|kndHZJ_b-EPrpoHo=3gs0<(Sbsd90y-NVGK$X1jsfh2`>@haq}J!@!5& zHDa0v0i^czG3ilgNP`ut`KA&vw;2;wM`fSjM zWPuvf?8qr>28cr;MKI4TF1GwZg{oYvgYsi^o^z%rD;>$Ij< zEt-5~=FRzLSF#JSAQS_5>y2!Np%&Wh5F;rWU4H4UIV#S=uu@e}un7g{{3o3_$X1d; z)VMZ>N}Jz5gF8DFq(eaUKuj(8MaYCicsst7$Br?CZfP*j5Tr?sGxz=T2BJz|lr`@K8pk&b_Az{nwM`%c&GP={aKd7X_f<0RFB! zRX%;eFLo$0h<`LEL#<#QxT8q!W$AFI+B?x+zs<&7*IUmdeGpu{QN!=LZRzG#!J7pM zom|^hf$DaeVFExuupVPmE`EOJ-#^diFf|S2>dSyK0=(zK(G-XV+5PFUO`ut&i=eB= z_o-W4;`;>(tJ%8>Lsn%Xo%AcwUumhaU4MFjJCva~ZTL)4z(a?NPWoXUV!BS6&37$F|2h` zkXKy$3!ohoY-i4#c}Knk8x5~s1!c+wzU}YdJ^B~SjE$RI-3eZO2mP2kRC&p=rwZb( z=Fr^DZjrsx4?@FU`6l2C_rDX6NQG zNz|H*4KdQ^{U(dW=RdK3oI7#-dTHxluh8Z7J7sKd-mJ7k@sie0!Y?2IHGyi;aLigd zOB4a%(`FE{3$U}tzug2Oi55s|!{_qE0hW9hyhlWgLY;o?Rwoqhdz(U;$*SlBE8d>^ zmKE<6ut;(=4-ZdKe*P5!dzR~it2?e&uU-tgoXVTH}t;^ znhH29FV4-unGn2z8HJfrAczOHSXoIaIsG#fqo7WjTmd&bcJxT_mc5;wQ^go4pd`y{ z!KOlHjx%<&z3WkCEsWVh_4d+9gJ$}IH*Me3sSsPqn4M>Lngw+t+h4cO2sm$iu^&dt z!@*j9<23Uk7XZMp!pzz`zPwp*dB=*eSFesphXDEq5C>wPT16NdRLHKd-Ju6J0=(Je zG*q0R$OdUm%o?h!v=sjAQFLm)T|}3?4pj+k5Rn%00U^;j@4WyES23ZjO!cCNRaMm2 zjO52b;0Tw#&}Q9q7bzEVQZDa>hm&!5z*K;zEzZtLIbdsXSd7W|`?)^wEZBet0<7~` zkYt-R1xptK9cke@MKx(*Wt=bBGcZtWGb1;LSrH0q;ACfiZ|w*m$q0A7@apBu#ml7& zJ>B!`I{;E)b0TNl_;YqRyX}Jo<7#0!cDbzX{g5eaFW4Gf+A{Rb3{p5nW5$Jx^u-VQPDCv*cyE8+~f6Y30YZkR3(K) zMX)Ovs%L_EwpGFtCj_bG!SzdhY9=7R$*TC+Q)_^FsAl5atG2oBRA{MOCH5{&v5&hM zB&Eq$vIdF5l=!h`W?GtbK#-eoT*FjnJ$>W_Ivo&kQJ(irE!VQJ-Z#k`1RDUMKX_1g@5PhH%C)kkOhW;UxWU#o zAsDJBiSU;gg9S_8{01!*l?c~xrBf?3u=uV^3%)PT6GUB`ID1ozfBMt>2dA83{bsVI zR0B1@E|X_dwzM8=E8qscUVPboHhVXL?8O2?phLSy!E@Vk(6q2+rr*3MwVAbU_z5<{ zu}0p-DDm>ruzdM2ObjG$xceZWP;sL4al+Ce@PKo5MRje`R0S`17e}6Smpm267!Jb^2F^b@&vGM%u?70Yv ze4F;eSXz!0dZI^il*dgo2hUgT4$%n01{~>W@+Vnr&DJbr-36Z^BykY1zTh{!cJ4A2 zRd@ZB`JO^3Pt72b&Os=Wa}l_k`DUkNPC1nBC2X(*oO6|ulHY#uBW1*8>mG`>`_x}B zKcbRADh^>({a6O%Yt>~%TOH~Cx4ev3h$WjcI5n9`luEzJ%QinU`Z{KsqIMI7I@AIT z4Sg+qE>|R0K+()n-SKgF~FhI2fl{YFWZtbAg=(dBDd?g?xrI z46904Qz54GO7#28`b+t^_8A!-BHg4?{jo(&7M{Bd|@U}p~4yjf2UC6a~kdgLzTYPa+T^4G=jA!})iB;uOuutyBwbL*WU<%NC7~hb8s!1*G>+~B z0N>qByUde(_8n9$y>ZX|GPRthFXbpfDX=@2YIvT~ceoYTErJVu23(KqtasEs>uJE% z;76Qlk7Tb2+5GXhpY6K3nJ?}#`GMmRxUMT%NSqsVyCBZ~Osx$#O)m{*`K**RD-!lO zLXa26ZFCz?w~OFixgC^yPG~{;2YS?NK&&+VYU9c<le-+{PgEkpkRUI#n-@qGIkqYdM?dw|bq&n}*x z0;0ao@SbbqG0%IOqT9jncaeNc0YE+raf)5sIYL?6)K!cV4M923j#VX z@+&g>KdYXDEi$gtVJbaQN7{5c3>aU~)ulRuYI)AUG8M95Zz zW1+k!T`H>Kgg4&)T-c1|x4Wg8jbY&uGEC&QvZ zt>C<<0qnMfPGDSNrrDbOXXxf}^76tyI>02r8+H}&4pg`}Kw282FUiSSl@!?xe)rOT zd1SD+m%em_o15Dm_6xcLd)t#qC;g;ocdMu>Mt@oj-~|xU0Fu8k&{aVH=Ve1FbPpE~D^;q~b}z#}<> zrbr`xF{hz+7O9xX0;1@ppA4^kHG8eDrj{F7Q4BZ*$~)n;pS1R6IHa4%rcoR}D%KQ! z6mCys`v(cN%Ob~WpHf-U2+-1#=gUTs_UDYCr`&e=yXBgj{UC&g9l;mHjEja9x0q(Z zsbI%AbmJ~ilQ%w>l=Lpn4QT@`30r+w>c1u@>%*p{LD|=GJmosCztKtSRObEk5~2>` zPeS+2sDylmVj4Cj__p6)OOxLIJYv_cInPKxk#t+S3$|a6m}3S8p20Q-z=bxe@rC$$ z=ql7kmB3Y@oi=nQWSm=-+qChIP^3_&Ae2r@qlSO#NdZN_C^)0ddsBq%32Ws!xCn6i zJ3Wf>J|^y`v?Whz>#I_~+rX|M%H&8nbfH=%ry32YqwBc%_+HwcJ$^fiDh${gb)XfP z2Czk|&uaAwhx*;COh?h}q^UXk-5->kl~lOS8|&_7uuGWv$E9eC+t_ z51pK!xmSq2FG+BelDY9+A%=>J`)O-s0W&JzG9UU`1qD8_or4){&AGkC)AgR;UrU7F zQQT9%C8lgGVIXyg5PMbimlvbp97_`aesF{u|fH@_-yA9rD?T zYaW9ocqPhvMVYj2)*7Xkt`^c3Ct_iXY^*`CYd7Ya7N`O8Y@W`2X4{y?sPtBIVjJbg=Ffn#0$QZ>ZQ-!nHPwT~zOTKQ${h3T((dM(U8sVX6Y!qH^ z)Ks=!T3geMB44R`xwB?m);ay0GNrLmI?05|h}fHU2D`y(l2P81$*NW1+&i-l&vxOB zk&9|&Xtr+CKJOuSLAVVWlLv(YDLv`wN3yd10Rcdg0A~G=-nvVpO(L>6KQY7qM_KoX zlM%kKs??%CKV6?X%UgSve?(bOF^zgCZ#jEkCUh+O!pUPB?mkXt2gcRn!Tp4VjOy~` z_xeuIw)%bWE2!rQK+jG9Z5RjvYhvO>P=lhjwgvZ0ykQ1fXFi<-Keg)MJX4uoZM9tz z=+Jg{Z(SSb)2zzx*NS`fQw6(W zRs(40(*muD97K^U&9EbcJId_&^J|O(ER`Foy!&!1Dm42IgJ$Zkb9t6~H51wU8x9Xt zUge*RKWno&ucR8@jmrl%6OQKzK;;DM?|>@FW~9AFzC!H-@|Bn|?>J|>6m_A!`zi`4 z?bWL!-ZbtB&9k@OQy$6ZNbLO5K9ioAdBCQ^MLdo^QCDAB-%0G8&A?c$`c{416;8E< zyYo}o8_Y&q(LIxOi&NwuvEQHkxmVWs`@Sg3>`c|Twt~XFnSbd1+p6Krw{Gq2Q+!T8 z6^re6uz4o`<5TAgVN8_YuF?};vY#*71desopFDOH0z_Mm)um>`F>4WC)N|k|J9}Vne%pN4O=;6-9VnLs;0P~ zgvTv3xbMI1yfn8wh%It8!LHf1@7H9rjp2UJx%PP;{nwBEuD{{ERGIC*oTVAXosp3f z{u7#Hv=8eNpCm0bd0Uui?=NHpk4Hb^=-x>Q=p7%bj7B-XK$!G{a`VbwpbbWzne7X&1ib2bH_UB;>*eeNKcy$e7hH}pJehw zke!3W3RG{1-t6s(B$ESD|G7K1y~up`PlUPAOJhquMR}XRx~YAi$Fb>6Q8dfLmfiKn zOSBZZ2AgT}o}Jl(yWy2#3D3`owi5u~3A?t2B%up;zM)k(F1x760YJWTdhe{7?>*it z`w)O(v;35OSwPmIK=FnZmL?OMt^KlchclfyIvKi%v({&9y6fkc9^uc6n&9QacT`E4 zkIp*j!v69bqEAVuLGK2M)I8@werRaugNVmZt0yeED0_t`ZTX=){UB-n@E-Pu#>(fb z-1VCS+p|@Ouq8F@6|2x>`ZF!t2dM=;2z#jxb6{) z)>GAj9iUKiu?q^y5*>^ag{m;qP;$YKg|oT2kLK1*=MF}j7eFcsk}+7X{&WsE_tbVb zjO+EpN4c#4~P6KdHX{g_fOWu>n^c{#BFW5wn+2nN#x3+hLJ!=u5&R{8$ z>7qH-5p1;BE!w%|5l;6~@LXi)?vHPYiu6pf$2!(Owi?Jp3j6g~gUCiMl`}!xJCP}Z zRr3kKqhJFNxcJa1uzdY7+0pU7RQ`NF((Wu!%e*lwX!A$5RJi99rg+^^aHR@g$(?=X zgeQ&LMlh4~;u4q!5Le&SGU1^IhT`!VXGQDyStCpU4I0M`#q&`HY;9~HdJf~q=}MZK zLV|*Va37-y<15;^dOnVy%gS7AZJ+u-Yr+}o>gjz3HsJlg>hJLAseBR&RpPx6jf@fw)VwDffG5(#M zyP#zX=HdFTvx)T3u+?#~;84w2s%*`0K@ooTOYSp^`Y|S4AtLN~6dkWRTKXDXBeoa` znDl-;x75+8HZNinxy!*ZvFZp9PGL(FQw>KR91!q8LRVikoX2z^KE)TF>IAo{rzg%5 z=wk+rsFol~sWc;W7QK*C5zTwKGXK&?MRm6LM2|_Q;mI9oU%x)-9tQEm zQo2+KDY3Uo#S42c(!^z2CcWu`t}tRxgsPi2mlhX`I@<)5sY2g_cD+q)rEc(dd5Y%$$Lm z9*(;C{unp3?XNkiEs;$UwyFgzmxSvC1vPqdRvqGBm>T({M*He>y<2xYrCVQ}U9MDZ0&-yS1j%zP_iI zOu;gY&C(|6pO~}ags1w*;)N353}~Knkx_Z_kTrt2M(BE|HoZ-2`Ey;2wlbO;A&MV?!Y1AaS#d3k%-JJxm6bo|=|Kj{$CpoVci zmz{}JE!?hIzY5E(MReO!-vnN)7RhBOc-$ujgM#B(~#vyJ!1(!Gm%Xv4cc)qblZt@!3&^0(o-dqj2LOD zH7-zN2?xVioH?Hw{hsxmN~5*NoStx=&YiN$TNHNn=W5bh3%pAo=Pp49o}6Pusu%%%N71FVV3 zj)+RRPw$zH^{wclul^x8x03|d>A%+6-9F*bpcrM`)Zn+OBifJLqpz8 z(U!aLAcZxJQUbp}B$F2Dz4O*3fCqXc4J7^j`xiE`xFNFMn?Z8Jma5|UL*JeuLYy>; z3TJEd(wiUT*X{1tW@w76M?P}Ao!TAwUO$znLy&Z4W9}&0)C; zXjzsCNx)3BUqws;#uq~S3?qTL{5YwKkGr}Lh zxcF@-3NNOz?Tz|b_(-(>-HyX8HTzkJ?2QxSo6BYh!=ABRQFvP};ePSHbV$9YYSFK2 z9$JpGM>O9+Dgud5^&=vQ@S%>5b7xNUrouCEzEp%oCo&=Vaa(=3K^5W5GKHB7AN6+* zEZM_D0(JmW7`wwNi-*UYP3<1+=(y%fTBL5v9|QabJOt^LGp%^%6mRp@ukg@1Z*L?k z1V1jh%(0Y%JYirsS=>LWv>LQy2D@FkEXbn(A@j~_9^)||;pOAwDlD|G^Sv|_EmRX3w*QwqY$$v%mnB=HpFKbF9@Y-;s3wVPla8-1bhwk3dqed*GK>uF6coCkv#f4~I z;P6OXyWfx;#ERaw$(=eVwW>;cxc48?6rb&G^}rmF zF4T?p#Zdv1{T;~h-1gEI0H;(p80?3;L5^AP=t{!k#5 z2%lE9ShgOrc_xdLcBNP7|E-v+2h5f5_~*YWJ2@uOT(wN7y>PxmSI88<8Q6YYSZLl% z|7^+fz9!n?WPAWr*bEE|AOblUK>RORSy?E#RW0`G{uKOE3sx0d<@|dssKMGlFmPG( zaOXQJpHzZ^ipr)$BX6vx@+!~oZ?UiMCe@5!B>&sbya3}d$Nc^K4bd7Q2kCpL>5GQV za-JDfc=v8*xnRtjUq~OH9jh2t?r*=Aq+K!KzLI6P+pBnTd9=WMbMkoSz6(-W%07A& zkjMlNHv3oE{Mr(VaR~)z*8a&2y%S|S8&y_lt`(a8^>k}rs|m52jY_21iKq{QpT%50 z6=tywEf1*o_frpbb0u-y8#^aTg*+vK^z>O`;x50#3Y109H}MQH@&DD{wZ}u5@8MzX zmRl~(N|uw2ZKP=^>)K{*E}KhIb~EHss_F6(W0L#nU@bCoJxEbPsobgLmWruH3R!KI zXqeEa2wl*sInO&YcH49Id_Mhm_K!1v4fDS5yzlS&{XXC4c_wDvDHQok$Ikzxo9=d6 zOF>|}iP65*xXg5N{AApY*1pCh?%IBY8YtU!NzNa}UG7Xjrp^m1)avvGbbdR>v1Fy4es`-7uc! zw^Db-!OT0sTZ6JL-13~fK6OdORWC6ts3eRRwsldrJ!93fTMC_genE#J`9e)^fT70o z${Q{A4}bXmuWRCodd!?uoDnmD?)A-meeRtVFhoLx< zSyijV%V*DZM7RTqT)esHJFn$4AKl~sRG;K-zw=!2cdR=hC$x7+u085-29fUA@;Qi& zyfV|0Wm>@eE?##t^`O<3_SR+GrS|u;8NmtCtfGPfZS@6(Vetz^6Dwlfun=+N+}QVd zleI6xcCcm7gvAGmnSL^ALH!ak-Jq#!+fM%aHf6ol%K`l?2k>YS>v!Z?15y-B}FU3h~zRV9PK0$Rf_Q&dO;#n=I-)?h?M`+gG1FS=&d6{(FGnw^@m_) z;{4pgQ(z5#A$~OdPbMCzD_LyRf>+(3zi1uDN6CJGSCtQ|dT?fKNNXFICO$xPMX$*t z+n8gxN}-OO4%z!}VpeLo1vXzN9V@|+nkrR~tFyT}cKWJRL}|OaeU@DmsRT*!&Quj= z@{e~vt64`{`%%jJ=pdmbG_JezFt=WO{SN_?h&q?(^#RD!{||njJN(yc4SE7?oB8rT z9V&1=R`pn)q8V{Fo_ZSy7+kqLgqjC*zDCv^2DsD}bikW=IyRXG&sSvyg;r8~e2nJE zx!WQ!&y9|607@sDHH9&|`=Z+snh;9>xAcmV++7WcmtWf1*&Y2=D2%~Ow^){TuArcK zM0dzoa%8Xm9q|CAm&eT;MsTFYFt8M%#w4m!^A87_$HiA8{Wsk6uh-u1VYS;=)kb_w zQDGlRN`lv!RelEo{kVAuRJT`k(>&`<3=PxkFNe>#e{*vjzdP`n_>F)`s267SXfVb^ zqDJ+s4Op%!H*}!l)TwJcrZzS7E70MR1so^>Haa>w)WwFw5Gg7l!dTpxg_bM{apkB$8p z$q<(fH$!C0Z&r#59UJQC&dvZsUo_HDk#}{?DmPiZdf-*4yq7vK8x4&Op2es^pls^b z@$?l?9fvwPI<1pd z>t4!XFB<+wMSXoZKpkoJ;NT!I!x%ltataY+a@p{*B8;iSJDvpKK{q-$fYQf5P*Gg$ z^W@%XX*Lq9Og%57@KuWQvCZ+VW1Bh*fo35d&iD^?oK=UGuW`-9CW1t??e8;=b)H-><;6t3T)v%wayCkN4ckx`m~n5 zc{5OBkc_?{E2qsrpPbmlyT$8UMxn&U#DMA9GYaazX|I?@U!DwhKLjYTO-gJ|>yW$A zA;r#xDUL%@=_=|WyDI2gO5M>rwr|AKk)Pb{qhfpZ=AhEow{1ShGLS~6s>po;h8y*^ z%^Z=QT!JLkvxa5;QaX}1KljB>A0LN+&0=IfcZq8T{LdXo9-72QfB_$~7KrJ%J{XYDLO11}yh5Zg4OtZ6IN9@6v$9L0#2rr${1 zR^s-e?Zv5tMPIv8wS}U^uym%o=ok|9OPq53-%k;*KIG5L22|!tj(o*S6<1l3jRTrj zqE_*rr~LjcBo(zkJ4O73@$wq}vQl^=CRY?^k#2p!w~%Dz|KyZ8%ntHC$7%EE9EsXkVnN+Coz5$v8y*dSbV!(w1**@T1{dgUXug*dn~ zSIak&7!~LS!m3x8IwiZAsGGwj8k_`_W+$r<$RI&bAYMU%foAH7u+$wx%^-c3so*qm z7cb)WvRL;<)MVUB88pZR55Wc5w3{g5vUF6~e;>a#527+p6ZI1vF7&& zbZM*(wcF6qi$5dfH=gpqWi?$cOK&HdW1+Am$%#PFA@E+Se3-v4wqwsrpps+HpSN`_ zz_Aen7Ni`CI}2}h6g|tT^YgW0pHwLxwlXs_1E0}12*W6%`;@SGF_W%J!W0^!?Sr*R zA)1ON{sU45gepu%;{73#t4P==W1+k6C-5u-m7~AQPk^ItDgUvHRne+QC~`~>48a$% zg6{D1&meX{G63OS$E)vgM)KlD`xV4vv#@-^5>hx}uSP9g5KHK7A-dI9U(gn?HH~}0 zf6&G@72e68CEcv*)g{XSTiOqQzB776#7SNIu)Acah1sgZM5wzXP z{&Cy_<})UERhS3PY1JSeNhqdriuSH}0KOo!(T80N+psn_!o8T&Psk~8R#kr>i&Y6l zMqnAySlQySC~AB@(p-vTHaIISSNv?E@C>M{IA`pt(78Q{4%_|HrcrEf@p-LdUFZV+ zpx)^6->3Ve<)(iEEyG^i8mfRY0aW-dox8uucs$xu?c(CHyCw|@&O5kdptN-2Ad%%9 zm3@EyTO7(>6I6TZM8(4 zv&~a$Zt(E0J$m%$oO2;s;F%HJAo}5T+VavBNbed@3OgQtOrcP`puU6ZE=%wYLbBwI zqV)!s(NT`itB01G#|#cUzD7L}dU~)Cc6yfN<$1%`*I<8nAlx9K;M?ot;}duh&%f== z9~QvSIz$auuU>t23E~FATsmXa5DeyOzkW|QH?#aq)g^?vz*J~Ml`aA67+8MWi$bt? zIJfs;7sLLG)ss2<4nGUAiwDMb`LOIy?Ql{ zWy?bk3Vx8wrcFG7B}^uN=>h>qdCS?eX9-4i>rmEPy?vmH5kEqZ=SVc1c_nh5b=QQe zvj^51iq#F*c#bzpS$Ih&`So6&koB65stty*W7Y1qyXTtwpUw?=F1HVtNG2wUltC_9 z!$}Um)8HK--7WLuI;=b=I*2JjHhsT`BT8X`oj}|E3HGQVmuvX^#e(;zqJ>M!`sNX3 zhNi0A0y#M$0%fr(U$j_?2I$DXK>1vG{zNdgIGodjAU!=XjQsyThzY*h0opO literal 94641 zcmeEtRZv{rvo8(_5Fi8z?g4^3L$Kf)+}+(>0|a+>*BRVlaCaL9_uwAf$<4WSzWR)%bg1iL!TY|SRFfiy+lA=m5Fi1x*F!1guNPq4?S<%q{ ze8D;^NqmK=nj|{>bAezkEGG;DQyYu=Y>4>h`i+C6rZWu8dzb(Ez^L&wK05_Bg2H}?(R;8;O-7L5P`F-q_w$GFrcfosvD6g^d5!0f9=I6 zHBv9ZS@YwA;Q5JOU8@cPN-OGf=V?t_VfA>sy};EfUtw@B7M|$e>wekqv-FZfwh{A_=3}sf$#Uvi(@L!IfU>39$}4o6 zgqHp#B)_NDmeS3^~$flb?a!@V3&o| z7an6(pT!clh$%7!hEdP*!PhcBa_j*R`xU~;ltc|3+xrBj9|y6>;*k%1H5&_Pc?Lx~ zTIHJkYE8lO02aU*X_dj$5BUe(2@r>BdIn2!z=yS9wa`^MjHa9atT2Aevwiu|(yc+H z8;V6Y5ExE*v8ugc?F0rkGe!)7_0pUop-}qtl2{F1( znXD~RF`y8$Y2Q95aTVr9d7Su4ZnpRGg7jyP`uujUcU9k;)8n36{IV57oU*#lL=4%) zJ@*$}PGbQ-zC#nU%;do#l(B3xUF884CUov&1F_Wztd%+8$IQ2ss`6{_A2&b@P735V zIqCU5agXbcYSvg^+$9jgKOs9zU?KG)(<+o&I>q3&pb!w$L4Jn3fB%;-&wKUYToFC4 za{UJmy!JQG%0tQMJ_M8>hb7q$MLa0~+G-=E?7D08IBdF_VUvnhn>Dz%VIaMsukt^T zxaNKOlI3Y)-!LjIG%`>0e7`H{Bfw5t`uoaI$KAvDpy?FxE63V*?exfnGha+n#@d)~ zq3T=@HCY6@>3VHKBdhwhe|NrnlPK8#NqM1%6YZb`md~C`&*PBKMS)@Vo=hpPvoqT<{K7Embtno4HFlpz^KHRsWiYu^)`T-!@yE1f?N55*#YFbTEn-M z)84}@I#03we}rh?Z79bn3JP?|AW|>(973JpkcuU@Tyj-W9+=muSW=$VUENg6srH=yF=SF!_l<>A~X*{n{FjXDvvBqEv7F+k! zyE!1XyZOVg`_1Ag!K7JZOh%XS9BAY6P_A}K+7wHHcJB0qjXNfTc%^ETB$hC<%UJcH zz;9?F@=;4^$H-dhn)_b|zRD^NNuyuo5;l*JC?W+0(5EK)58VZ$tsl*M61GJp3I}4S69o%lwYbYMx;|eSOH~vE zDs99$_vecuHtWwZERUx(jM~~67NQve)lnlpKcW}PP7>Dl zxxTlodN1_Ij9%BGrQ&pp9NX+V?&td!^X6n)rbq_(#rPKoYaRw5QeOzy=70prV9-=o z=)%V9hYcpCuVYNaGgOQ)jJUxNba=HT+^F9NIICV8>>d}D-+rx!U1*eYN;Eh4eDNIX z(^#${d~5{mcPvzT?0)#-v;p68*LA+xfg#w(M@d7*ofam4s!ch!`fxu8kU;Joqx5*t z#tUM;-7bm&R7hUy+Jb0{j@r*hZd^H1qy~DSZL3cET8n9Ag`Ne`QyXu$BWG|sNzDx> z0myP0gu}A&U&Bqc5XVl~2(dfgZlC@ZV{vN_zL;f9%B=e~(d)0@u-U{qLnFc!Z)~eY zKW5IC#!emRh|Tz+=sM#N=dh<~%rrTElH5DsHRbaVlqt>4FGj_)c-gX@+tH8MWjw zCXO_4@Zmcy@_Ul|QJ8Gx+{AX`wZhz85Q<2Q6Mn~*7 z5lYQ{0nPndHZiwj|&3_vkXh0aX;yV2Nt~lVVf+%TQrWY znq)kYbmwr~L7G=_Y}s<0({^!CxnHpLSnb#sjbVKGi0R{`AjfTp$24@+y#BqdQzKZ< zTqh)yq@BugZ@ZavWYn@#ME za%ZZK=udI9dH#?sgLw`@ucnwnFVP?uPL&R52Q(;)Pjw7UFs_R%@J@UYrGeYAhgu-Z zwIHD33-Z(#yxvhIO^Le7Iy*cqkGo(@aYjTiv9Zy;m_Qcv+tASkc{}^^V3QIa7H|rr zR7Z^-4LIU?QX&n}-|REArP8g%R+6W#$gtUZ!D51VWwhKIkxmLs3Xx3>COWOQX09H1 zQu0OGByS6l_HvhhhaDPm41IDaAmY9YWy$g9j+)rYraac+6|4t*Y>|X;>0Cm}N=my+#hnHOOKT!5DSBA))zq=j zi4iq&lw-!%J7&gJDQ~PwM!4vImB6vg0VQMsKsm?BMs6daE%*B?^xxfbcLI^-f&`(S z$A!Xt`Zgjr2g9c_5?krtaJO4lat5xNb-ouq-F7Q}(*Hj6di8-xE;nXK?-|SVs5Z@N zqbg*_t8%(pog3|70k!hzspEx~-XDG0P55^wKWm;lA5?~`um^|nk!SS22sGoll7hQA zto=JWw3-(u^zJyYXDB=NQ%*a4&`I(A$#a*swQg2#JRYV26XP11EvljUoIaecrIgQx3?t>^vfmo+o^ zLsgj=iUlCDS>B^)OpbX>7|#f-X;a>himW6@56wOpGA(_Y?g$Epm3VZVYoy6QAdbp4 zmGUgNa;UYiK|rN<)wxZDVd7a1wbPYF<7fR&Sbfvi+#_Zwdg}NI=Z9QXX`!;jlD#>$ z96JZlT$AiGFAme&CfFb*jLpkL)7BH~O+`PQr%!@NpIYy)BT=Mxp4m8&6L+5|l!tQd zP&OMVeGisL`r}Ix9AZNd3H853AH}v%-?y6|O?(8!xcA{i|?>w1Y10UZEJPB1<6X;wu=L@hwE@ipeuB%Wc>HBKENSr`=v zwDgXW!w(6Ws2A)fwfB6d%ndK#oJx#>*M!J%=*Ds8`|A1Sp|sGTDmNrG*AHPq_v%xY z?@iamOgjZLI@sEH31cI_;zqOOy2VT{PMm*_?R^Z{2aA}kQuM)P;D*7Z+YM98;<_$! zowtUXjfx)8P;Cd6dWx+CJivQX8ZHqzJACV5cJfzAS=BRJ%1Id{ZBrcBCz57rG4@-PZ8f z(6aop0;U0ksLdmYjRS+%b4-s!q(a9kaz{*H=9_fLcz=xmV1BYj@nFs{3Q$s^{F3&6 zxNXydP`2IjVoGWd(0R}35}M97(c3_m`0LU5cXx2Ce2*cXZZv8L$9H3~!mnVvJex6s z@hkyS<@THHQr{!6*h>H?_vYj6e`o4=ztDAsxzzYq?1i~)a!coF`^tK zvuS{}R6pz~*Yk~Fj0axz1}Bj%{r=|b%I{!|y$>SjyJHh;KN^rU%f@ws9-PgLp{1l- zGsx#Xq)Z(|sa-6{(OTx+OoBBt@9nR3uf%_?%m_`tOTb&(4?1g^@xjG2G=4K(=<0iK z;`j_oc`nxl{&xCt}v0TKx8x zX)`3Qu4^bc8hy!~e<_u2!+j|gk&1IZj+Px2l}N=7IWQqbtl)`OS0Wd8ojWJne#li zL%sLqTR&E1=d6?O7@G{)k`b~p!v&xVAjWn)!!eDmhgElGhZv}8AajnDU+eR6-9n7aWQnut zIC>CO!sO-vDW;k=WaE`b!2ii{)oo^LxS`2lJTrD{{PS7&6KbTvSx2{3^7hfeys*ds z068)Ob^|u-^VxfZn;ukAGawuSsx3={jBMZ871@OcDJ70T8z+ z(y8O2ssb-;g8Z-2!A5n*4+)(g^w_Ki*X0+G$Zy*dsnBbxb+1EMon=0pO!Z1J%hfQN zfwzb=PK&`Fd!Cx-yQLQ)Huc5{8}E1;3UW5pzmV&|WuA7(Cx(?VT3B)N;@yad9gX zyFKt^-=oj;g#>RYlt7o-cr<<2n{aOyz4OJUf_(PeXynUME5k}H$cJ@{ z%)BxywN9^o5m-1`NDwRaxA}TLMR5Ry0=a>ksb1k~YU-;lL)5g-m!{9|WTxb#1QxiK$Cn#1h0q*>qDBIR;MjIrczLZmwh!5q zcQ5c15ffL5K4o)#*xhLVu|KreJCDXpO0S^W;zg0;ed}BC7AX&jilC;(@KlLGrw%7x zgM>l*h?X47(hJ6q6OI+U50=6)4GDLinp0#UkK*nOBNN0&)dEUYzOQ9{@E309>it?v zxm#g&_DPQ0VTTpqRH~KMG8|$p8sthZ+3f0vpg*oc2{8VQD);s??|_(aKqMsZ^Dfg+ ztajC?&x8+QVaX!On7h3`jY7khK#V~^*~XY?stKMNH~{C@sp2vBrO-;=qjab9;!=_7eB=xEc3t+8!}|vqnD&M^dCyNOMuQ!@YDAc zDYs92q8{(9Ra|%wOI@bIkbV&nqx0Q~{Kieb|2e1rt2Z^V#%BVzLc0@Y!|xTA#A6G4 zd_yfJP8CBU+vf|w+%m4{gzAYCtEUb~`Rf_=e7o?Xz6QWd${T6El^HA8P01W9wj6i$ z>+W-mu3?tjYJKw zgO_vp6XT8H%gr@32&Y-|N&~)sFjc)(VX>LxKyCd~-@9RLzk?KW&&Mko8tfZ91+Td@DJIt7d#RPuIbM?nQW!vodG=f19^Uy&BstmB6x}ns(-QpprTd|Z}HQ4^Vv{2B<)imt*DLP^^ z(~=30do*PE61B*O;##y=mkX@(*Q_>Z4F(eGx>(tE+lyX_{bhh%y519$=yT!Yb zwrlA8EAydP+|Z$b7^KQJ%5oE&CX1y5NEP`rTXR7J+Uv>f!2*!(y^Fg9Cbe{;uegHF zd-3f_-mQ>b1lISQT zIB@)SWGy5=U@@)znh24V$aKkbXbWMZi&P^b8H1Gj zt%Gnf<4l^fI5ZTYc-^xN9Gn|#fVhU}+Xp0wxX>=t7$_%Bz}(FZwnD~n7~1nbB9 zfTre?I0{de;>!Ya6&$>j51Ky7_yTfAB6C13@Mq-|;80x1+!)Gs+1>C*tKmQDng@a7 zLCJvF9DP+ZslJ#?mt7twA6KHTla=IChKJf?Pi1GVq@y9#l?4|%v8xZ1Ls1>Q@qZO$ z{I+l3mlX!dT)}Y>e)5m-etmkwW$xDF7T*B~K8gKvi>Bjw-BV!(g zWChP(k}VLygLid}ItG-jCI#7dKEBnT(uppJV99%mTPopx^_U}d5M43hsl7w|tKg0` zlOGuX21w4j;Fsu~vc?@bW^kQY#qLr4AB?g8Z)7U(F^}=DCnzWYDF1W3D**Sc#Ok7Z z0YAjGo5e`*Antv5f9GGwieh#_FeNU$YlAu%=>tyNV#`cOmESbEu%IAL% zu8O)vI5??n3KHJCckqcNEglc1$Zy`DL5b;gTI-aR10+Zw9p`@qGL>cvPr@rt)ZAP8 zO=Poi=*7CX0K$7CcHmj!XFlGK_X~m|4j#$a|Ng-oua+rGzxI8DI|1e!cpMx8j(25z zCS|DF{6G%Qp|)8FR?*SG(-Foj}$|JDfBjBx+| z9OwI=i~s)#-1z^O@voJsp#Hy*M&2Lm;=U!$C>i@Ig2^lR4*)-4|zfR zk0HLqlxbI=k(_m%p+FD8??luC0PDik6p#FM#CT1%Lw>{tkVbs))(!i=3-0e|Vg~mUkY%s52UA+R0&kp`ZXQ<>XNN&@Sf2&(iy29e4A? z&61*EVai?Qd#b;qhW>J{nd)sk#3LdJ8g4-T z*?J}ofC}glUec;i9j!72DqdE{`BDCMUMlm=oSE8-fC}_OR(S@W4iD4SkQ`;p+r z69tELriVJUg6Sm~?D9}$_}}kbATl=f77B#XHN~ebN3bDOc-ll)DS($*v*vU`F4-cZ zgO!wL$s=rs)y@P3I>vAWTG2BwNd!H9TyGbN9}OB!d{kj(tn_lU)J1^;6W$FxraK_i zP`z0v4aS_YgHl@$K1S|{lCc_e8~FdRf&Wb|lJ_G40GzbXOtI@C`;X9s2htkpOr$hw4!6_>j0L$|(1_0)o z%LGb99|_L+K_}*6x3L_`8E0f8q~cx~7y>~ri2rS3in@6Izg=@h4xtmX1xk4h6a-%s z?i@X@k`5M%tJu|}q-{|WSZbSub>`0el>FbT^2B9`1s~>e#PUw4-@J7ZmWVFPLI-KwbqY_t3PyM9n?P< z-Svf@)8RWtsxATge8-DCW^plJkgFBd+Obm>?28Y+pCa6Q>^omL=WFzTM2Mw$%5nxi zKJ&ij7o~s}GLO;Uk=Q5;gKv&)#xp+l{1n4=O3~IJpRF^+y)^MNkCv*OgY_Nq^jm%rJcq1!+o9vm!& zEZo5)AQpqZb)Vd99Bv)U;}vEJbA9~sw2+zfHPQNp52*zK0~K}IW-~Ka=sOymVWq@v zo-oP2w5%k2?YcGvlwnZm*8MizdLiAMC#OjbDg4t@TtY=*eAYLR+-jx*ZIna*S1aj+ ztTE2&l?Gry1q1O$EUNLgIN@4brbbP7@3FmFf%`Y?RK0|j1+whXSW`{N;K8~IXGHap z$1N<;o|(tYc?R$pRP^Q67(*q&Z}QsNb46YExb{V5Lw}AHG*ykRdKh-j>}OX8x>Zh9 z-EF3?77T7)6F9n6D1D4HsqQKp6S4e9vwj;t&%3NrOxwJ{;aVTzxbQ^nEl$HfKM_2r zB4%)~$EO3mdr(~ys~Z*vvNR3T?Ki_H;U%SJ41NdRVIGDMjk#e}L?L0$IjYpTus^<1 z=MvQ)`Fgqa2UBm%P4MJ?ds^aV@aN(+Xz3Me(ej)49`Yj>(_HZZMVh#*JtJZ02f$%U zF4?kZaj@kZeGM6@q}6$!t2y<_1yOA;Q_*t!Bm0DY7t_z(#Jrsit5p-;y57c{FAxZ=KQ;k#l&FC>3jEu=T|I{FlygUWKx zC@h&_DFDAfa4t<8di0W|aN0P@h9(PBQI)mso}OGeul9&lZ8UFAS4Q46=SIkkDyZp& z(?lN`e+vN9&zfu8Zx%{3TQr+SMz41iP#(U&z0?DU?S8^X>t#W+kHDCxqgU|AyT5zS zqOhQo(?IQAkbwObuJxE9DWz=Qfz%`!%7M&<2g###orpYEXpJna zOOjARCd-GM%3n9kPxRlau7OnhKov!2Ad=(LxymF}p*dTfY|>E`|M+ZJDjJ&py?=6Y zMO|$F-#2*CWy-LvKaKEsKQo{a;|jSo1$f=KoQKrDjRR(aj%YN_2neGIeqU#nsH$Aj zM^dD*vxX^B2I2EDMbTmRrExJvA&xP8Pz6Xzakd+fQ84XA)Ab!l88WOI6*?2#lf#%E z?$;N}MwKs)q@L~5=#c9)Kj3K(tsZn1oux>7c0}yVCFosNB9_biD477hSFIx(a_?g@ zi)HG=r#l|dCU%Jj-;F$|Q)+Iw5HVz_guij#ZPph_yrm(SUzn-S#h@$ZN~vJY}pzP`n1%Dl^)ek*)`l-gf6)0SM1mLDc_gv7C8tRHhhsx-F z8(Aj_wdqYEwDP*r8BesPviIiIH-Ua3n*0g$`QfFNr{JF?vFt3e>o@qkt@Y_AczP~c z*VXy6-KAaD$jK}Gi{#Yw#Wn`{dCAS|I!9%JT}`k|FaDJtwYZPHv2j4ez=)qLo_xQU za83|u7pn1|G-`?0P#Q4bB{9;~+9ctdeZTI_UDU`O@i~>U@|^d`&oaV5)iKqK^6IW; z#gK@683I$K9$D2ds)-T;a!aAj=N)rGt8RQvYpAMf^fC3-E}$Dwj6Nq&@uN3DRL->c zHvh7z)Jb9N1r~2*UWUKIt=6y{RlNxv!Ky6M$})x-oB3O<>D)Q**>KRZIHX!!EvbL5 zqgkhfU`%M9n-P8Bf$6K&ia5=ygDu{B4o;dI`z%1vhrgv~YOM7l9_4(9{Uux_dN;egpQT>ZW% zi?tg*{A2oh8EC5O?Oa-nNW9hES8}|8fp6&~tH({h@h|XsU zpJv_Yy~Gf<1?-ILJJVY6CNgsqfzRa9 z(nvQhbTsbswpWtluC^{Xv)`2mcBk>y>0?{WZ^YKiOADQ^`&}RHiewHRnSlaG`rstT zP`4AY40mjtpxuB2LxUG-!Yefm$ziS;7P23JFw3_`azdDh$fNtqSa z7EDdm=!U(EEHrY_24`6)DN+FJdNdR1&c?-8PR(i;W&BycFoS{$Zf`Y&D_rp!&Yc23 z5#l#Wp{k(@H)UIu>zXtR9dp z6VXT%woCk{n;H*b@Xovd^lI!?RX1taZz%XzU%$l?lddf+9RWcZIL;Th8pEhxB0+ED z+awqA>B_KY$>p&|+AmMc4HIwk^@TVVvQQl)?iB+Td1`OMR9*`|HHcouBGW+_aLtf?gG2?BvrrSkR2K% zB=_)`GG*?b%AY7^ub2}VzxhV>=Ih6(a<_Z$4V4;B_L;*Z32dm$tR!U; zaBja-9$77PibHWyTRiSw$RN8~))clnI5<4o4U1p(Evwns-1MksceNgG`8Xky*Cd%5 zYp*e7nS4{df70&)ZFSPB_AglI1o-!7Pnb=^o@<5P6Hir6v{7ZsQg3AVI++%3NeZ(V z7KWiCh86M0S~tvA4Q|g30#g^__T{hUi4BO1S{Wz^w`&_^-7kCTeJSOYiteQH z{EV>hPtt<3E1~ZuioXOBJmwM!nW&=Z`{D}g#b;qdnlwSxiA}~cMueJcvPGk>Dw&vc zXvPr@Qi8HEO+zxp{I|tY{$=8P(la18Cz%+ct8ErDNj`;~ynG8ol#S>_F2&^#98ZY- z9J_-4BrpT?buNK8<&}Ctn0BI@Zl8Md7i_{h380tI=7Y4?94Q4yTsz0R#Wa5UFhz(! z@~~ruOm6*3Iyw8bktICdGA#z&M@Ctfct3vOWr(J}y3sYlVJsKG;%77K?g|Qxy2WednOA{fm=Arc* z>5Ak>AKI~fafK|ag5rk8+8u+#Md_$#y=CbKZgFSe?_OSEADWk2u0!u6Nnj*VYbO-t zNNoZg0hD`wIXi7LPE6c?0A!-LFcP> zjV0>t240Fy*`sIjpF=g?e#kE%^ofC&`D=QeRLphihDO=2OO1Tf%WEjh-dJlEJ9OWkQTZ`^)avTBWB3O(jO0|lVtKdJYtRRCa2>8g-h`D%U%CG>kQ z9B&PSHAJNYV?^6^L&sna%93K2DhCe;$$2ldF-db|emD%W&og8v1-RXM2BbyI8J?RN zDT-dbvgt>{Xmc$MS3Ar|&s)w4_1z{X-*04+kLy?&Z_^-aHv{Tof>a|JZQ}bI?JYfG zWf(L_dSabcCmxvQE9+`S=Ql95{mXrr^f860R)ortobuM};H)L3)nTQH)oOzWu5^Bk zTjW*Q^P!LVp?Pw%F4#LL&?C-(3&WGF+8*cv0DZ1Cn{98O?TdnwTe-OG!*{KXIcik0 z$_k^CTBlrU+lzBU~iF#7cky-sx zM%$7Km-oF0%S~3k-jGo83d#b5*5F;c=d&?nXs@N-<)b~ z6{*)B$^Imz?(+kC(OfCa38&8htwxr(pNRl%oWTZ9t|prlcrKQid^(ojnbJ||Hijbc zkxjnv7S-DLFUAPPDY(QPm`GeQ8O=z$9!hr;?2XyyRtdZEdMLSm_W8XrRp%i(HVMY| z49+vfnHZW0cd=cO28mEbHFv>f&Ur(;Q{toD1S;5&J+piV~E zjtTli<{$V=BI-rp2Rh*`Nd=z@oGX^)kg6a=qpB6Evewq=a%ip6>6W>3sc@W}Dp9@v zf)4!%;i}3Z8DyiYcLg%Hi(6@dhb?=GEEBTy9rhhulTEZ)l{u$Zv|o19oL{Ifa+=qajT; z3jV_ibGW0Apcg&*W0A>h+-Q|T92uLa&%ucawguK0A|)^Y$&F-c*Ia5_6E97K`uUdW z;u8Y5RJUTDsMzAQ4F=oEN1r@zLDC)QYVvaxtHP%z4VJ3+_3C`INn9QmaD<7vAEf#Vq>|H`#)Vr8^QXEs(t&^tlAZ%atcui5F3rdBNyZ`pQ$Iy{Oj2LW zUNp-m?lLaDQCj^`gWdNdA{ru@O(A&G43ntaZM9f7@n7tdI^6@34z&*3j@@qxTXhlz zP=$7Won9R zTMDqXkd&XvE6-DG!ti~3nr7ex`*W*`YzPaW#cy-TE?};c^m6Bl)_^g8nI9a`T!j+t ziKNaGjMqd`8wcGRww|u}s19-ZI_W&1iG3eWp?Mqwzj$X_#GxU`=*qrQgdsjK$+})u z5#aRKneb@~Y`0d0P)VhFxnxkTP*ab&^A7sq4%xZP4ul*E`|!^QOW{k}TNP|zk|JUJ z$``}MOmUecQ4D{>%jKdjvE2lK6t!!)#kWf?p^==3?C&MOSS+#$21>~=*h2&8-sHae zVqQ5F6k^tCW#ZEwzdOJtW>q4&e5X%Y{Ysd2;G7nI@_k2cluThtXevMeEy0CTHtBK& zb1=1#Vv*K$D-nD^eibSS=4*!LBkGPnbA@SsHkF`5dL-SJI8R9kBV*BqT=%uG&`h~2 zkXxRyP<=N>PL~`_*p~9;ikagtit2d2xggiRY@?Jc9d_cExun>aPIPe<9zLf+@1XJz z9TTWC(QX94%`2nB^nI`cmB}pXozblpve$4=LaSlLA{zc1 z0QmaOIIdzrA2OXE9fN!7YeBz12OAYVEnGyRTT0TU)uF3wn@u3B^MZj5^*Lwl_U386 zHJw~V(eo;P@9Xv8I5H3}sl+T9`H^D3BcoLOvg1MxvlxQ(oN}GoWE7`Uu9|3-snC1=1UsctTzBIH!^yL=7H~%|RLJTx8?=Lp(4Yt0hkp^O8ahek4|4?mgmoE0mns;1WTcBBVsbKgf$*Ta2&2ig(~$KV~@a>%Ug?n>?=}B zF3AFtuXKiEj>;MM_)aYig}Sl`&>Orp8)T7pOxihqkz))8SCMp^f5UP_bt8Fq(bz=98i!@beOrSnQtWQC-`0 zYsT}FQO^!p$?(I@Fc`#C-c-$y=%@Ya%X;7-sAc-mf_mTSGw;H4LeGN}y1Han>1fYL zbB#lEP*@Sr6qz?SL%OTvDD$5R?>o<~(LXhg3+~FfXdwErtp@HigI)iKzzCD_v9$xL z&Cafj*IQi%ef$eJS8FFmd#I=)=CG9k8Skvn&^v)7CR9V!0Gs8-?&MUe!f)!>=_B2v z7v#?^;{k~k#>2pHp5NEs@9xLXh&ex0*LTDtnbuXyzxdRcY@n(ymql4vEY5`1hkp7Q z=I3{PU`9c0WPA!3LRI61)ooIDpD{X&F0`w{v5hde=*^})^5SBr5(Ly8eYBXF5zW=3 zA)q>aRlfj9-|`m7ubq63+&qpqw4rs2hfepu+5hzON4}rS@tMe&37++W`fYx5>iv55K#gQ7u3u6k>jLS@@P$~zhr*rMl(A9qxS0&e||vtmO8cJ z_ln$vJn{y*D7pjzSISN?d%}OtObK0Av=8i0*;s>4%1ZmW;r>lt7b= zK#|R(bf?-xq(SlXJ4s?9e?^Lzj|URxodPbl_66ZZ>iiFNT5Rr{Vc_M)YAn~12H(h3 z4UHy>$93MD*Vfh5jM6!Xo_scm@GK4BQR-#k!}m8xA}FDhk}`5B*47JduHi)c1M2iM zB2^cHItiQ6Mdk9*N6M5TgFbi(zEYVqw$QWFt9pm4527E|;r7Uf09&=puL*GG*hUsI_BMe9elAr>W7Zw%il_t|GHuhn#^76s;C}vPQ_qwUF-TT>U)XPx36s=76$o^9_%!P_>&{Y2`Vl z+d2XgCT3pAX)Pz?+%}CY%|j&xXLXlHIJ9FrM#V%4mh-Oh*0d!<>zaKeH6L`4WoXn4 z#$^wki>hrQL$Q?JPjO7AHk5~ygbmk*-GB0-1ZT5m-H}_)n6ku|iX6N_@Ow=$KneLi z%~r)W+yT?6pL5^fb23C?qzfBPPPW0`xI4)oyCRKraL@@oM1N=+HPPWwv92?#`ZNr6HuZ|q`HPoy?)*ro+981& zqgt%>{B%(@sk-!1bEQ4ny zvs@QHC34&jvR4XcjW^axW7`B8^Lz3A^>d3&lN#4-B86j&8d4M4a5uqGk*gnG%dTSq z*+Ao&zvHR{HXUPWTQ45v!G_lkPt4Vt8oA5>XK$?Qf~&yX9!l`YpGJBOL{Ku%ZZ zM(k@*Ce9-I9pXb6TzB>jg}?WP@BlCzTNv^<)1K3LY*C#n6g^nBdSc`qyfZlyx3(&D zoC08AWG^KN=~<$LTOc-XI^m&?MPtv4TQX9b z(MBDXc|;>fUZ7zj;Tf6wcS?v^Huf|y>Jcr5}X5}Ms%68tD$6*dm3i;Fq z#s`5PcsU8+xQ6yeMf_GLo#c>&=y#(ge4iLRj}yNC=|x@VLr^8j`anFClkk0B2?NwQ z_u8klaN$~g#LwWWGF#ZAVlbzzFTh_HliJzBah^=|wb~(0KN(|e;OEf;ZA6t8#lEgs z9Ra5~nQ{{<9v}XL6>pAj3DH!SM!T zJ`c9|^d)71p>r!uVGXuhYz8a1cvwsFiCl)03&|e0;Z9#DZU0Q6iPvbRlmjL<3_J_- z-Kdo3?bq19n3b)A=Hf~uq*F-12m`$d8|aI%B+)wlsyLN-hS?S zkwZhXGw)VA9!pxBj$J=3o=6#Jy3Wp|$`%YJ;4E9d2KANAEL%|(9I0E4Cori0Mp773 zwp2FN#BMx|Vacs1JaYC2>VoTDZ(V1;mnSi)$)jzF9ipmqbltn5lsC(C(cg5+T@rzk z0>g_%vPI;;GmAO$rDLs|B8j3))jG%QxMG_r0FBn(uL0M=bd0jsHZ(8KdXvci z^ltzf%F-mVOYO|4i;gX32V%)TQE!hDDqq}bCpattX$Z_M96sxg|CtdVtUR-eos8&W zZl;7WZYyJORZu!9DC!q8%m;jrL5hY-auW+YXf}eD1Kt{_o~li#3%7%Wa=@NoJKSnBUwnOjBI*`?&5q||;0xpKa&UBMaR7wGotPYmOJq(o24a8Ze`% zY`LrxUA??BdEVaR|7P`Ino3iU?{xe`!0wED3AE7F=oNSK$gk=TGC zNn+jH2>;cTAD<_=XF&N4o=(rnR}>m`GdJsPnv_eS3j!kns+kHjr$WGM)`3f)IQ%gX zLc@{dN7JF@ph$DG-f`am+EqP0(F<}>YCf6)vk$3kUY*;1kXb2smE0ajeLuscd?k+P zeHf3yYQoye9-~Z`PzBByH*!G9 z))ais*i;>t_MzW>)M4AAJyX>6+H7b9U3jODbROt8i0Q6F(q~+}c8kkg`b|}K=>#mT z8bMy_^hH%2ok9YU5jq-|=aC^w=Qr#(x+@+_C4MPT#go#4R}ZTCe2na6v17Yqotw@c z7PYE`wWk0k_~-TQ((;7=^uLF8-fN4yCV(ByR7Gn{><|c1iPDaA{|re(jx!H@S};!Q zILB?gxoN>MQJA@#Omz@0~QS{qZUv14Si{os;!chfpoLmL9Nu3PraZ)KR<`Hg>-ne{ba zM|oU@RbJYk$;F50K0Pr6A_FLHX#(zVMr+Y-3U7@EH&Vf+#J_FwJBG!1vd{H2Cx17j7Tbs+L|$_q{i&~|vtE2sh}tPC z*TcWtA{reYt1>cp6Gs=sCK)NGo`Mt01I`NRhC4SW zn3*Ysth?n$6#hF|#pxK$KN3lBa3CuzK&ag!hFk%&`n1;cOflDJOoL*5|JxFg_WPO& znk^Q)pD`D4jzbyaT`>+y=1N_j%-C1+D3RM zjuYitOfwD(%_FaN>jUxL7e<$Mg@j@-$x1JF`09I_Ve`rIDagiI!%r9Qk1$du3+kQI z*=>vR(rpsNqEGDDI;iQnxb@>>&uzWa-TYy-V~cMLW!W zrzr!`6f7wn-dOYyzY1oEZvF8gyZ|GWn=OW--)(ahi5Hfnu2~K=^et=9oK9mnKu7gY zuGWpBc|fh>7*p6cC{SV63a$yWD^GA%^D*iIRoPb`MV6}vs}D_+#ZJX_%+W3KR|Hki z(XKgELku6~3lCnBL*y%z`|Ku)IAboHD^C*RE))d!Yk^4c^WQ6taw5q(L&;$p{x3vC z&&KBzD$0MVIEqK_HC9#dl_PiOZrN8bRZ!fbP(M4_eBl4grvZ6}7DDO8yY-hW&VbnuJ zJd`By(M}9mo(rCNF;(Ugj^@Dhh0#tTI|tMn$(#nTjfJvik#(0hzi&veIW4dx@c!~z zchEvZguy`Zr1)S`v%e3f-ak4atC{P4xY)Ct96j~tJZt@YSK%q0kry`Ps?s*_bC6`f z%T~8qY!IfGY5F3`&o&i*09R{FG~1O$a>~;vv;Wkl+sc;r^-PcX>PyJ7{im_2nLtKi zcC0YZ4>Te<$|O7VnvSoOMZAA4Q#Gn2HGld0CnPa5j2S!U&rS1vDD-7}aRmZmHHNZ@ zIz9aRrN!C1_a0c~tZLD36E5)Q8%|?V3m1mP{a&v(A8`K6SW{hg_}uhWV+Y>ll?xTg|89;V_S<` zyY!pVmRW30(}ELE7qtSdIJ8Qj>IHuYr5>dhL>g!U58wI_bL@q&(Rm)S*u*rm?%5VV>44BVElI!=B!;O}f0{WJ5eHh*KCP;p9JJv8VKM&WC3J zIEZ76fk20o9=r|!hwW?a%kmekh!#~SX6ZAoPS{ zZ-CeGwV8n}dCRlvrej}F*vHl0UrG<3lZKZIdc`2|x+z|$9 zwjt7dXZ)(?9x9ithvZaVk-R>ABl5Oln9-dkZ{eZTlgc+wX<$n{0&MRBl6GtE1!xp$ zFfEqHG&G9B?@4d9h3vEtsTNc+E*ly9p-b88Y z?dQK#z4C0}YU`g*wAb>nzZ_4|EBHg7cA@p1SL zF|>_^ON3ksFA>5ofE;!d2eL#%=Tym5J{7pksj`qo7QG38R--{n+*-mi zt!$%bThV?v9|_G(nX3!b-y( zcdD{f&KG*_Xp+rQ6cZ1j&utnAd$#33JCk~aW@3E_eRNrLMH29rSAZI@We{RdRke9+ zz&Uc2aeZLIkR8tz8fbq@x=aC#Gj>L**ko)661GEFyUq6j-R9$y$m-WN_%{_jv zz{pT6eu!>e3P@NDB&JMm{T;}>m}nR4{rgPg^HdsaiLzu7j`$FK{1q)yjbbWC+92y| z+CHtSv^bQzcj&DDith5N`!kgX3Uj%^NpAVC-cp+P?~Ci0Tt8DFHBPreCcq>!liyfX z#G$t6da!_iHSuYCQJbb;zg6=GqZy@#+KJUfEgH#6EesIl>8cZCZ}NVwEJM{P`W&TYD%;X}AG1A|%t0Aa!l=JH0L$UUeKYn! z=g~7;7QkQRUPPwZw_N_@3Ds+Vvb@5^$1~<#{Q|BWx*Kn(hhwkEY`gfx)NhC@HLBT_ zl$-a`)g-dKjZDiDUy}r?+l5N=;}tW4s!R>uq%m%$+t?a~Or;6;h{SPdjmlrm$qxh% zihB!f674UH1jmPj^WumlH%>@qGO6vHSr53ZaA|ZBrGn3(#Xa80GC!?bkcLEnN2;Pf zN|lFPW8WeF=2f4Rh3tl4PL-5YAnwPDz$iz~=v-C|vwF7H3rnX`?OU>X?VtMls!XuQ zI@4f9!n;k(=$UzBf(9o z2K1uDsIx7|NwSNy2iw2Fk@vKS*QXSWyD5Q|=v#PSWjDF=W%BV|2cdR;Jrlw}NK@SVr=;69o)plaPqPg`7nZ?JEqD;jhF60Hne*urY%?%(483nJD0S`u7- zYL~45QgfiEE3Jt8<_fhc*kOqKEe+wQQzgRYDiA-)qt;TggiJ+*bG@iZnZaG$y`)bC zOzMj^L9nf}M*ZlT)~Zp1xb!(&A|aIKDT2n%8^UYf!ZFr?o9!K5&UyrqY<^4=xpFHb z;fV3!QjE*!pKbP@#RhTMAlfEy;L3CiqU3v@+>&_tu_k9Ye--j3-8=cR zsD@Y`g@c`Nj@BnzPQLl_hXL7#DQqS*tDU0k?xiu2oQw>YkzbX!x^}0{fkt>;o+Se6<|!JWlvd=j+?vagYk(u(XL8rxyN-fe>-p(jnvLXE3xho!g zFg-h7nt`cUi=NMYnq$KaAS^ss4v|-gJ-Gta?d|oaTLdsbs|qTXc52VtuZJ zm!hLoNv9p!NJr^xE*LY24Ix(Iq;(#i?D$qXiEzne98NoN>f-FTk7k+SUpMQgvtKD4n#l=)IIME~ zgROBxX?Kdl`$o`Ae?L#wC~^#aZ%MkCDcUy}Y)FXrx3W?_Ox`6`uQ1vx4%3gb7g&c$zqK);p+SZS?w zjoo5v(J^~o-WY}%!{;KY$>}SR3|(m&izL|vgDd3jlu0Vfm5G>+UJ;%)m%Ha1wwekm zyR`ryD~<8~yEP3%T?zUn>Jbb~Bp zs--=Y_Nsu{dPBUj;*;S_eS~%z`P1hl3g2{JvDZYjvV`)n#VV8G_)W-newjryMvhe3 zOe9AZMDo5$JnUEj{PGVB4<$-htTYpdrR1N<6|b;rfP=?AlBIRy0iJ2d-$-y}y{2z& zU$w~Sx>3naFyt>U?*6}utK4mcQ= zhDHHFx27)z##TiISxgB}rC86UBx*ErPg)~2wrZv3RxN={;YCWYlO@%Nm)VAgH5VjiaxjK9=n{aESPOxZ7 zC!Lx3l^yA>tuwPS*B<+Xz8G5>AioS!bIqsVQU-MF7dk7p5xagmQhdCgz9>go(KBGq zum*5ysO&kLQB?|<0`an!MMty_PUfr*-Mw@>5vMPzt&@@g&_(260YiG-q_T^)^NOXpR#WUj5PJ)^lU8!M4zC#Q4N(X2qr|8UUPM1 zl&=V@*M}PL(`Lnr5wErZgLKk3sok#EXXJ6(eM-Ucg7t0&&1L6dPb&q6*aij1CHma#6P^H?m;&6 z?S$`%_odmi^VVnNDf-eE6-mSXa0oa5Yw*d!hu9&{F3A#}_V9)s@&yVJgvHX=V0LaR znd6O~A3ZoXcMc%|vIXPulhwA0$gAFfpG)Lx%pjRaig!%u_a&yK1 zz!9q4=;hl#dUAi=YDJlPA|3j=j}Zksw`%0=mqu5V>Aru_Nzy>rE-e&{EL6itzyb$< z0uL5Tla;O|-t6|T$_VG3ZvrvC7x@UK*1j+Kq;-QEA=cqRa#PB)ZEj&IN5!NO;h0nO zU_otsvTtRi$>D~YTSf@`>eY5IeAt)TEDFAN|FK-;cttn<1J})Mp@*Hk49Min2v`() z&PZQmB%etbZ2|19WUh;zXQuBt!Nac`PTk7^SVnUEaC(916I9Q@H zexDUv9a5d7QvOfrw0h$q=jWmWB2{e8+hIpr3@m`ceqot%rG)bre>XwZWfi<#KJ}s& zYMuKOYRDUw0I-B`+Kn{8bIgky2yE^9M|86MbA4w^-MK0vh0I-@Uxp)KY_p9nH%ki0Z2*a-ehSR`um5m7N1~Q#ku$=lbBG$Z z=Ciuy+o4ZJ{6bA@4>cja04@bVw8^^IL=q9Pu0P{gpt0W0;mf4+lO2;=rbQ_g+0RkT zs~W=4Qs1B*q47+~2h^-tLu&GOr`4V0|10mo_vY6sWcfrSW21$IA;9>X7#0tU^eomB z(UhVlCnY7D3c~L72`oSxv!?p2MS~mV?h|R}0s@S$RQ(--|1dI;Xb_0XM<`62OtopD&b zu#ZOb*^S&`7#oHFkemf4cZxy&jh7x*I=eBv+x=_eh253^BLMaPkq6o<`$6H>Y@z-u zhr>$Lz|@zEf2Pj=6rK1I!NP&rkGbo`j=#=@ASWRD=n?kLU#%D49ecO>b2E?SWUa=Q zU{SU^nW2(iu%nnbiA%R6Pk|aDiF6{)xF}Upo_-M#=uUmz8H~W!fVz10{zQ0wj)Sx5 zq4ozyoDgrJU9%$%To9IEgPXVPSLTOeYZUc`FC(Rm`XgFH)1Iv{fkM0;jnTX5z~YA9 zY8wO14B9|rxzzFy5<0Gvq6skY+G*_p{wf&#?-RzwgITANkoK2$N36hc+RL`j8YB@W zzu0_e6WhFqSzb8c$$T6cb%~rEd%7lWPP)ie^#gpR$pW{{P*t4fgdKN6N7nd+Ir&6j zE?4UchDM~bBP6`!RH~yybZYJ~$J+r}o>Ift-qqJ%3^~LX;_UF;j|a$WloY#9DTE|U z4LQdZ(G%oNmm9@-)~`7@w1Iw=9cU)Kv|Wi(B{~i;mL&UIJEG+8~P-{ z+Fv~Efr}VXjYIN{$))me1%gN3h)59aDzvVvS z<%}fN*{w^K5xDv71%841%U&Uw?AvGUOtf8H7NKWTwgrPzR)L4gva@K2a1?}eFp7m8`|b(Nt#Lq*Z~ud1w zs`S;>5pwpl%=L~s-&_-|KW|>x*_4g(8@^N?izYHUvWh%+YntPhGJ;yIppX0WVr{1h zk%_@~j>o;eB(sJ+_YRDXH$4T~dYi`zY2FK>#KaAJmfY&+6bG3;ehT>aPAl`Yd^0X= z`*+v^&z`)4OS`+>?ccY8P=xw?|AHP=mT!h$gFk85>xPjPRr)0w{VZBpq*Z&2{IVS4 zRg}iv8%-QK9$|g(+jM`$7X%dFgAuJi`tUM5@!x1cM=S+f6x#WexS`j0jh)Mu_fa4% zMNWJ<8h%8QC=$CbJ5g`p|y*iT17P_$qapXNeFB3%i>P` z$3g~M@faP3q*fC?U5TN$vxd)s2lkKrq>t{1a<-hW6)#9BblR*LGCg+T^ly@0deGej zB5&w+#Mp-cdEy+)ujE|L2Evus|3F@Z^>1eRoep{&jD|P7B`z%;M=t4dZLe$vyoUfy z$AybdCuz0sK1BfG9csvYo=lfm7e`r<8kk9Zy@{W5AkcRvbc*q;jm zMvw{n;Bmh^VdyjVcR&MUwKY+wi!BWW6oajh*!HGdouDHO8LY-q#$%||*yEz)KN2*J zOy+6v1(ju`eC_}ZQ~XQ%-|W!QTj+8P)$sHBkg!y#yl)(>h=Rg%HSliN4v_B+)BSsN zcUQdDu7k^(cfCOL-m*jN!-%n90rW58;vb?-^ixVj14`(oAt%;wWQxxI;V@Yk9X(?cZGrB*Q~4 z{s=7VEcf3M(+bF}ssymrSh^$9B+vx8Ur&k{v z{;Jo2i}|?4CBVjjWW~&v>u_U^%>01y;(3Y?0n~h}IJwTp6_F3qWX90jTR>tDg(We} zZUDGEt1%QUlgr|hv%7~s8__S`0LF3T_EtB4hXY#y5ER^WbYxW=1NG!S1;Zt&dr-pqA5T+y@} zw%&BSFyyoy(;#!!JlAmPC2)ROWa%_db$dANb=t_t(*5%EfaQHQRo{iF3|(h+$w?zh zgj_RpEUzH^@x3KE?mYXX@6kK${#Y>k@w&P5i)Gm>{aHT~JsJC@_ZIrj6cw%JJfr-j z05q=~i`KnDOwBDhZyp9rEunJ~gXSGReU~jW_rv+5v%$`&-bag*UP#ea&lEWML+_6A zN;GS@=h^_bVYj@Seq>o?)B6{XBLo3Yu~&Ig%Dwj}kfq z3$%CCJ`qz`09>v$(}#~uWl{8GNk%G}p1%rEJL|aZCEut?X&7}|cNrd&LolO#)uY?e zpOgrFBW@kLB}0oVTaoRfjqc=#@oGkdpn9o6Q|i11=t=F{2pF8yrTvyOb@~X<%~!2V z-&b`D!nfEN+4h1}A8*_96${@&hfeRp5!_ETP=m9wzWFkve{<{5194kR7_>ucP8T4b z#^~25Yl0vDiHotYRF=<&;K8K$QBVoGyMjegrd#_rT#fu>-092sd#S0FUT`&aEJbV2 zsQf1TJBDAtZ%&)v|1^J-BZr=v(-R}h{Ujr+C|x6bT-Z<+MUk~n;Dr)3;LRQ5kMrz^ zGkTnewpe4#&qdkbM_Tj*H%qemDsP1>_={|Cz4>dQOmk>_n*Fa>MfebH3|qY!NtF96 zZ@$spRU)a6H3y2|oY%phyTxVuNmWH9I#KWwk9x+!RUIX)-$bt%gzRK8b|0`Ntfd5_ zoXvZqOJ{CK<4lV7+DF$+8QtpJrW-^x((&>)cYHA^_#J6EA=RrX zEHfp;!@c?C2phH5T;ijJCMcDYB0^^fc=80ceq-zuSUcOVQ~IP{@#9xsrmsM-r6U9kx^pprc5xP?4q@lIC#pjXZ9{pFbL(aBh6-NqTce=BfaeE*l6u zVp)D^mCvZpuB(uTP`aZIihT<+JfxLbknf2d(72otb@1oLq+74x)>iI~v8;untlFN2Uvn7E zmitA3oNW{W38uNw-!;9vQ{u}1&~>_kv6X0aoIHCLEUX_m8EhC$Oy-58#?`#*d^>PE zB>2*;_0mr0^T~EMGI*^c$)Fb}3EjWbkw(aS@2x&~kIeCTq|(T^8uf{8#crnV&6}yb zQ)}d^J8z+rQFn(>$o^O!Gr!u!yt5YPjDJ#j~8dc<=jsK1c0+WGiQ^astyh&%SB5^67wQ+%$qNkk3?L zeDT^7jT#+Vx0S9J=dNHnqx4 zRL=A*Pvb35+C33UWzWxaIZN}$K_c&}D(fpL9v$>?h4<4A<7lCH#B3HO1m@lwobWAb z-^>^E<5V1Ja)fmEl3A*t#LrAt4Tc;6FU%ZSK{{Gg80?`+x5tt0=oB#@nH z2Ms8aKRfj)a6j}@6`;)e6J4d!K<|%9%G@n}llbKl+pY2ITqVnUH8n-x^rkzN59gFk zzBr>(oA>cVDdAVQ*M*T9WP#sxf*06&pfA_L~er{JP6h z*GR8LWU(50zFDQ7mPu>gzut8k-m?VEhoIAlv`@Xl_O@<@Q=5(9w;)QyFB_*oWN5Gz zQF-$b1ZXjK7wtd<2CwZN)6eOCj))=YQMjbie=lLC4YQ!04s@v7Dkg!a*?g*F9a24S z^d+%`AUm8)EF5cE{pxw!rh)!uCZ|h9jN?ZUCS!Bp*r_u|lGzg6llG6VJ+PMbv;d0^ z)^mIo94k-0q+y(^EZlve#TYT+6DO1TfH5N(aih|c5Tm}Gtz-kEwfwx-$i>L5b@oB) zVw{+M2 zGWp?%2`+i&A{r*R&4F93&rQK)$SY;u9uT_FTnm`J2T+!~Jt=q>D z{Il&g5v zEgz-o?98B((X=_*H{y(|L9%(mUE-MS!RNmCP(Pa}sZ`^hGqYTqRt9jMCNzzI8xWr*%8TsgVK|Y&{ns8%84D^^d(-M_SexN7+i`Ky^GRKJ0$p??e*`$Oh2Fj^_`imp$4#`JM3KlCpuCmDwtN=MHKQ5QYLq*zfh z_egl!4#p2G;r?pm<&C`v{|9Xq%MYta$wT~Cktvo?mR*{i$lz8MF*Nw#lF z)E#mX1UH@=&-|)9>tyZhZ=!dJDQfKHYk!H^r`N zx#pb%q!T9EyKyVY=>#haJrZO^kw)v-FtDBGcCH;CC`%4e<2SSfuvjVE&SCTfT?KTw z(mvnxeuyCA2b7f~68QEo^AydS4===g!%^N1=V1M;^(`gOqG#8KDat+zG$y_DF7~ru z{vdMV2%7QZFJyC=ZNFQU*dB*0Xf-(sr{$r|_-6&&w~HPEZa%bH7s~sF*lW=W9q@AL zA3$6=u|8SfFgnpa8J0*QmJL~{iXWGa&6`^M#a)msW$MhkVpInVlj~2gqlk<|57BFI z4!ZHM4*I1a1>DuNf^0D!n#pq`*ZZxxWOm@SFcsIpjpUH6Wu&6Mw+~8?>!mDw`+-o| zTt!fM101Nt!dNmiD*QP$tVfq%CGQi2;op7De0^Xt66a9Oa%o!eddQZ~GlvGZ>-59+ z6l<4?A?`_GCB_K&%LT`YRMjI{6!lL%L&0-Z-_|JGlPBc{m4n|4D}3Vsj|UZ~bg3&~ zac(h*@xcv@)0cbWbSO-_#_AvtU7PyqJ7<01!h(DrmZ2P@>XS>I^j*7;`4S1w)X0d3_ z(`#uNcD196h1OJ8gbyA4-qY0fK+eAkA1p|F^=5xR(XuP@Z-f#OQf$xvD!QXw{P11a z5T%TQqTdTQzCShPJ@{-q>F+tPvjhHdcZlzojEK28(QCAczr(@;X21xI#*90hUd8dZVD*>VE}4pEBUaY!3R~?Pr&5NlUS5d$p8@2bu`z%#m9w`K9P)E z8zKeRfmmHvN9=f58A|C2{6 zwjM(KfB~G$-qStAhl$|!GRm;-fQS8lTQ@1y4u1#3jnb}maDrg~#&T@gJLd5+ZZ;b{~`b z@7#w?DzB3QlmEz(UqJufqcBibBLB&_{LjSt|EgExzh^7#LjF%KV{Hd%Umq#KtoH+k zZZ)CTcF>TDx_*$%;SI$fl-w#K{)SD5-b6cu4-A;S?@2txwhHDNz-gOi#0~G`vz=F` zJ-W0Pcc9~)zP=dIbGeW)?-)$$Z&^MA5tv? z4?HDeUqP0{wse;>GN-*CFWpIMS~<60XEe|$(@vL;Fpbn@y zEqYjcUg#I@VnAs$p1gMEWh@Jw`}nCgCZE5R^`EpO?0)C2a{f26iH z$4Xf++U8+8dSp4F)$BD(>m}RN%jA&z`EXx#Gm_X_DWPYAKJ)ZQx|g|!Y=Iv3WOkyY zn7hZ_gaW;kt1ri?=Sy5ZmpX!R||~lm1poamQ8RC`l<>mhl`W5?;q1nwoS4|KWz@-fBK& z1~{tYqdGu@Y%rti>pns+VzoD2yUC)2V?_CSyRWaBwPPWtQd5c#kHB&}N7Brvn*^R_ zh*zEE41rzibuU5^Fu=ml9XaVzCi(|q_x7|+m9-bZd(OfVFSF(8?PD=|k36g3x0lvU z4z#@qG{rPd@k-JD4c5k$aPDU>_9(5`RztO4enAq{h!`8STo6e-y_qjy1FWSvhunZs zuu20j8D4eP-$ku{^Hk$!>6gfJ_v;xM^76hzyV2|OAhT{*bRVaolJRA-8}(jMH)Se%FPnAo*^dv1#Km{Nv%T3&+hpt&Pe*(S0qs88)fU z5}VH_1|%+n%{+(2#Ao~Q_581CT(?{!yr53{j@N8Osw&DJcdUw*6LF?_(gn_tf`hT;NcZg);k9I((O=+|+j$VW=Sq2@ zhTW$J23wBFwH|=IN#;B13ZME#X zi)TrDQY?PoxlJ_l+;4Pll*ERsba-G_5c&dyv`K-Os;YjbMti@soVH8wjppd_p`+t^ zUxv<4e082?3W$H40Yx-I)e zSG+7Cky!t>kB=xZk@_MfOPB8A&aNIbMT^TLpTv|L^^)(7WB+=CVd9Y6B10(zLl5i z@R#mOSZ~6)&&y;DK`@b`D%#s~zjeT0`(+Z8?bdHDqs)QDFw(8oxa@~t6#VhW#%?>{)$JzI(1eVjK3|Ji>>H z(9_Z1W-Jx{wVzE{Z{+!1*h!D>7qU0;WGn(ReFgA?}7Yzm{n3U9%{~BiP_?3JY{?Y8G zswuz;v7-GJ4ej8&{~gt8FXQTt$(joM*he-^>)x_cVfrD_*+&3(h8I&5jHi8+mTx>1 z5qs-y#xTlLPxfib{>LZphoz&Yt@?pt;al7#$E{wTW-WHxOipapX#J=O<}#J^X;eB) zD|8K8y~L-GKeSpK#cG(+aS|so{J7f@{YW@rj+-UYdCj`!NyamC4j(rb>2F>4G%xwK z1@v9-jV-l=@yyLwNZYT>AKktOn$WDy0^L#Wrz*R?{Soe;OOMpJmI=0M zv#;9>Pa{FE`73^F(f;Iuvf^>f_IT7W%jPS|_9^YAN5|q)%-S}=@Tcqb`^y(A z-(zUP-$33s{6F6~=dDF4%T6OIc2(v}FCXcJxs*$W`{z_laxnQ3q`73jucn=LTCDcY zjN-5T&^q~*(2p*1BHH5!g?(K@o}a1qHgLW&?^oo>pi^C4j19!5kV5Q!*B4mdg>Zq0$3aCy7L?a{1#bA zf(+U{H|oMGVfwj{zh9cr&nJfwkrX#-^g^tkgFin8rF*ewwqAWfI~XTTnSL#xaSuGj z)L*D2ft8%mF1hbOPA^-j*v#hph)5syN{hxQYQfx>dwCNx80g;>$@RZEWO}`4kHhFk ztrYx5{D;J!wEbbNO?c(o&GQB1cjncnOp5{2#BwQPI+0iulidM_QOT?j*`w z!5?%$-YP#OM?7Rg&|aRQ0|XO|#7gbEo^8(y%$8M~E^R!XoYrl<25oP7Vf4pq8~n6t zRvLn<;ka84@7F87>u^i%*PkzATX-(cJJ*+I?Zx`qqR)1B22N#8x2omk2OduU+yiU!t-&9cg6Efg9x1bpv*49 zJ$Z)ZJ%nxo!AbVbxc!$WVWR!%5v`;Ul1B93{dCYqitbMK&jkb5Efw-3GhK^rkDmZS zd2d`$C(dqKNxJ}Ancq6>h^a(fxo@CFr{m}E-8ZNEogNDMjlWJoL-C*ARtosA?{Be) z7+BG4Ko_I(C?Ny~Av5 z*yW-{!k6x*KWC>IZX033D-prMDZ#3@>)hzPv(^ItRFZHhlt{9dZljvEo7k)HX%}U=W9yJaAmj zgf4`sWc?8jSF7yO0jPet$bB7&w3ePnyzwxureYw-7G~C6uNIq9oi*LzAtaQp8NpuB zhMQ5^Uc*u{AKP?(*^qp7^=7j_YD)$JXh-iC)}QZw9~qBH znq3=>fNc}s^=;9(qz($vzqq(gH`&lxrZpPH;Y&;z@xQe+5JoLIgUc`yN__uqtXu`f z*~O*W^qIu_>ddQYC+j8Hg{2KK+P4#c{@o{}-P}$~mR{!6p{GUz4|ZI{Uy2F$cZ9#( zQ<)671l2jrz79RmG}h241mZ{Pas~-syC89XAF&XrT?Y6q`KTuBNK|>+3F+3`MIv$v zF0$EjpsNra*NCZpslBW!vR>p7yeS& z{G#uzRVrqoRZore%F5edV?sXf6|d

iN(jDSnW!>LmMEz6)gOatv}hcgKA8G?Zlm zv~^Mh&G(Jj&3O?>ng@$JdLt^Zekj1EwkpqLSr*AHG@eL{hBiE8{_J6W#9U=%kSL=l zg8KDm{Va|pTPy2l70XL~tw+zy1~cC>-5QgGeW|^cy1C02ph6BbVJ{7ppmcon{S}zPG8;2r7T^*_Cb%Im4F8tf>T9P75lp zxrjN!mVQa>CcmmU%T4f}n(^=$=@19ggB1B!VE@@Ph)JXHld#aqWotGe?RT}wv8!)G zXDdZyYXz+={eMS6ytfp2kS0gdho(>quZRmkm`Ep~h^e`LlG#FE1nT(B}IHqT0B#p#qKC)!-aUfhx z-HR+mxP;l0oB3C+hy0qLXie&LoCSIMG75mG90W@%HYg%|&O|;h4}A$ZE)t zGO@qX$b^cTw|x-@3FPeTx4|u5CP6hVm&2dOAN;xCb60O+-yMD!MY+GcnKz{FB*<0R zVi(#`d}t#CI$j~+4c{>Vql>>e`q@js-fonXZKLYUmmc=x1N~zA_3WRlfS-cxoh$}& zNSAgSM&tQboKXt;Tpkz=8sDZxx&i8A3X`SLbBjRLA{NVtt6O8aM^r!=uJJ^9{i)l5 z-}`D_2`{t!^$?fJetA}b(3cem`>1f+cI>ae_g$SEP$iNPyMQa5<;wK&9=vq3rCP^x zXiKs36wZLS5%CN@ zue=`Wv=iW9A{raEK}}Az9Q`=W=!U4KkMAjllkWSBrF}i1cWY|ktKT#vIo_u1cw%A0 zwt09lM#0AA)rq$T4L<1~5-#`CINCB*lEhdctwy%56tdhIkriv_Ku^=GSi*`?^W!&k zl}`6jwTSRJLXQ9f-|OWz&7Jm?xqE+kLznJ^O?1L}n&2N-$%B5?pwy?g%DlYwWNPoo zEOfVlR6bX8khZo_;K~bdGhD*U~{%WrEL7u zocCsf5=T)FB&OvewDM#uyYz6XAJ6oNdjqL?-Eo_6Q%pMA>`ZXp;fv#-xKMS#Tjz*N zdh7UFj`B>D7eGiD`g^ql?oIis+K$tJHziMhCu7 z_i%z<6{i~>tQ&5p`(;Kif%nzO?SoaumpocVwu7O{D)M?^7T(9pEGs>&7QioTu4$X~PEprAV7{-DHSb+*nqxY#9-9%XhfCy#Fy z{D# zu0z2>_q-BEPjAp?+>V&m5@wzDT~1AAH%{<%ug$8tIVynaCb2U=EffKf3KVZY-?Ts0 z(+=z9(6H0)wQFULzX)Yj<~g(gwMe!3Gqa^Ra$bsvHtfn!y|^4*W*DdDo>wjNEetcv zRu(;JXs+M8Y7Ya!ED~J&zK2m~cXCW4NU#3q=PM+A^P#f!V{u#?9>a%a;?IGIg5a~z z5!2cj)6Dqu{jrPB;L@~Qgvlc7`?i8X<;pU($~L9O%qcDyrpkBPep;~~RFbz8$j)Kh zC-y^y2H@dD-%s+VT_T_?|TQvxjoj_DE`x zNBI9m+k1z@*>!!xlR^>+DcXe~k*14i(Ss;aqRkM!#bC4~jNVcZ1QC6d5oL@SjOcxc zAX-H4L}x|`V=zYf=DP3jT+jU;$NPTA`#sPmReXhOtTKinPebq$f83g@2b0I{e{+QeF)8-uP>-pr>g2k`#a-W4WE*U4}lYSvZ&`MP%9a$H;F26 zCk46;*`6DE!u#?=-V5h)N=}wkS$>&raJ*_fn};4;DTSxlDeA5k9Np|_0}1%<;Pbns zZyt_1E(xfhQldO6KM(I>36;(n{- z!DOM~h%U2yot(kyl@>PWvQ^{|tuwE3d<~?a7hh(Oc$ZOv2G{B9_zH;daD+PPjZi^o zm?7R#+5*yRn7#QyS66P&GMWeYJbVCgh&?)g&)MBQ`e3fx9Qi~a>}Yg9!zI@&k1S-L z`*P~?N7o}csbbsanHG*{l-PRq857_DEVx9>ZCh_1A<>_6RHd6+rcO!H%akg2{T%Mn zbgRAN2WZ5npugT~KJs)F!+=C9ImrW5>bEVbm0qV7JP%t+bu1hTo^W&^;@*#JKSkM< zNZhK^Q-V6_KPBAL9$2weh76J(m5Gp9b)GpU%%hD{=teOfwIBK{49M)GCg%n}K&l<& zZ4MuhAfCO@iYa7Cb^qwE>)g-kMr()E&xvEM!4t3OAL5*-AQeDE0*CN<53x5=|K>{o zH%r@&h3xc;+bvqal`n6Ujnh&vo#95!rz}2=_i!%F)dUPwY0KpFl?U?c15B=7x^ua! z+U+z?;7;@(&IVYn$M_w^XVCeU{_6eWvzgwyrEld7mCr!a*#nj9Uqq=y|CQ|;$c`~? zo%p;cgwOt9rq4igBPxUL4+ZMf3hal%)?eP!7z*{QpV?cUg}2xrqWrvT&2xaL+}IyA zYPrxt0r{KkMIBfBPyPF@M{PHdX8=x&3haM+iTgm2uiNf_J^=uac>Zp1e5s#0X$S!5 z(45B|{{RTmd!%zb5&$6X^vT2k02vHdjvq&9o)dIJF#y0#bryQk#eY2NwFKhX{v7~7 zkH2N=;-5xE(q37*JndRWfJdgctq-k_?~QwiVQ`bM&C37;)ukMm+;BRS`^D+{=hud{ zX&#!8!HF26cikrJ0-&fOtm>uK*-g*1n4f28z2?{wboNLS^?9c3PH@f0N#k)5&s z&p;@wv8rX%oXvpkrgEv^$CkX$^*kH3$m-85h7qmbstgFGo7Kq(u$z z6)~~0L*w~Rp9r$Ii=T${9EI_kySa(^7nj8kFZ_ALFtpY@I&rb*6ku?JIN4} zH_F|E8+(2!{wnN^$D*`iY`#`=8t}<`Mm|>Qxci1eRatg`rO5fEKb>Y|d)s`m|C`a9 zGkb_l=6XiQlY8fF`0-uGLL{!hc-(uO%zqOxFXhIw0LzO=)E{9D`lO_M-1Og~8l4R^9b2@RNhrnq$bz z8(xE2V|*WHqOqs{%+Fb+>0Q1zQu~p^wV!={PfUTHG3Zr#_?r2Tenur%S=|KR_WiQw zzZDW9u*LT=p-|}u4$T3baa{{ClCLkkDCqkr@VOmQxm9Cpxxx^wuwU03C*fbTXYO7& z36DcS$k&mE0bWl+4L@M3jFegXj@T#skN8++OZg^IOI50PFooldr{G?-JKUC(FkGWl#FAEi>GkC3zlgC zDkkw_9mt65u5%cqJq@2=(lx^GNbVrK8qk1pq zHh9HkClQ(<6r*TAFnP7B)1V8U!&2gWC(Tf7TlR?sEoXl6Y+w)DVugV*Sk{d6O&TWd z@J^4oKdAciC`nH^oBC z!|G}6s>k~|hhF5(y{!mL=c~Qtz5uNKaH&Az24f*Cw^9BGsVzsY_Nucu$ey|@^mFfP zuNI>~N)9LX=qYE?Hm9GY4p!SlkV@tML4owCVz;#XS{ zVqefYF=lKBL+p`|cU;O2=Kavjc09fyy(SS{5vWJrVxXS64pmk~MN=-RBwzpHtubiq zx3~W1W_su%qaBX8amqH155s28hO8%#cn&U+`l9<*vHW_aulUje-1!R99-6+rC!lA} zeb>T=O3Z2GX)D|hB`8Y(+~H^zxBDqwJu})W`pUT)wi;MJNt$~w5P_|D)!ywhozuP5 zL^*^-DllI0YV^K?EeQQm1SyU{*@yIYeTz_;Y@AI0X@42jSDY0!jI9S>I(QV= zNzAh@DBPP$>_*xd?Atua{2U{d+uXb-xB>#JplZJ)aeY>tXJq1G$p0*jJy0APOf7Ki zmej>WT}0}YMfs2xEBevlq~DdE{fA$fOTCl}d9T09lk-{OEmP0GWrvvbA({Da%-)ZL zRjXAszL()~KpA23syRkXhD=|4GBya~Oj$BM*l+Xo*(u=^d%n+K&`I0w6e8o0j4IPU zQznyY3tzPCRO2?u#6Sz$p3-Kj_V1l~@w`Uunql_K{lY#+ z@%Nxz&PRR7_~v=*FKUmo&efv31CujAH{r#+2EciBBVLkC<4y(~9WX?rSXfAwcN4@xKJQ}Kg7Q#Cv0 zbqzjrJBSwz0o4S!jd+K{Jq-m*@*i{{j#de4)>S22U*GQFqK^Cvr*ipYdMQi#N*TmI zJJc>ozctSnjsIFR)$^gOune*PJBGvO_tDnC_mTE!>Nrr|Csfk5QJ%0svWz1AfX4&EH zJk)GhO20?#87f_StqPOz$*2l zCvG6reqzdV%`t}S(pqM)A^nIbRs3tawZ1li4qhBBw{6dPe|hke+Jg|M7lxu%8FFd0 zUj?7MHYVevLe~au>qgPK6CqL?#U@+(Vs*fqUAdwH0}VHu#B4y{{CGjmWYgS3xE#8s zTTYYRcu=>gJah@6x<0@{HW0bE{k`Nx%z^+frGlB!Eh0Y2F9+cjXRz}~c5mfY^~z5s z)KF>s)2fA2vfiM}pzvVEgRysAn5_r#@$t#L7zd}6-0TFyflVWnX;rP>D#)(&^EkCj zTw^BV-a#v8X#>5^4O>pLz3=+f`1~^i8472hvN%)Oqh}Un80&;Zo}b-tIz1OrNO{9O zO!7RDJWvE;8$cU?%GkvE6zYETA015DR4)uOneHF&n{f};25j&$lQZo?iGMZE@f$*g*baVv!Mzz*Wb z;oFRix4z%z+@hm%z-tqg?anqBOUWriAgFq~w-Ge4@(?rXj?dW^dqbl#TT%|04dA9* zDNkGuXx=n{41O5cB-et+D5r4Y4yt!iDegV-63K80{9Z10C_Jf^{q53+WY}!C-Xh)V zK&c;)%3E@8|3csPCY!s}l(c)7Su{R=z13k}97uhv(FA=`)(I`x1Y!zd9B@f}nN5D9 z!e-^hU1_|s_0@T1rckJywipPbClgqgMDly&aHz$(mS9xqw;$gsEAvvr%HZL3J$@#! z^)V|ReOpdK`6&^hzThx0sIf@VLxGAw9EUP5=cm92FS`2Y31P`kru>ew>X+r!W)tR< zL9p#|l;B~Nnr_USZBK&K$K>gSI|PzmJPf=IR=rk{b|oU1fsWDDk-uEgCgf6|5cZ%@ z;q7Yo&As{PFUA~s8SO^%z8G_grS;D21!j3p`DkCB zbT9i+NrY;d)oF#@I%Waxn`fcJyxPPTh`vI&&;F(W3x>1BMM`GAI2r5MFACp3M9Q#4 z4j_&9BhN+~*8fzDOsW=@yLD6h%((;G$!%_?4F@=@QHtzM=Tu;x50s?WJHIpUxYy4E zEAwmmm}fb2Hd2UNH1d{u9!R;2H`{Tt!H}qS#M0h&5$(|(vDLgY<*@bh6sz>U#)O-i znK#$;Zf->$b@baLj4+o+_9 z@UPd^HQJ% zn7Ao=UESmbsY+tY1q)%ews_YuGj_-4?J1u!8M6Wh^}Fi}u8Jg}@5f>CZUyw$X$NM` zdn-`gvh6QB2tZj#D!b)A)&FMqbiX!k z82D0Xs30Qm`ZcloAWulldeWGRFlfK|ONcnNC^s0VW7!9(zs67|ZGkA3a$n8v)KQ!M zE>pMlQ)F*0x5v0rJm2yMB$BgNJ6e7tBp{{A!wmYX{esd~yBh{q_xhHY&6C%>{ixih z=iEWj{^(?Jxtv`C$LUQ!=Q!mPDVlhnK^NGBKBHY{o?F%pH2;~ExqJq??(DfV!{_po zlsJ{m=NbUy+%GdeRQoJLd%1_!&K(HZk7fIf;C`?uA?w$=TD??^hc?Dn;Hq1(gEKFl zHTq#j*~d&mD^PPV7mw#{n+5K(jCH_?)Gu=%bM;!1tj<)0lKie}k*CGd{4`ymQu}@rd!z7nf(pYWCd@5{u*+G= zpYoc74x(M8>q=1Bq`5E#NDK1w-gctAsIBcI0hp*^8n4{^0wb!3J9`OC-%QMm`Mq~= z-WgcK6u}Dw?wecb>co62Ua4nJ_Fixc22auBxlJk}4$0qErSkmQC0tI3C@e0ep)yhZ1Tz7o)w*=!hohsxHO7-Hz{i zFCJH5DEQE9hn9lE>e@_@8V{syxW-NH`c)<{J&w~WMEG3y%zU~&0&KIRfQ0h6WM;X+ z{{0}FxSU?a_<%;+TWco4E=j$i(dZv@vu9ud^vvLe1kT|BaaX zzog70Nmsqwfdh^A(hADa_5Fk?Z9{Vf5c|}QVp>m*xgLbQXI#F1r~`hKZ`2Vn_e9Gu zw$R^wKBjEzAmyl!*~Sg@#n{mdI>iVv5kHL4dMvhD(Y>`2ArEK9_}yDdzciZ`sHiG(&E455^~2`%}Y&}MoIN07(s_O&5O{<%*o?hfV_4D$rLJv26T zxo>|yIJ`OggRy|SCfueUJt1ez8LLtT!KwKV9^H#ftyc3^1}6w*gwq=1=jfvhcfdx? zT|r&%lj$9`wv8zXwcd5+JAHI$SW>y+4cl0uV?brtk?P-ILL71~qcFxL@aO_@)mY+? zBS`ii0BGu5M$~7aFKh>0&ti_(5a4_DjZ|Dl($_{VMg|60p4v3l7kOe6HkiN5pu5AZ zOe=LAX|BqSm~3?iWvR1lkWOy@#R3Hc(NfHkKi_am%48kmwiFc9_5>>XKLD<^o8K%> z0j?}={7yVpMB_v;)*FTZz+YdUb!suaKE{0FuDgn|nt3WQ0RZHk<2^HD&bx^MxLtK= zKMe?{dV6yx7IH$7QDJ5w_g^25XaK`0%3N2ex~49)-DLl-{w`sr-&9+^FQNl%gFXJ5 z+kc~|+@($Sd`r7@?Iyd`@nctfCckwJ)zBAX{uj5%+z_9oteo+*mM{{y1_{K;05l+3Uq?!@bzcPgF))McFjNS1Y4Z=}f9Zl6s%_w9~6 zF&A(E3atJ1S6A)Uu>49#c7~vp;XC*L8eV>df{r1>rDk!uY12oB`)`6MkZF$T6}ys$ z1TpMO?h3S`RKyATJ~(z88G-dUzd%`}Qog3D=O;|~H|o6@d1Tr;iPZW}OxQ_}0{%7> zK0W~#^D74a0aF6*M4vq1e-H2f4`AI5`joNGU>GBs=Lp{Ogilka8YaI6?1bLZRn8Lj zu1nt?H@o+r%!2mPv2*PmO)23Ca6z1InB--RWmY zy9g+fcooRJ>t7rwbS%>9>4xEqm!VY=eB6|^mJ!eAd)*^+Y*D97mvv|Jh1p%BVD8EL z0lfKtez5fX?ekde9N?L&q5O&b7kN+aY9DtC0MuMP;nx3a7fS;@ovO9mRAC3+T3Dpn z;>>Vd>(lMamY10>1U{3P^&+x?s!w3VFV6%4ZIS!vgvGj&Qi(TEiZ>U`-7-R zGJM<9Qv30VYv+zlA~jU}V+Y?y%4_h96mM`)fyVP;mOMQJ#N&|Hvj0WujAXi^-HH)X(zzXvapkA~5(cZBBV11U@Lu;myXOG+9W z{CGM6+q}|@{UbMpoW-tCyjQcEG&x^&_L-08tY~O>Iz#YU49(jO3&2W|eW$60GUdNN zSA)4t=vyi2@f`m__U@cB9-pd`KBcoAL;8U2YtkJ3&EfX|($S8`Uk256Kb6gLG2BcN zxR9-$2j0}nsTd=j(KCjTmLs~4I|e)o8@=^L3NsS^M$GC!E<@)akOl(MDK<)@UDrFU z*PwY;?G88Vz?PLVZV02M&!lR$7#n}Q69eM%doSahxEZ1xw%4il_G-8ehOs(R2Up!0 z&C$8k#2&P_&vd-sf%wgQhQNK1JKsaz`sypgFFsjhSdPG4Bz81#`5$V1E`Iw(PXo6i za(PwUd2>a{H{HqI$tmSqsffqGs}nlQL+@RW`sP=^+KpGVa+ux}2xXgO;PExqu?PSm zgGn5uS@iHPT~l3=fo(i4R$t8DGF|R3Yw)ft#OE1IP>MH$q(noU911qRPDJ5_m!%4P z!^0JuIVY})1B?$0>krp!-e0S!b@A;C$d2DZIWID-PH`#Q_R55htdbK;{6C3ET6EXa z>nVKC>JO14%E~J-o(3Oz?zr`J3je0gf=QRDHHCyukQl|sei5J8@_G+g2VzG>X_0;* zt*M?jBUhOFJ6Dn(;Hljoj-~{iMyVxI||Cr%!=ZY+91u zy2^Soka1ZpRpNQ1TQrcpPb}j-`|BR3w5j!%@$MaNYj1j1vCax{r|Qd0yPybF=jJM5OwSL|)LA}E92vq7J3rwGs81hb zBDh8D8vH0k*lG#Asg=qy zfO2y4>l8LF;YE@9zOw#)5>wB@p%-ohHullHn5!acrTJKYv6Vr%qpV}+j&s~97CPpX z8@bnPlR*FeTdAebrA#lGF!jXmAfML@nXgIBw2z8p};f9pK5t%Hnx%Uub+8JTiTzw_RJBd1PQJBPqt zmJVlte)b*RXerM4!r05)g-Ak@_?P`6#D{46#?7S8F`76c(hAS8s^5lA8Y%ArJsxSW zA9yb#djNXO(U;*88Z_F!Q9ab71|+ne6?0M8s27vx*Sg1kO~)Oo6G#`vJLFfcf)B#I zQ)HDELxgvofq=Tv3?lN7a@&RM_X{xR3n>SLY#(6VYi+Xlmd29-Bk}Q#Bx<*NDccuq zXh!SRHjpa*aAUFU4!epbadGl2;c=NgL`{F;8}NB~E8A}*%n0mO()p-1%`AOlcm%-- z7Kp%%ZE2}cVPK`d(Z+}&MuEMq;swo14lVn;QeBI^?cV(v$rfdNWQMlO%a}$mrm~FiK< zCH(n$BG$N9<{_+eA!0s?&t_GE%f32k!$v~y()#mSv}7%q>4W$toFYaRgZq?VauSF) z8IsiCt z=ERw|_%AlCm8Gdu-R`ZMIFv8Z*xKC&$IyQ-W`c?(MH5eWD;*75hWabE$ z$>n``Dm#q%{WT+Czgmr+X&BewWp6v0Q1A=dIIXLio%e<$3*O@?y-R_{#(LbW+kb@d zI1S-NoI8BWv^yBs-85mvnpp2}#XVvNS0s#Mpnc`pOGPL2cLgNR5RmqIcmw(t|p2Ad5xs6-LB+*>#3P>d4 zzOg+RXk^nnS`QcXFa|o#95^)2tD14nN&TDl{77=ELkD#2l*#5{;3kLMGhQax|~_vRjG_qQ>a z{X>N1Pxtr9;o;dI+QE{xoory)>CB|em?RnV=+hoGy)x|<96iPa-?|pLfx;zSCucSJ zKm+7e5u0`iX>k_TPxawOr5v)n&ap{@eXX}y%d3O z=dFo3XTk1sAscO))CtO>Gh)@z2mH+oIgw66~n;}MaFtE z(hXS_9?tQH+cz|D2^iB;ip^nR{W~blhEzjg9nGXYhGx%GpZp657548b zPc|&Wp9AJN0b@hYi8-e)V>acW@;A+LiBI)&3s0?9V6=&jd%&tUhBr%^Y(iDa23OR* z302~>b(_ggQy&OtE02J8dS!9X<$e8NQWZOV==<3`Ze5a=%o|FD7S#?UZ&Ywt;AY(r~d@=b!Os2;8if>4{zB4S}&xFamt zMxtq}v35YuI6-@LEVuHr+^RRQbku2V3HfGW2Uc(f0{Jntd@DlKHkfw5bEER%vR9Ig zDcbyzQC@l%T^nS?vPjcspV{*Am671Q%_yMrAH1gl3)(ywn()}^r+aCX#Hp{)8)soD zp~42_b&hOh>nSx_I3sNG`d&dQ)8G&%Bkrx|MX?u;AA5uA{ePHDZXy#HlZQ-8`oiIvV{rt!RM9bub+c>)yF)`H4Ot0%7a4PoO0ok12@&Dw!3SN`?+96} zTMqSddl`@50p>{Ih|Y3@`(un7C>MWtdlFRTRTisQ(W7;&-Dr|Ohok^viyzbVS4I1p zr%Hl0xqPF;aa-Uyy4jdCe&u8Nwc?+h1g4g_x(5guLXG?{5FsiLWmzs_A1X<^J5X9@ z0qP1D-|;JluJ|VNNX=xsr+Jy8=!U1V)aPwzQsj*9OQJaIxqqHpU&QCe+zjq4&RTd_ zV3O)>6#L;(x6-HFf5Y&|+=uXg&s!Az!b0Y;PZ^()nt& zcGWszR?OciTGgoJ!wYjDHi^*25#7DA@r(;3+G=w^eGKOT?03#W#aB?{x0RVKEDQ`y z+2MseXvf0uIt1M0%wm7@n2VXF@=Zb$^$U`)7CU`!k(EN^b&md`&sGBFcwKc=!Q_4X zeo9fZoSoB1+{X|)-ripd3bgDTJ-wzPQ-1YCg*+)_fyH5@ILcZ~deC~GSsBFe0V`Uf z-?b^Tv9odre(p6cC9_aQOI#_LSh2GS;IX{xTI6~5J!NaeFSak}dn2ec{5MN6F}`hgaG3E$Js5VMV;^6JjR zHIcFBJ3L^I%=_{>GfWzp2sz|cG%K~J}l zr|up7bwoIl^;-Xj_|eUu2rzs7*P?(uehbb(BjjhL2+{Gaso(N%SVfxhZ`9L<0X%$f zyB@WWt?5!&-9=kx&th@U3*9&Mn9b~FeAR!?@%+(4p7yw;i`KfG}*wZGf9}o5c&(ho{n-&xIQATxM+)A!b zHQf?a8J~RV_KDTYY>p3{E^4K6TwV?s$b*dfSZ~fhAtXKq&BgXQnP>we7OBhZ9e|&c zRE?6&4ZkF7>kdX^vhDQ9httEtp_!Vd|RS4Smn? zgwOQVyK#ElTJURwyLmTrcDo+1A#fkiBGAP;L|2-v`+{&l4s}jW>{nKa?);&ZCIiB) zKIX71y(3Fd(XK3rm>yx&EwF?!S8SMltoRw6@026R_6Meo)JZjr|B``RN`xRlzIo9U8xVYg zw{-=Cx9NqF59ZyYllRisJi9A3CjH17){+TR0!>cn5p%YdsvEtXm+e%`N|C!sRQtjO zsVtzvWcZTA*j*fDBN=lybNAo(8mFAMnTNzv?Fqx5=~J$! z7P<-XWD$hK`@C)Y$wi_A#SXC}2<(_3+2i0~ALRn)V+vp1;iM6>f1yJ;JPz`^BA8W5 z43(*-84n?WDiRFKj7gx}R8*K}7^PKhC% zM}xDC0;!3sLdvYJ+pd&T7QN>2W1QioU-N|ol#QjT9$EMSGd`CaJDEP~TToX9cOUQw zyII;4o)!!Q$H*Psw!qTy^)7uubahhDkbMTXxz$Ds3DdbL=&-@?qhf=;pwTkXi^3fy zV)rtm&D;Afb}ueBAMy2DQ1)tkdCP31!@xVPIxD%IHbW}b2e*+i+sPqd)(7{IJQuB9 zzISZ7V^D@f*y=X7tfib?f=MNR_67o@5=jHmjT%|k=?~7AgR43$m_89Z_^ep69?Qk6 zKB+A-M(rlYmYzIRrE!Z@cszfn`{UqQElyi?Ml|fO^puqgMFr<^Z*}_i>ok#*18Qad z#vd^;6Sm;xmx^uNINux{>dZP}{OZy%G_fsJU$a(C=2)kWw;pSuV^503XoUbYUz}um-f;03oh(yA#48F1L{bVaFwtPj0QVJtT-MXU8(n_rfk>uIw z(QnV9lZ7&G=4vLFynJYfF5etVT4lWo?Y2Po@>4XLu8U(oG*2Wn8XeEGptOnp_U9>RVL1$fOGT5@SJr?jyRksCXbHg<#R|^wHwf zCkZnu%kUbGVVdZ+9T(OWXvrd|JpHeQT?{4pVfxM~NxhDH(1zD}sMKpcNmE=8 zYBDxyPeaCj@>v7`_3yqbyl#Yn@4Vo!!U?X?G6X^>#{nX=Ygx<( z3g`6=RDh0#-{MTTgEg_>RU3IcXj)OEd+R01Ct!7|p~As}+$m0X0Wu@wCdeO^WWzcb z*9M+yO>yu6N-BfdmpP!4Tw3G-3AU?5lxEDx8jEo7Ma@8%++PwrnTl?jQevC-gZ)JY z`SHbxNwXqC`C(a#thU=@>A3A@Ll68XhC^N~FQnebr`VKt%SpTpS#G+3z=*)I?K$0C z;s~vxfhv#D7dwlYLMD7;?9o{U0!&}02v{A9SmZH3llG%wcZv-@_iQ{6B9>RhO2^7 z&h{547^&6P6~v-+q?KtMQ!Hq*+Dqn*!Br0PvXJqc8FH-%v;wr`ib)w0fx9HcB9hH( z?s99KiO#rqS-vW946)?ZS8gPL;j)VE<=Y!TCL6}{M zkt1or<1p5Ueq?y3Pu8}PCqk>*$g_lw2M2v#jRXGXDJfniZzU-zgC!A>t$u z9&Jtj7LvG7x!0=&Uz;_I5IR2x`gV_(86+-?KpV#0|ZoVR493Od10LL zQD34X7uXjXiSuPJWdS|?c?&a=9!~Vb1ypKN_1W;=Uv(_MzZfHl zbDeemLWbl72=rrV>sfH@NL#VV)pAcMao7fiBsp; z9v9FpXH^i0L#AX+@v6n*k;Pw(RI<&}DWt3?-M}=LyVB3+S{R|eB5$Nt!cN?*8P6#_ zOdP(W8-50)Ip>G-^FsNQ?x55W$;=zbqVKn&El|(K(CPQG@$c47k-At^DK5M}R}noU z)T2_tMEBf7j<)sKH&nv&^R45g>CJ~C@^=QZI*U0>yeb-hf8thAX5W5%k}~qBZTOCo z#pn%i8!wM_!08KZ-5zc><}K*O^1bnfy42OgSGL*V<=3tbE1jfH{kg?SNFb+@^}Jfq zqLL^poC|N^26Ia866rhLj5&64G)(^O@iGyMk8xqLOn7wu%*sQy-W>YlbQM6+(8zSg zk9M{Ac0>yEZD@l>$v~)FH1M-i?~BjJbRSJ2<(yVZ%$?|kyxA{Gf;^GpOHOZRAJ5)HiZjOZiB95`NTyKt9MENDhS!cm7mkx-j%okjWtjbb*A$b~UE$_!#?KT%$uWr&k8Y(8ESvo|X*`F! ze1Cd*M6t5j>2RN#B*~+be?_TeZAb!wCfe`ma}uzd`hVY6xq%C9!u-CleEkhCEMxG6~tcA#CRMPLix_)`u}(L_y5}r z!~Y(FAVW#sTp%3yn=?l=k26H7Fw%ui;J3E6c5B95+FN&)q9iA_mIHm35|4a!rrup1 zsxyRwj0lr67E?M?s*^)AY6*J&93uA&_ObVr8QDq)!*aKKdeUG>#RdULcadN1n(vH8 znG7DnZef?ch9%3c`p&<_^$jz-QhfgKgD636?vGRJiQS8_t+PfF&JmW}*8|H4r~iW` zC(J3pYxiJ#S@0S~@hu;mV;bTA5yk-9k%@#7(<<;`=hpjYFOTQl-C0htE8I+io6P@aE$P!d`6zJw!RqVCQ%-06_x7s|snYv++YI$fe(v zN1wb!P-v|h@ljW-0c5DKj65>`bF%ti?Huv&Xd>&tt}YAB;ioNfafXu{%26V3UShE^ z)p_V_(@1mD<+gQ=EHa`wGyJhLZJNb=M-HEW0I!InEIHsgfzgdI#iLI)`aqR2G%#Nx zFo_+`mRO@^QNM}&A-S%X=JkWIdamA+*t);Jn{vUCQf|N#zih#jXSsG?_El?_6sc1K zy4hTTOLo17yrDaqm~WpG;J3q*=GJH{wUOW9KJwPp@muX&T#?s9of-}EK&tqT@T7wX z>~ZcJ0N~WbgbuJ&6|d5-+~AA6u7 z3%?kS{g=t3>;b7i)+Uq~6Atq0bItN|15NUYRYq!ry9YT!lm6yI;~ODNAgA0CZE15^ zHqzF7lRvbj=}i3fRplYgp<9Ng876XRh`ltrzL9&J`QZ@(y(H!dBy$Kqze^k}Ps-UO z?`f5&cP$IbEEo1*C);l6SOx)>(Q_9St zGcR-YDc9>xbCyYMolUhu_T0Pe!0e$K}CT=k`3C&we>3b?9 z2TdCV@w(h&T#A<9?2ord>~X}5XP3wCQ)1tW7K^5OsHmtm$p@PxzfCV0Ji<9<|9)Q@ zquR9ZwLkoEjiTGEZHHHzHNs(tuluLx8J@(nP2mY6V$inWx=)Hyd>M3hra4)TKL%PY zwDewXm;n`BS<2*B&Zd22=ry9HlcvNaQ=DA7=x{rJwmi?n_qcwvpcc(?zB^P7e~Gvi z*&BK*v32oe0s&)-oGksp1VWs-xRQ}=PV?;DC!iFEsTQ_u_m>0x1Ak39)a~M#5p&eU; zCa$=#%jUi4%jTE}O0kFXNJpd}I7CnXFox8%T}QD| zm5&y8kn;;U6_R@Q`crcgr}h>2xLgcOfwNdY-GhZe5Pa6ySwUYw0*Q@fL@gwX@y>7I{#WC!-8MdVhYr-$V z5U)VI+`iMNyI^95C5xRyb20MERm>KHN0Y2KrmTi=&3GR)*m%QE=5TYg!aG65Z#O@k z6i_`!gb-K4e0C_u%6{_|fv%+O;J$9h&nlRu2axCZ3hcd-q8v+JMozHO`W#6YKwiF! znyR3I*R-9m$x-&RcaE;fPDH*osF)M?O>=3JET zEX|W{j7fpSnwQSSYSRMg=sY%fV$oyFgekr)5YewP;dXSCekI&l^y$GD+T z;Y8E%MNp~(d{Cu6S-<~@MF0Q(8a~l*0RPuD1*94Jy=Br*R5=Q<($=}WxXd2=eMW$# z_O-CbyQ;@=@I|3xg&994x3n}9>aJb_43~Vf77lKjrmZibpBno!%ploTI>Y{vl={y) z-zP3n16Ej^)aT$^ZIQQ9{@393bbOuCg6A`fcMc z>ZyUYsi~6=|Ab^Yci>kde|>w77F&6KMv5#V+(-6`kU7m$En> zW@YaG8MSAokQRR)cdgOXT75tL{L*C~m@e~9L0k!J5VVhe?~ zQJz#|`07SQ&d}J~6E#=8MUWELD$YE73gCf>JX6I^2@F12%Z%qMy_A-}*Og9K>F@9k zw8UN1F8~0q%YLjf)`Uv1M*b=A@bFsXNfvnAZA?Mx`&ar*0Lw}&edd1JAEE#?Z--_# zxUWU>QbOG9C~B>Wc=p9)?8TLnk`Y0l7Rv&MQ;I!yJ2EZRVz^(k16H)Ioif`zqj~&N zxYJDuB;5#-qD1ulx6%l0%JhYq$EFSd(6FX_LjC_tiHrYn&OZSS{tt2FCkykx3YoU< zeTE3E0Y8(zC@ZLc|7?*rM|7(x|8`&@eVkl0$<=N|e{^=E9X~;`b$7RpqiW z{CsSRa(h#<>=S1Qe9hX;X8By7|AZpigtOp;<2H<$0F@fP{l{$6`-m zu6XP1jmH2$rPck2qht3_oT%&my<-lgn2nT7rISzpDtS`G{QF;8(Z=zP0Py|0V#;nJ z!KaQ@*!mqp8@_g0`p-QL>3>_khdDAF-PIpm2J?+rr9JY&okBTF^5WFHKPWx$}^z8@Z9S0p;lM;mzmqT(j zzIS!#D=17?Tdj01@W={)Y&#NOVr_Ozs(4Kp+wH2|3bP+`T`lNp0}b!MrV6^x z7!UHwCQpBUMB{ohEUrluFETLVgm-=IU@T7a3AFza-uhZ}EK21jrI2O-OEKz%R z?k9fwG*d2wRwV6?&pcX~ZiV+nVCoBagx zv_-%?XyxwiDrq#EkSZf?9&z;47r<)7@7P9;aMTE0b#u83pAM5no1sH$nA|#{q8?wL zdj^boZ2Q;zKg_**IF#$#KCabDtqNwq8D zjSg;rS9)m`Yg2vGJVDv{&R>ztSJZpGy{e~A2kViN{V2(AGy@vdn|RiI5dWv#h;iA& zX98bj2N-`bT^}#g)WF-vBr7s!UD$2Uvp=Nj=qtSI4BVZef7q>lycLsOPIP1-Hj_aSq8$ojMBk0v$r(u0X6E{OsxF`BO9BMNC;U%- zrZ-C(9h#r_pCbjHEX$7((ZReC6_5cVZ~FzB4{ap6iupOcH*sj_V#7>?iQ=%abM$J~ zdZud(`8d=6Hq=vfU_4SBv%V2)1E)UqY-jTFKDP2q@p|ptP#WR}Cqn_$--~5p-APJ9 z!`~R54wb&~ez^uQg?Wf?+F}N30q^r83IT0D$DQI*oo^Q$H_h_^HOB!7Ve=d~9~a6Q*Y0a7ER+Jh&+Ujqe5t0a#akP@3wk`clV3bu=AgJ3snY z-aZ`GPj;+Bc?EC)5%Vd1+x!oU+Am72Z(726eZw`CWLRRx2m+#Fu=n2bi6nA?6J08= z(o=}O6C2@wvOCWZe@Yd#!La@~g1E&=;f9Hx6LQ@%^t?N@dh!fnP!nYko3~9F>JEi( z@15B+|MqKsC?Wkc#D876jEthx-MozYaB>dhDT;BeT~g_{NDvSb>E$(g28p(E%IM1>FP^ zUEg58`{YQOZGA+%YN%AQl7r&-4&t9mMy-`!%MPaqiJxn*i8%J( zS@WYP^{LW= zmthyl`xBgFl5!6Qfc-kFGx6f_EoA#09)wGp@bbpe?q+wbSQ<&MQjtMYf9X;qV!SHa!LuRk4P4`S9)#zNPzYG=x^ zp{T9cAuOfzu%UlT1med4cY``e% z;hg$qZJ(6~*c>xlb6nu|7{#(Z8coe*%eFY4qDkJn?1iZ7sU^LEP``^|KxkoqEwa!N zZKBt6f)aYqZ>gx+R@+(_3ZEGmsq-QS@kf8N!Ppg17@N1-^P=*?F?IQzK$H?z{uxQgK*9%rf-wKiPzTlgaokk5Y?LA<;5n2Rv3%W-yJy zDBLvK8w`d|T(|Xw!T23a#)kzdobSfxNza;|TwP}yMo7l^1CcA*wz5-->(%mSNYg*n>n!~n@R;TpbVmiE${Ey)e7>tW-Jnq|Wz779 zavrQXGgmQsLyw~TOTX8Z=6hevbd0g0Y2AF9{nM|*BR*Jy(u2zuMq9`OYIZCERUj&G zj-hC8)jKexR`te%4c>5XnwpU-O%zzjt~hD=LJlVfj&K!qA6}gLgHQZsF4pq;5b2+V zR_r?4z8-7Qs3jIxM7Cv#Z#|>7e+XZ|YBZ8RpR&{r8n7a6i~<%hEhI;Y^KK*8rR2Y5 zuv52)t8xBK!ryw{XqyyP%$MB0?%HM^!u*`Q5bf9d@|A*4zvS&HWzx`d>aupg#N_Ij zf+y<4o;`bd!Z#=Iu*s&`jf(az>%;b;xjGN9qJWfq!xT<{Uh)o( zdXDzw;%=uH@?-GKW@R0R);jd>6Bu!Q&98d>y;T$Nr@=Fhbq0vGvQ@_)3829I`0qnh z76!-Ivg%4l2e#;emgk5(I|m$n2zUmpf{n+C;$~)5i@ArJ+ey@AJ-`AGJ03F?&fl7Z zK;YE1o8L;jsLy1xF1Tko%r3N;_;e14)Do|(** z0S!KzQ6-lMU#_p@W~}hr1Ligj0<3yi~CW4QFyloUY;31I0flKdSr6Xu?*47$9 zpZ{*Wyt7ME)u7Y5P^|57Iq0JBGt56??i+Iqzb{#T;~rnMVy2rL-5I1_fNlAi_rayb zWS;QI2xZ;FikRM{kx86oWeC{Cj*@#Qka@ujO2(P8=9xeFxGbe2^iRz&B6}D7wLP?&3(vY^ltuv-+@kzbpc)zi5>}~K&=O$Ek2FSFt zizwrp*X!5T^!r!VJQs+p^`%~i@;kCH1p(WrZwH-_N^&i~J18fxa`sK;o98V3xXT+; zY1eOi<}2yCENRIFk6dI`rzTsL2t@UP#jf$<{lA!Kxp#)>ZsejX!q=!K{6$0D2VlKN zLa_KqY3s~$hsy7F#f1+y>`1h--EZZQc@97M7E69VwW&Y-b^Dl`r9P!|qq?yDUB_VB z%a=Tr1|SvEfrQMMMQmhdjKK!|(x}~bFo2vD(&m*e~X`T;eq_Ayi5s>b7oAZc`t4_le{if4b@w_?jSA1&R2% zF-X041$ zM(Vg#lJc-FucR(Tgf0~G==$@E#?k|JCH#K762Tqzfu6C&$1fjzb=&?D4!-YU{l~8- zqlvM8HRHb!QwA{GL4z1Z46C@=D5}_0AA_;sST_-7*iw-pI{2A5cx0jc$d4&cF*w1y4zn9L(x5U!r7i)+p$(|cJrC@&QR{$Wg>59NU(8WUgU6x z{y6=VgZP#y0GY|<@xM# zR)}lp2MdCl!7&XZi#xhciYbnJdU|U5$dElIMS`F94}qyXFgl$@Lka=(ru z2lcgmKQn)MMeqzMVfFtJk|-Zc@`2zZe0=~)IU_TD2+-etDwvH-y}tvjl`0i+Si7vGM2Tplae@CLFLH8Q?ikaS9wnC5U;jJrriFzi_1yeYL_<02 zi+MUaFRLl=B~6g)vb_YVP}>J0^i+wwQO|Xh4b`A@9cp7G*{`@5!k_p@E_u%lZkv~=zM1!+;Lz@7P%#;+g7j_$a*kBw?5sMz6t%RKasBh@a$?aO1udji~11|1=a?yuTgKFa@fJ|-h*MS4yrrqQlBAh#veEXbw7E+ zP$mSrQ%0fOl0+OM2E-TSXt~Fs`t&zm@XCXs>)lj!$12$eP11_Rm_~&H??DfwM-oe2 z;sFpLRt5`>j^f<+07>Y*QQ5giCi&!`5~l)J3y{UW6F>)|p3d#6N4Zj4(!6o<(<}Li zZ|LL;6Q^-JS95(ErgLa+c^xklC}8f&ni)8_Pg{KR9T=LQCCU$pUN6J>Nnt8RO@WZ> zckecC1@+;hBY`x7-dOHLyhdRJV`#~uP0{OTkB<^J)+6<2Kf^*k&C7W=P;DH{R=E%J z>6^4t=Pf1rl0Mw4)N*{7?&uQ5-wDz7s!npcC5Oj^f~Zw7rM=z|M5h zVk+-E4WZY0q-$A&;FMw{8qnG4L_6l!o7piqds3HT4fB8CJ@RW1Z==<|A*2=m(8nel z1WQ)uiIn)~M526}QXu$M=t$>QBp(`fL<7Hct;eD6ixfR#G~5R7y1_}r)Ca=q^-4rp zFdXFubH0tn(?w*U^lN;PiFz6wz#uPGBV_fPCqDWDo}`EWxH84Zjp67mg~{1xZj|J} z>K$-IYv}p&KzWuMe#u#PAt?+L%;n6uVvfqsjxPL67zcSemRU4yv_UN>y<9TVU6TSh zSNzxUkVp>Q3D@baX!&XMsH*assFCe zAYzJ6*RpT#Sok{pixSeY^sRoFAG5YyI@~*6u6kQnD>J{eNP6KE#!i@D)K&-I+HAir z_p259ZpYJ|t=!(*s97_*=Gb?%p>j$&bB0@@U*l8v;lsh&)<6W02eNfNwy`XE+K$FF)pyxMh7tiLLw=7hhs!($Dgy2A2B9k*)N zNmWNjrEHuhKV@D`%$UnyL_6a+Wo1h$sg;NprvoyV?BVmJI?T_U2Vqfa_Ep_EO|Wi3>saG)3!RJI z+K98EpR{J)z+xgFI}$LV49?~##i2?^my%NwgP+=B&IOeR_vgu_cEeqnQrZjDnUPSz z1`cr_bBX4WHqZR+n2B7B08^=c9hc^(>lBtr2rYXVbHI5|%eCjhA}>rnSLz)%LQ+a` zwwIWRW#XrBA@jjfhjc!vMXr}kdXCpGw{tM`@eWJRI$>POs;LVWB3OI1RWArmyI~Rb zKMLt^YfMMNj-R64-<7m_Lg{tiQu+N%XTt9D22b3pZl8IlNGHtKxk_gs)YGK^AZ5)) zY7OxYfUwhguG^3lch)__xi!gbSpo%&9bh8j)5K`hpL#t72^0=T=6j7lzN*jv>`Jb& zjFfv3C78hhrSIE4W4(G}x9{1KQuKP-8nLR^K(wD+=G3D$C<$W79GSlSi|9hbP_-3& z!G0sEeA?%fL%;AFQ(PUjrov^^)+8?=o`A%Y20#klpdaeR&YP)0@7}YAKT1>b?yLry zw(iOC=Mh5(VqY-gQwBVp5%OD5<4P|8vD6-}_spAotl{A8rMji_D{PCs>Y5TLN;?A| zy~Wqex2t}Txu#Y;TH`bKdS5b_J4zE?sJLpH&guyR4YN3Yd?M#3G8$ruvc~jp5e3QV z>X5?jszdryL;Hr_+U=A3NcE&79no&vQQFSeA5*}bbyc1trZk0slZnatw~RKIvH_gu zy)OoDt7=`=M^K6`WRa0`ou<9u7+6`fqSnxkD2M@fq1L5iz)V@b36o|qR0A?#0J4|yuY42r_u0GQmizmY}`bN_X!?Z zKEm!pSiBKJW2mlSoc{QF*91t*W|4%0cLTAIJ-mPaQ=qJ`PqM%$92i*@{je<;+;8q% zzq1`@@L~HV+Hc29`eppj*(UCuOyqs*db3Yj1zP_hK#WWQ%kcq;&r~LwI`Jn?hx*Ya z64m!(8I{n9|3A5eA8-91_UZo{xut)(#P5>52md^b?@LN=(}K%P7G z?Xxr2DU|rWhUW(Oj6T}mB&yd-5T=%>)wGVgI`lHGWKW&~!8BjdxVt;LXrsZnn%o)` z4VogAaVWq3r4;Qii*J(IFTJpO^6IWF-ZmrTu77E&JO%S4zO?iJ2C0j(4+ zu0`=SAgM&_!oc&OFhfzJI|e;Ac420Qsd~V{$S6|^VO+R~4;5sM-O*X8WbcwR?(FGu zz;$2$>(8xgfB`cHzqGUjF`$7C?!||8O!Tk2OD57XI~r@3b^MvSlqx`2U;US7yVR3QjwaTdi4Xm(D{SdSnapp=TcG%a0R?liO}W7#-_D+S`V z{^L5XiE)`<9^;rh@X`>GJC^n_{$XqBE%bXvm zHn^x}aHI^TQE%O}p);iBNqiHnaN(AxQ(zjctyrY+K~tmQrt#M5ATisvxO?mS^5vN+ z^vm|(;t3BU`(C)HUxz_!W4R?{-(l7~)A>$GY((j|M*0wSBMRU-szaztaD=~zRpKlO z+;ms+5U|o-GXSc^hod1NE9WSBOzh@L1Xpk~AHmN}o`==ug->Y9eLSA-iG%U8{BlEi z?mmq%Dex|=Wo$ZRl3{l#90!Z}-7nCalmd`7@2rbSTwx;p1m)2`aqMBtI|cK4zk432 z8ypf{?Xa3tm^W5;Rvf)`;?iR=-I`ctUt{l?8~th>u~ghB+E75O&MuE)%v7I#>Tr=1 zNa}R2AwHjkMDx{Ks=*+2wsOPL-_F}APsN^n@Hno$jDN2GjU;I@-Rb1k6}MNR)VKRw;dUDy3TlJG0Y?3v~DkhQDK zxwkOVZ6~Ml@&hy564J~$(nU)0sTuFx0tt+b#pL2G(5BlBcrz}fg>5LCwVvd8i`r* za*{gUV;3iStlWN-=$m#?xUmSDp+FLg7OWYRL!L3fPr4l{t{ZJ^ppx~g<9#T|qZq)C z=x2jQ$H{Be(13tja*R5SXhtt>WOnqp|2yT!7e$TeYs(KB)}y~_?2bRCKl{b_S-!}F zmhH~De%mEhr^~wdui>U|cr`*&Ef>$#Y$9bhUEdykbRn^L{;Zzn!{VP;v6Z(8W#<(o9b!lee*B4}1zOL`91cDaOK>{V(0v ziLsUxv4tYG(NmG=tehWM4bhtd37pMehp^$k6QE3410-Z$CBo)i$TV%uodGTE*aK4m0my~$=gw2(32 zt_lWeTJ8*GPsb_4X)eP}s1cZ^ni@QV%%U#;&{5nzy$0IUVHnov4sC7tmPs23w74-J z@8CJC?3%!d`w*Y#&^e;f2LI{UZ;mrS%(9#vd?UZTcr~;M1V-$`@uhreo*u**Y&{E8y%{q4+zx45 z`FIbq8?!s^lYg|(KNs@V06w}`u3)5sGeugT|2)+L51wi5{BoV~hn2Yp2_o{V{?0`J z6Z0w;uQ(rz8@o^Y%6-wf-(Ti#*0Za{Wfo^4-CoilKPqXhB!c zDQj$m_WR;u6*aYf8CMmu(Z@nEEFKX?f0`_WxgXirv&n0`cqcTX5npso=S$WEJ+C8ssh zPFc3~B))O?@bpCaQLBnm9^2RZB&BCgRe3;@*IqrZpYotaR~2rUVwzAb5uub*hzReV zEI;(QRC?L;LQYM+pdS7kwoz+Fc7NRr`b?MF?zpjYVNVR>YIiTL*^Cb~(+TzWbQ&_5 z|4M~7!vRSjR`;Xj;9W4vLyC;H>;kWiWD5bF@@x}j803+MF6+8(0h_pTa)0o|sBsF2 zjr2P_%xDK^Q(ac_BIl)qoYMP>a`#7=x^{eFX%T>y_-uSGRgaR_4XTQY2I^_6J^*Pa zz%x`cN>`qTPy(PF5sXPXbXu%x{4o4HwzRu8z23hM{K`X$fQo*Q&B zZryXTe81Xm$r_IydrC1lvG_Z^1MiZuLHb?G`t8@?kVQ%{<)p?9b7P>MO(_;1vj8oWH&3M)e##y1%D{w>y#0m>8fga{*0LUeip0{D zw3?9l-IU>IqY-Y!w&|}f$N1upmm*d=2~02|*P zH`QJhzVtueyA{cWNUcNP^cpT(k<0G%a1$ZLQfIkK`JyHY-RO6((bTuEx$C({>DlHDAF%ey;i9R zGlC*VrE3BxqdizZ^4~<9c)5VV|B_0YUq@*S-icS( zA7isVhmGbH^EFe93X(;*JJHdP)ay1R!KGGrNQV5c_}y=wCT7&{ev5Fx1#ZhiU%pwl zyxC4)#^=Q>9X(DT43#yYMQo>yOE`O{2X#_2P!O6mo@<1F$fl@8gl|o0d`~Rq+E5yl zr_4Mgrb*swci(2(p7+$`J+q`nnCEpW1{+{xI zNNs1AH;e#(UTYn60hdi)MK@yeGe()^RDh2!RP1}z5CGIH1|%mW&b^7x)5dz}Ek z-g!&Q-n(VGyF%--T#xrJg{P0-fCquRd)wuFsVY_6nQ^by0xz9N6& zo+-?5gr|;H`xNSkYC|4PRA*wf7E2gox8H<~1?T=wP6*A1Hcc~`7@8a*7l3`N(t3o2 z=;CV8Sr5(?!<7*%e!Vj>q+p)Gsi_B&UmE${o$d4pOJ|$RL(&JRse@`^HYfkPuxB@> zgr!UU@k8>``n&KOnj3K~m-v#?)}zJcYT`IOs{r$4P1o0^U#3SEClQ-B=H(bH;lwfe z;wzoGDzct{`R>%9mapJ~jSD5Z2|+hQh@K-~#w3yFg7#^Rb>urR>Y9*xZ)5lwow3(Og`Ug4TmKsM&0zm_qY2`B>xN@}`FS z(!{Cpr2`;ExWJ&_;}ugSO+8!e)%d3M*`;Mv6{9KFspa(e&>H-|b`dJ@_d zrmD_6z}M*C$sUD}d(6q@=BTDR-LT*C{^Mh)tT#By53?|AeUyj3+|kn$M)H>5WO|A9 zBFYGNj6%n*=!U#U@2?aNaix}3H4L2jEOLK}(N?kU_*x$rb0nHGkDo(sc}+|nKOUW` zt9L;)Iy#R+f2V%(bVx4P*6Q84`9n)t+=>O|QL5)6)YLYXm};nd`Q_h{jfLog^d@d2 z&k^gAX*lC|4`-2^m31cknE^?~ruG1U%M0ksK zhVRf^koxUvnqP~cu;tpvp=uuqiJ-I_S1wtxmP~uehf0oMS|USj7K&sB6>yHI=jRSZ ztN|o{O#rHr19yv0DYHZsH?FOyA;cZoyhBlf%+%1ju4i_#V}g-|$G|>-<+%dN%Xr1< z;VnQiQCBfXmoWk~#Dxy&QeOy4eK(AwH*W4T5hK4Hiy8iGv~xaH=oCcCGkucLDjN^d z-6(_P<*qx0=Jo0=|E2q6y6%@$3Iqqy=ac~a&)ws&C{9o%+i$$OQX2FjJN>j>@kYZ> zayQEX1#+TC666mh9SS1os~u3(jSoth`lyP_vb2%k4e0N%t$W4$w4?MyofquqWb=&F9JD;AVsn)p7?*py77h9-CY=hN0lZvXt#sZk8nRIIbO$Lz=&wr%rUQJd7(jeXsK zwH^Y6wJ_>1gmw4~*F6G7EbkBtS~4Q0-j~3cJ(M{B_8(F@cCEg%;j`2#cc!)_PQs(= z(bx397}K?n9ugD$^9$WkPdLkkcY8<;kAK1`2*g(mM7Q)+rYnvM#&*3~?giza@#s}B z7<#L)*_l7nN!;_g+;UWCO4s{heeK9sere9gV@g9bS9$i@I(lFDamwmc1p& zd;mycvg8#5r#dglA0By@Z?tuoofmG?uy>xe5HsNHdcvzo2MzhqUlqO&m;0T{Lb+9P zIUuo5Ej4a`0(q2A6LNk7U0gh@NY8 z_2^RK6&{n(khe9I9j!NyR@C;KGFLf8uSlLb*M+><`po}LZ$P8y4nQ}C zQW}ZlaeL*x*C!(3UJPrpip}t+r=}q^!!7cb(*wGcoNtCFWveT|xJR&2YOn26cbgQK zlTMFufYvsx#%Fv;N|*^~bZRM13!!huuWX&Rt+QQ-S$Vo=H*}X*({Iow$LJolb?986 z(C{AZyUYq9L*0E@lwZmQjYUVQ%sARbH+1P*AE%}F3c^V)*4U#nu zy?()$S|!`r(L;8X886*VkTr@wH30zPgWp>#(tEP%Dl~wVD_KD>=q{Jb9(HY!Hi(&c z+p0fNy$2%O_g#dis;))_Bytsr)p^|_Rc#4niN!~sTUM;!3RMX53Pamvqe?b~uOg-MrNDkl?pd3q zI14Cl{6J3@%ZLJ*ja#D)H@4-q=IQ!Y!Cu?xwB)+dK1sF-vwbI0Nup99`~z@_B>>PC z-^!lIYYznm8UN8Be6H<~rY9Sy!Gk77r?!Y)xhaApRqSe#dDIfXu0z{S%e|uh+6UTK zUSHqa`TEqA@H4&dadV0QN zYaHDo!<`Vgc8?_Bm+%gd7asXngWWiqs6Zab#+!AR40EP@)hGU`tf|*BncD3mMq-shV9$ zY9?^=;=$+$Ss*}D`ITH7hi$7ySYBHai}oJB@B$T32T9C0ZMknysg{(k;as|7PsUtx zr2jCWXVaooYen{6&<(*DM%U&-gsWua`bq`)e$k|6xH;97Vq zkmZzB=}vK%D^$S9ugH#m}Z3^`IR_FYP;XoX@~lofULa#PP}l0btY#7OIz5glk22l6R3f3td$=6UNqmS`$u%i9f&JcrDUEb1*A+egvbHRT3~qgfpGx<2 zEbzd{8c=;}nU{9fxg2fJUJff=^UgmW#FMRF2o)WPgX zJV3ZeG%XQCtZdST3dPa9!`QAeyCS4rm@x#HSpp+(^zhHElpWX2GQdouk#dWwn> z1RbBD4~^H74;R@1qL7<7*W?RVWU@k&xXYuj%D12{@u{4CVj`g8W#?f$W2=y4WE~5u zX+A%-4*CsfXCLSTD#c%$cC3Aq67 zbhkwk&R;~}SobuI`cuBsfGOMYCmB(4Bb&7hGd`PJ6dBOek08k^%{0l(|ITStwh*KW zBE8Q0Jmm5871*_ZO5U&ef2}nNrNKkM#Jbl~ z@k!d$CMtEjYJuvmjJq+`XD-d~gFkD>x)=~n%RBvrGp*o03)t|l8=FQFo_y9Bo%u_n zs~21N8Nj5bKK+fgU{4{ZpF#1wB;46brO@alP{61%8|p3pjpUNT6<81131HcT1022d z^I@-!kDeIy^Ydk9x|6VNXEs5+q!ZVsLSd5XguYkZq4p*mP}_Gy?V zSBKu%g9(fLD;?Ji%H~ycnXZ^Vrd`8Yc$khYIM@q|#g=0!g8HzI#Ifp_-?`5+2&yAf z&J9UGU-hM@+`bI?;68cm8>ID8|vMm z^CyUqa)ePmQM@txS2{sWcY~s_u<-rLe4srinQxM1Ii>#~1?i);fqf zfQJy0GQ4M@4PVSe%KiS!?GYv%zpWn<3&Gw<@dLLG@P!{Q06oD~md4JTgH&-1sdUfJ zcgJ9yhfS`rbcI9fzn?s$c=jKsn~z(#FgXeHoL}VI^Z&~3!GGoPtP5K0KiqA)8x|%M zpqz*%GZD}Q7SF%>VvwqTqUfhRsvp;{F|#{P{F?ss&pqDMtrT<3%uKg)shsQ%QMX81 zVuNU9N6Cl@vz~jE^ZKNXpP$!mi6e^l@86#e(HEY#rQmw30KL6lPhaq4GBoSbv6Qf3uVd?mfeK=)V z?AN5!v3veiYO40Vj}pg^ALrf`{Oc+8vDuiCArW{G`xw`jB=7t6!BjpPxT`&h7e{vZ?*%sNB4eDLiIGBS$q?&5Jk3na%%p1H!Rj3& zIfwWCGt7aDz~A}k(z$!~u#Vrsz3j5v>S}6cCJOM^zyJRGE}$d#zmBgO4}HJ?Yv2F6 zPTwK$bl~}FID7b>?wf>!gl69%?k|*oXI*SOaPCFz`#WIPzWCMERU@e%Un3DO4K&JH zZMU|`sLuLFAM&s%{oZXGFg^PRKqVnDV$fWWtn)*o2sDQmUIj+r$8mB0YupFTy|7*5 zKU#DpBm#H6b}s7m(A)|8Tv2Z0nFJ&M7;y=SulK5;JsMh^(CkUa@_SwQdsjR+Qx1fF zEH7)4Xuss!tcPC7y44lgY|Pa z#9uA`vvn4Obo3Y2fz!4Z`y@E==DGU64Y^?cBXeR{xZTD%i`hY9J&6wy))^(0m6cDb z|GAFaW*pdUxqbVJw0(!aFqgqDd!!{kjF-Le?@Cw80)$(WQ&Yy4Kkh|576JGc@K%8a zrT0{<=iuaa3FPfM`K}D7h#P|?7Rvv2K>OUo#OkAtL(ad$s=W3#+bIF=Qt?C&cecdK zP@W&s+{OOq*T0^@*<8v6hc`smyj-B?D$oqVe#<^i@dUtk&&;6f4=?WggfRjkaOKy z6L0qIm-yK1kwyeua+=l8psjoUIp<*uDPIyk%zu@9Vd*+h*l3=YmN>t9)6|q)Vxa_q z^=PZ8s_rrS;bL^eT(Eu^zFl(fA7$Gg#&)v=1G{IF&rj~6M=lU)7`q=c@cae`eA^TJ zyFsqG0(|HW0H+;r0WRw=;8AW)4+$F&_pA_^Uj8vB&qr~Avw!*dr)Pzk{6heCdo`RBw%Pfln&M4GLC zz3>-_hjR&`rZ<;OkB+?kPaL6kj`H7ae8 zc(y59lNsEqO1vi3+GY*0HTn0kt}u3^Way+&eCy~=K-{9b!F|2K0>yh5?}08{{35}T zBgoeEiT)5gmhF|5l|@xlRBXVoO_x*P`mSlOU%h?zgYO zwX@S0OqtraL>?d5aWJyw*}#}$%9y`!n>)c;?n}9ChU2#&;5MYTc;wdJf&c5>a4-HQ zFrU9YksSz+bx}Z1NPN{Gokqq=F{CT_+2nI1yme`di;P*R{lCv`h4K4K5L>V#bC<-T zF!&56;rhRZ76zQNKyWNj`Oot!{qhpMd7k;NjjqH;#ed%fuHeqae?KcBapiyN;C|il z?+*et6#M^u=Kt4xNC(&M`=resM|H!}E(7LgH@_&%nWIscAW;Pu<`KeTH9E5~(FcNc3z zWwaK&!+o<)+$7VoFg&bAv*P^nr1<*yz@N|#jhBpFX8+7_;QrUREw63FwFF6o=QdqD z1WWgs8onGZwfsv@%&Vg<9yU?-3N=2oYe|;(J2U7?jZzV-pR#M9BY|QOXSOgQMuCbt zSM6n1ezPH(dPJS<+S)%brI`s9-KD(`mua_X-PD7&G=vizZjK9=<*)w~oemZN2 z_oHvpls)OzheKFKA%k`%C-qidASP)ACK`=kT5p7th7?U$;LpywYH$4pS)MDeqOOI$ z`9OCrTi0kH-pI_-af4|ijC}_k);%;is8l}var(ogSK^*&u zw3(GM^?1y@HQM%!sVTh%-dzAfvUjUBDqK={*tGLsVea|!fTbvka~NL!(T1v>V16O*Tdi!HwxL{5Ypoj|3Gb@7_Q;pQ-g*Kx?o$8;k3Z&uR?Y7)>YK zK-pl(tnSlLHENJrUB$z-4QjTIWIp3?)ozutW}7lIc?!^747bltQx=w6RPJ$YxkWU( z|0wRlq6H)}#8VrUihQzq{y60Wh8JtzAKso3A2O`XS_)pas@ApeS<3JqT)zc#F6$jy zAG^$@7fD>~?Cm8#etvvuZ8Ms({Ljw24_V00$NI-A2(}&N_J$;KYKCs#7fiHO+wfxW zHmJS+rL6|Ojue5V1BqFN;1`h&@GX>)ulpshmxjCc&v1!>koSKp)U&guxqSBn*SMLD;+NwCWMTG zaqO+}%!2NpCiNu1DG_n^<+HPp;0=K&PiASwLK5 zWQ0<-rx>AS7!fxxMh~8{0yfQcK}(MgFnORqme-?puZn{=?4H*3oQQ8ZD#|~N3m?wF zi;|hI$*jGG1{->SY_n;F!*5p!TS>$(^Ivub#UJ?j;R`hf+;cCbw(i}9=8}15 z487>ECWs=xFSt|5HX-MI(aP+CJAPvm=%TbtvFci{Ay-2*Ac2EM{DF>bh{aJ{=U#0T zwmg&h%oJQSstv|PE4)|M?=C%_wK$`Ec;PT|3lu~&=M7p*v4$*5ip={gE$FCi@AvV0 zKkNZTI~u8dkyhJVfLB#l?`H=@7XX4@CyNWh0JD5;3w4{*z{2w1$A*+b zRe`zAa?``3?EI3D`fDBs^*6^X!A2{)8{S_XZEzu`Lrc-=QnlW_Z}e2u^1+WRP|9X6 zOQwSKQ8vMAf3=`KsZs5*`j)?-vydUwD`w8vqV2a@cD+e?_M++9lt(~+!5(v)t|v=3 zwpkq7gU^=#AjKiIe8eW;TLy~}(-Z@h~cM_|^DFuF-T z0^&QcbN+K4-zuD$8a?9{kap)4szbi&V*8V?C%~UD7YmYjQG8j1 zh)P?Z9n|faP!lhbX%YNKu%RKJYFx%{+%!t74OQmt$|u_}SWSR4K$L<8tFXkmZweEI znQSxEszWHKGJlpJqZ}LnE0F@w)?yQW`jd;#NI~Pxo2EVI64-YCh-N=BdXDa9?N!TP z&pww*`(TOPugC4Ux#SOQ@MGf5%Q~Gl`f8|_auE_q%Z*^iXw3IyC=KyFL+;$4>ov)57E-ve9gDM^;2NX^9{(tPfXH-*b)Gmr$L}Xh)1%a)I2ndMut{@=2ClClI zNbe;e-3BNKsPx`T2uY+QK&XoJPDlcTfC`}`bO<%Tjo-K3-#O>U`Ekd+=iEESS-%*t z-o?sVZ{By#XFks}8_W$94;m&zmn>MDH*V2fzC?NauzIhpJD&fkS$DWvohxxA#;hPN z)bFcQ&4LQjoEommmW{Q9R^*X|?MBJb@xaWUqLUlavbW8*?Y0C>V+?eStrmPjkBGPX zEVD#vgw?s@Z{%kz2R-yW^sJq{c36tw95L-5TxWW^Fp5BB@6VlJkakM^HXy*eQvB-H zNEyki|4if607LsnS$*jJ7iG`K27+e#?Byxdc@(Qwt0GukP=KysEI@Msq2M_P2ioMa z7;u+n!?}dM2C2W|CILVXyB4ZYYQ&l% zH)R!8620(E%KdWv_8pQxNgcEK;U@hUQ z+l1JN%tG6c|CpNoEnjx8bR)2F_D6z@Vr@@iN)mKIg${JIK1jcG+PXYgY!NW|;2s*S zU$?BfTO0ySVVJ@8HrNJ+DkpKwIFI$N_)YQI2nt&f++xg(b<^+;_weXT$O#@K=e8o4 zur;1funshtPIfnbmS&?I0QHeZ3xz0J`F}DI0?Yq1BQwFjT z5(7t349TQ+z=3B2z_UMvNKGkuo`LVZR>a7X)l6h~KXiL!e4d_0K}Zkay}cY z|3q1b#%{_vQ(m$c{zeLC7xm~j>%_Qh8Z{S-o_Y}`1C_{#(MH$YZE zUlu1RP%3<6ZjazPC#G=!@)JgQiqlT><&S?a+ zzb}F8&7J3%EoTYo3(P1EgKJvvYqRJKk3gN3+OEK{zfkHfp}*8I`_TzFA=LUoWgtBq zvPCQKYwg^ISQiuyj)0S%>*L7;y5y+{)rTY0179?&E z?caAm5m7qmdP3%x1`shQ)6_;&*UHP1GnK*+>=!<>1WW)^)W;4gODuP-xxlGE3GZ8V zd^67r!?_;pjJO0!$T5T79(TL{%UF_ORUT$@9}pDx+;3&M;yToOJ-socs$n)UQN9lsZfJmmd%4$r zUgYOgdY0$Yz3766BKIJ!xl72qDkgR%{n+sd+w0Nm}Z;|kWMW3yyVDet&-Hz-AE-W*bDmi>goUUVdNOeZO0 zC4ujz*;3FAsIo}+2~8&1#DBBcA3sXR8L5=lC?gL3&&A>88@|fJq_=;z1SY0;zrFGP z9DkqtcPPWW@Gir&?$2MC^8Z`o@BeoPwRm)%jXlov_w{2nxW&|Q<&FK92{QlO_K(3YbxM;{_&vh;c)wB0oBvmo+-oBL+ z^|34egrQ8bR~7+1%piu5{vn7poRWz$Q#$5-7u1wM>k z3;IP_Ia;#Ko*B+z+tm{!=|2@9-CuMhH+fSiBP<^HH%I$=!ahn-dViaI zx_I{PQ`1UFUCid>RTbA6dimhn`PCE$*Iz&+byx>EqIFdJSRf}fqQ8Fn)WL&*Hw~`_ zRnHh{;ZKr6u|4VO8M)GZzp8_l&yi;kgPmf^Ln|>GGi)N;HBXxcj%^$u7MJ>Mq|A`v z(x55{7B!PUH^;QqJG-&V8a2(dfyuuYx;AbSv>bI0Ma2R;c(9rx#*5w7^@e<5Ipw>; ztl*tRFOrXGRx6A-r`g<2`Gr9Ym8Sp}14LO1D3L0(p`d3Ls4_ndaXn8n0noU&hXBEcc>$*`4h&ae%p zx^w4_tl=eF8Z9tLRf)?apxL zlWgu~18jjobN-5TotBcznQHj_4{4b1)!@X;;fJi;cv5yrjua~m$SU^a-8E(fD@t(=A6?|q~$Cyk9Z}{XjX}o@-RZv#otr> zHU%-8g&x`$PCZoELD~MQ30kQM>=s-Vgle71l5k7ilZ;{OlSy(P1g;$24WFv=QytcB zT>HAc?ysmOla14K{yHa&YcNt-h#B%tn!9kf!dn3Bb$B1*-8;0(#DxTzI{nv5_b$e>y#UO^=QPU} z)PXRAwcCvcI9ZVaTBlh7RA1gPRe$lX-9s&w$p(wL#uS)xV}>yP{>XBDi;Do0ARRwlGW%a8x=PSDTN-%*1vH^iILNrK`*%>oO7Kk=$I6}@T z;WJ=H4C3;|Dj@Xs57VU!2Y#*U0-epPmZ6F=MpwZhuQ@Tj0rqXTNn!hMKnY=Q(@V4k z`|7FdFj3R}E71F{n904T(9>37pMr4j&r;73sP$sa!>Ok-X2iFhTl+ia*L}}GjlOpb z*F}#=otlnC9+7f=nl=`Yvyf-Wi%;osr;MOrORbcD0D0i)cFbwph~ z6Fr6%;-?TjJiAtD-m$oP9m_V)Ysh_%Is@c^tTms8KET0y7i<3EC0G!`1@lENQ^xYL z(r-ry6~-#+?KxLFI0kmFWY7a+HtFybYNX5^;bNm;+VZH?u5a_O)ZA3=kv{PTsQVXw zSg@IzFffTz3Q&ek$q2w4w?$njk~Ty>8EWO#_R<$@u?`c$Vb~~^O!@8gs%`ngp6sJ4 zk6OA0D{oejMivIDiGAL;yT_m9{dGG1%NQ4^dwkMLk9#d~GsOd{U( z-?*amf+iJ2pOebt@3A2G`tsw9_IbcKn6Cf|ZXc$vT`6Th^Fm~_!A^6p(vk8?*mI=u zDC4d{`fesT1M2iu2J>^aV{>wTVb7f&;_e$IJ$`qvx1vBY(CjBD0;8Jr2LL zQlk)LFtp)aSASkYV8F>G(mfIoI|b8`>fDXjm^ktPq8`BR@P*;$2s@IT{P3 zGzeJ(C;BC<(<}HT;Uhbg{*4dCS^DBUJN}=3 zj#YfA@EG22f8VGtlf7@}>l8g;zB$yYu`-R2R<jZX+}L z&vVAx1ebln(|o3@K8RdIe!cI|6i_ncyb5oxVQEEv{UsxM-F$l06wqUruXh=ia}@-A`Asn_16a zfB$H=J>FpIxu4QcBR;U_hyjjR3huXhHSG95dsuUKL|=9-=EbuzC6C(zP%>C6Sbb3E<2 z8P-zCZ>6hVjG0eYJ_BCM=_i7eZ>shWi^WRl-&c_l?#Y?Ts2P3nqah&{n@z`cz8~d1 zHgIS9&AyoAp3VIU^S^>bCh0}(q&FBYkw|4HdiA~aI-L*V12&$U*L>o|^X^MBRTAh4KpvM4LB1E#P8Jpu$&I!H5Kf;;3TKqRZgQaV0-D z-woO#Blm92+;Rj4ZNNF<0Y_d7pIyZODd`vMNy)j#Qb_qGTyVk}8>JCriH`;HAmuMAjf~lH zQM-dhfd%EC_S`_F(A1wPL^I1b1By0kApT?n7VROys~5}?T2%GS0na?IwexHn{QSJY zS$8`(V;S?9+k_9ZSS5#BD`bK64bGmsvICFrJQOE*`i|hP1G7rd%nI3sPnT{S$o9ct zo|uPOF6veR{d>LV>#b#IPXKG8R{@eW{Hu;D=<4NoURv-oAjy08DNe|<;p%f=JWf4; zR6J`but4;-AYbka1v$V9@>=_cz2VsIl2dKeRN(<5U-(FMgaXJRKd7gVn6H_A4Z+as z7{CaKl-8eT{UOc;g8u@`OE=67uTCenT9jvs1YfxLpy>!1*(~+Q^3AriOZ%;O(DqW% zm5uCyETp|}38huWn&4hDRRR>S?`{rjtbEg>9ZAaj?!Pp0-!EJtxWT1EpbH4P)ZqFY zj*ref$vO{gDlGY1^E$SDz7Lp9=&Mmdcs{7C_QnEEDcq<&HN#d|=2^1tni{d|!{PGG zh!vugUvM`a760B5TjMc@-Y4@3ccl(@XaBgDpmhl8Pd%vm^Cm#C#{So`ouMX;Du{6=?8 z&VdR@Jg;?9yK#JFg3W1i{M2~nur(M&VAgy8)WC>v3`}0%-CjFDqXlyzz9~X*Jx3J0 znJUngJgk2reXKfYE&8bpa_gnd;ijLI+r(3XD_b$nP|4RIB{3=TPA*1ATX%$&U|m-4 z*=()V<#2s|W6A8rpECRoRqjhJLPJTae0pfHE(4|0T5wj}n^{};9V@&S`OrD-aAzk+ ze(qZjq`OXrnw`>w%m4O*Rd)Bh(pFBuCJmNSYUDNMgHC-101&UAGZ!gxLqCgY$}iq?Tk z(rt19RhF3*Nhek~}rnIOzk*{U+A{vZWp{!3itV2FzU(XbGCJFqVRD_CE*5@2sq zj+(OjuwEl_*Qen@QlDR^(X$f=v;JJOT*eC?(gar-LVZRJ)!^Ff{t{5P*MrA%IoCW2 zZ0(x^3YYl`moAaJk>nQcl>@5U_AdGLCF$c|NWLIH+(E!(X-;#HO^THl+SueXnYQ~n z@6Xra$I5ieYk8numSQOkPv5C%cOMUpi5pGr{FsLrm#PwcDv5I+P%1$-hXAJ<_RF)I zHXdX_+-x8oLxwaL{-qo5mgJI{Pha1XXEhV9(~UsI6MV%EHB~~k;+kI2a)MSdRZ@Nr zucAEg47QlQFhd*7FRBx2fT3;kFFx31WOuX z9c~wVF|6aNdP6$4>_*sDfInm5pF`l%^^#Efn@(!&50X-;V~3OJ22V%bzUKODmnV(l zYW38}<(7Hn`Rh=Hwdbyr8a15NPC~GNVeWlN$>WQs>UYQ7#v0yd2W?u1ZY&VxJx84x zr4VcO1NT4HAr~S}uuyuV0+h(AtgaYchG9!tu>**E*UXU7jXIg5Ym_ITkTO~o5BQeM z-hIzUaA@J(S>^6(ul7siq$N$*$7;NM@KensgmvNT)GKZBO9~+Wx{Ye5HDfY^{C#U| zT7F?O!NqA&GspQ5oh4k~NEM)_vE6aftbdTuhj@{a2X8APgI$(iRbjp58y_W49zIP( z14=b*RLCbn+4NvR5cy5q5X5(rLTq;Obpn|mHcV|U15K*? z*gZZu`;3^?%rc}U&V_ihP9?|K{YQ2jSG(8)AGMN7hbF8%sfwYiUy=0<0&)_~xSlDo z&l33%`wv(_M+@k>V)&K^ais8wnetIZ-Sgm!>=n((UC9HOj3lbMAdbtBC+rgJ$3upv zjjou=cCAO#vJc4S-||x@gGNxCn2b57=zvrn1%Lx-aLn8S+h$L7V7i=Lq;jxbEuwMu zLZd>BAt2@5v)CxBBBH)7URZ`ySZ{r(k>DUQSY4q)dB49h!X6)<&mW=i>sBjePs}hd zv0|g)1+96}0c;Z9#IW9#Ryv&YOqUL3G)7y0zg+hdK%&i>+k1L`u|J7icn-;`yKMO( zuAc7Cov<2(sG~C!^h?kC(xh~z8a7@dYRSzHf_RJM{fTk+h3`Cq2iIW7S5&Z66vtUIE34z_NZI5m8mE zVXP#n>yZ$pmS`5BxhSd#(Lg=NuNQ2P0zP;UMB5hBcZkXre*N)4=R4HHxd))U(%=&Ck zQtgiuU@w}<-jD6NbP4zT$*`YBq@5X07un}8KPxpYtpvCoXix@ab8BoufX(_@d7fc( zv*`!x-HssY{TjiBxbGU8l%|Y0d?nioVVAA3$OtPT8T##s(m{B@JpKqV9JoDIR>1Yv zZVbFlyg)`at_QaCyDxq!Ab zU?=55ztY!(hOSu1%$Irah=`OT^8_G(N*Yse<~VHunaxU1-|C6qqt6X-Dh1}9quzi< z@z&~UO6mdvJ=4PA>B>ehY|wo9BrBZxq9W+9gBO%D2`<_8FwB<44CAQtHz`*VpgG|; z@>VqI2yqFZP+bocvprjdYjP-mok1ZLTVOfv79SXKjtGU8cnEVd(@@P=$>{FdI83#KuD7K zSkl!jYT9>9pwzQq^oMT;&KY+3v|mE-?qi17dtfhcQBp+MJr}mzXd{z@Z=&x7KH?Ec zm((J1ZWFq|Xo2T9`9ZSFZ0yz7a`UoW-xiy-BSsvQ{Kg0Q%m|Gj75ZjI*mb%7iGix> z%JN9jhP4YyUTaLam>MCPdl_;0Eb9WuGNy!{AK!eaS0Er_Xw(=Mw75ouMDB8a+wWYR za~kgCck$X}rkFLW?yDG}+hh9-r;n9D9UIrr%X+Sv_K7-`=Lz_k+ImGpZ);+UMNQyM z2RPU(6ra0xDM>?sm#D6;b7!8qqUrQKH4U`nUP|S~sGZ1!Neb5^`V~Ta9BJ`m6 z2}@zu?kAznO#{t(sYS0jH5Y!?_b2#H8waNI<`od}=fXtkbv;r+4zKpI#h>VbKaExE ztwF|}=5x2!5t#^sCi}* zoQ;}p8DRdW#`3aN{rLf`Z*GIjH2>DPB2SL3UF(opp=x9Bl-^$BEPv2J|4r@Lf^#=6 z@vMaTJ!;^p=|KR9Y#d_N zW;YhgVwi{)?S?=FB7R!~F9}92}*ky|9e(CUX#k=i;Ov#{^B^h`UfUqV*K{p z0{s=!5`~HB>&l7di)TrH$}Kl^8Ki#;xIl=q3j8ipZhEEu`0DP7^^6ma9)DJ;yl1|_ z4GLym<8EjM-8gsezTcOiM{)lnqaL93A7!^&{{n`LvK!{bQo}A0VTwCYn~n*nD2T5&G+F z>e0V49mBrsk@T;7!`#1V9%sy2=<92{Cm#Qit(S?w!X(WNTo$cNTo_CDNY-(t3-TEZ zjwy+w;U7KBOUgHw-lvd23-Q9Le~bN$H*fUtzge;WZ^{xz86>QWoM)JR80ab0KlIkpG*(RKJ-NF16phh?bvS_e-UZ88(hG_PWs~Vv78UZXgq`Z@C-D*t=Sk z|AT(>5MpR?>OoRB(*5$o(NKb<^#{`2B;A9E4(9znm~TBK0}0C@Rp z*&wTmCxc5zQcRXQinuv?Nh-{ZYyB}K!D?4q6U0D0BW7B?TxQ< z$ZHJ2;m_|v+vV#xA4!e5;B z`fK=@?4m`-N4%EJS3B*h36_+IpX=i0qNh8Um}H-n+j(55XUrC_`{rO4v$hI{=jvKs z_d8mX1epv;=C88MPJ_OYe;vLHvDyhNKSv!k7Z6ZkD)oO&wu$&6`r$vmau$xfzvH1Z zF_1_fv$McdbNGU7uia>B>bkZIQ)%3yzvh=V%i0L<=Mn2o=Ts&+%e7rp<(Y74SjG10 z=tnT!^8pcBOM*;4a(^-XZhCqK`-;aiQaGnIsWZ9k+!51{m2M#ryz;(P+6@c9a?Gk7 zyxR%ZWBYq%fxq0VCe}px0j_Ixon;JVVk#?&X7B&G?-$6-^jXQAWBkflQ)|Ycr|Q*> z^j)T03~cm(pE1LJAEo$j?EK^;elo%E_)ChY)g?}i!0nWuc-~;)N;edb&e%C7vYY?D zzWwcg20VOMvBIj(^kbFX=olVPKJa`8VQZZ7B?@J7_YbM$Fgo8}9~ zoR7x)C&vD~DH|%*)4gi<8oqYd`q)zinAQ2@*gOBkb1y$O!b48Kr2Kdo{ThI-@!&sO`t?w+G>My_P z>@dO{e^}SKJ}AV;%vk3B^QcFu+~j8ZG4QX-3Kyrzl`Jy2qZRHnnXb6|Tk_bF_pea% z#yR`6z1ah{&0oz&usM}OFsx=rRpJ7CqK?TZ+ct7$0w)Ps(gWonuFI0a8r zX@dVe3e*uhob&CQu;NHRMpt4iX<|#s$C!fMDPydvcEg`+JkG~1S^pM7`FpurCg%n2 zR#1E%yB0+iGiG9L(CUkArNz;(zS|qpf1N#frg=&%Lr#RBUrv?{_p#=oZ%5o<%Ji%Y zSYK=Pd_b^qWjdM)Pl)F#hOel`WUy0~bOloNc~Kv}C~)9D7i|ANmP zl`Oo!@OA1$bcM{Yok|$OGT2bzZ8PF!AU!R|=1YMUKjXbE#;9?PoHE6TXzFm>BXm#P zvcTPu-mvW9NXhqKB9`hE_|;{SV_Rj8#2*z@O-s&rhEw}F}Cnf^uAbt}iEh*NIv#4BL0 z_9N$C7^@_-oP`a2?is5GJ8yd6{0>@l$kpILXGM=xj_$C@7g(W3fl@n$`7L}mMt@T^5ITh^R zZy_X7Up8bvu!mb0zxTy=+7x^W3RrpCwCvz=Wo78YdlN}Hy97XQ8bT`ueYeir*SEfR zmpi>IyUK7{(_pB9wJmS%o4>|Ge)}+-$AoDp24cA4vGhhQXW1PG)zS-;5yK&;=Ujcg z{V*38h-u4|m|8eaXM%ZDJm`$h6{1E zjOCk1T}_zOuTeZob}-m8%25vSzh78&`@5WJVhUU@!#6R@B>x&?;f+ca@PrnUc)OJ&$oMrEsw!eITti*CKBB@Xx zC(y{LYI|HrY{KFH3>oiO(iZq`9J-e@G-9H-#(@b)T>$BPC`oIM_n)an0&3w2L zE!O_W#~FEeo;vr15zGqjP)mLIr)MmkIG2yh%?CoPOE^>K*`IVYD~#dd=%$yMQ{o|D zmFv4tkWi8J5J}rfXv$13x>lkFKa+!eb`dUfT)mF-fFzKfP5vm1h|_QKOO-E7%|Uc> zK->+m+sMg05VgeVdxMnFM*H0+z-3ltk1z6s_#N(>Ie4n1VSo?#$buwb`-B1GRE^Lr)&koz;jvI9D@B_{mu#u4?0#k8A zWqth7Z~~|#$2-#v+k{IonWJp;YC2oIC(%U+rX|)7p3sh4OD!syww^{C9X~UUyC6dG zdQ!KsImW_9ISeT7c1^aaKv~YuZ1m&C)Zz^1W@%)hv0MuYN6J3i)Q}@fVa&==$?^@n z8tQ|Jr_tPGt+`5kS}}R7?ZJ$pPr(KGt=ce`j3(Dcv*og}z0mFuc#nv9JnQ+k+l8g> z+Xa4BQ_0cyvWghxhhHYKmcY9~8a zDP8=pL9~>xq@z7LI&>D8$EQI{uLjgaO84c|=%OPvOM|iNglvW2$}nq;o}iS;jAZkZ z1PTh5i+}`RgFG4biZ1AG3YyUZ7;bvgCNaEKHB{a|!iC|JHXZMT>7j{CxH?c_*LNxM zPe01zVs_XUMMu`vdr-GZp3I8duKryCuJ?BVD6IcAgq&btV+{3X9o)T=nE26DhkING z*eD6~Vh&gi>H}Bn*VzMPl%?BhFX0;OB~UI_b{($XSDgae&%w|7%eWwfMb(!Ss)67Y z7d;E-QZ>R>*;S(ea(|C;`!H_Yd_Lt_m9StVBFNvRxF!8mIvAv;>E~QFv&4I%84U*) z)d@?(I}FcmOpbvi#k40hz%M}E^j}F=y8f0EoX;9{Hj0-Xnw=; zqzw+&x^4#mr2l?miXXGKzk1lBE zsUBx~*(~{5{zmEAKxTo?q$PiI2OL3Y$X{7j2uiUF+$9Oo+Jq+eUR~-WY#WereGio^ zy}c{s7rItxV{Lt^Ea{pF87cFJJ^Hxt+3!O9s03^7jkL(LW+^nIgW!gJYO_kDq!t#_ zwjmilExmg}+=suY%$uq&e=wY}*)o=;+JUm?q_wMIb`Dp}k~4E>HCX{04eIGlZjI53 z$K*1Z%W%0Badk7&V4`1L_Rfkyw>erPwjS>W%fN6>l?r_@+pJ)7)Mcn zo$`9=$=^BO;ji;#k8WdtQ6^@~sR4Ks{E<}r@KLsdUTX1&%$JlAC|pD8^2}JZHlM~p zp4sl?;OpX-Kv;>vFq;XmwB?=I8w|=!WpWUs?knlvZVMkvEpcxnv&$s^c$6k#O`{S3 zSPW|9Z>6vSu-o_z?I-$^w5Y}|BN?Nv3Q>N6qQy(Y7MZjJcF!hd17{1T=(@viZhG7b z7A61>BR6AKDQLnSM`y&H17&LyAjq)`#Il>D zEmoa1kO+-_*jI z7aO&cicK<6JAAdz)|(>7r!$3+ZT+DeLfps`jx2Qn^~ORQH$b~G3jYXCH<&|-VVqL3nCx+ z9X5Kp{Gvn772<(J4puXXA?U*{K;Pc1l+xCkDf_8##neS{?I&Ll8_%0GHsrV~g;CEW zP_DQqT^IOvxLO?yjH}Ra98^sKRhnV7uyU2D^nnKSL9qe8+)Yit9!z8CWorP8$?=N{ z8LT*=dXtoWjYltZ!w7t|IG5h!log#+ct->T183GUnXwO%jW`s`_=XyZmV%y6St7ppG}5MTSs%nn_~%^ zI3Aqyh;ARH_@t%WiQqS70!%G7A1qH5?XL5WnA#C0kFHdwspRNP@K=_g8FAFM*?`9q zt`TU!$~_F$vqd}gzg}EY`JzL3?lj(gi_t~SvV@mtsf0n941{T>O@rB2L`|(L47T3Q zg7FskxScDg`3e23dWd1h9l<=gq^fU)qp9m0ZuMTHeY^{^@j=04qC+Yba?-BF3q!_I z>U>cX8Hk%5Y$G2FL2ud@6JKQ*& z=CX*T)7oakzC@_%si|`Too7G8T_y^e^;HllL4FIl$|v>~K0L}I_ky9$F}`TvW7GC$ z8_pZ!xb&K*?-{Lct(1Fh@>Pm;v87!wGnkqHs_e3h;GD=%z8njx-$qAGpR@d{roV9s zB1>|0)XvX%`}BCj^G~ym9Cx2KVP;OdOMMBgOz!I%9CtsqapH5qm~V0qE*};orTz1_ z^1tf$D!Zb?4Pf+h3H0YOduW&77u^0qaH^ zHDXDRXS|Va3;b4mUR;ND{!h<_yd34dlx1aD)ur=Mzrg-|JI@*bfFQlOmvo+#(~~r@ z(4)y#&eY`uqRS()ysB8pc3OM>8iP{$Ps}V!+tXGat5m6qf#~-Joi6nE6ckjI#JR@6Hsa8szoJ723Q{$NK~fEuPP%gi%xy=PSq3FcU;BvD957^FF`0* zcqBXKoSyqQpphcb=_w_4{(T7y60 z)a;AEo|a+y4S4r@*_*m}3jZQqj=^lO2D=7T+k!m;g_SKoP|(!;Eoc68aXz&wM=?{6 z5I$~PUA9p6H3kUxvem?m)D-k9;tYf&NSgT!Z8>hX zZ9>jzfP;@rihPdcOnCSCb!BbOBe20Rtt>y(ayTYClp;?vh)?Rf4x1QbkqmCcQPt*i zM-(}>&$zj~m~=vi3I+Nzb3l{4-}gN8!iPnFU}geeZXdN8Q%Zq zef4g0sqT9er_fE=aL4!1F70#0`a|!?{^WrkqYA!9P%XjreI=nzQlb6EtSh}%aBra| zqFYERW$jAWRTDL^CBbAY^3h&Pjgj&IU#?AZA8Vvhzmyuff?6 z1Rw;z>X|P(N08@OJQqLTZ*+H|= zz0JWP&21d>fr8*oPiT9;wHR`@PHuRU()E@T50>#+|s*wW{8}!@c=pe;)k%Q@876_wR z@xphk*oc4ZulFE_1R3fysH06Wqct;d>fYAf zNoSLmFVNgVTXKYrvJD@1@N-{G{r9ln1q6CqZDMjNVzg2bURG_HtTfseKfM~?^TWz@ zgwuapZ9Kgp!(!-ZRa2SLsr2oRlc7gFw_|th-UDW?2#XdaV;~ENsz`36U9hw?+ey2` z$)=LXte`}BnJ0nNXvTL-JHcVn}&1mkyGC(_0rcVs;Nk)jRM8)1`>HrF|w z!p18Rbm9{Ulp3`zC8J_fOJ1J*T!8ArqZOE<~2=s`D2Ki9j;c zpwZ_@(OO6tnwwZ=lHuU;G5K$Ut1kC@|9 z)Vo7AlFG^`E(oOweJwm?XaL;%qo&K zZj8%kR%kyd%|n!ckqUc+j~Blnq`-v@svD!@?~L4R^R14lMXKcXerSs>HxTXZ9&BLU zBKa42_;Qj1UZH@}IG^c5B#?u-LRx_5|YD7vt z^Dfp-ESz*C<%>0o+4!%=BNy7I(@RU;43bRu0h7l+{jSZ1D=|yqP_Djm(qcok*IldO zpC7}YlA@myVgrNcg(`gByO7G`s_o&k<&}u-FK>)6>Kn-pF0fq|1^V=eEyVq=1~gF_ zNo|qKq>WTK=cC=bfE7kRUF&5vb%1W7Yy-#Jy2q@(__jF2n2|J~K6?jWsfS6kv!%8k z3?GDSmv$`=gNNectRs1OYQibygT?3I*JGsb5kAl&zVa@U-L&0?wONm6NO=x{>H_2e z5Z8nJQy?@>o_H^DWb8&6p)Ac<=~TDKnoBVH+6=as9o7elTwea_QGaDUMJRYWC`r~N z={Z7JxhM{V_q}U(_#omz_tH6l7?wV94dn#VVaAVngdy?$;9RL}5CscM0zp>tIVvnA z?fN+bJAW5-%1#ZGO+hvo(IZA7dq{ZYsBA%p;Ix6i<=zK^+NkU!m3Qni0`E~|;;k}D zL$Q(@Es`3n88v{7YdL|nT!WEAIrF=`vqAwxfW7Xakpb(Gu`oMd$Fh%Ty$Rwod(d{< zRoY%@;P`Wd>r(Hhpv63br1XGD1_8E-$QmiAfW#J)aF2mXDZ&g-MY-oE7uSP0s2QG) z&Igf6={0T%ujeF=w>&bfat{ZG=!fIpt##k+B9tB}A*eii$fmSPb1rnE6ETv1?`MLH zUyu9te5AsmL41Q0Afw!sv@%R52+7#tsHMv;ld?wsHc$>a9fJs8zZ{!Z1K3?Y&e&iA z8h66k=KHrxV053|i19@5-FTv?jjbMJg<#-5oE7aY+gq}6H^3xr@l%reL-t<@rn2=hUB7pF{{}ThtQ%BgX2nK!raBe~&=?f-d-2Zbx~BO-isds?djT;E z@K^JVZ=ciACONarAnDL_4c*j&Q`d#7pSGVrkn|~DcU#led1uzbTM<$2mJTO$YX$?+kzn`E>v;hrMZ>;{HiiG|n<>ks6MHj1n7-4@0G0 z8`byiia5l~bH(jSTr6@U$8kEW)?8og38F6fZn4+pj4nIe8yfz=pDzC_KP&^S-$ip3 z!cTnhl6s_T97>}Msb=?{4^{DrcIdTyraY$_%dKRQkWyz>64^7sMwEH1#cBx+sF#Fi zNvW%RNXrYHbv1w9R4mTd;kr1H)GfG^pe%A8=_4W0=Pp60x@(&%VZ+x)*zdr+l=pK* z_>}~{T-G(*_uw`qq!bx7>kocU)2VhFGS=f((Q59sDi}#-WT4(|;q#SV0mT7wxm;LFxGz-i@O&~+{ur$rfF>X>5=9zr0M`HI#W zAJuO9lV~Hv^i+#m0cm*vf$Y4hJ;@8oWsnqQ3bvql!Xrl;EsD>GpF}>pYE{s)@&b^f zN6Jmj8qg<3kQNO4=|e%Yhrw^Cm{oKMZFW9sJy=oby(`q0p|Z|$wKxFqU``lo2600q z<6{myYR=od3JY3eKqzf8lwkoC0)~Eq^5z25p_Oj5KIWG`DcD7zO@Y4d)-y71Fgbi8 zU=qF|)?npm?Vqw7N?{FEkpT*gm3rF=2unOb2duc)D z%Jri^{|dMMNaRdYIPN5evz z`Z!89IicETAze-t0(j#oyFI84)wHh01D|h}2C6E0tfeQvQ2mwrIV5tr9IN|gKLCJz zE*P*;{G$n!1}+;Z=Js3sw3Nu1QHD4Of&mju1J9wIBI^2d@Z_#6Ljz5=`8*C3m`v4% zBVVcQK@AdayZ7`D=Sy)VP5VQ!pER8x5;3#N`X7bq!|u|)IgsJOf_j*9128R(TKxiy zP+Vh`oOxR?a!*tPQD|QG%`>_WssRq7KHs3-=4u0>9N@_wohHz#$URn`(m0w>@OJkY z10XXn09ZWAW|;4RFhLz(Zo9^Wtf59c4OM6!My){>c)l53*YcZcXHs)`OVAFUrtOq_ zzLvHlbt@0E24T=ly#I?D=LL+d6lKtQiA%to?R+$psM@F?lN7MwVxRQsH{0jN|JB&N zhcns#aR9%2DjlS#%ORF};z@0_iic-O3ZbYRx3!r=Y7UX+FcPsW7HNwdLQJK`oN8!F zXgRcJj$sucr%75QZOEbLJJ+x4>H1y2KYsUryRYwcZ@cdMd*Ao>`nm7TEOw@yHKM3|IKNRChLBDg!?3*|-w>xmb~`Vk$*_vf}AZjP}WY0fBaJL){j zCr;+=@hlu}bj@i@ydV+xIBil-e|cxd9F9cZ+#P!s|0WoY@G)W&W^GqhyVkHEwu+$pq}9;YV=;8TZ42=O^NVO5xmnpR#^NT6$I9 z_EJR%W+VmE3I@{t`Zh-WkZbwLf}Jtcnxc@e{~mr*wfL1vv9%i`0SjH!$2-Cf79Ptwk3)-}qWVqxT)=5B-zoPeiDSU{@Ug>t8DWFL2PRqCmetp96-`So{h4i6 z*i)Ylp3G?8V9%8COo5B)3ocJnsgXB6wSwVfZ=Xn=wCzz7xa_qVWHrA^EbD|%$Z1!Y zO?Z=qgSn@Jx|NPQMqGT?EHKzD3^^FzF()x!U0cgGp;x9|TR9FZ`vL48E5 zu^Y|(bFv-Hu|HqAP?_>Ug)a=I6zwa^z|d#xs&{v7x7lK`ii{u0aj@x7I-Y3S;x}>j zW0LE)eun%Ioy?w~1dhd9L|1oEuan#v$zPP;e5z}ZrtSDn$P@Z3f z7P)uFCC7vQAv)RvB($UBPA1O45JvBFBhRZ=X`2vJw_PpJ8=1!qOTP*~OoD-|OVL|f zF6z0{I%%JG=X_rnJUf5y9=R(hw!V7yfkgo_={*+9BIq5XrsJGdmWFvq4kiDq zHsqrWuWTGMB_7Kt)VGKmeQkxV>{zyM2p{>%SKsDk`s3X$O@j?dPr}o-9cSjpwySy#50t^Jz4Pu^hHTCG4=6v_)M0*7XRc2KsUvyr-*rjKf*%7Zfh z;^y{m96I>?UtI8iA|uf3|AC8#ZZ)Z>FM+N!9ClJpx}H=7*&;bcrHjr__QfEBN?99i zw@Lw`ZPuFpF~3VkYqO`CsAV}YTU!)YUjONjd_B=gYL_W<5s@JB$I%sI99xM2MdV>j;QPmRvNO0+jyiOgL|YDjS|#BESGlb;HQ7<%H;W z(6BF?)2s+?oY2}7wo&gIW~W$s!5h6I8()HNPfmV99h}wZFgI>iXM~@Ca*MkC7|)EE z;^0|nK;j~3^M5RJXlozLALw3#jX%jvprXCqql5mqPaF;$FOOIWIqG0GdS$LuDmwC5 z&bsd1!3{`k4VS3N5OiN5Lu?&pOK}^AHO;{4PD1(FM!e9G)pl_?G z;Bf8=7$Qe#?i68S_a)vF%0(8Cin1Tt`fl~JQ)v`5Aur8IP`(wAB4 zn?wg?7%qltoZnWf&h@J?E1$Jywdb7-yc)ZgL2NqR#m4DDbNVNVOmuH3m~(3iWEY&i zX?Q&3lL6t5Zx6m;q9+#g*V($5zCi3wry5bdB)sVDTepp$NN;yIErA)l5T&7eafKt@ znl@jaGW&JQ+-Iz0bW&$_^CDM`JOGN+9Nm2!`%d<;$73|JiZ1~W3AGZVNQ>g~R}Yc) zsNxcQV+mEd$;-(pq%=Hk=InJ^>Jl&)ru-34IT;-bmA{ub4PEdMYBF1C6eV5h@L~` z${o=!wLSp#neDz$9aj}Y``rHP|HGqJ$zbjak5 z2{7^^1f`~KOz`SJaFRc1>;PfhE|>~_K_TB26;k;+&UWTS^lR8kZ~sWe!K zH@vfVu@Oa}@Ey5d#*c4q!Y=)}C;&jC{MNg_X6XHDlWDiV{djT#-q=?8Q8rSAj+&o1 JTx9AN^$$&d-=Y8j diff --git a/EdgeCraftRAG/assets/img/create_pipeline.png b/EdgeCraftRAG/assets/img/create_pipeline.png old mode 100644 new mode 100755 index 53331b2b7da917847d43953e19a2d5aadeae1093..c443fafc59b5dd536c9d9dc056c61241ba72dd22 GIT binary patch literal 86636 zcmd?RWmJ`G7cRU+r9?qG6a*=0k&qTdxFzG2MMObB8bm<4yHr4gMdJc#q#J4J zGuM9Kz4tkP&NzR*F}`um8e_*|#q&J(Gw(UCd0p3B;VMefS8*wE5eURpSs6)H1Oh7n zfw(A&jRAjgQH}_O|KK>tXgebiqzvePF376V|As$acX_1c^2FZE#ofrs6!FB)(!t2t z#Z;fHX98X%L%*o*Z0dCDp}noGshtbr$ulQYQ#)r1dzS;(tI`O>ErhJ3xSB`u+Jw6v z(ZM;^Z)H*(S{#B9Zb@aWKC{8N!1M33pO-K! z{(V-0bE)s&XHSW+)Bin#&y4?f9MI?)J|b0;f4idD;QIpg5%c=g%Em<4>^~P8;D7s4 zUsVE~Es@9X|9CGFLm)wXNbQL=KW`{iPGPWavxE>*6GLf-Q7m|gGkNH?EJ z*3=Kh7;XC)sfttA|6aqhM_p#+lPX4^xF{pdLX5QHDwoH$JcLElI0lR(uSrHZ<0)goMY)P* z{@DIiyE~9LEsMUyS1adt>GY>IjcY@@lNk~W_pLugik;yuDZPGiY46nVYOl~t zHpFm14lc5cCAQOq`?^G{aj71M{n~Z(&G{VDW5Y)lA4d@xyFwf+@m!CwGCr(OgY@?y zhEUT_rH^mqk$WNBkv*h$D>O6>jCfd5=9a!cc1N#l#IFnSn|byN);P2&y7#*&riXvt z_LA@&lDeU5EH}gz^yb>@7s|Ss=Wz5B-4s#ZquD62Y%zQ)KAVnchDgC=t0fn~l#R*D zPPu9Ki=~$&tTv}jRV`c!mX>!~xPk~o(~4D?gT4x$ zJ}w+wHtSAw{q^;-L|Aq8fwI`?+{%j8Sm|h2G=uy#8c_xRvrYf=v3mE-&r6ZGgb2ig$}Tf!qta2EB4RrEEJCK>wdcjKO6v?Ws+CeXgZzI!zmj?audC-iz|@z9 zr9nv+@ONM*hiU7Kt?-kwLc8L!e&vcmGtb;LBE9a&ph<)OqRHa0eV zkN15SQ{0nNQdTRKdkJFYDxDX5#LoWQ77|L*WhRXfa^E=FAG620e0j@*cmGAb&*ApL zl)wMpiu#6wDoZR8qXM~z*Y3q;QF#&)68NZy^8%hPGhw(!fmVN`_kq_sfe|?)BjbYq z`LUs@YV`c`FN-^k4Gm^J$s;++X-Bpc2*fmnrk;-3o^*Vdc-|W^$?JhJ65kfLcy%TM zHGO3&c*eT3zKs47omkQNPYOL(WsI?09V%=I#h+^Otye!%!0umm(JVILCC97sSe=`j zgT>PQ7HnW(pkDrUW@hI5_wQw8Wk-!yTDTyPw6(QMOH0%I&pfrXeufhh7&;QIu zm6Vi3Qt|6#_XZIJesO>mub%W;-RTuu?A$4ol#)_P;$y#iw}0DM%F%HPR!z*j=SHN| zlY_V8wa$wxsQmGVnf>H9Z|Viz4Hs)O9}8wvy76>wS15pQV~K zZf?($sAg5jUUjl7e!0T-rYuWIa%hR6b^PR++|drCr)Qd1YI3Boyf1lumvi}+sd=Sn z-h!~alRR$Eu zT2EC^?*!Jp+GeC|wD2xox$Ei>jsHpMg@~Foao5~nbMggvwHJvzK3Xj{Et9JrGBY^jSOXVF)`D?vX@Z>qH*Q?f30ZGY28uQx!=_r1G_~f+fUJ zJyZpPA?zY;tO7j;sj?o%jOG{f1zTb;x8DV_X*#QZXJL88w}NHMSz2CRuJr3OF1P5| z(d|3u?N6#}LnSWs(gkoFu0!dyS8U zB|6gTb~9CcMddlBP)y%93%gmP3@Iod@|x?&^Dhta3%VTSyVNgb?v8Vn&#fkf%va-O zeaq9m%N*2DjKQ#ZJgP0?*zkF`yZiXc;}Da9?wBh$L>@xbacdQe6B3EjGJf}B=J-Dv^QDae8RULo>+K<;f-fZ+BJ=q&JTpcc66qPbJ zU%dP5o6GvBI!C-=lh0wip`WaxVwQ5-w{JxTjYl!cqNO%@@87-^ztVN+bJpOw6M{z} zfxV2%AO4a11RfM2wJyu5H!oU5N;$9nZ2ZcstlazJ1x9B7``B35!|i#v>>s{S@Ksg` zqC1Bte72)O!NDTuCp)USy9?b(nnij7Ub_|({C}A7Fdr%?pm#WR@0j{(|6e=$b+!}a zvbV^*<%TbDIfN2gwjTuMeZP`jFOMVmx-ytobSGX`Ue4CZ{k4>4ktE{u*7rZu)4kT0 zD}{x)`U*29mt<7WCgo96Rh~JLV#fL>s7HIt_qDV68!J|PY`iQLSwE64VHOGcugn(p z9{(6(DSh3=h7*ON?zo4lcrn_lB+o3S{q%@IlBLuF(IV_KcWUkzxha87vGn>G6#_vm zo3BxLbg((~Gt)!u?%lioCtFwtCI07UX?{n!5*g|E6B84GFJ6S;(->%InAzJ;$vpF+ zzjp1~xUk=5<3VbN$!MAB72o$UF;5H(?08WbBQ}5lcp}9`L_~ajeQ(pzVbkX3=2}`> z!riPM^^=q9++QE7{mFCx{vWaP!zfeR-28lNWE^`aK1~tI%G%Q4!Z zzq{>hXDq9HJV)WDPYy0FdUYKw!>%1YTKjx*L@$e5PeeduZg-_*=u2&cs8u2CFr7~J47!dh9jUuyR!Ac!KEP0*CFZ@<9DAz+S;()!SMo^@ z2_znRL;P_08tt(L%-|0>bN5s)xO%7 z@o;_B#LVnlTU&oxU|=BN8K0R}oDic59a|Td3Ux$x+1i&ttlBsIy}kB;LwQ~K92^`x zeqJDwfRi%xKdl5v;3bSl$z#zPc3mThJ3-;jojXvCWo2a28}pX+A~>7=7^C=2$rnZ| zt^Lmq&24PhPFF5sVAQ#9PB#1dZ+yKN8z1jJ{rUOQl0~HsJ849{%kp+7YcY@gWW&lW zc6^;uEn%CVC6J2_Gu-ZRaBRSu?wa%SR|80JTE53X8X@*)I`GGjAFZvD5@8zoB5rG1 z^=|7m!quKT3s8KZ2+q&XL$t2$I5c^Mhe(#{QJ$=BA_0%UR>$wbC;I%7p^u^B);s~n8k zK0HZWXfe^=2gHJgj)3B{*fs!G{w#e~_`0Yr+1k|9RJY3J4hKii_wVWQok-;QLG!tQ zfIxWL4Aem<&&I5;U%yV&yGuWMM1dCqi7<*&^ugYcE{NVVzVi44@1P)R;B24Fw~()9(RkwvYpmI=Ll=j9 z%~bo9W#;gP$2BJ&M{Nou`hL}t=Mx>TdP5GUsF-q<@s}hkJ)O<6rmjoZ3*b4C5lWQp zb|sM^qsYd4v;7r{wSMTRMvbF!FHT0kMbn>|h}rfiXL-O1O3C;8iw&C-4V-4bMG81C zn3$S6t{OJ`iLkK9v5`X6OymH_0W~}&HC5DpQn=`8OcM(qUzI5`4ie+e-d>?;5F}6@ z3Z|g@_wP6C^a!D{d*dvA`u?{of`DuUG?f?^CxxBC9*smI7rT=V00QYYv9PkT($X58 z9_`7;v1QcT*WT%!T z@4;8dmIUeEeuD33*Qxluz5U_&al$=w1OHQ3rkvv^35#=ceXYp^Ul(_#JQr``<41YD z%3~ZE8G)0Yt$WuuYjm(NG3B#!)g6zRjI8-!(tDS6y6@T3i8@zSjRI^uytz0s5)$}k zWy-> z@kh;H;cQT`J<>ZSbzA#cYK3CiV3m=V7jl~W?mp#9MocUj|6@z<9yuP9QgTW1Bf#}i zQc{Ex#JFuxqw+BOpy0sz|6=s+>g;?O9`3oE6)$?a`vhrtoiS#4*wBBX#xZlg;>|Ae z1b~wgqZYH?R7L!Yv2vyne<4fl5Zu*;3gfL)YBHM*QVr+jG-+YKh&IWkuW>h+UZv*T z`26JB(*ZGtSFcfe#=P-iE3ZDEOWWP198;jM_{2pQP^8bRS)1|r{^*rwqRXPIkESwu zYoo62o%p#-=LG6%7db4yuFD=CGWnH3j10|hzr=nOe`%@EA$}|pXX|7qP{q6dl+ zTA}UsL0yenZC7n|ghQX0FINNo$&^Hv8)>Mn_~oRmb_G!c;`iKuRb_B&Y+Rhv0bhbb zwtBvXc6RSOr5qujKMs{tT^$|TH4Yi3Z=?a6IW0S*;sGqh%ctGEd9&846Otk1KGdY$ z9uyosdSY^N3w5p22U3}iuyHi{nUG~9BnL(OJBL15svQ&dp9YuXI1TM8&t~7n#FRTP zD%sftV&PYn=ku%weR(>KF;B{_bLrBhn%Y`NA`&wblRk>|`w+`nH$ULeLQ1?T7LooZKbapO4jI}0rV2N!4={e^?` zOt*QC^k4@^io)CK>xE7UyUljsq}|s?+Y0pt#gmegeYR%W1UGW!vS!?#&=c-059VYW zF!YKX+=#!w2#I}e&Ov$0qXFGeoNOa~M5k={Hm91Q9-f+&YqOC;m9`uz(AKl)DPt2+ zCczC(O4^%v+0oIFkdUyu2+RTOv*3^rr+(o=>>?y}|JZeGS|3sWzI1VQPS+nCbaXTA zH2xJ0hzHn^zYX9_`4Lnp)__#Ct85ZXvlVUv1Je?6b-eg~`A7qlY#P5m87ql%Y0Grz zj%U3b@J`fsiwe7Uw8WFW#eY&aiOw2zS3K=FiQ-IE#Y`gn%;o9c{}^d?2%XyP_8*$x zYBbZRzpjfKeT{*S0w*Pvx8aswwy9o^peYCg#CmH%^h}6dYQEPyX z!qibl?*UW}SRUcx;zA9$>$z=Gs&&#MwAlbS0W#a`sHhyyWFAXG_F>z>>_|yE?FJ7= zpcDXbSO(>(vE(SFSnA7{KC6prME@dPjDM+@nGyQNdC-ox5B)<8V580_=wY)yrU9_sy$?Ojb z4UMAWSE46e|5+Nym>2Zs-8*8zJ9O0}6!*;VsQ46e)d&GET^!~(gcXBxb6IHsAmWhW zaJ;|Xe6+$k+AE&f-`d*h&@ISe*wkP-$mkX3#9oN9+Ji*alP1>7)m5Zlf3&~8y1c9% z6^O3cT@yNxlgeN*Wb)GwBN2ViVW6j4K z^^g}NGD@tZ-@;`w6TgINy}N>9<>z-t<*DnTJS8PtfxWQw_0rQD?@19_{#0Y=NcWZ* zXeB6myH_H^!_kcilCybjejarz=N@b$^*wvb=cZ}Q^^87I9STD%64np10snY^LGP?I=_2M9mz}L&fJ{!X2HNXg>lf0dfT1+TC55dUXaO zaeU$c7&&<4*!M$v0@96$Nr^xL__uFSG=@*g3i`*JiS;Xq?DYQ}9u{hqek~ixZ)`l} zpYpj)IMIA1x_2>({O$~}I1mtwwCGL(2t|Oe5UfFO?wN16;{lcrG|hQIX5f)gQu3y% zm?22!FK*N^#ddWGR(n^;AGxK)hgIUTF)lEAqDfEax;dG`VKbP0bvXF8prF^upDjHk z1`@xoyE`%|r*sta_S~=ftFNHU!)~tdfbCKX2NZH!OIJ=2^Y;S+uww3TG0ne(|*&78&U2`w%+C!_Jq>i;9Z$_E};&mT-dzAV0cq8XVCw$96(A^u5u^P*F$~ z7Px!2w4p&*NJvOPU}SKxoEpm(*yDQ#hT4_t9Vl+fNUz4!GaK%NAqaopV>imah&E#^ zEG(~|dJP_1k^GYY{qYSv=SireK&U!stgsmwD9ga5rRU^~vX-NU5N)=|RqJ&9#mvDW z>bX64@uCGXs?MrQpG;H~NI~|@i|+v}0ZF;^8TOE~v$G{azuh@%1rpMS3l%c- zxL2>*0H@CFAKmW4tnRky>ij8`fH{Q;V@tgip+@zy;qaIbQYIpq^yx`n(?*$qnBv|(#m)`J*EMFX#8h9WZ!X5wp?ppIusw{~`pSMrPVt`eZETon}X zPDk3?K^^cO?C;~a!4uHa-K{7mm*+N6_2C0uqd~7}UzAdY%Bvd!(eiPg!cbvP_cx~u z4Gff|KcK8ga4)Cz=7{ijc6M&>>;S_Q6BBcfo4W>AkAopzhJL)AnuKHz(i7Z34tn}l z?gouqHNB*_pmac4choOG^l1tVyy)w$Gocq1xc{7oQW0{mzLTuCxA!PTpy>#pDo3dN zqi*i*Q4tZkbuN|~8l>yej|~h&jlKq{D>W{ua&{$Z|itB9wfrz7A^-sA!%5FER#2fqx zpth|f(;mLu5*-!YshNe!)eW29B-CTYKm;&#kJ&e$!j015xQ-jlLqX8Y% z_PBg#ccGpm4{NY#p+?NmEe8Z3isD(JtDLR0=>M>SV5`5#Y0ywJDsIJ-_~FBc#IKyl zdFZ7pxN1jv^RrZ#btcXUaCBLFXyvaWD}#^7Q2E*D?M~Ufx_5*o zN8@oqxO69UNxI)4!t>7G>JXx6lA^+bUf z1sNiuYPv+&1;}h}=sm3XAJx+j&!!i~0Ym041vG>mLOnJ%)=|CJxb(e&wyc7Ju*dIP z`b&$8)g~Q;I+ZsmDUU$M7?dTW2+?l#Ys|?pmyp0HMA|`hlaUFV|8^C#r9V@8xtB0p zG9NXt!0w_pj%}G4HG1qj*h2^$LLwx2ZP9{d;;Ob8(8!NN$36ao%zY~o(Z0&u%$+18r!4a2;R5+-%@$|$VehDqP?CkWkO1z*VQmyDU zwOdn}S7xJLF5KKnAY-04US?AtW2fcjcKY=-c)altI!O)=jtdtqY;A4nRj$qcfVu~Y zQldgO?AL+R36Pp1NZF5mh0?G+6p1qhdaD*VTn_y@AdxGn8hj3&%*+^1mxP6d0lQh! zC8wqW7Od>2LWg-ZNXz2*D5b#}l*y;IDDm2cmOWEonSEKSZoewirb8QAb%9fjT zpPe1s=kt?bUjoIF05`b#bf2m^1^5KeP?GOE8n?Y9K^Z80%9IwcFy+<;Fl@P8+HZkA zJZ$oO{d)oe3HjgMoSieL(&q~iftXjffB&8ya(8q*1X%>$LPbSo%|%Yiru9IK`K`?E zH!6Dveq0r3kmmEe_PkC(-pIT8(9`n}EvDh8&pXv7CFvR%P)}q6uhZAphso*dWwWTG zzo1_Mgh7qyJrK@KxKT~e1b%KI48Nq|PfUR)RXWl5@&cbfQ*|;b=WYV>Pd8Z^No`Px za4COGc`9l=jU}ODiv15$03SJZNh}yxj9sT6^lAI*e@40Kr9-6abj}|(Z@BzD4P8U>8XCq~{DI0VUAw8Ljxx9W zl1sv|O@te4-?*nyIpSCu^U4te1LK_>gKdY7o?e6XFh|j>!>I*NB3hrQ>1=ObgvMe2 z;D9LzC{pcmGhR7_B2MsE93n=&CZ7Z)kN1g*bs(-ri5`U^FF89oeaOsQEz1QW>e{H=tvy)Kpoj*UFm}*JaOD8!de% zZTF%TNZYGad<9l%L@7$^4K=m-IJN^?{+A)yZmeB_y>Ug@ZfJV^NIn z?Um+7s_BBbjuebFh$K8XjD)qZQd6)>cPhsJ-P>ZH`=lgGd#A0ug(pi=USU#R(|A~j0IBJPg>SB}x}gYl zc7Eu#>JW>qQs0a}*xz69dmRx0Y>Zya2mn^-Q}Q;yPJFcIPDoBp2F>HVWY%V~!Ba*C zoB#IA^NUz|`ubB|tA%;jt+cfh0K{I*M4`K#$Ximc$i<27N`y&e{yG{EB`*51l?k;H z^pD9<8b6t5BOnv(1JeYO!|n)R61#80G&D5%3wS|ql9O-PeQjJ?T7sSoARrS{4lv*Y zh(rRXKfjG=XwqPF@qK?E@$x0~b!_=-kV}xAolCtCL;L&tQ1tb}gzd)fbCC7)_G;#; z@!F1{Pbl?&Oq z1uxrY8+Qlf(NfZ>O+-Wl8AO_K8!poO8gqydU?Pj?GZpH6z1b}fS7ElnfdPOawG+b@ zJmZxVqG`&=Ilv?nas+AhY80p z)f$7b1mL^ypAk&^blXo=Ytp#9zAkCoM$aGl9F&f$1*z3eKCLIcLFhQE;J?lJ0&#Q! z6LZKib%RNzw{yO$nQ3mBrAxKm>ZP9AJfrk6=6%AVSardeP`*lALE*le(wqpy&o>(R z^K};-{<73)G7~p7H9?|e2>OIt0p`SLq|6jI_=V{Rn(;lgVm~}Os;jFTxXQ@Dus%{A zy6_$|@E6cAa|1WzVI6^Q1c5LlBt++%E+`@mp5>5cD!U#q6XUi=Qjw97Ne7CB81dQs zB)EAKP0A{z2!>12g&0N0!~mh{4VfGE`;V-7Z0Ri3Qa0r57I}yOG|Z=r0`Fw#6%%{glFP&V{&MBZRi;If^ ze9+R<(mt=UM|*EhpySbLE6|&<;Uf1tUgzh?TOcMSrNqN5qDyU`1TJWUacl@ARZCCj& zx&oL3PiRvr0M(0( zUh5PpDJWDV^9-muIcjOJ&B&tgZCF+td!k?yXuekkQ-2A0Lg#VTe6Os-=jPR5uT_JWkH6bN7FqH@-Vm z6_jG{UZ%r-H-D{d^O+bv{`>JAKUdjPouBGzb~jRrOzQ-vXiQ#)$PTHoV3phGMhXC_ zFfCgSJP!xwED%I!ix>3PP*)%w8#vl}S z_J{fE-HSPjpWsZWiHL5;BV(eY+3Hmt;0>aGrY~lx3P8v}(uZdCw)ATULq+K1=bZ$i z+P&8+hXF`Hs$9Mt9u^jM?WSPgc_Q>$MTX6ywxc|W5<`IUaiZFjxQrwqlBWF672{1z zOuEv3cbD!OeZ90{Kl}QoP)JZv-;qn;7cT#^6KIs;dN8%&dw5Uz$?>LOJAR9# z0?2yK3u1U!li(lC4ndd(5X< z30^$lQe`qsGnA$_PKvI9pk%NXoX@d(FUw zf~laQ=;Y*Z2grFhM@MMH&PjnbHczSJ zEL;+*X+d4ngIgZQ>(#YIltFI*V-ysy#mdbuE}DCK*U!sO`W{S7Pd`2Cd&(sVNdkBb zu(n;le%*D=4Dxo!RjPfIrlF&v1ob0;@7{YWkNo+Py1_p;32$tRAOksAXPra%4G`K~ zPXlCJHjC~AetDCUqG*3oOamSjAbuTv5uixDjgBUyq3QM21HRb~5DkiKbYukVIg#ET zz?l`TI;yDj0rR%X?LYgl@)m?UsBY=?$w^7btX{b+r*E$)Djv!|c!1!iN(H6H@8k~@ ze~21DwRzoA6d|(m^16_Z!QKXy+ymNCFM(E&)q#TNF};S3jpnjiXS^>D_y2j>M@E#EskXQEYd)dZav zx}Y{Ws()sQ1<^9XZxb|sJ%z5to#Wl*C$^T(p#Rsn(Li^vr|$qMLJyQ5gQ&^-u8*Uf z3t?5qE$#S{fCmG$CeHxhdZ-{lHMio#=M!{gXtsTvY4+EzQs{vP2UnWM%144Y zVi_6h-sYri%d3Kg_t=k2$VvYsBgkhXH}>tatIukpg5~2oM9l`@E=gNnYR;EXvLw|T z&Q~0wLg~==eFAfOfInG=%6;%BfB>sQdJA~Rzeo%Kk~45&AmI3&9%*AbKoZiab;>f0 z*OiABb_&7|6i?20WML|pI)TS6u4`bSjvB6ax7WF%XWL>3L6q?@bjb7x#`C8T1#uJ~ zAlA4|zC+Vc^I*paRI5d>E*3PtKF+|S7Je_k4k-%Fi$I}s=ye0`C;I^R+O-AIv?PA} z$vW2v@6O@j1GME~8_9Wka+0@@1Iku!?`CF{=qNZ`+1X@(Y0pG^r^z^jOhMJ|lq0$}R?WuuA)nI(zqrVPEWwn5T(2$bE)H&Zb+{RraM? zi_H&sH>eCfyxBPz7catkDI90$t`Ebd!T;X?t+R3qQ!EAQtMef6uy1Y{a;X$u)L$Qjnf_~>Mr1k|&p7UC+FCRY%@~5UQdFsG=wA1;0H_ETR zM`X{8e>06qgT?&2#&YArBOMN-;cKNnH8YY&YmLLQBx93nTunQtXBub?=xQ>80^w zXSWR~Ec~0jQi8*V4Qwi^@$Q!p2n#)6_+g_$4uf1&o0GG6adGQ*(7V`JQ!}%by(1vx zYV`~rKfdd}p*MOWo}p5pU7l^VqY+{ReZb)0V5JU7@y!a1`SJ#OgZ1(6-hIr<0#Zp# zOe{e`BdX$4LBT7x=N7s|(PZZ5ug2VXs;;I6SR4{;d|Vtb<3Czj9%iWgnD;Km zIpty?1lOo?r6DV_43Y`}W2f5jsi~=+qAG2>noUiP_~WA^K>-1js$;?6GVYZt(2(IqO(Z;&6=btQCJhN(Jv1jk$_39v5}$z84!8@`TVF_u`hvl5kK=ZtTSlMWLb zjg}Hw2tjGn-^uO+PT}eZMQl%J>#qytLxuMWFf1g#k5oi=G}AMf>MCnIjNDw?o5;7- zH$6Dt5PtWDrit&hx7Goo?gi~m@oKX&uU4#(&`S?_orsim4|_efMbdp|lGHWFO}YLC z>!af9FlO`l9;~?~y2y;=H#eUHX`DY)iXptmL)o2^=4X{4?(OO6yVK3D!Tt&)tLX$| zzN83wBzWynIYAQ7CD}+L9>MU$=S3n3AeFfju3~0RJ}uGTRlvR;9wf&kFtAwHLLXGc zyDQ+%AMv!>J^#4+s@tBcf$)zs?>`MuuI5s;k`}cX&2<+M>J9~%IMztga$hz)eixn7sGFxL_oy)GeuW*~>ey&oUJ4KGM^_>&WcB{z zp~|$nn`u5&^a-1sqHnBUvEMacA@`>c`zcE>sW!(p6H7kQR=9bP0nM#h9JWUg!S;tF z730UZ1_m!nGlGfGHaa@Gti1eqZ`E8|*aMny0GH@GGf-`B__6o5O>~C|nC034$w4I6 z27@;l*gza5nAQp~k9%;-!P!wYw^B#H7WD*p)n@~i6;a+Y*Hpb-COx#!bpmWvHBHCs^w5_&6b=vSTF*1Sy!l66}t$~6n06k!&kP^Hb2zA9r!H+XphgYrFj*rNp z&EW15)Z+Yf?=t$ke8A8ufquL|Z7-&JZvZzYb%clhoJwp;ii{-_cT%_4!{CgsK%h~C zl)7rON43xH9MZjgV8aw0w%+HQIr*QtC*f;ySp$QE_k@JTtk5X|1b$VYve~t-R@5C6i$_) zcXB~Lh;c7r0B-mh2>JcLj{yHPus{q9s7gQ;o+jnSd!njJM;IRDBTDl>ultWcaKP;3 z+wbXVXfV!gdHl~qap}}BaHEKZ{a@*W{{CJFE#=BC&hKSx|F6x~{cmqrqzK#|>L?~K zA7fmAO}6C`DRSeVHM2mlADw7%%<^Vh>88~YN*?yRp1yb{np`|JKcqm~|+a7O4r z%KYR05OIo-pdB^&=SAU60(?=rOMUnO3!o$Z_tDGN8W@FrVfSMCzXP1p3Bew}uhRzj zA#ND`{XWEh-}ryqbmqAnFe~*Aza-dMVO|0rB2`-t)W3eC{yfumWv<8aF6;d`dIBKl+1m6SDf8c+y1<&Hu`#(nS;t`RMR07(i;7e?nn;90+bOL~E6sDFSbhFIo=TIto?X1U^F@Ji5RL?6RWWbg zJiy-C-6f7_gVqsfAeUemH9G}whF4?L##lkS{6Ba1elfIh@9#bXXJnyjZb7ObbiF{~ zkg;h!H8&q-es&R;q_2tuT5Bq*d^MKl!-XV(-VoLyMs##^;LoD;w!mMUX^Q}J1Q0|Z z-+6%t^N)`xvJ6OeP~##+{s|lpx9&tmX?;*#T0D_KLvpi6OWvLkdXW7=KzogDey4s zw2$(x-JU4Oq7f7S$k*fq zW*u#U$jHb59)^RB?FMkXdFuJtv`~ky!5M=o7A)f}TwodleP{z{VHp`2a2B5*Pn{2B z4DCh(meAR_&c6+vrhcUp) z>x1KO=U$OvATnUW4`Lil>S+BR$X0N$gC7?PAle(Za|kzI`t^;w1pPxpbgVyv)p>R$jl0kC&E|1XA0}!vp-KS{q>_HXN+1_b4Ob z-QdqTJwC99k2*M%NM)i~Wm#ESXkL?et>Y3BJYk#&BwX;rm$K=u%LAwX?AbFIoW#6# zLq^O~&`IJuSQtV4Fn;z7-R#H248qm{wH4~@^9K)-RQ^tupUdHNP%SObKm`WJ7BnOf zt-!^+=QfplXnXF+_w|&2z=EezID+&>6)(gOTJ!r^0hbSS9u6+$dc^sI@1+ofU1~VHQO?DH4pk_(PJbrvVt=HYnEtS_=10--WGcz!& zg5g;P+emvKOw7ajKR{{4G!?f{#cXrNjTd@~hZ z2$-G}(?nDD@5w4F`?qdl|DA2l)?q*_IyqSwZUG3t<8`jqfSDsj=xVUo@7u^T#+E zz?eLQ0Jy@LwKcuH?%zKd%L?fQ%#K;`QSF1(Ez}ElgEM}svQ*U z3M;K#wI(1`B{JT$&-UvQIypIk9Gl-1tiqfO6atx{&#$#LAJ}C0VkZuuZ&ZAdQJiz~ zw6I{liBC(<%q)|u_8-s7kW0apZ{A*Vf)4#?4h*fo)wIWJI3muGzM@15$#=@98g)Ve0Ga%}q=iAdf<<6`4-$ zY;VKtQoOS0A-%A$53B(2-1rA0>X7NMf!hCzvj2VONjCU)O4*LX#{LGg0XZ!|c=6ed zFHKJa1bTi<@IZ_aS|qga0AkIHz(AlCf$6s5O9Jv7_A>ZeA)f(VF*`d8Q*LSA8vI8QYszit`R<8XO1?<>V+J_fa!#;2i$wUz$?QHz&aHfE=?a1+DZ@d3Dm%lhzNNfpA%jC#!A~UK9hfb zbK(KE}Is^Dv2FnNADbq-$0wW;QFP@0SDw1Xv#?CrtYCJHqd?yqRbO&Ftv@u->q%(Cx! zH7YQc>wpm$0(ZiKhXXWw2<4zR;9-C*tfOI~IlsQHh)NR6DLRvB7bsH5~S! z7--Tk#SX4{6gZ~f7KMz9{JjYg7TAC*3PI}x>fA9mU#CzMMxbb68}(UDfm050ZPv$+ z=tK7O*nJMT;L;5wc#&oq5GXLYicKq+(&ID>G*rc(YH*4JF$&}hR1=sc-_GO^p`xqyhrhC14B$%-iDP;wW}!PTF(& zWvJ+MgPIx4_YK(T-(&#-2P{Y z_$>mMO1O*nL`&;C*DKXr9~jkNTU*1X1y%{Fv#9qzpa)AZkEEs1_!`;SZM>qmhmITo zTwq853+dxrH8}0}qEzJM9^hw%WljZk9{5jiq1=Ed0UZwi#{J)rmC<9WL2qE7B|Sae zs3nB=`)g>`A(VW#+Q>jRgY!*K|7^ntWC2XF#7lL-Y%i2Rv^_|`=_3?YNYoSr1P%cE zAfHx%yZ|ZJl>8e^B9vi_t}sBJwnyc~!t9KM#8lc_6Ft1X!?v=q*=kCElGoVnoe=wIj3M2d=cQ_VP!X|@}n;;1zKR-}rz%>Oo z4JhU(#RiSLUZ*hkM*_AdLc;A1rZjLFV~)H9G}8zJ7yvTZ+1TLNu^lF`nt=PBfu(Z^ z7YO52aF9yUb@cWw?i><^n*gf5h>6#?>;i>udw(CM>7JvW78uYGV8cu}z&f}=;QIwM zQ|Go`XVFjhR;IO){?;v!_o-nj0sPGWnKA%9#p~(fe}4J^9r6~$!i5@RXi&tf7NCp% zeFlXTL6ix1p?>|)@bK`EJq&aH_u)FMqG*?<0kI9`9d}6zCbVH&#w8`eh0V;)4*qQU zJB!2DaxLliS%8*9%4whn{H31W+QIqSglDoED&hB*^XgC&=S24hwijh}0en)=Fv7`k z!kH;iJd3o1%)Bk%F>BgS| zsAT|;zicy3J04%$05=T;=+#|^zuyqObiwz=Y2U^F{`}4Z)Ca8H(*D1nBDfGT|NMrJ zi#W#LKhl$ah47z@sQ>0KN(}vlOD`_{jqo3k(SzUkKaaXw7ykaOpT5gDf8UM(j{gt8 z`qf2|fG-U3$oU;l#X(H3TZjOw9Ojo_o~o!{w|PSyblK}^f=(7c$k6xpX?hLWiBA(9wlQS)C$k=RF|&mIP7BCy+vSRV!Y)gLp7?JqIQfB5t_w-nvl8iT)@_ zM!PMMI4(lvPAj~QqQerImU*xmLB)u8kfkP^O#oj=BKg-B8h3uCyHJ>^$HCwC5P|p+ ziz`L?0?vCT>cEOodO8fIS$Ex;7XA`#HQ-m8VxPPNIxTT@VfVf6WEXlTes z)X?V(7PZGT7&ZY7yGIQN6M)_5mwWT^1f=jb<5KEk=mXEmn*TiFp+xPqU?8j{G&P4o zYv-(;ixzkUJ?hx)aQNksPJ5qOs=R`n9j(664esG=& z;vAT5e7EO1&}m@_fe4)V^fli1L)f2R&}i7 z6c{xNw6@?vvsLdv+XYY%J*cIQ0uO2ZW)s>41{)mP5%?CcsSMgxBg=Jkb^Z6t(>hXr zfU)Z970#0unllGjK?8>`mz8p!U~j=}{S*{=z|RB{$({=dQ^%7B(CB=af*hchjIt7$ zeRVI@Yb6)zU){t8n(hxKGx*aV<;Mf~`O#yy&B@cjw6luLOvpmeTOR;1aQ4eO?R-1F znN7crC#m~vR_tt+!aVu06~Vi|{=bdP5ST%?`co{p``YFy497tQfMbB}C<}IRcHRNh zO-V^PcPB$-39br$T}7HNsdB`IOy8v+-Bg)&zJqyC&qf2!&rn#;kA|CJHWC&Cx=dv- zf0c;hAXI|)!4n4o19G?rgfK8ptPN+Q*2Rh`f|Xp#cRXez`Oo$Y&(-}zz__DpSNq7X z5xN<)qX&o$fAHOa(e74|g^i6~nMpW-F*v}{6p;js_#C#2ojGOJ6#yv^Y%W5&;q}%@FQwI%Ylzyx#TLr5-y+<+Tu8@ z?@r=(5e5$bfYIw3jaF275J9}Y+X&oC)*$8^PflvR^hgl_7!ctPxf7t{%%9qfPZ*g@ z+k?C2-DsfiozSsXo^6S4?d0W=`tB2z&^-VcR$xr?+3wi%Srk$h-jalu;K%P8frvYS z8xKBAi7*JTN!MY0_(dl)eFtw~uO8oqODlz#br=CKHtxOIXb6kzzmqft7D1)Kico5= zrRgt$N4^Az0BWWCzk*D-@05476(iSn_?xyOnbLg9fIb2IQCO%bTEZr}@EnVpT-4_e z*bUql63pXLe9yoVvQfL3iVmVLSTy5+?&;nVNSvGQ+VM+Y6q&}G41U4;2x zi7?1`a9>7js^Ivava(FUTjqa=G!#8r%KR$r0T9`)c0LY#v|`)eZ?m(r!@$@t^zSMG z$OHj(p%@3z#|cK={vSUoU~>K|E4XJy0UrLE?*-OFAAS}FNJluT20%gJBdDp4PFI`H zRy`L}nEY9Sv}zzP2Vb$TU$2UbgB~ERvZ|^`ueR(r>9C%2B*DyZ^O-BGU9tbEHEk<| zt)P(56-{q20Z79U2L{SKwv3C1gqH6dST&u#h_Bm@r~EO48r`dQqG`N zrQh+`6t$3B0bA4AZjP9UacdZyq$ziT2{+oN!OzRvohmZ6?skP%36L_{RXnf?tOj^m zk>FzR{YGJ~Rj3iw?%YPA3Gf^mE|CO&q$blhD;E7N8nF zh2dwr>f<3@ds{0j7S1=W?(X0WTYdLL34Xxu!>)0K2<2D@|=x77UAdez)r>sU6E%T0rbv|;OKe&;3-_+b{iB`9v+?s zC<$+6KK3?~BLZIA@Qp!^9WFLBlsU75%Yd^IS#g6Vvl6lie4$zsHE84Y8|b#1%v{sF z>%AkyYG+yx(7wRF0WyyUdWgTMw$>B;h1>f7gSs~l$8zo4hA$1GGE`DYgp?^H37L|i z5He?Kq9jwun4yrONTxD`WDc2UD#^5jkYq~8Oy-Q&yPvK5d7k&r_mA)0-tDuk?cUaM zU(V}1kMlTw|wEz?sC-N_u1px*d{*oNYZ%>8%FFIkAw>p{<#0?(DUkqYa1^D(2IzYcuyVF}W|d!~`&l zX;h5cy$4kTp~965cJHBuw?bZRByabToiRvji>NiZ|NgRpL1#+ROZ`&EpI8$yqiw5YjeP^a}Xi zM#%;T9Oc`UyHZ8GUTx-Wi^DGf}9gLXYDW?ogE!TNX^DFN;0aC zWy=*8^64MG;(sfw5$x%on#{D-J4(*h4@|+YZfy|urgtbQgUS#*(^sZ#Y;Y-lULh8` zM>W4;m!O25o}M0VXnU?BMxQDR{vKtEy4TKuI~c`%Yv|Z}=knT(Z$aHf)fr6HtH?tY zR}4iW)8pb!yC<_y>>&x(1bJrA*j`_OC02~T5MF=hF~<>==Qaq~7>_v2i{Le(Ou61i z=kau{84;e{V5lzqCmxnCBYAlT%E!Q^gqmN~t`!z!?R&TSfP(wAe;$=xp@J=*-?Kf) z?iIZrb=*mF-%dL&&y5hh4j5&}h7;}ICKfuaN{BC+nE8R7Suqmr^!n8m)ETI8ylwi&LpYC6)j=uB< z4hQBwF)^{y;MyLK3^>m@(XyC$T4GoQI((Lbj<>RbwfMj!gJwV4QWf(AUs&`2h5k;f>0>R#e4M#Ph z%chbBYpLYwTGHjzbUnvMHV&xD#ca1B)Cf?Ibio48=>^b`l8?T>{2LCF>G*Y(GiS~O zHuW#A5aAaQM-GNx3aIW!$EbKDF%s*==HY^%i|8<+jxoKmhI+c#RlZ@6KP%E5lJ^V?G&Xt1P^`E z?w5*@<3*&Za#r$$xEjux18iJXUhS8MJ2(c)A@Y>01$*+F+3L1S!BelvD|I29c z6M(axTpKEegV}aKfomkcOmw20giO)uf8` z`Ah@8FbpVWE;)H zq5l=s`ude4Ajd}U)~zupV)W_Y zr`PWClSly%H`zD5)lVn?rXyb#iL{0hM!d88`ghbgmxnyohkl-$X-0K0v8ZSXW^*Nz zyeuGAsdhLR51XsUhZ3{Y_w+`@@yT3uFcCJ$h?W>}LY-HqtFZ!-pj2pU8yFZ6-co`Z znE7CYzb+rBkD!1DM#hn7RYylhLZ}MX;HXNs^h!q~3ShI3j=X$25r;n_BZAn&FN6ot zRFj;?zUj%m8QgWp2A2j!n+$rCUer~j7wMM;3opOF59;D7a!D2hi+x7i#sE5@T~i;G zX%C{?9>+2~DAB$~Fpe}3Z#(e0mz34h(aA~vk4z)pyHug;VRGh-SF;&(#7gddSsIQH zEXP5dxs}r)f#e$HTTmz+?d_@9dVqB&f?AYSj87|M5%&Tg_)H*CBV`N9`ml`0PYrI6 zIsdl90S12DFXxVQ6K4S4nC4{H z!IrPc!;nu(&Hi)z{NZ4F!qeei_ZX|73 zo^m*UqC5gP0!!NS!2PdFjfvBKm+*fo)A%31ONgsdyR{e^OhX&pn+ z@D`H#8ruHtD4bv8c)uaCAs;P7^z0eG%j{W9;p#7gCT0;*hD`VQw3$#;HzA$yr|?Lf zLmNmje~_uVk5{rd4+P;)jNRtjeD;t>V%AfdGQ{br9Xh*^;NppYLDen*rU&9xPvr}druiAO>->(k;|%1t6|R_dcT958HVz&}UX zIT&h!oEneR+FrhpXNO&2#+ye$QtaH9skJHPS?StBC-!B`@f)8v?=k#gFTTY^l%!tv zP&1UsPdhS;nDXbx(2PV5LdpF2fjwnAB*ROX!i)w-BCpGTF@_FzCsHhW1Fk$i{ z1t~cM+3dGL+b8>uKVZ5+lBpOOAIGs=$uf{%OakN;=}&IeZKF@kVdRdny&q^c65=6y3{tg2L%8EMa z`faVP_DHGdYD$P?0tba}lD7;^eP9*TUI>}WeA-A#Hp=}WN~>=(d;`5;pYCmLQoV@R z(WS2W$U+KulF+c$Pkeb>raR{;MHHF#TNU%g-ywUUjXT{9FRq_89welao-iiDd7T|wM@;F`cEOPr-QnVJY(gq4V1Husj=U7R994nTU^gQJvqPh zTIt>y`MxNpkGFnF$8q`9ntZYQL-G_)x~VT7t(u@=D|Ki5_3W{lY?ku zO4&q4(K-1!ZV_1W6fKA!nFtjbPW~C13rM_)dkd~*Fvi2iPb%;0qO?s#WKK z^HfuzAXOs;FfG3LD;f_j&(E87e?S;_RJ+IYbl`RkfC?{!T)VW1v>dQW*i)+Zdd?#` z*8vLml$?<8sB;}U-i-3;)vTwFC_PW}?9#b9 z_2{>O{+|P!yWae++T|)Yq;O^HKspB&oQpwRfw@)q{O8$gS4dX~&gC=IEurz3gE^2Z(O%>6;;iYt&SHd?jh9oqWk73;%qE7a-7m}wWKm>9oK+B-Vd ztK?Sp2{IM!-{Se6J-8OcVxHPltg630g5TN}&<5d<^o_td-??vKOe%Y(nuNOs_r=)M zupzqx{NPj42wbQBMVlU zRY9kPh|UB_=792{owimQJz?W(DwN!8-l(^MrJB@oHETP1bFCuio(0IaS7}Ho^YIi40f>rm*ofK4V(LK!t1B~5ZV@E}hV zu;)4_a5t?H6f!jRNg>}{v0-+&E1FcUk=I{X@JBZ2rq(JR4yHzR<6=tv;MLmWq}F1U z!@GbZYc@OZ&uedbtO&#?4Eet_k?PKDV3JX@RU1HttCV1V^@-jTps#qpN(pat((pwc z78@nmNivUSZtm#dr!9N)vA4zc(LE-g(svWu|Lg&!r+!G| z5LhLm%%6v5H%f*neGxvN7}t%3ow!c7k;K`jM(Ih8zZucRoTAYFa9%?y=I;jn(r1JF z`Fd)jOvA%T247-b39J{aB%Z{ils;-4W-w#Q`gx1_X&r!9xA+gvABnF z>>DMYuIrpVgQ|8xr({zlg?bx|o|lo(3v@N-+b4?-VOyl7DKkbi2W=#!`h@N7CdV_c zMxV{4nN#<3N;G?!;zy538YV)}kSpsI3{|>gZ~tw=p`f$js~SmN8wVISDq^q{ppwUq zYK?=#-xf->%K?&N<)tl`f7cD#W}mz0X3@~4I8%TEx+%BtSPkj&W!UlDGXjr(M6&Ci zIAQvASJm612*-%k&)0KqMlf!tT#0K>+76>!*FP;9tybyN!UhEtDPgdG@k-U^umuq>i%jZ#5@oA+AI{VCTY7jU4R!ya>Ih~b#RDnDY>q9?c zU!0c48?9Q>ZI${D=`xAta-ZFmg+mojXSv1hJe%d7uKk!Avjudc@132JEP;(f@mXx1 zEQNNPDdXkX**dM}8aq=`@l$mb#ar9=*`+tc^I^#A2ZcP+SpD)&oX4i`u=qMQKSN?A zEgOt_4*!WCb~#i3V7afGA^uFx_GZ2#T=~0%$84@hUU*&_@oFoZi`Z@A803iIz4TAJ z58nu_ANc38p7LH$$qfru=xeXJ6a^c~rk6p34uw4=0J`vd!u* z%QvMsNf^HmYK=j4iMc=N-vDs*oOpDY z_m`1s8fS)|k5QL={w>~^yZOM+_#^V;l13R;g4CU78}6k3WXim#NF85p8Ddvth)dRD za7ubhzlPWQ%Qz@3bf>p!`-u9uK(h`ncD|(iuaiKf(!<=>)cdbjl1(arA(e`0G)eu( z%Q%)DoqOL+_AyUA6}rzJ4*u)*byn+AQ@>k?_*OQr?7kAp*&dD;<|L7!c=-F(EC8Y- z8+4|5H=16)8d&_=yLTt6wB}M`qu*&gwWh8cW|PMmR=BRb)%vqs_OyNwI3b?UFC({8 zpnbY@m$9C7SNO-nhnmUCuy{E+nt{BFc0Efr2$%qYxq3`es97Ptd1|rc8-1JCrd;>4 zGKlLCxjaDo~B5zVZ;d%d&WnOW_ z=_2!8zXM_TuwKuAAm|Qk$(k^_<+bvnhEa#8`nRyk7uG-W2s2!|46YH-uim zqxo0y=-mRu2JuM!U~22;dozMrie(+-!Y|X$YIKLmPbq_K9`EQ;qi$qqU`YPw1g{dK zT>XgKW=B5Z7b=8vaL^sSFVlf_IgV~Z+W7MBj`tNVGoJ}L7_=<1H#eiC`Oj;08oaEb zZ`8x7T>6`0aW)umuMqD*C<-?$rYAqwwj`$<(CMH1^0ritujRsI~7uvCRZK9W5d?QW0-X7e2(|E$MnV4_Y8KV zriwGumUN@;W;&Eyo)m{|F3POK;4>mg(vsVp(}?H|tK~6T^xR_SPlpotTKTZg^@qQ2 z;=XK?`a*7}r-pW__KT=PC59PFN0Y~IPCh)7v7(T6u2H8Sa$7V6vzdUcZ~x7rUL+OG_j&G%f~M8#k?#@tmrwI01Q`iM>~| zgioB1XAeS?y<8xTa|_LO{X?<$mKk9D%k}Fn3x2Z@Dd;rHD|_kMXd!|=#)}7&*G^en zyn0=5@3(UsNp+ep#&hQ7ngxw!UmjH{-}?CG4=%r}(0p~TXfvN!`Kkt6w)0VqZ*?dW z8)>`9?pi4WK1N)SRg@{SBcRre%)lyKWMVc7-zqX*}D%!GAjf3cC&qj`ee9Ae^VqIsFG@d(C_9_&=O$QOnO!TAb5 z2wMMO;^u)^irDS=R#yp@ir(JEF)h0I=5OdPwlukoTYhV6c zNMF=#F=-l`|Ej}#lC)Q^t~~G6Y@3wPhv6Of8vPzL zQrlhgq149oE;|(Kx=t4eBmgca4kT}CotFlQ>s)V6V`}R1+WIorU8$(bK2-f6_~T}= zfLlno8PPtL-ujNDC5GoWqV}!C84>8X0es@@(&(uabHTmi4v_uX>Lc51rqz-=lM#&F zByE08)Y|yg&w)5rMq4W{uh!uPJ^iJv0?5bd#HG@4N2U;{!v=Hd@PGvS9 zMYoHL_RReBR!08>&?3sozf)70s5YUQjQHWw;g_f#USJrFy>}YWF?DIY$3r{ed`vIB z@3TPzbGiHq59~dRt%V4hY1w;%ICoN?Z^65U{}}&GitVvqKR2>0a8%QM>TH8u;$Csx zZ;EXTiJQteyh02!aohDJXW4_OD8119F%i;sa3ZODpxDw>&GLXhoR^-eb-72Z>`;O8 z)adAF{i`n-8T>gNhP_p4q(hZ5bU;-d3a+&g?FzRtv?_3Y+jcG`b=0B6R3Il?0NKKd z-W^)fe5R}amubZybAefVFZum5oTeX7`nYMy`)ao#(yHwHy|U&;jg03Nq6wINo6EQX z^tD%>odrnto=j|9+?A&$F`THa+7CBLSV20mY(A+C3#-k%8gsMW2# z(DqP@lKXs%NhKFc$cA^#2+hmwg%SUsd9WST$KA{tG}B9WaU;Y$yn}iJDcN9zKVjB% z2RZhi7NcFcY~fqmz5N1GK0R-^9#|cJQy*znzx`6I)})}}bQXn7I?{XiZMzv+?|q z#g+OJHx;xtZ?;gA=9Q=|8G|g@V|e-v8{CRT?)DsPlu~wko5n(UN(f!nBu@7p$Hl}EanVRaPwGGXNx_PKq z z?9dYqCJ~G0wI)3Ma`?0PC(_|EG;n}}dG1J_tqLN?XPc4AX0{uAqA}$U$N%%@HYMq3 zxW;`Cot^1}QDahPwwGv$)=075OHDP-Y{#SA<|iIvv)B6>)m&^DU)rPbmrb7ntg@?4 zxj`|$Ih&Q5lxz!Irq5|09iEj^h8CYonMJtccE4Am%mFiDVl59hlDsX}n7{L7H*2W1 zM(z6e)F0pAWw95Ui*=$WLzR3guK>~*(tX$K`%u3IgXy=oR|cb%kDJ9BPb=<9K8MhT z(Lc0(xubvjshbqPwl<-_CuTNUPugytWu!C|BRA9kOig$xisL4=rrfgU#_U!K~o1PQG0P^ z$qOPsxm;V0l9D&V#+r^h-Vnx)!IrU+pc2C>Fe%hliSK5m^Smd&b8E6B>>Ep9c$WKD z2n!g(-n^+!9(@LOQ|3j7!@-Xf;!TCmQh4eo$>no5Te9meUw2%$^SSj*z3|k;(WThd z>p7t=SH+_dqL@m2qVyD}e|RWTN{RS(A?;#v{IM zF&2CZmoij;0xz^3OKTgQc2Y{WuNj&Z{z($tYuc~$C*3x}`p@UDO#Jj=Lg8by`q+>Y z4&lW+boj$j&3MyFAxL#ILdW>dQOoW1yYTMPe#KESB_GGjXX3c=I3esS#yVneDUAER z!tA@MLEF&p1u&n8N<~~Y2qtqiTk6|~hw)~Wd0mTQe9mpbF}F`E8ubp-alNOiNgP2` zym#QQ5qFhpyqqaMCvzc0=Ainhmj;`hC^Im)^WJa}%1Dq8?c|<+c`pCa>xCibM4V^u%?ekaTYj9f9$I!Q!{TFH^;1%;I21O8M$NUg zIaj*&Ad{k}3ULXGS{gmZLfS|;AZ3Kxhl9E2`LGc_dAXSgGu3~4iikO^U0^fGT*D;t zZ0@7Gk9eg^Z|pNn;-P%`EUeK?7$gXx`&xXAy#3|ChHBydt~s+?ZePhLrJ+Xoss7x( z;?uHQllR%a(lg(lug8Z#&`fw#IQ9J*tFwCM$BD^}Dq)2T2T0~mpN(h~pA7xJcF`_6 zVlg`D#eRFHwjXT!Z-+#%W7pW#rCa)+t$pq2uU^v*~0uRWJ#a9tXlL8d2R`s)_htU9;$xk~+bIKVWd zk+fK37{Jd=8Z{KjkSp*vxk)`-x?^3RrRPNR}64R);iKcZ6f)GnY#tGMB2ERIoNb80*oAKSbXXjj|Ns4{IHN7ZT zdfqIdl}oIeS}jL#$$o$9Wkkt!PD*u32S|4Wn4XwN=zQU}Dky2KNEXeQ@rzCQMKMak z*`}_EF*xDweiue7y_VX`a6F^1OLOm}*jr zz()1(?OBU?1MhoSGhdYYk(p5=s9>YHo|;<6sm>W)MXbx?nW0oT=aeU+16~@@pEkyN z0H>dd0uZX*oRTvreJ1Eg#~m`=b9#1|MJD=!#uj-K-)A(THh#CBy?wHiG9I@&fVy+T zb>0(ar$ZQIj8NQ8_{God@ih`v3?}6gnupc>;xv>pi;$*;Yc7bEQV(|f<-(n-r}JqWc4uKSeg9$X$7 zvZ<+I->Jn}0$X^tCf|f;jslqm2W0gA z;gGm-cC)nyn@DR%vlKr%_#0C1lEwagD?ECV{jA?BFpr+ua=}PLZfw?hO<^->?akw| zE^d!WR@)A%b4gfHb0)m0R5Wm()x|tfh;O*gLD4OZWvlEPcH3c3R0&XKS7^ULAw%ZQ zTD_R_GP&^Z)@wHn=;9>c$4!Jn?YF}2weBmH);%6rt*H^k(|>z|=P0Na$;L&78LeEm zvh-@C7LUX?>y3Ut_G^JO>J`?=^+<=arF%t_p?dF?#MPphJt(D zA*EumL$SY4D$!`zCIq2Pev`+YK!XD8d0zPmSxXy`rQv5@{22y-hwBGPqHwpN^vH@X8q7I-)QDr8P{^wW_!Z=ckcU?<_vfnN-*-z^ld`){rr_K zXy>(V61HP3Bev zYKX&csbdqlQ@q&sbN6lPq^gVAg0n#^gnSbSB=d22gte2D>gTA&oadi!DI$TBP z+-W7M_bOYH<#D{s50wbdxmyO+ny{}(cn)mz>|u>*xX#VLQN7ZZJ4lMlrD9{;on%4h zIcEBRs>^~Qb0HxjcsQAJ9*Z?nbZ<0v;r?P7iLG@(B=TrVPH1W>bA#pBFzy60CG{|cso-9vVzq;3DUvsGo(~wVIQCYlyEXdY=2mtg9hnoLG#`q+ zrJQ0Ji=qX}xp&TzVs~-w(wX6$``W5FkZR2Qz7Xk$A{Zoei~oAMgiy4 zSkWRSi@*wn-)Vs3@zPji8{p=VXMAAD33X*UKY2fz^N^> z|A_+LZWPLJc>5mPWi|_YGKJA{FBxaLpI8#!s2sWFxy8orm3~xgM?Jf(SOL~LrgZB7 zts>?dc+gTSUK3_g=1G3FZHu~{lAa)KY;Q>G#t6zLZQ1oRbJ-@37a~aJdLn!-mnbY} zs5#PbHN(VsE+v5}e&`mdvw{;2>@5m(e2gGiS7o9`krce3uKm$OXkIE^1@|aM*j!C> zHC~?BG|YFxQ1G&-tiOA7j%6nZ3}KHRS{Uvqb5lxvQ7W4^CVpt{L6`5_@Hj=Qu(ZXe zBpK_LyO zb4S~SDMn`7uli1V!~~A2GS?rSuPZqnq$_GgYt%!zdx&Zv(mdkurK#yx62=!8#J;;1 z+G!ANTF#wi`eDWz`Ano#-)AE`mwGu2YnyZv4gRFQ?Dw>Koc= z4B9o>UVOb;O#gDAd|J|&Z)&+}Hu2h`j`-c3y2Dew{fa&txAtk5ze-JgEay|UX)e+r zH1{I%xn!|b#cf&#hCJT3{udSiY4wWdck8v2hz_m=N;&{{2)W0y-#;0O%y>~PK& zP640kReA$;He=wv37C_UM340DpPZJ{S}pQpB+BCr;wpQ)%Opgtx0-#bv;T$&31d|B zKqtU;w4A2F{UYcgtgho(#!{=NB8WuvQaL7yk)46JYz}~O9kAl!VBtpo{Q-Vzl|aQk zcn%T*NZ4n~-GLV-W`No1Typx5nu-b+!84McebFd<@+26m^MJyTHL0#hFV(6=L`G&7 z{`AA(@;$QOz$5`tBM#+<6sHSO^kNx54aG>UVMn_ks82C z!z`-|mcZq~)za!~t3Q1%`|Js>9RPu=2pC4dWFIq~Foy^SC{>RehwMta31G4ze|t`& zR0-g((5{IBMwCP0u@eYCsLg|~cCDauC8@*b{pF`*=CX_VZ@84Ec2SVj1T%k_rq>M*ceP9;x8G`29W)O|FqLxfOMe& zQ4AE8H$C_>AFbVI_p^P(gwjEI6Drp}pFQ9~B5Ju!t-3piGk)$>EiuSQJ0N$+g z^f4i<1Jt$~K$;~*+13F(Dq`K!)!cjx6Twk6O`+E@e|pu5E>^*9FE5+qcc$IJE-UJJ zGh2^7clnqoyO$?GPVw=@8Fu1-#0dpW%rwJ4v>uKxJ^;L%v$_ntUO6}biE1$f!Zr!) zETEUbqp6f$t;8n}t44imZl1>QGIERb8sg!W6h$Hyh-@euPbVDd-}A)bUxip^EAQ)o zE0=dQN&Z@p47*+Hn!-|`jz{zP*@50mVwZjZ@;{;1;k4IYA|WfWrfBIDezSR9m~@x}EjbX~db`pEC?e z=D}{S8K)6IR7-zxxqp3bRQlkB;foiT)wfRY(bAoH#ntujm7vzyQkJYdt!eqKWI#E~ zXF@X>%GbJvq}qFW$^kqDpsm_GudZNncvuwD4*=sMi0G(eRk5oAI1tuWjbOmidtc7R>!pD1jrt`Wesp(cs!mar_$ z#&+qvIUVEn;(;q?-~FusuWSzAwa~0IX#2SOpXbyZm zo%?Su9NyYJc>SQ1uY&1W4C@+jYVuowb8BEswY&N{Hg^mp#ly~ zRn+j8#&1X)OcU=>aBtKASD(O*YaSnyIS{a9mM^DT*s-l#t9hP><&@1#f}u){;VLC0 zO!kPg^Y{@)53_H`5E_}7rqYozAtYdvR9@KPl(R^A8uIHhmj>v`82xbphY;FAr7+3! z8RMuxmvLJHx}q~iTC1nv($yYFC(R*7f}zT?YeFSz&{n%~^)1Fl0R7B@;{iPAz-rlW#q^d8JLJ`_IR2{Y&cNvlhC4JW`njls;vOoEZT z3QGds127*Tr-7jBT+q1&Xl*{!e84ON%MbM?0ywLz?B70y(MQ(+-16v@O_ljvi5CdQ z;(K|<%-Uzd&Y-gcq6M`GVss*o$%KPzoci~vMYTwg1~XxJ9bC~4%$0bBC*oihj61lB zM1nJ6jm8TO7*4LDB5*!X>;)9>jT05eS&F^eKmRsAA7*^QgX#LN;I6f=heydZAE!Zj z0Li%o#I8uI;+tUJ`n@Uz)EFpoy&~H|z`6m+E`fp#oJq}_IS}_UAV&?}7kQzeg3xqd zoal5MA4^Nc-(;49k#!QBk!D(6KHX62N$eo3!IJDlSpSOYPx}#q!DT`TL+C*d@X8J) zMh-h#ttm3zU*0P;AnHhqz0f~B43&Z9xXoMC6JS2kry>wAtG|c-`L~9id%>#*4FmQd zVBB*%EI?ZV|0TGVAl>Ms&EMa~Z)Ro&a<0Son;vUpm%tY7$5m95Aa5fDxt;jtKQAzQ z4R?^Gr_z#(5fX3@7w>+#A8)h9!7si`cGHG?aGoo1zAR9I{>#Kp5O7N=E9rSl!s`hjZ^uIk#j>l%*|mBb-RuD|tr6r0O@DM-t<=NXTj@C%I5 zk@aY{t!0r$g*>+}oHrnN#ZC@Q%B8zKm_7026DVDOqj1RX4%mxBGKstXiQ1`&Tr4># ziQwpL6p6G1FA6+~R^bW4Nq)tep zIsLc?>WLvNHvuYcUm|es79y@4rr}1|^}DxUsixIH;@m$CX@XYaNFM(EK^_hj%hS_L zv_`HPpO4Ax3qmZ%?Z@uO@Eoo$@h09QMz#?^cEE|bE4=>Ou?rx32nYQKGr?<~b5_c+#oet&P<6ZdfGwkg@XR@L#D^G7AO!*HU%#$*1e{s; zNiPckyChKgcB0I(4`HpOnQ3ukECKu0Nz)cle)GrrlZ8ki(}_{gbA31!tGcWlrCsFDh7nffCucUFf*3w`!{4(4y5v zNckWryC`abNs5H_+vKDU!{F;+RSoSEF^zx?g0R%mke&Xm&+&M5VZ{q5KLKmc>y5f- z#oB~*|8!x81we)bHCgDzT__3>K`ZE>zaPq5x~MZ$_1U0U1&TrV^`Tj3)NyS@2kd;@ z_iiK&pFe)`(XNUFu_OL9r<7(mf@sVYGOzAW5v$&|IO12UJMB20C2dIu%>NA32c5>! z3otS<)ML@nV-M9PxD7}hz}Uus81~@`_|1DUA<4iXvSkD#>tAn|1a39o>5K5h!)Oo$ zvH4cRJ(m9dvqP2u#8<2i3cx|Rfdf|ag1GUYkB+dwBmm1#5Czb7EZj)2cZhw5DLwp< z@gB4V!d?>qe^KycDt`M20H99HezA@J07)_+_`pO3MOFfInHMU*z$bG7K^QC>qxbX< z9}k*BJ*tl=tR8qi{`Hf1b|$k{k3UmS;KG}$x0$Nw(-sE%mg2rraCw9sJ|A(&`_Iok z6;3VZPr8AYt;N8}0_xUuHByhKfjkNR{KZ@>frL#W2%?6<94%jc^7wqK`Lwn1<{{Vs zD`);fjvtN@XqF~jeg46qPCCXfbu&l7Hkj*YCF|*!OT*erFM~<~MhVCawvc2ZcIPfN ziQC|P)W*K47O1`~h#?;UfF!ou+?S-KeWwV_$S0ro%S^CS&}g0L@Q-IQ2Y@cPaj3J% z?gor^U)9f4lgNez5DXoWG!Vo@_}jHX0gt6ymsTKFPoV#GE0)`WILMg>$!{L>&v%y@ zXSR&+xi=5mq5vP?%*9#(a}3fr5Rkuq{rdb_HNIICsyW0%Oq@U}=4z*$5PS-!qJ29iku4$IXg zi#)-j0GlA#;SUE3@S+gEN{mz)LI6afWDu7vb2?_0N>fvDWn90gNQ`0Y%?J_PJ|4p2 z8*284hkD#rH!sLM-Jst4CiTx3oAVF5rs>-5HC*2h1tSD)>Shgw64qaYkJ3{?aN@Yo zQVHh0L2*Md@%0pVUMv^;KQL}Ct6iuoAbdIO5Uj6OR6P-~oRmth>?H4zX!|eSj=g1|yvttzIiCaE)qlyoKgWPds_2bTO+r)P zif=MJdOb@MP&%OasU-d^dwJqKfMNho!-D}}cv%q1)evOn7MAWp*ntW#`L054VCJ5@ z+!lRu+f>@(`JE`+%!mQ`Oy?tO`yCh~%vD0*g;`rbmu@Gh+Jr0?ERH+;3}8T1o22l7 zlCBG@9t)F_{B`1FGTBx1I8PGAj%gdOk4-AN>ay`4a}R0g7_I$K2)niA#4E$2YZlED zi6<;0LpvJj&gH!r_?%kelaXX$r)*iSl<$;k;`iYo-NDCECZbgTOa(3`joD57Blt?t zbGeY5b@G>Fd!L(8r9)rvp3H)^IEz8VfL?M?t*7Cc>D&IxiI##kINHe-7E&*`>XCvx=xBA|k?s6!)oe z8!1B`x|OD;rf8H(F`$12bvtZ!)5pBzAX^kP)>nGTFX5Cx8%=C8ul(>qYGwR*WIv`t za@L16zJX!qk&JC?n{6v&hkpV1_ev%NAy4c=-9( zfBFj$Q8!cxR4)d)F*KH}@{0AVh-Y;31pnHh|)c zq-PWli(wqwsi=NqejX$Wpc{u-0+=Emd)mvy`l(eZndG?;Os-N@RfT?D7fMfDtnAlw zoI#Ju4t_un;dan*WvH%X=|Oamor5D!A4YPe6S`-0x7j`-DAre3Kj>WuF5W<)EpyR5 zyqXs(sF{EaQzvjfom_j}3U3lO{|&>y6yld4fPl!3XIn<-k#LCsiDp9Jq@86uH8nRR zJNWrCGBbOT${D87lQO7|ijoXpu1?FRwSA5}DS%<#l9FZl`8re>XOj)GDCR4rwTVN! z)B{p2#Is^pC#EDpwMP(?N;Wxj^HC(k$3dHB`$&-gzy_tnjO8s`1oYniKxDNJ%5ne1 zgd}8S5D8MX3ABZJ%<_5|OpJ|1Mn$cHlRSu&hF(?$#uH^P2=8W(bwFPPgmMYIy@wB} z(%M`)Eer5`l5pu52uk=f6nTo@K;b;LrB-?iEe(y)+sp5X+rWWOOgxO~PN)TVpy-If zRS4-5BduIevcw)A0r5Q~B&4-f3qbvr5d_3UJ`axc8LDH5gNd480I-ZGCmrGDuI=rW z=H?!O2naZ>I^@A84a}&E;~iRASzWspkLgK-ry)`^gfQsFU@wAujCoJ_+0&r#G4dO7 zxjRSEKmrg?s8`|cW^0>G&n_f1gTVNukg4DJhZiw1G2!8RpzMm7CVBdhqtw*Wnrk{k zx3mIX9P%1`bf%Lx#k&)7QaKR7ghq=rp}V5Qy1m2r%=+S8+&B(+sB$JT91h-^p~AlQ zsh6#5kyIa4K|p>r7n{V`S&aST^wQxP!S2s-X{zie3$d;!)`HcEy_izUEO_}HDE2pi zo5|86axLr@SAcA=Rm7tw5p9P~oHDt7G&4)nYLDlB??f>BONN@_YkSpnRP51=F@o!hKUcS*sm^-p0L#|G z#M=Dh2dd9(ZD;OquKd-(My}5cu`+Tn%w0M5+P+e2eY7Y#P|US1Or)eBH%Lj5N!hCx6xR6?Rws=vdYmgO%TD%LE39lusSahe zAD}xks#&v<=MwKFt~k$zI1;IyeXw_8v?B1rmiPQF+f!I&M5+tu_Kb4}Ywg@JKi5jG z=h#ll*e|q0ha7&%nUoS^`ZGM~ahG!R`7~BWk$#aO^E8%=OXrhZsu!g5KMFJ+T%sR% zzbTdjO8kcV=GD$->~(Y>&37-(W*wjlqxezWs#6=9#(IXfpyJiCz`od1%mvbW0tHEh zw7KMp)$s21QJ#RzV%3gcLDgQ=GF!5!*#2?`Nu+n2|D)~C|38x%Y^XPw{pS;&%Kr|7 zApXzZ~-ZJv$a0 zw0>5`lf2t9XQjU^X6pD>w$ie)Uu6EHLRa6e8Hk)>jOH;Wm&avgohv_KBSS__zp1@i z(e2ang9nbiYnN-D)(p5FVHI<^+Vaa#%)S2iurpml&?Q}M?aYQ{yYuJM*SKGED5_x` zuSQ4D`XA{SDc(!p)r=?0Z`w^-5B>9apJaCWfwtKTlg~_YI7p=bJhRQ2jws!KGHe9?leUw%~1AsrsBY-YFVnm^jDfykp&iR!*OCW3E}Nz z%5vNKqXzjMA3tv1+^gH(8o+uoABcXle}Zp>Kbn0I)lm1Tr^nczasBgQQO4Bgf5Q5j zRr!0*DH!%PT^9b;vAu5i?rLG-k>f0yR-Y#W7Z3NIx>&9?GmNJguNh%qc6jaKe}1DPAu~`-R(7>9sZmQ?JFOZ^BxWPyB;)CUw-LNAur_DQra!&mcMH}& ziGYgIzTQ|u490#bb!u^G=u_$_leM+L{1qCHHJjB!QBEsH*+aO`HH~8y-vk79->n={Qaz{dQA~7wGT|BDzw>HucKoo`?b^m9h;?3NJ$Xt? z-=(KFGq51AKnU3Qwb_y59E!A#<^4W&sSh3~tW0~7$fjyHL}lCph|+VfD$M zrA6k;T}#Wz);e40+yq^D=*dRQIML63nyZnchS zE#~{?A(`EJ{aOd(86AcC9V;J8*RW@qMM^WQ&At`Q1|Q3+$bR$|6}!Dchq9wnTnxfV zq{8>apdI{J9o)l{_BmX)bZ%zIf#9zLw{wBOPI+!myoZ&wznc36b z$K!+OYb?7reE)Xlrb`D}>k?UM<9o2Tq6IRai{J6P3vjKGu~pbEyF$O%*=?CADcZkc zttgMK_^L!bZ%XFL3~Nv8Vx8CU9+hXPki6+~eQE39^9hwS3erj1f6x58C$b_ZSZp<4 zzcK8=j`#AVMO`oDT33F4(B{wm)m61mI5;Q+S@$ef(#g-$(1_d!GJ58(x|>|}+tF10 zqPIHBY+ieg&==OA`I@26q1xK*a*B!_Dt9cu#sI_E#%=e{aFCLDDNgoS@5x+guYkvU z-+1(0T$&GxKO$oJIdqGJ?c7J~K$F+fKgar8hLbCN2gdq2+Sx+t1$p{Mg?{H<^uSt7 zy*(pL`td-X?ysGC_vF>ERu=u$<;v!_J5XXU4ICT!ZNj>F>O zqu3|`%#v*0T;keye>e4cdpliXWxd22gLG!~_qAri!W7}iMn*(1tM>nh%k-ufN)Nl! zBr@12t9?IRM_W6+b&tn-QC4xH!)jrcYDNLD{#D_|y=&9Pk#2#^lGBxe@-QTymnNQB zDiE(JoKNiWk17Fw-h7W{S<*Y2`&<9>B0jWN4^t_S=tek~>$!^PqS)>_4s~;-ypba_{I4=ZtNk z+SnxE^{3E%S+Dn)n12%y@jaivD-QM@YsMY^CQA3*i#C6H3~Y^s<9YYszoLzsdEe7| z3etP;Y$VyVAwLiw6qSGqYQ{UeQKed+t*mDwyFd3FAbF;J4e|wdY~u#klP4q^96$=%UW)X~~q(B1ri6$J0vY zzh@i$U-qDTFeji}ax>}OorGO6e8i(g2MjTgVyl1FetAL_YgE(lq;nf-SS9+8Ngo^3 zY3(JT7b(f87H0J^-H{D~`jam=kbV?@d8Ee30(DWw8}yN{a2rv^5C{3Gv0}e?Q~%_cOLtXSV8kR~IEWPbK~r z7NCCm+AdP8m*L{ZU33tqt^Rl2UN=foG|pta3p|f^@N1Oyxzkw+lKB0|LLWF4^L8e>7hLYxlu3?9Mv}Am4lp))x@g( z-QOgrM*5P9{Q+MnKw`4U2T7~D05Ty}PDrRi&7jhY_2}QFB>OTl8{M1RN3!8CLr@K& z^GWDm;m3u3gIPESCJJ?l*8FR{%(0o|F?y`9`RVDuo{r(aenBz)=kG7hZffFh{qp5Y zw%3auMzX2=$wo!|f*zkT&C@S&#n>@>I+pvy?@0e!w2EnYe}9FC z_`eyg{(t`t3!nalLL(V0Ir5X|$&)AON0`EKfO$4q^q)tPq@@rxOh`#Nm)S0GFFJwHZ?u{TL+Pckun^LzJdr0vM5Ive$H)ZuUz@^k z`m2y?g3ysVgyL@A96{j$Gus!T*GR<%?J=|piDC}sLhspAfEosZT5d(-B3n}k$!S zF)D=U_Cwfor|dWe58;ClEk!wy>Ne^ZXgA!15HoHR10O_YLrx{WejO#@Y>QSDp)3l> z0E$#fN_9xjouNYyRlRGdb>rgz5g6XN1KnMBsBmGT1%6_Xi;s?)J2PWz;59=FN6w0w zm6eQFR~NuJV}|)JbzM-EhoG~qc|z8s(u6uiL1?bwqVX9SpVu>3pruAM8L_G;zCfe5 zUg*KUFQjn2P#W#dZb@I3^tA^6iM-rg!})Szv%0vrU_nvaC5jQ++Henq0x&TbX!R~; zL!h{a5-OV4uy%0qhn)|5)yoj^~BHwjBP@x9~}(* zK+FkEdt&$ov`7JHf{4-pA+m`L0#t&ti_0ifAJ`zMi_*QOoSZlPt5>hC0#t-f9MMZd z`^lRgpLaZawnln=IuPPmm`#s<&8{++`D{HtO!q)tkbt^i;7AKFJgAaG+Aj+PKD=S3 zCyC|d>x7^v{sbjPTpJT2jc^c=*wq*qp^Jv1)P)OdLk`de#)zfX*4FW^qL%9F8$iG4 zr+)?X#sFgh8R_YNOic+K`3+mqi*1N{s7`MiPESo`Wju(K`S}azSP~E%Pl}Bh%lj!w zE(iY;9pKH|-$%LV&LV@0aL1R}qxNFMaZ->&gh(cu9fFdQM)A$Lxw)%o?#e#I<61#r z97hvn+`hg(*@t)s-@kt!7#P41i~I$pbLSrU`C;_UHYzH;Ql~7{q+PNMyLY?&e!m3+ zxF9*XAJyqL^M=}5XS58rQ&ysi97?+va9Lp8h?X=owMrlxg*Fe61~_FUsQHm$<_HiU z4&zk$3iRx5Pn70j??Gps0I$GnK{}BYRjVaJ)H+)H1jZ0UrVG!F2HydU;X>n`m(Y>^ z=kw=`H*Zuh3<*-m|BJb|49jY5yGBu$f&t<}Q4mlJL@6Z&Bm~7J2%-o`8+13)EJZ|- zkdy|Il9X;#6zP;MgAP%;gl|mH_u21{@7Vj@d;j?E<2lxY#C^|s&Fea2oZ}oP5;qlP zWen=U$WLPYI!GayTT)pTQagxSA>I*S?Nlt~3uw&(Oh(Q0KB!ei1>mWi!|6sPm_VI@ z^MMwUhLOh!85x|+%vkWdIPj?8U;qWRSjCtSpnU9BZir>h%*?o2X=@yW?G$g7euV%#P29)C8x9lMDdY8kKs9Ij`f zUJM4bAOfNFmnv0)%lBv}SRD+snuIlj3n2CfQE_otF!humAZaMIZd4kiAs>!uN!S#G zfc~zKuFmQ&CpFDxKve~z@`Ay{*jQoyCzz8frQPDXxpC~K8dYbE36pLXTavHb(`N!Sc+q;$rZ5*MFV}Q;> zLyL=wQiq>z+^*i7t+PDgJ`7UO_geKSxh)Av>q5w&(8UFNLSsDHkB)%U6+6_roP*5-<4elVQ zr+Re(yxVz!IQxt z2m}5**~%Dg1qajmaK9bR1I)nT)Wpm=ConIy$~;jBQY*A35Our}14Nf~!V?hU&Dl}x z+U0NNfI&EfidK^$Z<300!iVy5xY_T-6txn^4A09}ReeLJ2`IFHK4#eWVF z-DcX<}|$fa)G>HQ_h5hY$r4SQ^Z4+R2C^UjcViPS`&L8OhC!djEc6E+N=T zn4||q!+e-hj9QqhIO=^tLISYXIEW|gCr{d0?mmp02|OAhcn`rvC8c0c86gN<3?>eKWb^$o=xRS; zo_Fx)qNp|0=`Fxd&kYtm+S!27k-wC2X9S~NTCUeT}(LSMnZ4c{h-C= z^#|XEq0z{nh8s8Vi>)&}iBVCXQ4b8oZUoh|xyRyn^i1**!+AW3=LkOgH2frX2XWr9 z>^(w2y4PQ(Pf3<&}+6fFu*tQ*U`h&`h7 zme?nF+C5Os<-CqEAEv&n{q%M=gaUi{<4vL;gKGydPS0Br(tpeMvD)r6r(Xp?F%D#@7r z2AOqs)x7k=q0J=WQ8=j(clY-4vd(~Xoe~tAVV5Y6@bb0}_Q1>?+_f!SW~iSGQ`eG{ z4Phh^cHTOD4qFR1CSwhIi}GgXA&!$L^O5+0@yX(fPI$yG)M77VW4ALRvw=O^&2Wyn zR6Wl$T0O14rpA&p>3fUc(0Vy3&scVoBb%PC?#*0dlwZNMSGTF(N+NI`913aaS}i`7 zlP43ClDeRaZ)aCfs{((D$ga+YFZc>LrBYX~w&_DU7Synru{INE8_QUCU|zeO-4cv4 z+)c|jOie`%T(%@a(8gF@JtDcHb;Ocz7Y=YP1ZfPR0$*rtP_e+qLfyR~sv)&LdE_R> zj=;Zi-o#^p<*Hn`i(C*>8O7XTPoZWEP5Q5($>2JX)xpL39Tju$?p9@VRY-C~H^R}Q zM+eOM*-^1Aro2OVd7W#-Ogl_1{ItB;Me5?EOKIRgy|SsVugCDrgWS444v8)LNL}Gj z2r=`cdiEIt3mHFu0^Jo87pG@6X=|K#NLAh1!;Ud7D>IFP#l-(_^HBYt7)o(b2Hod`HZ$nL{^R5s7}$rXOg3RO zs>Sa$KhyduD0=P~HjqH~&h6V@UR?C}lyB!9vE6yf2=7z3rvWr3ET8Fp}V0 zl-@4e7OU~-X#}5PyH9wy5SRgA@>WYUVRsWbtgLJUqDQPMB@8qQ&~{h0m}WnDvcpq? z>hHhJTQ2Edtur`8h0(44fq|H*rZH>#kP?JfU2Y71s9uFQSQdIL5SZp{HvTE~Sy2B| z0=yo>xqGL1cz8~nxa9Z)WDy;0ZG6WRn*y=9+xHCCgdI|b> zLxZes4Y=L|)j=$5nFuE_NAu3<1L!BYOT`{~B@R&w*eTWY1Z&i*n`V7M0BN(xaolAaKME38KFS6onwbfPR8NrHVXDc>*RmMz{`ra|4Lc>XB%}K|2CA zVSEqDKBMbD($WE;v2k(8Hb4z%{i2DCz1MDxttxVp7_eU}y;QbQYf| z?56{kA|Lh-`ajKcz|qOPY>viZ2Nt}D)SK~br5SFZ8Cwe7I~73)8ZZZN%E03Tp*`Q< zuW9>NN5fq#jO0aYP+i9VdeNef&=^rULE<^{kP;HszM*J(O32EZL4U%O7Tl7){wv2J zyoE28&&TJmNAg2L3O;USeMuTll0~JF6RtlkE$GZ^;v^vYCWe`C9{Hws+=XJtjtspR zV!o%=g^oNd76{SE4X-yDX=rGK%bZxFJV>H?f|2pdXAbBGXB&2KT7{d~!(^Pi^c(>b z24$tOe<*pv#l*zqbx1_rI?Knu8pB{C`7N|1B;2k$8kXI8-9IqU(WX_+0eiS(=T0OA z0l6SV0Qk^AlDR^FIYFN$RPvucKX9U$!}XQfqefk*rU!GD=(n62(yHdA zQqR^8P)~yekl8^jPq(}I1UOH!lck=&!ROOU??c|Zz*~*S6j8&jif~MlMYMvDO3WQQf6TG~8sc#e9?kyziHX(kfp?U> zdO%X3Xs@u>vd*daJmY)Bs{!nlUxViM*Cly(U6!b)FnEN}4=1qe$){{(OdUyh`k}=y zNMjc*H!~|MtBA*Khy6-qM1~Ow%J7G@zI)Fj$VC+BQq&N!P z?wX7aZ)l&xbS;=rRRRM9KH2tyb!_Nm^hXGpV|NZ{_V)+0(wOVLg;9=;%&yc_21KM4 zALQ>(q;-aR7S5e_oC|QkaxC~StuJ+0%+`-QR@}8m3pzA6;Mj}s{%90^^dU>AY#=9 zaxpP646A#|ZQ$kU3G69)!-AQHN0ZKI5ko9plEgPE%x$E0qJoz;zQ#L<^8LT&y!?-1 zUDrpo21zZQY}2On;7k%#lJ%F%!N#?14U*r-382#Ym;Xi7Z1Ac|yZqvJtRKZ<>#otH z0}AWatMy|i9RpGN;ZQ61)ZJ~Etz5ZLp1}%M->aDEz)5Rv&MZ9f?QbyEdV72EUo8B8 z%_wfr>V9t(F3fTcOuwR{_p|N&e@H-fOnktHdgAMSuz$9G@N)gT zS0exG!``%PRGTQ&K0?+=KigM${W)PV_x~^j)r_X#^yybBaZOr9JG^!<-t?jTeY2`n zWnXj~>L03|^Jcf``8f9b+YRFFfg*im!|(oNH)@|Hc7Yy6iV}l*+T?d7RmHIG>+n&} zp1mdCNU#*2kQBs&qN|EL?_2lf7j z%jRBL!1*rSIS9U(ejazwVBh! zQ3J(3?pdwQ$i~5==!_^@DSfaopz?0@B?duvY%K16v+galLbWjGusg1lFgSL_!h(YK zC@S;_os1#{l=}~AF4opL3Q7CKTpvAx2HTjW0Hf==tLeZLH@K2OR;3Jd$RZ<$TkSY> zjE!vqW2T5n59znd`b)Gk7i;i;0PmW@^S|A|OnrJI}x+ zB;<&B7gfo`2ot~#S`q6z$#UnZsI)Uk$BB6(fGd#~7ud6lUIn`rXdWk!!bd+fU?0dHD>lk<9q+OJfZ`0ARP zT|llL0pQQ1x#7K2o-wBkejUi?RqgVdf4Bf}#RF|s4z84oi;F~whlq?3C|o6rzM_Z7 z6>Qws!HlUU8L^^w?tDdf+V9S~HQ-`6LU}QF1Uf7alSY5e%ve79PjM+DdP;R-vkXJgXMbHYtGw5#{eW!w+Jiu;M zU;7YGlQ9CT49Lp?Du@*7%$YOvVMUdim|gu`J~zpL|&=oj=Nah)5A~D0Df4 zp(o(DWQbUi$hxYWOJ2gM%`eZ6*~IExV;)Ig>f+?( z%|YCRS{HWIJ8UP+%V~>H0jD0ZqYg&N$PJ>}&=Eg``vaco*XP%S&7j+`!0c zD=K`>yBTipgnH=O9U(!%E(8!*cFvPKckLpkfZ%-f-@A708Ec{SSzQatMRV}YCAX<* zXslsyOA)3`Rn?f4br##_Qi@q?LrFAa;38&vBW=|@mvZ6WsPtMJS4AN@ux!HvIe%JN}?HV7oHu9#38tdiW3?f z&=bIa&g<8&EawqTDOel4!om!lYp{3GXoJ?Mno8u5On#T+7Drj2R84dOVBCwmm3#*4 z3E;5K^p)1cedYvDvcp-9~HBD>=!9RKoV5&CKUMf!9Rl?B96z8=&COchgCinRU%F-}x|Q*gUGkzk$;x_z z7j;SWHZilYVThHx`ORRp9gL_J+_`&Ki*y@}m*C?I7&xONcc`bepI$pepLbr4>={5v zypg4OdIu*;jveL*c}E#)^)piWMMV0(EZwUrutjAXb=c9DD&=_?U7zc1fGmZRiwi}4 z4#h8We5wjpGm?{QDu;|qf1%H=Q^U|W1}RW#s!VwaYz__bzar|zLMndtEnlam%!eNN z_9^4eGiW$CIC7O!#K^Sr#ZIvN<>u-NErCmCtKzyc(4G8q?efX?+{lO}!9&CYbn%MX zywvFv8k&@qBRd*&9EjD*WqY)WphpIi;65!Am1F<`aM&4^or7wO?-ey1c>sst0apj1 zwuQ1cs*)Qy(glQI_wMDHOFVioN4iIB0g$z20lFz9jj_0ij&3hgl>_*d&CFgq?lFcD z1Y@M&F!$NB?fTh)K|x7Jr5>H{N3c(Hx|YgZy^2nZwNxj*vuE?b#m9}OxTm0=92Zw! zUw>8TrmU>2ii$z=w%c%Nv$L}>2q@!SS3Fd0yjwES99MDuX!1Jxq~*@)p>9+c5gEzh zoOYBhyCQGA|F~5WdH%7x$f|39?R_OfG3wg3S|@0t&ZxpyF>0c8fZgfjjjK4(vELE$ z03^eDNXqAp)Ah89IbgR;xlD^WaUl%W5+J!Bv{{!?`Ejd|Cd5g zTeaxkwe8(4Rx^s0JBrmg)+$6-hUUG0&HF;@H{|4H^Ojl)-kJ3?Bq4Dpz#Fb`4EeZALeV5ERm)nfOj+B`vz z+S(C7jP^Tx9#YyBLrS`en%&6AFF5#QUqHYDN;q&GbD@>ydi>;}eN~!B|FpGFu^FUR zKzYWMa>6iv9nTQ8rxevN0rvqYQOmWojLzp&Fw;gC_W{MkE5mjMVc&o@1u2JW9ssc@ z%?2dPNaAF58r}qq_7vup&F!3{Yn9!}2vlR-LUYJ}S87p_1FF#oY2@`y0#v)4IFuTb z!^B#(a!))+O-Ts|cbH&P%Fx8Xs0r1!y4!4-#JsT1u%s6+z9L2o63UyN?qp?4*z4BS zZ5rF@h>j9$NI{Fir`JPS(JIM?=nW=DW{I9fQ_!%(H};oNl;Fc9UTRd$dB#zTMb2#A z?xr&GcLZ9*TIX*AvcPd`X|zT$6a~P91k;<#lS4x%CVeGhD(mYf@6~320f$mOLgIo2 zr*+^x)nvw{Bo$>Z6p5-Nbv|GbVqUxe8A>ZjD_=XR6RHSsu|{(ldk!ANrCp&pBS?q3 zJQVQIBc<4zl9mR*80?S<*PUhLdJR$M!4$|Sen1tJ!lRCnj__1)_yNy|Fsj%tkxTAj z&1cP0ZKawd%kV%UZn~T#5LsE-*s>Q_0X<1yxne17JCwrqsV4#6?Kwy*3C#J{FT_zy z#5ob({ca>;;gFt5yA_w?Q zl>aNrJ5Ye4nAXadNSf}u52G{(BtYnRY)!{16;?E;Z<|24G3X~0_AwfJloryA_iXI7 z_`_IKcqy6Zr@z3x$z2~r?U$ukb)pZm8(gy+~1& zO{&G8`74SjW}nB1SghXueftJZS6#zM<6mHZTtI&?2$+$eY0?8G9%t`jq zDGl;d*u@#$g&wy`P2ZzXNTO&P%dG#b`TnHAivcsFiRN@@UEbgA@%ZrwhSurI6})i`^fW|zhY1w%cI!$QxO&2wiq@k=8h^m4&)DjI}vE;M}!Q; zA5gp|e~iMHL(YI+8di`vIVG`ceI=$qx zQMScA#fl*GUOHlAy?@Suxk-Xih^P`I8z{#p6qo`Eyd)vP%gw!l^3~{DM3@3}d&L$O z7S1h?X99eyQju9w*kW{9QnC{@=l)7Rt+8HLS62{tP8vlK@KmzW-O`}(#m_JIm2HHL zM910qxFMk8M;;ZG&%t0K=B-vm8HyHR%Yc-UmBpKlqOeaJuU z4mN7mZd0LhIhxO;e;gk;SjHb}JSgT)+?|x%DLLQpi~-V@5#CEw*ZMS1a&cJ@H{x}D zxjIw-I7VNk=ABkL~|G_#DN;!r~_I3+%S(Sad{sF%}rMzLIu7;$;0^Z-MXy^3o z$Li{&LeR?b3Kv~0R5|-ep-*N+U)j_F8ZDeiZ$YJtP8rBl{Q_Po@$d1z(lvakX01;m z6gaMSxa;WIzUw(W&$;!k*Kaqq66gUBBy7f5Vdf1!Qqyx0`794eH$O|CgAalj>Yoi7 zsOhl&W?idm>AAVNp(<@WzH>7(H>v_*O5)>ng2FLAiW)Gv>rFg~>0Qb>K7m3G_Fh1x zkyckrypfrysqAyX6Q)>=x6V;nU)|5!EO~^Y5VP*4J9{`t4AgpP0E7!A@nSTZ=b&?! zTMcTDAmy!Jc(5UzLr1Nh_Ei-nMSFZP*uQ~4h2@2i*ny6NnwN;(jIOdW=6EkbIUxfr z_f{eUkl{y$=yjF-@ZmRIT`IOpcu-hwfOfqQ7cnWGRlqYXOrD1l&HC%Jq?$Adn=7bc zPdz;BA#;J*m5YQZ6Q+Y^{6Y<;mKYn3ywkd>AHdui2DC!rO=BBD=V=6O6v4$R)Kdh{ zohy=)`Gb&|KBLGsLACQc7)}7SS<;HkL6-o87{H=vt+2%EN+?!M4OD}=bJUgj)G74o z2TXdVXTkVa@cVmEzy;3_T=MuUQs_-br4OCA@nY_%lr%d0 z#t2DLx6_kHfu{!LknQ0zZ6`C$*w38#S+e|n>^{`wHeLeXPwi*C;o;@wgayNo1Vhb$ zeqJJGdR}6nK^?Uad@2fJf+Yeigx^-EKl7R(%N@ghC!(s4CwBdQtKrS`DK3GKw6O;r z@ztf3U%za|+kZAmaa_D`A&U!}`JTN!=FNX6o&p$@CesSa%CrW%e{-GQ@Vpe?2c4-| zLfHqF5A-3#5uZ$o0iJ=9lDQp(fFda8GrTA@;C66cEe}v6$4wu2HZk0 z)BE#q3PgvpKBt=@*ChSxFEU0jQTpsB zg|v#&(|<&+t;E|^12`l~E;CAG*VNRC!Ea1zIpqptJ=~TS$rAUQ{z5XmnSFZ{QN-WR zp?<^rhN5DU2FO2{w=(+=P=7f&KPjE9|Mh(Et5B^&V>RU+1X>wXS^Z;Uf}m&31|kv+ zIcJ+;cMOF$oHgPjL6{wSGV*7i6u3xj)`bEx(Ix-m=TBQ)Hz5z?N=8AU4k!uT;)c9n z$-hMvcZ5LL2j>WhAIxhvUxD%kn4*>+HWCh8j9Qeq2W3ud2_zw_0{Hp)QI`YK`3rsR zVTR-1zGb4BS}R&?1eH@1^UOYU|Gs__wK933sQJl05m-2Y2;U`b4E6LD@%-bZL$#aP z{y^-eMrN0{X3fmDX<)Y*!@dwaSP&rXtr5`@@@=&luD{>X#o*b_ExIF6wIOr}lj?u( zMUr7X)t38zo|Nf>G1;w_sJdgYbULa+V3=&(woRZLO=JS-^baRZPD~^aUkrh%%|jp! zS}=6+Zr%C}5L}Q`=l4g6whhaYOpJ`=!{y?XRtBNAp4o8=ENz&dYoel}%ih5#h5Gt7 zb$2H-jDu5&LZu+5{GUJHZ(+343yc#YTX$Z@VE|DjFp#=U`ga2WNdu2+A=wv@%l0C= zY5G(4AhBbjX8emp5z{+>(L zYa&kM_usf5IB{O~#;yYTOi#6 zXqvl|QAETUo2QBb^S^8A>vR7}d{cZ3OyZfe+6W5 zL`{3_)fuZ4wkttnM~@ugrme*zGLY&DRYKbWavq3JwN~V5^m+S-3N_Vq&BixE@2jg>1Oz%ie7MM7{^0|WvYkD956;7rrWNTH zT#od$&ufkP*@z4=YFOBE*yp+_updBUqM|MtDj*+}mX=0FYJ97|2lyUjq|cu}cUqH? zk?BrVCwmsHfUCghieLG%grp6hP-*`+`ZZ7}0PXhEJyK9B07at8zh(1g<&pOjg)s_A zC@%!DtJxtVDvMvY94_U$U#aQ9?ggJ#!_So#Ed-^DwI+lB4RUopr>xebtSn1lAb<{V zUY{vtQogIK)B~$l%pGKY#Z*Fo1=Q34STaH|6rCShKY>0Hf_$jGhNM}3W`H{13OFv8 zE?&&eqRHTN!pxR6oNMAw_MkI{mi84ZK?`kb<2!Zg)n%pvrV}T?SjoFwGQrmikysB8 z4+vCh?sdmd4G>ZWBG8^w*4Cyt_^Aa>fgPnD6lAqIatGi@0M&PA>g@L0sOL$?KHj1m zAUTTni=4iZ{3P&Je*;5*^~UH{_;B-dfl%(Pjo$~i4a<@<|ySh z+-dx&<^3>qi?zCKU%ydMy~^w*<}+umQLyhPH(5p&fi*+`Z@k2?ZUben7P+R!qC|oI znX_acC$oT~Oo$jzf6lDM}L0Hrye)KPp%`+1c5T93h%I0Inht z(m|WY?QM<-1(Bm5;K3@}f!+nJ;8zi{2HYVyzjPu_D?kZRirOy*9O0A-+E~EcLM>{5 z06`PYrmTScL+54msm3~ELkWHcGCs5&H3Fy`p-BL4e^%sWLc%bz!d>8y&%YAp&-3u8 zMny}@=alO|Qc{B0VEEUVMJ!K~es->L&nPVRtTe4B)6t`4>5h*mkAOXngmQ}IEan&zCMk?Jv=3ml%jrPy~TwqhjlhV{zWt=`UhA8 z>9_v9O|^xT^bvuR)tir6&Qg=&IY4)3>S2<8HV$*XWC0t_{FOms=oE2TVGeA;Wby`> z$G!`sUVsaHKMI_!%2D2YO2?lOJ^7fYND)dfl5h0y!;wb_VWm!;(nm(^ska33?rLX< zA^KFv7!h9i^MJNOumbIlRBs>5NSa06tf{&A!Gi|@9IE&3-Ggj3zT@xczkP&L0?Nxp zw>9)q6b>>lpjqKH6nL|h+2XZX`1-Maz=3M3s(h!Pq3l&?Ir4R4!f2@|V-Ud{ZO@T` zJ%IcIDaAT&<9SPvU$z4StZT+}-9gemAzAAEf({07k0DA!uq3nWw}pUpr4OX{OIQb- zDXO~Qhg4jIQdOC|H6FH`3OQ0&WN}(~y-3X9Xr?l2m`!>Lt2ASESb zX?`+!pOZ27{3ZUgXAdwmVXt?7cYB8N7wX}}(q){tz~e(*# zzI^>k)S!avnT{O;_ydtB7L^I~MVvc_bSg{P2>Ybb)e@Yu07#+B!OrL1MY; zK6S5R4-tezX{u&TLIr}Dk6L9)`y(jJ(ovG&(7jT91XDv_k?(YMP|dXD9$ga=*{G%{oM2)qfgBw9t9Uz&yCNbY{M-4ktH2;#Vl-F&35t8_1wZl(Br<~z zzoKy55Z4gwGlWQm1|^IdP0z!_9iekYb(C2{p%N`iEiKtO;hs)~hf8-OEUqq7a+)FF2X$IquD;^A*N>3i%En zz6EH9+(_qkMLL_6Q02BgHjVD$F^$kCdpk2m8X71OLomLVXGY|T|Gg}Q^gBd2th#=r7%t{NmEl(!q&-A@Ru!NE2HuQlZo{*7oI|y zTtihgdF9oc8Qb-MT9=uFetbT^WKJUe zA^mYWrxuot+@7wwzf*Tf->OCBT-&4MOLcasC-_qq5G1?l%*@SwZ;JJ3G#GwLYfL9( zmld6Mh&yFz*LR%G7cY)z5jtcH3=FvGYIJQ)7eAEHOd3GZ58VtQz0G(mICpLa07!S4 ze1JP8N>OW1Mtg2(zCUN&eKuPem8XAVo}UFdG_CER_2FOa6*N-?(G{d0*!uH^RgN~NpxI8@_T{n_F#Oi1!p=HXk@*T0A?NSfN zUU%3REyf>?`s|Y@Qz-jaE`#KVLxlq1K##&5MU z`0%7$8wB03#Lf#}NEZvq?d_LAr$Rd-Ghg?N?ZHVj@y$Ai9$#OrxW+Bf7eBxI!v!eV zXKHS4hw?NaQ^aVmk|8&CCQL%{`t=Os9sw-^QUERZwBbQEdvU(2Z{G-^0BnIygsMBH zrayqFlG{8NaW0`0)7vsJFtCrF-Y9PkD29YL0~;S6o4m;;=-@rYj<%(Cc3wl(1+7|R z?2-YFk%*O{@S2be(a-J-Esw|PXPKFYp~&WDuzB-lNWmK}u)jy8);3to9ehx97_}`h zy#bR0;JcO?JlrgY@L}R5+ADzuSF9+MUN4e8L;3Y#MSLY zN&Q-+v8~F}ueXBd5bd#y;c-ASmw}o(5Oo{Z=MeYe^hC8A*1F|~@xk8X*b#8W03@n8 zii-}maG{xq=sUqTpe!r>{WHuiN;e+Q1mWL^s_0^b$x@N$%SK&bEu&Pe>&8X)(5Yf| zldMx|p=r}sN2n9yX!@KUdYXP&6$bc!b?<2 zHjM3V*CF92wmk>!sftBuEE z{(u9t^#yf8FdyKg|A78CQClu3=xlE%V?>65IH&tL)giP7V#lC#;YoAB3>7VK&IxMo zWr_70$E9$q({IWGspp#~9~2!k!AAwxrw;PprYsY=e?BQY5j0MOXGBvivJV_E+SqzG zFhaqKDHS1f-O#bFsrmBnE12=1uGf7=uL(?l%rOrN2@wDZ4XuMa8Ih8mL8~BAnv>!K zgM+CcrJ|olLVRR)2{1d{mnXUa&|nX&4UF_Qur_wj0x@=_L`N7JzI74PeEa?xss_MT_XJJI=^>8$l>iM{uvP8e^zj0D^r; z2_yn=pW&4&HE?85JZwOo3Wm`xv@b)`RodEQeV5`tp0e{&Wb&s1uJ{<-H8BwZ#SBbm z#`lm1oE#p$hjarS!(ct%y$jFzmY`>Wp9hnN1{cf*d@qYciAY-eNARi8Hz2@FWXps) zUx}kV=n(|gKJ&fD9Tz0Uf}2F2pfnWQZWOfNxEpW%mZ)Grm82Svf>t^DtUI zRFstH&zgd6oe(XKjfA|~8^IFDCx|n@qY+?a1Y(?cKb+DMeLFGbF z22~2yI}LR=qoJ(nc$O~AG*eEi5a`$&FPFmq=q#Py`5)~UrTw6JgAaS?#?XdS(rqQIhpXaj47R^p9l1MpV( zB^u+u#SqP-&#N#m?|DLWPA2o%LrS^0E8EG*c^uhLvqVW^D#*0TmRyncmC>7xOa@Wr zjlPdRnt*Tt<2P9{jj2BKdfjllNE~r7rbirA`fE<5>^(mdQ+dX59)jdmx;o1trMM}o z1*icro(Eaf2ab^mfB*s9{6~&_g;60$vq>!oJ3&?8Y~peBPtdrToG`(Pj!(?wn8N>S z<8Be!Fu4T;3x7>RQR*-V#6nIwnGd zCCSX-x|a6e`pSy@MJkZ>Sz&%&9;-^uI6OFRGAA0sra|H1^n3Rn+yCJHeKae8BCF_F zk4ptcn^)dXSPq~EcvW3Sg)j>67Husc8?^f~eY=|vtABaXf@Z+j)y?fxbFog%^Dl_a zPH2R-ukT{fZ%R;UQ&OV&*kIe^z;>PdRdq$hKM!8z#xdRhH#*RW)wX}YAXAkJ*C-nfDxCse7_0msRiX1>LgiH)- zx7p-q13L|H;P{v9+}!yeKak$|=w*mW^kU<*Tx!tehOGsiIyj>o2^z2_KwT{@@@Z{_ z1qHOmzOM^WX+YmKDLL&8N9&xTFj^{Gs?hbrp0~3VVV10!15 zRvnKMvPCvR#AphwDU3FdkwHGc2ssjh09(j4v<_e+5=V8VP|O|e=vesUvNDpAe~U4P zA{rR@_U-7h-lx_ropp7_@X!)MQNVC_&Evg0a*RTsmJ*GGsA(YRb3jRjz=O@eIL`nE z&3X@kBj-Y3><-3VT*I9qvRm?kuw2Y%#*l^nU?pQWKv5nsFHBi zv*+##)f3eY*k2GuE%h(XPFK0k<0!)1{a~hQjwI)}umOvF!!uV^Qv)CGshD*>q?%H4 zlDN$HXzgOw*XDHXE6^I7c(@~5@4eoD*HX=vEF2N)?m8sgov!o<{idkJ2iR&K6ePXS5+5buCxg81;8v%U-TF9+D>UIwO~>c02T0RwZ5W$#f4!Y1o4pU znzAyAvXL-m*NqXYh2Zm0X_6@;kCFza4_;sP!%Q)EAX9DAM6LwC>dhb|fF8K$Qg7~a zQg`mWpgRH34Oo_hMex`sl$c_8%)O=oe2ntn50gNtdwLq$F!z8}XzixNrl+Q6(l`Nc zM#vwxwujjPT!te+)xo%dY5n%U-w%+9X8}dilBPDgs7N)wInn9%hTx;NE!QO5#S)qEAlwoRe{?E9_F@- z#(e)r)cYxDxpk4VVl5(&ECEt_S~79)G0Npjy8Xn~%z|*P-VOE*|1Qsr`2) z1j1#KT5_6(`H^8F9Q)369;pM%c5@>G#J8GBp(MB<)TUfB}uKcTRuLSSKqV|u_7|}m~B;(5h`8RB>btVzXmDnlMs9^?7ZDUMynmFi4Q#DUpWyf)`I*jYQhj8x>)Wh=5xx{+JToV=Vk9 z+pDT@6bFZf;HU$7#gXTs3ngGCpJ%R*XyQa-;!Q88@J&HbP*6u_2-kDU0Kb4%*irtx z@bDg#Ls14peLFO&HEs}uIz(GQ>Fz|ry5QtD`ESYlh}0M)fltqm!Zw4jK(wNmUz0f$ zXq);t=IMdj!>M)em7Zb??xm&XS2zs>@dZtZi@+2)djgHP{RMc|i=2q5-gEdR>(qwGJDUFX6T0UNXg(+_DUFa< z7*#z|xKtS=-{P(iAWZ15rxnjBgJX*leta?@(_9C$NW{E=k+Xp`F}ecmg=3}lKQGA7 z4-iq}!iAMDyPbUuwClq3J{>EN0FJbX83F*QI+5@J`#Ladf=u`4?5r9niyN`wn1=Tr z;7mR~lmlt+lDu=kqbX#d6O{@K3xk5!g)$!Cz)-A5l?(qxduMj{t7L9Rf6A}8H=rW{ zIQTQkr_HEQk-g|aebOG9_z-33doK$ohv)}K7ji_FvvSq>?TE^F9Qih|s$}lxykP_3 zetuklngr645)vxX4H;cyV!k0bxnhW&%cZt&-Nsm5$Leoq^zZIC?NUN|z^%?ENSJRh zz{Y%)hlIpP4jCPr+Ij%Yy=k%Sb%Hed#~!Dw*>8{j;{U~-)rEJZVRZ1n`t`=N*qpY9 zPga_?{&?}da_4Vd5EtrdmQ4rqU>~DWv0|WybbQ{EK(btp7?nl z-){wEeEt6@V$Y&Uym3kj+Q}|jKfDhUpCC#p8X6F=FLyslgT&AOd`lkfP^`oa1-=!z zPy8O_B8`k|);5!5onM(==bQK}EH)Hjw8J7J9Z5snP2fC`J1kfaMMXVCdG)Q)0O^wR z0792|-EzN`#ZQP>c5ZH3CqcOEagsD-nj)P7ivhVTkhT%ru`cJS0DR*~F2(vn$-|r3Uf27?1`&$nS4A0sTtG51n!|6_X z@G(gN#kt!X-w+MGc&;5noK^|R$yt_vpdi)<_tKw{MU6#a&dlE!1nY-*wg*U>|RnQ|#9UD*a05PRcPKrGAp z0+Wz5{I%ZhnUy*?ZG?9H{OfH>cxi2H~#s*U!nh!5B(pfkiS3W z|M<5iN@^)jC1W!{&TC5zmyE?S2D1MqUWrnnn+q~75G+EwHnCSMCJ(z9lORIH>pzeA zaphthU@xEoFeZ2frVcZlCsE@#74V9{ggUb`k)Lh6Y6!{~jz&>YSYm_crQO4Jj8KMM ze|35x_Rcox>ur+n-iaNfefO?7L}J%pSts$Mm_sJSjyWHD{b$(kEn(yXaa3Hr<-U}I zn=I**Q}q0PKj5ru?;m(xAWoIO=6lQ`10HFsNPgF1g>nl@**5j179w z(?hCs@%VtrIRA8GV%DTn4g1Mvp4$&~gU5|h4h;|dTqtv@ucIHhg+(36(tGvw>m8}K zh8Iwc`F-p~Rctm>8z=0kBHqm!!X}ry!P$$n=cVO#~~bjOyNH?JMX@|`{FeHD_7Mhm{5Tt{P?eiO>FbV=)a!+ur;y{tD< zovYX4QeT{U(-BGzvfNN2m%`)`N8`*4o{RWO;D_z2*qwTXzn4a|TK>kXopxhIu}Pv* zduCb}Uo>swZeTuja(AaggTri8Hd&`bdcw=@Ci$#uzww)>0lRM%-ZhSHN0x^&_`Ftb zuKnt3w{Xj^rxepPc+%T$QU2e4d1*L_RVv6OF!sFn`%CdnHHzDn_cLshV*LHo+1hN{ zcEYkL2W}JkyYH9oXpgS0+`R*=e;#s{>T$4Q*f+6BJz#k3o3V}iX?d~dqqo=p?;q{| z1%-Oo*|arF59C(pNT#gI6)Gr5uT{GM_(V*Ocw=#C>D7%_m_S491s8?)t>v$hG9CF8 zm!GP1j^!&$1wClhxv-X6Tp$pyRx33BOIoUJa~5rlOU_HgMgq*lYZ$d_W8Y2CYyp&W z=)i&Jp`k3U3&33gg%Goj0rB!yF0Ajhtn1DPhQjBTOq|+zM>H3n8+0A|wK8z#8)=?W zL4r1qlS`N98sFkqkrTYyKJ@3scl3tle7`n@ zo%Uc=`zMoDGY*dC>Wvr4e*P#g@MXi+`6H^#ccj% zOTBdQTFpSw+A%E#nYz{5H+6p7tb||l-YP4Z6z?m&QGcHL+&F#qy@rlMQzufTxDB%7 zQbN0I7ELDmrhe|`n7Eu`Ha7aEcOqrYr2Na{Npn_}(@&FCIB(D1KQm#}S@~h=NtN_# zjiQ;s$4oR1E3%REt-{Z&d2H@f2zn+0Mf#fb)Qw{ECwG$0aiHx;%Z&^TkvJRol|cGq z8sp5scp&5(5vw8i;Ot98x-{kvVXI^2;b}~|1%wtbABh=j;YrC>RJwZ zH#DOr*p-%WVcvTShNIv1vlm?L{5mo8p2K$LXV7M*RH>&|D@T^%W2El|2F<-}J#fFw z)noEecVEkg(`GVW+cOL=yY&h`FS_?_(@!Y|M~AUyD)HXOYXUK2S&AV)SUHMqUnFj} z$=Z?i`LrNdHEn3%2D=vx&g}rcfD|`3HDM~kt&t6Tq)W9bT)M_W+6=ndp4IcC-`9Gd z^OdtY`*({KWu2JyxKaCKIb5@O*Ui=s^Wp;h?Lu?i_HI9u*4|C9o%VDbsio2gE9EUJ z4KfVb#J!CxDzpV;IlNGtu!TYaawVuHLiqz==7ozFXCaK4Gmbsnj?@_Ry8TpCkOXW) zx{N3rgT^IoJ#Ilc7}p6#95S6DjCzLrE@0;N&Q1jpH4tJ<9k)W##)X>t*?Aap(7=Y(N`Z;y{_(ifa(4Ge3y}Ynckpe0Zn6JXZtN0L*@sT|` z0xCh_qn(v+_xho9TU=gQ<@jq(rfy-e$FEJl`*!t5k(K3++aedQs_&Urn|1Bv?zpmu zjj2;^vtMj);P8DL)dO99^6Y_QuXyS6SWl2mlF@SN__uZLzOE=!*jL@5PWCKVacViG z^5;o@x0u8xY8~tCQg2jqZV_mMYrKVe2pdV7kw}5 zgXlOb${E9zK-hJYmr&2O#xlMBD+tyCKypRr)m%Eo}4`H)YB@CD_|+j zIm1zMs3}cHhw(ym4(mcd+3U!#G$)q8HRDhCM0LAq*Oj8_hJ2-O%l7&FwCap*J-mOo z0A3lHO$S|552x(KjtPa`qI~h{m1J65TiyfpG@^xlwQp_N88{c?X4^I|Ilty_27 z?=Rw$CW{U!)}Uyq{m}bE{@)(e@o<6m1ntR5SOlTfm64@ly@$sOX6>7AWX`t*W2lc^ zUgA#4Q=HS1uS{sWWBigec^r@9xlPyV2_v=(vuZE3`q2B$B}( zCEAoA+a$f1LqFJ=*`?0fe0VS1(+#iInJD3A6+S1q(nj+!SGXANV9bUoLBcJiFE}0l znEK$jli^_ZhnCy#GH2B}#+}nPgxaI0I17kY``P(Zy>(pzWE{%N>7?SPk zyh&P1S2EqeuN2+oNZ-mz$HF&plj=Z@uIl)SA^D?aPHR)nZu9d3*yKxly{R9P6evM3 zO1;EEiL5bJYCn)c+IjS>k2ge@p{^;iG#mr09O`Q+5Icl+93}ulc?l)kQZSHFO6YT( z?*YvODkcETodJ~wvA4{JZwD=n;30q+MsY~E2#pU14~7DJ1)MCmUYHooEyfGuiDNp>6}j23pDcep z!9O%Fayi&*j`!PAVUw|*TNRJO3m7%_-6L-%5nrx;Dp$B2T} zx4|_Ejr;mnjq8^AYPnYWc9uHdBzt{VI;zo5O-nvJ#>&hpPws|t*Nn~wSeO1sTfNB{ zdi%yarrR_hP;^k;onr3$DeibjguidCGQ~Urrp+tj^5kG|VtH8G`@?b7?XQ_kVfkWo znhaR;vQ+m^b$*m z?uuQW*P&4^r-b_cYb8Oc-GR+XcSZQu-u~QMa6d~T>$9#fNTh&uglwl?Vg@_h?W?R$ zr=fB%je;z{fWQt$OhbU!I|{bkpF@dwh)XFLxWFS0v%6HAp7HIbXOGD=J zWty|5w;Zb=M;wXfO!UW4-q~mlLg~7yvT_9UUl8eKWaAQ6Mu9n3`R;~R-LP&<*f+{l@9@9ds6>oEKJ>PyIRD*NQHK^9kc)df*~ z?b}TQ=d|RmB^sN0%a99wxEk_z?3st-md_&ORlh|H6ub)EX5mA{KFChnSNtf{ZiR$T zgoCuq%$3sIr_ph>jV&{0v6|@|S4m;hcFkYCS)Jb)(=Y|)%s+eq9911OU z8!qycPNof5hpr|CO$rG#*?bIn^Tp#uruk0tw!S>IGVU*hx6XdMH~v7#&?xz;^StwJ zxADh}%QxILY`!pmYF*hsYDICUKX_y{p?b$r>ypm1z4)e`nkGB+c(_Ip3 zU>h2`qZMcaGiLKdvM^8JdC6$Tj_}(LmGUpb&e`VkFX8eE)5;pTMyldM^B;ceK?n<8)@|?xkwGHF(#@bCA*F)SAqWV_fYL)LQX(lO-QArtbDpJppY!|v zah1aFWA#n4DR@DONBWE`t-pRxp&1 zz+D6I7-T6wqD!EEfU>PHf4_@|9&r>(=O{rF@Gz_4)f=CgF|)MH*31UT7%WcKw)$-i z%U$gTu}*)&J(VDn)$-3SsuSs=V$e@6Fj1X79`S2Ee@)@L^lhb*Xig6!HV2RI(;JI( zM$*4}?wnZNHjI5JaD>OyyV zK=gCnX-0r({*0>+fwln*CA%Qm0trBn*!pd(WbhZ?3*B%&C}glbZ|nMnQ0{{Iy4Y{_U;1n(3lH zhtBygUTvybDgQzp7;nwxVRzN_cYhvlgMI`~J~E8Sx*KCWtV}<-U&D->qt5WgH+P9r z91rozyOq$p`tdkc(EAn9YsuIrZ=}3Gh8vP;iMr@+jXhZTJ;EZYUtP)>oK8xOyRNW& zK1^)&Y&B&Ds0ktc;wmTSpXdmL<*BZYdL9ek_C$q3O zmht?}%c|C8VQ-(ff!Oj{^6ybhLbqW@Y+o;Cl^7{vHVf3tx3L+XPzo2o9%!=Q=R>Fn zo7%$L%EwmU`{%YRgc0i$KbG0V!7+s1GMai`PsFL~@`+fYa?^ZOA?mScLzv(*oV&A- zMDU~jCltN{{!ww`u`i4#rv*vsv4cIM(yTnhjLsMLi;_zgmeKgowKw1UFc_(c{qwA+ z)_Xs*+{0+Y@3&XH#~gfX-I&NR7)ovj@I`?!7EDEfYmWE>fbbu3YXA&{lLpdSCg4~C zzQj`?Y5^nm1Y_mF+D{nOL-uUpYQ_>6F2K2n#I+;oC?MGcSCMTXLt?-$9)!^+5n~a= zs10a@qT=EiFjGoP1B-X)3bR(H|4b7`hBP0f?2jl;g(V*!CRwGDiQnNi9mC3&vTt>6 zHkH4D8hVqV<&~wP%hNDPC$CWv+ur-7i4}>K4r68U+{(^tkox&s*rZpoT$NT7PKFD zl|A3K*LK}VDlj-*0&TnKq$u1_qyq!U;eElgO2pT%{Z!%v7 zTUtLOWAsDjs>UEG0gWuUC!aZgUcd)5n1SfcOoQo<8ji^rX+m0g&-9E0^X22n~?jkn{wZ6A8)) z4Hzj?VLn~}DLZRx?y&Db^e7omoNInxxHvL5hlOE&Ul^pV;HRYt+T-b97gy&7~v zyHg+MJIvvnQ->T$?whv$3;3GJnSnnk@}a{m9_RcIu1H+ArC3=*Sh@D#Hh~R8A0Uy*^$^xn`Yy-BF(62<=r;yfBZb9%FN`HnI*iO z`z4}%+=*@3_i8jd3SnaGkQq^z0!=!}%i{e>EM4ruil@Bh&)LS+jI# zv_M0?ul|v+xreMQp$q#{!&AIu*k*eNd)IZ_b@)twGV5`rz$ocIke<)GpDpqtS+yT- zB$Dkqd2dgI*HHHWk+keUEu5NePzFmHqDkvvb}xcKO=&r?dk%}PMu&0V~nm?YFv6QQR67w{g7UE=x! zc=P?U9ts?}FTThLWotW`E>)}~wCIcJ9Xwm~s4hM0_;Acl*l{yM#|4H}lqf$NhKNK~k>m zzgHUt(}jQe@NILGK;wrooyET-NT2TJ-3&(n`jbumkH-=J1+)HNVov_^8vpN%y^z_I z^=&}OH8jBdy7=E2(pO1^G|1h658llpIhaSChQ0{C2Oh7#f4*cJNbWEv1WE@uO~6n| z8yjB~Hhd%+yw{K%!apCez!e6bP`=}NYoO%2&4Tl<%|EZ65;(&@lDUFWeIXp2oOk{W znIm~~QS_?5T)!O3THbN!pYOdI7TxmvL~`?s`hWHXGsYA1Ff8cPbNy?yjq;&{HYTHxw`rfb4oryt3zz^>ISQ5!A!e@clkzHDu;6YW~m9 z{_{%y9~{Jbt zbojdQU%b@>?7$H3!QtU|-1>Xql0m`d;(cUL zz5^P~P#QrC`2CQU<-Sxp&23Qrv1dN6^la+#HIOJScV;3A-<%w4eyzi(>!C0sM&W^M zZUSDaAX9tCeH0aY2Otw*YpFrx6iCp){sle&#>oX022_T>(hgnsg~bD|)}V@dX;@hW zkjv!61Q^gV)_}kb4XF$euF!4z5wK{8xjTK;8xF$(0L2~)Zll(eN<5q zTq<{gL!!>1k$>KJ<6>QJuP8Wu_O^ghCIQ58M?nS$+;NZ~S;A4=-R-H0Dj( zyTFmA23-z78;BnYOaz3tYmZnfg1~Aa86+b-2s=^PSu;}|8bVTH;!Ma`2a62lj0nOi zQhkE50&*OS4jwInFN>PHW9L>!%$nK|RGr`k1?u4taLu7XU;eZzuDIzO=0E%t=^4Kr zXJC_NyYS{1mI6ag%>DD05GiK zK~wL6!Z-{Wm=i*B0=A`e2oVE{E^za}V2HnC@Ca%kLUAJz z2n`3J4hm55mO{`I1Z%RhuuSbv6VM*h^oAOCn>fM!_*cQQ+(ai1J-d`lbm-aPU-ypG zsgwJ9C0{zjG}-=3hLP!B(B>6Q*FGa!Zk3la6fX+u8kBB?zNGo(epiMu+TUI@+3YM2 zaa4`KK=qaB1vf%;#dK5Y#Doll9ac^nM^u=$8t~_Dfv{Vp+hELY3rvS%;HjxKVoU+L zI3B!yq@RxX{64BAi^WJoJSpq|2zU(E%1G9xC>jP&H)UkxLunxM;tT8-poQ`VYZ^EJ zAu$u&xQldh$Rt)!)1I&^AnGl!`YyD&1~e+;LLGoclw{%hTUcuY=c*mt_TWwwK$;Xh z(a?awMydnA2e9~HHVIxL@a+{)*27xHO$k?#PaIAEWEaN*O z;IL>*lImSwiNzwaZ;*`*!jU2IbmThY0c^Km&kYVBJ>ZhF0pAQJ6$94AgXxJty2npMsDO3e zI*0?yfLA^GRD|yIZsQ*0;88=K3^~~3z~+SH9KopvxSMPzumM3f4-FC6#FY&iz~4o3 zx?s6lOafUDtEmLUICR6d1+bJT!T7Ns*7BRUxVYF@r0WGaCznCFE7&JUZTl7g&mW9E z;CF*Zgs9uv<>_*B!;2w-fgf{n5OEwtja`HEHV6RBwmFU(v1k`isys;pTQu-dYg$2x z2a4&Rh@lmnb?#cGU_3=tweuoSO3h258Vhc_5J6H3zQPbF<81cI~`8ItQ} z6ReH7-=CSu2Vv)edo~DNdpRGJ6`?86C$3FqZ?hjw&SeG@Oh8P~pd(M?2vA`^HNaEKKCl zl(%fKvn;E=T^5^WLTTiI6aKh){(S1K`VV zy1c0Cy7fk%CBS(K$=U@)640B=kF9W8e&o*f(v^l}|^Xw~oUThoHeQ z0WXF*5p=*14YKrB%V2d!R}6K19bm3)7!ZI$04cnnxSL~-Yj#GAc~j+)`#%==JL9SW z31VXi({NDuKsB(QFQ(>Tb|SX?ayeu}L57Z!Qt0>5SG;DbXxL9ubx*_f=u`q3)QOuO zzTsyKTqM>Sm|@0%c*WFGy$YIvZ6bahB-N|)aNbR3$}|4511A;*@6H|AUxeavv}@pv zvwRnoh9wT)o2yp}GGe{mjC8op{vRqWBcj&>8UAiaB6oMbOMc+S*oU~lEGPUJk16e z1#ZADVEQ$$VPgjO94EL+-~_MLj_~QOGk-ljGam-DSwx`%dUn{`K|KuL?7qF^1=}gPDx3(g=Rs>!D_RAA2~6lN z7=x-^8g^DVR)D3y*Ru)|9VA_A3r=}1DXB_$BgifQn;Rr8K`N;R7)0yqb2S>4+E0RZ z8Z>G!<6J<(31HJY`W96pq!=%<3e`Otw2-h-wZRpJKd29`%{%;{FQ7=GI|p=6?gx1f zVQ2;CJ-mN-OyJ5r4iU6)nnK^1u}M4B{Vx~b(Ic2_kh^68%^CjZ1$n%Pb{9%aaDxJa zcf>Id#ws(QJXF|15<*4}iznyag3}UANG6ao2dTrrA^_|(q(SWq8WlV|>gFr01 zkVFdU*%l~wI3E%04e(9af@YtJ01GM}Owz#G1WLIDcu#^^!Oj> z&rbLx|1f+Ar;xS59k|x+sp;uNNtuUGp3Bd2j*Qg3cT3tvdO2jv*%=K;0*}F|EKq`c z)%aEL;jk>EvUilf@oZg@er9z1Uxp?8YB-T84;}g!8~gJK6xZRyZGZpkpMUBco$$dN zp8&d^5B9o&I_hgE%IYFY__XWLA=HQmnf>+RGYyCSW%>U3ndI}=k;e8qh51^xF`Otj zzjhybI_@)l5x#0hdnlmsyF4PZNrL- zPM}aLl1Kl?q8K4M#;8SHy$Y8>=F)$saR2?$|Fy;QFS{2-Q;)Uw5H=yqk5FaAV~6^_ zbcV}y5N*$I4X=KmQYp=N7G))|)}G!f^%=cg7*YI;$o=6XoldTb^kJ#3s^d~^uhyt~ za>@$qn8`J+byued9y9s;fQan$-ZJ5IqI6Z^lS3$<^#8IT+D@2oq36}RUI?m4o|7zz z>iR{5&-*>vxlZPd3#h^y6WYI2r7L2JJ%6$CXy{4gdu|vrV@dNY-dSFHv_96Bu&?5k z=sc2e#r%m@&TxaH%Wp{kst}IncZm3`Ab8)(`gM(PGahf=c!uZ!n{s;18=`WC1DPtnGf;u_9&LuRe$RC ziH<(a;QbMd>0zAOO$jF-4XN`#3%ZSI2D{`aCOVD9tk0WkT{Kqg5U7i;HEJ&?B9}Tc zviHb%tEyXK^PZ(byKG{N)3TbS)JjsIaffNrRR-K|L5fVdJJ|i#k?6`PR`U9^P}=@n zZ3iwnER9T}+th)Y3M2V(RC4pPe;>i-kQ0({J-R7tgyS7DI_}7x4)|_S$aISIdaB4y zZRX6k3LV{-EN3N4dcN@}@A4QEmgdO^iS%gQbYkJ(zpM89R1Qsl^X;CQv1Do0FbWNG zrkD8LcOSW+H~wLK`%Kl0K7~N(qM~!8wMb(S=Hr9PKCf^DMdPQ|(|>o0)bC9ZSSgisu7&@up`bh zK$$I$k1j16o)Bc^XzkMF6JNeiW=dUVs_i`VIem~*eEGqVWjOVAxvq@{Ae&~j14Jwr$=J!JUtD&}b@7FbI zYU)r9^dd+!5YsxR?61OCjfCxxd(v01^5<2KVAGlp(uh|^WG72Ax)(f!1sT1YE;ipE z-3`{F-*xIflDa3Eu#;+%2?z0IXa`^p1Q8@HP>{?0F3!tqOI7srJOHDG+EzqT+uJJ( ze7pkaIl)RtTLtXGN`$~@2n_PBUR8{51aYo{LPTMv=@V;umrSa{7J%zu?DxXYue0kq zdvvROCSRv+f0uD_Kf;^=ak;;6H(4`Nd9#Twz-aY1!dEv{ZqiKho`FCN03~c9<3nWv zL5FoEd>rcV(4Zj3ZB5~O7na4ISdZ-8>e(!+3@Rxq((DkJ_$Rik`NZKi>$fW5Z^_%q zz#+NsIjjI2Vnz(%D$pY_r9cu1^jI2_mc{*4fEQW!39MQ|JpuI$P?kaQ4D~q_v~q0P zHI#@%Z4%xz9Hthh`b?jK)de3|F$3e{ARcNNIP+iSeHyd@>yLtxdjn!%Lx%ZkSrMA@ zj?6#HYN)(doWosEuVMu{%@qDCMrfBKsjZ!P@?f9>-n_m>S7k3p1?ph>p`63-wxY9c zf$a}yy};?_hOAH=7qqreGlIFKYBgl_%|nqA`I5BnOv|#IfpSJi&_Bz*JwW{(CFtYoBQ|Al3x_Hmp#zAe}6204e&}N zQ7|b2V&0dBhhLSS1OYGrQs<=u{QR!!6wH9u5V}rlnN&EcfapW(I@-m&mk0Gm^L92WuR2x@8$8efE>0n!ItmW74ocXO+FRb*ps^V_XxQD{RRrV z3HZw0g@aSCn?;?y>v~C|y7FJtH;UKe4B~Od>7UJZdtWY6L_PM>);zzu!T_JITzRk? z)aZa?$tADt{M0s95_`GNbZB~^-1z;t=D}jeSQ?La(~|a9{`r30l6I62ZD2@92>3z* zc6sIU~MI!*F3O(hS%24&1VjKaYzvL+XB11PtfFWdw(f~EG9|xMrcwnwt^` zc1UduPzhH6Yd_%qyC*5#xkIkuW-6?yBA6HB z#F$M3K0f;9pSn;LUt8IBY$F+9Rf0=H=P2}q)f6fcqxr>h_R(0PYJ&C+i0%bpcFoqb zyZJ)H$xeekKs^rPH)p>4pTar4O{w6r4=t8X60sZ8s@m&uCaw#!roamuVo_jr z4TG48*UjC`XJ#+*!fgbONj4PJFJICC#s)2)+yauvL9}1PJqX(r;mz+wFr9ZSy`hl@ zJ<1^9R=h)g_w9Uvfq4?pNbp)_X){!V{!uFDTt*ua5@J_f!WqT!gp zcPLFl7wnM$=sd9skScgh&WDoO^nSSlE*qweBcKKa$F`U6Aj_Ev98Qli$(d<iuf_~y1unPNe(s;nGl78tP9 zYgGgb98mg~27Oqk?1d{y%eTut*ddJ-nhx;je_Y`;!&?atK#t`M7$ty92yNGo>n|L< zxolmgnyAYLxz&`x#SsULNBV0Iuji|KhxQ*vnVmgOv7}uw*WX=MJSHjM-0SALZE1s-?Oym=BaN5#`Zeajb?&j@Rdp{WOaGBIs%#GJ>e9WR zhfrU)k0L9oDdD)g%W{d$kMCnWJG?GUvDefOJn>_13*!i)meu!GW4JM`(mO65)4<-{_=Ps?A(-Y5o_1`PedqH0r3p45xmuu=k^D={&5m=D2Y zK~Eh6>IqMqdvE(?0F+LMi<>8}M?ezT7i(K#^9D#85_yKo4B@MbpN)$@heGJY`|b1* z*g(OYGFTHX=ViF$)G`;00TTTr!f7vJ1txGR_%m_MMNadYuj6Zc;Y5`H_VJg+l#mAN zFF7$0B{st?jNXt<62#1u(+J}QSSTt}Q%}g0UQxXad&776$`yFCK_pk0Xqx~~>cYUV<|fRC zpvaIb(cVBqP=twziKl1zj>cnu)mue*dH2|Mp{iJPF6&c(36J_D*~ybx#bj_EpeGv0 zlgzCFo=rLJ_29v8K(uIow|YU`PgT34qOP!_y)=@l>S9c9@=l z0m2hNOFx4cA;;6luLHs;z{q%mjxgd&2KXY_IW53BlLH_Hc%H&2jKT?mr4dsBJZQGr zHrJ7^8IVqx7Bq>%TR;D{zP{e$1COn}E(#@lwX}9OVy`8M?lqxbGnk~92uB&Py3G}% zkX2=74gGDh(leB!r9G^Z@GUPp%i`0EzWqOZRlPzMoszsXv^RQ#evIpJP6y`2iLTlk z9x~(0x6bo;7xb8hiv!5m0wke#weM+bi-I*9>=LE71R;aaZ@NYpRcrS-AMvZ=i)WQW zYpJVWE<$uW^cRfU=B6Q6J0B#vDvwIL8=yvbkD{MdoJdem23B2DkGaI($aox#wpyE-5SwfA#i0shI02B03Z$A1E_+j5xF7;~Hn8!6 z7a&p_z*!3rMuZ@k2=GBpG=yiWmjK=kHRmSODJJ`@Js+y_9NWS3^tpO|O!Fz?uKa}Kv|pUZv1 zW6j4+rWJNMvcqow&C13V9%}D<-y}A<;q? zuOL!k7xbgK^GcU*(^ZCs!9%DWsplxid1mGoshE<>3MS6@Z>D}TR_tjXmw&jz8kQ>P1u(lj_Yf?D)4)Fe#7FMf|_GC6-} zRLVU}2ohg#_OP|U<>rF<*H*m!dv)116p@*n~4{!Q(~e9!4dJ|l)R4Pp=(X7ei*CqmIssS%VE+m+<7Tk--~(RBeB1jY##bT4 z4GuQMRDS@L8xR`7*lGssD&QD^SzK6HSho{^kBflQB?($;g2^;scRBO!H*Og77avB& z1|{^f++!Lpd%mo67iX{AIX~7x0>@sjd?shj>LZ5!u*Nkvf>5|h%xdW(VUm2ae7Mr3 z@zh-8-qcBaAMUm6mE7E={ne4NKlM@VIwyb~fCZUK@_$saILv zY46|V_x>_F5wFBLEg|R|m@<`g@S_udEi7FPD9WAX_c_iabxpc*P{c?Ycm#g@;C|1r-ndp@WVt%{ z$V(Q1%gumI6>xppq&k#yDzTfm1*Oe>e+b7 z8w56BV?6-PCoVEdU0wZJ9T+NeIA6)muoBV*t1~1ral8bas90GMQp1o&(1C{W)yxZ8y%-IdbT32T+Xy;Gm`bvq2G6Kb@SUUKdwd%(J(~OZT^0$~9#!+VN~nP?H}qi_E?fY! z7KjgmOb|d{EMT2}`Ow6$n9oIObpn;l9+G5~#Lg1P9Xx&N!5jSYo0#oeQ@H)mL%g-d z4I=pcXi{m(N0AH>(N7KpL>?U!T}eAHP&b}tU%M4f;5Tl;jinh)H0_%XtS^%#(jMGX z5KLZ3Lr<&F@FfKX$}5LGSnqHBIpw@~L!M5%CNRisVu071W0rTco4%Eb>Qgw)9rYDW zw{U7^k2B;>xZF+sw}-o)$QmdoR6O*U={#&qc`=5rQ1IKi5uJ#H$#TxZir0nn??6wyf5|!9+usxb*>+!epdIbA*?D%_4DG9m$skVkDz?IQR2zE5nJCDZ5F#nQ@ucD z>MBQH5JfXT(eDwkIXjd|oEa~-bb8fcM(J5anTv$Zn2txb^byo`&M{;q!ITotst*MP z>IWP`=YA^1!gFiU`3c;)rlWoKT1{SQoTTsBIHL)YHD$1GD% zO<}znZZ#iQsX0?lUclY~&yjO_o zMuvNh>2s^kv^onRHtMN!a(;4!nRr-FSaW# zTQ9?YrfNKVe;a8cV0`K%a&T!F?Bjo>VX-=I4b|DaG1+86Y(hfa#-6mlx!WE*rYrq% zfF%BG^_n8a;o$tdO297@WW$i@G_Z4lNDX3h;k-m>$}6A+M;08g-GR0Q!+scm0`*!D ziY^z}{v;#7>$zADYNf=1?yl{m_>Ea_R&w&Uk(!;_{v4W;CI_kuehE?}@jE@!8+B*- zmFgw?hmv2Wa&3E6cO0{gPhE7ACkpYi7qb~=w14919Y*A2V|J;eCpxoE_|zh7R_U>i zEE~OAKe~Mwbg>D|&AvN3(EWNAV^p7=YfgE({(Y)dTY{|P>I3qN2KGZud3N~O)TtIT zUw4u-w92?;=WrV%T&q>){ zPs?6($n$uujzbSQc6;>m+6Rqg53RH0l#PKw_cj5*N;KGRN4MxlzLmI^m2}|v2)^ZQ z?&21EP@wmmrytVyq(>^z*Y5=^#L@OY?oRfDkUhN zzVGep*YCsRkOKZe(oSJ|IvAbH4c!i%Qh=%r6g3lzol;bWaf*P!&5C?D?9myIBnC$? zpZk?6!%K-cbQhB$Rbgc=+mU84M}5vV)s6w?Wq877vZ@{qGYbFR0v2RnVHYg28RV#g zyjfSM(E&t)Y4R>`gg_BMv6|k94BJhL`=L)DEEe0?fofMQ^rE;BE9*408sK%rq*f1Y z1@dK}`*e4AYeF>GP&t@I{^aVRMA{E|OIjJduVyC%Q0j6zi=v*H(ZSeeL5!7Xd2AJgY9Z_E1 z_HkE0Bz1HEuY1#QninSPpkOAq2b1@+Dusk#2RMS~=JO`u#AnH967VzW+#OYNBf@CmE@;{g!cDrgR% zvJMXkIrqh_p!hR*QjfyyR=eDF750=L6~Wcj)sV>yKK8Yteitn2{IP!9cfd;xi~(ng zcXoCFiEyF-vHs`18xlbu)xshhSlazezy~Y1<(#Y4Lo%rO0fh(;r?QhQmmAZ`WCidHO2>@`3nf_?ytYEz`PxS>?lrZa0?kn5@u>k)TUuUolM!T5 ziErL8wSc2II6&X83(H)R+Hg(xyDak>UL^R$=fI2U1Jce~Z;XgtO1e3CBhG=BCPFiek5`mMo*5P;cOZR$@?{13?%vcapre$p*3p84 zan(GgtmZYlTKbuG5c>i@?qQ)01LR_W`@ghxX{`lzxkE?tbPD&NnqJEK~gwb2?xbcVb=2<}F{|^YF2S zH+ydV054uhwtg#Ana$wVWP1dI5C`Faj&!d7=1`!Fmp-MS{r#a>w=JdlxEV_QW4*-Z zDTedPf0CAtQCy{9RaY8K7@ub|v^~T$C;a2C7tewD{gvF+M?`?gP&$M4L-)H;Z2Aw{ z=0AglucUI;){7c-$+^cNT@0)TEfo%Zbs~jf$M8enOLHg`xyc>b8|r1%wy%AOx2-(B z@06;|#bIy_f6Cy=kxm|K1u0f0tb_NP7_!)zetlKx^TYHDk+EW~-``MVdQdQ&jrz1t zMovx|&5K`8=?_r|89!q5(y&bW&5ZB#+oKo9GghV}U%3pfX$|&Otqa#np7hI5krp+L zSPovCuU==fZ1F61)Tt{zt~Tv_{#eMiX++%x-m!u0bk;};ISHq=QFE#P>O_%TCi9NAU%n9bR7RyE&%& zLOK8L&ocbs?yRpUA4_7sp51!%(esC9-2WZT3LMq%!A%sJaRT{|3C8Vs^Xn+AGC;M1 z+dublvH`h=w9fn3`08n(_V$jzpjC(DXfkeG`uWM}vwv=ikID6pf}%hGL6*)0zU$lud26mMfpRLhqRD9V^l?jOqF3T{r-p-$ zfk)f#ma0i!qj-n93#<+t6E#8nMl}tK7s!nj=^lmHy><;GpVl*`iE;~U%I@U*LRsuP z@G*v$rRM3Ky66@s7R=+iLX3M;UyZiB;xsm0Tck%)hb?wWHQK>Ovq{8Gi<(J)n#LHD zKX%SozS*Mnl`g6L^zCoa*~j&iShUcM22W}(bcpnX5-ezXoZ4CMo1}5k#1vDwm@1Cf z{3s7rATs(Q94$AFRjrzFX}QxOpx;ByY!Y@+Eq0bsqCQ_L|#ENLN_I4$a%Y&-%>+>y0y=%_8e^jSUA`}Rz!wu9=o z=L9;O22;!jIH{srO6x9=&&pgjO4+UnywI2JkbVc(^P-wIJPX6GWGst4qrs7=&R=(X z8XJI*drlq8Ge@9Q`i5#I;$sfl5;tRF`oG(peiwk}AE%mne)mK!61o)jrtY$>+HTou z(`uB&r@j!fBzcu8>Pl_;On;ps`QG$C%Jjppm&e=E2r0vli=(x8pX@zNT8J*jlaz73 zy=9(iG1+M+B=F% zr(=x>lNuJ(+WGr8Y>EQe$we|57da5kR5i=et#rRJDm z*F9reS0kI=q-Eb0VsK^h<3U#%LCl5Pf=8gaeMTiJ>3jSM^ENGcdm-0~ zA8nb@OzU-_$L$`)V#HtMRX?B}3Qtkb%C(FZRLcmNVsqJO@4uUL=C#Sz+h|ikrS)jD zCO-f0$DKFyO4}SUiu<{o%-Ou@^Vp5=KQc_&x8f zNu7L>7tggLP48t6Y*;s)-d;lwbXRTT*QN=sqWUqsPIUsa3fY{@g`!hzKkEl(j%A*e z=?HAll`@h#K-(AJwm;$5vp7n)o!c~s=S){|$U*C8^DBL788|>=_6V_*xOLVwL$|l{ z11spjh}{v_CYq)}3xzv1fo56OS_ceK-n`$3SBeNy8-`{jSp=Gn1ZCBP&-RaF1C3oT zIHY>NzdLnm#7>Pji9dZNi#@{xgBFaUQNZPTe9`QFVRO8c_mh9-MjK(C6`x9tS79EL zUNd0wd^FTPzA88_&Ha@{W6D`veo-n*=4^l2x`Jn^wjP zRoS`&nESMrjW-+BZ)Xuy62aF1b{Lh_mo#;4x)r_{be_#`a zza`coqCM22mF;_-2IsULPi=LH@*RP(CAHv>_d5jAOE)y!ltaYN&gyNuo_Jv`NUu+3 z8cR!XbjUHHi+nMdevV0=uX-ju)xXN(Wn)PG81ZYNYS%Amb8$;&_>EOegsG(+dOnPM zGxO_JJL#9Uu#MQ?Q8HAUy)LZOe*RlX7++aC+t-9I&In0_CchzU%;ccwllG2gibKua zx+}!fzb*>yy`z{_*55O*FL@!}P|8kkm%&|CW;vvl6^V)CXb64EAH{erQ+i<2&`$OO zyL@~}_UP0V#TM+{hG%gv8-nJ4bvbN);#G|aJK@5Gy1@Hc{Sqb!KdR?4Hs+hWv z9QUxGTO!+>rbB=lEl}6uxezk>3d28n=o+P}$<0#%{TpX%yaF8tH@ZU7`M$TRX`O?L zfRJ@Q^2nk)N|r)EiDK}{z#ZO`V_Konbpja9J$U?{%!Sv7)^w)P?&WTL*u4}Spv-UO zo9!{_@H1#0+esHi^HE4~TFh;xYOF1IR7hNNN6sJ5Ul(H95g8*gja|x$a{ z<{rL0t7ja`k=T*R&DbEgCo4A1ca(>PsAW*I$X}E<H4>4>Qv2 zM<&`;xyv}ls-_04D(!f?d1KRqjt|dv{;+&-Cz_l+*QW1!k9$}2p@Uwu{%1l{;DX1G zOd@ot!ZAhjhRr-zWfT?@M}1q86}9w>()jQz@3{R6$25Co|D+5~vWR;3W{1PCmY&-@ zkS;#_i~eQV?SPP)@rM`(5xWeGMD*0-Tq3o2tisuhjvM~?IE(8av;`h<%H9jQ!*Js> zfg7i_@y?HCMoc)sH@o&6j&%&1Ntn84FUQq9oe*3fYNs3y^5WQ(x5#FyP-GrjzgD=k2U(#&a%*D(){&g^~EOVC-ql>KqfYU)Gb> z3VC!yXFH8nH&}?;F|)KN?W;`uqIBSkVB**^>3U|r`5`2A7Jp%xeQ?7tCcOg3u(p9d*Ub2< z%VkPdubf4-&fmeF24A)2+_perBlJv<6fgF~#v9!sI>XfG-8Q?bAk)$$`#JZ^$umQF~GOaJXBs7LB6{4md_o3aW zt{8k`H?C_IeIEDL;w3ft43-S%LVfc4d%?ZN6?_^_nC`3^nNsaL+>g@!JuU32o^Y{4 zOnAlyT91(-_u07rm)GUN^$X)$hVoNtmo^MF?0G#?&fhh=Lwq&bA^WE!Wj4c2La73w z)TU-N^mv4M?B^9b2fNAEx_Qb+uLf|!d>cu;{I71on-IfbTh8OQPdzH@W*-jU3D1AM z%J4alKl?4wySWfTN%c9CgIj8>cjX9zd|IfKv#TuqSE};X@0wLVJ<=esF{1*QVS#nPXW9tYH^y-V_s52FgQw&#%chPGT`3mr5~hqExw78ie|p^Bn|o z%7)n`LJ8@svnakev-qy2os9T9FL1)vVZ4vIStFudv6H?F?+M&VdhKW%qR1{`Qj?wf zG=jlY(g)V+XsgORf(^ck6K}jnXmZ`QRiECADdW(xl}*Rf9=f!Bc zSHI>hU1EVV(|QAPlo6&N^{q8htq`*zy7*z9{c-+-tM{*U6`g!Uclx-oBKPuO_pS+b z&$*BDurEoe3Qv7}9+WMc6ew;%**yESapat^A=$}nM#^_fMLs{po5LBZ!UvCeX0e!gY?pa#rHPxi*=wcwM5u{QE%L^>+{U@M&q;50qZ2G+r|xYUQr~MS<&M;* zDvOV%b6LwI)RYwC;{<0^coh!{YA8?!> z+(;@D%YVPOywZ)n1z+B8M;DW_OKl|eq!9|0+R+&*`^%jIB*r0u!W-q?RpI zsZpT?jY@(^H_>|EwKDKMkcBBtgE>>yGRMn0-;1nL$#7q>r$PO;ruQy9`J9eOZ>{@@Q3vapFHMgOS zMRVt> h7#8ZOQfea*$X*CKA9@y1brWgm@5m|3Vq_jZ|8MF*d0YSh literal 171708 zcmb@tbx>Tv5-%LwEw~2>uE9NcaCZ*`*Wey>aRMR1J-8DrxGye27KgN*DshOJT>FMd7)BT%1v6||NIG7ZeuU@^vQTiaK{puCU?yFZwjp(RFs{ezL`->zQ&9)t~BTEADXUP_eY zq;>twPjgYe=q^BkJ#L>nBG%uM7`Ata>Ea`I48cs6$UdOQMa2{?#l{xJym`yHzCOw2 z?5j5YkbiAU^26H~gW&7t{oyCOjkQ|ebAM=V)zT?0arhhb|1OVJhn=A)()h^#RbZq9 zQJw$W)f8Edj{ARxqmv?NjbQyZN%VgC_e59!(-!VdfdBuVq^7eE;P{CH{YAtpZu?1B z#8t2T_kXjU{)1hgF4V`-`2Bp-i&|SSwJWi_KC|gfD?N;~*j)NcGPwFX^nZALK{fa( zlebX2tlk?CoeG(L7Jc&}^bRu!eAR0|W96%jprE>*_{18;b3ESKZR6QC+wIE2w6=^6 z4l7muzlkyZCARSs$yK;~V8XQ5Ft&947?mbT7hv_T11mR6!D^*mu>7@iw9fyE)|8%Y=HoN8q2kN;D65xc=x2mGy<^N2gVRBy!?PD1_j_WldGZ{Yt{h2>|6aGLcI^`N z6%y_tj~Jj^w=~up394A-*HIbw`dyd zECdD5C;OwjmP0+WM{0^U=dN6KP3~eXD%{7j4)d>a@^b!XOKG%5UVv5)kT@+ovI6NExFFUik!Eo|h zDBJI4fBuP2KmSXS``;?gdxg8B^pRYt{4WQjMPEt(+hXt|1*m!{SiJ`?R?|vf&Vlu@wU7%t>Y4Y) zXVkMIXx;yaAm`su-LP|n-gT?q-0y94;P+xu@9Sb@%wZS(cm*;Jj6DWsLt+c+zkNs5 z=zVq2{l6M0{GYQUgHt`_v#UiYndAJxJ-ATt8>CCWv^d~c8*{Oq7oN6NdJ@-v*?M}; zUqj+Xe^IIL&i#JACqgRnpDm3HO(>dqIqls1|CcUoCr9_fU7QIi4IT5Q;Wi*LE&V(` zHnp=Ciph1I`5!?(f-YG;P0#udJyQW^gHSlf+(M(T3B?@h}%NPD%v_ppAkx~^;Vt;y+qwYneApyxQe^o{R@jF4Yf4_ zD$B5>{ar@SAAf)K@E4Ylyci0NotqQQ<|jVvaw%$7*~{Z5Bsx~%2l`)6SgZd#_nuv9 zmpT%YT%Q%%GxJ6pdakx8J^Y_l#3E{g5R3{JVx=|u(Vx8AbPZ2kcZtPPzPl;;eqk`)o^i&(0Y1vyohrPQ)qG9_r`7!Xix0xLF3%WfR|}?{^?WMqXE@7PcXcD-PJ& zMW|Og0gvpuFX|^u;+uB(E6`KGz6(u1b5(M`(leJA2{#$$x^)^+5Omn{aK3@o*q1WF z#N0emhOHC(pLT;c{Atkb1h$=pMo5yAy8l68@EHaOrqAy z`qnxJH4qA&We^=iQ%G$R;eYsq5=ejnb?_HFRpsbS`vi0st|60WY!h~%8ygECT;F<{ z4B)RIh|FFE7KJRlmqX1Y_yB3}Czt{)TfRGn>?b^tJzbl}Ws?%^+5C7{S$t#O{af<< zkP7h8+%GMBoPfYa#_-E%;NB^V)}Tqbsp+)ohDf${gQW>-V?4F4FISr~>()~+b~z>G z7ZcIJLb~rl^fMoBUH{M!2~SyvxfaRKln(rEbGouU75M=qzIZCZPV1RcW8ghI5{$tW zbi$K<#u4@=G3M0mTHbv}-CHTMm28SKdhU4M(mdTYq^h>3`0SJsX43yf|Cy}_ z`+faJalt`E>@>=q`G*BVK`JP^I$-X>SNY#D@~F`_h6SPvx*F?%eLNy)J9;)xx`7Cp zdf_EdiogCv>m>hqQDhB@leSS?ariugw&Y1qJDlYL*^b2HZED8w(U*HdgPaQbz7cy% z4D@juzWMFh;JzSjLt+dwc#Zssuis1}dfS5u@^wuf_AbKTFY7_xdrO{V^G$+}d8-=~ zBCDN8=~|Y%EBJj^aV*|~u=DA`uC2=TRcwK1>V0RQDVOk+?8dfPb6^Nn6tK)$rglJA zA*DuUHXGld3isn<@R1vXm*Xn>LKU!3;rsi;+$p|s^)=u1ql~9zxz4)*T*D6}(0FG6 zNRm42T-w#;+h8W$efqAOffZL0^ZCHakP7d0oxqX@BNhpJ9VJU@FZz7tCN}8iEW(wV zvRK-gEa)~bk<~7m_)3r?^uT=W>Ry`6Oh3{>3YE{iom>FPffK?#+VCQ8xGeE7v_U4y zfVL>Z2bX}p8jQU;CmuY>YwU38d%n=dYFR*WWV~wN zhpSLg>_c7JW=A=O3DB>Q3LbL(Hg>Mvx3s(F!AN7-1 zv3XNw^tecZ6Q*0^8BmeCC!)^aZ6B({wlQ{Z_RfW(&fM}Hcx?7q*82_L8Ty^`O3V@8 z%iysY;H%mD`=3$8A4|QhCx0A{UlZP5z+jc=J59f-;%Jh#iT#=eJPi+<`cf?jr-QFi z^G5l;=jgIQSj%FDn52jQY@)BU zQqGx^;w0Q)vn$McuA?|hcFIyLwisu*aizcr{ROU0o3|_Z3N1P_XF0GF;ygF+DyGY$ z{#w8pG*!yy-cFK@#eQe=@#tKYW9>)SB*3%qO8LXZt(Wn~w|loi*tqYCHNLrfRnjIo z8Os&BI=@+S6*4`jA89isx-zvx{TwwPOhU|JXoAkcu3I0Q^=w17Q2#gceIjb;zNjBY^{OyE!DF9VPjyxn!h#Z@^$5ywZk!AprS`}=i{#CpZfor89{ zKWBXjUhC2fO7t4=Na@tOryf`Otex1gF%Zp5Z6%eC2C4qK`9v5Xc|~y7C?na86P=u& zp!i(;SGYb$)yqru9|a{JEHU@broxJZWa`SZICg*QU7!26XN#S&yc^nDF*)yl!6A>F z5(wIAi}|FVkOq>yc7M@vvwU{NQk6v2RkDxThlFWTp^RXjhbb}C$tKj|2MX2>5AW!| z2I@u%oJO1D$W#y=F!|NXggn^E^4eUn*@#Wh{7yUm$zQFzV<@mk5!(&sgoBqA-DF?5 za0n_q^((*b!aSiIeV2;SR{I!)LAOMJK0E^{=v&E-GNv56N)vg-t1>KdK&$nlkj~|^ zTh&)vFo1;dQCm_)5Lz!)bMGujwr#vs`*1PvvObTo5KKgYW>-mAClm7LsqHCVmL}@4 zU^YV%t}uueuu)4a{+G3>k*Z7Bg-AVdmixjlYsmi0#=+08pxMW54$-jhd?5pXO;upl zfIkmeqVeV)WaM&ysYZetw;W-?W3mRi-4F;baBzNAQ=GGR^LaxA_or)S)PaI(e^Idi ziV3;vH$3c0YA)tx-rKV!^;XON6OBTX>LcCF3L<%v2x_Ctw=|Rjz!ka=WsQ9rXSJ!7#*z{pWQ#>S? zlt1petWisXxMpyMi`z1#^lLJ(zRm(j*mYuV6D7XTFdH`LV$$Go;9=-33PYWjOAk>a zP}mdZq&1Iv+$9BW=UTK5jA7n)b6diHhNa8VO?#VsZ6{UHKLyAa-UK@p86*$KT=6+x z=lyeC_B6vtu6yt{Yu6YPD|TUS{z)YQ)2=sjZmw*cak|~+!=Vlb7$+E`9lgsO1m12FMR{bHwegIXYrb1fN6j3)`Du{~p*KP47(CJ~+t zD>nz6@F3heD)gPU3>+^qXioO`*D-C+T3JKp?&x1rb8q0}evXe2gVNp0kzjRP#@ol+ zF{}65U37(MNClwfONI&*!Ej=Wyf|QYIO2~J_TrFJmd*yA6q}>#31+n7`n+!WDSPor z$hm2%rIl}+B@gA-o-@1zzt6}_lS>If8_OrLdl&WE3tBPjPPRLB4D)ss?#47{B+i+9 z#7>^xJbG-=Y>bT^9zg?cWOsw^3Ge`v#$1I;22u_|;^$e9^35;sfG*K;y&+n`cpTRV za^-#Xpt}_oalwxicMf8QU9%YJRPu_!Lg_09d$VMscYM`qjlIsgmcK*`@RDSxUCt}x zVPsNEr-E%n3<)iS8MjB-eviu-8Qlzk=%Gz9oK^Z(yR%`?MNnka1E$qwq729++HyC3 z8vH#yZ#Wp#^qx3WLRd{*nCC9gN``*?%cz^iY0Oga$+wG(pHhCH-r z9L|^vC(xh*IM4oRzoJ{>fOWCtaOHpxEW2EiLthxFNGl!ggLJ&pToodddu$%_Wm+k0maa!Gl}_73_S zn=+s<8w97P@!umo9dFSI{XQ!D*V|Oy=_x^wKx;A~dGTa2Am6*^K0{yREjw6oKiE|x zgABM*sd1v(hq)hjt^fMsR&OhT8K09O{(z6#(?!D$wRI>p=OTOSUptbE5OUb-#@qux zp;?1(D5z%nzsmIY;oQu!g`3Wh7!pnW1V#rN`5jlCwPhc>je||wA2$zE8T08=;$C!9 zaeZmDE+KIWmD{1vh9>}22+{Vu5r9!I_VHzMgD0<>bm zk>C88#*<_u6JAFhZ0<%O+zh3r?Uca1FxZmJfMJ@`w#tP2$I(&dOAqEmw4HgP6{vs34^#8_jfB<3S!X&W7GG%itL8#^7~6r$YO}E2#)d z#}C2c!pAtO_kM_yKSj6OvmU3iVeq+D-n(;Gbdbm)>IFUd0Gida=mird8^hk7+uoKa zzh7-Glk6YwN#t;-Pz}d!Q$R(5a8~1MlAk~K>6rnNIuz0n>RG-%gVpBq&y)lt+hv(e zvzZB(kXywsJ;3{ijHbl5d5^gkxfSYYY2^WJ4?`F*$hGg~0+3Ln-Ayt-Az5KFEV@wP z^`IJ$Ve!Q(sa zk6FP@D&t5k?vNP&j?b2u{IcHUv#{fb%PGZ+?Rcm;@PTj`+|_E)2e{CpkqSbA-d3aJ zuA@}?M-D3V=6_lXy?P&a-0*W<1`vM zBA7JARFK{Xo?hn)^`9~Hcnco)aNFuSGSPCr1N2z|+`6iGj?Y>lx4S|Pk~rewk0 z<6Ut&#IkOMr_H;d0uKP17+fOV=np3XZo*sD4`hZLTnrl|>qsDjr;i zBYg4tvZvR=7MsHbP+lNWRUR1we&PDzidxw%fk-5rODI@1g=+*>G1@mxMC`*Yw@&o| zCW7+1V1|mZS=AJ3VC#i?f213zuy_&^*8i z)rXQQe*vl~m8Gj5);Q3^SKM4RVUwY7jbT|2xHvB*%%skKw*43;ANhYYM9 zDkg_GR70WclK{7M1=2huq&vqM<{vXF|Y%bW4|5xaUm+nYX&#? z=gBf-Hhc`lt&Dys&}uH=>uw+I*Tmej9)=Uy?CWi)FBCs{=m3O~VPtJ>FX-w~>)}CL z5IJhw13#Zur&O`ew6wyHzJ&I3jxH^Jk$$DJqHqytQ@uKdX0=JUK zt4v=56r`Fb39<8kC=WQlFV0i!j_V$a-G{O7(o8`*5bCkKy) zKS}ZkCLX9h9J%pzp}AoiJunS?D9a~u*()lI{{KoNQfMK zMA`C#HWg^<^YeT)(rB&>JjW`vQ+1#1o}jjm9Q{Ug!a^r0XWS=L|Gn+mBe6!(BY7+? zp=umfNFo}z%h$ZuEZDeEI*8*V{Q$wJH!*P{g@9AxCt#92rHibOrHc5=_D)-63+?*2 zH%~iXL%?rw@VS_&!CgMqvG%+FO9Io{;ZRi#tKeV5TUp9ROVda`|!8~CnP|#|l z+6ZZ^YYLY(uTVBwhW*(gbX=WZ|0%`l<;&uV85s6IdCsPj5*f55TEt7@d@{fM{~aS z*#`U(oF2M3`KGD*uDx3sY3<#vwCQRQo8vAVC z0~>Mdk~TQ<6*%A6PuaB$sjyBn@P` zlOs8-p`=t#DYw5PXYR`b3An};zj2`b!IcAW;B#;ek{}MPUQfyeHf|A;Mb+b5oCQ@i zJ{bkJ$)8oMV7l8sussU82$rfp;O0*oN7bz2(TS9i?QgjsOFdpjKeN9Lb#(B|ab-{S zts8Qid+9Z>2uZ$hprW4Q_`)lU2&M~3<3eHcrShji%%OQkdu;QgR3{Q9Afb-Be8q>$ zb&u-_H3m1Hq|7vk1+>u zPkH77Pq7*J25(#07!8M*A-1AE>glZaY=bwV%#GdLwDLfyJCXCttxhL=8PTTi!-=wl zBeacr$0C%Z)yE}4bV2;dq$>H4t=y;z*Vy|s?gwww2du+bNZneZewIX#wX6=&?o5q#v&FZqpn04g!&&tP8Gv zkgjZ={DV63jeDn6bt>?Em9lVn@MOt8H%*ngbJo+_5aV{-szuj<=lP(vk%%H;Eqk?r zq-M%X(AdHUfw@Oo>YukB5?A#hkayLEzo(R*=;#ZvTzg7s=Bup@C4F(b;c%xQ__*?^ z`c(KaWq3^e8`!0KIp#USny<*k;;CG9!KIYi6UoI~u*>K^!w3g*fAIjp;dtqo`rfLg z*T;+gOei4Wg&x~|za#&51~wkV zwa~p>)jc#t$RE)dNDko!Jvp0TAN2|D8dWJHQmm(aYveq9fg7KS8ppbPHfHg1B0-UK z{{*@7QZ6o%n<6xB;;$l45aDi7{BB8WO!e56IW-g=6eYY)=4&S_E>w`z@|80ilUrG* zvBvMb!wlvyG!N=q2x7uW9ox93rEK=oD)FAfDY?P#CSOv`^k&bLM3A)WF z^nvVH>EeK_o@q&x)i~BvZOCO;% z17sXlwRPg2_l-VAt?e-GPyG#u`*N7s)4eU`+*4ibR9)}ane&@WhVml;G9ebKi%>ZG z9n)UFW)hCSAVJtR5eNL4uk-Hgb-BNWP1U#`J%?$F2*0OnTXs^!91Nkr^6@8LxGvcs#q!1I?G zAv*kXWt;Ch_3e$XbZ96e8&o)X*nEU{YKH-|lF4B>GT_hJk9&iK5;mH-^&-S%iK59O z@{`do(wzd%*peAtl#cd{s9nMYPkf=h8Ph-LCe^poKEAKxm9iddM&-za6PxE>F(>T` z7qnC9wyIZBzRvFK5cF1%YHB<8NC-W4oJW7>xQ|U-hN6?-fyvGFIV@EIrCBjH2HYez zb3ME2u?;X<%C;zPmL65NlDyPgW!Y{iGroYnV^`8&Pvo>2$9aPM8c00*Viw~4`l&f=L0H2 z;*oB6#QjOg;7D_RK%*AJ3lvw5H85yP6qB)&`d9?!yE*6D{u!Mq!P6flXsrDNw@qE z67ebnC0*xjY@+gL=$coyX9)(}6|>1Qh(ZFxiVKm=NT`eikm|aHh)!P-Ri?QD%()q(p79Ea59mVuuK1_KH(*$|e z(@pzpScE^E2U24RyD?^Ans1F!H&_%<9XCFS(CWSxu zpx$5kFENFaH=PNyGjni`z?Xc#{U0fYy1UV@gNY?y92wyY7(W}nCmH3d_O~VPGmMrh z)&y=0rxN5kK6k+A-^p2ZH`QZG`CQV`SNej1`N4;+c+{esuzsNSF)=39QMVfE7D}>0F zOsmRVau?~nXeDs+*A~!2P_rylmcO=U25Z#;pZwY;wSq3e`5zj~SMCvl@U{YD^Y&Y3 z_K|o#F9jchiZE~Z^OOznE@FItUI1^VHwOlfOuouKjmlg9c&Jx*n?F{z!<0>b$QxE^ z^_8B~loV4hIZ>aWxy8@RZNZ5W4f$}K>sE1PwvzqH_4q)vsprI@^I@!}yrzQ(0K@zn zt)?ga`PkylHG>nf9!6EYlq+wc?C_qO$cu0zF9YCfOA9wBlA?rJ$>L%>Pyb-qF|vMf zyCbXA31PPyOzx=nQ*397F;#bWjZ>n5v*}oLzsrlHr)^*4(j0PH(4A1yY-q}(44v6j zS_lL4vMCH!!pfs5Jx^xbyv<-GEpZR@x^m>K?nZ@i+uONUZY6AnW-)W5L;tzf%Pg3A zQqc%i$^OoAlJM!`C=MaI$QpJxS>^#c;dAHN{Oc9-o}z}z$tk=MU&5=Li<`&cgWQ<( zY`d4rep4CiuY<7<0WT+=7%mTCnZjAtQ48A0RjX)pS}8G@c##`M2|4p!u+I}ZWDh`( zza3gX&G!BDwVV#0c37mM_3Amux)-=o3`3@Xo4-2&7${hq0uJpxJcW~iN@?&R56nu6XgK67u8hTV1 z*WVvt_<`3^gnzh6OBbT}=s%L_GP>l%V-S{oJb{?Dn2c)q;Va%?@Zm+^4khbguH6H^0=$B7Eh9lz3sXt8GD15L-@0W=HI~Bci z>@H^tM#Lu}QRc!XC`8~Pf9`&`r7aGa*yne~uM7T^ODUDlTE|74%I&E?F_^$e35$Fv z*PoniF!IhQ4g}94=z;%`TdFTd`HwGd#mEyXs`QEF#n=0t2b>{%b@Zh#*P~q%tj8)! zq!|j8|1^htRPtQ7gA_QeO#b5Qr9it|_ei19#fSBv?WwgZNGHsOsMr6v)kIvpWRcGF_RS5qxnfFm54Vpuzs;Z3}i#gZCYhjxc= z`s`_oXy~;k!N*fdEWizg43`6NBEyP=DCaUM{_sB-(aBS>Md3CR8N#m#4T$_GUZ}S2 zBsy&+aBcUs7(@!A4HP%~S_@>Gim0L z$$96yTY3U+m~>xgZnktkDo}YK+}U_RQ32HsG1nyxlqc`gH++z{ml4A5DN7Xe&BF+B zqW3sb>q0;j5xWM zQC@_lAkkBJ#i`IKLiy0h;Z6}mKtdinq8Zv||JLYLu z)80eB=s>+@t-F)D0X3nUApt6l9ntb*gM=qE zzOM$R)n}$Y9XT-6z)+LyNDSA%2UJ7?iuKnnIBkN|E%#M%#0ixvsCvOs&6@mpxTt|Bjf)2z7EF z2(Pz58p&f7=LUM5WUtjDSafT&^esbC&q-GMGflr^v)n^~Ro2=*Epc+ox(UNrR@nOd z6iMZ;-kBCM_;EzrZxg$l(^=wNV}99=ACt4q-iY&QXGV|r4-cHnjo1^&sSy8nn4y=ntiKg8^)y-7d-~kMsAa>Ku zUjiSMXvB{4eQm^&!8;hVA1719iB5`V9-_rO-14({QmIG@3N2?Z*i)*6J@`@Vmn%3a zbRp1@1!F%X>?CT~RhLu)dk4Cp0P_07W=sd#e1yi4**Yn2_)<%=i1@wSE)5R5GrH<*@gvGV+%37scF zeT8_^tKA@Q;Y$Z8-nsx8$ZQ>@<^;7Sh^m0rz@HnJ)AQRKg$?LmQXh`){?r)TkV69f+V= z@v>fGTtpHupcS@PGzM_chKTi_vG5wn-u6B%J46&3i>;{!yIp#Nw%hYuW}2+@SSl3V zJ9|-zjlUEt-N(2r;d1UcTwX;IbHhNPI5)#mCRn$=S~aK;~M=IhPeX##a%b9Z22@^DeqkN{sI&4^T@P#*sdw*?WAt? zvW#`Drw~7G$!8r7rf~vQ8a_WOG2?*aUjFWnJ!K(kidLGR#>B9)HPa^gbSu~;Ia znJn&$OD5;hM7e3MM0{1kRMsZUtQHrQBg6WT0bXHy^Uuiqz~X`Et(Iqr*$z89FV|IM zh;ym3>f*n=hXaDB$9uNj%&QRkz6s}fI>{B>NIR*Sbob zZ@=c#XA?>?Nz^)!pLO}gd1{YbUvvtCn^eTc-0<{AKvHIO)0h_yPN|S>ybcx?e{r1< zRvN;+UxFBHru(i$cED(`y)ioxFSvUpDO-jueonnAf9yDI_Sa+p)>VA&;T%WhTJBZ4 zGTrn%Peh~z@i^vkG|QGoK5ZHF1S zNc*DXtMVOL`73kwu&y{6^S^*Yn{b-;$b~pUnG%}HqE@xhZ;E)*rzk1Xmnf7^Yn}J; zUroA%0_L$wW?a=A0M=NS+kZV``#dD|Tq%m%-)MxJe#k;iZjk}aPDQS{qV@~jNbN#b z19@s4?)-ByMKPEqX)+r%Z1uKo{T%*eQ448|Di954JMH>F8QpYhe8oFDWyG#jGo;=! z%S2(Ix#iDUGLnRG(qZdl35yZWco*F__^!X(8AO{M!cessuQL#}IJ*hjZ-1k<;+)6& zBs$V{JMvc!>dFr=P0ln2yZo#+jm#BEi|sTebK-vwyz?fIX_Wo2U|g=lfZGybQg3r_ zPDFu&j(ZYgW$mVj=9(XA9D{|Yc$+1l$t?H^C^!KudLmI%(k?1T>G5-`%?i`aqRus@VF~`11dN6Ff1fytQxwvQ2 z&(=;ilg+$~My?v8Og{VIgjV;NYIkf?-t@;iO#kyL-4;}kR8p)pP2X11tycbncP`w1 zvygCQ5u-1*X56EG1kDjna0+pYk#=!(Ktt`Ylrdd!v|lV?cI`N#xN%&6g|T3bZ}OI+ z`B*^aV5){7uLbqmXRmK}NwT+?DX6!y zBKTncA-GOf&Boaf5PXC(GCA=@vYohn&XE7`%ZftRqCKE2x+gNl*H`g6 zKp;i4gM>oJA;~p&s_;?`Jb90EjhdON1QPHNu>2ib`NmDFlACHXS z)J|qG>62mZX*4Szff0t8yGw+-+acfkVBL$6KDd17R%=bg+PYQTMMR-5zoO>y(1_8c zP*mqCZlEc1r?9PXUe7Pm6~W5HfZO9@m(`?0O;92ZRtv*GNpFf#!X-R$wp$Me=J8OW z%PrZ*A1d%3)Xg7?a~yk@sIp%t%wKR{MLB6PT18@IfO|hZ72T~b>*Z*I)~6e-nbq^U z(jdxZ)0V@mix!0+h^FW@$XDzt4-n<&UZ_ZHv)s6m7nt&NbchnKk-b-|`4N#f_`tmm z=&o^?SV$@h)HZ86`$JA`y}s0G&7FE}>b^&#nDvVb4*kii2u_;Fi~(;thvlm2MJ)`w zXe9JyC*RDCnF01EQE@1Xom!L1iy1!--4)lbFE?wPvi%bVDRMV^yRMfjQBr=sLxd4` z6VNxgxgYof!p@hP$JWVrRs*eo0c9%%NQXoEQQVUjEDm+m3~zj{x&lKw#*z9LlXl%9+JRN; zVb8P6+hWeksr5w2{#oMXomt}L(7+^QwZ0`ostAeZ&}6Py%nqTv@#s$~V;R*_MQZCe88My0#n5Y(E^)C- zI~q+n$aulONQ8^jP|jThD<#hlF;;2x0~;v!y+R6*OXwoQ#>4g^?p8SU8(zo=6_o)k zR6-n?vp<;sVg2s)fiC{~wt~37C%fEz$8+iYr=tRUkn&?@ZLebUfN?S3v)f3sJ#noV zF#tr@$>kbyNOx0p3}4M;bM_wgah7Viko-bFg#4`n)AjcY7k~VG#Fx_vKWxO?Gavl*4IW`2n-QS9U zf4Wy1;M&~g=e}?%MQmHSxlVH3SG0II9lUDOSPWhk5}(G^APe#9CmzW*R;FU-&OM^! z^?+q(1kNDJd-qcqQm(Hyzn^?R=B@N_b?yW8c!aM8d47 z2$Osw9_ad%f(LMsb>|pZ1=*O-Zv{LZ|D9dj((2ggehk#b;gw zO@nvL$Y3|qA4>VV^!)GIPTdE6fxqzZ4rm85PD>B6mah+U{Yio{djroQByiccmGfW&*`Khl$@<3ah! zlDAtOLc#A#6OR|?Lj8gZmPw!(3SasjN?&CkC@p_E68l`}D^{RiliVky(;b05uOYD$ zLk+l2b^;unpTxcjx*NIp5dEeqIND=e%d-RV=lsC1m1?oRxS!eykMnxH5t-I(hZ`*^ zjJR2?8lKH8PV*3ls}RVAOQj~KAPHqs{&qZ(mz%KXY4`B~!?pFDL_EBHwjIkNxQBX3 z9gU+~Z??n4w<$@-e&aSF)Kk^P>wKAj7q`;WT2f!_p77zY^%pyybnvDtXY~52h`{RQ zfo2J@?9;+WIbc}p*1ozxLD{okCC+;J%Cq{~d=bad8(aJCP z<$2z;yrc?C0r~*g+7Hlcl1*5z*sdwop$FS#4$7|sqr}Z1vb?X9DHUWko@OxZ=4p(+a2keOmpS^^#!>!yy+$S{ z4=KEO*|ns`9m5)-WUUn&7>|wFw$SeZhOhydzXsDK(XEa;5rcOyO}_@ z*Yw6pB!zR|jd@8NHAo6>B4zCGx(oN*A- zYn{%DBn5URrPL*G?`;fT8Zd3eiE(Bs|-6*vE z!rf`r*c>cRm++?%a7Bzxri#c$H3KApBqH!gXv2xQ3|(xSF6L;3ZYuB?7F?8g4Xv_z z9NzwL-AqQ*!+V*rHx1Sa)x?{1mbflVjr%AU^sS1gK<5>eIzw85{Cqcnby3^<_3LQ& zh}e}38%KQ>F_`AUm`QRd`;=N=ZQ^RZ!?lm|#<$9@NxxdoDYR#Or)5+edL;gmI|ofE zJ%>kKmsym=i8St91UR9R@K_xw597eenmUn9>Bp0?d)Q%3D^HOwEt&M0)OohR8wZPV zgIMeEfy!cU0Gu&rWlKM<;l9I{r){&@e*FMZ=GItL}gcD&nKcNXLdpGb!;zjbu zDR)70L3mLUCoJ{&F#7q){b6@u#PjkuQXZ@guZ-g!No85}$9m$SD9%ik8#^o&?QF}* zWlsCeiqC2E9nPftZ)_GxLBdUriGuo#4{Q-9y*A+%n&^*RZtms$c1smDb_8(j`YU1F z8cM|hz?cWzQ$!M}KfyM!DzKZadT<{tzYJbr_uR;kyId<$3P&VP{*{gHTLF~nU+oK-MG&0dgI4EdRu{4%!$uT|s2!2JmgiW|k zXck8I!G8HQB<=ScUPdXSU(!4N_pNs+JlS46<@X;nBofjPrN>$h`GhqB`}}jx;GnQ!&Jej{Lht-gVk>ZX+Kj^Y)S}+IrTl2IV&)cQi2K(T8t2l*b>|x13YaM+3_@YrGSG#v4K@Uih566l@TEoH5_d>8^=)zLIZN#K@LinQ` z)%49;4}17EiCM8|0;A6_iTloSolJH}%5W?Yf=UBzMjn561%7 zop-ov_opVXt&?}3?yt}$)l7!_EN~kTdAq<<8ChQav}y zGKzj1u|!a~^(>^lBfl9&)9Xm;FEd6$?l2>oNW_uS0M^pCu&cQT5$I`wlHs%q%^Rsv3;^6M3tub7H!`3%N==grq2nmM@_B`%0>0tU)~hY(TQZkzb0$26v-IvmlCCU+1Q92i6){9 zB{7Wm03}*C;==F}`knHhW4OngURr;(^zA2}F?Ldlydgol2&K@#bB@e-Lo1BwneyXD z93cSuaj$<~f6N=ywti``l-~+i`0gY!rvd(gx)C&c{a(Um6^r+#}qz>2N-N z0XeF3`ufcPJ$@*h@JM82tPGKabU^v&dX-VZ!UBd0)2ZH@!cNnm{#Y5Ew@Oyj7WQF2 z2}imBZ%@bL4WApg4^!o1YHud^zT{BYhp~))x};drKGMP+N4LbeT@ML+D3AxT$K4>% zq4_I+b8n*o2025U?U0q~yq~pSn_+p)6jy40%`eU$^zO&G(r{nRcmvm5Sid-mKqcxN z)t-yf`QOB=3^-8#=bk8wOK2oqp+MM!^GDj>90L81p3x|DTF&P~IIYF_Ri%@I1ZClT zB(a3=;0&Tgm_j~;k)Ji+y_ zXZGtLu&pXu^%L2A9onqlVFKWTd8DZ?nBOs7ld8h;hpr1sIdXbt>Uqc}AHOEGS1%}> z$x6mM*NajcljWws(f_vFX0;W%=?yaVm!~BFoCrp zF+Ks4nyeWknzT(va5rqLyL1I?;j@wAz5mgvUaDg*DvKb*@8NvdUOWEB7V-D|v0TMZ z&kna3hR$3rDM2EA@{+RSxqPm1O{_JQ#fEei(Lw3qNctM>|FQKW*5BNLM5NW?)ib$1 z*Y5^4r_qs|;!V{J`EUoI>XW37@SGBj%9~6+>CZ(HbjVcuHH#Mj^q2QfGbSlc8k^t-tKNBtlqNi1?pa(a>@t7^- zso+ve0hY3@YzU$1Yy=aT(a+EM6g7TszTConYz{(tcgs|hhowzoz1t=0@qnbUHTCVB zcP;4R+V3QNvi_b-prk{>TckTL6jAhyF__d;5~HnBmXRnmK5_Wt=~@%+iSM3P3B}Y5 z*9(;-A8Y3K4t2+c&#!j?O1O%2SMf_Zc;7e46Sm$XB?P>20?8x>Q)i~KLb z$lmSV-1Ynl^{gN3aK^LzMYHed0_l&0LYAvEwh9ABxoOLni*bw><0FhN!XvbOY2a9O zNkJ(2oDzkT2+twU^S}xh80Xg+VJ}KAF`90uZdREHg_Nux70z!Ys09cEqfqxfZ758O z(tfA80R8^|?k>UI-QC@_ai?*IkKE6D z&KUcDSMO0>z1FHZuh|R}--v+2#|ak+5a$zXUP2}hH(jV9gA^KLS-}2Fe%7BA#@bWi zPE4f~4vA?*>bB!8^nT7;bUaxg zs`a^O1DU>^3!$+-G%&>8YW6=b3e7S%JKjm z>m-~ZMeEnv6?JCYiTs*M3+g5x78??t|Ai{RcU0J_vMIX8htgY9-0Mb4Hlm1EYYyso zzA}n)9@)0Zu}ZxGv{-8x%mtH2_9}Cogh1M)0&|VK zI>OgTEXRR>l`2M9d{iZOM2){0*xzr<`PhxSC3!9Ug)Oxb74k;39JSfXc{kt0!a>Om zJdY{%D5uxKcMP()^36ikZi^QAXO5@ypgSeH@j=$m4hl%M)2%dLq4qzh$<$xXBsvKM z9q-4?AvkJQ(9pbQrkV&@Uv`$p8|Uy9c7=*?F@Y$^#B18>D6%~H7s|CuaxHpo%Lxzs zGoYh!>IE;C=6{03!mgO8Z)e$KRRVg=`ZpGtDlJ`U)n}0cRI9%0)&5LqSNdnw`JBW? z*MuJK4v+pB!9cz@WUh=yt+iM5N<_U{$yZ#y+G>XwYhV414C?Z1s!~9*PVf239n;p9 zgEyrbpF;*=P{Oj(Ma1NffNHQ)r&M>k5Nz_unxT3MF)l8!$vlU+7M_#D9N81WrkxW5 zlgiGflNF&dstqQ>(@iLdyux07eJYT(cOA;S7bK^5Q)@U{Mw!g*!k7LOnxJE;(jT7{ zY&AHe-%rVmW8&vx+cBfs40l-%KY!;kr<$@7*JF+mH{y=6nBh*6;qM=)GCXxUQp` z@Az3Riz!);uZ9@l&4kS5K=l4Zkk09kKDcPv^_KqqNr8-3MXS~7*IFBN+BIdR5{l~W ziYdPZCM#mS(T;f;=c!-i2C`5X0mIw4D)_ExcJ)0$fAMSj>LkIQnVhGL_LtQAc09wu zwqTb{VxY?=ewfQ(RR3SLtigzg?|LkkG7pAo^|lCZ717%ANX(x{XH9h-HD_T7-aPsHsxoX^8BqjktY-ckzC4jW{ z;A>1r=CD07Iqq^9=}Kb}B3~h9=&CO=pO4>UB2l5Gk`}Tsxfh%0~x{<7%c~FdvgfO_@aAP+%D#sJpYXmwr&foQwFj>pNxynL)*$xyk66%>{PhS~k=GbZ{R^H?egd-Ic zUHihDE{U+Nrn@E&TOsySs>jK}6~IP+G%nxZcKdQXmrBQpe0&`-Alw~=gjX|=J93kt z0~$k2FT4_{f{gzy9(As^KyRS0k>Q?!`dLX@Tz6Y-jhed#%Ax@v+K^I4^^GcsXMOZ$VA&^pc9( zDrTf(EAdmO$^Vj^CFX5f7cgDVA?`W6F0KJjj9HdxrR^d+dRILZu%)3BVjwggFB~^4 z?X#EDkQPu(z$5?+9)iB+*jdz>PD2u{)CoVUj88R4`fxu^(V8j>Rh|WJ^1Ms7J@vyb z6?aIO1Fd9nO$d8q)g_~Im=$w;=}~tA9Y)y~uzdZH*rje>Kv&z3ujC-a{N*RRC64~Os&&`V5 z_`DO6?|L%Z&wehKozes@$YVQBJVemg)|mITA;N8CP(|oq#niKnyuRG)8`lhhXp;pa0?ST8FiMv(G`x*Y$ zKPyYMdmNZIvln}*c;~GExwUVeKZ+6RB*C1PDtnc7JJYl5y$^7fz(ZvofafW{dSg-*>4oa*YCYw_ z!X=T+QpI)AzKOBOkT7)-Y5Fvs*)wjUk^OQfbZ2NNKS2eC%_1@xrc56SV~U1X_yfD4 z1*&tE9l0P-h|v`;4Gf+6?&|Aq_^hTBb;FgSN_Q~QuCCTX9fzar7_m)_+F@7Q{*+iR zNNM@_xEuY!NP25PdJz;%z)^LZmS-Y50q(pJUslPz+Eq`E$LQ@jSt|1OO$KLCw*N(v zgxOQoqjk>JUT1JD!j%D0VEA39#v1mA-%<{ZmHy43h&Y~pYbq8Zg!KGnd!@7v6BxHY z{;Xw0$U}|q3O^~R%V@NyaCJ{Z1k}|o(i5^vVVYj#k9E2TVfQLT^V#+9=@?ar77eSP zIU8tYsz89%c=p)-cRdr}6BM}{WpJuihc)Vw`6t80?iq%Jdm#A63uw1BB~M5*NW|_6 zEf#gq%T|Ek=SX!UUymMJgCN;Z_y=`rjJFPNw1F7NNCO-5CdzIfTYYs;&-gS>U%j0y zvPxUA<1goLINdDER~ItxKe;C#4pD>y@b_CEt-cH(auMK1)x9iQp*Xy4ZjI*ci{h>bjl=xPUBYv*e zCk)f8dLSyVg3mhkPHd=~8I0^oD9(%fS|s=%OZIuGJR z&pS??%xXkdWz@#m$e{u4=}t+^0D|SdXu&hf!EPt&YSUW?EIec|s(G-$y`vz@{AMvm zEgxLu=~^(1MWg`ZWlk*RXR~S8TS2E^>-D&Ur*oFy1mX)vPQ=CRtycD%AH2LUhLaH- z_2;L{O>9`A$WkQYIELjr1WOhLmAWCyHXJ_((ox4^V4_b|ajhud&+ke?-B$J`Y;Gq| zu7c*cKxH#$e$H*mO~oRG37 z8~J7&7ppHqG*x?8Jrwa4$*%_2*Dict7Jro5jDnGD_^KTuti|3h>0~x4(Z9lxwR2gW zwN;oh8iTotvwnPa=Xwqh?OXV1xO1b-KEPyUN3}q&=`W<>qZEZ4lzHW5aG>kz@-abk zeUrhmOvD)R85&%OV9uyBNCH8yy|*Z4IJ z+QDK#z;ZP!Vo1XY9kUBns0hGL8~>i~YO{^?BJQAz&KI1`lq~n>j4P6)NQQ)^h!_R1 zYdBtJ10GHnnrqZz__5~aqT7L9Zmvw*_+GQat5s^pe)sx@dxyJ1>2=JfY@Vn13=J)h zQXGukQ*aBlO_1I2il$=8X=CQ<6Hx1)A@i#FZ~4J zW5pL)whL89JT7!wU^T@)09^Tw?$D_b&})sG-hk#vIEQ#<9%x~M9v=kI%D2gkD7Kj8 zzLt$UKiq6a7nDXAHxgNDbi+$Lr3)JV5P6kSJ=K*%05ss5De39R$91zhf|7hTrLS?v zk5E>J!5}h~j#(a2n*bgoE}V+xaHKS16*1T=2=PKtYY~}Az!+02N z(JEtEd?kE(Tb!N4hI+MjayZ{<`Omq}L8R2^h@EbzlhVnNUT!b^qsZ@v`>w{P{-y#J zv_MT@@RM@PIXp4%mx6hYVxHQUP{pbfJ*xEmUjdJH96U~x|5RUW z@|MnLJT24+yzFLc^d6W0oM`khxV2{%3|vdtqE{`57_jsWNYpyn%Qw|rdI>< zd^SqYI4M=XRl@(^^t|#J2lHQ~Vl5oKf*e_ln3U&Fur@~(@2dt{d)&jKZiImD`kBB_ zd-CE@1~K(F7UN}Vx*g<*!)@bbxz075j5jTut_PU|8X0yqFH}$eAA_p6?EI9@(HcL* zOtw+rWA@;o{3s}}otl{CS@M)ux5RC3LU4hA5{Kp;AL;HQ`Zhnk(GKc$SxloUQ`ciT}5Ae!fof-g>a4Q9oSCp9(vH;VFs|JGrN z>2wFG1{ETMn+yVyHFZnW0|a1w9l9elu8k5a%FnA)wvYMZEz&BaUk=Ir3sT#2^IXLI zc&nRLRkKv^3HX!u!>-QU{j3E-HgoP_Q8#5*iCX4tQ{84IoqH(>Th)K)vAzufHNr@ZU01E>W4s0XLMy@Y>^p~Uo7G*x?}esjx4P|S&< zoIYJsbKmVAEFDet0)LCrW+u|29ZB>@61B!8oBR8ieobd3AWRfwX>$6(OK3-M{3QR} z^k@|*IOg*%jpXA&H{&%PT;dK@K;w-fO(9EMoTA@)+!ce(XRxGrVW1pI++j@hAqLGi zcTh)^zb>dwX4-~Tx_9xrb;s^LYkIxT`NHJ(v!b{xRAQ^4aCr{3XgDJ(F}5%KOujK7Cgkx|Q1(c5Mv7D7 zy>Cp_ZZ=u{*M0=kH0|Z~$qr+kMbiR`Wk+vJ8&Ouc4-|lQB?_`MY-OUTp1{T%6v3MM zcXWN?EGTXziY@B_HYm@ylN|h->>WH)8#R*cU9?i=`9J&*!%_I>NTh(D)R>PyJJUlaWNmwYBxWx}pyRNB6mX>MFNTT>$szY!& zXA(ukE_x=p=+^=o-5ks-v;*iob;5yXA)v61Ll-+9p*Gq(Sdm`jYB1^;Z=d3+RbrK(`fo%^Z?^gBXVG`(>_Rpb1MnXH?8Rztr-~&U`;YKP zoc--EMhX?{Yy}Jq0V_nEGz!%aRSZm%aI1o!mA$!xyk0po54Dr_X&24BvJRdyviJgg z;~J)46#oP@Ms`L5~p^Sfl>1w zN6r1)+24OZaK5pSH5I*(i9F#Wc{mex=q5nMp#sKEi3&@gBDbs`o|2NLd00P*-ZM&h ze>mnJow(KO4i`2`X=-EJcwUMcHK#9N<4Y*6 zHzz{B)V`m3&NQ^!=CGii{6yVk|byZ2NM*wA^lzU+`coOYnOvmCD=r<;hjnjERgCK zGt4P9Jg4tXSC#Xt8zRjQ>8U-qWL089^e8i|^zhbYBk3P)Lq;OuHSy9jQq-x*Uvl1^%BqWn}WbZz@vwf)?4To5Y85T#&Gv zdPB%@lPTWMBBWchf zaJ0ffoEN`b35`xm018U@W0%SRp;SufWoFU7_>+~d7=XQGx9W4e&`iF^{K|Q9F)q<}jZ>bZv`9;xVQHXosU9cUkbWsCK$#x#a31W<>14x^A zhym`*oC9#?p>-o$Kqmd4rCW!79BM*L7V;9DSFfenLX8E%bG28i$X;?toK6-|8or#d zY1$NHqx{Al2~TB0igH)F1k@_poY11GVquAAjIKk5uGb*q;IfyYAM~MjWJ91mKG3T2 z@Eoca!`9^>W043nLL6pHawc3@#xn`WQ|VJsj&&_v(|xtrC3vggvq9ju{BAFfuO(FG z!Z3MY*YA-9Cf_Tw(T>DW4NlJe*<&idSIBeSbe0pv_Y+iYDg^jVi8ht`kk5T)>q<=v z-GQ-AE?x_4ZXTkfxGOlqgJW&Q+y@_Z2OK_GO*MOeK7D)~ z8XgsJch)ZktEgp&3iq+`PkQ@y3K6$O9CpSQqqO8ch9~0PJC_C9% zo^!e0rg>4+;r6KCa<;vUDHjo0{^k9kY0Cp5i_jyjI>);jw8#-qi397MnH?>3pu9c7 zWs91RRk_+xCFL!7vb|_n*i)XrM@8IGZaGF+sB`L@)jcG8;Io49O_wVWt$iNExqV#3 z@JU|23d1E`QxtTvVYV*qd+B`T+?c{8!uUS2{w=G<8NWj-_A}zV@~=4JFH80C-^Zv3 zsm68ZOG!9~BL5l=v-s@Is9lfgwzglZGN*Q)pJMrmSlrFB5A)&iruSHgPk)v3x5##j zhd3G*vt_4@TS(`2ZbD*lzHy&B;Y73A@if&=2*dUn8{C#2lq+M)-O1ZwsLO-`Dk{XJ zrl5x16YJHs<{$>=dQTC%@bY2E-WN#>$ET`NNE@p0+#Ktm25oy{;p~0;Ng`JJS{Yb4 zVd0NjvS1Z3@p|4hxe8 zy2y~a|5AH)@~^_(3qFnJ+a$U|VbhpH?r)5$7u+%TO=Wua+-h?su7@mjHd0E*S+^8| zl75J|T6xu7`8Fxx!TZ^Csgdh)uZ`p@{Mz%V(WiRp#e(VND^tWivBw$J+t{<{1N{BJ ziTl*9AA{Z+$gfdnv#xu$K55VZY)t}%N6PZpSleME;6Grzt=&|&9?d{xppX9E`ET}L zI2>uK0P0w&H@ib8A>AJjHG&%s87@%x`eHK`PHJuXfEoL;)T;z5IXDqzk=66{sR9XN zvpUq%Yu=b3<4VnqNR`E&g^GjcD)>}xU8O>!*IsJJfSL z(3{WQTXJPkj1cc_cZ;`oE&O@y|`sr{|8`!2z$ zf2r1*G|`eH$WM?!OX&jEf#JB}BAj@K3*j&o^QlCdAzO?~3H5iN4QYr6 z=*`t~eloc>FZwB(XrM%Tx>5{o?(^>Ys*WtDIaF-9mHJ?n{hs+LdtxO~Q%Z)kU``gF z9~2}=mafHF$kA&3OLg~?;+YE$MG2xff$`2gEythH09c-z)hVgl8Yhdmu3S*4CZT)?bQ-f4;o=0PKHOW!!6`k zTFqy%^|`?hb>#rBl^jpsXSVu6bdx=wV!O*fG$twRPr$e>H zD`(kSf?sYri=|rgr%~sFUMY+38&gX8lf572;%}yCzpmkok_xdUSBPmgw8CXn8->6A z3{9lFXk#lH;%Pzlq1%ki!xOQ^G5S_N6khbpC)ukCfy$@2wjX zK&IWAE5KfLJ6Wt|KZo2tisJA}*HrJoGo=8gc|sfane%a`dqW>ZY0YUW1D9_Su_-+0 zjmL*AuHtQF+;K`mI9sYm{A`VlPP6c>XCPQc;S&yc>|Hz!JLHs}Zg)UbLX@k6vfwdN z-Et-z%-W>sMrTVagBN{SWVJqhpV+-jXuto36xlQ z$uE4gxYnS_f26LRd)WghBJxxm2WdQ>Tt2ozr4FZd2HqQF*X#n810E$CzEd{-@=!Bo znR(53mb|&xz*2sBAdUytORDItZ?H5SHU?O;8SoR~I+ttvVsphtEACERRj@sMZ^IuS zU0=zzh8`bPI-fD3@;zYkZ~X$-P}W^j4yvskfC}_nh1Gv>HkF(Udp1FM zq{~x{1}6!)Jbw6-(#|@|qYsF_x+wu+t(`KnJb%SslMu%Qj&k}|rbYgkzuRzDqb|2Q z)4RFocqPrw8q;MgE92kehT?ncxd$YoflExwkT_1!A-)?kbEPGn8?2ud6jw>pBClmi3R*+bRU8(gnm63o=H^787M7LRo|9>l znogoR8Ezp}O7N~coR`RT%t(Bg=`cIXl>^X(3@iGOyXYq!agCXQ;CaJxYDm#md-v;= zL*8e571!U+zV#fLqE@uwJ!q=QB##V`{rFiFQ|G<^>OKK>{oAoqOvN_CI^H*=%Gix~|>nyQ}|zxQVDa%mIW%+XGCV7K7e)*}1(=o6c}{SCKv= zu(+>AKe0a2;;po4rVmCQq?S;|$lh8^cF&kFdrPqMaL*&o&o>7wg7=Gm5Es@b?~`Nt zy&#*I`mkE4Z2Q~Gy16Jtv!{h{E_?6o)uwXQfG2y9UQ)G{tYSiH(FH^X`87=g}|QLr5n)pgBRYG`Rw-}UW(^E!m0?th(}uI~BQ z=%LZcs)-NZjZdy8l(H$QH}5}YV9SA)(50;T3PSmu{;Sa41$>Ee4qxCw=p_-oZ{ywgV3n`pDs2_@X%IBT%v zHV7F|m=uP+o$pQFiA}YWqVPy1K8ksMn&4X&&7DHr=w!5_aSkMZbx$|;=#(Nn$PG7< zSS%-Ry9Y8=30Cz1sQaWlcp2n@G;?x0V|V^v+kHhJH!*-B~@yh=n9k z=r-FVh7-{>N}7mfXX#jk#l9pFuUPu0znT}c1`T4aPp*_N#^dI2_%B|kHknJqCRLgLQ5 z#kMO|`FRR`W8hT=LV6Ep6N+$CZ#_=U59TU)YAg2NKx6wX*cpm=v8k}hO|Dp3|J_bx zuxb0JVg&O+Q$zDic;qm-lK8fNX~muFTI&@Vnw08k5MgGtewaOUnZGq%Y^GmvB>3|I zWE%w5x_m7kf-ju+1y}ep-t1ckUpr;V=M1quD7rndk)nrgj&nV$7Yk>#tJ43#EhHqS zQ{hZP)KXC?F$em*t@ht7oN)=LzxmqXGHFpHCIt|1eeRhK><8)cCuO!Ky8Mt>0@qJuNRSLK54N`wt&K@q=2}y zcUdM|9gu9kM~8S8g3ku}OD*_c^=uUKk>G;r=yRZY3+WZ%qsp4(JKq2Bh)T?fwZJOJ*xVAz)uVias4~s?$@CbJE52JlZ>)#8MV=O^v?1meMTZ=llI1h7{HviruDJAxyYt6CJq9bxhPU%*ceH6fJr#{(eHo!*XRKD7NvgK^uEP>r*@|KGA_9z+t~+_K%$v`~6I(myHv@g6=D4Fu-JP+!}DThlJiq(F%pOrw8|*ndwF{zsWJ z2!F}eqclD7gkgH?Ml+jB*)C~C_)O3?+9@qa!8wOmD*O$V{ftgZGsBNZ9EJgJvJ>%= zj@5ho+ExSI5~cAD|8NhM?_|i~qjmGhVNnu^@PhCexr5Q%QgmMp^$ov5iTo!RfCin) z$rAaIbGrUp)4zv~{38fRHwY(1-q$#J@-G(KVU?Z-r!#>5Pq$ok3f{(_iI4G=DOThk zJ;iXmSh_|79R0AE+|NH#^t)hbcV-0E zlqpmykviB!icieMehP7OeS6k+H9IE8!r@|cyAxQu1{qs&#KIA|m06O%_Xl0)R%Yux zw#8+X9Az0N_0~?jx6;6zjB<=A|KH-?IB3EM^okB&NCl^l>ZQ4p;<(>r7YTKVMVWRy;U9D++6|ufQW44qu0EKV59ZVx>D9P=6~B7?%8!1H?9K}D zuxnG5h&}C8SkLS2&Pq`$+|)8y$brXeglQ*G_lX3n=f#I9Qe!`)wnmsNrw_Fa*BNh; zV^v%I0Oy@Vae~Xoo;=&j22bgo@?PClX1mc=0!AwB+UwL=%3=mm{I&mh6|a zf|AjLYsdDa30xA9s$#Y7#grQm6Gh{H`>iJd9`I=_O+kXSDBb?0;1CobDZk3PAzI@E!DfH~N%L!tH;1 z=x1*9JDh3F8Y$M{nl<+HH{Q=5H))khkbATHw)*1}Qc{_2Xn6(a-Ba`pJCVZ#j24EW z#uWvBDP$T6qVCE=Dk8rB@}?D2p!bC~dvluYAmo03wbtVDrX_y1=h}FB{Zv3BIk)}l zB^27g{GMR;{6ShD|HAG~ME24FXpP);XWdHryz?Sfxy5flO#E)nHSpE~z-c9Dkz5m6 zwhxy_RXU)eTV1&E(&3FhoywF>$0ZC06rSevcgKl$*`?j*{_lrq>pT5#D6+BVvt1k2 zIRMo4%xgyFeaFv|%JARgn%$Cf#8rd$!t3VRM7-ZAy&upUlI+}Rns<6yEx!?|mf_V>#Q*GOvzM%xo~q6M5On798sO&)ONeyP;z zAN24sxNQSn@AD`hz;TohhqE@9Z+(rI;ApNX!=Y!jg!8Q=HrZSX%k7x)CRgX_tQisy z=hfQfff}%rx5LeiF8AA`JA7#5Hd%A>&f98*-P8Hy#p9S81yN$c8tJ2fh?5)GUF5pg z=2I1J5{wynq`r>ny~d7>C1Rc=6i+hq^tfByU<3<#QM!T z&qVojhqv_Rr6yY{9in%{h5~H!MTujTQfm9c5eF(0OKVY+@ov;SYr#(3j-CZ1qHs&P zT1UwI(c--mC7l$%_&lY3rCl|*)^RhAfAH~-F7hy*z8U@@Gl{$1ZJ_uI*Q!q}y^A9jX=NC!S$ z9G2{8WzSZB{mqh4g0^M1HNHth^BL;r_}(0ZQH0Cz`=&gRvj*1EQS9BAf2ltqZi>Ss zBJ5kmtLTKtq73_=Mg?pw4>{mjzfKeqsp6IV|BkVyswOSEB`cJ=76&X|?|W%aZZpUY zf>)oW5_O81#n)hcS+DSQ6o0bT>JV%anjtCa{+M1dJMP-(U3Ms`o4n(Ouhbk+954&B zE*_smBxZ__@P^>@kHnGjVX3EM$!TjCSs)G00^nWw-%}4 ziH)W4-&+NJR}l5Xc(bSq#yhp?(atD^aNfAI18rrHHr7S;#=TnfCM8^zDe)1gP0SO% z=5buy*9JmgtuR8kzIPNH&M+WVF!LZiB-xW(L%JVj zpYMhWtC2NxERviL*Q&AIbzDBi2DU7)t^TUd=KTkt{C^&R;~mm~aQn+`Y}P+#Ay3@Y!?3F30vU( zL`>}I_(AM#m3^kYpCWq!9|?2cboG95ynbzZe{8%Qt~`m}SEw7O;mjj$xFJb?o;`5A zJ1S(bo$_jmyL!7#ryZhF`q}R&l1uiA;6LH_cO3C@BYC>%wqcd=(D#7Rpyyt#iW#{i z=SvtbaTIw_XW%BPM!h|As9@mOT6pzjWWOd~r5JC&L(5y*?BDl??-${P)XRg3JN06s z+X=Eh-AP$qe+G%&uol;uu%e56Nv$pepsx z8Pgg_Hd4KiuDu_ido$fIKJ?0fMjl?AN@?}rbu*lD;{O#lbZo-kd`Pn)h1^xdN}oVe zmeX2a*GfY9b)iIcbWvqde*MWxfnM=ZS~rbH^Ao@S**{LSy|rJyn*>o7@57?d zza{sipo2Zt@_o3%z>~@L{u_}sS6Q!@n3L;nGD=je$6>OcM-)X$C}-M910a_X(T0y3 zdr$`xQHt`));GvtXXbJ2_cZD>4scNy92#b zTG!7?jC=8R;`^!Q8%f9dk+7pBp}ne)uSupG`UM>&|3Nn!C?{>Tr)Hag`)%xtXC0NY zq6*iQVwK4%&e*FlMv$})q}XrnW0kN#W+Pg%a5o59SsmaMtC!t-yT4qaDUl8VBV;e1 zE-K;y$vuU)mk$fEH%PH;G^|_tUK@DF8}b15r~e3d`6iX$IFlq|6W{~Ve|eZDf7pwL z_p)w6d^KOL-l}B-4^`cR2ho`|$cofjjqwU(R{ubXRMJv8!Lm8aJVVYp69?+v4g@o* zbYuaIv^51j+_cL9M}YPSyUtT#sW9R%YxPu-G%tj`nLfC)jla9Fbr7{zSj0S|5jo#Y zS*)fZcsPFwjiWJZ!H7Sct((|wQg?o(&E_EXJY~I!q#dBI4;E*N17sdBz$cIZ_zNxNyt*-aJ;fm-a~R# zG^oF7+~S{C5dPk*{HcJM8nJLp4~=1@f;9b|d4~9Z5wktbc`2^bCoI?WmT`n)s)pw| zf?uQsiU{=kv-rG#7w;nioBB3eJ8YSQL;A9yb<~ZnsyC-9n}Lt`6>i+}XWb}w@;k%V zyY(Rw(1a0Rz+>k*kmEc4+j3g9{j>hAqDI3D7R|dM>ao*FsLUna!>PY+ReKSU&Z8su z0MGNMOwWA|&#NQbn{Eit=MBfxLIzmgfO*qDqX~cX0c~NE8?KOzPM7_$Hr{WWlXs-Hpy4Be5_s za0@L-xtlhr?A<+jmdkf%LpmoC>N!yxU&x z*ri;{FlAm?uxA%P?pw=vJpL=H0wlvfD#C9`VYY)KCy}>3;N@Q0sdDvra^14&J)*M% z1I-RXHOfg~+s1l1r|}1r+1JW2n~1L8u9i!;O~Q3TH-p_rs;!Zc_o;K=RPLu-t^PY| ztyedQRD`zH@4lqCiA`bNxi+aZ`C!CkurtZYh%21U6VYd@G}W4$sNB5rwS_J%lk~q% z)poDj*rMU(4St~gCw!%BdH`sWH?ZTrEyReD-!qAefq@e{&G}S7R2X;7$q;T1dA1w$ zBZWN|-kM;F_!1QOVBv+4yS=)Ty1%T}WDJL560_E30$)%7mve&bvJ&q1=Y7GkUvb!~o;-*HjO6e5MHw@<|otUEXrqe3nA6v%|uhTy#aOtirK9>dRsGak7|;`e-lu9I&Gcc{; z5Q9u8%~bqe!_Atv=4P&7Dc~kx0qG+Wk13Ol{Y{z#+1~a10no|W2eZ+xi`=Wp2X(GM zuEnUu;+tEkDk7J+N}+YRmlr95mn$jFx(dwDDSk|>3EaU+I|tDR54T9fF0)wOz_+L8 zV|nHkz$8*pgpna9D8#~?B>igiwtIs*Ff}}Hhn1qTLxL+|W;XQ*^7FQHXkiQdLsL*F zYDuZ-7UIJ-#r8up@npQ++xm$orIs{6?{`^fpp*6&!X7whw<#P3LS4K;X*GF_m5Jw> zx8jGjhMWIV4WI=mU{;xASDK)ijVA_enD-4qEw2;|!aK)sIz3T46{1z^kZI$v$_I$L z?YYA*RPZXEywRPVsNrN#r!X+!kVv6tUTwKLOAhBB#Jzw{x<7PHR3DNG6-TQL!j4f~ zLnnKD)-iP?k6VqyxpKOj=Eon6F`l~dp|_(d2A4I;YBiQHZ(4OQ{UBxeQe*67N+Rhx zhgNi404fQ5M{1c#b@#cgpmJ!H&$tGSdXK)Ue)o&fk`275ry!)@c)V(of1d!H-zn3x8Ipp$4lx`xQ=y z*n`xs?|P1BVee5&M#JD}PgCp^J0Q`BFh0d~{2uRq+=Fs*T6qcrUA%57@ngDR%0Fs9|RQQUFk70}I>XLnL^@%{IlPMaCPwfUZRwg|a=OSt)OF8fE zQ$w(*Yj(B59>;-SpqlQ;ELgy)3Y;IA%v47zs-+9J=n^UfN2S>Npr;gdCzfCf5DrL2 z7nZ3-_QA1b_2Jd_O!8;A)GyYjEzRU*HW^8Q&kA64?ILOKRl z{$Vc8%=`%ky-HI=5rG#nK3sv8cb}h4)RSobz&+wm1+diEx$ZV$zFF&>b`P zAXw`ltr1LwI$~$_mo{+;iiCaox;Atx#OEG*+WnkT37r035Rw(DNqPF$oC;^o$HjBG$|U|s){yM8UOCBzbmLT_m?{ZhyQ;a zfe&HgCfC|MQ^%9-ciyZQoC42Vj<@t{%ntkoI1r6xph%8+L@HQ%+0Sh?vN!+mS0+o> zAILJbPRwC5=f1^K0Y7yL{QAS-1_3zmNjPP4&75EGQ~Na80HA!eUerIC=ixC40NzW7 zR4Sc?seAcTA*$W?@Z62Igf;2O4Kup`9@@Dyo)!e?e89N&Ne(zEg;Kew2*sFIz!W4Y z&9SqDep&NvdRlI*a?NBrebHsO9z|t{x?*tWrwvdo1+MIqJXoB0|6Wo zg&M5MHsQJ^G~p<6hZ@5ZSF3lXR!A>VvF1rzj_6z`KEOb}|eh3}GALudId>np@x1K9|s=|iLN7;g!>w0`r? zc1P_j@)fw9Y^#pFxgi)!XGO{qn*!zXv^8V@HzC!cqhM`qSlkZv*Oe^$HAEAky1+$* z;MyQdMHgT@h;VLiFAv*K671fYfy|k)&CwUUZ35%`k^Vj0ta4@(vz~t@Yy#uq-^Bmp zgH0U7>A`N^u2a~4m6l@Dn@G~Ul%)_{1_>8xNypyKWf;CYI9Vo?+zDTw`#N1GidG^D z_D9oDjRs8gKIT$(PC_;slV3x!sNk`CL2;amiJf&Net)uny3$4nJ#t~IPs{YamREwt zQ<`I_79=h*ybH*83T&7yp>cmIZyWrf zUXr%G^qSwidva_vQ!SB9mU=c>3~Q$T5F)@P)O{o&_Sv^8W2_sP;&RUwO@>eWJD&V> ztwk8thNJ8AhS8$(vsLhXEDt%3Ck zgXvv~;Xd*+VEJlscJFGt#HF@l83a5(jClA3=t>M5zgF#9eixT&sfAe+w*X(o(Qn0N zbT{2j_?l|75eBUpCbK!(#4!@3pi`RgX&A zr7Ed3?RGK9pM56cD8vVa+=-QWEZP0b5kuDEHS{OW)t9?_cfRFfj#n4Ej71jon2iUP zc^w3YC(hmw)s23ahRFxu1EdF5-nkbQZ$H>|a8J9U812 z_&f9AARqmJ*v|eH-%UjB(=tU+u5K^Wtu2OsmOSZ{?hn+%buJb3*poge;%>=m>XRqT zS+plmdrQih8d8S$I#?g6cM6#0A+$5hr&dTX{^2_~wJBPuIRQ6>f(>tf2OolDU_wwi z%zGuDCTd3VkE@Re-rBYO@LB~>y z68DKWdGm(o;M-PQVM-SXEW5{An5G1OxB&axCWXJk1vbJ$ErC|22YmQW^`NfegGh>q zN(wh_Pw5G>{hmU}uMTFf-G-8~AIcsSYWAxrcP7q@E$$SZ?=B=sbRhT}jZ;a78N232 zPrTVMCbEl3o>vZg>g$s=KTlrGK;bR@Z6SxbWlNPNxC+mtt(#KcZ&QY}%=SHS88Eg% zHdV*b9CMp?`mS%9Ar$wuqJ=p{3Cm?8Hr`*cBe)@QwSsq*QDr zo;g78XwmklC`ohG5AeA6XGB9q!%)HGN>OVm8$M{;ctmHHan&o3Xnq(lH>Fir41+z3Smhu;2KL-sL%Vb3!qjk{_polyapdK{qjjUtN(f zC7dQr9vLaoNlD9zOCa@sS2aRH5P|3?z%~w=sqEHH2}3VUb%0VAt&|cR@eX4v4g5tq zA}iZ-70HKzq5JI;R>;8}!Fm8n)TM<>byrec*qrBSSi~Tjps2<{Ee0;QkQknu!*{j} z73@dqLcr3xhq44|G)6iK9wCgR50(mb`^=Z1oJj|@h^U~hEFu~u0|clqnWZWB_YuvY zK)%SQEdqoyEabo$?dVlmmy1Rc+Q4HL$&kgzFF{X7TU6uX)4%Ax=h#PYEqs9w8Nnkz}~)8jKSF$+>dK6q({A2}zwbSXF0&{$x^4K7xA z8FPXrnLRl8wJI{%lC^|dqL$d>RqCg8+rK8#df;Lte|4^pm@o)|r-RTV6TE3Oj5UkD zj4G|eWlWB=I_tFpLh?NZPuaMMV_y7Cb43ogEY52cG^Z(NMM^WF;|IY7Ubb;bR|5n96>T*Cu z`8OW_iWL#-LHu76|BEMVu>YTtRQk}c<2SGV+_JKL1P^{^4+4w}mwyBCFMJpEYTwgs4V%Zr{9xC?}Q7CTP=6L&8bpJ2A5k~WqX7s%d z>i>n>|JkHZPU2QIcOHLlu4GEhNT~G6frzueRplh!nDCz!d?-xq5<%I+jW6yxo;@2b zpW%3*G3~d8&kV^AA1PEsqQ2)9-QVPL2!bg7L+FoPNE5WX1Pa8(pK6xCIOl>q#AgQ* zk##9q4*i6DoqVz`{-subT_m%DX5fB?zz>bwokn5<&V4bCJ4pP69Q+>I{-Klq5E^JU z0{V^klxM6w06NHR9AyMQMY?eNPgD~?+5o=vGNZPu<#~@SLBw=i8mfDr|H41wpG71^5;id$n?Lq`21giEtUgc4U&nFrV+3zO(eUfhrs{J3TtluGV>Q1ZhrkX zVB>84o0slo{->mRS{UhTj^d~27LavQmX}`K@N^u+>>t~g+SYUOxK^Sn z_`3HhNitDk5T;0m9bh}bWVpUbpEg0Li3odX#Ij+V7G@t@R(euVjI3}7E{IXE=IZKOfEht9B z^iF28rp-}LY-k7V@ioI`EY^>Qi__{$jcXd^2-)gxFzSSyb3W(G1`v*uQr_GD5GAq@ zsJWR6nXq@3`*wZxiIX3zrb}XT&jIU z=4~{mIrPaVvTNx@=8SL4Z6X9S7;EW4NICkU4d~3OO8sfUrGGFXLH5(NHJ7HW(IjK6 z=Zm)*hQ%_orwp?OfdiO;U+J#4h&0zn{=iX00{w#n1j_haWD1!TF-M%?Y(hd`igEac zqJ6_+kwTkrPTQYfsNTtRD^l)?;h&)>R>k&qoYr027uoZe^#Zucf#iX+Fe`19yP^S35dEq>Z7 zmy(6yNZeG8KROp88fkDQ%}86qaCtguOf~G}c8*4axOf~FZoNg+{g|`1b|N835|7Gk zrt&kTRco?sEj3b9 zd1TG8jOMW*Ir-0G#;tNza1d^h1$Qd*DRUzc_k(k~zgJ z*El7jABkgBrvq^?)vUgWOfX1%K|f1*NJjL$FRwYFmMcK=SHC-)S&P(aBAD@U1Zrho z&us*ul;I_!$!{n{rnafE+1X1h3_xkImYidO1U)UlzKF-Yfi9#>w?$SR-2UgI_03=0 z)xUpD8%t|-ZEcMpK-BLGNzf<933rg;ogA=F>)!6d#|>0Iu|}7gb$lGBS-tM5uU{#l zl|fv!Y{B`-c6GO8VHW%xPC6R%#tXm~(!1rrg^%>Fzb8dW>(^wHAV`nOvrn==h>xPM zk_pRyrFjQ(M9GQ_5T9qVV+eaIsLoDCZ4$`B+{#J$Z%}m&rmU`mGipxVgf~ztY{~3b74qtI_ zX$7}niv24tzd+FO<>@b|hjXjkBp9WlL)2@%C{etb+JgirqBuyYJ8)y5OS6kIz-Hq2 z_NWeP?g^Z;H!U7D1`ofTY8o+(t`poGtGu5c2>`M%StWIe#=UV3X=PwAa-{Uz-Yje= z+|;I3ETMH-TSJsz!3580TTUNfMduJBhew$QYt2FtO351&E8q(o_AK~1^wA|r+;YI& zpb}}p;34XlJW001H4(`(AHA^j31_K%2v?*81@cW3@I7&}VXl9H_%BG`(_`h+IlPOm zEkbKQM#D!zgP-)0<-GBNJbMu-uow)QB&65F)$&E$V1R2%#AxQuPtNFI2m z_l!wI`0;u)b1LPi;z7#%l)55U3yYV$b13;9-N}zpqP&EUIXX-nF}rPE;}-z022+yN0X92@yTYk%G|$hT!-&+K{SU2n}#x^a9CFR z&WSps0+K5JHWXoI`(2*hZcUvm@|dnEp?f)aN;Y^a8cw9v%JW8a;idRKX>!y}vc4AF|uWqLsK@D?>asIwL4R3&gV;;w#ay z(LRl6DV%wYIqgF!a-8U#J3j0q7N;nVY{!>HMb}>@LT->WNAzazhG^(PvIZ0~3CogTZv^P{rkg5>#-T|{?vZEQyjRBLIF;%>`9hrC zh7+C`yV;A*o)>qkRBUy5Sw%Q5zDu=ff`n`jYx@=a9AB=dC;zC~RTMy3?JQ;fGP+yW zr<(^hEpT`#&fxFTO2~LY?RZv)=0<3ZN-LEUJxRi%j7)!*Vt~xmI;517g8)9NXctY| znNS)9ApqFR$qUmx)WV7b2Kvk^FV!0+eOEL&H}h}oz&M;A2cM%!&sP(sRuN%atQSBG zu`A6r*6rWb(0GQSA--*ZYO1Do+$lqQAcUWsa#_0Pqj}zSUFRH@VuYDok=Z(6@Mlm9 z{Jt{daPNhC&NoSXh>jWhSg!r!-#h&Opfm0Z&PSY$H%p@u27$D`U$P>M@ z812QiXzP)leI;DrBBzc}N)bIamWue7fFmq8-P*3@^enDUjlc1E_sNR2mdR9R+N#NA2xKD^+1G2yZA>Aj!aZu0RCMkJ^yzMMA(H( zvlkVC=Y7G}l7+CI^2ERT=|2GheZTbcaxD@6s*JnpQv$Ro{L~Ts-Tq^Jb~OUwI2zUt zR&ut@9oi!%hbR~k@na)ea@DcX7KP}qI43D~KN^C|GSle&@0f>^Yq){D+Bvc4D3?+Z zUq|B2dT|AJ0EH^>LR%z(^#*_y|g=w8TAw&j&(w)T$)G%>^mv?@ulP^2DIk3OIIO|u-r!m^ZJQuT-fMHlxgkqFM) zm;4iJy{;Hdhe}Lq%dK%bwwfnICH8~Qp~1`DVqj& zCvoTw=t6qR5cL7V>r*J9a3sYl8prcsB*kp2OA4vLNo-ub)}nznMsP1{?Hmne?mz4v zUPx1d0fE199K! zZTg15RbePjO>SqRId{yW!M0Btn(XImQ^a&oO0wja^ya}T8a(*R#eJzZxItdCTV(p6 z?=?t{9@J&?tRXR<)fd^W6)*XBx&MjvKirBFy7SN)4I`l{bn@B^4~!?9+X*DeRH5~^7l+4~KrZl} zCE?SHxs35kO~xmu)=x{s?=L|>O6ZCpiY25b4nIy#K;XiP?Hv_|iV08yedQNWeq*zy z3Cu)@DwDYaOg~~K<;R9VSU>aAKuJZ;kVTI#(3+O451`G#jS!DwMOs|?{WlvYPo+My z3-n^-=)%g{vNh@!TLmsKhy&izLSjcIN+et!r~*4gUy#rR@sU{%kVZ$Xf!w13{yh}! zCEAK4p{aYQ5i9n~gQ}6gkE;N$u>3-tt_(?j^{Ipq)G02@Vea&&{e2ZjPyBhDQ@&C) zWck)lsQ|gx@x}zyruAJ$PJ}n7qOr1;`A;~Bs zg}sEjTZ$K;X+nHh_^KYoxygja-l(Z_bR1fm@>(R~@H=hN+@rWe>e266jRaWPUONq@ z0yTy(3B?wfL;K#&hYGq=xBFj81WWfq9t}u3GGvW0S52$mTpnE6ef~D=Wq^d-h1O9d z-zn&tvuZhc;T4K~^P&(kpSAK$1w_AbL!aVqSengz?D3#Hc&nUj z=o@{TbMmpk0p0(o@A-WFA6krbAV@$MYeD%AspX&*U(jD}fg9vIFM3QY-V?V^bw-Zf z6t?xc(f!|^%LEHHHKx$1DX?~#2Ea^_2->wamTxeS`aRsnE$IR1p~gI zgKsa15uUZ6&!I0vCySl?XerX}Oz(nVSg{BHAA9!$iwuzIyT#{gbCz!4>BIvC2-NOCEqwQVa0m&(4-T1-TK8kHhTa3ceDf}1g2q29MF#o1f&C{Z_~Tviprz#2w7lt0)K)^RX=jM`d_zw$ z1Ah=|8d3Txtry>~UW4KO)6RTS41wN_I=l;LCgOuXh2v{|A&Lp&$*2zho2Vf9*9qud z`M`_dE=XoB&>io8lKOxH#0Q%39J0t+ILVzk>Qj_p%KDpJTp9nvC+4;79WQ(zE6aQ0 zu<3S}%6*dN|IH)*rDN|B@04T{bYHS-2@v52WWS;f?Kv@{Oaxy24cgu(%}YUSZ;mzW ztnbRb-+J7!6820)xvZ^EhPGh_) z5Hw70!vcQe)#d(omS5W*v%{p)tD-F!oyN3C{%a+Hpj1CT_|9~wy(d@p|00RowP2ft ztc1gV!_F7_e*pfQ5r!0Ua}iMeclG=bd7~R-EA$tsalsf4cGL?&^1zM#{r}US@eKeE zIoOt=N-hXAxwH%R^DnVLOcTg--N6RQ9WDW5BaPH3 z?o+dL1}|{>vADP>L^4AxKmut^Pr%9=!k#!c!q*gW_$w_O*~Iw~$9iOS{+q`Zb(z#MD)u{_PI46o~a?`zLDSi$?+Z zSK@v(K6137U6!FeEHxtjIP~e9+)gIe`T~x+fF(gpM6?p>nOLE{dF7t3vxHdl95wZS zVyVH(eybG}m6jL;lcAL+JS%~D+Q>?=R?FYXs*2>W=%kH0L6oo^eHE<<$|S|sHoJ~j zIHO<5dm*j5!9kW8%9)bCgN7EhOGRQTUKiTuEbp z=)b*FnrwH$&b!HU;A_3+i*GFPJ@Wb)A1q3;Yz7G|W3}N4TR}tw7e1g48hX z?l+EFfaTy&!#q_g2VH=7E7wT=iG}+J;)g8PFM+wznf|MsVYR$qM-v-+WYu4ox3=Tb z!f7(H)FldRl8ehr({~76tPmr#%!wmrEE=hHtW5mouTDrF@R+JcLVT0JUR^xU+CT(%UN#1s$@JIq{1)RC}xd2Hk?#Lfina? z$IG3%Z>sF~m8qC}W1dF0D-JzY#wuluG;w&ih@_2J)y*h0`&1a8b_*m}AauCBlC0lV<#jyDjYcAew!Y8 zCUsvulJm*T9By$bZY@c_$WpH;l!sPTIsCp3IGwVH$H4`)L0^-(CBDPdKZ`#{B>!d0K0;I?EOWP$!f>J=Yjdq#zpc*L_==vbrj{6 ztx+nZ+|oAeIBMyF%?D%Nbj9o{f-A2|sZf-*;ZLvTOS&||;M57@mVT80&5QYvFm9G% zN8p1wV6s5NM=Q=d{np|Q8-%MnE}5&nqH|d{V)h4?r(g!ZI+41sHXV~dC2rcPo%)+N z$5L0a>0@?V#Fqm!ORXj7{enSNshDCLgvL+hRZ`IOfSoJkkw$1m|e6P{>aNzVY0kYU{{=m7(l9U@kW+W3wqZwFBJZJHUfO(LZ z00?Pl-S`ShxjLRsTdQ~d3kW4;Tv6()XVjZD83zcM9m zyd;XJ9dVYX=kxTR)m>e7i3)8#0Rmq0PrZd3CD7EBjPf6Ev>@l$*~#aapjlFLM@LEs zPHnYQ-)~K+hO6Yw9jUJP>eP?Ymd2!2AqbXQY%5qr6PDLjr*(e6v7RBPwaug;_Kk@0 zbPvpD4OCols#>%aXL=#fAb+eo^8S_>!I(P}>nrgrB@auE^Zk;buJ=|8ci1dVC%wfw{YjC96)R>G z_u!FX+rYTDg^JBq6%Sxr-~UBE)XNzbo3vso&&s8l`aQVunRSRkURqi-&G8jyJNau& z>K)lyv6=`ab08`9i)>`6r72qyA_xAB(M*F1F0HA(1*1U3L%~UDHI?rUNpOx=u(r2b z&NWfQxU2VA8KVtX}ghRgGXoD@w6_d%~o8WJ{i)fJ7n%3b&FHPd)_OzDqf;;aG@LVDi-6un0mbYvqOPE3@gdI#d+OU(a zS%~E4-%~>h?MJ^YyGa5sQOmBsa5kW_(#!#|Z zRH-;m+Hen`4&!=zajNLLjGHbL-I@c#b3+cuzS2|$GE@!*p<730@<6cb$$vgAf(b=# ziHJ&n*mM%2bN0aX64;vpg&0^WuLnjF%H$#rth)9(s5k_o6MO>}7i;&nEHN%}w#TJJ z_?zOVbAZ`ok%x{%sZsAbA(9&S`q>N~J?6{uBt@@8F~MUBbT9_&HSOnz=^N}lv#Pm2 zJYpf?3r3|g8t2Sgk)&w%l)FRzwpoiqXauiGmNeR_30u5nA#{C80h3ioD+EVkFeq7U z=5A%L1V@&Q6Cpv6m827wd&^B(YGA8NK70mNr7%SH+)1-XNy;e!Om+$i-Brr4lMu_& z&h!Wv#&N>uG(OjY4$^8KyCxZZ{SWYha~~R+np?QiAutOj$$k1Ptkf&`J5gL%+8=)_ z?tZoFzsDnyl9~hlnI`m^!!?qsPNK?;8GDw2(BliZqkKgzOvtxB~xu;3!_LaqCCE938tPj(dH`yU4@-!skn>gI&u?_;zv7vkaT^ z&_cl>XN}xsOQYgqf^$5+U^Lp|NM*)%pB>Ke25Wxzis&Xi{AHFF=b;sX&$n1OeF?9o zEI-CQfy5I1YJ0#8pU)FLM$|ZmBp;Pyykx$PNW!%gj)@3A2YG)1Qj7;gTYeqd6t+*#RJ=U`}gtBJ&({ z1up036Z?VgvEFDI+mx(A7O*g*Tq!#)ct)&+-av17Cfg$uu9sy|8I2r+a{oj(>Btlg zbonS@$M14fReB#@=;Z8eZ!b6;psDFHcb;B@`Qd6OA&vV=_)#Z*b~Hs7KcKC?k}C!- z07K(`UWw~D=jXxsp-LM{MQ5(P8nQR7U+{S#S;%F%Jq*D=7HBx~|J!a=? zugJ>ei4->|8a+KeoErs5k$ti6H0C)XI=M+&ybV}(^7_H~*~Tb0{`=iMCbi$h&i-7N zqHXLvAF*b`hl`Tld+{-JEucZA>Bs5xB$QdkAt)WXm#+n^BwqN7| zPJJY8T6Pmp_5ggl(L_9#+pc+7uzK8ihSq9RHH^O@Zuq*Buv_KO<77EkM`*2i2>_<> zT%Bt~u3f_NV=D@oB-^j>I9eR!t*wPOv(%IS>CE?XRBSTozEyf{^~JI_*PE0YiJ1A) zcudL@lNNJd_;DIi7xY{RZ%V@*hoX&x^q%D)ZUY z;~`zG#%pF=1zZ4)uaSq6;9>xhWP#S54H1R|8#4kru*T|&boJ^eVoq>=C?NubH|_BA zjz&*iy>BrauXo9vO)VdiW0Lid$A^_v1(CB) zy6u4g;vHd|GxdG8xt);d_$z$~btK%n>cFI}T0Pf`kYXU^KOcb|kF=*iX- zEX%j+p14hX3Y*aXz0K0a?lC1f>!5RZoi>8c#4`VAaZ7X;vW(hi@uGkJxh$ zjwT9(^Le!GUDk}5cRA-IrE!m@>^?K}o}5maU%w{?@nEYm2Kv-YO*2(Qk1n|y<;gE| zImbtu(@~(g!}b#gmmDz3NBsJb>{{1iqT9YW@Y|N1;u(p5OCkAbrk-+{ovO3FHaz6% zZfrqke;fdsxO@fM1oy>Tb4~r~*JDndurOy^^^t0p@qFK%P`Xvh{Orxfav*PLa`nbsSlK*+nED{xeh z>NMMweOFN(w}Q4P7{=gbJe@PS2}H|&iWEurWtzOyKAjSHDPR##ea2cBpR6)Lb}rLJ zng7WaO~ZqsshB;`NsD|%9p>d5hB1%iQ!@RVnTNf~-RipHMn3p;51X z$&6W>>uVwIXlT%>4Uh>(I$SS_VjaioN{yzD3HSA)c8))VQ~fO57ADkU!Bj?9Rn`cXr@pg%og3)r>HOa2*e3`s(SXNvxx_p!?n?S47nOGY<}a$}+{W_>{#2XA6w$m39jzW}kKKbUTcIDPbIkqZw!K&|(~@O9J^ zbb~o}F(k6#eRvn+&Fx+=;ammVM~U!U?&lm??Gc5JKx0IPlDS4fmvSAlSJ0*M%)!Lu zk#m4VKI-($O#j3~E{1{LBDq{jr*JcB8kME9hj4i$>i8S8x+TxIXa{y+EX{~-@npvu zvC;4nr%1R03Y&B;u(MwfHtK~AGph`OBkJedya9<>Qd8ViQi zCG=r7jSDk?;kiqnEE^=v5z z9KOchWoA7GyoYs>AbjUm<-$&^JwL;F;p)@dERVhs`J#%U8#c>p^+`}rKBTxk7qjSmW|8E z(9>O1FA)SaUg>fqTmYg3V^D+bJF`bybhfrLdPH6dn;^{?xgO101(G76BKo+8{JfdBpBc%H7zfIC6f9`5* z$v3u^ZX3Y(MrJcqG+)Yus{fB)0#GtY8Pt=>1Dk&z5@z)Z(junoAe3l{<8XHte%y)4 z1lr_(2{u?ImV18-kNx(T27d$$qwWm~d)bJ6r>u41Om4@f;94z;6n(5VTWn> zkA5biU_0w-ahS9+d2h&0Q$Yz3q*Kghx=3fRvRqHm!~-OYIbrpAZ!wYtpXV2+;SNd+ z;u$4yyYQxb(eB4G82&7}sKUr<%z$Oa;+`wISW^Dha*vTglEBx zmrB0RN&`>gZGsy02VrdNy*(9PIhcXb%w2Fd$X#eE{hj7NYIUR@F=#SY+r2`KaiURzI}%WxOx}CTLiNDD8u>}gHY0((`fKLtXSftlblT>n9#}HC){)2 z!FOIWU)55&EQ=jzsZu)&;1v}rG562MYH~VZ+72er=D(a}bp;&vVCByBeFwHRN21E~ zLBZis#$b6c^3e9B;Jm>y418IRf7cO#sZ$8^0Fc02)(*XalZ<?BN;DU5kn!szN7je{r9VZ z8)slSyl&7hL3?Zw@s?u8q37z3T+nUNTa$6zq~2+lb1f(VSrZn~cadHUcVbjnCbq`w zRS{S+)&%H_6e;RJpO+_~k6dw5lzNnf1o0}tze_H_niLVD2z(jMcsvSVYFtCuvgWJj zN4ROHxXO1hpK;FBKFA)DEOyOvPMZ`J2~p^RWlTs=;A7>51-^%=1}*Ut!G^IJh59sW zkb8N!h^O$?cNv_|QW@V&-Gc(pH)u~i6)Hz(LirK2~Xyx(Rp*)Q7~4a`TFa>k#2@ijV( z0|ID07ht;n=+Wc*pd{LOCtRGvL>!EvYh=Tsp(QwX8jy(zEYmvMvkGZB(8Qkh*isgD zp=-nt5<6$LiZ(wm&pGga55GJY&B#F$wbt1hJ8}_8&N2yd;p`1^KE_r5gJ(2XfGTnT zs|(Pi9`Gp359)=NfRFN!N1e|mBchR~A6QAq-hkot&1R6(@iln-hM*?~osH&m-l*g* z6s;6Z)5ni85Uo#j>hrqSHYi?oxC5v!r-1wJDUYU+~LQ;=58rd>5}Evrf1?8Srezy znIbq;Fk}Ebs|5VnvphDE;Dr9W)83nO?#^Z+Fmqzy{ZWwOeN5a)4fHNzUpHI@F#u!D zvCl@c)dBi=v8QkYs|K?nolL^SwpN#=MSQX)fS3C3A~KC|l!gX$v*}K1QZk%}tDMl- zu$jI~RPB*t;z(KcsK;z6)T?T#??qVAF+^YgG>W(xnS^_9SwO;72K2%Wd7f1&VAa@Ix&9wwkfP?R^5x)wB`S z>7B==h@6r7EYkc8)_XrwH;3AFpx45ZKh_M$Ci7hN<5#f*00lVQ9UqX?2r-q|yO3Ur zCq(5&`?V%(bDI18XMw1NwCVxR16Uc+j1jI;<@rV(xKt-L(5$5pXkj=R@ECsA?^b+? zNs@tZRjD!NCE(E}tIwL2Jmv^Gn2+%;N)>!`f*o-7gll3!qKIsaR*HiKxA921F;$xl zc|6NWE^Jw`n2paF{8{;t9%1!vzb#i=Nbw`Pm~U(CYmE98ENu{hm*$YMgT_Xr1P@5x zds6$XwU|2S+ZQX9N+H`v5=1%=J-MEU5!G>%0`zZXPX#lB2Li3TGpoi%tR8C!XUVP; z1??cod-_KqlV-DVx@>@U#`{3k%R_!BK{c16_acvaRp)GnYb7cf=Mn&rn=tJ4hnjt~ z(d@RTPA~B>610gSGe_ z;kj1a{Pzr9uwCO(n-*DqZ5HRG^uA`ZjCT!+tnR)qC{?g?Q(fZ=b1kWdXjhNd{9dl~ zf##U)b(yrpw*%CCY~_CA>|VslmRIk4m=zn%*n{I6eS6T1sN`JT6EoVYb);v#KkygZ zh}vQd`<`r&Q3{;Iwq6;OOp#FO8}7c_F0~Scj2rYfSY+p3ItyQyOJ0Jgh`V9}jnr!? zFeey@fUReg4#&s*?b0=3zdkHNK9WL-?kF4gG~>^LnvHq4w8sftRGG3l!Z?bxm0~|5 z+oEtrKTp~Mzp1%!+$Wx6YS4Lw2hT|4mBxJRO!=XkP@;`*&sou5ZcML4bx%M#L#I?J zp{!MZ=Ohir^_WY%oH)I`xW6M+HH(xv(qg)1zM1`%|B>R}r^e4@NL8wgQ`oWB*qDr` zy~CC8z`jE`&y#t!Qk@Wxi8xAnH~g#F*%X=MLNMe*nkDs(a`IE~gsJt|{M)AJ416y!~R}XN#)s z>s}NoeeYizOYYeE=pk(j=cHoP;aq* zH&6D=S9Qb#FAo-KtwzD{x!n*SfN`6Lx)^o?kg?RcZVsoyEZ5kssM)|LxS1%!JK4Uy zs7Dt=e5eV+KwJgw1GrQm<{a)Ewx)M2z4{_L(>&qRR`M&t%Pak4E&$#x<1upjZlrrc z8Vt`H!5}z9_}BmvGI62ivfIKLX9^SY+Ut-8%nfNBnr2qwjPyP>J%RQ_1{+ajTg`#m z0N1SjEb_>Y@neE78%qiGUL?2Y)$*km?laB*Rc4A`ppLKNgOwt#TJ zQ??3af_0>Qio0)S?1ye)6tFR`Z@JeMVLX7k5$Xmw1|lECmsWM9xs{yw`V2<%9Pz`Q zVi@JNT%R%Unw+1%^Nsxd&Y_YMC=Fi z^#VoQn|ukzhtoHcH6S~E%@R=ccU4b)>4isi%(l8IPJ@(eU%fF27diSx%0gs|;#5g5 z*CA~eJzxeI*ZX#)*!8^Ofn1ovQ8qz`qLE%jO6tiMA7j@Gp$j1BdGdk~tx^PX3< zO(TyG*34zN=Km}KX3cw<+jOZN_u6NHNa8jL;@d6y&BO^p5z`F8EmyiF0kyW%xRyGR z{XsEsz_rngHa;6S;qrbGFrXaS)SZNH92mTs`CM(X!cV?b?Y~&_`e|MFDJLZa z;Q{HK@`(Z7UN_E$*^Gsda3uusrGzi}e9I~ULmbW>_{=D1la1*j_i0!j7uTgSz{@Fh zF~kY*Sw@2f`Dx{(8RI+@YtHL(e^-GoelO&HiiA$!KHyf|$elM$I4`2N;f=TUyjwe? z;~C{b4T&mC&~UyIVLQ;lVZ!H>OYVMgZ1C}GgDA;{m&6-lfscFW_Z8QO`{6rmqTEXg zebeOHvtMY8c3Iue-z6Q6r{(p$(Sq3EH6L5!VGNhv<=ej(%M7aPvQSV!PfIyhNXgk}XfQ8_gGaE{EnsF)~Dym#x7(K@;QW346P+-K-?-=(1u&VomixkMIIS^dur&{+bHg69BRE>@10lRzj!PX=r^h6$v>Zctkb$V9q5@cwO>aLPT1!aq@?XtB7b=ppKF7e8~OEo zOf$wP>2SSG&leWg)0V?|I?RmyLuGJlA8-?T`#mz+Ouw(SOl5uR`3NTATnAggv^umg z2tXgMMh7j~x<5zEZ9Pi^f#=C97H9XQRlrZvwpSEyY1i(B9cEyrRF6?_NFAjHg_jI& zskR1Htf?2|9Z^JOE^O9;`Ki=UTaEEh=m{p&#~Xgd?Y!4tb;04+ew#ahQ6_FYRw~Z% z;tzAh`cJ}Y5do6dJJr>nFF4}>*&g{{ezYs&BqI%<+Ee8E+k4*eglih7ch*Zg7M(Zo zqaun!nSY3bT_64F2Zw)ry(g(!e}*{7Bdi&zfLJ2awA*5i@)cx^6PBm5BjgRE;ze%f zow%K|$|z)^O!ds;S}F>=c&jxzAw1=E!C5dE=q?j-bYC$XEJ%w7#)dNBX8UYnsT zD4pX#4HtrZ;Ks>M+a)oI!bZxv6&08#yO zUL57p$}R88_T)Ibpf!hG=!nnhmsm6}Z0vme$(%G23F40&#sa?Y{$m{*?bs zXBE?z6Vi-Nnq$6JcKLgJ9;=qIc1Mlv__iVhoTA)^RSKys0uisOr}`!4%TnaN56wo$ zNs|M}>fIKuh(|uN+hF)LguNrIHo@2ksP|9fbSNzkH(MK8BSG^5geUqt27;qPwkBQqeC&29 zj}35ipY)eMD^6sHmdH!bGMxJ5pvy9!)FBpuI4}uln!|LJY8`2thC}M+PI83{!D~$R za_MHIEH+xvK2o^Y0OP%S=+1H?`=0PFHnporkH~Gr__=~oZKAiY8+4nU?Rl?PqDgkfHY44i`>Y2m=ik=$gF%-go~Z7G+lzO)xiLPX zRg{!HFf)R(Y?jiKC+Yc41|?j*3c{`KU~x&>)c#_rE_uIDw)r~pRjMiOa`bStPYAzR z+6Q&1Fc*6`Y26dNxLQEGc^tLhD}(i*i*bTsXXFH<*&WAAL*5C3fp9J2JM_dB-O?Pg zV2N_hZ=%!1FGiGN{J>|{?5&f`;2YliZzsxFy)Z)U-ohA^7Ry|wb#eINF0%8)?T-8|W9HruKrep%Jea3#(b{&AATRAm9!MxMa+_LQN=AFlGZZ!v) ztRD096w9%-U`>wXBL2@VG~NA+45?!L{qsM;GKXzi)En@zIYwkC$Fh%d#vqd^#O3Ej8S+LBe9tnK%5IYpn=#j?3$-uuTF zItoO4N4zZ<0D(Q|3KT1GV1wyHh0N2Lw*m38?69b-dzP-FD;o>o-r4M(#9MDyPFtsAg zf7hUz>F}UIa@(E_^+@@I+Zgpv=6-J^E-xgrW}^!!X>&s*AskAAMMKFnhP2R(_m0aW zG}+p3qgp?cysge)|MFl_Gk9Jfzf!9=OrOaCTij$PyiY=Oqrm}P&^VkTD3p#owSoRZK7I(Zm zeLoyjjyIE#IWnBLTTMg$A`sLjPa~MovFu zkZ0{joZ7wyt1gCsy;{K(pE^A;zHON-ef8?dF@zRwg7&62nIu~1^g5_b`&%Rp8YLO6 z_rl<0FNI#4f+P()E-t*j)v)hd`R5c|)=Zrs_FYK|cd6*#u$0^*Fv z8M?R-T)y%U#aj196&DHA?9n`SJT?$(7cRuXKyy^8TL*Q^x#7)$<(RzjHc5wpq<{-p zx_UX5Z+#5w5|z=gS}Eur@59(}Yw^e0h1mQ|in6t8p>_!;+}$zo=FDjjjb3u zV<#eQOVDIp8x>tsuxtKwEWHqB?5`9A+&zlP6BlB)e=>Yam4LTgi$}MP;^?D9F^>$A zgCFDEWh$51U(+!eh?*?=36`whf!A7Z)TZ*(ujq^PxDR;$F_nHy!^Jf#aXiEfV{g>UiHhkA?VaD{5oX`Hrx({QcS1DtMl71 zW7QSN^H)HVdNtsa@&pTKFU0vULq>1TZo#58m+@KrmOGl@bC|a)2u(+gMN98I!l&6M zb)6ou@rkf@&j)MLW3G0wh*BB^Rh6!TRwISIO$>JZ;g3>XdZ6MrqcyPzPqKlV*Prb> z?u>@rx}kkVlOV4?0*B6pz^!^!6w60tOjjZ)Ara1QZpH-Jcy|MPUOHm%52TB8sVcLb zWX7OPCZa)=6}>hNp`mFgR?3QDmpXR?#Sa3!1SOEhF2u3+K)#ON59 zD=fu*%fCgQQ&@ZqA}lSC&zg)$ivj$cee$Dv(OD7Tv@qA{I4w``6T zBR&?N)6$U7m7FBwp?{RT=4glVEikOM15WS0Lo*{0{Q^6VKSr#cN|P3WU7Oe8%zH~z ztydR~s+EAX_7h$O$BXA?M<0F{(`Ro-kc9_4?7l65d17pY;B#v+`$z~1R;qHEUd5i7K*WmQa z=se})!^PG3{Ye2#n%DzHvKuc_Clv6NAo%4Qd`dSSBa}hGc$KV!p9eWU#$%?K_LaoJ z4r$5hP^YpI>7mr5AkEnoy3o_u9pQ<7UCU>A|HNN~IXy`KjGZf1;F@_YOdZ)0Zkaxo|>Y!PzV!)H#7&7++J{r3y z16IYfpvzdad>d{>WbX}f(w@JA%)1(DmNeW@6I7bK$29TfOA=h{Tww8MlX_?Xo=JVs zwvu1AG3bddjlJ-a-QYE>z4-=lIJjsHLM!}$>D^0yW5LOvp~0Ket1xQmd1$;Up>e&M zsNj;0L-QtJ(uz~K5fEcYCdA^%!bw>DK#h_$>!NYB{7AaG4nHiujEs>R`RoREES-nZ zn_t4ZSOwItR{{2smoa(fc07wUc913-r`IjTqB9XFP`)Ob)F=+sll_>y-~isF7+O~w zejC4$>?~M$9_b!MQOK2?V|@bd9Y2JJU-Sl9R)=8Cuha0`jZ_q?S_6%ljrVjf#;iVv zFX`XvrSFI3k|ifID|q_(!JOtWB?-^l3x~06(R?hr^cjAI3(`Dm0ZH0REFLozn_kFJ zk?f%+)J_p+R%4(je}>#r-(10z$-m)hH2GO|>Y!oie0aWR6P6u(3f2Uv&-Y=(v^|J) zD2Yb(YNLX~NBl8yCbqvwrSb^mpzp)l{mes3W=ZN!|FliN@ zrRibmz4QHpJ0M|1gybE`4p_sfv@lo##$BAREIJ^2q@{*aQj zFEUW75c&BNyo%%}Qy{C-%8{OJ=gInyY@;;xt7TD?E&LhJUs2w9)XSn+8_e!s7-v>& zz~S@9vF&0iI``|1;g^(7G?Q>>eZ;&}E)T)O@6bQykH`v9rN#`<(~8-AF+ z6CbUMp+WsRsN#`kw13|j_G{Or;`O~_m@+ZYW$*h9_^s^Q@~#Zoj#eL`icM;*)azm=<;! z^H*O%x_fCfs9P5`ee^i``*bY7mq0U%1$>H^N9CeqYq%Fc-MY0y zkH4r4mrM-Dg_G+re*8K-O?E-`T6IywSHSTVi?D|LG?iFr0nx!vaN)vp#Aj^7Cz6iW zw{PR>qYvcI36SU#ad+1mOkT1L7eCvgYVBI6?46Fof6T_XWgGGP>XV3frLk433Oti< zVeZt0xDsL9gFg!r6e{3_$XCx17H^{a(*khwjh1A|G0S_85Pc6bR^LIHVPny|gd)={ zkNlVdFS2=MxtT9gOSpJBBJIsH1f+eflO{S0kAvdjli!!@>8yRNe{};VK09GRj|T9_ zWnVuDS?XEtN#@$&_NY|U0`Upt;pO)8_o7Y0o%5%mtlAAd$Zt%4a|&aBT7(m?Vxdwe z?1e?XW~%_uq>N z^UorZy@-nu6M*fzH`1KB4zJ7$qCxGtsO}n%9dl-3+mobR=cG@s@8HR24|MBR9S*tp zb8Ikv)u95kYCSy4b;82F`EmUBCD{8m{k!J6?;-TU1<2~xM#=BoBUBlB2RqLNLS-yK z*qw7&|J!{0x-Sear7NOgy)uvm9Kx7chw(nmsJG&RapLklydyiybhsx6-Nmt6FA%2^ zVCU$F_~)na$G)dX7xPZS*_A7BHOU^X7P{fhuGv49+3Zs(8l4y(KASM zYk(#d$U_PaL5MUT+;i*`Du?@lK{lo)x^O5Ez!JDJ#^^Y4MY03LUHEA+Ly$LaU(EgP$#r+UJtFi4#fO^W$?`Z zHa;c@P=C3PH5+fC%)nnUV_++^Y}5#yyZ6P9JzByuB@{~XgxS2E{BS>3KP!wG^GBg) ztNLiwtTo0=`586SPGjlGPbA*H7_x9JHVmj>TrvL@_990162(lC?qkCx9oqJ&0VmV8 zmXM0)CHTv~kyZ{!N31V_NCdd*9Il<&jr9SQFu*qy zN1jKcQq#Jy`tlMt9|j_dc}ss6q>c3w8&qiC4TaK9Veeg|Lk%5Xoj-tUik9f#$QfEC z88G66q{gFj$8b;92=itPK&Phl(578?Od8u6(w9fD|1k?vv!rIQj?<&^h#&Do+a_q$ zt}Q0d83NzOM{zQ^1SU-Aht4e;p+77fs> zMLSHK+ZVo%F5~J4Dr@>XY}CqS@ScC)!jo zeUd>cu5wBZxq(SDwn0^+AEpnei_F&*{6pwScU*@aIIYll-~hC+yp2AcdZF*cr5HV|KN^kRhXU;f zp=;GVZg!dCSp#*2ARIb!5*IF?!O1h9Nd|4P|7ZltHD~~Dbp-C*X76txr^I;5cfmXX z7L3Wqk&f+;_EmgfAr~M~0}8zj8jKu4b3{Y5Zru*!Cyhd>gtORw-4GT|xw8$+ zLW^Sh%#r9rbAMAd_s{qlb>k1=k4s-D#n)D6b+aka0Xz3BL-#_4W0?qPi20ldm6avl zU)h45ox791>W;c?24LJKa&W0ES5(X?L;z)<-c-_=5|$6kvzpQ^uhm^Kr@z)04gn~;6oi*#Ivig2>f;VH?;d0idMnmG(T zT9Dn-vI8bf`303zFJbMGAVUB`W(M=5NVqf|ju|66p+RXkNNh@B-j3~<)xhPuo|=*9 zrLatrpz4n!@I%KYXx*|U+251UNO~3P&yoFY?}73Ss=?FN5vA+YL950!QN927@@m8E=g3mU(BhI?8&I639)d`pjNm^AAbjP6PP7}dW+|DQ0wgDXxS zx{Fku8N3P>BN=jrm1hZ($3|#Uqa55twI_u+=~nZM1r~C1SbT_qV}o`WGO!am(75O{ zU<{^paL0v<*YJV-KUO{|*%FrQCDmDq!d^a0=bIaCPil{Jy#h|274h?MvhkZY5`B&- zZS84}xdpq%KVW=M@;_TP$H*~b(c1nk4qqnu&m{uq*>VVeu9bpc`}afl5i>Dq;uzHJ zI0ND3y5Of4C18 zhzm7xq>IG;N2%~~cFyIu??;vR8SetE$$lsSnJxee7oSDG)QfXq*Bw9rlVxV$hd)Gg$?ljqq$`cPdT2#`Gk@U^aKF6@%Wo!%HkXXX zs-?M^crO5QTh!@20rN)pMyD1H(Te%oGY7)=gc#NpvcSGWn!A=K(^eQA4dYJx>p{N&~O@W3jNg8f#}R z!L9h*Zqof#KI7a~8`Q7j4a@9{$?`xVw;+G%>kT4vvcWz^=fluZ{m{2vQ_{0-Fm=H& z6c0It1MG$bOC)74i6tK^TVyh!1^&8@KM@`pw(8a_1Ey zvwJDuKp*=a_wU}twVQWv<@!Us42w0atj&dp%PTSM81U2hE+}AJHl$09MNG0fkHJNP z$Ye2>*@2nlOfL#LH6*o0A^R!`}hMqdmV(ph-fIy z6^0dgax1u4Y7ibAgja#Vc=IljWJm|y$0&SBPeW|z8+?&eM*GHDUq0zxq9PjBE(4v~ z(7G@0T!m1qFl6CDc>eM=ecmA8LmCPe^+Cvmdx(b&4lb^6v$y<4i4D9ue+7ZwCD67= z_7`4CPG z>U~h7Ml}?7&gc;ZDp#uvq`ZLtOGBVi`6UdOpM;`pn?_l-r^ePeZ_1DXw+i(5w*qj&z#_-*7*Sb976mBa@0hWycHe&ny zfpC4j7b9lx#%mT){4>ajU9xEp9-QBUvq!e#(7I{p==>JHPFR9dZ}SMslNHb0&AeJC z=H(gUWAY3yJJ$%%E;kYWgyxw z)GoRFZXELLFZ#&f>*b3N)aFO-zQn7yAqe}D4l8o**>Bd?3J8gx@alCSjnj7s`w#_X zd@73@LEE=^(6VtavORZ3_UI(--NGerek&L&va^C`le9aIM{5Es$304(C zRS$oheV9mtz!Le~UEyGrbJXiU-oUiQ*HE(0k7(^{*gr+0NkLRX3e=hRdUSYu{UXl1 z)u3M2ei;2jUyL0-06nW&cxOCC0soei2nEl@91(;>QQ^Ad}!XdipiWIKr5%Yrcwjcas3)KG9#VB9?wpQ zGhD_0`eQg3^r3MZf)~d8UxunsvAhSaUVcI;vW+1NJ9xOez&3jjmt>W_`0>`Ut=Rt7 z5wkb$z?HKHaenPju)ezmlQv$@rYq&BF=#r@Tse-*WFwy2yB@24tcl>=b1{C~Er{_X zBdi2@syU|B$d-~)KPN?o;MVoqxOVFcR(ek8v$%Jr)^G#?5mUnUVqF%m4B&&d6hI}6N(6owU&fo<#oCNQ2^{y06Th~RQj4cES zGdoWd^>s9E7xzy09>b*tHhiZN3)5+@>f7buNwv*PkQdLnvPB zpOTR);Og!MXRGgPSiO|`uY4mkB3m)D`)s^Wy{r!sk zSd0DPzUbVp3gpJN_j2_i-NAI6Soz!$Vh|Rifx=9J;DC(P+KdD|yzvU&MZ6Fblrb*e zB0R&*{I<~I^NAbE6 zxaV_+Iwm;}AC$GB7O{~LaP>vpQG2o z!od+0YgZGuW-wp|k193L&b=V2R`>X-e{J%ijGreylRXfh*{@$oeQ_20uSUSHetV1@ zG5{lo4#CJaxrOCXK?;iN{*Gf zY-O(?Ml=5$ww0hJ=`+bj7y?5`_mJ-&^!hP&&HoLD%vxYjL&NrVqVJSfmZJFNK4+yX zBk)mI2IUk|q}{!OXTG)2uU;|I73PqU-m$PIAEs+t6mfAftjKY5hThx~NuT4tB@OuO ze+aYIKSkLtebA^d=@HXwlPpN(vDm$2Dek#;#QsCOaQVz39Gp-M7Z=UMO8?K|3vQW8 z$fZjS`|YTI6fgELRBm6qc}H0cJBZ z$a8%JA`O>UuEaIxE|}aUuNNvCp-Qr!GyK@RU|r8-g2m_TO0>bs1ADP^N>AiZe~#rd z#-K&-30Qe83@Y*g6!y;Wa4hR*$q0O7(9JIcf{|oZ zjO@JYxbc<-Iq46btSWl6DGn!-m5%04)zGK0Ev`Na$RW#uL=GEUDTgRLyUl#EY+F>Zaw#A&896p7U{FJ{SuU^EzO!;1ChvKUWYYq% zcBSt%hRwl*tgvMEC0k;0*TOhAu`7CB^26-$t>I;1I(dFCdYZ3~ zV9c`Xs5o@YpZd`pQ#4<2cJ1$2_1Xth7K}tqXTxEMUqKX=1R{pT0J1DF&>@+O(GONm zu+1FE0s_u%#@yYZXf}2N`d2iBqY_^n$FvRi;oGqcg0Gywa}7BYIwhXIj6$USdF(SU zicU%uQptc**`P-I?kJSsIj0bO zzKW@fk3(IqH{x!ez%IHLyF>Kxc?_b`Z(`TMc=#2qh{hG%GX>k+!i5}lGlayjND6rc zCd5WXAvrM}3ggxUGMQZbn3-9?s%!&H`%Q-6wp(%feM|K56O))AftiIB99-C^EtYv| z(D{e`xY@e`8hq=irSHNMCHyql82E(@xgvj}Y)LW6QgG+QetdpvPfCi6o3vCYH40Sf zJ{==l7d9r7!%8L~UYGk_L7heq8^=6WoXSaYTax0=^<3&Rr3U%j6ykhbux>9*mpNd| z!M#{=QV)rx6$({ojDFo4p@2RF+y2;wEALZbo6iyEQaOa^SNN1%R5YLkn((A#VB`p8 zn43`_*+3yqvOr35JR+YQ#*+7zpkf;lpwn5RN+su9>z6C30XTZ=3u==w=9}BL9uhk= z8@U)whHJ?I(8G+%AtSAOcSRf=ixz+wx{=$VaJdR7OzrUQ#x6XGE057_K4STCA9U;3 z7Ny-(@Nzhc^QT{<$-oj~w*NHL4m#@TQjMCS;GNAlb@>6_K0bux_9M`oiMTQNZx{m- zq{V%})3dv<;Gx8%y{O+bPH0fyj{YKOl97A5XH^q~xm+p+v$A3^a@dV)rBVe;Ym&Ph zon{Xw8y&*gvM1(L+1VnWyE|iM@kiWmJ(Sm^Il7n|GhVs^ur4GqZ$C{z~ZDt37J?2zan_E!N!p40C&H zSX-DuB8|i2BqbV=3`ztE8D9#>&2mmb3Nk{?%;@4&CA~HYp`rfRzWfU;ne%B(Bk0Jn zE8YMuIocyPq_DKH6+iWRjDm+`CdO|)i@vpbiu>SaMUP1TgP6SZHi{3PgNcpUE!VFo zoPANVwl5GLf~^O_QL#}^*gjc_@P@-NzcJ-j(He`6h2u?_1~u}V7LJTWDckqbpk&P5 ze9?H|uSosy6OLYei&mv-kkcdnl!fH5SGe*j4K>Er7gv>MAR+xp>Vo_uGdL@((PH2z z$P%Yu%aMENKCmQQ*IdWb#5DLibVDvM7Pm?HBoh0@ZE zxEDlDY7yf>6PV@`-!Wrs9346CjxL2!w@O8PT{B%BjobA`=$Yl1aQGA|79EN*MqA)- zMbO~W>p;YLHG#Me-&iJrDTI6_FnrPk?B8_|oBy~0joch|uEo%*YiG2i!m!m`s<)>x zYlQ^55@jH~x{X^Q)Tf1VpERWkb6DwAqGOvK9T~D_h(WmJdNWw%vZ7fku_1>-AP0hk zLr6!W{tNEy-GMi@bWIaWSt2isGT(d>?K8Ho+<@~b`7t`-5%wHR6^xGutId&fvM>ov+G*@oWeD77fthh~I z)Qj`D_F0N{Lp*R|{tNUO*$I_BY*5U~3Rj=sz~?G$k^Odbu25e}pB%@o;)AelM?bg* zU&h2ao3Y9>AI7z0@9`0!)g|K1%a_==EEMM1^rCIWw&MFjbb^$eTrake^((9`NoQEP zxf;e@26UJ?)`qo_I;4<|7=HgCet*m4d870M7|4w&bl;gWIB8dyo>Ey??FZK$oBMXI;J`brQgHKwTF>dq&+@9 zzJ}fRwUC)>5cn(zZ=>UI@SqeWOVmcA(%Hw0fRI~9u=&1?&bG0dKHf(9A3q zIc+MkwXCeIjUuc;aQZ&*@tJ=steo^Ei>(6^?D$I_6Xmc!SE>~Znn-wU(h zQsQ1<$Er=R7geeXN%85h_AZCF?GF3_iIQlQt|BcRxG+n^KVAk$evZpoKdHmH4@U(VQFQXyZvjc#Rnyw#O!rD z8{~5!`!`!%#L$k(F|n>D-sCn>XQ>IoU>}C*q!8hrhHTHg24VhTg{gwMEE> zqOxUcLL=lSnyK*g(s9Iu-=n_ML!(SZYN`Y!8;{2F&Xr+^+hjdk74ZF)w5)t}iw*&y3MpU)Xi3WaI)#H(cqtI+`_L#AoO6eh>mITqxLIcDrFZbXon zk*lbHBD>kgI!?%KW(ddDZI@x&d?XfiFQ2*JbJr0i(J=pfblh_Rt;+NxJ;bz-6biD_ z*{4RE0hnu+imCff;cbgyC_#<#bzX2nsrtRJBJ~5>pSgpNH553s{{o8i`yDfyX1tHX z9RJL!IKoUb0Y`e z*hLwLH~UC7GFn8hqxs+74OQxtMp_i^MP?IWItCqQ1u1$G&~%PrwuraFeP50y6X>?(|2 zx(KfMSE7p_wRbMrd>08H5WwE8=IaIRjqxZ8{=8ML}bhgs7M#_$gJW-eD3pcQBsx7qKeW0ZW&E z`(!3mg5ZiaLx$jUyb@nx*y>GYWN1hMryVd z_FzxlvKUe&mq59WW)Ja>Eg5@S2ecVB2BR915o!1}7raoWpc$?Oy+?|=ER0Vo8_)&g z`ZIG>9i4}fC3ixB!uGft@*bbm)ltCoT=iSrerJIqja`gRpE-)v>VQS`VP#7UyoCje z)?&45SxmD!gu`)uSiN*G{K(D_dASE&mxMs2lfuEf5S%o-ap6HShE>f-Ks^=y8dt8o zfaSn8q8xY?E(&)^Cv2EeNBsVe68^v+Z`2+bTBF1_6^@iWE|d|gNqC6;w-o zrff{Mjy9sh`}+qmcg64xK%;-3ifz@Js!oH8{5gjiHwv8 z2x)AI^*1+_CoEt`hP6hMb-%i=sPXpBX?$?1iz(&)r1E4$u!EA@UD&YU z3aZR&o71IZhMtgZr4$*>Ig!G~qX1krXYhqh1E%*TRH3*ToC=Q$WnmIa%R$P=n^-vK z0Me@U!_2-_slLV^^1{Mgd;yP&3H?8WIjpHG^eUZUtYyccELzvoV&0kqXtua(ju)89 zNRKdS&DtF~M-;=y)0LVn3@NFkah(4yjq{b{r^m4J{AVipD|W+c_Fz6q6dBs2tCnGbw3I@-(z)w+zJ5B;rEBJAltj2gWZUVh^ce)&F}TaCr4HU$l7 z_0c%IU@=ap4MF2V70V%J!*M+BJPs92rrS8mBb{Ss=Z>J~bXn?Q@OA7sd% z-vQ2cJ+NtfwHzy(42Zi&e+QQE@hygOzL5~J8BG>uX7u^K2DVg4As4jJm^$kDOQNFe z3@$ui5P*`k0a$?TRKn`k`>DfP~M{BqS2*nlwa4 z#UMs&jl{Tk#HHu<^r;jz2QQ+J@vD)Pq*8GGa0bVNTB1*puj422<5L{D9D*W0u*zGY zU?FFCl^TgH!`T*3xg)kK`%i8S%sNwM4l5Tw)UEA@*)uzt+$hY7xPLw+&n+9nL3b2! zi^Kglu}H7v1(S7Se+z6euY-GC%v|&%ie*`Pos%pwD)&Jtw~x5>Bpz+O*$YaI#|i)2 z0kHRL0xu^G%&31t!#_bN;FZH!hL!pLBvF0E`Iglu zueSdyPDw3l=A>BFan%L}FiNCN*eDfM6q$VLM%?!3Kb~$b? zvi{6$bz9hJC;4{FoXZAy9-Il zdT8Nr85<8=N5u&(;hZDTs}mPwvF6ch(qUh*J*JK5fdYBz6Bgw6fg<$=LZcE<#WibL zf*!ntS<#yCW-mN79?QUt1z^cuiKl9M<@bZ?b{vwFI=EYzTr21Wo}{Ls9xG;gDh+)l z1V!$C!H28cFm=l_R2wuA!^n@xed`tq5>vti?Ti&Ee)&6khqxi>~AQp-i@$L%HLL&Rq+l-_BjQQ+_<^xMy!~9Re?&#@#drjIL2! z3=-!=f+F?1pvk@w2n`+N^q!g`~oyS2%R@O1e5zlU4#tjowGRDMvTbe(cvB!w2`n*rEL~W@tZ*8QcSnOFE!X z{hk>0V>{IF&F$Xm-wTshiAO1P>eLM*s4QcQW%+5~V010#3E%4NFlJa!bg7*`a~?OK zK;^oP;Tv%f(>L5jf_S4`hqvdqW9{Ws)a%>^zS6f?{_A|KIQ0xk8NQAx1}B$q$D87< z(YlC$_YW>(_u1!&i+O^%zbwSoYoSQVh|Q#b#Dt>d7-1lQyqCIAG_rl@X zzhd5Df5c@hZ_y;-&cPMLY%RlI&#?e%~^&apT7?NZuQZY9484W=AgT0u;*GB(!{or zz=kG&d-6!l$jLP`GZ**jPs#{tsiJW1(nAD%B9n&%$jY-SdUWu^sl^j9XYX}9dj1BN z4z9%Yj9oMQav3HPaMX92Op8= z{SoCyT)puSZsqHvTITXVa(-Ul#jZow@j+~FC1Rp88lDW{uzQ$4X*r^*4nUXUrp7`P z4sN^#-?qI`jdZ2V#}6sdsYr=uS67iS`3@=hz5ljgyeal1_q|&6!i0&<;G4O`*9g<{ zQhU^E+#D(Amtn<~@XQs2aj$M+(}sfx5Vyei#}F5t6l;cl1!J&m`EI-rAGxRFDPh<; zXCuPOwnWRq)K1@+ZOqHHMl0tJ*s*Fk4oVwiScSZuCq>nEXlL;fvli{cvlwH~lCl2a z&|=KK>xLhDRupaHkI!%6@Tt3*FJP8hJHXkR92IjiDClD$O+!jThGb~*=Ij~VNsbrI zTv!)H-wt&UeR>(jEIx?q&tK#I)svVre-o~TE1^(GMd#41OfA&)jll9*2N0Yggrqah z9NdjPH$RDA6JMRdYBCtI?{bp#cn>+zYjGtcKC?em;SaEB&rP^Cs184p+D9i1^2_|t zPY(}Z{>A{BAje^JBi}q@XK`=u;!J=eT6U@-ZUs@Gge#uDFmAjM5e_q%78XvXfj%`J z9o~Ye^LFCl#|$~wBkJj4Y<*^i2DOSpp`$Undq1wd{~83+CcVb-#M;tP=(*Zs%^KBh*UCkH)7(#0|+p@ zfP~N^{Kion3AI7Z8ih@LudhfHs6SvVTBuK9*e|PbAs~)PqlhjcSlr6(gH~a3Bh|2K zIJ?@{#)o3-t{c$V7J{eCKdy1ukYy{OGBpZ)SH``^)jiAb!?fdY?=l8MD`%|<3%(>p zyuz_VXAzX~avDoVxZ1HNPRzxzDXs)9-v)#0v+$4&>eBq(6t^*d^DU(1q9!aHi=t^W zR~%Zh2i7uu#ZgM$S}f zs!AMBjKnwhaE$aJ+eV39*11+cj3^w4@uTKo%ejYm{^|+#u3d;dMbOz)Hrdl#?3HJW>i)vAlq(x+Iu<^5ixj)X>nL_LvY+1Ms_v2V=e4PW?U%JF#?B8+-k8;@w%)5F6)bU8bp}pr2oiQCG zhho!#8?en^8ihS&`1If~Hk}PcV)#8Qo;L@Zu7_p5D@d0dhE03#z_vt16m?S|{PrPi zIU7WM;)4O5%OZH+TCBJbV(24HBCc;*jKk@bFuYNbJRKZsmqKXOqqit0V}_BO3?aE3 z(hIGt7e}cIbuesbA9Sh0v~GrMd>flWbup}i9kxyU6-y4?!sD0ZMDAIQ@rzC&#@vR= zVQ9oEJ^RA>${b8P_S*2GWXRqoyCO?bwLZZxapm)h6J==H*ztm)ay>BmCSn z4NJzY!<|T_u{~4>IJp7y&qqS9jlu0pXK?CKl(F6t*t$5t#)7@4BNcZ}p2EfFpNz2~ z2^{ULVPPSI!onW4JGMdk+267DUZkYGRdifCvo0DlQ_M_>nVA_9Gc$IZnVFfH8DeH; zJ7#vw%m#e4NRBO&ssU(%^ld2RYKH?~LMZRRQv7ITRxVS&V z+H>`IV^4*cSY=;x8k|E5R&WK0*ZL3U=q>aXC}H2)djyB~ocFJ3>`B>VXxG?i1#ZL^ zIxHBQI7n*n#t{5mF4_F11 z>oo23$TK2*&v_72fW0E(2I;%85T*gqjQlElgKeT3uGEMkA3-2}u@{3COVSY4?+FK@ zAB-CCDt^Sk{oHj**UCS+JyAk{;k-uVYOSk{IE7m}@k`7)gbJN6NU$ke2Uy&;8 zY`;v~KC$c1vKv6@58(vJn9n2!5tE&hSN{EIN>J!V2u7d1nSLv1+;DX0#R;~p8?%Q< zTJ&g&-BEXEvbpo7&Ns@FnT?0GQJl2_Mx*uoGudF(6s$$$7RrXOlU1b~P>}hUT*Y4K zEnty5WWA$ejoDhCDI+#L!Z6ya9%^p|T`#ou+$*m1!Q$Z@I#+$T_DL%4kxXC0W8~`u zH(D%~GrZ`b62atY($9_P9>dEyvp@kAallqamD`>Q*7@(paBJnxyI33*GK_}dsot11 z!*AoT<(5GEy;WUEulT;}nvBt;I)-OlmkVFN`VE)@UAX4ejmNdz?F9?2p_IshTYJ{J zSFDxKxCh@~{N_#%f$KT0X0tVnzvIqHqr0LZ z{9RG!ra2@GqTC05^jhq1-(i1*qHhZJEv_gt9VOuLL%T{Ic+FonV@^k&1!X6y&uWas zp&;q$Yz7vEZsFL3)#Fz_v9-3oxSsjO8<;tgVz#(W3}5;OGZ&cm5=4GkPLWcg);3GX zpM!+VS_wsfYqw%~$F8Fd0#k9R@U;9h_wxEDHm_}d8PXgL#r{y3D{24de&^sVW{VU* zbBDSipZ9U>x4G*ivw)ZjdyPAg+WK=`#tN1#Lv{Z^gHTO%2>vHH#x1gyA!u0_V z`NMhCca>Ke!=(q>aGFE35C39#ebGS1w{uJ@K^M%+1uNdK?d5R+H=d(<4OM}9-mBZG z`n;l!7EoCFu~QDByy|9jQr`+Ydz$bOTtf=4()5`+W7H3o_lg-s$l$U-EAc6RD2|7@i`O3G_!kahxe`oxw-)7i1osFT3|e z#oNkqX+7r(g`6$@G8Tg5~0IS&H>tcC~{Q-1bWHPl*DyW7tX2TTwpXCU=047UeCzt{(b)DbmM@3PPUxgZjgz?{>AR;*y|q4k=c{C5oAm-@!?=GfY^gHq*ARa zm|be&GVGu$voCOikQ_pH?D|(a+K!(9VFYme7p0jC>yW| zj*Fu;(p7Hom~~}#V*PRPlE4bh*$w^K>M^kUdtRYshcf=lYzsryi2Li)Qh6G$Hxu%A zdXg49u_6lRkHB6CNuLAXoN8v^q&FhAXNx>r-LBat=yVnG$4OM2SNtZwTl5#&VO<M0XmI#_ z__Q$ca?~Vejl594ddi4s0(09rjzd=~-h?g<*_tP+W*U!#R3_>%s}Rv36v^1%7avx9zpBjG{2)p6`3KgKhmuoC`(MpOE-LJ*I7)8n)&Um& z9n9{J!nLmO<6n77&pT&|A0QEHY>qY4J&k2__=rtWhzAyN$?HfaHGISABNolV2OBa? zZ0Lp#qxWwjXKpvO&~d*!v9w5z;#D#G%Agy0s|f|?_IBOFO7z0uh<=2^Ty#kW?G1(fF08ZoQF;cB8rs2>)X0T#u{cuaXT(K;P9G4 zj}h<(Ayrz>LMi_G{RJMQYJ&j%xiKV{YWK4S6`PxP>d9^*6XkdslK8zoMPQLdOW#X3 z{Z6VQOvj$X?g=#q{vAIt27!Ou;ryM?%VB7Vd2b4>7TU!Lx!4-gkv}dm7 zxZ5z{mELYC{mSRNFFtF05YdN4GtOU_9BIQDAYj7eHGfEk?(E0B{=rc~zKi5m8u!Ad z*UqU@R^(BGANx6r66iyOGCtYCT&!+uCSD^cKe-*y&MyJ{wl>SO^mS)3Uzh7(KkPml z0q?Lt=hk(#qJR0Nn~p-B!A9rDc8Ffr$06fO)>>&}xNsmd;%k{T*2b;uGW!6f2BNj0 zP)GuyB=Gl9cp5*D)k_VjuhyR2u)&!U+Ii#a1tX#(Op?tyS|NgO*AC-Xd|Ju1FlVN> zY_PANhczWO&S(K+0rtzSCiotW`cg0>_YDrB=N#VCy^m?qL#+VQAEBE=hu-SbRT7da z4ihPDZH&+FTo{tBR>D0DYb(u5Y>V^?t-_K~mkX#d^B0uzXM^2baH~yb{wB%bwJtYo zXS$W-`0>`jFM*6Ja>~zaA(82LpiyKlu_ISAMaY;Y5FlO8KiHC@@I;CR`$TlO9*N0Z zi!o{30p^cq-u+_2ge8(v0^CC4WEuDB3`QBd#Y1|O>Y&ZeETJ!fXQ6+#%8VxB{+mlxXT@1?~*I!e?sM6~0{q6#nNrx_RYkdkGYU|V8)q3R< z{j9ZJ3q4B?c9oqJxboFNZ)PXPCS2?-J`^l3kcXeNlg@$;=B(?i_H50%S>Ra0p;=N{ zj$cK+TQ%{YpNK=6@U?gDDVf$N*WwDUwumiDa!`kT_nyqzhPQEzAPA<)rjv>ijq2`LM8F_?VpP0RTbcY*QYl!#d($|)pC ztY0|8kC4`cf*s)%J=5sdS4u0xbkF!| z_$cLjv}_VUdQZ$dWfXDyoL2m3KV!$HMI&4Xoy~lb8gtBJ4o>H4GhEr*g-RR4Dg8Ka z7M%M^5XD)B_(s$LEm~e17~6tx6A~g?s*|Ssac;7@laoYrVMg#2l`cej>>kxo3BpTY#nmdI)GMHYV|N&-B%ARs%w?RpU4FGC!ZeW zPhO7BV}SpN37#qYCFB-~WE|Pt6BYXOMySEo$XPJ@wxFSrud)%Nu~-7fmj))hF>>JHW zZW~`RDZom7ujx8OiD~0|dab=i;~nZPyiTyqhTP8_`toarK#qlqUQi+_u^L@yh&R*A zDj(v-H=i(X;al7%TIdBbV&2S{4{vl=k!bdrD$~vo=lXy_p6p z{7TcYpAoy>i(5p)wfiTv;^Gqmh8lnE_|fn7`XbV4O<=UCoSoPOcitT&RX;N(BKh^R|HYyIU92rznrlG59h$xAU~>|7b9{%xHUd@K%z9^ zL^j!Q2-5frKo)$(Yo3t>X8x2HE3l&8`uq#kh6|I4IVW&tJ$|_$MI>%yTfGpXIMyN% zfV;$bBU1C-QStcJ-b#lPX${*eWpl8H|o6%p&Unjs$mm*SgX2riz=8)4~mbc zn@GE+N4R_Yq_s^?ey)K?stc2*=?x7bROR#ey^i6t@tKU94xhg@?~%c9pGpEXbvEPA zLK*U?41NOV2p;^~cwJ2=+q;i@`V76*TLt1EEqMn6S)VO=R>^|qXuQ@Xi4RsZi1W?)7^H~?HT645f0tAm;FW4 zcwV^Nq2@V%G(tL3s@SsPp|4_v_PKdnTuH!1v=BwKU5YY46M7bmyQR^6gh!$52kc4E z5pknBnpeVpDG5+EzfETXD9 zsZ><~mP*$w{mzWC4#y68<3TKwUIZEpdsi0QQO%$tRV;}#w#ues zTu#{zEgtDT8VfvypYr;rnj~S^9?;D(HHDjjsRbJW1 z!MflBkvyDvzR)xodjIt0t$Z`_y?&%!+l%pTw5ml-N#Q157=>Fa7bEXR3_cGwwp)*i zQy8Pes0G+=VfLKnt5L^~fk(0xrmbm+8g?cR-ruWOnB~pfInO_9;^|r-Ti##CSY?k? zSIxxv&0ZdI`%XY7Ctfeb7nx_IBGYEhERAG)~6)J?i5A4lTJg5 zj0t{)qc(zE)zXe&^kVI5mqq+f71G z(YI)iFp`953o!D>C>nI-qC%vnjZAnB5^zXBLP$_WsEdwb(kx^(1!)wX7;{I=7;`MaSj^k6q<(!;{{>+PXh0hw`ZQ)Gy{%%{U&`Y>#`p`-LH;2bh< zs`Cu16Q>$VzPKXwGMguObnko|+oleKDpBcBcO_S8r1I?ZJmHUI*clYmz6mg6(uIbe zuv#L9-}gwhc~9qQj`vO2;4+&7vCL2KXyBs7xwB2*yw5?VwO8e2+!GagWR)mvJnhx% zKPo*EGy|;~?g5P8z}KAkNTI#>a{~s}=s(`pzL1OySQxy1m4Ko+l+6=vJ6cVc}>2!m_^roeJ|s>US--DQIMrgp`m-M*c3;`H1e5Tm3ouO z;6Pb{AF;$IS-)6$UWEv2$$-%pZCDn1J_kNH08!>X_#93On6N*0s|DU9N>f2q)I`!1 zpBZ7qk}S70tw3Dn_kLweY^E{UHB7Y#Ps;fWV=T@WXX^ z*?==hRjV9kO*-*&9e?6rJ?&gxl1A^)Sio)biOx?aq)~z<#EzK^_q2SW%FS%LMb!~Yqd|kbZcC16}3+>qj@Kr=M$yu!1FlWu+mN% zsgW#sH54`1doyE+wXw-EJd}1O2Eu#`=*L1*7;aB=q7L)3oQEnSthNFDy?ZR$aml$~{n9Gd1os zehv3eq+=nKAhJ_6t*o8`#sZ|h6}jM1bgh6OfEZf5?Q(8C%sJB0pR0PN9}ff67k3mo zVmiB8bNF^->)zm@|1In2p@WQm%67K{7bBWiQ()wm(WpAg_Z$SVmE$;2#jQ*+7bQrf zpc30S!j1mR7`nS$od84uAM6r(0qf{ZCROhK11XC{!p(@@B50`)S;rSK{6?#eYfJM; z^eRK!tH(%+`nyk!5$Zg}Loi=xn8+wY%tLSYXVNQ8Ml8^({&6rKb|lUfQNb3wt`|yG zdr_sd}leiZ3Tlq0A}1HP`&fg4vkj8boWuR=X%j}0rkkQD_MkE zM-Urx{WT`B(iB8yUc~m4oqFyU*u7FuEuATnaNznaOZ2`1BnG+f&abnAmU-4Zr49OU z0vukD1s8@MWp#U@{~otSGNroPL61#()W9u{i(UoNFa|p5xf3Pc?T`4N)r>xe(4qJ1 z)7wYwyD~gW3|yt;irPA@4IBx0Tr`C}FbK5ocA~vT!8@d1XMMfz0`^?y+&*0DKYG%1 z&wWKn`txvU3oBsXopQvPSt{6iz^xRMaufmZeFS9V>|xyHbe*HLy%ywhUo@uWn* zAreo@4xlsY#Iyt21BnGSDI!^DzX;gwHehOj=o#GVKiR(7(Q1V~-Fun>`0J5r=jdKv z`NuE2N>RdpHL4zt*XuQVlI4nuPmrH}166hGdwxFRZ7F@fcwIlMqD?A)AA6t~FYL1B zBtGb3r-A9#%Z-MT@L z`?dFU+f%${v>jW)jk-Sj>l(}T0NPvU%0RtXX4U>1_R${rRc`|KMRvUng@A*wqOXz| zXlb>pUI!yrvqDzMVQ;{#2gi1+<|Wa%3B{)l8P}{n@-jv=8H`m^{j+6bvm)IZg`=RP z37Szh3)QMqksx-+x;!9i&*130y}$@T)7DyI*c#Mc#FZHP7Gi7-kU8vD(;RwPUgTQ4yA@QpmallLE_)SpL z`+@7YR&yF-y_q`$(in@-Sx-;$C=6N35#~XUl_`vhA|vJhlM_@KtxdH>WONldR@93<;pwpsXkiCS&gbb%Ra0oPNUe3 zpla@0{R{o)En4@^^E&&m#G|+be9MI=BddyU^K;@W^sAzvC1SDx=7c-J9x7l&V1{Z* zbL8{9&(VIMmk-@FD@lxPqihw-fB?o3i9692wg z=4UPZB}1i;ez`(wGB4gnKo6{JPQ$_0eck#Eca1YzFE`I$dBR^X zNy7+DT`)zlHRM0`<$H6LI3~z;PdffiN7Faa8n%aKHFI~LyoVeFAl`Xa{^mu4vdoaM zH3>1OB8D$L#I}DVy~MYh#VEjd-9U<+*+695_0Dd$`fjJb_4qnv&eBW`lmHDH{nuog zTOa-_y;4&0RB-1x(Tp~g=N_iqSEEh{cL_C?U9&iXCy=!J{W_$-9tkh*s3ZXiczUyk z4O2DnRQ|YG#V|{MIvA3-We3<&mBl1?|c(v&| zhh{O4MyNe6TDVI6Evlh6Gdt$oJh*hOn^vocU^|scOjTj#B025P5?*81BTM54qoAhC zgTdSjEQZDIQ=T3#N=BE{i2QiJi~7V6UnHfp9!HRJ8DH9#lqs<~TV>r6Zf3S4xY%zx zws0f7w-r!+j^#UWZi6?n9ypHQWAb!lK-5NZdpV(Un(sYH)d`7gC*}*~MN?H1w#vA} z@6M^k&z6MQi4~t!KNm;AhCw^0 z1AH8I>%|*r^SsN7#=g|+hs7XVZO}|j3b~-6CdT;%Y3MlJjl!5fnk5s zF_(c*&fi?vF;{6bn0wLeVHf$|8U#u=V<#;}8h$HBKOJ=T=tQ|+j^E#_f5y$@Z*k1ADv83lH=+y`M zAp9>u$7vO6U{7YNc(_rifyL%(HtxB(+LzUPM;2S!{`=;pdgNbH7!k<-|Mjgvn(X?JGU;*@BGrc0_4?`7EdNIlmXB*BgnGt|XPw&q}4sMvwS9H`N7$mX&R% z)px9{BU#VGYs9Z2`n#HQ%I$ubZ7pG0Z)nwONKhXiJu<_mn(>hEB&`$Rq9mBK45@SNg_p6O2-k;bvfE#S%o9}zfxNeQ8Na) zZyaNvuANm5ZnqY*pguY44Qf3hjlg^)j8crL;kXdrkf_I zUIKc*V&c`jEJ{Vy-#mou*Wc1Gn(Fd7Q9b0SVPf_VZY`-$#UEj2mzPuSx6($ypW|Fm zy6-?X^E8Dg2-nE2eBIV2bY#C`3EW4jnGVfcASJ@BZgmrNr zn^H4|NmB|50u8Fl(?_J&LhiI5ewKteiw&tuI`pjVUs2a>S!+FR(?r1OD`L&gI_z+w zc)uX$YUJPvEl#l+VKPJgoqZHl3mzeQX=ZBHVS^PE6r$KUTOy*so?kj?uS+XcxXyyp z?&j=X{v0`AEL#Jzz%%EfwW4{q`^KRQ)C;+A+HLMeVwEtVG;e%=)>8jOa@7yKXXd$m ztuGzPYY(_(XJrOVe4$&<3XMynZ1(ZgaXa~nGtGepi|c!OQdNp40>#U;Gfeykbj^lw zLXia*J$-ZLkK-jJJwTKH+Pr6=G6`Mxd7APzYa&|&$*SaUP-%QlTBCD$Lyw<=_%5~iKqo_IrE zaNuob0A2*}^a2IxUnWuboXDfIrEB_UD+X+ZobR5=&{aZYOWN zSGTsAhE!i&GqWgR_5@$SfhDrNf6U9&K6x>!29hU5!x0NaR}j4-n=|V&?T?Myzs5z{ zS!5%6{n_7eGdZJxR4kWlFZfBd9)?2XjnhA8xtCyR^V8R+4O%^S5w@e)JNPmcqu(ZS zg0wif%!G}eZv+8y=Zc1agE%GTm7vCpAAhADyN99})Avqy{OWh+rNK2;&jLt084RR= z=%MGptJiuUGw+BuZJebC{Q)~Yg;4aj<3QSaIAO>j>N~B` z)C2pGje2PGqv{3*ub@%}_K(0JywA&H7InbO~J>Xk?4C^`)bSJME$Fsw(^X~JB(FY`Wdt17rHb6n0-HudP)_U^@=lyXbl#sW{|0l^EEhx(SPfWIpbG1p zC#0&34_IaAQ;@!OP4z#^`fj^h&$#zz#q}cwT{5)^Jnlz`QAWfCq*U3MIg;j2=_j}& z)W{+;Sz+?ug(q<`7eE`iO<4VcZ-_m{&4aHG$Rhlq=+x}l%N#(uoVVZdw_Vpgt)0qr zjX~|-!d20z?#Xaww`1%@*hT+f9g-VF22v)4Hcp5f1IP*L;7SueqlB9OtmV-3m$i~M zTqHUhMPH-XY+>b4ltG8kSL2vyH5ypOyy^}d!CFh#CHRP-7rcQUxgUgQw-)=-9)~ED zs*d6=3*pdQ-5({+tI%SJ;$(pldt97e3|B~k&+UO|vCv5By{1IBp#nj9XXTI6fCDB< z#EQg_ijJH)VaezcwameK9J)k|_zn=^a}9gFM1z@PLp*NxKOe^e>a|C(f|`Tp3uh>z9; z`V1{4rjYP21z9L7HaOerxSk&zE35bCUam){`mTi&kVDJjQII!ZwYlxXW7_gP(PI@I zk<{6s&qp-vGXLtF1C+>=cJE^k&wTE3GOo9U3^o7;O%8HN%ssnEmHejJ&V3#d+ zJMk|Droko+(Ls_;wBNE>p)Y-3VK@%MA1Iq?If`&SZY`r5>-uNTc8Hl8vx!eBV}5E2 zP6}BFf72WG`#cA!zV61(a5l57H5KoLI!;f*=nJ+z*`Kg-Q_EY%FLyeN%21?2gIC9Z zJEpckGTT!1bC;cTV7P*6gx;^3=8ro>Sj8M-m%K#;SDk!RflQ@VBi33Z;?>oR!E+ZA z0rl}_2Lzrw_;S4sVakW4tGk+nx)&momZ)mgReW5lgW0o>#w`)QFFi5DO1E;J4HcHE zbgh7$r>PpEge<0M*|d$91F(~!OQm#HD=LHTkY`_^c0Aolk#t_cY$%I&eGP6mIM`?1 zQs~pcV5YzHb#plW%=+)3=fRWFR90j8d`d4NAryER5mjG7rNe^Nv+vY@u#uRzXdVJ) zZ2>%3H(x%3vWke|=T#%^H;Z`dIt9z`fyp-B-mqCB`z3{nKGe*QYsz!97d|ZbGqteT z@G6Ba?K-64y105b*Mzm!=gidYriLKP`+@fHZa9|t_go3{NEgAc_br@C6dt=x->m`Q z`|n61i0Yx?E$^U9{Z2?i9{nH`y*9!R#)Yos_}~lr-jH4#?=-8?v`8kG8SQh7bY8y^ zRA%raxK~y89qD@q=!g17r83z1!R2;EGiy7Y7MBED4(+6?`v#w8@%|Z1%{82NCEktJ z_8_~d$-OddcJpNkl;8Z^d2^uD*m6vi%=TIBUhyLKB`{6eJOz;j);8U^8aRfZ;wkTCT zQTxM5G{O`&6ofo4+@uw5S`Qtjo;G5J%kP-(D$8D6b$ruYXBz07O(@w!k$8Q=pjyit zj%B>YffES1pzbJ38~mnRn;_uV&Y`!bE?MTfLOFT9$Ng-aWtG9=;KV-{k-v5id2^9Hehm6nn`dQn5Zx_68M$biH0f*WNnEU*QU`0^y)U=Z0h`PU*>7xi^$KDwT z@v)NX34CIfoeo~)|JAFC-6)Qve8}#af8*E7)zO9`t$vO+hm$7wLb-q2g(R=Hj$2ob z{8SkE!^LzvA7)B2o6OQRf^bLNuogD>$p^B^*s^q~n|+mC%D-!W(nKCryi=WgAMSAS+Jt zL&JVEK_mpswXnMNelhdv4}|M(ucytB^HIydYOY@YoPG^>r}zlXx?e1jCn`2l8xb}l zG79$FDQ=WD2TFyjX83qN0`KQyIPzj>rHm$#0yjLS1F98i9bp+O0*q7mKw?}?;l}gn z&*jUTc9%Bjr2LRYHk7yJ6+h#s)s;MU;Oz`fv& zoue zjPR{NCLiyVWWDctNnDS3Mz9+(8BL!nL7#$ZkUli#@a283*8IW|;-IYxGJ#Pj*-CXf zB^k05%hh;PY#2y{K@ViNeQ5^3c~9PTZXuumh0PQ52_)MjsTn_pBme=2XH&g1GMSo1jd`3bA<%;0wbZ-{*j zySAtgZ9^>IKwJVoA0B+(52bf@?$ocwA7|VwgzBSyheEW+zZ^MKUDsf0Y=I^`o_-G8 z)2I!sN5F@UZZ{wfY2UahROBF7F4_21o7bAtsHj4>s6x)F1Fb_4ouwHX`MhEEJiKuP zd>^>^JWsh{W!_<3d)SfJx*U2m(6gBwh~3wT=5KJ8Q$FH0i3Y3fYK#*881rs*Re7{O zc{P#n%!|AVwoC!e5MiEsxD1zhw-@;4UK}rR37^F(vBE9lJPC4#iBqv0>m8%iD0fr5 z)r6hHjp7ePm^!>$Sit)9$OJapx;rF#`{zAYt_(Jhu0e*zvi{$F09v*i;>dgS8JLZz z!>feZTsy87r)_|mH{;Ja*N4GX=0YZSX=bZ>*OwvjwAy-{C1en6kM}E)&+{1}VrqZM z~9PTSTKDBi;1LtDEaag#B($5tl;JXy~;LMT;0`cqtb~IG?F!vYc zPfHYAEicCYooS@5Dz}nI0I#8Y_T}VDhyhFFt8~_ghFU9cth4V;3Uh>k8F&06o3#kM ztX|XmrczcV@wpM~Ww@pBN+@!ttR^3!b`9v&`2PeG|D#L@G*afcU7w8XNy*+nB0~m@ z5ZuTaAx_(kK*UQL7m3V8)S8{K8O>jOoD@FoGdb9R{+!V)G1lDk=OT#j_IL}H))$04 z*Pyi%V);z^9qH%|Ik8L{Fw<&6(f#N~ftO>C_3UIGN-;+9a(!c))^35I#A(}C|5cjl zq~vL3^U1f=C|d-N0r?SRp}VW5Kz%X5=$7)}RzwZg~MzRWAe21l~pGkwKb+p#3YQ_V3(vccg8NB$G*9p@{jm=?8-Wu#6S31|bUSrB4jWRW( z8q0;;$hL-JQr05DDRXR;c`N^Fc1LEL`CN^5Zp^i%nxXl0>22#Ro;Cmb6w0?WAO*^ss3KMzZhdc9r4VP4(t6Bap0~x zdTlJ~1VGLr>nqrT=^}zCeGpzzUA$Nx> zKwgmmk3&i%ODSsp-2B_aZ>q6zagX0FR^#+S`Aab&=Xhj#orWcc{W*jk@rNj6jI7vt z8~*!`B+XT8(RuwbXS%o}wD)9`j+`8o>wa|F@a;~P!CaXkcsUwr?rU`}+9AtwjP)!o zz}E~FjMm=;k|ob!&S5|({D=lufT;_b*C6+Db_wa%Ui!V)zxE`jEM^k@h=MXZkKLJC zW951IYq5ZR3dGjwPHW&Yyz>vXDuk9@;YdC2nJRx64=(#>0f4p+Qbu7}e^7%V>wKhh zp?%5b@lqM`?K<85K4wAQvL9<1+Emvyp3Y1MDcjt#APId+{Bk*^oNc1t%LM<(2!l`^ zbBF{Q zL3z&J=V?TXMsids@KS?KW-j_?TKjN7@SO{^@A56YsXuWYu|(eu7rKdh;4waP`XoZw zOuK{v?@N1pYO?=28Dpm~rmRO!Y{jq32qVwZ0jTJHSfb~MCC zv_HWHk?5d&acL);biiw+m?B|Vc%&qtb1g$Is_syoof2R`r^PzRtp$m+PMJLu2`-#H?^yWJ z846jRz+$LlS@|Cg6;k2Fe;X}CgcfVXvm-AL$I`|I6Bpw4$z6SaW=Fwbj;)jlm5R_z zoqd@ds@U*n(%eX<ecRq-!FcjA*i$(822R7rC6bI7U^dBHVIW@;Xy> za@-h;aTr~owsm6_fe(4ZgOr7uGEVLSG9IUfosHd$-R`ciN&_#~fZ?}H^N6QGmKZiX z#t}a+Ih$^-iz;$p%x&xCudyvg4z+lH+p9&)D~;~0W1m_*^}aQIhtq>MW$CCTHf=8B z*e@wxu}B5+7O&ubQF<<3j;k};?$hc8Ojd_e?u1KE*@i7Wtm*DL%O0L@dE)e+rhtJ; zyDr7~I7A(uzG4gg9WuiR5Z9Dpm;^|ImHQY}63`Neo?yfiTY>w<^a{VqNW8~+esr|& zov2^-o;aT^LiquAG@$k$l%0{W{#jau5J!}sn2;c^S?i3oX1tzghP zkJUNofM5Upp4b6N24djxF%=%cmGgA7w0stO9_1>hT!5%odyMfvJCCqZmJif(EPf)M z;bx5ZNmlEx^*XGa=3i#!4{YUsTl6>QlWCeDHs+fy!y5mmbso&UeBMY%fR_@*GPF57 z3QX?VMtlEnJ3f%GUOzrn54~Js=^AT1cW9><0Pz1^T%Y=KHhwCR!)%Q7x2mel2Mdy! zOzpZx$p3BJPn#dn|GUPKk@1B5FP!UtWy_tA-OF>*{sjooklp`JWbA*}&>71Z2;xdJ zNp}0ktS+V!%=F)e`vZyicZ^f8|E4p!&yVjvzmg%KGKz&%jf8S${u|*bfG|XM{|}iw zG%TNN8_K3tl6(J8iKu~)9;&sZ*=o12K&t4;{eInVxA&HV`ar>t7pY zKbE0TwR4E;`g08h1Aw{~EU!yYsBVYI$Ou#Z>CUIFED3?Pye@3Ws9g@a*ea8Hct05_6k({cx8?H0JU9KDyu(SHHi@ zxzXmrpGFTOTn9bb%AXkTQg0YK45IhdjNiLc?NHdC1m{*c$Y=Anw5nz6^upnT2V9aq zOb{PEP;O$yqKecp{NINEWL${rc)!lIs_j06Gc_T-gs$C&dKs^4%}+iiFPuNLV^D>Y3udM`{S*>2A(NI4`{@D+YdurXVQ#PFYxauLAu zxGqkgf2jk{HxlXwN@8}*$&BlQR`Vn0owjK{MKe$ zdZJXH6;iEtknj;FaxC*om&HUn1C*cFriCUbF)`EAxTS|p>Z-2Gg+&4aE;iWMu-7{D zlm0VJmqroPyMq5m@IFKj*B3}Yy7x!z@^OU3GsTk+9smTZk-wLRS*$`J1;)aOpRxB4 zYqU@N#MXz7V;IWzh(PM{T&diC`m-ZU+ZBy?KkjwA$@o*l2FSi8nLjkzaDUKh{T?v~k!A!GkBquXyB%+Z42N7{i$Zc6q2#8T{ z>|qqqs#(s(Na5k|-5^pk>jp1IU)lLO?~ZUEZ)?xg_;~8gR5xHz%fG^~ZA{6;t(P!2zIjfsU8?r$8Tp(&%PAqG;{Cu+&_D~! zAr~c$TU)ZB7`__kb55}>F_I|sTAeu%Hj^V1;S=1P`_nqv(A_z(-*4_* zO$&bo*rThBB%Sgbpev-O_pOpLERw;+;3qCeqjiY@A)r`5-1)0QniUe!rDJjz7WD*sbg#~ML^gu2|3eT6=5Q) zO3Uc=o%D*0LroauLIVI890a9`D%A%Z0-m!!#_}YBp`-<@OGm%~Eh3#>IdGH(0vDj` z@XL0S0RBNvPz=R6{isWh9&<9YOl@1IS`_-~bV0e&iQVDb8WQcJz5;4rLXiW1$bzT7 z#4T$>KWAZ;uJ&N(Q+qOx>7W#0$E?ihZb0X!f0ie)VWR#zG+vhwCe!Mewi~C3oGRdx z6vRfmIS$vf4OEr%16k@XYrJhI8c>l}JPIKo>&pjpFZ2|GL|QTY(iaSk71kr2iuX-M zVLH$Mb9dV*>4aQW`#D6;Iq1`fD>2-CK1C%V0qgkv$ zFc>1Mrjr$b6AQSL5iF|+$>-#;AD;w4w^bvbTkRMV8$5xy z2iP3m_P;=de`j%y0rohL7>*c(Iy-8#(VOP%{G87G@Pv=-guccGEcS0~oZe*P`)5&g zI%6VX6oa^6DJ3Ed@Zf6IW5LnqVX&FQn8+&AE;BGv(7We&zo{Ww#i&R6}iRqj-B{4P!q=yz1^Nv%shWgo@rDqA>nzyu{F-hg0Tw@m#km zvC5+R3!&Piva%Fg8P(i7pCWSU!fUZDA>(8<2ON?bOyWT}TU3x=t<)f!ulv@`z> zi6;HeFg2fZqJ^3?PUV@QOStJ$qsJT_g)f$wxf+HfoGU&ZSwcjF-Jl3B7hF` zMFH%7foM27w&TILy)5ZHSF7`ug!Pq!)uOucGL3{kIM4-YQ7D%lTnlxT{;(Zt~V$0 z+|pyECz&h}GoCpd;WS%5_`@h1RISM@>`OdeXpHRVG2fp5l1)OG{5_f6y%phRvdkDyxR|($w)p7Sg%|dfeVvL`z@kWxX9%?{e&~$ z!>{yl@ap@|pAmJ$yDP-X%*R8drK!o$TCaSMC!_JDj{SkHCBEk)PLwHnal!LmNOz}{ zLS7hsx8N^-)*;d|z4-N7OBey*BE|BGl(6qF2sow1ZN@Xp`9G~ zMGnq&gNV=LGLMQVq4%Fgms7-iA<1PvSLWjw4CS@yfY7xF&Wp6rCSk<%^|}+#%RT93 zwdIRq#qS_jPVHGB*Bdm=%-Fv0geeTCWx3`W(PW2?6?}(qSk(;z)d*tio-X;7VH+&$ z*FAbk@IdQuV_>^fqNP7mrEB8c&0uSk<^*HgoDkq@UY(^sQdB?F;BI+2(*Kqi`mZ%q z6-hkpc_*;-q=K?>ont*R0jCHw1kT8xQ*ZQpA!IG?w4z_uiORgMiqo71Fc^ZC zhgytdIdJ1>G87tIe#j29BJQ|@0-=`=s>2$Y7{KLz^Pg3XKJ`U;gc^6V_MYb;0x0)) z>={#mi3&|pUYoTK_Wk?w8b9KFb=w!Iy79Xj5LH+WB1lPz`XbcZH3U6)x%)XK^nbJ5 zt{ylLh(_ALvU8N3Rk8<0Yn5}-e z95|iu8I{~{1E?zJsB!`J>vjWc6%PW`=nd1CGT<~bEbyao?<**u7k0z#+wlSid0xtl zoyjCcP-IkJC`W_Qt-6T*?d_hS$ggglMhc?$p&nIn0iRCVqgZH|`hx2?t+ng_QJLJB zXspJ$po!^oQ77gs!4|fao^XL)gJNz?FSBl)`cV!giT>4GRu z=a#ve8Od*XvWB`Z?O@^_pN*zsa87uw5WG|BUtqM7JIOpo>X`y^exp|@CGozQQb4N0XWN5!sVi5+yLSf!>(#4Gq5>uQZ=>TF zU@A&IZxqfXd`c{Q*aAtzeLX_Lyi;CLVIVuHxCC!$q&4784F$E7pMj+AGqNhz$E7P1 z&4IW0JKFihMY&nC7QPWQ%PDp{X-p2cEG^V51RZ2_0u5jK#m(ES$irD{*Sn}F7%;ZI ztpUgF)5SwPNLZfYf;~uq>pyKru##cuwm=6)m%X@8xn=kE|19FmcZX^Ul&5VWcr)H@ zp|;(gr@7ggVy9+I;R*52^l!7-t#>q!L-+_q-}jyo$jFs5$wCUFk{$#mL5JY;Spy>d&Jwd@*Rg8VC`D@G)nt}v*;JdOTn8%ndA zIjVfV^8h$}Zb?xmJ=Y5P7FnNWZ_YYpg!_g`Fu&E8MU0P9-$D<_tAten%fKtiMpI_S z+if8<;_7d3@o`HotC$KV%WhJ2Or3{oaTrDv_Q_lMcqJ_2?MTod_57L%C?q9IFaRi= z9+PaEF;S>!_RCnI4+L{Z?6hkXCNX?mWb+dps_j1sThdO|WfpS_ zXfq>oT8Sg368etqNG+ls_xMx_kq3+l5YMhS@Q}6r!YJ^5Lfi(tyoXkv-rt+LLBm?@ zK~iRht6iuLFcp~CNo3aP=;rc_;|*=%fql;gql3!VG-AYVnx>tn1BdBsC;Pp8^^KrH zmVl>+B!Wf>1ts4z@)${lRtmR!h?T>9Hi!Kg9*@DC&SSO5Cc!X7$C|0&*VJ^WzRO+| zM#%7~ks&0BT>tZM3c>a6NR#F2eADVWh1>d_z=XwxtDBX$+XJbBfJ?!^YFl^G#&s!k zm(&hMPkYQt(1p?LMp9U*@+;)jtUu&BX|rO>+q4GbgoPVz%-^$%OOj_13+2^dsHb2C zI|GYJhSLQJSbqY-Sho~F*T0p#iTH6?6}P#rou{cj%w79>x0bG&F$VSewEM*&Mw6>C zWy8ul=+3c~!*9)Uk&P0;rGj*m!^*W(s!a~Kt>@a%(|8yMLSaDk72KQdD>I%@IgKB0mQS-S zz_s>bS-%E(>5-N_&_y?HnKatih-J-~rAO38{32D$G+SSk&Q=VxkWyS8>ypRQGDi(N znU^d_Gw)EB{`m+e!#r*UD|Jd0U&f zBjvG}^ktp41lhQv?{<%^$#{^yi2M^==ghLMd%W9*6}V3Ms`o;PHPoZsBS$f<<>AA* zC6%*X{Hs3oS3Z|WFfjxq@pV1=a5uevemTv=$ZqvZ=^C>Y7rym9XY}#I!H2?Z@;X{I@ zR7~@ZefYt7N@L3KYbQ!fU8;?-T9tsDIAJPuioz ztCjq$H)Vn3SuCsxi$0NXQM`C#s0n2G5Q@w$+zA7IDI>cC4usjTThUPPnd*(YQURGd zyzB_bJfFPUuFllm4nQoYXDpM|)aEkpGkj9qBEBEoJ{`69&EjSuiiDfYq$+(Z6CwPa z>2;8_3B;0N@?3hJsi6w7_dB{lPKz^sD5R#@$uIs5TzQL&i5T){#O&z0VUxpFP;WQO z)laVgSgDEXQdFQoT@?wu4@5OAPK4bNDp#hF&u1!u2qPi(<39syWMmE5_xrheO>TrES%e`zLMxt78ry>etjvbM zz<4A>Utw%JSktxd-EjF8A5pe^xlVE|XX_Mqs)itt>uoA`@RESZ{F4NGUz$>Co_Yjl zB-BKr7_#SnYjvyoi_rA_T!r|gobe_Aocti%u{^hZBhEs&muP2r!bwq<$%S3p$knfM z((Z?;>rQ0XdHOzh*&UC6{_(YR0SEegh+Y{LwC?Ara0 ztLL-{$O=z1qZ=W&C@J3+K`A3&IXicW>NL)|$?m<`9CMq$)y!;w)jRwB(-CN;`54(D zrFFgX0IsenSK(fsh@M8_1a?47pRCBJ)XaVPf zpUCPHvR?NXSo?PN$gAr5;^wTcO($-)U4v)MOHhD=Oaj?8GE(3hyQ|4d=60*s+LOAV zau%3xC%#QrO^VOz7OigoT)9{gP-TX0ccBoWAsR;3SrNX(JKIUz>BIeUvT>_x3S;J{ zl76kl>wVolBO4SKzK?y73!OnB7+o$Pv=wc=o1L-@MP@#C6SFe*HVDh2!5~>p+sfIp(26 zT(NYnO3L^MLEVwSD*%stpNd3fpC&*z$gi;@#8A1~Ypde|RBZjWW#bG03upm%^n6Xy z4Z2`%LD^ZlB5Rnj2-vQEI4-he;(CjdL@)o#JXE{l;$%_HQOkG1Y&kT^cF(_${1MYk z&y8!e`Bty#EFiyVr4D-?GZsm@WZH>Um)M18;rI^yM%#;Pv_oUqb*-M@e)R3su@{ko z#hRFPlGW)kReRN8Ev-X=_OVpsYFPD)rMo4Vy>k4=XrgLD#_g%dctJ4x4l-~}SDANO-tfdB3^}AlyRiKJpH?jJ3UrY>yvf)OL_aYTgUFAZX zKJkk=eeHcRz7>oPctg36$^RPQGSrVVs}}>dYeP(0d8FW;J`B)}LVHD3OYPD^Fnb9J z3t?${S*Z8)KA2_nA5Mq#-kNJ$OKlm!%Rd3;jdy0+v@ylJ$LfM?Z$GJvL>4mJ5~k3# zJ$*UhZcFzWTK*e=cGjMo*m=rnx^JU^Cxv!nq#1aqaK**M*xur->yur&_Wylzx+K?L zV#hV?QSV*`7ey}|g|v-Xe3S!B^*LkkW%$W=wnaW9T@4JHDM)3%gLD1Rq-sGp8-97J zfcV%>f*^?gZE!&eDOm z1yUu;NXz~?@&W+pCdMVEEXgbZMG-V2EKqE8I(m8IKD<2L0)Q@?`aE<_-O?YnzTN-; zJq)gIap$zzLLDrC0V5A~Fk#1Le(Uw*K@0g^Ki{h*JTEHK*JATJ%QTPx0z4F4P-(ag zJ_EPOm@~uc3jN2~o3#c!GlSvM9V%|YLjCNf?2jZochWX9!s>nA9B{Jgwbh`n-XU@t zZ^5^)03sA5rOhj`QBzWeYM-C6yhW*QopDi^d+^=m-heI@?tHj)zrMv*J+wbsoM%PQ z#l;01S&CQlVI4ReY0Zkwu-IMiKWiw*x|d2$iAKRwVK?-K%2emtA3 z&>? z=T1pJ2j9pkcIcR|zxlZ(f#0OjT-b%19>&%CgcYU!=)PTAdfD6Wk!?X;`|uW1V6579$q$sww{g=3rbao?%1&A9$at+B3xzX9k0IFYW z`Ae{|NyCQA1mC~!{RX&xjzg98C!5l}O~#zH@I05!%k6OF+^j*9@H3rVb>s)U`Ls*qe096A(HHnH3CBCD}! z_$saD^*)92O|=%^R*bafml@Wt|G=Nrjxpr9-O>i3O%uCXeWj3PgAn;A@3iNC^N;h| z;QFdMJJg)i)@G-mGz;b${4WUwEBjVh7T3Q2b)w6pfC%2T#=gI`5l$u}@cpj7#UDNjKgKyiUZAb{*ANe2yXOk(rGsueS7OXDT znqmI<@q^L`qk{6)rtHk@rV$b^`~>!KFk(V^Vij8bdQb*TpQ!oVW|_Vq!uj*PcSdlx zd}vVwadQj(n+dD;_2InvpI>kIU60XpW~nNlmcuXxWdaBtLy;8WgAuJ-Cm5DUKgm*T z^Oz9L{f<0}WzaGCxnl{)aOPMnZLVWKJoA^(D1o@iaW(o01hKD@a@a!4Caj*eyep6{@#GQkF*V04WJ>y(84g4|Ff?%WqR%^7IcSlAO2 z;fd4}j$2BdbOI#QU?%q$ni!0=@S!mOT@(1%jm!P~QEjuPriRu1-1yd0n8941AQ+D~ zHb%WtH;P&|@i60BG8~C(Q~#@|?B)YCCFKW}Ej-Sc#C%8nckolp5V6Y4PYM%x1zX`6 zySYG7)~3dH@WW=Sj`?B# zByDwtzj0dmdwjQ|Dms|q4FUNjj-;{bOGyGr8BIpo+%L-BLAzffiaAv5jprbI`SPV~ zFL!flfWc&hKJQO?XU*~Z6^S}zDs1eD>UZF@AAFl8e9pXE3Y?km!19{pW*P8hGDM7U zZt{8%#%IC*By_23Y-*Ax9d&x{*%e3f z{h9KpMm;Zlg+eNJF6^&(G9J2o8loMH^cI`^va?aquA;gXA}x=2^ide_K%sm-I0uA2 z2xgbXs2S!@cZaW&A^#)~aJEWF!|mp>`q@(&OYWyUB=eODu8zA+`L??u4)cXyao<6l zg`lwQ5u%N9^~^q=tPT?)?~8Utw)@66ars!vCX@`Tr8Cz3qdGD!I7L2`4A-_XsBC1I zrlTn_I<1pA#i;~4!Qd@48E|OP=Ql{-%QV`tf$TMK97JYW0CZ9*F&bsy%kM5EAmLj$ zVUq@&Qya1~ zY41m>M}-kEDuko1>y1qYJSFSx>oZ(B!CAYKQkjQ$F)l78Knkp8bk)N2`jvlN^%fW{ z#eA-c%+c|j)oJvlc|+$-(Mo(=7(myH7=ZQ+UhN>O?0m#NKSwQ_g=51k9zAv**7EGZ z%b05iCo=kL>sy9PKBr4yyKg*hKhMw4PvQI0S`P(}2An+y8Y|a(4S$q`1_11M={X}(_FVrBmyF8XY-~5LZNvTMGX{$h09;M}=MsMDvF%;oWxuTFAlY>O?JQg!Q< z#U%-PPZMUk{bqWPp(9!iMvwg?Zabi{vn4RhOkPDvDUcehww^}r)BSU=_N=c_WUanY zxcAT<^$awY2agN5Fu1`6RY0fri{wV20;ZYMel7BiK6ME8!NoT)MXr%z!HiaeN5I#Z z*qxutKz6E1Rm=%U7@947>7U^r!SxG(tki`Mg%&J`hgViTAbIc1fc*vq&-_ z{II=+feE3cM3YDIXZt293K#}tWVq+Doa)*uyzHyNv#NFh(ace{Z<-K`LT*ogjb@xq zEP|>myN%_tPEjpqD@^2DX{o?N78bRO4j12HJ;-()p_f9MO^&BhZdZ2$<+F|#C~RiO zlyb?O=sem2ufRJBy?^F?)_eTSj4qo*SnbxnSn0rb^YU-4^Pj&`bSNgXqqK;*Iq`- zS<5furQ||096swEB=Jm|4Tt@TRxi`7bEsx(x(}CKy?8j=OA^uFtnAd<`T2)O|7USs zItKEREkMwgN05!os`a}BV6h#9@`ML$zr8NhJ6%3iR#Su?3AkvsZ^1d(40InFMj0lg zJD}dja*I)<1G{8+cz6LM@Eu;WY8Y;rjly2s8U1RSi|Fxx$xzug%%Lcm0NVea78jR* zDmCxw@iQi=a>4g79XgnNPQ~zWzCEtPUq4Wn;#AGSwA>O}82&F zF{%zECA6+R;=OP0)PJ4)mYf|WjZIHCkD|&AXW|e55c3p%{!Phw%oY7qK+OsxmPpFRdSD6 zXjh%VZX*v0zFKqUQ|91%BVpG>bhG4ubAP=beSdY33pajoVPU%(^YsD)Y1m$zV_WFws9SkYnmdDg3l9 z;4E?{h}L}U0^c`Yfz`|UB^29ynHInFsXn~)Jg2EMrCZ9A+3tW>&$~K~=1@2aQS5}0 z2`Q2W+CGkf@?k{bjgUv&10Hr0^oi>Cog3#5OYK$y=`$xMI|-A$%>|}kUZRADQ#xT` zlfFSXFpKQ2n(ge5B<-6)>@?~@qu9O4EgHnsnc{h`cTEGmJzl)s9ZMKwi=Iyn3#IEB z)(FIeop^b2xg&VywySU99`*b>Spij@Y*Xdk&b4})W($AJ?XZnvi>uyuRZjE{ z`c%px^{_zn$$8jZQSfYYGHxIt6;1HBe(K^@WF#NwU9E#V=xYj5k8Pt=KiS;zCEBoF z6K+Xzp-VaLds^zA<0>j@>iHHzBXLiX?GZggCYfz}7yk6!#a01b?`!Njn`QPIT*{Kc z%^R(4>&ldLeZ)Zw&iAx^pe^W*lU(&IW)j$8Q1|dj^dGQVnRH$-`gp61I>QYR?kup^ zI`v+}kJkd#Im`4gzgy0VTkb13*g9++YgG*N1&sFHN2MvLr2A~%QQO>*rT=N9H59Zh zTdX21OkY|YqGh=NVe;0Kr*hhcK6$lhwQp@T%VicS94{g$3f=z$_e4nMi75H;H~F%q zwGS!A7(Rua&A47?z3S!56ZFZUV!21Wise=3lgaP^XIVl~#w&wy;t( zuGlup^uHwyiUj~&YzOx5YR}+RV>If_3Ds;y{Kx9Lzfxh4Eynp2#01g{uXrehM9P3q zyxhx{=|=ndxfrT!wfJ3*mx4$ICd(+em~i-{_(^g{*Bhidd=N{|u4O@|=Vp0TU6bSz z57V{#TwBamT)MCmuBOf7;vyjA_wE-4q_jdGzt&?Qsz~_@1>t(0RBFDVtw$Wv^2ebZ z3QP?X9Zj%+rtWV?*_kZZnR>A#^)7^@Os56qzy-Gp8xTT}bTl{`w$4H4Q(sO@p;;B` z4aOT)(bvM@3aPrm7ArH0i zdM}(m^Id)QqaN#gq{mycM)-_g|Mjg_H(`Uy?PRKP}jTwE$PiV8V&HBABl zJ|2jkD3~mUfju#Fch|&pi-Lk8k;j=P*HJ}7!yFNLD8gDB_s)OD%l7jM-Mu|8#-Cz? zg}!>q7~dzV$)3*F>}bk6=WtFtX-<#xFwqTH=0hhro0Tl~(kHGs0@Tf8NPwZ14chHZ zxtsCs*6#G_?es}44|7iuwvWCnsP=SqOGpXiemQc}!xirU(`j5ExMH)n+=f7a9$gXH zg>RUjZqRSX#wm7{)scXXtb2;c~?V}*l#+B%tojPA|!MoTFBt7qQdQ)5%5C~Hz*d=V#=q>I_^C9w*=hNhz z{SyMg*`JZ!FO$73D=XcoRF2~(;idLRn&qo}G(q&r1pl79`2L=|;6A#fKfFSEmT(BH zjT_&Bp4Mo0b#lDHLTE}sDK?`8{o;whFKec6?bAeT=f9fs?8WZHq&dGDdI~(={DdSB9cm9 zu{~eU)^EA;cG)`AOefXgWi(MxA;e*F+>41!|r2VcvOdg zdDHOxMsK6Gdar2}BD^~>l&|33+)1s1b4HYznxsN?lC}Ka+S=#)R_n5UGSE+W_(gH9$VdxBv5&Zd)DPdx=77?YkO>fN_ z6@m%116)r$0UHfveM2@1iVqEaEh<@uzLj^#8{vcJfFnHK)EH@sHRZ=xoMcfhR(k_lvV*Mj*``7WUIE?%ep;1SgB@{0gS2gZX6hN zZOyLldtZU+5wX7F$%RV)jT_>nWyYAO^y_=H5S!_EvM~#}!}Qy5X^95l;9a-hAx8&^ z2J64r-0uBkPS@2t4SJ^fG~&1MDv!DfV+2DVbV;Mbz}%q1;<;vrH{MTf1h!2_MvQUs zuZx{vT_IVX0da;M@LaCp*^o)H;2BRF#kQAwk4vB&-h3;!k%=WzxE%3Ru8>B%0fL(K zLbRZd8)2z*M-VZ&8(a-c!#7G7&-6)QnCyayto_0kx>B}uDjD+jPoKbiXKh@ek@{s( z%0JV?(w3?A>P=&Mj&6XkiAIe z?W(%m$`8ymP}fW|&e_>T%vuLezrg`pTMjw4^4+}O2_z*D`b*Ne_10y{NzBgKLM;+| zMNV4U9)^LprvmAY1?qM07!?@y|KQSS_CPFK9&j;(p7Z}Zia|T-1D{&+TY)y86i1TJabPlv%jOGd2xb|I#6{2Jw zep<_sTroi&*IngwTKYds;xlE&oPk98%Q3wo_ZA~^N_4GPst^}z+N!K5j8EqMwyb#c zzgp5j9jl*4Zf{ct9AFG920ghMr&e9ZOcAG(AuMa>4;{TF=66n_HND(95j765c4eGp zfdZqz-2Mc>Ae%nM)QqN|F48xZO!&Ckq@%g zlU_}C8NUb5LXxbHb^Q!tV1O8(Q`7?M!C*t4DyKe9&+B!nrlYHss=QNuQf)co3O(Oy zoE?J^X60i|I~q&|)6wq1lg>UB#m+#Mr*DycR_cZeWhn$_(?B=tIb@Ak14&3@m=1?< z_xDDT7{vjm{suUzgat3&hd5g1>E^xeqt~SOKuGAnSzXGKaq$jAYmIt>j$W7Dm0~_i zm{H+ZMDA7u+s1H8AHfPQqgO7K1VKC%RuNlp?!Th9-m9^c7(=+2G{N1TKr*`xr*Lxv z0z1GfK)plDiYZ5VhvT#hNLB5)=UkS4C!jW#7P_MQ$U}t8y?&8p_D*28%lAP9lvN4w zjkd%HZ-EkT_z;IQQ$avmOua7~6nYQzqI%!z$+U4jDGuMQkwRGwaZP<(kiQWVpZ#Xg zGnf>4JeB&Ye9v8+Q+`m*-uVv&7e2b>5L@nqoomn0pKQs+`*4)BNpv31*YzDFopSdG z5e1P0_jgohqxCx2bMOMc9Z*!qzWd3Y=mA>2=-%i8bt7n<2-{Nx&snJ(`9@YJNtl4l z#=PQo7clln6^}UGmdE-WX>6uNzQ_e2m>TV$9PPFpp{yZf_EK$_EWowmBy{}x_pQYr z_FtMJf0%m?$)7MQ73jN~F*|~&<2(Cj$-P3GYo%BBIE>iNz~pQOd(rLPK+4?k&sjY9 zdSxu4AZ~z>JrDY|?c+qJkyaqSr(evDM|qmJ7JI?!nYxUTdkb+u*2CM1k44mG}9M+kMTo+CO8eNA?04xz?;o*8KLP zL51tT8p1Sg9?cm~gDAV##G(9_Gz~k8Y^%ZDA04J78!zkzDmMlUCjCXK4Y%FOQiR^J z=uV*BIr$9C^)0SWEA&|Tc>1Zsriu9Y6w-Z8z{x_7JkYzLzK=6Bde5v)#PANHNiQDc^S7bfDXK8*K&CiRA0~R$IOn7!bZdhy#w*f0U#HEQy{lH&n}XEh*KT@j zGj-40_U~#i5Ozduu$ty;Pl(ZdQ1#)*XHJeGblNspC__vZvLPY!IY$_V^Zt;8D*EQj zb92zE6-;OwPB4a&o22}rv;Pob3c8@T%s6)2s$}yLuZHIU@>FVH;WDQA61JjLS^hR; zTp)_*@JfAWwLvu8a6Q0ZR8N#+3o^DQzN1atMaHBsPYoBEcc;7lZN`r0`=YWDX|xk6 zo<7@-acGD$oqW&hzG|`yFw&Rxz-T)MypmdEH90ioO|6%nd{4~HtVwx2W;nc<9@L8= zG|uTt)<3!kk5k4m!j}jgvL1{V!M&S^ccUO;Tr_@XtG{Xck8M$sQs9J0w3N&Hqt!$U zj;aq#IG==`q6&lSLS%oEyWoZHIU{=Cp2;sQS7MaSALVLSE1Z08(#9UMnL(YQD+oB$oTbYgDuC> z!}NMyQm!;k$#z`gg6LHLONm#THVs%Vb{YYiwE%;`rNjKc4R_7QR7!wmq-()r%_E;OAVoT#5A8jP91rZRq!{G@6#W!IsP%dW3pj%88<7F8eyW)M|V!wBDWVv8z9 z)}OM9L9HN&RA3)#qf6T$y6W#lcJ=*{5D{5sJy_1D0VS3$SeU3$R(ZehhgEUvh3TlC zKNlj`M5CEaSz71Yh}rz0Nkr^7@KhQ+%80ETRKQDOKHl^uvh8T@XJ+vF;H?iOaYqz! zU;uU^l;juwR-VR%;XaSuTHKCqx6xQ<_&}x?HtfNZd3~Q#bc5fe)Zu7Z5c5O?hJ~c( zg<9dW>MnS@tKxQgG@((w7ziR86$9G$4Gz+YddDG1fu+}e1A^C-n=^Mp)_h`of;Gi3 zi5kn0M_6i+$piO&set%o9e2{{e5ZQ65B?OqSL?vD?w2ecdfqsh?(2q}OG!;?O4&(= z*Jebo`=(?zctbobvKFq!2e+i@>WD2C;r#5hFcO|BPbd#ui#iFkcz1Zwv@=pYs2V@s zP>f7%Na;fbU^3c%5JzLjY{Vfeq+MEM>sfQA;8sv5((uF5SqYfpq$O!4KD{urTlY}} z8=aw$c|y6`ZWf7}78*r=sNkT+F#5v5LVuEAeiPuv9<%+svqdSL z_-HWwhC^6~L;<@&AP-B}=XHZ=6t>ok=ekeC%7-=ObrN#?7*yn}K;u?_zC0O9oi<*x zeQQQN%|I2~CM+AFbINK!DeYq@vRx+vVA-Dh*d6K)-SJ^r%b19WWkPsDUikUrfQeCD zOpVO?SX=ICn@aHM5Ue(Tx2rO?zE-&-5n~rPsbsu6#6j zzS92Br0m;{l!$;7=Cpg!_ItvJw-4*?D-wPL)IrUeXT7dl$sG;f67~0i@`!o&hLBIs zO2<~0ld@`pZ*<*XWVJs~>Uv+ktwT349F8G3nrl25_u2*B?oT$JUG0#s9+*{1!~}ES z-hkg&2hUm{s^`kvU|LSgi+i?_f4;al-LEn`)ioA6>42gMds`7e$gf{9+c||U3FN%H zB+{`T;}~0?ywUBJIcW>76!u15@xDcYc%LbxPv5@d8ISm4J&wYyKa#orIN5Os3ChrA z;jDi_RCoHW*_7!$0FbwqGP@p?lnDZYC>Qw-`K$y)+%}}0BZQz>{!0wh>I~#EjS@(_ z*4O>!e|ke+s^g$_NWm+LWJMij6`pk;u&hsFLV7j;uA=D3Ab_N*^440?ikabLwD_xV zp4IrmtX?5Z;V>ZVhR)W@fS2107nGJ2osw6O{V^RzG%M>5wvn(u!Z$v0HDs1*!}q$& zTEBLzY3hLWB4DmH>{p_zQMe#3|J)I}6Tq``pciWv;KV7D^ z+X%5+=~9`^2*>rQ>iw?RTz48?E?Rr_F*HKr%V(QpUBvdvHsX`ebBb>+9S{4mw~{~c z?1Z?nuId92Tji^LIGvkUk)SIAl^gM{!E3cQD?P4oT`9#C_cw~#5W?#79aZG~o`U-- zZ&@0Y-sq*qo$zWaMP?Kfc}^Q8WVTG>sZuMMocVu}Ry;%?UQ8_+itFt>x?M6S9ez9& z$1jsUxSn<7Lvi?u-HV(B*Ykf#%j;hdf3$_5!hqM*o|m7EVWH=+0G^dDb-R@)v5bSz z$9qiVpC#!3v8IdsYjboeog7BGW(Kv{F<|WvCZ6r~g;%vs#cnt3p3vKXru&5fR*%xd zQLic_I~8qvOBv3Omqy)1Z|1NU(MT5?>w~5HJ>(V)Wx%HNzAS)Zpku7rjN}c+{l?GH z^6?5Oqp5K;TvL4U`G;1IuC22!Jcfl0&6Giu`9awG zNkwiM`!zCac;o6tqm1E1qM#dpW*CGE&J@tnyR^+Sn6~pV%xA$qS`2Qim6yXeEoTvR z43I7|V5){{IAAI2qrds!226Q7UQg;A-0vP7@JcSg9K()Cn%mQ`3qoH4~S0^q;)n@TDeh5 zsL6Zx$7#GnE%GbT_SUw(;y2+#^u>+Kr}l|r8`(N*B~(iXnm>KGK*icr($QqMrns2p zd8xIu=Fic=yDO#0Ex(M7&WrXw#@Iw+#Icv#PIwG2mS-E(^4pVh&RJ6ujE|QM&^PVt zCBYM5nD|=@eFW_0|J*l}y_pq(0Qv(*jwJVQD^Gp$0C~JWhO8gTux~Ag(GIz_43%Is zRn)T*SuYUAdF}9|6*L2s%4y<`tT9Y>On>2jF#nU?CR=Bpg1=~jRdJlH4=%}1Zv8-h zrPWw{t@t3#RRug^H>Ty5`kc;N5a%BHAALPHoTZxU*!$37$;(@1voh6pppx7_dRaj# zt)eg&Jm0v{r>AD9s9SBmHrLxrgQgR*0uaqs#W=@LfA70pqZ0K-L=}f;xzmedi@J4em%-SHqj8jh@=F$ElXFX}A5^t#P7^ zAoZ+Pf={_z1U`S< zv+={A)d>bxTKTt4gV_uYRX1BaR__Wc#S8Dnpm+DnyO1N!x+4!1h2}?uVck|dsZjzA z*H2+qK`M(}(()#lkj|}_&ig{IVID5;T|#kkO^IF)Iu#3^d|Sj$bQ>~hs_BT1JR+{1 zoI#gRu!`CoWQnxowv5s(%y_larsyy)@2c5;%hCz;5ES;LMw3rZyUDDy%GP{{A1|#4 zFSZj3R6#})ue>1Z?$t)HVQE5ucqMeTwB65`68OAXSjHssJC9t z*y#^cXKauHeLh9iy(>g8FC$u5c0iWw#t1Nr|q5lpgamvi^Cq z;3^wrv9nP}L-fZf+`!dCiK+sL$e9oQM&$kDTe5A4OPWk~_0G-sSKpuE4jCE0MZKgWgQF|rtk0~b(B z0c0y00e-@Z53rwM4lhsPD|&n4d?UU)EoS@pxg<*Mk+~byP$UB~jr+JSfZVqSG83tH(HR@oc$2=#YPIM!UCdieKl4o}Z9t zKGDy$yKs9G36a~_-I5;@kE7j53HSE*QRO>WX}gT&xt)jG<_bnLPEvytIL4W~V}>aa z$dqp)Ws8Eia}R;n9SjrgnTz}iPq{+5&>>OJ*^hTyaK)LsdI}cocEy7zvhstjXz?<= zCPcVMe_`elpgev{UT5pGINis^z^_2TU4d+t7Ou)SX>Jq+x&ml|wW#SqxuaNL)?5a{ z*gNS*o1gHPeB?!@j8V8hh@Bz(e_ekYMS(K>FR41AxcdX@Mo+FGZ(q{n2KI^|Wv=6o zl!TDLK#c=`c(B=>j*+p>A7M+18#&xl?0xU7Z_A221!l~~y9}ZN9c_m5*SjL@=q(i> zR>{161LDN7?{Y7>YTmRrpM`y>wIq*C5|S}qWlUXE{V;~ag8fzG9NKg78g6-iym0?I zovqOcW)guP;J!MT!UYG(#Uz3zH|Ej?GKyN@GW|s*tRNn(s{^G&g(i|3T{+de>X5eE z9p|}n@ua^1={LxmUkP$4zGMRKpWUF`lE~*UcgbLhm#A*TM;!&3W^Tu4crqLrUHb~b z`>(q`^fwd^|IK&Nj;cW`F7ah%+P}OpRRct5$7gtl_g?5QCw)bRWAFJB;ox5uFjLwH zQtL9AdjcFL4ONB1lHo)%Tfl!e-FdOU)AM%UjByneCR(!RB~kN*i#6tH8@53e;Ny&kie_NGw9v;k= zu5-;fbfA021{+Bs{gI8|QquJKME=N7c=PZ$;{?Ba-usMegl}Ui>{Zz9Y?)&_0{T^f zfAPQ095w0m#m2FZN}m;IkyF-buwJv$!QtFn(f=j___vMqC6KmEVnDgMl-Ka07P*ep zlexdo^9BFpN!h(*05et2My#(JGR(nmQYHJ>$;>sPSpc)%t0VCkD@h>9qoAUeLqgCT z;b_FTP>3(rw}L@EeJP(fJiMbbUHp7-+3q{wZ}1j)wGHi-qY>UyQf5t;LC4QeKV+ZR zeC~TTNil7>@_uh%6Z=ju1LL{43SQGPGk(!Cpx)q4Du09%%wo<2dMnlO?(MpjDBkxR zs;#O;%9v}OqxElbGF^W`!V@Uw&ap4h@8C0{NN}*LdBn6f*j=4FJQ6)g!UTO#c4E#K zu5nZI;XdiP`o;$qw84MLMR?jK7=u~-9x-3X!ZG0Ehdu-zpne`~5XIsZI~EJDPX<^E z9&ifg43)!X4#7xbjL+Rt+hdm%j_!bG8&%b6hQkxJ{!eR|@c&v3BFPntGVh|gSI2-q zg$c#t**;?Irxan?E#_ zV8tfoUNI34$8KKykZ>k}qxy1wqFMhW$=&!r!0~2WxPFAb=OqjL>+YY-|M&j4u`I>T z%Mst%;1l@m#6KapUfPL{yY>l;g&L;tU&n_AwrvQ8v|dd zADRNL;3i|kWYue?OmlE#q~8BxBJ)hA)kP`-p9u>MiAv?fsSNmEZJ7zb>X7FZpiGU2 z60~~eHr{Bb_nwAEeD*s7!AiMSV*t{B?1bIke`&czZ{zpO4Gj&;vOS}SxZwDDxa{Bm zNc)!f{yWP{HjR}aM=l$i@ru%LEO}Yxe^P6zXa6!uR1pU+xtIYb5fXoM&Z=>2f5~d9 zQmh33@+DNO|Go0}Yd8ymwIAZuQuCX^k(_`dVW-uypSSgG!Dd~+SeaaF3b9RYY}@~d6p$< zEX#-Ua2XnH1>}FNp%^_K<8Y&g%1>I5YFzqH{pqmiDFft9sqdxPMFcHq{Xp%wxPOOb zTILF45CAlIgQF(c35y zB?zKK3ll^ejOg7Eo#@@f=yfo9nZfKk{Jw8@|J>c@+56A%miyjw?&&W} zE*%}sTr78X_M+6S!O`>sFyNtha*x05b}1uHgD&FA2|v*ya8P9ws--RIH}Lb7Hd+Y( zOw0Q!L=mR7{VUx^%v$clCr>OKvo}qkE|Nydx%p74$V7yf07Cu($F9$U5$?pFYW+GZ zN$R|Cgf)9+ynekYg4%?l|DH#DeFbzU>Z={_Zdg@Q^RxAEPBZF2&y55l_T22xlf#7G z`^q~l80rhmX_=Kyvx-r6MJbdX+!cK*`>2wD;+4?pVpM#?yiTF+d%>TRG=NUa%zBKX z2DG?Fj$+BDz0;-X^Tl8i+OTl}Z7sYO4$E`?wqsg1O{8=xK2RJq+5O^LO^aFa@lsEs ze#7Dg8)ojZqNpSw6vMeHRq8{(b5AAsmB@m2F2K=D?oyVy5zbbY?Q)Z!na8JyVO1X3 z-47UqH;IC@P132)TNIz0X&GF6Dvaa#WP99GJNb_9v@xJ-lNv>`(TrA;rP-spPXM?% zcuLiuXpHV}BB;segr3c$R%(9;Pq6=nA7xnHZro<-5EnhsA^>Fk5^$P_y4-5;YAsCG zhy{-`=TbiwAB7*z-A~`j$;}1h?%KKkPtHTNr5qcMc=a_o+8Er8!JnscY-yjLD5YGe z-q*5=P7SfE<{C& z^GCJ3<;jZh)=@UPa(AA>P^S!G=Z|`&uDu%?>BrY;s85Sk>5LAM-cSz}=l^tpG7fRW z@+GMZFj>3}T}(6wa0YDSArJZ@fuiPMJa8_?S!7zfmL99NY+KBJ#jB>8l}D91#6*CZ z$*YL}+o)QNg19m>H_uyf>OY(Ln*R|3%EMNEGL*fh6Kwm?F?+Z+9kc(;7 zkv-;CzsOrCCzMK-oh(suV(Yihn9l26@y?9S8~3~)KD;E_?!3dN zwwd)3AVpN}EOuyI^P4*del#EWozjzm4Q8Oyc)?wwLuXnjltrjAgesD!7`ekHYW~$g zQQJtw$#8Vteiz6#F4=pnM8y+lcrjh_5ib@gq$=lH8zp)3^sjhL54^CW8fGLw+a;tGo+Sn_ZM-j*5K5a0N{Aus|&75AP3G* zscJbx@#0QSR$o{&iI^#dy60Pj=)`*ALD+ayFA}Joa;b2&{4W1ctI1ov;&oyxYe-FH zRCje}#xKz0L{GIb=gT;URTQ9rE-=RdAG+ctxQjQ{!}7X+zfj!JU2VhmFD=xqT{(VJ zB>Z4SM}XP?8+9=SXOnQ;sZ)F{jG5<3b3Yt-+L^!j@~Tf$te>f$Zh!yO&SoORL;$NmUuJ0K&WYnSY;(lneF@9 z?I2d0(J@c0NXj7Tmt*pykmDy zJThmnhZ@xVl$xaVLtl}-0;?Yy|JAg4I@U)`HTCF@peXoaln2JczEW%#DaY{0Z$!g#aQI)RWT2j3)Bd3u_B#QBZ8a10g zzi!DkgFMP9ynqMFdF_$A>w6QA>%j=pNXTl(*wrTO%=`5G0mnJ+q_vYlSoGJnuT`z{ z>4=xB)2hwgw6E+qZ&O+EHP~n{i}lo{zSu^KefWqq~ED%mzb$vy;Rm*nY z1G(02{WY*c!FA&;#g}dm0u6t%54pOX$-=Gk&I zbm=}pYt2;d<%UG9)bjomj462(2G6+>lN}N6RL<6zBok4RVe#XtKM<;&7Ig*_(en}i z2!7)1>f(rYGt`72T+6fM9#Vb0^L+(TcH{yTz>8>114b1;4$NJlpDJ zpn(Y|lFs4bDMEy6)}p1QQen2tIbF|7Tq2@axqcj8WSI$r3SBZ=p8RuZ{#gfEG>PHr z7=bRREqaZUbah@ZnNb$mBnZx?FmBX{ES9~-yHiZ?L~*3VFY#8JHmfRk)-E}7J4{k= zQ9P0$wKrsfhH{IBOo-2DIfCMyN0`>nF4`sHt52lnRbHh8(tenvWFy)hhg|tL)fJq1bz13P`cx+tv}dI^36(#>9wZnRt>)9z5A_djc5#Zi*>qSzR>m` zjkgc!R`p;oWA8nh4=GmY~|G~wXQ3NUhNeqaVrYs(g*Te zK4_Z%`r%Wo_+s>D-OqZ%fzQ34vQNNFHza5{f6=|>TU^{w|F_eWUd{w8EuR_hQ5mJOey5{-`elieg;E0U zt5i?1Q(NC`%WJ|qD?CfJ*V<||(ApW3$5Ft2{lneQt~<5dm1@hOhJUR@ zwC%DYxW^f8_muRcAK94ICkf3`G@}L0=DjDuv7Z&2|BZKu$f8M4v;jIP)1>Z)!a)z(Qk(unm{;Xi69GHdk4 zYZ_MQJ`}w8PDoym>z^bS8uIaC$dCAWd3=8AP?fIVM+Leqj6ml=6H7qQR>ue?y=VJv-wt-Kz1o8pdIf=1E9L2b0> zptos1J`FWgd9fE&pq{uZA{-NYo7_Z}aF74tdviLogpti_u?t6rW|vMDG~0V-L`%`K z%zCW93eOi%O0S^ttfFh_t3d70C(xjq59Z(r-+X7>m5g}5L|(omU%anmSl}ipaT;e| z>4`@s=GGTXENLv$*YNxQrQU{G$=_Hka08;EwTAe#=V}L8{Od{=D!}z0pAHFr)R%4W zqiT`HSSMYr&D1yJxR?_;phj95A-x;<80DM8uW?WJ8H7&C_O?avP%&P8CBD1b`Jbi) z!X|v7zPDs4V|Wg)d~nNT;T$(SA>0+>^E8k&i2TPhhc73aXH(F9<)n1%#s$}hk-vVL zn=VD#ndtQE%ER1^Wm-`y1jA>g!fXR1svBGvwB`mTxKxO^mqO# z>AMCG1)ROR_;i>$Hy6V*ps{s*O9bk7fzpetcndU79?Eg$WObKRZ7?A^i^}}JkjCoU zcsq?es14cBw`)tZF+6i`ukO~BUGZn${RO=XDG-$lh!ge#ylUtWR7%o_TY;QKjXF~M zyZPf6qL*2B8DP71mht4B&oz$U-^CUWA)6`+TUQi)2UzhL5W+ZqS}p041EH6coOjaY zrHYqx^T^Ccs**EvlVox^tP-^7Bmf*TB{IVn?_cB>UXmu7$;wQ4pW17o6@Q7)?g-Mq zK;2(E$Ii-7yNJ$@j}ebTBZ|H|KSBiLy#J$bmnp*U34Y@GkU#LJO@M{iDRTqscdiu; z+JAVDcnTpcG4(uMeEy=ZRo!oC{)?)N;)}`B=e#bR=)VM?LMA?bTTu)DkZGkuQ1;}6 z&$8&tPZ919_v}iDYe}8vB9Y{+5qI9zNpFAI*{KllG$R1<A-U*`Ya-rc?8 z5gy`cxfuNLwtWGk4Ru$%j9Xlbum?UI0WBz9j)^8($hzVc8kBG!7!P0>R+=Fm-oCk! z_@wL^Lr{|RIZX^!Jc+LL*zqi>LHbRP1Njf%Qo=LLi-F3F@IQt`;jI!L9YlPaC@J6F z8ftwq=60&J9d4pJD_PMW0Yt;=w|EcjhYq|Q0#lZ`QLVRfo z&tzXt%hGGr6vke0>*a<-m~)m=CtLTLE)3%72}thnIqQ|zJ6gO;3k+_|Y+HOv!KhL; z>Yh($qBMmUKo_Lee4QYV<_TV+C-Mu*OSaDgkRp z*`g_qTGz^~%IT(*Y|%=~lx?gMyU=x*8QHYAXw0K!L~VCyzKiRxoRqxD<|vt`91p_b z)9Wd^CG25EYs8}Io3F(u<&^3fz|m3HIbH9y+0qR3k}GYbrlCIRHgrj)kc;KAmDyEfs|2ER;Kga}WZf4u+DUrA88LHfs*pnZ$EJ^ILW z><@D#TIjdMM*@uwEU)tqrZ<_*$*K2Wdq-LMhA*u|SXbHIVy^4_JWcw3p_Cv;ao04O zn;iP`ke?Y+WICwGWbuqbN<(Qw57ki&!t#PkqBh0IemM;C`Fv;6FH;g}-M`^@_d;U7 z91N&$y>L>2C^t{(*`3Wtk3t`2i0`z%f4$Uar%5OH{m zc;`{7!po8l4kt5O-HOMbdpX_yLYP`pibQ9x#F`3NeH1ERB{1U>jLEX0aWZ_?w`M2% zNk8yz*3b7YbbTVj!CTX5tJC_gOd`=o3Sa`S$F zA)Bn$lhup*beJouir^M{T`u?a83X4y-8S17^UBfR)-a~HVNCvObEy1>(h}e9s9CaH=|SaFf$e@+Ur=aDH~j zD#Sz6ORkUN0%i}$F|+>Hiu}ZXorS4%isu{K)w+8LH)wx)Ob#B&Q#io?M16Oo5^3$+ z=oR%lX^n{X+x?Kp6!t~UB(6i2R5|);v_i>lt8FPYn2hQ=t*Co8(QmQg&(pN&LSjt6 z>!?J`F*GHNsYx9V)5Xkw*S7|~q|u;%@A2u=gY(-tXG<+3@BHi(gm2&DnRtyo7+|6X z_#-oX?X6GNf6}ig%Q_lJU8HPgqVHmphiLu^q@15m$G)s~2qR3RYPczIxIK; z#!zFhJe7p7|93{zjfq23@%?}jdn|_ZR|BjdZmo19G1rQJrijinP|xe>9HG25(J^}x7Ov4I7z5E5H<<~N zpYTN+F}<6>k0IrjkQ$>?Om39Dk5p&B@V5%F&-VE&j|8f+6nyAfZH^Ah6BkXF@m2*C z!{Gu9w@&Tc{Ic{0tEiC~-J=v~BjoJCTW4U^$5>g|&8?N)iHc6Qaj5B#fn{TY@z3GN z?fq+HD@Kcd2#^bS82UKU(k&`J1xwRBS^qVH4J*w%6hmSxM2$OwbDyf}tLMSO#> zNm9B^L<(r3c>A8rv!!k`1VV@J&2!`>~6I6H6VD|fxo zH{F|IDqxj6*L#N94lShz;S!wTKPO6lPBeWIupYV=E9qUFTG~*xcGQRa@wCh07K^z@ zS(hKkDLGlp!{LR8#fueT%f8X9=SKM+48~junro+tX*uDiWjBiT;pgar(PLx(0n7ff zgz9pP&y8PizsQ&B?B5_DAaD|y6ne}0t<#;fG~@LA+OGB9gVfRc^tuneQd{(l#5aIe zwbYW06GpEj-1qt0)g`WDM zVC(}H3_w93LdqPp_CBEdVWJuu^g4tDJ#?V4Jw~djkBb+qO^O%;)pivt6g>q@8 z#0XW_?RMCfhr^&Ulfa#^tkdW%}dHg17S=u*Xs!>g}A?y?O8{($lda-P??`u=(ZJ_o@JQ9BI6#j;I!f6d3To`}{8s@_w3XPwM(mBD-Q2tHmEZ5KoNOzTZ+ z78%8vw(_-yCJ!P1KEq{J>VfVJ%n@d=n% zHvF%9xR#l}%q?JivtcsK{s9}R`LaI6kHmiTUO9IHuA0~>c%Y2^Prm|Xp>@ao4@@xw z3Al_SIPL*>+l{)Q(7wps^Q`+^gtu0&>TH{@*Q+CUKbr!sfv#C+z$|u1WD1Ylk1ByY zHB;Q{x&4Q)LyMA%bQ%tdym1Jl{O6YA>(voPrr)tK=PT<~YvO2dbmPvBFf`9E|GWY! zUgxDEWJ1v+{3~jS_}%=C z-{2BK+!$zD;{Y1>XWiCb!$l!=3KNuqi`^6SIs;Wy&pcZcwUFohW*9^@7Qf~O-R&41 z8_O7j`DLHBaiGy-m4}BL-W1Sc>8Y-ez5HpqtN023{4fS2`S^`GZwv`6TIeuoBSyAu znTEpTtuDY8eEn|fyWUr!h4gw|Pt~=0?~PVR(eHH*BeLvMnMPJXo<%UscJ4tV?xY-v zZH`7w6;J1nb&V1~aw(qDFyKU=eU)(D%-s*{2OC0aw+b-m;&TIv!VOIk)rs_cEAyBA zSgOnpRJIg3Fp|iJBRGl}vO>0a`9sWmSq*l_=r*C-Jqn6tuDId|5@4vWZ-#W=+ixqk z23z_mrwdEl0+YR|+V1J@}xY&SjyZ5aI0 z-+EWPhg-qzk3+z+@bhNj{Rlb`Y&=aX%Zjz?cae0?4%RE!=P9YlayDxq^hgnfz3P*sIuo1Y$VPi>?)`QDOpMT7s ze>H>TTnZ!}4au^xR!ZX1bvVBd;IZjjcih{YU8PSCQx|I2Q4!G6OwHu8joPg}?ZrMY%dYYHDi6`&)oo>-JkAj-%1%HF|`}0iV)iEYiR!Hg#16y)IA*OfaWF z7x!4LOcieTsMg=DEPv%Be#cDUecws#Zc5eKpW2`0OUe!F;{B_H9=>mi3!;*ANUgez z9mhRzwW@5zFgZfl9-_4;J!MrmUaP|WnwcQ(KLXfTfNz~?G7)jtDvHsi4-QYQx>h#;%4 z^3~lhpRWc6_)0*m>pV+`8-MlM8X%sFMoXsOdjOE4#sYUtl{#OAhbvTgxpU#5pwK3> zyI!tsh>%CqMjAm3KlMKA5xC_*kzQ_`)!7YivJ~;WhRqlR(SwS z;zes>XO6*(7<@Ci2WTi>vUg>0#5F))Y4RM#b)L1w2;%Sk#?eUo#k5d$QXubVyd)r? zu@2(&34QRFvXc^n@al*^ue7J7Qo&R~J@!;r)lQFsg z3gtlDpUZ@WzRI{`trB2=b4p(oJh)59_bg{n*Xkw!NTh)N%5N%= z{*96&68KozaO0<~tUCuh_>LYnhinN?yT+xYrXFm0!1W#dv4}Dj-;AKc4Ca^_m0%)R zLj{>J;%4R7_kbX5Y1Y0&p68U+q@Et%OTge>kt0~4sHn)h-v^m>o#g&Y zp3k6o)pW^of(ass{q_{|xryI>K)mt(BL>CM7mF|6x|lUmY6J|kn+PwAiXv|~YCw+y z^@@@j!fYKK(J;GX)jysQnqHuZKa4LYkygNWOaqHZr=qR~dxvXPi|bKwG#!d;>h9vf zaLW5ol5~#7ovRoc8rEXg3(CY7H1ahMGhv4%4S59&FS2J%g1oMtoSD)21|xUyeaX!w z`YdE5`eqF2AG3aVdCrgTr)j&C+Noiuoo*C~Oy8t+Z1EI}s&B|c^`(g&bn7%akJIc{ zGVoU$I{&=6X#AOL@s9vK2+vG34#60?j6U^h|MJof+uT1qgm%SL_zp#A2CX!OXMD@_ z^z__=cPhVr?BS}i>iN#)Jzj^*E`ZhAM7zJuAam!vxkwRUzfI-+6PWVI$CXkdjDLAl=KhGacbw>(3g&I( zWgs0|A{1bJPz|s}rzwhkAwOOH=E9~~=)H2&E(9cAT?bFxzu|WJ^WmJZfm>`pV$z6N zuqb1kH&FIOg)PlhT`01uxxuh}DvJD`J2ov?bJF73j02Xu!0^$QL=Lb(q^B!|!HA{hEyE5GQrCR-X00vXmDK2QesZ zh!SgLz_hEIV#Uv;Uf{uNYMgMFexJA7yh)mRoq8F;i&PGF9}6rO9Lh~4Ug(2X>f4@$HYYoP?N-WcG3Rt(hm>BW#c8j5SkbN{J-ly zAWP~=++;D{1|=Y;#t$msp{A{EfvA|+UUYkI#w+Y>2UZLZLX*`6nnkYK|EpGmoG!8* zG@!veAj{K9h8)Yqo$ARhRPQ32kN;}C4|mnZ7H_&L9+D=joky|xS((@8t6>;SCbW=! z8ZDF1k4osTLZcsT)EbmIh-Irk4Q_cq9JAm<3NZe0;kz=K8Ql4ipk*I@n$K2E?e-s% z1jQ#yB2iZZB`aa`J9x$RNMr4^A&8Rnc<;!_Ky)*&AXkNI2@5N@e(F~?b8-Xw+lK*` zsivaukB!{0E& zA%XSwvpK`ocw9K?PV%1Tq&f_J>|n?Rl0K>@BqU6p2*p&n-G|TkkKEWjJaFIje7?RV zk2-?W4!|LA^NQN|R2p@Er+?#_9o%@~*D$a3-tfHrgSkOJ48zcjpXh)&0f88{cSwrL z_u5tyfU!!?tzT)^MXvVT5&5zKJB>zCwNnT&u5;E%g0js1jDpZ#)-te^>!_~0yv~Bm z5B(8}M%M`$?TbD3$HhMAm9jgmd)?sw?(4U4a+)i2)H*2HN_hy9J=rUJ@B8LWu(!AO$jl4`EP*if{o%$) zL*wK+1NtDJyE(Cal8l`dja`^#4T=wz7iGs32Xcuk~=n4#CGlk*LJ&eLB}nDfKr)! zZwm}hW_Y_$J@;68d8{&dd6{?bzt#d*eE>m>El6$W+b^2-1e*^H&Ta3pJA3 z-9KZTPjrT%n;9iLvZJ~8{f1jN9Vo}g?;IrBFqR1MU{{}VI^wEb=A?iMgH}{G(TNvw z-{=Rr7S-qpK0m+HD|9cmE1q@kUt7&#D6qw?0SOk?4HdEiRF?zs988pd9x$x!J zA0Bm`vhicD62qzk39+`!s4Yyy=3FD1`j_=P>hP|mT-hB(`*wCxG4|1+Lw>jKgThbrhEE?mB3wNRGwADXwoc21B#*Az9lvm27 z@QD>tc|6nZ>fs*l(vptCPe4;dDMO$gA}fG#7uzNDmzeC|Yu0b}*8uj3{H0Z(ZO$`m z6PoV@=->ph1!231pLC8uEuD5a?hHMiooEErefGU5H!{~2IZ-=-47p3M2UH@M{42(t zXA63l;lgvXAZIL*0rsThd%`=FcJNNEAyk?K;-`+Bq+Xl9PL1mJ;=)Xd)9tA&w+W*x zjnDltTMelR;Klu??j=M95UGrqtH7#!^H0`qzyAdOiz7S#1ROI-hy zvEkj&7$Eh=f=TL-70zg$NrqWCYRrI$S89aEv;S}y1^Hv_!3R93Jw&(F*)(WO7QVw~ zzgiOUd87>E2R_I_?AzrsSVAoD`hu5!l?BHtzF3q&rjIx-qZD7dLeTWKq+xdRx@BZY@JFC<1UYn)>~SSBkBjhIzb~t|Edn$ zxYz~clyG13ZQP2t8~UrqQ|GlFH$^#zLbq$pFrF6&cKVA>!ax&_1Ew6qZFX*2CTI;hWbb#T|bB=1b_mnGVh%|;v7%{8pE zUnVygJB?}%iH};Y9#DVYKN6!}vXoE7Ij5%UZ;Li$$n5GUj}^*24nK5-7G=qTZ{?SE zYa#lGDmpfYEFO224_T1Zg;aZpRvhmvrFe8@L*ojovSOv&7N=-lqzRuSp*hW@pw zDzNg-xM40Xh3BziiPuY`1K4TmIhqmD2<{j^i2eC)xyWN}9#A*JeVrcmOdOT(g#Fho zMjaL@wBT+tY3u{Wg89#e51@0$RtTV~yyyyI8ah?z7c&C#|lG$-wKdDg4w3?S{8I3*!rt46L?|PlN!sAcJ7s#>bk(@|1*u+DBH3A%#2Fj z(TcfPH^>jXr!?iff!z`i6Pn`wz5E>JG+>{dFiXJcBwCZu!u%VrAPoM|i&e z{js&Z8NaIKkT|`QGM+U`1(o5=s`|%Ok;C~25ca*e$6T~q3suc~=a8xW+~QS#NOA^U zqi?pw_hm~x1bV0rx#wYs$8~0IIoSIURG2t0t=Odo`p*hQ`!Qy^? zk~59@e>WhG#et_eAU_!GgR zJF!_Mc825(9o&l80W8AOGBnTn-Kh@vBnNKgQ~H)lgJ6OMX z7g&!OU-N-rn9d$yjBAgUIR(XLN*XL%7BGX+xL?7W5=wbD*#$nrmPJtIEH0T6hxrAi zfqtdnB8QW8VOMnDPkH(1Q7jDW*x>B0%8ABh`T4F4+eFpuB0g?JJXy=sfoq|?rhO-S z4;}sVK)AI{Zzox)$2sX6oGFa}g3@zTOh961Pc){*GvtIY5KrU>^3C}n z=xms{D$ZWh0AwZ8lW}e5dY4%@j2*W+OAnKjpzaove>8+aFsMzRg#LO5Zy50N%bU=#_2WPwEd zB5r0<4FnI~Csv3rQn)|6aKgV7Vd>W=cCLqt#Z;koJPpW5e;xeG{OrEMTLPsVKN^y=*uEcL>4?UzAL8i#AUKJQ6d(QNVfut=2@jbnj)%QrD5m?W~Xk@rOu z*)oGhVo&En#oorkhWe!Wky%G*RmWBW%JZ+S$BmuO_-Gn|%%1_t^MM;GAmWLL+9@fy z?4gqvz-3u#IwQ6{x3nx4kIK24v<)5xumqgGBUf`M*OlWp>hL7af874bf@KmAZU!RK zQH;CNW`Dg)zly=Fe|jA^;=SET`i?GTpY>2|V01XxbU=auLOJqG`x$uor&pLURAVam&k;7{a>%7clS{@@>X-6C;F9JFQ^zmvWmoatr=8Gsf&Bsd1x`vBFcXFbt{5M&_%q821wtaa|Lev<#p`GSv-m(^)Ihe&8ez7 zJjpT}V!_iH0WbH3KX0OWd?rCf9ZaS0C|>>rYuv>xq$gm?seLr(>Tnw!r`9rmlh04f zeIJ-d-l0!d{LhhrU)tir+vkao!Nk|DPR`lY8_feu08kqnoetpT(|0_+z-RM|2}z{< zV~q!4Z3&a3D(M)&dIrLI$!w|m7oMkN04LE5Um&l! z#I4fvWxkmx*Z9Z33#XL$uQ=V?@fP33_!8eByqGKzka|T?4at}l+Xe=aEE}V<*+JE`f}3wcLVEXjfxi!Jj<#M`SATkzNt)GsI}m) z!rJU@r@qHu5oe4(l_*a3-SxJ=t%6q=1ST7$@bwC5*D3c%;qff1;>!UqFLIQ8-o|$l z^-L&fM#m@QQV@i1YR#Mf^~B$hR@tj`ZP>#hh4e8z zm$j4rxqN{HA?;_y9J;kWBJ3{%!nU4-A0NU8V*xhb+7i0$hNm0)LH=DUlatNb zEv&x|SODZgEjyU**OqqLBK@9Y4z@6r8Wa}}6?Be79%ZMx;*9;W;b(Kyn7yyK0uSYx zd=%)A_H4B)ZCMwOk;t4bHn@~dW1$R#7-9rDnH|-jXunysP67E@FyjLIbK`b9`s@E0 z5IiggM0ogT=?fJ}BPTI)ezf^pTx$L@oVVxIaO0n}M5RkYx&z-qQx2CmtXQ5pjnA;I zYABoS{P$Zb9{*%z7{|WH$IR@N(40!`$IYk1+E5+TeGtrt>D{k4 zoI+4G)ol6PcM3XH)?2~UHhPb$bTj>P(~nmkdLOT?l=~_~yrj&rR{z3tw5WQfgc)~9 z)%2NvENW`;uJYNRH%lpQ-ETx$PRjin+4B|p^?wdY$}8EXk2X-8>8pUM)H#Jb*m=Eb zpU4njA1>C4<{p+b&lZLFyv6AvGl!*Y2|)+obgUyi={Wrny5 z#-Tvt&o_;zQnikR?FXih&h_(Yb8Y>4XI(+X@OMD6u7)Rn#dUH=$QUI$vgEderW^V5 zbs%MugH-Kz+S-)IhrSI3muA+poapRVA(MemC>q1;n4O{4{ zu4kej#w5tdc`5Y3A95TZ0W5c~6^t7fXOn9g6+r!_>={37|J3S~ zd5HJi73=jXp;UQ))kaB4^XJOLKA`<8;PWi>c!$KUA9zy%GSpKaz5YGaeKGOyDb3n& zX~?bG^*G3on@)rLUbxVDRh8p-vJP0+_;_Lzvl@yl?)RU6(6BWkM_k)byTS(2UZ}4^ zaYA}GQ8>T#hqcRpll;If###`=T;sV+KdwomOkDlp0%l0Ccs#%2!W^dIk#}q3;{7q)3T=SMu)TY4n8fDPuM^S_mHx$=MR0r`q!F$3}vE&@9T$)}B>$KbX<(Z0;-;9ZFjVNm} zFIh2vi;Z&@EP*}2DF&(}Pz?)~b@yuZnNCk38}w$dO@iLjV#wmEMiikp%HJ?cxZ#Mz z`*)RgquH~g(q`XRRBnZXJdJyg8y4W8q5_Fs>xPByqxG~IZ%q;;_UT!FPn4h!kDEQF zs&N+}>LRA&U$B!_w_1!TWLOxD$Pe)19@ptO_o!X&rGNRXkJQkoWyRIu^bU9)j4Kju zSkVJ-+rkG;Q?c=K`+1G_`HgP9J_l2$jg3p7;xT=RZQTBG)31$>2qczPX|(>``v-XU zf~HE@tEbMjxtznx&Jp6Ox%OoR#ZWA6JRDB$o0mc@1;(4Uved@GMkg>^S3PtPrac-x zFlP1y#0}1WqE%yKZKpjRGiX{XWUs-hUU;kRPQvR?QFlsL-HD0du*#%er4&HE&2iE z0$i%AJ#wr^hcq0Ag!hFk$Dqg+@-WW#5GS(SKk5qA( zKDSIJKHyZ=vgtj;jDmzV9-KcOJ1uWKLoDezZPBJ^7(z*S8l=)Y{9q6s>&XN!k0LK! z(Ag+_M4djSzweoZH10sO*zcgt*yf!rYgNo(`R<*wWk~9t_+)4)&pyZPn{tfsKN&Gn zflV>#LbTZovVj9z5~s=SmTnyyX}enmR0DNd6rje}`n9Em>z(?V1(YgD`K)YE*ArSy zw9cdbJ)SdVTT%7Nm7+}rrXe(A-`9(^d}hnBC8lv7l_JfZzb*$%XwQb6YoWqYM;A-v zOp*V)uZ_0$5I{_1wMi5C?^6FEYuO9icE-byYxZcxrwk46%k|Ig|JR_QbJmYr|0j2n z?8QamUrfOs%C42GeRpN3K>!zy`2L??#)477Gpr zwv5L?4vnVYYX1dDS!g@0+c9X+j`lIoHZ1HxhHEG{K9JQO30JK}M?6@X_-V*!-kq~r zT4j*W*`Jn#2y9MCAGY+{re?rSsjM|v>MUB4C#bb#!q@acHO576MmdfwH6^U0EZVi# zU?WSlDBHy3O}kNHqa%AQ1DqRpc4YToPbOl&&iGXONTNz3-Q=y2hOn}Qyz35PCi|#< zU*!7VJ%8+yquWaC{o$+I-SMTGFAR%MqdnF(Ej5#=Hb%ZWVsl>N_O2WHoNw(M0CFc_ zhb?Qf$G?~f`TU*n`AmdT>fFFpFT`BCn@3gPrNmKfmuE(|`ivqW+V9$~y-BZ9kgE`K zeX99fZK?TBxKcw&ksJbnQDhjRm+iWyl*X_2I4IQ2BW~T!MpIfxkv3YTO)E~YVJH`w zo)iv+-s?K77LLd!8mZ2bOB;-zbcC=b#0gl+Pa-=K6@fjf(~xN+FHfcfZ{Z?X4uj2V z1!|w&fKP6?d)S8IW7mHi=a(EZy4+M@Bes2LuOOtw0`~> z$#?~Zur5p7`lA1e2iLP2PGdJOuZ=zU{q5r$dojv6?WhKI#ea+JyB_hXz}`R2e%((M zBmA#+^W?uP)-TGXzrS>TB>Q~~A07NZ8LD55uq#z#A1+4A+0qM+e8i0__eUazM|8<7idzN^I|3*I7t`@kqBDiXZ6eVDO$)sHNFV=4=*El~>9@fp*3erB0s@T1(y z-W{h)kh;g={QWz+w^rF5>e|KPCB=@6dFe{ zMz5!ZMbd-=+PVTduv;5S?UgcIPapb!d{l4?(9b60|MZBv$S>TN&lS)p&8T*Jsl`K! znWY^2&XO(ph=M$1VTPUftF6B%{~@!EqzPn5c6ffCbsc-HCBq@LFr*}T#YXK|LCDf< zNUn}uA$j~v2ZQ4}YncNYPR;P|TgMogW2X|D|7FS+k^)`}(?B2ew>YX3WDX;XTbHj# zo6Z%e1A(wGhnH>sepZ$P36qR7ZT%V!uwb`r_N={&7dk7Q>^s9d0d5(KU!fjoi2z>% zp*gQf18^9VJ;2W-jU?lC>IaJ|c+MQie{_Uf<aynm135kmc{G$EuyoT6_aQDw zol&^z17&1m4w=D%Fcp=$8W}bA9uEQM!Gpj6EDm(Q+|NpWdS*60Ci(c|Tfdwl=J}h( zSjq=l2Quhx`1ZX|GV7i^{<;^#z_))1Q%DiZI@SBY39(G|9N-fcPtM^rnoWv~yZ>B` zH^|5$zE=gzRWeDo(~jEeZe42`d|oZorB%Vjv8p!SuN|tbdL6FOXw5d+ zF|Z-Uj6dTUuZb{VBFUu58lMKoKoFl#gHyguYu-5wq32gw0g3%JZhd2|M|`3 z6>FoAI{FhKKK|D(4jOvTzjK~%8%sn1_r;gb3Yh8MD+5DrQb52IxRI!ZNl`HWFm(@0pQmOq*oWO-GPsQ=K z8e*m{uYm&5xhM5`xg$;F#E|oI^|T)7H9};JZV%V+>CifGhwf(R?LNIu9nxl_GMD?c zEsa>;0NI_XI0`*MT#Rx4ABZcGT?#)bF;LtKT`)IDR@0{Rq=&=mIy&?X%pT|wc1^CU zV!7M+!>A0V<1F7S4>1(8-~G#7M~0U7Sm>ko)7HbWQ5_a_{CkLdm;t}tqgK&-piqO* zxSZ(ARQIe@(a?qsXy&h&t$xCbAe~pIg2i^NeFl zIlG}hk~?+bpN7!^J&%6|l~NQ%LkHNz*Jg?DV14FBtyP5k`drw{4S4c5@Gqb6bX%In z@lI)hZ9Ce*xNbXE%2+qk6?L#!Mk$EJz#M2w;q$chHz>QrW%9Gmne0wlb(cUt$MMz^1DK| zJrBSVhU~F+KW`vEJZTnkKzUxW@(ByXLzXluftpqQx5`{ZW#q=|b5m!0buCjbVHch; zurSYgWhy{Y$>Z*W!nOdw&%#{DhQW{kX>wHwtN6FXRCUTfnvGTIsihN>f=-w=m`O#l zHN;qH<*nGvJ1y5s&te7byPNgSDD>W1F{h*~Z&x$f#raSfQbjPZSUo!1a50z;i$btB zIJW8)0o8N3ZG2kLZBj4%uiz&u32oBUs28V-6fQ}NTN~WNY|dTN>S%TfRIkZv5P-EI zkvYkHr^sg%y{4f*?U>QsQk=FWTv#ahOI5Pc(w0*Fgsz|cbhlfl{SzN9sIvDS#xqgW z%q9`-(g%Rn_}tE36$O9IpIOzDMB1vGh$Jg3VIE`hdeJ(E<6A1&%ic|>-otK8JqpTz z7r_E`TiN9at@d(#Idxo%3@8mh<~x|->``2>suE}w>vAu<&d42aVNRjf9(Eh1)2Lb| z_T7yJg+t;Ho7Z`bvyG4Y{>nwa!TtkN@!Lz1=Cid0tKJN7-pU4YS+iYf@nOwm$-7QlB~Bx1s@DDfE0p7~e)V5aXCT`m3&SI-9H9vKeZly*8wxRemmGr|#=z zI9GADoNf2KPguJ~idsi*lK-!Qrl--}7qMVlrssT#ljr=!keK-lo4}_zkZs~I78Vu| z{cM;TT0M(=q&4_jfL8*=ZJ&K$^pozyXyu)hhJf0x|`I7E>c6-LJ@n(9qP}Vx#Kh}Gd z<8ymwi#UlK&HP~{56@YY_ufgh@i%vfxT_t1`yjz3jv_=UaPxiC^?+KT2H&3RfE3B= zDHuJmZ*%WV$!M^wZa3meyp2Oc#Q59%X6(F3IcnKfDpfY6C4R83R^tI<7RuOyT)kBa zri`hZDy)k7OnyCkT(Hbn)GP~&tUt(F2pb)=w6c2=jVyX8CA)Kd2?-1vIc-uVZ$9N| zY~{ajNr~a<=etzCloncPDQT;CQ=Sr865rnpR1a%e75#u|d6sH%4g`j4Rsz8aq2$vF z@fcp~Jvt1g4can5N}T$@8beFanM62QM9kO1;K96~jZc6l+!-+1Mc!&1k;_eFOWAO2 zsBH&?VUE}O;Q6zs8sQ410;U zp`b3rtPnDJt(QI{HIW_A2VbnrRtmwmE98Qsr_=*XYvB^TQX`i8OFvLK~cY>~M!gBmQ*5v=j0u}lQk0ShW z(q_ohCPj`{YwOI}YA9i&2HCBy``@?PT|VD(fqp99mia?=pmI^;VO0z!hKTUgsZfBP28VCY6l}AH9n;sS?)f zG+#=FwDmObnUp+M32Q_P^^Nv(M6#Ttp3neO*5b$yb`JErKu=tV6SW9hQTt%o z1ppz87DL=X?9M>rmUi7@`;PtDC4FJBPXG7+^rVzF5sqF|v}Bs@ut*ck7qEva73+LzHuaAkQezCXQi z>02giNfOpbns7Z^=7vawDjmf<8L&3y{$;tMt0xVf4f$wI@;_Be6N!UClx}^7CK%Dv=l9qU!5%;FVP`R?ZjU| zHR!YsQ;9q~BGZ?K$*`TM=dq(n-@{KOG1X}zt|hk1=0WXsjg3%%%2z6>2tZxi=-aMy zSfE#+4|oCv$G6zUq8*8W`a6g9ws4f)d)5K+;8jb(-Pw7Qy=2? zCr}+45FH?FM-Y6}W=Z zIhbw|=us}m?@QH4YxKv);W0z@>1akcjFHMq44Q1CmtdM3Og{a5pQR;eNYzIweMeSf zF7aVYWRE|zaIO)ztb69_LW_x9qOOaS%VjDE>1!bn7(0BqAXa0EuDMDg4#~WC=tsQ! z@T}6)xTttdYeNnzE?rh{x0GRzRj*EP8GIfHBhP}@sOhUjBZ<-!@xl=&%a6h--ai*F zO0+Osh4O8z!KyRm)}&P!LdYd3u$WPr6kc=vs+trR;gLxP`NBiHx3+=-C1|%`>-vi! z64v&M0=)gpg#&086Ed)De01ihNwKzr$DQ67+Ti0{(_-yAf{65Iv$bzNRt!r5<$Bom4#c^82yYyGM_2gyj;J-?ftCmq*nxph6F~^6EV0h!1mEG5 z+!U??hB$S*Xs1YxHha4>+#lU2Xd+0^5_*Hh`(<2j?}ot95M_@NEgn52(3>WL@ra+w z!V5FZ;`sMJowCkT_qFSFc33T%9>>*HGX=7u0gH)s)VwC9vCiRF56Iy9FYj;ehphAf zZ)oQq3sjodMS&-VHQBN13s#xxc{QPjFAc)bBr|S^685|FeBQK9T-o%hbLMD_-J_vKK3%?nadp?vS~8yRIIn{R<5=!BUwfi)k#yj5HRxJ%1n)H z_(n#xG9w1Qh;;3yak+MHpDx3k7Y#+71C#pn&)zQJ_C%+Baf4#3EyL2&QBy3?xW~LZIOOp~+vZ1(6OoQgLYq#h z4YhgTUE^GR&_hqJF-y^she6jWj&?dKg1p@;3BnGNnOw4n{(#J%{7-hGn=DQSIJ8+A zwS^Wkk&i;hJn?JV=nGnI&aAawPaSU?0Co>*4Gp=E zsG(u+)q+6vFQ4DhT`3TR#kTFVsdp+xUpAeK39m{fcMt{N>u7){>WNCA?Pu6iiPz9I zoa8PoZk-e8aTB?=7xMSis{SF^dW{Y9Yg&swm>J)=aky;;k2kQS?L{st z&&Lm}(GR40$9V!fU5rNXUGpBuI*}c1ut}_(QsZuZ++McJO207Eus8N`BHV&OCtVTI z2w#2Mi)$1vpyIYkoz&^1()|Ng&E_6eLBGh-9%K(bSd@RQ<8(PUc`>A>!AvZ4%*{Xp zI4mvXav#fI3sn+DbY7`P^+;5=j-4dddES0tJE1O$zE_wQZprft0$ z)sd+-lI9`hdhJ>L9CK$2iN{zo={YkIhF(tR$7}fMVHjeZ6FntiR1u4BQLLdIjJ&B1 zR1vt(n(ru#v~S=t&|;2#$Wu1T<#R@w{sM1(c;kGS_ue_~txIh&0c9jj@mp6Ew=1*B z=8{FH$wMrMde&<%N;fV5Uf)_3IRcXIy z0B#2KDXN(3UaU4F2JAV_SEP2vL?f_~Fs3rq<(qrnx(;;HmOZbVYFY{rkJCRmITiZC zY1CL@IMqtI+3{fViM6$%M!V8d(u&-Zc`1wqzOuSQcpJcL_LX+21!O4>qpAYcBYMsk zKdm=a?YV!n^HcXNax;nOqd*F*g1ECZ3d{pD`9^$6ZH});PDN*o9wmC6JtwX*rKFv49ND(~4YZw<6 zt&w*uS`Zy6|2Cd?d>ibsiiVp@wTX4|ZA?xg<{^bPMI^c2;RK9B7Em@e&s1lP*C}Qw zRBgV2HLevb$**pBk7N3u@3&d{+FLd?K1c_lCm^(DUG!Ea*5#h2%MC^Ib92%Gm{}>Y z-i*-zKUj~Ll*U}PRfS!)uJ0q5N%!E~vB02I8>O#`QGmphR8cSnRoC#+8QE?BT~fr3$FM$C*q+F(ffEI?%MB<|52PLaISgq(n05 z({kj^vVrRZUCWbuI+{eQUO1w8X0)W1WoFUKAC+?a5jl%e+tp_bF4n39s#LSbZMFPFGE}9b`c4X{X=;zfZ zGmZFy5@ml(3(AKI=R#v+V_i=j(LdM!he^Dnw%Y$@!;l2dZPfILD~WY|Fks~|?YQAR z576&+rC>}N<-_8n9(^~bROTp0M1_(Q&j`9)6`9J)-h8K-JGoHjXl=R$xw+uGW=NdP ze;wvnqn}|X8lddR8v+AVRek-dSGtg7$sDS-Q8WOD+YX zkD5r!X*~KSKH_BNWoTBoc6$2*=DkEi%@_z8>SJGr@?4ybq_&`dOyHt9J}+nez|Zjz zFj7B7rd(^qm3Vd&bM!-GxWS?&6a6&?Fsof^R_hB19>7R*q}AoEo5&4s>M4BzR4vRm z%HAjs#GAfrl^Y6mhgJ2wn|?3akjQCizn_Mtows#j16@w-QeTJ+ajdh%j4vaNT+Y~} z+qtLIs6=gM>Zm#cslWhs(FxxSBT`|8^LLYUkQ z*y)?K-Yp=T*EuW1Ijy7^^JIzZ!3{2Oo4OHzasU`L6RvLwGPMf6Qa+S&O;4cRcM#Ly zsfC64L}y+@hC%917otG05SFfj!MqzS&{u*jNo-s0(7U%Z=Iwg{*~&pGr1=>SQ_s7m z#2j+LkC!z+wS0pF->AG9pMe!Y11ZV@fhInLWU-GJ*?l7=Kf!;P{5nrDy*Ab2S9B}+Cb zcEZEcO}0!6tELDuo?jSFld%~1vxf79@^eo{`{)6J5d#AmB7I9sD~ z#NXh9Gj%FY5<<`|!3}wVzOvYP5D51Ma@S{XsMb403I#cNDtwxRwswF`d$)bN{W1h5i15%wLS^*=BSz(FEMR6or zZn3D{KUcxEZB^n~H@+q?zz)?k6lH1c>N#tdoL7R$yNQ&W8eWu-JtxB+;vFYu>HDFPxHhSDVnjp@ zGEaU?tlGfq)i$A>eH!3p6>%#>=@6y|AgLYrn?3Esn;fp95IkO?m^wCfC#8LvD?Ej(1z;zMq%n}}=6RFiO*r++fow;2O9sUyrOuc)QLofH_G&uno0a9zwP zZlU!r)EK~kz%dEA#6qpAwhGsEj%(T`Tb+RKGUoH`$htZmtNIRWA#`f{ zdivVDy;G^Vfo#pA$!7fkf~T?FalTXQJq1;y>YN9QNfFI%Q?Kq;kl#pLv>a^{YaT1x znDD1v&wX&Sa7~%?7RCS2{b3~;WP%o(LdVh~_dT_q$7IGa;O^l!FKP5gs(Ry6!^2DG zO|QKjqQ0jA*NUt?M@VepV(iIbK@&f{Iu$A8DJAj0ZHLIOQN_FO(Wb7kMEASNmq#wt#hp6rPG(9~3>)ASD##9I^Q zg&8%eQoLp2RiE_b9`qxGfGncz@N(^rrcsv%%y?pkx z#A6&whvp1(S-#i3UaYRi>GqSMOVA_?f9pD#Md0$3nI$3C-7vTQ{#L5rIfka>Y2_2T zPKzIswy~;)Rbk6=`uCd|v+2rT=7a=N~5d72*IZM+~9G#5P{$#D2@w3(rRs z;e&<{J>L{nc$XG_{sde=rQwed@IoXYp zI}FdEYYR#nwfoJBxLN|`aprY$ezub>?VeYrlsE3sS*6DN)?4OrGoRqEmr3h8dq@2~ zm<7e-%anE?epDPM2z=wLg^$5ka;BP4t>w=CE(cs^TYY}FG#kNyndrofF`z^)9b|-b z6I$t#c^Anwc0V8{q(L&UoD*wt+8W;AMklG~7SD80%@ZA3=I z(Iax5X?L6$S@7jUr-6c&$iZjK4gMaGwI53I?-TB%V>E?lM7?Da>2M>N9f_Kdh8xF0k>Ty!g72WZ6^(< zh7Fx$WhEAdVd*_~S1zf%KD+5nutnyc-F<;|Oak2ZX1^m}?C2z^Dl zz2u*iYl*Tr`6fZ`Sy=G0^?SMT z_-b0a|K4&&@JjwwweTp1K$Zrv^Li|4@8!AqI~#AWwYqe&JmzDlji?M0yT$f^%z?Oz z@0&uND7)w31^XRnu`tQB_LhpMxDxc*ns!hRIoj)6&+!q){&?|#ZeSo_e5uWo=7y#d89^!AYAZ`2mxwmP~6>( z5A+4V7>7QWR7QHo!XrIkcs2I-@yrYUi;lJYc?XKlU{;BC(yY%J5B zW#Dk@`>xLNFdUK9MqmC2`?Bhu^DXmEKu3nHf=tHBk{4jn=tD}a|EvU`ApZ=X%qIBS z>Ix5j17)H@BSq_&Lc|x5doo-^sDX2$7klWZzBrSxDf$$`vv7k(WayZA>u2hR&yns; zTC1;3mx1_0J?L${Y*xK~E8j4&)SoI!{RDW-9X!osm3WP}LYLHy)PQNyDg`hnj>R>UPSA+CoMWO~_a@jdp0B^g?I1~O3|o-Xw8q=(Ic@03I* zxlGkEltS8E!aSWC(d&yB2|ae6DTS+|QDNj4<)%6^?<@(seDZyoiue|Wz2?>Srfh_I zi`LKA-`u+(C99Su>1Ieiq2sAB>IB8s(FHc3-1TyzCq4xM_*|oHqQsE@;*a7^&z55d z$`6~$H?bxm>m*tad|F?q6Gsw8)uhT#-Z2X=X1aM6g_$Wb{PPO6CCm2+v!k6?-VL5H zYvbs_*Sn>lN6AF(#GMtcF0oFMo~yWfQ+O2buj46-6>!HNx)-z-A1p~Vj8_j;o?)JL zGiwth!vn*2({(QbuJ;hdnX%THAkzl9_z=x4{UK!QPa&!8QpllZr%h4bt!dH2;61s; zD!{t%&dsoXhtSc*`S{kU-4qXsup@Q2Qe5hHGictf5vUA+P)-ibr7)U!S6R(f03}C zx!&P+`u8mBpMLQ8(Vu?c9BoXP$;|@K)gRMHGy15#{2+Xm^!d#0h-Q%z*100QFf9t_ z+!cyC16+>L#(tTau~k4{RWybnGv?k3&h*9IdH7w0G{lV}yN~#3hM!oSYopmXda4OL z#(E^JlY~ZSYk;-Lc^c<2N(;FMy;ghHvzIc4C98VOWUtNp@)3$kLPzAH5C^mTAP%W) zsdJR`@0X`&&)XS+WfA`Qu?zsESSwItpVmlSPmFDZI7&zkjUD+_*;1yVphjQ=$_qP3 zn;b7=x`n)v#~wBW9WHLzTj8&l?R|%Pc7Gb=3`OWSPKa3kqz~`}O?UrLS>doPd!kKh zfGr7-oZ)84*(MG}s9W+h` z!Hu6&$GkUZxEcCp&EM}i(c$gHdE4T5rx+2>hyj4LjVk(#SmWd->`v6#62SH-o`mA5 zA2~QRK(2^c8dynbKX{Fc`4M{jq;+%I?=s9zInh|LXDJ73k9a2r9Caoud&j>uw*=kkJkoXZf6UPzf3BIt znyzP^q0Sp&DJmVagtwxidMf-(C6*tf zmx(s;)yt0LGu#6wLWJ(pif3KHzh@DH{|mued+ReG(XQ1pxX9CVcOx=ddqte{vZe-i zTgj2G51+jf{dkV47XM4|HqHv6k$i-iUX2rNYtLB7-HOn+kl;z9(g6B53 zyo_cho=9MGJIufgDy@AFRS(d%)jobKjrkWjmDzP8+aRhNNA~JZ+M;P6NrOUt{6Dm9 zzSaMO=)Qbb&9QalX|=?@V3;u*o_&iJta=%Ql6lX=${Z}a*bDaKJr<>{aOD0M7Kq$* z>^C@jMD+#l-LH3fTv)V2`I(P6G>(O$zT%KQ|DR;Lr+-PXY^d!`YfL{j{Uc-fhBYlN zjuQ$(A0PcE67S;~(BBf=r9U@SD>P+aFSv7ajAVC=mw%f5S)=g8}G{9!DB_2V5@&L=B+4n00`;3W0e z8!M*?-3=*4ciXeSFw>UR0o;AM`oV znKM3Ohq}YBI>~pJcK5rs>OaDl``JEvU;iT%{tMo^JzWX>5*h7KaEkK0%IWG;VGPT? ze)QvyKNptZdWY4dZo}ihVt?TsFoPF(PE|0*9a3+zp2R=Ng8$jz+#SvS){0LWcUXNm z{g2V>%L~?zeq~Jmg{Bifj;4wHABvaN+V6u{pl;Bg5Z&L6ewOjRRkIcN$2cFyUy zRrJ1fMV!-#5~*mC9G~Jd3kNeq?#pf|O&US6Au;N@I15UjBWv^gX+zbU`p>QeMO+2G zp;uX4%jF~ld&huG&T$*|=~;Z8%jl0{yqwuB!>v?@^ej<-Eo2w{R+a>&S zj~Y5*Zl09_So%AKp;S1BWoLfu*xday;6hOP!;QljoDG#FvTDa~XZ;1|mtP_Ur1|^r z4IDHtg?+diHTpXyy+7T3$k+$apQ-536YhV&`x|%q_+IkWK%xVnU%2>g2ba{?+xveW z``|CMIqhjc+rV@=+tEQCppuVSToDK%dkZfzOFgczRTRcr|M$asMZOYizqHY$f$&PN z{fg!#+Z7QV`w#CVS%2jodK+nu!W{)yp{%-gU|drmxz1+&@1?;tRzq_@M)Eo|ybXTJ z_?MqOaz!zr*xMf4)^W;)q1Z7a*2~pN852BhcCEo9)pC`G5%yX9sP*w@E;gsdI(smN z;G4z2gU1tGT}e~4&(*|G`{%bG{{{HALj+?F%Lj@qw%$Du>)G~zZ&!<-DoO!};_CW% z@O5G;Q1Qlc(J>9(pvWa3$gK6JaA<5_Hej)k>Ps{->a$iSiR2SsbRbWqJ=QvWjF#0d zG+w{-GX91%c}UOc_W}98H*1Y^SlDkpb&TH)Fz)S?0%7C%r?fI}Oy>>VypRJ|sonmL zcLTVa0IZUFngfa=9V{Xa6@L0bkKC*(>+XM~ zpk?EJDavat#-2(_wYC;`$ehYt!NoVA68t>H9JU8GV)S$XEhaVDAm5qAlfF*dV$2O{ z{5buUQ@LJz=N^f^ZzgP`D^a;MWN{pTBI&yv0OPhFRau0;mFs8!r8^v2z1P^eo{Cy# z1r7MSie!uHvFP64?vk_{WkH9Mhi&=~G~Zh7>aF@}PnBO7e1pyK6Ed1jiHPgJ5`Gi7 zGV>vBhf$r^Xz1rqRn-`-pU#yL9vB7UOxsOD`;n59B{kxz9Wnt%G|zdCm+jQThWFJiy>HTy`#V%8xoo#BkB}$PyG1 z3+#4MCQp3z^&z}SAQZW^klv5+1Y6tg9;wFzJ7PW&B3VO?>T1J~*@lB+|Gc#g6n|S!ov_19UPcbFYlb2!I z!JX^ArB$}YnuzIT$$e6wemcJ+q#L@mlOG|TYim+q$5qX&~8 zN8_|tjhN87HS{vJ?TEs zY`WXXE76imN$cuyY}$I%TSp^QUvijST~|oIrZ6V$TI+tRs&qXq>t*yYHG-07N9j>Z z@%Bta$mT&Bz4Bt;+SZzZl9DKLW=X;)PYQaRjA;C*sq@9pm&){H&UXjWeuDS(A~u0i zH%7wY_LjcB49zvCfGSk+t&E1Va9*sTcx%!#Y z(|4d(Xc}0pA2Oz?$4M}@@_zU(cJk!ZB8hBK3;ri#TSz!DLK8 z@SY%~t&W}|=`@}I&uzR#Ex*-SK~{GhEr~O{jnosO`WyLC&+8jl9y(GUJMw5Qt#s68 z1Lz$&G3S^-XTLAe`kWgA<1+ENRujadCXy8Q*I$AKvf&gJHu~@-+vT!L4?(DjSe=b2 zd|nNViGFpk_tbt&%9OY%X5njKsNlJ0efiDL5^*WOj5XT7H$&%BQ`W1xFf)CYOt)Y^ zl|2Eq(>D1hGCZAagW^67Q)}#HDnp`$jr{sjpyv;<%ir6|<*XQ?u;-1IuD=R1CDKt1rDuFr3UN?T`WK;0wn;FJ%6q4;z)d3qVM|Q1{4#t|z}LwJWjl9VcmljxMBj+HZPW z#di&u4~pOE-c0$&(>5|!di!fTmK*|IY0jCciG_b9<4L%`XX>2!CXajlW#zPraIRL? zm`g95gX${{uu~^dNuhhBy8vq4KG53pl4x9}DNy*K&s#LETW-Exu?oEWY8!*Xdb|MU zg@*2w1UIaG`vx6gUF1{V;ew&ywLaSwgyjq|i4CAH1gc(Q5lqgDwhi(S6;euJ>@fSjvgU@7 z-w!r(Ce_t8x0am>8vX*bULWKRvV5E$mSr({`CSI zWz{cIFj)piOF}&#c$M)@zj}0EL!+8l%Ryb`e3=qjv;A!=4YItuonm`Dbkyf)No?@L zrl@4wLTOu)ZSB%7BAh~E$toWw2BvS-eADcvD3pymm`YoAHQ9Evoy;B4v=xpVlT_^e z+LZOgCC*2(p@C-}B7^JEl{p;mq;8>WIBfK_Q~u5BjT3)->BL2pwL`S@zL4)zvxp*8 zzA?3Ka!&N8g`-gJea)7tUDfb38r$ROIu1xTM11pR$v3XL1uj5~E^>Ag$LYeoJF?IZ zBa-`^+z8FFjg@oRf;w@so!T3JH4(}PsITwpULT%E1@!*2`dWp1MI}OP+Lsow;;l6JUNUpQGELL_ClwUR(!J0^ z+MeNgg1X(Zag1?L64yw;k#!Z4L9K#e&B(}APIQ=uM#D1dt)S?@9Cu**Y-e+Ue#j`5 z^pz+S)?h@$)moyPi96o-0)BO&CeKo*YMwawBjFzTYRV`LtVr}7WIL$^lV~VFfB!CI zh~8Yoo}&+IiPu140ewZ+fKHyJvKmV#qZiGL`aBOrns|s@LGN|HzuKm^zuNxf_XY4p zB#uvHi@|#66N3AQ4JfS$P3c3CrsbPiIW2#98ut-ozsT#Vw9>@GX^G=z%_<|Zc^dRtC_FA{|VhC)t zusWO^It{;GlbD)~!K}aask@Y>ZCf$=K`8ky4v&|if>P7=? zO-o$Fik|r>Ej0%_+&}oo?0JGMSX-YeC8ReLv&HQ!%zAV)&R$dlS7(Wfq4qwT_zbk@ zhwdzl`EF+A+VAda(dn$o5wW{6r~!O<8h8OPy+4isF!k4G$f#b!)=DaX)=^`>TVHry zW+JaKwBm5^1EF&rF5_I*8-K6W_Ppu;%zNpnSSTgGBzf|h-QbIR3kQyo=3Co$WLv*{ z`g#^aQkK`#MwI-VxLsBkAUO%xO0F)ARR(*NKJYOqQjE%TY4zIW^EV=)kv>ti9HFCf z9iceVu=nz0j?Ko1_s>Dr_&ibdB;?(MsXi$WMBV;eT^c7^lX27`@FF#_3%;0EuDO{u zBW>v2iLX^KJN>TZ8;Cf0+s}UZJSZ1QFQ@z3@eS^HC;TVA6ie-O#KkOe#tpTzg*4 zkQG$tK^r8GQZCujf(`DS+6~EMy)4UW`g7ZVpaSWwCHD=x;%|RSRtViMzI1%ad%<#N zHG^^3P|M#mo<&6fBL}0mk%v9^&b0=-bd~~yri0zii?_Lzlsdr;jq;mDO~<4hDn`hh zVOuvWIfOpZ51zMptf*#ys~973=&KRWrk*}SC~wu?8fGmZlQdJ3`5=P?`L}>{3NrjX z6c&F!dZXe0Z^zY#zQFyS@g^kwgBEBSvnpUr+4thi9omZoo%@BiBrJ%?xu`hn{K#pT zR7049)81-~dKgH?p@T+uRB8%=!j!637hbA^hS(P__5mmDThZo+33fNhQ_}`QHI)}( zaOdAJ9PZwI2b-GdOWUhoqqK8hYoR%t`_-o~?}y$hh?F01HC_Oz)QA`_0Uu z3h~gm$@fRXVQKj3&$~QiR(!*Cg|Oqn0h!^J;wqE?)gaPD8wY?kE-0?Ha;#BXVT{|$ zO=i`XTg7jumOV!EHl!s3S4b=#ag^32cy1p}YTqi}O$+M})f z0ImZ5T7(}b_SP@L&OjhmNub5;2R62qKL}RQ>r}=yZ+Wt{-foPOt1nxgQK+@PuZ{kQ z6H3rx?{spwAN?m3Om_d&^DYt`{Sb${f9>sltLnq%N4Eq%VsGOPCKFhZ^nWfSt+7h! z4gRH524ty})@uCc|Kk1r@3(QZU)=03x3hFkw2`w0-B~>SypQCK=r`)|HQ?xzW%L*CD}ADlsariYHFNUcP^Hy zW#yW8>sT!GZ*cx`8})Nx_dIw>m<6EV`EMB-zyFK>us)H=a!$_&ymn~b3Ah>b(o;ZH zp?as(KZ^H7q=V_qxU27d$s8}1md=XB%DEnVdN0z#cSD)608DurCzMpC)%PjE%F=br zCr*^LMUqK{X+RL?!XUq>z%3r`!hsMQ_1)dIh7$j#-Oq}bo@@Rqk-b+v%NAAFt-qRy zITaDdo>ZdsWl6-M=C>(dRz8Uxp>=Dmu9q*O;H(`2q*3K0RmP zK%Yad|Ep$v`T2lrvBAo4b$yJ>)s58ua&&II+y+xUT{H;y z8um&J;;Kw#Tr)GXM$w{nf}RV-{qT%0fb!bn(8vo z)M+)9s>(^Ptz5Hj4Xv?hn6FN!b{lr=iM`=x@qjSQq}sFN=hnO@R>RU)O%}a=pVoiQ zaN&W4hHcYymQ<`n69fqzw3bMZsB5k;YED~}DXZV(1l3JvyTsg6TlK*-COx`0Pz)&c zG?924RNY>*Iti@tgSLv5mdVZ3Orxf%a9_WLxkQ{j85TZ}vpD=W>mX^z@L{OfP5M6b zy1i(&xqP9;jmty7GaNoX*xIA@2EG2D17WV$4PD}ajg%9*8gh=``xEq}gp8!6bwD~q zB8Bf)bgsd`G-_Ec^Gixtn>RL(U8y|`+}&*%fGD~2lubgMYf#db6*mxopDS}+g!`0o1A4B0J;&D7gZHL{xxv>aCqKC+lJ^{V zO~W23kbO_Ze4qDB|>`6yR%KDpZBCFvL)#V((pAW#;;LAZ71^az{o+TZWf?&%_-oDd$YXK>sSP2R4!JT7b zvj*E>A5ypyv>^7^`K6)H*;bSpPq1%`;Q|5!@6cmj>@0nHFr1nLcfd>+Z9`_smNuBn zUb?C5T_O^vqH>6V$UIi|oR*x@^u|Ls{ks)ULVC75@>phB%YUulSZh@RRpSz}koz6f z`&aPqj>j>IaCbkLPyExDIhOu8y_jpIuiFafX2o=KOMIskF1*i1H%2MX!|xnAW1HY= z=ay3SmHsv{bf(0pL+A(%rHAU~3U#T|t)9t3_VW@h)sG5G?`yR;TlXy?C1kdil(X{< z+I)x$Mo`c&E42M~1C<1Q=4FV?hVU9tA+T15SmGOqWD2je^t>R;mIeD+=mmp(LB5)N z0|18mMmH%2SakfP`cc1S0s5FCr$p7EG4tFtFLQy9gmR|Z%rm7i{~vE}8P`_#wGCEj z3#C+`g4TfIE$&b#P~4s18eD=~OMwc~7T1*G2?U2C0Sb4axCRMUTtc8ofCQV9aQ~lo zp7}j9U*?_R3nA>Y&p!L?z1Ci9UF%voZHbJ(0$-XA zY-~eX*b6!1njG^pO@Am+c6{zc0^rdmMXs&es%ZvK)kDHEO&d&c6UL}TGMesP`^9qA zwGTQDN;YHbj!HdrdTTSvbPLo8zwbGsQUxfrY%*XU36+G^T%&XN!xO!Iq6Wcyvgsf_>j{Gt&c(PQ_QLHsp({4}V z&S*>(W;mI9B!2Cy)3{BypK+5VlJJQItmP-2f^~$PZeziY8(N+5isi2PcM(oH9m8y} z>;3?%Mz)C5xb*Zr)ZU_URZdSy3Lk4%I*}&v&*%S0*Dr4xN?z04iZ!Ypb?mCX3)#P3 zAhq%A<_!+|znxgefhEBb+wPQt@2|_);cy*Y5X{;UX>_r|tG7^_dJSFFuD9=aQfmi3 z>84dp6_rDkWf)CAM5!V)%=O3C?GL{$q8TOSd8Ax3Sx|aFS{?XTjf)Oa8!^rYrlGgj zuJQSjO!bfqhN8|ju-8@x(og_ zQZRl>n;Sq|>7<&plexi!0wD@KMRAu>|mtqAKELPgW8l&Q4 z>^*e1Hp=*{oc+BGkxH>*y30L66Y2blLwP!doC`vmNl~eSX+dlfFH0KD5iF_*R8dE; zGpJtnX9@(in~NGB`)*+Wb`2mMJ!Y2P>Ama6qSA4B8%t?~;?6kwx>MDhUQ)NJtvgv} zNEZO}>W+6qLI{ zTxxZ%jt@HaGB%Bdvs=$J40E@obwC5>bK{WcAGWI1>3WE%yvBIPJ-*^eW-rh5=>p3X zOI1#;X9m!X+%4vHlS$Vqh zuPxoz`fxHSt8MF2B+HS+IW*WK!JZ+gt-HHLR9{!m7;=QEb^Nf_Jrc9Npun~l*6IgWNbuJ%c#B3U4 zQmk)`y`V_5*}KmuWmWwJDKPQk@Vk!I2d}-9mq+BjJ=EOVxPQ=2BD$k0!$o*j`=8zH z;bb)LdH%$Ws&XufyE6OPik%!yU8uNN@6A?^m=a4qUUWhWczXBk9FR$sBjdXp0<*>@ z-fXDZu&54&FRSY2AvN(~w)mqHY;EIYpUYH~er|4@?&9ij1cXaT*fv-ck-?!4R@Ij0 z%U6l#LFfLc5vJQyN04_#wMnjtm@uO z+g-MlXja$Pxp&^Kf$jB|LsP6ezuPU7Id6D-yFRHe2`G9PVuG(nWV@lR_drE-?_kvW z9U$&pA>LI+mbJ^oiJtxEK9S!Eh(<2l-?lw`^D1@(VF0J0t8r*;D@IRlfHxG|9lcdw zZAjNuOqW#l?CAbb-@ zJE~&8v0=z^(n*V|%P=YrMCv&sM)NW*ruxra$atQsV_N&s$EQFcH&wLIDb^I5xe!`i zYz&u!SKGHPIin|Uj>gxr+M7C;u?)vQn=X~>E1E2Tq*cQj`Et?3ME!Mr7Vy5*HwQkL z&5jqx&_otP$72Susupo}t#G9M8VqNIEYuwML*0|1ovRTNcbCUFU%-}0Xk%T=T^F?X zy;y0luuP&D{a|T~{F+aZf*3(O^+lD^8;vhu_9ttT4Qii8N790(vbbGyW1#wQH^p4= zMa^QvGWjkQ!;*>N9D^J!zO_!8vFzldrR2_qYL}t!G(eI^mmqeX!oF=iCH1BvJiAQ0 zaZMDJNNq~3pT^ey-JuG$eMLvhwzE8-{%*-+4ld0H#-*9lp?jW2cLY{9%!74566rt) zR>5T|SdyLz^yedf^u^C!lDi<8prhM0R3;nUQMv~6p~?n4uR!I%EUVuE08Hca#$yGR z|KKzn1k(XyrTI5GSwzf}?Uw&e@)zqw46*_Q`T#@P_zk7Yhcm?_@@V3B^!)!7DF6R4 ze1Gm=;_3U@yeTRtrW=ib=?pN7`I`6=0W{5Hq^R(@sF&fPL2g&gd`*%$#X|lbw|Edo z+{OrSn2WN5*JB*s?UE|_C_Mq<=H}Fw{_nV@g$0ETYAgbHQ*=1*c1@$(_e4rY;nEMk zX*|nu6XtAU6YGlOX$G(!z}9GG#R})$XsV|?qZkX5c__t8ZqbkdRKg`39;)JVb%D~p zl+(^vU0c1|b=I{D~dD%cY zp{%T|tEZP}X1NKFvJ4D-L7EvFreL{TWdmf0*7cviw6wpqWq}uYt}PTOv;eTBwa>fj z>gzYz12|#RovNy;@ma^Zy1H_SHLiMYpxHUADDd*rEZ(Oh8Lpu( zT1@{zMQjUxmNFpLNxe`4I5hq*_Z)XDAp~mirf6ttx(9FktN=>>nr?@eEnrEd-y@-< zl#nJfh>8ALwSnz86QZ&EKY@vYqrJQvg6gyV`D`YSqFTEXBQfZXf$j$qsgg)hx&kKh z`rh7NH4!!W+1F$7g+g?lcSr>eL`+K*W;GB`x?U@ZsZ}?codI9MJf4zm<8U#-K>&?{ zkZ5_p3&A-Dy0&_cL5HBdA@>Y#v<}~9f5>m}MSSmuDgOA8Ul0kN(QFQ|MsjnrYxx;! z2KW9ZK}&$L5w|bmBN^4!M$Fo&uT8j#O6`<(;bLlRc0a_*LADi#dKzSBY7q!Eh&PVy z=G9Kj&RKLrHkr1Z1*LX>z@PVDo|iB8e@oRo%{4pX&3kiR`xax^o1=C_G-h{*r3*84 zS|0Q$BZFaWZ6m3VDL$5wD4?4eou-8EUoC*a0$+uQD3g#97Vf~a6~^zbSSd?ge8bOp~XUmrPYP~rpJ3?G0viKlLfMUPg`SX{vg>B)U zV>%3RD%VdIuo*@ralunlQzzHMsAJnvQ&Yv|<-FlCkNl9eB(tu01z46$1k`t8PxSnU zm&50+%aQw7-^~u|!AT*i8!v%v74bK?DP;8CLoXvh!<@SzMUpCip0i(Pwz{}0{qQq( zU>b!=2-#}}a1ON5upd8!>w)O^E)mT1pVsD4rL%rdZ+Gl{-dtVn=$i`Jg9vt0MYaNf zF5;cr?*7{+2Y|wgqZ|Y_6ACCVNjb_8fE&MO5t&^!-&Rvq#diNXtf0VNf72J>-~%rE zchx_aANxDGykjE^0Z15U?a&hw0?(g+^AbI7aR1|M0Db)wsO($SH9qU&bl-JU^FFNY zF{utk8yQL1AT}2MQ*m_hEs)oAF8ksAvC~JI|6R^HHrTd@dz94(+WV*;k3{^fUarqr zKiC$uXQD?;EUrHn5DX-Wfo%%pI^d170N^kHN-U=WayaGukQALgXC?hexTYq2*4F8& zD^IVHrsfxVTly|f99?^4k z&gzU2{kc5N1$+zkk6WW#)AOdAEoW63tnQIk@pH0q^B}k1F8MMCrdQ|ptBH*Np=AK3 z4ETM&mHg%5^o9Lx4IYYN(XK)NTel^*{WQ0$!ft^YNe(yM3DDPl8&OC%{k zx`OlnR8sfudD5d3zvp~XYvM9a$x30yo+09=v_mb7fUmCpfWbS}VdOvs5(8LEmig?! ze3valw7MD`wq4eTlKJ^9u(5?yNJD*ma%U62JlFL|0oPOC{7e!mb@Hl1G{$8AwZvJ? z5ukDpanZ6$1)PBBm!!;!GzhD_J`C8tIjbhA8TW3>A}P{f?*3EA_}h-&!Cy})r>8c@ zFG^eX;lp1oT7HG&X5Cgm58|Su5UNwoh#~LNH@lYa23RlFPkTQb2@f(cIh}KRdiOi>>oed&B43ls0rAt#?Qzs$|`LmDUz` z+;q2J8h$Q60$YrK+i9cy4tXFUK7)=&9`E3*nZ<;Gnx^e>sGJKhhTi{AQQdo z*IbXC$8DZQY7u68+^5O}vUOITN_`Ltr^sc%Hu51{19`IEj8A?a2G(e^hyHr-FQX{) zaZ%riXv)sHx8VYnh5S%Ge>$d^<*GCuVjzPYwc^E7hI=pTWwu` z_bryrIqpcHE2q^DjVzv(Q%x1|TO>YEPi^-!t>9sFe4xOl!WYt*D1`WX@NB1jVmW-) z;Nv`%`h03#dIAe;Zucb5ZE8~~@hNf7cBv6B;AQ)I{R+juitOe~a~+VH8p~U5#Fcnq zVfH63u}g$joUE-R@q~sqt~Eqq_D)tji$wI7R{_?!c)RM02Lg zm*##-B@R+G#SO;DiU^oc4@^vh@=qurow1GX*quXtJqTZX_Wmu<_iiu}CQ1Jz%i_)qaI zN9$A*$NBd(?#Yj}kH`DPK8W$lHrnxTj1uWlOXiKS6G-c>G1*S>TXe#A2UIK}r^T0l zr+V0_q^zT4f<+K{3Vf;QUJd2`UHf^JOm`RJ`7s-*_#Ew+JXMhJ9%N_QGAY*^o$e=* z<|Sa$#rsvaK|?5#XR`aU20FgISISP$zHJ0!uXl|v1B7fH0+#5XRiO%PCc-~cb$?2x zv!q0d7gjt4e2ku0NoIJDkR{N&fNu4F9O~2bbkmu7GF>=RzNnnr^MG*sPcld7nOMAs z9U?;K7CGt3?=ng>I*kk*EL6hSnGQ|3sIU8<|NDVgfFoK>f_IYgoeB0t(`2T8RJZ5y zE#%(ft%NkoyUJ^#2$7jxHu{~y{oZ$tIC!Op<6GE(M-VA2V%3ywUyhEoR53^!V z%;d+*L-is+SZq&9mte(aax{dkJ{&Cy*dW0DSuRL<;3_%QJYrXMNSCL=`ZX-Y$tgy?NDF$Pcsc=OKL|*v*cF?%n=Aw73Ar6hd-I#L=jR&EE;ZIUEnY0D0B^x%04lWzGrr4gb#qZ zOrnAzN1}zBGbvi@>z~!Alp@g#5H-b@74(NcWe-7!x#;uGlM6$CL5e{bJ%u-rm!SRh z(oEd?XT2+)b-2?x^oXloOOai#N$J5IM0M4&PScB`o!)DO!zOOsAjpKMUdX|Z z#_ed=)#J^8*pod+Y~8|Q-%#7%j$3U~AkvAabR9paHAseDdi%W&ZDQB08pkA8DNvBq z#GZHWhHNnpuKX;zk@7QQYWyTVxJfFX1`~!h_&S6%W?*K&=-v(|jJnGQ1?$BbQO`@+uiLT_s3aW$;w>9}0tm5|(N;OuutI$(!s`Y)e ziRJsU_>=FL%sVJ1(@lNh%ptc}IGeGpSh1gWCAx&plws&oCLIdLQ)zRfaJ*rpmJf&o zI&3UI1}5D=C1&sFKt9MqS%)VCM^RP%l!@!`DEM;gc~9PDiGgw89jdP|)(s}xLr zHPg7$={&0Akkf3crq#z&o?Y#@)@s8|jX0L-&w|Gc;|EQ@{ zd<_}Oyf-h7jqe}`#=ga7o@w{-xK-+<;JX_Fmxi@wlhP8k40ptrw8OD-ZQbw#AhqKe zhd1X!bNudpB!BY1bAU;JW$krG|9h zIY|F(1k$fZ(?_#{Ad^X`O2b&nYJr>|lYH9q+wV;imb*>O`1%aCVZF6>N*~=m6em(;a=(AI zM>2Xt36n%+M{`}$(tA6teEgi*R(c-q5~QB7TGpHHV4`_aJT~FAF2n*z3{}sluUMN% z8$@35fPu-^-r|B?@xe{_kbg>cN^Y#y*}a1lV>6c{-k& z>kzM=+{(%c-ph_6@CIji>b~yt%}eR?lcWiNmErUko_Z=Ji{4MebCYFpz(0xOwzx6oYleQy?+LuEUPdGDRE*cRyxkuZDnGPA~lUUIeT1 zlwY!B_C5BFM<9!HZ#GtOdb_G&lxV{}FISVkXL4sSm3Flm#1=HatVpR^ms;8Ct0xvL zkli;K-~V$p(AR~gbw}a({M38(gapZi0}wX&S@ys!{h=p@&f|$ASr#+1e$+IrhwXDY zxl0tS{wEa5xXyc*QPf`rPuUCWZ-|;6m|kCW{&Xmvpw2NurX#tuGP<8HAP!EH!rGsP zIbg?@J41{#(!NXLJCBV#y*0QhVuIZr<&oDM=4mWYV}AUHRh`#QG!+h60%5M?uq@^R zw6t2bA{0z}VOmw6tA1U32y{uSNSgs(f(=JkY>`@dKAW^%iK z8L4S&=XKZERdx*rs9H(GF-VLsFa_e2fhVP4W24GyNR^x)LA zkx-SrE=Z@0!tA>~Kr?z}CZJ+tI#}Zhl{d`llsjlX4oJ)N;>S=y0}_waVX^JG%zrcY z8}?@?V$~h?Vt%Z8$(I!zeI`F^YiJh-M=rp4C zG@X)7?)K-|I(Yp?Wnm4c{1l9+CZQu!z<{&$VfI(;ajD-dvkfIG#W-eQpSfo~$L1Wr zN%ya|t*mFwkMPVE&HkBxo6p7a_rNrl;gr>qXCLSRE_$3IC4KHi6QceZrx|ZPk}7YQ z;rhy)rwoky#bv%z+sJbCp0wU7%_ERks$rY2ky2R1hqF(+a}{=VvJZ`!_@9t;-f=@a5TgJ4m3estPE zZ%n#O^dscL8Hy!Y2YqL;LA1d75VL;jYG*flrCn787d*4E=!TOb2CBA^93F2bR1BJC}Z&0Cym$VbyZ$wiJ$2!N7ZBMrUVUZXyPjy?h89av`Hdg z+LyamIP9C${NCoWkO?FgjW8!c5wOs{bjPICuXa+(x>Gfn#LW{Gsah=e@ls3=Ev#N; zGjKT#g8ld>u6}v&nl+}wo!-;=&hy1;n?5Rv4qUau?1x$9T)rJO2X)mLN8{svbV0ho zzLCI|ZT@g)k3i*TS#GE>A2$T}_trhDKG&KX{MnhwR_Rg7*2X7}wOiIhI@D|{0k7y! z_tAJ`@^{A{Q2;bvydPxb-U_c`3j z(I6?^O)%LAdlB3`~TarkXe!laTxtW_>X zh4R+ZFB*l3?tlN*iM~zfALS!2y1I(jCmTjBjWK#tb5q(6g}*F@XZq}*FH9iSopW_G zweABiw=}92fyr>S>*;_7(SLZguunMuPnk9EgA&C0m3yAD`0#j0dN!e(m6TcOZuk)Z z8y*@{MRcos4z7<$OIoXU_K4>6h~%@?QDSAQ1*-jerbDZqnYV>IeS?;X z_+o=heK@>L1v%8wmE+pcB4fy~L;xWGW<(u=Hg`Ks2fzADK&~HTcgQF{V2!FXAT-J-kjfOnX z4L39pw95n3T~}8xhv%0JA)*gSy0NOpCd^liI&A5)kIY9a#izGT&$(hcg5ApQziALb z5|L^e)`#YOIZo0WaU9Tl{1Y0rW4a$HziM}6xB7NWp{JZ%u0Hj_7__tY|XU*l-vYym!X@otx%3GkF1@qLbku9%kK(&s)5 zNQdr=Vq^a0Hq}ztawaZfIp^kEzaILz6qq=%#VV8+&pS_1V;o(O8L!WL``NxxZAa?8 zGCj*|2Jaz;pZ%zV8Go5P5#?y{pHH*$pJIy8nMC%%pEeR2s~)?t@Z z#$O1yGiuGtHu0axDmk2XK0n_cfMAr2SfP==m z$ofr~`In_7F1AH%3cgp*A2O9N@N``ObLleVBd08)oIzK8 zPwIWLSo+HI>9auBTVdUcejH!Ew&ksNxSxpR$L7b(o!laSjiS+tdJY^c{%DSSZqDI- zw#pb1tsaIHWgVbDoJAroyR%=}PLyEH{Gi-XDU!r_X-61;`GOWdXF@{1D&WRzzN0CA zntAA1yO_Vzy8nxOo{JO);C!zwK?)uue!?&R>3SiRLD$G5f79&WPKl|{z)!4m@Iz=* z1}RiV{98Ed|Iyw1=UpdT+UpL(l?nPq6Pq}Pp4FJ!?SFE&KImkV9D`?w?Spsrkm%|fWi@!vgek7)t~}>WRz5_vvooy<@L6vB z0nXdYzH3hc-C)&pI9cZ01qTruIZj9RcP}FCMCX2zKup5 z+T2gPv9kOhiA}%FrhAOgL66$>!PO|sf!7wm+5Mbaj`W`oHCVT)^shis#aK_o-M@#6 zJ2aW;4`l;|7@YwQ%mW%1%cd1@xynQf44P_$@34W3Q8QM>2zxJs*mbY{I*c+W_=#;%~4lv71@%A#9nhH}| zcx=niy*urzUm=^vIsW-ViaU<}e*de^VNb&kZ+Auq?rG#S$it2F?d4>v*3#B!RBx>P z3005gLWzRi6CKy8@{Ol1<}p$WOguHCvI0f&8`l|eJhobs6#mxX)u?tcd@(n#P2fd> z=5MFgBVh$TtL#*z1KrycY)R=FGv+Pgy$+Rd1<(_BIb|j4YczYO!%>Dy@$5a4IyNIi z2rEm=*RoF@$aPtg3oBB3Lkjzb<1n$w_)AoqM3l4vVbVlxX~yC|g24JAlzwn8Ll!K$ zsZw0RAwnNT==ONBND0ea8?JV5(reGFgzOh6(bDPI^681tV#nbiB*Msu19jYdT(|ir z{_(9QGOCTy#@X~WIv5R34DB;B<-gLO?0+R+cSI*2c3UCnFp>r|1taW+GX8Ld@2UNO zUA6Nv_es0abE)!su9BdnWFd4o|~fB(nPqoy{0s+HLowyGkSer58*dp~-k{P!U6 z0H@cof(CuRP-Q;Q8{P3>kY|cvO;KI7^N2)1B+dH>fq|*445Xs2zz7>Pbh!U_LEf_BuDNg|j(SiY;T0KP3*FwS*7J}F2lric>inhbE z+&$zK{?8u@avN2&r>=O}+iQR>YIovIDCsA&i-QJJ69mZdpy>TdM;~>)DlYHC6u(*r z+~!1f*j9lae|(qJNY=!rtG@c?w>I8j5gr@Vj^szp z05hAOJq-LEm(t?%QZoS^)9c0aJyBQooVi;u6D?1;?JW?o(J6~cOK}|MBA;5>jQbz zJdM&qlWvXdWIm3{W@_V^-q;7%*2*2Zhkh+*AUF{9uhQOTqlYBF{q58%hP^K+_s;DV zcZ^=^v;>1R5g(={{Uk6sZ36rW+Qm}vy`MkyPySh`w9P00r*;neopP%Rq-QOIJYxU` zX7MSvMWbI6dxdr^xvN>nXd$ETj))4JXElN!SNU^w09nt{ZPLA{z@2L-u=ZdSgiz*@ zROs^^2?zJyDD({MH15srh}y-NyUJ5yz`BzA%I8uCd#4yYQu{`5eLtj^(|sB<{jhSP zaJD7Tdh(h+i$OL9(WH`O>XDflbX*HgUqzkBKsT4Re>+ZkTF>H_r-e5Q;ZfsxRm&z> z7=*Aw5-6xDv#hP88gz^wm5{^-K2U{hqArTDRIN2oRF*v|@|`JV38SoAIO2n}`7N&! z=(tB8cry`AlJc|isEbej?7wxIP+BzbNQp85!WS9|BmD99r6DF2F-TXt*t+rrQD>W1 z$5s{O2fS;O^=9=x8$+4Z7R){3ad9!_hHUp^VAYLsI{`W`Ce|>@p(|CeD*4`nQd@GXk51G|#(;_5edFhV_ImZ2gHA~` zWn0E37X~ZrHd{}|8V(J^Zg0{&(J6%8u0ZB7N}NEA1!~jPX9vMmf=qboZ;h~5t14T; z7bPpwqX_%hK?y3KXCS0rzQD3^a}s~Gy{?zmMD==qA{LAGw12g=o6U0%*nY}PqVfuRacKQ%;Vz+D`LW3bgp8q^zrz$zwN_Dj@2p9cKA6 zuxl&(@CvM-Kx2X1SRMR5_U^)B0@%e_H>|C)NeZ#`CWk$ZuE*<5AawhURzeCsF+Sm$ zkybHe8JY!2u)ufMT{%7<&e5d{#w=)emtoe{N!8x>Bqa+S25gejb0Mm_AIS!3MN0MSZ^Wz#Z?*y{ zSE0;j6iZ!^C0ff)PJ=p!f>ZTAIxb~p9+oBn8$quRE0y%q^%9u$%VpBm0>Lwls$zDX zkP(8G>n=>UikvxbFu0n^G#H)DR;bm5Jq59@S}Lh>!a(3n1!`0F|asD=^< zq+*SOq33D(lUW1?ywdN&K+pE7xy<8eiuN!aZsAg;(mq*WQZuj}*I?;QZA1 zWI0te}%S9+O3hs3>UbQw|CwoPn!KOVz;Pd+e8&!C(uw$4J<^nArE-`Sok7$sT{^PE# z5B#Q2(%#NPL^Ku27eiiBZn_Q~*P~G*?yu{`2#Y$!qgD4x^!*lIpYQAhB-Lns{~Cpt zo_1qGRY|2@be1A^vQBvdEn1ad3@|kJ!xF*a$B2l!!v(mbu67yxuZhe>(EP{hB0Xi< zS1Dh(oyTj<0>u1>rBXE&Ig2_8@~Fhi0wa*uY7lsOud~1T{v)i2aM5cFz>E+@%kv7_ zH4Wgagc0`PX4pJVv3(|;!&3MVcb!2R_^no@7Ue(=+Gl+@5gA!oQ>-kzTA){C|JNmx z7I7Wyw$CV0nQwfDT{8%j((*X&+b$lsh&mr#iDMu3j~N+Vda`BjjO=8s z^Xgj)vX!{uSKQbNp`2C)$(DJb{?S4s{qn0fHixH=Bl*|#z^!ZG@W2?BE0OW7X%M&1A#Ct}E7x9d!#GZg*!Awnmf1N96s#HIQ=M!KP8%hCK6m|XQE%^5= zrJn;XQ%zN>0X<&Wi);c2qr#=m_bdc|;I!IF(Yn*U?H{o)k~+$9fm}Dfu=~MiB3D2q z9I2fVR`F3cu`s7{u%4}mNAi2v6Uk!JQMUzS^0-zo^crS&#E7rn5EuPaXJWXZ^Xs?5 zJ#d0eWM#-hc)7GvfX~Kr+)SaMHq~(lZ$`j~r~oj~Ym&gLPb|d?L2$W)Dfnm!HN_YV z*k2T}2J_${;b1?|+>qGPc##I-mlP?6b+$|DQ(O*X75@yyxEXfKKzA*%A0V??*u13= zX#p}Wu4G2O_Up*wQCb(jM8x<4hl=MeCU)t?sFj*Xtbo+vG^_rJ$j2SYHjibO<@t=J zjIjOQcEiZ;Pr3pvae|}Tew$_Dz}0asKecH7Hzd~N;(!b> zhS7Y@qIGZ1JnW#bkS#hY%GV-LQSrmt+S>5g*vc^{56`|)+r(&ldwYUfz5^TJCJ1Fd ztNYx1P4oJY^AAZu!KLG(l9K(1!9lGOxueO%Edc=mu#3yyv}1k^*XpO8heT*#v)0+f zn*XFRu9%BmdZ9TNi zB2ZTs6}Lt)UHz@3px|%Ik)_$rnGuFML!B5r|Mh3`#+I+i8|h|>k=RJeMdxHGF|mN* zK;T>+s!j86hjy-W;ER3}xF~GIk}m)j4wxDl(<8x03uDgkn{UgIgJVY$X{rH2PF1bi z7OjC3O4AYB>UmwnlEHth7XT-<@|+m-zcx{08vv~;z1ot+%2;Vz5!1$+ud9n6TLx#D z_fvSO=lp!BA9TDIksW@^;5nY##zaI-Eth!+3}Vpple3vc&D+%%rKG28g{LI-ZN|+k zvy!%?wq+k{>n+b?ixaUgcGV)ujf#v)S@xzaZJv*{P&sdY2b&}U%ZHmQ1iPNKst~j=M$_=E@6YZx)9XzEbS>b`~*p-HgJN_M?vSDzzV2OCu z$IO(Jft8gXM?Y_J5zDNWK79M;TW6PY<}jpZ>suEs&^O@Y!sQ`^zpPhRg~7+i--$>; zbwQwvx3=|?G_ta?cU*SjVc3jn6sZM(2mt^oG z+kk%vHE-p*0zUj4`|9!Z(K;{VJRdP3>`%@a*53T(n2YttFsxBtWu?xYJ9j$K?@olr z&ILO;sgxP4yXgyf=>t1CRQunJ4lLtEXCc1o`qY5|&D!eKqd0y^$qi&5+4e6#(w5{U ziKTNCu=|-4BkPNc_(QVmR8#?=KJi;I^S+OQ0WiLt+-0tL;*(pGylLlv_1@ob|MvvK zb}3?;kB`svxhu5K20DRop4<8%lz|PHZsY;B{SKR&EI_shj3tF9p_lW6@neyHL73aD zhrkmFm;H|z%-_aIWD}^UsS)o|u4%02#M~6whG0(3)Ofcu&c&=o$gL_Xke^m!(zStM zo!^rPOa7zt1=kz?Ut%SnGu0x!!O6{Z9h`T+C@DU^pso&0G3k8H)%Yjt?Q5>30JL`% z4udtM{X{n|V*@_!JS2IghnycFr0j_-Nx)L)t^8cG9C}{LSw&6nmowdgTpXg~^j~Q3L|_D+pmYs=L8gI7VF{qM z8*iyT!<%O9*dY>@(5-X(Fv!ECJ)QneCv^Rz1<%WIW>=P;&@2YjoY7qm(83 z63ib7BQPMVOFg(YnKL9PTVHiP_Z};>*LsJw7oP;QIrryU1=n0fOty+(n(I-;xvAmq zGsBemr?;o&q1W}ux6_Jz8JWbUzU2T;+W4Qyw?FR;COjs|L*m!}QBcH>p=bIpG)_Jy zbA~tuPUanMcaoHF8~j~M+W6-ZBTP2FsK)&Jt(&LIEWmmSAT-ep($m*HNz(XFjzym? zLnKL-Hy8HQSBta>g`Ulwmb=M*GAn|3g&TXRKA&N`-z$vo877*QVHDC}o@qeRONIaR zEG~#E-FN-BNA2~xw#?sk8&3{FL}KyKIK^I*nr>u-AAh5cXvI^6ixUOx3`SRL4s1Sy zOS2lKe^<6_{*}mnN6X;G7Ior%g~!DQ;Pj*|N1ycT22q{>%U6K}$`qarloUH)+<~VN zTb2%=-TYmJMc~6f`vh&$ofppz46NxhoBGf0>#!&68FdjAceBTtKV9xt^t;6+52btt z8DgT!xvtx!$-XQ1{Jw^RqRIyyr4$uCYofqBmq_VL_mT;AEI09QOjILB$Z*Ua(iar z>lL^zV*EHoiv{!Bou2ReU!tTv$N5OIN&Stq^cT0!Bz51uTzF}F>y=W));k$n#+RO! zV08`B=3@gUxEGSpYkf9-fibL~Yw{h2D%@?lKiDhsKW?I&@0cTPk&Ao&?#k zqD@9or;-)z6iI~D>#q|%S%Opke=Le_!!_ED}#eMvbYeaWuXk03?+uajkBD9}8 zDjsNeANY=y399~6y7_uADM8{KtHfdMbm)xPj|jiru?*5YvNF8&C5|xv@x;`U6{zp5 z?;K0lBXB*DhZ#^nx({y(FCk0+l^f53dx)a@oSKc`M=JX41aW<$U>0_`Ny|MwV{kS@^+KfoQ(I}fxqf; zze0W}l{MdKvM1#`{1!Mn^|&c{RzjbBo{amgcrexYH{JZ+&D5kd-K(^*s!|Y(L zhtSd^HNX%czZBHUpMUUM@_093X4ua4DAncY=O*gK7ft?y-`Wd}#Z9B5P2Lrfhl{Ql zu_o6cug=~M)bEp?I&AM$YLEI-ir~%>&N)pWoh#}!P+S2Tq7FU%%GNFvoOdYJzDRAV zFz>xcUyi3_B8?ivb1NNP~(bK(NHlsY!|`{)8aDk$qtoLz?# zP{E$Kt>LpkqM~Bd#!!vq!6uH!_-DJOD`M!%o~`jxNbjS6>~2HvJ*ARx0z z^W~{_^gKz#l>T$+?IYC~8)}${s_RK)&C2O94vH~5!7}YGCoKYK>=4+E57{GPgxMNpAADplAaa2xkPk55h;v#&((qPIerhH3`;2AYuA2e9Fy;KOm_%6!7O zKTw7kc``H8U>b6|7t-Ir`w(=bZM#rnh&|(^93eWFqq})7a=4IiJ2$Os}$wOL!@Q

{children}
+ + ); + }, + code({ inline, className, children }) { + const lang = /language-(\w+)/.exec(className || ""); + return ( + Loading Code Block...}> + {/*@ts-ignore*/} + + + ); + }, + }} + /> + ); +}; + +export default ChatMarkdown; diff --git a/ProductivitySuite/ui/react/src/components/Chat_Markdown/CodeRender/CodeRender.tsx b/ProductivitySuite/ui/react/src/components/Chat_Markdown/CodeRender/CodeRender.tsx new file mode 100644 index 0000000000..3fb833c90f --- /dev/null +++ b/ProductivitySuite/ui/react/src/components/Chat_Markdown/CodeRender/CodeRender.tsx @@ -0,0 +1,78 @@ +import styles from "./codeRender.module.scss"; +import { Light as SyntaxHighlighter } from "react-syntax-highlighter"; +import { + atomOneDark, + atomOneLight, +} from "react-syntax-highlighter/dist/esm/styles/hljs"; +import ContentCopyIcon from "@mui/icons-material/ContentCopy"; +import { IconButton, styled, Tooltip, useTheme } from "@mui/material"; +import { + NotificationSeverity, + notify, +} from "@components/Notification/Notification"; + +const TitleBox = styled("div")(({ theme }) => ({ + background: theme.customStyles.code?.primary, + color: theme.customStyles.code?.title, +})); + +const StyledCode = styled(SyntaxHighlighter)(({ theme }) => ({ + background: theme.customStyles.code?.secondary + " !important", +})); + +type CodeRenderProps = { + cleanCode: React.ReactNode; + language: string; + inline: boolean; +}; +const CodeRender = ({ cleanCode, language, inline }: CodeRenderProps) => { + const theme = useTheme(); + + const isClipboardAvailable = navigator.clipboard && window.isSecureContext; + + cleanCode = String(cleanCode) + .replace(/\n$/, "") + .replace(/^\s*[\r\n]/gm, ""); //right trim and remove empty lines from the input + + const copyText = (text: string) => { + navigator.clipboard.writeText(text); + notify("Copied to clipboard", NotificationSeverity.SUCCESS); + }; + + try { + return inline ? ( + + {cleanCode} + + ) : ( +
+ +
+ {language || "language not detected"} +
+
+ {isClipboardAvailable && ( + + copyText(cleanCode.toString())}> + + + + )} +
+ + +
+ ); + } catch (err) { + return 
{cleanCode}
; + } +}; + +export default CodeRender; diff --git a/ProductivitySuite/ui/react/src/components/Chat_Markdown/CodeRender/codeRender.module.scss b/ProductivitySuite/ui/react/src/components/Chat_Markdown/CodeRender/codeRender.module.scss new file mode 100644 index 0000000000..596004846e --- /dev/null +++ b/ProductivitySuite/ui/react/src/components/Chat_Markdown/CodeRender/codeRender.module.scss @@ -0,0 +1,36 @@ +.code { + margin: 7px 0px; + + .codeHead { + padding: 0px 10px !important; + display: flex; + flex-direction: row; + flex-wrap: nowrap; + align-items: center; + justify-content: space-between; + + .codeTitle { + } + + .codeActionGroup { + display: flex; + flex-direction: row; + flex-wrap: nowrap; + align-items: center; + justify-content: flex-start; + } + } + + .codeHighlighterDiv { + margin: 0px !important; + white-space: pre-wrap !important; + + code { + white-space: pre-wrap !important; + } + } +} + +.inlineCode { + background: #fff; +} diff --git a/ProductivitySuite/ui/react/src/components/Chat_Markdown/markdown.module.scss b/ProductivitySuite/ui/react/src/components/Chat_Markdown/markdown.module.scss new file mode 100644 index 0000000000..e86902eed3 --- /dev/null +++ b/ProductivitySuite/ui/react/src/components/Chat_Markdown/markdown.module.scss @@ -0,0 +1,29 @@ +.tableDiv { + &:first-of-type { + padding-top: 0px !important; + } + + table, + th, + td { + border: 1px solid black; + border-collapse: collapse; + padding: 5px; + } +} + +.md { + li { + margin-left: 35px; /* Adjust the value based on your preference */ + } +} + +.markdownWrapper { + > p:first-of-type { + margin-top: 0.25rem; + } + + > p:last-of-type { + margin-bottom: 0.25rem; + } +} diff --git a/ProductivitySuite/ui/react/src/components/Chat_SettingsModal/ChatSettingsModal.module.scss b/ProductivitySuite/ui/react/src/components/Chat_SettingsModal/ChatSettingsModal.module.scss new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ProductivitySuite/ui/react/src/components/Chat_SettingsModal/ChatSettingsModal.tsx b/ProductivitySuite/ui/react/src/components/Chat_SettingsModal/ChatSettingsModal.tsx new file mode 100644 index 0000000000..732e5a2123 --- /dev/null +++ b/ProductivitySuite/ui/react/src/components/Chat_SettingsModal/ChatSettingsModal.tsx @@ -0,0 +1,43 @@ +import * as React from "react"; +import { + Box, + Typography, + Modal, + IconButton, + styled, + Tooltip, +} from "@mui/material"; +import SettingsApplicationsOutlinedIcon from "@mui/icons-material/SettingsApplicationsOutlined"; +import PromptSettings from "@components/PromptSettings/PromptSettings"; +import { Close } from "@mui/icons-material"; +import ModalBox from "@root/shared/ModalBox/ModalBox"; + +const ChatSettingsModal = () => { + const [open, setOpen] = React.useState(false); + const handleOpen = () => setOpen(true); + const handleClose = () => setOpen(false); + + return ( +
+ + + + + + + + + Response Settings + setOpen(false)}> + + + + + + + +
+ ); +}; + +export default ChatSettingsModal; diff --git a/ProductivitySuite/ui/react/src/components/Chat_Sources/ChatSources.module.scss b/ProductivitySuite/ui/react/src/components/Chat_Sources/ChatSources.module.scss new file mode 100644 index 0000000000..1a6a0d76e6 --- /dev/null +++ b/ProductivitySuite/ui/react/src/components/Chat_Sources/ChatSources.module.scss @@ -0,0 +1,47 @@ +.sourceWrapper { + display: flex; + flex-direction: row; + justify-content: flex-end; + flex-wrap: wrap; + width: var(--content-width); + margin: 0 auto var(--vertical-spacer); + max-width: 100%; +} + +.iconWrap { + border: none; + border-radius: 6px; + margin-right: 0.5rem; + width: 30px; + height: 30px; + display: flex; + align-items: center; + justify-content: center; +} + +.sourceBox { + display: flex; + flex-direction: row; + align-items: center; + justify-content: center; + margin-left: 1rem; + padding: 5px; + border-radius: 6px; + margin-bottom: 1rem; +} + +.title { + margin: 0 0.5rem 0 0; + white-space: nowrap; + display: inline-block; + max-width: 150px; + overflow: hidden; + text-overflow: ellipsis; + font-weight: 400; +} + +.chip { + border-radius: 8px; + padding: 3px; + font-size: 12px; +} diff --git a/ProductivitySuite/ui/react/src/components/Chat_Sources/ChatSources.tsx b/ProductivitySuite/ui/react/src/components/Chat_Sources/ChatSources.tsx new file mode 100644 index 0000000000..2bf0858254 --- /dev/null +++ b/ProductivitySuite/ui/react/src/components/Chat_Sources/ChatSources.tsx @@ -0,0 +1,28 @@ +import { Box } from "@mui/material"; +import { conversationSelector } from "@redux/Conversation/ConversationSlice"; +import { useAppSelector } from "@redux/store"; +import styles from "./ChatSources.module.scss"; +import FileDispaly from "@components/File_Display/FileDisplay"; + +const ChatSources: React.FC = () => { + const { sourceLinks, sourceFiles, sourceType } = + useAppSelector(conversationSelector); + const isWeb = sourceType === "web"; + const sourceElements = isWeb ? sourceLinks : sourceFiles; + + if (sourceLinks.length === 0 && sourceFiles.length === 0) return; + + const renderElements = () => { + return sourceElements.map((element: any, elementIndex) => { + return ( + + + + ); + }); + }; + + return {renderElements()}; +}; + +export default ChatSources; diff --git a/ProductivitySuite/ui/react/src/components/Chat_User/ChatUser.module.scss b/ProductivitySuite/ui/react/src/components/Chat_User/ChatUser.module.scss new file mode 100644 index 0000000000..3a5b5079ee --- /dev/null +++ b/ProductivitySuite/ui/react/src/components/Chat_User/ChatUser.module.scss @@ -0,0 +1,27 @@ +.userWrapper { + display: flex; + justify-content: flex-end; + margin-bottom: 2rem; + position: relative; + + .userPrompt { + max-width: 80%; + border-radius: var(--input-radius); + padding: 0.75rem 2rem 0.75rem 1rem; + overflow-wrap: break-word; + word-wrap: break-word; + word-break: break-word; + } + + .addIcon { + position: absolute; + right: -16px; + top: 3px; + opacity: 0; + transition: opacity 0.3s; + } + + &:hover .addIcon { + opacity: 1; + } +} diff --git a/ProductivitySuite/ui/react/src/components/Chat_User/ChatUser.tsx b/ProductivitySuite/ui/react/src/components/Chat_User/ChatUser.tsx new file mode 100644 index 0000000000..37971f87cb --- /dev/null +++ b/ProductivitySuite/ui/react/src/components/Chat_User/ChatUser.tsx @@ -0,0 +1,44 @@ +import { IconButton, styled, Tooltip } from "@mui/material"; +import React from "react"; +import styles from "./ChatUser.module.scss"; +import AddCircle from "@mui/icons-material/AddCircle"; +import { useAppDispatch } from "@redux/store"; +import { addPrompt } from "@redux/Prompt/PromptSlice"; +import ChatMarkdown from "@components/Chat_Markdown/ChatMarkdown"; + +interface ChatUserProps { + content: string; +} + +const UserInput = styled("div")(({ theme }) => ({ + background: theme.customStyles.user?.main, +})); + +const AddIcon = styled(AddCircle)(({ theme }) => ({ + path: { + fill: theme.customStyles.icon?.main, + }, +})); + +const ChatUser: React.FC = ({ content }) => { + const dispatch = useAppDispatch(); + + const sharePrompt = () => { + dispatch(addPrompt({ promptText: content })); + }; + + return ( +
+ + + + + + + + +
+ ); +}; + +export default ChatUser; diff --git a/ProductivitySuite/ui/react/src/components/CodeGen/CodeGen.tsx b/ProductivitySuite/ui/react/src/components/CodeGen/CodeGen.tsx deleted file mode 100644 index 29c96f61cb..0000000000 --- a/ProductivitySuite/ui/react/src/components/CodeGen/CodeGen.tsx +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright (C) 2024 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -import { KeyboardEventHandler, SyntheticEvent, useEffect, useRef, useState } from 'react' -import styleClasses from "./codeGen.module.scss" -import { ActionIcon, Textarea, Title, rem } from '@mantine/core' -import { IconArrowRight } from '@tabler/icons-react' -import { ConversationMessage } from '../Message/conversationMessage' -import { fetchEventSource } from '@microsoft/fetch-event-source' -import { CODE_GEN_URL } from '../../config' - - - -const CodeGen = () => { - const [prompt, setPrompt] = useState("") - const [submittedPrompt, setSubmittedPrompt] = useState("") - const [response,setResponse] = useState(""); - const promptInputRef = useRef(null) - const scrollViewport = useRef(null) - - const toSend = "Enter" - - const handleSubmit = async () => { - setResponse("") - setSubmittedPrompt(prompt) - const body = { - messages:prompt - } - fetchEventSource(CODE_GEN_URL, { - method: "POST", - headers: { - "Content-Type": "application/json", - "Accept":"*/*" - }, - body: JSON.stringify(body), - openWhenHidden: true, - async onopen(response) { - if (response.ok) { - return; - } else if (response.status >= 400 && response.status < 500 && response.status !== 429) { - const e = await response.json(); - console.log(e); - throw Error(e.error.message); - } else { - console.log("error", response); - } - }, - onmessage(msg) { - if (msg?.data != "[DONE]") { - try { - const match = msg.data.match(/b'([^']*)'/); - if (match && match[1] != "") { - const extractedText = match[1].replace(/\\n/g, "\n"); - setResponse(prev=>prev+extractedText); - } - } catch (e) { - console.log("something wrong in msg", e); - throw e; - } - } - }, - onerror(err) { - console.log("error", err); - setResponse("") - throw err; - }, - onclose() { - setPrompt("") - }, - }); - - } - - const scrollToBottom = () => { - scrollViewport.current!.scrollTo({ top: scrollViewport.current!.scrollHeight }) - } - - useEffect(() => { - scrollToBottom() - }, [response]) - - const handleKeyDown: KeyboardEventHandler = (event) => { - if (!event.shiftKey && event.key === toSend) { - handleSubmit() - setTimeout(() => { - setPrompt("") - }, 1) - } - } - - const handleChange = (event: SyntheticEvent) => { - event.preventDefault() - setPrompt((event.target as HTMLTextAreaElement).value) - } - return ( -
-
-
-
- CodeGen -
- -
- {!submittedPrompt && !response && - (<> -
Start by asking a question
- ) - } - {submittedPrompt && ( - - )} - {response && ( - - )} -
- -
-