diff --git a/.github/workflows/azure-dev.yml b/.github/workflows/azure-dev.yml index 5e1376d9..9da32aa9 100644 --- a/.github/workflows/azure-dev.yml +++ b/.github/workflows/azure-dev.yml @@ -48,7 +48,7 @@ jobs: uses: actions/checkout@v4 - name: Install azd - uses: Azure/setup-azd@v1.0.0 + uses: Azure/setup-azd@v2 - name: Install Nodejs uses: actions/setup-node@v4 diff --git a/.github/workflows/evaluate.yml b/.github/workflows/evaluate.yml index a5f97566..537a2834 100644 --- a/.github/workflows/evaluate.yml +++ b/.github/workflows/evaluate.yml @@ -31,13 +31,15 @@ jobs: AZURE_OPENAI_NAME: ${{ vars.AZURE_OPENAI_NAME }} BING_SEARCH_ENDPOINT: ${{ vars.BING_SEARCH_ENDPOINT }} BING_SEARCH_KEY: ${{ secrets.BING_SEARCH_KEY }} + AZURE_AI_PROJECT_NAME: ${{ vars.AZURE_AI_PROJECT_NAME }} + AZURE_LOCATION: ${{ vars.AZURE_LOCATION }} steps: - name: checkout repo content uses: actions/checkout@v4 # checkout the repository content - name: Install azd - uses: Azure/setup-azd@v1.0.0 + uses: Azure/setup-azd@v2 - name: setup python uses: actions/setup-python@v5 diff --git a/azure.yaml b/azure.yaml index 8f56e760..00384ff3 100644 --- a/azure.yaml +++ b/azure.yaml @@ -39,11 +39,12 @@ pipeline: - AZURE_SEARCH_ENDPOINT - BING_SEARCH_ENDPOINT - AZURE_OPENAI_NAME - - AZURE_CONTAINER_REGISTRY_NAME - - AZURE_RESOURCE_GROUP + - AZURE_CONTAINER_REGISTRY_NAME - API_SERVICE_ACA_NAME - WEB_SERVICE_ACA_NAME - AZURE_SEARCH_NAME - WEB_SERVICE_ACA_URI + - AZURE_AI_PROJECT_NAME + - AZURE_LOCATION secrets: - BING_SEARCH_KEY diff --git a/docs/workshop/LAB-MANUAL.md b/docs/workshop/LAB-MANUAL.md index 45f81dea..71c4817d 100644 --- a/docs/workshop/LAB-MANUAL.md +++ b/docs/workshop/LAB-MANUAL.md @@ -29,7 +29,20 @@ To participate in this workshop, you will need: 4. Once your Codespace is ready, **run the following command**: -++./docs/workshop/lab_setup.py --username '@lab.CloudPortalCredential(User1).Username' --password '@lab.CloudPortalCredential(User1).Password' --azure-env-name 'AITOUR@lab.LabInstance.Id' --subscription '@lab.CloudSubscription.Id'++ +```bash +# 1. Install the required dependencies +pip install -r requirements.txt + +# 2.a. Run the following script if you haven't already provsionned the environment with "azd up" +./docs/workshop/lab_setup.py --username '@lab.CloudPortalCredential(User1).Username' --password '@lab.CloudPortalCredential(User1).Password' --azure-env-name '' --subscription '@lab.CloudSubscription.Id' + +# 2.b. Else run the following command if you have already run "azd up": +azd env get-values > .env + +# 3. In all cases, you need to run the following commands: +sed 's/^/export /' .env >> ~/.bashrc # export variables to your shell +source ~/.bashrc # refresh your shell +``` > [!IMPORTANT] diff --git a/docs/workshop/researcher/researcher-0.prompty b/docs/workshop/researcher/researcher-0.prompty index f269cc15..b080fa65 100644 --- a/docs/workshop/researcher/researcher-0.prompty +++ b/docs/workshop/researcher/researcher-0.prompty @@ -11,6 +11,12 @@ model: type: azure_openai azure_deployment: gpt-4 api_version: 2024-08-01-preview + parameters: + temperature: 0.2 + max_tokens: 1000 + top_p: 1 + frequency_penalty: 0 + presence_penalty: 0 sample: instructions: Can you generate queries to find the latest winter camping trends? Use 'en-US' as the market code. --- diff --git a/docs/workshop/researcher/researcher-1.prompty b/docs/workshop/researcher/researcher-1.prompty index 3761d7c1..cbd71ead 100644 --- a/docs/workshop/researcher/researcher-1.prompty +++ b/docs/workshop/researcher/researcher-1.prompty @@ -11,6 +11,12 @@ model: type: azure_openai azure_deployment: gpt-4 api_version: 2024-08-01-preview + parameters: + temperature: 0.2 + max_tokens: 1000 + top_p: 1 + frequency_penalty: 0 + presence_penalty: 0 inputs: participant: type: object diff --git a/docs/workshop/researcher/researcher3.py b/docs/workshop/researcher/researcher3.py index 48c9a2c4..42426f4c 100644 --- a/docs/workshop/researcher/researcher3.py +++ b/docs/workshop/researcher/researcher3.py @@ -1,8 +1,9 @@ import os import json +import sys +sys.path.append(os.path.abspath('../../../src/api/agents/researcher')) +from researcher import execute_research from typing import List -import requests -import urllib.parse from dotenv import load_dotenv import prompty import prompty.azure @@ -10,69 +11,24 @@ load_dotenv() -BING_SEARCH_ENDPOINT = os.getenv("BING_SEARCH_ENDPOINT") -BING_SEARCH_KEY = os.getenv("BING_SEARCH_KEY") -BING_HEADERS = {"Ocp-Apim-Subscription-Key": BING_SEARCH_KEY} +global_prompty_file_path=os.getcwd()+'/researcher-2.prompty' +def find_information(query, market='en-US'): + # You can use another api to find information, here we are using "Grounding with Bing Search" + print("Executing 'Find Information' ToolCall in %s.." % market) + return execute_research(instructions=query, return_raw=True, prompty_file_path=global_prompty_file_path) -def _make_bing_endpoint(endpoint, path): - """Make a Bing endpoint URL""" - return f"{endpoint}{'' if endpoint.endswith('/') else '/'}{path}" +def find_entities(query, market='en-US'): + # You can use another api to find entities, here we are using "Grounding with Bing Search" + print("Executing 'Find Entities' ToolCall in %s.." % market) + return execute_research(instructions=query, return_raw=True, prompty_file_path=global_prompty_file_path) +def find_news(query, market='en-US'): + print("Executing 'Find News' ToolCall in %s.." % market) + # You can use another api to find bews, here we are using "Grounding with Bing Search" + return execute_research(instructions=query, return_raw=True, prompty_file_path=global_prompty_file_path) -def _make_bing_request(path, params=None): - """Make a request to the Bing API""" - endpoint = _make_bing_endpoint(BING_SEARCH_ENDPOINT, path) - response = requests.get(endpoint, headers=BING_HEADERS, params=params) - items = response.json() - return items - - -def find_information(query, market="en-US"): - """Find information using the Bing Search API""" - params = {"q": query, "mkt": market, "count": 5} - items = _make_bing_request("v7.0/search", params) - pages = [ - {"url": a["url"], "name": a["name"], "description": a["snippet"]} - for a in items["webPages"]["value"] - ] - related = [a["text"] for a in items["relatedSearches"]["value"]] - return {"pages": pages, "related": related} - - - -def find_entities(query, market="en-US"): - """Find entities using the Bing Entity Search API""" - params = "?mkt=" + market + "&q=" + urllib.parse.quote(query) - items = _make_bing_request(f"v7.0/entities{params}") - entities = [] - if "entities" in items: - entities = [ - {"name": e["name"], "description": e["description"]} - for e in items["entities"]["value"] - ] - return entities - - - -def find_news(query, market="en-US"): - """Find news using the Bing News Search API""" - params = {"q": query, "mkt": market, "count": 5} - items = _make_bing_request("v7.0/news/search", params) - articles = [ - { - "name": a["name"], - "url": a["url"], - "description": a["description"], - "provider": a["provider"][0]["name"], - "datePublished": a["datePublished"], - } - for a in items["value"] - ] - return articles - - -def execute_researcher_prompty(instructions: str): +def execute_researcher_prompty(instructions: str, prompty_file_path=global_prompty_file_path): """ Executes the researcher prompty to find information, entities, and news, and runs the selected function given the query and returns the results @@ -80,7 +36,7 @@ def execute_researcher_prompty(instructions: str): # Execute the researcher prompty function_calls: List[ToolCall] = prompty.execute( - "researcher-2.prompty", inputs={"instructions": instructions} + prompty_file_path, inputs={"instructions": instructions} ) return function_calls @@ -152,13 +108,16 @@ def extract_findings(research): } -def research(instructions: str): +def research(instructions: str, prompty_file_path: str = None): """ Calls the execute and process functions above to run the research agent and return the results to the user in a readable format. """ - - function_calls = execute_researcher_prompty(instructions=instructions) + + global global_prompty_file_path + if prompty_file_path is not None: + global_prompty_file_path=prompty_file_path + function_calls = execute_researcher_prompty(instructions=instructions, prompty_file_path=global_prompty_file_path) research = execute_function_calls(function_calls) - findings = extract_findings(research) - return findings + #findings = extract_findings(research) + return research \ No newline at end of file diff --git a/docs/workshop/socialmedia/social.py b/docs/workshop/socialmedia/social.py index 4ca9ac46..43c9f730 100644 --- a/docs/workshop/socialmedia/social.py +++ b/docs/workshop/socialmedia/social.py @@ -5,7 +5,7 @@ from prompty.tracer import trace, Tracer, PromptyTracer # Import the researcher agent to use here -sys.path.append(os.path.abspath('../../docs/workshop/researcher/')) +sys.path.append(os.path.abspath('../researcher/')) from researcher3 import research #initiate local prompty tracing @@ -22,13 +22,13 @@ def execute_social_media_writer_prompty(research_context: str, research, social_ return reseponse -def run_social_media_agent(instructions: str, social_media_instructions: str): +def run_social_media_agent(instructions: str, social_media_instructions: str, research_prompty_file_path='../researcher/researcher-2.prompty'): """ Run the social media agent Execute the researcher prompty to find information, entities, and news execute the social media writer prompty to create the twitter thread """ - research_results = research(instructions) + research_results = research(instructions, prompty_file_path=os.path.abspath(research_prompty_file_path)) thread = execute_social_media_writer_prompty(research_context= instructions, research=research_results, social_media_instructions = social_media_instructions) print(thread) diff --git a/requirements.txt b/requirements.txt index a3e83105..bd1d9b1a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ click rich-click +marshmallow==3.26.1 \ No newline at end of file diff --git a/src/api/agents/researcher/researcher.py b/src/api/agents/researcher/researcher.py index 7f0af8f9..65c7e099 100644 --- a/src/api/agents/researcher/researcher.py +++ b/src/api/agents/researcher/researcher.py @@ -21,7 +21,7 @@ @trace -def execute_research(instructions: str, feedback: str = "No feedback"): +def execute_research(instructions: str, prompty_file_path='researcher.prompty', return_raw=False, feedback: str = "No feedback"): ai_project_conn_str = os.getenv("AZURE_LOCATION")+".api.azureml.ms;"+os.getenv("AZURE_SUBSCRIPTION_ID")+";"+os.getenv("AZURE_RESOURCE_GROUP")+";"+os.getenv("AZURE_AI_PROJECT_NAME") @@ -30,8 +30,7 @@ def execute_research(instructions: str, feedback: str = "No feedback"): conn_str=ai_project_conn_str, ) - prompt_template = PromptTemplate.from_prompty(file_path="researcher.prompty") - + prompt_template = PromptTemplate.from_prompty(file_path=prompty_file_path) instructions = instructions feedback= feedback @@ -45,7 +44,7 @@ def execute_research(instructions: str, feedback: str = "No feedback"): # Initialize agent bing tool and add the connection id bing = BingGroundingTool(connection_id=conn_id) - prompt_template = PromptTemplate.from_prompty(file_path="researcher.prompty") + #prompt_template = PromptTemplate.from_prompty(file_path="researcher.prompty") # Create agent with the bing tool and process assistant run with project_client: @@ -106,6 +105,8 @@ def run_agent(): messages = project_client.agents.list_messages(thread_id=thread.id) # print(f"Messages: {messages}") research_response = messages.data[0]['content'][0]['text']['value'] + if return_raw: + return research_response try: json_r = json.loads(research_response) except: diff --git a/src/api/evaluate/evaluate.py b/src/api/evaluate/evaluate.py index a78bd6ab..b337d108 100644 --- a/src/api/evaluate/evaluate.py +++ b/src/api/evaluate/evaluate.py @@ -136,9 +136,8 @@ def run_orchestrator(research_context, product_context, assignment_context): } @trace -def evaluate_orchestrator(model_config, project_scope, data_path): +def evaluate_orchestrator(model_config, project_scope, data_path): writer_evaluator = ArticleEvaluator(model_config, project_scope) - data = [] eval_data = [] print(f"\n===== Creating articles to evaluate using data provided in {data_path}") @@ -150,17 +149,33 @@ def evaluate_orchestrator(model_config, project_scope, data_path): print(f"generating article {num +1}") eval_data.append(run_orchestrator(row["research_context"], row["product_context"], row["assignment_context"])) - # write out eval data to a file so we can re-run evaluation on it + # Write out eval data to a file so we can re-run evaluation on it with jsonlines.open(folder + '/eval_data.jsonl', 'w') as writer: for row in eval_data: writer.write(row) eval_data_path = folder + '/eval_data.jsonl' - print(f"\n===== Evaluating the generated articles") - eval_results = writer_evaluator(data_path=eval_data_path) - import pandas as pd + retries = 0 + max_retries = 5 + while retries < max_retries: + try: + eval_results = writer_evaluator(data_path=eval_data_path) + break + except Exception as e: + if 'rate_limit_exceeded' in str(e): + wait_time = (2 ** retries) + random.uniform(0, 1) + print(f"Rate limit exceeded. Retrying in {wait_time} seconds...") + time.sleep(wait_time) + retries += 1 + else: + raise e + + if retries == max_retries: + raise Exception("Max retries reached. Exiting...") + + import pandas as pd print("Evaluation summary:\n") print("View in Azure AI Studio at: " + str(eval_results['studio_url'])) metrics = {key: [value] for key, value in eval_results['metrics'].items()} @@ -456,4 +471,4 @@ def make_image_message(url_path): # eval_image_result = evaluate_image(project_scope, img_paths) end=time.time() - print(f"Finished evaluate in {end - start}s") \ No newline at end of file + print(f"Finished evaluate in {end - start}s")