From 617a48f2756e6dcb3835610a669e721f7e171b05 Mon Sep 17 00:00:00 2001 From: Jason Bouska Date: Wed, 13 Mar 2024 15:17:56 -0400 Subject: [PATCH 1/6] Update Priority Queue Cloud Pattern Update to .NET 8.0 Updated README --- .../PriorityQueueConsumerHigh.csproj | 22 +-- .../PriorityQueueConsumerHighFn.cs | 19 +- .../PriorityQueueConsumerHigh/Program.cs | 7 + .../local.settings.json | 8 - .../local.settings.template.json | 8 + .../PriorityQueueConsumerLow.csproj | 23 +-- .../PriorityQueueConsumerLowFn.cs | 19 +- .../PriorityQueueConsumerLow/Program.cs | 7 + .../local.settings.json | 8 - .../local.settings.template.json | 8 + .../PriorityQueueSender.csproj | 25 ++- .../PriorityQueueSenderFn.cs | 21 +- priority-queue/PriorityQueueSender/Program.cs | 26 +++ .../PriorityQueueSender/local.settings.json | 8 - .../local.settings.template.json | 8 + priority-queue/Readme.md | 179 ++++++++++-------- 16 files changed, 238 insertions(+), 158 deletions(-) create mode 100644 priority-queue/PriorityQueueConsumerHigh/Program.cs delete mode 100644 priority-queue/PriorityQueueConsumerHigh/local.settings.json create mode 100644 priority-queue/PriorityQueueConsumerHigh/local.settings.template.json create mode 100644 priority-queue/PriorityQueueConsumerLow/Program.cs delete mode 100644 priority-queue/PriorityQueueConsumerLow/local.settings.json create mode 100644 priority-queue/PriorityQueueConsumerLow/local.settings.template.json create mode 100644 priority-queue/PriorityQueueSender/Program.cs delete mode 100644 priority-queue/PriorityQueueSender/local.settings.json create mode 100644 priority-queue/PriorityQueueSender/local.settings.template.json diff --git a/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHigh.csproj b/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHigh.csproj index 1ac631dd..5dd483eb 100644 --- a/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHigh.csproj +++ b/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHigh.csproj @@ -1,22 +1,20 @@ - net6.0 + net8.0 v4 + Exe + enabled - - true - - - - - - - PreserveNewest - PreserveNewest Never - + + + + + + + \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHighFn.cs b/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHighFn.cs index 5ce7ad67..9cfe92f2 100644 --- a/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHighFn.cs +++ b/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHighFn.cs @@ -1,14 +1,21 @@ -using Microsoft.Azure.WebJobs; using Microsoft.Extensions.Logging; +using Microsoft.Azure.Functions.Worker; namespace PriorityQueueConsumerHigh { - public static class PriorityQueueConsumerHighFn + public class PriorityQueueConsumerHighFn { - [FunctionName("HighPriorityQueueConsumerFunction")] - public static void Run([ServiceBusTrigger("messages", "highPriority", Connection = "ServiceBusConnection")]string highPriorityMessage, ILogger log) + private readonly ILogger _logger; + + public PriorityQueueConsumerHighFn(ILogger logger) + { + _logger = logger; + } + + [Function("HighPriorityQueueConsumerFunction")] + public void Run([ServiceBusTrigger("messages", "highPriority", Connection = "ServiceBusConnectionString")] string highPriorityMessage) { - log.LogInformation($"C# ServiceBus topic trigger function processed message: {highPriorityMessage}"); + _logger.LogInformation($"C# ServiceBus topic trigger function processed message: {highPriorityMessage}"); } } -} +} \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerHigh/Program.cs b/priority-queue/PriorityQueueConsumerHigh/Program.cs new file mode 100644 index 00000000..cd97ae1f --- /dev/null +++ b/priority-queue/PriorityQueueConsumerHigh/Program.cs @@ -0,0 +1,7 @@ +using Microsoft.Extensions.Hosting; + +var host = new HostBuilder() + .ConfigureFunctionsWorkerDefaults() + .Build(); + +host.Run(); \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerHigh/local.settings.json b/priority-queue/PriorityQueueConsumerHigh/local.settings.json deleted file mode 100644 index 80410be8..00000000 --- a/priority-queue/PriorityQueueConsumerHigh/local.settings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "IsEncrypted": false, - "Values": { - "AzureWebJobsStorage": "UseDevelopmentStorage=true", - "FUNCTIONS_WORKER_RUNTIME": "dotnet", - "ServiceBusConnection__fullyQualifiedNamespace": ".servicebus.windows.net" - } -} \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerHigh/local.settings.template.json b/priority-queue/PriorityQueueConsumerHigh/local.settings.template.json new file mode 100644 index 00000000..078ee8ad --- /dev/null +++ b/priority-queue/PriorityQueueConsumerHigh/local.settings.template.json @@ -0,0 +1,8 @@ +{ + "IsEncrypted": false, + "Values": { + "AzureWebJobsStorage": "UseDevelopmentStorage=true", + "FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated", + "ServiceBusConnectionString": "SERVICE_BUS_CONNECTION_STRING" + } +} \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLow.csproj b/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLow.csproj index 6ce569f0..5dd483eb 100644 --- a/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLow.csproj +++ b/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLow.csproj @@ -1,23 +1,20 @@ - net6.0 + net8.0 v4 - 736bb6a2-68b4-463b-a8fb-3a90cba7cd4f + Exe + enabled - - true - - - - - - - PreserveNewest - PreserveNewest Never - + + + + + + + \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLowFn.cs b/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLowFn.cs index 36c1c593..0a652603 100644 --- a/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLowFn.cs +++ b/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLowFn.cs @@ -1,14 +1,21 @@ -using Microsoft.Azure.WebJobs; using Microsoft.Extensions.Logging; +using Microsoft.Azure.Functions.Worker; namespace PriorityQueueConsumerLow { - public static class PriorityQueueConsumerLowFn + public class PriorityQueueConsumerLowFn { - [FunctionName("LowPriorityQueueConsumerFunction")] - public static void Run([ServiceBusTrigger("messages", "lowPriority", Connection = "ServiceBusConnection")]string lowPriorityMessage, ILogger log) + private readonly ILogger _logger; + + public PriorityQueueConsumerLowFn(ILogger logger) + { + _logger = logger; + } + + [Function("LowPriorityQueueConsumerFunction")] + public void Run([ServiceBusTrigger("messages", "lowPriority", Connection = "ServiceBusConnectionString")] string lowPriorityMessage) { - log.LogInformation($"C# ServiceBus topic trigger function processed message: {lowPriorityMessage}"); + _logger.LogInformation($"C# ServiceBus topic trigger function processed message: {lowPriorityMessage}"); } } -} +} \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerLow/Program.cs b/priority-queue/PriorityQueueConsumerLow/Program.cs new file mode 100644 index 00000000..cd97ae1f --- /dev/null +++ b/priority-queue/PriorityQueueConsumerLow/Program.cs @@ -0,0 +1,7 @@ +using Microsoft.Extensions.Hosting; + +var host = new HostBuilder() + .ConfigureFunctionsWorkerDefaults() + .Build(); + +host.Run(); \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerLow/local.settings.json b/priority-queue/PriorityQueueConsumerLow/local.settings.json deleted file mode 100644 index 80410be8..00000000 --- a/priority-queue/PriorityQueueConsumerLow/local.settings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "IsEncrypted": false, - "Values": { - "AzureWebJobsStorage": "UseDevelopmentStorage=true", - "FUNCTIONS_WORKER_RUNTIME": "dotnet", - "ServiceBusConnection__fullyQualifiedNamespace": ".servicebus.windows.net" - } -} \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerLow/local.settings.template.json b/priority-queue/PriorityQueueConsumerLow/local.settings.template.json new file mode 100644 index 00000000..078ee8ad --- /dev/null +++ b/priority-queue/PriorityQueueConsumerLow/local.settings.template.json @@ -0,0 +1,8 @@ +{ + "IsEncrypted": false, + "Values": { + "AzureWebJobsStorage": "UseDevelopmentStorage=true", + "FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated", + "ServiceBusConnectionString": "SERVICE_BUS_CONNECTION_STRING" + } +} \ No newline at end of file diff --git a/priority-queue/PriorityQueueSender/PriorityQueueSender.csproj b/priority-queue/PriorityQueueSender/PriorityQueueSender.csproj index 1ac631dd..e0d4619e 100644 --- a/priority-queue/PriorityQueueSender/PriorityQueueSender.csproj +++ b/priority-queue/PriorityQueueSender/PriorityQueueSender.csproj @@ -1,22 +1,21 @@ - + - net6.0 + net8.0 v4 + Exe + enabled - - true - - - - - - - PreserveNewest - PreserveNewest Never - + + + + + + + + \ No newline at end of file diff --git a/priority-queue/PriorityQueueSender/PriorityQueueSenderFn.cs b/priority-queue/PriorityQueueSender/PriorityQueueSenderFn.cs index 9429b6d6..405c710e 100644 --- a/priority-queue/PriorityQueueSender/PriorityQueueSenderFn.cs +++ b/priority-queue/PriorityQueueSender/PriorityQueueSenderFn.cs @@ -1,31 +1,34 @@ using System; using System.Threading.Tasks; using Azure.Messaging.ServiceBus; -using Microsoft.Azure.WebJobs; +using Microsoft.Extensions.Logging; +using Microsoft.Azure.Functions.Worker; namespace PriorityQueueSender { - public static class PriorityQueueSenderFn + public class PriorityQueueSenderFn(ILogger logger, ServiceBusClient client) { - [FunctionName("PriorityQueueSenderFunction")] - public static async Task Run( - [TimerTrigger("0,30 * * * * *")] TimerInfo myTimer, - [ServiceBus("messages", Connection = "ServiceBusConnection")] IAsyncCollector collector ) + private readonly ILogger _logger = logger; + private readonly ServiceBusClient _client = client; + + [Function("PriorityQueueSenderFunction")] + public async Task Run([TimerTrigger("0,30 * * * * *")] TimerInfo myTimer) { + var sender = _client.CreateSender("messages"); for (int i = 0; i < 10; i++) { var messageId = Guid.NewGuid().ToString(); var lpMessage = new ServiceBusMessage() { MessageId = messageId }; lpMessage.ApplicationProperties["Priority"] = Priority.Low; lpMessage.Body = BinaryData.FromString($"Low priority message with Id: {messageId}"); - await collector.AddAsync(lpMessage); + await sender.SendMessageAsync(lpMessage); messageId = Guid.NewGuid().ToString(); var hpMessage = new ServiceBusMessage() { MessageId = messageId }; hpMessage.ApplicationProperties["Priority"] = Priority.High; hpMessage.Body = BinaryData.FromString($"High priority message with Id: {messageId}"); - await collector.AddAsync(hpMessage); + await sender.SendMessageAsync(hpMessage); } } } -} +} \ No newline at end of file diff --git a/priority-queue/PriorityQueueSender/Program.cs b/priority-queue/PriorityQueueSender/Program.cs new file mode 100644 index 00000000..c393da75 --- /dev/null +++ b/priority-queue/PriorityQueueSender/Program.cs @@ -0,0 +1,26 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Azure.Messaging.ServiceBus; + +var host = new HostBuilder() + .ConfigureFunctionsWorkerDefaults() + .ConfigureAppConfiguration((hostingContext, config) => + { + config.AddJsonFile("local.settings.json", optional: true, reloadOnChange: true); + }) + .ConfigureServices(services => + { + var configuration = services.BuildServiceProvider().GetRequiredService(); + + services.AddSingleton(configuration); + + services.AddSingleton(sp => + { + var connectionString = configuration.GetValue("ServiceBusConnectionString"); + return new ServiceBusClient(connectionString); + }); + }) + .Build(); + +host.Run(); \ No newline at end of file diff --git a/priority-queue/PriorityQueueSender/local.settings.json b/priority-queue/PriorityQueueSender/local.settings.json deleted file mode 100644 index 445c47d3..00000000 --- a/priority-queue/PriorityQueueSender/local.settings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "IsEncrypted": false, - "Values": { - "AzureWebJobsStorage": "UseDevelopmentStorage=true", - "FUNCTIONS_WORKER_RUNTIME": "dotnet", - "ServiceBusConnection__fullyQualifiedNamespace": "mgransb1.servicebus.windows.net" - } -} \ No newline at end of file diff --git a/priority-queue/PriorityQueueSender/local.settings.template.json b/priority-queue/PriorityQueueSender/local.settings.template.json new file mode 100644 index 00000000..078ee8ad --- /dev/null +++ b/priority-queue/PriorityQueueSender/local.settings.template.json @@ -0,0 +1,8 @@ +{ + "IsEncrypted": false, + "Values": { + "AzureWebJobsStorage": "UseDevelopmentStorage=true", + "FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated", + "ServiceBusConnectionString": "SERVICE_BUS_CONNECTION_STRING" + } +} \ No newline at end of file diff --git a/priority-queue/Readme.md b/priority-queue/Readme.md index 01e0ef99..92e686a1 100644 --- a/priority-queue/Readme.md +++ b/priority-queue/Readme.md @@ -1,100 +1,129 @@ -# Priority Queue Pattern +# Priority Queue pattern example -This document describes the Priority Queue Pattern example from the guide [Cloud Design Patterns](http://aka.ms/Cloud-Design-Patterns). +This directory contains an example of the [Priority Queue pattern](https://learn.microsoft.com/en-us/azure/architecture/patterns/priority-queue). -## System Requirements +This example demonstrates how priority queues can be implemented by using Service Bus topics and subscriptions. A time-triggered Azure Function is responsible for sending messages to a topic, each with a priority assigned. The receiving Azure Functions read messages from subscriptions that have the corresponding priority. In this example, the _PriorityQueueConsumerHigh_ Azure function can scale out to 200 instances, while the _PriorityQueueConsumerLow_ function runs only with one instance. This example simulates high priority messages being read from the queue more urgently than low priority messages. -* Microsoft .NET 6 -* Microsoft Visual Studio 2019 or later version -* Azure Functions Core Tools version 4x +This example also demonstrates operational aspects of applications running on Azure. Monitoring tools need to be used in order to understand how the sample works. Azure Functions in the solution **must** be configured to use the diagnostics mechanism. If not, you will not see the trace information generated by the example. -## Before you start +## :rocket: Deployment guide -Ensure that you have installed all of the software prerequisites. +Install the prerequisites and follow the steps to deploy and run an example of the Priority Queue pattern. -The example demonstrates operational aspects of applications running in Windows Azure. Therefore, you will need to use the monitoring tools in order to understand how the code sample works. You **must** ensure that the Azure Functions in the solution are configured to use the diagnostics mechanism. If not, you will not see the trace information generated by the example. +### Prerequisites -## About the Example +- Permission to create a new resource group and resources in an [Azure subscription](https://azure.com/free). +- [Git](https://git-scm.com/downloads) +- [Azure CLI](https://learn.microsoft.com/cli/azure/install-azure-cli) +- [.NET 8 SDK](https://dotnet.microsoft.com/download/dotnet/8.0) +- [Microsoft Visual Studio 2022](https://visualstudio.microsoft.com/vs/) or later version +- [Azure Functions Core Tools v4.x](https://learn.microsoft.com/azure/azure-functions/functions-run-local#install-the-azure-functions-core-tools) -This example shows how you can implement priority queues by using Service Bus Topics and Subscriptions. A timer triggered Azure Function is responsible for sending messages to a topic. It assigns a priority to each message. The receiving Azure Functions read messages from subscriptions that have the corresponding priority. In the example, the PriorityQueueConsumerHigh Azure function can scale out to 200 instances, whereas the PriorityQueueConsumerLow worker runs only with one instance. This simulates high priority messages being read from the queue more urgently than low priority messages. -## Running the Example +### Steps -You can either run this example locally from Visual Studio or you can run it by deploying it to Azure. +1. Clone the repository -* From the Azure Portal, provision an Azure Service Bus Namespace. -* Once the Service Bus Namespace is created, add a new topic to it, name it "messages" (leave all the properties default). -* Once the topic is created, add a new subscription to it, name it "highPriority", set "10" as Max delivery count. -* Add a new filter of type "SqlFilter" to the subscription, name it "priorityFilter". -* Set the expression __Priority = 'highpriority'__ in the filter body. -* Click on "Save changes". -* Add another subscription to the topic, name it "lowPriority", set "10" as Max delivery count. -* Add a new filter of type "SqlFilter" to the subscription, name it "priorityFilter". -* Set the expression __Priority = 'lowpriority'__ in the filter body. -* Click on "Save changes". + Open a terminal, clone the repository, and navigate to the `priority-queue` directory. -* Start Visual Studio. -* Open the solution you want to explore from the subfolders where you downloaded the examples. -* Edit the local.settings.json file in all the projects and change the ServiceBusConnection__fullyQualifiedNamespace setting by changing the placeholder "" to your Azure Service Bus Namespace name. + ```shell + git clone https://github.com/mspnp/cloud-design-patterns.git + cd cloud-design-patterns + cd priority-queue + ``` +1. Log into Azure and create an empty resource group. -* If you want to run the example in the local Windows Azure emulator: - * Set the PriorityQueueSender project as startup. - * Press F5 in Visual Studio to start the example running. The function will start sending messages to the topic, every 30 seconds. - * Stop the execution, change the startup project to either the PriorityQueueConsumerHigh or PriorityQueueConsumerLow - * Press F5 in Visual Studio to start the execution of the consumer function - * In the Azure Functions Core Tools Console, you can view the diagnostic information generated by Trace statements in the code. + Create an empty resource group to hold the resources for this example. The location you select in the resource group creation command below is the Azure region that your resources will be deployed in; modify as needed. -* If you want to run the example on Azure: - * On every function project, right click and select publish. - * Select "Azure" as publishing target. - * Select "Azure Function App" as specific target. - * In the Functions instance step, select you Azure Function app or create a new one using the link button. - * Since your app settings are different you need to deploy every Azure Function in a separate Azure Funcion App. - * You can select an existing resource group and storage account or create new ones. - * In "Hosting" section, click on the the three dots (...) in the upper right corner. - * Select "Manage Azure App Service Settings". - * You need to add these two settings: + ```azurecli + az login + az account set -s - - ServiceBusConnection__fullyQualifiedNamespace: + RESOURCE_GROUP_NAME=rg-priority-queue + az group create -n $RESOURCE_GROUP_NAME -l eastus2 + ``` +1. Provision a Service Bus messaging namespace. - Set the value to: + ```azurecli + SERVICE_BUS_NAMESPACE_NAME=sbns-priority-queue-pattern + az servicebus namespace create -g $RESOURCE_GROUP_NAME -n $SERVICE_BUS_NAMESPACE_NAME + ``` +1. Create a Service Bus topic named "messages". - .servicebus.windows.net + ```azurecli + SERVICE_BUS_TOPIC_NAME=messages + az servicebus topic create --name $SERVICE_BUS_TOPIC_NAME --namespace-name $SERVICE_BUS_NAMESPACE_NAME --resource-group $RESOURCE_GROUP_NAME + ``` +1. Create a Service Bus subscription for high priority messages. - Replacing the placeholder with you Azure Service Bus Namespace name. + ```azurecli + SERVICE_BUS_SUBSCRIPTION_NAME_HIGH=highPriority + az servicebus topic subscription create --name $SERVICE_BUS_SUBSCRIPTION_NAME_HIGH --namespace-name $SERVICE_BUS_NAMESPACE_NAME --resource-group $RESOURCE_GROUP_NAME --topic-name $SERVICE_BUS_TOPIC_NAME + ``` +1. Create a Service Bus subscription for low priority messages. - * Once the Functions are deployed you need to restrict the maximum number of instances your app service consumption can scale out to: + ```azurecli + SERVICE_BUS_SUBSCRIPTION_NAME_LOW=lowPriority + az servicebus topic subscription create --name $SERVICE_BUS_SUBSCRIPTION_NAME_LOW --namespace-name $SERVICE_BUS_NAMESPACE_NAME --resource-group $RESOURCE_GROUP_NAME --topic-name $SERVICE_BUS_TOPIC_NAME + ``` +1. Add a filter to the high priority subscription. - * From the portal go to the Function App that contains the "PriorityQueueConsumerLow" Azure Function - * Navigate to Scale Out on the left menu - * On the App Scale Out dialog set the "Enforce Scale Out Limit" to "Yes" - * Set the Maximum Scale Out Limit to 1 instance - - You don't need to modify these settings for the Function App containing the "PriorityQueueConsumerHigh" Azure Function since the default setting is 200; this ensures that high priority messages are read from the queue more quickly than low priority messages. + ```azurecli + az servicebus topic subscription rule create --name priorityFilter --namespace-name $SERVICE_BUS_NAMESPACE_NAME --resource-group $RESOURCE_GROUP_NAME --topic-name $SERVICE_BUS_TOPIC_NAME --subscription-name $SERVICE_BUS_SUBSCRIPTION_NAME_HIGH --filter-sql-expression "Priority = 'highpriority'" + ``` +1. Add a filter to the low priority subscription. - * Now you need to configure the managed identities role assignments: + ```azurecli + az servicebus topic subscription rule create --name priorityFilter --namespace-name $SERVICE_BUS_NAMESPACE_NAME --resource-group $RESOURCE_GROUP_NAME --topic-name $SERVICE_BUS_TOPIC_NAME --subscription-name $SERVICE_BUS_SUBSCRIPTION_NAME_LOW --filter-sql-expression "Priority = 'lowpriority'" + ``` +1. Retrieve the primary connection string for the Service Bus namespace. Copy and retain this value as it will be needed in later steps. - In the Azure portal, navigate to the Azure Service Bus Namespace that was provisioned in the first step. - Select Access Control (IAM). This is where you can view and configure who has access to the resource. - Click Add and select add role assignment. - From the list, select "Azure Service Bus Data Sender", click Next - In "Assign access to", radio button list, select "Managed identity" - Click on "Select members", the "Select Manage identities" dialog will show up - In the Managed Identity dropdown list select "Function App" - Find the PriorityQueueSender function app click on it - Click "Select" - On the main dialog, click "Review + Assign" + ```azurecli + SERVICE_BUS_CONNECTION_STRING=$(az servicebus namespace authorization-rule keys list --resource-group $RESOURCE_GROUP_NAME --namespace-name $SERVICE_BUS_NAMESPACE_NAME --name RootManageSharedAccessKey --query primaryConnectionString --output tsv) + echo $SERVICE_BUS_CONNECTION_STRING + ``` - For the Azure Consumer Function Apps, repeat the process but in this case use the role - "Azure Service Bus Data Reader" - * Once the Functions are deployed you can configure monitoring by following these steps: +## Run the example locally - - From the Azure Portal, go to the Function App Service - - Click on "Functions" and select the azure function - - From the "developer" left menu click on "Monitor" - - Turn on application insights - - Refresh the screen - - Once refreshed you will see two tabs, "invocation" and "logs" - - From the invocations tab you can see the twenty most recent function invocation traces. For more advanced analysis, run the query in Application Insights. - - From the logs tab you can see the logging information that your functions are sending. \ No newline at end of file + +1. Open the `priority-queue.sln` file in Visual Studio. + +1. Edit the `local.settings.json` file for each project set the value of `ServiceBusConnectionString` to the **SERVICE_BUS_CONNECTION_STRING** retained from earlier. + +1. Set the `PriorityQueueSender` project as startup. +1. Press F5 in Visual Studio to start running the example. The function will begin sending messages to the topic every 30 seconds. +1. Stop the execution, change the startup project to either the `PriorityQueueConsumerHigh` or `PriorityQueueConsumerLow`. +1. Press F5 in Visual Studio to start the execution of the consumer function. +1. In the Azure Functions Core Tools Console, you can view the diagnostic information generated by statements in the code. + + +## Deploy the example to Azure +To deploy the example to Azure, you need to publish each Azure Functions to Azure. You can do so from Visual Studio by right clicking each function and selecting `Publish` from the menu. Use the same resource group and region from earlier. Be sure to enable Application Insights. + +Once each function is published, a new App Setting must be added to store the connection string to the Service Bus namespace. This is the same value that was used in the `SERVICE_BUS_CONNECTION_STRING` variable in the previous steps. + +For each function, run the following: +```azurecli +az webapp config appsettings set -n -g $RESOURCE_GROUP_NAME --settings ServiceBusConnectionString=$SERVICE_BUS_CONNECTION_STRING +``` + +Once the functions are deployed you need to restrict the maximum number of instances the `PriorityQueueConsumerLow` function can scale out to. + +From the Azure portal: +- Visit the Function App that contains `PriorityQueueConsumerLow` +- Navigate to Scale Out on the left menu +- On the App Scale Out dialog, set the `Enforce Scale Out Limit` to `Yes` +- Set the `Maximum Scale Out Limit` to `1` instance + + +Once the functions are deployed you can visit Application Insights to view the most recent activity for each function. + + +## :broom: Clean up resources + +Be sure to delete Azure resources when not using them. Since all resources were deployed into a new resource group, you can simply delete the resource group. + +```azurecli +az group delete -n $RESOURCE_GROUP_NAME +``` \ No newline at end of file From ea84f2ace501b93486b714d2b5d33402cd1a33bf Mon Sep 17 00:00:00 2001 From: Federico Arambarri <62260324+v-fearam@users.noreply.github.com> Date: Mon, 7 Oct 2024 17:47:16 -0300 Subject: [PATCH 2/6] Priority Queue Update (To branch) (#160) * Improvements. Bicep file. Settings. Moved to managed identity. Run func start * Removing VS requirement * deleting any --------- Co-authored-by: Federico Arambarri --- .../PriorityQueueConsumerHigh.csproj | 10 +- .../PriorityQueueConsumerHighFn.cs | 11 +- .../PriorityQueueConsumerHigh/host.json | 17 +- .../local.settings.template.json | 2 +- .../PriorityQueueConsumerLow.csproj | 10 +- .../PriorityQueueConsumerLowFn.cs | 11 +- .../PriorityQueueConsumerLow/host.json | 17 +- .../local.settings.template.json | 2 +- .../PriorityQueueSender.csproj | 48 ++-- priority-queue/PriorityQueueSender/Program.cs | 9 +- priority-queue/PriorityQueueSender/host.json | 17 +- .../local.settings.template.json | 2 +- priority-queue/Readme.md | 116 +++++----- priority-queue/bicep/main.bicep | 216 ++++++++++++++++++ 14 files changed, 354 insertions(+), 134 deletions(-) create mode 100644 priority-queue/bicep/main.bicep diff --git a/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHigh.csproj b/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHigh.csproj index 5dd483eb..0eef047b 100644 --- a/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHigh.csproj +++ b/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHigh.csproj @@ -5,6 +5,9 @@ Exe enabled + + + PreserveNewest @@ -12,9 +15,8 @@ - - - - + + + \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHighFn.cs b/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHighFn.cs index 9cfe92f2..0544fb7a 100644 --- a/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHighFn.cs +++ b/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHighFn.cs @@ -3,17 +3,12 @@ namespace PriorityQueueConsumerHigh { - public class PriorityQueueConsumerHighFn + public class PriorityQueueConsumerHighFn(ILogger logger) { - private readonly ILogger _logger; - - public PriorityQueueConsumerHighFn(ILogger logger) - { - _logger = logger; - } + private readonly ILogger _logger = logger; [Function("HighPriorityQueueConsumerFunction")] - public void Run([ServiceBusTrigger("messages", "highPriority", Connection = "ServiceBusConnectionString")] string highPriorityMessage) + public void Run([ServiceBusTrigger("messages", "highPriority", Connection = "ServiceBusConnection")] string highPriorityMessage) { _logger.LogInformation($"C# ServiceBus topic trigger function processed message: {highPriorityMessage}"); } diff --git a/priority-queue/PriorityQueueConsumerHigh/host.json b/priority-queue/PriorityQueueConsumerHigh/host.json index beb2e402..5df170b6 100644 --- a/priority-queue/PriorityQueueConsumerHigh/host.json +++ b/priority-queue/PriorityQueueConsumerHigh/host.json @@ -1,11 +1,12 @@ { - "version": "2.0", - "logging": { - "applicationInsights": { - "samplingSettings": { - "isEnabled": true, - "excludedTypes": "Request" - } - } + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + }, + "enableLiveMetricsFilters": true } + } } \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerHigh/local.settings.template.json b/priority-queue/PriorityQueueConsumerHigh/local.settings.template.json index 078ee8ad..8cbbeb91 100644 --- a/priority-queue/PriorityQueueConsumerHigh/local.settings.template.json +++ b/priority-queue/PriorityQueueConsumerHigh/local.settings.template.json @@ -3,6 +3,6 @@ "Values": { "AzureWebJobsStorage": "UseDevelopmentStorage=true", "FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated", - "ServiceBusConnectionString": "SERVICE_BUS_CONNECTION_STRING" + "ServiceBusConnection__fullyQualifiedNamespace": "{SERVICE_BUS_CONNECTION}" } } \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLow.csproj b/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLow.csproj index 5dd483eb..0eef047b 100644 --- a/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLow.csproj +++ b/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLow.csproj @@ -5,6 +5,9 @@ Exe enabled + + + PreserveNewest @@ -12,9 +15,8 @@ - - - - + + + \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLowFn.cs b/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLowFn.cs index 0a652603..a9213d2c 100644 --- a/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLowFn.cs +++ b/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLowFn.cs @@ -3,17 +3,12 @@ namespace PriorityQueueConsumerLow { - public class PriorityQueueConsumerLowFn + public class PriorityQueueConsumerLowFn(ILogger logger) { - private readonly ILogger _logger; - - public PriorityQueueConsumerLowFn(ILogger logger) - { - _logger = logger; - } + private readonly ILogger _logger = logger; [Function("LowPriorityQueueConsumerFunction")] - public void Run([ServiceBusTrigger("messages", "lowPriority", Connection = "ServiceBusConnectionString")] string lowPriorityMessage) + public void Run([ServiceBusTrigger("messages", "lowPriority", Connection = "ServiceBusConnection")] string lowPriorityMessage) { _logger.LogInformation($"C# ServiceBus topic trigger function processed message: {lowPriorityMessage}"); } diff --git a/priority-queue/PriorityQueueConsumerLow/host.json b/priority-queue/PriorityQueueConsumerLow/host.json index beb2e402..5df170b6 100644 --- a/priority-queue/PriorityQueueConsumerLow/host.json +++ b/priority-queue/PriorityQueueConsumerLow/host.json @@ -1,11 +1,12 @@ { - "version": "2.0", - "logging": { - "applicationInsights": { - "samplingSettings": { - "isEnabled": true, - "excludedTypes": "Request" - } - } + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + }, + "enableLiveMetricsFilters": true } + } } \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerLow/local.settings.template.json b/priority-queue/PriorityQueueConsumerLow/local.settings.template.json index 078ee8ad..8cbbeb91 100644 --- a/priority-queue/PriorityQueueConsumerLow/local.settings.template.json +++ b/priority-queue/PriorityQueueConsumerLow/local.settings.template.json @@ -3,6 +3,6 @@ "Values": { "AzureWebJobsStorage": "UseDevelopmentStorage=true", "FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated", - "ServiceBusConnectionString": "SERVICE_BUS_CONNECTION_STRING" + "ServiceBusConnection__fullyQualifiedNamespace": "{SERVICE_BUS_CONNECTION}" } } \ No newline at end of file diff --git a/priority-queue/PriorityQueueSender/PriorityQueueSender.csproj b/priority-queue/PriorityQueueSender/PriorityQueueSender.csproj index e0d4619e..81ebc955 100644 --- a/priority-queue/PriorityQueueSender/PriorityQueueSender.csproj +++ b/priority-queue/PriorityQueueSender/PriorityQueueSender.csproj @@ -1,21 +1,29 @@  - - net8.0 - v4 - Exe - enabled - - - - PreserveNewest - Never - - - - - - - - - - \ No newline at end of file + + net8.0 + v4 + Exe + enabled + enable + + + + + + + + + + PreserveNewest + Never + + + + + PreserveNewest + + + + + + diff --git a/priority-queue/PriorityQueueSender/Program.cs b/priority-queue/PriorityQueueSender/Program.cs index c393da75..e2037b64 100644 --- a/priority-queue/PriorityQueueSender/Program.cs +++ b/priority-queue/PriorityQueueSender/Program.cs @@ -2,6 +2,9 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Azure.Messaging.ServiceBus; +using Azure.Identity; +using System; +using Microsoft.Extensions.Azure; var host = new HostBuilder() .ConfigureFunctionsWorkerDefaults() @@ -15,10 +18,10 @@ services.AddSingleton(configuration); - services.AddSingleton(sp => + services.AddAzureClients(builder => { - var connectionString = configuration.GetValue("ServiceBusConnectionString"); - return new ServiceBusClient(connectionString); + builder.AddServiceBusClientWithNamespace(Environment.GetEnvironmentVariable("ServiceBusConnection__fullyQualifiedNamespace")) + .WithCredential(new DefaultAzureCredential()); }); }) .Build(); diff --git a/priority-queue/PriorityQueueSender/host.json b/priority-queue/PriorityQueueSender/host.json index beb2e402..5df170b6 100644 --- a/priority-queue/PriorityQueueSender/host.json +++ b/priority-queue/PriorityQueueSender/host.json @@ -1,11 +1,12 @@ { - "version": "2.0", - "logging": { - "applicationInsights": { - "samplingSettings": { - "isEnabled": true, - "excludedTypes": "Request" - } - } + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + }, + "enableLiveMetricsFilters": true } + } } \ No newline at end of file diff --git a/priority-queue/PriorityQueueSender/local.settings.template.json b/priority-queue/PriorityQueueSender/local.settings.template.json index 078ee8ad..8cbbeb91 100644 --- a/priority-queue/PriorityQueueSender/local.settings.template.json +++ b/priority-queue/PriorityQueueSender/local.settings.template.json @@ -3,6 +3,6 @@ "Values": { "AzureWebJobsStorage": "UseDevelopmentStorage=true", "FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated", - "ServiceBusConnectionString": "SERVICE_BUS_CONNECTION_STRING" + "ServiceBusConnection__fullyQualifiedNamespace": "{SERVICE_BUS_CONNECTION}" } } \ No newline at end of file diff --git a/priority-queue/Readme.md b/priority-queue/Readme.md index 92e686a1..e7c9ceaa 100644 --- a/priority-queue/Readme.md +++ b/priority-queue/Readme.md @@ -1,6 +1,6 @@ # Priority Queue pattern example -This directory contains an example of the [Priority Queue pattern](https://learn.microsoft.com/en-us/azure/architecture/patterns/priority-queue). +This directory contains an example of the [Priority Queue pattern](https://learn.microsoft.com/azure/architecture/patterns/priority-queue). This example demonstrates how priority queues can be implemented by using Service Bus topics and subscriptions. A time-triggered Azure Function is responsible for sending messages to a topic, each with a priority assigned. The receiving Azure Functions read messages from subscriptions that have the corresponding priority. In this example, the _PriorityQueueConsumerHigh_ Azure function can scale out to 200 instances, while the _PriorityQueueConsumerLow_ function runs only with one instance. This example simulates high priority messages being read from the queue more urgently than low priority messages. @@ -16,114 +16,110 @@ Install the prerequisites and follow the steps to deploy and run an example of t - [Git](https://git-scm.com/downloads) - [Azure CLI](https://learn.microsoft.com/cli/azure/install-azure-cli) - [.NET 8 SDK](https://dotnet.microsoft.com/download/dotnet/8.0) -- [Microsoft Visual Studio 2022](https://visualstudio.microsoft.com/vs/) or later version - [Azure Functions Core Tools v4.x](https://learn.microsoft.com/azure/azure-functions/functions-run-local#install-the-azure-functions-core-tools) - ### Steps 1. Clone the repository - Open a terminal, clone the repository, and navigate to the `priority-queue` directory. + Open a terminal, clone the repository, and navigate to the `priority-queue` directory. + + ```shell + git clone https://github.com/mspnp/cloud-design-patterns.git + cd cloud-design-patterns + cd priority-queue + ``` - ```shell - git clone https://github.com/mspnp/cloud-design-patterns.git - cd cloud-design-patterns - cd priority-queue - ``` 1. Log into Azure and create an empty resource group. Create an empty resource group to hold the resources for this example. The location you select in the resource group creation command below is the Azure region that your resources will be deployed in; modify as needed. - ```azurecli + ```bash az login az account set -s - RESOURCE_GROUP_NAME=rg-priority-queue - az group create -n $RESOURCE_GROUP_NAME -l eastus2 + LOCATION=eastus2 + RESOURCE_GROUP_NAME=rg-priority-queue-${LOCATION} + az group create -n $RESOURCE_GROUP_NAME -l $LOCATION ``` -1. Provision a Service Bus messaging namespace. - ```azurecli - SERVICE_BUS_NAMESPACE_NAME=sbns-priority-queue-pattern - az servicebus namespace create -g $RESOURCE_GROUP_NAME -n $SERVICE_BUS_NAMESPACE_NAME - ``` -1. Create a Service Bus topic named "messages". +1. Deploy the supporting Azure resources. - ```azurecli - SERVICE_BUS_TOPIC_NAME=messages - az servicebus topic create --name $SERVICE_BUS_TOPIC_NAME --namespace-name $SERVICE_BUS_NAMESPACE_NAME --resource-group $RESOURCE_GROUP_NAME - ``` -1. Create a Service Bus subscription for high priority messages. + ```bash + CURRENT_USER_OBJECT_ID=$(az ad signed-in-user show -o tsv --query id) + SERVICE_BUS_NAMESPACE_NAME="sbns-priority-queue-$(LC_ALL=C tr -dc 'a-z0-9' < /dev/urandom | fold -w 7 | head -n 1)" - ```azurecli - SERVICE_BUS_SUBSCRIPTION_NAME_HIGH=highPriority - az servicebus topic subscription create --name $SERVICE_BUS_SUBSCRIPTION_NAME_HIGH --namespace-name $SERVICE_BUS_NAMESPACE_NAME --resource-group $RESOURCE_GROUP_NAME --topic-name $SERVICE_BUS_TOPIC_NAME + # This takes about two minute + az deployment group create -n deploy-priority-queue -f bicep/main.bicep -g "${RESOURCE_GROUP_NAME}" -p queueNamespaces=$SERVICE_BUS_NAMESPACE_NAME principalId=$CURRENT_USER_OBJECT_ID ``` -1. Create a Service Bus subscription for low priority messages. - ```azurecli - SERVICE_BUS_SUBSCRIPTION_NAME_LOW=lowPriority - az servicebus topic subscription create --name $SERVICE_BUS_SUBSCRIPTION_NAME_LOW --namespace-name $SERVICE_BUS_NAMESPACE_NAME --resource-group $RESOURCE_GROUP_NAME --topic-name $SERVICE_BUS_TOPIC_NAME - ``` -1. Add a filter to the high priority subscription. +1. Configure the samples to use the created Azure resources. - ```azurecli - az servicebus topic subscription rule create --name priorityFilter --namespace-name $SERVICE_BUS_NAMESPACE_NAME --resource-group $RESOURCE_GROUP_NAME --topic-name $SERVICE_BUS_TOPIC_NAME --subscription-name $SERVICE_BUS_SUBSCRIPTION_NAME_HIGH --filter-sql-expression "Priority = 'highpriority'" - ``` -1. Add a filter to the low priority subscription. + ```bash + # Retrieve the primary connection string for the Service Bus namespace. + SERVICE_BUS_CONNECTION="${SERVICE_BUS_NAMESPACE_NAME}.servicebus.windows.net" - ```azurecli - az servicebus topic subscription rule create --name priorityFilter --namespace-name $SERVICE_BUS_NAMESPACE_NAME --resource-group $RESOURCE_GROUP_NAME --topic-name $SERVICE_BUS_TOPIC_NAME --subscription-name $SERVICE_BUS_SUBSCRIPTION_NAME_LOW --filter-sql-expression "Priority = 'lowpriority'" + sed "s|{SERVICE_BUS_CONNECTION}|${SERVICE_BUS_CONNECTION}|g" ./PriorityQueueSender/local.settings.template.json > ./PriorityQueueSender/local.settings.json + sed "s|{SERVICE_BUS_CONNECTION}|${SERVICE_BUS_CONNECTION}|g" ./PriorityQueueConsumerHigh/local.settings.template.json > ./PriorityQueueConsumerHigh/local.settings.json + sed "s|{SERVICE_BUS_CONNECTION}|${SERVICE_BUS_CONNECTION}|g" ./PriorityQueueConsumerLow/local.settings.template.json > ./PriorityQueueConsumerLow/local.settings.json ``` -1. Retrieve the primary connection string for the Service Bus namespace. Copy and retain this value as it will be needed in later steps. - ```azurecli - SERVICE_BUS_CONNECTION_STRING=$(az servicebus namespace authorization-rule keys list --resource-group $RESOURCE_GROUP_NAME --namespace-name $SERVICE_BUS_NAMESPACE_NAME --name RootManageSharedAccessKey --query primaryConnectionString --output tsv) - echo $SERVICE_BUS_CONNECTION_STRING +1. [Run Azurite](https://learn.microsoft.com/azure/storage/common/storage-use-azurite#run-azurite) blob storage emulation service. + + > The local storage emulator is required as an Azure Storage account is a required "backing resource" for Azure Functions. + +1. Launch the Function PriorityQueueSender to generate Low and High messages. + + ```bash + cd ./PriorityQueueSender + func start ``` +1. In a new terminal, launch the Function PriorityQueueConsumerLow to consume messages. -## Run the example locally + ```bash + cd ./PriorityQueueConsumerLow + func start -p 15000 + ``` + > Please note: For demo purposes, the sample application will write content to the the screen. -1. Open the `priority-queue.sln` file in Visual Studio. - -1. Edit the `local.settings.json` file for each project set the value of `ServiceBusConnectionString` to the **SERVICE_BUS_CONNECTION_STRING** retained from earlier. +1. In a new terminal, launch the Function PriorityQueueConsumerHigh to consume messages. -1. Set the `PriorityQueueSender` project as startup. -1. Press F5 in Visual Studio to start running the example. The function will begin sending messages to the topic every 30 seconds. -1. Stop the execution, change the startup project to either the `PriorityQueueConsumerHigh` or `PriorityQueueConsumerLow`. -1. Press F5 in Visual Studio to start the execution of the consumer function. -1. In the Azure Functions Core Tools Console, you can view the diagnostic information generated by statements in the code. + ```bash + cd ./PriorityQueueConsumerHigh + func start -p 15001 + ``` + > Please note: For demo purposes, the sample application will write content to the the screen. -## Deploy the example to Azure -To deploy the example to Azure, you need to publish each Azure Functions to Azure. You can do so from Visual Studio by right clicking each function and selecting `Publish` from the menu. Use the same resource group and region from earlier. Be sure to enable Application Insights. +## Deploy the example to Azure (Optional) + +To deploy the example to Azure, you need to publish each Azure Functions to Azure. You can do so from Visual Studio by right clicking each function and selecting `Publish` from the menu. Use the same resource group and region from earlier. Be sure to enable Application Insights. Once each function is published, a new App Setting must be added to store the connection string to the Service Bus namespace. This is the same value that was used in the `SERVICE_BUS_CONNECTION_STRING` variable in the previous steps. For each function, run the following: -```azurecli -az webapp config appsettings set -n -g $RESOURCE_GROUP_NAME --settings ServiceBusConnectionString=$SERVICE_BUS_CONNECTION_STRING + +```bash +az webapp config appsettings set -n -g $RESOURCE_GROUP_NAME --settings ServiceBusConnection__fullyQualifiedNamespace=$SERVICE_BUS_CONNECTION ``` Once the functions are deployed you need to restrict the maximum number of instances the `PriorityQueueConsumerLow` function can scale out to. From the Azure portal: + - Visit the Function App that contains `PriorityQueueConsumerLow` - Navigate to Scale Out on the left menu - On the App Scale Out dialog, set the `Enforce Scale Out Limit` to `Yes` - Set the `Maximum Scale Out Limit` to `1` instance - Once the functions are deployed you can visit Application Insights to view the most recent activity for each function. - ## :broom: Clean up resources Be sure to delete Azure resources when not using them. Since all resources were deployed into a new resource group, you can simply delete the resource group. -```azurecli -az group delete -n $RESOURCE_GROUP_NAME -``` \ No newline at end of file +```bash +az group delete -n $RESOURCE_GROUP_NAME -y +``` diff --git a/priority-queue/bicep/main.bicep b/priority-queue/bicep/main.bicep new file mode 100644 index 00000000..ba5d4b23 --- /dev/null +++ b/priority-queue/bicep/main.bicep @@ -0,0 +1,216 @@ +targetScope = 'resourceGroup' + +@minLength(5) +@description('Location of the resources. Defaults to resource group location.') +param location string = resourceGroup().location + +@minLength(15) +@description('Service Bus Namespace Name.') +param queueNamespaces string + +@minLength(36) +@description('The guid of the principal running the valet key generation code. In Azure this would be replaced with the managed identity of the Azure Function, when running locally it will be your user.') +param principalId string + +var logAnalyticsName = 'loganalytics-${uniqueString(subscription().subscriptionId, resourceGroup().id)}' + +var senderServiceBusRole = subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + '69a216fc-b8fb-44d8-bc22-1f3c2cd27a39' +) // Azure Service Bus Data Sender +var receiverServiceBusRole = subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + '4f6d3b9b-027b-4f4c-9142-0e5a2a2247e0' +) // Azure Service Bus Data Receiver + +resource queueNamespacesResource 'Microsoft.ServiceBus/namespaces@2023-01-01-preview' = { + name: queueNamespaces + location: location + sku: { + name: 'Standard' + tier: 'Standard' + } + properties: { + geoDataReplication: { + maxReplicationLagDurationInSeconds: 0 + locations: [ + { + locationName: location + roleType: 'Primary' + } + ] + } + premiumMessagingPartitions: 0 + minimumTlsVersion: '1.2' + publicNetworkAccess: 'Enabled' + disableLocalAuth: false + zoneRedundant: false + } +} + +resource queueNamespacesResourceRootManageSharedAccessKey 'Microsoft.ServiceBus/namespaces/authorizationrules@2023-01-01-preview' = { + parent: queueNamespacesResource + name: 'RootManageSharedAccessKey' + location: location + properties: { + rights: [ + 'Listen' + 'Manage' + 'Send' + ] + } +} + +resource queueNamespacesResourceNetworkRules 'Microsoft.ServiceBus/namespaces/networkrulesets@2023-01-01-preview' = { + parent: queueNamespacesResource + name: 'default' + location: location + properties: { + publicNetworkAccess: 'Enabled' + defaultAction: 'Allow' + virtualNetworkRules: [] + ipRules: [] + trustedServiceAccessEnabled: false + } +} + +resource queueNamespacesResourceTopic 'Microsoft.ServiceBus/namespaces/topics@2023-01-01-preview' = { + parent: queueNamespacesResource + name: 'messages' + location: location + properties: { + maxMessageSizeInKilobytes: 256 + maxSizeInMegabytes: 1024 + requiresDuplicateDetection: false + duplicateDetectionHistoryTimeWindow: 'PT10M' + enableBatchedOperations: true + status: 'Active' + supportOrdering: true + enablePartitioning: false + enableExpress: false + } +} + +resource queueNamespacesResourceTopicHigPriority 'Microsoft.ServiceBus/namespaces/topics/subscriptions@2023-01-01-preview' = { + parent: queueNamespacesResourceTopic + name: 'highPriority' + location: location + properties: { + isClientAffine: false + lockDuration: 'PT1M' + requiresSession: false + deadLetteringOnMessageExpiration: false + deadLetteringOnFilterEvaluationExceptions: true + maxDeliveryCount: 10 + status: 'Active' + enableBatchedOperations: true + } +} + +resource queueNamespacesResourceTopicLowPriority 'Microsoft.ServiceBus/namespaces/topics/subscriptions@2023-01-01-preview' = { + parent: queueNamespacesResourceTopic + name: 'lowPriority' + location: location + properties: { + isClientAffine: false + lockDuration: 'PT1M' + requiresSession: false + deadLetteringOnMessageExpiration: false + deadLetteringOnFilterEvaluationExceptions: true + maxDeliveryCount: 10 + status: 'Active' + enableBatchedOperations: true + } +} + +resource queueNamespacesResourceTopicHigPriorityRules 'Microsoft.ServiceBus/namespaces/topics/subscriptions/rules@2023-01-01-preview' = { + parent: queueNamespacesResourceTopicHigPriority + name: 'priorityFilter' + location: location + properties: { + action: {} + filterType: 'SqlFilter' + sqlFilter: { + sqlExpression: 'Priority = \'highpriority\'' + compatibilityLevel: 20 + } + } +} + +resource LqueueNamespacesResourceTopicLowPriorityRules 'Microsoft.ServiceBus/namespaces/topics/subscriptions/rules@2023-01-01-preview' = { + parent: queueNamespacesResourceTopicLowPriority + name: 'priorityFilter' + location: location + properties: { + action: {} + filterType: 'SqlFilter' + sqlFilter: { + sqlExpression: 'Priority = \'lowpriority\'' + compatibilityLevel: 20 + } + } +} + +resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' = { + name: logAnalyticsName + location: location + properties: { + retentionInDays: 30 + features: { + searchVersion: 1 + } + sku: { + name: 'PerGB2018' + } + } +} + +resource diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: '${queueNamespacesResource.name}-diagnostic' + scope: queueNamespacesResource + properties: { + logs: [ + { + category: 'OperationalLogs' + enabled: true + retentionPolicy: { + enabled: false + days: 0 + } + } + ] + metrics: [ + { + category: 'AllMetrics' + enabled: true + retentionPolicy: { + enabled: false + days: 0 + } + } + ] + workspaceId: logAnalytics.id + } +} + +// Assign Role to allow sending messages to the Service Bus +resource serviceBusSenderRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(resourceGroup().id, principalId, 'ServiceBusSenderRole') + scope: queueNamespacesResource + properties: { + roleDefinitionId: senderServiceBusRole + principalId: principalId + principalType: 'User' // 'ServicePrincipal' if this was a managed identity + } +} + +// Assign Role to allow receiving messages from the Service Bus +resource serviceBusReceiverRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(resourceGroup().id, principalId, 'ServiceBusReceiverRole') + scope: queueNamespacesResource + properties: { + roleDefinitionId: receiverServiceBusRole + principalId: principalId + principalType: 'User' // 'ServicePrincipal' if this was a managed identity + } +} From e521b85839d1d4c36855118d0c04fc601cd59f77 Mon Sep 17 00:00:00 2001 From: "Federico Arambarri (CLARIUS CONSULTING SA)" Date: Tue, 2 Dec 2025 10:56:08 -0300 Subject: [PATCH 3/6] [Update Priority Queue Cloud Pattern] Update to .NET 9, Library Upgrades, and Azure Deployment Enhancements (#373) * Some updates and rewording * Update API and Readme * Readme Improvement * Moving forward * Changing names * Improve Information * Net10 --- .../PriorityQueueConsumerHigh.csproj | 10 +- .../local.settings.template.json | 2 +- .../PriorityQueueConsumerLow.csproj | 10 +- .../local.settings.template.json | 2 +- .../PriorityQueueSender.csproj | 11 +- .../local.settings.template.json | 2 +- priority-queue/Readme.md | 89 +++++++---- .../bicep/azure/azure-function-apps.bicep | 82 ++++++++++ priority-queue/bicep/azure/sites.bicep | 148 ++++++++++++++++++ priority-queue/bicep/main.bicep | 31 ++-- 10 files changed, 324 insertions(+), 63 deletions(-) create mode 100644 priority-queue/bicep/azure/azure-function-apps.bicep create mode 100644 priority-queue/bicep/azure/sites.bicep diff --git a/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHigh.csproj b/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHigh.csproj index 0eef047b..7d8d94d1 100644 --- a/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHigh.csproj +++ b/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHigh.csproj @@ -1,6 +1,6 @@ - + - net8.0 + net10.0 v4 Exe enabled @@ -15,8 +15,8 @@ - - - + + + \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerHigh/local.settings.template.json b/priority-queue/PriorityQueueConsumerHigh/local.settings.template.json index 8cbbeb91..643657ce 100644 --- a/priority-queue/PriorityQueueConsumerHigh/local.settings.template.json +++ b/priority-queue/PriorityQueueConsumerHigh/local.settings.template.json @@ -3,6 +3,6 @@ "Values": { "AzureWebJobsStorage": "UseDevelopmentStorage=true", "FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated", - "ServiceBusConnection__fullyQualifiedNamespace": "{SERVICE_BUS_CONNECTION}" + "ServiceBusConnection__fullyQualifiedNamespace": "{SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}" } } \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLow.csproj b/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLow.csproj index 0eef047b..7d8d94d1 100644 --- a/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLow.csproj +++ b/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLow.csproj @@ -1,6 +1,6 @@ - + - net8.0 + net10.0 v4 Exe enabled @@ -15,8 +15,8 @@ - - - + + + \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerLow/local.settings.template.json b/priority-queue/PriorityQueueConsumerLow/local.settings.template.json index 8cbbeb91..643657ce 100644 --- a/priority-queue/PriorityQueueConsumerLow/local.settings.template.json +++ b/priority-queue/PriorityQueueConsumerLow/local.settings.template.json @@ -3,6 +3,6 @@ "Values": { "AzureWebJobsStorage": "UseDevelopmentStorage=true", "FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated", - "ServiceBusConnection__fullyQualifiedNamespace": "{SERVICE_BUS_CONNECTION}" + "ServiceBusConnection__fullyQualifiedNamespace": "{SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}" } } \ No newline at end of file diff --git a/priority-queue/PriorityQueueSender/PriorityQueueSender.csproj b/priority-queue/PriorityQueueSender/PriorityQueueSender.csproj index 81ebc955..7fb11ec8 100644 --- a/priority-queue/PriorityQueueSender/PriorityQueueSender.csproj +++ b/priority-queue/PriorityQueueSender/PriorityQueueSender.csproj @@ -1,15 +1,18 @@  - net8.0 + net10.0 v4 Exe enabled enable - - - + + + + + + diff --git a/priority-queue/PriorityQueueSender/local.settings.template.json b/priority-queue/PriorityQueueSender/local.settings.template.json index 8cbbeb91..643657ce 100644 --- a/priority-queue/PriorityQueueSender/local.settings.template.json +++ b/priority-queue/PriorityQueueSender/local.settings.template.json @@ -3,6 +3,6 @@ "Values": { "AzureWebJobsStorage": "UseDevelopmentStorage=true", "FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated", - "ServiceBusConnection__fullyQualifiedNamespace": "{SERVICE_BUS_CONNECTION}" + "ServiceBusConnection__fullyQualifiedNamespace": "{SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}" } } \ No newline at end of file diff --git a/priority-queue/Readme.md b/priority-queue/Readme.md index e7c9ceaa..baa3a1cf 100644 --- a/priority-queue/Readme.md +++ b/priority-queue/Readme.md @@ -2,9 +2,13 @@ This directory contains an example of the [Priority Queue pattern](https://learn.microsoft.com/azure/architecture/patterns/priority-queue). -This example demonstrates how priority queues can be implemented by using Service Bus topics and subscriptions. A time-triggered Azure Function is responsible for sending messages to a topic, each with a priority assigned. The receiving Azure Functions read messages from subscriptions that have the corresponding priority. In this example, the _PriorityQueueConsumerHigh_ Azure function can scale out to 200 instances, while the _PriorityQueueConsumerLow_ function runs only with one instance. This example simulates high priority messages being read from the queue more urgently than low priority messages. +This example demonstrates how priority queues can be implemented by using Service Bus topics and subscriptions. A time-triggered Azure Function is responsible for sending messages to a topic, each with a priority assigned. The receiving Azure Functions read messages from subscriptions that have the corresponding priority. -This example also demonstrates operational aspects of applications running on Azure. Monitoring tools need to be used in order to understand how the sample works. Azure Functions in the solution **must** be configured to use the diagnostics mechanism. If not, you will not see the trace information generated by the example. +For local execution, the sample demonstrates the producer/consumer model, where each consumer processes only one type of message based on its priority. + +In the Azure Deployment, the _PriorityQueueConsumerHigh_ Azure function could scale out to 200 instances, while the _PriorityQueueConsumerLow_ Azure function could scale out only to 40 instances. It simulates high priority messages being read from the queue more urgently than low priority messages. + +The Azure deployment also demonstrates operational aspects of applications running on Azure. Monitoring tools are essential to understand how the sample operates. Azure Functions in the solution **must** be configured to use the diagnostics mechanism. Otherwise, trace information generated by the example will not be visible. ## :rocket: Deployment guide @@ -15,7 +19,7 @@ Install the prerequisites and follow the steps to deploy and run an example of t - Permission to create a new resource group and resources in an [Azure subscription](https://azure.com/free). - [Git](https://git-scm.com/downloads) - [Azure CLI](https://learn.microsoft.com/cli/azure/install-azure-cli) -- [.NET 8 SDK](https://dotnet.microsoft.com/download/dotnet/8.0) +- [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0) - [Azure Functions Core Tools v4.x](https://learn.microsoft.com/azure/azure-functions/functions-run-local#install-the-azure-functions-core-tools) ### Steps @@ -49,73 +53,104 @@ Install the prerequisites and follow the steps to deploy and run an example of t CURRENT_USER_OBJECT_ID=$(az ad signed-in-user show -o tsv --query id) SERVICE_BUS_NAMESPACE_NAME="sbns-priority-queue-$(LC_ALL=C tr -dc 'a-z0-9' < /dev/urandom | fold -w 7 | head -n 1)" - # This takes about two minute - az deployment group create -n deploy-priority-queue -f bicep/main.bicep -g "${RESOURCE_GROUP_NAME}" -p queueNamespaces=$SERVICE_BUS_NAMESPACE_NAME principalId=$CURRENT_USER_OBJECT_ID + # This takes about two minutes + az deployment group create -n deploy-priority-queue -f bicep/main.bicep -g $RESOURCE_GROUP_NAME -p queueNamespaces=$SERVICE_BUS_NAMESPACE_NAME principalId=$CURRENT_USER_OBJECT_ID ``` 1. Configure the samples to use the created Azure resources. ```bash # Retrieve the primary connection string for the Service Bus namespace. - SERVICE_BUS_CONNECTION="${SERVICE_BUS_NAMESPACE_NAME}.servicebus.windows.net" + SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE="${SERVICE_BUS_NAMESPACE_NAME}.servicebus.windows.net" - sed "s|{SERVICE_BUS_CONNECTION}|${SERVICE_BUS_CONNECTION}|g" ./PriorityQueueSender/local.settings.template.json > ./PriorityQueueSender/local.settings.json - sed "s|{SERVICE_BUS_CONNECTION}|${SERVICE_BUS_CONNECTION}|g" ./PriorityQueueConsumerHigh/local.settings.template.json > ./PriorityQueueConsumerHigh/local.settings.json - sed "s|{SERVICE_BUS_CONNECTION}|${SERVICE_BUS_CONNECTION}|g" ./PriorityQueueConsumerLow/local.settings.template.json > ./PriorityQueueConsumerLow/local.settings.json + sed "s|{SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}|${SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}|g" ./PriorityQueueSender/local.settings.template.json > ./PriorityQueueSender/local.settings.json + sed "s|{SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}|${SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}|g" ./PriorityQueueConsumerHigh/local.settings.template.json > ./PriorityQueueConsumerHigh/local.settings.json + sed "s|{SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}|${SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}|g" ./PriorityQueueConsumerLow/local.settings.template.json > ./PriorityQueueConsumerLow/local.settings.json ``` 1. [Run Azurite](https://learn.microsoft.com/azure/storage/common/storage-use-azurite#run-azurite) blob storage emulation service. - > The local storage emulator is required as an Azure Storage account is a required "backing resource" for Azure Functions. + > Azure Functions require an Azure Storage account as a backing resource. When running locally, you can use Azurite, the local storage emulator, to fulfill this requirement. +Alternatively, you may configure the AzureWebJobsStorage setting to use a real Azure Storage account if preferred. -1. Launch the Function PriorityQueueSender to generate Low and High messages. +1. Launch the Azure Function PriorityQueueSender to generate Low and High messages. ```bash cd ./PriorityQueueSender func start ``` -1. In a new terminal, launch the Function PriorityQueueConsumerLow to consume messages. +1. In a new terminal, launch the Azure Function PriorityQueueConsumerLow to consume messages. ```bash cd ./PriorityQueueConsumerLow func start -p 15000 ``` - > Please note: For demo purposes, the sample application will write content to the the screen. + > Please note: For demo purposes, the sample application will write content to the screen. -1. In a new terminal, launch the Function PriorityQueueConsumerHigh to consume messages. +1. In a new terminal, launch the Azure Function PriorityQueueConsumerHigh to consume messages. ```bash cd ./PriorityQueueConsumerHigh func start -p 15001 ``` - > Please note: For demo purposes, the sample application will write content to the the screen. + > Please note: For demo purposes, the sample application will write content to the screen. ## Deploy the example to Azure (Optional) -To deploy the example to Azure, you need to publish each Azure Functions to Azure. You can do so from Visual Studio by right clicking each function and selecting `Publish` from the menu. Use the same resource group and region from earlier. Be sure to enable Application Insights. - -Once each function is published, a new App Setting must be added to store the connection string to the Service Bus namespace. This is the same value that was used in the `SERVICE_BUS_CONNECTION_STRING` variable in the previous steps. +This Bicep template sets up the core infrastructure for a priority-based message processing system using Azure Functions. It creates a secure Storage Account, an Application Insights instance for monitoring, and uses a previously created Service Bus namespace to enable communication between the sender and consumer functions. The deployment includes three Azure Function Apps: one sender and two consumers, each with different scaling limits to simulate message prioritization. +The funcPriorityQueueConsumerHigh function can scale out to 200 instances, allowing it to process high-priority messages quickly. The funcPriorityQueueConsumerLow function is limited to 40 instances, handling lower-priority messages with less urgency. All function apps use the FlexConsumption plan and are connected to Application Insights for diagnostics and monitoring. Role assignments are configured to securely grant access to the Service Bus and Storage resources using managed identities. +All Azure Function Apps share the same Storage Account and Application Insights instance (It is essential to understand how the sample operates), which centralizes observability and logging. -For each function, run the following: +```bash + # This takes about three minutes + az deployment group create -n deploy-priority-queue-sites -f bicep/azure/azure-function-apps.bicep -g $RESOURCE_GROUP_NAME -p serviceBusNamespaceName=$SERVICE_BUS_NAMESPACE_NAME +``` +After deploying the infrastructure, you need to publish each Azure Function to its corresponding Function App using Azure Functions Core Tools: ```bash -az webapp config appsettings set -n -g $RESOURCE_GROUP_NAME --settings ServiceBusConnection__fullyQualifiedNamespace=$SERVICE_BUS_CONNECTION + cd .\PriorityQueueSender\ + func azure functionapp publish funcPriorityQueueSender + cd .. + cd .\PriorityQueueConsumerLow\ + func azure functionapp publish funcPriorityQueueConsumerLow + cd .. + cd .\PriorityQueueConsumerHigh\ + func azure functionapp publish funcPriorityQueueConsumerHigh ``` -Once the functions are deployed you need to restrict the maximum number of instances the `PriorityQueueConsumerLow` function can scale out to. +You can view the maximum scaling configuration in the Azure Portal. Go to each Function App, then under Settings, select Scale and concurrency. There, you'll see the Maximum instance count setting. -From the Azure portal: +Once the Azure Functions are deployed, you can use Application Insights to monitor their activity. In the Azure portal, go to the Application Insights resource, then select Logs. Switch to KQL mode and run the following queries to view trace logs for each function: -- Visit the Function App that contains `PriorityQueueConsumerLow` -- Navigate to Scale Out on the left menu -- On the App Scale Out dialog, set the `Enforce Scale Out Limit` to `Yes` -- Set the `Maximum Scale Out Limit` to `1` instance +``` + // Traces for the High priority consumer + traces + | where operation_Name contains "High" + + // Traces for the Low priority consumer + traces + | where operation_Name contains "Low" -Once the functions are deployed you can visit Application Insights to view the most recent activity for each function. + // Traces for the Sender function + traces + | where operation_Name contains "Sender" +``` +In addition to viewing trace logs, you can also use Application Insights to analyze request-level data for each Azure Function. The following queries help you inspect recent requests and understand how frequently each function is being called: +``` + // Recent requests with key details + requests + | project timestamp, operation_Name, cloud_RoleName, id, success, resultCode, duration, operation_Id + | order by timestamp desc + + // Count of requests by Function + requests + | summarize RequestCount = count() by operation_Name + | order by RequestCount desc +``` ## :broom: Clean up resources Be sure to delete Azure resources when not using them. Since all resources were deployed into a new resource group, you can simply delete the resource group. diff --git a/priority-queue/bicep/azure/azure-function-apps.bicep b/priority-queue/bicep/azure/azure-function-apps.bicep new file mode 100644 index 00000000..007f4b67 --- /dev/null +++ b/priority-queue/bicep/azure/azure-function-apps.bicep @@ -0,0 +1,82 @@ +targetScope = 'resourceGroup' + +@minLength(5) +@description('Location of the resources. Defaults to resource group location.') +param location string = resourceGroup().location + +@description('The name of the existing Service Bus namespace used for message queuing between the sender and consumer functions.') +param serviceBusNamespaceName string + +@description('Defines the name of the Storage Account used by the Function Apps. It uses a unique string based on the resource group ID to ensure global uniqueness.') +param storageAccountName string = 'st${uniqueString(resourceGroup().id)}' + +@description('Sets the name of the Application Insights resource for monitoring and diagnostics. Like the storage account, it uses a unique string based on the resource group ID.') +param appInsightsName string = 'ai${uniqueString(resourceGroup().id)}' + +var senderRoleId = '69a216fc-b8fb-44d8-bc22-1f3c2cd27a39' //Azure Service Bus Data Sender +var receiverRoleId = '4f6d3b9b-027b-4f4c-9142-0e5a2a2247e0' //Azure Service Bus Data Receiver + +resource storageAccount 'Microsoft.Storage/storageAccounts@2025-01-01' = { + name: storageAccountName + location: location + sku: { + name: 'Standard_LRS' + } + kind: 'StorageV2' + properties: { + defaultToOAuthAuthentication: true + publicNetworkAccess: 'Enabled' + allowCrossTenantReplication: false + minimumTlsVersion: 'TLS1_2' + allowBlobPublicAccess: false + allowSharedKeyAccess: false + networkAcls: { + bypass: 'AzureServices' + virtualNetworkRules: [] + ipRules: [] + defaultAction: 'Allow' + } + supportsHttpsTrafficOnly: true + encryption: { + services: { + file: { + keyType: 'Account' + enabled: true + } + blob: { + keyType: 'Account' + enabled: true + } + } + keySource: 'Microsoft.Storage' + } + } +} + +resource appInsights 'Microsoft.Insights/components@2020-02-02' = { + name: appInsightsName + location: location + kind: 'web' + properties: { + Application_Type: 'web' + } +} + +module functionApp './sites.bicep' = [ + for name in [ + 'funcPriorityQueueSender' + 'funcPriorityQueueConsumerLow' + 'funcPriorityQueueConsumerHigh' + ]: { + name: name + params: { + location: location + functionAppName: name + storageAccountName: storageAccount.name + serviceBusNamespaceName: serviceBusNamespaceName + roleId: name == 'funcPriorityQueueSender' ? senderRoleId : receiverRoleId + appInsightsName: appInsights.name + scaleUp: name == 'funcPriorityQueueConsumerHigh' ? 200 : 40 + } + } +] diff --git a/priority-queue/bicep/azure/sites.bicep b/priority-queue/bicep/azure/sites.bicep new file mode 100644 index 00000000..96865af1 --- /dev/null +++ b/priority-queue/bicep/azure/sites.bicep @@ -0,0 +1,148 @@ +targetScope = 'resourceGroup' + +@minLength(5) +@description('Location of the resources. Defaults to resource group location.') +param location string = resourceGroup().location + +param functionAppName string + +@description('Defines the name of the Storage Account used by the Function Apps.') +param storageAccountName string + +@description('The name of the existing Service Bus namespace used for message queuing between the sender and consumer functions.') +param serviceBusNamespaceName string + +@description('The built-in role definition ID to assign to the Function App for accessing the Service Bus namespace.') +param roleId string + +@description('Sets the name of the Application Insights resource for monitoring and diagnostics. ') +param appInsightsName string + +@description('Specifies the maximum number of instances to which the Function App can scale out.') +param scaleUp int = 1 + +@description('Generates a unique container name for deployments') +var deploymentStorageContainerName = toLower('package-${take(functionAppName, 32)}') + +@description('Built-in role definition ID for Storage Blob Data Owner') +var azureStorageBlobDataOwnerRole = subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + 'b7e6dc6d-f1e8-4753-8033-0f276bb0955b' +) // Storage Blob Data Owner + +resource serviceBusNamespace 'Microsoft.ServiceBus/namespaces@2025-05-01-preview' existing = { + name: serviceBusNamespaceName +} + +resource storageAccount 'Microsoft.Storage/storageAccounts@2025-01-01' existing = { + name: storageAccountName +} + +// Define the blob service under the existing storage account for deployments +resource blobService 'Microsoft.Storage/storageAccounts/blobServices@2021-09-01' = { + parent: storageAccount + name: 'default' + properties: { + deleteRetentionPolicy: { + enabled: false + } + } + + // Define the container inside the blob service + resource deploymentContainer 'containers' = { + name: deploymentStorageContainerName + properties: { + publicAccess: 'None' + } + } +} + +resource appInsights 'Microsoft.Insights/components@2020-02-02' existing = { + name: appInsightsName +} + +resource appServicePlan 'Microsoft.Web/serverfarms@2024-11-01' = { + name: '${functionAppName}-plan' + location: location + sku: { + name: 'FC1' + tier: 'FlexConsumption' + } + kind: 'functionapp' + properties: { + reserved: true + } +} + +resource functionApp 'Microsoft.Web/sites@2024-11-01' = { + name: functionAppName + location: location + kind: 'functionapp,linux' + identity: { + type: 'SystemAssigned' + } + properties: { + serverFarmId: appServicePlan.id + siteConfig: { + minTlsVersion: '1.2' + appSettings: [ + { + name: 'AzureWebJobsStorage__accountName' + value: storageAccount.name + } + { + name: 'ServiceBusConnection__fullyQualifiedNamespace' + value: '${serviceBusNamespaceName}.servicebus.windows.net' + } + { + name: 'APPINSIGHTS_INSTRUMENTATIONKEY' + value: appInsights.properties.InstrumentationKey + } + { + name: 'APPLICATIONINSIGHTS_CONNECTION_STRING' + value: appInsights.properties.ConnectionString + } + ] + } + functionAppConfig: { + deployment: { + storage: { + type: 'blobContainer' + value: '${storageAccount.properties.primaryEndpoints.blob}${blobService::deploymentContainer.name}' + authentication: { + type: 'SystemAssignedIdentity' + } + } + } + runtime: { + name: 'dotnet-isolated' + version: '10.0' + } + scaleAndConcurrency: { + maximumInstanceCount: scaleUp + instanceMemoryMB: 2048 + } + } + httpsOnly: true + } +} + +resource roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(functionApp.id, roleId) + scope: serviceBusNamespace + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleId) + principalId: functionApp.identity.principalId + principalType: 'ServicePrincipal' + } +} + +resource storageBlobDataOwnerRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(functionApp.id, storageAccount.id, 'Storage Blob Data Owner') + scope: storageAccount + properties: { + roleDefinitionId: azureStorageBlobDataOwnerRole + principalId: functionApp.identity.principalId + principalType: 'ServicePrincipal' + } +} diff --git a/priority-queue/bicep/main.bicep b/priority-queue/bicep/main.bicep index ba5d4b23..b8328b2c 100644 --- a/priority-queue/bicep/main.bicep +++ b/priority-queue/bicep/main.bicep @@ -9,7 +9,7 @@ param location string = resourceGroup().location param queueNamespaces string @minLength(36) -@description('The guid of the principal running the valet key generation code. In Azure this would be replaced with the managed identity of the Azure Function, when running locally it will be your user.') +@description('The principal ID used to run the Azure Functions. In Azure, this should be the managed identity (system-assigned or user-assigned) of the Azure Function. When running locally, it should be your user identity.') param principalId string var logAnalyticsName = 'loganalytics-${uniqueString(subscription().subscriptionId, resourceGroup().id)}' @@ -23,7 +23,7 @@ var receiverServiceBusRole = subscriptionResourceId( '4f6d3b9b-027b-4f4c-9142-0e5a2a2247e0' ) // Azure Service Bus Data Receiver -resource queueNamespacesResource 'Microsoft.ServiceBus/namespaces@2023-01-01-preview' = { +resource queueNamespacesResource 'Microsoft.ServiceBus/namespaces@2025-05-01-preview' = { name: queueNamespaces location: location sku: { @@ -48,10 +48,9 @@ resource queueNamespacesResource 'Microsoft.ServiceBus/namespaces@2023-01-01-pre } } -resource queueNamespacesResourceRootManageSharedAccessKey 'Microsoft.ServiceBus/namespaces/authorizationrules@2023-01-01-preview' = { +resource queueNamespacesResourceRootManageSharedAccessKey 'Microsoft.ServiceBus/namespaces/authorizationrules@2025-05-01-preview' = { parent: queueNamespacesResource name: 'RootManageSharedAccessKey' - location: location properties: { rights: [ 'Listen' @@ -61,10 +60,9 @@ resource queueNamespacesResourceRootManageSharedAccessKey 'Microsoft.ServiceBus/ } } -resource queueNamespacesResourceNetworkRules 'Microsoft.ServiceBus/namespaces/networkrulesets@2023-01-01-preview' = { +resource queueNamespacesResourceNetworkRules 'Microsoft.ServiceBus/namespaces/networkrulesets@2025-05-01-preview' = { parent: queueNamespacesResource name: 'default' - location: location properties: { publicNetworkAccess: 'Enabled' defaultAction: 'Allow' @@ -74,10 +72,9 @@ resource queueNamespacesResourceNetworkRules 'Microsoft.ServiceBus/namespaces/ne } } -resource queueNamespacesResourceTopic 'Microsoft.ServiceBus/namespaces/topics@2023-01-01-preview' = { +resource queueNamespacesResourceTopic 'Microsoft.ServiceBus/namespaces/topics@2025-05-01-preview' = { parent: queueNamespacesResource name: 'messages' - location: location properties: { maxMessageSizeInKilobytes: 256 maxSizeInMegabytes: 1024 @@ -91,10 +88,9 @@ resource queueNamespacesResourceTopic 'Microsoft.ServiceBus/namespaces/topics@20 } } -resource queueNamespacesResourceTopicHigPriority 'Microsoft.ServiceBus/namespaces/topics/subscriptions@2023-01-01-preview' = { +resource queueNamespacesResourceTopicHigPriority 'Microsoft.ServiceBus/namespaces/topics/subscriptions@2025-05-01-preview' = { parent: queueNamespacesResourceTopic name: 'highPriority' - location: location properties: { isClientAffine: false lockDuration: 'PT1M' @@ -107,10 +103,9 @@ resource queueNamespacesResourceTopicHigPriority 'Microsoft.ServiceBus/namespace } } -resource queueNamespacesResourceTopicLowPriority 'Microsoft.ServiceBus/namespaces/topics/subscriptions@2023-01-01-preview' = { +resource queueNamespacesResourceTopicLowPriority 'Microsoft.ServiceBus/namespaces/topics/subscriptions@2025-05-01-preview' = { parent: queueNamespacesResourceTopic name: 'lowPriority' - location: location properties: { isClientAffine: false lockDuration: 'PT1M' @@ -123,10 +118,9 @@ resource queueNamespacesResourceTopicLowPriority 'Microsoft.ServiceBus/namespace } } -resource queueNamespacesResourceTopicHigPriorityRules 'Microsoft.ServiceBus/namespaces/topics/subscriptions/rules@2023-01-01-preview' = { +resource queueNamespacesResourceTopicHigPriorityRules 'Microsoft.ServiceBus/namespaces/topics/subscriptions/rules@2025-05-01-preview' = { parent: queueNamespacesResourceTopicHigPriority name: 'priorityFilter' - location: location properties: { action: {} filterType: 'SqlFilter' @@ -137,10 +131,9 @@ resource queueNamespacesResourceTopicHigPriorityRules 'Microsoft.ServiceBus/name } } -resource LqueueNamespacesResourceTopicLowPriorityRules 'Microsoft.ServiceBus/namespaces/topics/subscriptions/rules@2023-01-01-preview' = { +resource LqueueNamespacesResourceTopicLowPriorityRules 'Microsoft.ServiceBus/namespaces/topics/subscriptions/rules@2025-05-01-preview' = { parent: queueNamespacesResourceTopicLowPriority name: 'priorityFilter' - location: location properties: { action: {} filterType: 'SqlFilter' @@ -151,7 +144,7 @@ resource LqueueNamespacesResourceTopicLowPriorityRules 'Microsoft.ServiceBus/nam } } -resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' = { +resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2025-02-01' = { name: logAnalyticsName location: location properties: { @@ -200,7 +193,7 @@ resource serviceBusSenderRoleAssignment 'Microsoft.Authorization/roleAssignments properties: { roleDefinitionId: senderServiceBusRole principalId: principalId - principalType: 'User' // 'ServicePrincipal' if this was a managed identity + principalType: 'User' // 'ServicePrincipal' if this was App Service with a managed identity } } @@ -211,6 +204,6 @@ resource serviceBusReceiverRoleAssignment 'Microsoft.Authorization/roleAssignmen properties: { roleDefinitionId: receiverServiceBusRole principalId: principalId - principalType: 'User' // 'ServicePrincipal' if this was a managed identity + principalType: 'User' // 'ServicePrincipal' if this was App Service with a managed identity } } From 104c09e75e133cc421f97f8db807da5a588896f3 Mon Sep 17 00:00:00 2001 From: Jason Bouska Date: Tue, 2 Dec 2025 12:18:12 -0500 Subject: [PATCH 4/6] small edits --- priority-queue/Readme.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/priority-queue/Readme.md b/priority-queue/Readme.md index baa3a1cf..a5f5d707 100644 --- a/priority-queue/Readme.md +++ b/priority-queue/Readme.md @@ -111,14 +111,17 @@ All Azure Function Apps share the same Storage Account and Application Insights After deploying the infrastructure, you need to publish each Azure Function to its corresponding Function App using Azure Functions Core Tools: ```bash - cd .\PriorityQueueSender\ + cd ./PriorityQueueSender func azure functionapp publish funcPriorityQueueSender cd .. - cd .\PriorityQueueConsumerLow\ + + cd ./PriorityQueueConsumerLow func azure functionapp publish funcPriorityQueueConsumerLow cd .. - cd .\PriorityQueueConsumerHigh\ + + cd ./PriorityQueueConsumerHigh func azure functionapp publish funcPriorityQueueConsumerHigh + cd .. ``` You can view the maximum scaling configuration in the Azure Portal. Go to each Function App, then under Settings, select Scale and concurrency. There, you'll see the Maximum instance count setting. From 04615fb80abcf17959ebd3a27592b733fca29a76 Mon Sep 17 00:00:00 2001 From: Jason Bouska Date: Tue, 2 Dec 2025 13:00:16 -0500 Subject: [PATCH 5/6] Update main.bicep --- priority-queue/bicep/main.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priority-queue/bicep/main.bicep b/priority-queue/bicep/main.bicep index b8328b2c..fc0b8636 100644 --- a/priority-queue/bicep/main.bicep +++ b/priority-queue/bicep/main.bicep @@ -131,7 +131,7 @@ resource queueNamespacesResourceTopicHigPriorityRules 'Microsoft.ServiceBus/name } } -resource LqueueNamespacesResourceTopicLowPriorityRules 'Microsoft.ServiceBus/namespaces/topics/subscriptions/rules@2025-05-01-preview' = { +resource queueNamespacesResourceTopicLowPriorityRules 'Microsoft.ServiceBus/namespaces/topics/subscriptions/rules@2025-05-01-preview' = { parent: queueNamespacesResourceTopicLowPriority name: 'priorityFilter' properties: { From e8ff0fc6e8f8f426a124d7012879f30a18b8f4f2 Mon Sep 17 00:00:00 2001 From: Jason Bouska <82831332+skabou@users.noreply.github.com> Date: Tue, 2 Dec 2025 13:01:22 -0500 Subject: [PATCH 6/6] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- priority-queue/bicep/azure/azure-function-apps.bicep | 4 ++-- priority-queue/bicep/azure/sites.bicep | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/priority-queue/bicep/azure/azure-function-apps.bicep b/priority-queue/bicep/azure/azure-function-apps.bicep index 007f4b67..7ff9fe27 100644 --- a/priority-queue/bicep/azure/azure-function-apps.bicep +++ b/priority-queue/bicep/azure/azure-function-apps.bicep @@ -13,8 +13,8 @@ param storageAccountName string = 'st${uniqueString(resourceGroup().id)}' @description('Sets the name of the Application Insights resource for monitoring and diagnostics. Like the storage account, it uses a unique string based on the resource group ID.') param appInsightsName string = 'ai${uniqueString(resourceGroup().id)}' -var senderRoleId = '69a216fc-b8fb-44d8-bc22-1f3c2cd27a39' //Azure Service Bus Data Sender -var receiverRoleId = '4f6d3b9b-027b-4f4c-9142-0e5a2a2247e0' //Azure Service Bus Data Receiver +var senderRoleId = '69a216fc-b8fb-44d8-bc22-1f3c2cd27a39' //Azure Service Bus Data Sender +var receiverRoleId = '4f6d3b9b-027b-4f4c-9142-0e5a2a2247e0' //Azure Service Bus Data Receiver resource storageAccount 'Microsoft.Storage/storageAccounts@2025-01-01' = { name: storageAccountName diff --git a/priority-queue/bicep/azure/sites.bicep b/priority-queue/bicep/azure/sites.bicep index 96865af1..1ab4950e 100644 --- a/priority-queue/bicep/azure/sites.bicep +++ b/priority-queue/bicep/azure/sites.bicep @@ -15,7 +15,7 @@ param serviceBusNamespaceName string @description('The built-in role definition ID to assign to the Function App for accessing the Service Bus namespace.') param roleId string -@description('Sets the name of the Application Insights resource for monitoring and diagnostics. ') +@description('Sets the name of the Application Insights resource for monitoring and diagnostics.') param appInsightsName string @description('Specifies the maximum number of instances to which the Function App can scale out.')