diff --git a/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHigh.csproj b/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHigh.csproj index 613bc056..7d8d94d1 100644 --- a/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHigh.csproj +++ b/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHigh.csproj @@ -1,22 +1,22 @@ - + - net6.0 + net10.0 v4 - - - true + Exe + enabled - - + - - PreserveNewest - PreserveNewest Never - + + + + + + \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHighFn.cs b/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHighFn.cs index 5ce7ad67..0544fb7a 100644 --- a/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHighFn.cs +++ b/priority-queue/PriorityQueueConsumerHigh/PriorityQueueConsumerHighFn.cs @@ -1,14 +1,16 @@ -using Microsoft.Azure.WebJobs; using Microsoft.Extensions.Logging; +using Microsoft.Azure.Functions.Worker; namespace PriorityQueueConsumerHigh { - public static class PriorityQueueConsumerHighFn + public class PriorityQueueConsumerHighFn(ILogger logger) { - [FunctionName("HighPriorityQueueConsumerFunction")] - public static void Run([ServiceBusTrigger("messages", "highPriority", Connection = "ServiceBusConnection")]string highPriorityMessage, ILogger log) + private readonly ILogger _logger = logger; + + [Function("HighPriorityQueueConsumerFunction")] + public void Run([ServiceBusTrigger("messages", "highPriority", Connection = "ServiceBusConnection")] string highPriorityMessage) { - log.LogInformation($"C# ServiceBus topic trigger function processed message: {highPriorityMessage}"); + _logger.LogInformation($"C# ServiceBus topic trigger function processed message: {highPriorityMessage}"); } } -} +} \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerHigh/Program.cs b/priority-queue/PriorityQueueConsumerHigh/Program.cs new file mode 100644 index 00000000..cd97ae1f --- /dev/null +++ b/priority-queue/PriorityQueueConsumerHigh/Program.cs @@ -0,0 +1,7 @@ +using Microsoft.Extensions.Hosting; + +var host = new HostBuilder() + .ConfigureFunctionsWorkerDefaults() + .Build(); + +host.Run(); \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerHigh/host.json b/priority-queue/PriorityQueueConsumerHigh/host.json index beb2e402..5df170b6 100644 --- a/priority-queue/PriorityQueueConsumerHigh/host.json +++ b/priority-queue/PriorityQueueConsumerHigh/host.json @@ -1,11 +1,12 @@ { - "version": "2.0", - "logging": { - "applicationInsights": { - "samplingSettings": { - "isEnabled": true, - "excludedTypes": "Request" - } - } + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + }, + "enableLiveMetricsFilters": true } + } } \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerHigh/local.settings.json b/priority-queue/PriorityQueueConsumerHigh/local.settings.json deleted file mode 100644 index 80410be8..00000000 --- a/priority-queue/PriorityQueueConsumerHigh/local.settings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "IsEncrypted": false, - "Values": { - "AzureWebJobsStorage": "UseDevelopmentStorage=true", - "FUNCTIONS_WORKER_RUNTIME": "dotnet", - "ServiceBusConnection__fullyQualifiedNamespace": ".servicebus.windows.net" - } -} \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerHigh/local.settings.template.json b/priority-queue/PriorityQueueConsumerHigh/local.settings.template.json new file mode 100644 index 00000000..643657ce --- /dev/null +++ b/priority-queue/PriorityQueueConsumerHigh/local.settings.template.json @@ -0,0 +1,8 @@ +{ + "IsEncrypted": false, + "Values": { + "AzureWebJobsStorage": "UseDevelopmentStorage=true", + "FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated", + "ServiceBusConnection__fullyQualifiedNamespace": "{SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}" + } +} \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLow.csproj b/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLow.csproj index 408f55dd..7d8d94d1 100644 --- a/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLow.csproj +++ b/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLow.csproj @@ -1,23 +1,22 @@ - + - net6.0 + net10.0 v4 - 736bb6a2-68b4-463b-a8fb-3a90cba7cd4f - - - true + Exe + enabled - - + - - PreserveNewest - PreserveNewest Never - + + + + + + \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLowFn.cs b/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLowFn.cs index 36c1c593..a9213d2c 100644 --- a/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLowFn.cs +++ b/priority-queue/PriorityQueueConsumerLow/PriorityQueueConsumerLowFn.cs @@ -1,14 +1,16 @@ -using Microsoft.Azure.WebJobs; using Microsoft.Extensions.Logging; +using Microsoft.Azure.Functions.Worker; namespace PriorityQueueConsumerLow { - public static class PriorityQueueConsumerLowFn + public class PriorityQueueConsumerLowFn(ILogger logger) { - [FunctionName("LowPriorityQueueConsumerFunction")] - public static void Run([ServiceBusTrigger("messages", "lowPriority", Connection = "ServiceBusConnection")]string lowPriorityMessage, ILogger log) + private readonly ILogger _logger = logger; + + [Function("LowPriorityQueueConsumerFunction")] + public void Run([ServiceBusTrigger("messages", "lowPriority", Connection = "ServiceBusConnection")] string lowPriorityMessage) { - log.LogInformation($"C# ServiceBus topic trigger function processed message: {lowPriorityMessage}"); + _logger.LogInformation($"C# ServiceBus topic trigger function processed message: {lowPriorityMessage}"); } } -} +} \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerLow/Program.cs b/priority-queue/PriorityQueueConsumerLow/Program.cs new file mode 100644 index 00000000..cd97ae1f --- /dev/null +++ b/priority-queue/PriorityQueueConsumerLow/Program.cs @@ -0,0 +1,7 @@ +using Microsoft.Extensions.Hosting; + +var host = new HostBuilder() + .ConfigureFunctionsWorkerDefaults() + .Build(); + +host.Run(); \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerLow/host.json b/priority-queue/PriorityQueueConsumerLow/host.json index beb2e402..5df170b6 100644 --- a/priority-queue/PriorityQueueConsumerLow/host.json +++ b/priority-queue/PriorityQueueConsumerLow/host.json @@ -1,11 +1,12 @@ { - "version": "2.0", - "logging": { - "applicationInsights": { - "samplingSettings": { - "isEnabled": true, - "excludedTypes": "Request" - } - } + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + }, + "enableLiveMetricsFilters": true } + } } \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerLow/local.settings.json b/priority-queue/PriorityQueueConsumerLow/local.settings.json deleted file mode 100644 index 80410be8..00000000 --- a/priority-queue/PriorityQueueConsumerLow/local.settings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "IsEncrypted": false, - "Values": { - "AzureWebJobsStorage": "UseDevelopmentStorage=true", - "FUNCTIONS_WORKER_RUNTIME": "dotnet", - "ServiceBusConnection__fullyQualifiedNamespace": ".servicebus.windows.net" - } -} \ No newline at end of file diff --git a/priority-queue/PriorityQueueConsumerLow/local.settings.template.json b/priority-queue/PriorityQueueConsumerLow/local.settings.template.json new file mode 100644 index 00000000..643657ce --- /dev/null +++ b/priority-queue/PriorityQueueConsumerLow/local.settings.template.json @@ -0,0 +1,8 @@ +{ + "IsEncrypted": false, + "Values": { + "AzureWebJobsStorage": "UseDevelopmentStorage=true", + "FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated", + "ServiceBusConnection__fullyQualifiedNamespace": "{SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}" + } +} \ No newline at end of file diff --git a/priority-queue/PriorityQueueSender/PriorityQueueSender.csproj b/priority-queue/PriorityQueueSender/PriorityQueueSender.csproj index 613bc056..7fb11ec8 100644 --- a/priority-queue/PriorityQueueSender/PriorityQueueSender.csproj +++ b/priority-queue/PriorityQueueSender/PriorityQueueSender.csproj @@ -1,22 +1,32 @@ - - - net6.0 - v4 - - - true - - - - - - - - PreserveNewest - - - PreserveNewest - Never - - + + + net10.0 + v4 + Exe + enabled + enable + + + + + + + + + + + + + PreserveNewest + Never + + + + + PreserveNewest + + + + + diff --git a/priority-queue/PriorityQueueSender/PriorityQueueSenderFn.cs b/priority-queue/PriorityQueueSender/PriorityQueueSenderFn.cs index 9429b6d6..405c710e 100644 --- a/priority-queue/PriorityQueueSender/PriorityQueueSenderFn.cs +++ b/priority-queue/PriorityQueueSender/PriorityQueueSenderFn.cs @@ -1,31 +1,34 @@ using System; using System.Threading.Tasks; using Azure.Messaging.ServiceBus; -using Microsoft.Azure.WebJobs; +using Microsoft.Extensions.Logging; +using Microsoft.Azure.Functions.Worker; namespace PriorityQueueSender { - public static class PriorityQueueSenderFn + public class PriorityQueueSenderFn(ILogger logger, ServiceBusClient client) { - [FunctionName("PriorityQueueSenderFunction")] - public static async Task Run( - [TimerTrigger("0,30 * * * * *")] TimerInfo myTimer, - [ServiceBus("messages", Connection = "ServiceBusConnection")] IAsyncCollector collector ) + private readonly ILogger _logger = logger; + private readonly ServiceBusClient _client = client; + + [Function("PriorityQueueSenderFunction")] + public async Task Run([TimerTrigger("0,30 * * * * *")] TimerInfo myTimer) { + var sender = _client.CreateSender("messages"); for (int i = 0; i < 10; i++) { var messageId = Guid.NewGuid().ToString(); var lpMessage = new ServiceBusMessage() { MessageId = messageId }; lpMessage.ApplicationProperties["Priority"] = Priority.Low; lpMessage.Body = BinaryData.FromString($"Low priority message with Id: {messageId}"); - await collector.AddAsync(lpMessage); + await sender.SendMessageAsync(lpMessage); messageId = Guid.NewGuid().ToString(); var hpMessage = new ServiceBusMessage() { MessageId = messageId }; hpMessage.ApplicationProperties["Priority"] = Priority.High; hpMessage.Body = BinaryData.FromString($"High priority message with Id: {messageId}"); - await collector.AddAsync(hpMessage); + await sender.SendMessageAsync(hpMessage); } } } -} +} \ No newline at end of file diff --git a/priority-queue/PriorityQueueSender/Program.cs b/priority-queue/PriorityQueueSender/Program.cs new file mode 100644 index 00000000..e2037b64 --- /dev/null +++ b/priority-queue/PriorityQueueSender/Program.cs @@ -0,0 +1,29 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Azure.Messaging.ServiceBus; +using Azure.Identity; +using System; +using Microsoft.Extensions.Azure; + +var host = new HostBuilder() + .ConfigureFunctionsWorkerDefaults() + .ConfigureAppConfiguration((hostingContext, config) => + { + config.AddJsonFile("local.settings.json", optional: true, reloadOnChange: true); + }) + .ConfigureServices(services => + { + var configuration = services.BuildServiceProvider().GetRequiredService(); + + services.AddSingleton(configuration); + + services.AddAzureClients(builder => + { + builder.AddServiceBusClientWithNamespace(Environment.GetEnvironmentVariable("ServiceBusConnection__fullyQualifiedNamespace")) + .WithCredential(new DefaultAzureCredential()); + }); + }) + .Build(); + +host.Run(); \ No newline at end of file diff --git a/priority-queue/PriorityQueueSender/host.json b/priority-queue/PriorityQueueSender/host.json index beb2e402..5df170b6 100644 --- a/priority-queue/PriorityQueueSender/host.json +++ b/priority-queue/PriorityQueueSender/host.json @@ -1,11 +1,12 @@ { - "version": "2.0", - "logging": { - "applicationInsights": { - "samplingSettings": { - "isEnabled": true, - "excludedTypes": "Request" - } - } + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + }, + "enableLiveMetricsFilters": true } + } } \ No newline at end of file diff --git a/priority-queue/PriorityQueueSender/local.settings.json b/priority-queue/PriorityQueueSender/local.settings.json deleted file mode 100644 index 445c47d3..00000000 --- a/priority-queue/PriorityQueueSender/local.settings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "IsEncrypted": false, - "Values": { - "AzureWebJobsStorage": "UseDevelopmentStorage=true", - "FUNCTIONS_WORKER_RUNTIME": "dotnet", - "ServiceBusConnection__fullyQualifiedNamespace": "mgransb1.servicebus.windows.net" - } -} \ No newline at end of file diff --git a/priority-queue/PriorityQueueSender/local.settings.template.json b/priority-queue/PriorityQueueSender/local.settings.template.json new file mode 100644 index 00000000..643657ce --- /dev/null +++ b/priority-queue/PriorityQueueSender/local.settings.template.json @@ -0,0 +1,8 @@ +{ + "IsEncrypted": false, + "Values": { + "AzureWebJobsStorage": "UseDevelopmentStorage=true", + "FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated", + "ServiceBusConnection__fullyQualifiedNamespace": "{SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}" + } +} \ No newline at end of file diff --git a/priority-queue/Readme.md b/priority-queue/Readme.md index 01e0ef99..a5f5d707 100644 --- a/priority-queue/Readme.md +++ b/priority-queue/Readme.md @@ -1,100 +1,163 @@ -# Priority Queue Pattern +# Priority Queue pattern example -This document describes the Priority Queue Pattern example from the guide [Cloud Design Patterns](http://aka.ms/Cloud-Design-Patterns). +This directory contains an example of the [Priority Queue pattern](https://learn.microsoft.com/azure/architecture/patterns/priority-queue). -## System Requirements +This example demonstrates how priority queues can be implemented by using Service Bus topics and subscriptions. A time-triggered Azure Function is responsible for sending messages to a topic, each with a priority assigned. The receiving Azure Functions read messages from subscriptions that have the corresponding priority. -* Microsoft .NET 6 -* Microsoft Visual Studio 2019 or later version -* Azure Functions Core Tools version 4x +For local execution, the sample demonstrates the producer/consumer model, where each consumer processes only one type of message based on its priority. -## Before you start +In the Azure Deployment, the _PriorityQueueConsumerHigh_ Azure function could scale out to 200 instances, while the _PriorityQueueConsumerLow_ Azure function could scale out only to 40 instances. It simulates high priority messages being read from the queue more urgently than low priority messages. -Ensure that you have installed all of the software prerequisites. +The Azure deployment also demonstrates operational aspects of applications running on Azure. Monitoring tools are essential to understand how the sample operates. Azure Functions in the solution **must** be configured to use the diagnostics mechanism. Otherwise, trace information generated by the example will not be visible. -The example demonstrates operational aspects of applications running in Windows Azure. Therefore, you will need to use the monitoring tools in order to understand how the code sample works. You **must** ensure that the Azure Functions in the solution are configured to use the diagnostics mechanism. If not, you will not see the trace information generated by the example. +## :rocket: Deployment guide -## About the Example +Install the prerequisites and follow the steps to deploy and run an example of the Priority Queue pattern. -This example shows how you can implement priority queues by using Service Bus Topics and Subscriptions. A timer triggered Azure Function is responsible for sending messages to a topic. It assigns a priority to each message. The receiving Azure Functions read messages from subscriptions that have the corresponding priority. In the example, the PriorityQueueConsumerHigh Azure function can scale out to 200 instances, whereas the PriorityQueueConsumerLow worker runs only with one instance. This simulates high priority messages being read from the queue more urgently than low priority messages. +### Prerequisites -## Running the Example +- Permission to create a new resource group and resources in an [Azure subscription](https://azure.com/free). +- [Git](https://git-scm.com/downloads) +- [Azure CLI](https://learn.microsoft.com/cli/azure/install-azure-cli) +- [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0) +- [Azure Functions Core Tools v4.x](https://learn.microsoft.com/azure/azure-functions/functions-run-local#install-the-azure-functions-core-tools) -You can either run this example locally from Visual Studio or you can run it by deploying it to Azure. +### Steps -* From the Azure Portal, provision an Azure Service Bus Namespace. -* Once the Service Bus Namespace is created, add a new topic to it, name it "messages" (leave all the properties default). -* Once the topic is created, add a new subscription to it, name it "highPriority", set "10" as Max delivery count. -* Add a new filter of type "SqlFilter" to the subscription, name it "priorityFilter". -* Set the expression __Priority = 'highpriority'__ in the filter body. -* Click on "Save changes". -* Add another subscription to the topic, name it "lowPriority", set "10" as Max delivery count. -* Add a new filter of type "SqlFilter" to the subscription, name it "priorityFilter". -* Set the expression __Priority = 'lowpriority'__ in the filter body. -* Click on "Save changes". +1. Clone the repository -* Start Visual Studio. -* Open the solution you want to explore from the subfolders where you downloaded the examples. -* Edit the local.settings.json file in all the projects and change the ServiceBusConnection__fullyQualifiedNamespace setting by changing the placeholder "" to your Azure Service Bus Namespace name. + Open a terminal, clone the repository, and navigate to the `priority-queue` directory. -* If you want to run the example in the local Windows Azure emulator: - * Set the PriorityQueueSender project as startup. - * Press F5 in Visual Studio to start the example running. The function will start sending messages to the topic, every 30 seconds. - * Stop the execution, change the startup project to either the PriorityQueueConsumerHigh or PriorityQueueConsumerLow - * Press F5 in Visual Studio to start the execution of the consumer function - * In the Azure Functions Core Tools Console, you can view the diagnostic information generated by Trace statements in the code. + ```shell + git clone https://github.com/mspnp/cloud-design-patterns.git + cd cloud-design-patterns + cd priority-queue + ``` -* If you want to run the example on Azure: - * On every function project, right click and select publish. - * Select "Azure" as publishing target. - * Select "Azure Function App" as specific target. - * In the Functions instance step, select you Azure Function app or create a new one using the link button. - * Since your app settings are different you need to deploy every Azure Function in a separate Azure Funcion App. - * You can select an existing resource group and storage account or create new ones. - * In "Hosting" section, click on the the three dots (...) in the upper right corner. - * Select "Manage Azure App Service Settings". - * You need to add these two settings: +1. Log into Azure and create an empty resource group. - - ServiceBusConnection__fullyQualifiedNamespace: + Create an empty resource group to hold the resources for this example. The location you select in the resource group creation command below is the Azure region that your resources will be deployed in; modify as needed. - Set the value to: + ```bash + az login + az account set -s - .servicebus.windows.net + LOCATION=eastus2 + RESOURCE_GROUP_NAME=rg-priority-queue-${LOCATION} + az group create -n $RESOURCE_GROUP_NAME -l $LOCATION + ``` - Replacing the placeholder with you Azure Service Bus Namespace name. +1. Deploy the supporting Azure resources. - * Once the Functions are deployed you need to restrict the maximum number of instances your app service consumption can scale out to: + ```bash + CURRENT_USER_OBJECT_ID=$(az ad signed-in-user show -o tsv --query id) + SERVICE_BUS_NAMESPACE_NAME="sbns-priority-queue-$(LC_ALL=C tr -dc 'a-z0-9' < /dev/urandom | fold -w 7 | head -n 1)" - * From the portal go to the Function App that contains the "PriorityQueueConsumerLow" Azure Function - * Navigate to Scale Out on the left menu - * On the App Scale Out dialog set the "Enforce Scale Out Limit" to "Yes" - * Set the Maximum Scale Out Limit to 1 instance - - You don't need to modify these settings for the Function App containing the "PriorityQueueConsumerHigh" Azure Function since the default setting is 200; this ensures that high priority messages are read from the queue more quickly than low priority messages. + # This takes about two minutes + az deployment group create -n deploy-priority-queue -f bicep/main.bicep -g $RESOURCE_GROUP_NAME -p queueNamespaces=$SERVICE_BUS_NAMESPACE_NAME principalId=$CURRENT_USER_OBJECT_ID + ``` - * Now you need to configure the managed identities role assignments: +1. Configure the samples to use the created Azure resources. - In the Azure portal, navigate to the Azure Service Bus Namespace that was provisioned in the first step. - Select Access Control (IAM). This is where you can view and configure who has access to the resource. - Click Add and select add role assignment. - From the list, select "Azure Service Bus Data Sender", click Next - In "Assign access to", radio button list, select "Managed identity" - Click on "Select members", the "Select Manage identities" dialog will show up - In the Managed Identity dropdown list select "Function App" - Find the PriorityQueueSender function app click on it - Click "Select" - On the main dialog, click "Review + Assign" + ```bash + # Retrieve the primary connection string for the Service Bus namespace. + SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE="${SERVICE_BUS_NAMESPACE_NAME}.servicebus.windows.net" - For the Azure Consumer Function Apps, repeat the process but in this case use the role - "Azure Service Bus Data Reader" + sed "s|{SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}|${SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}|g" ./PriorityQueueSender/local.settings.template.json > ./PriorityQueueSender/local.settings.json + sed "s|{SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}|${SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}|g" ./PriorityQueueConsumerHigh/local.settings.template.json > ./PriorityQueueConsumerHigh/local.settings.json + sed "s|{SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}|${SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE}|g" ./PriorityQueueConsumerLow/local.settings.template.json > ./PriorityQueueConsumerLow/local.settings.json + ``` - * Once the Functions are deployed you can configure monitoring by following these steps: +1. [Run Azurite](https://learn.microsoft.com/azure/storage/common/storage-use-azurite#run-azurite) blob storage emulation service. - - From the Azure Portal, go to the Function App Service - - Click on "Functions" and select the azure function - - From the "developer" left menu click on "Monitor" - - Turn on application insights - - Refresh the screen - - Once refreshed you will see two tabs, "invocation" and "logs" - - From the invocations tab you can see the twenty most recent function invocation traces. For more advanced analysis, run the query in Application Insights. - - From the logs tab you can see the logging information that your functions are sending. \ No newline at end of file + > Azure Functions require an Azure Storage account as a backing resource. When running locally, you can use Azurite, the local storage emulator, to fulfill this requirement. +Alternatively, you may configure the AzureWebJobsStorage setting to use a real Azure Storage account if preferred. + +1. Launch the Azure Function PriorityQueueSender to generate Low and High messages. + + ```bash + cd ./PriorityQueueSender + func start + ``` + +1. In a new terminal, launch the Azure Function PriorityQueueConsumerLow to consume messages. + + ```bash + cd ./PriorityQueueConsumerLow + func start -p 15000 + ``` + + > Please note: For demo purposes, the sample application will write content to the screen. + +1. In a new terminal, launch the Azure Function PriorityQueueConsumerHigh to consume messages. + + ```bash + cd ./PriorityQueueConsumerHigh + func start -p 15001 + ``` + + > Please note: For demo purposes, the sample application will write content to the screen. + +## Deploy the example to Azure (Optional) + +This Bicep template sets up the core infrastructure for a priority-based message processing system using Azure Functions. It creates a secure Storage Account, an Application Insights instance for monitoring, and uses a previously created Service Bus namespace to enable communication between the sender and consumer functions. The deployment includes three Azure Function Apps: one sender and two consumers, each with different scaling limits to simulate message prioritization. +The funcPriorityQueueConsumerHigh function can scale out to 200 instances, allowing it to process high-priority messages quickly. The funcPriorityQueueConsumerLow function is limited to 40 instances, handling lower-priority messages with less urgency. All function apps use the FlexConsumption plan and are connected to Application Insights for diagnostics and monitoring. Role assignments are configured to securely grant access to the Service Bus and Storage resources using managed identities. +All Azure Function Apps share the same Storage Account and Application Insights instance (It is essential to understand how the sample operates), which centralizes observability and logging. + +```bash + # This takes about three minutes + az deployment group create -n deploy-priority-queue-sites -f bicep/azure/azure-function-apps.bicep -g $RESOURCE_GROUP_NAME -p serviceBusNamespaceName=$SERVICE_BUS_NAMESPACE_NAME +``` +After deploying the infrastructure, you need to publish each Azure Function to its corresponding Function App using Azure Functions Core Tools: + +```bash + cd ./PriorityQueueSender + func azure functionapp publish funcPriorityQueueSender + cd .. + + cd ./PriorityQueueConsumerLow + func azure functionapp publish funcPriorityQueueConsumerLow + cd .. + + cd ./PriorityQueueConsumerHigh + func azure functionapp publish funcPriorityQueueConsumerHigh + cd .. +``` + +You can view the maximum scaling configuration in the Azure Portal. Go to each Function App, then under Settings, select Scale and concurrency. There, you'll see the Maximum instance count setting. + +Once the Azure Functions are deployed, you can use Application Insights to monitor their activity. In the Azure portal, go to the Application Insights resource, then select Logs. Switch to KQL mode and run the following queries to view trace logs for each function: + +``` + // Traces for the High priority consumer + traces + | where operation_Name contains "High" + + // Traces for the Low priority consumer + traces + | where operation_Name contains "Low" + + // Traces for the Sender function + traces + | where operation_Name contains "Sender" +``` +In addition to viewing trace logs, you can also use Application Insights to analyze request-level data for each Azure Function. The following queries help you inspect recent requests and understand how frequently each function is being called: + +``` + // Recent requests with key details + requests + | project timestamp, operation_Name, cloud_RoleName, id, success, resultCode, duration, operation_Id + | order by timestamp desc + + // Count of requests by Function + requests + | summarize RequestCount = count() by operation_Name + | order by RequestCount desc +``` +## :broom: Clean up resources + +Be sure to delete Azure resources when not using them. Since all resources were deployed into a new resource group, you can simply delete the resource group. + +```bash +az group delete -n $RESOURCE_GROUP_NAME -y +``` diff --git a/priority-queue/bicep/azure/azure-function-apps.bicep b/priority-queue/bicep/azure/azure-function-apps.bicep new file mode 100644 index 00000000..7ff9fe27 --- /dev/null +++ b/priority-queue/bicep/azure/azure-function-apps.bicep @@ -0,0 +1,82 @@ +targetScope = 'resourceGroup' + +@minLength(5) +@description('Location of the resources. Defaults to resource group location.') +param location string = resourceGroup().location + +@description('The name of the existing Service Bus namespace used for message queuing between the sender and consumer functions.') +param serviceBusNamespaceName string + +@description('Defines the name of the Storage Account used by the Function Apps. It uses a unique string based on the resource group ID to ensure global uniqueness.') +param storageAccountName string = 'st${uniqueString(resourceGroup().id)}' + +@description('Sets the name of the Application Insights resource for monitoring and diagnostics. Like the storage account, it uses a unique string based on the resource group ID.') +param appInsightsName string = 'ai${uniqueString(resourceGroup().id)}' + +var senderRoleId = '69a216fc-b8fb-44d8-bc22-1f3c2cd27a39' //Azure Service Bus Data Sender +var receiverRoleId = '4f6d3b9b-027b-4f4c-9142-0e5a2a2247e0' //Azure Service Bus Data Receiver + +resource storageAccount 'Microsoft.Storage/storageAccounts@2025-01-01' = { + name: storageAccountName + location: location + sku: { + name: 'Standard_LRS' + } + kind: 'StorageV2' + properties: { + defaultToOAuthAuthentication: true + publicNetworkAccess: 'Enabled' + allowCrossTenantReplication: false + minimumTlsVersion: 'TLS1_2' + allowBlobPublicAccess: false + allowSharedKeyAccess: false + networkAcls: { + bypass: 'AzureServices' + virtualNetworkRules: [] + ipRules: [] + defaultAction: 'Allow' + } + supportsHttpsTrafficOnly: true + encryption: { + services: { + file: { + keyType: 'Account' + enabled: true + } + blob: { + keyType: 'Account' + enabled: true + } + } + keySource: 'Microsoft.Storage' + } + } +} + +resource appInsights 'Microsoft.Insights/components@2020-02-02' = { + name: appInsightsName + location: location + kind: 'web' + properties: { + Application_Type: 'web' + } +} + +module functionApp './sites.bicep' = [ + for name in [ + 'funcPriorityQueueSender' + 'funcPriorityQueueConsumerLow' + 'funcPriorityQueueConsumerHigh' + ]: { + name: name + params: { + location: location + functionAppName: name + storageAccountName: storageAccount.name + serviceBusNamespaceName: serviceBusNamespaceName + roleId: name == 'funcPriorityQueueSender' ? senderRoleId : receiverRoleId + appInsightsName: appInsights.name + scaleUp: name == 'funcPriorityQueueConsumerHigh' ? 200 : 40 + } + } +] diff --git a/priority-queue/bicep/azure/sites.bicep b/priority-queue/bicep/azure/sites.bicep new file mode 100644 index 00000000..1ab4950e --- /dev/null +++ b/priority-queue/bicep/azure/sites.bicep @@ -0,0 +1,148 @@ +targetScope = 'resourceGroup' + +@minLength(5) +@description('Location of the resources. Defaults to resource group location.') +param location string = resourceGroup().location + +param functionAppName string + +@description('Defines the name of the Storage Account used by the Function Apps.') +param storageAccountName string + +@description('The name of the existing Service Bus namespace used for message queuing between the sender and consumer functions.') +param serviceBusNamespaceName string + +@description('The built-in role definition ID to assign to the Function App for accessing the Service Bus namespace.') +param roleId string + +@description('Sets the name of the Application Insights resource for monitoring and diagnostics.') +param appInsightsName string + +@description('Specifies the maximum number of instances to which the Function App can scale out.') +param scaleUp int = 1 + +@description('Generates a unique container name for deployments') +var deploymentStorageContainerName = toLower('package-${take(functionAppName, 32)}') + +@description('Built-in role definition ID for Storage Blob Data Owner') +var azureStorageBlobDataOwnerRole = subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + 'b7e6dc6d-f1e8-4753-8033-0f276bb0955b' +) // Storage Blob Data Owner + +resource serviceBusNamespace 'Microsoft.ServiceBus/namespaces@2025-05-01-preview' existing = { + name: serviceBusNamespaceName +} + +resource storageAccount 'Microsoft.Storage/storageAccounts@2025-01-01' existing = { + name: storageAccountName +} + +// Define the blob service under the existing storage account for deployments +resource blobService 'Microsoft.Storage/storageAccounts/blobServices@2021-09-01' = { + parent: storageAccount + name: 'default' + properties: { + deleteRetentionPolicy: { + enabled: false + } + } + + // Define the container inside the blob service + resource deploymentContainer 'containers' = { + name: deploymentStorageContainerName + properties: { + publicAccess: 'None' + } + } +} + +resource appInsights 'Microsoft.Insights/components@2020-02-02' existing = { + name: appInsightsName +} + +resource appServicePlan 'Microsoft.Web/serverfarms@2024-11-01' = { + name: '${functionAppName}-plan' + location: location + sku: { + name: 'FC1' + tier: 'FlexConsumption' + } + kind: 'functionapp' + properties: { + reserved: true + } +} + +resource functionApp 'Microsoft.Web/sites@2024-11-01' = { + name: functionAppName + location: location + kind: 'functionapp,linux' + identity: { + type: 'SystemAssigned' + } + properties: { + serverFarmId: appServicePlan.id + siteConfig: { + minTlsVersion: '1.2' + appSettings: [ + { + name: 'AzureWebJobsStorage__accountName' + value: storageAccount.name + } + { + name: 'ServiceBusConnection__fullyQualifiedNamespace' + value: '${serviceBusNamespaceName}.servicebus.windows.net' + } + { + name: 'APPINSIGHTS_INSTRUMENTATIONKEY' + value: appInsights.properties.InstrumentationKey + } + { + name: 'APPLICATIONINSIGHTS_CONNECTION_STRING' + value: appInsights.properties.ConnectionString + } + ] + } + functionAppConfig: { + deployment: { + storage: { + type: 'blobContainer' + value: '${storageAccount.properties.primaryEndpoints.blob}${blobService::deploymentContainer.name}' + authentication: { + type: 'SystemAssignedIdentity' + } + } + } + runtime: { + name: 'dotnet-isolated' + version: '10.0' + } + scaleAndConcurrency: { + maximumInstanceCount: scaleUp + instanceMemoryMB: 2048 + } + } + httpsOnly: true + } +} + +resource roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(functionApp.id, roleId) + scope: serviceBusNamespace + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleId) + principalId: functionApp.identity.principalId + principalType: 'ServicePrincipal' + } +} + +resource storageBlobDataOwnerRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(functionApp.id, storageAccount.id, 'Storage Blob Data Owner') + scope: storageAccount + properties: { + roleDefinitionId: azureStorageBlobDataOwnerRole + principalId: functionApp.identity.principalId + principalType: 'ServicePrincipal' + } +} diff --git a/priority-queue/bicep/main.bicep b/priority-queue/bicep/main.bicep new file mode 100644 index 00000000..fc0b8636 --- /dev/null +++ b/priority-queue/bicep/main.bicep @@ -0,0 +1,209 @@ +targetScope = 'resourceGroup' + +@minLength(5) +@description('Location of the resources. Defaults to resource group location.') +param location string = resourceGroup().location + +@minLength(15) +@description('Service Bus Namespace Name.') +param queueNamespaces string + +@minLength(36) +@description('The principal ID used to run the Azure Functions. In Azure, this should be the managed identity (system-assigned or user-assigned) of the Azure Function. When running locally, it should be your user identity.') +param principalId string + +var logAnalyticsName = 'loganalytics-${uniqueString(subscription().subscriptionId, resourceGroup().id)}' + +var senderServiceBusRole = subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + '69a216fc-b8fb-44d8-bc22-1f3c2cd27a39' +) // Azure Service Bus Data Sender +var receiverServiceBusRole = subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + '4f6d3b9b-027b-4f4c-9142-0e5a2a2247e0' +) // Azure Service Bus Data Receiver + +resource queueNamespacesResource 'Microsoft.ServiceBus/namespaces@2025-05-01-preview' = { + name: queueNamespaces + location: location + sku: { + name: 'Standard' + tier: 'Standard' + } + properties: { + geoDataReplication: { + maxReplicationLagDurationInSeconds: 0 + locations: [ + { + locationName: location + roleType: 'Primary' + } + ] + } + premiumMessagingPartitions: 0 + minimumTlsVersion: '1.2' + publicNetworkAccess: 'Enabled' + disableLocalAuth: false + zoneRedundant: false + } +} + +resource queueNamespacesResourceRootManageSharedAccessKey 'Microsoft.ServiceBus/namespaces/authorizationrules@2025-05-01-preview' = { + parent: queueNamespacesResource + name: 'RootManageSharedAccessKey' + properties: { + rights: [ + 'Listen' + 'Manage' + 'Send' + ] + } +} + +resource queueNamespacesResourceNetworkRules 'Microsoft.ServiceBus/namespaces/networkrulesets@2025-05-01-preview' = { + parent: queueNamespacesResource + name: 'default' + properties: { + publicNetworkAccess: 'Enabled' + defaultAction: 'Allow' + virtualNetworkRules: [] + ipRules: [] + trustedServiceAccessEnabled: false + } +} + +resource queueNamespacesResourceTopic 'Microsoft.ServiceBus/namespaces/topics@2025-05-01-preview' = { + parent: queueNamespacesResource + name: 'messages' + properties: { + maxMessageSizeInKilobytes: 256 + maxSizeInMegabytes: 1024 + requiresDuplicateDetection: false + duplicateDetectionHistoryTimeWindow: 'PT10M' + enableBatchedOperations: true + status: 'Active' + supportOrdering: true + enablePartitioning: false + enableExpress: false + } +} + +resource queueNamespacesResourceTopicHigPriority 'Microsoft.ServiceBus/namespaces/topics/subscriptions@2025-05-01-preview' = { + parent: queueNamespacesResourceTopic + name: 'highPriority' + properties: { + isClientAffine: false + lockDuration: 'PT1M' + requiresSession: false + deadLetteringOnMessageExpiration: false + deadLetteringOnFilterEvaluationExceptions: true + maxDeliveryCount: 10 + status: 'Active' + enableBatchedOperations: true + } +} + +resource queueNamespacesResourceTopicLowPriority 'Microsoft.ServiceBus/namespaces/topics/subscriptions@2025-05-01-preview' = { + parent: queueNamespacesResourceTopic + name: 'lowPriority' + properties: { + isClientAffine: false + lockDuration: 'PT1M' + requiresSession: false + deadLetteringOnMessageExpiration: false + deadLetteringOnFilterEvaluationExceptions: true + maxDeliveryCount: 10 + status: 'Active' + enableBatchedOperations: true + } +} + +resource queueNamespacesResourceTopicHigPriorityRules 'Microsoft.ServiceBus/namespaces/topics/subscriptions/rules@2025-05-01-preview' = { + parent: queueNamespacesResourceTopicHigPriority + name: 'priorityFilter' + properties: { + action: {} + filterType: 'SqlFilter' + sqlFilter: { + sqlExpression: 'Priority = \'highpriority\'' + compatibilityLevel: 20 + } + } +} + +resource queueNamespacesResourceTopicLowPriorityRules 'Microsoft.ServiceBus/namespaces/topics/subscriptions/rules@2025-05-01-preview' = { + parent: queueNamespacesResourceTopicLowPriority + name: 'priorityFilter' + properties: { + action: {} + filterType: 'SqlFilter' + sqlFilter: { + sqlExpression: 'Priority = \'lowpriority\'' + compatibilityLevel: 20 + } + } +} + +resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2025-02-01' = { + name: logAnalyticsName + location: location + properties: { + retentionInDays: 30 + features: { + searchVersion: 1 + } + sku: { + name: 'PerGB2018' + } + } +} + +resource diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: '${queueNamespacesResource.name}-diagnostic' + scope: queueNamespacesResource + properties: { + logs: [ + { + category: 'OperationalLogs' + enabled: true + retentionPolicy: { + enabled: false + days: 0 + } + } + ] + metrics: [ + { + category: 'AllMetrics' + enabled: true + retentionPolicy: { + enabled: false + days: 0 + } + } + ] + workspaceId: logAnalytics.id + } +} + +// Assign Role to allow sending messages to the Service Bus +resource serviceBusSenderRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(resourceGroup().id, principalId, 'ServiceBusSenderRole') + scope: queueNamespacesResource + properties: { + roleDefinitionId: senderServiceBusRole + principalId: principalId + principalType: 'User' // 'ServicePrincipal' if this was App Service with a managed identity + } +} + +// Assign Role to allow receiving messages from the Service Bus +resource serviceBusReceiverRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(resourceGroup().id, principalId, 'ServiceBusReceiverRole') + scope: queueNamespacesResource + properties: { + roleDefinitionId: receiverServiceBusRole + principalId: principalId + principalType: 'User' // 'ServicePrincipal' if this was App Service with a managed identity + } +}