feat: Provide support for scraping Azure Synapse workspace, Apache Sp…

…ark & SQL pools (#1513) * Add basics Signed-off-by: Tom Kerkhove <[email protected]> * Provide documentation & changelog Signed-off-by: Tom Kerkhove <[email protected]> * Add discovery groups Signed-off-by: Tom Kerkhove <[email protected]> * Add new metrics Signed-off-by: Tom Kerkhove <[email protected]> * Implement scrapers Signed-off-by: Tom Kerkhove <[email protected]> * Less verbose logs Signed-off-by: Tom Kerkhove <[email protected]> * Information logs Signed-off-by: Tom Kerkhove <[email protected]> * Cleanup Signed-off-by: Tom Kerkhove <[email protected]> * Code cleanup Signed-off-by: Tom Kerkhove <[email protected]> * Provide unit tests Signed-off-by: Tom Kerkhove <[email protected]>
tomkerkhove · Feb 18, 2021 · 3acf6c2 · 3acf6c2
1 parent a5cc8ba
commit 3acf6c2
Show file tree

Hide file tree

Showing 54 changed files with 1,572 additions and 56 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,3 +1,4 @@
+<!-- markdownlint-disable -->
 <!-- For new scrapers, make sure to follow https://github.com/tomkerkhove/promitor/blob/master/adding-a-new-scraper.md
 
 When implementing a new scraper; these tasks are completed:
@@ -28,3 +29,5 @@ promitor_ratelimit_arm{tenant_id="T",subscription_id="SUB",app_id="APP"} 11996 1
  -->
 
 Fixes #
+
+<!-- markdownlint-enable -->
diff --git a/changelog/content/experimental/unreleased.md b/changelog/content/experimental/unreleased.md
@@ -12,6 +12,12 @@ version:
  | [#343](https://github.com/tomkerkhove/promitor/issues/343))
 - {{% tag added %}} Support for scraping Azure SQL Elastic Pool ([docs](https://promitor.io/configuration/v2.x/metrics/sql-elastic-pool)
  | [#319](https://github.com/tomkerkhove/promitor/issues/319))
+- {{% tag added %}} Support for scraping Azure Synapse (Apache Spark pool) ([docs](https://promitor.io/configuration/v2.x/metrics/synapse-apache-spark-pool)
+ | [#1477](https://github.com/tomkerkhove/promitor/issues/1477))
+- {{% tag added %}} Support for scraping Azure Synapse (SQL pool) ([docs](https://promitor.io/configuration/v2.x/metrics/sql-elastic-pool)
+ | [#1477](https://github.com/tomkerkhove/promitor/issues/1477))
+- {{% tag added %}} Support for scraping Azure Synapse (Workspace) ([docs](https://promitor.io/configuration/v2.x/metrics/sql-elastic-pool)
+ | [#1477](https://github.com/tomkerkhove/promitor/issues/1477))
 - {{% tag added %}} Support Prometheus Operator in Helm chart ([PR #31](https://github.com/promitor/charts/pull/31)
  | Contributed  by [@DaveOHenry](https://github.com/DaveOHenry) 🎉)
 - {{% tag added %}} Support for affinity in Helm chart ([PR #30](https://github.com/promitor/charts/pull/30)
@@ -40,6 +46,12 @@ version:
  | [#343](https://github.com/tomkerkhove/promitor/issues/343))
 - {{% tag added %}} Support for scraping Azure SQL Elastic Pool ([docs](https://promitor.io/configuration/v2.x/metrics/sql-elastic-pool)
  | [#319](https://github.com/tomkerkhove/promitor/issues/319))
+- {{% tag added %}} Support for scraping Azure Synapse (Apache Spark pool) ([docs](https://promitor.io/configuration/v2.x/metrics/synapse-apache-spark-pool)
+ | [#1477](https://github.com/tomkerkhove/promitor/issues/1477))
+- {{% tag added %}} Support for scraping Azure Synapse (SQL pool) ([docs](https://promitor.io/configuration/v2.x/metrics/sql-elastic-pool)
+ | [#1477](https://github.com/tomkerkhove/promitor/issues/1477))
+- {{% tag added %}} Support for scraping Azure Synapse (Workspace) ([docs](https://promitor.io/configuration/v2.x/metrics/sql-elastic-pool)
+ | [#1477](https://github.com/tomkerkhove/promitor/issues/1477))
 - {{% tag added %}} Support for affinity in Helm chart ([PR #30](https://github.com/promitor/charts/pull/30)
  | Contributed by [@t3mi](https://github.com/t3mi) 🎉)
 - {{% tag added %}} Support for priority class name in Helm chart ([PR #30](https://github.com/promitor/charts/pull/30)

diff --git a/config/promitor/resource-discovery/resource-discovery-declaration.yaml b/config/promitor/resource-discovery/resource-discovery-declaration.yaml
@@ -46,6 +46,12 @@ resourceDiscoveryGroups:
   type: SqlServer
 - name: storage-accounts
   type: StorageAccount
+- name: synapse-apache-spark-pools
+  type: SynapseApacheSparkPool
+- name: synapse-sql-pools
+  type: SynapseSqlPool
+- name: synapse-workspaces
+  type: SynapseWorkspace
 - name: virtual-machines
   type: VirtualMachine
 - name: virtual-machine-scale-sets

diff --git a/config/promitor/resource-discovery/runtime.yaml b/config/promitor/resource-discovery/runtime.yaml
@@ -3,9 +3,7 @@ server:
 telemetry:
   applicationInsights:
     instrumentationKey: ABC
-    isEnabled: true
-    verbosity: warning
+    isEnabled: false
   containerLogs:
     isEnabled: true
-    verbosity: trace
-  defaultVerbosity: trace
+  defaultVerbosity: information
diff --git a/config/promitor/scraper/metrics.yaml b/config/promitor/scraper/metrics.yaml
@@ -10,17 +10,6 @@ metricDefaults:
     # Every minute
     schedule: "0 * * ? * *"
 metrics:
-- name: promitor_demo_frontdoor_backend_health
-  description: "Average percentage of memory usage on an Azure App Plan"
-  resourceType: FrontDoor
-  labels:
-    app: promitor
-  azureMetricConfiguration:
-    metricName: BackendHealthPercentage
-    aggregation:
-      type: Average
-  resources:
-  - name: promitor-app-plan
 - name: promitor_demo_appplan_percentage_cpu
   description: "Average percentage of memory usage on an Azure App Plan"
   resourceType: AppPlan
@@ -248,4 +237,31 @@ metrics:
     aggregation:
       type: Average
   resourceDiscoveryGroups:
-  - name: sql-elastic-pools
+  - name: sql-elastic-pools
+- name: promitor_demo_synapse_apache_spark_apps_ended
+  description: "Amount of apps ended running on Apache Spark pool in Azure Synapse"
+  resourceType: SynapseApacheSparkPool
+  azureMetricConfiguration:
+    metricName: BigDataPoolApplicationsEnded
+    aggregation:
+      type: Total
+  resourceDiscoveryGroups:
+  - name: synapse-apache-spark-pools
+- name: promitor_demo_synapse_sql_pool_dwu_limit
+  description: "Amount of DWUs defined as limit for SQL pool in Azure Synapse"
+  resourceType: SynapseSqlPool
+  azureMetricConfiguration:
+    metricName: DWULimit
+    aggregation:
+      type: Maximum
+  resourceDiscoveryGroups:
+  - name: synapse-sql-pools
+- name: promitor_demo_synapse_workspace_builtin_sql_processed_bytes
+  description: "Amount of bytes processed in Azure Synapse workspace"
+  resourceType: SynapseWorkspace
+  azureMetricConfiguration:
+    metricName: BuiltinSqlPoolDataProcessedBytes
+    aggregation:
+      type: Total
+  resourceDiscoveryGroups:
+  - name: synapse-workspaces
diff --git a/config/promitor/scraper/runtime.yaml b/config/promitor/scraper/runtime.yaml
@@ -19,7 +19,7 @@ metricsConfiguration:
 telemetry:
   applicationInsights:
     instrumentationKey: ABC
-    isEnabled: true
+    isEnabled: false
     verbosity: warning
   containerLogs:
     isEnabled: true

diff --git a/docs/configuration/v2.x/metrics/index.md b/docs/configuration/v2.x/metrics/index.md
@@ -153,6 +153,9 @@ We also provide a simplified way to scrape the following Azure resources:
 - [Azure Storage (Blob)](blob-storage)
 - [Azure Storage (Files)](file-storage)
 - [Azure Storage (Queue)](storage-queue)
+- [Azure Synapse (Apache Spark pool)](synapse-apache-spark-pool)
+- [Azure Synapse (SQL pool)](synapse-sql-pool)
+- [Azure Synapse (Workspace)](synapse-workspace)
 - [Azure Virtual Machine](virtual-machine)
 - [Azure Virtual Machine Scale Set (VMSS)](virtual-machine-scale-set)
 - [Azure Web App](web-app)

diff --git a/docs/configuration/v2.x/metrics/synapse-apache-spark-pool.md b/docs/configuration/v2.x/metrics/synapse-apache-spark-pool.md
@@ -0,0 +1,44 @@
+---
+layout: default
+title: Azure Synapse (Apache Spark pool) Declaration
+---
+
+## Azure Synapse (Apache Spark pool)
+
+![Availability Badge](https://img.shields.io/badge/Available%20Starting-v2.1-green.svg)![Resource Discovery Support Badge](https://img.shields.io/badge/Support%20for%20Resource%20Discovery-Yes-green.svg)
+
+You can scrape an Azure Synapse Apache Spark pool via the `SynapseApacheSparkPool` resource type.
+
+The following fields need to be provided:
+
+- `workspaceName` - The name of the Azure Synapse workspace.
+- `poolName` - The name of the Apache Spark pool.
+
+All supported metrics are documented in the official [Azure Monitor documentation](https://docs.microsoft.com/en-us/azure/azure-monitor/platform/metrics-supported#microsoftsynapseworkspacesbigdatapools).
+
+The following scraper-specific metric labels will be added:
+
+- `workspace_name` - The name of the Azure Synapse workspace.
+- `pool_name` - The name of the Apache Spark pool.
+
+Example:
+
+```yaml
+- name: promitor_demo_synapse_apache_spark_apps_ended
+  description: "Amount of apps ended running on Apache Spark pool in Azure Synapse"
+  resourceType: SynapseApacheSparkPool
+  azureMetricConfiguration:
+    metricName: BigDataPoolApplicationsEnded
+    aggregation:
+      type: Total
+  resources:
+  - workspaceName: promitor-synapse
+    poolName: sparkpool
+  resourceDiscoveryGroups: # Optional, requires Promitor Resource Discovery agent (https://promitor.io/concepts/how-it-works#using-resource-discovery)
+  - name: synapse-apache-spark-pools
+```
+
+<!-- markdownlint-disable MD033 -->
+[&larr; back to metrics declarations](/configuration/v2.x/metrics)<br />
+[&larr; back to introduction](/)
+<!-- markdownlint-enable -->
diff --git a/docs/configuration/v2.x/metrics/synapse-sql-pool.md b/docs/configuration/v2.x/metrics/synapse-sql-pool.md
@@ -0,0 +1,44 @@
+---
+layout: default
+title: Azure Synapse (SQL pool) Declaration
+---
+
+## Azure Synapse (SQL pool)
+
+![Availability Badge](https://img.shields.io/badge/Available%20Starting-v2.1-green.svg)![Resource Discovery Support Badge](https://img.shields.io/badge/Support%20for%20Resource%20Discovery-Yes-green.svg)
+
+You can scrape an Azure Synapse SQL pool via the `SynapseSqlPool` resource type.
+
+The following fields need to be provided:
+
+- `workspaceName` - The name of the Azure Synapse workspace.
+- `poolName` - The name of the SQL pool.
+
+All supported metrics are documented in the official [Azure Monitor documentation](https://docs.microsoft.com/en-us/azure/azure-monitor/platform/metrics-supported#microsoftsynapseworkspacessqlpools).
+
+The following scraper-specific metric labels will be added:
+
+- `workspace_name` - The name of the Azure Synapse workspace.
+- `pool_name` - The name of the SQL pool.
+
+Example:
+
+```yaml
+- name: promitor_demo_synapse_sql_pool_dwu_limit
+  description: "Amount of DWUs defined as limit for SQL pool in Azure Synapse"
+  resourceType: SynapseSqlPool
+  azureMetricConfiguration:
+    metricName: DWULimit
+    aggregation:
+      type: Maximum
+  resources:
+  - workspaceName: promitor-synapse
+    poolName: sqlpool
+  resourceDiscoveryGroups: # Optional, requires Promitor Resource Discovery agent (https://promitor.io/concepts/how-it-works#using-resource-discovery)
+  - name: synapse-sql-pools
+```
+
+<!-- markdownlint-disable MD033 -->
+[&larr; back to metrics declarations](/configuration/v2.x/metrics)<br />
+[&larr; back to introduction](/)
+<!-- markdownlint-enable -->
diff --git a/docs/configuration/v2.x/metrics/synapse-workspace.md b/docs/configuration/v2.x/metrics/synapse-workspace.md
@@ -0,0 +1,42 @@
+---
+layout: default
+title: Azure Synapse (Workspace) Declaration
+---
+
+## Azure Synapse (Workspace)
+
+![Availability Badge](https://img.shields.io/badge/Available%20Starting-v2.1-green.svg)![Resource Discovery Support Badge](https://img.shields.io/badge/Support%20for%20Resource%20Discovery-Yes-green.svg)
+
+You can scrape an Azure Synapse workspace via the `SynapseWorkspace` resource type.
+
+The following fields need to be provided:
+
+- `workspaceName` - The name of the Azure Synapse workspace.
+
+All supported metrics are documented in the official [Azure Monitor documentation](https://docs.microsoft.com/en-us/azure/azure-monitor/platform/metrics-supported#microsoftsynapseworkspaces).
+
+The following scraper-specific metric labels will be added:
+
+- `workspace_name` - The name of the Azure Synapse workspace.
+
+Example:
+
+```yaml
+- name: promitor_demo_synapse_workspace_builtin_sql_processed_bytes
+  description: "Amount of bytes processed in Azure Synapse workspace"
+  resourceType: SynapseWorkspace
+  azureMetricConfiguration:
+    metricName: BuiltinSqlPoolDataProcessedBytes
+    aggregation:
+      type: Total
+  resources:
+  - workspaceName: promitor-synapse
+    resourceGroupName: promitor-sources
+  resourceDiscoveryGroups: # Optional, requires Promitor Resource Discovery agent (https://promitor.io/concepts/how-it-works#using-resource-discovery)
+  - name: synapse-apache-spark-pools
+```
+
+<!-- markdownlint-disable MD033 -->
+[&larr; back to metrics declarations](/configuration/v2.x/metrics)<br />
+[&larr; back to introduction](/)
+<!-- markdownlint-enable -->
diff --git a/docs/configuration/v2.x/resource-discovery.md b/docs/configuration/v2.x/resource-discovery.md
@@ -112,6 +112,9 @@ Dynamic resource discovery is supported for the following scrapers:
 - [Azure SQL Elastic Pool](metrics/sql-elastic-pool)
 - [Azure SQL Managed Instance](metrics/sql-managed-instance)
 - [Azure Storage (Account)](metrics/storage-account)
+- [Azure Synapse (Apache Spark pool)](metrics/synapse-apache-spark-pool)
+- [Azure Synapse (SQL pool)](metrics/synapse-sql-pool)
+- [Azure Synapse (Workspace)](metrics/synapse-workspace)
 - [Azure Virtual Machine](metrics/virtual-machine)
 - [Azure Virtual Machine Scale Set (VMSS)](metrics/virtual-machine-scale-set)
 - [Azure Web App](metrics/web-app)

diff --git a/src/Promitor.Agents.ResourceDiscovery/Docs/Open-Api.xml b/src/Promitor.Agents.ResourceDiscovery/Docs/Open-Api.xml
diff --git a/src/Promitor.Agents.ResourceDiscovery/Graph/ResourceDiscoveryFactory.cs b/src/Promitor.Agents.ResourceDiscovery/Graph/ResourceDiscoveryFactory.cs
@@ -60,6 +60,12 @@ public static ResourceDiscoveryQuery UseResourceDiscoveryFor(ResourceType resour
                     return new SqlManagedInstanceDiscoveryQuery();
                 case ResourceType.StorageAccount:
                     return new StorageAccountDiscoveryQuery();
+                case ResourceType.SynapseApacheSparkPool:
+                    return new SynapseApacheSparkPoolDiscoveryQuery();
+                case ResourceType.SynapseSqlPool:
+                    return new SynapseSqlPoolDiscoveryQuery();
+                case ResourceType.SynapseWorkspace:
+                    return new SynapseWorkspaceDiscoveryQuery();
                 case ResourceType.VirtualMachine:
                     return new VirtualMachineDiscoveryQuery();
                 case ResourceType.VirtualMachineScaleSet:

diff --git a/src/Promitor.Agents.ResourceDiscovery/Graph/ResourceDiscoveryQuery.cs b/src/Promitor.Agents.ResourceDiscovery/Graph/ResourceDiscoveryQuery.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using GuardNet;
 using Newtonsoft.Json.Linq;
 using Promitor.Agents.ResourceDiscovery.Configuration;
@@ -45,6 +46,23 @@ public virtual GraphQueryBuilder DefineQuery(ResourceCriteriaDefinition criteria
 
             return graphQueryBuilder;
         }
+
+        /// <summary>
+        /// Gets the name of the parent resource from a resource URI
+        /// </summary>
+        /// <param name="resourceIdentifier">Identifier to split on, ie servers/ for Azure SQL DB to get the server name</param>
+        /// <param name="resourceUri">Uri of the child resource</param>
+        /// <returns>Name of the parent resource</returns>
+        public virtual string GetParentResourceNameFromResourceUri(string resourceIdentifier, JToken resourceUri)
+        {
+            Guard.NotNull(resourceUri, nameof(resourceUri));
+            var rawResourceUri = resourceUri.ToString();
+            Guard.For<ArgumentException>(() => string.IsNullOrWhiteSpace(rawResourceUri), nameof(resourceUri));
+
+            var positionOfServersSection = rawResourceUri.LastIndexOf(resourceIdentifier, StringComparison.InvariantCultureIgnoreCase) + resourceIdentifier.Length;
+            var sqlResourceDetailsParts = rawResourceUri.Substring(positionOfServersSection).Split("/");
+            return sqlResourceDetailsParts.FirstOrDefault();
+        }
 
         public abstract string[] ResourceTypes { get; }
         public abstract string[] ProjectedFieldNames { get; }

diff --git a/src/Promitor.Agents.ResourceDiscovery/Graph/ResourceTypes/SqlDatabaseDiscoveryQuery.cs b/src/Promitor.Agents.ResourceDiscovery/Graph/ResourceTypes/SqlDatabaseDiscoveryQuery.cs
@@ -6,17 +6,17 @@
 
 namespace Promitor.Agents.ResourceDiscovery.Graph.ResourceTypes
 {
-    public class SqlDatabaseDiscoveryQuery : SqlDiscoveryQuery
+    public class SqlDatabaseDiscoveryQuery : ResourceDiscoveryQuery
     {
+        public const string ServerSectionInResourceUri = "servers/";
         public override string[] ResourceTypes => new[] { "microsoft.sql/servers/databases" };
         public override string[] ProjectedFieldNames => new[] { "subscriptionId", "resourceGroup", "type", "name", "id" };
 
         public override AzureResourceDefinition ParseResults(JToken resultRowEntry)
         {
             Guard.NotNull(resultRowEntry, nameof(resultRowEntry));
 
-            var serverName = GetServerNameFromResourceUri(resultRowEntry[4]);
-
+            var serverName = GetParentResourceNameFromResourceUri(ServerSectionInResourceUri, resultRowEntry[4]);
             if (string.IsNullOrWhiteSpace(serverName))
             {
                 throw new Exception($"Unable to determine server name from resource URI '{resultRowEntry[4]}'");

diff --git a/src/Promitor.Agents.ResourceDiscovery/Graph/ResourceTypes/SqlDiscoveryQuery.cs b/src/Promitor.Agents.ResourceDiscovery/Graph/ResourceTypes/SqlDiscoveryQuery.cs