Merge remote-tracking branch 'origin/develop'

docs/SelfDiagnostics.MD

@@ -2,13 +2,24 @@
 
 ## Enable self-diagnostics
 
-Enabling self-diagnostics helps troubleshoot issues and determine whether the problem is with service code/configuration, or with the `ApplicationInsights.Kubernetes` package itself.
+Enabling self-diagnostics helps troubleshoot issues and determine whether the problem is with service code/configuration, or with the `ApplicationInsights.Kubernetes` package.
 
-If problem with ApplicationInsights.Kubernetes package is suspected, please open an [issues](https://github.com/microsoft/ApplicationInsights-Kubernetes/issues) and we will look into it.
+If problem with `ApplicationInsights.Kubernetes` is suspected, please open an [issue](https://github.com/microsoft/ApplicationInsights-Kubernetes/issues) and we will look into it.
 
-Application Insights for Kubernetes is instrumented with DiagnosticsSource. To see the logs, create a diagnostic source observer that subscribe to `Microsoft.ApplicationInsights.Kubernetes.Debugging.ApplicationInsightsKubernetesDiagnosticSource.Observable`. The observable(DiagnosticSource) will emits logs in different levels.
+## With 2.0.4-beta1 or above
 
-To make diagnostic easier, a default observer that outputs logs into the console is provided and and could be enabled like it below before `AddApplicationInsightsKubernetesEnricher` is called:
+```csharp
+// LogLevel.Error is the default; Setting it to LogLevel.Trace to see detailed logs.
+builder.Services.AddApplicationInsightsKubernetesEnricher(LogLevel.Trace);
+```
+
+Refer to [Program.cs](https://github.com/microsoft/ApplicationInsights-Kubernetes/blob/c8a906080f368570078d873b61f58dd742277916/dev/F5WebApi/Program.cs#L10) for a complete example.
+
+
+## With versions before 2.0.4-beta1
+
+<details>
+<summary>Click to expand!</summary>
 
 ```csharp
 using Microsoft.ApplicationInsights.Kubernetes.Debugging;
@@ -17,6 +28,11 @@ var observer = new ApplicationInsightsKubernetesDiagnosticObserver(DiagnosticLog
 ApplicationInsightsKubernetesDiagnosticSource.Instance.Observable.SubscribeWithAdapter(observer);
 ```
 
+Refer to [Startup.cs](https://github.com/microsoft/ApplicationInsights-Kubernetes/blob/36e31d39e4ef867fadbb3a4191f82565af3cb5b0/dev/F5WebApi/Startup.cs#L26) for a complete example.
+</details>
+
+## Log example
+
 When the observer above is enabled, you will see diagnostic logs like this in the console:
 
 ```shell
@@ -40,22 +56,18 @@ A similar log will be output to the terminal.
 
 ## Logging Levels
 
-There are 5 levels of the events, each is assigned with a number like:
+There are 6 levels of the events, each is assigned with a number like:
 
+* None - 6
 * Critical - 5
 * Error - 4
 * Warning - 3
 * Information - 2
 * Debug - 1
 * Trace - 0
 
-When minimum level is set at the creation of the observer, issues that has a number greater or equal to it will be printed. For example, if the minimum level is set to `Error`(4) like this:
-
-```csharp
-var observer = new ApplicationInsightsKubernetesDiagnosticObserver(DiagnosticLogLevel.Error); // Set the minimum level to Error 4.
-```
-
-Logs with the level of critical or error will be printed to the console.
+When minimum level is set at the creation of the observer, issues that has a number greater or equal to it will be printed. For example, if the minimum level is set to `Error`(4), logs with the level of critical or error will be printed to the console.
+Set LogLevel to None to complete turn off the logs.
 
 ## About the Diagnostic Source
 

src/ApplicationInsights.Kubernetes/ContainerIdProviders/CGroupContainerIdProvider.cs

@@ -1,53 +1,21 @@
 #nullable enable
 
 using Microsoft.ApplicationInsights.Kubernetes.Debugging;
-using System;
-using System.IO;
-using System.Text.RegularExpressions;
 
 namespace Microsoft.ApplicationInsights.Kubernetes.ContainerIdProviders
 {
     /// <summary>
     /// Gets the current container id by using CGroup
     /// </summary>
-    internal class CGroupContainerIdProvider : IContainerIdProvider
+    internal class CGroupContainerIdProvider : FileBasedContainerIdProvider
     {
         private const string CGroupPath = "/proc/self/cgroup";
-        private const string CGroupPathPatternString = "cpu.+/([^/]*)$";
-        private static readonly Regex CGroupPathPattern = new Regex(CGroupPathPatternString, RegexOptions.CultureInvariant | RegexOptions.Multiline, TimeSpan.FromSeconds(1));
-
         private readonly ApplicationInsightsKubernetesDiagnosticSource _logger = ApplicationInsightsKubernetesDiagnosticSource.Instance;
 
-        public bool TryGetMyContainerId(out string? containerId)
-        {
-            containerId = FetchContainerId(CGroupPath);
-            return containerId != null;
-        }
-
-        private string? FetchContainerId(string pathToCGroup)
-        {
-            if (!File.Exists(pathToCGroup))
-            {
-                _logger.LogWarning("CGroup file doesn't exist. Path: {0}", pathToCGroup);
-                return null;
-            }
-
-            string content = File.ReadAllText(pathToCGroup);
-            return ParseContainerId(content);
-        }
-
-        internal string? ParseContainerId(string? content)
+        public CGroupContainerIdProvider(
+            CGroupV1Matcher lineMatcher) : 
+            base(lineMatcher, CGroupPath, providerName: default)
         {
-            if (!string.IsNullOrEmpty(content))
-            {
-                MatchCollection matches = CGroupPathPattern.Matches(content);
-                if (matches.Count >= 1 && matches[0].Groups.Count >= 2)
-                {
-                    return matches[0].Groups[1].Value;
-                }
-            }
-            _logger.LogWarning("Can't figure out container id. Input: {0}. Pattern: {1}", content, CGroupPathPatternString);
-            return null;
         }
     }
 }

src/ApplicationInsights.Kubernetes/ContainerIdProviders/ContainerDMountInfoContainerIdProvider.cs

@@ -0,0 +1,14 @@
+#nullable enable
+
+namespace Microsoft.ApplicationInsights.Kubernetes.ContainerIdProviders;
+
+internal class ContainerDMountInfoContainerIdProvider : FileBasedContainerIdProvider
+{
+    private const string InfoFilePath = "/proc/self/mountinfo";
+
+    public ContainerDMountInfoContainerIdProvider(
+        ContainerDMountInfoMatcher matcher)
+            : base(matcher, InfoFilePath, providerName: default)
+    {
+    }
+}

src/ApplicationInsights.Kubernetes/ContainerIdProviders/DockerEngineMountInfoContainerIdProvider.cs

@@ -0,0 +1,14 @@
+#nullable enable
+
+namespace Microsoft.ApplicationInsights.Kubernetes.ContainerIdProviders;
+
+internal class DockerEngineMountInfoContainerIdProvider : FileBasedContainerIdProvider
+{
+    private const string InfoFilePath = "/proc/self/mountinfo";
+
+    public DockerEngineMountInfoContainerIdProvider(
+        DockerEngineMountInfoMatcher matcher)
+            : base(matcher, InfoFilePath, providerName: default)
+    {
+    }
+}

src/ApplicationInsights.Kubernetes/ContainerIdProviders/EnvironmentVariableContainerIdProvider.cs

@@ -0,0 +1,24 @@
+#nullable enable
+
+using System;
+using Microsoft.ApplicationInsights.Kubernetes.Debugging;
+
+namespace Microsoft.ApplicationInsights.Kubernetes.ContainerIdProviders
+{
+    internal class EnvironmentVariableContainerIdProvider : IContainerIdProvider
+    {
+        private readonly ApplicationInsightsKubernetesDiagnosticSource _logger = ApplicationInsightsKubernetesDiagnosticSource.Instance;
+
+        private const string EnvironmentVariableName = "ContainerId";
+
+        /// <summary>
+        /// Try get the container id by environment variable named "ContainerId".
+        /// </summary>
+        public bool TryGetMyContainerId(out string? containerId)
+        {
+            containerId = Environment.GetEnvironmentVariable(EnvironmentVariableName);
+            _logger.LogWarning($"Getting container id by environment variable {EnvironmentVariableName}. Result: {containerId}.");
+            return !string.IsNullOrEmpty(containerId);
+        }
+    }
+}

src/ApplicationInsights.Kubernetes/ContainerIdProviders/FileBasedContainerIdProvider.cs

@@ -0,0 +1,67 @@
+#nullable enable
+
+using System;
+using System.IO;
+using Microsoft.ApplicationInsights.Kubernetes.Debugging;
+
+namespace Microsoft.ApplicationInsights.Kubernetes.ContainerIdProviders;
+
+/// <summary>
+/// A common framework to get container id from a file, providing consistent implementation as well as logging.
+/// </summary>
+internal abstract class FileBasedContainerIdProvider : IContainerIdProvider
+{
+    private readonly ApplicationInsightsKubernetesDiagnosticSource _logger = ApplicationInsightsKubernetesDiagnosticSource.Instance;
+    private readonly IContainerIdMatcher _lineMatcher;
+    private readonly string _providerName;
+    private readonly string _targetFile;
+
+    public FileBasedContainerIdProvider(
+        IContainerIdMatcher lineMatcher,
+        string filePath, 
+        string? providerName)
+    {
+        _providerName = GetType().Name;
+        if (string.IsNullOrEmpty(filePath))
+        {
+            throw new ArgumentException($"'[{_providerName}] {nameof(filePath)}' cannot be null or empty.", nameof(filePath));
+        }
+        _targetFile = filePath;
+
+        _lineMatcher = lineMatcher ?? throw new System.ArgumentNullException(nameof(lineMatcher));
+    }
+
+    public bool TryGetMyContainerId(out string? containerId)
+    {
+        containerId = FetchContainerId(_targetFile);
+        return containerId != null;
+    }
+
+    private string? FetchContainerId(string filePath)
+    {
+        if (!File.Exists(filePath))
+        {
+            _logger.LogWarning($"[{_providerName}] {nameof(_targetFile)} doesn't exist at: {filePath}");
+            return null;
+        }
+
+        using StreamReader reader = File.OpenText(_targetFile);
+        while (!reader.EndOfStream)
+        {
+            string line = reader.ReadLine();
+            if(string.IsNullOrEmpty(line))
+            {
+                continue;
+            }
+
+            if (_lineMatcher.TryParseContainerId(line, out string containerId))
+            {
+                _logger.LogDebug($"[{_providerName}] Got container id by: {line}");
+                _logger.LogInformation($"[{_providerName}] Got container id: {containerId}");
+                return containerId;
+            }
+        }
+        _logger.LogWarning($"[{_providerName}] Can't figure out container id.");
+        return null;
+    }
+}

src/ApplicationInsights.Kubernetes/ContainerIdProviders/Matchers/CGroupV1Matcher.cs

@@ -0,0 +1,35 @@
+#nullable enable
+
+using System;
+using System.Text.RegularExpressions;
+using Microsoft.ApplicationInsights.Kubernetes.Debugging;
+
+namespace Microsoft.ApplicationInsights.Kubernetes.ContainerIdProviders;
+
+internal class CGroupV1Matcher : IContainerIdMatcher
+{
+    private const string LogCategory = nameof(CGroupV1Matcher);
+    private readonly ApplicationInsightsKubernetesDiagnosticSource _logger = ApplicationInsightsKubernetesDiagnosticSource.Instance;
+
+    private const string MatchPattern = @"cpu.+/([^/]*)$";
+    private static readonly Regex MatchRegex = new Regex(MatchPattern, RegexOptions.CultureInvariant | RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1));
+
+    public bool TryParseContainerId(string? line, out string containerId)
+    {
+        containerId = string.Empty;
+        if (string.IsNullOrEmpty(line))
+        {
+            return false;
+        }
+
+        Match match = MatchRegex.Match(line);
+        if (!match.Success)
+        {
+            _logger.LogDebug($"[{LogCategory}] No match for containerId. Input: {line}, pattern: {MatchPattern}");
+            return false;
+        }
+        _logger.LogTrace($"[{LogCategory}] Matched container id.");
+        containerId = match.Groups[1].Value;
+        return true;
+    }
+}

src/ApplicationInsights.Kubernetes/ContainerIdProviders/Matchers/ContainerDMountInfoMatcher.cs

@@ -0,0 +1,44 @@
+#nullable enable
+
+using System;
+using System.Text.RegularExpressions;
+using Microsoft.ApplicationInsights.Kubernetes.Debugging;
+
+namespace Microsoft.ApplicationInsights.Kubernetes.ContainerIdProviders;
+
+/// <summary>
+/// Ths is a heuristic matcher for container id in containers by containerD using the mount info.
+/// More info about MountInfo: https://man7.org/linux/man-pages/man5/proc.5.html
+/// </summary>
+internal class ContainerDMountInfoMatcher : IContainerIdMatcher
+{
+    private readonly ApplicationInsightsKubernetesDiagnosticSource _logger = ApplicationInsightsKubernetesDiagnosticSource.Instance;
+
+    // An example of container id line:
+    // 1735 1729 0:37 /kubepods/besteffort/pod3272f253-be44-4a82-a541-9083e68cf99f/a22b3a93bd510bf062765ec5df6608fa6cae186a476b0407bfb5369ff99afdd2 /sys/fs/cgroup/hugetlb ro,nosuid,nodev,noexec,relatime master:19 - cgroup cgroup rw,hugetlb
+    // See unit tests for more examples.
+    // This is heuristic, the mount path is not always guaranteed. File issue at https://github.com/microsoft/ApplicationInsights-Kubernetes/issues if/when find it changed.
+    private const string MatchPattern = @"/kubepods/.*?/.*?/(.*?)[\s|/]";
+    private static readonly Regex MatchRegex = new Regex(MatchPattern, RegexOptions.CultureInvariant | RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1));
+
+    private const string LogCategory = nameof(ContainerDMountInfoMatcher);
+
+    public bool TryParseContainerId(string? line, out string containerId)
+    {
+        containerId = string.Empty;
+        if (string.IsNullOrEmpty(line))
+        {
+            return false;
+        }
+
+        Match match = MatchRegex.Match(line);
+        if (!match.Success)
+        {
+            _logger.LogDebug($"[{LogCategory}] No match for containerId. Input: {line}, pattern: {MatchPattern}");
+            return false;
+        }
+        _logger.LogTrace($"[{LogCategory}] Matched container id.");
+        containerId = match.Groups[1].Value;
+        return true;
+    }
+}

src/ApplicationInsights.Kubernetes/ContainerIdProviders/Matchers/DockerEngineMountInfoMatcher.cs

@@ -0,0 +1,35 @@
+#nullable enable
+
+using System;
+using System.Text.RegularExpressions;
+using Microsoft.ApplicationInsights.Kubernetes.Debugging;
+
+namespace Microsoft.ApplicationInsights.Kubernetes.ContainerIdProviders;
+
+internal class DockerEngineMountInfoMatcher : IContainerIdMatcher
+{
+    private const string LogCategory = nameof(DockerEngineMountInfoMatcher);
+    private readonly ApplicationInsightsKubernetesDiagnosticSource _logger = ApplicationInsightsKubernetesDiagnosticSource.Instance;
+
+    private const string MatchPattern = @"/docker/containers/(.*?)/";
+    private static readonly Regex MatchRegex = new Regex(MatchPattern, RegexOptions.CultureInvariant | RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1));
+
+    public bool TryParseContainerId(string? line, out string containerId)
+    {
+        containerId = string.Empty;
+        if (string.IsNullOrEmpty(line))
+        {
+            return false;
+        }
+
+        Match match = MatchRegex.Match(line);
+        if (!match.Success)
+        {
+            _logger.LogDebug($"[{LogCategory}] No match for containerId. Input: {line}, pattern: {MatchPattern}");
+            return false;
+        }
+        _logger.LogTrace($"[{LogCategory}] Matched container id.");
+        containerId = match.Groups[1].Value;
+        return true;
+    }
+}

src/ApplicationInsights.Kubernetes/ContainerIdProviders/Matchers/IContainerIdMatcher.cs

@@ -0,0 +1,17 @@
+#nullable enable
+
+namespace Microsoft.ApplicationInsights.Kubernetes.ContainerIdProviders;
+
+/// <summary>
+/// Matches container id.
+/// </summary>
+public interface IContainerIdMatcher
+{
+    /// <summary>
+    /// Matches the container id.
+    /// </summary>
+    /// <param name="value">The value to match.</param>
+    /// <param name="containerId">The container id when matched.</param>
+    /// <returns>Returns true when matched. Otherwise, false.</returns>
+    bool TryParseContainerId(string? value, out string containerId);
+}