Merge branch 'main' into 1454-pipeline-does-not-fail-on-pester-fail

.github/workflows/convert_baselines.yaml

@@ -0,0 +1,98 @@
+# Purpose: Converts the baseline Markdown files into HTML and text so that they can be ingested into Drupal.
+
+name: Convert Baseline to Text and HTML
+
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - ".github/workflows/convert_baselines.yaml"
+
+permissions: read-all
+
+jobs:
+  convert:
+    name: Convert
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+      - name: Convert AAD to Text
+        uses: docker://pandoc/core:3.5
+        with:
+          args: >-
+            --standalone
+            -f markdown
+            -t plain
+            PowerShell/ScubaGear/baselines/aad.md
+            --output=baselines/aad.txt
+      - name: Convert Defender to Text
+        uses: docker://pandoc/core:3.5
+        with:
+          args: >-
+            --standalone
+            -f markdown
+            -t plain
+            PowerShell/ScubaGear/baselines/defender.md
+            --output=baselines/defender.txt
+      - name: Convert Exchange to Text
+        uses: docker://pandoc/core:3.5
+        with:
+          args: >-
+            --standalone
+            -f markdown
+            -t plain
+            PowerShell/ScubaGear/baselines/exo.md
+            --output=baselines/exo.txt
+      - name: Convert Power BI to Text
+        uses: docker://pandoc/core:3.5
+        with:
+          args: >-
+            --standalone
+            -f markdown
+            -t plain
+            PowerShell/ScubaGear/baselines/powerbi.md
+            --output=baselines/powerbi.txt
+      - name: Convert Power Platform to Text
+        uses: docker://pandoc/core:3.5
+        with:
+          args: >-
+            --standalone
+            -f markdown
+            -t plain
+            PowerShell/ScubaGear/baselines/powerplatform.md
+            --output=baselines/powerflatform.txt
+      - name: Convert Sharepoint to Text
+        uses: docker://pandoc/core:3.5
+        with:
+          args: >-
+            --standalone
+            -f markdown
+            -t plain
+            PowerShell/ScubaGear/baselines/sharepoint.md
+            --output=baselines/sharepoint.txt
+      - name: Convert Teams to Text
+        uses: docker://pandoc/core:3.5
+        with:
+          args: >-
+            --standalone
+            -f markdown
+            -t plain
+            PowerShell/ScubaGear/baselines/teams.md
+            --output=baselines/teams.txt
+      - name: Install HTML converter
+        run: npm i markdown-to-html-cli -g
+      - name: Convert to HTML
+        run: |
+          markdown-to-html --source PowerShell/ScubaGear/baselines/aad.md --output baselines/aad.html
+          markdown-to-html --source PowerShell/ScubaGear/baselines/defender.md --output baselines/defender.html
+          markdown-to-html --source PowerShell/ScubaGear/baselines/exo.md --output baselines/exo.html
+          markdown-to-html --source PowerShell/ScubaGear/baselines/powerbi.md --output baselines/powerbi.html
+          markdown-to-html --source PowerShell/ScubaGear/baselines/powerplatform.md --output baselines/powerplatform.html
+          markdown-to-html --source PowerShell/ScubaGear/baselines/sharepoint.md --output baselines/sharepoint.html
+          markdown-to-html --source PowerShell/ScubaGear/baselines/teams.md --output baselines/teams.html
+      - name: Upload
+        uses: actions/upload-artifact@v4
+        with:
+          name: output
+          path: baselines

PowerShell/ScubaGear/CheckVersion.ps1

@@ -0,0 +1,45 @@
+function Invoke-CheckScubaGearVersionPSGallery {
+
+    # Retrieve the installed version of ScubaGear from the system
+    $InstalledModule = Get-Module -Name ScubaGear -ListAvailable -ErrorAction 'Stop'
+    if ($InstalledModule) {
+        $CurrentVersion = [System.Version]$InstalledModule.Version
+    } else {
+        # If we are here, ScubaGear is not installed from PSGallery.
+        # Or it may have been installed a different way in a nonstandard folder,
+        # or is running in an extracted release folder. Check github instead.
+        return Invoke-CheckScubaGearVersionGithub -ErrorAction 'Stop'
+    }
+
+    # Retrieve the latest version from PowerShell Gallery
+    $ModuleInfo = Find-Module -Name ScubaGear -ErrorAction 'Stop'
+    $LatestVersion = [System.Version]$ModuleInfo.Version
+
+    if ($CurrentVersion -lt $LatestVersion) {
+        Write-Warning "A new version of ScubaGear ($LatestVersion) is available on PowerShell Gallery. This notification can be disabled by setting `$env:SCUBAGEAR_SKIP_VERSION_CHECK = `$true before running ScubaGear."
+
+    }
+
+    # Store the current time in the file to mark the last check time
+    (Get-Date -ErrorAction 'Stop').ToString() | Set-Content $VersionCheckFile -ErrorAction 'Stop'
+}
+
+
+function Invoke-CheckScubaGearVersionGithub {
+    $ScubaManifest = Import-PowerShellDataFile (Join-Path -Path $PSScriptRoot -ChildPath 'ScubaGear.psd1' -Resolve  -ErrorAction 'Stop' ) -ErrorAction 'Stop'
+    $CurrentVersion = $ScubaManifest.ModuleVersion
+    $LatestVersion = $(Invoke-RestMethod -Uri "https://api.github.com/repos/cisagov/ScubaGear/releases/latest" -ErrorAction 'Stop').tag_name.TrimStart("v")
+    if ($CurrentVersion -ne $LatestVersion) {
+        Write-Warning "A new version of ScubaGear ($latestVersion) is available. Please consider updating at: https://github.com/cisagov/ScubaGear/releases. This notification can be disabled by setting `$env:SCUBAGEAR_SKIP_VERSION_CHECK = `$true before running ScubaGear."
+    }
+}
+
+# Do the version check if the skip envvar is not defined.
+if ([string]::IsNullOrWhiteSpace($env:SCUBAGEAR_SKIP_VERSION_CHECK)) {
+    try {
+        Invoke-CheckScubaGearVersionPSGallery -ErrorAction 'Stop'
+    }
+    catch {
+        Write-Warning "The ScubaGear version check failed to execute. This notification can be disabled by setting `$env:SCUBAGEAR_SKIP_VERSION_CHECK = `$true.`n$($_.Exception.Message)`n$($_.ScriptStackTrace)"
+    }
+}

PowerShell/ScubaGear/Modules/Connection/Connection.psm1

@@ -222,7 +222,7 @@ function Connect-Tenant {
            }
        }
        catch {
-           Write-Error "Error establishing a connection with $($Product). $($_)"
+           Write-Warning "Error establishing a connection with $($Product): $($_.Exception.Message)`n$($_.ScriptStackTrace)"
            $ProdAuthFailed += $Product
            Write-Warning "$($Product) will be omitted from the output because of failed authentication"
        }
@@ -299,7 +299,7 @@ function Disconnect-SCuBATenant {
        # Suppress error due to disconnect from service with no active connection
        continue
    } catch {
-       Write-Error "ERRROR: Could not disconnect from $Product`n$($Error[0]): "
+       Write-Warning "Could not disconnect from $Product`n: $($_.Exception.Message)`n$($_.ScriptStackTrace)"
    } finally {
        $ErrorActionPreference = "Continue"
    }

PowerShell/ScubaGear/Modules/Orchestrator.psm1

@@ -638,7 +638,7 @@ function Invoke-ProviderList {
                     $ProviderJSON += $RetVal
                 }
                 catch {
-                    Write-Error "Error with the $($BaselineName) Provider. See the exception message for more details:  $($_)"
+                    Write-Warning "Error with the $($BaselineName) Provider: $($_.Exception.Message)`n$($_.ScriptStackTrace)"
                     $ProdProviderFailed += $Product
                     Write-Warning "$($Product) will be omitted from the output because of the failure above `n`n"
                 }
@@ -686,7 +686,8 @@ function Invoke-ProviderList {
         }
         catch {
             $InvokeProviderListErrorMessage = "Fatal Error involving the Provider functions. `
-            Ending ScubaGear execution. See the exception message for more details: $($_)`n$($_.ScriptStackTrace)"
+            Ending ScubaGear execution. Error: $($_.Exception.Message)`
+            `n$($_.ScriptStackTrace)"
             throw $InvokeProviderListErrorMessage
         }
     }
@@ -768,7 +769,7 @@ function Invoke-RunRego {
                     $TestResults += $RetVal
                 }
                 catch {
-                    Write-Error "Error with the $($BaselineName) Rego invocation. See the exception message for more details:  $($_)"
+                    Write-Warning "Error with the $($BaselineName) Rego invocation: $($_.Exception.Message)`n$($_.ScriptStackTrace)"
                     $ProdRegoFailed += $Product
                     Write-Warning "$($Product) will be omitted from the output because of the failure above"
                 }
@@ -782,7 +783,8 @@ function Invoke-RunRego {
         }
         catch {
             $InvokeRegoErrorMessage = "Fatal Error involving the OPA output function. `
-            Ending ScubaGear execution. See the exception message for more details: $($_)"
+            Ending ScubaGear execution. Error: $($_.Exception.Message)`
+            `n$($_.ScriptStackTrace)"
             throw $InvokeRegoErrorMessage
         }
     }
@@ -850,6 +852,7 @@ function Format-PlainText {
         $CleanString = $CleanString.Replace("<br/>", " ")
         $CleanString = $CleanString.Replace("<b>", "")
         $CleanString = $CleanString.Replace("</b>", "")
+
         # Strip out HTML comments
         $CleanString = $CleanString -replace '(.*)(<!--)(.*)(-->)(.*)', '$1$5'
         # The following regex looks for a string with an anchor tag. If it finds an anchor tag, it reformats
@@ -866,7 +869,6 @@ function Format-PlainText {
         # Group 6: '(</a>)' Matches the closing anchor tag
         # Group 7: '(.*)' Matches any number of characters after the closing anchor tag
         $CleanString = $CleanString -replace '(.*)(<a href=")([\w#./=&?%\-+:;$@,]+)(".*>)(.*)(</a>)(.*)', '$1$5, $3$7'
-
         $CleanString
     }
 }
@@ -1012,9 +1014,7 @@ function ConvertTo-ResultsCsv {
             }
         }
         catch {
-            $Warning = "Error involving the creation of CSV version of output. "
-            $Warning += "See the exception message for more details: $($_)"
-            Write-Warning $Warning
+            Write-Warning "Error creating CSV output file: $($_.Exception.Message)`n$($_.ScriptStackTrace)"
         }
     }
 }
@@ -1115,10 +1115,17 @@ function Merge-JsonOutput {
 
                 # The date is listed under the metadata, no need to include it in the summary as well
                 $IndividualResults.ReportSummary.PSObject.Properties.Remove('Date')
-
                 $Summary | Add-Member -NotePropertyName $BaselineName `
                     -NotePropertyValue $IndividualResults.ReportSummary
             }
+            foreach ($Product in $Results.PSObject.Properties) {
+                foreach ($Group in $Product.Value) {
+                    foreach ($Control in $Group.Controls) {
+                        $Control.Requirement = Format-PlainText -RawString $Control.Requirement
+                        $Control.Details = Format-PlainText -RawString $Control.Details
+                    }
+                }
+            }
 
             # Convert the output a json string
             $MetaData = ConvertTo-Json $MetaData -Depth 3
@@ -1153,12 +1160,15 @@ function Merge-JsonOutput {
                 $PathLengthErrorMessage = "ScubaGear was likely executed in a location where the maximum file path length is greater than the allowable Windows file system limit `
                 Please execute ScubaGear in a directory where for Windows file path limit is less than $($MAX_WINDOWS_PATH_LEN).`
                 Another option is to change the -NumberOfUUIDCharactersToTruncate, -OutJSONFileName, or -OutFolderName parameters to achieve an acceptable file path length `
-                See the Invoke-SCuBA parameters documentation for more details. $($_)"
+                See the Invoke-SCuBA parameters documentation for more details. `
+                Error: $($_.Exception.Message) `
+                Stacktrace: $($_.ScriptStackTrace)"
                 throw $PathLengthErrorMessage
             }
             else {
                 $MergeJsonErrorMessage = "Fatal Error involving the Json reports aggregation. `
-                Ending ScubaGear execution. See the exception message for more details: $($_)"
+                Ending ScubaGear execution. Error: $($_.Exception.Message) `
+                Stacktrace: $($_.ScriptStackTrace)"
                 throw $MergeJsonErrorMessage
             }
         }
@@ -1356,7 +1366,8 @@ function Invoke-ReportCreation {
         }
         catch {
             $InvokeReportErrorMessage = "Fatal Error involving the Report Creation. `
-            Ending ScubaGear execution. See the exception message for more details: $($_)"
+            Ending ScubaGear execution. Error: $($_.Exception.Message)`
+            `n$($_.ScriptStackTrace)"
             throw $InvokeReportErrorMessage
         }
     }
@@ -1572,7 +1583,8 @@ function Import-Resources {
     }
     catch {
         $ImportResourcesErrorMessage = "Fatal Error involving importing PowerShell modules. `
-            Ending ScubaGear execution. See the exception message for more details: $($_)"
+            Ending ScubaGear execution. Error: $($_.Exception.Message) `
+            `n$($_.ScriptStackTrace)"
             throw $ImportResourcesErrorMessage
     }
 }