Merge branch 'main' into python/stdlib-start-modeling-using-mad

.bazelversion

@@ -1 +1 @@
-7.0.2
+7.1.0

.gitattributes

@@ -67,11 +67,6 @@ go/extractor/opencsv/CSVReader.java -text
 # for those testing dbscheme files.
 */ql/lib/upgrades/initial/*.dbscheme -text
 
-# Generated test files - these are synced from the standard JavaScript libraries using
-# `javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py`.
-javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge
-javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge
-
 # Auto-generated modeling for Python
 python/ql/lib/semmle/python/frameworks/data/internal/subclass-capture/*.yml linguist-generated=true
 

.github/labeler.yml

@@ -15,7 +15,7 @@ Java:
   - change-notes/**/*java.*
 
 JS:
-  - any: [ 'javascript/**/*', '!javascript/ql/experimental/adaptivethreatmodeling/**/*' ]
+  - any: [ 'javascript/**/*' ]
   - change-notes/**/*javascript*
 
 Kotlin:
@@ -46,6 +46,3 @@ documentation:
 # Since these are all shared files that need to be synced, just pick _one_ copy of each.
 "DataFlow Library":
   - "shared/dataflow/**/*"
-
-"ATM":
-  - javascript/ql/experimental/adaptivethreatmodeling/**/*

.github/workflows/cpp-swift-analysis.yml

@@ -0,0 +1,55 @@
+name: "Code scanning - C++"
+
+on:
+  push:
+    branches:
+      - main
+      - 'rc/*'
+  pull_request:
+    branches:
+      - main
+      - 'rc/*'
+    paths:
+      - 'swift/**'
+      - '.github/codeql/**'
+      - '.github/workflows/cpp-swift-analysis.yml'
+  schedule:
+    - cron: '0 9 * * 1'
+
+jobs:
+  CodeQL-Build:
+
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      security-events: write
+      pull-requests: read
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@main
+      # Override language selection by uncommenting this and choosing your languages
+      with:
+        languages: cpp
+        config-file: ./.github/codeql/codeql-config.yml
+
+    - name: "[Ubuntu] Remove GCC 13 from runner image"
+      shell: bash
+      run: |
+        sudo rm -f /etc/apt/sources.list.d/ubuntu-toolchain-r-ubuntu-test-jammy.list
+        sudo apt-get update
+        sudo apt-get install -y --allow-downgrades libc6=2.35-* libc6-dev=2.35-* libstdc++6=12.3.0-* libgcc-s1=12.3.0-*
+
+    - name: "Build Swift extractor using Bazel"
+      run: |
+        bazel clean --expunge
+        bazel run //swift:create-extractor-pack --nouse_action_cache --noremote_accept_cached --noremote_upload_local_results --spawn_strategy=local --features=-layering_check
+        bazel shutdown
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@main

.github/workflows/swift.yml

@@ -6,6 +6,7 @@ on:
       - "swift/**"
       - "misc/bazel/**"
       - "misc/codegen/**"
+      - "shared/**"
       - "*.bazel*"
       - .github/workflows/swift.yml
       - .github/actions/**
@@ -22,10 +23,12 @@ on:
       - "swift/**"
       - "misc/bazel/**"
       - "misc/codegen/**"
+      - "shared/**"
       - "*.bazel*"
       - .github/workflows/swift.yml
       - .github/actions/**
       - codeql-workspace.yml
+      - .pre-commit-config.yaml
       - "!**/*.md"
       - "!**/*.qhelp"
     branches:

CODEOWNERS

@@ -1,4 +1,5 @@
 /cpp/ @github/codeql-c-analysis
+/cpp/autobuilder/ @github/codeql-c-extractor
 /csharp/ @github/codeql-csharp
 /go/ @github/codeql-go
 /java/ @github/codeql-java
@@ -11,9 +12,6 @@
 /java/ql/test-kotlin1/ @github/codeql-kotlin
 /java/ql/test-kotlin2/ @github/codeql-kotlin
 
-# ML-powered queries
-/javascript/ql/experimental/adaptivethreatmodeling/ @github/codeql-ml-powered-queries-reviewers
-
 # CodeQL tools and associated docs
 /docs/codeql/codeql-cli/ @github/codeql-cli-reviewers
 /docs/codeql/codeql-for-visual-studio-code/ @github/codeql-vscode-reviewers
@@ -36,9 +34,7 @@ MODULE.bazel @github/codeql-ci-reviewers
 
 # Workflows
 /.github/workflows/ @github/codeql-ci-reviewers
-/.github/workflows/atm-* @github/codeql-ml-powered-queries-reviewers
 /.github/workflows/go-* @github/codeql-go
-/.github/workflows/js-ml-tests.yml @github/codeql-ml-powered-queries-reviewers
 /.github/workflows/ql-for-ql-* @github/codeql-ql-for-ql-reviewers
 /.github/workflows/ruby-* @github/codeql-ruby
 /.github/workflows/swift.yml @github/codeql-swift

MODULE.bazel

@@ -14,7 +14,7 @@ local_path_override(
 # see https://registry.bazel.build/ for a list of available packages
 
 bazel_dep(name = "platforms", version = "0.0.8")
-bazel_dep(name = "rules_pkg", version = "0.9.1")
+bazel_dep(name = "rules_pkg", version = "0.10.1")
 bazel_dep(name = "rules_nodejs", version = "6.0.3")
 bazel_dep(name = "rules_python", version = "0.31.0")
 bazel_dep(name = "bazel_skylib", version = "1.5.0")
@@ -31,6 +31,8 @@ pip.parse(
 use_repo(pip, "codegen_deps")
 
 swift_deps = use_extension("//swift/third_party:load.bzl", "swift_deps")
+
+# following list can be kept in sync with `bazel mod tidy`
 use_repo(
     swift_deps,
     "binlog",

codeql-workspace.yml

@@ -6,28 +6,18 @@ provide:
   - "*/ql/consistency-queries/qlpack.yml"
   - "*/ql/automodel/src/qlpack.yml"
   - "*/ql/automodel/test/qlpack.yml"
+  - "python/extractor/qlpack.yml"
   - "shared/**/qlpack.yml"
   - "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml"
   - "go/ql/config/legacy-support/qlpack.yml"
   - "go/build/codeql-extractor-go/codeql-extractor.yml"
-  - "javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml"
-  # This pack is explicitly excluded from the workspace since most users
-  # will want to use a version of this pack from the package cache. Internal
-  # users can uncomment the following line and place a custom ML model
-  # in the corresponding pack to test a custom ML model within their local
-  # checkout.
-  # - "javascript/ql/experimental/adaptivethreatmodeling/model/qlpack.yml"
-  - "javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml"
-  - "javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml"
-  - "javascript/ql/experimental/adaptivethreatmodeling/test/qlpack.yml"
   - "csharp/ql/campaigns/Solorigate/lib/qlpack.yml"
   - "csharp/ql/campaigns/Solorigate/src/qlpack.yml"
   - "csharp/ql/campaigns/Solorigate/test/qlpack.yml"
   - "misc/legacy-support/*/qlpack.yml"
   - "misc/suite-helpers/qlpack.yml"
   - "ruby/extractor-pack/codeql-extractor.yml"
   - "swift/extractor-pack/codeql-extractor.yml"
-  - "swift/integration-tests/qlpack.yml"
   - "ql/extractor-pack/codeql-extractor.yml"
   - ".github/codeql/extensions/**/codeql-pack.yml"
 

config/identical-files.json

@@ -251,13 +251,6 @@
     "cpp/ql/src/Security/CWE/CWE-020/SafeExternalAPIFunction.qll",
     "cpp/ql/src/Security/CWE/CWE-020/ir/SafeExternalAPIFunction.qll"
   ],
-  "XML": [
-    "cpp/ql/lib/semmle/code/cpp/XML.qll",
-    "csharp/ql/lib/semmle/code/csharp/XML.qll",
-    "java/ql/lib/semmle/code/xml/XML.qll",
-    "javascript/ql/lib/semmle/javascript/XML.qll",
-    "python/ql/lib/semmle/python/xml/XML.qll"
-  ],
   "DuplicationProblems.inc.qhelp": [
     "cpp/ql/src/Metrics/Files/DuplicationProblems.inc.qhelp",
     "javascript/ql/src/Metrics/DuplicationProblems.inc.qhelp",
@@ -372,4 +365,4 @@
     "python/ql/test/experimental/dataflow/model-summaries/InlineTaintTest.ext.yml",
     "python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.ext.yml"
   ]
-}
+}

cpp/BUILD.bazel

@@ -1,4 +1,4 @@
-load("@rules_pkg//:mappings.bzl", "pkg_filegroup")
+load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup")
 
 package(default_visibility = ["//visibility:public"])
 

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

@@ -203,6 +203,8 @@ internal class TestDiagnosticWriter : IDiagnosticsWriter
         public IList<DiagnosticMessage> Diagnostics { get; } = new List<DiagnosticMessage>();
 
         public void AddEntry(DiagnosticMessage message) => this.Diagnostics.Add(message);
+
+        public void Dispose() { }
     }
 
     /// <summary>
@@ -250,12 +252,7 @@ void EndCallback(int ret, string s, bool silent)
             EndCallbackIn.Add(s);
         }
 
-        CppAutobuilder CreateAutoBuilder(bool isWindows,
-            string? buildless = null, string? solution = null, string? buildCommand = null, string? ignoreErrors = null,
-            string? msBuildArguments = null, string? msBuildPlatform = null, string? msBuildConfiguration = null, string? msBuildTarget = null,
-            string? dotnetArguments = null, string? dotnetVersion = null, string? vsToolsVersion = null,
-            string? nugetRestore = null, string? allSolutions = null,
-            string cwd = @"C:\Project")
+        CppAutobuilder CreateAutoBuilder(bool isWindows, string? dotnetVersion = null, string cwd = @"C:\Project")
         {
             string codeqlUpperLanguage = Language.Cpp.UpperCaseName;
             Actions.GetEnvironmentVariable[$"CODEQL_AUTOBUILDER_{codeqlUpperLanguage}_NO_INDEXING"] = "false";
@@ -265,22 +262,7 @@ CppAutobuilder CreateAutoBuilder(bool isWindows,
             Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_DIAGNOSTIC_DIR"] = "";
             Actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
             Actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = "win64";
-            Actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa";
-            Actions.GetEnvironmentVariable["SEMMLE_JAVA_HOME"] = @"C:\odasa\tools\java";
-            Actions.GetEnvironmentVariable["SEMMLE_PLATFORM_TOOLS"] = @"C:\odasa\tools";
-            Actions.GetEnvironmentVariable["LGTM_INDEX_VSTOOLS_VERSION"] = vsToolsVersion;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_ARGUMENTS"] = msBuildArguments;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_PLATFORM"] = msBuildPlatform;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_CONFIGURATION"] = msBuildConfiguration;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_TARGET"] = msBuildTarget;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_ARGUMENTS"] = dotnetArguments;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_VERSION"] = dotnetVersion;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_BUILD_COMMAND"] = buildCommand;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_SOLUTION"] = solution;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_IGNORE_ERRORS"] = ignoreErrors;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_BUILDLESS"] = buildless;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_ALL_SOLUTIONS"] = allSolutions;
-            Actions.GetEnvironmentVariable["LGTM_INDEX_NUGET_RESTORE"] = nugetRestore;
+            Actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_OPTION_DOTNET_VERSION"] = dotnetVersion;
             Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = isWindows ? @"C:\Program Files (x86)" : null;
             Actions.GetCurrentDirectory = cwd;
             Actions.IsWindows = isWindows;

cpp/autobuilder/Semmle.Autobuild.Cpp/CppAutobuilder.cs

@@ -26,9 +26,6 @@ public class CppAutobuilder : Autobuilder<CppAutobuildOptions>
 
         public override BuildScript GetBuildScript()
         {
-            if (Options.BuildCommand != null)
-                return new BuildCommandRule((_, f) => f(null)).Analyse(this, false);
-
             return
                 // First try MSBuild
                 new MsBuildRule().Analyse(this, true) |

cpp/autobuilder/Semmle.Autobuild.Cpp/Program.cs

@@ -17,7 +17,7 @@ static int Main()
                 try
                 {
                     Console.WriteLine("CodeQL C++ autobuilder");
-                    var builder = new CppAutobuilder(actions, options);
+                    using var builder = new CppAutobuilder(actions, options);
                     return builder.AttemptBuild();
                 }
                 catch (InvalidEnvironmentException ex)

cpp/downgrades/BUILD.bazel

@@ -1,4 +1,4 @@
-load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix")
+load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix")
 
 pkg_files(
     name = "downgrades",

cpp/downgrades/aa7ff0ab32cd4674f6ab731d32fea64116997b05/exprs.ql

@@ -0,0 +1,13 @@
+class Expr extends @expr {
+  string toString() { none() }
+}
+
+class Location extends @location_expr {
+  string toString() { none() }
+}
+
+from Expr expr, int kind, int kind_new, Location loc
+where
+  exprs(expr, kind, loc) and
+  if kind = 363 then kind_new = 1 else kind_new = kind
+select expr, kind_new, loc