From 798efde66810fe547ad4799695553577975dd1d0 Mon Sep 17 00:00:00 2001 From: Cornelius Riemenschneider Date: Mon, 29 Jul 2024 09:30:06 +0200 Subject: [PATCH] `crate_universe`: Enable modext isolation. This should allow us to build our python and ruby code independently - in particular, we can now do shallow checkouts of one without the other. Previously, the modext introduced cross-dependency. This also reduces the amount of work we do in the crate universe processing for the other language, even though it's unused. This does need renaming the module, as otherwise the generated paths from rules_rust get too long for Windows :( --- .bazelrc | 1 + .bazelrc.internal | 1 + MODULE.bazel | 22 ++++++++++++++++------ java/kotlin-extractor/BUILD.bazel | 12 ++++++------ ruby/extractor/BUILD.bazel | 2 +- shared/tree-sitter-extractor/BUILD.bazel | 2 +- 6 files changed, 26 insertions(+), 14 deletions(-) diff --git a/.bazelrc b/.bazelrc index e2ccb69be902e..9abda3e670a01 100644 --- a/.bazelrc +++ b/.bazelrc @@ -24,5 +24,6 @@ common --registry=file:///%workspace%/misc/bazel/registry common --registry=https://bcr.bazel.build common --@rules_dotnet//dotnet/settings:strict_deps=false +common --experimental_isolated_extension_usages try-import %workspace%/local.bazelrc diff --git a/.bazelrc.internal b/.bazelrc.internal index f7718959c9de7..dcc8dafa6c7f0 100644 --- a/.bazelrc.internal +++ b/.bazelrc.internal @@ -8,3 +8,4 @@ common --registry=https://bcr.bazel.build # its implementation packages without providing any code itself. # We either can depend on internal implementation details, or turn of strict deps. common --@rules_dotnet//dotnet/settings:strict_deps=false +common --experimental_isolated_extension_usages diff --git a/MODULE.bazel b/MODULE.bazel index 3352e004375b8..5f16293101585 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -1,5 +1,5 @@ module( - name = "codeql", + name = "ql", version = "0.0", ) @@ -30,11 +30,13 @@ bazel_dep(name = "rules_rust", version = "0.49.1") bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True) -crate = use_extension( +# crate_py but shortened due to Windows file path considerations +cp = use_extension( "@rules_rust//crate_universe:extension.bzl", "crate", + isolate = True, ) -crate.from_cargo( +cp.from_cargo( name = "py_deps", cargo_lockfile = "//python/extractor/tsg-python:Cargo.lock", manifests = [ @@ -42,15 +44,23 @@ crate.from_cargo( "//python/extractor/tsg-python/tsp:Cargo.toml", ], ) -crate.from_cargo( - name = "ruby_deps", +use_repo(cp, "py_deps") + +# crate_ruby, but shortened due to windows file paths +r = use_extension( + "@rules_rust//crate_universe:extension.bzl", + "crate", + isolate = True, +) +r.from_cargo( + name = "rd", cargo_lockfile = "//ruby/extractor:Cargo.lock", manifests = [ "//ruby/extractor:Cargo.toml", "//ruby/extractor/codeql-extractor-fake-crate:Cargo.toml", ], ) -use_repo(crate, "py_deps", "ruby_deps") +use_repo(r, "rd") dotnet = use_extension("@rules_dotnet//dotnet:extensions.bzl", "dotnet") dotnet.toolchain(dotnet_version = "8.0.101") diff --git a/java/kotlin-extractor/BUILD.bazel b/java/kotlin-extractor/BUILD.bazel index f95661f8128c0..49cf5e1664b5a 100644 --- a/java/kotlin-extractor/BUILD.bazel +++ b/java/kotlin-extractor/BUILD.bazel @@ -16,7 +16,7 @@ will build a default variant: * the version will be taken as the last supported version less than the version of the currently available `kotlinc`, or `CODEQL_KOTLIN_SINGLE_VERSION` if set. -If building from the `codeql` repository, `@codeql` can be skipped. +If building from the `codeql` repository, `@codeql` needs to be skipped. It is recommended to use the `kotlinc` wrapper in `dev` (which is also available in `tools` from `semmle-code`), which takes care about providing a sensible default version and keep the version of the default target up to date. @@ -32,7 +32,7 @@ version. Possible workarounds for that: # This file is used in the `@codeql_kotlin_embeddable` external repo, which means we need to # reference explicitly @codeql load( - "@codeql//java/kotlin-extractor:versions.bzl", + "@ql//java/kotlin-extractor:versions.bzl", "VERSIONS", "get_compatilibity_sources", "get_language_version", @@ -107,7 +107,7 @@ kt_javac_options( kt_jvm_library( name = "%s-%s" % (_extractor_name_prefix, v), srcs = - ["@codeql//java/kotlin-extractor:generated-dbscheme"] + + ["@ql//java/kotlin-extractor:generated-dbscheme"] + glob( [ "src/**/*.kt", @@ -124,7 +124,7 @@ kt_javac_options( kotlinc_opts = ":kotlinc-options-%s" % v, module_name = "codeql-kotlin-extractor", # resource_strip_prefix is very nit-picky: the following makes it work from - # `codeql`, `@codeql_kotlin_embeddable` and `semmle-code` + # `ql`, `@codeql_kotlin_embeddable` and `semmle-code` resource_strip_prefix = ( ("../%s/" % repo_name() if repo_name() else "") + ("%s/" % package_name() if package_name() else "") + @@ -145,7 +145,7 @@ kt_javac_options( # if in main repository, alias the embeddable versions from the modified @codeql_kotlin_embeddable repo alias( name = "%s-embeddable-%s" % (_common_extractor_name_prefix, v), - actual = "@codeql_kotlin_embeddable//:%s-embeddable-%s" % (_common_extractor_name_prefix, v), + actual = "@ql//:%s-embeddable-%s" % (_common_extractor_name_prefix, v), visibility = ["//visibility:public"], ) if not _for_embeddable else None, ) @@ -155,7 +155,7 @@ kt_javac_options( ( genrule( name = "generated-dbscheme", - srcs = ["@codeql//java:dbscheme"], + srcs = ["@ql//java:dbscheme"], outs = ["KotlinExtractorDbScheme.kt"], cmd = "$(execpath :generate_dbscheme) $< $@", tools = [":generate_dbscheme"], diff --git a/ruby/extractor/BUILD.bazel b/ruby/extractor/BUILD.bazel index e3c2a387af59d..bb39608f3fc13 100644 --- a/ruby/extractor/BUILD.bazel +++ b/ruby/extractor/BUILD.bazel @@ -1,4 +1,4 @@ -load("@ruby_deps//:defs.bzl", "aliases", "all_crate_deps") +load("@rd//:defs.bzl", "aliases", "all_crate_deps") load("//misc/bazel:rust.bzl", "codeql_rust_binary") codeql_rust_binary( diff --git a/shared/tree-sitter-extractor/BUILD.bazel b/shared/tree-sitter-extractor/BUILD.bazel index e57a9f47ca139..5d9c4ca1ccf84 100644 --- a/shared/tree-sitter-extractor/BUILD.bazel +++ b/shared/tree-sitter-extractor/BUILD.bazel @@ -1,4 +1,4 @@ -load("@ruby_deps//:defs.bzl", "aliases", "all_crate_deps") +load("@rd//:defs.bzl", "aliases", "all_crate_deps") load("@rules_rust//rust:defs.bzl", "rust_library") package(default_visibility = ["//visibility:public"])