From 0da4130bb9c3a354e20042b730bea8ce7f279394 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 3 Apr 2024 06:42:44 +0200 Subject: [PATCH 01/81] Bazel: add LFS rules --- java/kotlin-extractor/BUILD.bazel | 0 misc/bazel/internal/BUILD.bazel | 0 misc/bazel/internal/git_lfs_smudge.py | 16 ++++++ misc/bazel/lfs.bzl | 73 +++++++++++++++++++++++++++ 4 files changed, 89 insertions(+) create mode 100644 java/kotlin-extractor/BUILD.bazel create mode 100644 misc/bazel/internal/BUILD.bazel create mode 100755 misc/bazel/internal/git_lfs_smudge.py create mode 100644 misc/bazel/lfs.bzl diff --git a/java/kotlin-extractor/BUILD.bazel b/java/kotlin-extractor/BUILD.bazel new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/misc/bazel/internal/BUILD.bazel b/misc/bazel/internal/BUILD.bazel new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/misc/bazel/internal/git_lfs_smudge.py b/misc/bazel/internal/git_lfs_smudge.py new file mode 100755 index 000000000000..397975473c35 --- /dev/null +++ b/misc/bazel/internal/git_lfs_smudge.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 + +import sys +import pathlib +import subprocess +import os + +sources = [pathlib.Path(arg).resolve() for arg in sys.argv[1:]] +source_dir = pathlib.Path(os.path.commonpath(src.parent for src in sources)) + +for src in sources: + with open(src, 'rb') as input: + lfs_pointer = subprocess.run(["git", "lfs", "clean", "--", src], + stdin=input, stdout=subprocess.PIPE, check=True, cwd=source_dir).stdout + with open(src.name, 'wb') as output: + subprocess.run(["git", "lfs", "smudge", "--", src], input=lfs_pointer, stdout=output, check=True, cwd=source_dir) diff --git a/misc/bazel/lfs.bzl b/misc/bazel/lfs.bzl new file mode 100644 index 000000000000..a5559d55a744 --- /dev/null +++ b/misc/bazel/lfs.bzl @@ -0,0 +1,73 @@ +def _smudge(repository_ctx, srcs): + for src in srcs: + repository_ctx.watch(src) + script = Label("//misc/bazel/internal:git_lfs_smudge.py") + python = repository_ctx.which("python3") or repository_ctx.which("python") + if not python: + fail("Neither python3 nor python executables found") + res = repository_ctx.execute([python, script] + srcs, quiet = False) + if res.return_code != 0: + fail("git LFS smudging failed while instantiating @%s" % repository_ctx.name) + +def _download_and_extract_lfs(repository_ctx): + attr = repository_ctx.attr + src = repository_ctx.path(attr.src) + if attr.build_file_content and attr.build_file: + fail("You should specify only one among build_file_content and build_file for rule @%s" % repository_ctx.name) + _smudge(repository_ctx, [src]) + repository_ctx.extract(src.basename, stripPrefix = attr.strip_prefix) + repository_ctx.delete(src.basename) + if attr.build_file_content: + repository_ctx.file("BUILD.bazel", attr.build_file_content) + elif attr.build_file: + repository_ctx.symlink(attr.build_file, "BUILD.bazel") + +def _download_lfs(repository_ctx): + attr = repository_ctx.attr + if int(bool(attr.srcs)) + int(bool(attr.dir)) != 1: + fail("Exactly one between `srcs` and `dir` must be defined for @%s" % repository_ctx.name) + if attr.srcs: + srcs = [repository_ctx.path(src) for src in attr.srcs] + else: + dir = repository_ctx.path(attr.dir) + if not dir.is_dir: + fail("`dir` not a directory in @%s" % repository_ctx.name) + srcs = [f for f in dir.readdir() if not f.is_dir] + _smudge(repository_ctx, srcs) + + # with bzlmod the name is qualified with `~` separators, and we want the base name here + name = repository_ctx.name.split("~")[-1] + repository_ctx.file("BUILD.bazel", """ +exports_files({files}) + +filegroup( + name = "{name}", + srcs = {files}, + visibility = ["//visibility:public"], +) +""".format(name = name, files = repr([src.basename for src in srcs]))) + +lfs_archive = repository_rule( + doc = "Export the contents from an on-demand LFS archive. The corresponding path should be added to be ignored " + + "in `.lfsconfig`.", + implementation = _download_and_extract_lfs, + attrs = { + "src": attr.label(mandatory = True, doc = "Local path to the LFS archive to extract."), + "build_file_content": attr.string(doc = "The content for the BUILD file for this repository. " + + "Either build_file or build_file_content can be specified, but not both."), + "build_file": attr.label(doc = "The file to use as the BUILD file for this repository. " + + "Either build_file or build_file_content can be specified, but not both."), + "strip_prefix": attr.string(default = "", doc = "A directory prefix to strip from the extracted files. "), + }, +) + +lfs_files = repository_rule( + doc = "Export LFS files for on-demand download. Exactly one between `srcs` and `dir` must be defined. The " + + "corresponding paths should be added to be ignored in `.lfsconfig`.", + implementation = _download_lfs, + attrs = { + "srcs": attr.label_list(doc = "Local paths to the LFS files to export."), + "dir": attr.label(doc = "Local path to a directory containing LFS files to export. Only the direct contents " + + "of the directory are exported"), + }, +) From 341816c280f5a41ffeaf0fbf4fee1b57daaa0ccc Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 3 Apr 2024 06:45:45 +0200 Subject: [PATCH 02/81] Kotlin: add dependencies as lazy LFS files --- .gitattributes | 10 +--------- .lfsconfig | 3 +++ MODULE.bazel | 7 +++++++ java/kotlin-extractor/deps/kotlin-compiler-1.4.32.jar | 3 +++ java/kotlin-extractor/deps/kotlin-compiler-1.5.0.jar | 3 +++ java/kotlin-extractor/deps/kotlin-compiler-1.5.10.jar | 3 +++ java/kotlin-extractor/deps/kotlin-compiler-1.5.20.jar | 3 +++ java/kotlin-extractor/deps/kotlin-compiler-1.5.30.jar | 3 +++ java/kotlin-extractor/deps/kotlin-compiler-1.6.0.jar | 3 +++ java/kotlin-extractor/deps/kotlin-compiler-1.6.20.jar | 3 +++ java/kotlin-extractor/deps/kotlin-compiler-1.7.0.jar | 3 +++ java/kotlin-extractor/deps/kotlin-compiler-1.7.20.jar | 3 +++ java/kotlin-extractor/deps/kotlin-compiler-1.8.0.jar | 3 +++ .../deps/kotlin-compiler-1.9.0-Beta.jar | 3 +++ .../deps/kotlin-compiler-1.9.20-Beta.jar | 3 +++ .../deps/kotlin-compiler-2.0.0-Beta4.jar | 3 +++ .../deps/kotlin-compiler-2.0.255-SNAPSHOT.jar | 3 +++ .../deps/kotlin-compiler-embeddable-1.4.32.jar | 3 +++ .../deps/kotlin-compiler-embeddable-1.5.0.jar | 3 +++ .../deps/kotlin-compiler-embeddable-1.5.10.jar | 3 +++ .../deps/kotlin-compiler-embeddable-1.5.20.jar | 3 +++ .../deps/kotlin-compiler-embeddable-1.5.30.jar | 3 +++ .../deps/kotlin-compiler-embeddable-1.6.0.jar | 3 +++ .../deps/kotlin-compiler-embeddable-1.6.20.jar | 3 +++ .../deps/kotlin-compiler-embeddable-1.7.0.jar | 3 +++ .../deps/kotlin-compiler-embeddable-1.7.20.jar | 3 +++ .../deps/kotlin-compiler-embeddable-1.8.0.jar | 3 +++ .../deps/kotlin-compiler-embeddable-1.9.0-Beta.jar | 3 +++ .../deps/kotlin-compiler-embeddable-1.9.20-Beta.jar | 3 +++ .../deps/kotlin-compiler-embeddable-2.0.0-Beta4.jar | 3 +++ .../kotlin-compiler-embeddable-2.0.255-SNAPSHOT.jar | 3 +++ java/kotlin-extractor/deps/kotlin-stdlib-1.4.32.jar | 3 +++ java/kotlin-extractor/deps/kotlin-stdlib-1.5.0.jar | 3 +++ java/kotlin-extractor/deps/kotlin-stdlib-1.5.10.jar | 3 +++ java/kotlin-extractor/deps/kotlin-stdlib-1.5.20.jar | 3 +++ java/kotlin-extractor/deps/kotlin-stdlib-1.5.30.jar | 3 +++ java/kotlin-extractor/deps/kotlin-stdlib-1.6.0.jar | 3 +++ java/kotlin-extractor/deps/kotlin-stdlib-1.6.20.jar | 3 +++ java/kotlin-extractor/deps/kotlin-stdlib-1.7.0.jar | 3 +++ java/kotlin-extractor/deps/kotlin-stdlib-1.7.20.jar | 3 +++ java/kotlin-extractor/deps/kotlin-stdlib-1.8.0.jar | 3 +++ .../kotlin-extractor/deps/kotlin-stdlib-1.9.0-Beta.jar | 3 +++ .../deps/kotlin-stdlib-1.9.20-Beta.jar | 3 +++ .../deps/kotlin-stdlib-2.0.0-Beta4.jar | 3 +++ .../deps/kotlin-stdlib-2.0.255-SNAPSHOT.jar | 3 +++ 45 files changed, 137 insertions(+), 9 deletions(-) create mode 100644 .lfsconfig create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.4.32.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.5.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.5.10.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.5.20.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.5.30.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.6.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.6.20.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.7.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.7.20.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.8.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.9.0-Beta.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.9.20-Beta.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-2.0.0-Beta4.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-2.0.255-SNAPSHOT.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.4.32.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.10.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.20.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.30.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.20.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.20.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.8.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.0-Beta.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.20-Beta.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.0-Beta4.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.255-SNAPSHOT.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.4.32.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.5.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.5.10.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.5.20.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.5.30.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.6.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.6.20.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.7.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.7.20.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.8.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.9.0-Beta.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.9.20-Beta.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-2.0.0-Beta4.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-2.0.255-SNAPSHOT.jar diff --git a/.gitattributes b/.gitattributes index 37484ad742a3..ec56b50dc944 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5,10 +5,8 @@ # git add --renormalize . # git status [just to show what files were renormalized] # git commit -m "Normalize line endings" - # Anything Git auto-detects as text gets normalized and checked out as LF * text=auto eol=lf - # Explicitly set a bunch of known extensions to text, in case auto detection gets confused. *.ql text *.qll text @@ -40,7 +38,6 @@ *.lua text *.expected text *.go text - # Explicitly set a bunch of known extensions to binary, because Git < 2.10 will treat # `* text=auto eol=lf` as `* text eol=lf` *.png -text @@ -49,10 +46,8 @@ *.gif -text *.dll -text *.pdb -text - java/ql/test/stubs/**/*.java linguist-generated=true java/ql/test/experimental/stubs/**/*.java linguist-generated=true - # Force git not to modify line endings for go or html files under the go/ql directory go/ql/**/*.go -text go/ql/**/*.html -text @@ -60,21 +55,18 @@ go/ql/**/*.html -text go/*.dbscheme -text # Preserve unusual line ending from codeql-go merge go/extractor/opencsv/CSVReader.java -text - # For some languages, upgrade script testing references really old dbscheme # files from legacy upgrades that have CRLF line endings. Since upgrade # resolution relies on object hashes, we must suppress line ending conversion # for those testing dbscheme files. */ql/lib/upgrades/initial/*.dbscheme -text - # Generated test files - these are synced from the standard JavaScript libraries using # `javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py`. javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge - # Auto-generated modeling for Python python/ql/lib/semmle/python/frameworks/data/internal/subclass-capture/*.yml linguist-generated=true - # auto-generated bazel lock file ruby/extractor/cargo-bazel-lock.json linguist-generated=true ruby/extractor/cargo-bazel-lock.json -merge +/java/kotlin-extractor/deps/*.jar filter=lfs diff=lfs merge=lfs -text diff --git a/.lfsconfig b/.lfsconfig new file mode 100644 index 000000000000..9525078f2763 --- /dev/null +++ b/.lfsconfig @@ -0,0 +1,3 @@ +[lfs] +fetchexclude = \ +/java/extractor-kotlin/deps diff --git a/MODULE.bazel b/MODULE.bazel index e8c79e8377fa..ad47c834c17b 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -48,6 +48,13 @@ node.toolchain( ) use_repo(node, "nodejs", "nodejs_toolchains") +lfs_files = use_repo_rule("//misc/bazel:lfs.bzl", "lfs_files") + +lfs_files( + name = "kotlin_deps", + dir = "//java/kotlin-extractor:deps", +) + register_toolchains( "@nodejs_toolchains//:all", ) diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.4.32.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.4.32.jar new file mode 100644 index 000000000000..531a86135a62 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-1.4.32.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e784cdef6ce21554c1696b4fea8efa476728447b65054e8f1057a1dfd5ca771a +size 48246906 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.5.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.5.0.jar new file mode 100644 index 000000000000..5aad4918db7a --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-1.5.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6e4a2c4394e77f937fcffda0036531604f25cc7c8de8daea098e1aa31f1d248 +size 47816554 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.5.10.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.5.10.jar new file mode 100644 index 000000000000..04a54b334a24 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-1.5.10.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfefb1aa8bec81256617c8ceb577373e44078b7e21024625da50e376037e9ae5 +size 47822093 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.5.20.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.5.20.jar new file mode 100644 index 000000000000..29a6d4df3e26 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-1.5.20.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0ae437d1b670a5ba6da7893b7023df649c4ab2e6c19d5e9b4eee5332e1cde1f +size 49012794 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.5.30.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.5.30.jar new file mode 100644 index 000000000000..502c844ce5cb --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-1.5.30.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:487d8ff9766a6ba570cd15c5225c1600654e7cf1b6ef2b92ed6905528a3e838a +size 49509580 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.6.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.6.0.jar new file mode 100644 index 000000000000..4a98879e43e6 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-1.6.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bd7a92568fd89c23b7f9f36d4380886beed18d3d54ea6adf49bebae627db805 +size 51408858 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.6.20.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.6.20.jar new file mode 100644 index 000000000000..5a467a5af518 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-1.6.20.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90567c5cf297985d028fa39aa3a7904dc8096173e1c7f3d3f35fe7074581098e +size 53370229 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.7.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.7.0.jar new file mode 100644 index 000000000000..ef54ce1e6532 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-1.7.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce85fafb3e24712d62a0d02d277c2d56197d74afdd4f5ca995eaf33d2c504663 +size 53906809 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.7.20.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.7.20.jar new file mode 100644 index 000000000000..442aea3e3425 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-1.7.20.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e36d98c56f7c9685ab9d9e1fac9be36a5214939adb3f905b93c62de76023618 +size 54773087 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.8.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.8.0.jar new file mode 100644 index 000000000000..7884c5361b56 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-1.8.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f19f247d337387cbdd75d54e10a6865857c28a533fced50c0c5c9482b3ab9af +size 55128997 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.9.0-Beta.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.9.0-Beta.jar new file mode 100644 index 000000000000..8da3d202a5e4 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-1.9.0-Beta.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c7e3972e0ce0be8aa5c8ceeb8eb795f6345685bb57c6f59b649ed70c6051581 +size 60202336 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.9.20-Beta.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.9.20-Beta.jar new file mode 100644 index 000000000000..57215c331e61 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-1.9.20-Beta.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b90980de3a16858e6e1957236d7bb9a729fcd0587a98fb64668866e1975aaa6f +size 61641342 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-2.0.0-Beta4.jar b/java/kotlin-extractor/deps/kotlin-compiler-2.0.0-Beta4.jar new file mode 100644 index 000000000000..cd550d37b971 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-2.0.0-Beta4.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b702954abb1cd62e24f84d0e452262bd8bf8c21a1023e3f9b4bfa72d9870865 +size 63232450 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-2.0.255-SNAPSHOT.jar b/java/kotlin-extractor/deps/kotlin-compiler-2.0.255-SNAPSHOT.jar new file mode 100644 index 000000000000..bb5d18b825f6 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-2.0.255-SNAPSHOT.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b648fd72c8c9e18c822881a0085342e6a7734a500867f409b3f64365b358f01 +size 190091169 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.4.32.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.4.32.jar new file mode 100644 index 000000000000..a3f0e4f87b35 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.4.32.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:083d80ea6262faac293d248c32bf89e062a4e44d657ea6a095c8066e31791e5e +size 46724405 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.0.jar new file mode 100644 index 000000000000..d6e7ef8585bc --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7b85448039e468daf3b9462a172244477fa3eb890f199ec77703992f36ade44 +size 46308890 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.10.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.10.jar new file mode 100644 index 000000000000..05f594d6af13 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.10.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9965f7c958efb17f2a90a19b5e60325d4f4e644d2833dbfb4a11edd8dddf244 +size 46314547 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.20.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.20.jar new file mode 100644 index 000000000000..3bb8ee5e52f7 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.20.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11d51087eb70b5abbad6fbf459a4349a0335916588000b5ecd990f01482e38ff +size 47526900 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.30.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.30.jar new file mode 100644 index 000000000000..647b08b7b7ab --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.30.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5051dc92725b099c41710bd3f213cd0c1d6f25056d31b2e8cae30903873b741 +size 48031832 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.0.jar new file mode 100644 index 000000000000..eafb6508d449 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0366843cd2defdd583c6b16b10bc32b85f28c5bf9510f10e44c886f5bd24c388 +size 49978431 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.20.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.20.jar new file mode 100644 index 000000000000..f426dae65b1e --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.20.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be634faaafb56816b6ef6d583e57ab33e4d6e5180cde2f505ccf7d45dc738ef8 +size 51976423 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.0.jar new file mode 100644 index 000000000000..4abc782ed05f --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6dcd02ced3bdbfabc7da046a69f4e730c8f907fc29bb5b2fea44083ffea22dc8 +size 142740530 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.20.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.20.jar new file mode 100644 index 000000000000..b95097b6935e --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.20.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ec2be1872dc47b9dcb466f1781eded6c59d9eee18657d4b0f1148e619caea36 +size 53395419 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.8.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.8.0.jar new file mode 100644 index 000000000000..5494dd88f974 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.8.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9b3a56dbbfdf1e0328d4731b7d7ca789ce0f1f263372ad88dd8decbd1602858 +size 53769272 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.0-Beta.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.0-Beta.jar new file mode 100644 index 000000000000..32ea1ec13146 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.0-Beta.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7429076f8dd2ccd1cce48d7e5bf5b9fadde8afab110f9f4cfe0912756f16d770 +size 58697460 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.20-Beta.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.20-Beta.jar new file mode 100644 index 000000000000..5c1db1782f8a --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.20-Beta.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c9de79f0f8d97f2aa4d877449063b1cc2828c17f25a119fc32c776401f058de +size 60157805 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.0-Beta4.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.0-Beta4.jar new file mode 100644 index 000000000000..430ba404b88c --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.0-Beta4.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58688bfeddd277767a18bc80f6b2db25fe7b45f8d9a1eb3e20562c4f882ae6bc +size 61765489 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.255-SNAPSHOT.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.255-SNAPSHOT.jar new file mode 100644 index 000000000000..848cbbbf971a --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.255-SNAPSHOT.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91e879b8f58b3fa28cdc6fe61c26bba1235502bff0a8417e492c3eb465d7d342 +size 188200654 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.4.32.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.4.32.jar new file mode 100644 index 000000000000..caf6fe8a0f01 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-1.4.32.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13e9fd3e69dc7230ce0fc873a92a4e5d521d179bcf1bef75a6705baac3bfecba +size 1495021 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.5.0.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.5.0.jar new file mode 100644 index 000000000000..83d4a9f9c031 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-1.5.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52283996fe4067cd7330288b96ae67ecd463614dc741172c54d9d349ab6a9cd7 +size 1497598 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.5.10.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.5.10.jar new file mode 100644 index 000000000000..a0b08eaf499a --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-1.5.10.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca87c454cd3f2e60931f1803c59699d510d3b4b959cd7119296fb947581d722d +size 1497600 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.5.20.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.5.20.jar new file mode 100644 index 000000000000..90844bea3812 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-1.5.20.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80cd79c26aac46d72d782de1ecb326061e93c6e688d994b48627ffd668ba63a8 +size 1497567 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.5.30.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.5.30.jar new file mode 100644 index 000000000000..b19606b84967 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-1.5.30.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c55608e9eb6df7327e74b21e271d324dc523cef31587b8d6d2393db08d6e000c +size 1505951 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.6.0.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.6.0.jar new file mode 100644 index 000000000000..3ad56eecdc2d --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-1.6.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:115daea30b0d484afcf2360237b9d9537f48a4a2f03f3cc2a16577dfc6e90342 +size 1508076 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.6.20.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.6.20.jar new file mode 100644 index 000000000000..b170c1da738b --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-1.6.20.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eeb51c2b67b26233fd81d0bc4f8044ec849718890905763ceffd84a31e2cb799 +size 1509405 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.7.0.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.7.0.jar new file mode 100644 index 000000000000..bed18f8429b7 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-1.7.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa88e9625577957f3249a46cb6e166ee09b369e600f7a11d148d16b0a6d87f05 +size 1524619 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.7.20.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.7.20.jar new file mode 100644 index 000000000000..9998f1e65920 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-1.7.20.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7779ec96b9acbf92ca023858ac04543f9d2c3bdf1722425fff42f25ff3acfc9b +size 1537347 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.8.0.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.8.0.jar new file mode 100644 index 000000000000..47dd2710887b --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-1.8.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c77bef8774640b9fb9d6e217459ff220dae59878beb7d2e4b430506feffc654e +size 1636558 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.9.0-Beta.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.9.0-Beta.jar new file mode 100644 index 000000000000..c6f3f947029f --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-1.9.0-Beta.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af458cc55cf69e966668e6010c7ccee4a50d553b3504a2e8311dd0c76242d881 +size 1708001 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.9.20-Beta.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.9.20-Beta.jar new file mode 100644 index 000000000000..f9c853bbb068 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-1.9.20-Beta.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:788e48813dd76ad598fff7bef3b1e038d7291741810bd04c6c57037c4d75dac2 +size 1715901 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-2.0.0-Beta4.jar b/java/kotlin-extractor/deps/kotlin-stdlib-2.0.0-Beta4.jar new file mode 100644 index 000000000000..50f32cb0f8d6 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-2.0.0-Beta4.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:710ca6626ffa4f86af055aa4f3ebf63dd438a478c7cee43290e46506cecedf9a +size 1729213 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-2.0.255-SNAPSHOT.jar b/java/kotlin-extractor/deps/kotlin-stdlib-2.0.255-SNAPSHOT.jar new file mode 100644 index 000000000000..fb6056d106f2 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-2.0.255-SNAPSHOT.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eab3a26cc18a853f7a67d39b79ea43099dd0e69d60c00835b55b61ace791dc2d +size 4419987 From c242466d316c8e029420dea01d1179e975b19b14 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 3 Apr 2024 16:29:16 +0200 Subject: [PATCH 03/81] Kotlin: first support for Kotlin extractor build --- .bazelrc | 3 + MODULE.bazel | 30 +++++++-- java/kotlin-extractor/BUILD.bazel | 64 +++++++++++++++++++ java/kotlin-extractor/deps/BUILD.bazel | 0 java/kotlin-extractor/deps/empty.jar | 3 + java/kotlin-extractor/extension.bzl | 63 ++++++++++++++++++ java/kotlin-extractor/generate_dbscheme.py | 3 +- java/kotlin-extractor/rules_kotlin.patch | 32 ++++++++++ .../semmle/extractor/java/OdasaOutput.java | 10 +++ java/kotlin-extractor/versions.bzl | 49 ++++++++++++++ misc/bazel/lfs.bzl | 6 +- 11 files changed, 254 insertions(+), 9 deletions(-) create mode 100644 java/kotlin-extractor/deps/BUILD.bazel create mode 100755 java/kotlin-extractor/deps/empty.jar create mode 100644 java/kotlin-extractor/extension.bzl create mode 100644 java/kotlin-extractor/rules_kotlin.patch create mode 100644 java/kotlin-extractor/versions.bzl diff --git a/.bazelrc b/.bazelrc index 12232b4bbd68..6592f65f83a3 100644 --- a/.bazelrc +++ b/.bazelrc @@ -14,4 +14,7 @@ build:linux --cxxopt=-std=c++20 build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64 build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor +# emitting jdeps does not work when building the 2.0.0+ kotlin extractor +build --@rules_kotlin//kotlin/settings:jvm_emit_jdeps=false + try-import %workspace%/local.bazelrc diff --git a/MODULE.bazel b/MODULE.bazel index ad47c834c17b..c8c2804c0dad 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -21,6 +21,14 @@ bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "absl") bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json") bazel_dep(name = "fmt", version = "10.0.0") +bazel_dep(name = "rules_kotlin", version = "1.9.4") + +# we patch `rules_kotlin` to allow passing `-language-version` at a jvm_kt_library level +single_version_override( + module_name = "rules_kotlin", + patch_strip = 1, + patches = ["//java/kotlin-extractor:rules_kotlin.patch"], +) pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") pip.parse( @@ -48,11 +56,23 @@ node.toolchain( ) use_repo(node, "nodejs", "nodejs_toolchains") -lfs_files = use_repo_rule("//misc/bazel:lfs.bzl", "lfs_files") - -lfs_files( - name = "kotlin_deps", - dir = "//java/kotlin-extractor:deps", +kotlin_extractor_deps = use_extension("//java/kotlin-extractor:extension.bzl", "kotlin_extractor_deps") +use_repo( + kotlin_extractor_deps, + "kotlin_extractor_dep_1.4.32", + "kotlin_extractor_dep_1.5.0", + "kotlin_extractor_dep_1.5.10", + "kotlin_extractor_dep_1.5.20", + "kotlin_extractor_dep_1.5.30", + "kotlin_extractor_dep_1.6.0", + "kotlin_extractor_dep_1.6.20", + "kotlin_extractor_dep_1.7.0", + "kotlin_extractor_dep_1.7.20", + "kotlin_extractor_dep_1.8.0", + "kotlin_extractor_dep_1.9.0-Beta", + "kotlin_extractor_dep_1.9.20-Beta", + "kotlin_extractor_dep_2.0.0-Beta4", + "kotlin_extractor_dep_2.0.255-SNAPSHOT", ) register_toolchains( diff --git a/java/kotlin-extractor/BUILD.bazel b/java/kotlin-extractor/BUILD.bazel index e69de29bb2d1..b51624104999 100644 --- a/java/kotlin-extractor/BUILD.bazel +++ b/java/kotlin-extractor/BUILD.bazel @@ -0,0 +1,64 @@ +load( + "//java/kotlin-extractor:versions.bzl", + "VERSIONS", + "get_compatilibity_sources", + "version_less", +) +load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") +load("@rules_kotlin//kotlin:core.bzl", "kt_javac_options", "kt_kotlinc_options") + +py_binary( + name = "generate_dbscheme", + srcs = ["generate_dbscheme.py"], +) + +genrule( + name = "generated-dbscheme", + srcs = ["//java:dbscheme"], + outs = ["KotlinExtractorDbScheme.kt"], + cmd = "$(execpath :generate_dbscheme) $< $@", + tools = [":generate_dbscheme"], +) + +kt_javac_options( + name = "javac-options", + warn = "off", +) + +[ + ( + kt_kotlinc_options( + name = "kotlinc-options-%s" % v, + include_stdlibs = "none", + jvm_target = "1.8", + language_version = v[:3], + warn = "error", + x_optin = [ + "kotlin.RequiresOptIn", + "org.jetbrains.kotlin.ir.symbols.%s" % + ("IrSymbolInternals" if version_less(v, "2.0.0") else "UnsafeDuringIrConstructionAPI"), + ], + x_suppress_version_warnings = True, + ), + kt_jvm_library( + name = "kotlin-extractor-%s" % v, + srcs = + [":generated-dbscheme"] + + glob( + [ + "src/**/*.kt", + "src/**/*.java", + ], + exclude = ["src/main/kotlin/utils/versions/**"], + ) + get_compatilibity_sources(v, "src/main/kotlin/utils/versions"), + javac_opts = ":javac-options", + kotlinc_opts = ":kotlinc-options-%s" % v, + module_name = "codeql-kotlin-extractor", + deps = [ + "@kotlin_extractor_dep_%s//:kotlin-compiler" % v, + "@kotlin_extractor_dep_%s//:kotlin-stdlib" % v, + ], + ), + ) + for v in VERSIONS +] diff --git a/java/kotlin-extractor/deps/BUILD.bazel b/java/kotlin-extractor/deps/BUILD.bazel new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/java/kotlin-extractor/deps/empty.jar b/java/kotlin-extractor/deps/empty.jar new file mode 100755 index 000000000000..fe3604a050f4 --- /dev/null +++ b/java/kotlin-extractor/deps/empty.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 +size 22 diff --git a/java/kotlin-extractor/extension.bzl b/java/kotlin-extractor/extension.bzl new file mode 100644 index 000000000000..8034a1768712 --- /dev/null +++ b/java/kotlin-extractor/extension.bzl @@ -0,0 +1,63 @@ +load("//java/kotlin-extractor:versions.bzl", "VERSIONS") +load("//misc/bazel:lfs.bzl", "lfs_smudge") + +_kotlin_dep_build = """ +load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_import") + +package(default_visibility = ["//visibility:public"]) + +kt_jvm_import( + name = "kotlin-compiler", + jar = "kotlin-compiler-{version}.jar", +) + +kt_jvm_import( + name = "kotlin-compiler-embeddable", + jar = "kotlin-compiler-embeddable-{version}.jar", +) + +kt_jvm_import( + name = "kotlin-stdlib", + jar = "kotlin-stdlib-{version}.jar", +) +""" + +def _kotlin_dep_impl(repository_ctx): + _, sep, version = repository_ctx.name.rpartition("_") + if not sep: + fail("rule @%s malformed, name should be _") + + sources = [ + # "empty.jar", + "kotlin-compiler-%s.jar" % version, + "kotlin-compiler-embeddable-%s.jar" % version, + "kotlin-stdlib-%s.jar" % version, + ] + sources = [repository_ctx.path(Label("//java/kotlin-extractor/deps:%s" % p)) for p in sources] + lfs_smudge(repository_ctx, sources) + + # for some reason rules_kotlin warns about these jars missing, this is to silence those warnings + for jar in ( + "annotations-13.0.jar", + "kotlin-stdlib.jar", + "kotlin-reflect.jar", + "kotlin-script-runtime.jar", + "trove4j.jar", + ): + repository_ctx.symlink("empty.jar", jar) + repository_ctx.file("BUILD.bazel", _kotlin_dep_build.format(version = version)) + +_kotlin_dep = repository_rule(implementation = _kotlin_dep_impl) + +def _kotlin_deps_impl(module_ctx): + deps = [] + for v in VERSIONS: + dep = "kotlin_extractor_dep_%s" % v + _kotlin_dep(name = dep) + deps.append(dep) + return module_ctx.extension_metadata( + root_module_direct_deps = deps, + root_module_direct_dev_deps = [], + ) + +kotlin_extractor_deps = module_extension(implementation = _kotlin_deps_impl) diff --git a/java/kotlin-extractor/generate_dbscheme.py b/java/kotlin-extractor/generate_dbscheme.py index fb891fb105c7..be0c5622ed11 100755 --- a/java/kotlin-extractor/generate_dbscheme.py +++ b/java/kotlin-extractor/generate_dbscheme.py @@ -8,6 +8,7 @@ tables = {} dbscheme = sys.argv[1] if len(sys.argv) >= 2 else '../ql/lib/config/semmlecode.dbscheme' +output = sys.argv[2] if len(sys.argv) >= 3 else 'src/main/kotlin/KotlinExtractorDbScheme.kt' def parse_dbscheme(filename): with open(filename, 'r') as f: @@ -152,7 +153,7 @@ def genTable(kt, relname, columns, enum = None, kind = None, num = None, typ = N kt.write(')\\n")\n') kt.write('}\n') -with open('src/main/kotlin/KotlinExtractorDbScheme.kt', 'w') as kt: +with open(output, 'w') as kt: kt.write('/* Generated by ' + sys.argv[0] + ': Do not edit manually. */\n') kt.write('package com.github.codeql\n') kt.write('import java.util.Date\n') diff --git a/java/kotlin-extractor/rules_kotlin.patch b/java/kotlin-extractor/rules_kotlin.patch new file mode 100644 index 000000000000..3bc9ff4aac27 --- /dev/null +++ b/java/kotlin-extractor/rules_kotlin.patch @@ -0,0 +1,32 @@ +diff --git a/src/main/starlark/core/options/opts.kotlinc.bzl b/src/main/starlark/core/options/opts.kotlinc.bzl +index 9b15fb8..c0ac2cd 100644 +--- a/src/main/starlark/core/options/opts.kotlinc.bzl ++++ b/src/main/starlark/core/options/opts.kotlinc.bzl +@@ -28,6 +28,11 @@ def _map_jvm_target_to_flag(version): + return None + return ["-jvm-target=%s" % version] + ++def _map_language_version_to_flag(version): ++ if not version: ++ return None ++ return ["-language-version=%s" % version, "-api-version=%s" % version] ++ + _KOPTS_ALL = { + "warn": struct( + args = dict( +@@ -349,6 +354,15 @@ _KOPTS_ALL = { + value_to_flag = None, + map_value_to_flag = _map_jvm_target_to_flag, + ), ++ "language_version": struct( ++ args = dict( ++ default = "1.9", ++ doc = "-language-version", ++ ), ++ type = attr.string, ++ value_to_flag = None, ++ map_value_to_flag = _map_language_version_to_flag, ++ ), + } + + # Filters out options that are not available in current compiler release diff --git a/java/kotlin-extractor/src/main/java/com/semmle/extractor/java/OdasaOutput.java b/java/kotlin-extractor/src/main/java/com/semmle/extractor/java/OdasaOutput.java index 830b2012c983..4bb30527fb84 100644 --- a/java/kotlin-extractor/src/main/java/com/semmle/extractor/java/OdasaOutput.java +++ b/java/kotlin-extractor/src/main/java/com/semmle/extractor/java/OdasaOutput.java @@ -547,6 +547,16 @@ public boolean equals(Object obj) { } } + @Override + public int hashCode() { + int hash = 7; + hash = 31 * hash + majorVersion; + hash = 31 * hash + minorVersion; + hash = 31 * hash + (int)lastModified; + hash = 31 * hash + (extractorName == null ? 0 : extractorName.hashCode()); + return hash; + } + private boolean newerThan(TrapClassVersion tcv) { // Classes being compiled from source have major version 0 but should take precedence // over any classes with the same qualified name loaded from the classpath diff --git a/java/kotlin-extractor/versions.bzl b/java/kotlin-extractor/versions.bzl new file mode 100644 index 000000000000..473825682c16 --- /dev/null +++ b/java/kotlin-extractor/versions.bzl @@ -0,0 +1,49 @@ +VERSIONS = [ + "1.4.32", + "1.5.0", + "1.5.10", + "1.5.20", + "1.5.30", + "1.6.0", + "1.6.20", + "1.7.0", + "1.7.20", + "1.8.0", + "1.9.0-Beta", + "1.9.20-Beta", + "2.0.0-Beta4", + "2.0.255-SNAPSHOT", +] + +def _version_to_tuple(v): + v, _, tail = v.partition("-") + v = tuple([int(x) for x in v.split(".")]) + return v + (tail,) + +def _tuple_to_version(t): + ret = ".".join([str(x) for x in t[:3]]) + if t[3]: + ret += "-" + t[3] + return ret + +def version_less(lhs, rhs): + return _version_to_tuple(lhs) < _version_to_tuple(rhs) + +def _basename(path): + if "/" not in path: + return path + return path[path.rindex("/") + 1:] + +def get_compatilibity_sources(version, dir): + prefix = "%s/v_" % dir + available = native.glob(["%s*" % prefix], exclude_directories = 0) + + # we want files with the same base name to replace ones for previous versions, hence the map + srcs = {} + for d in available: + compat_version = d[len(prefix):].replace("_", ".") + if version_less(version, compat_version): + break + files = native.glob(["%s/*.kt" % d]) + srcs |= {_basename(f): f for f in files} + return srcs.values() diff --git a/misc/bazel/lfs.bzl b/misc/bazel/lfs.bzl index a5559d55a744..b58831f72eaa 100644 --- a/misc/bazel/lfs.bzl +++ b/misc/bazel/lfs.bzl @@ -1,4 +1,4 @@ -def _smudge(repository_ctx, srcs): +def lfs_smudge(repository_ctx, srcs): for src in srcs: repository_ctx.watch(src) script = Label("//misc/bazel/internal:git_lfs_smudge.py") @@ -14,7 +14,7 @@ def _download_and_extract_lfs(repository_ctx): src = repository_ctx.path(attr.src) if attr.build_file_content and attr.build_file: fail("You should specify only one among build_file_content and build_file for rule @%s" % repository_ctx.name) - _smudge(repository_ctx, [src]) + lfs_smudge(repository_ctx, [src]) repository_ctx.extract(src.basename, stripPrefix = attr.strip_prefix) repository_ctx.delete(src.basename) if attr.build_file_content: @@ -33,7 +33,7 @@ def _download_lfs(repository_ctx): if not dir.is_dir: fail("`dir` not a directory in @%s" % repository_ctx.name) srcs = [f for f in dir.readdir() if not f.is_dir] - _smudge(repository_ctx, srcs) + lfs_smudge(repository_ctx, srcs) # with bzlmod the name is qualified with `~` separators, and we want the base name here name = repository_ctx.name.split("~")[-1] From 47ff1c1ee616fd9b63c05ed28806317e22e6f3ca Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 4 Apr 2024 10:23:26 +0200 Subject: [PATCH 04/81] Kotlin: working standalone extractor built with bazel --- MODULE.bazel | 58 ++++++++++++++++++++------- java/kotlin-extractor/BUILD.bazel | 46 ++++++++++++++++----- java/kotlin-extractor/deps/empty.jar | 3 -- java/kotlin-extractor/deps/empty.zip | Bin 0 -> 22 bytes java/kotlin-extractor/extension.bzl | 50 +++++++++-------------- java/kotlin-extractor/versions.bzl | 1 + 6 files changed, 100 insertions(+), 58 deletions(-) delete mode 100755 java/kotlin-extractor/deps/empty.jar create mode 100755 java/kotlin-extractor/deps/empty.zip diff --git a/MODULE.bazel b/MODULE.bazel index c8c2804c0dad..2d0a910a51ae 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -57,22 +57,52 @@ node.toolchain( use_repo(node, "nodejs", "nodejs_toolchains") kotlin_extractor_deps = use_extension("//java/kotlin-extractor:extension.bzl", "kotlin_extractor_deps") + +# following list can be kept in sync by running `bazel mod tidy` in `codeql` use_repo( kotlin_extractor_deps, - "kotlin_extractor_dep_1.4.32", - "kotlin_extractor_dep_1.5.0", - "kotlin_extractor_dep_1.5.10", - "kotlin_extractor_dep_1.5.20", - "kotlin_extractor_dep_1.5.30", - "kotlin_extractor_dep_1.6.0", - "kotlin_extractor_dep_1.6.20", - "kotlin_extractor_dep_1.7.0", - "kotlin_extractor_dep_1.7.20", - "kotlin_extractor_dep_1.8.0", - "kotlin_extractor_dep_1.9.0-Beta", - "kotlin_extractor_dep_1.9.20-Beta", - "kotlin_extractor_dep_2.0.0-Beta4", - "kotlin_extractor_dep_2.0.255-SNAPSHOT", + "kotlin-compiler-1.4.32", + "kotlin-compiler-1.5.0", + "kotlin-compiler-1.5.10", + "kotlin-compiler-1.5.20", + "kotlin-compiler-1.5.30", + "kotlin-compiler-1.6.0", + "kotlin-compiler-1.6.20", + "kotlin-compiler-1.7.0", + "kotlin-compiler-1.7.20", + "kotlin-compiler-1.8.0", + "kotlin-compiler-1.9.0-Beta", + "kotlin-compiler-1.9.20-Beta", + "kotlin-compiler-2.0.0-Beta4", + "kotlin-compiler-2.0.255-SNAPSHOT", + "kotlin-compiler-embeddable-1.4.32", + "kotlin-compiler-embeddable-1.5.0", + "kotlin-compiler-embeddable-1.5.10", + "kotlin-compiler-embeddable-1.5.20", + "kotlin-compiler-embeddable-1.5.30", + "kotlin-compiler-embeddable-1.6.0", + "kotlin-compiler-embeddable-1.6.20", + "kotlin-compiler-embeddable-1.7.0", + "kotlin-compiler-embeddable-1.7.20", + "kotlin-compiler-embeddable-1.8.0", + "kotlin-compiler-embeddable-1.9.0-Beta", + "kotlin-compiler-embeddable-1.9.20-Beta", + "kotlin-compiler-embeddable-2.0.0-Beta4", + "kotlin-compiler-embeddable-2.0.255-SNAPSHOT", + "kotlin-stdlib-1.4.32", + "kotlin-stdlib-1.5.0", + "kotlin-stdlib-1.5.10", + "kotlin-stdlib-1.5.20", + "kotlin-stdlib-1.5.30", + "kotlin-stdlib-1.6.0", + "kotlin-stdlib-1.6.20", + "kotlin-stdlib-1.7.0", + "kotlin-stdlib-1.7.20", + "kotlin-stdlib-1.8.0", + "kotlin-stdlib-1.9.0-Beta", + "kotlin-stdlib-1.9.20-Beta", + "kotlin-stdlib-2.0.0-Beta4", + "kotlin-stdlib-2.0.255-SNAPSHOT", ) register_toolchains( diff --git a/java/kotlin-extractor/BUILD.bazel b/java/kotlin-extractor/BUILD.bazel index b51624104999..b6c6e87e56cb 100644 --- a/java/kotlin-extractor/BUILD.bazel +++ b/java/kotlin-extractor/BUILD.bazel @@ -5,7 +5,7 @@ load( "version_less", ) load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") -load("@rules_kotlin//kotlin:core.bzl", "kt_javac_options", "kt_kotlinc_options") +load("@rules_kotlin//kotlin:core.bzl", "kt_kotlinc_options") py_binary( name = "generate_dbscheme", @@ -20,10 +20,13 @@ genrule( tools = [":generate_dbscheme"], ) -kt_javac_options( - name = "javac-options", - warn = "off", -) +_resources = [ + ( + r, + r[len("src/main/resources/"):], + ) + for r in glob(["src/main/resources/**"]) +] [ ( @@ -40,8 +43,27 @@ kt_javac_options( ], x_suppress_version_warnings = True, ), + # * extractor.name is different for each version, so we need to put it in different output dirs + # * in order to put it in `resources`, we need to define `resource_strip_prefix` to strip this version + # * `resource_strip_prefix` is unique per jar, so we must also put other resources under the same version prefix + genrule( + name = "resources-%s" % v, + srcs = [src for src, _ in _resources], + outs = [ + "%s/com/github/codeql/extractor.name" % v, + ] + [ + "%s/%s" % (v, tgt) + for _, tgt in _resources + ], + cmd = "\n".join([ + "echo codeql-extractor-kotlin-standalone-%s > $(RULEDIR)/%s/com/github/codeql/extractor.name" % (v, v), + ] + [ + "cp $(execpath %s) $(RULEDIR)/%s/%s" % (src, v, tgt) + for src, tgt in _resources + ]), + ), kt_jvm_library( - name = "kotlin-extractor-%s" % v, + name = "codeql-extractor-kotlin-standalone-%s" % v, srcs = [":generated-dbscheme"] + glob( @@ -51,12 +73,18 @@ kt_javac_options( ], exclude = ["src/main/kotlin/utils/versions/**"], ) + get_compatilibity_sources(v, "src/main/kotlin/utils/versions"), - javac_opts = ":javac-options", kotlinc_opts = ":kotlinc-options-%s" % v, module_name = "codeql-kotlin-extractor", + resource_strip_prefix = "%s/%s" % ( + package_name(), + v, + ), + resources = [ + ":resources-%s" % v, + ], deps = [ - "@kotlin_extractor_dep_%s//:kotlin-compiler" % v, - "@kotlin_extractor_dep_%s//:kotlin-stdlib" % v, + "@kotlin-compiler-%s" % v, + "@kotlin-stdlib-%s" % v, ], ), ) diff --git a/java/kotlin-extractor/deps/empty.jar b/java/kotlin-extractor/deps/empty.jar deleted file mode 100755 index fe3604a050f4..000000000000 --- a/java/kotlin-extractor/deps/empty.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 -size 22 diff --git a/java/kotlin-extractor/deps/empty.zip b/java/kotlin-extractor/deps/empty.zip new file mode 100755 index 0000000000000000000000000000000000000000..15cb0ecb3e219d1701294bfdf0fe3f5cb5d208e7 GIT binary patch literal 22 NcmWIWW@Tf*000g10H*)| literal 0 HcmV?d00001 diff --git a/java/kotlin-extractor/extension.bzl b/java/kotlin-extractor/extension.bzl index 8034a1768712..9bd076190cf2 100644 --- a/java/kotlin-extractor/extension.bzl +++ b/java/kotlin-extractor/extension.bzl @@ -4,39 +4,22 @@ load("//misc/bazel:lfs.bzl", "lfs_smudge") _kotlin_dep_build = """ load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_import") -package(default_visibility = ["//visibility:public"]) - kt_jvm_import( - name = "kotlin-compiler", - jar = "kotlin-compiler-{version}.jar", -) - -kt_jvm_import( - name = "kotlin-compiler-embeddable", - jar = "kotlin-compiler-embeddable-{version}.jar", -) - -kt_jvm_import( - name = "kotlin-stdlib", - jar = "kotlin-stdlib-{version}.jar", + name = "{name}", + jar = "{name}.jar", + visibility = ["//visibility:public"], ) """ +def _get_dep(repository_ctx, name): + return repository_ctx.path(Label("//java/kotlin-extractor/deps:%s" % name)) + def _kotlin_dep_impl(repository_ctx): - _, sep, version = repository_ctx.name.rpartition("_") - if not sep: - fail("rule @%s malformed, name should be _") - - sources = [ - # "empty.jar", - "kotlin-compiler-%s.jar" % version, - "kotlin-compiler-embeddable-%s.jar" % version, - "kotlin-stdlib-%s.jar" % version, - ] - sources = [repository_ctx.path(Label("//java/kotlin-extractor/deps:%s" % p)) for p in sources] - lfs_smudge(repository_ctx, sources) + _, _, name = repository_ctx.name.rpartition("~") + lfs_smudge(repository_ctx, [_get_dep(repository_ctx, name + ".jar")]) # for some reason rules_kotlin warns about these jars missing, this is to silence those warnings + empty = _get_dep(repository_ctx, "empty.zip") for jar in ( "annotations-13.0.jar", "kotlin-stdlib.jar", @@ -44,17 +27,20 @@ def _kotlin_dep_impl(repository_ctx): "kotlin-script-runtime.jar", "trove4j.jar", ): - repository_ctx.symlink("empty.jar", jar) - repository_ctx.file("BUILD.bazel", _kotlin_dep_build.format(version = version)) + repository_ctx.symlink(empty, jar) + repository_ctx.file("BUILD.bazel", _kotlin_dep_build.format(name = name)) -_kotlin_dep = repository_rule(implementation = _kotlin_dep_impl) +_kotlin_dep = repository_rule( + implementation = _kotlin_dep_impl, +) def _kotlin_deps_impl(module_ctx): deps = [] for v in VERSIONS: - dep = "kotlin_extractor_dep_%s" % v - _kotlin_dep(name = dep) - deps.append(dep) + for lib in ("compiler", "compiler-embeddable", "stdlib"): + dep = "kotlin-%s-%s" % (lib, v) + _kotlin_dep(name = dep) + deps.append(dep) return module_ctx.extension_metadata( root_module_direct_deps = deps, root_module_direct_dev_deps = [], diff --git a/java/kotlin-extractor/versions.bzl b/java/kotlin-extractor/versions.bzl index 473825682c16..454e25c10205 100644 --- a/java/kotlin-extractor/versions.bzl +++ b/java/kotlin-extractor/versions.bzl @@ -1,3 +1,4 @@ +# when updating this list, `bazel mod tidy` should be run from `codeql` to update `MODULE.bazel` VERSIONS = [ "1.4.32", "1.5.0", From 5313288b8e6d12cd195797a77657765dcee9b49e Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 4 Apr 2024 11:02:29 +0200 Subject: [PATCH 05/81] LFS: do non-matching fetchinclude rather than explicit fetchexclude --- .lfsconfig | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.lfsconfig b/.lfsconfig index 9525078f2763..87c9472c023e 100644 --- a/.lfsconfig +++ b/.lfsconfig @@ -1,3 +1,2 @@ [lfs] -fetchexclude = \ -/java/extractor-kotlin/deps +fetchinclude = nothing From 55ff7109fa53004a424bdfe238d6d49a2d375f6a Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 4 Apr 2024 11:26:54 +0200 Subject: [PATCH 06/81] Kotlin: support embeddable build in bazel --- MODULE.bazel | 3 +- java/kotlin-extractor/BUILD.bazel | 36 ++++++++++++--- .../{extension.bzl => deps.bzl} | 45 +++++++++++++++++-- 3 files changed, 74 insertions(+), 10 deletions(-) rename java/kotlin-extractor/{extension.bzl => deps.bzl} (50%) diff --git a/MODULE.bazel b/MODULE.bazel index 2d0a910a51ae..40a9bd1b3038 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -56,11 +56,12 @@ node.toolchain( ) use_repo(node, "nodejs", "nodejs_toolchains") -kotlin_extractor_deps = use_extension("//java/kotlin-extractor:extension.bzl", "kotlin_extractor_deps") +kotlin_extractor_deps = use_extension("//java/kotlin-extractor:deps.bzl", "kotlin_extractor_deps") # following list can be kept in sync by running `bazel mod tidy` in `codeql` use_repo( kotlin_extractor_deps, + "codeql_kotlin_embeddable", "kotlin-compiler-1.4.32", "kotlin-compiler-1.5.0", "kotlin-compiler-1.5.10", diff --git a/java/kotlin-extractor/BUILD.bazel b/java/kotlin-extractor/BUILD.bazel index b6c6e87e56cb..7aacfed14485 100644 --- a/java/kotlin-extractor/BUILD.bazel +++ b/java/kotlin-extractor/BUILD.bazel @@ -1,5 +1,7 @@ +# notice that this is used in the `@codeql_koblin_embeddable` external repo, which means we need to +# reference explicitly @codeql load( - "//java/kotlin-extractor:versions.bzl", + "@codeql//java/kotlin-extractor:versions.bzl", "VERSIONS", "get_compatilibity_sources", "version_less", @@ -7,6 +9,15 @@ load( load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") load("@rules_kotlin//kotlin:core.bzl", "kt_kotlinc_options") +_for_embeddable = repo_name().endswith("codeql_kotlin_embeddable") + +_common_extractor_name_prefix = "codeql-extractor-kotlin" + +_extractor_name_prefix = "%s-%s" % ( + _common_extractor_name_prefix, + "embeddable" if _for_embeddable else "standalone", +) + py_binary( name = "generate_dbscheme", srcs = ["generate_dbscheme.py"], @@ -14,7 +25,7 @@ py_binary( genrule( name = "generated-dbscheme", - srcs = ["//java:dbscheme"], + srcs = ["@codeql//java:dbscheme"], outs = ["KotlinExtractorDbScheme.kt"], cmd = "$(execpath :generate_dbscheme) $< $@", tools = [":generate_dbscheme"], @@ -56,14 +67,14 @@ _resources = [ for _, tgt in _resources ], cmd = "\n".join([ - "echo codeql-extractor-kotlin-standalone-%s > $(RULEDIR)/%s/com/github/codeql/extractor.name" % (v, v), + "echo %s-%s > $(RULEDIR)/%s/com/github/codeql/extractor.name" % (_extractor_name_prefix, v, v), ] + [ "cp $(execpath %s) $(RULEDIR)/%s/%s" % (src, v, tgt) for src, tgt in _resources ]), ), kt_jvm_library( - name = "codeql-extractor-kotlin-standalone-%s" % v, + name = "%s-%s" % (_extractor_name_prefix, v), srcs = [":generated-dbscheme"] + glob( @@ -75,18 +86,31 @@ _resources = [ ) + get_compatilibity_sources(v, "src/main/kotlin/utils/versions"), kotlinc_opts = ":kotlinc-options-%s" % v, module_name = "codeql-kotlin-extractor", - resource_strip_prefix = "%s/%s" % ( + resource_strip_prefix = "../%s/%s" % ( + repo_name(), + v, + ) if _for_embeddable else "%s/%s" % ( package_name(), v, ), resources = [ ":resources-%s" % v, ], + visibility = ["//visibility:public"], deps = [ - "@kotlin-compiler-%s" % v, + "@kotlin-compiler%s-%s" % ( + "-embeddable" if _for_embeddable else "", + v, + ), "@kotlin-stdlib-%s" % v, ], ), + # if in main repository, alias the embeddable versions from the modified @codeql_kotlin_embeddable repo + alias( + name = "%s-embeddable-%s" % (_common_extractor_name_prefix, v), + actual = "@codeql_kotlin_embeddable//:%s-embeddable-%s" % (_common_extractor_name_prefix, v), + visibility = ["//visibility:public"], + ) if not _for_embeddable else None, ) for v in VERSIONS ] diff --git a/java/kotlin-extractor/extension.bzl b/java/kotlin-extractor/deps.bzl similarity index 50% rename from java/kotlin-extractor/extension.bzl rename to java/kotlin-extractor/deps.bzl index 9bd076190cf2..65532770031a 100644 --- a/java/kotlin-extractor/extension.bzl +++ b/java/kotlin-extractor/deps.bzl @@ -34,13 +34,52 @@ _kotlin_dep = repository_rule( implementation = _kotlin_dep_impl, ) +def _walk(dir): + res = [] + next_dirs = [dir] + + # loops must be bounded in starlark + for i in range(100): + current_dirs = next_dirs + next_dirs = [] + for d in current_dirs: + children = d.readdir() + next_dirs.extend([c for c in children if c.is_dir]) + res.extend([c for c in children if not c.is_dir]) + if not next_dirs: + break + return res + +def _embeddable_source_impl(repository_ctx): + src_dir = repository_ctx.path(Label("//java/kotlin-extractor:src")) + for src in _walk(src_dir): + contents = repository_ctx.read(src) + contents = contents.replace( + "import com.intellij", + "import org.jetbrains.kotlin.com.intellij", + ) + repository_ctx.file(str(src).replace(str(src_dir), "src"), contents) + repository_ctx.symlink( + Label("//java/kotlin-extractor:generate_dbscheme.py"), + "generate_dbscheme.py", + ) + repository_ctx.symlink( + Label("//java/kotlin-extractor:BUILD.bazel"), + "BUILD.bazel", + ) + +_embeddable_source = repository_rule(implementation = _embeddable_source_impl) + +def _add_rule(rules, rule, *, name, **kwargs): + rule(name = name, **kwargs) + rules.append(name) + def _kotlin_deps_impl(module_ctx): deps = [] for v in VERSIONS: for lib in ("compiler", "compiler-embeddable", "stdlib"): - dep = "kotlin-%s-%s" % (lib, v) - _kotlin_dep(name = dep) - deps.append(dep) + _add_rule(deps, _kotlin_dep, name = "kotlin-%s-%s" % (lib, v)) + _add_rule(deps, _embeddable_source, name = "codeql_kotlin_embeddable") return module_ctx.extension_metadata( root_module_direct_deps = deps, root_module_direct_dev_deps = [], From 5d6baea174bc46fccd282c772b549a7b6c49ead7 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 4 Apr 2024 15:34:48 +0200 Subject: [PATCH 07/81] Kotlin: add aliases for default versions --- MODULE.bazel | 1 + java/kotlin-extractor/BUILD.bazel | 39 +++++++++++++++++++ java/kotlin-extractor/deps.bzl | 38 +++++++++++++++++- .../kotlin_plugin_versions.py | 2 +- java/kotlin-extractor/versions.bzl | 2 + 5 files changed, 80 insertions(+), 2 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 40a9bd1b3038..48609e84fdab 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -61,6 +61,7 @@ kotlin_extractor_deps = use_extension("//java/kotlin-extractor:deps.bzl", "kotli # following list can be kept in sync by running `bazel mod tidy` in `codeql` use_repo( kotlin_extractor_deps, + "codeql_kotlin_defaults", "codeql_kotlin_embeddable", "kotlin-compiler-1.4.32", "kotlin-compiler-1.5.0", diff --git a/java/kotlin-extractor/BUILD.bazel b/java/kotlin-extractor/BUILD.bazel index 7aacfed14485..fdcb9b56c582 100644 --- a/java/kotlin-extractor/BUILD.bazel +++ b/java/kotlin-extractor/BUILD.bazel @@ -8,6 +8,7 @@ load( ) load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") load("@rules_kotlin//kotlin:core.bzl", "kt_kotlinc_options") +load("@codeql_kotlin_defaults//:defaults.bzl", "kotlin_extractor_defaults") _for_embeddable = repo_name().endswith("codeql_kotlin_embeddable") @@ -114,3 +115,41 @@ _resources = [ ) for v in VERSIONS ] + +# default aliases, based on the kotlinc version installed on the host +# * default version can be overridden with env variableCODEQL_KOTLIN_SINGLE_VERSION +# * setting CODEQL_KOTLIN_SINGLE_VERSION_EMBEDDABLE=true overrides the default variant +# * when a new kotlinc version is installed, you'll need to either `bazel clean` or +# `bazel fetch --force @codeql_kotlin_defaults//:all` to refresh the default +( + alias( + name = "%s-standalone" % _common_extractor_name_prefix, + actual = "%s-standalone-%s" % ( + _common_extractor_name_prefix, + kotlin_extractor_defaults.version, + ), + visibility = ["//visibility:public"], + ), + alias( + name = "%s-embeddable" % _common_extractor_name_prefix, + actual = "%s-embeddable-%s" % ( + _common_extractor_name_prefix, + kotlin_extractor_defaults.version, + ), + visibility = ["//visibility:public"], + ), + alias( + name = _common_extractor_name_prefix, + actual = "%s-%s-%s" % ( + _common_extractor_name_prefix, + kotlin_extractor_defaults.variant, + kotlin_extractor_defaults.version, + ), + visibility = ["//visibility:public"], + ), + alias( + name = "kotlin-extractor", + actual = _common_extractor_name_prefix, + visibility = ["//visibility:public"], + ), +) if not _for_embeddable else None diff --git a/java/kotlin-extractor/deps.bzl b/java/kotlin-extractor/deps.bzl index 65532770031a..48aae95fd9ef 100644 --- a/java/kotlin-extractor/deps.bzl +++ b/java/kotlin-extractor/deps.bzl @@ -1,4 +1,4 @@ -load("//java/kotlin-extractor:versions.bzl", "VERSIONS") +load("//java/kotlin-extractor:versions.bzl", "DEFAULT_FALLBACK_VERSION", "VERSIONS", "version_less") load("//misc/bazel:lfs.bzl", "lfs_smudge") _kotlin_dep_build = """ @@ -70,6 +70,41 @@ def _embeddable_source_impl(repository_ctx): _embeddable_source = repository_rule(implementation = _embeddable_source_impl) +def _get_default_version(repository_ctx): + default_version = repository_ctx.getenv("CODEQL_KOTLIN_SINGLE_VERSION") + if default_version: + if default_version not in VERSIONS: + fail("overriding CODEQL_KOTLIN_SINGLE_VERSION=%s not known, must be one of:\n %s" % + (default_version, " ".join(VERSIONS))) + return default_version + kotlinc = repository_ctx.which("kotlinc") + if not kotlinc: + return DEFAULT_FALLBACK_VERSION + res = repository_ctx.execute([kotlinc, "-version"]) + if not res: + fail("kotlinc -version failed: %s" % res.stderr) + out = res.stderr.split(" ") + if len(out) < 3: + fail("malformed kotlinc -version output: %s" % res.stdout) + host_version = out[2] + for version in reversed(VERSIONS): + if version_less(version, host_version) or version == host_version: + return version + fail("no relevant version found for host version %s among:\n %s" % (host_version, " ".join(VERSIONS))) + +def _defaults_impl(repository_ctx): + default_version = _get_default_version(repository_ctx) + default_variant = "standalone" + if repository_ctx.getenv("CODEQL_KOTLIN_SINGLE_VERSION_EMBEDDABLE") in ("true", "1"): + default_variant = "embeddable" + repository_ctx.file( + "defaults.bzl", + "kotlin_extractor_defaults = struct(version = '%s', variant = '%s')\n" % (default_version, default_variant), + ) + repository_ctx.file("BUILD.bazel") + +_defaults = repository_rule(implementation = _defaults_impl) + def _add_rule(rules, rule, *, name, **kwargs): rule(name = name, **kwargs) rules.append(name) @@ -80,6 +115,7 @@ def _kotlin_deps_impl(module_ctx): for lib in ("compiler", "compiler-embeddable", "stdlib"): _add_rule(deps, _kotlin_dep, name = "kotlin-%s-%s" % (lib, v)) _add_rule(deps, _embeddable_source, name = "codeql_kotlin_embeddable") + _add_rule(deps, _defaults, name = "codeql_kotlin_defaults") return module_ctx.extension_metadata( root_module_direct_deps = deps, root_module_direct_dev_deps = [], diff --git a/java/kotlin-extractor/kotlin_plugin_versions.py b/java/kotlin-extractor/kotlin_plugin_versions.py index d385eb9e6137..e7e193cdbc53 100755 --- a/java/kotlin-extractor/kotlin_plugin_versions.py +++ b/java/kotlin-extractor/kotlin_plugin_versions.py @@ -62,6 +62,7 @@ def get_single_version(fakeVersionOutput = None): raise KotlincNotFoundException() versionOutput = subprocess.run([kotlinc, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True).stderr if fakeVersionOutput is None else fakeVersionOutput m = re.match(r'.* kotlinc-jvm ([0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z][a-zA-Z0-9]*)?) .*', versionOutput) + if m is None: if m is None: raise Exception('Cannot detect version of kotlinc (got ' + str(versionOutput) + ')') current_version = version_string_to_version(m.group(1)) @@ -87,4 +88,3 @@ def get_latest_url(): print(get_single_version(*args[2:])) else: raise Exception("Unknown command: " + command) - diff --git a/java/kotlin-extractor/versions.bzl b/java/kotlin-extractor/versions.bzl index 454e25c10205..f5ee9520bb44 100644 --- a/java/kotlin-extractor/versions.bzl +++ b/java/kotlin-extractor/versions.bzl @@ -16,6 +16,8 @@ VERSIONS = [ "2.0.255-SNAPSHOT", ] +DEFAULT_FALLBACK_VERSION = "1.9.0-Beta" + def _version_to_tuple(v): v, _, tail = v.partition("-") v = tuple([int(x) for x in v.split(".")]) From 3a0a219ce737491a54e5e0bdd8ab5078b2ba394a Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 4 Apr 2024 15:49:37 +0200 Subject: [PATCH 08/81] Kotlin: remove obsolete 1.4.32 version --- MODULE.bazel | 3 --- java/kotlin-extractor/deps/kotlin-compiler-1.4.32.jar | 3 --- .../deps/kotlin-compiler-embeddable-1.4.32.jar | 3 --- java/kotlin-extractor/deps/kotlin-stdlib-1.4.32.jar | 3 --- java/kotlin-extractor/versions.bzl | 1 - 5 files changed, 13 deletions(-) delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.4.32.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.4.32.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.4.32.jar diff --git a/MODULE.bazel b/MODULE.bazel index 48609e84fdab..0f6546b4e364 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -63,7 +63,6 @@ use_repo( kotlin_extractor_deps, "codeql_kotlin_defaults", "codeql_kotlin_embeddable", - "kotlin-compiler-1.4.32", "kotlin-compiler-1.5.0", "kotlin-compiler-1.5.10", "kotlin-compiler-1.5.20", @@ -77,7 +76,6 @@ use_repo( "kotlin-compiler-1.9.20-Beta", "kotlin-compiler-2.0.0-Beta4", "kotlin-compiler-2.0.255-SNAPSHOT", - "kotlin-compiler-embeddable-1.4.32", "kotlin-compiler-embeddable-1.5.0", "kotlin-compiler-embeddable-1.5.10", "kotlin-compiler-embeddable-1.5.20", @@ -91,7 +89,6 @@ use_repo( "kotlin-compiler-embeddable-1.9.20-Beta", "kotlin-compiler-embeddable-2.0.0-Beta4", "kotlin-compiler-embeddable-2.0.255-SNAPSHOT", - "kotlin-stdlib-1.4.32", "kotlin-stdlib-1.5.0", "kotlin-stdlib-1.5.10", "kotlin-stdlib-1.5.20", diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.4.32.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.4.32.jar deleted file mode 100644 index 531a86135a62..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-1.4.32.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e784cdef6ce21554c1696b4fea8efa476728447b65054e8f1057a1dfd5ca771a -size 48246906 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.4.32.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.4.32.jar deleted file mode 100644 index a3f0e4f87b35..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.4.32.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:083d80ea6262faac293d248c32bf89e062a4e44d657ea6a095c8066e31791e5e -size 46724405 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.4.32.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.4.32.jar deleted file mode 100644 index caf6fe8a0f01..000000000000 --- a/java/kotlin-extractor/deps/kotlin-stdlib-1.4.32.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:13e9fd3e69dc7230ce0fc873a92a4e5d521d179bcf1bef75a6705baac3bfecba -size 1495021 diff --git a/java/kotlin-extractor/versions.bzl b/java/kotlin-extractor/versions.bzl index f5ee9520bb44..186a9bc6c356 100644 --- a/java/kotlin-extractor/versions.bzl +++ b/java/kotlin-extractor/versions.bzl @@ -1,6 +1,5 @@ # when updating this list, `bazel mod tidy` should be run from `codeql` to update `MODULE.bazel` VERSIONS = [ - "1.4.32", "1.5.0", "1.5.10", "1.5.20", From 44f3c0289a4c8bf3be675d4acdf93c9bcd1b9a2c Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 4 Apr 2024 17:31:40 +0200 Subject: [PATCH 09/81] Kotlin: revert accidental modification of `kotlin_plugin_versions.py` --- java/kotlin-extractor/kotlin_plugin_versions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/java/kotlin-extractor/kotlin_plugin_versions.py b/java/kotlin-extractor/kotlin_plugin_versions.py index e7e193cdbc53..1978ac84bc37 100755 --- a/java/kotlin-extractor/kotlin_plugin_versions.py +++ b/java/kotlin-extractor/kotlin_plugin_versions.py @@ -62,7 +62,6 @@ def get_single_version(fakeVersionOutput = None): raise KotlincNotFoundException() versionOutput = subprocess.run([kotlinc, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True).stderr if fakeVersionOutput is None else fakeVersionOutput m = re.match(r'.* kotlinc-jvm ([0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z][a-zA-Z0-9]*)?) .*', versionOutput) - if m is None: if m is None: raise Exception('Cannot detect version of kotlinc (got ' + str(versionOutput) + ')') current_version = version_string_to_version(m.group(1)) From 7aefd22e34ae75e121b8b8cb073d1ebc7b57452f Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 4 Apr 2024 17:32:23 +0200 Subject: [PATCH 10/81] Kotlin: tweak `BUILD.bazel` file, add documentation --- java/kotlin-extractor/BUILD.bazel | 30 ++++++++++++++++++++++++++++-- java/kotlin-extractor/versions.bzl | 10 ++++------ 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/java/kotlin-extractor/BUILD.bazel b/java/kotlin-extractor/BUILD.bazel index fdcb9b56c582..4270770cc8c8 100644 --- a/java/kotlin-extractor/BUILD.bazel +++ b/java/kotlin-extractor/BUILD.bazel @@ -1,9 +1,35 @@ -# notice that this is used in the `@codeql_koblin_embeddable` external repo, which means we need to +""" +# Usage overview +Building the extractor can be done via +``` +bazel build //java/kotlin-extractor:codeql-extractor-kotlin-- +``` +where `` is either `standalone` or `embeddable`, and `` is one of the supported versions. + +For the moment both variants where tested by replacing them into `target/intree/codeql-java` and running one relevant integration test. + +``` +bazel build //java/kotlin-extractor +``` +will build a default variant: +* standalone, unless `CODEQL_KOTLIN_SINGLE_VERSION_EMBEDDABLE` is set to true, in which case it will go for embeddable +* the version will be taken as the last supported version less than the version of the currently installed `kotlinc` + * if `CODEQL_KOTLIN_SINGLE_VERSION` is set, that will be used instead + * if `kotlinc` is not installed, `1.9.20-Beta` will be used + +If `kotlinc` is updated, bazel won't be aware of it and will therefore keep the same default version. Possible workarounds for that: +* `bazel clean` +* `bazel fetch --force @codeql_kotlin_defaults\\:all` +* `CODEQL_KOTLIN_SINGLE_VERSION= bazel build //java/kotlin-extractor` +""" + +# notice that this file is used in the `@codeql_koblin_embeddable` external repo, which means we need to # reference explicitly @codeql load( "@codeql//java/kotlin-extractor:versions.bzl", "VERSIONS", "get_compatilibity_sources", + "get_language_version", "version_less", ) load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") @@ -46,7 +72,7 @@ _resources = [ name = "kotlinc-options-%s" % v, include_stdlibs = "none", jvm_target = "1.8", - language_version = v[:3], + language_version = get_language_version(v), warn = "error", x_optin = [ "kotlin.RequiresOptIn", diff --git a/java/kotlin-extractor/versions.bzl b/java/kotlin-extractor/versions.bzl index 186a9bc6c356..b7a11dfdd587 100644 --- a/java/kotlin-extractor/versions.bzl +++ b/java/kotlin-extractor/versions.bzl @@ -22,15 +22,13 @@ def _version_to_tuple(v): v = tuple([int(x) for x in v.split(".")]) return v + (tail,) -def _tuple_to_version(t): - ret = ".".join([str(x) for x in t[:3]]) - if t[3]: - ret += "-" + t[3] - return ret - def version_less(lhs, rhs): return _version_to_tuple(lhs) < _version_to_tuple(rhs) +def get_language_version(version): + major, minor, _, _ = _version_to_tuple(version) + return "%s.%s" % (major, minor) + def _basename(path): if "/" not in path: return path From 38a7bc0580b20a5372266266b9b691f0a6b8804e Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 8 Apr 2024 08:38:43 +0200 Subject: [PATCH 11/81] Bazel: optimize LFS to use symlinks when the file is not an LFS pointer --- misc/bazel/internal/git_lfs_smudge.py | 15 +++++++++++---- misc/bazel/lfs.bzl | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/misc/bazel/internal/git_lfs_smudge.py b/misc/bazel/internal/git_lfs_smudge.py index 397975473c35..83edcd37ddfd 100755 --- a/misc/bazel/internal/git_lfs_smudge.py +++ b/misc/bazel/internal/git_lfs_smudge.py @@ -8,9 +8,16 @@ sources = [pathlib.Path(arg).resolve() for arg in sys.argv[1:]] source_dir = pathlib.Path(os.path.commonpath(src.parent for src in sources)) +def is_lfs_pointer(fileobj): + lfs_header = "version https://git-lfs.github.com/spec".encode() + actual_header = fileobj.read(len(lfs_header)) + fileobj.seek(0) + return lfs_header == actual_header + for src in sources: with open(src, 'rb') as input: - lfs_pointer = subprocess.run(["git", "lfs", "clean", "--", src], - stdin=input, stdout=subprocess.PIPE, check=True, cwd=source_dir).stdout - with open(src.name, 'wb') as output: - subprocess.run(["git", "lfs", "smudge", "--", src], input=lfs_pointer, stdout=output, check=True, cwd=source_dir) + if is_lfs_pointer(input): + with open(src.name, 'wb') as output: + subprocess.run(["git", "lfs", "smudge", "--", src], stdin=input, stdout=output, check=True, cwd=source_dir) + continue + pathlib.Path(src.name).symlink_to(src) diff --git a/misc/bazel/lfs.bzl b/misc/bazel/lfs.bzl index b58831f72eaa..ab40e6ac3d07 100644 --- a/misc/bazel/lfs.bzl +++ b/misc/bazel/lfs.bzl @@ -7,7 +7,7 @@ def lfs_smudge(repository_ctx, srcs): fail("Neither python3 nor python executables found") res = repository_ctx.execute([python, script] + srcs, quiet = False) if res.return_code != 0: - fail("git LFS smudging failed while instantiating @%s" % repository_ctx.name) + fail("git LFS smudging failed while instantiating @%s:\n%s" % (repository_ctx.name, res.stderr)) def _download_and_extract_lfs(repository_ctx): attr = repository_ctx.attr From fd77f1a7cb77ace0e0a12d0249d91fffa1f4e69c Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 8 Apr 2024 10:01:32 +0200 Subject: [PATCH 12/81] Bazel: fix lfs to do checkout rather than smudging On Windows `git lfs smudge` was not working as expected. --- misc/bazel/internal/git_lfs_checkout.py | 25 +++++++++++++++++++++++++ misc/bazel/internal/git_lfs_smudge.py | 23 ----------------------- misc/bazel/lfs.bzl | 4 ++-- 3 files changed, 27 insertions(+), 25 deletions(-) create mode 100755 misc/bazel/internal/git_lfs_checkout.py delete mode 100755 misc/bazel/internal/git_lfs_smudge.py diff --git a/misc/bazel/internal/git_lfs_checkout.py b/misc/bazel/internal/git_lfs_checkout.py new file mode 100755 index 000000000000..689b2fcf096d --- /dev/null +++ b/misc/bazel/internal/git_lfs_checkout.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 + +import sys +import pathlib +import subprocess +import os + +sources = [pathlib.Path(arg).resolve() for arg in sys.argv[1:]] +source_dir = pathlib.Path(os.path.commonpath(src.parent for src in sources)) +source_dir = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], cwd=source_dir, text=True).strip() + + +def is_lfs_pointer(file): + lfs_header = "version https://git-lfs.github.com/spec".encode() + with open(file, 'rb') as fileobj: + return fileobj.read(len(lfs_header)) == lfs_header + + +for src in sources: + if is_lfs_pointer(src): + relative_src = src.relative_to(source_dir) + subprocess.run(["git", "lfs", "fetch", f"--include={relative_src}"], stdout=subprocess.DEVNULL, check=True, + cwd=source_dir) + subprocess.run(["git", "lfs", "checkout", relative_src], check=True, cwd=source_dir) + pathlib.Path(src.name).symlink_to(src) diff --git a/misc/bazel/internal/git_lfs_smudge.py b/misc/bazel/internal/git_lfs_smudge.py deleted file mode 100755 index 83edcd37ddfd..000000000000 --- a/misc/bazel/internal/git_lfs_smudge.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import pathlib -import subprocess -import os - -sources = [pathlib.Path(arg).resolve() for arg in sys.argv[1:]] -source_dir = pathlib.Path(os.path.commonpath(src.parent for src in sources)) - -def is_lfs_pointer(fileobj): - lfs_header = "version https://git-lfs.github.com/spec".encode() - actual_header = fileobj.read(len(lfs_header)) - fileobj.seek(0) - return lfs_header == actual_header - -for src in sources: - with open(src, 'rb') as input: - if is_lfs_pointer(input): - with open(src.name, 'wb') as output: - subprocess.run(["git", "lfs", "smudge", "--", src], stdin=input, stdout=output, check=True, cwd=source_dir) - continue - pathlib.Path(src.name).symlink_to(src) diff --git a/misc/bazel/lfs.bzl b/misc/bazel/lfs.bzl index ab40e6ac3d07..3674fbd2a443 100644 --- a/misc/bazel/lfs.bzl +++ b/misc/bazel/lfs.bzl @@ -1,13 +1,13 @@ def lfs_smudge(repository_ctx, srcs): for src in srcs: repository_ctx.watch(src) - script = Label("//misc/bazel/internal:git_lfs_smudge.py") + script = Label("//misc/bazel/internal:git_lfs_checkout.py") python = repository_ctx.which("python3") or repository_ctx.which("python") if not python: fail("Neither python3 nor python executables found") res = repository_ctx.execute([python, script] + srcs, quiet = False) if res.return_code != 0: - fail("git LFS smudging failed while instantiating @%s:\n%s" % (repository_ctx.name, res.stderr)) + fail("git LFS checkout failed while instantiating @%s:\n%s" % (repository_ctx.name, res.stderr)) def _download_and_extract_lfs(repository_ctx): attr = repository_ctx.attr From e963b84a5ab3ea4119e3859dc1faf961128937fe Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 8 Apr 2024 14:03:15 +0200 Subject: [PATCH 13/81] Kotlin: fix error in building extractor from internal repo --- java/kotlin-extractor/BUILD.bazel | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/java/kotlin-extractor/BUILD.bazel b/java/kotlin-extractor/BUILD.bazel index 4270770cc8c8..d40452e3fde6 100644 --- a/java/kotlin-extractor/BUILD.bazel +++ b/java/kotlin-extractor/BUILD.bazel @@ -113,12 +113,12 @@ _resources = [ ) + get_compatilibity_sources(v, "src/main/kotlin/utils/versions"), kotlinc_opts = ":kotlinc-options-%s" % v, module_name = "codeql-kotlin-extractor", - resource_strip_prefix = "../%s/%s" % ( - repo_name(), - v, - ) if _for_embeddable else "%s/%s" % ( - package_name(), - v, + # resource_strip_prefix is very nit-picky: the following makes it work from + # `codeql`, `@codeql_kotlin_embeddable` and `semmle-code` + resource_strip_prefix = ( + ("../%s/" % repo_name() if repo_name() else "") + + ("%s/" % package_name() if package_name() else "") + + v ), resources = [ ":resources-%s" % v, From a970c2d11c327f97eddb7ec62aabc0f243d9ba74 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 8 Apr 2024 14:37:53 +0200 Subject: [PATCH 14/81] Kotlin: move empty zip from git to internal bazel repo rule --- java/kotlin-extractor/deps.bzl | 6 ++++-- java/kotlin-extractor/deps/empty.zip | Bin 22 -> 0 bytes 2 files changed, 4 insertions(+), 2 deletions(-) delete mode 100755 java/kotlin-extractor/deps/empty.zip diff --git a/java/kotlin-extractor/deps.bzl b/java/kotlin-extractor/deps.bzl index 48aae95fd9ef..724f5a8a4e8d 100644 --- a/java/kotlin-extractor/deps.bzl +++ b/java/kotlin-extractor/deps.bzl @@ -11,6 +11,8 @@ kt_jvm_import( ) """ +_empty_zip = "PK\005\006\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" + def _get_dep(repository_ctx, name): return repository_ctx.path(Label("//java/kotlin-extractor/deps:%s" % name)) @@ -19,7 +21,7 @@ def _kotlin_dep_impl(repository_ctx): lfs_smudge(repository_ctx, [_get_dep(repository_ctx, name + ".jar")]) # for some reason rules_kotlin warns about these jars missing, this is to silence those warnings - empty = _get_dep(repository_ctx, "empty.zip") + repository_ctx.file("empty.zip", _empty_zip) for jar in ( "annotations-13.0.jar", "kotlin-stdlib.jar", @@ -27,7 +29,7 @@ def _kotlin_dep_impl(repository_ctx): "kotlin-script-runtime.jar", "trove4j.jar", ): - repository_ctx.symlink(empty, jar) + repository_ctx.symlink("empty.zip", jar) repository_ctx.file("BUILD.bazel", _kotlin_dep_build.format(name = name)) _kotlin_dep = repository_rule( diff --git a/java/kotlin-extractor/deps/empty.zip b/java/kotlin-extractor/deps/empty.zip deleted file mode 100755 index 15cb0ecb3e219d1701294bfdf0fe3f5cb5d208e7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 22 NcmWIWW@Tf*000g10H*)| From 60febcdf1ee2c348f5c472f2623e758ebf5c0579 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 8 Apr 2024 14:54:15 +0200 Subject: [PATCH 15/81] Revert "Bazel: fix lfs to do checkout rather than smudging" This reverts commit fd77f1a7cb77ace0e0a12d0249d91fffa1f4e69c. --- misc/bazel/internal/git_lfs_checkout.py | 25 ------------------------- misc/bazel/internal/git_lfs_smudge.py | 23 +++++++++++++++++++++++ misc/bazel/lfs.bzl | 4 ++-- 3 files changed, 25 insertions(+), 27 deletions(-) delete mode 100755 misc/bazel/internal/git_lfs_checkout.py create mode 100755 misc/bazel/internal/git_lfs_smudge.py diff --git a/misc/bazel/internal/git_lfs_checkout.py b/misc/bazel/internal/git_lfs_checkout.py deleted file mode 100755 index 689b2fcf096d..000000000000 --- a/misc/bazel/internal/git_lfs_checkout.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import pathlib -import subprocess -import os - -sources = [pathlib.Path(arg).resolve() for arg in sys.argv[1:]] -source_dir = pathlib.Path(os.path.commonpath(src.parent for src in sources)) -source_dir = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], cwd=source_dir, text=True).strip() - - -def is_lfs_pointer(file): - lfs_header = "version https://git-lfs.github.com/spec".encode() - with open(file, 'rb') as fileobj: - return fileobj.read(len(lfs_header)) == lfs_header - - -for src in sources: - if is_lfs_pointer(src): - relative_src = src.relative_to(source_dir) - subprocess.run(["git", "lfs", "fetch", f"--include={relative_src}"], stdout=subprocess.DEVNULL, check=True, - cwd=source_dir) - subprocess.run(["git", "lfs", "checkout", relative_src], check=True, cwd=source_dir) - pathlib.Path(src.name).symlink_to(src) diff --git a/misc/bazel/internal/git_lfs_smudge.py b/misc/bazel/internal/git_lfs_smudge.py new file mode 100755 index 000000000000..83edcd37ddfd --- /dev/null +++ b/misc/bazel/internal/git_lfs_smudge.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 + +import sys +import pathlib +import subprocess +import os + +sources = [pathlib.Path(arg).resolve() for arg in sys.argv[1:]] +source_dir = pathlib.Path(os.path.commonpath(src.parent for src in sources)) + +def is_lfs_pointer(fileobj): + lfs_header = "version https://git-lfs.github.com/spec".encode() + actual_header = fileobj.read(len(lfs_header)) + fileobj.seek(0) + return lfs_header == actual_header + +for src in sources: + with open(src, 'rb') as input: + if is_lfs_pointer(input): + with open(src.name, 'wb') as output: + subprocess.run(["git", "lfs", "smudge", "--", src], stdin=input, stdout=output, check=True, cwd=source_dir) + continue + pathlib.Path(src.name).symlink_to(src) diff --git a/misc/bazel/lfs.bzl b/misc/bazel/lfs.bzl index 3674fbd2a443..ab40e6ac3d07 100644 --- a/misc/bazel/lfs.bzl +++ b/misc/bazel/lfs.bzl @@ -1,13 +1,13 @@ def lfs_smudge(repository_ctx, srcs): for src in srcs: repository_ctx.watch(src) - script = Label("//misc/bazel/internal:git_lfs_checkout.py") + script = Label("//misc/bazel/internal:git_lfs_smudge.py") python = repository_ctx.which("python3") or repository_ctx.which("python") if not python: fail("Neither python3 nor python executables found") res = repository_ctx.execute([python, script] + srcs, quiet = False) if res.return_code != 0: - fail("git LFS checkout failed while instantiating @%s:\n%s" % (repository_ctx.name, res.stderr)) + fail("git LFS smudging failed while instantiating @%s:\n%s" % (repository_ctx.name, res.stderr)) def _download_and_extract_lfs(repository_ctx): attr = repository_ctx.attr From b71ffc658b665c1c5adeb9a4a6e3d9015598d39d Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 8 Apr 2024 14:58:56 +0200 Subject: [PATCH 16/81] Bazel: properly fix lfs smudge script --- .lfsconfig | 2 +- misc/bazel/internal/git_lfs_smudge.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.lfsconfig b/.lfsconfig index 87c9472c023e..76924691a3a8 100644 --- a/.lfsconfig +++ b/.lfsconfig @@ -1,2 +1,2 @@ [lfs] -fetchinclude = nothing +fetchinclude = /nothing diff --git a/misc/bazel/internal/git_lfs_smudge.py b/misc/bazel/internal/git_lfs_smudge.py index 83edcd37ddfd..e565854b63b7 100755 --- a/misc/bazel/internal/git_lfs_smudge.py +++ b/misc/bazel/internal/git_lfs_smudge.py @@ -7,6 +7,8 @@ sources = [pathlib.Path(arg).resolve() for arg in sys.argv[1:]] source_dir = pathlib.Path(os.path.commonpath(src.parent for src in sources)) +source_dir = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], cwd=source_dir, text=True).strip() + def is_lfs_pointer(fileobj): lfs_header = "version https://git-lfs.github.com/spec".encode() @@ -14,10 +16,14 @@ def is_lfs_pointer(fileobj): fileobj.seek(0) return lfs_header == actual_header + for src in sources: with open(src, 'rb') as input: if is_lfs_pointer(input): + lfs_pointer = input.read() + rel_src = src.relative_to(source_dir).as_posix() with open(src.name, 'wb') as output: - subprocess.run(["git", "lfs", "smudge", "--", src], stdin=input, stdout=output, check=True, cwd=source_dir) + subprocess.run(["git", "-c", f"lfs.fetchinclude={rel_src}", "lfs", "smudge", "--", rel_src], + input=lfs_pointer, stdout=output, check=True, cwd=source_dir) continue pathlib.Path(src.name).symlink_to(src) From 4a4bd16eabd8a28b33b7d40dc9a7ef4472e624ee Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 9 Apr 2024 13:27:27 +0200 Subject: [PATCH 17/81] Java/Kotlin: prepare for internal bazel packaging --- java/downgrades/BUILD.bazel | 12 ++++++++++++ java/kotlin-extractor/BUILD.bazel | 9 +++++++++ misc/bazel/internal/git_lfs_smudge.py | 7 +++++-- 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 java/downgrades/BUILD.bazel diff --git a/java/downgrades/BUILD.bazel b/java/downgrades/BUILD.bazel new file mode 100644 index 000000000000..e96f322ec898 --- /dev/null +++ b/java/downgrades/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") + +pkg_files( + name = "downgrades", + srcs = glob( + ["**"], + exclude = ["BUILD.bazel"], + ), + prefix = "downgrades", + strip_prefix = strip_prefix.from_pkg(), + visibility = ["//visibility:public"], +) diff --git a/java/kotlin-extractor/BUILD.bazel b/java/kotlin-extractor/BUILD.bazel index d40452e3fde6..7ad61640df48 100644 --- a/java/kotlin-extractor/BUILD.bazel +++ b/java/kotlin-extractor/BUILD.bazel @@ -178,4 +178,13 @@ _resources = [ actual = _common_extractor_name_prefix, visibility = ["//visibility:public"], ), + filegroup( + name = "many", + srcs = ["%s-%s-%s" % ( + _common_extractor_name_prefix, + variant, + version, + ) for variant in ("standalone", "embeddable") for version in VERSIONS], + visibility = ["//visibility:public"], + ), ) if not _for_embeddable else None diff --git a/misc/bazel/internal/git_lfs_smudge.py b/misc/bazel/internal/git_lfs_smudge.py index e565854b63b7..4e86cecc3abf 100755 --- a/misc/bazel/internal/git_lfs_smudge.py +++ b/misc/bazel/internal/git_lfs_smudge.py @@ -23,7 +23,10 @@ def is_lfs_pointer(fileobj): lfs_pointer = input.read() rel_src = src.relative_to(source_dir).as_posix() with open(src.name, 'wb') as output: - subprocess.run(["git", "-c", f"lfs.fetchinclude={rel_src}", "lfs", "smudge", "--", rel_src], - input=lfs_pointer, stdout=output, check=True, cwd=source_dir) + subprocess.run( + ["git", + "-c", f"lfs.fetchinclude={rel_src}", "-c", "lfs.fetchexclude=", + "lfs", "smudge", "--", rel_src], + input=lfs_pointer, stdout=output, check=True, cwd=source_dir) continue pathlib.Path(src.name).symlink_to(src) From 11729aaf6e980f9f935e02dafc7ed01cada90ced Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 9 Apr 2024 13:55:30 +0200 Subject: [PATCH 18/81] Kotlin: add licensing links about kotlin dependencies --- java/kotlin-extractor/deps/LICENSE.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 java/kotlin-extractor/deps/LICENSE.md diff --git a/java/kotlin-extractor/deps/LICENSE.md b/java/kotlin-extractor/deps/LICENSE.md new file mode 100644 index 000000000000..a93f66f0aa30 --- /dev/null +++ b/java/kotlin-extractor/deps/LICENSE.md @@ -0,0 +1,7 @@ +The Git LFS files contained in this directory are mirrored +from [org.jetbrains.kotlin packages in the Maven repository][1]. Please refer to [the kotlin Apache 2.0 license][2] for +details about their license. + +[1]: https://mvnrepository.com/artifact/org.jetbrains.kotlin + +[2]: https://github.com/JetBrains/kotlin/tree/master/license From 4b205ff96df83e053af6e7ce5705150893549482 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 9 Apr 2024 14:44:44 +0200 Subject: [PATCH 19/81] Apply suggestions from code review Co-authored-by: Cornelius Riemenschneider --- java/kotlin-extractor/BUILD.bazel | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/java/kotlin-extractor/BUILD.bazel b/java/kotlin-extractor/BUILD.bazel index 7ad61640df48..0a18f046ae66 100644 --- a/java/kotlin-extractor/BUILD.bazel +++ b/java/kotlin-extractor/BUILD.bazel @@ -6,7 +6,6 @@ bazel build //java/kotlin-extractor:codeql-extractor-kotlin-- ``` where `` is either `standalone` or `embeddable`, and `` is one of the supported versions. -For the moment both variants where tested by replacing them into `target/intree/codeql-java` and running one relevant integration test. ``` bazel build //java/kotlin-extractor @@ -23,7 +22,7 @@ If `kotlinc` is updated, bazel won't be aware of it and will therefore keep the * `CODEQL_KOTLIN_SINGLE_VERSION= bazel build //java/kotlin-extractor` """ -# notice that this file is used in the `@codeql_koblin_embeddable` external repo, which means we need to +# This file is used in the `@codeql_kotlin_embeddable` external repo, which means we need to # reference explicitly @codeql load( "@codeql//java/kotlin-extractor:versions.bzl", @@ -143,7 +142,7 @@ _resources = [ ] # default aliases, based on the kotlinc version installed on the host -# * default version can be overridden with env variableCODEQL_KOTLIN_SINGLE_VERSION +# * default version can be overridden with env variable CODEQL_KOTLIN_SINGLE_VERSION # * setting CODEQL_KOTLIN_SINGLE_VERSION_EMBEDDABLE=true overrides the default variant # * when a new kotlinc version is installed, you'll need to either `bazel clean` or # `bazel fetch --force @codeql_kotlin_defaults//:all` to refresh the default From 9c73a9a1db33dd84be1b2525095a729df4e4f0d9 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 9 Apr 2024 15:10:13 +0200 Subject: [PATCH 20/81] Bazel: move shared `bazelrc` settings to a `exported.bazelrc` file This will be `importe`d by the `semmle-code` `.bazelrc` file. --- .bazelrc | 4 +--- exported.bazelrc | 4 ++++ 2 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 exported.bazelrc diff --git a/.bazelrc b/.bazelrc index 6592f65f83a3..7c65160af7d8 100644 --- a/.bazelrc +++ b/.bazelrc @@ -14,7 +14,5 @@ build:linux --cxxopt=-std=c++20 build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64 build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor -# emitting jdeps does not work when building the 2.0.0+ kotlin extractor -build --@rules_kotlin//kotlin/settings:jvm_emit_jdeps=false - +import %workspace%/exported.bazelrc try-import %workspace%/local.bazelrc diff --git a/exported.bazelrc b/exported.bazelrc new file mode 100644 index 000000000000..900f660bba0b --- /dev/null +++ b/exported.bazelrc @@ -0,0 +1,4 @@ +# these settings are shared between `codeql` and `semmle-code` + +# emitting jdeps does not work when building the 2.0.0+ kotlin extractor +build --@rules_kotlin//kotlin/settings:jvm_emit_jdeps=false From 662fd5c04a76d1b1f0fb6b746a8ee1f73e9deab4 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 9 Apr 2024 15:32:46 +0200 Subject: [PATCH 21/81] Add explanatory comment to `.lfsconfig` --- .lfsconfig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.lfsconfig b/.lfsconfig index 76924691a3a8..cb0a8e352e86 100644 --- a/.lfsconfig +++ b/.lfsconfig @@ -1,2 +1,5 @@ [lfs] +# codeql is publicly forked by many users, and we don't want any LFS file polluting their working +# copies. We therefore exclude everything by default. +# For files required by bazel builds, use rules in `misc/bazel/lfs.bzl` to download them on demand. fetchinclude = /nothing From e7c680e9638aeee918896f6e9b2374c1ef1ece39 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 9 Apr 2024 16:33:17 +0200 Subject: [PATCH 22/81] Kotlin: reuse generated dbscheme in embeddable, and tweak comments --- java/kotlin-extractor/BUILD.bazel | 24 +++++++++--------------- java/kotlin-extractor/deps.bzl | 4 ---- 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/java/kotlin-extractor/BUILD.bazel b/java/kotlin-extractor/BUILD.bazel index 0a18f046ae66..8e9a138c91c8 100644 --- a/java/kotlin-extractor/BUILD.bazel +++ b/java/kotlin-extractor/BUILD.bazel @@ -6,7 +6,6 @@ bazel build //java/kotlin-extractor:codeql-extractor-kotlin-- ``` where `` is either `standalone` or `embeddable`, and `` is one of the supported versions. - ``` bazel build //java/kotlin-extractor ``` @@ -49,14 +48,6 @@ py_binary( srcs = ["generate_dbscheme.py"], ) -genrule( - name = "generated-dbscheme", - srcs = ["@codeql//java:dbscheme"], - outs = ["KotlinExtractorDbScheme.kt"], - cmd = "$(execpath :generate_dbscheme) $< $@", - tools = [":generate_dbscheme"], -) - _resources = [ ( r, @@ -102,7 +93,7 @@ _resources = [ kt_jvm_library( name = "%s-%s" % (_extractor_name_prefix, v), srcs = - [":generated-dbscheme"] + + ["@codeql//java/kotlin-extractor:generated-dbscheme"] + glob( [ "src/**/*.kt", @@ -141,12 +132,15 @@ _resources = [ for v in VERSIONS ] -# default aliases, based on the kotlinc version installed on the host -# * default version can be overridden with env variable CODEQL_KOTLIN_SINGLE_VERSION -# * setting CODEQL_KOTLIN_SINGLE_VERSION_EMBEDDABLE=true overrides the default variant -# * when a new kotlinc version is installed, you'll need to either `bazel clean` or -# `bazel fetch --force @codeql_kotlin_defaults//:all` to refresh the default ( + genrule( + name = "generated-dbscheme", + srcs = ["@codeql//java:dbscheme"], + outs = ["KotlinExtractorDbScheme.kt"], + cmd = "$(execpath :generate_dbscheme) $< $@", + tools = [":generate_dbscheme"], + visibility = ["@codeql_kotlin_embeddable//:__pkg__"], + ), alias( name = "%s-standalone" % _common_extractor_name_prefix, actual = "%s-standalone-%s" % ( diff --git a/java/kotlin-extractor/deps.bzl b/java/kotlin-extractor/deps.bzl index 724f5a8a4e8d..77378a412271 100644 --- a/java/kotlin-extractor/deps.bzl +++ b/java/kotlin-extractor/deps.bzl @@ -61,10 +61,6 @@ def _embeddable_source_impl(repository_ctx): "import org.jetbrains.kotlin.com.intellij", ) repository_ctx.file(str(src).replace(str(src_dir), "src"), contents) - repository_ctx.symlink( - Label("//java/kotlin-extractor:generate_dbscheme.py"), - "generate_dbscheme.py", - ) repository_ctx.symlink( Label("//java/kotlin-extractor:BUILD.bazel"), "BUILD.bazel", From 3bdab70451aeb459c4de7a644842cf307973f9c5 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 10 Apr 2024 10:18:10 +0200 Subject: [PATCH 23/81] Bazel: rename `exported.bazelrc` to `.bazelrc.exported` This makes the `.bazelrc` files be near each other in the directory listing. --- exported.bazelrc => .bazelrc.exported | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename exported.bazelrc => .bazelrc.exported (100%) diff --git a/exported.bazelrc b/.bazelrc.exported similarity index 100% rename from exported.bazelrc rename to .bazelrc.exported From aca8d047ca192a55eeb1751f448decfd816ffb1e Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 10 Apr 2024 10:24:15 +0200 Subject: [PATCH 24/81] Bazel: fix `.bazelrc` --- .bazelrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index 7c65160af7d8..2e4711826449 100644 --- a/.bazelrc +++ b/.bazelrc @@ -14,5 +14,5 @@ build:linux --cxxopt=-std=c++20 build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64 build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor -import %workspace%/exported.bazelrc +import %workspace%/.bazelrc.exported try-import %workspace%/local.bazelrc From c9565b365751e9e162fbca1a989ac6cc9d430e7d Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 10 Apr 2024 11:29:58 +0200 Subject: [PATCH 25/81] Bazel/Kotlin: fix version comparison logic, add default version printing --- java/kotlin-extractor/BUILD.bazel | 15 ++++++++++++--- java/kotlin-extractor/deps.bzl | 27 ++++++++++++++++----------- java/kotlin-extractor/versions.bzl | 10 +++++----- 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/java/kotlin-extractor/BUILD.bazel b/java/kotlin-extractor/BUILD.bazel index 8e9a138c91c8..3a1d563c004c 100644 --- a/java/kotlin-extractor/BUILD.bazel +++ b/java/kotlin-extractor/BUILD.bazel @@ -145,7 +145,7 @@ _resources = [ name = "%s-standalone" % _common_extractor_name_prefix, actual = "%s-standalone-%s" % ( _common_extractor_name_prefix, - kotlin_extractor_defaults.version, + kotlin_extractor_defaults.extractor_version, ), visibility = ["//visibility:public"], ), @@ -153,7 +153,7 @@ _resources = [ name = "%s-embeddable" % _common_extractor_name_prefix, actual = "%s-embeddable-%s" % ( _common_extractor_name_prefix, - kotlin_extractor_defaults.version, + kotlin_extractor_defaults.extractor_version, ), visibility = ["//visibility:public"], ), @@ -162,7 +162,7 @@ _resources = [ actual = "%s-%s-%s" % ( _common_extractor_name_prefix, kotlin_extractor_defaults.variant, - kotlin_extractor_defaults.version, + kotlin_extractor_defaults.extractor_version, ), visibility = ["//visibility:public"], ), @@ -180,4 +180,13 @@ _resources = [ ) for variant in ("standalone", "embeddable") for version in VERSIONS], visibility = ["//visibility:public"], ), + genrule( + name = "default-version-printer", + outs = ["print-default-version.sh"], + cmd = "echo echo %s > $@" % kotlin_extractor_defaults.version, + ), + sh_binary( + name = "print-default-version", + srcs = [":default-version-printer"], + ), ) if not _for_embeddable else None diff --git a/java/kotlin-extractor/deps.bzl b/java/kotlin-extractor/deps.bzl index 77378a412271..dd4aeee36c6b 100644 --- a/java/kotlin-extractor/deps.bzl +++ b/java/kotlin-extractor/deps.bzl @@ -1,4 +1,4 @@ -load("//java/kotlin-extractor:versions.bzl", "DEFAULT_FALLBACK_VERSION", "VERSIONS", "version_less") +load("//java/kotlin-extractor:versions.bzl", "DEFAULT_VERSION", "VERSIONS", "version_less") load("//misc/bazel:lfs.bzl", "lfs_smudge") _kotlin_dep_build = """ @@ -71,33 +71,38 @@ _embeddable_source = repository_rule(implementation = _embeddable_source_impl) def _get_default_version(repository_ctx): default_version = repository_ctx.getenv("CODEQL_KOTLIN_SINGLE_VERSION") if default_version: - if default_version not in VERSIONS: - fail("overriding CODEQL_KOTLIN_SINGLE_VERSION=%s not known, must be one of:\n %s" % - (default_version, " ".join(VERSIONS))) return default_version kotlinc = repository_ctx.which("kotlinc") if not kotlinc: - return DEFAULT_FALLBACK_VERSION + return DEFAULT_VERSION res = repository_ctx.execute([kotlinc, "-version"]) if not res: fail("kotlinc -version failed: %s" % res.stderr) out = res.stderr.split(" ") if len(out) < 3: fail("malformed kotlinc -version output: %s" % res.stdout) - host_version = out[2] - for version in reversed(VERSIONS): - if version_less(version, host_version) or version == host_version: - return version - fail("no relevant version found for host version %s among:\n %s" % (host_version, " ".join(VERSIONS))) + return out[2] + +def _get_available_version(version): + for available_version in reversed(VERSIONS): + if not version_less(version, available_version): + return available_version + fail("no available version found for version %s among:\n %s" % (version, " ".join(VERSIONS))) def _defaults_impl(repository_ctx): default_version = _get_default_version(repository_ctx) default_variant = "standalone" if repository_ctx.getenv("CODEQL_KOTLIN_SINGLE_VERSION_EMBEDDABLE") in ("true", "1"): default_variant = "embeddable" + available_version = _get_available_version(default_version) + info = struct( + version = default_version, + variant = default_variant, + extractor_version = available_version, + ) repository_ctx.file( "defaults.bzl", - "kotlin_extractor_defaults = struct(version = '%s', variant = '%s')\n" % (default_version, default_variant), + "kotlin_extractor_defaults = %s\n" % repr(info), ) repository_ctx.file("BUILD.bazel") diff --git a/java/kotlin-extractor/versions.bzl b/java/kotlin-extractor/versions.bzl index b7a11dfdd587..42f7bffba83d 100644 --- a/java/kotlin-extractor/versions.bzl +++ b/java/kotlin-extractor/versions.bzl @@ -15,18 +15,18 @@ VERSIONS = [ "2.0.255-SNAPSHOT", ] -DEFAULT_FALLBACK_VERSION = "1.9.0-Beta" +DEFAULT_VERSION = "1.9.0" def _version_to_tuple(v): - v, _, tail = v.partition("-") - v = tuple([int(x) for x in v.split(".")]) - return v + (tail,) + # we ignore the tag when comparing versions, for example 1.9.0-Beta <= 1.9.0 + v, _, ignored_tag = v.partition("-") + return tuple([int(x) for x in v.split(".")]) def version_less(lhs, rhs): return _version_to_tuple(lhs) < _version_to_tuple(rhs) def get_language_version(version): - major, minor, _, _ = _version_to_tuple(version) + major, minor, _ = _version_to_tuple(version) return "%s.%s" % (major, minor) def _basename(path): From 5bdd72422166e1b0659a89ada37c1709e6538db5 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 9 Apr 2024 18:03:41 +0200 Subject: [PATCH 26/81] Bazel: move patching of `rules_kotlin` to a registry override --- .bazelrc | 3 ++ MODULE.bazel | 9 +----- misc/bazel/override_registry/README.md | 8 +++++ .../override_registry/bazel_registry.json | 3 ++ .../rules_kotlin/1.9.4-patched/MODULE.bazel | 31 +++++++++++++++++++ .../patches/add_language_version_option.patch | 0 .../patches/module_dot_bazel_version.patch | 12 +++++++ .../rules_kotlin/1.9.4-patched/source.json | 9 ++++++ .../modules/rules_kotlin/metadata.json | 11 +++++++ 9 files changed, 78 insertions(+), 8 deletions(-) create mode 100644 misc/bazel/override_registry/README.md create mode 100644 misc/bazel/override_registry/bazel_registry.json create mode 100644 misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/MODULE.bazel rename java/kotlin-extractor/rules_kotlin.patch => misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/patches/add_language_version_option.patch (100%) create mode 100644 misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/patches/module_dot_bazel_version.patch create mode 100644 misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/source.json create mode 100644 misc/bazel/override_registry/modules/rules_kotlin/metadata.json diff --git a/.bazelrc b/.bazelrc index 2e4711826449..3b8604e4d9e0 100644 --- a/.bazelrc +++ b/.bazelrc @@ -14,5 +14,8 @@ build:linux --cxxopt=-std=c++20 build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64 build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor +common --registry=file://%workspace%/misc/bazel/override_registry +common --registry=https://bcr.bazel.build + import %workspace%/.bazelrc.exported try-import %workspace%/local.bazelrc diff --git a/MODULE.bazel b/MODULE.bazel index 0f6546b4e364..18f3810a9b04 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -21,14 +21,7 @@ bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "absl") bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json") bazel_dep(name = "fmt", version = "10.0.0") -bazel_dep(name = "rules_kotlin", version = "1.9.4") - -# we patch `rules_kotlin` to allow passing `-language-version` at a jvm_kt_library level -single_version_override( - module_name = "rules_kotlin", - patch_strip = 1, - patches = ["//java/kotlin-extractor:rules_kotlin.patch"], -) +bazel_dep(name = "rules_kotlin", version = "1.9.4-patched") pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") pip.parse( diff --git a/misc/bazel/override_registry/README.md b/misc/bazel/override_registry/README.md new file mode 100644 index 000000000000..1728ff88946c --- /dev/null +++ b/misc/bazel/override_registry/README.md @@ -0,0 +1,8 @@ +This was taken from https://github.com/bazelbuild/bazel-central-repository and readapted: +* The latest version was renamed with a `-patched` suffix +* The above rename was done also in the files: + * `modules/rules_kotlin/metadata.json` + * `module/rules_kotlin/1.9.4-patched/MODULE.bazel` +* `modules/rules_kotlin/1.9.4-patched/patches/add_language_version_option.patch` was added +* the above patch was added in `modules/rules_kotlin/1.9.4-patched/source.json`, with integrity SHAs generated via + `echo -n sha256-; cat | openssl dgst -sha256 -binary | base64`. diff --git a/misc/bazel/override_registry/bazel_registry.json b/misc/bazel/override_registry/bazel_registry.json new file mode 100644 index 000000000000..ea3f94f7a1e4 --- /dev/null +++ b/misc/bazel/override_registry/bazel_registry.json @@ -0,0 +1,3 @@ +{ + "mirrors": [] +} diff --git a/misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/MODULE.bazel b/misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/MODULE.bazel new file mode 100644 index 000000000000..a54dab518939 --- /dev/null +++ b/misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/MODULE.bazel @@ -0,0 +1,31 @@ +module( + name = "rules_kotlin", + version = "1.9.4-patched", + repo_name = "rules_kotlin", +) + +bazel_dep(name = "platforms", version = "0.0.6") +bazel_dep(name = "bazel_skylib", version = "1.4.2") +bazel_dep(name = "rules_java", version = "7.2.0") +bazel_dep(name = "rules_python", version = "0.23.1") +bazel_dep(name = "rules_cc", version = "0.0.8") + +rules_kotlin_extensions = use_extension( + "//src/main/starlark/core/repositories:bzlmod_setup.bzl", + "rules_kotlin_extensions", +) +use_repo( + rules_kotlin_extensions, + "com_github_google_ksp", + "com_github_jetbrains_kotlin", + "com_github_pinterest_ktlint", + "rules_android", +) + +register_toolchains("//kotlin/internal:default_toolchain") + +# TODO(bencodes) We should be able to remove this once rules_android has rolled out official Bzlmod support +remote_android_extensions = use_extension("@bazel_tools//tools/android:android_extensions.bzl", "remote_android_tools_extensions") +use_repo(remote_android_extensions, "android_gmaven_r8", "android_tools") + +bazel_dep(name = "rules_proto", version = "5.3.0-21.7") diff --git a/java/kotlin-extractor/rules_kotlin.patch b/misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/patches/add_language_version_option.patch similarity index 100% rename from java/kotlin-extractor/rules_kotlin.patch rename to misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/patches/add_language_version_option.patch diff --git a/misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/patches/module_dot_bazel_version.patch b/misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/patches/module_dot_bazel_version.patch new file mode 100644 index 000000000000..d5981f575cf9 --- /dev/null +++ b/misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/patches/module_dot_bazel_version.patch @@ -0,0 +1,12 @@ +=================================================================== +--- a/MODULE.bazel ++++ b/MODULE.bazel +@@ -1,7 +1,7 @@ + module( + name = "rules_kotlin", +- version = "1.9.0", ++ version = "1.9.4", + repo_name = "rules_kotlin", + ) + + bazel_dep(name = "platforms", version = "0.0.6") diff --git a/misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/source.json b/misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/source.json new file mode 100644 index 000000000000..bc0f0f2f4b09 --- /dev/null +++ b/misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/source.json @@ -0,0 +1,9 @@ +{ + "integrity": "sha256-dsD8wsI+33NjIK3tGs2d3guuQY5XMd8Skz2IbLqGt5U=", + "url": "https://github.com/bazelbuild/rules_kotlin/releases/download/v1.9.4/rules_kotlin-v1.9.4.tar.gz", + "patches": { + "module_dot_bazel_version.patch": "sha256-8Fu6NiXzckWm8oaVY/rPonGEqWG5ijV0r2GUf7ajKHM=", + "add_language_version_option.patch": "sha256-uF6LFpkqe9ye7+avviY1NE5ZhxJ3WMLenS+mbg4XFTM=" + }, + "patch_strip": 1 +} diff --git a/misc/bazel/override_registry/modules/rules_kotlin/metadata.json b/misc/bazel/override_registry/modules/rules_kotlin/metadata.json new file mode 100644 index 000000000000..14ddf135bece --- /dev/null +++ b/misc/bazel/override_registry/modules/rules_kotlin/metadata.json @@ -0,0 +1,11 @@ +{ + "homepage": "https://github.com/bazelbuild/rules_kotlin", + "maintainers": [], + "repository": [ + "github:bazelbuild/rules_kotlin" + ], + "versions": [ + "1.9.4-patched" + ], + "yanked_versions": {} +} From a15681a181bcebe81e90b8eb550d78bbe3f2b234 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 10 Apr 2024 10:12:29 +0200 Subject: [PATCH 27/81] Bazel: ease update of local registry --- .bazelrc | 2 +- misc/bazel/override_registry/README.md | 8 --- .../modules/rules_kotlin/metadata.json | 11 ---- misc/bazel/registry/README.md | 3 ++ .../bazel_registry.json | 0 misc/bazel/registry/fix.py | 53 +++++++++++++++++++ .../rules_kotlin/1.9.4-patched/MODULE.bazel | 0 .../codeql_add_language_version_option.patch} | 0 .../patches/module_dot_bazel_version.patch | 2 +- .../rules_kotlin/1.9.4-patched/source.json | 4 +- .../modules/rules_kotlin/metadata.json | 5 ++ 11 files changed, 65 insertions(+), 23 deletions(-) delete mode 100644 misc/bazel/override_registry/README.md delete mode 100644 misc/bazel/override_registry/modules/rules_kotlin/metadata.json create mode 100644 misc/bazel/registry/README.md rename misc/bazel/{override_registry => registry}/bazel_registry.json (100%) create mode 100755 misc/bazel/registry/fix.py rename misc/bazel/{override_registry => registry}/modules/rules_kotlin/1.9.4-patched/MODULE.bazel (100%) rename misc/bazel/{override_registry/modules/rules_kotlin/1.9.4-patched/patches/add_language_version_option.patch => registry/modules/rules_kotlin/1.9.4-patched/patches/codeql_add_language_version_option.patch} (100%) rename misc/bazel/{override_registry => registry}/modules/rules_kotlin/1.9.4-patched/patches/module_dot_bazel_version.patch (89%) rename misc/bazel/{override_registry => registry}/modules/rules_kotlin/1.9.4-patched/source.json (53%) create mode 100644 misc/bazel/registry/modules/rules_kotlin/metadata.json diff --git a/.bazelrc b/.bazelrc index 3b8604e4d9e0..eb3efb676fec 100644 --- a/.bazelrc +++ b/.bazelrc @@ -14,7 +14,7 @@ build:linux --cxxopt=-std=c++20 build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64 build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor -common --registry=file://%workspace%/misc/bazel/override_registry +common --registry=file://%workspace%/misc/bazel/registry common --registry=https://bcr.bazel.build import %workspace%/.bazelrc.exported diff --git a/misc/bazel/override_registry/README.md b/misc/bazel/override_registry/README.md deleted file mode 100644 index 1728ff88946c..000000000000 --- a/misc/bazel/override_registry/README.md +++ /dev/null @@ -1,8 +0,0 @@ -This was taken from https://github.com/bazelbuild/bazel-central-repository and readapted: -* The latest version was renamed with a `-patched` suffix -* The above rename was done also in the files: - * `modules/rules_kotlin/metadata.json` - * `module/rules_kotlin/1.9.4-patched/MODULE.bazel` -* `modules/rules_kotlin/1.9.4-patched/patches/add_language_version_option.patch` was added -* the above patch was added in `modules/rules_kotlin/1.9.4-patched/source.json`, with integrity SHAs generated via - `echo -n sha256-; cat | openssl dgst -sha256 -binary | base64`. diff --git a/misc/bazel/override_registry/modules/rules_kotlin/metadata.json b/misc/bazel/override_registry/modules/rules_kotlin/metadata.json deleted file mode 100644 index 14ddf135bece..000000000000 --- a/misc/bazel/override_registry/modules/rules_kotlin/metadata.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "homepage": "https://github.com/bazelbuild/rules_kotlin", - "maintainers": [], - "repository": [ - "github:bazelbuild/rules_kotlin" - ], - "versions": [ - "1.9.4-patched" - ], - "yanked_versions": {} -} diff --git a/misc/bazel/registry/README.md b/misc/bazel/registry/README.md new file mode 100644 index 000000000000..5d1723d0eacb --- /dev/null +++ b/misc/bazel/registry/README.md @@ -0,0 +1,3 @@ +Versions to be patched can be taken from https://github.com/bazelbuild/bazel-central-repository. After adding patches +inside `//patches`, and eventually renaming ``, run [`fix.py`](./fix.py) to align all metadata +to the renamed version and added patches. diff --git a/misc/bazel/override_registry/bazel_registry.json b/misc/bazel/registry/bazel_registry.json similarity index 100% rename from misc/bazel/override_registry/bazel_registry.json rename to misc/bazel/registry/bazel_registry.json diff --git a/misc/bazel/registry/fix.py b/misc/bazel/registry/fix.py new file mode 100755 index 000000000000..c904ebf49517 --- /dev/null +++ b/misc/bazel/registry/fix.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +""" +Fix metadata in overridden registry, updating `metadata.json` to list correct versions and `source.json` +to list correct patches with sha256 hashes. +""" + +import pathlib +import json +import base64 +import hashlib +import re + +this_dir = pathlib.Path(__file__).resolve().parent + + +def sha256(file): + with open(file, 'rb') as input: + hash = hashlib.sha256(input.read()).digest() + hash = base64.b64encode(hash).decode() + return f"sha256-{hash}" + + +def patch_file(file, f): + try: + data = file.read_text() + except FileNotFoundError: + data = None + file.write_text(f(data)) + + +def patch_json(file, **kwargs): + def update(data): + data = json.loads(data) if data else {} + data.update(kwargs) + return json.dumps(data, indent=4) + "\n" + + patch_file(file, update) + + +for entry in this_dir.joinpath("modules").iterdir(): + if not entry.is_dir(): + continue + versions = [e for e in entry.iterdir() if e.is_dir()] + + patch_json(entry / "metadata.json", versions=[v.name for v in versions]) + + for version in versions: + patch_json(version / "source.json", patches={ + p.name: sha256(p) for p in version.joinpath("patches").iterdir() + }) + patch_file(version / "MODULE.bazel", + lambda s: re.sub(r'''version\s*=\s*['"].*['"]''', f'version = "{version.name}"', s, 1)) diff --git a/misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/MODULE.bazel b/misc/bazel/registry/modules/rules_kotlin/1.9.4-patched/MODULE.bazel similarity index 100% rename from misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/MODULE.bazel rename to misc/bazel/registry/modules/rules_kotlin/1.9.4-patched/MODULE.bazel diff --git a/misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/patches/add_language_version_option.patch b/misc/bazel/registry/modules/rules_kotlin/1.9.4-patched/patches/codeql_add_language_version_option.patch similarity index 100% rename from misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/patches/add_language_version_option.patch rename to misc/bazel/registry/modules/rules_kotlin/1.9.4-patched/patches/codeql_add_language_version_option.patch diff --git a/misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/patches/module_dot_bazel_version.patch b/misc/bazel/registry/modules/rules_kotlin/1.9.4-patched/patches/module_dot_bazel_version.patch similarity index 89% rename from misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/patches/module_dot_bazel_version.patch rename to misc/bazel/registry/modules/rules_kotlin/1.9.4-patched/patches/module_dot_bazel_version.patch index d5981f575cf9..7a33385b1702 100644 --- a/misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/patches/module_dot_bazel_version.patch +++ b/misc/bazel/registry/modules/rules_kotlin/1.9.4-patched/patches/module_dot_bazel_version.patch @@ -5,7 +5,7 @@ module( name = "rules_kotlin", - version = "1.9.0", -+ version = "1.9.4", ++ version = "1.9.4-patched", repo_name = "rules_kotlin", ) diff --git a/misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/source.json b/misc/bazel/registry/modules/rules_kotlin/1.9.4-patched/source.json similarity index 53% rename from misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/source.json rename to misc/bazel/registry/modules/rules_kotlin/1.9.4-patched/source.json index bc0f0f2f4b09..8f652c667d11 100644 --- a/misc/bazel/override_registry/modules/rules_kotlin/1.9.4-patched/source.json +++ b/misc/bazel/registry/modules/rules_kotlin/1.9.4-patched/source.json @@ -2,8 +2,8 @@ "integrity": "sha256-dsD8wsI+33NjIK3tGs2d3guuQY5XMd8Skz2IbLqGt5U=", "url": "https://github.com/bazelbuild/rules_kotlin/releases/download/v1.9.4/rules_kotlin-v1.9.4.tar.gz", "patches": { - "module_dot_bazel_version.patch": "sha256-8Fu6NiXzckWm8oaVY/rPonGEqWG5ijV0r2GUf7ajKHM=", - "add_language_version_option.patch": "sha256-uF6LFpkqe9ye7+avviY1NE5ZhxJ3WMLenS+mbg4XFTM=" + "module_dot_bazel_version.patch": "sha256-0GnFHOv9wuuv3jFcHBSXrdo7JMFP7y66O5C4rccy5wg=", + "codeql_add_language_version_option.patch": "sha256-uF6LFpkqe9ye7+avviY1NE5ZhxJ3WMLenS+mbg4XFTM=" }, "patch_strip": 1 } diff --git a/misc/bazel/registry/modules/rules_kotlin/metadata.json b/misc/bazel/registry/modules/rules_kotlin/metadata.json new file mode 100644 index 000000000000..a271e0425881 --- /dev/null +++ b/misc/bazel/registry/modules/rules_kotlin/metadata.json @@ -0,0 +1,5 @@ +{ + "versions": [ + "1.9.4-patched" + ] +} From 35a2ed87b6e0922ec899f4844fd9b155cf7f9a1c Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 10 Apr 2024 14:57:02 +0200 Subject: [PATCH 28/81] Bazel/Kotlin: patch jdep emission --- .bazelrc | 1 - .bazelrc.exported | 4 ---- MODULE.bazel | 2 +- .../{1.9.4-patched => 1.9.4-codeql.1}/MODULE.bazel | 2 +- .../codeql_add_language_version_option.patch | 2 ++ .../patches/codeql_do_not_emit_jdeps.patch | 14 ++++++++++++++ .../patches/module_dot_bazel_version.patch | 0 .../{1.9.4-patched => 1.9.4-codeql.1}/source.json | 3 ++- .../registry/modules/rules_kotlin/metadata.json | 2 +- 9 files changed, 21 insertions(+), 9 deletions(-) delete mode 100644 .bazelrc.exported rename misc/bazel/registry/modules/rules_kotlin/{1.9.4-patched => 1.9.4-codeql.1}/MODULE.bazel (97%) rename misc/bazel/registry/modules/rules_kotlin/{1.9.4-patched => 1.9.4-codeql.1}/patches/codeql_add_language_version_option.patch (88%) create mode 100644 misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/patches/codeql_do_not_emit_jdeps.patch rename misc/bazel/registry/modules/rules_kotlin/{1.9.4-patched => 1.9.4-codeql.1}/patches/module_dot_bazel_version.patch (100%) rename misc/bazel/registry/modules/rules_kotlin/{1.9.4-patched => 1.9.4-codeql.1}/source.json (61%) diff --git a/.bazelrc b/.bazelrc index eb3efb676fec..8cac362d7c19 100644 --- a/.bazelrc +++ b/.bazelrc @@ -17,5 +17,4 @@ build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor common --registry=file://%workspace%/misc/bazel/registry common --registry=https://bcr.bazel.build -import %workspace%/.bazelrc.exported try-import %workspace%/local.bazelrc diff --git a/.bazelrc.exported b/.bazelrc.exported deleted file mode 100644 index 900f660bba0b..000000000000 --- a/.bazelrc.exported +++ /dev/null @@ -1,4 +0,0 @@ -# these settings are shared between `codeql` and `semmle-code` - -# emitting jdeps does not work when building the 2.0.0+ kotlin extractor -build --@rules_kotlin//kotlin/settings:jvm_emit_jdeps=false diff --git a/MODULE.bazel b/MODULE.bazel index 18f3810a9b04..4872f5ae08f0 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -21,7 +21,7 @@ bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "absl") bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json") bazel_dep(name = "fmt", version = "10.0.0") -bazel_dep(name = "rules_kotlin", version = "1.9.4-patched") +bazel_dep(name = "rules_kotlin", version = "1.9.4-codeql.1") pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") pip.parse( diff --git a/misc/bazel/registry/modules/rules_kotlin/1.9.4-patched/MODULE.bazel b/misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/MODULE.bazel similarity index 97% rename from misc/bazel/registry/modules/rules_kotlin/1.9.4-patched/MODULE.bazel rename to misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/MODULE.bazel index a54dab518939..4982ad4b9a4d 100644 --- a/misc/bazel/registry/modules/rules_kotlin/1.9.4-patched/MODULE.bazel +++ b/misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/MODULE.bazel @@ -1,6 +1,6 @@ module( name = "rules_kotlin", - version = "1.9.4-patched", + version = "1.9.4-codeql.1", repo_name = "rules_kotlin", ) diff --git a/misc/bazel/registry/modules/rules_kotlin/1.9.4-patched/patches/codeql_add_language_version_option.patch b/misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/patches/codeql_add_language_version_option.patch similarity index 88% rename from misc/bazel/registry/modules/rules_kotlin/1.9.4-patched/patches/codeql_add_language_version_option.patch rename to misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/patches/codeql_add_language_version_option.patch index 3bc9ff4aac27..bb8fa5e4fcaf 100644 --- a/misc/bazel/registry/modules/rules_kotlin/1.9.4-patched/patches/codeql_add_language_version_option.patch +++ b/misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/patches/codeql_add_language_version_option.patch @@ -1,3 +1,5 @@ +We need to build different extractor variants with different -language-version options, which is not allowed +in current kotlin_rules diff --git a/src/main/starlark/core/options/opts.kotlinc.bzl b/src/main/starlark/core/options/opts.kotlinc.bzl index 9b15fb8..c0ac2cd 100644 --- a/src/main/starlark/core/options/opts.kotlinc.bzl diff --git a/misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/patches/codeql_do_not_emit_jdeps.patch b/misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/patches/codeql_do_not_emit_jdeps.patch new file mode 100644 index 000000000000..1a01bd2c91bc --- /dev/null +++ b/misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/patches/codeql_do_not_emit_jdeps.patch @@ -0,0 +1,14 @@ +Emitting jdeps is broken for the 2.0.0 kotlin extractor, and we don't need those files. +Patching it here rather than passing `--@rules_kotlin//kotlin/settings:jvm_emit_jdeps=false` +allows us to not have to specify that option (and therefore pull in `rules_kotlin`) in `semmle-code`. +--- a/kotlin/settings/BUILD.bazel 2000-01-01 01:00:00.000000000 +0100 ++++ b/kotlin/settings/BUILD.bazel 2024-04-10 14:51:16.060085986 +0200 +@@ -16,7 +16,7 @@ + # Flag that controls the emission of jdeps files during kotlin jvm compilation. + bool_flag( + name = "jvm_emit_jdeps", +- build_setting_default = True, # Upstream default behavior ++ build_setting_default = False, + visibility = ["//visibility:public"], + ) + diff --git a/misc/bazel/registry/modules/rules_kotlin/1.9.4-patched/patches/module_dot_bazel_version.patch b/misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/patches/module_dot_bazel_version.patch similarity index 100% rename from misc/bazel/registry/modules/rules_kotlin/1.9.4-patched/patches/module_dot_bazel_version.patch rename to misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/patches/module_dot_bazel_version.patch diff --git a/misc/bazel/registry/modules/rules_kotlin/1.9.4-patched/source.json b/misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/source.json similarity index 61% rename from misc/bazel/registry/modules/rules_kotlin/1.9.4-patched/source.json rename to misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/source.json index 8f652c667d11..5941e8379534 100644 --- a/misc/bazel/registry/modules/rules_kotlin/1.9.4-patched/source.json +++ b/misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/source.json @@ -3,7 +3,8 @@ "url": "https://github.com/bazelbuild/rules_kotlin/releases/download/v1.9.4/rules_kotlin-v1.9.4.tar.gz", "patches": { "module_dot_bazel_version.patch": "sha256-0GnFHOv9wuuv3jFcHBSXrdo7JMFP7y66O5C4rccy5wg=", - "codeql_add_language_version_option.patch": "sha256-uF6LFpkqe9ye7+avviY1NE5ZhxJ3WMLenS+mbg4XFTM=" + "codeql_do_not_emit_jdeps.patch": "sha256-x/HsujFlR1FGrgmbAbRZag9V4vKZZinBcs73tgRS478=", + "codeql_add_language_version_option.patch": "sha256-qFpP/hIvqGzjJi0h8LAQK0UuWqwlj/oCecZYGqlMVP8=" }, "patch_strip": 1 } diff --git a/misc/bazel/registry/modules/rules_kotlin/metadata.json b/misc/bazel/registry/modules/rules_kotlin/metadata.json index a271e0425881..d1270e0eeb1f 100644 --- a/misc/bazel/registry/modules/rules_kotlin/metadata.json +++ b/misc/bazel/registry/modules/rules_kotlin/metadata.json @@ -1,5 +1,5 @@ { "versions": [ - "1.9.4-patched" + "1.9.4-codeql.1" ] } From 5df1abcd267800170522ddd9bb7f24d09987d416 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 10 Apr 2024 15:41:27 +0200 Subject: [PATCH 29/81] Bazel/Kotlin: use `"all"` for extension metadata direct dependencies --- java/kotlin-extractor/deps.bzl | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/java/kotlin-extractor/deps.bzl b/java/kotlin-extractor/deps.bzl index dd4aeee36c6b..93884d66b6f6 100644 --- a/java/kotlin-extractor/deps.bzl +++ b/java/kotlin-extractor/deps.bzl @@ -108,19 +108,14 @@ def _defaults_impl(repository_ctx): _defaults = repository_rule(implementation = _defaults_impl) -def _add_rule(rules, rule, *, name, **kwargs): - rule(name = name, **kwargs) - rules.append(name) - def _kotlin_deps_impl(module_ctx): - deps = [] for v in VERSIONS: for lib in ("compiler", "compiler-embeddable", "stdlib"): - _add_rule(deps, _kotlin_dep, name = "kotlin-%s-%s" % (lib, v)) - _add_rule(deps, _embeddable_source, name = "codeql_kotlin_embeddable") - _add_rule(deps, _defaults, name = "codeql_kotlin_defaults") + _kotlin_dep(name = "kotlin-%s-%s" % (lib, v)) + _embeddable_source(name = "codeql_kotlin_embeddable") + _defaults(name = "codeql_kotlin_defaults") return module_ctx.extension_metadata( - root_module_direct_deps = deps, + root_module_direct_deps = "all", root_module_direct_dev_deps = [], ) From 02257eead0aa9568de5017735cc6b6b55a437604 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 11 Apr 2024 16:28:49 +0200 Subject: [PATCH 30/81] Bazel: use triple slash in `--registry` `file://%workspace%` works on POSIX systems, but not on Windows. `file:///` works on both. --- .bazelrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index 8cac362d7c19..0a49f682da37 100644 --- a/.bazelrc +++ b/.bazelrc @@ -14,7 +14,7 @@ build:linux --cxxopt=-std=c++20 build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64 build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor -common --registry=file://%workspace%/misc/bazel/registry +common --registry=file:///%workspace%/misc/bazel/registry common --registry=https://bcr.bazel.build try-import %workspace%/local.bazelrc From f0fc8110a0b60cb35922f1b1e8247898e88c04d5 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 11 Apr 2024 16:38:04 +0200 Subject: [PATCH 31/81] Bazel/Kotlin: add `2.0.0-RC1`, remove `2.0.0-Beta4` --- MODULE.bazel | 6 +++--- java/kotlin-extractor/deps/kotlin-compiler-2.0.0-Beta4.jar | 3 --- java/kotlin-extractor/deps/kotlin-compiler-2.0.0-RC1.jar | 3 +++ .../deps/kotlin-compiler-embeddable-2.0.0-Beta4.jar | 3 --- .../deps/kotlin-compiler-embeddable-2.0.0-RC1.jar | 3 +++ java/kotlin-extractor/deps/kotlin-stdlib-2.0.0-Beta4.jar | 3 --- java/kotlin-extractor/deps/kotlin-stdlib-2.0.0-RC1.jar | 3 +++ java/kotlin-extractor/versions.bzl | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-2.0.0-Beta4.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-2.0.0-RC1.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.0-Beta4.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.0-RC1.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-2.0.0-Beta4.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-2.0.0-RC1.jar diff --git a/MODULE.bazel b/MODULE.bazel index 33d984889007..2a0ff78306bf 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -69,7 +69,7 @@ use_repo( "kotlin-compiler-1.8.0", "kotlin-compiler-1.9.0-Beta", "kotlin-compiler-1.9.20-Beta", - "kotlin-compiler-2.0.0-Beta4", + "kotlin-compiler-2.0.0-RC1", "kotlin-compiler-2.0.255-SNAPSHOT", "kotlin-compiler-embeddable-1.5.0", "kotlin-compiler-embeddable-1.5.10", @@ -82,7 +82,7 @@ use_repo( "kotlin-compiler-embeddable-1.8.0", "kotlin-compiler-embeddable-1.9.0-Beta", "kotlin-compiler-embeddable-1.9.20-Beta", - "kotlin-compiler-embeddable-2.0.0-Beta4", + "kotlin-compiler-embeddable-2.0.0-RC1", "kotlin-compiler-embeddable-2.0.255-SNAPSHOT", "kotlin-stdlib-1.5.0", "kotlin-stdlib-1.5.10", @@ -95,7 +95,7 @@ use_repo( "kotlin-stdlib-1.8.0", "kotlin-stdlib-1.9.0-Beta", "kotlin-stdlib-1.9.20-Beta", - "kotlin-stdlib-2.0.0-Beta4", + "kotlin-stdlib-2.0.0-RC1", "kotlin-stdlib-2.0.255-SNAPSHOT", ) diff --git a/java/kotlin-extractor/deps/kotlin-compiler-2.0.0-Beta4.jar b/java/kotlin-extractor/deps/kotlin-compiler-2.0.0-Beta4.jar deleted file mode 100644 index cd550d37b971..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-2.0.0-Beta4.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8b702954abb1cd62e24f84d0e452262bd8bf8c21a1023e3f9b4bfa72d9870865 -size 63232450 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-2.0.0-RC1.jar b/java/kotlin-extractor/deps/kotlin-compiler-2.0.0-RC1.jar new file mode 100644 index 000000000000..fca4ae9fd36e --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-2.0.0-RC1.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60e31163aa7348166d708cdc9cb47d616b0222e5b11173f35f1adfb61bc3690a +size 58604870 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.0-Beta4.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.0-Beta4.jar deleted file mode 100644 index 430ba404b88c..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.0-Beta4.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:58688bfeddd277767a18bc80f6b2db25fe7b45f8d9a1eb3e20562c4f882ae6bc -size 61765489 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.0-RC1.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.0-RC1.jar new file mode 100644 index 000000000000..2f65ae3aaed0 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.0-RC1.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12d330c6e98ba42840f1d10e2752a9c53099d4dfa855bc4526d23fe1a1af9634 +size 57125659 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-2.0.0-Beta4.jar b/java/kotlin-extractor/deps/kotlin-stdlib-2.0.0-Beta4.jar deleted file mode 100644 index 50f32cb0f8d6..000000000000 --- a/java/kotlin-extractor/deps/kotlin-stdlib-2.0.0-Beta4.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:710ca6626ffa4f86af055aa4f3ebf63dd438a478c7cee43290e46506cecedf9a -size 1729213 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-2.0.0-RC1.jar b/java/kotlin-extractor/deps/kotlin-stdlib-2.0.0-RC1.jar new file mode 100644 index 000000000000..fbf0a8eff06f --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-2.0.0-RC1.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbb2c9b813e6196f9afa9f9add8b395ee384ab1763a0880e084d2942214f1c30 +size 1729731 diff --git a/java/kotlin-extractor/versions.bzl b/java/kotlin-extractor/versions.bzl index 42f7bffba83d..2d5e5ca7a9a3 100644 --- a/java/kotlin-extractor/versions.bzl +++ b/java/kotlin-extractor/versions.bzl @@ -11,7 +11,7 @@ VERSIONS = [ "1.8.0", "1.9.0-Beta", "1.9.20-Beta", - "2.0.0-Beta4", + "2.0.0-RC1", "2.0.255-SNAPSHOT", ] From 54156650d747a09b84a7247e0e89bec041f455b8 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 11 Apr 2024 16:40:37 +0200 Subject: [PATCH 32/81] Bazel/Kotlin: make `_embeddable_source` more robust --- java/kotlin-extractor/deps.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/java/kotlin-extractor/deps.bzl b/java/kotlin-extractor/deps.bzl index 93884d66b6f6..dae0b376002d 100644 --- a/java/kotlin-extractor/deps.bzl +++ b/java/kotlin-extractor/deps.bzl @@ -54,6 +54,7 @@ def _walk(dir): def _embeddable_source_impl(repository_ctx): src_dir = repository_ctx.path(Label("//java/kotlin-extractor:src")) + repository_ctx.watch_tree(src_dir) for src in _walk(src_dir): contents = repository_ctx.read(src) contents = contents.replace( From c18b556f3d38b1f25a6a1db9ab912f44137919fb Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 11 Apr 2024 16:50:09 +0200 Subject: [PATCH 33/81] Kotlin/Bazel: remove `2.0.255-SNAPSHOT` version --- MODULE.bazel | 3 --- .../kotlin-extractor/deps/kotlin-compiler-2.0.255-SNAPSHOT.jar | 3 --- .../deps/kotlin-compiler-embeddable-2.0.255-SNAPSHOT.jar | 3 --- java/kotlin-extractor/deps/kotlin-stdlib-2.0.255-SNAPSHOT.jar | 3 --- java/kotlin-extractor/versions.bzl | 1 - 5 files changed, 13 deletions(-) delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-2.0.255-SNAPSHOT.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.255-SNAPSHOT.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-2.0.255-SNAPSHOT.jar diff --git a/MODULE.bazel b/MODULE.bazel index 2a0ff78306bf..016aa4759899 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -70,7 +70,6 @@ use_repo( "kotlin-compiler-1.9.0-Beta", "kotlin-compiler-1.9.20-Beta", "kotlin-compiler-2.0.0-RC1", - "kotlin-compiler-2.0.255-SNAPSHOT", "kotlin-compiler-embeddable-1.5.0", "kotlin-compiler-embeddable-1.5.10", "kotlin-compiler-embeddable-1.5.20", @@ -83,7 +82,6 @@ use_repo( "kotlin-compiler-embeddable-1.9.0-Beta", "kotlin-compiler-embeddable-1.9.20-Beta", "kotlin-compiler-embeddable-2.0.0-RC1", - "kotlin-compiler-embeddable-2.0.255-SNAPSHOT", "kotlin-stdlib-1.5.0", "kotlin-stdlib-1.5.10", "kotlin-stdlib-1.5.20", @@ -96,7 +94,6 @@ use_repo( "kotlin-stdlib-1.9.0-Beta", "kotlin-stdlib-1.9.20-Beta", "kotlin-stdlib-2.0.0-RC1", - "kotlin-stdlib-2.0.255-SNAPSHOT", ) register_toolchains( diff --git a/java/kotlin-extractor/deps/kotlin-compiler-2.0.255-SNAPSHOT.jar b/java/kotlin-extractor/deps/kotlin-compiler-2.0.255-SNAPSHOT.jar deleted file mode 100644 index bb5d18b825f6..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-2.0.255-SNAPSHOT.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6b648fd72c8c9e18c822881a0085342e6a7734a500867f409b3f64365b358f01 -size 190091169 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.255-SNAPSHOT.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.255-SNAPSHOT.jar deleted file mode 100644 index 848cbbbf971a..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.255-SNAPSHOT.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:91e879b8f58b3fa28cdc6fe61c26bba1235502bff0a8417e492c3eb465d7d342 -size 188200654 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-2.0.255-SNAPSHOT.jar b/java/kotlin-extractor/deps/kotlin-stdlib-2.0.255-SNAPSHOT.jar deleted file mode 100644 index fb6056d106f2..000000000000 --- a/java/kotlin-extractor/deps/kotlin-stdlib-2.0.255-SNAPSHOT.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:eab3a26cc18a853f7a67d39b79ea43099dd0e69d60c00835b55b61ace791dc2d -size 4419987 diff --git a/java/kotlin-extractor/versions.bzl b/java/kotlin-extractor/versions.bzl index 2d5e5ca7a9a3..827ea9adf3b9 100644 --- a/java/kotlin-extractor/versions.bzl +++ b/java/kotlin-extractor/versions.bzl @@ -12,7 +12,6 @@ VERSIONS = [ "1.9.0-Beta", "1.9.20-Beta", "2.0.0-RC1", - "2.0.255-SNAPSHOT", ] DEFAULT_VERSION = "1.9.0" From 7f495b185112b4bb2a6f0f0d7841096fb303730c Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 11 Apr 2024 17:17:57 +0200 Subject: [PATCH 34/81] Bazel/kotlin: make version detection more robust --- java/kotlin-extractor/deps.bzl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/java/kotlin-extractor/deps.bzl b/java/kotlin-extractor/deps.bzl index dae0b376002d..1695bbeab0e5 100644 --- a/java/kotlin-extractor/deps.bzl +++ b/java/kotlin-extractor/deps.bzl @@ -80,9 +80,8 @@ def _get_default_version(repository_ctx): if not res: fail("kotlinc -version failed: %s" % res.stderr) out = res.stderr.split(" ") - if len(out) < 3: - fail("malformed kotlinc -version output: %s" % res.stdout) - return out[2] + kotlinc_jvm_index = out.index("kotlinc-jvm") + return out[kotlinc_jvm_index + 1] def _get_available_version(version): for available_version in reversed(VERSIONS): From 59fdbdbb20c644307820cd86f6304a7ec668a01f Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 11 Apr 2024 17:31:37 +0200 Subject: [PATCH 35/81] Kotlin/Bazel: move defaults to separate bazel package This makes calling specific targets in `java/kotlin-extractor` not depend on `@kotlin_extractor_defaults`, avoiding its overhead. --- java/kotlin-extractor/BUILD.bazel | 45 +++++++--------------- java/kotlin-extractor/defaults/BUILD.bazel | 36 +++++++++++++++++ 2 files changed, 50 insertions(+), 31 deletions(-) create mode 100644 java/kotlin-extractor/defaults/BUILD.bazel diff --git a/java/kotlin-extractor/BUILD.bazel b/java/kotlin-extractor/BUILD.bazel index 3a1d563c004c..49713b89633a 100644 --- a/java/kotlin-extractor/BUILD.bazel +++ b/java/kotlin-extractor/BUILD.bazel @@ -32,7 +32,8 @@ load( ) load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") load("@rules_kotlin//kotlin:core.bzl", "kt_kotlinc_options") -load("@codeql_kotlin_defaults//:defaults.bzl", "kotlin_extractor_defaults") + +package(default_visibility = ["//java/kotlin-extractor:__subpackages__"]) _for_embeddable = repo_name().endswith("codeql_kotlin_embeddable") @@ -141,31 +142,18 @@ _resources = [ tools = [":generate_dbscheme"], visibility = ["@codeql_kotlin_embeddable//:__pkg__"], ), - alias( - name = "%s-standalone" % _common_extractor_name_prefix, - actual = "%s-standalone-%s" % ( - _common_extractor_name_prefix, - kotlin_extractor_defaults.extractor_version, - ), - visibility = ["//visibility:public"], - ), - alias( - name = "%s-embeddable" % _common_extractor_name_prefix, - actual = "%s-embeddable-%s" % ( - _common_extractor_name_prefix, - kotlin_extractor_defaults.extractor_version, - ), - visibility = ["//visibility:public"], - ), - alias( - name = _common_extractor_name_prefix, - actual = "%s-%s-%s" % ( + [ + alias( + name = n, + actual = "//java/kotlin-extractor/defaults:%s" % n, + visibility = ["//visibility:public"], + ) + for n in ( + "%s-standalone" % _common_extractor_name_prefix, + "%s-embeddable" % _common_extractor_name_prefix, _common_extractor_name_prefix, - kotlin_extractor_defaults.variant, - kotlin_extractor_defaults.extractor_version, - ), - visibility = ["//visibility:public"], - ), + ) + ], alias( name = "kotlin-extractor", actual = _common_extractor_name_prefix, @@ -180,13 +168,8 @@ _resources = [ ) for variant in ("standalone", "embeddable") for version in VERSIONS], visibility = ["//visibility:public"], ), - genrule( - name = "default-version-printer", - outs = ["print-default-version.sh"], - cmd = "echo echo %s > $@" % kotlin_extractor_defaults.version, - ), sh_binary( name = "print-default-version", - srcs = [":default-version-printer"], + srcs = ["//java/kotlin-extractor/defaults:default-version-printer"], ), ) if not _for_embeddable else None diff --git a/java/kotlin-extractor/defaults/BUILD.bazel b/java/kotlin-extractor/defaults/BUILD.bazel new file mode 100644 index 000000000000..f68bcb912b66 --- /dev/null +++ b/java/kotlin-extractor/defaults/BUILD.bazel @@ -0,0 +1,36 @@ +load("@codeql_kotlin_defaults//:defaults.bzl", "kotlin_extractor_defaults") + +package(default_visibility = ["//java/kotlin-extractor:__pkg__"]) + +_common_extractor_name_prefix = "codeql-extractor-kotlin" + +alias( + name = "%s-standalone" % _common_extractor_name_prefix, + actual = "//java/kotlin-extractor:%s-standalone-%s" % ( + _common_extractor_name_prefix, + kotlin_extractor_defaults.extractor_version, + ), +) + +alias( + name = "%s-embeddable" % _common_extractor_name_prefix, + actual = "//java/kotlin-extractor:%s-embeddable-%s" % ( + _common_extractor_name_prefix, + kotlin_extractor_defaults.extractor_version, + ), +) + +alias( + name = _common_extractor_name_prefix, + actual = "//java/kotlin-extractor:%s-%s-%s" % ( + _common_extractor_name_prefix, + kotlin_extractor_defaults.variant, + kotlin_extractor_defaults.extractor_version, + ), +) + +genrule( + name = "default-version-printer", + outs = ["print-default-version.sh"], + cmd = "echo echo %s > $@" % kotlin_extractor_defaults.version, +) From 3d1465439addb4f1af77e7770847081c04e4e8be Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 11 Apr 2024 17:39:09 +0200 Subject: [PATCH 36/81] Bazel/Kotlin: tentatively fix version detection on Windows --- java/kotlin-extractor/deps.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/kotlin-extractor/deps.bzl b/java/kotlin-extractor/deps.bzl index 1695bbeab0e5..36293112df6f 100644 --- a/java/kotlin-extractor/deps.bzl +++ b/java/kotlin-extractor/deps.bzl @@ -79,7 +79,7 @@ def _get_default_version(repository_ctx): res = repository_ctx.execute([kotlinc, "-version"]) if not res: fail("kotlinc -version failed: %s" % res.stderr) - out = res.stderr.split(" ") + out = (res.stdout or res.stderr).split(" ") kotlinc_jvm_index = out.index("kotlinc-jvm") return out[kotlinc_jvm_index + 1] From fc62ed5af005e0079d75ef79b7a86d295a36c835 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 12 Apr 2024 08:18:53 +0200 Subject: [PATCH 37/81] Kotlin/Bazel: port `custom_plugin` test to use bazel --- .../kotlin/custom_plugin/build_plugin | 53 ------------------- .../kotlin/custom_plugin/plugin/BUILD.bazel | 26 +++++++++ .../linux-only/kotlin/custom_plugin/test.py | 23 +++++++- 3 files changed, 47 insertions(+), 55 deletions(-) delete mode 100755 java/ql/integration-tests/linux-only/kotlin/custom_plugin/build_plugin create mode 100644 java/ql/integration-tests/linux-only/kotlin/custom_plugin/plugin/BUILD.bazel diff --git a/java/ql/integration-tests/linux-only/kotlin/custom_plugin/build_plugin b/java/ql/integration-tests/linux-only/kotlin/custom_plugin/build_plugin deleted file mode 100755 index c38ab33eb75e..000000000000 --- a/java/ql/integration-tests/linux-only/kotlin/custom_plugin/build_plugin +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 - -import subprocess -import shutil -import os -import os.path -import sys -import shlex - - -def run_process(cmd): - try: - print("Running command: " + shlex.join(cmd)) - return subprocess.run(cmd, check=True, capture_output=True) - except subprocess.CalledProcessError as e: - print("In: " + os.getcwd(), file=sys.stderr) - print("Command failed: " + shlex.join(cmd), file=sys.stderr) - print("stdout output:\n" + e.stdout.decode(encoding='UTF-8', - errors='strict'), file=sys.stderr) - print("stderr output:\n" + e.stderr.decode(encoding='UTF-8', - errors='strict'), file=sys.stderr) - raise e - -root = '../../../../../../../../..' - -sys.path.append(root + '/ql/java/kotlin-extractor') -import kotlin_plugin_versions -defaultKotlinDependencyVersion = kotlin_plugin_versions.get_single_version() - -builddir = 'build' -dependency_dir = root + '/resources/kotlin-dependencies/' -dependencies = ['kotlin-stdlib-' + defaultKotlinDependencyVersion + - '.jar', 'kotlin-compiler-' + defaultKotlinDependencyVersion + '.jar'] -classpath = ':'.join([dependency_dir + dep for dep in dependencies]) -srcs = ['plugin/Plugin.kt'] -output = 'plugin.jar' - -if os.path.exists(builddir): - shutil.rmtree(builddir) -os.makedirs(builddir) - -run_process(['kotlinc', - '-J-Xmx2G', - '-d', builddir, - '-module-name', 'test', - '-no-reflect', '-no-stdlib', - '-jvm-target', '1.8', - '-classpath', classpath] + srcs) - -run_process(['jar', '-c', '-f', output, - '-C', builddir, '.', - '-C', 'plugin/resources', 'META-INF']) -shutil.rmtree(builddir) diff --git a/java/ql/integration-tests/linux-only/kotlin/custom_plugin/plugin/BUILD.bazel b/java/ql/integration-tests/linux-only/kotlin/custom_plugin/plugin/BUILD.bazel new file mode 100644 index 000000000000..3e5195df88c2 --- /dev/null +++ b/java/ql/integration-tests/linux-only/kotlin/custom_plugin/plugin/BUILD.bazel @@ -0,0 +1,26 @@ +load("@codeql_kotlin_defaults//:defaults.bzl", "kotlin_extractor_defaults") +load("@rules_kotlin//kotlin:core.bzl", "kt_kotlinc_options") +load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") +load("//java/kotlin-extractor:versions.bzl", "get_language_version") + +_version = kotlin_extractor_defaults.extractor_version + +kt_kotlinc_options( + name = "kotlinc-options", + include_stdlibs = "none", + jvm_target = "1.8", + language_version = get_language_version(_version), +) + +kt_jvm_library( + name = "plugin", + srcs = ["Plugin.kt"], + kotlinc_opts = ":kotlinc-options", + module_name = "test", + resource_strip_prefix = "%s/resources" % package_name(), + resources = glob(["resources/**"]), + deps = [ + "@kotlin-compiler-%s" % _version, + "@kotlin-stdlib-%s" % _version, + ], +) diff --git a/java/ql/integration-tests/linux-only/kotlin/custom_plugin/test.py b/java/ql/integration-tests/linux-only/kotlin/custom_plugin/test.py index 98610959b582..0928ff58234d 100644 --- a/java/ql/integration-tests/linux-only/kotlin/custom_plugin/test.py +++ b/java/ql/integration-tests/linux-only/kotlin/custom_plugin/test.py @@ -1,6 +1,25 @@ from create_database_utils import * -import subprocess -subprocess.call("./build_plugin", shell=True) +import pathlib +import shutil + +this_dir = pathlib.Path(__file__).resolve().parent +cwd = pathlib.Path.cwd() +builddir = cwd / 'build' + +builddir.mkdir(exist_ok=True) + +try: + runSuccessfully(['bazel', f'--output_user_root={builddir}', '--max_idle_secs=1', 'build', + '//java/ql/integration-tests/linux-only/kotlin/custom_plugin/plugin', '--spawn_strategy=local', + '--nouse_action_cache', '--noremote_accept_cached', '--noremote_upload_local_results', + f'--symlink_prefix={cwd / "bazel-"}'], cwd=this_dir) +finally: + # rules_python creates a read-only directory in bazel's output, this allows cleanup to succeed + runSuccessfully(['chmod', '-R', '+w', builddir]) + +shutil.copy( + 'bazel-bin/java/ql/integration-tests/linux-only/kotlin/custom_plugin/plugin/plugin.jar', 'plugin.jar') + run_codeql_database_create( ["kotlinc -J-Xmx2G -Xplugin=plugin.jar a.kt b.kt c.kt d.kt e.kt"], lang="java") From 4822de3313bdb2f29b905a96fae3f7ee25217269 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 12 Apr 2024 10:19:10 +0200 Subject: [PATCH 38/81] Kotlin/Bazel: add fail printouts to `_default_version` --- java/kotlin-extractor/deps.bzl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/java/kotlin-extractor/deps.bzl b/java/kotlin-extractor/deps.bzl index 36293112df6f..b51e62c94542 100644 --- a/java/kotlin-extractor/deps.bzl +++ b/java/kotlin-extractor/deps.bzl @@ -80,7 +80,11 @@ def _get_default_version(repository_ctx): if not res: fail("kotlinc -version failed: %s" % res.stderr) out = (res.stdout or res.stderr).split(" ") + if "kotlinc-jvm" not in out: + fail("kotlinc -version output does not contain 'kotlinc-jvm': %s" % out) kotlinc_jvm_index = out.index("kotlinc-jvm") + if kotlinc_jvm_index + 1 >= len(out): + fail("kotlinc -version output does not contain a version after 'kotlinc-jvm': %s" % out) return out[kotlinc_jvm_index + 1] def _get_available_version(version): From a834101d7e6a39bf2357cff5b652bea058a8f923 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 12 Apr 2024 10:41:46 +0200 Subject: [PATCH 39/81] Use `fetchexclude=*` in `.lfsconfig` rather than `fetchinclude` --- .lfsconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.lfsconfig b/.lfsconfig index cb0a8e352e86..bad6617b880e 100644 --- a/.lfsconfig +++ b/.lfsconfig @@ -2,4 +2,4 @@ # codeql is publicly forked by many users, and we don't want any LFS file polluting their working # copies. We therefore exclude everything by default. # For files required by bazel builds, use rules in `misc/bazel/lfs.bzl` to download them on demand. -fetchinclude = /nothing +fetchexclude = * From c64d02d6df712507e7ec72011373fce4051abe15 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 12 Apr 2024 10:44:18 +0200 Subject: [PATCH 40/81] Undo `.gitattributes` formatting, move `LFS` entry --- .gitattributes | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index ec56b50dc944..a35508431033 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5,8 +5,10 @@ # git add --renormalize . # git status [just to show what files were renormalized] # git commit -m "Normalize line endings" + # Anything Git auto-detects as text gets normalized and checked out as LF * text=auto eol=lf + # Explicitly set a bunch of known extensions to text, in case auto detection gets confused. *.ql text *.qll text @@ -38,6 +40,7 @@ *.lua text *.expected text *.go text + # Explicitly set a bunch of known extensions to binary, because Git < 2.10 will treat # `* text=auto eol=lf` as `* text eol=lf` *.png -text @@ -46,8 +49,11 @@ *.gif -text *.dll -text *.pdb -text + java/ql/test/stubs/**/*.java linguist-generated=true java/ql/test/experimental/stubs/**/*.java linguist-generated=true +/java/kotlin-extractor/deps/*.jar filter=lfs diff=lfs merge=lfs -text + # Force git not to modify line endings for go or html files under the go/ql directory go/ql/**/*.go -text go/ql/**/*.html -text @@ -55,18 +61,21 @@ go/ql/**/*.html -text go/*.dbscheme -text # Preserve unusual line ending from codeql-go merge go/extractor/opencsv/CSVReader.java -text + # For some languages, upgrade script testing references really old dbscheme # files from legacy upgrades that have CRLF line endings. Since upgrade # resolution relies on object hashes, we must suppress line ending conversion # for those testing dbscheme files. */ql/lib/upgrades/initial/*.dbscheme -text + # Generated test files - these are synced from the standard JavaScript libraries using # `javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py`. javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge + # Auto-generated modeling for Python python/ql/lib/semmle/python/frameworks/data/internal/subclass-capture/*.yml linguist-generated=true + # auto-generated bazel lock file ruby/extractor/cargo-bazel-lock.json linguist-generated=true ruby/extractor/cargo-bazel-lock.json -merge -/java/kotlin-extractor/deps/*.jar filter=lfs diff=lfs merge=lfs -text From bc897429797dddd6ea775f78733a140384cd3d58 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 12 Apr 2024 10:50:02 +0200 Subject: [PATCH 41/81] Kotlin: fix space indentation in `OdasaOutput.java` --- .../semmle/extractor/java/OdasaOutput.java | 308 ++++++++++-------- 1 file changed, 181 insertions(+), 127 deletions(-) diff --git a/java/kotlin-extractor/src/main/java/com/semmle/extractor/java/OdasaOutput.java b/java/kotlin-extractor/src/main/java/com/semmle/extractor/java/OdasaOutput.java index 4bb30527fb84..97318bbd8bb8 100644 --- a/java/kotlin-extractor/src/main/java/com/semmle/extractor/java/OdasaOutput.java +++ b/java/kotlin-extractor/src/main/java/com/semmle/extractor/java/OdasaOutput.java @@ -67,7 +67,9 @@ public class OdasaOutput { private final Logger log; private final Compression compression; - /** DEBUG only: just use the given file as the root for TRAP, source archive etc */ + /** + * DEBUG only: just use the given file as the root for TRAP, source archive etc + */ OdasaOutput(File outputRoot, Compression compression, Logger log) { this.trapFolder = new File(outputRoot, "trap"); this.sourceArchiveFolder = new File(outputRoot, "src_archive"); @@ -77,14 +79,16 @@ public class OdasaOutput { } public OdasaOutput(boolean trackClassOrigins, Compression compression, Logger log) { - String trapFolderVar = Env.systemEnv().getFirstNonEmpty("CODEQL_EXTRACTOR_JAVA_TRAP_DIR", Var.TRAP_FOLDER.name()); + String trapFolderVar = Env.systemEnv().getFirstNonEmpty("CODEQL_EXTRACTOR_JAVA_TRAP_DIR", + Var.TRAP_FOLDER.name()); if (trapFolderVar == null) { throw new ResourceError("CODEQL_EXTRACTOR_JAVA_TRAP_DIR was not set"); } - String sourceArchiveVar = Env.systemEnv().getFirstNonEmpty("CODEQL_EXTRACTOR_JAVA_SOURCE_ARCHIVE_DIR", Var.SOURCE_ARCHIVE.name()); + String sourceArchiveVar = Env.systemEnv().getFirstNonEmpty("CODEQL_EXTRACTOR_JAVA_SOURCE_ARCHIVE_DIR", + Var.SOURCE_ARCHIVE.name()); if (sourceArchiveVar == null) { throw new ResourceError("CODEQL_EXTRACTOR_JAVA_SOURCE_ARCHIVE_DIR was not set"); - } + } this.trapFolder = new File(trapFolderVar); this.sourceArchiveFolder = new File(sourceArchiveVar); this.trackClassOrigins = trackClassOrigins; @@ -104,6 +108,7 @@ public boolean getTrackClassOrigins() { * Set the source file that is currently being processed. This may affect * things like trap and source archive directories, and persists as a * setting until this method is called again. + * * @param f the current source file */ public void setCurrentSourceFile(File f) { @@ -130,7 +135,7 @@ public void writeTrapSet() { private File trapSetFor(File file) { return FileUtil.appendAbsolutePath( - currentSpecFileEntry.getTrapFolder(), PathTransformer.std().fileAsDatabaseString(file) + ".set"); + currentSpecFileEntry.getTrapFolder(), PathTransformer.std().fileAsDatabaseString(file) + ".set"); } public void addDependency(IrDeclaration sym, String signature) { @@ -185,7 +190,8 @@ private File getTrapFileForJarFile(File jarFile) { return null; return FileUtil.appendAbsolutePath( currentSpecFileEntry.getTrapFolder(), - JARS_DIR + "/" + PathTransformer.std().fileAsDatabaseString(jarFile) + ".trap" + compression.getExtension()); + JARS_DIR + "/" + PathTransformer.std().fileAsDatabaseString(jarFile) + ".trap" + + compression.getExtension()); } private File getTrapFileForModule(String moduleName) { @@ -213,13 +219,13 @@ private File trapFileForDecl(IrElement sym, String signature) { private String trapFilePathForDecl(IrElement sym, String signature) { String binaryName = getIrElementBinaryName(sym); // TODO: Reinstate this? - //if (getTrackClassOrigins()) - // classId += "-" + StringDigestor.digest(sym.getSourceFileId()); + // if (getTrackClassOrigins()) + // classId += "-" + StringDigestor.digest(sym.getSourceFileId()); String result = CLASSES_DIR + "/" + - binaryName.replace('.', '/') + - signature + - ".members" + - ".trap" + compression.getExtension(); + binaryName.replace('.', '/') + + signature + + ".members" + + ".trap" + compression.getExtension(); return result; } @@ -229,16 +235,21 @@ private String trapFilePathForDecl(IrElement sym, String signature) { /** * Get a {@link TrapFileManager} to write members - * about a declaration, or null if the declaration shouldn't be populated. + * about a declaration, or null if the declaration shouldn't be + * populated. * * @param sym - * The declaration's symbol, including, in particular, its fully qualified - * binary class name. + * The declaration's symbol, including, in particular, its + * fully qualified + * binary class name. * @param signature - * Any unique suffix needed to distinguish `sym` from other declarations with the same name. - * For functions for example, this means its parameter signature. + * Any unique suffix needed to distinguish `sym` from other + * declarations with the same name. + * For functions for example, this means its parameter + * signature. */ - private TrapFileManager getMembersWriterForDecl(File trap, File trapFileBase, TrapClassVersion trapFileVersion, IrElement sym, String signature) { + private TrapFileManager getMembersWriterForDecl(File trap, File trapFileBase, TrapClassVersion trapFileVersion, + IrElement sym, String signature) { // If the TRAP file already exists then we // don't need to write it. if (trap.exists()) { @@ -250,7 +261,8 @@ private TrapFileManager getMembersWriterForDecl(File trap, File trapFileBase, Tr // don't need to rewrite it only to rename it // again. File trapFileDir = trap.getParentFile(); - File trapOld = new File(trapFileDir, trap.getName().replace(".trap" + compression.getExtension(), ".trap-old" + compression.getExtension())); + File trapOld = new File(trapFileDir, + trap.getName().replace(".trap" + compression.getExtension(), ".trap-old" + compression.getExtension())); if (trapOld.exists()) { log.trace("Not rewriting trap file for " + trap.toString() + " as the trap-old exists"); return null; @@ -261,11 +273,12 @@ private TrapFileManager getMembersWriterForDecl(File trap, File trapFileBase, Tr if (trapFileBase != null && trapFileVersion != null && trapFileDir.exists()) { String trapFileBaseName = trapFileBase.getName(); - for (File f: FileUtil.list(trapFileDir)) { + for (File f : FileUtil.list(trapFileDir)) { String name = f.getName(); Matcher m = selectClassVersionComponents.matcher(name); if (m.matches() && m.group(1).equals(trapFileBaseName)) { - TrapClassVersion v = new TrapClassVersion(Integer.valueOf(m.group(2)), Integer.valueOf(m.group(3)), Long.valueOf(m.group(4)), m.group(5)); + TrapClassVersion v = new TrapClassVersion(Integer.valueOf(m.group(2)), Integer.valueOf(m.group(3)), + Long.valueOf(m.group(4)), m.group(5)); if (v.newerThan(trapFileVersion)) { log.trace("Not rewriting trap file for " + trap.toString() + " as " + f.toString() + " exists"); return null; @@ -285,7 +298,8 @@ private TrapFileManager trapWriter(File trapFile, IrElement sym, String signatur return concurrentWriter(trapFile, relative, log, sym, signature); } - private TrapFileManager concurrentWriter(File trapFile, String relative, Logger log, IrElement sym, String signature) { + private TrapFileManager concurrentWriter(File trapFile, String relative, Logger log, IrElement sym, + String signature) { if (trapFile.exists()) return null; return new TrapFileManager(trapFile, relative, true, log, sym, signature); @@ -299,7 +313,8 @@ public class TrapFileManager implements AutoCloseable { private String signature; private boolean hasError = false; - private TrapFileManager(File trapFile, String relative, boolean concurrentCreation, Logger log, IrElement sym, String signature) { + private TrapFileManager(File trapFile, String relative, boolean concurrentCreation, Logger log, IrElement sym, + String signature) { trapDependenciesForClass = new TrapDependencies(relative); this.trapFile = trapFile; this.sym = sym; @@ -325,6 +340,7 @@ public void close() { writeTrapDependencies(trapDependenciesForClass); } + private void writeTrapDependencies(TrapDependencies trapDependencies) { String dep = trapDependencies.trapFile().replace(".trap" + compression.getExtension(), ".dep"); trapDependencies.save( @@ -340,56 +356,77 @@ public void setHasError() { * Trap file locking. */ - private final Pattern selectClassVersionComponents = Pattern.compile("(.*)#(-?[0-9]+)\\.(-?[0-9]+)-(-?[0-9]+)-(.*)\\.trap.*"); + private final Pattern selectClassVersionComponents = Pattern + .compile("(.*)#(-?[0-9]+)\\.(-?[0-9]+)-(-?[0-9]+)-(.*)\\.trap.*"); /** - * CAUTION: to avoid the potential for deadlock between multiple concurrent extractor processes, - * only one source file {@link TrapLocker} may be open at any time, and the lock must be obtained + * CAUTION: to avoid the potential for deadlock between multiple + * concurrent extractor processes, + * only one source file {@link TrapLocker} may be open at any time, and the lock + * must be obtained * before any class file lock. * - * Trap file extensions (and paths) ensure that source and class file locks are distinct. + * Trap file extensions (and paths) ensure that source and class file locks are + * distinct. * - * @return a {@link TrapLocker} for the currently processed source file, which must have been - * previously set by a call to {@link OdasaOutput#setCurrentSourceFile(File)}. + * @return a {@link TrapLocker} for the currently processed source file, which + * must have been + * previously set by a call to + * {@link OdasaOutput#setCurrentSourceFile(File)}. */ public TrapLocker getTrapLockerForCurrentSourceFile() { - return new TrapLocker((IrClass)null, null, true); + return new TrapLocker((IrClass) null, null, true); } /** - * CAUTION: to avoid the potential for deadlock between multiple concurrent extractor processes, - * only one jar file {@link TrapLocker} may be open at any time, and the lock must be obtained - * after any source file lock. Only one jar or class file lock may be open at any time. + * CAUTION: to avoid the potential for deadlock between multiple + * concurrent extractor processes, + * only one jar file {@link TrapLocker} may be open at any time, and the lock + * must be obtained + * after any source file lock. Only one jar or class file lock may + * be open at any time. * - * Trap file extensions (and paths) ensure that source and jar file locks are distinct. + * Trap file extensions (and paths) ensure that source and jar file locks are + * distinct. * - * @return a {@link TrapLocker} for the trap file corresponding to the given jar file. + * @return a {@link TrapLocker} for the trap file corresponding to the given jar + * file. */ public TrapLocker getTrapLockerForJarFile(File jarFile) { return new TrapLocker(jarFile); } /** - * CAUTION: to avoid the potential for deadlock between multiple concurrent extractor processes, - * only one module {@link TrapLocker} may be open at any time, and the lock must be obtained - * after any source file lock. Only one jar or class file or module lock may be open at any time. + * CAUTION: to avoid the potential for deadlock between multiple + * concurrent extractor processes, + * only one module {@link TrapLocker} may be open at any time, and the lock must + * be obtained + * after any source file lock. Only one jar or class file or + * module lock may be open at any time. * - * Trap file extensions (and paths) ensure that source and module file locks are distinct. + * Trap file extensions (and paths) ensure that source and module file locks are + * distinct. * - * @return a {@link TrapLocker} for the trap file corresponding to the given module. + * @return a {@link TrapLocker} for the trap file corresponding to the given + * module. */ public TrapLocker getTrapLockerForModule(String moduleName) { return new TrapLocker(moduleName); } /** - * CAUTION: to avoid the potential for deadlock between multiple concurrent extractor processes, - * only one class file {@link TrapLocker} may be open at any time, and the lock must be obtained - * after any source file lock. Only one jar or class file lock may be open at any time. + * CAUTION: to avoid the potential for deadlock between multiple + * concurrent extractor processes, + * only one class file {@link TrapLocker} may be open at any time, and the lock + * must be obtained + * after any source file lock. Only one jar or class file lock may + * be open at any time. * - * Trap file extensions (and paths) ensure that source and class file locks are distinct. + * Trap file extensions (and paths) ensure that source and class file locks are + * distinct. * - * @return a {@link TrapLocker} for the trap file corresponding to the given class symbol. + * @return a {@link TrapLocker} for the trap file corresponding to the given + * class symbol. */ public TrapLocker getTrapLockerForDecl(IrElement sym, String signature, boolean fromSource) { return new TrapLocker(sym, signature, fromSource); @@ -403,10 +440,11 @@ public class TrapLocker implements AutoCloseable { private File trapFileBase = null; private TrapClassVersion trapFileVersion = null; private final String signature; + private TrapLocker(IrElement decl, String signature, boolean fromSource) { this.sym = decl; this.signature = signature; - if (sym==null) { + if (sym == null) { log.error("Null symbol passed for Kotlin TRAP locker"); trapFile = null; } else { @@ -422,21 +460,25 @@ private TrapLocker(IrElement decl, String signature, boolean fromSource) { // in a single directory. This makes our directory listings later slow. // To avoid this, rather than using files named .../Foo*, we use .../Foo/Foo*. trapFileBase = new File(new File(normalTrapFile.getParentFile(), baseName), baseName); - trapFile = new File(trapFileBase.getPath() + '#' + trapFileVersion.toString() + ".trap" + compression.getExtension()); + trapFile = new File(trapFileBase.getPath() + '#' + trapFileVersion.toString() + ".trap" + + compression.getExtension()); } } + private TrapLocker(File jarFile) { sym = null; signature = null; trapFile = getTrapFileForJarFile(jarFile); } + private TrapLocker(String moduleName) { sym = null; signature = null; trapFile = getTrapFileForModule(moduleName); } + public TrapFileManager getTrapFileManager() { - if (trapFile!=null) { + if (trapFile != null) { return getMembersWriterForDecl(trapFile, trapFileBase, trapFileVersion, sym, signature); } else { return null; @@ -445,7 +487,7 @@ public TrapFileManager getTrapFileManager() { @Override public void close() { - if (trapFile!=null) { + if (trapFile != null) { // Now that we have finished writing our TRAP file, we want // to rename and TRAP file that matches our trapFileBase // but doesn't have the latest metadata. @@ -458,12 +500,13 @@ public void close() { String trapFileBaseName = trapFileBase.getName(); List> pairs = new LinkedList>(); - for (File f: FileUtil.list(trapFileDir)) { + for (File f : FileUtil.list(trapFileDir)) { String name = f.getName(); Matcher m = selectClassVersionComponents.matcher(name); if (m.matches()) { if (m.group(1).equals(trapFileBaseName)) { - TrapClassVersion v = new TrapClassVersion(Integer.valueOf(m.group(2)), Integer.valueOf(m.group(3)), Long.valueOf(m.group(4)), m.group(5)); + TrapClassVersion v = new TrapClassVersion(Integer.valueOf(m.group(2)), + Integer.valueOf(m.group(3)), Long.valueOf(m.group(4)), m.group(5)); pairs.add(new Pair(f, v)); } else { // Everything in this directory should be for the same TRAP file base @@ -490,10 +533,12 @@ public int compare(Pair p1, Pair }; TrapClassVersion latestVersion = Collections.max(pairs, comparator).snd(); - for (Pair p: pairs) { + for (Pair p : pairs) { if (!latestVersion.equals(p.snd())) { File f = p.fst(); - File fOld = new File(f.getParentFile(), f.getName().replace(".trap" + compression.getExtension(), ".trap-old" + compression.getExtension())); + File fOld = new File(f.getParentFile(), + f.getName().replace(".trap" + compression.getExtension(), + ".trap-old" + compression.getExtension())); // We aren't interested in whether or not this succeeds; // it may fail because a concurrent extractor has already // renamed it. @@ -528,7 +573,9 @@ public long getLastModified() { return lastModified; } - public String getExtractorName() { return extractorName; } + public String getExtractorName() { + return extractorName; + } private TrapClassVersion(int majorVersion, int minorVersion, long lastModified, String extractorName) { this.majorVersion = majorVersion; @@ -540,34 +587,37 @@ private TrapClassVersion(int majorVersion, int minorVersion, long lastModified, @Override public boolean equals(Object obj) { if (obj instanceof TrapClassVersion) { - TrapClassVersion other = (TrapClassVersion)obj; - return majorVersion == other.majorVersion && minorVersion == other.minorVersion && lastModified == other.lastModified && extractorName.equals(other.extractorName); + TrapClassVersion other = (TrapClassVersion) obj; + return majorVersion == other.majorVersion && minorVersion == other.minorVersion + && lastModified == other.lastModified && extractorName.equals(other.extractorName); } else { return false; } } - @Override - public int hashCode() { - int hash = 7; - hash = 31 * hash + majorVersion; - hash = 31 * hash + minorVersion; - hash = 31 * hash + (int)lastModified; - hash = 31 * hash + (extractorName == null ? 0 : extractorName.hashCode()); - return hash; - } + @Override + public int hashCode() { + int hash = 7; + hash = 31 * hash + majorVersion; + hash = 31 * hash + minorVersion; + hash = 31 * hash + (int) lastModified; + hash = 31 * hash + (extractorName == null ? 0 : extractorName.hashCode()); + return hash; + } private boolean newerThan(TrapClassVersion tcv) { - // Classes being compiled from source have major version 0 but should take precedence + // Classes being compiled from source have major version 0 but should take + // precedence // over any classes with the same qualified name loaded from the classpath // in previous or subsequent extractor invocations. if (tcv.majorVersion == 0 && majorVersion != 0) return false; else if (majorVersion == 0 && tcv.majorVersion != 0) return true; - // Always consider the Kotlin extractor superior to the Java extractor, because we may decode and extract + // Always consider the Kotlin extractor superior to the Java extractor, because + // we may decode and extract // Kotlin metadata that the Java extractor can't understand: - if(!Objects.equals(tcv.extractorName, extractorName)) { + if (!Objects.equals(tcv.extractorName, extractorName)) { if (Objects.equals(tcv.extractorName, "kotlin")) return false; if (Objects.equals(extractorName, "kotlin")) @@ -578,56 +628,57 @@ else if (majorVersion == 0 && tcv.majorVersion != 0) return tcv.majorVersion < majorVersion || (tcv.majorVersion == majorVersion && tcv.minorVersion < minorVersion) || (tcv.majorVersion == majorVersion && tcv.minorVersion == minorVersion && - tcv.lastModified < lastModified); - } - - private static Map> jarFileEntryTimeStamps = new HashMap<>(); - - private static Map getZipFileEntryTimeStamps(String path, Logger log) { - try { - Map result = new HashMap<>(); - ZipFile zf = new ZipFile(path); - Enumeration entries = zf.entries(); - while (entries.hasMoreElements()) { - ZipEntry ze = entries.nextElement(); - result.put(ze.getName(), ze.getLastModifiedTime().toMillis()); - } - return result; - } catch(IOException e) { - log.warn("Failed to get entry timestamps from " + path, e); - return null; - } - } - - private static long getVirtualFileTimeStamp(VirtualFile vf, Logger log) { - if (vf.getFileSystem().getProtocol().equals("jar")) { - String[] parts = vf.getPath().split("!/"); - if (parts.length == 2) { - String jarFilePath = parts[0]; - String entryPath = parts[1]; - if (!jarFileEntryTimeStamps.containsKey(jarFilePath)) { - jarFileEntryTimeStamps.put(jarFilePath, getZipFileEntryTimeStamps(jarFilePath, log)); - } - Map entryTimeStamps = jarFileEntryTimeStamps.get(jarFilePath); - if (entryTimeStamps != null) { - Long entryTimeStamp = entryTimeStamps.get(entryPath); - if (entryTimeStamp != null) - return entryTimeStamp; - else - log.warn("Couldn't find timestamp for jar file " + jarFilePath + " entry " + entryPath); - } - } else { - log.warn("Expected JAR-file path " + vf.getPath() + " to have exactly one '!/' separator"); - } - } - - // For all files except for jar files, and a fallback in case of I/O problems reading a jar file: - return vf.getTimeStamp(); - } + tcv.lastModified < lastModified); + } + + private static Map> jarFileEntryTimeStamps = new HashMap<>(); + + private static Map getZipFileEntryTimeStamps(String path, Logger log) { + try { + Map result = new HashMap<>(); + ZipFile zf = new ZipFile(path); + Enumeration entries = zf.entries(); + while (entries.hasMoreElements()) { + ZipEntry ze = entries.nextElement(); + result.put(ze.getName(), ze.getLastModifiedTime().toMillis()); + } + return result; + } catch (IOException e) { + log.warn("Failed to get entry timestamps from " + path, e); + return null; + } + } + + private static long getVirtualFileTimeStamp(VirtualFile vf, Logger log) { + if (vf.getFileSystem().getProtocol().equals("jar")) { + String[] parts = vf.getPath().split("!/"); + if (parts.length == 2) { + String jarFilePath = parts[0]; + String entryPath = parts[1]; + if (!jarFileEntryTimeStamps.containsKey(jarFilePath)) { + jarFileEntryTimeStamps.put(jarFilePath, getZipFileEntryTimeStamps(jarFilePath, log)); + } + Map entryTimeStamps = jarFileEntryTimeStamps.get(jarFilePath); + if (entryTimeStamps != null) { + Long entryTimeStamp = entryTimeStamps.get(entryPath); + if (entryTimeStamp != null) + return entryTimeStamp; + else + log.warn("Couldn't find timestamp for jar file " + jarFilePath + " entry " + entryPath); + } + } else { + log.warn("Expected JAR-file path " + vf.getPath() + " to have exactly one '!/' separator"); + } + } + + // For all files except for jar files, and a fallback in case of I/O problems + // reading a jar file: + return vf.getTimeStamp(); + } private static VirtualFile getVirtualFileIfClass(IrElement e) { if (e instanceof IrClass) - return getIrClassVirtualFile((IrClass)e); + return getIrClassVirtualFile((IrClass) e); else return null; } @@ -635,7 +686,7 @@ private static VirtualFile getVirtualFileIfClass(IrElement e) { private static TrapClassVersion fromSymbol(IrElement sym, Logger log) { VirtualFile vf = getVirtualFileIfClass(sym); if (vf == null && sym instanceof IrDeclaration) - vf = getVirtualFileIfClass(((IrDeclaration)sym).getParent()); + vf = getVirtualFileIfClass(((IrDeclaration) sym).getParent()); if (vf == null) return new TrapClassVersion(-1, 0, 0, null); @@ -646,12 +697,12 @@ private static TrapClassVersion fromSymbol(IrElement sym, Logger log) { // We want to use the latest one that there is. Field asmField = null; int asmNum = -1; - for(Field f : Opcodes.class.getDeclaredFields()) { + for (Field f : Opcodes.class.getDeclaredFields()) { String name = f.getName(); - if(name.startsWith("ASM")) { + if (name.startsWith("ASM")) { try { int i = Integer.parseInt(name.substring(3)); - if(i > asmNum) { + if (i > asmNum) { asmNum = i; asmField = f; } @@ -662,26 +713,29 @@ private static TrapClassVersion fromSymbol(IrElement sym, Logger log) { } int asm = asmField.getInt(null); ClassVisitor versionGetter = new ClassVisitor(asm) { - public void visit(int version, int access, java.lang.String name, java.lang.String signature, java.lang.String superName, java.lang.String[] interfaces) { + public void visit(int version, int access, java.lang.String name, java.lang.String signature, + java.lang.String superName, java.lang.String[] interfaces) { versionStore[0] = version; } }; - (new ClassReader(vf.contentsToByteArray())).accept(versionGetter, ClassReader.SKIP_CODE | ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES); + (new ClassReader(vf.contentsToByteArray())).accept(versionGetter, + ClassReader.SKIP_CODE | ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES); - return new TrapClassVersion(versionStore[0] & 0xffff, versionStore[0] >> 16, getVirtualFileTimeStamp(vf, log), "kotlin"); - } - catch(IllegalAccessException e) { + return new TrapClassVersion(versionStore[0] & 0xffff, versionStore[0] >> 16, + getVirtualFileTimeStamp(vf, log), "kotlin"); + } catch (IllegalAccessException e) { log.warn("Failed to read class file version information", e); return new TrapClassVersion(-1, 0, 0, null); - } - catch(IOException e) { + } catch (IOException e) { log.warn("Failed to read class file version information", e); return new TrapClassVersion(-1, 0, 0, null); } } + private boolean isValid() { - return majorVersion>=0 && minorVersion>=0; + return majorVersion >= 0 && minorVersion >= 0; } + @Override public String toString() { return majorVersion + "." + minorVersion + "-" + lastModified + "-" + extractorName; From 9c3a6157943eb414ef7818da698b0b2c2655867d Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 12 Apr 2024 10:59:17 +0200 Subject: [PATCH 42/81] CI: add non-blocking kotlin extractor build check This is to check that the build from `codeql` works. The "official" build will still be checked from the internal repo with QLucie, once we integrate the bazel build there. --- .github/workflows/kotlin-build.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/kotlin-build.yml diff --git a/.github/workflows/kotlin-build.yml b/.github/workflows/kotlin-build.yml new file mode 100644 index 000000000000..cb98949a1be5 --- /dev/null +++ b/.github/workflows/kotlin-build.yml @@ -0,0 +1,28 @@ +name: "Kotlin Build" + +on: + pull_request: + paths: + - "java/kotlin-extractor/**" + - "misc/bazel/**" + - "misc/codegen/**" + - "*.bazel*" + - .github/workflows/kotlin-build.yml + branches: + - main + - rc/* + - codeql-cli-* + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: | + bazel query //java/kotlin-extractor/... + # only build the default version as a sanity check that we can build from `codeql` + # the full official build will be checked by QLucie + bazel build //java/kotlin-extactor From 2d16192bab05c838860b79b39d651fa35e75870f Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 12 Apr 2024 11:19:17 +0200 Subject: [PATCH 43/81] CI: fix typo in `kotlin-build.yml` --- .github/workflows/kotlin-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/kotlin-build.yml b/.github/workflows/kotlin-build.yml index cb98949a1be5..0ecc63a50859 100644 --- a/.github/workflows/kotlin-build.yml +++ b/.github/workflows/kotlin-build.yml @@ -25,4 +25,4 @@ jobs: bazel query //java/kotlin-extractor/... # only build the default version as a sanity check that we can build from `codeql` # the full official build will be checked by QLucie - bazel build //java/kotlin-extactor + bazel build //java/kotlin-extractor From 911413112813aabdd58a16828d21fa65a61b28f0 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 12 Apr 2024 11:17:27 +0200 Subject: [PATCH 44/81] Bazel: add settings required by internal repo --- .bazelrc.internal | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .bazelrc.internal diff --git a/.bazelrc.internal b/.bazelrc.internal new file mode 100644 index 000000000000..cdffa9ccdea6 --- /dev/null +++ b/.bazelrc.internal @@ -0,0 +1,4 @@ +# this file should contain bazel settings required to build things from `semmle-code` + +common --registry=file:///%workspace%/ql/misc/bazel/registry +common --registry=https://bcr.bazel.build From 65df2bb16d3f9c3b0eb1ca460fc25c631567f511 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 12 Apr 2024 11:41:08 +0200 Subject: [PATCH 45/81] Bazel/Kotlin: add `.exe` to `kotlinc` on windows --- java/kotlin-extractor/deps.bzl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/java/kotlin-extractor/deps.bzl b/java/kotlin-extractor/deps.bzl index b51e62c94542..272764e2af4e 100644 --- a/java/kotlin-extractor/deps.bzl +++ b/java/kotlin-extractor/deps.bzl @@ -76,6 +76,9 @@ def _get_default_version(repository_ctx): kotlinc = repository_ctx.which("kotlinc") if not kotlinc: return DEFAULT_VERSION + if repository_ctx.os.name == "windows" and not kotlinc.basename.endswith(".exe"): + # for some reason the result from `which` can fail to have exe + kotlinc = repository_ctx.path("%s.exe" % kotlinc) res = repository_ctx.execute([kotlinc, "-version"]) if not res: fail("kotlinc -version failed: %s" % res.stderr) From 0ad8ed38221e2d4d8f0d9423b5896f202945431e Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 12 Apr 2024 12:15:26 +0200 Subject: [PATCH 46/81] Kotlin/Bazel: use `kotlinc.bat` instead of `kotlinc.exe` on Windows --- java/kotlin-extractor/deps.bzl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/java/kotlin-extractor/deps.bzl b/java/kotlin-extractor/deps.bzl index 272764e2af4e..198f9ecede73 100644 --- a/java/kotlin-extractor/deps.bzl +++ b/java/kotlin-extractor/deps.bzl @@ -76,9 +76,8 @@ def _get_default_version(repository_ctx): kotlinc = repository_ctx.which("kotlinc") if not kotlinc: return DEFAULT_VERSION - if repository_ctx.os.name == "windows" and not kotlinc.basename.endswith(".exe"): - # for some reason the result from `which` can fail to have exe - kotlinc = repository_ctx.path("%s.exe" % kotlinc) + if repository_ctx.os.name == "windows" and not kotlinc.basename.endswith(".bat"): + kotlinc = repository_ctx.path("%s.bat" % kotlinc) res = repository_ctx.execute([kotlinc, "-version"]) if not res: fail("kotlinc -version failed: %s" % res.stderr) From 24c7ad5d28ca6a919de9a56a1067c7451ac5c038 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 12 Apr 2024 14:00:46 +0200 Subject: [PATCH 47/81] Bazel/Kotlin: use plain `kotlinc` instead of result of `repository_ctx.which` --- java/kotlin-extractor/deps.bzl | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/java/kotlin-extractor/deps.bzl b/java/kotlin-extractor/deps.bzl index 198f9ecede73..07fd63f6dbce 100644 --- a/java/kotlin-extractor/deps.bzl +++ b/java/kotlin-extractor/deps.bzl @@ -73,12 +73,9 @@ def _get_default_version(repository_ctx): default_version = repository_ctx.getenv("CODEQL_KOTLIN_SINGLE_VERSION") if default_version: return default_version - kotlinc = repository_ctx.which("kotlinc") - if not kotlinc: + if not repository_ctx.which("kotlinc"): return DEFAULT_VERSION - if repository_ctx.os.name == "windows" and not kotlinc.basename.endswith(".bat"): - kotlinc = repository_ctx.path("%s.bat" % kotlinc) - res = repository_ctx.execute([kotlinc, "-version"]) + res = repository_ctx.execute(["kotlinc", "-version"]) if not res: fail("kotlinc -version failed: %s" % res.stderr) out = (res.stdout or res.stderr).split(" ") From a78124b2d5ce79176879788fb3546429567b2e19 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 12 Apr 2024 14:47:21 +0200 Subject: [PATCH 48/81] Bazel/Kotlin: use a wrapper to get the current kotlin version --- java/kotlin-extractor/current_kotlin_version.py | 13 +++++++++++++ java/kotlin-extractor/deps.bzl | 16 ++++++---------- 2 files changed, 19 insertions(+), 10 deletions(-) create mode 100644 java/kotlin-extractor/current_kotlin_version.py diff --git a/java/kotlin-extractor/current_kotlin_version.py b/java/kotlin-extractor/current_kotlin_version.py new file mode 100644 index 000000000000..51c5229029c7 --- /dev/null +++ b/java/kotlin-extractor/current_kotlin_version.py @@ -0,0 +1,13 @@ +import subprocess +import re +import shutil + +kotlinc = shutil.which('kotlinc') +if kotlinc is None: + raise Exception("kotlinc not found") +output = subprocess.run([kotlinc, "-version"], text=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + check=True).stderr +m = re.match(r'.* kotlinc-jvm ([0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z][a-zA-Z0-9]*)?) .*', output) +if m is None: + raise Exception(f'Cannot detect version of kotlinc (got {output})') +print(m[1]) diff --git a/java/kotlin-extractor/deps.bzl b/java/kotlin-extractor/deps.bzl index 07fd63f6dbce..4fb1d0b38af2 100644 --- a/java/kotlin-extractor/deps.bzl +++ b/java/kotlin-extractor/deps.bzl @@ -75,16 +75,12 @@ def _get_default_version(repository_ctx): return default_version if not repository_ctx.which("kotlinc"): return DEFAULT_VERSION - res = repository_ctx.execute(["kotlinc", "-version"]) - if not res: - fail("kotlinc -version failed: %s" % res.stderr) - out = (res.stdout or res.stderr).split(" ") - if "kotlinc-jvm" not in out: - fail("kotlinc -version output does not contain 'kotlinc-jvm': %s" % out) - kotlinc_jvm_index = out.index("kotlinc-jvm") - if kotlinc_jvm_index + 1 >= len(out): - fail("kotlinc -version output does not contain a version after 'kotlinc-jvm': %s" % out) - return out[kotlinc_jvm_index + 1] + kotlin_plugin_versions = repository_ctx.path(Label("//java/kotlin-extractor:current_kotlin_version.py")) + python = repository_ctx.which("python3") or repository_ctx.which("python") + res = repository_ctx.execute([python, kotlin_plugin_versions]) + if res.return_code != 0: + fail(res.stderr) + return res.stdout.strip() def _get_available_version(version): for available_version in reversed(VERSIONS): From b36cabb39632719b0769f46a08cb1ff09320c013 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 12 Apr 2024 16:04:15 +0200 Subject: [PATCH 49/81] Anchor `.gitattributes` entries at root where it makes sense --- .gitattributes | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.gitattributes b/.gitattributes index a35508431033..8ec5fa046599 100644 --- a/.gitattributes +++ b/.gitattributes @@ -50,32 +50,32 @@ *.dll -text *.pdb -text -java/ql/test/stubs/**/*.java linguist-generated=true -java/ql/test/experimental/stubs/**/*.java linguist-generated=true +/java/ql/test/stubs/**/*.java linguist-generated=true +/java/ql/test/experimental/stubs/**/*.java linguist-generated=true /java/kotlin-extractor/deps/*.jar filter=lfs diff=lfs merge=lfs -text # Force git not to modify line endings for go or html files under the go/ql directory -go/ql/**/*.go -text -go/ql/**/*.html -text +/go/ql/**/*.go -text +/go/ql/**/*.html -text # Force git not to modify line endings for go dbschemes -go/*.dbscheme -text +/go/*.dbscheme -text # Preserve unusual line ending from codeql-go merge -go/extractor/opencsv/CSVReader.java -text +/go/extractor/opencsv/CSVReader.java -text # For some languages, upgrade script testing references really old dbscheme # files from legacy upgrades that have CRLF line endings. Since upgrade # resolution relies on object hashes, we must suppress line ending conversion # for those testing dbscheme files. -*/ql/lib/upgrades/initial/*.dbscheme -text +/*/ql/lib/upgrades/initial/*.dbscheme -text # Generated test files - these are synced from the standard JavaScript libraries using # `javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py`. -javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge -javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge +/javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge +/javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge # Auto-generated modeling for Python -python/ql/lib/semmle/python/frameworks/data/internal/subclass-capture/*.yml linguist-generated=true +/python/ql/lib/semmle/python/frameworks/data/internal/subclass-capture/*.yml linguist-generated=true # auto-generated bazel lock file -ruby/extractor/cargo-bazel-lock.json linguist-generated=true -ruby/extractor/cargo-bazel-lock.json -merge +/ruby/extractor/cargo-bazel-lock.json linguist-generated=true +/ruby/extractor/cargo-bazel-lock.json -merge From bdc8a7f59d806382c7c8fdbceafbd76b19774938 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 15 Apr 2024 08:26:19 +0200 Subject: [PATCH 50/81] Revert "Use `fetchexclude=*` in `.lfsconfig` rather than `fetchinclude`" This reverts commit a834101d7e6a39bf2357cff5b652bea058a8f923. Using `fetchinclude` allows for easier local customization of what to fetch, because fetchexlude has precedence over fetchinclude. --- .lfsconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.lfsconfig b/.lfsconfig index bad6617b880e..cb0a8e352e86 100644 --- a/.lfsconfig +++ b/.lfsconfig @@ -2,4 +2,4 @@ # codeql is publicly forked by many users, and we don't want any LFS file polluting their working # copies. We therefore exclude everything by default. # For files required by bazel builds, use rules in `misc/bazel/lfs.bzl` to download them on demand. -fetchexclude = * +fetchinclude = /nothing From b07fa70133168666307419d44bc52348ca2f84d6 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 15 Apr 2024 10:48:01 +0200 Subject: [PATCH 51/81] Kotlin/Bazel: provide wrapper for managing versions of `kotlinc` By adding `java/kotlinc-extractor/deps/dev` to `PATH`, one gets a `kotlinc` wrapper that takes care of downloading and extracting the desired version of `kotlinc` on demand. The desired version can be selected with `kotlinc --select x.y.z`, or left to the current default of `1.9.0`. Moreover, this default version is integrated with the Bazel build, so that when using this wrapper, changes in the selected version will be picked up to define the default single version kotlin extractor build, without needing to do anything else (like `bazel fetch --force` or similar). Selected and installed version data is stored in `.gitignore`d files in the same directory, and can be cleared with `kotlinc --clear`. --- java/kotlin-extractor/deps.bzl | 19 ++- java/kotlin-extractor/deps/dev/.gitignore | 3 + java/kotlin-extractor/deps/dev/kotlinc | 156 +++++++++++++++++++++ java/kotlin-extractor/deps/dev/kotlinc.bat | 2 + java/kotlin-extractor/versions.bzl | 2 - 5 files changed, 176 insertions(+), 6 deletions(-) create mode 100644 java/kotlin-extractor/deps/dev/.gitignore create mode 100755 java/kotlin-extractor/deps/dev/kotlinc create mode 100644 java/kotlin-extractor/deps/dev/kotlinc.bat diff --git a/java/kotlin-extractor/deps.bzl b/java/kotlin-extractor/deps.bzl index 4fb1d0b38af2..8d737211967b 100644 --- a/java/kotlin-extractor/deps.bzl +++ b/java/kotlin-extractor/deps.bzl @@ -1,4 +1,4 @@ -load("//java/kotlin-extractor:versions.bzl", "DEFAULT_VERSION", "VERSIONS", "version_less") +load("//java/kotlin-extractor:versions.bzl", "VERSIONS", "version_less") load("//misc/bazel:lfs.bzl", "lfs_smudge") _kotlin_dep_build = """ @@ -73,11 +73,22 @@ def _get_default_version(repository_ctx): default_version = repository_ctx.getenv("CODEQL_KOTLIN_SINGLE_VERSION") if default_version: return default_version - if not repository_ctx.which("kotlinc"): - return DEFAULT_VERSION kotlin_plugin_versions = repository_ctx.path(Label("//java/kotlin-extractor:current_kotlin_version.py")) python = repository_ctx.which("python3") or repository_ctx.which("python") - res = repository_ctx.execute([python, kotlin_plugin_versions]) + env = {} + repository_ctx.watch(Label("//java/kotlin-extractor/deps:dev/.kotlinc_selected_version")) + if not repository_ctx.which("kotlinc"): + # take default from the kotlinc wrapper + path = repository_ctx.getenv("PATH") + path_to_add = repository_ctx.path(Label("//java/kotlin-extractor/deps:dev")) + if not path: + path = str(path_to_add) + elif repository_ctx.os.name == "windows": + path = "%s;%s" % (path, path_to_add) + else: + path = "%s:%s" % (path, path_to_add) + env["PATH"] = path + res = repository_ctx.execute([python, kotlin_plugin_versions], environment = env) if res.return_code != 0: fail(res.stderr) return res.stdout.strip() diff --git a/java/kotlin-extractor/deps/dev/.gitignore b/java/kotlin-extractor/deps/dev/.gitignore new file mode 100644 index 000000000000..a65f75fad6fb --- /dev/null +++ b/java/kotlin-extractor/deps/dev/.gitignore @@ -0,0 +1,3 @@ +/.kotlinc_installed +/.kotlinc_installed_version +/.kotlinc_selected_version diff --git a/java/kotlin-extractor/deps/dev/kotlinc b/java/kotlin-extractor/deps/dev/kotlinc new file mode 100755 index 000000000000..4f23c5e1d3da --- /dev/null +++ b/java/kotlin-extractor/deps/dev/kotlinc @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 + +""" +Wrapper script that manages kotlinc versions. +Usage: add this directory to your PATH, then +* `kotlinc --select x.y.z` will select the version for the next invocations, checking it actually exists +* `kotlinc --clear` will remove any state of the wrapper (deselecting a previous version selection) +* `kotlinc -version` will print the selected version information. It will not print `JRE` information as a normal + `kotlinc` invocation would do though. In exchange, the invocation incurs no overhead. +* Any other invocation will forward to the selected kotlinc version, downloading it if necessary. If no version was + previously selected with `--select`, a default will be used (see `DEFAULT_VERSION` below) + +In order to install kotlin, ripunzip will be used if installed, or if running on Windows within `semmle-code` (ripunzip +is available in `resources/lib/windows/ripunzip` then). +""" + +import pathlib +import os +import urllib +import urllib.request +import urllib.error +import argparse +import sys +import platform +import subprocess +import zipfile +import shutil +import io + +DEFAULT_VERSION = "1.9.0" + +def options(): + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--select") + parser.add_argument("--clear", action="store_true") + parser.add_argument("-version", action="store_true") + return parser.parse_known_args() + + +url_template = 'https://github.com/JetBrains/kotlin/releases/download/v{version}/kotlin-compiler-{version}.zip' +this_dir = pathlib.Path(__file__).resolve().parent +version_file = this_dir / ".kotlinc_selected_version" +installed_version_file = this_dir / ".kotlinc_installed_version" +install_dir = this_dir / ".kotlinc_installed" +windows_ripunzip = this_dir.parents[4] / "resources" / "lib" / "windows" / "ripunzip" / "ripunzip.exe" + + +class Error(Exception): + pass + + +class ZipFilePreservingPermissions(zipfile.ZipFile): + def _extract_member(self, member, targetpath, pwd): + if not isinstance(member, zipfile.ZipInfo): + member = self.getinfo(member) + + targetpath = super()._extract_member(member, targetpath, pwd) + + attr = member.external_attr >> 16 + if attr != 0: + os.chmod(targetpath, attr) + return targetpath + + +def check_version(version: str): + try: + with urllib.request.urlopen(url_template.format(version=version)) as response: + pass + except urllib.error.HTTPError as e: + if e.code == 404: + raise Error(f"Version {version} not found in github.com/JetBrains/kotlin/releases") from e + raise + +def get_version(file: pathlib.Path) -> str: + try: + return file.read_text() + except FileNotFoundError: + return None + + +def install(version: str): + url = url_template.format(version=version) + if install_dir.exists(): + shutil.rmtree(install_dir) + install_dir.mkdir() + ripunzip = shutil.which("ripunzip") + if platform.system() == "Windows" and windows_ripunzip.exists(): + ripunzip = windows_ripunzip + if ripunzip: + print(f"downloading and extracting {url} using ripunzip", file=sys.stderr) + subprocess.run([ripunzip, "unzip-uri", url], cwd=install_dir, check=True) + return + with io.BytesIO() as buffer: + with urllib.request.urlopen(url) as response: + print(f"downloading {url}", file=sys.stderr) + while True: + bytes = response.read() + if not bytes: + break + buffer.write(bytes) + buffer.seek(0) + print(f"extracting kotlin-compiler-{version}.zip", file=sys.stderr) + with ZipFilePreservingPermissions(buffer) as archive: + archive.extractall(install_dir) + + +def forward(forwarded_opts): + kotlinc = install_dir / "kotlinc" / "bin" / "kotlinc" + if platform.system() == "Windows": + kotlinc = kotlinc.with_suffix(".bat") + args = [sys.argv[0]] + args.extend(forwarded_opts) + os.execv(kotlinc, args) + +def clear(): + if install_dir.exists(): + print(f"removing {install_dir}", file=sys.stderr) + shutil.rmtree(install_dir) + if installed_version_file.exists(): + print(f"removing {installed_version_file}", file=sys.stderr) + installed_version_file.unlink() + if version_file.exists(): + print(f"removing {version_file}", file=sys.stderr) + version_file.unlink() + +def main(opts, forwarded_opts): + if opts.clear: + clear() + return + if opts.select: + check_version(opts.select) + version_file.write_text(opts.select) + selected_version = opts.select + else: + selected_version = get_version(version_file) + if not selected_version: + selected_version = DEFAULT_VERSION + version_file.write_text(selected_version) + if opts.version: + print(f"info: kotlinc-jvm {selected_version} (codeql dev wrapper)", file=sys.stderr) + return + if opts.select and not forwarded_opts: + print(f"selected {selected_version}", file=sys.stderr) + return + if get_version(installed_version_file) != selected_version: + install(selected_version) + installed_version_file.write_text(selected_version) + forward(forwarded_opts) + + +if __name__ == "__main__": + try: + main(*options()) + except Exception as e: + print(f"{e.__class__.__name__}: {e}", file=sys.stderr) + sys.exit(1) diff --git a/java/kotlin-extractor/deps/dev/kotlinc.bat b/java/kotlin-extractor/deps/dev/kotlinc.bat new file mode 100644 index 000000000000..3d75e9396107 --- /dev/null +++ b/java/kotlin-extractor/deps/dev/kotlinc.bat @@ -0,0 +1,2 @@ +@echo off +python3 %~dp0/kotlinc diff --git a/java/kotlin-extractor/versions.bzl b/java/kotlin-extractor/versions.bzl index 827ea9adf3b9..728672ead6de 100644 --- a/java/kotlin-extractor/versions.bzl +++ b/java/kotlin-extractor/versions.bzl @@ -14,8 +14,6 @@ VERSIONS = [ "2.0.0-RC1", ] -DEFAULT_VERSION = "1.9.0" - def _version_to_tuple(v): # we ignore the tag when comparing versions, for example 1.9.0-Beta <= 1.9.0 v, _, ignored_tag = v.partition("-") From e53ef4acd2f9f216c2266f32d8b25686cffc8ff2 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 15 Apr 2024 13:13:31 +0200 Subject: [PATCH 52/81] Kotlin/Bazel: fix wrapper on Windows --- java/kotlin-extractor/deps/dev/kotlinc | 11 ++++++++--- java/kotlin-extractor/deps/dev/kotlinc.bat | 4 +++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/java/kotlin-extractor/deps/dev/kotlinc b/java/kotlin-extractor/deps/dev/kotlinc index 4f23c5e1d3da..f74ed42c5095 100755 --- a/java/kotlin-extractor/deps/dev/kotlinc +++ b/java/kotlin-extractor/deps/dev/kotlinc @@ -15,7 +15,6 @@ is available in `resources/lib/windows/ripunzip` then). """ import pathlib -import os import urllib import urllib.request import urllib.error @@ -29,6 +28,7 @@ import io DEFAULT_VERSION = "1.9.0" + def options(): parser = argparse.ArgumentParser(add_help=False) parser.add_argument("--select") @@ -71,6 +71,7 @@ def check_version(version: str): raise Error(f"Version {version} not found in github.com/JetBrains/kotlin/releases") from e raise + def get_version(file: pathlib.Path) -> str: try: return file.read_text() @@ -108,9 +109,12 @@ def forward(forwarded_opts): kotlinc = install_dir / "kotlinc" / "bin" / "kotlinc" if platform.system() == "Windows": kotlinc = kotlinc.with_suffix(".bat") - args = [sys.argv[0]] + assert kotlinc.exists(), f"{kotlinc} not found" + args = [kotlinc] args.extend(forwarded_opts) - os.execv(kotlinc, args) + ret = subprocess.run(args).returncode + sys.exit(ret) + def clear(): if install_dir.exists(): @@ -123,6 +127,7 @@ def clear(): print(f"removing {version_file}", file=sys.stderr) version_file.unlink() + def main(opts, forwarded_opts): if opts.clear: clear() diff --git a/java/kotlin-extractor/deps/dev/kotlinc.bat b/java/kotlin-extractor/deps/dev/kotlinc.bat index 3d75e9396107..f6e59843f76f 100644 --- a/java/kotlin-extractor/deps/dev/kotlinc.bat +++ b/java/kotlin-extractor/deps/dev/kotlinc.bat @@ -1,2 +1,4 @@ @echo off -python3 %~dp0/kotlinc + +python "%~dp0/kotlinc" %* +exit /b %ERRORLEVEL% From 1b5675eb214a20d332e39d9f8d34b1655c0bd9b1 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 15 Apr 2024 13:31:29 +0200 Subject: [PATCH 53/81] Kotlin/Bazel: tweak wrapper --- java/kotlin-extractor/deps/dev/kotlinc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/java/kotlin-extractor/deps/dev/kotlinc b/java/kotlin-extractor/deps/dev/kotlinc index f74ed42c5095..509ed30cb7cb 100755 --- a/java/kotlin-extractor/deps/dev/kotlinc +++ b/java/kotlin-extractor/deps/dev/kotlinc @@ -25,6 +25,7 @@ import subprocess import zipfile import shutil import io +import os DEFAULT_VERSION = "1.9.0" @@ -85,15 +86,15 @@ def install(version: str): shutil.rmtree(install_dir) install_dir.mkdir() ripunzip = shutil.which("ripunzip") - if platform.system() == "Windows" and windows_ripunzip.exists(): + if ripunzip is None and platform.system() == "Windows" and windows_ripunzip.exists(): ripunzip = windows_ripunzip if ripunzip: print(f"downloading and extracting {url} using ripunzip", file=sys.stderr) subprocess.run([ripunzip, "unzip-uri", url], cwd=install_dir, check=True) return with io.BytesIO() as buffer: + print(f"downloading {url}", file=sys.stderr) with urllib.request.urlopen(url) as response: - print(f"downloading {url}", file=sys.stderr) while True: bytes = response.read() if not bytes: @@ -141,15 +142,12 @@ def main(opts, forwarded_opts): if not selected_version: selected_version = DEFAULT_VERSION version_file.write_text(selected_version) - if opts.version: - print(f"info: kotlinc-jvm {selected_version} (codeql dev wrapper)", file=sys.stderr) - return - if opts.select and not forwarded_opts: - print(f"selected {selected_version}", file=sys.stderr) - return if get_version(installed_version_file) != selected_version: install(selected_version) installed_version_file.write_text(selected_version) + if opts.version or (opts.select and not forwarded_opts): + print(f"info: kotlinc-jvm {selected_version} (codeql dev wrapper)", file=sys.stderr) + return forward(forwarded_opts) @@ -159,3 +157,5 @@ if __name__ == "__main__": except Exception as e: print(f"{e.__class__.__name__}: {e}", file=sys.stderr) sys.exit(1) + except KeyboardInterrupt: + sys.exit(1) From 8c705adbe99ec2a4ee86123a1fb701704fead3d4 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 15 Apr 2024 15:38:06 +0200 Subject: [PATCH 54/81] Kotlin/Bazel: fix java release to 8 for java code --- java/kotlin-extractor/BUILD.bazel | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/java/kotlin-extractor/BUILD.bazel b/java/kotlin-extractor/BUILD.bazel index 49713b89633a..b197bce07903 100644 --- a/java/kotlin-extractor/BUILD.bazel +++ b/java/kotlin-extractor/BUILD.bazel @@ -31,7 +31,7 @@ load( "version_less", ) load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") -load("@rules_kotlin//kotlin:core.bzl", "kt_kotlinc_options") +load("@rules_kotlin//kotlin:core.bzl", "kt_javac_options", "kt_kotlinc_options") package(default_visibility = ["//java/kotlin-extractor:__subpackages__"]) @@ -57,6 +57,11 @@ _resources = [ for r in glob(["src/main/resources/**"]) ] +kt_javac_options( + name = "javac-options", + release = "8", +) + [ ( kt_kotlinc_options( @@ -102,6 +107,7 @@ _resources = [ ], exclude = ["src/main/kotlin/utils/versions/**"], ) + get_compatilibity_sources(v, "src/main/kotlin/utils/versions"), + javac_opts = ":javac-options", kotlinc_opts = ":kotlinc-options-%s" % v, module_name = "codeql-kotlin-extractor", # resource_strip_prefix is very nit-picky: the following makes it work from From aee3c0d249f7c70183f9b043f342196cdaede124 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 16 Apr 2024 12:33:00 +0200 Subject: [PATCH 55/81] Add license and notice information to patched registry --- misc/bazel/registry/AUTHORS | 7 + misc/bazel/registry/LICENSE | 202 ++++++++++++++++++ misc/bazel/registry/NOTICE | 3 + misc/bazel/registry/fix.py | 2 + .../modules/rules_kotlin/metadata.json | 24 ++- 5 files changed, 237 insertions(+), 1 deletion(-) create mode 100644 misc/bazel/registry/AUTHORS create mode 100644 misc/bazel/registry/LICENSE create mode 100644 misc/bazel/registry/NOTICE diff --git a/misc/bazel/registry/AUTHORS b/misc/bazel/registry/AUTHORS new file mode 100644 index 000000000000..42818b292e7c --- /dev/null +++ b/misc/bazel/registry/AUTHORS @@ -0,0 +1,7 @@ +# This is the list of Bazel's significant contributors. +# +# This does not necessarily list everyone who has contributed code, +# especially since many employees of one corporation may be contributing. +# To see the full list of contributors, see the revision history in +# source control. +Google LLC diff --git a/misc/bazel/registry/LICENSE b/misc/bazel/registry/LICENSE new file mode 100644 index 000000000000..d64569567334 --- /dev/null +++ b/misc/bazel/registry/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/misc/bazel/registry/NOTICE b/misc/bazel/registry/NOTICE new file mode 100644 index 000000000000..95329ae18f60 --- /dev/null +++ b/misc/bazel/registry/NOTICE @@ -0,0 +1,3 @@ +The files in this directory where originally taken from http://github.com/bazelbuild/bazel-central-registry and are +a derivative work thereof, distributed under the Apache 2.0 license, with the following exceptions: +* the `fix.py` file was added under the MIT license as the rest of the `codeql` repository. diff --git a/misc/bazel/registry/fix.py b/misc/bazel/registry/fix.py index c904ebf49517..a2b947e19e20 100755 --- a/misc/bazel/registry/fix.py +++ b/misc/bazel/registry/fix.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +# Copyright (c) 2024 GitHub, Inc. + """ Fix metadata in overridden registry, updating `metadata.json` to list correct versions and `source.json` to list correct patches with sha256 hashes. diff --git a/misc/bazel/registry/modules/rules_kotlin/metadata.json b/misc/bazel/registry/modules/rules_kotlin/metadata.json index d1270e0eeb1f..e399c3f5b0a2 100644 --- a/misc/bazel/registry/modules/rules_kotlin/metadata.json +++ b/misc/bazel/registry/modules/rules_kotlin/metadata.json @@ -1,5 +1,27 @@ { + "homepage": "https://github.com/bazelbuild/rules_kotlin", + "maintainers": [ + { + "email": "ben@ben.cm", + "github": "Bencodes", + "name": "Ben Lee" + }, + { + "email": "corbin@mcneely-smith.com", + "github": "restingbull", + "name": "Corbin McNeely-Smith" + }, + { + "email": "nk@snap.com", + "github": "nkoroste", + "name": "Nick Korostelev" + } + ], + "repository": [ + "github:bazelbuild/rules_kotlin" + ], "versions": [ "1.9.4-codeql.1" - ] + ], + "yanked_versions": {} } From 27ab4875fdd4aee0d78519ab76ac0ea407775109 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 16 Apr 2024 13:37:06 +0200 Subject: [PATCH 56/81] Kotlin: simplify `kotlinc` wrapper --- java/kotlin-extractor/deps/dev/.gitignore | 3 +-- java/kotlin-extractor/deps/dev/kotlinc | 21 +++++++-------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/java/kotlin-extractor/deps/dev/.gitignore b/java/kotlin-extractor/deps/dev/.gitignore index a65f75fad6fb..74a035783b57 100644 --- a/java/kotlin-extractor/deps/dev/.gitignore +++ b/java/kotlin-extractor/deps/dev/.gitignore @@ -1,3 +1,2 @@ +/.kotlinc_version /.kotlinc_installed -/.kotlinc_installed_version -/.kotlinc_selected_version diff --git a/java/kotlin-extractor/deps/dev/kotlinc b/java/kotlin-extractor/deps/dev/kotlinc index 509ed30cb7cb..95d9ebfdb1e6 100755 --- a/java/kotlin-extractor/deps/dev/kotlinc +++ b/java/kotlin-extractor/deps/dev/kotlinc @@ -40,8 +40,7 @@ def options(): url_template = 'https://github.com/JetBrains/kotlin/releases/download/v{version}/kotlin-compiler-{version}.zip' this_dir = pathlib.Path(__file__).resolve().parent -version_file = this_dir / ".kotlinc_selected_version" -installed_version_file = this_dir / ".kotlinc_installed_version" +version_file = this_dir / ".kotlinc_version" install_dir = this_dir / ".kotlinc_installed" windows_ripunzip = this_dir.parents[4] / "resources" / "lib" / "windows" / "ripunzip" / "ripunzip.exe" @@ -73,9 +72,9 @@ def check_version(version: str): raise -def get_version(file: pathlib.Path) -> str: +def get_version(): try: - return file.read_text() + return version_file.read_text() except FileNotFoundError: return None @@ -121,9 +120,6 @@ def clear(): if install_dir.exists(): print(f"removing {install_dir}", file=sys.stderr) shutil.rmtree(install_dir) - if installed_version_file.exists(): - print(f"removing {installed_version_file}", file=sys.stderr) - installed_version_file.unlink() if version_file.exists(): print(f"removing {version_file}", file=sys.stderr) version_file.unlink() @@ -133,18 +129,15 @@ def main(opts, forwarded_opts): if opts.clear: clear() return + current_version = get_version() if opts.select: check_version(opts.select) - version_file.write_text(opts.select) selected_version = opts.select else: - selected_version = get_version(version_file) - if not selected_version: - selected_version = DEFAULT_VERSION - version_file.write_text(selected_version) - if get_version(installed_version_file) != selected_version: + selected_version = current_version or DEFAULT_VERSION + if selected_version != current_version: install(selected_version) - installed_version_file.write_text(selected_version) + version_file.write_text(selected_version) if opts.version or (opts.select and not forwarded_opts): print(f"info: kotlinc-jvm {selected_version} (codeql dev wrapper)", file=sys.stderr) return From 306f0f172d325326a5ff5b166ba1dcc28da13549 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 16 Apr 2024 13:55:20 +0200 Subject: [PATCH 57/81] Kotlin: accept `--select default` in kotlinc wrapper to select the default version --- java/kotlin-extractor/deps/dev/kotlinc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/java/kotlin-extractor/deps/dev/kotlinc b/java/kotlin-extractor/deps/dev/kotlinc index 95d9ebfdb1e6..bf2d922e686a 100755 --- a/java/kotlin-extractor/deps/dev/kotlinc +++ b/java/kotlin-extractor/deps/dev/kotlinc @@ -130,7 +130,9 @@ def main(opts, forwarded_opts): clear() return current_version = get_version() - if opts.select: + if opts.select == "default": + selected_version = DEFAULT_VERSION + elif opts.select is not None: check_version(opts.select) selected_version = opts.select else: From aaa29d8982bb33dcfc693bfeb6a984abe88f355b Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 22 Apr 2024 18:09:41 +0200 Subject: [PATCH 58/81] Remove unneeded slash in `kotlinc.bat` --- java/kotlin-extractor/deps/dev/kotlinc.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/kotlin-extractor/deps/dev/kotlinc.bat b/java/kotlin-extractor/deps/dev/kotlinc.bat index f6e59843f76f..b19940d995d0 100644 --- a/java/kotlin-extractor/deps/dev/kotlinc.bat +++ b/java/kotlin-extractor/deps/dev/kotlinc.bat @@ -1,4 +1,4 @@ @echo off -python "%~dp0/kotlinc" %* +python "%~dp0kotlinc" %* exit /b %ERRORLEVEL% From 5b143cee96a397005822aa32ff753c2f77b3ae19 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 23 Apr 2024 15:09:11 +0200 Subject: [PATCH 59/81] Kotlin: make wrapper install quietly unless `--select` is explicit This allows `kotlinc -version` to always produce something parseable. --- java/kotlin-extractor/deps/dev/kotlinc | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/java/kotlin-extractor/deps/dev/kotlinc b/java/kotlin-extractor/deps/dev/kotlinc index bf2d922e686a..25df88988dbe 100755 --- a/java/kotlin-extractor/deps/dev/kotlinc +++ b/java/kotlin-extractor/deps/dev/kotlinc @@ -28,7 +28,7 @@ import io import os DEFAULT_VERSION = "1.9.0" - +DEVNULL = open(os.devnull, "w") def options(): parser = argparse.ArgumentParser(add_help=False) @@ -79,7 +79,8 @@ def get_version(): return None -def install(version: str): +def install(version: str, quiet: bool): + info_out = DEVNULL if quiet else sys.stderr url = url_template.format(version=version) if install_dir.exists(): shutil.rmtree(install_dir) @@ -88,11 +89,12 @@ def install(version: str): if ripunzip is None and platform.system() == "Windows" and windows_ripunzip.exists(): ripunzip = windows_ripunzip if ripunzip: - print(f"downloading and extracting {url} using ripunzip", file=sys.stderr) - subprocess.run([ripunzip, "unzip-uri", url], cwd=install_dir, check=True) + print(f"downloading and extracting {url} using ripunzip", file=info_out) + subprocess.run([ripunzip, "unzip-uri", url], stdout=info_out, stderr=info_out, cwd=install_dir, + check=True) return with io.BytesIO() as buffer: - print(f"downloading {url}", file=sys.stderr) + print(f"downloading {url}", file=info_out) with urllib.request.urlopen(url) as response: while True: bytes = response.read() @@ -100,7 +102,7 @@ def install(version: str): break buffer.write(bytes) buffer.seek(0) - print(f"extracting kotlin-compiler-{version}.zip", file=sys.stderr) + print(f"extracting kotlin-compiler-{version}.zip", file=info_out) with ZipFilePreservingPermissions(buffer) as archive: archive.extractall(install_dir) @@ -138,7 +140,8 @@ def main(opts, forwarded_opts): else: selected_version = current_version or DEFAULT_VERSION if selected_version != current_version: - install(selected_version) + # don't print information about install procedure unless explicitly using --select + install(selected_version, quiet=opts.select is None) version_file.write_text(selected_version) if opts.version or (opts.select and not forwarded_opts): print(f"info: kotlinc-jvm {selected_version} (codeql dev wrapper)", file=sys.stderr) @@ -149,8 +152,8 @@ def main(opts, forwarded_opts): if __name__ == "__main__": try: main(*options()) - except Exception as e: - print(f"{e.__class__.__name__}: {e}", file=sys.stderr) + except Error as e: + print(f"Error: {e}", file=sys.stderr) sys.exit(1) except KeyboardInterrupt: sys.exit(1) From c014cd84f43f7320d99d3578975aa4c2240ab7a8 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 23 Apr 2024 15:09:48 +0200 Subject: [PATCH 60/81] Bazel: fix kotlin wrapper version dependency --- java/kotlin-extractor/deps.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/kotlin-extractor/deps.bzl b/java/kotlin-extractor/deps.bzl index 8d737211967b..284e12494469 100644 --- a/java/kotlin-extractor/deps.bzl +++ b/java/kotlin-extractor/deps.bzl @@ -76,7 +76,7 @@ def _get_default_version(repository_ctx): kotlin_plugin_versions = repository_ctx.path(Label("//java/kotlin-extractor:current_kotlin_version.py")) python = repository_ctx.which("python3") or repository_ctx.which("python") env = {} - repository_ctx.watch(Label("//java/kotlin-extractor/deps:dev/.kotlinc_selected_version")) + repository_ctx.watch(Label("//java/kotlin-extractor/deps:dev/.kotlinc_version")) if not repository_ctx.which("kotlinc"): # take default from the kotlinc wrapper path = repository_ctx.getenv("PATH") From 4aa0a8ebae2e6c1bb23f6bd7accf1f284e5b77ce Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 24 Apr 2024 13:39:32 +0200 Subject: [PATCH 61/81] Kotlin: make wrapper more robust for windows --- java/kotlin-extractor/current_kotlin_version.py | 9 +++++---- java/kotlin-extractor/deps/dev/kotlinc | 14 +++++++++----- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/java/kotlin-extractor/current_kotlin_version.py b/java/kotlin-extractor/current_kotlin_version.py index 51c5229029c7..464745cf4964 100644 --- a/java/kotlin-extractor/current_kotlin_version.py +++ b/java/kotlin-extractor/current_kotlin_version.py @@ -5,9 +5,10 @@ kotlinc = shutil.which('kotlinc') if kotlinc is None: raise Exception("kotlinc not found") -output = subprocess.run([kotlinc, "-version"], text=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, - check=True).stderr -m = re.match(r'.* kotlinc-jvm ([0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z][a-zA-Z0-9]*)?) .*', output) +res = subprocess.run([kotlinc, "-version"], text=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) +if res.returncode != 0: + raise Exception(f"kotlinc -version failed: {res.stderr}") +m = re.match(r'.* kotlinc-jvm ([0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z][a-zA-Z0-9]*)?) .*', res.stderr) if m is None: - raise Exception(f'Cannot detect version of kotlinc (got {output})') + raise Exception(f'Cannot detect version of kotlinc (got {res.stderr})') print(m[1]) diff --git a/java/kotlin-extractor/deps/dev/kotlinc b/java/kotlin-extractor/deps/dev/kotlinc index 25df88988dbe..929ceb2c9d0d 100755 --- a/java/kotlin-extractor/deps/dev/kotlinc +++ b/java/kotlin-extractor/deps/dev/kotlinc @@ -28,7 +28,6 @@ import io import os DEFAULT_VERSION = "1.9.0" -DEVNULL = open(os.devnull, "w") def options(): parser = argparse.ArgumentParser(add_help=False) @@ -80,7 +79,12 @@ def get_version(): def install(version: str, quiet: bool): - info_out = DEVNULL if quiet else sys.stderr + if quiet: + info_out = subprocess.DEVNULL + info = lambda *args: None + else: + info_out = sys.stderr + info = lambda *args: print(*args, file=sys.stderr) url = url_template.format(version=version) if install_dir.exists(): shutil.rmtree(install_dir) @@ -89,12 +93,12 @@ def install(version: str, quiet: bool): if ripunzip is None and platform.system() == "Windows" and windows_ripunzip.exists(): ripunzip = windows_ripunzip if ripunzip: - print(f"downloading and extracting {url} using ripunzip", file=info_out) + info(f"downloading and extracting {url} using ripunzip") subprocess.run([ripunzip, "unzip-uri", url], stdout=info_out, stderr=info_out, cwd=install_dir, check=True) return with io.BytesIO() as buffer: - print(f"downloading {url}", file=info_out) + info(f"downloading {url}") with urllib.request.urlopen(url) as response: while True: bytes = response.read() @@ -102,7 +106,7 @@ def install(version: str, quiet: bool): break buffer.write(bytes) buffer.seek(0) - print(f"extracting kotlin-compiler-{version}.zip", file=info_out) + info(f"extracting kotlin-compiler-{version}.zip") with ZipFilePreservingPermissions(buffer) as archive: archive.extractall(install_dir) From 8e1d77bcb2637d07591050334aaf89a9c1b609a7 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 25 Apr 2024 11:05:57 +0200 Subject: [PATCH 62/81] Bazel: format file --- java/kotlin-extractor/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/kotlin-extractor/BUILD.bazel b/java/kotlin-extractor/BUILD.bazel index b197bce07903..16cff368e177 100644 --- a/java/kotlin-extractor/BUILD.bazel +++ b/java/kotlin-extractor/BUILD.bazel @@ -30,8 +30,8 @@ load( "get_language_version", "version_less", ) -load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") load("@rules_kotlin//kotlin:core.bzl", "kt_javac_options", "kt_kotlinc_options") +load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") package(default_visibility = ["//java/kotlin-extractor:__subpackages__"]) From d7ecaae245446b1dc26a3a3f4a87f29c3ea5d0ab Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 29 Apr 2024 17:21:46 +0200 Subject: [PATCH 63/81] Kotlin: back off from lazy LFS rules Those have shown to cause problems with too many concurrent downloads. This changes kotlinc dependencies fetching to: * use `resource/kotlinc-dependencies` if available (which is the case for the internal repo) * otherwise, download them from maven. This means sha256 hashes need to be written down for bazel. --- .gitattributes | 1 - java/kotlin-extractor/deps.bzl | 28 ++++++- java/kotlin-extractor/deps/BUILD.bazel | 0 java/kotlin-extractor/deps/LICENSE.md | 7 -- .../deps/kotlin-compiler-1.5.0.jar | 3 - .../deps/kotlin-compiler-1.5.10.jar | 3 - .../deps/kotlin-compiler-1.5.20.jar | 3 - .../deps/kotlin-compiler-1.5.30.jar | 3 - .../deps/kotlin-compiler-1.6.0.jar | 3 - .../deps/kotlin-compiler-1.6.20.jar | 3 - .../deps/kotlin-compiler-1.7.0.jar | 3 - .../deps/kotlin-compiler-1.7.20.jar | 3 - .../deps/kotlin-compiler-1.8.0.jar | 3 - .../deps/kotlin-compiler-1.9.0-Beta.jar | 3 - .../deps/kotlin-compiler-1.9.20-Beta.jar | 3 - .../deps/kotlin-compiler-2.0.0-RC1.jar | 3 - .../deps/kotlin-compiler-embeddable-1.5.0.jar | 3 - .../kotlin-compiler-embeddable-1.5.10.jar | 3 - .../kotlin-compiler-embeddable-1.5.20.jar | 3 - .../kotlin-compiler-embeddable-1.5.30.jar | 3 - .../deps/kotlin-compiler-embeddable-1.6.0.jar | 3 - .../kotlin-compiler-embeddable-1.6.20.jar | 3 - .../deps/kotlin-compiler-embeddable-1.7.0.jar | 3 - .../kotlin-compiler-embeddable-1.7.20.jar | 3 - .../deps/kotlin-compiler-embeddable-1.8.0.jar | 3 - .../kotlin-compiler-embeddable-1.9.0-Beta.jar | 3 - ...kotlin-compiler-embeddable-1.9.20-Beta.jar | 3 - .../kotlin-compiler-embeddable-2.0.0-RC1.jar | 3 - .../deps/kotlin-stdlib-1.5.0.jar | 3 - .../deps/kotlin-stdlib-1.5.10.jar | 3 - .../deps/kotlin-stdlib-1.5.20.jar | 3 - .../deps/kotlin-stdlib-1.5.30.jar | 3 - .../deps/kotlin-stdlib-1.6.0.jar | 3 - .../deps/kotlin-stdlib-1.6.20.jar | 3 - .../deps/kotlin-stdlib-1.7.0.jar | 3 - .../deps/kotlin-stdlib-1.7.20.jar | 3 - .../deps/kotlin-stdlib-1.8.0.jar | 3 - .../deps/kotlin-stdlib-1.9.0-Beta.jar | 3 - .../deps/kotlin-stdlib-1.9.20-Beta.jar | 3 - .../deps/kotlin-stdlib-2.0.0-RC1.jar | 3 - .../{deps => }/dev/.gitignore | 0 java/kotlin-extractor/{deps => }/dev/kotlinc | 0 .../{deps => }/dev/kotlinc.bat | 0 java/kotlin-extractor/versions.bzl | 80 +++++++++++++++---- 44 files changed, 91 insertions(+), 133 deletions(-) delete mode 100644 java/kotlin-extractor/deps/BUILD.bazel delete mode 100644 java/kotlin-extractor/deps/LICENSE.md delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.5.0.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.5.10.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.5.20.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.5.30.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.6.0.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.6.20.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.7.0.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.7.20.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.8.0.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.9.0-Beta.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.9.20-Beta.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-2.0.0-RC1.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.0.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.10.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.20.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.30.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.0.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.20.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.0.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.20.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.8.0.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.0-Beta.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.20-Beta.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.0-RC1.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.5.0.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.5.10.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.5.20.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.5.30.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.6.0.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.6.20.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.7.0.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.7.20.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.8.0.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.9.0-Beta.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.9.20-Beta.jar delete mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-2.0.0-RC1.jar rename java/kotlin-extractor/{deps => }/dev/.gitignore (100%) rename java/kotlin-extractor/{deps => }/dev/kotlinc (100%) rename java/kotlin-extractor/{deps => }/dev/kotlinc.bat (100%) diff --git a/.gitattributes b/.gitattributes index 1c344d3c035c..6d87d26f5476 100644 --- a/.gitattributes +++ b/.gitattributes @@ -52,7 +52,6 @@ /java/ql/test/stubs/**/*.java linguist-generated=true /java/ql/test/experimental/stubs/**/*.java linguist-generated=true -/java/kotlin-extractor/deps/*.jar filter=lfs diff=lfs merge=lfs -text # Force git not to modify line endings for go or html files under the go/ql directory /go/ql/**/*.go -text diff --git a/java/kotlin-extractor/deps.bzl b/java/kotlin-extractor/deps.bzl index 284e12494469..39325f5bbe4f 100644 --- a/java/kotlin-extractor/deps.bzl +++ b/java/kotlin-extractor/deps.bzl @@ -1,5 +1,4 @@ load("//java/kotlin-extractor:versions.bzl", "VERSIONS", "version_less") -load("//misc/bazel:lfs.bzl", "lfs_smudge") _kotlin_dep_build = """ load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_import") @@ -16,9 +15,28 @@ _empty_zip = "PK\005\006\000\000\000\000\000\000\000\000\000\000\000\000\000\000 def _get_dep(repository_ctx, name): return repository_ctx.path(Label("//java/kotlin-extractor/deps:%s" % name)) +_local_path = "{root}/resources/kotlin-dependencies/kotlin-{kind}-{version}.jar" +_maven_url = "https://repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-{kind}/{version}/kotlin-{kind}-{version}.jar" + def _kotlin_dep_impl(repository_ctx): _, _, name = repository_ctx.name.rpartition("~") - lfs_smudge(repository_ctx, [_get_dep(repository_ctx, name + ".jar")]) + kind = repository_ctx.attr.kind + version = repository_ctx.attr.version + filename = "kotlin-%s-%s.jar" % (kind, version) + local_path = _local_path.format(root = repository_ctx.workspace_root, kind = kind, version = version) + if repository_ctx.path(local_path).exists: + url = "file://%s" % local_path + else: + url = _maven_url.format(kind = kind, version = version) + + sha256 = VERSIONS[version].get(kind, "") + res = repository_ctx.download(url, output = filename, sha256 = sha256) + if not sha256: + fail('\nPlease add\n "%s": "%s",\nto VERSIONS["%s"] in java/kotlin-extractor/versions.bzl' % ( + kind, + res.sha256, + version, + )) # for some reason rules_kotlin warns about these jars missing, this is to silence those warnings repository_ctx.file("empty.zip", _empty_zip) @@ -34,6 +52,10 @@ def _kotlin_dep_impl(repository_ctx): _kotlin_dep = repository_rule( implementation = _kotlin_dep_impl, + attrs = { + "kind": attr.string(), + "version": attr.string(), + }, ) def _walk(dir): @@ -121,7 +143,7 @@ _defaults = repository_rule(implementation = _defaults_impl) def _kotlin_deps_impl(module_ctx): for v in VERSIONS: for lib in ("compiler", "compiler-embeddable", "stdlib"): - _kotlin_dep(name = "kotlin-%s-%s" % (lib, v)) + _kotlin_dep(name = "kotlin-%s-%s" % (lib, v), kind = lib, version = v) _embeddable_source(name = "codeql_kotlin_embeddable") _defaults(name = "codeql_kotlin_defaults") return module_ctx.extension_metadata( diff --git a/java/kotlin-extractor/deps/BUILD.bazel b/java/kotlin-extractor/deps/BUILD.bazel deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/java/kotlin-extractor/deps/LICENSE.md b/java/kotlin-extractor/deps/LICENSE.md deleted file mode 100644 index a93f66f0aa30..000000000000 --- a/java/kotlin-extractor/deps/LICENSE.md +++ /dev/null @@ -1,7 +0,0 @@ -The Git LFS files contained in this directory are mirrored -from [org.jetbrains.kotlin packages in the Maven repository][1]. Please refer to [the kotlin Apache 2.0 license][2] for -details about their license. - -[1]: https://mvnrepository.com/artifact/org.jetbrains.kotlin - -[2]: https://github.com/JetBrains/kotlin/tree/master/license diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.5.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.5.0.jar deleted file mode 100644 index 5aad4918db7a..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-1.5.0.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f6e4a2c4394e77f937fcffda0036531604f25cc7c8de8daea098e1aa31f1d248 -size 47816554 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.5.10.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.5.10.jar deleted file mode 100644 index 04a54b334a24..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-1.5.10.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:dfefb1aa8bec81256617c8ceb577373e44078b7e21024625da50e376037e9ae5 -size 47822093 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.5.20.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.5.20.jar deleted file mode 100644 index 29a6d4df3e26..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-1.5.20.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a0ae437d1b670a5ba6da7893b7023df649c4ab2e6c19d5e9b4eee5332e1cde1f -size 49012794 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.5.30.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.5.30.jar deleted file mode 100644 index 502c844ce5cb..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-1.5.30.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:487d8ff9766a6ba570cd15c5225c1600654e7cf1b6ef2b92ed6905528a3e838a -size 49509580 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.6.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.6.0.jar deleted file mode 100644 index 4a98879e43e6..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-1.6.0.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4bd7a92568fd89c23b7f9f36d4380886beed18d3d54ea6adf49bebae627db805 -size 51408858 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.6.20.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.6.20.jar deleted file mode 100644 index 5a467a5af518..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-1.6.20.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:90567c5cf297985d028fa39aa3a7904dc8096173e1c7f3d3f35fe7074581098e -size 53370229 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.7.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.7.0.jar deleted file mode 100644 index ef54ce1e6532..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-1.7.0.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ce85fafb3e24712d62a0d02d277c2d56197d74afdd4f5ca995eaf33d2c504663 -size 53906809 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.7.20.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.7.20.jar deleted file mode 100644 index 442aea3e3425..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-1.7.20.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0e36d98c56f7c9685ab9d9e1fac9be36a5214939adb3f905b93c62de76023618 -size 54773087 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.8.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.8.0.jar deleted file mode 100644 index 7884c5361b56..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-1.8.0.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1f19f247d337387cbdd75d54e10a6865857c28a533fced50c0c5c9482b3ab9af -size 55128997 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.9.0-Beta.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.9.0-Beta.jar deleted file mode 100644 index 8da3d202a5e4..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-1.9.0-Beta.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4c7e3972e0ce0be8aa5c8ceeb8eb795f6345685bb57c6f59b649ed70c6051581 -size 60202336 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.9.20-Beta.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.9.20-Beta.jar deleted file mode 100644 index 57215c331e61..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-1.9.20-Beta.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b90980de3a16858e6e1957236d7bb9a729fcd0587a98fb64668866e1975aaa6f -size 61641342 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-2.0.0-RC1.jar b/java/kotlin-extractor/deps/kotlin-compiler-2.0.0-RC1.jar deleted file mode 100644 index fca4ae9fd36e..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-2.0.0-RC1.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:60e31163aa7348166d708cdc9cb47d616b0222e5b11173f35f1adfb61bc3690a -size 58604870 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.0.jar deleted file mode 100644 index d6e7ef8585bc..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.0.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d7b85448039e468daf3b9462a172244477fa3eb890f199ec77703992f36ade44 -size 46308890 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.10.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.10.jar deleted file mode 100644 index 05f594d6af13..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.10.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b9965f7c958efb17f2a90a19b5e60325d4f4e644d2833dbfb4a11edd8dddf244 -size 46314547 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.20.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.20.jar deleted file mode 100644 index 3bb8ee5e52f7..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.20.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:11d51087eb70b5abbad6fbf459a4349a0335916588000b5ecd990f01482e38ff -size 47526900 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.30.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.30.jar deleted file mode 100644 index 647b08b7b7ab..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.30.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b5051dc92725b099c41710bd3f213cd0c1d6f25056d31b2e8cae30903873b741 -size 48031832 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.0.jar deleted file mode 100644 index eafb6508d449..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.0.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0366843cd2defdd583c6b16b10bc32b85f28c5bf9510f10e44c886f5bd24c388 -size 49978431 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.20.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.20.jar deleted file mode 100644 index f426dae65b1e..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.20.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:be634faaafb56816b6ef6d583e57ab33e4d6e5180cde2f505ccf7d45dc738ef8 -size 51976423 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.0.jar deleted file mode 100644 index 4abc782ed05f..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.0.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6dcd02ced3bdbfabc7da046a69f4e730c8f907fc29bb5b2fea44083ffea22dc8 -size 142740530 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.20.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.20.jar deleted file mode 100644 index b95097b6935e..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.20.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5ec2be1872dc47b9dcb466f1781eded6c59d9eee18657d4b0f1148e619caea36 -size 53395419 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.8.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.8.0.jar deleted file mode 100644 index 5494dd88f974..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.8.0.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e9b3a56dbbfdf1e0328d4731b7d7ca789ce0f1f263372ad88dd8decbd1602858 -size 53769272 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.0-Beta.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.0-Beta.jar deleted file mode 100644 index 32ea1ec13146..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.0-Beta.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7429076f8dd2ccd1cce48d7e5bf5b9fadde8afab110f9f4cfe0912756f16d770 -size 58697460 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.20-Beta.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.20-Beta.jar deleted file mode 100644 index 5c1db1782f8a..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.20-Beta.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5c9de79f0f8d97f2aa4d877449063b1cc2828c17f25a119fc32c776401f058de -size 60157805 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.0-RC1.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.0-RC1.jar deleted file mode 100644 index 2f65ae3aaed0..000000000000 --- a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.0-RC1.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:12d330c6e98ba42840f1d10e2752a9c53099d4dfa855bc4526d23fe1a1af9634 -size 57125659 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.5.0.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.5.0.jar deleted file mode 100644 index 83d4a9f9c031..000000000000 --- a/java/kotlin-extractor/deps/kotlin-stdlib-1.5.0.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:52283996fe4067cd7330288b96ae67ecd463614dc741172c54d9d349ab6a9cd7 -size 1497598 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.5.10.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.5.10.jar deleted file mode 100644 index a0b08eaf499a..000000000000 --- a/java/kotlin-extractor/deps/kotlin-stdlib-1.5.10.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ca87c454cd3f2e60931f1803c59699d510d3b4b959cd7119296fb947581d722d -size 1497600 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.5.20.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.5.20.jar deleted file mode 100644 index 90844bea3812..000000000000 --- a/java/kotlin-extractor/deps/kotlin-stdlib-1.5.20.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:80cd79c26aac46d72d782de1ecb326061e93c6e688d994b48627ffd668ba63a8 -size 1497567 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.5.30.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.5.30.jar deleted file mode 100644 index b19606b84967..000000000000 --- a/java/kotlin-extractor/deps/kotlin-stdlib-1.5.30.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c55608e9eb6df7327e74b21e271d324dc523cef31587b8d6d2393db08d6e000c -size 1505951 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.6.0.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.6.0.jar deleted file mode 100644 index 3ad56eecdc2d..000000000000 --- a/java/kotlin-extractor/deps/kotlin-stdlib-1.6.0.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:115daea30b0d484afcf2360237b9d9537f48a4a2f03f3cc2a16577dfc6e90342 -size 1508076 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.6.20.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.6.20.jar deleted file mode 100644 index b170c1da738b..000000000000 --- a/java/kotlin-extractor/deps/kotlin-stdlib-1.6.20.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:eeb51c2b67b26233fd81d0bc4f8044ec849718890905763ceffd84a31e2cb799 -size 1509405 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.7.0.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.7.0.jar deleted file mode 100644 index bed18f8429b7..000000000000 --- a/java/kotlin-extractor/deps/kotlin-stdlib-1.7.0.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:aa88e9625577957f3249a46cb6e166ee09b369e600f7a11d148d16b0a6d87f05 -size 1524619 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.7.20.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.7.20.jar deleted file mode 100644 index 9998f1e65920..000000000000 --- a/java/kotlin-extractor/deps/kotlin-stdlib-1.7.20.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7779ec96b9acbf92ca023858ac04543f9d2c3bdf1722425fff42f25ff3acfc9b -size 1537347 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.8.0.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.8.0.jar deleted file mode 100644 index 47dd2710887b..000000000000 --- a/java/kotlin-extractor/deps/kotlin-stdlib-1.8.0.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c77bef8774640b9fb9d6e217459ff220dae59878beb7d2e4b430506feffc654e -size 1636558 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.9.0-Beta.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.9.0-Beta.jar deleted file mode 100644 index c6f3f947029f..000000000000 --- a/java/kotlin-extractor/deps/kotlin-stdlib-1.9.0-Beta.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:af458cc55cf69e966668e6010c7ccee4a50d553b3504a2e8311dd0c76242d881 -size 1708001 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.9.20-Beta.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.9.20-Beta.jar deleted file mode 100644 index f9c853bbb068..000000000000 --- a/java/kotlin-extractor/deps/kotlin-stdlib-1.9.20-Beta.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:788e48813dd76ad598fff7bef3b1e038d7291741810bd04c6c57037c4d75dac2 -size 1715901 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-2.0.0-RC1.jar b/java/kotlin-extractor/deps/kotlin-stdlib-2.0.0-RC1.jar deleted file mode 100644 index fbf0a8eff06f..000000000000 --- a/java/kotlin-extractor/deps/kotlin-stdlib-2.0.0-RC1.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:bbb2c9b813e6196f9afa9f9add8b395ee384ab1763a0880e084d2942214f1c30 -size 1729731 diff --git a/java/kotlin-extractor/deps/dev/.gitignore b/java/kotlin-extractor/dev/.gitignore similarity index 100% rename from java/kotlin-extractor/deps/dev/.gitignore rename to java/kotlin-extractor/dev/.gitignore diff --git a/java/kotlin-extractor/deps/dev/kotlinc b/java/kotlin-extractor/dev/kotlinc similarity index 100% rename from java/kotlin-extractor/deps/dev/kotlinc rename to java/kotlin-extractor/dev/kotlinc diff --git a/java/kotlin-extractor/deps/dev/kotlinc.bat b/java/kotlin-extractor/dev/kotlinc.bat similarity index 100% rename from java/kotlin-extractor/deps/dev/kotlinc.bat rename to java/kotlin-extractor/dev/kotlinc.bat diff --git a/java/kotlin-extractor/versions.bzl b/java/kotlin-extractor/versions.bzl index 728672ead6de..b3f2d4dca147 100644 --- a/java/kotlin-extractor/versions.bzl +++ b/java/kotlin-extractor/versions.bzl @@ -1,18 +1,70 @@ # when updating this list, `bazel mod tidy` should be run from `codeql` to update `MODULE.bazel` -VERSIONS = [ - "1.5.0", - "1.5.10", - "1.5.20", - "1.5.30", - "1.6.0", - "1.6.20", - "1.7.0", - "1.7.20", - "1.8.0", - "1.9.0-Beta", - "1.9.20-Beta", - "2.0.0-RC1", -] +# when adding a new version, start out with {}, then run `bazel fetch //java/kotlin-extractor:all` +# (`bazel fetch @codeql//java/kotlin-extractor:all` from `semmle-code`). +# Repeat while it will fail printing what to add here. +# Alternatively, run `sha256sum` on the files to get what to write here. +VERSIONS = { + "1.5.0": { + "compiler": "f6e4a2c4394e77f937fcffda0036531604f25cc7c8de8daea098e1aa31f1d248", + "compiler-embeddable": "d7b85448039e468daf3b9462a172244477fa3eb890f199ec77703992f36ade44", + "stdlib": "52283996fe4067cd7330288b96ae67ecd463614dc741172c54d9d349ab6a9cd7", + }, + "1.5.10": { + "compiler": "dfefb1aa8bec81256617c8ceb577373e44078b7e21024625da50e376037e9ae5", + "compiler-embeddable": "b9965f7c958efb17f2a90a19b5e60325d4f4e644d2833dbfb4a11edd8dddf244", + "stdlib": "ca87c454cd3f2e60931f1803c59699d510d3b4b959cd7119296fb947581d722d", + }, + "1.5.20": { + "compiler": "a0ae437d1b670a5ba6da7893b7023df649c4ab2e6c19d5e9b4eee5332e1cde1f", + "compiler-embeddable": "11d51087eb70b5abbad6fbf459a4349a0335916588000b5ecd990f01482e38ff", + "stdlib": "80cd79c26aac46d72d782de1ecb326061e93c6e688d994b48627ffd668ba63a8", + }, + "1.5.30": { + "compiler": "487d8ff9766a6ba570cd15c5225c1600654e7cf1b6ef2b92ed6905528a3e838a", + "compiler-embeddable": "b5051dc92725b099c41710bd3f213cd0c1d6f25056d31b2e8cae30903873b741", + "stdlib": "c55608e9eb6df7327e74b21e271d324dc523cef31587b8d6d2393db08d6e000c", + }, + "1.6.0": { + "compiler": "4bd7a92568fd89c23b7f9f36d4380886beed18d3d54ea6adf49bebae627db805", + "compiler-embeddable": "0366843cd2defdd583c6b16b10bc32b85f28c5bf9510f10e44c886f5bd24c388", + "stdlib": "115daea30b0d484afcf2360237b9d9537f48a4a2f03f3cc2a16577dfc6e90342", + }, + "1.6.20": { + "compiler": "90567c5cf297985d028fa39aa3a7904dc8096173e1c7f3d3f35fe7074581098e", + "compiler-embeddable": "be634faaafb56816b6ef6d583e57ab33e4d6e5180cde2f505ccf7d45dc738ef8", + "stdlib": "eeb51c2b67b26233fd81d0bc4f8044ec849718890905763ceffd84a31e2cb799", + }, + "1.7.0": { + "compiler": "ce85fafb3e24712d62a0d02d277c2d56197d74afdd4f5ca995eaf33d2c504663", + "compiler-embeddable": "573935b492e65b93a792eaa6270295532f580cd4f26f9f6eb105ecbafcd182d4", + "stdlib": "aa88e9625577957f3249a46cb6e166ee09b369e600f7a11d148d16b0a6d87f05", + }, + "1.7.20": { + "compiler": "0e36d98c56f7c9685ab9d9e1fac9be36a5214939adb3f905b93c62de76023618", + "compiler-embeddable": "5ec2be1872dc47b9dcb466f1781eded6c59d9eee18657d4b0f1148e619caea36", + "stdlib": "7779ec96b9acbf92ca023858ac04543f9d2c3bdf1722425fff42f25ff3acfc9b", + }, + "1.8.0": { + "compiler": "1f19f247d337387cbdd75d54e10a6865857c28a533fced50c0c5c9482b3ab9af", + "compiler-embeddable": "e9b3a56dbbfdf1e0328d4731b7d7ca789ce0f1f263372ad88dd8decbd1602858", + "stdlib": "c77bef8774640b9fb9d6e217459ff220dae59878beb7d2e4b430506feffc654e", + }, + "1.9.0-Beta": { + "compiler": "4c7e3972e0ce0be8aa5c8ceeb8eb795f6345685bb57c6f59b649ed70c6051581", + "compiler-embeddable": "7429076f8dd2ccd1cce48d7e5bf5b9fadde8afab110f9f4cfe0912756f16d770", + "stdlib": "af458cc55cf69e966668e6010c7ccee4a50d553b3504a2e8311dd0c76242d881", + }, + "1.9.20-Beta": { + "compiler": "b90980de3a16858e6e1957236d7bb9a729fcd0587a98fb64668866e1975aaa6f", + "compiler-embeddable": "5c9de79f0f8d97f2aa4d877449063b1cc2828c17f25a119fc32c776401f058de", + "stdlib": "788e48813dd76ad598fff7bef3b1e038d7291741810bd04c6c57037c4d75dac2", + }, + "2.0.0-RC1": { + "compiler": "60e31163aa7348166d708cdc9cb47d616b0222e5b11173f35f1adfb61bc3690a", + "compiler-embeddable": "12d330c6e98ba42840f1d10e2752a9c53099d4dfa855bc4526d23fe1a1af9634", + "stdlib": "bbb2c9b813e6196f9afa9f9add8b395ee384ab1763a0880e084d2942214f1c30", + }, +} def _version_to_tuple(v): # we ignore the tag when comparing versions, for example 1.9.0-Beta <= 1.9.0 From 450f6518e2912f1e2fa57bce58ed310b12b8cf88 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 29 Apr 2024 17:26:48 +0200 Subject: [PATCH 64/81] Bazel: remove generic lazy lfs files --- .lfsconfig | 5 ---- misc/bazel/lfs.bzl | 73 ---------------------------------------------- 2 files changed, 78 deletions(-) delete mode 100644 .lfsconfig delete mode 100644 misc/bazel/lfs.bzl diff --git a/.lfsconfig b/.lfsconfig deleted file mode 100644 index cb0a8e352e86..000000000000 --- a/.lfsconfig +++ /dev/null @@ -1,5 +0,0 @@ -[lfs] -# codeql is publicly forked by many users, and we don't want any LFS file polluting their working -# copies. We therefore exclude everything by default. -# For files required by bazel builds, use rules in `misc/bazel/lfs.bzl` to download them on demand. -fetchinclude = /nothing diff --git a/misc/bazel/lfs.bzl b/misc/bazel/lfs.bzl deleted file mode 100644 index ab40e6ac3d07..000000000000 --- a/misc/bazel/lfs.bzl +++ /dev/null @@ -1,73 +0,0 @@ -def lfs_smudge(repository_ctx, srcs): - for src in srcs: - repository_ctx.watch(src) - script = Label("//misc/bazel/internal:git_lfs_smudge.py") - python = repository_ctx.which("python3") or repository_ctx.which("python") - if not python: - fail("Neither python3 nor python executables found") - res = repository_ctx.execute([python, script] + srcs, quiet = False) - if res.return_code != 0: - fail("git LFS smudging failed while instantiating @%s:\n%s" % (repository_ctx.name, res.stderr)) - -def _download_and_extract_lfs(repository_ctx): - attr = repository_ctx.attr - src = repository_ctx.path(attr.src) - if attr.build_file_content and attr.build_file: - fail("You should specify only one among build_file_content and build_file for rule @%s" % repository_ctx.name) - lfs_smudge(repository_ctx, [src]) - repository_ctx.extract(src.basename, stripPrefix = attr.strip_prefix) - repository_ctx.delete(src.basename) - if attr.build_file_content: - repository_ctx.file("BUILD.bazel", attr.build_file_content) - elif attr.build_file: - repository_ctx.symlink(attr.build_file, "BUILD.bazel") - -def _download_lfs(repository_ctx): - attr = repository_ctx.attr - if int(bool(attr.srcs)) + int(bool(attr.dir)) != 1: - fail("Exactly one between `srcs` and `dir` must be defined for @%s" % repository_ctx.name) - if attr.srcs: - srcs = [repository_ctx.path(src) for src in attr.srcs] - else: - dir = repository_ctx.path(attr.dir) - if not dir.is_dir: - fail("`dir` not a directory in @%s" % repository_ctx.name) - srcs = [f for f in dir.readdir() if not f.is_dir] - lfs_smudge(repository_ctx, srcs) - - # with bzlmod the name is qualified with `~` separators, and we want the base name here - name = repository_ctx.name.split("~")[-1] - repository_ctx.file("BUILD.bazel", """ -exports_files({files}) - -filegroup( - name = "{name}", - srcs = {files}, - visibility = ["//visibility:public"], -) -""".format(name = name, files = repr([src.basename for src in srcs]))) - -lfs_archive = repository_rule( - doc = "Export the contents from an on-demand LFS archive. The corresponding path should be added to be ignored " + - "in `.lfsconfig`.", - implementation = _download_and_extract_lfs, - attrs = { - "src": attr.label(mandatory = True, doc = "Local path to the LFS archive to extract."), - "build_file_content": attr.string(doc = "The content for the BUILD file for this repository. " + - "Either build_file or build_file_content can be specified, but not both."), - "build_file": attr.label(doc = "The file to use as the BUILD file for this repository. " + - "Either build_file or build_file_content can be specified, but not both."), - "strip_prefix": attr.string(default = "", doc = "A directory prefix to strip from the extracted files. "), - }, -) - -lfs_files = repository_rule( - doc = "Export LFS files for on-demand download. Exactly one between `srcs` and `dir` must be defined. The " + - "corresponding paths should be added to be ignored in `.lfsconfig`.", - implementation = _download_lfs, - attrs = { - "srcs": attr.label_list(doc = "Local paths to the LFS files to export."), - "dir": attr.label(doc = "Local path to a directory containing LFS files to export. Only the direct contents " + - "of the directory are exported"), - }, -) From d4e0a5629d02dc1c5c8531348763b934d488a4f9 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 29 Apr 2024 17:33:40 +0200 Subject: [PATCH 65/81] Kotlin: update wrapper location --- java/kotlin-extractor/deps.bzl | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/java/kotlin-extractor/deps.bzl b/java/kotlin-extractor/deps.bzl index 39325f5bbe4f..93556805c8b6 100644 --- a/java/kotlin-extractor/deps.bzl +++ b/java/kotlin-extractor/deps.bzl @@ -12,9 +12,6 @@ kt_jvm_import( _empty_zip = "PK\005\006\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" -def _get_dep(repository_ctx, name): - return repository_ctx.path(Label("//java/kotlin-extractor/deps:%s" % name)) - _local_path = "{root}/resources/kotlin-dependencies/kotlin-{kind}-{version}.jar" _maven_url = "https://repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-{kind}/{version}/kotlin-{kind}-{version}.jar" @@ -98,11 +95,11 @@ def _get_default_version(repository_ctx): kotlin_plugin_versions = repository_ctx.path(Label("//java/kotlin-extractor:current_kotlin_version.py")) python = repository_ctx.which("python3") or repository_ctx.which("python") env = {} - repository_ctx.watch(Label("//java/kotlin-extractor/deps:dev/.kotlinc_version")) + repository_ctx.watch(Label("//java/kotlin-extractor:dev/.kotlinc_version")) if not repository_ctx.which("kotlinc"): # take default from the kotlinc wrapper path = repository_ctx.getenv("PATH") - path_to_add = repository_ctx.path(Label("//java/kotlin-extractor/deps:dev")) + path_to_add = repository_ctx.path(Label("//java/kotlin-extractor:dev")) if not path: path = str(path_to_add) elif repository_ctx.os.name == "windows": From 52a015fb118d331495d5912c6ada4b1a90d3d013 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 30 Apr 2024 09:07:11 +0200 Subject: [PATCH 66/81] Bazel: remove further unneeded files --- misc/bazel/internal/BUILD.bazel | 0 misc/bazel/internal/git_lfs_smudge.py | 32 --------------------------- 2 files changed, 32 deletions(-) delete mode 100644 misc/bazel/internal/BUILD.bazel delete mode 100755 misc/bazel/internal/git_lfs_smudge.py diff --git a/misc/bazel/internal/BUILD.bazel b/misc/bazel/internal/BUILD.bazel deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/misc/bazel/internal/git_lfs_smudge.py b/misc/bazel/internal/git_lfs_smudge.py deleted file mode 100755 index 4e86cecc3abf..000000000000 --- a/misc/bazel/internal/git_lfs_smudge.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import pathlib -import subprocess -import os - -sources = [pathlib.Path(arg).resolve() for arg in sys.argv[1:]] -source_dir = pathlib.Path(os.path.commonpath(src.parent for src in sources)) -source_dir = subprocess.check_output(["git", "rev-parse", "--show-toplevel"], cwd=source_dir, text=True).strip() - - -def is_lfs_pointer(fileobj): - lfs_header = "version https://git-lfs.github.com/spec".encode() - actual_header = fileobj.read(len(lfs_header)) - fileobj.seek(0) - return lfs_header == actual_header - - -for src in sources: - with open(src, 'rb') as input: - if is_lfs_pointer(input): - lfs_pointer = input.read() - rel_src = src.relative_to(source_dir).as_posix() - with open(src.name, 'wb') as output: - subprocess.run( - ["git", - "-c", f"lfs.fetchinclude={rel_src}", "-c", "lfs.fetchexclude=", - "lfs", "smudge", "--", rel_src], - input=lfs_pointer, stdout=output, check=True, cwd=source_dir) - continue - pathlib.Path(src.name).symlink_to(src) From 514e24ce6283c5b3ed8d7a91476eadf5e648a657 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 6 May 2024 17:31:50 +0200 Subject: [PATCH 67/81] Kotlin: bring back lazy LFS dependencies This reverts commit d7ecaae245446b1dc26a3a3f4a87f29c3ea5d0ab. Problems with lazy LFS rules were solved by https://github.com/github/codeql/pull/16393 and https://github.com/github/codeql/pull/16434. --- .gitattributes | 1 + java/kotlin-extractor/deps.bzl | 29 ++----- java/kotlin-extractor/deps/BUILD.bazel | 0 java/kotlin-extractor/deps/LICENSE.md | 7 ++ .../deps/kotlin-compiler-1.5.0.jar | 3 + .../deps/kotlin-compiler-1.5.10.jar | 3 + .../deps/kotlin-compiler-1.5.20.jar | 3 + .../deps/kotlin-compiler-1.5.30.jar | 3 + .../deps/kotlin-compiler-1.6.0.jar | 3 + .../deps/kotlin-compiler-1.6.20.jar | 3 + .../deps/kotlin-compiler-1.7.0.jar | 3 + .../deps/kotlin-compiler-1.7.20.jar | 3 + .../deps/kotlin-compiler-1.8.0.jar | 3 + .../deps/kotlin-compiler-1.9.0-Beta.jar | 3 + .../deps/kotlin-compiler-1.9.20-Beta.jar | 3 + .../deps/kotlin-compiler-2.0.0-RC1.jar | 3 + .../deps/kotlin-compiler-embeddable-1.5.0.jar | 3 + .../kotlin-compiler-embeddable-1.5.10.jar | 3 + .../kotlin-compiler-embeddable-1.5.20.jar | 3 + .../kotlin-compiler-embeddable-1.5.30.jar | 3 + .../deps/kotlin-compiler-embeddable-1.6.0.jar | 3 + .../kotlin-compiler-embeddable-1.6.20.jar | 3 + .../deps/kotlin-compiler-embeddable-1.7.0.jar | 3 + .../kotlin-compiler-embeddable-1.7.20.jar | 3 + .../deps/kotlin-compiler-embeddable-1.8.0.jar | 3 + .../kotlin-compiler-embeddable-1.9.0-Beta.jar | 3 + ...kotlin-compiler-embeddable-1.9.20-Beta.jar | 3 + .../kotlin-compiler-embeddable-2.0.0-RC1.jar | 3 + .../deps/kotlin-stdlib-1.5.0.jar | 3 + .../deps/kotlin-stdlib-1.5.10.jar | 3 + .../deps/kotlin-stdlib-1.5.20.jar | 3 + .../deps/kotlin-stdlib-1.5.30.jar | 3 + .../deps/kotlin-stdlib-1.6.0.jar | 3 + .../deps/kotlin-stdlib-1.6.20.jar | 3 + .../deps/kotlin-stdlib-1.7.0.jar | 3 + .../deps/kotlin-stdlib-1.7.20.jar | 3 + .../deps/kotlin-stdlib-1.8.0.jar | 3 + .../deps/kotlin-stdlib-1.9.0-Beta.jar | 3 + .../deps/kotlin-stdlib-1.9.20-Beta.jar | 3 + .../deps/kotlin-stdlib-2.0.0-RC1.jar | 3 + java/kotlin-extractor/versions.bzl | 80 ++++--------------- 41 files changed, 135 insertions(+), 90 deletions(-) create mode 100644 java/kotlin-extractor/deps/BUILD.bazel create mode 100644 java/kotlin-extractor/deps/LICENSE.md create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.5.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.5.10.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.5.20.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.5.30.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.6.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.6.20.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.7.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.7.20.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.8.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.9.0-Beta.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-1.9.20-Beta.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-2.0.0-RC1.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.10.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.20.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.30.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.20.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.20.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.8.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.0-Beta.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.20-Beta.jar create mode 100644 java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.0-RC1.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.5.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.5.10.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.5.20.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.5.30.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.6.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.6.20.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.7.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.7.20.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.8.0.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.9.0-Beta.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-1.9.20-Beta.jar create mode 100644 java/kotlin-extractor/deps/kotlin-stdlib-2.0.0-RC1.jar diff --git a/.gitattributes b/.gitattributes index 6d87d26f5476..1c344d3c035c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -52,6 +52,7 @@ /java/ql/test/stubs/**/*.java linguist-generated=true /java/ql/test/experimental/stubs/**/*.java linguist-generated=true +/java/kotlin-extractor/deps/*.jar filter=lfs diff=lfs merge=lfs -text # Force git not to modify line endings for go or html files under the go/ql directory /go/ql/**/*.go -text diff --git a/java/kotlin-extractor/deps.bzl b/java/kotlin-extractor/deps.bzl index 93556805c8b6..049c4a25cada 100644 --- a/java/kotlin-extractor/deps.bzl +++ b/java/kotlin-extractor/deps.bzl @@ -1,4 +1,5 @@ load("//java/kotlin-extractor:versions.bzl", "VERSIONS", "version_less") +load("//misc/bazel:lfs.bzl", "lfs_smudge") _kotlin_dep_build = """ load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_import") @@ -12,28 +13,12 @@ kt_jvm_import( _empty_zip = "PK\005\006\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" -_local_path = "{root}/resources/kotlin-dependencies/kotlin-{kind}-{version}.jar" -_maven_url = "https://repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-{kind}/{version}/kotlin-{kind}-{version}.jar" +def _get_dep(repository_ctx, name): + return repository_ctx.path(Label("//java/kotlin-extractor/deps:%s" % name)) def _kotlin_dep_impl(repository_ctx): _, _, name = repository_ctx.name.rpartition("~") - kind = repository_ctx.attr.kind - version = repository_ctx.attr.version - filename = "kotlin-%s-%s.jar" % (kind, version) - local_path = _local_path.format(root = repository_ctx.workspace_root, kind = kind, version = version) - if repository_ctx.path(local_path).exists: - url = "file://%s" % local_path - else: - url = _maven_url.format(kind = kind, version = version) - - sha256 = VERSIONS[version].get(kind, "") - res = repository_ctx.download(url, output = filename, sha256 = sha256) - if not sha256: - fail('\nPlease add\n "%s": "%s",\nto VERSIONS["%s"] in java/kotlin-extractor/versions.bzl' % ( - kind, - res.sha256, - version, - )) + lfs_smudge(repository_ctx, [_get_dep(repository_ctx, name + ".jar")]) # for some reason rules_kotlin warns about these jars missing, this is to silence those warnings repository_ctx.file("empty.zip", _empty_zip) @@ -49,10 +34,6 @@ def _kotlin_dep_impl(repository_ctx): _kotlin_dep = repository_rule( implementation = _kotlin_dep_impl, - attrs = { - "kind": attr.string(), - "version": attr.string(), - }, ) def _walk(dir): @@ -140,7 +121,7 @@ _defaults = repository_rule(implementation = _defaults_impl) def _kotlin_deps_impl(module_ctx): for v in VERSIONS: for lib in ("compiler", "compiler-embeddable", "stdlib"): - _kotlin_dep(name = "kotlin-%s-%s" % (lib, v), kind = lib, version = v) + _kotlin_dep(name = "kotlin-%s-%s" % (lib, v)) _embeddable_source(name = "codeql_kotlin_embeddable") _defaults(name = "codeql_kotlin_defaults") return module_ctx.extension_metadata( diff --git a/java/kotlin-extractor/deps/BUILD.bazel b/java/kotlin-extractor/deps/BUILD.bazel new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/java/kotlin-extractor/deps/LICENSE.md b/java/kotlin-extractor/deps/LICENSE.md new file mode 100644 index 000000000000..a93f66f0aa30 --- /dev/null +++ b/java/kotlin-extractor/deps/LICENSE.md @@ -0,0 +1,7 @@ +The Git LFS files contained in this directory are mirrored +from [org.jetbrains.kotlin packages in the Maven repository][1]. Please refer to [the kotlin Apache 2.0 license][2] for +details about their license. + +[1]: https://mvnrepository.com/artifact/org.jetbrains.kotlin + +[2]: https://github.com/JetBrains/kotlin/tree/master/license diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.5.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.5.0.jar new file mode 100644 index 000000000000..5aad4918db7a --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-1.5.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6e4a2c4394e77f937fcffda0036531604f25cc7c8de8daea098e1aa31f1d248 +size 47816554 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.5.10.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.5.10.jar new file mode 100644 index 000000000000..04a54b334a24 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-1.5.10.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfefb1aa8bec81256617c8ceb577373e44078b7e21024625da50e376037e9ae5 +size 47822093 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.5.20.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.5.20.jar new file mode 100644 index 000000000000..29a6d4df3e26 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-1.5.20.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0ae437d1b670a5ba6da7893b7023df649c4ab2e6c19d5e9b4eee5332e1cde1f +size 49012794 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.5.30.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.5.30.jar new file mode 100644 index 000000000000..502c844ce5cb --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-1.5.30.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:487d8ff9766a6ba570cd15c5225c1600654e7cf1b6ef2b92ed6905528a3e838a +size 49509580 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.6.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.6.0.jar new file mode 100644 index 000000000000..4a98879e43e6 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-1.6.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bd7a92568fd89c23b7f9f36d4380886beed18d3d54ea6adf49bebae627db805 +size 51408858 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.6.20.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.6.20.jar new file mode 100644 index 000000000000..5a467a5af518 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-1.6.20.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90567c5cf297985d028fa39aa3a7904dc8096173e1c7f3d3f35fe7074581098e +size 53370229 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.7.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.7.0.jar new file mode 100644 index 000000000000..ef54ce1e6532 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-1.7.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce85fafb3e24712d62a0d02d277c2d56197d74afdd4f5ca995eaf33d2c504663 +size 53906809 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.7.20.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.7.20.jar new file mode 100644 index 000000000000..442aea3e3425 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-1.7.20.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e36d98c56f7c9685ab9d9e1fac9be36a5214939adb3f905b93c62de76023618 +size 54773087 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.8.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.8.0.jar new file mode 100644 index 000000000000..7884c5361b56 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-1.8.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f19f247d337387cbdd75d54e10a6865857c28a533fced50c0c5c9482b3ab9af +size 55128997 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.9.0-Beta.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.9.0-Beta.jar new file mode 100644 index 000000000000..8da3d202a5e4 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-1.9.0-Beta.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c7e3972e0ce0be8aa5c8ceeb8eb795f6345685bb57c6f59b649ed70c6051581 +size 60202336 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-1.9.20-Beta.jar b/java/kotlin-extractor/deps/kotlin-compiler-1.9.20-Beta.jar new file mode 100644 index 000000000000..57215c331e61 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-1.9.20-Beta.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b90980de3a16858e6e1957236d7bb9a729fcd0587a98fb64668866e1975aaa6f +size 61641342 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-2.0.0-RC1.jar b/java/kotlin-extractor/deps/kotlin-compiler-2.0.0-RC1.jar new file mode 100644 index 000000000000..fca4ae9fd36e --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-2.0.0-RC1.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60e31163aa7348166d708cdc9cb47d616b0222e5b11173f35f1adfb61bc3690a +size 58604870 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.0.jar new file mode 100644 index 000000000000..d6e7ef8585bc --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7b85448039e468daf3b9462a172244477fa3eb890f199ec77703992f36ade44 +size 46308890 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.10.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.10.jar new file mode 100644 index 000000000000..05f594d6af13 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.10.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9965f7c958efb17f2a90a19b5e60325d4f4e644d2833dbfb4a11edd8dddf244 +size 46314547 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.20.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.20.jar new file mode 100644 index 000000000000..3bb8ee5e52f7 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.20.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11d51087eb70b5abbad6fbf459a4349a0335916588000b5ecd990f01482e38ff +size 47526900 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.30.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.30.jar new file mode 100644 index 000000000000..647b08b7b7ab --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.5.30.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5051dc92725b099c41710bd3f213cd0c1d6f25056d31b2e8cae30903873b741 +size 48031832 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.0.jar new file mode 100644 index 000000000000..eafb6508d449 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0366843cd2defdd583c6b16b10bc32b85f28c5bf9510f10e44c886f5bd24c388 +size 49978431 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.20.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.20.jar new file mode 100644 index 000000000000..f426dae65b1e --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.6.20.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be634faaafb56816b6ef6d583e57ab33e4d6e5180cde2f505ccf7d45dc738ef8 +size 51976423 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.0.jar new file mode 100644 index 000000000000..4abc782ed05f --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6dcd02ced3bdbfabc7da046a69f4e730c8f907fc29bb5b2fea44083ffea22dc8 +size 142740530 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.20.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.20.jar new file mode 100644 index 000000000000..b95097b6935e --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.20.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ec2be1872dc47b9dcb466f1781eded6c59d9eee18657d4b0f1148e619caea36 +size 53395419 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.8.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.8.0.jar new file mode 100644 index 000000000000..5494dd88f974 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.8.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9b3a56dbbfdf1e0328d4731b7d7ca789ce0f1f263372ad88dd8decbd1602858 +size 53769272 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.0-Beta.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.0-Beta.jar new file mode 100644 index 000000000000..32ea1ec13146 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.0-Beta.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7429076f8dd2ccd1cce48d7e5bf5b9fadde8afab110f9f4cfe0912756f16d770 +size 58697460 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.20-Beta.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.20-Beta.jar new file mode 100644 index 000000000000..5c1db1782f8a --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.9.20-Beta.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c9de79f0f8d97f2aa4d877449063b1cc2828c17f25a119fc32c776401f058de +size 60157805 diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.0-RC1.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.0-RC1.jar new file mode 100644 index 000000000000..2f65ae3aaed0 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-2.0.0-RC1.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12d330c6e98ba42840f1d10e2752a9c53099d4dfa855bc4526d23fe1a1af9634 +size 57125659 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.5.0.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.5.0.jar new file mode 100644 index 000000000000..83d4a9f9c031 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-1.5.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52283996fe4067cd7330288b96ae67ecd463614dc741172c54d9d349ab6a9cd7 +size 1497598 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.5.10.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.5.10.jar new file mode 100644 index 000000000000..a0b08eaf499a --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-1.5.10.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca87c454cd3f2e60931f1803c59699d510d3b4b959cd7119296fb947581d722d +size 1497600 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.5.20.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.5.20.jar new file mode 100644 index 000000000000..90844bea3812 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-1.5.20.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80cd79c26aac46d72d782de1ecb326061e93c6e688d994b48627ffd668ba63a8 +size 1497567 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.5.30.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.5.30.jar new file mode 100644 index 000000000000..b19606b84967 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-1.5.30.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c55608e9eb6df7327e74b21e271d324dc523cef31587b8d6d2393db08d6e000c +size 1505951 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.6.0.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.6.0.jar new file mode 100644 index 000000000000..3ad56eecdc2d --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-1.6.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:115daea30b0d484afcf2360237b9d9537f48a4a2f03f3cc2a16577dfc6e90342 +size 1508076 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.6.20.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.6.20.jar new file mode 100644 index 000000000000..b170c1da738b --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-1.6.20.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eeb51c2b67b26233fd81d0bc4f8044ec849718890905763ceffd84a31e2cb799 +size 1509405 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.7.0.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.7.0.jar new file mode 100644 index 000000000000..bed18f8429b7 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-1.7.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa88e9625577957f3249a46cb6e166ee09b369e600f7a11d148d16b0a6d87f05 +size 1524619 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.7.20.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.7.20.jar new file mode 100644 index 000000000000..9998f1e65920 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-1.7.20.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7779ec96b9acbf92ca023858ac04543f9d2c3bdf1722425fff42f25ff3acfc9b +size 1537347 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.8.0.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.8.0.jar new file mode 100644 index 000000000000..47dd2710887b --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-1.8.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c77bef8774640b9fb9d6e217459ff220dae59878beb7d2e4b430506feffc654e +size 1636558 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.9.0-Beta.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.9.0-Beta.jar new file mode 100644 index 000000000000..c6f3f947029f --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-1.9.0-Beta.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af458cc55cf69e966668e6010c7ccee4a50d553b3504a2e8311dd0c76242d881 +size 1708001 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-1.9.20-Beta.jar b/java/kotlin-extractor/deps/kotlin-stdlib-1.9.20-Beta.jar new file mode 100644 index 000000000000..f9c853bbb068 --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-1.9.20-Beta.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:788e48813dd76ad598fff7bef3b1e038d7291741810bd04c6c57037c4d75dac2 +size 1715901 diff --git a/java/kotlin-extractor/deps/kotlin-stdlib-2.0.0-RC1.jar b/java/kotlin-extractor/deps/kotlin-stdlib-2.0.0-RC1.jar new file mode 100644 index 000000000000..fbf0a8eff06f --- /dev/null +++ b/java/kotlin-extractor/deps/kotlin-stdlib-2.0.0-RC1.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbb2c9b813e6196f9afa9f9add8b395ee384ab1763a0880e084d2942214f1c30 +size 1729731 diff --git a/java/kotlin-extractor/versions.bzl b/java/kotlin-extractor/versions.bzl index b3f2d4dca147..728672ead6de 100644 --- a/java/kotlin-extractor/versions.bzl +++ b/java/kotlin-extractor/versions.bzl @@ -1,70 +1,18 @@ # when updating this list, `bazel mod tidy` should be run from `codeql` to update `MODULE.bazel` -# when adding a new version, start out with {}, then run `bazel fetch //java/kotlin-extractor:all` -# (`bazel fetch @codeql//java/kotlin-extractor:all` from `semmle-code`). -# Repeat while it will fail printing what to add here. -# Alternatively, run `sha256sum` on the files to get what to write here. -VERSIONS = { - "1.5.0": { - "compiler": "f6e4a2c4394e77f937fcffda0036531604f25cc7c8de8daea098e1aa31f1d248", - "compiler-embeddable": "d7b85448039e468daf3b9462a172244477fa3eb890f199ec77703992f36ade44", - "stdlib": "52283996fe4067cd7330288b96ae67ecd463614dc741172c54d9d349ab6a9cd7", - }, - "1.5.10": { - "compiler": "dfefb1aa8bec81256617c8ceb577373e44078b7e21024625da50e376037e9ae5", - "compiler-embeddable": "b9965f7c958efb17f2a90a19b5e60325d4f4e644d2833dbfb4a11edd8dddf244", - "stdlib": "ca87c454cd3f2e60931f1803c59699d510d3b4b959cd7119296fb947581d722d", - }, - "1.5.20": { - "compiler": "a0ae437d1b670a5ba6da7893b7023df649c4ab2e6c19d5e9b4eee5332e1cde1f", - "compiler-embeddable": "11d51087eb70b5abbad6fbf459a4349a0335916588000b5ecd990f01482e38ff", - "stdlib": "80cd79c26aac46d72d782de1ecb326061e93c6e688d994b48627ffd668ba63a8", - }, - "1.5.30": { - "compiler": "487d8ff9766a6ba570cd15c5225c1600654e7cf1b6ef2b92ed6905528a3e838a", - "compiler-embeddable": "b5051dc92725b099c41710bd3f213cd0c1d6f25056d31b2e8cae30903873b741", - "stdlib": "c55608e9eb6df7327e74b21e271d324dc523cef31587b8d6d2393db08d6e000c", - }, - "1.6.0": { - "compiler": "4bd7a92568fd89c23b7f9f36d4380886beed18d3d54ea6adf49bebae627db805", - "compiler-embeddable": "0366843cd2defdd583c6b16b10bc32b85f28c5bf9510f10e44c886f5bd24c388", - "stdlib": "115daea30b0d484afcf2360237b9d9537f48a4a2f03f3cc2a16577dfc6e90342", - }, - "1.6.20": { - "compiler": "90567c5cf297985d028fa39aa3a7904dc8096173e1c7f3d3f35fe7074581098e", - "compiler-embeddable": "be634faaafb56816b6ef6d583e57ab33e4d6e5180cde2f505ccf7d45dc738ef8", - "stdlib": "eeb51c2b67b26233fd81d0bc4f8044ec849718890905763ceffd84a31e2cb799", - }, - "1.7.0": { - "compiler": "ce85fafb3e24712d62a0d02d277c2d56197d74afdd4f5ca995eaf33d2c504663", - "compiler-embeddable": "573935b492e65b93a792eaa6270295532f580cd4f26f9f6eb105ecbafcd182d4", - "stdlib": "aa88e9625577957f3249a46cb6e166ee09b369e600f7a11d148d16b0a6d87f05", - }, - "1.7.20": { - "compiler": "0e36d98c56f7c9685ab9d9e1fac9be36a5214939adb3f905b93c62de76023618", - "compiler-embeddable": "5ec2be1872dc47b9dcb466f1781eded6c59d9eee18657d4b0f1148e619caea36", - "stdlib": "7779ec96b9acbf92ca023858ac04543f9d2c3bdf1722425fff42f25ff3acfc9b", - }, - "1.8.0": { - "compiler": "1f19f247d337387cbdd75d54e10a6865857c28a533fced50c0c5c9482b3ab9af", - "compiler-embeddable": "e9b3a56dbbfdf1e0328d4731b7d7ca789ce0f1f263372ad88dd8decbd1602858", - "stdlib": "c77bef8774640b9fb9d6e217459ff220dae59878beb7d2e4b430506feffc654e", - }, - "1.9.0-Beta": { - "compiler": "4c7e3972e0ce0be8aa5c8ceeb8eb795f6345685bb57c6f59b649ed70c6051581", - "compiler-embeddable": "7429076f8dd2ccd1cce48d7e5bf5b9fadde8afab110f9f4cfe0912756f16d770", - "stdlib": "af458cc55cf69e966668e6010c7ccee4a50d553b3504a2e8311dd0c76242d881", - }, - "1.9.20-Beta": { - "compiler": "b90980de3a16858e6e1957236d7bb9a729fcd0587a98fb64668866e1975aaa6f", - "compiler-embeddable": "5c9de79f0f8d97f2aa4d877449063b1cc2828c17f25a119fc32c776401f058de", - "stdlib": "788e48813dd76ad598fff7bef3b1e038d7291741810bd04c6c57037c4d75dac2", - }, - "2.0.0-RC1": { - "compiler": "60e31163aa7348166d708cdc9cb47d616b0222e5b11173f35f1adfb61bc3690a", - "compiler-embeddable": "12d330c6e98ba42840f1d10e2752a9c53099d4dfa855bc4526d23fe1a1af9634", - "stdlib": "bbb2c9b813e6196f9afa9f9add8b395ee384ab1763a0880e084d2942214f1c30", - }, -} +VERSIONS = [ + "1.5.0", + "1.5.10", + "1.5.20", + "1.5.30", + "1.6.0", + "1.6.20", + "1.7.0", + "1.7.20", + "1.8.0", + "1.9.0-Beta", + "1.9.20-Beta", + "2.0.0-RC1", +] def _version_to_tuple(v): # we ignore the tag when comparing versions, for example 1.9.0-Beta <= 1.9.0 From e546560d7cdc8dbca5898859a4fe2dd07b88c67d Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 6 May 2024 17:37:16 +0200 Subject: [PATCH 68/81] Kotlin: restore `kotlin-compiler-embeddable-1.7.0.jar` to the maven central version --- .../deps/kotlin-compiler-embeddable-1.7.0.jar | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.0.jar b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.0.jar index 4abc782ed05f..0857bed8c9e1 100644 --- a/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.0.jar +++ b/java/kotlin-extractor/deps/kotlin-compiler-embeddable-1.7.0.jar @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6dcd02ced3bdbfabc7da046a69f4e730c8f907fc29bb5b2fea44083ffea22dc8 -size 142740530 +oid sha256:573935b492e65b93a792eaa6270295532f580cd4f26f9f6eb105ecbafcd182d4 +size 52514265 From 99f70a64e9c98b96d5ce33ef09a76ba149aea24e Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 4 Jun 2024 09:00:35 +0200 Subject: [PATCH 69/81] Kotlin: address review comments --- java/kotlin-extractor/BUILD.bazel | 7 ++++--- java/kotlin-extractor/deps.bzl | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/java/kotlin-extractor/BUILD.bazel b/java/kotlin-extractor/BUILD.bazel index 16cff368e177..a37c04683d66 100644 --- a/java/kotlin-extractor/BUILD.bazel +++ b/java/kotlin-extractor/BUILD.bazel @@ -2,12 +2,11 @@ # Usage overview Building the extractor can be done via ``` -bazel build //java/kotlin-extractor:codeql-extractor-kotlin-- +bazel build @codeql//java/kotlin-extractor:codeql-extractor-kotlin-- ``` where `` is either `standalone` or `embeddable`, and `` is one of the supported versions. - ``` -bazel build //java/kotlin-extractor +bazel build @codeql//java/kotlin-extractor ``` will build a default variant: * standalone, unless `CODEQL_KOTLIN_SINGLE_VERSION_EMBEDDABLE` is set to true, in which case it will go for embeddable @@ -19,6 +18,8 @@ If `kotlinc` is updated, bazel won't be aware of it and will therefore keep the * `bazel clean` * `bazel fetch --force @codeql_kotlin_defaults\\:all` * `CODEQL_KOTLIN_SINGLE_VERSION= bazel build //java/kotlin-extractor` + +If building from the `codeql` repository, `@codeql` can be skipped. """ # This file is used in the `@codeql_kotlin_embeddable` external repo, which means we need to diff --git a/java/kotlin-extractor/deps.bzl b/java/kotlin-extractor/deps.bzl index 049c4a25cada..7ba3ec1f8b7a 100644 --- a/java/kotlin-extractor/deps.bzl +++ b/java/kotlin-extractor/deps.bzl @@ -49,8 +49,8 @@ def _walk(dir): next_dirs.extend([c for c in children if c.is_dir]) res.extend([c for c in children if not c.is_dir]) if not next_dirs: - break - return res + return res + fail("%s directory too deep" % dir) def _embeddable_source_impl(repository_ctx): src_dir = repository_ctx.path(Label("//java/kotlin-extractor:src")) From fcd326eb11ebf77b747bd584c5f84590d7f5c331 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 4 Jun 2024 14:14:23 +0200 Subject: [PATCH 70/81] CI: reword comment --- .github/workflows/kotlin-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/kotlin-build.yml b/.github/workflows/kotlin-build.yml index 0ecc63a50859..565c3d3a8ba4 100644 --- a/.github/workflows/kotlin-build.yml +++ b/.github/workflows/kotlin-build.yml @@ -23,6 +23,6 @@ jobs: - uses: actions/checkout@v4 - run: | bazel query //java/kotlin-extractor/... - # only build the default version as a sanity check that we can build from `codeql` + # only build the default version as a quick check that we can build from `codeql` # the full official build will be checked by QLucie bazel build //java/kotlin-extractor From 6a9cb90a5738482ce8eb4264a3e916871792868f Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 4 Jun 2024 14:21:58 +0200 Subject: [PATCH 71/81] Kotlin: rework header comment --- java/kotlin-extractor/BUILD.bazel | 9 +- java/kotlin-extractor/deps/LICENSE.md | 6 +- java/kotlin-extractor/deps/license | 1675 +++++++++++++++++++++++++ 3 files changed, 1683 insertions(+), 7 deletions(-) create mode 100644 java/kotlin-extractor/deps/license diff --git a/java/kotlin-extractor/BUILD.bazel b/java/kotlin-extractor/BUILD.bazel index a37c04683d66..18136727c8ad 100644 --- a/java/kotlin-extractor/BUILD.bazel +++ b/java/kotlin-extractor/BUILD.bazel @@ -1,6 +1,9 @@ """ # Usage overview -Building the extractor can be done via +Building the extractor can be done with bazel. If building from the internal repository, it is recommended to use +`tools/bazel` from there. + +A specific kotlin extractor variant can be built with ``` bazel build @codeql//java/kotlin-extractor:codeql-extractor-kotlin-- ``` @@ -16,8 +19,8 @@ will build a default variant: If `kotlinc` is updated, bazel won't be aware of it and will therefore keep the same default version. Possible workarounds for that: * `bazel clean` -* `bazel fetch --force @codeql_kotlin_defaults\\:all` -* `CODEQL_KOTLIN_SINGLE_VERSION= bazel build //java/kotlin-extractor` +* `bazel fetch --force @codeql//java/kotlin-extractor` +* `bazel fetch --force @codeql_kotlin_defaults//:all` (only from `codeql`) If building from the `codeql` repository, `@codeql` can be skipped. """ diff --git a/java/kotlin-extractor/deps/LICENSE.md b/java/kotlin-extractor/deps/LICENSE.md index a93f66f0aa30..81a6185e3c00 100644 --- a/java/kotlin-extractor/deps/LICENSE.md +++ b/java/kotlin-extractor/deps/LICENSE.md @@ -1,7 +1,5 @@ The Git LFS files contained in this directory are mirrored -from [org.jetbrains.kotlin packages in the Maven repository][1]. Please refer to [the kotlin Apache 2.0 license][2] for -details about their license. +from [org.jetbrains.kotlin packages in the Maven repository][1]. A copy of the license is included as +the [`license`](./license) file. [1]: https://mvnrepository.com/artifact/org.jetbrains.kotlin - -[2]: https://github.com/JetBrains/kotlin/tree/master/license diff --git a/java/kotlin-extractor/deps/license b/java/kotlin-extractor/deps/license new file mode 100644 index 000000000000..28ffe454312c --- /dev/null +++ b/java/kotlin-extractor/deps/license @@ -0,0 +1,1675 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + kotlin/license at master · JetBrains/kotlin · GitHub + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + +
+ Skip to content + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + +
+ +
+ + + + + + + + +
+ + + + + + +
+ + + + + + + + + +
+
+
+ + + + + + + + + + + + +
+ +
+ +
+ +
+ + + + / + + kotlin + + + Public +
+ + +
+ +
+ + +
+
+ +
+
+ + + + +
+ + + + + + +
+ + + + + + + + + + + + + + + + +
+ + + + + +
+ + + +
+
+ +
+ +
+

Footer

+ + + + +
+
+ + + + + © 2024 GitHub, Inc. + +
+ + +
+
+ + + + + + + + + + + + + + + + + + + +
+ +
+
+ + From 2fe0718e495baccfde6a807116670ffee216b189 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 4 Jun 2024 14:27:08 +0200 Subject: [PATCH 72/81] Explain `.lfsconfig` choice in the comment --- .lfsconfig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.lfsconfig b/.lfsconfig index cb0a8e352e86..d8a03ef83f23 100644 --- a/.lfsconfig +++ b/.lfsconfig @@ -2,4 +2,6 @@ # codeql is publicly forked by many users, and we don't want any LFS file polluting their working # copies. We therefore exclude everything by default. # For files required by bazel builds, use rules in `misc/bazel/lfs.bzl` to download them on demand. +# we go for `fetchinclude` to something not exsiting rather than `fetchexclude = *` because the +# former is easier to override (with `git -c` or a local git config) to fetch something specific fetchinclude = /nothing From 4c91bdce48e632b04ebb190fa111d1cb6d8debde Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 4 Jun 2024 14:27:33 +0200 Subject: [PATCH 73/81] Kotlin: `tgt` -> `target` --- java/kotlin-extractor/BUILD.bazel | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/java/kotlin-extractor/BUILD.bazel b/java/kotlin-extractor/BUILD.bazel index 18136727c8ad..b626ea40ac5b 100644 --- a/java/kotlin-extractor/BUILD.bazel +++ b/java/kotlin-extractor/BUILD.bazel @@ -90,14 +90,14 @@ kt_javac_options( outs = [ "%s/com/github/codeql/extractor.name" % v, ] + [ - "%s/%s" % (v, tgt) - for _, tgt in _resources + "%s/%s" % (v, target) + for _, target in _resources ], cmd = "\n".join([ "echo %s-%s > $(RULEDIR)/%s/com/github/codeql/extractor.name" % (_extractor_name_prefix, v, v), ] + [ - "cp $(execpath %s) $(RULEDIR)/%s/%s" % (src, v, tgt) - for src, tgt in _resources + "cp $(execpath %s) $(RULEDIR)/%s/%s" % (source, v, target) + for source, target in _resources ]), ), kt_jvm_library( From e7cec01a81839df67bb90ace6a7b6e63c22e93f5 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 4 Jun 2024 14:31:59 +0200 Subject: [PATCH 74/81] Kotlin: make `current_kotlin_version.py` executable --- java/kotlin-extractor/current_kotlin_version.py | 2 ++ 1 file changed, 2 insertions(+) mode change 100644 => 100755 java/kotlin-extractor/current_kotlin_version.py diff --git a/java/kotlin-extractor/current_kotlin_version.py b/java/kotlin-extractor/current_kotlin_version.py old mode 100644 new mode 100755 index 464745cf4964..9211042b02a3 --- a/java/kotlin-extractor/current_kotlin_version.py +++ b/java/kotlin-extractor/current_kotlin_version.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + import subprocess import re import shutil From e693c2719d4cdd1f5f650d03096a433b27b3cfbc Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 4 Jun 2024 14:39:02 +0200 Subject: [PATCH 75/81] Kotlin: remove unneeded patch --- .../patches/module_dot_bazel_version.patch | 12 ------------ .../modules/rules_kotlin/1.9.4-codeql.1/source.json | 1 - 2 files changed, 13 deletions(-) delete mode 100644 misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/patches/module_dot_bazel_version.patch diff --git a/misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/patches/module_dot_bazel_version.patch b/misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/patches/module_dot_bazel_version.patch deleted file mode 100644 index 7a33385b1702..000000000000 --- a/misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/patches/module_dot_bazel_version.patch +++ /dev/null @@ -1,12 +0,0 @@ -=================================================================== ---- a/MODULE.bazel -+++ b/MODULE.bazel -@@ -1,7 +1,7 @@ - module( - name = "rules_kotlin", -- version = "1.9.0", -+ version = "1.9.4-patched", - repo_name = "rules_kotlin", - ) - - bazel_dep(name = "platforms", version = "0.0.6") diff --git a/misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/source.json b/misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/source.json index 5941e8379534..5089476d754a 100644 --- a/misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/source.json +++ b/misc/bazel/registry/modules/rules_kotlin/1.9.4-codeql.1/source.json @@ -2,7 +2,6 @@ "integrity": "sha256-dsD8wsI+33NjIK3tGs2d3guuQY5XMd8Skz2IbLqGt5U=", "url": "https://github.com/bazelbuild/rules_kotlin/releases/download/v1.9.4/rules_kotlin-v1.9.4.tar.gz", "patches": { - "module_dot_bazel_version.patch": "sha256-0GnFHOv9wuuv3jFcHBSXrdo7JMFP7y66O5C4rccy5wg=", "codeql_do_not_emit_jdeps.patch": "sha256-x/HsujFlR1FGrgmbAbRZag9V4vKZZinBcs73tgRS478=", "codeql_add_language_version_option.patch": "sha256-qFpP/hIvqGzjJi0h8LAQK0UuWqwlj/oCecZYGqlMVP8=" }, From c6039b345b49a4168ddd275af0bfb83e99ea5ee8 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 4 Jun 2024 14:39:16 +0200 Subject: [PATCH 76/81] Kotlin: update default `kotlinc` version --- java/kotlin-extractor/dev/kotlinc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/kotlin-extractor/dev/kotlinc b/java/kotlin-extractor/dev/kotlinc index 929ceb2c9d0d..176544f12945 100755 --- a/java/kotlin-extractor/dev/kotlinc +++ b/java/kotlin-extractor/dev/kotlinc @@ -27,7 +27,7 @@ import shutil import io import os -DEFAULT_VERSION = "1.9.0" +DEFAULT_VERSION = "2.0.0" def options(): parser = argparse.ArgumentParser(add_help=False) From be5c82cb88f592cc9c93d67afe282fe112f8eb3b Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 4 Jun 2024 15:11:59 +0200 Subject: [PATCH 77/81] Kotlin: add other tools to dev wrapper --- java/kotlin-extractor/dev/kapt | 3 + java/kotlin-extractor/dev/kapt.bat | 4 + java/kotlin-extractor/dev/kotlin | 3 + java/kotlin-extractor/dev/kotlin.bat | 4 + java/kotlin-extractor/dev/kotlinc | 164 +------------------------ java/kotlin-extractor/dev/kotlinc.bat | 2 +- java/kotlin-extractor/dev/wrapper.py | 170 ++++++++++++++++++++++++++ 7 files changed, 187 insertions(+), 163 deletions(-) create mode 100755 java/kotlin-extractor/dev/kapt create mode 100644 java/kotlin-extractor/dev/kapt.bat create mode 100755 java/kotlin-extractor/dev/kotlin create mode 100644 java/kotlin-extractor/dev/kotlin.bat create mode 100755 java/kotlin-extractor/dev/wrapper.py diff --git a/java/kotlin-extractor/dev/kapt b/java/kotlin-extractor/dev/kapt new file mode 100755 index 000000000000..19528d866512 --- /dev/null +++ b/java/kotlin-extractor/dev/kapt @@ -0,0 +1,3 @@ +#!/bin/bash + +"$(dirname "$0")/wrapper.py" kapt "$@" diff --git a/java/kotlin-extractor/dev/kapt.bat b/java/kotlin-extractor/dev/kapt.bat new file mode 100644 index 000000000000..bb94ea0f72aa --- /dev/null +++ b/java/kotlin-extractor/dev/kapt.bat @@ -0,0 +1,4 @@ +@echo off + +python "%~dp0wrapper.py" kapt %* +exit /b %ERRORLEVEL% diff --git a/java/kotlin-extractor/dev/kotlin b/java/kotlin-extractor/dev/kotlin new file mode 100755 index 000000000000..cc1a4507e5a3 --- /dev/null +++ b/java/kotlin-extractor/dev/kotlin @@ -0,0 +1,3 @@ +#!/bin/bash + +"$(dirname "$0")/wrapper.py" kotlin "$@" diff --git a/java/kotlin-extractor/dev/kotlin.bat b/java/kotlin-extractor/dev/kotlin.bat new file mode 100644 index 000000000000..8cdf794bd5a0 --- /dev/null +++ b/java/kotlin-extractor/dev/kotlin.bat @@ -0,0 +1,4 @@ +@echo off + +python "%~dp0wrapper.py" kotlin %* +exit /b %ERRORLEVEL% diff --git a/java/kotlin-extractor/dev/kotlinc b/java/kotlin-extractor/dev/kotlinc index 176544f12945..2e3ebc134452 100755 --- a/java/kotlin-extractor/dev/kotlinc +++ b/java/kotlin-extractor/dev/kotlinc @@ -1,163 +1,3 @@ -#!/usr/bin/env python3 +#!/bin/bash -""" -Wrapper script that manages kotlinc versions. -Usage: add this directory to your PATH, then -* `kotlinc --select x.y.z` will select the version for the next invocations, checking it actually exists -* `kotlinc --clear` will remove any state of the wrapper (deselecting a previous version selection) -* `kotlinc -version` will print the selected version information. It will not print `JRE` information as a normal - `kotlinc` invocation would do though. In exchange, the invocation incurs no overhead. -* Any other invocation will forward to the selected kotlinc version, downloading it if necessary. If no version was - previously selected with `--select`, a default will be used (see `DEFAULT_VERSION` below) - -In order to install kotlin, ripunzip will be used if installed, or if running on Windows within `semmle-code` (ripunzip -is available in `resources/lib/windows/ripunzip` then). -""" - -import pathlib -import urllib -import urllib.request -import urllib.error -import argparse -import sys -import platform -import subprocess -import zipfile -import shutil -import io -import os - -DEFAULT_VERSION = "2.0.0" - -def options(): - parser = argparse.ArgumentParser(add_help=False) - parser.add_argument("--select") - parser.add_argument("--clear", action="store_true") - parser.add_argument("-version", action="store_true") - return parser.parse_known_args() - - -url_template = 'https://github.com/JetBrains/kotlin/releases/download/v{version}/kotlin-compiler-{version}.zip' -this_dir = pathlib.Path(__file__).resolve().parent -version_file = this_dir / ".kotlinc_version" -install_dir = this_dir / ".kotlinc_installed" -windows_ripunzip = this_dir.parents[4] / "resources" / "lib" / "windows" / "ripunzip" / "ripunzip.exe" - - -class Error(Exception): - pass - - -class ZipFilePreservingPermissions(zipfile.ZipFile): - def _extract_member(self, member, targetpath, pwd): - if not isinstance(member, zipfile.ZipInfo): - member = self.getinfo(member) - - targetpath = super()._extract_member(member, targetpath, pwd) - - attr = member.external_attr >> 16 - if attr != 0: - os.chmod(targetpath, attr) - return targetpath - - -def check_version(version: str): - try: - with urllib.request.urlopen(url_template.format(version=version)) as response: - pass - except urllib.error.HTTPError as e: - if e.code == 404: - raise Error(f"Version {version} not found in github.com/JetBrains/kotlin/releases") from e - raise - - -def get_version(): - try: - return version_file.read_text() - except FileNotFoundError: - return None - - -def install(version: str, quiet: bool): - if quiet: - info_out = subprocess.DEVNULL - info = lambda *args: None - else: - info_out = sys.stderr - info = lambda *args: print(*args, file=sys.stderr) - url = url_template.format(version=version) - if install_dir.exists(): - shutil.rmtree(install_dir) - install_dir.mkdir() - ripunzip = shutil.which("ripunzip") - if ripunzip is None and platform.system() == "Windows" and windows_ripunzip.exists(): - ripunzip = windows_ripunzip - if ripunzip: - info(f"downloading and extracting {url} using ripunzip") - subprocess.run([ripunzip, "unzip-uri", url], stdout=info_out, stderr=info_out, cwd=install_dir, - check=True) - return - with io.BytesIO() as buffer: - info(f"downloading {url}") - with urllib.request.urlopen(url) as response: - while True: - bytes = response.read() - if not bytes: - break - buffer.write(bytes) - buffer.seek(0) - info(f"extracting kotlin-compiler-{version}.zip") - with ZipFilePreservingPermissions(buffer) as archive: - archive.extractall(install_dir) - - -def forward(forwarded_opts): - kotlinc = install_dir / "kotlinc" / "bin" / "kotlinc" - if platform.system() == "Windows": - kotlinc = kotlinc.with_suffix(".bat") - assert kotlinc.exists(), f"{kotlinc} not found" - args = [kotlinc] - args.extend(forwarded_opts) - ret = subprocess.run(args).returncode - sys.exit(ret) - - -def clear(): - if install_dir.exists(): - print(f"removing {install_dir}", file=sys.stderr) - shutil.rmtree(install_dir) - if version_file.exists(): - print(f"removing {version_file}", file=sys.stderr) - version_file.unlink() - - -def main(opts, forwarded_opts): - if opts.clear: - clear() - return - current_version = get_version() - if opts.select == "default": - selected_version = DEFAULT_VERSION - elif opts.select is not None: - check_version(opts.select) - selected_version = opts.select - else: - selected_version = current_version or DEFAULT_VERSION - if selected_version != current_version: - # don't print information about install procedure unless explicitly using --select - install(selected_version, quiet=opts.select is None) - version_file.write_text(selected_version) - if opts.version or (opts.select and not forwarded_opts): - print(f"info: kotlinc-jvm {selected_version} (codeql dev wrapper)", file=sys.stderr) - return - forward(forwarded_opts) - - -if __name__ == "__main__": - try: - main(*options()) - except Error as e: - print(f"Error: {e}", file=sys.stderr) - sys.exit(1) - except KeyboardInterrupt: - sys.exit(1) +"$(dirname "$0")/wrapper.py" kotlinc "$@" diff --git a/java/kotlin-extractor/dev/kotlinc.bat b/java/kotlin-extractor/dev/kotlinc.bat index b19940d995d0..6a8857082a07 100644 --- a/java/kotlin-extractor/dev/kotlinc.bat +++ b/java/kotlin-extractor/dev/kotlinc.bat @@ -1,4 +1,4 @@ @echo off -python "%~dp0kotlinc" %* +python "%~dp0wrapper.py" kotlinc %* exit /b %ERRORLEVEL% diff --git a/java/kotlin-extractor/dev/wrapper.py b/java/kotlin-extractor/dev/wrapper.py new file mode 100755 index 000000000000..13a4d1aca602 --- /dev/null +++ b/java/kotlin-extractor/dev/wrapper.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 + +""" +Wrapper script that manages kotlin versions. +Usage: add this directory to your PATH, then +* `kotlin* --select x.y.z` will select the version for the next invocations, checking it actually exists +* `kotlin* --clear` will remove any state of the wrapper (deselecting a previous version selection) +* `kotlinc -version` will print the selected version information. It will not print `JRE` information as a normal + `kotlinc` invocation would do though. In exchange, the invocation incurs no overhead. +* Any other invocation will forward to the selected kotlin tool version, downloading it if necessary. If no version was + previously selected with `--select`, a default will be used (see `DEFAULT_VERSION` below) + +In order to install kotlin, ripunzip will be used if installed, or if running on Windows within `semmle-code` (ripunzip +is available in `resources/lib/windows/ripunzip` then). +""" + +import pathlib +import urllib +import urllib.request +import urllib.error +import argparse +import sys +import platform +import subprocess +import zipfile +import shutil +import io +import os + +DEFAULT_VERSION = "2.0.0" + +def options(): + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("tool") + parser.add_argument("--select") + parser.add_argument("--clear", action="store_true") + parser.add_argument("-version", action="store_true") + return parser.parse_known_args() + + +url_template = 'https://github.com/JetBrains/kotlin/releases/download/v{version}/kotlin-compiler-{version}.zip' +this_dir = pathlib.Path(__file__).resolve().parent +version_file = this_dir / ".kotlinc_version" +install_dir = this_dir / ".kotlinc_installed" +windows_ripunzip = this_dir.parents[4] / "resources" / "lib" / "windows" / "ripunzip" / "ripunzip.exe" + + +class Error(Exception): + pass + + +class ZipFilePreservingPermissions(zipfile.ZipFile): + def _extract_member(self, member, targetpath, pwd): + if not isinstance(member, zipfile.ZipInfo): + member = self.getinfo(member) + + targetpath = super()._extract_member(member, targetpath, pwd) + + attr = member.external_attr >> 16 + if attr != 0: + os.chmod(targetpath, attr) + return targetpath + + +def check_version(version: str): + try: + with urllib.request.urlopen(url_template.format(version=version)) as response: + pass + except urllib.error.HTTPError as e: + if e.code == 404: + raise Error(f"Version {version} not found in github.com/JetBrains/kotlin/releases") from e + raise + + +def get_version(): + try: + return version_file.read_text() + except FileNotFoundError: + return None + + +def install(version: str, quiet: bool): + if quiet: + info_out = subprocess.DEVNULL + info = lambda *args: None + else: + info_out = sys.stderr + info = lambda *args: print(*args, file=sys.stderr) + url = url_template.format(version=version) + if install_dir.exists(): + shutil.rmtree(install_dir) + install_dir.mkdir() + ripunzip = shutil.which("ripunzip") + if ripunzip is None and platform.system() == "Windows" and windows_ripunzip.exists(): + ripunzip = windows_ripunzip + if ripunzip: + info(f"downloading and extracting {url} using ripunzip") + subprocess.run([ripunzip, "unzip-uri", url], stdout=info_out, stderr=info_out, cwd=install_dir, + check=True) + return + with io.BytesIO() as buffer: + info(f"downloading {url}") + with urllib.request.urlopen(url) as response: + while True: + bytes = response.read() + if not bytes: + break + buffer.write(bytes) + buffer.seek(0) + info(f"extracting kotlin-compiler-{version}.zip") + with ZipFilePreservingPermissions(buffer) as archive: + archive.extractall(install_dir) + + +def forward(tool, forwarded_opts): + tool = install_dir / "kotlinc" / "bin" / tool + if platform.system() == "Windows": + tool = tool.with_suffix(".bat") + assert tool.exists(), f"{tool} not found" + args = [tool] + args.extend(forwarded_opts) + ret = subprocess.run(args).returncode + sys.exit(ret) + + +def clear(): + if install_dir.exists(): + print(f"removing {install_dir}", file=sys.stderr) + shutil.rmtree(install_dir) + if version_file.exists(): + print(f"removing {version_file}", file=sys.stderr) + version_file.unlink() + + +def main(opts, forwarded_opts): + if opts.clear: + clear() + return + current_version = get_version() + if opts.select == "default": + selected_version = DEFAULT_VERSION + elif opts.select is not None: + check_version(opts.select) + selected_version = opts.select + else: + selected_version = current_version or DEFAULT_VERSION + if selected_version != current_version: + # don't print information about install procedure unless explicitly using --select + install(selected_version, quiet=opts.select is None) + version_file.write_text(selected_version) + if opts.select and not forwarded_opts and not opts.version: + print(f"selected {selected_version}") + return + if opts.version: + if opts.tool == "kotlinc": + print(f"info: kotlinc-jvm {selected_version} (codeql dev wrapper)", file=sys.stderr) + return + forwarded_opts.append("-version") + + forward(opts.tool, forwarded_opts) + + +if __name__ == "__main__": + try: + main(*options()) + except Error as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + except KeyboardInterrupt: + sys.exit(1) From 8205f86365f8dea6b2356a557f24f52d9fb31849 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 4 Jun 2024 15:12:23 +0200 Subject: [PATCH 78/81] Kotlin: use `tools/bazel` in integration test --- .../linux-only/kotlin/custom_plugin/test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/java/ql/integration-tests/linux-only/kotlin/custom_plugin/test.py b/java/ql/integration-tests/linux-only/kotlin/custom_plugin/test.py index f8da2c838e83..eba36ddca0c2 100644 --- a/java/ql/integration-tests/linux-only/kotlin/custom_plugin/test.py +++ b/java/ql/integration-tests/linux-only/kotlin/custom_plugin/test.py @@ -3,17 +3,17 @@ import pathlib import shutil -this_dir = pathlib.Path(__file__).resolve().parent +root = get_semmle_code_path() cwd = pathlib.Path.cwd() builddir = cwd / 'build' builddir.mkdir(exist_ok=True) try: - runSuccessfully(['bazel', f'--output_user_root={builddir}', '--max_idle_secs=1', 'build', + runSuccessfully([f'{root}/tools/bazel', f'--output_user_root={builddir}', '--max_idle_secs=1', 'build', '//java/ql/integration-tests/linux-only/kotlin/custom_plugin/plugin', '--spawn_strategy=local', '--nouse_action_cache', '--noremote_accept_cached', '--noremote_upload_local_results', - f'--symlink_prefix={cwd / "bazel-"}'], cwd=this_dir) + f'--symlink_prefix={cwd / "bazel-"}'], cwd=root) finally: # rules_python creates a read-only directory in bazel's output, this allows cleanup to succeed runSuccessfully(['chmod', '-R', '+w', builddir]) From 10584b3a7a2af8016a456ae77fdd738c2367dfce Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 4 Jun 2024 15:36:35 +0200 Subject: [PATCH 79/81] Kotlin: improve posix dev wrappers --- java/kotlin-extractor/dev/kapt | 2 +- java/kotlin-extractor/dev/kotlin | 2 +- java/kotlin-extractor/dev/kotlinc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/java/kotlin-extractor/dev/kapt b/java/kotlin-extractor/dev/kapt index 19528d866512..c6025a06cffe 100755 --- a/java/kotlin-extractor/dev/kapt +++ b/java/kotlin-extractor/dev/kapt @@ -1,3 +1,3 @@ #!/bin/bash -"$(dirname "$0")/wrapper.py" kapt "$@" +exec -a "$0" "$(dirname "$0")/wrapper.py" kapt "$@" diff --git a/java/kotlin-extractor/dev/kotlin b/java/kotlin-extractor/dev/kotlin index cc1a4507e5a3..cabef3a8b2bb 100755 --- a/java/kotlin-extractor/dev/kotlin +++ b/java/kotlin-extractor/dev/kotlin @@ -1,3 +1,3 @@ #!/bin/bash -"$(dirname "$0")/wrapper.py" kotlin "$@" +exec -a "$0" "$(dirname "$0")/wrapper.py" kotlin "$@" diff --git a/java/kotlin-extractor/dev/kotlinc b/java/kotlin-extractor/dev/kotlinc index 2e3ebc134452..c63707432125 100755 --- a/java/kotlin-extractor/dev/kotlinc +++ b/java/kotlin-extractor/dev/kotlinc @@ -1,3 +1,3 @@ #!/bin/bash -"$(dirname "$0")/wrapper.py" kotlinc "$@" +exec -a "$0" "$(dirname "$0")/wrapper.py" kotlinc "$@" From e4653a80d4977b6168ed6a41b4f6e4bed276bb55 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 4 Jun 2024 17:55:19 +0200 Subject: [PATCH 80/81] Kotlin: remove `kapt` wrapper, probably unneeded --- java/kotlin-extractor/dev/kapt | 3 --- java/kotlin-extractor/dev/kapt.bat | 4 ---- 2 files changed, 7 deletions(-) delete mode 100755 java/kotlin-extractor/dev/kapt delete mode 100644 java/kotlin-extractor/dev/kapt.bat diff --git a/java/kotlin-extractor/dev/kapt b/java/kotlin-extractor/dev/kapt deleted file mode 100755 index c6025a06cffe..000000000000 --- a/java/kotlin-extractor/dev/kapt +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -exec -a "$0" "$(dirname "$0")/wrapper.py" kapt "$@" diff --git a/java/kotlin-extractor/dev/kapt.bat b/java/kotlin-extractor/dev/kapt.bat deleted file mode 100644 index bb94ea0f72aa..000000000000 --- a/java/kotlin-extractor/dev/kapt.bat +++ /dev/null @@ -1,4 +0,0 @@ -@echo off - -python "%~dp0wrapper.py" kapt %* -exit /b %ERRORLEVEL% From b7e16ca55dd1a9544d3b599775e1a1acc5d98e69 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 4 Jun 2024 17:59:13 +0200 Subject: [PATCH 81/81] Kotlin: fix `custom_plugin` test --- .../linux-only/kotlin/custom_plugin/test.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/java/ql/integration-tests/linux-only/kotlin/custom_plugin/test.py b/java/ql/integration-tests/linux-only/kotlin/custom_plugin/test.py index eba36ddca0c2..55d24e016076 100644 --- a/java/ql/integration-tests/linux-only/kotlin/custom_plugin/test.py +++ b/java/ql/integration-tests/linux-only/kotlin/custom_plugin/test.py @@ -3,17 +3,18 @@ import pathlib import shutil -root = get_semmle_code_path() +this_dir = pathlib.Path(__file__).resolve().parent cwd = pathlib.Path.cwd() builddir = cwd / 'build' builddir.mkdir(exist_ok=True) try: - runSuccessfully([f'{root}/tools/bazel', f'--output_user_root={builddir}', '--max_idle_secs=1', 'build', - '//java/ql/integration-tests/linux-only/kotlin/custom_plugin/plugin', '--spawn_strategy=local', - '--nouse_action_cache', '--noremote_accept_cached', '--noremote_upload_local_results', - f'--symlink_prefix={cwd / "bazel-"}'], cwd=root) + runSuccessfully( + [f'{get_semmle_code_path()}/tools/bazel', f'--output_user_root={builddir}', '--max_idle_secs=1', 'build', + '//java/ql/integration-tests/linux-only/kotlin/custom_plugin/plugin', '--spawn_strategy=local', + '--nouse_action_cache', '--noremote_accept_cached', '--noremote_upload_local_results', + f'--symlink_prefix={cwd / "bazel-"}'], cwd=this_dir) finally: # rules_python creates a read-only directory in bazel's output, this allows cleanup to succeed runSuccessfully(['chmod', '-R', '+w', builddir])