Bazel: add codeql specific packaging library

misc/bazel/internal/install.py

@@ -23,18 +23,18 @@
                     help="The wrapped `pkg_install` installation script rlocation")
 parser.add_argument("--build-file", required=True,
                     help="BUILD.bazel rlocation relative to which the installation should take place")
-parser.add_argument("--ripunzip", required=True,
-                    help="ripunzip executable rlocation")
-parser.add_argument("--zip-manifest", required=True,
+parser.add_argument("--ripunzip",
+                    help="ripunzip executable rlocation. Must be provided if `--zip-manifest` is.")
+parser.add_argument("--zip-manifest",
                     help="The rlocation of a file containing newline-separated `prefix:zip_file` entries")
 parser.add_argument("--cleanup", action=argparse.BooleanOptionalAction, default=True,
                     help="Whether to wipe the destination directory before installing (true by default)")
 opts = parser.parse_args()
+if opts.zip_manifest and not opts.ripunzip:
+    parser.error("Provide `--ripunzip` when specifying `--zip-manifest`")
 
 build_file = runfiles.Rlocation(opts.build_file)
 script = runfiles.Rlocation(opts.pkg_install_script)
-ripunzip = runfiles.Rlocation(opts.ripunzip)
-zip_manifest = runfiles.Rlocation(opts.zip_manifest)
 destdir = pathlib.Path(build_file).resolve().parent / opts.destdir
 
 if destdir.exists() and opts.cleanup:
@@ -43,10 +43,13 @@
 destdir.mkdir(parents=True, exist_ok=True)
 subprocess.run([script, "--destdir", destdir], check=True)
 
-with open(zip_manifest) as manifest:
-    for line in manifest:
-        prefix, _, zip = line.partition(":")
-        assert zip, f"missing prefix for {prefix}, you should use prefix:zip format"
-        dest = destdir / prefix
-        dest.mkdir(parents=True, exist_ok=True)
-        subprocess.run([ripunzip, "unzip-file", zip, "-d", dest], check=True)
+if opts.zip_manifest:
+    ripunzip = runfiles.Rlocation(opts.ripunzip)
+    zip_manifest = runfiles.Rlocation(opts.zip_manifest)
+    with open(zip_manifest) as manifest:
+        for line in manifest:
+            prefix, _, zip = line.partition(":")
+            assert zip, f"missing prefix for {prefix}, you should use prefix:zip format"
+            dest = destdir / prefix
+            dest.mkdir(parents=True, exist_ok=True)
+            subprocess.run([ripunzip, "unzip-file", zip, "-d", dest], check=True)

misc/bazel/pkg.bzl

@@ -21,19 +21,34 @@ _PLAT_DETECTION_ATTRS = {
 
 _PLAT_PLACEHOLDER = "{CODEQL_PLATFORM}"
 
-def _process_path(path, platform):
+def _expand_path(path, platform):
     if _PLAT_PLACEHOLDER in path:
         path = path.replace(_PLAT_PLACEHOLDER, platform)
         return ("arch", path)
     return ("generic", path)
 
-def _detect_platform(ctx):
-    if ctx.target_platform_has_constraint(ctx.attr._windows[platform_common.ConstraintValueInfo]):
-        return "win64"
-    elif ctx.target_platform_has_constraint(ctx.attr._macos[platform_common.ConstraintValueInfo]):
-        return "osx64"
+def _platform_select(
+        ctx = None,
+        *,
+        linux,
+        windows,
+        macos):
+    if ctx:
+        if ctx.target_platform_has_constraint(ctx.attr._windows[platform_common.ConstraintValueInfo]):
+            return windows
+        elif ctx.target_platform_has_constraint(ctx.attr._macos[platform_common.ConstraintValueInfo]):
+            return macos
+        else:
+            return linux
     else:
-        return "linux64"
+        return select({
+            "@platforms//os:linux": linux,
+            "@platforms//os:macos": macos,
+            "@platforms//os:windows": windows,
+        })
+
+def _detect_platform(ctx = None):
+    return _platform_select(ctx, linux = "linux64", macos = "osx64", windows = "win64")
 
 def codeql_pkg_files(
         *,
@@ -89,9 +104,9 @@ def _extract_pkg_filegroup_impl(ctx):
     for pfi, origin in src.pkg_files:
         dest_src_map = {}
         for dest, file in pfi.dest_src_map.items():
-            file_kind, dest = _process_path(dest, platform)
+            file_kind, expanded_dest = _expand_path(dest, platform)
             if file_kind == ctx.attr.kind:
-                dest_src_map[dest] = file
+                dest_src_map[expanded_dest] = file
         if dest_src_map:
             pkg_files.append((PackageFilesInfo(dest_src_map = dest_src_map, attributes = pfi.attributes), origin))
 
@@ -124,9 +139,10 @@ def _imported_zips_manifest_impl(ctx):
     manifest = []
     files = []
     for zip, prefix in ctx.attr.zips.items():
-        _, prefix = _process_path(prefix, platform)
+        # we don't care about the kind here, as we're taking all zips together
+        _, expanded_prefix = _expand_path(prefix, platform)
         zip_files = zip.files.to_list()
-        manifest += ["%s:%s" % (prefix, f.short_path) for f in zip_files]
+        manifest += ["%s:%s" % (expanded_prefix, f.short_path) for f in zip_files]
         files += zip_files
 
     output = ctx.actions.declare_file(ctx.label.name + ".params")
@@ -146,7 +162,10 @@ _imported_zips_manifest = rule(
         {CODEQL_PLATFORM} can be used as zip prefixes and will be expanded to the relevant codeql platform.
     """,
     attrs = {
-        "zips": attr.label_keyed_string_dict(doc = "mapping from zip files to install prefixes", allow_files = True),
+        "zips": attr.label_keyed_string_dict(
+            doc = "mapping from zip files to install prefixes",
+            allow_files = [".zip"],
+        ),
     } | _PLAT_DETECTION_ATTRS,
 )
 
@@ -156,11 +175,11 @@ def _zipmerge_impl(ctx):
     platform = _detect_platform(ctx)
     filename = "%s-%s.zip" % (ctx.attr.zip_name, platform if ctx.attr.kind == "arch" else "generic")
     output = ctx.actions.declare_file(filename)
-    args = [output.path, "--prefix=%s" % ctx.attr.zip_prefix, ctx.file.base.path]
+    args = [output.path, "--prefix=%s" % ctx.attr.prefix, ctx.file.base.path]
     for zip, prefix in ctx.attr.zips.items():
-        zip_kind, prefix = _process_path(prefix, platform)
+        zip_kind, expanded_prefix = _expand_path(prefix, platform)
         if zip_kind == ctx.attr.kind:
-            args.append("--prefix=%s/%s" % (ctx.attr.zip_prefix, prefix.rstrip("/")))
+            args.append("--prefix=%s/%s" % (ctx.attr.prefix, expanded_prefix.rstrip("/")))
             args += [f.path for f in zip.files.to_list()]
             zips.append(zip.files)
     ctx.actions.run(
@@ -188,12 +207,15 @@ _zipmerge = rule(
     attrs = {
         "base": attr.label(
             doc = "Base zip file to which zips from `zips` will be merged with",
-            allow_single_file = True,
+            allow_single_file = [".zip"],
+        ),
+        "zips": attr.label_keyed_string_dict(
+            doc = "mapping from zip files to install prefixes",
+            allow_files = [".zip"],
         ),
-        "zips": attr.label_keyed_string_dict(doc = "mapping from zip files to install prefixes", allow_files = True),
         "zip_name": attr.string(doc = "Prefix to use for the output file name"),
         "kind": attr.string(doc = "Which zip kind to consider", values = ["generic", "arch"]),
-        "zip_prefix": attr.string(doc = "Prefix posix path to add to the zip contents in the archive"),
+        "prefix": attr.string(doc = "Prefix posix path to add to the zip contents in the archive"),
         "_zipmerge": attr.label(default = "//misc/bazel/internal/zipmerge", executable = True, cfg = "exec"),
     } | _PLAT_DETECTION_ATTRS,
 )
@@ -203,9 +225,10 @@ def codeql_pack(
         name,
         srcs = None,
         zips = None,
-        zip_filename = "extractor",
+        zip_filename = None,
         visibility = None,
         install_dest = "extractor-pack",
+        compression_level = None,
         **kwargs):
     """
     Define a codeql pack. This macro accepts `pkg_files`, `pkg_filegroup` or their `codeql_*` counterparts as `srcs`.
@@ -221,6 +244,9 @@ def codeql_pack(
     The distinction between arch-specific and generic contents is made based on whether the paths (including possible
     prefixes added by rules) contain the special `{CODEQL_PLATFORM}` placeholder, which in case it is present will also
     be replaced by the appropriate platform (`linux64`, `win64` or `osx64`).
+
+    `compression_level` can be used to tweak the compression level used when creating archives. Consider that this
+    does not affect the contents of `zips`, only `srcs`.
     """
     internal = _make_internal(name)
     zip_filename = zip_filename or name
@@ -238,25 +264,37 @@ def codeql_pack(
             kind = kind,
             visibility = ["//visibility:private"],
         )
-        pkg_zip(
-            name = internal(kind + "-zip-base"),
-            srcs = [internal(kind)],
-            visibility = ["//visibility:private"],
-        )
-        _zipmerge(
-            name = internal(kind + "-zip"),
-            base = internal(kind + "-zip-base"),
+        if zips:
+            pkg_zip(
+                name = internal(kind + "-zip-base"),
+                srcs = [internal(kind)],
+                visibility = ["//visibility:private"],
+                compression_level = compression_level,
+            )
+            _zipmerge(
+                name = internal(kind + "-zip"),
+                base = internal(kind + "-zip-base"),
+                zips = zips,
+                zip_name = zip_filename,
+                kind = kind,
+                prefix = name,
+                visibility = visibility,
+            )
+        else:
+            pkg_zip(
+                name = internal(kind + "-zip"),
+                srcs = [internal(kind)],
+                visibility = ["//visibility:private"],
+                package_dir = name,
+                package_file_name = name + "-" + (_detect_platform() if kind == "arch" else "generic") + ".zip",
+                compression_level = compression_level,
+            )
+    if zips:
+        _imported_zips_manifest(
+            name = internal("zip-manifest"),
             zips = zips,
-            zip_name = zip_filename,
-            zip_prefix = name,
-            kind = kind,
-            visibility = visibility,
+            visibility = ["//visibility:private"],
         )
-    _imported_zips_manifest(
-        name = internal("zip-manifest"),
-        zips = zips,
-        visibility = ["//visibility:private"],
-    )
 
     pkg_install(
         name = internal("script"),
@@ -276,18 +314,20 @@ def codeql_pack(
         data = [
             internal("build-file"),
             internal("script"),
+        ] + ([
             internal("zip-manifest"),
             "//misc/bazel/internal/ripunzip",
-        ],
+        ] if zips else []),
         deps = ["@rules_python//python/runfiles"],
         args = [
             "--build-file=$(rlocationpath %s)" % internal("build-file"),
             "--pkg-install-script=$(rlocationpath %s)" % internal("script"),
             "--destdir",
             install_dest,
+        ] + ([
             "--ripunzip=$(rlocationpath //misc/bazel/internal/ripunzip)",
             "--zip-manifest=$(rlocationpath %s)" % internal("zip-manifest"),
-        ],
+        ] if zips else []),
         visibility = visibility,
     )
     native.filegroup(