From 0cf34494760df382194a4d56fb5cc95c2c9a49ab Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Tue, 9 Jul 2024 13:54:17 +0000 Subject: [PATCH] ci: use `bzlmod` in most Bazel builds (#14401) --- .bazelrc | 4 +- .gitignore | 3 + MODULE.bazel | 75 ++++++++++ WORKSPACE.bazel | 4 + WORKSPACE.bzlmod | 23 +++ bazel/bzlmod0.bzl | 43 ++++++ bazel/bzlmod1.bzl | 26 ++++ bazel/curl.BUILD | 8 ++ bazel/development0.bzl | 14 -- bazel/development2.bzl | 43 ++++++ bazel/googleapis.modules.patch | 136 ++++++++++++++++++ ci/cloudbuild/builds/asan.sh | 8 +- ci/cloudbuild/builds/bazel-oldest.sh | 9 +- ci/cloudbuild/builds/checkers.sh | 2 +- ci/cloudbuild/builds/generate-libraries.sh | 12 +- ci/cloudbuild/builds/lib/bazel.sh | 6 +- ci/cloudbuild/builds/lib/integration.sh | 4 +- ci/gha/builds/lib/bazel.sh | 5 +- .../run_integration_tests_emulator_bazel.sh | 9 +- .../run_integration_tests_emulator_cmake.sh | 4 +- .../bigtable/tools/run_emulator_utils.sh | 14 +- 21 files changed, 411 insertions(+), 41 deletions(-) create mode 100644 MODULE.bazel create mode 100644 WORKSPACE.bzlmod create mode 100644 bazel/bzlmod0.bzl create mode 100644 bazel/bzlmod1.bzl create mode 100644 bazel/development2.bzl create mode 100644 bazel/googleapis.modules.patch diff --git a/.bazelrc b/.bazelrc index 2591f36f278e4..58ddbc9077436 100644 --- a/.bazelrc +++ b/.bazelrc @@ -39,9 +39,6 @@ build --experimental_convenience_symlinks=ignore # frustrating when they fail and don't give any output. So, remove the limit. build --experimental_ui_max_stdouterr_bytes=-1 -# TODO(#11485) - enable bzlmod once it works -common --noenable_bzlmod - # TODO(#13311) - remove once gRPC works with Bazel v7 or when gRPC stops using # `apple_rules`. common:macos --repo_env=BAZEL_NO_APPLE_CPP_TOOLCHAIN=1 @@ -84,6 +81,7 @@ build:asan --copt=-fsanitize=address # may be assuming that some compiler flags are set when compiling with Address # Sanitizier (-DADDRESS_SANITIZER=1 is one possibility). build:asan --per_file_copt=com_google_protobuf//@-Wno-error +build:asan --per_file_copt=protobuf~//@-Wno-error build:asan --linkopt=-fsanitize=address build:asan --action_env=ASAN_OPTIONS=detect_leaks=1:color=always build:asan --action_env=LSAN_OPTIONS=report_objects=1 diff --git a/.gitignore b/.gitignore index e4ab153f432d6..b8e44a997ef9b 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,6 @@ cmake-build-*/ # Ignore the file with user-defined CMake presets CMakeUserPresets.json + +# Lock files created by bzlmod. We do not want to accidentally commit these. +MODULE.bazel.lock diff --git a/MODULE.bazel b/MODULE.bazel new file mode 100644 index 0000000000000..fd5578e889e12 --- /dev/null +++ b/MODULE.bazel @@ -0,0 +1,75 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Idiomatic C++ client libraries for Google Cloud Platform.""" + +module( + name = "google_cloud_cpp", + # TODO(#11485) - automatically synchronize from (or to) the CMakeList.txt + # file. + version = "2.26.0-rc", + # TODO(#11485) - automatically increase on each major release. + compatibility_level = 1, +) + +bazel_dep(name = "platforms", version = "0.0.10") +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "rules_cc", version = "0.0.9") +bazel_dep(name = "abseil-cpp", version = "20240116.2", repo_name = "com_google_absl") +bazel_dep(name = "protobuf", version = "27.2", repo_name = "com_google_protobuf") +bazel_dep(name = "boringssl", version = "0.0.0-20230215-5c22014") +bazel_dep(name = "grpc", version = "1.63.1", repo_name = "com_github_grpc_grpc") +bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "com_github_nlohmann_json") +bazel_dep(name = "crc32c", version = "1.1.0", repo_name = "com_github_google_crc32c") +bazel_dep(name = "opentelemetry-cpp", version = "1.16.0", repo_name = "io_opentelemetry_cpp") + +bazel_dep(name = "rules_proto", version = "6.0.2", dev_dependency = True) +bazel_dep(name = "googletest", version = "1.14.0", dev_dependency = True, repo_name = "com_google_googletest") +bazel_dep(name = "google_benchmark", version = "1.8.4", dev_dependency = True, repo_name = "com_google_benchmark") +bazel_dep(name = "yaml-cpp", version = "0.8.0", dev_dependency = True, repo_name = "com_github_jbeder_yaml_cpp") +bazel_dep(name = "pugixml", version = "1.14.bcr.1", dev_dependency = True, repo_name = "com_github_zeux_pugixml") + +# Our `curl.BUILD` file uses these. +bazel_dep(name = "zlib", version = "1.2.13") +bazel_dep(name = "c-ares", version = "1.15.0", repo_name = "com_github_cares_cares") + +# `google-cloud-cpp` uses this indirectly in the coverage build. And we need +# to configure it for our CI builds. +bazel_dep(name = "rules_python", version = "0.33.2", dev_dependency = True) + +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain( + ignore_root_user_error = True, + python_version = "3.11", +) + +bazel_dep(name = "googleapis", version = "0.0.0", repo_name = "com_google_googleapis") +archive_override( + module_name = "googleapis", + integrity = "sha256-M8YsA/lHlyi9qhplU9izX6Jz0BBwbHXqhc2N/haHWGw=", + patch_strip = 1, + patches = ["//bazel:googleapis.modules.patch"], + strip_prefix = "googleapis-622e10a1e8b2b6908e0ac7448d347a0c1b4130de", + urls = [ + "https://storage.googleapis.com/cloud-cpp-community-archive/com_google_googleapis/622e10a1e8b2b6908e0ac7448d347a0c1b4130de.tar.gz", + "https://github.com/googleapis/googleapis/archive/622e10a1e8b2b6908e0ac7448d347a0c1b4130de.tar.gz", + ], +) + +switched_rules = use_extension("@com_google_googleapis//:extensions.bzl", "switched_rules") +switched_rules.use_languages( + cc = True, + grpc = True, +) +use_repo(switched_rules, "com_google_googleapis_imports") diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index 90e37fed6a8b9..0576121167b7c 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -45,3 +45,7 @@ gl_cpp_development0() load("//bazel:development1.bzl", "gl_cpp_development1") gl_cpp_development1() + +load("//bazel:development2.bzl", "gl_cpp_development2") + +gl_cpp_development2() diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod new file mode 100644 index 0000000000000..78cf2d70acd1d --- /dev/null +++ b/WORKSPACE.bzlmod @@ -0,0 +1,23 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +workspace(name = "google_cloud_cpp") + +load("//bazel:bzlmod0.bzl", "gl_cpp_bzlmod0") + +gl_cpp_bzlmod0() + +load("//bazel:bzlmod1.bzl", "gl_cpp_bzlmod1") + +gl_cpp_bzlmod1() diff --git a/bazel/bzlmod0.bzl b/bazel/bzlmod0.bzl new file mode 100644 index 0000000000000..7f7e61104692f --- /dev/null +++ b/bazel/bzlmod0.bzl @@ -0,0 +1,43 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Load dependencies needed to use the google-cloud-cpp libraries.""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") + +def gl_cpp_bzlmod0(name = None): + """Loads dependencies need to compile the google-cloud-cpp libraries. + + This is a workaround until all dependencies of `google-cloud-cpp` can + be managed via bzlmod. + + Args: + name: Unused. It is conventional to provide a `name` argument to all + workspace functions. + """ + + # TODO(#11485) - use some bazel_dep() from BCR. + # We need libcurl for the Google Cloud Storage client. + maybe( + http_archive, + name = "com_github_curl_curl", + urls = [ + "https://storage.googleapis.com/cloud-cpp-community-archive/com_github_curl_curl/curl-7.69.1.tar.gz", + "https://curl.haxx.se/download/curl-7.69.1.tar.gz", + ], + sha256 = "01ae0c123dee45b01bbaef94c0bc00ed2aec89cb2ee0fd598e0d302a6b5e0a98", + strip_prefix = "curl-7.69.1", + build_file = Label("//bazel:curl.BUILD"), + ) diff --git a/bazel/bzlmod1.bzl b/bazel/bzlmod1.bzl new file mode 100644 index 0000000000000..538dd847aaf8b --- /dev/null +++ b/bazel/bzlmod1.bzl @@ -0,0 +1,26 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Load dependencies needed to use the google-cloud-cpp libraries.""" + +def gl_cpp_bzlmod1(name = None): + """Loads dependencies need to compile the google-cloud-cpp libraries. + + This is a workaround until all dependencies of `google-cloud-cpp` can + be managed via bzlmod. + + Args: + name: Unused. It is conventional to provide a `name` argument to all + workspace functions. + """ diff --git a/bazel/curl.BUILD b/bazel/curl.BUILD index 41d2b06f76578..f1005431f8433 100644 --- a/bazel/curl.BUILD +++ b/bazel/curl.BUILD @@ -3,6 +3,7 @@ # Description: # curl is a tool for talking to web servers. +load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") load("@bazel_skylib//rules:write_file.bzl", "write_file") licenses(["notice"]) # MIT/X derivative license @@ -26,6 +27,13 @@ config_setting( visibility = ["//visibility:public"], ) +# This is unused, it is here just for compatibility with the Bazel Central +# Registry modules. +bool_flag( + name = "http_only", + build_setting_default = False, +) + # On Linux, libcurl needs to know, at compile time, the location for the # Certificate Authority (CA) bundle file. By default we dynamically guess the # location for most common Linux distribution, but this guessing makes the build diff --git a/bazel/development0.bzl b/bazel/development0.bzl index fda15dc02cd20..732a21455f08e 100644 --- a/bazel/development0.bzl +++ b/bazel/development0.bzl @@ -65,20 +65,6 @@ def gl_cpp_development0(name = None): strip_prefix = "benchmark-1.8.4", ) - # An XML parser and generator, this is only used in //docfx. - # This is an internal tool used to generate the reference documentation. - maybe( - http_archive, - name = "com_github_zeux_pugixml", - urls = [ - "https://storage.googleapis.com/cloud-cpp-community-archive/com_github_zeux_pugixml/v1.14.tar.gz", - "https://github.com/zeux/pugixml/archive/v1.14.tar.gz", - ], - sha256 = "610f98375424b5614754a6f34a491adbddaaec074e9044577d965160ec103d2e", - strip_prefix = "pugixml-1.14", - build_file = Label("//bazel:pugixml.BUILD"), - ) - # A YAML parser and generator, this is only used in //docfx and //generator. # Both are internal tools used for development only. maybe( diff --git a/bazel/development2.bzl b/bazel/development2.bzl new file mode 100644 index 0000000000000..00215e02a9a18 --- /dev/null +++ b/bazel/development2.bzl @@ -0,0 +1,43 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Load dependencies needed for google-cloud-cpp development / Phase 2.""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") + +def gl_cpp_development2(name = None): + """Loads dependencies needed to develop the google-cloud-cpp libraries. + + This function assumes the repositories needed by our dependencies are loaded + + + Args: + name: Unused. It is conventional to provide a `name` argument to all + workspace functions. + """ + + # An XML parser and generator, this is only used in //docfx. + # This is an internal tool used to generate the reference documentation. + maybe( + http_archive, + name = "com_github_zeux_pugixml", + urls = [ + "https://storage.googleapis.com/cloud-cpp-community-archive/com_github_zeux_pugixml/v1.14.tar.gz", + "https://github.com/zeux/pugixml/archive/v1.14.tar.gz", + ], + sha256 = "610f98375424b5614754a6f34a491adbddaaec074e9044577d965160ec103d2e", + strip_prefix = "pugixml-1.14", + build_file = Label("//bazel:pugixml.BUILD"), + ) diff --git a/bazel/googleapis.modules.patch b/bazel/googleapis.modules.patch new file mode 100644 index 0000000000000..37c2a4d170c75 --- /dev/null +++ b/bazel/googleapis.modules.patch @@ -0,0 +1,136 @@ +diff --git a/BUILD.bazel b/BUILD.bazel +index 95e4c12e5..a901fd573 100644 +--- a/BUILD.bazel ++++ b/BUILD.bazel +@@ -1,3 +1,7 @@ ++package(default_visibility = ["//visibility:public"]) ++ ++licenses(["notice"]) # Apache 2.0 ++ + genrule( + name = "build_gen", + outs = ["build_gen.sh"], +@@ -16,3 +20,15 @@ EOD + fi + """, + ) ++ ++# This build file overlays on top of the BUILD files for the googleapis repo, ++# and it adds a target that lets us include their header files using ++# angle-brackets, thus treating their headers as system includes. This allows ++# us to dial-up the warnings in our own code, without seeing compiler warnings ++# from their headers, which we do not own. ++cc_library( ++ name = "googleapis_system_includes", ++ includes = [ ++ ".", ++ ], ++) +diff --git a/MODULE.bazel b/MODULE.bazel +new file mode 100644 +index 000000000..169133e43 +--- /dev/null ++++ b/MODULE.bazel +@@ -0,0 +1,29 @@ ++module( ++ name = "googleapis", ++ version = "0.0.0-20240326-1c8d509c5", ++ repo_name = "com_google_googleapis", ++) ++ ++bazel_dep(name = "grpc", version = "1.56.3.bcr.1", repo_name = "com_github_grpc_grpc") ++bazel_dep(name = "grpc-java", version = "1.62.2", repo_name = "io_grpc_grpc_java") ++bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf") ++bazel_dep(name = "rules_go", version = "0.46.0", repo_name = "io_bazel_rules_go") ++bazel_dep(name = "rules_proto", version = "5.3.0-21.7") ++bazel_dep(name = "rules_python", version = "0.31.0") ++ ++switched_rules = use_extension("//:extensions.bzl", "switched_rules") ++ ++# TODO: Enable support for other languages with bzlmod ++switched_rules.use_languages( ++ cc = True, ++ # csharp = True, ++ # gapic = True, ++ go = True, ++ grpc = True, ++ java = True, ++ # nodejs = True, ++ # php = True, ++ python = True, ++ # ruby = True, ++) ++use_repo(switched_rules, "com_google_googleapis_imports") +diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod +new file mode 100644 +index 000000000..8cf3fe396 +--- /dev/null ++++ b/WORKSPACE.bzlmod +@@ -0,0 +1,2 @@ ++workspace(name = "com_google_googleapis") ++ +diff --git a/extensions.bzl b/extensions.bzl +new file mode 100644 +index 000000000..9aa161841 +--- /dev/null ++++ b/extensions.bzl +@@ -0,0 +1,59 @@ ++load(":repository_rules.bzl", "switched_rules_by_language") ++ ++_use_languages_tag = tag_class( ++ attrs = { ++ "cc": attr.bool(default = False), ++ "csharp": attr.bool(default = False), ++ "gapic": attr.bool(default = False), ++ "go": attr.bool(default = False), ++ "go_test": attr.bool(default = False), ++ "grpc": attr.bool(default = False), ++ "java": attr.bool(default = False), ++ "nodejs": attr.bool(default = False), ++ "php": attr.bool(default = False), ++ "python": attr.bool(default = False), ++ "ruby": attr.bool(default = False), ++ }, ++) ++ ++def _switched_rules_impl(ctx): ++ attrs = {} ++ for module in ctx.modules: ++ if not module.is_root: ++ continue ++ ++ is_tag_set = False ++ set_tag_name = "" ++ ++ for t in module.tags.use_languages: ++ if is_tag_set: ++ fail("Multiple use_language tags are set in the root module: '{}' and '{}'. Only one is allowed.".format(set_tag_name, module.name)) ++ ++ is_tag_set = True ++ set_tag_name = module.name ++ ++ attrs = { ++ "cc": t.cc, ++ "csharp": t.csharp, ++ "gapic": t.gapic, ++ "go": t.go, ++ "go_test": t.go_test, ++ "grpc": t.grpc, ++ "java": t.java, ++ "nodejs": t.nodejs, ++ "php": t.php, ++ "python": t.python, ++ "ruby": t.ruby, ++ } ++ ++ switched_rules_by_language( ++ name = "com_google_googleapis_imports", ++ **attrs ++ ) ++ ++switched_rules = module_extension( ++ implementation = _switched_rules_impl, ++ tag_classes = { ++ "use_languages": _use_languages_tag, ++ }, ++) diff --git a/ci/cloudbuild/builds/asan.sh b/ci/cloudbuild/builds/asan.sh index 628d5ab048d62..62b45b669e47f 100755 --- a/ci/cloudbuild/builds/asan.sh +++ b/ci/cloudbuild/builds/asan.sh @@ -23,7 +23,13 @@ source module ci/cloudbuild/builds/lib/integration.sh source module ci/lib/io.sh mapfile -t args < <(bazel::common_args) -args+=("--config=asan") +args+=( + --config=asan + # TODO(#11485) - asan claims ODR violations when building googleapis and gRPC + # under bzlmod. Disable for now. + # https://github.com/bazelbuild/bazel-central-registry/issues/2334 + --noenable_bzlmod +) io::run bazel test "${args[@]}" --test_tag_filters=-integration-test "${BAZEL_TARGETS[@]}" mapfile -t integration_args < <(integration::bazel_args) diff --git a/ci/cloudbuild/builds/bazel-oldest.sh b/ci/cloudbuild/builds/bazel-oldest.sh index 40d0c0f87b7ee..907dd70fd13b7 100755 --- a/ci/cloudbuild/builds/bazel-oldest.sh +++ b/ci/cloudbuild/builds/bazel-oldest.sh @@ -26,4 +26,11 @@ export CC=clang export CXX=clang++ mapfile -t args < <(bazel::common_args) -bazel test "${args[@]}" --test_tag_filters=-integration-test "${BAZEL_TARGETS[@]}" +args+=( + # For now, we continue to test Bazel 6.x without bzlmod. Once the minimum + # supported version of Bazel is 7.x we can decide how to test without bzlmod. + --noenable_bzlmod + # Only run the unit tests, no need to waste time running everything. + --test_tag_filters=-integration-test +) +bazel test "${args[@]}" -- "${BAZEL_TARGETS[@]}" diff --git a/ci/cloudbuild/builds/checkers.sh b/ci/cloudbuild/builds/checkers.sh index 1b8af810182fc..ff5ad6b5c0699 100755 --- a/ci/cloudbuild/builds/checkers.sh +++ b/ci/cloudbuild/builds/checkers.sh @@ -151,7 +151,7 @@ time { } printf "%-50s" "Running buildifier (format):" >&2 time { - git_files -z -- '*.BUILD' '*.bzl' '*.bazel' | + git_files -z -- '*.BUILD' '*.bzl' '*.bazel' '*.bzlmod' | xargs -r -P "$(nproc)" -n 50 -0 buildifier -mode=fix } diff --git a/ci/cloudbuild/builds/generate-libraries.sh b/ci/cloudbuild/builds/generate-libraries.sh index db636b0c60ddc..418af44f8041d 100755 --- a/ci/cloudbuild/builds/generate-libraries.sh +++ b/ci/cloudbuild/builds/generate-libraries.sh @@ -25,8 +25,8 @@ bazel_output_base="$(bazel info output_base)" io::log_h2 "Running the generator to update the golden files" bazel run --action_env=GOOGLE_CLOUD_CPP_ENABLE_CLOG=yes \ //generator:google-cloud-cpp-codegen -- \ - --protobuf_proto_path="${bazel_output_base}/external/com_google_protobuf/src" \ - --googleapis_proto_path="${bazel_output_base}/external/com_google_googleapis" \ + --protobuf_proto_path="${bazel_output_base}/external/protobuf~/src" \ + --googleapis_proto_path="${bazel_output_base}/external/googleapis~" \ --golden_proto_path="${PWD}" \ --output_path="${PWD}" \ --update_ci=false \ @@ -38,8 +38,8 @@ if [ -z "${GENERATE_GOLDEN_ONLY}" ]; then io::log_h2 "Running the generator to emit protos from discovery docs" bazel run --action_env=GOOGLE_CLOUD_CPP_ENABLE_CLOG=yes \ //generator:google-cloud-cpp-codegen -- \ - --protobuf_proto_path="${bazel_output_base}"/external/com_google_protobuf/src \ - --googleapis_proto_path="${bazel_output_base}"/external/com_google_googleapis \ + --protobuf_proto_path="${bazel_output_base}"/external/protobuf~/src \ + --googleapis_proto_path="${bazel_output_base}"/external/googleapis~ \ --discovery_proto_path="${PWD}/protos" \ --output_path="${PROJECT_ROOT}/protos" \ --export_output_path="${PROJECT_ROOT}" \ @@ -71,8 +71,8 @@ if [ -z "${GENERATE_GOLDEN_ONLY}" ]; then io::log_h2 "Running the generator to update the generated libraries" bazel run --action_env=GOOGLE_CLOUD_CPP_ENABLE_CLOG=yes \ //generator:google-cloud-cpp-codegen -- \ - --protobuf_proto_path="${bazel_output_base}"/external/com_google_protobuf/src \ - --googleapis_proto_path="${bazel_output_base}"/external/com_google_googleapis \ + --protobuf_proto_path="${bazel_output_base}"/external/protobuf~/src \ + --googleapis_proto_path="${bazel_output_base}"/external/googleapis~ \ --discovery_proto_path="${PWD}/protos" \ --output_path="${PROJECT_ROOT}" \ --check_comment_substitutions=true \ diff --git a/ci/cloudbuild/builds/lib/bazel.sh b/ci/cloudbuild/builds/lib/bazel.sh index 0dc0597eb9289..96792f0bb7469 100644 --- a/ci/cloudbuild/builds/lib/bazel.sh +++ b/ci/cloudbuild/builds/lib/bazel.sh @@ -36,11 +36,7 @@ io::log "Prefetching bazel deps..." # build. TIMEFORMAT="==> 🕑 prefetching done in %R seconds" time { - "ci/retry-command.sh" 3 120 bazel fetch ... \ - @local_config_platform//... \ - @local_config_cc_toolchains//... \ - @local_config_sh//... \ - @remotejdk11_linux//:jdk + "ci/retry-command.sh" 3 120 bazel fetch ... } echo >&2 diff --git a/ci/cloudbuild/builds/lib/integration.sh b/ci/cloudbuild/builds/lib/integration.sh index 34d16b0c7228f..98e3fc6fc0986 100644 --- a/ci/cloudbuild/builds/lib/integration.sh +++ b/ci/cloudbuild/builds/lib/integration.sh @@ -249,8 +249,8 @@ function integration::bazel_with_emulators() { bazel_output_base="$(bazel info output_base)" bazel run --action_env=GOOGLE_CLOUD_CPP_ENABLE_CLOG=yes \ //generator:google-cloud-cpp-codegen -- \ - --protobuf_proto_path="${bazel_output_base}/external/com_google_protobuf/src" \ - --googleapis_proto_path="${bazel_output_base}/external/com_google_googleapis" \ + --protobuf_proto_path="${bazel_output_base}/external/protobuf~/src" \ + --googleapis_proto_path="${bazel_output_base}/external/googleapis~" \ --golden_proto_path="${PWD}" \ --output_path="${PWD}" \ --update_ci=false \ diff --git a/ci/gha/builds/lib/bazel.sh b/ci/gha/builds/lib/bazel.sh index 650f5d4d8e9da..ccdddf63fcc25 100755 --- a/ci/gha/builds/lib/bazel.sh +++ b/ci/gha/builds/lib/bazel.sh @@ -98,13 +98,10 @@ function bazel::prefetch() { mapfile -t args < <(bazel::common_args) local common_rules=( "..." - "@local_config_platform//..." - "@local_config_cc_toolchains//..." - "@local_config_sh//..." ) local os_rules mapfile -t os_rules < <(os::prefetch) - "ci/retry-command.sh" 3 120 bazelisk "${args[@]}" fetch "${common_rules[@]}" "${os_rules[@]}" + "ci/retry-command.sh" 3 120 bazelisk "${args[@]}" fetch "${common_rules[@]}" } io::log "Prefetching bazel deps..." diff --git a/google/cloud/bigtable/ci/run_integration_tests_emulator_bazel.sh b/google/cloud/bigtable/ci/run_integration_tests_emulator_bazel.sh index 121c34630bf80..447a1f20fb74a 100755 --- a/google/cloud/bigtable/ci/run_integration_tests_emulator_bazel.sh +++ b/google/cloud/bigtable/ci/run_integration_tests_emulator_bazel.sh @@ -37,7 +37,14 @@ fi # Configure run_emulators_utils.sh to find the instance admin emulator. BAZEL_BIN_DIR="$("${BAZEL_BIN}" info bazel-bin)" readonly BAZEL_BIN_DIR -export CBT_INSTANCE_ADMIN_EMULATOR_CMD="${BAZEL_BIN_DIR}/google/cloud/bigtable/tests/instance_admin_emulator" + +CBT_INSTANCE_ADMIN_EMULATOR_START=( + "${BAZEL_BIN}" + run + "${bazel_test_args[@]}" + -- + //google/cloud/bigtable/tests:instance_admin_emulator +) source module /google/cloud/bigtable/tools/run_emulator_utils.sh # These can only run against production diff --git a/google/cloud/bigtable/ci/run_integration_tests_emulator_cmake.sh b/google/cloud/bigtable/ci/run_integration_tests_emulator_cmake.sh index 85a4c29b00584..9621925f22514 100755 --- a/google/cloud/bigtable/ci/run_integration_tests_emulator_cmake.sh +++ b/google/cloud/bigtable/ci/run_integration_tests_emulator_cmake.sh @@ -38,7 +38,9 @@ if ctest::has_no_tests "${BINARY_DIR}" "^bigtable_" "${ctest_args[@]}"; then fi # Configure run_emulators_utils.sh to find the instance admin emulator. -export CBT_INSTANCE_ADMIN_EMULATOR_CMD="${BINARY_DIR}/google/cloud/bigtable/tests/instance_admin_emulator" +CBT_INSTANCE_ADMIN_EMULATOR_START=( + "${BINARY_DIR}/google/cloud/bigtable/tests/instance_admin_emulator" +) source module /google/cloud/bigtable/tools/run_emulator_utils.sh cd "${BINARY_DIR}" diff --git a/google/cloud/bigtable/tools/run_emulator_utils.sh b/google/cloud/bigtable/tools/run_emulator_utils.sh index 014bc42b18ebd..16d5be328b154 100755 --- a/google/cloud/bigtable/tools/run_emulator_utils.sh +++ b/google/cloud/bigtable/tools/run_emulator_utils.sh @@ -34,6 +34,16 @@ function kill_emulators() { echo -n "." wait "${INSTANCE_ADMIN_EMULATOR_PID}" >/dev/null 2>&1 || echo -n "+" echo "." + + for log in emulator.log bigtable-instance-emulator.log; do + echo "================ ${log} ================" + cat --number --show-nonprinting "${HOME}/${log}" | head || true + cat --number --show-nonprinting "${BINARY_DIR}/${log}" | head || true + echo "================ ${log} ================" + cat --number --show-nonprinting "${HOME}/${log}" | tail || true + cat --number --show-nonprinting "${BINARY_DIR}/${log}" | tail || true + echo "================ ${log} ================" + done } ################################################ @@ -62,7 +72,7 @@ function wait_until_emulator_connects() { # Wait until the emulator starts responding. delay=1 for _ in $(seq 1 8); do - if env BIGTABLE_EMULATOR_HOST="${address}" "${cmd[@]}" >/dev/null 2>&1; then + if timeout 10 env BIGTABLE_EMULATOR_HOST="${address}" "${cmd[@]}" >/dev/null 2>&1; then io::log_green "Connected to the Cloud Bigtable emulator on ${address}" return fi @@ -87,7 +97,7 @@ function start_emulators() { EMULATOR_PID=$! # The path to this command must be set by the caller. - "${CBT_INSTANCE_ADMIN_EMULATOR_CMD}" "${instance_admin_emulator_port}" >instance-admin-emulator.log 2>&1 instance-admin-emulator.log 2>&1