diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java index 4f8b091d55..3ab383899b 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java @@ -37,7 +37,11 @@ public class BuiltInMetricsConstant { public static final String GAX_METER_NAME = OpenTelemetryMetricsRecorder.GAX_METER_NAME; + public static final String SPANNER_METER_NAME = "spanner-java"; + static final String OPERATION_LATENCIES_NAME = "operation_latencies"; + + static final String GFE_LATENCIES_NAME = "gfe_latencies"; static final String ATTEMPT_LATENCIES_NAME = "attempt_latencies"; static final String OPERATION_LATENCY_NAME = "operation_latency"; static final String ATTEMPT_LATENCY_NAME = "attempt_latency"; @@ -114,6 +118,7 @@ static Map getAllViews() { ImmutableMap.Builder views = ImmutableMap.builder(); defineView( views, + BuiltInMetricsConstant.GAX_METER_NAME, BuiltInMetricsConstant.OPERATION_LATENCY_NAME, BuiltInMetricsConstant.OPERATION_LATENCIES_NAME, BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM, @@ -121,6 +126,7 @@ static Map getAllViews() { "ms"); defineView( views, + BuiltInMetricsConstant.GAX_METER_NAME, BuiltInMetricsConstant.ATTEMPT_LATENCY_NAME, BuiltInMetricsConstant.ATTEMPT_LATENCIES_NAME, BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM, @@ -128,6 +134,15 @@ static Map getAllViews() { "ms"); defineView( views, + BuiltInMetricsConstant.SPANNER_METER_NAME, + BuiltInMetricsConstant.GFE_LATENCIES_NAME, + BuiltInMetricsConstant.GFE_LATENCIES_NAME, + BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM, + InstrumentType.HISTOGRAM, + "ms"); + defineView( + views, + BuiltInMetricsConstant.GAX_METER_NAME, BuiltInMetricsConstant.OPERATION_COUNT_NAME, BuiltInMetricsConstant.OPERATION_COUNT_NAME, Aggregation.sum(), @@ -135,6 +150,7 @@ static Map getAllViews() { "1"); defineView( views, + BuiltInMetricsConstant.GAX_METER_NAME, BuiltInMetricsConstant.ATTEMPT_COUNT_NAME, BuiltInMetricsConstant.ATTEMPT_COUNT_NAME, Aggregation.sum(), @@ -145,6 +161,7 @@ static Map getAllViews() { private static void defineView( ImmutableMap.Builder viewMap, + String meterName, String metricName, String metricViewName, Aggregation aggregation, @@ -153,7 +170,7 @@ private static void defineView( InstrumentSelector selector = InstrumentSelector.builder() .setName(BuiltInMetricsConstant.METER_NAME + '/' + metricName) - .setMeterName(BuiltInMetricsConstant.GAX_METER_NAME) + .setMeterName(meterName) .setType(type) .setUnit(unit) .build(); diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsRecorder.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsRecorder.java new file mode 100644 index 0000000000..06834014ea --- /dev/null +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsRecorder.java @@ -0,0 +1,85 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner; + +import com.google.api.gax.core.GaxProperties; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.Meter; +import java.util.Map; + +/** + * OpenTelemetry implementation of recording metrics. This implementation collections the + * measurements related to the lifecyle of an RPC. + * + *

For the Otel implementation, an attempt is a single RPC invocation and an operation is the + * collection of all the attempts made before a response is returned (either as a success or an + * error). A single call (i.e. `EchoClient.echo()`) should have an operation_count of 1 and may have + * an attempt_count of 1+ (depending on the retry configurations). + */ +public class BuiltInOpenTelemetryMetricsRecorder { + + private final DoubleHistogram gfeLatencyRecorder; + + /** + * Creates the following instruments for the following metrics: + * + *

    + *
  • GFE Latency: Histogram + *
+ * + * @param openTelemetry OpenTelemetry instance + */ + public BuiltInOpenTelemetryMetricsRecorder(OpenTelemetry openTelemetry) { + Meter meter = + openTelemetry + .meterBuilder(BuiltInMetricsConstant.SPANNER_METER_NAME) + .setInstrumentationVersion(GaxProperties.getLibraryVersion(getClass())) + .build(); + this.gfeLatencyRecorder = + meter + .histogramBuilder( + BuiltInMetricsConstant.METER_NAME + '/' + BuiltInMetricsConstant.GFE_LATENCIES_NAME) + .setDescription( + "Latency between Google's network receiving an RPC and reading back the first byte of the response") + .setUnit("ms") + .build(); + } + + /** + * Record the latency between Google's network receiving an RPC and reading back the first byte of + * the response. Data is stored in a Histogram. + * + * @param gfeLatency Attempt Latency in ms + * @param attributes Map of the attributes to store + */ + public void recordGFELatency(double gfeLatency, Map attributes) { + gfeLatencyRecorder.record(gfeLatency, toOtelAttributes(attributes)); + } + + @VisibleForTesting + Attributes toOtelAttributes(Map attributes) { + Preconditions.checkNotNull(attributes, "Attributes map cannot be null"); + AttributesBuilder attributesBuilder = Attributes.builder(); + attributes.forEach(attributesBuilder::put); + return attributesBuilder.build(); + } +} diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java index af54515e7c..e82e85daca 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java @@ -1680,6 +1680,15 @@ public OpenTelemetry getOpenTelemetry() { } } + /** + * Returns an instance of OpenTelemetry. If OpenTelemetry object is not set via SpannerOptions + * then GlobalOpenTelemetry will be used as fallback. + */ + public OpenTelemetry getBuiltInMetricsOpenTelemetry() { + return this.builtInOpenTelemetryMetricsProvider.getOrCreateOpenTelemetry( + this.getProjectId(), getCredentials()); + } + @Override public ApiTracerFactory getApiTracerFactory() { return createApiTracerFactory(false, false); diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java index fe23b09798..aff5b8c318 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java @@ -357,6 +357,7 @@ public GapicSpannerRpc(final SpannerOptions options) { options.getInterceptorProvider(), SpannerInterceptorProvider.createDefault( options.getOpenTelemetry(), + options.getBuiltInMetricsOpenTelemetry(), (() -> directPathEnabledSupplier.get())))) // This sets the trace context headers. .withTraceContext(endToEndTracingEnabled, options.getOpenTelemetry()) diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java index 026f9b4ca9..de5a042ff7 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java @@ -25,6 +25,7 @@ import com.google.api.gax.tracing.ApiTracer; import com.google.cloud.spanner.BuiltInMetricsConstant; +import com.google.cloud.spanner.BuiltInOpenTelemetryMetricsRecorder; import com.google.cloud.spanner.CompositeTracer; import com.google.cloud.spanner.SpannerExceptionFactory; import com.google.cloud.spanner.SpannerRpcMetrics; @@ -94,12 +95,17 @@ class HeaderInterceptor implements ClientInterceptor { private static final Level LEVEL = Level.INFO; private final SpannerRpcMetrics spannerRpcMetrics; + private final BuiltInOpenTelemetryMetricsRecorder builtInOpenTelemetryMetricsRecorder; + private final Supplier directPathEnabledSupplier; HeaderInterceptor( - SpannerRpcMetrics spannerRpcMetrics, Supplier directPathEnabledSupplier) { + SpannerRpcMetrics spannerRpcMetrics, + BuiltInOpenTelemetryMetricsRecorder builtInOpenTelemetryMetricsRecorder, + Supplier directPathEnabledSupplier) { this.spannerRpcMetrics = spannerRpcMetrics; this.directPathEnabledSupplier = directPathEnabledSupplier; + this.builtInOpenTelemetryMetricsRecorder = builtInOpenTelemetryMetricsRecorder; } @Override @@ -118,8 +124,8 @@ public void start(Listener responseListener, Metadata headers) { TagContext tagContext = getTagContext(key, method.getFullMethodName(), databaseName); Attributes attributes = getMetricAttributes(key, method.getFullMethodName(), databaseName); - Map builtInMetricsAttributes = - getBuiltInMetricAttributes(key, databaseName); + Map commonBuiltInMetricAttributes = + getCommonBuiltInMetricAttributes(key, databaseName); super.start( new SimpleForwardingClientCallListener(responseListener) { @Override @@ -127,8 +133,13 @@ public void onHeaders(Metadata metadata) { Boolean isDirectPathUsed = isDirectPathUsed(getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR)); addBuiltInMetricAttributes( - compositeTracer, builtInMetricsAttributes, isDirectPathUsed); - processHeader(metadata, tagContext, attributes, span); + compositeTracer, commonBuiltInMetricAttributes, isDirectPathUsed); + processHeader( + metadata, + tagContext, + attributes, + span, + getBuiltInMetricAttributes(commonBuiltInMetricAttributes, isDirectPathUsed)); super.onHeaders(metadata); } }, @@ -142,7 +153,11 @@ public void onHeaders(Metadata metadata) { } private void processHeader( - Metadata metadata, TagContext tagContext, Attributes attributes, Span span) { + Metadata metadata, + TagContext tagContext, + Attributes attributes, + Span span, + Map builtInMetricsAttributes) { MeasureMap measureMap = STATS_RECORDER.newMeasureMap(); String serverTiming = metadata.get(SERVER_TIMING_HEADER_KEY); if (serverTiming != null && serverTiming.startsWith(SERVER_TIMING_HEADER_PREFIX)) { @@ -154,6 +169,7 @@ private void processHeader( spannerRpcMetrics.recordGfeLatency(latency, attributes); spannerRpcMetrics.recordGfeHeaderMissingCount(0L, attributes); + builtInOpenTelemetryMetricsRecorder.recordGFELatency(latency, builtInMetricsAttributes); if (span != null) { span.setAttribute("gfe_latency", String.valueOf(latency)); @@ -224,8 +240,8 @@ private Attributes getMetricAttributes(String key, String method, DatabaseName d }); } - private Map getBuiltInMetricAttributes(String key, DatabaseName databaseName) - throws ExecutionException { + private Map getCommonBuiltInMetricAttributes( + String key, DatabaseName databaseName) throws ExecutionException { return builtInAttributesCache.get( key, () -> { @@ -240,17 +256,21 @@ private Map getBuiltInMetricAttributes(String key, DatabaseName }); } + private Map getBuiltInMetricAttributes( + Map commonBuiltInMetricsAttributes, Boolean isDirectPathUsed) { + Map builtInMetricAttributes = new HashMap<>(commonBuiltInMetricsAttributes); + builtInMetricAttributes.put( + BuiltInMetricsConstant.DIRECT_PATH_USED_KEY.getKey(), Boolean.toString(isDirectPathUsed)); + return builtInMetricAttributes; + } + private void addBuiltInMetricAttributes( CompositeTracer compositeTracer, - Map builtInMetricsAttributes, + Map commonBuiltInMetricsAttributes, Boolean isDirectPathUsed) { if (compositeTracer != null) { - // Direct Path used attribute - Map attributes = new HashMap<>(builtInMetricsAttributes); - attributes.put( - BuiltInMetricsConstant.DIRECT_PATH_USED_KEY.getKey(), Boolean.toString(isDirectPathUsed)); - - compositeTracer.addAttributes(attributes); + compositeTracer.addAttributes( + getBuiltInMetricAttributes(commonBuiltInMetricsAttributes, isDirectPathUsed)); } } diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/SpannerInterceptorProvider.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/SpannerInterceptorProvider.java index c3c05b8af1..64c924c15e 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/SpannerInterceptorProvider.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/SpannerInterceptorProvider.java @@ -18,6 +18,7 @@ import com.google.api.core.InternalApi; import com.google.api.core.ObsoleteApi; import com.google.api.gax.grpc.GrpcInterceptorProvider; +import com.google.cloud.spanner.BuiltInOpenTelemetryMetricsRecorder; import com.google.cloud.spanner.SpannerRpcMetrics; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; @@ -44,12 +45,14 @@ private SpannerInterceptorProvider(List clientInterceptors) { @ObsoleteApi("This method always uses Global OpenTelemetry") public static SpannerInterceptorProvider createDefault() { - return createDefault(GlobalOpenTelemetry.get()); + return createDefault(GlobalOpenTelemetry.get(), GlobalOpenTelemetry.get()); } - public static SpannerInterceptorProvider createDefault(OpenTelemetry openTelemetry) { + public static SpannerInterceptorProvider createDefault( + OpenTelemetry openTelemetry, OpenTelemetry builtInMetricsopenTelemetry) { return createDefault( openTelemetry, + builtInMetricsopenTelemetry, Suppliers.memoize( () -> { return false; @@ -57,13 +60,18 @@ public static SpannerInterceptorProvider createDefault(OpenTelemetry openTelemet } public static SpannerInterceptorProvider createDefault( - OpenTelemetry openTelemetry, Supplier directPathEnabledSupplier) { + OpenTelemetry openTelemetry, + OpenTelemetry builtInMetricsopenTelemetry, + Supplier directPathEnabledSupplier) { List defaultInterceptorList = new ArrayList<>(); defaultInterceptorList.add(new SpannerErrorInterceptor()); defaultInterceptorList.add( new LoggingInterceptor(Logger.getLogger(GapicSpannerRpc.class.getName()), Level.FINER)); defaultInterceptorList.add( - new HeaderInterceptor(new SpannerRpcMetrics(openTelemetry), directPathEnabledSupplier)); + new HeaderInterceptor( + new SpannerRpcMetrics(openTelemetry), + new BuiltInOpenTelemetryMetricsRecorder(builtInMetricsopenTelemetry), + directPathEnabledSupplier)); return new SpannerInterceptorProvider(ImmutableList.copyOf(defaultInterceptorList)); }