Skip to content

Commit

Permalink
Add an http endpoint wrapper for health check (#12485)
Browse files Browse the repository at this point in the history
  • Loading branch information
kezhenxu94 committed Jul 29, 2024
1 parent ceaa88f commit 6462436
Show file tree
Hide file tree
Showing 11 changed files with 102 additions and 25 deletions.
6 changes: 3 additions & 3 deletions .licenserc.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -106,10 +106,10 @@ dependency:
version: 2.13.4
license: Apache-2.0
- name: com.fasterxml.jackson.datatype:jackson-datatype-jsr310
version: 2.16.1
version: 2.17.1
license: Apache-2.0
- name: com.fasterxml.jackson.datatype:jackson-datatype-jdk8
version: 2.16.1
version: 2.17.1
license: Apache-2.0
- name: com.fasterxml.jackson.dataformat:jackson-dataformat-yaml
version: 2.15.2
Expand Down Expand Up @@ -139,7 +139,7 @@ dependency:
version: 0.6.13
license: Apache-2.0
- name: com.aayushatharva.brotli4j:service
version: 1.15.0
version: 1.16.0
license: Apache-2.0
- name: io.vertx:vertx-grpc
version: 4.5.1
Expand Down
32 changes: 16 additions & 16 deletions dist-material/release-docs/LICENSE
Original file line number Diff line number Diff line change
Expand Up @@ -210,8 +210,8 @@ The following components are provided under the Apache-2.0 License. See project
The text of each license is the standard Apache 2.0 license.
https://mvnrepository.com/artifact/build.buf.protoc-gen-validate/pgv-java-stub/0.6.13 Apache-2.0
https://mvnrepository.com/artifact/build.buf.protoc-gen-validate/protoc-gen-validate/0.6.13 Apache-2.0
https://mvnrepository.com/artifact/com.aayushatharva.brotli4j/brotli4j/1.15.0 Apache-2.0
https://mvnrepository.com/artifact/com.aayushatharva.brotli4j/service/1.15.0 Apache-2.0
https://mvnrepository.com/artifact/com.aayushatharva.brotli4j/brotli4j/1.16.0 Apache-2.0
https://mvnrepository.com/artifact/com.aayushatharva.brotli4j/service/1.16.0 Apache-2.0
https://mvnrepository.com/artifact/com.alibaba.nacos/nacos-auth-plugin/2.3.2 Apache-2.0
https://mvnrepository.com/artifact/com.alibaba.nacos/nacos-client/2.3.2 Apache-2.0
https://mvnrepository.com/artifact/com.alibaba.nacos/nacos-encryption-plugin/2.3.2 Apache-2.0
Expand All @@ -222,8 +222,8 @@ The text of each license is the standard Apache 2.0 license.
https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind/2.16.0 Apache-2.0
https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml/2.15.2 Apache-2.0
https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-guava/2.12.0 Apache-2.0
https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jdk8/2.16.1 Apache-2.0
https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jsr310/2.16.1 Apache-2.0
https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jdk8/2.17.1 Apache-2.0
https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jsr310/2.17.1 Apache-2.0
https://mvnrepository.com/artifact/com.fasterxml.jackson.module/jackson-module-kotlin/2.13.4 Apache-2.0
https://mvnrepository.com/artifact/com.fasterxml/classmate/1.5.1 Apache-2.0
https://mvnrepository.com/artifact/com.google.api.grpc/proto-google-common-protos/2.29.0 Apache-2.0
Expand All @@ -238,12 +238,12 @@ The text of each license is the standard Apache 2.0 license.
https://mvnrepository.com/artifact/com.google.inject/guice/4.1.0 Apache-2.0
https://mvnrepository.com/artifact/com.google.j2objc/j2objc-annotations/2.8 Apache-2.0
https://mvnrepository.com/artifact/com.graphql-java/java-dataloader/3.2.0 Apache-2.0
https://mvnrepository.com/artifact/com.linecorp.armeria/armeria/1.27.3 Apache-2.0
https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-graphql/1.27.3 Apache-2.0
https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-graphql-protocol/1.27.3 Apache-2.0
https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-grpc/1.27.3 Apache-2.0
https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-grpc-protocol/1.27.3 Apache-2.0
https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-protobuf/1.27.3 Apache-2.0
https://mvnrepository.com/artifact/com.linecorp.armeria/armeria/1.29.3 Apache-2.0
https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-graphql/1.29.3 Apache-2.0
https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-graphql-protocol/1.29.3 Apache-2.0
https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-grpc/1.29.3 Apache-2.0
https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-grpc-protocol/1.29.3 Apache-2.0
https://mvnrepository.com/artifact/com.linecorp.armeria/armeria-protobuf/1.29.3 Apache-2.0
https://mvnrepository.com/artifact/com.orbitz.consul/consul-client/1.5.3 Apache-2.0
https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp/3.14.9 Apache-2.0
https://mvnrepository.com/artifact/com.squareup.okio/okio/1.17.2 Apache-2.0
Expand Down Expand Up @@ -297,12 +297,12 @@ The text of each license is the standard Apache 2.0 license.
https://mvnrepository.com/artifact/io.grpc/grpc-netty/1.63.0 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-protobuf/1.63.0 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-protobuf-lite/1.63.0 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-services/1.61.0 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-services/1.64.0 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-stub/1.63.0 Apache-2.0
https://mvnrepository.com/artifact/io.grpc/grpc-util/1.63.0 Apache-2.0
https://mvnrepository.com/artifact/io.micrometer/micrometer-commons/1.12.2 Apache-2.0
https://mvnrepository.com/artifact/io.micrometer/micrometer-core/1.12.2 Apache-2.0
https://mvnrepository.com/artifact/io.micrometer/micrometer-observation/1.12.2 Apache-2.0
https://mvnrepository.com/artifact/io.micrometer/micrometer-commons/1.13.0 Apache-2.0
https://mvnrepository.com/artifact/io.micrometer/micrometer-core/1.13.0 Apache-2.0
https://mvnrepository.com/artifact/io.micrometer/micrometer-observation/1.13.0 Apache-2.0
https://mvnrepository.com/artifact/io.netty/netty-buffer/4.1.108.Final Apache-2.0
https://mvnrepository.com/artifact/io.netty/netty-codec/4.1.108.Final Apache-2.0
https://mvnrepository.com/artifact/io.netty/netty-codec-dns/4.1.108.Final Apache-2.0
Expand Down Expand Up @@ -416,7 +416,7 @@ CC0-1.0 and BSD-2-Clause licenses
The following components are provided under the CC0-1.0 and BSD-2-Clause License. See project link for details.
The text of each license is also included in licenses/LICENSE-[project].txt.

https://mvnrepository.com/artifact/org.hdrhistogram/HdrHistogram/2.1.12 CC0-1.0 and BSD-2-Clause
https://mvnrepository.com/artifact/org.hdrhistogram/HdrHistogram/2.2.1 CC0-1.0 and BSD-2-Clause

========================================================================
ISC licenses
Expand Down Expand Up @@ -548,7 +548,7 @@ The text of each license is also included in licenses/LICENSE-[project].txt.
https://npmjs.com/package/nanoid/v/3.3.7 3.3.7 MIT
https://mvnrepository.com/artifact/org.checkerframework/checker-qual/3.33.0 MIT
https://mvnrepository.com/artifact/org.codehaus.mojo/animal-sniffer-annotations/1.23 MIT
https://mvnrepository.com/artifact/org.curioswitch.curiostack/protobuf-jackson/2.2.0 MIT
https://mvnrepository.com/artifact/org.curioswitch.curiostack/protobuf-jackson/2.5.0 MIT
https://npmjs.com/package/pinia/v/2.0.28 2.0.28 MIT
https://npmjs.com/package/pinia/node_modules/vue-demi/v/0.13.11 0.13.11 MIT
https://npmjs.com/package/postcss/v/8.4.33 8.4.33 MIT
Expand Down
9 changes: 9 additions & 0 deletions docs/en/api/health-check.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# Check OAP healthiness

This is an http wrapper of the [health checker](../setup/backend/backend-health-check.md),
make sure to set necessary configurations required by the health checker before using this endpoint,
otherwise 404 will be returned.

> GET http://localhost:12800/healthcheck
When the OAP server is healthy, the request returns status 200, otherwise, the request returns status 503.
2 changes: 2 additions & 0 deletions docs/en/changes/changes.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@
* BanyanDB: Zipkin Module set service as Entity for improving the query performance.
* MQE: check the metrics value before do binary operation to improve robustness.
* Replace workaround with Armeria native supported context path.
* Add an http endpoint wrapper for health check.
* Bump up Armeria and transitive dependencies.

#### UI

Expand Down
4 changes: 3 additions & 1 deletion docs/en/setup/backend/backend-health-check.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@ health-checker:
```
Note: The `telemetry` module should be enabled at the same time. This means that the provider should not be `-` and `none`.

After that, we can check the OAP server health status by querying GraphQL:
After that, we can check the OAP server health status by querying the http endpoint: `/healthcheck`,
see [the health check http endpoint doc](../../api/health-check.md).

You can also query the healthiness via other methods like GraphQL, see following.
```
query{
checkHealth{
Expand Down
2 changes: 2 additions & 0 deletions docs/menu.yml
Original file line number Diff line number Diff line change
Expand Up @@ -322,6 +322,8 @@ catalog:
path: "/en/api/promql-service"
- name: "LogQL APIs"
path: "/en/api/logql-service"
- name: "Health Check API"
path: "/en/api/health-check"
- name: "Customization"
catalog:
- name: "Overview"
Expand Down
8 changes: 6 additions & 2 deletions oap-server-bom/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@
<postgresql.version>42.4.4</postgresql.version>
<jetcd.version>0.6.1</jetcd.version>
<testcontainers.version>1.17.6</testcontainers.version>
<armeria.version>1.27.3</armeria.version>
<armeria.version>1.29.3</armeria.version>
<awaitility.version>3.0.0</awaitility.version>
<httpcore.version>4.4.13</httpcore.version>
<httpasyncclient.version>4.1.5</httpasyncclient.version>
Expand Down Expand Up @@ -548,7 +548,11 @@
</exclusion>
<exclusion>
<groupId>com.aayushatharva.brotli4j</groupId>
<artifactId>native-osx-x86_64</artifactId>
<artifactId>native-linux-x86_64</artifactId>
</exclusion>
<exclusion>
<groupId>com.aayushatharva.brotli4j</groupId>
<artifactId>native-osx-aarch64</artifactId>
</exclusion>
</exclusions>
</dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,11 @@

import lombok.Getter;
import lombok.Setter;
import lombok.ToString;

@Getter
@Setter
@ToString
public class HealthStatus {
// score == 0 means healthy, otherwise it's unhealthy.
private int score;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package org.apache.skywalking.oap.server.health.checker.provider;

import com.linecorp.armeria.common.HttpRequest;
import com.linecorp.armeria.common.HttpResponse;
import com.linecorp.armeria.common.HttpStatus;
import com.linecorp.armeria.server.ServiceRequestContext;
import com.linecorp.armeria.server.annotation.Get;

import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;

@Slf4j
@RequiredArgsConstructor
public class HealthCheckerHttpService {
private final HealthQueryService healthQueryService;

@Get("/healthcheck")
public HttpResponse healthcheck(ServiceRequestContext ctx, HttpRequest req) throws Exception {
final var status = healthQueryService.checkHealth();
log.info("Health status: {}", status);

if (status.getScore() == 0) {
return HttpResponse.of(HttpStatus.OK);
}
return HttpResponse.of(HttpStatus.SERVICE_UNAVAILABLE);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,16 @@

import com.google.common.util.concurrent.AtomicDouble;
import io.vavr.collection.Stream;

import java.util.Arrays;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.oap.server.core.CoreModule;
import org.apache.skywalking.oap.server.core.server.HTTPHandlerRegister;
import org.apache.skywalking.oap.server.health.checker.module.HealthCheckerModule;
import org.apache.skywalking.oap.server.library.module.ModuleDefine;
import org.apache.skywalking.oap.server.library.module.ModuleProvider;
Expand All @@ -36,6 +40,8 @@
import org.apache.skywalking.oap.server.telemetry.api.MetricsCollector;
import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator;

import com.linecorp.armeria.common.HttpMethod;

/**
* HealthCheckerProvider fetches health check metrics from telemetry module, then calculates health score and generates
* details explains the score. External service or users can query health status by HealthCheckerService.
Expand All @@ -48,6 +54,7 @@ public class HealthCheckerProvider extends ModuleProvider {
private MetricsCollector collector;
private MetricsCreator metricsCreator;
private ScheduledExecutorService ses;
private HealthQueryService healthQueryService;

@Override public String name() {
return "default";
Expand Down Expand Up @@ -75,13 +82,19 @@ public void onInitialized(final HealthCheckerConfig initialized) {
@Override public void prepare() throws ServiceNotProvidedException, ModuleStartException {
score.set(-1);
ses = Executors.newSingleThreadScheduledExecutor();
this.registerServiceImplementation(HealthQueryService.class, new HealthQueryService(score, details));
healthQueryService = new HealthQueryService(score, details);
this.registerServiceImplementation(HealthQueryService.class, healthQueryService);
}

@Override public void start() throws ServiceNotProvidedException, ModuleStartException {
ModuleServiceHolder telemetry = getManager().find(TelemetryModule.NAME).provider();
metricsCreator = telemetry.getService(MetricsCreator.class);
collector = telemetry.getService(MetricsCollector.class);

final var service = getManager().find(CoreModule.NAME)
.provider()
.getService(HTTPHandlerRegister.class);
service.addHandler(new HealthCheckerHttpService(healthQueryService), Arrays.asList(HttpMethod.HEAD, HttpMethod.GET));
}

@Override public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import com.linecorp.armeria.server.ServerBuilder;
import com.linecorp.armeria.server.docs.DocService;
import com.linecorp.armeria.server.encoding.DecodingService;
import com.linecorp.armeria.server.healthcheck.HealthCheckService;
import com.linecorp.armeria.server.logging.LoggingService;

import java.io.FileInputStream;
Expand Down Expand Up @@ -61,7 +60,6 @@ public void initialize() {
.builder()
.baseContextPath(config.getContextPath())
.serviceUnder("/docs", DocService.builder().build())
.service("/internal/l7check", HealthCheckService.of())
.workerGroup(config.getMaxThreads())
.http1MaxHeaderSize(config.getMaxRequestHeaderSize())
.idleTimeout(Duration.ofMillis(config.getIdleTimeOut()))
Expand Down

0 comments on commit 6462436

Please sign in to comment.