Percentile filter

.gitignore

@@ -10,4 +10,5 @@ dependency-reduced-pom.xml
 *.releaseBackup
 release.properties
 
+*.srl
 

README.md

@@ -110,7 +110,17 @@ The available command line options may be seen by passing `-h`/`--help`:
                               [--jmx-service-url=URL] [--jmx-user=NAME]
                               [--keyspace-metrics=FILTER] [--node-metrics=FILTER]
                               [--table-metrics=FILTER]
-                              [--exclude-keyspaces=<excludedKeyspaces>]... [-g=LABEL
+                              [--exclude-keyspaces=<excludedKeyspaces>]...
+                              [--ssl=MODE]
+                              [--ssl-client-authentication=CLIENT-AUTHENTICATION]
+                              [--ssl-implementation=IMPLEMENTATION]
+                              [--ssl-reload-interval=SECONDS]
+                              [--ssl-server-certificate=SERVER-CERTIFICATE]
+                              [--ssl-server-key=SERVER-KEY]
+                              [--ssl-server-key-password=SERVER-KEY-PASSWORD]
+                              [--ssl-trusted-certificate=TRUSTED-CERTIFICATE]
+                              [--ssl-ciphers=CIPHER[,CIPHER...]]...
+                              [--ssl-protocols=PROTOCOL[,PROTOCOL...]]... [-g=LABEL
                               [,LABEL...]]... [-l=[ADDRESS][:PORT]]... [-t=LABEL[,
                               LABEL...]]... [-e=EXCLUSION...]...
       -g, --global-labels=LABEL[,LABEL...]
@@ -180,6 +190,49 @@ The available command line options may be seen by passing `-h`/`--help`:
                                   or PORT will be interpreted as a decimal IPv4 address.
                                   This option may be specified more than once to listen
                                   on multiple addresses. Defaults to '0.0.0.0:9500'
+          --ssl=MODE            Enable or disable secured communication with SSL. Valid
+                                  modes: DISABLE, ENABLE, OPTIONAL. Optional support
+                                  requires Netty version 4.0.45 or later. Defaults to
+                                  DISABLE.
+          --ssl-implementation=IMPLEMENTATION
+                                SSL implementation to use for secure communication.
+                                  OpenSSL requires platform specific libraries. Valid
+                                  implementations: OPENSSL, JDK, DISCOVER. Defaults to
+                                  DISCOVER which will use OpenSSL if required libraries
+                                  are discoverable.
+          --ssl-ciphers=CIPHER[,CIPHER...]
+                                A comma-separated list of SSL cipher suites to enable,
+                                  in the order of preference. Defaults to system
+                                  settings.
+          --ssl-protocols=PROTOCOL[,PROTOCOL...]
+                                A comma-separated list of TLS protocol versions to
+                                  enable. Defaults to system settings.
+          --ssl-reload-interval=SECONDS
+                                Interval in seconds by which keys and certificates will
+                                  be reloaded. Defaults to 0 which will disable run-time
+                                  reload of certificates.
+          --ssl-server-key=SERVER-KEY
+                                Path to the private key file for the SSL server. Must be
+                                  provided together with a server-certificate. The file
+                                  should contain a PKCS#8 private key in PEM format.
+          --ssl-server-key-password=SERVER-KEY-PASSWORD
+                                Path to the private key password file for the SSL
+                                  server. This is only required if the server-key is
+                                  password protected. The file should contain a clear
+                                  text password for the server-key.
+          --ssl-server-certificate=SERVER-CERTIFICATE
+                                Path to the certificate chain file for the SSL server.
+                                  Must be provided together with a server-key. The file
+                                  should contain an X.509 certificate chain in PEM
+                                  format.
+          --ssl-client-authentication=CLIENT-AUTHENTICATION
+                                Set SSL client authentication mode. Valid options: NONE,
+                                  OPTIONAL, REQUIRE, VALIDATE. Defaults to NONE.
+          --ssl-trusted-certificate=TRUSTED-CERTIFICATE
+                                Path to trusted certificates for verifying the remote
+                                  endpoint's certificate. The file should contain an X.
+                                  509 certificate collection in PEM format. Defaults to
+                                  the system setting.
           --family-help=VALUE   Include or exclude metric family help in the exposition
                                   format. AUTOMATIC excludes help strings when the user
                                   agent is Prometheus and includes them for all other

agent/src/main/java/com/zegelin/cassandra/exporter/Agent.java

@@ -28,7 +28,7 @@ public Void call() throws Exception {
 
         final MBeanServerInterceptorHarvester harvester = new MBeanServerInterceptorHarvester(harvesterOptions);
 
-        final Server server = Server.start(httpServerOptions.listenAddresses, harvester, httpServerOptions.helpExposition);
+        final Server server = Server.start(harvester, httpServerOptions);
 
         Runtime.getRuntime().addShutdownHook(new Thread(() -> {
             try {

bin/generate_cert_for_test.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+RESOURCE_PATH="common/src/test/resources"
+
+# Generate a private key and store it both unecrypted and encrypted (password protected)
+# Create a self-signed certificate for the key
+mkdir -p ${RESOURCE_PATH}/cert
+rm -f ${RESOURCE_PATH}/cert/*
+openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -outform PEM -out ${RESOURCE_PATH}/cert/key.pem
+echo -n "password" > ${RESOURCE_PATH}/cert/protected-key.pass
+openssl pkcs8 -topk8 -v1 PBE-SHA1-RC4-128 -in ${RESOURCE_PATH}/cert/key.pem -out ${RESOURCE_PATH}/cert/protected-key.pem -passout file:${RESOURCE_PATH}/cert/protected-key.pass
+openssl req -x509 -new -key ${RESOURCE_PATH}/cert/key.pem -sha256 -days 10000 -out ${RESOURCE_PATH}/cert/cert.pem -subj '/CN=localhost/O=Example Company/C=SE' -nodes

common/pom.xml

@@ -14,16 +14,25 @@
     <name>Cassandra Exporter Common</name>
 
     <properties>
-        <version.picocli>3.6.1</version.picocli>
+        <version.picocli>3.9.5</version.picocli>
     </properties>
 
     <dependencies>
+        <!-- Bring in improved resource leak detection in netty for utests. See #46-->
+        <dependency>
+            <groupId>io.netty</groupId>
+            <artifactId>netty-all</artifactId>
+            <version>4.0.53.Final</version>
+            <scope>provided</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.cassandra</groupId>
             <artifactId>cassandra-all</artifactId>
             <scope>provided</scope>
         </dependency>
 
+
         <dependency>
             <groupId>info.picocli</groupId>
             <artifactId>picocli</artifactId>
@@ -46,5 +55,20 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <version>3.12.0</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>io.netty</groupId>
+            <artifactId>netty-tcnative-boringssl-static</artifactId>
+            <version>2.0.28.Final</version>
+            <scope>test</scope>
+        </dependency>
+
+
     </dependencies>
 </project>

common/src/main/java/com/zegelin/cassandra/exporter/FactoriesSupplier.java

@@ -793,10 +793,12 @@ public List<Factory> get() {
         // org.apache.cassandra.metrics.ThreadPoolMetrics
         {
             builder.add(threadPoolMetric(functionalCollectorConstructor(numericGaugeAsGauge()), "ActiveTasks", "active_tasks", null));
+            builder.add(threadPoolMetric(functionalCollectorConstructor(numericGaugeAsGauge()), "PendingTasks", "pending_tasks", null));
             builder.add(threadPoolMetric(functionalCollectorConstructor(numericGaugeAsCounter()), "CompletedTasks", "completed_tasks_total", null));
             builder.add(threadPoolMetric(functionalCollectorConstructor(counterAsCounter()), "TotalBlockedTasks", "blocked_tasks_total", null));
             builder.add(threadPoolMetric(functionalCollectorConstructor(counterAsGauge()), "CurrentlyBlockedTasks", "blocked_tasks", null));
             builder.add(threadPoolMetric(functionalCollectorConstructor(numericGaugeAsGauge()), "MaxPoolSize", "maximum_tasks", null));
+            builder.add(threadPoolMetric(functionalCollectorConstructor(numericGaugeAsGauge()), "MaxTasksQueued", "maximum_tasks_queued", null));
         }
 
 

common/src/main/java/com/zegelin/cassandra/exporter/Harvester.java

@@ -7,6 +7,7 @@
 import com.zegelin.jmx.NamedObject;
 import com.zegelin.cassandra.exporter.cli.HarvesterOptions;
 import com.zegelin.prometheus.domain.CounterMetricFamily;
+import com.zegelin.prometheus.domain.Interval.Quantile;
 import com.zegelin.prometheus.domain.Labels;
 import com.zegelin.prometheus.domain.MetricFamily;
 import com.zegelin.prometheus.domain.NumericMetric;
@@ -132,6 +133,7 @@ public boolean equals(final Object o) {
 
     private final boolean collectorTimingEnabled;
     private final Map<String, Stopwatch> collectionTimes = new ConcurrentHashMap<>();
+    private final Set<Quantile> excludedHistoQuantiles;
 
     private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder()
             .setNameFormat("cassandra-exporter-harvester-defer-%d")
@@ -145,6 +147,11 @@ protected Harvester(final MetadataFactory metadataFactory, final HarvesterOption
         this.exclusions = options.exclusions;
         this.enabledGlobalLabels = options.globalLabels;
         this.collectorTimingEnabled = options.collectorTimingEnabled;
+        this.excludedHistoQuantiles = options.excludedHistoQuantiles;
+    }
+
+    public Set<Quantile> getExcludedHistoQuantiles() {
+        return excludedHistoQuantiles;
     }
 
     protected void addCollectorFactory(final MBeanGroupMetricFamilyCollector.Factory factory) {

common/src/main/java/com/zegelin/cassandra/exporter/cli/HarvesterOptions.java

@@ -2,6 +2,7 @@
 
 import com.google.common.collect.ImmutableSet;
 import com.zegelin.netty.Floats;
+import com.zegelin.prometheus.domain.Interval.Quantile;
 import com.zegelin.cassandra.exporter.FactoriesSupplier;
 import com.zegelin.cassandra.exporter.Harvester;
 import picocli.CommandLine;
@@ -152,4 +153,27 @@ public void setExcludeSystemTables(final boolean excludeSystemTables) {
 
         excludedKeyspaces.addAll(CASSANDRA_SYSTEM_KEYSPACES);
     }
+
+    public final Set<Quantile> excludedHistoQuantiles = new HashSet<>();
+    @Option(names = {"--exclude-from-histogram"}, paramLabel = "EXCLUSION", arity = "1..*",
+            description = "Select which quantiles to exclude from histogram metrics. The specified quantiles are excluded from all histogram/summary metrics" +
+                    "Valid options are: P_50, P_75, P_95, P_98, P_99, P_99_9" +
+                    "'P_50' (Quantile .5), " +
+                    "'P_75' (Quantile .75), " +
+                    "'P_95' (Quantile .95),  " +
+                    "'P_98' (Quantile .98). " +
+                    "'P_99' (Quantile .99). " +
+                    "'P_99_9' (Quantile .999). " +
+                    "The default is to include all quantiles. "
+            )
+    void setExcludeFromHistogram(final Set<String> values) {
+        values.forEach( e -> {
+            Quantile q = Quantile.ALL_PERCENTILES.get(e);
+            if(q == null) {
+                throw new IllegalArgumentException(String.format("The specified exlusion quantile '%s' is invalid, value values are '%s'", e, Quantile.ALL_PERCENTILES.keySet()));
+            }
+            excludedHistoQuantiles.add(q); 
+        });
+    }
+
 }

common/src/main/java/com/zegelin/cassandra/exporter/cli/HttpServerOptions.java

@@ -1,11 +1,16 @@
 package com.zegelin.cassandra.exporter.cli;
 
-import com.zegelin.picocli.InetSocketAddressTypeConverter;
 import com.zegelin.cassandra.exporter.netty.HttpHandler;
+import com.zegelin.cassandra.exporter.netty.ssl.ClientAuthentication;
+import com.zegelin.cassandra.exporter.netty.ssl.SslImplementation;
+import com.zegelin.cassandra.exporter.netty.ssl.SslMode;
+import com.zegelin.picocli.InetSocketAddressTypeConverter;
 import picocli.CommandLine.Option;
 
+import java.io.File;
 import java.net.InetSocketAddress;
 import java.util.List;
+import java.util.Set;
 
 public class HttpServerOptions {
 
@@ -33,6 +38,91 @@ protected int defaultPort() {
                     "Defaults to '${DEFAULT-VALUE}'")
     public List<InetSocketAddress> listenAddresses;
 
+    @Option(names = "--ssl",
+            paramLabel = "MODE",
+            defaultValue = "DISABLE",
+            description = "Enable or disable secured communication with SSL. " +
+                    "Valid modes: ${COMPLETION-CANDIDATES}. " +
+                    "Optional support requires Netty version 4.0.45 or later. " +
+                    "Defaults to ${DEFAULT-VALUE}."
+    )
+    public SslMode sslMode = SslMode.DISABLE;
+
+    @Option(names = "--ssl-implementation",
+            paramLabel = "IMPLEMENTATION",
+            defaultValue = "DISCOVER",
+            description = "SSL implementation to use for secure communication. " +
+                    "OpenSSL requires platform specific libraries. " +
+                    "Valid implementations: ${COMPLETION-CANDIDATES}. " +
+                    "Defaults to ${DEFAULT-VALUE} which will use OpenSSL if required libraries are discoverable."
+    )
+    public SslImplementation sslImplementation = SslImplementation.DISCOVER;
+
+    @Option(names = "--ssl-ciphers",
+            paramLabel = "CIPHER",
+            split = ",",
+            description = "A comma-separated list of SSL cipher suites to enable, in the order of preference. " +
+                    "Defaults to system settings."
+    )
+    public List<String> sslCiphers;
+
+    @Option(names = "--ssl-protocols",
+            paramLabel = "PROTOCOL",
+            split = ",",
+            description = "A comma-separated list of TLS protocol versions to enable. " +
+                    "Defaults to system settings."
+    )
+    public Set<String> sslProtocols;
+
+    @Option(names = "--ssl-reload-interval",
+            paramLabel = "SECONDS",
+            defaultValue = "0",
+            description = "Interval in seconds by which keys and certificates will be reloaded. " +
+                    "Defaults to ${DEFAULT-VALUE} which will disable run-time reload of certificates."
+    )
+    public long sslReloadIntervalInSeconds = 0L;
+
+    @Option(names = "--ssl-server-key",
+            paramLabel = "SERVER-KEY",
+            description = "Path to the private key file for the SSL server. " +
+                    "Must be provided together with a server-certificate. " +
+                    "The file should contain a PKCS#8 private key in PEM format."
+    )
+    public File sslServerKeyFile;
+
+    @Option(names = "--ssl-server-key-password",
+            paramLabel = "SERVER-KEY-PASSWORD",
+            description = "Path to the private key password file for the SSL server. " +
+                    "This is only required if the server-key is password protected. " +
+                    "The file should contain a clear text password for the server-key."
+    )
+    public File sslServerKeyPasswordFile;
+
+    @Option(names = "--ssl-server-certificate",
+            paramLabel = "SERVER-CERTIFICATE",
+            description = "Path to the certificate chain file for the SSL server. " +
+                    "Must be provided together with a server-key. " +
+                    "The file should contain an X.509 certificate chain in PEM format."
+    )
+    public File sslServerCertificateFile;
+
+    @Option(names = "--ssl-client-authentication",
+            paramLabel = "CLIENT-AUTHENTICATION",
+            defaultValue = "NONE",
+            description = "Set SSL client authentication mode. " +
+                    "Valid options: ${COMPLETION-CANDIDATES}. " +
+                    "Defaults to ${DEFAULT-VALUE}."
+    )
+    public ClientAuthentication sslClientAuthentication = ClientAuthentication.NONE;
+
+    @Option(names = "--ssl-trusted-certificate",
+            paramLabel = "TRUSTED-CERTIFICATE",
+            description = "Path to trusted certificates for verifying the remote endpoint's certificate. " +
+                    "The file should contain an X.509 certificate collection in PEM format. " +
+                    "Defaults to the system setting."
+    )
+    public File sslTrustedCertificateFile;
+
     @Option(names = {"--family-help"},
             paramLabel = "VALUE",
             defaultValue = "AUTOMATIC",

common/src/main/java/com/zegelin/cassandra/exporter/netty/HttpHandler.java

@@ -10,18 +10,21 @@
 import com.zegelin.cassandra.exporter.Harvester;
 import com.zegelin.prometheus.domain.Labels;
 import com.zegelin.prometheus.domain.MetricFamily;
-import com.zegelin.prometheus.exposition.json.JsonFormatChunkedInput;
-import com.zegelin.prometheus.exposition.text.TextFormatChunkedInput;
+import com.zegelin.prometheus.exposition.json.JsonFormatExposition;
+import com.zegelin.prometheus.exposition.text.TextFormatExposition;
+import com.zegelin.prometheus.exposition.FormattedByteChannel;
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.ByteBufUtil;
 import io.netty.channel.ChannelFuture;
 import io.netty.channel.ChannelFutureListener;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.SimpleChannelInboundHandler;
 import io.netty.handler.codec.http.*;
+import io.netty.handler.stream.ChunkedNioStream;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.nio.channels.ReadableByteChannel;
 import java.time.Instant;
 import java.util.List;
 import java.util.Map;
@@ -296,7 +299,8 @@ private ChannelFuture sendMetrics(final ChannelHandlerContext ctx, final FullHtt
                 lastWriteFuture = ctx.writeAndFlush(response);
 
                 if (request.getMethod() == HttpMethod.GET) {
-                    lastWriteFuture = ctx.writeAndFlush(new HttpChunkedInput(new TextFormatChunkedInput(metricFamilyStream, timestamp, globalLabels, includeHelp)));
+                    ReadableByteChannel byteChannel = new FormattedByteChannel(new TextFormatExposition(metricFamilyStream, timestamp, globalLabels, includeHelp, harvester.getExcludedHistoQuantiles()));
+                    lastWriteFuture = ctx.writeAndFlush(new HttpChunkedInput(new ChunkedNioStream(byteChannel, FormattedByteChannel.MAX_CHUNK_SIZE)));
                 }
 
                 return lastWriteFuture;
@@ -308,7 +312,8 @@ private ChannelFuture sendMetrics(final ChannelHandlerContext ctx, final FullHtt
                 lastWriteFuture = ctx.writeAndFlush(response);
 
                 if (request.getMethod() == HttpMethod.GET) {
-                    lastWriteFuture = ctx.writeAndFlush(new HttpChunkedInput(new JsonFormatChunkedInput(metricFamilyStream, timestamp, globalLabels, includeHelp)));
+                    ReadableByteChannel byteChannel = new FormattedByteChannel(new JsonFormatExposition(metricFamilyStream, timestamp, globalLabels, includeHelp));
+                    lastWriteFuture = ctx.writeAndFlush(new HttpChunkedInput(new ChunkedNioStream(byteChannel, FormattedByteChannel.MAX_CHUNK_SIZE)));
                 }
 
                 return lastWriteFuture;