Skip to content

Commit

Permalink
ZOOKEEPER-4798: Secure prometheus support
Browse files Browse the repository at this point in the history
  • Loading branch information
purshotam shah committed Apr 8, 2024
1 parent 315abde commit f0f85b5
Show file tree
Hide file tree
Showing 8 changed files with 350 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,12 @@
import org.eclipse.jetty.security.ConstraintMapping;
import org.eclipse.jetty.security.ConstraintSecurityHandler;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.util.security.Constraint;
import org.eclipse.jetty.util.ssl.KeyStoreScanner;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -101,7 +104,8 @@ public class PrometheusMetricsProvider implements MetricsProvider {
private final CollectorRegistry collectorRegistry = CollectorRegistry.defaultRegistry;
private final RateLogger rateLogger = new RateLogger(LOG, 60 * 1000);
private String host = "0.0.0.0";
private int port = 7000;
private int httpPort = -1;
private int httpsPort = -1;
private boolean exportJvmInfo = true;
private Server server;
private final MetricsServletImpl servlet = new MetricsServletImpl();
Expand All @@ -111,11 +115,49 @@ public class PrometheusMetricsProvider implements MetricsProvider {
private long workerShutdownTimeoutMs = 1000;
private Optional<ExecutorService> executorOptional = Optional.empty();

// Constants for SSL configuration
public static final int SCAN_INTERVAL = 60 * 10; // 10 minutes
public static final String SSL_KEYSTORE_LOCATION = "ssl.keyStore.location";
public static final String SSL_KEYSTORE_PASSWORD = "ssl.keyStore.password";
public static final String SSL_KEYSTORE_TYPE = "ssl.keyStore.type";
public static final String SSL_TRUSTSTORE_LOCATION = "ssl.trustStore.location";
public static final String SSL_TRUSTSTORE_PASSWORD = "ssl.trustStore.password";
public static final String SSL_TRUSTSTORE_TYPE = "ssl.trustStore.type";
public static final String SSL_X509_CN = "ssl.x509.cn";
public static final String SSL_X509_REGEX_CN = "ssl.x509.cn.regex";
public static final String SSL_NEED_CLIENT_AUTH = "ssl.need.client.auth";
public static final String SSL_WANT_CLIENT_AUTH = "ssl.want.client.auth";

private String keyStorePath;
private String keyStorePassword;
private String keyStoreType;
private String trustStorePath;
private String trustStorePassword;
private String trustStoreType;
private boolean needClientAuth = true;
private boolean wantClientAuth = true;

@Override
public void configure(Properties configuration) throws MetricsProviderLifeCycleException {
LOG.info("Initializing metrics, configuration: {}", configuration);
this.host = configuration.getProperty("httpHost", "0.0.0.0");
this.port = Integer.parseInt(configuration.getProperty("httpPort", "7000"));
if (configuration.containsKey("httpsPort")) {
this.httpsPort = Integer.parseInt(configuration.getProperty("httpsPort"));
this.keyStorePath = configuration.getProperty(SSL_KEYSTORE_LOCATION);
this.keyStorePassword = configuration.getProperty(SSL_KEYSTORE_PASSWORD);
this.keyStoreType = configuration.getProperty(SSL_KEYSTORE_TYPE);
this.trustStorePath = configuration.getProperty(SSL_TRUSTSTORE_LOCATION);
this.trustStorePassword = configuration.getProperty(SSL_TRUSTSTORE_PASSWORD);
this.trustStoreType = configuration.getProperty(SSL_TRUSTSTORE_TYPE);
this.needClientAuth = Boolean.parseBoolean(configuration.getProperty(SSL_NEED_CLIENT_AUTH, "true"));
this.wantClientAuth = Boolean.parseBoolean(configuration.getProperty(SSL_WANT_CLIENT_AUTH, "true"));
//check if httpPort is also configured
this.httpPort = Integer.parseInt(configuration.getProperty("httpPort", "-1"));
}
else {
// Use the default HTTP port (7000) or the configured port if HTTPS is not set.
this.httpPort = Integer.parseInt(configuration.getProperty("httpPort", "7000"));
}
this.exportJvmInfo = Boolean.parseBoolean(configuration.getProperty("exportJvmInfo", "true"));
this.numWorkerThreads = Integer.parseInt(
configuration.getProperty(NUM_WORKER_THREADS, "1"));
Expand All @@ -129,12 +171,30 @@ public void configure(Properties configuration) throws MetricsProviderLifeCycleE
public void start() throws MetricsProviderLifeCycleException {
this.executorOptional = createExecutor();
try {
LOG.info("Starting /metrics HTTP endpoint at host: {}, port: {}, exportJvmInfo: {}",
host, port, exportJvmInfo);
LOG.info("Starting /metrics {} endpoint at HTTP port: {}, HTTPS port: {}, exportJvmInfo: {}",
httpPort > 0 ? httpPort : "disabled",
httpsPort > 0 ? httpsPort : "disabled",
exportJvmInfo);
if (exportJvmInfo) {
DefaultExports.initialize();
}
server = new Server(new InetSocketAddress(host, port));
if (httpPort == -1) {
server = new Server();
}
else {
server = new Server(new InetSocketAddress(host, httpPort));
}
if (httpsPort != -1) {
SslContextFactory sslServerContextFactory = new SslContextFactory.Server();
configureSslContextFactory(sslServerContextFactory);
KeyStoreScanner keystoreScanner = new KeyStoreScanner(sslServerContextFactory);
keystoreScanner.setScanInterval(SCAN_INTERVAL);
server.addBean(keystoreScanner);
ServerConnector connector = new ServerConnector(server, sslServerContextFactory);
connector.setPort(httpsPort);
connector.setHost(host);
server.addConnector(connector);
}
ServletContextHandler context = new ServletContextHandler();
context.setContextPath("/");
constrainTraceMethod(context);
Expand All @@ -156,6 +216,44 @@ public void start() throws MetricsProviderLifeCycleException {
}
}

@SuppressWarnings("deprecation")
private void configureSslContextFactory(SslContextFactory sslServerContextFactory) {
if (keyStorePath != null) {
sslServerContextFactory.setKeyStorePath(keyStorePath);
} else {
LOG.error("KeyStore configuration is incomplete keyStorePath: {}", keyStorePath);
throw new IllegalStateException("KeyStore configuration is incomplete keyStorePath: " + keyStorePath);
}
if (keyStorePassword != null) {
sslServerContextFactory.setKeyStorePassword(keyStorePassword);
} else {
LOG.error("keyStorePassword configuration is incomplete ");
throw new IllegalStateException("keyStorePassword configuration is incomplete ");
}
if (keyStoreType != null) {
sslServerContextFactory.setKeyStoreType(keyStoreType);
}
if (trustStorePath != null) {
sslServerContextFactory.setTrustStorePath(trustStorePath);
} else {
LOG.error("TrustStore configuration is incomplete trustStorePath: {}", trustStorePath);
throw new IllegalStateException("TrustStore configuration is incomplete trustStorePath: " + trustStorePath);
}
if (trustStorePassword != null) {
sslServerContextFactory.setTrustStorePassword(trustStorePassword);
} else {
LOG.error("trustStorePassword configuration is incomplete");
throw new IllegalStateException("trustStorePassword configuration is incomplete");
}
if (trustStoreType != null) {
sslServerContextFactory.setTrustStoreType(trustStoreType);
}
sslServerContextFactory
.setNeedClientAuth(needClientAuth);
sslServerContextFactory
.setWantClientAuth(wantClientAuth);
}

// for tests
MetricsServletImpl getServlet() {
return servlet;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.zookeeper.metrics.prometheus;

import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.lang.reflect.Field;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;

import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManagerFactory;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.zookeeper.metrics.Counter;
import org.apache.zookeeper.metrics.CounterSet;
import org.apache.zookeeper.metrics.Gauge;
import org.apache.zookeeper.metrics.GaugeSet;
import org.apache.zookeeper.metrics.MetricsContext;
import org.apache.zookeeper.metrics.Summary;
import org.apache.zookeeper.metrics.SummarySet;
import org.apache.zookeeper.server.util.QuotaMetricsUtils;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.hamcrest.CoreMatchers;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;

/**
* Tests about Prometheus Metrics Provider. Please note that we are not testing
* Prometheus but only our integration.
*/
public class PrometheusHttpsMetricsProviderTest extends PrometheusMetricsTestBase {

private PrometheusMetricsProvider provider;
private String httpHost = "127.0.0.1";
private int httpsPort = 4443;
private int httpPort = 4000;
private String testDataPath = System.getProperty("test.data.dir", "src/test/resources/data");

public void initializeProviderWithCustomConfig(Properties inputConfiguration) throws Exception {
provider = new PrometheusMetricsProvider();
Properties configuration = new Properties();
configuration.setProperty("httpHost", httpHost);
configuration.setProperty("exportJvmInfo", "false");
configuration.setProperty("ssl.keyStore.location", testDataPath + "/ssl/server_keystore.jks");
configuration.setProperty("ssl.keyStore.password", "testpass");
configuration.setProperty("ssl.trustStore.location", testDataPath + "/ssl/server_truststore.jks");
configuration.setProperty("ssl.trustStore.password", "testpass");
configuration.putAll(inputConfiguration);
provider.configure(configuration);
provider.start();
}

@AfterEach
public void tearDown() {
if (provider != null) {
provider.stop();
}
}

@Test
void testHttpResponce() throws Exception {
Properties configuration = new Properties();
configuration.setProperty("httpPort", String.valueOf(httpPort));
initializeProviderWithCustomConfig(configuration);
simulateMetricIncrement();
validateMetricResponse(callHttpServlet("http://" + httpHost + ":" + httpPort + "/metrics"));
}

@Test
void testHttpsResponse() throws Exception {
Properties configuration = new Properties();
configuration.setProperty("httpsPort", String.valueOf(httpsPort));
initializeProviderWithCustomConfig(configuration);
simulateMetricIncrement();
validateMetricResponse(callHttpsServlet("https://" + httpHost + ":" + httpsPort + "/metrics"));
}

@Test
void testHttpAndHttpsResponce() throws Exception {
Properties configuration = new Properties();
configuration.setProperty("httpsPort", String.valueOf(httpsPort));
configuration.setProperty("httpPort", String.valueOf(httpPort));
initializeProviderWithCustomConfig(configuration);
simulateMetricIncrement();
validateMetricResponse(callHttpServlet("http://" + httpHost + ":" + httpPort + "/metrics"));
validateMetricResponse(callHttpsServlet("https://" + httpHost + ":" + httpsPort + "/metrics"));
}

private String callHttpsServlet(String urlString) throws Exception {
// Load and configure the SSL context from the keystore and truststore
KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
try (FileInputStream keystoreStream = new FileInputStream(testDataPath + "/ssl/client_keystore.jks")) {
keyStore.load(keystoreStream, "testpass".toCharArray());
}

KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
try (FileInputStream trustStoreStream = new FileInputStream(testDataPath + "/ssl/client_truststore.jks")) {
trustStore.load(trustStoreStream, "testpass".toCharArray());
}

SSLContext sslContext = SSLContext.getInstance("TLS");
KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
keyManagerFactory.init(keyStore, "testpass".toCharArray());
TrustManagerFactory trustManagerFactory = TrustManagerFactory
.getInstance(TrustManagerFactory.getDefaultAlgorithm());
trustManagerFactory.init(trustStore);
sslContext.init(keyManagerFactory.getKeyManagers(), trustManagerFactory.getTrustManagers(),
new java.security.SecureRandom());

HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory());
URL url = new URL(urlString);
HttpsURLConnection connection = (HttpsURLConnection) url.openConnection();
connection.setRequestMethod("GET");

return readResponse(connection);
}

private String callHttpServlet(String urlString) throws IOException {
URL url = new URL(urlString);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestMethod("GET");
return readResponse(connection);
}

private String readResponse(HttpURLConnection connection) throws IOException {
int status = connection.getResponseCode();
try (BufferedReader reader = new BufferedReader(
new InputStreamReader(status > 299 ? connection.getErrorStream() : connection.getInputStream()))) {
StringBuilder content = new StringBuilder();
String inputLine;
while ((inputLine = reader.readLine()) != null) {
content.append(inputLine).append("\n");
}
return content.toString().trim();
}
finally {
connection.disconnect();
}
}

public void simulateMetricIncrement() {
Counter counter = provider.getRootContext().getCounter("cc");
counter.add(10);
}

private void validateMetricResponse(String response) throws IOException {
assertThat(response, containsString("# TYPE cc counter"));
assertThat(response, containsString("cc 10.0"));
}
}

0 comments on commit f0f85b5

Please sign in to comment.