Skip to content

Commit

Permalink
ZOOKEEPER-4798: Secure prometheus support
Browse files Browse the repository at this point in the history
  • Loading branch information
purshotam shah committed Feb 25, 2024
1 parent 315abde commit 47e9896
Show file tree
Hide file tree
Showing 5 changed files with 175 additions and 6 deletions.
102 changes: 96 additions & 6 deletions ...rics/src/main/java/org/apache/zookeeper/metrics/prometheus/PrometheusMetricsProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,12 @@
import org.eclipse.jetty.security.ConstraintMapping;
import org.eclipse.jetty.security.ConstraintSecurityHandler;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.util.security.Constraint;
import org.eclipse.jetty.util.ssl.KeyStoreScanner;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -101,8 +104,9 @@ public class PrometheusMetricsProvider implements MetricsProvider {
private final CollectorRegistry collectorRegistry = CollectorRegistry.defaultRegistry;
private final RateLogger rateLogger = new RateLogger(LOG, 60 * 1000);
private String host = "0.0.0.0";
private int port = 7000;
private boolean exportJvmInfo = true;
protected int port = 7000;
private boolean isSsl = false;
protected boolean exportJvmInfo = true;
private Server server;
private final MetricsServletImpl servlet = new MetricsServletImpl();
private final Context rootContext = new Context();
Expand All @@ -111,11 +115,46 @@ public class PrometheusMetricsProvider implements MetricsProvider {
private long workerShutdownTimeoutMs = 1000;
private Optional<ExecutorService> executorOptional = Optional.empty();

// Constants for SSL configuration
public static final int SCAN_INTERVAL = 60 * 10; // 10 minutes
public static final String SSL_KEYSTORE_LOCATION = "ssl.keyStore.location";
public static final String SSL_KEYSTORE_PASSWORD = "ssl.keyStore.password";
public static final String SSL_KEYSTORE_TYPE = "ssl.keyStore.type";
public static final String SSL_TRUSTSTORE_LOCATION = "ssl.trustStore.location";
public static final String SSL_TRUSTSTORE_PASSWORD = "ssl.trustStore.password";
public static final String SSL_TRUSTSTORE_TYPE = "ssl.trustStore.type";
public static final String SSL_X509_CN = "ssl.x509.cn";
public static final String SSL_X509_REGEX_CN = "ssl.x509.cn.regex";
public static final String SSL_NEED_CLIENT_AUTH = "ssl.need.client.auth";
public static final String SSL_WANT_CLIENT_AUTH = "ssl.want.client.auth";

private String keyStorePath;
private String keyStorePassword;
private String keyStoreType;
private String trustStorePath;
private String trustStorePassword;
private String trustStoreType;
private boolean needClientAuth = true;
private boolean wantClientAuth = true;

@Override
public void configure(Properties configuration) throws MetricsProviderLifeCycleException {
LOG.info("Initializing metrics, configuration: {}", configuration);
this.host = configuration.getProperty("httpHost", "0.0.0.0");
this.port = Integer.parseInt(configuration.getProperty("httpPort", "7000"));
if (configuration.containsKey("httpsPort")) {
this.port = Integer.parseInt(configuration.getProperty("httpsPort"));
this.keyStorePath = configuration.getProperty(SSL_KEYSTORE_LOCATION);
this.keyStorePassword = configuration.getProperty(SSL_KEYSTORE_PASSWORD);
this.keyStoreType = configuration.getProperty(SSL_KEYSTORE_TYPE);
this.trustStorePath = configuration.getProperty(SSL_TRUSTSTORE_LOCATION);
this.trustStorePassword = configuration.getProperty(SSL_TRUSTSTORE_PASSWORD);
this.trustStoreType = configuration.getProperty(SSL_TRUSTSTORE_TYPE);
this.needClientAuth = Boolean.parseBoolean(configuration.getProperty(SSL_NEED_CLIENT_AUTH, "true"));
this.wantClientAuth = Boolean.parseBoolean(configuration.getProperty(SSL_WANT_CLIENT_AUTH, "true"));
this.isSsl = true;
} else {
this.port = Integer.parseInt(configuration.getProperty("httpPort", "7000"));
}
this.exportJvmInfo = Boolean.parseBoolean(configuration.getProperty("exportJvmInfo", "true"));
this.numWorkerThreads = Integer.parseInt(
configuration.getProperty(NUM_WORKER_THREADS, "1"));
Expand All @@ -129,12 +168,25 @@ public void configure(Properties configuration) throws MetricsProviderLifeCycleE
public void start() throws MetricsProviderLifeCycleException {
this.executorOptional = createExecutor();
try {
LOG.info("Starting /metrics HTTP endpoint at host: {}, port: {}, exportJvmInfo: {}",
host, port, exportJvmInfo);
LOG.info("Starting /metrics {} endpoint at port {} exportJvmInfo: {}", isSsl ? "HTTPS" : "HTTP", port,
exportJvmInfo);
if (exportJvmInfo) {
DefaultExports.initialize();
}
server = new Server(new InetSocketAddress(host, port));
if (isSsl) {
server = new Server();
SslContextFactory sslServerContextFactory = new SslContextFactory.Server();
configureSslContextFactory(sslServerContextFactory);
KeyStoreScanner keystoreScanner = new KeyStoreScanner(sslServerContextFactory);
keystoreScanner.setScanInterval(SCAN_INTERVAL);
server.addBean(keystoreScanner);
ServerConnector connector = new ServerConnector(server, sslServerContextFactory);
connector.setPort(port);
connector.setHost(host);
server.addConnector(connector);
} else {
server = new Server(new InetSocketAddress(host, port));
}
ServletContextHandler context = new ServletContextHandler();
context.setContextPath("/");
constrainTraceMethod(context);
Expand All @@ -156,6 +208,44 @@ public void start() throws MetricsProviderLifeCycleException {
}
}

@SuppressWarnings("deprecation")
private void configureSslContextFactory(SslContextFactory sslServerContextFactory) {
if (keyStorePath != null) {
sslServerContextFactory.setKeyStorePath(keyStorePath);
} else {
LOG.error("KeyStore configuration is incomplete keyStorePath: {}", keyStorePath);
throw new IllegalStateException("KeyStore configuration is incomplete keyStorePath: " + keyStorePath);
}
if (keyStorePassword != null) {
sslServerContextFactory.setKeyStorePassword(keyStorePassword);
} else {
LOG.error("keyStorePassword configuration is incomplete ");
throw new IllegalStateException("keyStorePassword configuration is incomplete ");
}
if (keyStoreType != null) {
sslServerContextFactory.setKeyStoreType(keyStoreType);
}
if (trustStorePath != null) {
sslServerContextFactory.setTrustStorePath(trustStorePath);
} else {
LOG.error("TrustStore configuration is incomplete trustStorePath: {}", trustStorePath);
throw new IllegalStateException("TrustStore configuration is incomplete trustStorePath: " + trustStorePath);
}
if (trustStorePassword != null) {
sslServerContextFactory.setTrustStorePassword(trustStorePassword);
} else {
LOG.error("trustStorePassword configuration is incomplete");
throw new IllegalStateException("trustStorePassword configuration is incomplete");
}
if (trustStoreType != null) {
sslServerContextFactory.setTrustStoreType(trustStoreType);
}
sslServerContextFactory
.setNeedClientAuth(needClientAuth);
sslServerContextFactory
.setWantClientAuth(wantClientAuth);
}

// for tests
MetricsServletImpl getServlet() {
return servlet;
Expand Down
33 changes: 33 additions & 0 deletions ...est/java/org/apache/zookeeper/metrics/prometheus/PrometheusMetricsProviderConfigTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,4 +59,37 @@ public void testValidConfig() throws MetricsProviderLifeCycleException {
provider.start();
}

@Test
public void testValidSslConfig() throws MetricsProviderLifeCycleException {
PrometheusMetricsProvider provider = new PrometheusMetricsProvider();
Properties configuration = new Properties();
String testDataPath = System.getProperty("test.data.dir", "src/test/resources/data");
configuration.setProperty("httpHost", "127.0.0.1");
configuration.setProperty("httpsPort", "50513");
configuration.setProperty(PrometheusMetricsProvider.SSL_KEYSTORE_LOCATION,
testDataPath + "/ssl/testTrustStore.jks");
configuration.setProperty(PrometheusMetricsProvider.SSL_KEYSTORE_PASSWORD, "testpass");
configuration.setProperty(PrometheusMetricsProvider.SSL_TRUSTSTORE_LOCATION,
testDataPath + "/ssl/testKeyStore.jks");
configuration.setProperty(PrometheusMetricsProvider.SSL_TRUSTSTORE_PASSWORD, "testpass");
provider.configure(configuration);
provider.start();
}

@Test
public void testInvalidSslConfig() throws MetricsProviderLifeCycleException {
assertThrows(MetricsProviderLifeCycleException.class, () -> {
PrometheusMetricsProvider provider = new PrometheusMetricsProvider();
Properties configuration = new Properties();
String testDataPath = System.getProperty("test.data.dir", "src/test/resources/data");
configuration.setProperty("httpsPort", "50514");
//keystore missing
configuration.setProperty(PrometheusMetricsProvider.SSL_KEYSTORE_PASSWORD, "testpass");
configuration.setProperty(PrometheusMetricsProvider.SSL_TRUSTSTORE_LOCATION,
testDataPath + "/ssl/testKeyStore.jks");
configuration.setProperty(PrometheusMetricsProvider.SSL_TRUSTSTORE_PASSWORD, "testpass");
provider.configure(configuration);
provider.start();
});
}
}
46 changes: 46 additions & 0 deletions .../src/test/java/org/apache/zookeeper/metrics/prometheus/PrometheusMetricsProviderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -669,6 +669,52 @@ public void testGaugeSet_unregisterNull() {
() -> provider.getRootContext().unregisterGaugeSet(null));
}

@Test
void testSSLResponce() throws Exception {
PrometheusMetricsProvider provider = new PrometheusMetricsProvider();
Properties configuration = new Properties();
configuration.setProperty("httpHost", "0.0.0.0");
configuration.setProperty("httpsPort", "4443");
configuration.setProperty("exportJvmInfo", "false");
String testDataPath = System.getProperty("test.data.dir", "src/test/resources/data");
configuration.setProperty(PrometheusMetricsProvider.SSL_KEYSTORE_LOCATION,
testDataPath + "/ssl/testTrustStore.jks");
configuration.setProperty(PrometheusMetricsProvider.SSL_KEYSTORE_PASSWORD, "testpass");
configuration.setProperty(PrometheusMetricsProvider.SSL_TRUSTSTORE_LOCATION,
testDataPath + "/ssl/testKeyStore.jks");
configuration.setProperty(PrometheusMetricsProvider.SSL_TRUSTSTORE_PASSWORD, "testpass");
provider.configure(configuration);
provider.start();

Counter counter = provider.getRootContext().getCounter("cc");
counter.add(10);
int[] count = {0};
provider.dump((k, v) -> {
assertEquals("cc", k);
assertEquals(10, ((Number) v).intValue());
count[0]++;
}
);
assertEquals(1, count[0]);
count[0] = 0;

counter.add(-1);

provider.dump((k, v) -> {
assertEquals("cc", k);
assertEquals(10, ((Number) v).intValue());
count[0]++;
}
);
assertEquals(1, count[0]);

assertSame(counter, provider.getRootContext().getCounter("cc"));

String res = callServlet();
assertThat(res, CoreMatchers.containsString("# TYPE cc counter"));
assertThat(res, CoreMatchers.containsString("cc 10.0"));
}

private void createAndRegisterGaugeSet(final String name,
final Map<String, Number> metricsMap,
final AtomicInteger callCount) {
Expand Down
Binary file added ...trics-providers/zookeeper-prometheus-metrics/src/test/resources/data/ssl/testKeyStore.jks
View file
Binary file not shown.
Binary file added ...ics-providers/zookeeper-prometheus-metrics/src/test/resources/data/ssl/testTrustStore.jks
View file
Binary file not shown.

0 comments on commit 47e9896

Please sign in to comment.