lingwei.kong/hawkbit
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Add JPA statistics support for eclipselink and hibernate (#2202)

Browse Source 
Signed-off-by: Avgustin Marinov <Avgustin.Marinov@bosch.com>
This commit is contained in:
Avgustin Marinov
2025-01-20 13:17:55 +02:00
committed by GitHub
parent 357c81fbf4
commit 1f71d6ddb0
10 changed files with 436 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
hawkbit-repository/hawkbit-repository-jpa-eclipselink/pom.xml
View File

@@ -33,6 +33,12 @@
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-core</artifactId>
<optional>true</optional>
</dependency>
<!-- Static class generation -->
<dependency>
<groupId>org.hibernate.orm</groupId>

207
hawkbit-repository/hawkbit-repository-jpa-eclipselink/src/main/java/org/eclipse/hawkbit/repository/jpa/Statistics.java Normal file
View File

@@ -0,0 +1,207 @@
/**
* Copyright (c) 2025 Contributors to the Eclipse Foundation
*
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*/
package org.eclipse.hawkbit.repository.jpa;
import java.util.HashMap;
import java.util.Map;
import java.util.StringTokenizer;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import jakarta.persistence.EntityManagerFactory;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Timer;
import lombok.Getter;
import org.eclipse.persistence.sessions.Session;
import org.eclipse.persistence.tools.profiler.PerformanceMonitor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.Scheduled;
/**
* (Experimental) Report EclipseLink statistics to Micrometer.
* <p/>
* To be enabled:
* <ol>
* <li>The Spring property spring.jpa.properties.eclipselink.profiler=PerformanceMonitor shall be set - enables Eclipselink statistics
* collecting</li>
* <li>By default the stdout log is disabled by setting hawkbit.jpa.statistics.dumpPeriodMS=9223372036854775807 (Long.MAX_VALUE) -
* i.e. effectively <b>never</b>. If log is required it should be set to the required period</li>
* <li>The MeterRegistry shall be registered available - e.g. include org.springframework.boot:spring-boot-actuator-autoconfigure</li>
* <li>(?) When using in test the metrics MAYBE shall be enabled with @AutoConfigureObservability(tracing = false)</li>
* </ol>
*
* It encapsulates reporting the Eclipselink {@link PerformanceMonitor} statistics to the {@link MeterRegistry} and the Spring autoconfiguration.
*/
public class Statistics {
public static final String METER_PREFIX = "eclipselink.";
private static final Statistics INSTANCE = new Statistics();
private static final Pattern PATTERN = Pattern.compile("(?<type>(Counter|Timer)+):(?<key>[^ -]+)");
private static final Map<String, Long> REPORTED_TIMER_VALUES = new HashMap<>();
private EntityManagerFactory entityManagerFactory;
// if meter registry is unavailable, the statistics will not send to metrics
@Getter
private MeterRegistry meterRegistry;
private boolean flushing;
/**
* @return the singleton {@link Statistics} instance
*/
public static Statistics getInstance() {
return INSTANCE;
}
@Autowired
public void setEntityManagerFactory(
final EntityManagerFactory entityManagerFactory,
@Value("${hawkbit.jpa.statistics.dumpPeriodMS:9223372036854775807}") final long dumpPeriod) {
this.entityManagerFactory = entityManagerFactory;
// set stdout log PerformanceMonitor. By default, it is Long.MAX_VALUE (9223372036854775807) which effectively disable logging
getPerformanceMonitor(entityManagerFactory).setDumpTime(dumpPeriod);
}
@Autowired
public void setMeterRegistry(final MeterRegistry meterRegistry) {
this.meterRegistry = meterRegistry;
}
// flushes the statistics to the meter registry (if needed)
public static void flush() {
final MeterRegistry meterRegistry = INSTANCE.meterRegistry;
if (meterRegistry == null) {
// not a bean (i.e. no performance monitoring) is enabled or no meter registry available
return;
}
synchronized (INSTANCE) {
if (INSTANCE.flushing) {
// wait for flushing
do {
try {
INSTANCE.wait(1000);
} catch (final InterruptedException e) {
Thread.currentThread().interrupt();
}
} while (INSTANCE.flushing);
// flushed
return;
}
// flush
INSTANCE.flushing = true;
}
INSTANCE.flush0();
}
@Scheduled(initialDelayString = "${hawkbit.jpa.statistics.flush.fixedDelay:60000}", fixedDelayString = "${hawkbit.jpa.statistics.flush.fixedDelay:60000}")
void periodicFlush() {
if (meterRegistry == null) {
// meter registry available
return;
}
synchronized (this) {
if (flushing) {
// no need to wait for flushing
return;
}
// flush
flushing = true;
}
flush0();
}
private void flush0() {
final PerformanceMonitor performanceMonitor = getPerformanceMonitor(entityManagerFactory);
final Map<String, Object> opTimings = performanceMonitor.getOperationTimings();
opTimings.forEach((k, v) -> {
if (opTimings.keySet().stream().anyMatch(key -> !key.equals(k) && key.startsWith(k))) {
// it is a group, e.g.:
// Timer:ReportQuery 65,402,376
// Timer:ReportQuery:org.eclipse.hawkbit.repository.jpa.model.DistributionSetTypeElement:null:QueryPreparation 177,375
// Timer:ReportQuery:org.eclipse.hawkbit.repository.jpa.model.DistributionSetTypeElement:null:SqlGeneration 36,083
// Counter:ReportQuery 56
// Counter:ReportQuery:org.eclipse.hawkbit.repository.jpa.model.JpaTenantMetaData:null 56
// we want to report per tag/operation, not the group - the sum could be made on the metric collector side (e.g. prometheus)
return;
}
final Matcher matcher = PATTERN.matcher(k);
if (matcher.matches()) {
final String type = matcher.group("type");
final StringTokenizer stringTokenizer = new StringTokenizer(matcher.group("key"), ":");
final String name = METER_PREFIX + stringTokenizer.nextToken();
if (type.equals("Counter")) {
final double quantity = v instanceof Double d ? d : Double.parseDouble(v.toString());
final Counter counter;
if (stringTokenizer.hasMoreTokens()) {
counter = meterRegistry.counter(name, "entity", stringTokenizer.nextToken());
} else {
counter = meterRegistry.counter(name);
}
counter.increment(quantity - counter.count());
} else { // Timer
final long quantity = v instanceof Long l ? l : (long) Double.parseDouble(v.toString());
final Timer timer;
if (stringTokenizer.hasMoreTokens()) {
final String entity = stringTokenizer.nextToken();
stringTokenizer.nextToken(); // skip, what is this?
final String subOp = stringTokenizer.hasMoreTokens() ? stringTokenizer.nextToken() : "n/a";
timer = meterRegistry.timer(name, "entity", entity, "subOp", subOp);
} else {
timer = meterRegistry.timer(name);
}
timer.record(quantity - REPORTED_TIMER_VALUES.getOrDefault(name, 0L), TimeUnit.NANOSECONDS);
REPORTED_TIMER_VALUES.put(name, quantity);
}
}
});
synchronized (this) {
if (flushing) {
flushing = false;
}
this.notifyAll();
}
}
private static PerformanceMonitor getPerformanceMonitor(final EntityManagerFactory entityManagerFactory) {
return (PerformanceMonitor) entityManagerFactory.unwrap(Session.class).getProfiler();
}
// autoconfigure after CompositeMeterRegistryAutoConfiguration, so when the autoconfiguration is being processed the MeterRegistry
// has already been registered / resolved (if it is to be registered at all) - otherwise @ConditionalOnBean(MeterRegistry.class) may not be
// met event if the MeterRegistry is registered (if resolved later).
// 'autoconfigure after' relies on this is being an AutoConfiguration
@AutoConfiguration(afterName = "org.springframework.boot.actuate.autoconfigure.metrics.CompositeMeterRegistryAutoConfiguration")
@Configuration
public static class StatisticsAutoConfiguration {
@ConditionalOnProperty(prefix = "spring.jpa.properties.eclipselink", name = "profiler", havingValue = "PerformanceMonitor")
@ConditionalOnBean(MeterRegistry.class)
@Bean
public Statistics statistics() {
// injects the singleton Statistics, and start scheduler
return Statistics.getInstance();
}
}
}

1
hawkbit-repository/hawkbit-repository-jpa-eclipselink/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports Normal file
View File

@@ -0,0 +1 @@
org.eclipse.hawkbit.repository.jpa.Statistics.StatisticsAutoConfiguration

6
hawkbit-repository/hawkbit-repository-jpa-hibernate/pom.xml
View File

@@ -36,6 +36,12 @@
<artifactId>hibernate-core</artifactId>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-core</artifactId>
<optional>true</optional>
</dependency>
<!-- Static class generation -->
<dependency>
<groupId>org.hibernate.orm</groupId>

77
hawkbit-repository/hawkbit-repository-jpa-hibernate/src/main/java/org/eclipse/hawkbit/repository/jpa/Statistics.java Normal file
View File

@@ -0,0 +1,77 @@
/**
* Copyright (c) 2025 Contributors to the Eclipse Foundation
*
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*/
package org.eclipse.hawkbit.repository.jpa;
import io.micrometer.core.instrument.MeterRegistry;
import lombok.Getter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* (Experimental) Report Hibernate statistics to Micrometer.
* <p/>
* To be enabled:
* <ol>
* <li>The Spring property spring.jpa.properties.hibernate.generate_statistics=true shall be set - enables Hibernate statistics
* collecting</li>
* <li>If don't need log in the stdout set logging.level.org.hibernate.engine.internal.StatisticalLoggingSessionEventListener=WARN</li>
* <li>The MeterRegistry shall be registered available - e.g. include org.springframework.boot:spring-boot-actuator-autoconfigure</li>
* <li>Hibernate reporting to micrometer shall be enabled - include org.hibernate.orm:hibernate-micrometer</li>
* <li>(?) When using in test the metrics MAYBE shall be enabled with @AutoConfigureObservability(tracing = false)</li>
* </ol>
*/
public class Statistics {
public static final String METER_PREFIX = "hibernate.";
private static final Statistics INSTANCE = new Statistics();
// if meter registry is unavailable, the statistics will not send to metrics
@Getter
public MeterRegistry meterRegistry;
/**
* @return the singleton {@link Statistics} instance
*/
public static Statistics getInstance() {
return INSTANCE;
}
@Autowired
public void setMeterRegistry(final MeterRegistry meterRegistry) {
this.meterRegistry = meterRegistry;
}
// flushes the statistics to the meter registry (if needed)
public static void flush() {
// nothing to do for Hibernate
}
// autoconfigure after CompositeMeterRegistryAutoConfiguration, so when the autoconfiguration is being processed the MeterRegistry
// has already been registered / resolved (if it is to be registered at all) - otherwise @ConditionalOnBean(MeterRegistry.class) may not be
// met event if the MeterRegistry is registered (if resolved later).
// 'autoconfigure after' relies on this is being an AutoConfiguration
@AutoConfiguration(afterName = "org.springframework.boot.actuate.autoconfigure.metrics.CompositeMeterRegistryAutoConfiguration")
@Configuration
public static class StatisticsAutoConfiguration {
@ConditionalOnProperty(prefix = "spring.jpa.properties.hibernate", name = "generate_statistics", havingValue = "true")
@ConditionalOnBean(MeterRegistry.class)
@Bean
public Statistics statistics() {
// injects the singleton Statistics, and start scheduler
return Statistics.getInstance();
}
}
}

1
hawkbit-repository/hawkbit-repository-jpa-hibernate/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports Normal file
View File

@@ -0,0 +1 @@
org.eclipse.hawkbit.repository.jpa.Statistics.StatisticsAutoConfiguration

19
hawkbit-repository/hawkbit-repository-jpa/pom.xml
View File

@@ -72,6 +72,12 @@
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-core</artifactId>
<optional>true</optional>
</dependency>
<!-- Static class generation -->
<dependency>
<groupId>org.hibernate.orm</groupId>
@@ -116,5 +122,18 @@
<artifactId>javax.el-api</artifactId>
<scope>test</scope>
</dependency>
<!-- Enable metrics -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-actuator-autoconfigure</artifactId>
<scope>test</scope>
</dependency>
<!-- Enable metrics for hibernates -->
<dependency>
<groupId>org.hibernate.orm</groupId>
<artifactId>hibernate-micrometer</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

3
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/RepositoryApplicationConfiguration.java
View File

@@ -869,8 +869,7 @@ public class RepositoryApplicationConfiguration {
*/
@Bean
@ConditionalOnMissingBean
// don't active the auto assign scheduler in test, otherwise it is hard to
// test
// don't active the auto assign scheduler in test, otherwise it is hard to test
@Profile("!test")
@ConditionalOnProperty(prefix = "hawkbit.autoassign.scheduler", name = "enabled", matchIfMissing = true)
AutoAssignScheduler autoAssignScheduler(final SystemManagement systemManagement,

91
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/utils/StatisticsUtils.java Normal file
View File

@@ -0,0 +1,91 @@
/**
* Copyright (c) 2025 Contributors to the Eclipse Foundation
*
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*/
package org.eclipse.hawkbit.repository.jpa.utils;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.FunctionCounter;
import io.micrometer.core.instrument.Meter;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Tag;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import org.eclipse.hawkbit.repository.jpa.Statistics;
import org.springframework.util.ObjectUtils;
/**
* (Experimental) Utility class to get some statistics.
*/
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class StatisticsUtils {
private static final ThreadLocal<Map<String, Double>> LAST_COUNTERS = ThreadLocal.withInitial(HashMap::new);
// for test purposes we may want to flush the statistics and to get diff from the last get int THIS thread
public static Map<String, Double> diff() {
final MeterRegistry meterRegistry = Statistics.getInstance().getMeterRegistry();
if (meterRegistry == null) {
// not a bean (i.e. no performance monitoring) is enabled or no meter registry available
return Map.of();
}
final Map<String, Double> last = LAST_COUNTERS.get();
final Map<String, Double> current = counters();
return current.entrySet().stream()
.filter(e -> e.getValue() != 0.0)
.filter(e -> e.getValue().doubleValue() != last.getOrDefault(e.getKey(), 0.0).doubleValue())
.collect(Collectors.toMap(
Map.Entry::getKey,
e -> e.getValue() - last.getOrDefault(e.getKey(), 0.0)));
}
// gets the jpa related counters
public static Map<String, Double> counters() {
final MeterRegistry meterRegistry = Statistics.getInstance().getMeterRegistry();
if (meterRegistry == null) {
// not a bean (i.e. no performance monitoring) is enabled or no meter registry available
return Map.of();
}
Statistics.flush();
final Map<String, Double> counters = new HashMap<>();
meterRegistry.forEachMeter(m -> {
final Meter.Id id = m.getId();
if (id.getName().startsWith(Statistics.METER_PREFIX)) {
final double value;
if (m instanceof Counter counter) {
value = counter.count();
} else if (m instanceof FunctionCounter functionCounter) {
value = functionCounter.count();
} else {
return;
}
final StringBuilder key = new StringBuilder(id.getName());
final List<Tag> tags = id.getTags();
if (!ObjectUtils.isEmpty(tags)) {
key.append(" [");
tags.forEach(tag -> key.append(tag.getKey()).append('=').append(tag.getValue()).append(", "));
key.setLength(key.length() - 2);
key.append(']');
}
counters.put(key.toString(), value);
}
});
LAST_COUNTERS.set(counters);
return counters;
}
}

37
hawkbit-repository/hawkbit-repository-jpa/src/test/resources/jpa-test.properties
View File

@@ -8,28 +8,45 @@
# SPDX-License-Identifier: EPL-2.0
#
# Debug utility functions - START
### Debug & Monitor Eclipselink - START
logging.level.org.eclipse.persistence=ERROR
#incomment to see the debug of persistence, e.g. to see the generated SQLs
## Uncomment to see the debug of persistence, e.g. to see the generated SQLs
#logging.level.org.eclipse.persistence=DEBUG
spring.jpa.properties.eclipselink.logging.level=FINE
spring.jpa.properties.eclipselink.logging.level.sql=FINE
spring.jpa.properties.eclipselink.logging.parameters=true
#spring.jpa.properties.eclipselink.logging.level=FINE
#spring.jpa.properties.eclipselink.logging.level.sql=FINE
#spring.jpa.properties.eclipselink.logging.parameters=true
## Enable EclipseLink performance monitor (monitoring and profile)
#spring.jpa.properties.eclipselink.profiler=PerformanceMonitor
### Debug & Monitor Eclipselink - END
### Debug & Monitor Hibernate - START
## Enable the generated SQLs logging
#logging.level.org.hibernate.SQL=TRACE
#logging.level.org.hibernate.stat=TRACE
## Enable Hibernate statistics
#spring.jpa.properties.hibernate.generate_statistics=true
## Disables info log messages from Hibernate statistics
logging.level.org.hibernate.engine.internal.StatisticalLoggingSessionEventListener=WARN
# Debug & Monitor Hibernate - END
#logging.level.org.springframework.security=TRACE
#logging.level.org.springframework.aop=TRACE
#spring.aop.proxy-target-class=true
#hibernate.generate_statistics=true
#logging.level.org.hibernate.SQL=TRACE
#logging.level.org.hibernate.stat=TRACE
# Debug utility functions - END
### Debug utility functions - END
# Switch to mysql
### Switch to MySQL or MariaDB - START
#spring.jpa.database=MYSQL
#spring.datasource.url=jdbc:mariadb://localhost:3306/hawkbit_test
#spring.datasource.driverClassName=org.mariadb.jdbc.Driver
#spring.datasource.username=root
#spring.datasource.password=
### Switch to MySQL or MariaDB - END
# enable / disable case sensitiveness of the DB when playing around
#hawkbit.rsql.caseInsensitiveDB=true