Align rollouts and autoassign metrics (#2844)

* Refactor auto-assign locking and metrics
* Align rollouts and autoassign metrics

Signed-off-by: Avgustin Marinov <Avgustin.Marinov@bosch.com>
This commit is contained in:
Avgustin Marinov
2025-12-03 12:34:40 +02:00
committed by GitHub
parent 977b3fe40c
commit 904c8b180d
12 changed files with 211 additions and 136 deletions

View File

@@ -12,18 +12,18 @@ package org.eclipse.hawkbit.repository;
/** /**
* An interface declaration which contains the check for the auto assignment logic. * An interface declaration which contains the check for the auto assignment logic.
*/ */
public interface AutoAssignExecutor { public interface AutoAssignHandler {
/** /**
* Checks all target filter queries with an auto assign distribution set and triggers the check and assignment to targets that don't have * Checks all target filter queries with an auto assign distribution set and triggers the check and assignment to targets that don't have
* the design DS yet * the design DS yet
*/ */
void checkAllTargets(); void handleAll();
/** /**
* Method performs an auto assign check for a specific device only * Method performs an auto assign check for a specific device only
* *
* @param controllerId of the device to check * @param controllerId of the device to check
*/ */
void checkSingleTarget(String controllerId); void handleSingleTarget(String controllerId);
} }

View File

@@ -28,7 +28,7 @@ import org.eclipse.hawkbit.ql.jpa.QLSupport;
import org.eclipse.hawkbit.ql.jpa.QLSupport.NodeTransformer; import org.eclipse.hawkbit.ql.jpa.QLSupport.NodeTransformer;
import org.eclipse.hawkbit.ql.jpa.QLSupport.QueryParser; import org.eclipse.hawkbit.ql.jpa.QLSupport.QueryParser;
import org.eclipse.hawkbit.ql.rsql.RsqlParser; import org.eclipse.hawkbit.ql.rsql.RsqlParser;
import org.eclipse.hawkbit.repository.AutoAssignExecutor; import org.eclipse.hawkbit.repository.AutoAssignHandler;
import org.eclipse.hawkbit.repository.DeploymentManagement; import org.eclipse.hawkbit.repository.DeploymentManagement;
import org.eclipse.hawkbit.repository.PropertiesQuotaManagement; import org.eclipse.hawkbit.repository.PropertiesQuotaManagement;
import org.eclipse.hawkbit.repository.QuotaManagement; import org.eclipse.hawkbit.repository.QuotaManagement;
@@ -341,10 +341,10 @@ public class JpaRepositoryConfiguration {
@Profile("!test") @Profile("!test")
@ConditionalOnProperty(prefix = "hawkbit.autoassign.scheduler", name = "enabled", matchIfMissing = true) @ConditionalOnProperty(prefix = "hawkbit.autoassign.scheduler", name = "enabled", matchIfMissing = true)
AutoAssignScheduler autoAssignScheduler( AutoAssignScheduler autoAssignScheduler(
final SystemManagement systemManagement, final AutoAssignExecutor autoAssignExecutor, final SystemManagement systemManagement, final AutoAssignHandler autoAssignHandler,
@Value("${hawkbit.autoassign.executor.thread-pool.size:1}") final int threadPoolSize, @Value("${hawkbit.autoassign.executor.thread-pool.size:1}") final int threadPoolSize,
final LockRegistry lockRegistry, final Optional<MeterRegistry> meterRegistry) { final LockRegistry lockRegistry, final Optional<MeterRegistry> meterRegistry) {
return new AutoAssignScheduler(systemManagement, autoAssignExecutor, threadPoolSize, lockRegistry, meterRegistry); return new AutoAssignScheduler(systemManagement, autoAssignHandler, threadPoolSize, meterRegistry);
} }
/** /**

View File

@@ -14,14 +14,11 @@ import static org.eclipse.hawkbit.context.AccessContext.asSystemAsTenant;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.MeterRegistry;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.eclipse.hawkbit.repository.AutoAssignExecutor; import org.eclipse.hawkbit.repository.AutoAssignHandler;
import org.eclipse.hawkbit.repository.SystemManagement; import org.eclipse.hawkbit.repository.SystemManagement;
import org.eclipse.hawkbit.tenancy.DefaultTenantConfiguration;
import org.springframework.integration.support.locks.LockRegistry;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
@@ -34,18 +31,16 @@ public class AutoAssignScheduler {
private static final String PROP_SCHEDULER_DELAY_PLACEHOLDER = "${hawkbit.autoassign.scheduler.fixedDelay:2000}"; private static final String PROP_SCHEDULER_DELAY_PLACEHOLDER = "${hawkbit.autoassign.scheduler.fixedDelay:2000}";
private final SystemManagement systemManagement; private final SystemManagement systemManagement;
private final AutoAssignExecutor autoAssignExecutor; private final AutoAssignHandler autoAssignHandler;
private final LockRegistry lockRegistry;
private final Optional<MeterRegistry> meterRegistry; private final Optional<MeterRegistry> meterRegistry;
private final ThreadPoolTaskExecutor autoAssignTaskExecutor; private final ThreadPoolTaskExecutor autoAssignTaskExecutor;
public AutoAssignScheduler( public AutoAssignScheduler(
final SystemManagement systemManagement, final AutoAssignExecutor autoAssignExecutor, final SystemManagement systemManagement, final AutoAssignHandler autoAssignHandler,
final int threadPoolSize, final int threadPoolSize, final Optional<MeterRegistry> meterRegistry) {
final LockRegistry lockRegistry, final Optional<MeterRegistry> meterRegistry) {
this.systemManagement = systemManagement; this.systemManagement = systemManagement;
this.autoAssignExecutor = autoAssignExecutor; this.autoAssignHandler = autoAssignHandler;
this.lockRegistry = lockRegistry;
this.meterRegistry = meterRegistry; this.meterRegistry = meterRegistry;
autoAssignTaskExecutor = SchedulerUtils.threadPoolTaskExecutor("auto-assign-exec-", threadPoolSize); autoAssignTaskExecutor = SchedulerUtils.threadPoolTaskExecutor("auto-assign-exec-", threadPoolSize);
} }
@@ -58,7 +53,8 @@ public class AutoAssignScheduler {
public void autoAssignScheduler() { public void autoAssignScheduler() {
// run this code in system code privileged to have the necessary permission to query and create entities. // run this code in system code privileged to have the necessary permission to query and create entities.
log.debug("Triggered auto-assign scheduler."); log.debug("Triggered auto-assign scheduler.");
final long startNano = java.lang.System.nanoTime(); final long startNano = System.nanoTime();
asSystem(() -> asSystem(() ->
systemManagement.forEachTenantAsSystem(tenant -> { systemManagement.forEachTenantAsSystem(tenant -> {
if (autoAssignTaskExecutor == null) {// sync if (autoAssignTaskExecutor == null) {// sync
@@ -68,31 +64,19 @@ public class AutoAssignScheduler {
} }
}) })
); );
meterRegistry
.map(mReg -> mReg.timer("hawkbit.autoassign.scheduler.all")) meterRegistry // handle all tenants (some could be skipped if lock could not be obtained)
.ifPresent(timer -> timer.record(java.lang.System.nanoTime() - startNano, TimeUnit.NANOSECONDS)); .map(mReg -> mReg.timer("hawkbit.autoassign.scheduler"))
.ifPresent(timer -> timer.record(System.nanoTime() - startNano, TimeUnit.NANOSECONDS));
log.debug("Finished auto-assign scheduler run."); log.debug("Finished auto-assign scheduler run.");
} }
private void handleAll(final String tenant) { private void handleAll(final String tenant) {
final Lock lock = lockRegistry.obtain(createAutoAssignmentLockKey(tenant)); log.trace("Handling auto-assignments for tenant: {}", tenant);
if (!lock.tryLock()) {
return;
}
final long startNanoT = System.nanoTime();
try { try {
autoAssignExecutor.checkAllTargets(); autoAssignHandler.handleAll();
} finally { } catch (final Exception e) {
lock.unlock(); log.error("Error auto-assignments rollout for tenant {}", tenant, e);
meterRegistry
.map(mReg -> mReg.timer(
"hawkbit.autoassign.scheduler",
DefaultTenantConfiguration.TENANT_TAG, tenant))
.ifPresent(timer -> timer.record(System.nanoTime() - startNanoT, TimeUnit.NANOSECONDS));
} }
} }
private static String createAutoAssignmentLockKey(final String tenant) {
return tenant + "-auto-assign";
}
} }

View File

@@ -11,16 +11,24 @@ package org.eclipse.hawkbit.repository.jpa.scheduler;
import static org.eclipse.hawkbit.context.AccessContext.asActor; import static org.eclipse.hawkbit.context.AccessContext.asActor;
import static org.eclipse.hawkbit.context.AccessContext.withSecurityContext; import static org.eclipse.hawkbit.context.AccessContext.withSecurityContext;
import static org.eclipse.hawkbit.tenancy.DefaultTenantConfiguration.TENANT_TAG;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
import java.util.function.Consumer; import java.util.function.Consumer;
import jakarta.persistence.LockTimeoutException;
import jakarta.persistence.PersistenceException; import jakarta.persistence.PersistenceException;
import io.micrometer.core.instrument.MeterRegistry;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.eclipse.hawkbit.context.AccessContext;
import org.eclipse.hawkbit.exception.AbstractServerRtException; import org.eclipse.hawkbit.exception.AbstractServerRtException;
import org.eclipse.hawkbit.repository.AutoAssignExecutor; import org.eclipse.hawkbit.repository.AutoAssignHandler;
import org.eclipse.hawkbit.repository.DeploymentManagement; import org.eclipse.hawkbit.repository.DeploymentManagement;
import org.eclipse.hawkbit.repository.TargetFilterQueryManagement; import org.eclipse.hawkbit.repository.TargetFilterQueryManagement;
import org.eclipse.hawkbit.repository.TargetManagement; import org.eclipse.hawkbit.repository.TargetManagement;
@@ -33,6 +41,7 @@ import org.eclipse.hawkbit.repository.model.TargetFilterQuery;
import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Slice; import org.springframework.data.domain.Slice;
import org.springframework.integration.support.locks.LockRegistry;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.annotation.Isolation; import org.springframework.transaction.annotation.Isolation;
@@ -47,7 +56,7 @@ import org.springframework.util.StringUtils;
*/ */
@Slf4j @Slf4j
@Service @Service
public class JpaAutoAssignExecutor implements AutoAssignExecutor { public class JpaAutoAssignHandler implements AutoAssignHandler {
/** /**
* The message which is added to the action status when a distribution set is assigned to a target. * The message which is added to the action status when a distribution set is assigned to a target.
@@ -60,33 +69,98 @@ public class JpaAutoAssignExecutor implements AutoAssignExecutor {
*/ */
private static final int PAGE_SIZE = 1000; private static final int PAGE_SIZE = 1000;
private static final LockTimeoutException LOCK_TIMEOUT_EXCEPTION = new LockTimeoutException("Could not obtain lock for auto assignment");
private final TargetFilterQueryManagement<? extends TargetFilterQuery> targetFilterQueryManagement; private final TargetFilterQueryManagement<? extends TargetFilterQuery> targetFilterQueryManagement;
private final TargetManagement<? extends Target> targetManagement; private final TargetManagement<? extends Target> targetManagement;
private final DeploymentManagement deploymentManagement; private final DeploymentManagement deploymentManagement;
private final PlatformTransactionManager transactionManager; private final PlatformTransactionManager transactionManager;
private final LockRegistry lockRegistry;
private final Optional<MeterRegistry> meterRegistry;
public JpaAutoAssignExecutor( public JpaAutoAssignHandler(
final TargetFilterQueryManagement<? extends TargetFilterQuery> targetFilterQueryManagement, final TargetFilterQueryManagement<? extends TargetFilterQuery> targetFilterQueryManagement,
final TargetManagement<? extends Target> targetManagement, final DeploymentManagement deploymentManagement, final TargetManagement<? extends Target> targetManagement, final DeploymentManagement deploymentManagement,
final PlatformTransactionManager transactionManager) { final PlatformTransactionManager transactionManager, final LockRegistry lockRegistry, final Optional<MeterRegistry> meterRegistry) {
this.targetFilterQueryManagement = targetFilterQueryManagement; this.targetFilterQueryManagement = targetFilterQueryManagement;
this.targetManagement = targetManagement; this.targetManagement = targetManagement;
this.deploymentManagement = deploymentManagement; this.deploymentManagement = deploymentManagement;
this.transactionManager = transactionManager; this.transactionManager = transactionManager;
this.lockRegistry = lockRegistry;
this.meterRegistry = meterRegistry;
} }
@Override @Override
@Transactional(propagation = Propagation.REQUIRES_NEW) @Transactional(propagation = Propagation.REQUIRES_NEW)
public void checkAllTargets() { public void handleAll() {
log.debug("Auto assign check call started"); final long startNano = System.nanoTime();
forEachFilterWithAutoAssignDS(this::checkByTargetFilterQueryAndAssignDS);
log.debug("Auto assign check call finished"); final AtomicReference<Lock> lockRef = new AtomicReference<>();
try {
forEachFilterWithAutoAssignDS(targetFilterQuery -> {
if (lockRef.get() == null) {
// only if there are targetFilterQueries to process we try to obtain the lock (on the first one)
final Lock lock = lockRegistry.obtain(createAutoAssignmentLockKey(AccessContext.tenant()));
if (!lock.tryLock()) {
if (log.isTraceEnabled()) {
log.trace("Could not obtain lock {}", lock);
}
throw LOCK_TIMEOUT_EXCEPTION;
} else {
lockRef.set(lock);
log.debug("Start auto-assign handling");
}
}
final long startNanoPartial = System.nanoTime();
try {
checkByTargetFilterQueryAndAssignDS(targetFilterQuery);
} finally {
meterRegistry // handle single targetFilterQuery
.map(mReg -> mReg.timer(
"hawkbit.autoassign.handle", TENANT_TAG, AccessContext.tenant(), "targetFilterQuery",
String.valueOf(targetFilterQuery.getId())))
.ifPresent(timer -> timer.record(System.nanoTime() - startNanoPartial, TimeUnit.NANOSECONDS));
}
});
} finally {
final Lock lock = lockRef.get();
if (lock != null) {
lock.unlock();
// only if there is at least one targetFilterQuery and lock has been obtained then will be measured (as in rollouts)
meterRegistry // handle single targetFilterQuery for single target
.map(mReg -> mReg.timer(
"hawkbit.autoassign.handle.all", TENANT_TAG, AccessContext.tenant()))
.ifPresent(timer -> timer.record(System.nanoTime() - startNano, TimeUnit.NANOSECONDS));
log.debug("Auto assign check all targets finished");
}
}
} }
@Override @Override
public void checkSingleTarget(final String controllerId) { public void handleSingleTarget(final String controllerId) {
log.debug("Auto assign check call for device {} started", controllerId); log.debug("Auto assign check call for device {} started", controllerId);
forEachFilterWithAutoAssignDS(filter -> checkForDevice(controllerId, filter)); final long startNano = System.nanoTime();
forEachFilterWithAutoAssignDS(targetFilterQuery -> {
final long startNanoPartial = System.nanoTime();
try {
checkForDevice(controllerId, targetFilterQuery);
} finally {
meterRegistry // handle single targetFilterQuery for single target
.map(mReg -> mReg.timer(
"hawkbit.autoassign.handle.single", TENANT_TAG, AccessContext.tenant(), "targetFilterQuery",
String.valueOf(targetFilterQuery.getId())))
.ifPresent(timer -> timer.record(System.nanoTime() - startNanoPartial, TimeUnit.NANOSECONDS));
}
});
meterRegistry // handle single targetFilterQuery for single target
.map(mReg -> mReg.timer(
"hawkbit.autoassign.handle.single.all", TENANT_TAG, AccessContext.tenant()))
.ifPresent(timer -> timer.record(System.nanoTime() - startNano, TimeUnit.NANOSECONDS));
log.debug("Auto assign check call for device {} finished", controllerId); log.debug("Auto assign check call for device {} finished", controllerId);
} }
@@ -134,15 +208,19 @@ public class JpaAutoAssignExecutor implements AutoAssignExecutor {
do { do {
filterQueries = targetFilterQueryManagement.findWithAutoAssignDS(query); filterQueries = targetFilterQueryManagement.findWithAutoAssignDS(query);
try {
filterQueries.forEach(filterQuery -> { filterQueries.forEach(filterQuery -> {
try { try {
filterQuery.getAccessControlContext().ifPresentOrElse( filterQuery.getAccessControlContext().ifPresentOrElse(
// has stored context - executes it with it // has stored context - executes it with it
context -> withSecurityContext(context, () -> consumer.accept(filterQuery)), context -> withSecurityContext(context, () -> consumer.accept(filterQuery)),
// has no stored context - executes it in the tenant & user scope // has no stored context - executes it in the tenant & user scope
() -> asActor(getAutoAssignmentInitiatedBy(filterQuery), () -> consumer.accept(filterQuery)) () -> asActor(getAutoAssignmentInitiatedBy(filterQuery), () -> consumer.accept(filterQuery)));
);
} catch (final RuntimeException ex) { } catch (final RuntimeException ex) {
if (ex == LOCK_TIMEOUT_EXCEPTION) {
// expected - just stop processing further
throw ex; // throw in order to break
}
if (log.isDebugEnabled()) { if (log.isDebugEnabled()) {
log.debug("Exception on forEachFilterWithAutoAssignDS execution for filter id {}. Continue with next filter query.", log.debug("Exception on forEachFilterWithAutoAssignDS execution for filter id {}. Continue with next filter query.",
filterQuery.getId(), ex); filterQuery.getId(), ex);
@@ -153,6 +231,9 @@ public class JpaAutoAssignExecutor implements AutoAssignExecutor {
} }
} }
}); });
} catch (final LockTimeoutException lte) {
break; // lock not found
}
} while (filterQueries.hasNext() && (query = filterQueries.nextPageable()) != Pageable.unpaged()); } while (filterQueries.hasNext() && (query = filterQueries.nextPageable()) != Pageable.unpaged());
} }
@@ -209,4 +290,8 @@ public class JpaAutoAssignExecutor implements AutoAssignExecutor {
} }
log.debug("Auto assign check call for target filter query id {} for device {} finished", targetFilterQuery.getId(), controllerId); log.debug("Auto assign check call for target filter query id {} for device {} finished", targetFilterQuery.getId(), controllerId);
} }
private static String createAutoAssignmentLockKey(final String tenant) {
return tenant + "-auto-assign";
}
} }

View File

@@ -9,6 +9,8 @@
*/ */
package org.eclipse.hawkbit.repository.jpa.scheduler; package org.eclipse.hawkbit.repository.jpa.scheduler;
import static org.eclipse.hawkbit.tenancy.DefaultTenantConfiguration.TENANT_TAG;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@@ -21,7 +23,6 @@ import org.eclipse.hawkbit.repository.RolloutExecutor;
import org.eclipse.hawkbit.repository.RolloutHandler; import org.eclipse.hawkbit.repository.RolloutHandler;
import org.eclipse.hawkbit.repository.RolloutManagement; import org.eclipse.hawkbit.repository.RolloutManagement;
import org.eclipse.hawkbit.repository.jpa.utils.DeploymentHelper; import org.eclipse.hawkbit.repository.jpa.utils.DeploymentHelper;
import org.eclipse.hawkbit.tenancy.DefaultTenantConfiguration;
import org.springframework.integration.support.locks.LockRegistry; import org.springframework.integration.support.locks.LockRegistry;
import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.PlatformTransactionManager;
@@ -57,6 +58,8 @@ public class JpaRolloutHandler implements RolloutHandler {
@Override @Override
public void handleAll() { public void handleAll() {
final long startNano = System.nanoTime();
final List<Long> rollouts = rolloutManagement.findActiveRollouts(); final List<Long> rollouts = rolloutManagement.findActiveRollouts();
if (rollouts.isEmpty()) { if (rollouts.isEmpty()) {
return; return;
@@ -66,15 +69,13 @@ public class JpaRolloutHandler implements RolloutHandler {
final Lock lock = lockRegistry.obtain(handlerId); final Lock lock = lockRegistry.obtain(handlerId);
if (!lock.tryLock()) { if (!lock.tryLock()) {
if (log.isTraceEnabled()) { if (log.isTraceEnabled()) {
log.trace("Could not perform lock {}", lock); log.trace("Could not obtain lock {}", lock);
} }
return; return;
} }
try { try {
log.debug("Trigger handling {} rollouts.", rollouts.size()); log.debug("Start handling {} rollouts.", rollouts.size());
final long startNano = System.nanoTime();
rollouts.forEach(rolloutId -> { rollouts.forEach(rolloutId -> {
try { try {
handleRolloutInNewTransaction(rolloutId, handlerId); handleRolloutInNewTransaction(rolloutId, handlerId);
@@ -82,16 +83,16 @@ public class JpaRolloutHandler implements RolloutHandler {
log.error("Failed to process rollout with id {}", rolloutId, throwable); log.error("Failed to process rollout with id {}", rolloutId, throwable);
} }
}); });
meterRegistry
.map(mReg -> mReg.timer("hawkbit.rollout.handler.all", DefaultTenantConfiguration.TENANT_TAG, AccessContext.tenant()))
.ifPresent(timer -> timer.record(System.nanoTime() - startNano, TimeUnit.NANOSECONDS));
log.debug("Finished handling of the rollouts."); log.debug("Finished handling of the rollouts.");
} finally { } finally {
lock.unlock();
if (log.isTraceEnabled()) { if (log.isTraceEnabled()) {
log.trace("Unlock lock {}", lock); log.trace("Unlock lock {}", lock);
} }
lock.unlock(); meterRegistry // handle all rollouts of a tenant
.map(mReg -> mReg.timer("hawkbit.rollout.handle.all", TENANT_TAG, AccessContext.tenant()))
.ifPresent(timer -> timer.record(System.nanoTime() - startNano, TimeUnit.NANOSECONDS));
} }
} }
@@ -110,11 +111,9 @@ public class JpaRolloutHandler implements RolloutHandler {
return 0L; return 0L;
}); });
meterRegistry meterRegistry // handle single rollout
.map(mReg -> mReg.timer( .map(mReg -> mReg.timer(
"hawkbit.rollout.handler", "hawkbit.rollout.handle", TENANT_TAG, AccessContext.tenant(), "rollout", String.valueOf(rolloutId)))
DefaultTenantConfiguration.TENANT_TAG, AccessContext.tenant(),
"rollout", String.valueOf(rolloutId)))
.ifPresent(timer -> timer.record(System.nanoTime() - startNano, TimeUnit.NANOSECONDS)); .ifPresent(timer -> timer.record(System.nanoTime() - startNano, TimeUnit.NANOSECONDS));
} }
} }

View File

@@ -19,7 +19,6 @@ import io.micrometer.core.instrument.MeterRegistry;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.eclipse.hawkbit.repository.RolloutHandler; import org.eclipse.hawkbit.repository.RolloutHandler;
import org.eclipse.hawkbit.repository.SystemManagement; import org.eclipse.hawkbit.repository.SystemManagement;
import org.eclipse.hawkbit.tenancy.DefaultTenantConfiguration;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
@@ -54,8 +53,8 @@ public class RolloutScheduler {
*/ */
@Scheduled(initialDelayString = PROP_SCHEDULER_DELAY_PLACEHOLDER, fixedDelayString = PROP_SCHEDULER_DELAY_PLACEHOLDER) @Scheduled(initialDelayString = PROP_SCHEDULER_DELAY_PLACEHOLDER, fixedDelayString = PROP_SCHEDULER_DELAY_PLACEHOLDER)
public void runningRolloutScheduler() { public void runningRolloutScheduler() {
log.debug("rollout schedule checker has been triggered."); log.debug("Rollout scheduler has been triggered.");
final long startNano = java.lang.System.nanoTime(); final long startNano = System.nanoTime();
// run this code in system code privileged to have the necessary system code permission execute forEachTenant // run this code in system code privileged to have the necessary system code permission execute forEachTenant
asSystem(() -> asSystem(() ->
@@ -66,28 +65,21 @@ public class RolloutScheduler {
if (rolloutTaskExecutor == null) { if (rolloutTaskExecutor == null) {
handleAll(tenant); handleAll(tenant);
} else { } else {
rolloutTaskExecutor.submit(() -> asSystemAsTenant(tenant, () -> handleAll(tenant))); rolloutTaskExecutor.execute(() -> asSystemAsTenant(tenant, () -> handleAll(tenant)));
} }
})); }));
meterRegistry meterRegistry // handle all tenants (some could be skipped if lock could not be obtained)
.map(mReg -> mReg.timer("hawkbit.rollout.scheduler.all")) .map(mReg -> mReg.timer("hawkbit.rollout.scheduler"))
.ifPresent(timer -> timer.record(java.lang.System.nanoTime() - startNano, TimeUnit.NANOSECONDS)); .ifPresent(timer -> timer.record(System.nanoTime() - startNano, TimeUnit.NANOSECONDS));
} }
private void handleAll(final String tenant) { private void handleAll(final String tenant) {
log.trace("Handling rollout for tenant: {}", tenant); log.trace("Handling rollouts for tenant: {}", tenant);
final long startNano = java.lang.System.nanoTime();
try { try {
rolloutHandler.handleAll(); rolloutHandler.handleAll();
} catch (final Exception e) { } catch (final Exception e) {
log.error("Error processing rollout for tenant {}", tenant, e); log.error("Error processing rollout for tenant {}", tenant, e);
} }
meterRegistry
.map(mReg -> mReg.timer("hawkbit.rollout.scheduler", DefaultTenantConfiguration.TENANT_TAG, tenant))
.ifPresent(timer -> timer.record(java.lang.System.nanoTime() - startNano, TimeUnit.NANOSECONDS));
} }
} }

View File

@@ -19,7 +19,7 @@ import static org.eclipse.hawkbit.repository.test.util.SecurityContextSwitch.cal
import java.util.Optional; import java.util.Optional;
import org.eclipse.hawkbit.repository.AutoAssignExecutor; import org.eclipse.hawkbit.repository.AutoAssignHandler;
import org.eclipse.hawkbit.repository.Identifiable; import org.eclipse.hawkbit.repository.Identifiable;
import org.eclipse.hawkbit.repository.TargetFilterQueryManagement; import org.eclipse.hawkbit.repository.TargetFilterQueryManagement;
import org.eclipse.hawkbit.repository.TargetFilterQueryManagement.AutoAssignDistributionSetUpdate; import org.eclipse.hawkbit.repository.TargetFilterQueryManagement.AutoAssignDistributionSetUpdate;
@@ -27,34 +27,30 @@ import org.eclipse.hawkbit.repository.jpa.scheduler.AutoAssignScheduler;
import org.eclipse.hawkbit.repository.model.TargetFilterQuery; import org.eclipse.hawkbit.repository.model.TargetFilterQuery;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.integration.support.locks.LockRegistry;
class AutoAssignTest extends AbstractAccessControllerManagementTest { class AutoAssignTest extends AbstractAccessControllerManagementTest {
@Autowired @Autowired
AutoAssignExecutor autoAssignExecutor; AutoAssignHandler autoAssignHandler;
@Autowired
LockRegistry lockRegistry;
@Test @Test
void verifyOnlyUpdatableTargetsArePartOfAutoAssignmentByScheduler() throws Exception { void verifyOnlyUpdatableTargetsArePartOfAutoAssignmentByScheduler() throws Exception {
// auto assign scheduler apply stored access control context and the context is correctly applied // auto assign scheduler apply stored access control context and the context is correctly applied
verifyOnlyUpdatableTargetsArePartOfAutoAssignment( verifyOnlyUpdatableTargetsArePartOfAutoAssignment(
() -> new AutoAssignScheduler(systemManagement, autoAssignExecutor, 1, lockRegistry, Optional.empty()).autoAssignScheduler()); () -> new AutoAssignScheduler(systemManagement, autoAssignHandler, 1, Optional.empty()).autoAssignScheduler());
} }
@Test @Test
void verifyOnlyUpdatableTargetsArePartOfAutoAssignment() throws Exception { void verifyOnlyUpdatableTargetsArePartOfAutoAssignment() throws Exception {
verifyOnlyUpdatableTargetsArePartOfAutoAssignment(autoAssignExecutor::checkAllTargets); verifyOnlyUpdatableTargetsArePartOfAutoAssignment(autoAssignHandler::handleAll);
} }
@Test @Test
void verifyOnlyUpdatableTargetsWillGetAssignmentOnSingleCheck() throws Exception { void verifyOnlyUpdatableTargetsWillGetAssignmentOnSingleCheck() throws Exception {
verifyOnlyUpdatableTargetsArePartOfAutoAssignment(() -> { verifyOnlyUpdatableTargetsArePartOfAutoAssignment(() -> {
autoAssignExecutor.checkSingleTarget(target1Type1.getControllerId()); autoAssignHandler.handleSingleTarget(target1Type1.getControllerId());
autoAssignExecutor.checkSingleTarget(target2Type2.getControllerId()); autoAssignHandler.handleSingleTarget(target2Type2.getControllerId());
autoAssignExecutor.checkSingleTarget(target3Type2.getControllerId()); autoAssignHandler.handleSingleTarget(target3Type2.getControllerId());
}); });
} }

View File

@@ -21,7 +21,7 @@ import java.util.function.Supplier;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import org.eclipse.hawkbit.auth.SpPermission; import org.eclipse.hawkbit.auth.SpPermission;
import org.eclipse.hawkbit.context.AccessContext; import org.eclipse.hawkbit.context.AccessContext;
import org.eclipse.hawkbit.repository.AutoAssignExecutor; import org.eclipse.hawkbit.repository.AutoAssignHandler;
import org.eclipse.hawkbit.repository.jpa.AbstractJpaIntegrationTest; import org.eclipse.hawkbit.repository.jpa.AbstractJpaIntegrationTest;
import org.eclipse.hawkbit.repository.model.Rollout; import org.eclipse.hawkbit.repository.model.Rollout;
import org.eclipse.hawkbit.repository.model.TargetFilterQuery; import org.eclipse.hawkbit.repository.model.TargetFilterQuery;
@@ -45,7 +45,7 @@ class SecurityContextCtxTest extends AbstractJpaIntegrationTest {
private static final Set<String> AUTHORITIES = SpPermission.getAllAuthorities(); private static final Set<String> AUTHORITIES = SpPermission.getAllAuthorities();
@Autowired @Autowired
AutoAssignExecutor autoAssignExecutor; AutoAssignHandler autoAssignHandler;
/** /**
* Verifies acm context is persisted when creating Rollout * Verifies acm context is persisted when creating Rollout
@@ -97,7 +97,7 @@ class SecurityContextCtxTest extends AbstractJpaIntegrationTest {
try (final MockedStatic<AccessContext> mocked = mockStatic(AccessContext.class, Mockito.CALLS_REAL_METHODS)) { try (final MockedStatic<AccessContext> mocked = mockStatic(AccessContext.class, Mockito.CALLS_REAL_METHODS)) {
withSecurityContext(userContext, testdataFactory::createTargetFilterWithTargetsAndActiveAutoAssignment); withSecurityContext(userContext, testdataFactory::createTargetFilterWithTargetsAndActiveAutoAssignment);
withSecurityContext(userContext, () -> { withSecurityContext(userContext, () -> {
autoAssignExecutor.checkAllTargets(); autoAssignHandler.handleAll();
return null; return null;
}); });
@@ -118,7 +118,7 @@ class SecurityContextCtxTest extends AbstractJpaIntegrationTest {
withSecurityContext(userContext, testdataFactory::createTargetFilterWithTargetsAndActiveAutoAssignment); withSecurityContext(userContext, testdataFactory::createTargetFilterWithTargetsAndActiveAutoAssignment);
withSecurityContext(userContext, () -> { withSecurityContext(userContext, () -> {
autoAssignExecutor.checkSingleTarget(targetManagement.findAll(Pageable.ofSize(1)).getContent().get(0).getControllerId()); autoAssignHandler.handleSingleTarget(targetManagement.findAll(Pageable.ofSize(1)).getContent().get(0).getControllerId());
return null; return null;
}); });

View File

@@ -44,18 +44,18 @@ import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Slice; import org.springframework.data.domain.Slice;
/** /**
* Test class for {@link JpaAutoAssignExecutor}. * Test class for {@link JpaAutoAssignHandler}.
* <p/> * <p/>
* Feature: Component Tests - Repository<br/> * Feature: Component Tests - Repository<br/>
* Story: Auto assign checker * Story: Auto assign checker
*/ */
@SuppressWarnings("java:S6813") // constructor injects are not possible for test classes @SuppressWarnings("java:S6813") // constructor injects are not possible for test classes
class AutoAssignExecutorIntTest extends AbstractJpaIntegrationTest { class AutoAssignHandlerIntTest extends AbstractJpaIntegrationTest {
private static final String SPACE_AND_DESCRIPTION = " description"; private static final String SPACE_AND_DESCRIPTION = " description";
@Autowired @Autowired
private JpaAutoAssignExecutor autoAssignChecker; private JpaAutoAssignHandler autoAssignChecker;
@Autowired @Autowired
private DeploymentManagement deploymentManagement; private DeploymentManagement deploymentManagement;
@@ -81,7 +81,7 @@ class AutoAssignExecutorIntTest extends AbstractJpaIntegrationTest {
targetFilterQueryManagement.updateAutoAssignDS( targetFilterQueryManagement.updateAutoAssignDS(
new AutoAssignDistributionSetUpdate(targetFilterQuery.getId()).ds(secondDistributionSet.getId())); new AutoAssignDistributionSetUpdate(targetFilterQuery.getId()).ds(secondDistributionSet.getId()));
// Run the check // Run the check
autoAssignChecker.checkAllTargets(); autoAssignChecker.handleAll();
// verify that manually created action is canceled and action // verify that manually created action is canceled and action
// created from AutoAssign is running // created from AutoAssign is running
@@ -144,7 +144,7 @@ class AutoAssignExecutorIntTest extends AbstractJpaIntegrationTest {
assertThat(targetManagement.countByRsqlAndNonDsAndCompatibleAndUpdatable(setA.getId(), targetFilterQuery.getQuery())).isEqualTo(15); assertThat(targetManagement.countByRsqlAndNonDsAndCompatibleAndUpdatable(setA.getId(), targetFilterQuery.getQuery())).isEqualTo(15);
// Run the check // Run the check
autoAssignChecker.checkAllTargets(); autoAssignChecker.handleAll();
verifyThatTargetsHaveDistributionSetAssignment(setA, targets.subList(5, 25), targetsCount); verifyThatTargetsHaveDistributionSetAssignment(setA, targets.subList(5, 25), targetsCount);
@@ -173,7 +173,7 @@ class AutoAssignExecutorIntTest extends AbstractJpaIntegrationTest {
final int targetsCount = targets.size(); final int targetsCount = targets.size();
// Run the check // Run the check
autoAssignChecker.checkSingleTarget(targets.get(0).getControllerId()); autoAssignChecker.handleSingleTarget(targets.get(0).getControllerId());
verifyThatTargetsHaveDistributionSetAssignment(toAssignDs, targets.subList(0, 1), targetsCount); verifyThatTargetsHaveDistributionSetAssignment(toAssignDs, targets.subList(0, 1), targetsCount);
@@ -204,7 +204,7 @@ class AutoAssignExecutorIntTest extends AbstractJpaIntegrationTest {
targetDsAIdPref.concat(SPACE_AND_DESCRIPTION)); targetDsAIdPref.concat(SPACE_AND_DESCRIPTION));
// Run the check // Run the check
autoAssignChecker.checkAllTargets(); autoAssignChecker.handleAll();
verifyThatTargetsHaveDistributionSetAssignedAndActionStatus(distributionSet, targets, expectedStatus); verifyThatTargetsHaveDistributionSetAssignedAndActionStatus(distributionSet, targets, expectedStatus);
} }
@@ -232,7 +232,7 @@ class AutoAssignExecutorIntTest extends AbstractJpaIntegrationTest {
final List<Target> targets = testdataFactory.createTargets(25); final List<Target> targets = testdataFactory.createTargets(25);
// Run the check // Run the check
autoAssignChecker.checkSingleTarget(targets.get(0).getControllerId()); autoAssignChecker.handleSingleTarget(targets.get(0).getControllerId());
verifyThatTargetsHaveDistributionSetAssignedAndActionStatus(toAssignDs, targets.subList(0, 1), expectedStatus); verifyThatTargetsHaveDistributionSetAssignedAndActionStatus(toAssignDs, targets.subList(0, 1), expectedStatus);
verifyThatTargetsNotHaveDistributionSetAssignment(targets.subList(1, 25)); verifyThatTargetsNotHaveDistributionSetAssignment(targets.subList(1, 25));
@@ -281,7 +281,7 @@ class AutoAssignExecutorIntTest extends AbstractJpaIntegrationTest {
verifyThatTargetsHaveDistributionSetAssignment(setB, targetsF.subList(0, 5), targetsCount); verifyThatTargetsHaveDistributionSetAssignment(setB, targetsF.subList(0, 5), targetsCount);
// Run the check // Run the check
autoAssignChecker.checkAllTargets(); autoAssignChecker.handleAll();
// first 5 targets of the fail group should still have setB // first 5 targets of the fail group should still have setB
verifyThatTargetsHaveDistributionSetAssignment(setB, targetsF.subList(0, 5), targetsCount); verifyThatTargetsHaveDistributionSetAssignment(setB, targetsF.subList(0, 5), targetsCount);
@@ -309,7 +309,7 @@ class AutoAssignExecutorIntTest extends AbstractJpaIntegrationTest {
final int targetsCount = targetsA.size() + targetsB.size() + targetsC.size(); final int targetsCount = targetsA.size() + targetsB.size() + targetsC.size();
autoAssignChecker.checkAllTargets(); autoAssignChecker.handleAll();
verifyThatTargetsHaveDistributionSetAssignment(distributionSet, targetsA, targetsCount); verifyThatTargetsHaveDistributionSetAssignment(distributionSet, targetsA, targetsCount);
verifyThatTargetsHaveDistributionSetAssignment(distributionSet, targetsB, targetsCount); verifyThatTargetsHaveDistributionSetAssignment(distributionSet, targetsB, targetsCount);
@@ -334,7 +334,7 @@ class AutoAssignExecutorIntTest extends AbstractJpaIntegrationTest {
.name("a").query("name==*").autoAssignDistributionSet(ds).autoAssignWeight(weight) .name("a").query("name==*").autoAssignDistributionSet(ds).autoAssignWeight(weight)
.build()); .build());
testdataFactory.createTargets(amountOfTargets); testdataFactory.createTargets(amountOfTargets);
autoAssignChecker.checkAllTargets(); autoAssignChecker.handleAll();
final List<Action> actions = deploymentManagement.findActionsAll(PAGE).getContent(); final List<Action> actions = deploymentManagement.findActionsAll(PAGE).getContent();
assertThat(actions) assertThat(actions)
@@ -353,7 +353,7 @@ class AutoAssignExecutorIntTest extends AbstractJpaIntegrationTest {
enableMultiAssignments(); enableMultiAssignments();
testdataFactory.createTargets(amountOfTargets); testdataFactory.createTargets(amountOfTargets);
autoAssignChecker.checkAllTargets(); autoAssignChecker.handleAll();
final List<Action> actions = deploymentManagement.findActionsAll(PAGE).getContent(); final List<Action> actions = deploymentManagement.findActionsAll(PAGE).getContent();
assertThat(actions) assertThat(actions)
@@ -404,7 +404,7 @@ class AutoAssignExecutorIntTest extends AbstractJpaIntegrationTest {
testFilter.getQuery()); testFilter.getQuery());
assertThat(compatibleCount).isEqualTo(compatibleTargets.size()); assertThat(compatibleCount).isEqualTo(compatibleTargets.size());
autoAssignChecker.checkAllTargets(); autoAssignChecker.handleAll();
final List<Action> actions = deploymentManagement.findActionsAll(Pageable.unpaged()).getContent(); final List<Action> actions = deploymentManagement.findActionsAll(Pageable.unpaged()).getContent();
assertThat(actions).hasSize(compatibleTargets.size()); assertThat(actions).hasSize(compatibleTargets.size());

View File

@@ -9,6 +9,7 @@
*/ */
package org.eclipse.hawkbit.repository.jpa.scheduler; package org.eclipse.hawkbit.repository.jpa.scheduler;
import static org.assertj.core.api.Assertions.assertThatNoException;
import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
@@ -17,9 +18,12 @@ import static org.mockito.Mockito.when;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.Optional;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.locks.Lock;
import org.assertj.core.api.Assertions;
import org.eclipse.hawkbit.repository.DeploymentManagement; import org.eclipse.hawkbit.repository.DeploymentManagement;
import org.eclipse.hawkbit.repository.TargetFilterQueryManagement; import org.eclipse.hawkbit.repository.TargetFilterQueryManagement;
import org.eclipse.hawkbit.repository.TargetManagement; import org.eclipse.hawkbit.repository.TargetManagement;
@@ -35,6 +39,7 @@ import org.mockito.Mock;
import org.mockito.Mockito; import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.data.domain.SliceImpl; import org.springframework.data.domain.SliceImpl;
import org.springframework.integration.support.locks.LockRegistry;
import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.PlatformTransactionManager;
/** /**
@@ -42,7 +47,7 @@ import org.springframework.transaction.PlatformTransactionManager;
* Story: Auto assign checker * Story: Auto assign checker
*/ */
@ExtendWith(MockitoExtension.class) @ExtendWith(MockitoExtension.class)
class AutoAssignExecutorTest { class AutoAssignHandlerTest {
@Mock @Mock
private TargetFilterQueryManagement<? extends TargetFilterQuery> targetFilterQueryManagement; private TargetFilterQueryManagement<? extends TargetFilterQuery> targetFilterQueryManagement;
@@ -53,19 +58,36 @@ class AutoAssignExecutorTest {
@Mock @Mock
private PlatformTransactionManager transactionManager; private PlatformTransactionManager transactionManager;
private JpaAutoAssignExecutor autoAssignChecker; @Mock
LockRegistry lockRegistry;
private JpaAutoAssignHandler autoAssignHandler;
@BeforeEach @BeforeEach
void before() { void before() {
autoAssignChecker = new JpaAutoAssignExecutor( autoAssignHandler = new JpaAutoAssignHandler(
targetFilterQueryManagement, targetManagement, deploymentManagement, transactionManager); targetFilterQueryManagement, targetManagement, deploymentManagement, transactionManager, lockRegistry, Optional.empty());
} }
/** /**
* Single device check triggers update for matching auto assignment filter. * Single device check triggers update for matching auto assignment filter.
*/ */
@Test @Test
void checkForDevice() { void failToLockInHandleAll() {
final Lock lock = mock(Lock.class);
when(lock.tryLock()).thenReturn(false);
when(lockRegistry.obtain(any())).thenReturn(lock);
final TargetFilterQuery matching = mock(TargetFilterQuery.class);
when(targetFilterQueryManagement.findWithAutoAssignDS(any())).thenReturn(new SliceImpl<>(Arrays.asList(matching)));
assertThatNoException().isThrownBy(autoAssignHandler::handleAll);
}
/**
* Single device check triggers update for matching auto assignment filter.
*/
@Test
void handleSingleTarget() {
final String target = getRandomString(); final String target = getRandomString();
final long ds = getRandomLong(); final long ds = getRandomLong();
final TargetFilterQuery matching = mockFilterQuery(ds); final TargetFilterQuery matching = mockFilterQuery(ds);
@@ -75,7 +97,7 @@ class AutoAssignExecutorTest {
when(targetManagement.isTargetMatchingQueryAndDSNotAssignedAndCompatibleAndUpdatable(target, ds, notMatching.getQuery())) when(targetManagement.isTargetMatchingQueryAndDSNotAssignedAndCompatibleAndUpdatable(target, ds, notMatching.getQuery()))
.thenReturn(false); .thenReturn(false);
autoAssignChecker.checkSingleTarget(target); autoAssignHandler.handleSingleTarget(target);
verify(deploymentManagement).assignDistributionSets(Mockito.argThat(deployReqMatcher(target, ds)), any()); verify(deploymentManagement).assignDistributionSets(Mockito.argThat(deployReqMatcher(target, ds)), any());
Mockito.verifyNoMoreInteractions(deploymentManagement); Mockito.verifyNoMoreInteractions(deploymentManagement);

View File

@@ -41,13 +41,10 @@ import org.springframework.util.CollectionUtils;
@PropertySource("classpath:hawkbit-security-defaults.properties") @PropertySource("classpath:hawkbit-security-defaults.properties")
public class SecurityManagedConfiguration { public class SecurityManagedConfiguration {
public static final String ANONYMOUS_CONTROLLER_SECURITY_ENABLED_SHOULD_ONLY_BE_USED_FOR_DEVELOPMENT_PURPOSES = """
******************
** Anonymous controller security enabled, should only be used for development purposes **
******************""";
public static final int DOS_FILTER_ORDER = -200; public static final int DOS_FILTER_ORDER = -200;
public static FilterRegistrationBean<DosFilter> dosFilter(final Collection<String> includeAntPaths, public static FilterRegistrationBean<DosFilter> dosFilter(
final Collection<String> includeAntPaths,
final HawkbitSecurityProperties.Dos.Filter filterProperties, final HawkbitSecurityProperties.Dos.Filter filterProperties,
final HawkbitSecurityProperties.Clients clientProperties) { final HawkbitSecurityProperties.Clients clientProperties) {
final FilterRegistrationBean<DosFilter> filterRegBean = new FilterRegistrationBean<>(); final FilterRegistrationBean<DosFilter> filterRegBean = new FilterRegistrationBean<>();