lingwei.kong/hawkbit
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Action history cleanup/purge initial (#2728)

Browse Source 
* Action history cleanup/purge initial

Signed-off-by: strailov <Stanislav.Trailov@bosch.io>

* apply changes after review

Signed-off-by: strailov <Stanislav.Trailov@bosch.io>

* fix hibernate build by annotating delete methods with transactional annotation

Signed-off-by: strailov <Stanislav.Trailov@bosch.io>

* changes after review and new test cases for new requirements

* accept 0 for keep last

Signed-off-by: strailov <Stanislav.Trailov@bosch.io>

* Fix ManagementSecurityTest

Signed-off-by: strailov <Stanislav.Trailov@bosch.io>

* apply object utils check

Signed-off-by: strailov <Stanislav.Trailov@bosch.io>

* fix for oldestAction deletion

Signed-off-by: strailov <Stanislav.Trailov@bosch.io>

* remove unused comment

Signed-off-by: strailov <Stanislav.Trailov@bosch.io>

* rename action ids variable

Signed-off-by: strailov <Stanislav.Trailov@bosch.io>

* Fix access control handling

Signed-off-by: Avgustin Marinov <Avgustin.Marinov@bosch.com>

---------

Signed-off-by: strailov <Stanislav.Trailov@bosch.io>
Signed-off-by: Avgustin Marinov <Avgustin.Marinov@bosch.com>
Co-authored-by: Avgustin Marinov <Avgustin.Marinov@bosch.com>
This commit is contained in:
Stanislav Trailov
2025-10-21 10:34:58 +03:00
committed by GitHub
parent 9984c89183
commit f1c3d0175e
21 changed files with 662 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
hawkbit-repository/hawkbit-repository-api/src/main/java/org/eclipse/hawkbit/repository/DeploymentManagement.java
View File

@@ -298,6 +298,45 @@ public interface DeploymentManagement extends PermissionSupport {
@PreAuthorize(SpringEvalExpressions.HAS_UPDATE_REPOSITORY)
Action forceTargetAction(long actionId);
/**
* Deletes the current action by id.
* @param actionId - action id
*/
@PreAuthorize("hasAuthority('UPDATE_" + SpPermission.TARGET + "')")
void deleteAction(long actionId);
/**
* Deletes actions matching the provided rsql filter
* @param rsql - rsql filter for actions to be deleted
*/
@PreAuthorize("hasAuthority('UPDATE_" + SpPermission.TARGET + "')")
void deleteActionsByRsql(String rsql);
/**
* Deletes actions present in provided list of ids
* @param actionIds - list of action ids to be deleted
*/
@PreAuthorize("hasAuthority('UPDATE_" + SpPermission.TARGET + "')")
void deleteActionsByIds(List<Long> actionIds);
/**
* Deletes actions in scope of the target ONLY by list of action ids.
*
* @param target - target controllerId
* @param actionsIds - list of action ids to be deleted
*/
@PreAuthorize("hasAuthority('UPDATE_" + SpPermission.TARGET + "')")
void deleteTargetActionsByIds(final String target, final List<Long> actionsIds);
/**
* Deletes target actions and leaves the LAST N actions in the action history only.
*
* @param target - target controllerId
* @param keepLast - number of actions to be left/kept (NOT deleted)
*/
@PreAuthorize("hasAuthority('UPDATE_" + SpPermission.TARGET + "')")
void deleteOldestTargetActions(final String target, final int keepLast);
/**
* Sets the status of inactive scheduled {@link Action}s for the specified {@link Target}s to {@link Status#CANCELED}
*
@@ -372,4 +411,7 @@ public interface DeploymentManagement extends PermissionSupport {
*/
@PreAuthorize(SpringEvalExpressions.HAS_UPDATE_REPOSITORY)
void cancelActionsForDistributionSet(final ActionCancellationType cancelationType, final DistributionSet set);
@PreAuthorize(SpringEvalExpressions.IS_SYSTEM_CODE)
void handleMaxAssignmentsExceeded(Long targetId, Long requested, AssignmentQuotaExceededException ex);
}

5
hawkbit-repository/hawkbit-repository-api/src/main/java/org/eclipse/hawkbit/tenancy/configuration/TenantConfigurationProperties.java
View File

@@ -133,6 +133,11 @@ public class TenantConfigurationProperties {
*/
public static final String IMPLICIT_LOCK_ENABLED = "implicit.lock.enabled";
/**
* Configuration value for percentage of oldest actions to be cleaned if @maxActionsPerTarget quota is hit
*/
public static final String ACTIONS_PURGE_PERCENTAGE_ON_QUOTA_HIT = "actions.cleanup.onQuotaHit.percent";
private static final Map<Class<? extends Serializable>, TenantConfigurationValidator> DEFAULT_TYPE_VALIDATORS = Map.of(
Boolean.class, new TenantConfigurationBooleanValidator(),
Integer.class, new TenantConfigurationIntegerValidator(),

4
hawkbit-repository/hawkbit-repository-core/src/main/resources/hawkbit-repository-defaults.properties
View File

@@ -78,6 +78,10 @@ hawkbit.server.tenant.configuration.action-cleanup-action-expiry.dataType=java.l
hawkbit.server.tenant.configuration.action-cleanup-action-status.keyName=action.cleanup.actionStatus
hawkbit.server.tenant.configuration.action-cleanup-action-status.defaultValue=CANCELED,ERROR
hawkbit.server.tenant.configuration.actions-cleanup-on-quota-hit-percent.keyName=actions.cleanup.onQuotaHit.percent
hawkbit.server.tenant.configuration.actions-cleanup-on-quota-hit-percent.defaultValue=0
hawkbit.server.tenant.configuration.actions-cleanup-on-quota-hit-percent.dataType=java.lang.Integer
hawkbit.server.tenant.configuration.multi-assignments-enabled.keyName=multi.assignments.enabled
hawkbit.server.tenant.configuration.multi-assignments-enabled.defaultValue=false
hawkbit.server.tenant.configuration.multi-assignments-enabled.dataType=java.lang.Boolean

7
hawkbit-repository/hawkbit-repository-jpa-ql/src/main/java/org/eclipse/hawkbit/repository/jpa/ql/SpecificationBuilder.java
View File

@@ -345,12 +345,7 @@ public class SpecificationBuilder<T> {
}
private String toSqlLikeValue(final String value) {
final String escaped;
if (database == Database.SQL_SERVER) {
escaped = value.replace("%", "[%]").replace("_", "[_]");
} else {
escaped = value.replace("%", ESCAPE_CHAR + "%").replace("_", ESCAPE_CHAR + "_");
}
final String escaped = value.replace("%", ESCAPE_CHAR + "%").replace("_", ESCAPE_CHAR + "_");
final String finalizedValue;
if (escaped.contains(ESCAPE_CHAR_WITH_ASTERISK)) {
finalizedValue = escaped.replace(ESCAPE_CHAR_WITH_ASTERISK, "$")

8
hawkbit-repository/hawkbit-repository-jpa-ql/src/main/java/org/eclipse/hawkbit/repository/jpa/rsql/legacy/JpaQueryRsqlVisitor.java
View File

@@ -503,13 +503,7 @@ public class JpaQueryRsqlVisitor<A extends Enum<A> & QueryField, T> extends Abst
}
private String toSQL(final String transformedValue) {
final String escaped;
if (database == Database.SQL_SERVER) {
escaped = transformedValue.replace("%", "[%]").replace("_", "[_]");
} else {
escaped = transformedValue.replace("%", ESCAPE_CHAR + "%").replace("_", ESCAPE_CHAR + "_");
}
final String escaped = transformedValue.replace("%", ESCAPE_CHAR + "%").replace("_", ESCAPE_CHAR + "_");
return replaceIfRequired(escaped);
}

7
hawkbit-repository/hawkbit-repository-jpa-ql/src/main/java/org/eclipse/hawkbit/repository/jpa/rsql/legacy/JpaQueryRsqlVisitorG2.java
View File

@@ -425,12 +425,7 @@ public class JpaQueryRsqlVisitorG2<A extends Enum<A> & QueryField, T>
}
private String toSQL(final String value) {
final String escaped;
if (database == Database.SQL_SERVER) {
escaped = value.replace("%", "[%]").replace("_", "[_]");
} else {
escaped = value.replace("%", ESCAPE_CHAR + "%").replace("_", ESCAPE_CHAR + "_");
}
final String escaped = value.replace("%", ESCAPE_CHAR + "%").replace("_", ESCAPE_CHAR + "_");
return replaceIfRequired(escaped);
}

9
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/JpaRolloutExecutor.java
View File

@@ -56,7 +56,6 @@ import org.eclipse.hawkbit.repository.model.Action.ActionType;
import org.eclipse.hawkbit.repository.model.Action.Status;
import org.eclipse.hawkbit.repository.model.ActionCancellationType;
import org.eclipse.hawkbit.repository.model.DistributionSet;
import org.eclipse.hawkbit.repository.model.DistributionSetInvalidation;
import org.eclipse.hawkbit.repository.model.Rollout;
import org.eclipse.hawkbit.repository.model.Rollout.RolloutStatus;
import org.eclipse.hawkbit.repository.model.RolloutGroup;
@@ -856,8 +855,12 @@ public class JpaRolloutExecutor implements RolloutExecutor {
* @param target the target
* @param requested number of actions to check
*/
private void assertActionsPerTargetQuota(final Target target, final int requested) {
private void assertActionsPerTargetQuota(final Target target, final long requested) {
final int quota = quotaManagement.getMaxActionsPerTarget();
QuotaHelper.assertAssignmentQuota(target.getId(), requested, quota, Action.class, Target.class, actionRepository::countByTargetId);
try {
QuotaHelper.assertAssignmentQuota(target.getId(), requested, quota, Action.class, Target.class, actionRepository::countByTargetId);
} catch (final AssignmentQuotaExceededException ex) {
deploymentManagement.handleMaxAssignmentsExceeded(target.getId(), requested, ex);
}
}
}

15
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/management/AbstractDsAssignmentStrategy.java
View File

@@ -19,12 +19,14 @@ import java.util.function.BooleanSupplier;
import jakarta.persistence.criteria.JoinType;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.function.TriConsumer;
import org.eclipse.hawkbit.repository.QuotaManagement;
import org.eclipse.hawkbit.repository.RepositoryConstants;
import org.eclipse.hawkbit.repository.RepositoryProperties;
import org.eclipse.hawkbit.repository.event.EventPublisherHolder;
import org.eclipse.hawkbit.repository.event.remote.CancelTargetAssignmentEvent;
import org.eclipse.hawkbit.repository.event.remote.entity.TargetUpdatedEvent;
import org.eclipse.hawkbit.repository.exception.AssignmentQuotaExceededException;
import org.eclipse.hawkbit.repository.jpa.configuration.Constants;
import org.eclipse.hawkbit.repository.jpa.executor.AfterTransactionCommitExecutor;
import org.eclipse.hawkbit.repository.jpa.model.AbstractJpaBaseEntity_;
@@ -65,6 +67,7 @@ public abstract class AbstractDsAssignmentStrategy {
private final BooleanSupplier multiAssignmentsConfig;
private final BooleanSupplier confirmationFlowConfig;
private final RepositoryProperties repositoryProperties;
private final TriConsumer<Long, Long, AssignmentQuotaExceededException> maxAssignmentExceededHandler;
@SuppressWarnings("java:S107")
AbstractDsAssignmentStrategy(
@@ -72,7 +75,8 @@ public abstract class AbstractDsAssignmentStrategy {
final AfterTransactionCommitExecutor afterCommit,
final ActionRepository actionRepository, final ActionStatusRepository actionStatusRepository,
final QuotaManagement quotaManagement, final BooleanSupplier multiAssignmentsConfig,
final BooleanSupplier confirmationFlowConfig, final RepositoryProperties repositoryProperties) {
final BooleanSupplier confirmationFlowConfig, final RepositoryProperties repositoryProperties,
final TriConsumer<Long, Long, AssignmentQuotaExceededException> maxAssignmentExceededHandler) {
this.targetRepository = targetRepository;
this.afterCommit = afterCommit;
this.actionRepository = actionRepository;
@@ -81,6 +85,7 @@ public abstract class AbstractDsAssignmentStrategy {
this.multiAssignmentsConfig = multiAssignmentsConfig;
this.confirmationFlowConfig = confirmationFlowConfig;
this.repositoryProperties = repositoryProperties;
this.maxAssignmentExceededHandler = maxAssignmentExceededHandler;
}
public JpaAction createTargetAction(
@@ -292,8 +297,12 @@ public abstract class AbstractDsAssignmentStrategy {
.publishEvent(new CancelTargetAssignmentEvent(tenant, actions)));
}
private void assertActionsPerTargetQuota(final Target target, final int requested) {
private void assertActionsPerTargetQuota(final Target target, final long requested) {
final int quota = quotaManagement.getMaxActionsPerTarget();
QuotaHelper.assertAssignmentQuota(target.getId(), requested, quota, Action.class, Target.class, actionRepository::countByTargetId);
try {
QuotaHelper.assertAssignmentQuota(target.getId(), requested, quota, Action.class, Target.class, actionRepository::countByTargetId);
} catch (AssignmentQuotaExceededException ex) {
maxAssignmentExceededHandler.accept(target.getId(), requested, ex);
}
}
}

154
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/management/JpaDeploymentManagement.java
View File

@@ -37,6 +37,8 @@ import jakarta.persistence.criteria.Root;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.function.TriConsumer;
import org.eclipse.hawkbit.repository.ActionFields;
import org.eclipse.hawkbit.repository.DeploymentManagement;
import org.eclipse.hawkbit.repository.QuotaManagement;
@@ -44,6 +46,7 @@ import org.eclipse.hawkbit.repository.RepositoryConstants;
import org.eclipse.hawkbit.repository.RepositoryProperties;
import org.eclipse.hawkbit.repository.TenantConfigurationManagement;
import org.eclipse.hawkbit.repository.event.remote.TargetAssignDistributionSetEvent;
import org.eclipse.hawkbit.repository.exception.AssignmentQuotaExceededException;
import org.eclipse.hawkbit.repository.exception.CancelActionNotAllowedException;
import org.eclipse.hawkbit.repository.exception.EntityNotFoundException;
import org.eclipse.hawkbit.repository.exception.ForceQuitActionNotAllowedException;
@@ -62,10 +65,10 @@ import org.eclipse.hawkbit.repository.jpa.model.JpaActionStatus;
import org.eclipse.hawkbit.repository.jpa.model.JpaDistributionSet;
import org.eclipse.hawkbit.repository.jpa.model.JpaTarget;
import org.eclipse.hawkbit.repository.jpa.model.JpaTarget_;
import org.eclipse.hawkbit.repository.jpa.ql.QLSupport;
import org.eclipse.hawkbit.repository.jpa.repository.ActionRepository;
import org.eclipse.hawkbit.repository.jpa.repository.ActionStatusRepository;
import org.eclipse.hawkbit.repository.jpa.repository.TargetRepository;
import org.eclipse.hawkbit.repository.jpa.ql.QLSupport;
import org.eclipse.hawkbit.repository.jpa.specifications.ActionSpecifications;
import org.eclipse.hawkbit.repository.jpa.specifications.TargetSpecifications;
import org.eclipse.hawkbit.repository.jpa.utils.DeploymentHelper;
@@ -87,6 +90,7 @@ import org.eclipse.hawkbit.repository.model.TargetUpdateStatus;
import org.eclipse.hawkbit.repository.model.TargetWithActionType;
import org.eclipse.hawkbit.security.SystemSecurityContext;
import org.eclipse.hawkbit.tenancy.TenantAware;
import org.eclipse.hawkbit.tenancy.configuration.TenantConfigurationProperties;
import org.eclipse.hawkbit.utils.TenantConfigHelper;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
import org.springframework.boot.autoconfigure.orm.jpa.JpaProperties;
@@ -126,8 +130,6 @@ public class JpaDeploymentManagement extends JpaActionManagement implements Depl
static {
QUERY_DELETE_ACTIONS_BY_STATE_AND_LAST_MODIFIED = new EnumMap<>(Database.class);
QUERY_DELETE_ACTIONS_BY_STATE_AND_LAST_MODIFIED.put(Database.SQL_SERVER,
"DELETE TOP (" + ACTION_PAGE_LIMIT + ") FROM sp_action " + "WHERE tenant=" + Jpa.nativeQueryParamPrefix() + "tenant" + " AND status IN (%s)" + " AND last_modified_at<" + Jpa.nativeQueryParamPrefix() + "last_modified_at ");
QUERY_DELETE_ACTIONS_BY_STATE_AND_LAST_MODIFIED.put(Database.POSTGRESQL,
"DELETE FROM sp_action " + "WHERE id IN (SELECT id FROM sp_action " + "WHERE tenant=" + Jpa.nativeQueryParamPrefix() + "tenant" + " AND status IN (%s)" + " AND last_modified_at<" + Jpa.nativeQueryParamPrefix() + "last_modified_at LIMIT " + ACTION_PAGE_LIMIT + ")");
}
@@ -161,9 +163,11 @@ public class JpaDeploymentManagement extends JpaActionManagement implements Depl
this.auditorProvider = auditorProvider;
this.txManager = txManager;
onlineDsAssignmentStrategy = new OnlineDsAssignmentStrategy(targetRepository, afterCommit, actionRepository, actionStatusRepository,
quotaManagement, this::isMultiAssignmentsEnabled, this::isConfirmationFlowEnabled, repositoryProperties);
quotaManagement, this::isMultiAssignmentsEnabled, this::isConfirmationFlowEnabled, repositoryProperties,
maxAssignmentsExceededHandler);
offlineDsAssignmentStrategy = new OfflineDsAssignmentStrategy(targetRepository, afterCommit, actionRepository, actionStatusRepository,
quotaManagement, this::isMultiAssignmentsEnabled, this::isConfirmationFlowEnabled, repositoryProperties);
quotaManagement, this::isMultiAssignmentsEnabled, this::isConfirmationFlowEnabled, repositoryProperties,
maxAssignmentsExceededHandler);
this.tenantConfigurationManagement = tenantConfigurationManagement;
this.systemSecurityContext = systemSecurityContext;
this.tenantAware = tenantAware;
@@ -401,6 +405,71 @@ public class JpaDeploymentManagement extends JpaActionManagement implements Depl
return action;
}
@Override
@Transactional
public void deleteAction(final long actionId) {
log.info("Deleting action {}", actionId);
actionRepository.getAccessController().ifPresent(accessController -> {
// check update access
if (ObjectUtils.isEmpty(actionRepository.findAll(accessController.appendAccessRules(
AccessController.Operation.UPDATE, ((root, q, cb) -> cb.equal(root.get(AbstractJpaBaseEntity_.id), actionId)))))) {
// could be also InsufficientPermissionException but for security reasons we do not reveal that the entity exists
throw new EntityNotFoundException(Action.class, actionId);
}
});
actionRepository.deleteById(actionId);
}
@Override
@Transactional
public void deleteActionsByRsql(final String rsql) {
log.info("Deleting actions matching rsql {}", rsql);
final Specification<JpaAction> specification = QLSupport.getInstance().buildSpec(rsql, ActionFields.class);
actionRepository.delete(
actionRepository.getAccessController()
.map(accessController -> accessController.appendAccessRules(AccessController.Operation.UPDATE, specification))
.orElse(specification));
}
@Override
@Transactional
public void deleteActionsByIds(final List<Long> actionIds) {
log.info("Deleting actions with ids {}", actionIds);
actionRepository.getAccessController().ifPresent(accessController ->
actionRepository.findAll(
accessController.appendAccessRules(AccessController.Operation.UPDATE, ActionSpecifications.byIdIn(actionIds))));
actionRepository.deleteByIdIn(actionIds);
}
@Override
@Transactional
public void deleteTargetActionsByIds(final String target, final List<Long> actionsIds) {
log.info("Delete actions for target {} with action ids {}", target, actionsIds);
targetRepository.getAccessController()
.ifPresent(accessController ->
accessController.assertOperationAllowed(AccessController.Operation.UPDATE, targetRepository.getByControllerId(target)));
actionRepository.delete(ActionSpecifications.byControllerIdAndIdIn(target, actionsIds));
}
@Override
@Transactional
public void deleteOldestTargetActions(final String target, final int keepLast) {
final JpaTarget jpaTarget = targetRepository.findByControllerId(target)
.orElseThrow(EntityNotFoundException::new);
targetRepository.getAccessController().ifPresent(accessController ->
accessController.assertOperationAllowed(AccessController.Operation.UPDATE, jpaTarget));
final long targetActions = actionRepository.countByTargetId(jpaTarget.getId());
long oldestToDelete;
if (targetActions > keepLast) {
oldestToDelete = targetActions - keepLast;
} else {
return;
}
deleteOldestTargetActions(jpaTarget.getId(), (int) oldestToDelete);
}
@Override
@Transactional(isolation = Isolation.READ_COMMITTED)
@Retryable(retryFor = {
@@ -438,7 +507,7 @@ public class JpaDeploymentManagement extends JpaActionManagement implements Depl
startScheduledActions(groupScheduledActions.getContent());
return groupScheduledActions.getTotalElements();
}
}) > 0);
}) > 0) ;
}
@Override
@@ -529,6 +598,58 @@ public class JpaDeploymentManagement extends JpaActionManagement implements Depl
}
}
@Override
public void handleMaxAssignmentsExceeded(final Long targetId, final Long requested, final AssignmentQuotaExceededException ex) {
maxAssignmentsExceededHandler.accept(targetId, requested, ex);
}
private final TriConsumer<Long, Long, AssignmentQuotaExceededException> maxAssignmentsExceededHandler = (targetId, requested, quotaExceededException) -> {
int actionsPurgePercentage = getActionsPurgePercentage();
int quota = quotaManagement.getMaxActionsPerTarget();
if (actionsPurgePercentage > 0 && actionsPurgePercentage < 100) {
int numberOfActions = (int) ((actionsPurgePercentage / 100.0) * quota);
if (requested > numberOfActions) {
log.warn("Requested number of actions {} bigger than configured for deletion {}", requested, numberOfActions);
throw quotaExceededException;
}
int totalTargetActions = Math.toIntExact(actionRepository.countByTargetId(targetId));
if (totalTargetActions < quota) {
numberOfActions = totalTargetActions - (quota - numberOfActions);
}
log.info("Actions purge percentage {}, will delete {} oldest actions for target {}",
actionsPurgePercentage, numberOfActions, targetId);
deleteOldestTargetActions(targetId, numberOfActions);
} else {
throw quotaExceededException;
}
};
/**
* Deletes the first n target actions of a target
*
* @param targetId - target id
* @param oldestToDelete - number of oldest actions to be deleted
*/
public void deleteOldestTargetActions(long targetId, int oldestToDelete) {
// Workaround for the case where JPQL or Criteria API do not support LIMIT
log.info("Deleting last {} actions of target {}", oldestToDelete, targetId);
final String SQL = "DELETE FROM sp_action WHERE id IN(" +
"SELECT id FROM (" +
"SELECT id FROM sp_action" +
" WHERE target=" + Jpa.nativeQueryParamPrefix() + "target" +
" ORDER BY id ASC" +
" LIMIT " + oldestToDelete
+ ") AS sub"
+ ")";
Query query = entityManager.createNativeQuery(SQL);
query.setParameter("target", targetId);
query.executeUpdate();
}
private int getActionsPurgePercentage() {
return getConfigValue(TenantConfigurationProperties.TenantConfigurationKey.ACTIONS_PURGE_PERCENTAGE_ON_QUOTA_HIT, Integer.class);
}
protected boolean isActionsAutocloseEnabled() {
return getConfigValue(REPOSITORY_ACTIONS_AUTOCLOSE_ENABLED, Boolean.class);
}
@@ -777,14 +898,25 @@ public class JpaDeploymentManagement extends JpaActionManagement implements Depl
}
private void enforceMaxActionsPerTarget(final Collection<DeploymentRequest> deploymentRequests) {
final int quota = quotaManagement.getMaxActionsPerTarget();
final Map<String, Long> countOfTargetInRequest = deploymentRequests.stream().map(DeploymentRequest::getControllerId)
.collect(Collectors.groupingBy(Function.identity(), Collectors.counting()));
countOfTargetInRequest.forEach(
(controllerId, count) -> QuotaHelper.assertAssignmentQuota(controllerId, count, quota, Action.class, Target.class,
actionRepository::countByTargetControllerId));
countOfTargetInRequest.forEach(this::checkMaxAssignmentQuota);
}
private void checkMaxAssignmentQuota(final String controllerId, final long requested) {
final int quota = quotaManagement.getMaxActionsPerTarget();
try {
systemSecurityContext.runAsSystem(() -> QuotaHelper.assertAssignmentQuota(
controllerId, requested, quota, Action.class, Target.class, actionRepository::countByTargetControllerId));
} catch (final AssignmentQuotaExceededException ex) {
targetRepository.findByControllerId(controllerId).ifPresentOrElse(
// assume requested are always smaller than int size
target -> maxAssignmentsExceededHandler.accept(target.getId(), requested, ex),
() -> {
throw new EntityNotFoundException(Target.class, controllerId);
});
}
}
private void closeOrCancelActiveActions(final AbstractDsAssignmentStrategy assignmentStrategy, final List<List<Long>> targetIdsChunks) {

2
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/management/JpaRolloutManagement.java
View File

@@ -179,7 +179,7 @@ public class JpaRolloutManagement implements RolloutManagement {
this.repositoryProperties = repositoryProperties;
this.onlineDsAssignmentStrategy = new OnlineDsAssignmentStrategy(targetRepository, afterCommit, actionRepository, actionStatusRepository,
quotaManagement, this::isMultiAssignmentsEnabled, this::isConfirmationFlowEnabled, repositoryProperties);
quotaManagement, this::isMultiAssignmentsEnabled, this::isConfirmationFlowEnabled, repositoryProperties, null);
}
public static String createRolloutLockKey(final String tenant) {

9
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/management/OfflineDsAssignmentStrategy.java
View File

@@ -12,13 +12,17 @@ package org.eclipse.hawkbit.repository.jpa.management;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.function.BiConsumer;
import java.util.function.BiFunction;
import java.util.function.BooleanSupplier;
import java.util.function.Function;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.lang3.function.TriConsumer;
import org.eclipse.hawkbit.repository.QuotaManagement;
import org.eclipse.hawkbit.repository.RepositoryConstants;
import org.eclipse.hawkbit.repository.RepositoryProperties;
import org.eclipse.hawkbit.repository.exception.AssignmentQuotaExceededException;
import org.eclipse.hawkbit.repository.exception.InsufficientPermissionException;
import org.eclipse.hawkbit.repository.jpa.JpaManagementHelper;
import org.eclipse.hawkbit.repository.jpa.acm.AccessController;
@@ -48,9 +52,10 @@ class OfflineDsAssignmentStrategy extends AbstractDsAssignmentStrategy {
final AfterTransactionCommitExecutor afterCommit,
final ActionRepository actionRepository, final ActionStatusRepository actionStatusRepository,
final QuotaManagement quotaManagement, final BooleanSupplier multiAssignmentsConfig,
final BooleanSupplier confirmationFlowConfig, final RepositoryProperties repositoryProperties) {
final BooleanSupplier confirmationFlowConfig, final RepositoryProperties repositoryProperties,
final TriConsumer<Long, Long, AssignmentQuotaExceededException> maxAssignmentExceededHandler) {
super(targetRepository, afterCommit, actionRepository, actionStatusRepository,
quotaManagement, multiAssignmentsConfig, confirmationFlowConfig, repositoryProperties);
quotaManagement, multiAssignmentsConfig, confirmationFlowConfig, repositoryProperties, maxAssignmentExceededHandler);
}
@Override

9
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/management/OnlineDsAssignmentStrategy.java
View File

@@ -13,18 +13,22 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.function.BiConsumer;
import java.util.function.BiFunction;
import java.util.function.BooleanSupplier;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.lang3.function.TriConsumer;
import org.eclipse.hawkbit.repository.QuotaManagement;
import org.eclipse.hawkbit.repository.RepositoryProperties;
import org.eclipse.hawkbit.repository.event.EventPublisherHolder;
import org.eclipse.hawkbit.repository.event.remote.MultiActionAssignEvent;
import org.eclipse.hawkbit.repository.event.remote.MultiActionCancelEvent;
import org.eclipse.hawkbit.repository.event.remote.TargetAssignDistributionSetEvent;
import org.eclipse.hawkbit.repository.exception.AssignmentQuotaExceededException;
import org.eclipse.hawkbit.repository.exception.InsufficientPermissionException;
import org.eclipse.hawkbit.repository.jpa.acm.AccessController;
import org.eclipse.hawkbit.repository.jpa.configuration.Constants;
@@ -54,9 +58,10 @@ class OnlineDsAssignmentStrategy extends AbstractDsAssignmentStrategy {
final AfterTransactionCommitExecutor afterCommit,
final ActionRepository actionRepository, final ActionStatusRepository actionStatusRepository,
final QuotaManagement quotaManagement, final BooleanSupplier multiAssignmentsConfig,
final BooleanSupplier confirmationFlowConfig, final RepositoryProperties repositoryProperties) {
final BooleanSupplier confirmationFlowConfig, final RepositoryProperties repositoryProperties,
final TriConsumer<Long, Long, AssignmentQuotaExceededException> maxAssignmentExceededHandler) {
super(targetRepository, afterCommit, actionRepository, actionStatusRepository,
quotaManagement, multiAssignmentsConfig, confirmationFlowConfig, repositoryProperties);
quotaManagement, multiAssignmentsConfig, confirmationFlowConfig, repositoryProperties, maxAssignmentExceededHandler);
}
public void sendDeploymentEvents(final long distributionSetId, final List<Action> actions) {

17
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/specifications/ActionSpecifications.java
View File

@@ -24,6 +24,7 @@ import org.eclipse.hawkbit.repository.jpa.model.JpaDistributionSet;
import org.eclipse.hawkbit.repository.jpa.model.JpaDistributionSet_;
import org.eclipse.hawkbit.repository.jpa.model.JpaSoftwareModule;
import org.eclipse.hawkbit.repository.jpa.model.JpaSoftwareModule_;
import org.eclipse.hawkbit.repository.jpa.model.JpaTarget;
import org.eclipse.hawkbit.repository.jpa.model.JpaTarget_;
import org.eclipse.hawkbit.repository.model.Action;
import org.springframework.data.jpa.domain.Specification;
@@ -42,6 +43,11 @@ public final class ActionSpecifications {
cb.equal(root.get(JpaAction_.active), true));
}
public static Specification<JpaAction> byIdIn(final List<Long> actionIds) {
return ((root, query, cb) ->
root.get(AbstractJpaBaseEntity_.id).in(actionIds));
}
public static Specification<JpaAction> byTargetControllerId(final String controllerId) {
return (root, query, cb) -> cb.equal(root.get(JpaAction_.target).get(JpaTarget_.controllerId), controllerId);
}
@@ -101,6 +107,17 @@ public final class ActionSpecifications {
);
}
public static Specification<JpaAction> byControllerIdAndIdIn(final String controllerId, final List<Long> actionIds) {
return ((root, query, cb) -> {
final Join<JpaAction, JpaTarget> targetJoin = root.join(JpaAction_.target);
return cb.and(
cb.equal(targetJoin.get(JpaTarget_.controllerId), controllerId),
root.get(AbstractJpaBaseEntity_.id).in(actionIds)
);
});
}
public static Specification<JpaAction> byDistributionSetIdAndActiveAndStatusIsNot(final Long distributionSetId, final Action.Status status) {
return (root, query, cb) -> cb.and(
cb.equal(root.get(JpaAction_.distributionSet).get(AbstractJpaBaseEntity_.id), distributionSetId),

102
hawkbit-repository/hawkbit-repository-jpa/src/test/java/org/eclipse/hawkbit/repository/jpa/management/DeploymentManagementTest.java
View File

@@ -14,6 +14,7 @@ import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
import static org.eclipse.hawkbit.repository.model.Action.Status.RUNNING;
import static org.eclipse.hawkbit.repository.model.Action.Status.WAIT_FOR_CONFIRMATION;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import java.util.AbstractMap.SimpleEntry;
import java.util.ArrayList;
@@ -34,6 +35,7 @@ import org.assertj.core.api.Assertions;
import org.eclipse.hawkbit.repository.ActionStatusFields;
import org.eclipse.hawkbit.repository.DistributionSetManagement;
import org.eclipse.hawkbit.repository.DistributionSetTagManagement;
import org.eclipse.hawkbit.repository.Identifiable;
import org.eclipse.hawkbit.repository.event.remote.CancelTargetAssignmentEvent;
import org.eclipse.hawkbit.repository.event.remote.MultiActionAssignEvent;
import org.eclipse.hawkbit.repository.event.remote.MultiActionCancelEvent;
@@ -75,6 +77,7 @@ import org.eclipse.hawkbit.repository.model.DeploymentRequest;
import org.eclipse.hawkbit.repository.model.DistributionSet;
import org.eclipse.hawkbit.repository.model.DistributionSetAssignmentResult;
import org.eclipse.hawkbit.repository.model.DistributionSetType;
import org.eclipse.hawkbit.repository.model.Rollout;
import org.eclipse.hawkbit.repository.model.SoftwareModule;
import org.eclipse.hawkbit.repository.model.Target;
import org.eclipse.hawkbit.repository.model.TargetType;
@@ -1673,6 +1676,105 @@ class DeploymentManagementTest extends AbstractJpaIntegrationTest {
.isThrownBy(() -> deploymentManagement.assignDistributionSets(deploymentRequests));
}
@Test
void testManualAssignmentsActionsPurge() {
Target target = testdataFactory.createTarget();
for (int i = 0; i < 20; i++) {
DistributionSet distributionSet = testdataFactory.createDistributionSet("ds_" + i);
assignDistributionSet(distributionSet.getId(), target.getControllerId());
}
long actions = deploymentManagement.countActionsByTarget(target.getControllerId());
// quota in tests is set to 20 ...
assertEquals(20, actions);
// extract the first 5 action ids
List<Action> firstSample = deploymentManagement.findActionsByTarget(target.getControllerId(), PAGE).getContent();
List<Long> shouldBePurgedActionsList = firstSample.stream().map(Identifiable::getId).limit(5).toList();
DistributionSet exceededQuotaDsAssign = testdataFactory.createDistributionSet("exceededQuotaAssignment");
// should throw quota exception if not explicitly configured to purge actions
assertThrows(AssignmentQuotaExceededException.class,
() -> assignDistributionSet(exceededQuotaDsAssign.getId(), target.getControllerId()));
// set purge config to 25 %
tenantConfigurationManagement.addOrUpdateConfiguration("actions.cleanup.onQuotaHit.percent", 25);
// assign again
assignDistributionSet(exceededQuotaDsAssign.getId(), target.getControllerId());
// 16 actions should be present
actions = deploymentManagement.countActionsByTarget(target.getControllerId());
assertEquals(16, actions);
List<Action> actionsList = deploymentManagement.findActionsByTarget(target.getControllerId(), PAGE).getContent();
// first 5 should have been purged so the first actionId should be the last purged action id + 1
assertEquals(shouldBePurgedActionsList.get(shouldBePurgedActionsList.size() - 1) + 1, actionsList.get(0).getId());
assertEquals(firstSample.get(firstSample.size() - 1).getId() + 1, actionsList.get(15).getId());
}
@Test
void testRolloutAssignmentsActionsPurge() {
final Target target = testdataFactory.createTarget();
for (int i = 0; i < 20; i++) {
DistributionSet distributionSet = testdataFactory.createDistributionSet();
Rollout rollout = testdataFactory.createRolloutByVariables("rollout-" + i, "Description", 1, "controllerId==" + target
.getControllerId(),
distributionSet, "50", "50");
rolloutManagement.start(rollout.getId());
rolloutHandler.handleAll();
}
assertEquals(20, deploymentManagement.countActionsByTarget(target.getControllerId()));
List<Action> firstSample = deploymentManagement.findActionsByTarget(target.getControllerId(), PAGE).getContent();
List<Long> shouldBePurgedActionsList = firstSample.stream().map(Identifiable::getId).limit(5).toList();
DistributionSet distributionSet = testdataFactory.createDistributionSet();
Rollout rollout = testdataFactory.createRolloutByVariables("rollout-quota", "Description", 1, "controllerId==" + target
.getControllerId(),
distributionSet, "50", "50");
rolloutManagement.start(rollout.getId());
// don't assert quota exception here because rollout executor does not throw such in order to not interrupt other executions
rolloutHandler.handleAll();
//check that the old number of actions remain instead
assertEquals(20, deploymentManagement.countActionsByTarget(target.getControllerId()));
// set purge config to 25 %
tenantConfigurationManagement.addOrUpdateConfiguration("actions.cleanup.onQuotaHit.percent", 25);
rolloutHandler.handleAll();
assertEquals(16, deploymentManagement.countActionsByTarget(target.getControllerId()));
List<Action> actionsList = deploymentManagement.findActionsByTarget(target.getControllerId(), PAGE).getContent();
// first 5 should have been purged so the first actionId should be the last purged action id + 1
assertEquals(shouldBePurgedActionsList.get(shouldBePurgedActionsList.size() - 1) + 1, actionsList.get(0).getId());
assertEquals(firstSample.get(firstSample.size() - 1).getId() + 1, actionsList.get(15).getId());
}
@Test
void testThatOnlyNeededNumberOfActionsIsPurged() {
final Target target = testdataFactory.createTarget();
for (int i = 0; i < 18; i++) {
DistributionSet distributionSet = testdataFactory.createDistributionSet();
Rollout rollout = testdataFactory.createRolloutByVariables("rollout-" + i, "Description", 1, "controllerId==" + target
.getControllerId(),
distributionSet, "50", "50");
rolloutManagement.start(rollout.getId());
rolloutHandler.handleAll();
}
tenantConfigurationManagement.addOrUpdateConfiguration("actions.cleanup.onQuotaHit.percent", 25);
deploymentManagement.handleMaxAssignmentsExceeded(target.getId(), 5L, new AssignmentQuotaExceededException());
// only 3 actions should be deleted in such case :
assertEquals(15, deploymentManagement.countActionsByTarget(target.getControllerId()));
// should throw the quota exception if requested is bigger than the configured limit of actions purge
assertThrows(AssignmentQuotaExceededException.class, () ->
deploymentManagement.handleMaxAssignmentsExceeded(target.getId(), 10L, new AssignmentQuotaExceededException()));
}
private List<DeploymentRequest> createAssignmentRequests(
final Collection<DistributionSet> distributionSets, final Collection<Target> targets, final int weight) {
return createAssignmentRequests(distributionSets, targets, weight, false);

4
hawkbit-repository/hawkbit-repository-jpa/src/test/java/org/eclipse/hawkbit/repository/jpa/management/ManagementSecurityTest.java
View File

@@ -139,6 +139,10 @@ class ManagementSecurityTest extends AbstractJpaIntegrationTest {
method.getReturnType() != String.class)
// jacoco adds some methods with bytecode instrumentation
.filter(method -> !"$jacocoInit".equals(method.getName()))
// skip maxAssignmentsExceededHandler in DeploymentManagement since it throws quota exception
// because of actions.cleanup.onQuotaHit.percent not configured
// other option would be to configure it for all tests
.filter(method -> !"handleMaxAssignmentsExceeded".equals(method.getName()))
.map(method -> Arguments.of(clazz, method)))
// consumes the stream because scan result couldn't be used after being closed
.toList()