Feature action cleanup (#704)

* Initial commit
* Tenant configuration enhancements
* Update REST documentation
* Enhance System Configuration view in UI
* Add unit tests
* Fix delete query for H2
* Improve test coverage
* Fix Sonar findings
* Fix Hawkbit bot findings
* Fix PR review findings
* Fix peer review findings

Signed-off-by: Stefan Behl <stefan.behl@bosch-si.com>
This commit is contained in:
Stefan Behl
2018-07-19 15:23:14 +02:00
committed by GitHub
parent 5e9b0fb884
commit 19caff6e46
28 changed files with 1165 additions and 83 deletions

View File

@@ -10,6 +10,7 @@ package org.eclipse.hawkbit.repository;
import java.util.Collection;
import java.util.Optional;
import java.util.Set;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.NotNull;
@@ -457,4 +458,19 @@ public interface DeploymentManagement {
* if target with given ID does not exist
*/
Optional<DistributionSet> getInstalledDistributionSet(@NotEmpty String controllerId);
/**
* Deletes actions which match one of the given action status and which have
* not been modified since the given (absolute) time-stamp.
*
* @param status
* Set of action status.
* @param lastModified
* A time-stamp in milliseconds.
*
* @return The number of action entries that were deleted.
*/
@PreAuthorize(SpringEvalExpressions.IS_SYSTEM_CODE)
int deleteActionsByStatusAndLastModifiedBefore(@NotNull Set<Action.Status> status, long lastModified);
}

View File

@@ -61,6 +61,7 @@ public class TenantConfigurationProperties {
*
*/
public static class TenantConfigurationKey {
/**
* Header based authentication enabled.
*/
@@ -70,6 +71,7 @@ public class TenantConfigurationProperties {
* Header based authentication authority name.
*/
public static final String AUTHENTICATION_MODE_HEADER_AUTHORITY_NAME = "authentication.header.authority";
/**
* Target token based authentication enabled.
*/
@@ -125,6 +127,21 @@ public class TenantConfigurationProperties {
*/
public static final String REPOSITORY_ACTIONS_AUTOCLOSE_ENABLED = "repository.actions.autoclose.enabled";
/**
* Switch to enable/disable automatic action cleanup.
*/
public static final String ACTION_CLEANUP_ENABLED = "action.cleanup.enabled";
/**
* Specifies the action expiry in milli-seconds.
*/
public static final String ACTION_CLEANUP_ACTION_EXPIRY = "action.cleanup.actionExpiry";
/**
* Specifies the action status.
*/
public static final String ACTION_CLEANUP_ACTION_STATUS = "action.cleanup.actionStatus";
private String keyName;
private String defaultValue = "";
private Class<?> dataType = String.class;

View File

@@ -0,0 +1,21 @@
/**
* Copyright (c) 2018 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.tenancy.configuration.validator;
/**
* Specific tenant configuration validator, which validates that the given value
* is a {@link Long}.
*/
public class TenantConfigurationLongValidator implements TenantConfigurationValidator {
@Override
public Class<?> validateToClass() {
return Long.class;
}
}

View File

@@ -77,4 +77,18 @@ hawkbit.server.tenant.configuration.rollout-approval-enabled.defaultValue=false
hawkbit.server.tenant.configuration.rollout-approval-enabled.dataType=java.lang.Boolean
hawkbit.server.tenant.configuration.rollout-approval-enabled.validator=org.eclipse.hawkbit.tenancy.configuration.validator.TenantConfigurationBooleanValidator
hawkbit.server.tenant.configuration.action-cleanup-enabled.keyName=action.cleanup.enabled
hawkbit.server.tenant.configuration.action-cleanup-enabled.defaultValue=false
hawkbit.server.tenant.configuration.action-cleanup-enabled.dataType=java.lang.Boolean
hawkbit.server.tenant.configuration.action-cleanup-enabled.validator=org.eclipse.hawkbit.tenancy.configuration.validator.TenantConfigurationBooleanValidator
hawkbit.server.tenant.configuration.action-cleanup-action-expiry.keyName=action.cleanup.actionExpiry
# default: 30 days
hawkbit.server.tenant.configuration.action-cleanup-action-expiry.defaultValue=2592000000
hawkbit.server.tenant.configuration.action-cleanup-action-expiry.dataType=java.lang.Long
hawkbit.server.tenant.configuration.action-cleanup-action-expiry.validator=org.eclipse.hawkbit.tenancy.configuration.validator.TenantConfigurationLongValidator
hawkbit.server.tenant.configuration.action-cleanup-action-status.keyName=action.cleanup.actionStatus
hawkbit.server.tenant.configuration.action-cleanup-action-status.defaultValue=CANCELED,ERROR
# Default tenant configuration - END

View File

@@ -63,7 +63,7 @@ public interface ActionRepository extends BaseEntityRepository<JpaAction, Long>,
* the {@link DistributionSet} on which will be filtered
* @return the found {@link Action}s
*/
Page<Action> findByDistributionSetId(final Pageable pageable, final Long dsId);
Page<Action> findByDistributionSetId(Pageable pageable, Long dsId);
/**
* Retrieves all {@link Action}s which are referring the given
@@ -98,7 +98,7 @@ public interface ActionRepository extends BaseEntityRepository<JpaAction, Long>,
* the action active flag
* @return the found {@link Action}s
*/
List<Action> findByTargetAndActiveOrderByIdAsc(final JpaTarget target, boolean active);
List<Action> findByTargetAndActiveOrderByIdAsc(JpaTarget target, boolean active);
/**
* Retrieves the oldest {@link Action} that is active and referring to the
@@ -114,7 +114,7 @@ public interface ActionRepository extends BaseEntityRepository<JpaAction, Long>,
* @return the found {@link Action}
*/
@EntityGraph(value = "Action.ds", type = EntityGraphType.LOAD)
Optional<Action> findFirstByTargetControllerIdAndActive(final Sort sort, final String controllerId, boolean active);
Optional<Action> findFirstByTargetControllerIdAndActive(Sort sort, String controllerId, boolean active);
/**
* Checks if an active action exists for given
@@ -139,8 +139,7 @@ public interface ActionRepository extends BaseEntityRepository<JpaAction, Long>,
* assigned {@link DistributionSet}.
*/
@Query("Select a from JpaAction a join a.distributionSet ds join ds.modules modul where a.target.controllerId = :target and modul.id = :module order by a.id desc")
List<Action> findActionByTargetAndSoftwareModule(@Param("target") final String targetId,
@Param("module") Long moduleId);
List<Action> findActionByTargetAndSoftwareModule(@Param("target") String targetId, @Param("module") Long moduleId);
/**
* Retrieves all {@link Action}s which are referring the given
@@ -155,7 +154,7 @@ public interface ActionRepository extends BaseEntityRepository<JpaAction, Long>,
* @return the found {@link Action}s
*/
@Query("Select a from JpaAction a where a.target = :target and a.distributionSet = :ds")
Page<JpaAction> findByTargetAndDistributionSet(final Pageable pageable, @Param("target") final JpaTarget target,
Page<JpaAction> findByTargetAndDistributionSet(Pageable pageable, @Param("target") JpaTarget target,
@Param("ds") JpaDistributionSet ds);
/**
@@ -453,5 +452,6 @@ public interface ActionRepository extends BaseEntityRepository<JpaAction, Long>,
@Transactional
// Workaround for https://bugs.eclipse.org/bugs/show_bug.cgi?id=349477
@Query("DELETE FROM JpaAction a WHERE a.id IN ?1")
void deleteByIdIn(final Collection<Long> actionIDs);
void deleteByIdIn(Collection<Long> actionIDs);
}

View File

@@ -90,6 +90,6 @@ public interface ActionStatusRepository
* @return Page with found status messages.
*/
@Query("SELECT message FROM JpaActionStatus actionstatus JOIN actionstatus.messages message WHERE actionstatus.action.id = :actionId AND message NOT LIKE :filter")
Page<String> findMessagesByActionIdAndMessageNotLike(final Pageable pageable, @Param("actionId") Long actionId,
Page<String> findMessagesByActionIdAndMessageNotLike(Pageable pageable, @Param("actionId") Long actionId,
@Param("filter") String filter);
}

View File

@@ -48,7 +48,7 @@ public interface DistributionSetRepository
* @return list of found {@link DistributionSet}s
*/
@Query(value = "Select Distinct ds from JpaDistributionSet ds join ds.tags dst where dst.id = :tag and ds.deleted = 0")
Page<JpaDistributionSet> findByTag(Pageable pageable, @Param("tag") final Long tagId);
Page<JpaDistributionSet> findByTag(Pageable pageable, @Param("tag") Long tagId);
/**
* deletes the {@link DistributionSet}s with the given IDs.

View File

@@ -40,7 +40,7 @@ public interface DistributionSetTagRepository
*/
@Modifying
@Transactional
Long deleteByName(final String tagName);
Long deleteByName(String tagName);
/**
* find {@link DistributionSetTag} by its name.
@@ -49,7 +49,7 @@ public interface DistributionSetTagRepository
* to filter on
* @return the {@link DistributionSetTag} if found, otherwise null
*/
Optional<DistributionSetTag> findByNameEquals(final String tagName);
Optional<DistributionSetTag> findByNameEquals(String tagName);
/**
* Checks if tag with given name exists.

View File

@@ -18,8 +18,10 @@ import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.JoinType;
@@ -61,6 +63,7 @@ import org.eclipse.hawkbit.repository.model.TargetUpdateStatus;
import org.eclipse.hawkbit.repository.model.TargetWithActionType;
import org.eclipse.hawkbit.repository.rsql.VirtualPropertyReplacer;
import org.eclipse.hawkbit.security.SystemSecurityContext;
import org.eclipse.hawkbit.tenancy.TenantAware;
import org.eclipse.hawkbit.tenancy.configuration.TenantConfigurationProperties.TenantConfigurationKey;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -86,6 +89,7 @@ import org.springframework.transaction.support.TransactionTemplate;
import org.springframework.util.CollectionUtils;
import org.springframework.validation.annotation.Validated;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
/**
@@ -102,6 +106,8 @@ public class JpaDeploymentManagement implements DeploymentManagement {
*/
private static final int ACTION_PAGE_LIMIT = 1000;
private static final String QUERY_DELETE_ACTIONS_BY_STATE_AND_LAST_MODIFIED = "DELETE FROM sp_action WHERE tenant=#tenant AND status IN (%s) AND last_modified_at<#last_modified_at LIMIT 1000";
private final EntityManager entityManager;
private final ActionRepository actionRepository;
private final DistributionSetRepository distributionSetRepository;
@@ -119,6 +125,7 @@ public class JpaDeploymentManagement implements DeploymentManagement {
private final TenantConfigurationManagement tenantConfigurationManagement;
private final QuotaManagement quotaManagement;
private final SystemSecurityContext systemSecurityContext;
private final TenantAware tenantAware;
private final Database database;
JpaDeploymentManagement(final EntityManager entityManager, final ActionRepository actionRepository,
@@ -128,7 +135,7 @@ public class JpaDeploymentManagement implements DeploymentManagement {
final ApplicationContext applicationContext, final AfterTransactionCommitExecutor afterCommit,
final VirtualPropertyReplacer virtualPropertyReplacer, final PlatformTransactionManager txManager,
final TenantConfigurationManagement tenantConfigurationManagement, final QuotaManagement quotaManagement,
final SystemSecurityContext systemSecurityContext, final Database database) {
final SystemSecurityContext systemSecurityContext, final TenantAware tenantAware, final Database database) {
this.entityManager = entityManager;
this.actionRepository = actionRepository;
this.distributionSetRepository = distributionSetRepository;
@@ -148,6 +155,7 @@ public class JpaDeploymentManagement implements DeploymentManagement {
this.tenantConfigurationManagement = tenantConfigurationManagement;
this.quotaManagement = quotaManagement;
this.systemSecurityContext = systemSecurityContext;
this.tenantAware = tenantAware;
this.database = database;
}
@@ -388,7 +396,7 @@ public class JpaDeploymentManagement implements DeploymentManagement {
throw new ForceQuitActionNotAllowedException(action.getId() + " is not active and cannot be force quit");
}
LOG.warn("action ({}) was still activ and has been force quite.", action);
LOG.warn("action ({}) was still active and has been force quite.", action);
// document that the status has been retrieved
actionStatusRepository.save(new JpaActionStatus(action, Status.CANCELED, System.currentTimeMillis(),
@@ -689,4 +697,41 @@ public class JpaDeploymentManagement implements DeploymentManagement {
return distributionSetRepository.findInstalledAtTarget(controllerId);
}
@Override
@Transactional(readOnly = false)
public int deleteActionsByStatusAndLastModifiedBefore(final Set<Status> status, final long lastModified) {
if (status.isEmpty()) {
return 0;
}
/*
* We use a native query here because Spring JPA does not support to
* specify a LIMIT clause on a DELETE statement. However, for this
* specific use case (action cleanup), we must specify a row limit to
* reduce the overall load on the database.
*/
final int statusCount = status.size();
final Status[] statusArr = status.toArray(new Status[statusCount]);
final String queryStr = String.format(QUERY_DELETE_ACTIONS_BY_STATE_AND_LAST_MODIFIED,
formatInClauseWithNumberKeys(statusCount));
final Query deleteQuery = entityManager.createNativeQuery(queryStr);
IntStream.range(0, statusCount)
.forEach(i -> deleteQuery.setParameter(String.valueOf(i), statusArr[i].ordinal()));
deleteQuery.setParameter("tenant", tenantAware.getCurrentTenant().toUpperCase());
deleteQuery.setParameter("last_modified_at", lastModified);
LOG.debug("Action cleanup: Executing the following (native) query: {}", deleteQuery);
return deleteQuery.executeUpdate();
}
private static String formatInClauseWithNumberKeys(final int count) {
return formatInClause(IntStream.range(0, count).mapToObj(String::valueOf).collect(Collectors.toList()));
}
private static String formatInClause(final Collection<String> elements) {
return "#" + Joiner.on(",#").join(elements);
}
}

View File

@@ -8,6 +8,7 @@
*/
package org.eclipse.hawkbit.repository.jpa;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ScheduledExecutorService;
@@ -51,6 +52,9 @@ import org.eclipse.hawkbit.repository.event.remote.TargetPollEvent;
import org.eclipse.hawkbit.repository.jpa.aspects.ExceptionMappingAspectHandler;
import org.eclipse.hawkbit.repository.jpa.autoassign.AutoAssignChecker;
import org.eclipse.hawkbit.repository.jpa.autoassign.AutoAssignScheduler;
import org.eclipse.hawkbit.repository.jpa.autocleanup.AutoActionCleanup;
import org.eclipse.hawkbit.repository.jpa.autocleanup.AutoCleanupScheduler;
import org.eclipse.hawkbit.repository.jpa.autocleanup.CleanupTask;
import org.eclipse.hawkbit.repository.jpa.builder.JpaDistributionSetBuilder;
import org.eclipse.hawkbit.repository.jpa.builder.JpaDistributionSetTypeBuilder;
import org.eclipse.hawkbit.repository.jpa.builder.JpaRolloutBuilder;
@@ -567,7 +571,6 @@ public class RepositoryApplicationConfiguration extends JpaBaseConfiguration {
lockRegistry, properties.getDatabase(), rolloutApprovalStrategy);
}
/**
* {@link DefaultRolloutApprovalStrategy} bean.
*
@@ -613,11 +616,12 @@ public class RepositoryApplicationConfiguration extends JpaBaseConfiguration {
final AfterTransactionCommitExecutor afterCommit, final VirtualPropertyReplacer virtualPropertyReplacer,
final PlatformTransactionManager txManager,
final TenantConfigurationManagement tenantConfigurationManagement, final QuotaManagement quotaManagement,
final SystemSecurityContext systemSecurityContext, final JpaProperties properties) {
final SystemSecurityContext systemSecurityContext, final TenantAware tenantAware,
final JpaProperties properties) {
return new JpaDeploymentManagement(entityManager, actionRepository, distributionSetRepository, targetRepository,
actionStatusRepository, targetManagement, auditorProvider, eventPublisher, applicationContext,
afterCommit, virtualPropertyReplacer, txManager, tenantConfigurationManagement, quotaManagement,
systemSecurityContext, properties.getDatabase());
systemSecurityContext, tenantAware, properties.getDatabase());
}
/**
@@ -730,6 +734,46 @@ public class RepositoryApplicationConfiguration extends JpaBaseConfiguration {
return new AutoAssignScheduler(systemManagement, systemSecurityContext, autoAssignChecker, lockRegistry);
}
/**
* {@link AutoActionCleanup} bean.
*
* @param deploymentManagement
* Deployment management service
* @param configManagement
* Tenant configuration service
*
* @return a new {@link AutoActionCleanup} bean
*/
@Bean
CleanupTask actionCleanup(final DeploymentManagement deploymentManagement,
final TenantConfigurationManagement configManagement) {
return new AutoActionCleanup(deploymentManagement, configManagement);
}
/**
* {@link AutoCleanupScheduler} bean.
*
* @param systemManagement
* to find all tenants
* @param systemSecurityContext
* to run as system
* @param lockRegistry
* to lock the tenant for auto assignment
* @param cleanupTasks
* a list of cleanup tasks
*
* @return a new {@link AutoCleanupScheduler} bean
*/
@Bean
@ConditionalOnMissingBean
@Profile("!test")
@ConditionalOnProperty(prefix = "hawkbit.autocleanup.scheduler", name = "enabled", matchIfMissing = true)
AutoCleanupScheduler autoCleanupScheduler(final SystemManagement systemManagement,
final SystemSecurityContext systemSecurityContext, final LockRegistry lockRegistry,
final List<CleanupTask> cleanupTasks) {
return new AutoCleanupScheduler(systemManagement, systemSecurityContext, lockRegistry, cleanupTasks);
}
/**
* {@link RolloutScheduler} bean.
*

View File

@@ -144,7 +144,7 @@ public interface RolloutGroupRepository
* the page request to sort, limit the result
* @return a page of found {@link RolloutGroup} or {@code empty}.
*/
Page<JpaRolloutGroup> findByRolloutId(final Long rolloutId, Pageable page);
Page<JpaRolloutGroup> findByRolloutId(Long rolloutId, Pageable page);
/**
* Counts all {@link RolloutGroup} for a specific rollout.
@@ -154,7 +154,7 @@ public interface RolloutGroupRepository
*
* @return the amount of found {@link RolloutGroup}s.
*/
long countByRolloutId(final Long rolloutId);
long countByRolloutId(Long rolloutId);
@Modifying
@Query("DELETE FROM JpaRolloutGroup g where g.id in :rolloutGroupIds")

View File

@@ -30,5 +30,5 @@ public interface RolloutTargetGroupRepository
* the group to filter for
* @return count of targets in the group
*/
Long countByRolloutGroup(final JpaRolloutGroup rolloutGroup);
Long countByRolloutGroup(JpaRolloutGroup rolloutGroup);
}

View File

@@ -75,7 +75,7 @@ public interface SoftwareModuleRepository
@Transactional
@Query("UPDATE JpaSoftwareModule b SET b.deleted = 1, b.lastModifiedAt = :lastModifiedAt, b.lastModifiedBy = :lastModifiedBy WHERE b.id IN :ids")
void deleteSoftwareModule(@Param("lastModifiedAt") Long modifiedAt, @Param("lastModifiedBy") String modifiedBy,
@Param("ids") final Long... ids);
@Param("ids") Long... ids);
/**
* @param pageable

View File

@@ -36,7 +36,7 @@ public interface TargetFilterQueryRepository
* @param name
* @return custom target filter
*/
Optional<TargetFilterQuery> findByName(final String name);
Optional<TargetFilterQuery> findByName(String name);
/**
* Find list of all custom target filters.

View File

@@ -112,7 +112,7 @@ public interface TargetRepository extends BaseEntityRepository<JpaTarget, Long>,
@Transactional
// Workaround for https://bugs.eclipse.org/bugs/show_bug.cgi?id=349477
@Query("DELETE FROM JpaTarget t WHERE t.id IN ?1")
void deleteByIdIn(final Collection<Long> targetIDs);
void deleteByIdIn(Collection<Long> targetIDs);
/**
* Finds {@link Target}s by assigned {@link Tag}.
@@ -125,7 +125,7 @@ public interface TargetRepository extends BaseEntityRepository<JpaTarget, Long>,
* @return page of found targets
*/
@Query(value = "SELECT DISTINCT t FROM JpaTarget t JOIN t.tags tt WHERE tt.id = :tag")
Page<JpaTarget> findByTag(Pageable page, @Param("tag") final Long tagId);
Page<JpaTarget> findByTag(Pageable page, @Param("tag") Long tagId);
/**
* Finds all {@link Target}s based on given {@link Target#getControllerId()}
@@ -138,8 +138,8 @@ public interface TargetRepository extends BaseEntityRepository<JpaTarget, Long>,
* @return {@link List} of found {@link Target}s.
*/
@Query(value = "SELECT DISTINCT t from JpaTarget t JOIN t.tags tt WHERE tt.name = :tagname AND t.controllerId IN :targets")
List<JpaTarget> findByTagNameAndControllerIdIn(@Param("tagname") final String tag,
@Param("targets") final Collection<String> controllerIds);
List<JpaTarget> findByTagNameAndControllerIdIn(@Param("tagname") String tag,
@Param("targets") Collection<String> controllerIds);
/**
* Used by UI to filter based on selected status.
@@ -151,7 +151,7 @@ public interface TargetRepository extends BaseEntityRepository<JpaTarget, Long>,
*
* @return found targets
*/
Page<Target> findByUpdateStatus(final Pageable pageable, final TargetUpdateStatus status);
Page<Target> findByUpdateStatus(Pageable pageable, TargetUpdateStatus status);
/**
* retrieves the {@link Target}s which has the {@link DistributionSet}
@@ -163,7 +163,7 @@ public interface TargetRepository extends BaseEntityRepository<JpaTarget, Long>,
* the ID of the {@link DistributionSet}
* @return the found {@link Target}s
*/
Page<Target> findByInstalledDistributionSetId(final Pageable pageable, final Long setID);
Page<Target> findByInstalledDistributionSetId(Pageable pageable, Long setID);
/**
* Finds all targets that have defined {@link DistributionSet} assigned.
@@ -175,7 +175,7 @@ public interface TargetRepository extends BaseEntityRepository<JpaTarget, Long>,
*
* @return page of found targets
*/
Page<Target> findByAssignedDistributionSetId(final Pageable pageable, final Long setID);
Page<Target> findByAssignedDistributionSetId(Pageable pageable, Long setID);
/**
* Counts number of targets with given distribution set Id.
@@ -185,7 +185,7 @@ public interface TargetRepository extends BaseEntityRepository<JpaTarget, Long>,
*
* @return number of found {@link Target}s.
*/
Long countByAssignedDistributionSetId(final Long distId);
Long countByAssignedDistributionSetId(Long distId);
/**
* Counts number of targets with given distribution set Id.
@@ -194,7 +194,7 @@ public interface TargetRepository extends BaseEntityRepository<JpaTarget, Long>,
* to search for
* @return number of found {@link Target}s.
*/
Long countByInstalledDistributionSetId(final Long distId);
Long countByInstalledDistributionSetId(Long distId);
/**
* Finds all {@link Target}s in the repository.
@@ -221,7 +221,7 @@ public interface TargetRepository extends BaseEntityRepository<JpaTarget, Long>,
* the page request parameter
* @return a page of all targets related to a rollout group
*/
Page<Target> findByRolloutTargetGroupRolloutGroupId(final Long rolloutGroupId, Pageable page);
Page<Target> findByRolloutTargetGroupRolloutGroupId(Long rolloutGroupId, Pageable page);
/**
* Finds all targets related to a target rollout group stored for a specific

View File

@@ -0,0 +1,111 @@
/**
* Copyright (c) 2018 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.repository.jpa.autocleanup;
import static org.eclipse.hawkbit.tenancy.configuration.TenantConfigurationProperties.TenantConfigurationKey.ACTION_CLEANUP_ACTION_EXPIRY;
import static org.eclipse.hawkbit.tenancy.configuration.TenantConfigurationProperties.TenantConfigurationKey.ACTION_CLEANUP_ACTION_STATUS;
import static org.eclipse.hawkbit.tenancy.configuration.TenantConfigurationProperties.TenantConfigurationKey.ACTION_CLEANUP_ENABLED;
import java.io.Serializable;
import java.time.Instant;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.eclipse.hawkbit.repository.DeploymentManagement;
import org.eclipse.hawkbit.repository.TenantConfigurationManagement;
import org.eclipse.hawkbit.repository.model.Action;
import org.eclipse.hawkbit.repository.model.Action.Status;
import org.eclipse.hawkbit.repository.model.TenantConfigurationValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* A cleanup task for {@link Action} entities which can be used to delete
* actions which are in a certain {@link Action.Status}. It is recommended to
* only clean up actions which have terminated already (i.e. actions in status
* CANCELLED or ERROR).
*
* The cleanup task can be enabled /disabled and configured on a per tenant
* basis.
*/
public class AutoActionCleanup implements CleanupTask {
private static final Logger LOGGER = LoggerFactory.getLogger(AutoActionCleanup.class);
private static final String ID = "action-cleanup";
private static final boolean ACTION_CLEANUP_ENABLED_DEFAULT = false;
private static final long ACTION_CLEANUP_ACTION_EXPIRY_DEFAULT = TimeUnit.DAYS.toMillis(30);
private static final EnumSet<Status> EMPTY_STATUS_SET = EnumSet.noneOf(Status.class);
private final DeploymentManagement deploymentMgmt;
private final TenantConfigurationManagement config;
/**
* Constructs the action cleanup handler.
*
* @param deploymentMgmt
* The {@link DeploymentManagement} to operate on.
* @param configMgmt
* The {@link TenantConfigurationManagement} service.
*/
public AutoActionCleanup(final DeploymentManagement deploymentMgmt,
final TenantConfigurationManagement configMgmt) {
this.deploymentMgmt = deploymentMgmt;
this.config = configMgmt;
}
@Override
public void run() {
if (!isEnabled()) {
LOGGER.debug("Action cleanup is disabled for this tenant...");
return;
}
final EnumSet<Status> status = getActionStatus();
if (!status.isEmpty()) {
final long lastModified = System.currentTimeMillis() - getExpiry();
final int actionsCount = deploymentMgmt.deleteActionsByStatusAndLastModifiedBefore(status, lastModified);
LOGGER.debug("Deleted {} actions in status {} which have not been modified since {} ({})", actionsCount,
status, Instant.ofEpochMilli(lastModified), lastModified);
}
}
@Override
public String getId() {
return ID;
}
private long getExpiry() {
final TenantConfigurationValue<Long> expiry = getConfigValue(ACTION_CLEANUP_ACTION_EXPIRY, Long.class);
return expiry != null ? expiry.getValue() : ACTION_CLEANUP_ACTION_EXPIRY_DEFAULT;
}
private EnumSet<Status> getActionStatus() {
final TenantConfigurationValue<String> statusStr = getConfigValue(ACTION_CLEANUP_ACTION_STATUS, String.class);
if (statusStr != null) {
return Arrays.stream(statusStr.getValue().split("[;,]")).map(Status::valueOf)
.collect(Collectors.toCollection(() -> EnumSet.noneOf(Status.class)));
}
return EMPTY_STATUS_SET;
}
private boolean isEnabled() {
final TenantConfigurationValue<Boolean> isEnabled = getConfigValue(ACTION_CLEANUP_ENABLED, Boolean.class);
return isEnabled != null ? isEnabled.getValue() : ACTION_CLEANUP_ENABLED_DEFAULT;
}
private <T extends Serializable> TenantConfigurationValue<T> getConfigValue(final String key,
final Class<T> valueType) {
return config.getConfigurationValue(key, valueType);
}
}

View File

@@ -0,0 +1,97 @@
/**
* Copyright (c) 2018 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.repository.jpa.autocleanup;
import java.util.List;
import java.util.concurrent.locks.Lock;
import org.eclipse.hawkbit.repository.SystemManagement;
import org.eclipse.hawkbit.security.SystemSecurityContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.integration.support.locks.LockRegistry;
import org.springframework.scheduling.annotation.Scheduled;
/**
* A scheduler to invoke a set of cleanup handlers periodically.
*/
public class AutoCleanupScheduler {
private static final Logger LOGGER = LoggerFactory.getLogger(AutoCleanupScheduler.class);
private static final String AUTO_CLEANUP = "auto-cleanup";
private static final String SEP = ".";
private static final String PROP_AUTO_CLEANUP_INTERVAL = "${hawkbit.autocleanup.scheduler.fixedDelay:86400000}";
private final SystemManagement systemManagement;
private final SystemSecurityContext systemSecurityContext;
private final LockRegistry lockRegistry;
private final List<CleanupTask> cleanupTasks;
/**
* Constructs the cleanup schedulers and initializes it with a set of
* cleanup handlers.
*
* @param systemManagement
* Management APIs to invoke actions in a certain tenant context.
* @param systemSecurityContext
* The system security context.
* @param lockRegistry
* A registry for shared locks.
* @param cleanupTasks
* A list of cleanup tasks.
*/
public AutoCleanupScheduler(final SystemManagement systemManagement,
final SystemSecurityContext systemSecurityContext, final LockRegistry lockRegistry,
final List<CleanupTask> cleanupTasks) {
this.systemManagement = systemManagement;
this.systemSecurityContext = systemSecurityContext;
this.lockRegistry = lockRegistry;
this.cleanupTasks = cleanupTasks;
}
/**
* Scheduler method which kicks off the cleanup process.
*/
@Scheduled(initialDelayString = PROP_AUTO_CLEANUP_INTERVAL, fixedDelayString = PROP_AUTO_CLEANUP_INTERVAL)
public void run() {
LOGGER.debug("Auto cleanup scheduler has been triggered.");
// run this code in system code privileged to have the necessary
// permission to query and create entities
if (!cleanupTasks.isEmpty()) {
systemSecurityContext.runAsSystem(this::executeAutoCleanup);
}
}
/**
* Method which executes each registered cleanup task for each tenant.
*/
@SuppressWarnings("squid:S3516")
private Void executeAutoCleanup() {
systemManagement.forEachTenant(tenant -> cleanupTasks.forEach(task -> {
final Lock lock = obtainLock(task, tenant);
if (!lock.tryLock()) {
return;
}
try {
task.run();
} catch (final RuntimeException e) {
LOGGER.error("Cleanup task failed.", e);
} finally {
lock.unlock();
}
}));
return null;
}
private Lock obtainLock(final CleanupTask task, final String tenant) {
return lockRegistry.obtain(AUTO_CLEANUP + SEP + task.getId() + SEP + tenant);
}
}

View File

@@ -0,0 +1,27 @@
/**
* Copyright (c) 2018 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.repository.jpa.autocleanup;
/**
* Interface modeling a cleanup task.
*/
public interface CleanupTask extends Runnable {
/**
* Executes the cleanup task.
*/
@Override
void run();
/**
* @return The identifier of this cleanup task. Never null.
*/
String getId();
}

View File

@@ -0,0 +1,227 @@
/**
* Copyright (c) 2018 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.repository.jpa.autocleanup;
import static org.assertj.core.api.Assertions.assertThat;
import static org.eclipse.hawkbit.tenancy.configuration.TenantConfigurationProperties.TenantConfigurationKey.ACTION_CLEANUP_ACTION_EXPIRY;
import static org.eclipse.hawkbit.tenancy.configuration.TenantConfigurationProperties.TenantConfigurationKey.ACTION_CLEANUP_ACTION_STATUS;
import static org.eclipse.hawkbit.tenancy.configuration.TenantConfigurationProperties.TenantConfigurationKey.ACTION_CLEANUP_ENABLED;
import java.util.Arrays;
import java.util.Collections;
import java.util.stream.Collectors;
import org.eclipse.hawkbit.repository.jpa.AbstractJpaIntegrationTest;
import org.eclipse.hawkbit.repository.model.Action;
import org.eclipse.hawkbit.repository.model.Action.ActionType;
import org.eclipse.hawkbit.repository.model.Action.Status;
import org.eclipse.hawkbit.repository.model.DistributionSet;
import org.eclipse.hawkbit.repository.model.Target;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import ru.yandex.qatools.allure.annotations.Description;
import ru.yandex.qatools.allure.annotations.Features;
import ru.yandex.qatools.allure.annotations.Stories;
/**
* Test class for {@link AutoActionCleanup}.
*
*/
@Features("Component Tests - Repository")
@Stories("Action cleanup handler")
public class AutoActionCleanupTest extends AbstractJpaIntegrationTest {
@Autowired
private AutoActionCleanup autoActionCleanup;
@Test
@Description("Verifies that running actions are not cleaned up.")
public void runningActionsAreNotCleanedUp() {
// cleanup config for this test case
setupCleanupConfiguration(true, 0, Action.Status.CANCELED, Action.Status.ERROR);
final Target trg1 = testdataFactory.createTarget("trg1");
final Target trg2 = testdataFactory.createTarget("trg2");
final DistributionSet ds1 = testdataFactory.createDistributionSet("ds1");
final DistributionSet ds2 = testdataFactory.createDistributionSet("ds2");
deploymentManagement.assignDistributionSet(ds1.getId(), ActionType.FORCED, 0,
Collections.singletonList(trg1.getControllerId()));
deploymentManagement.assignDistributionSet(ds2.getId(), ActionType.FORCED, 0,
Collections.singletonList(trg2.getControllerId()));
assertThat(actionRepository.count()).isEqualTo(2);
autoActionCleanup.run();
assertThat(actionRepository.count()).isEqualTo(2);
}
@Test
@Description("Verifies that nothing is cleaned up if the cleanup is disabled.")
public void cleanupDisabled() {
// cleanup config for this test case
setupCleanupConfiguration(false, 0, Action.Status.CANCELED);
final Target trg1 = testdataFactory.createTarget("trg1");
final Target trg2 = testdataFactory.createTarget("trg2");
final DistributionSet ds1 = testdataFactory.createDistributionSet("ds1");
final DistributionSet ds2 = testdataFactory.createDistributionSet("ds2");
final Long action1 = deploymentManagement.assignDistributionSet(ds1.getId(), ActionType.FORCED, 0,
Collections.singletonList(trg1.getControllerId())).getActions().get(0);
deploymentManagement.assignDistributionSet(ds2.getId(), ActionType.FORCED, 0,
Collections.singletonList(trg2.getControllerId()));
setActionToCanceled(action1);
assertThat(actionRepository.count()).isEqualTo(2);
autoActionCleanup.run();
assertThat(actionRepository.count()).isEqualTo(2);
}
@Test
@Description("Verifies that canceled and failed actions are cleaned up.")
public void canceledAndFailedActionsAreCleanedUp() {
// cleanup config for this test case
setupCleanupConfiguration(true, 0, Action.Status.CANCELED, Action.Status.ERROR);
final Target trg1 = testdataFactory.createTarget("trg1");
final Target trg2 = testdataFactory.createTarget("trg2");
final Target trg3 = testdataFactory.createTarget("trg3");
final DistributionSet ds1 = testdataFactory.createDistributionSet("ds1");
final DistributionSet ds2 = testdataFactory.createDistributionSet("ds2");
final Long action1 = deploymentManagement.assignDistributionSet(ds1.getId(), ActionType.FORCED, 0,
Collections.singletonList(trg1.getControllerId())).getActions().get(0);
final Long action2 = deploymentManagement.assignDistributionSet(ds2.getId(), ActionType.FORCED, 0,
Collections.singletonList(trg2.getControllerId())).getActions().get(0);
final Long action3 = deploymentManagement.assignDistributionSet(ds2.getId(), ActionType.FORCED, 0,
Collections.singletonList(trg3.getControllerId())).getActions().get(0);
assertThat(actionRepository.count()).isEqualTo(3);
setActionToCanceled(action1);
setActionToFailed(action2);
assertThat(actionRepository.count()).isEqualTo(3);
autoActionCleanup.run();
assertThat(actionRepository.count()).isEqualTo(1);
assertThat(actionRepository.getById(action3)).isPresent();
}
@Test
@Description("Verifies that canceled actions are cleaned up.")
public void canceledActionsAreCleanedUp() {
// cleanup config for this test case
setupCleanupConfiguration(true, 0, Action.Status.CANCELED);
final Target trg1 = testdataFactory.createTarget("trg1");
final Target trg2 = testdataFactory.createTarget("trg2");
final Target trg3 = testdataFactory.createTarget("trg3");
final DistributionSet ds1 = testdataFactory.createDistributionSet("ds1");
final DistributionSet ds2 = testdataFactory.createDistributionSet("ds2");
final Long action1 = deploymentManagement.assignDistributionSet(ds1.getId(), ActionType.FORCED, 0,
Collections.singletonList(trg1.getControllerId())).getActions().get(0);
final Long action2 = deploymentManagement.assignDistributionSet(ds2.getId(), ActionType.FORCED, 0,
Collections.singletonList(trg2.getControllerId())).getActions().get(0);
final Long action3 = deploymentManagement.assignDistributionSet(ds2.getId(), ActionType.FORCED, 0,
Collections.singletonList(trg3.getControllerId())).getActions().get(0);
assertThat(actionRepository.count()).isEqualTo(3);
setActionToCanceled(action1);
setActionToFailed(action2);
assertThat(actionRepository.count()).isEqualTo(3);
autoActionCleanup.run();
assertThat(actionRepository.count()).isEqualTo(2);
assertThat(actionRepository.getById(action2)).isPresent();
assertThat(actionRepository.getById(action3)).isPresent();
}
@Test
@Description("Verifies that canceled and failed actions are cleaned up once they expired.")
@SuppressWarnings("squid:S2925")
public void canceledAndFailedActionsAreCleanedUpWhenExpired() throws InterruptedException {
// cleanup config for this test case
setupCleanupConfiguration(true, 500, Action.Status.CANCELED, Action.Status.ERROR);
final Target trg1 = testdataFactory.createTarget("trg1");
final Target trg2 = testdataFactory.createTarget("trg2");
final Target trg3 = testdataFactory.createTarget("trg3");
final DistributionSet ds1 = testdataFactory.createDistributionSet("ds1");
final DistributionSet ds2 = testdataFactory.createDistributionSet("ds2");
final Long action1 = deploymentManagement.assignDistributionSet(ds1.getId(), ActionType.FORCED, 0,
Collections.singletonList(trg1.getControllerId())).getActions().get(0);
final Long action2 = deploymentManagement.assignDistributionSet(ds2.getId(), ActionType.FORCED, 0,
Collections.singletonList(trg2.getControllerId())).getActions().get(0);
final Long action3 = deploymentManagement.assignDistributionSet(ds2.getId(), ActionType.FORCED, 0,
Collections.singletonList(trg3.getControllerId())).getActions().get(0);
assertThat(actionRepository.count()).isEqualTo(3);
setActionToCanceled(action1);
setActionToFailed(action2);
autoActionCleanup.run();
// actions have not expired yet
assertThat(actionRepository.count()).isEqualTo(3);
// wait for expiry to elapse
Thread.sleep(800);
autoActionCleanup.run();
assertThat(actionRepository.count()).isEqualTo(1);
assertThat(actionRepository.getById(action3)).isPresent();
}
private void setActionToCanceled(final Long id) {
deploymentManagement.cancelAction(id);
deploymentManagement.forceQuitAction(id);
}
private void setActionToFailed(final Long id) {
controllerManagement.addUpdateActionStatus(entityFactory.actionStatus().create(id).status(Status.ERROR));
}
private void setupCleanupConfiguration(final boolean cleanupEnabled, final long expiry, final Status... status) {
tenantConfigurationManagement.addOrUpdateConfiguration(ACTION_CLEANUP_ENABLED, cleanupEnabled);
tenantConfigurationManagement.addOrUpdateConfiguration(ACTION_CLEANUP_ACTION_EXPIRY, expiry);
tenantConfigurationManagement.addOrUpdateConfiguration(ACTION_CLEANUP_ACTION_STATUS,
Arrays.stream(status).map(Status::toString).collect(Collectors.joining(",")));
}
}

View File

@@ -0,0 +1,84 @@
/**
* Copyright (c) 2018 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.repository.jpa.autocleanup;
import static org.assertj.core.api.Assertions.assertThat;
import java.util.Arrays;
import java.util.concurrent.atomic.AtomicInteger;
import org.eclipse.hawkbit.repository.jpa.AbstractJpaIntegrationTest;
import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.integration.support.locks.LockRegistry;
import ru.yandex.qatools.allure.annotations.Description;
import ru.yandex.qatools.allure.annotations.Features;
import ru.yandex.qatools.allure.annotations.Stories;
/**
* Test class for {@link AutoCleanupScheduler}.
*
*/
@Features("Component Tests - Repository")
@Stories("Auto cleanup scheduler")
public class AutoCleanupSchedulerTest extends AbstractJpaIntegrationTest {
private final AtomicInteger counter = new AtomicInteger();
@Autowired
private LockRegistry lockRegistry;
@Before
public void setUp() {
counter.set(0);
}
@Test
@Description("Verifies that all cleanup handlers are executed regardless if one of them throws an error")
public void executeHandlerChain() {
new AutoCleanupScheduler(systemManagement, systemSecurityContext, lockRegistry, Arrays.asList(
new SuccessfulCleanup(), new SuccessfulCleanup(), new FailingCleanup(), new SuccessfulCleanup())).run();
assertThat(counter.get()).isEqualTo(4);
}
private class SuccessfulCleanup implements CleanupTask {
@Override
public void run() {
counter.incrementAndGet();
}
@Override
public String getId() {
return "success";
}
}
private class FailingCleanup implements CleanupTask {
@Override
public void run() {
counter.incrementAndGet();
throw new RuntimeException("cleanup failed");
}
@Override
public String getId() {
return "success";
}
}
}