From 6fd52d4b4a30156b754327a860fb6f48e7ce148c Mon Sep 17 00:00:00 2001 From: Avgustin Marinov Date: Thu, 1 Aug 2024 15:36:29 +0300 Subject: [PATCH] Involve all targets in dynamic rollouts (#1795) * involve targets into dynamic rollouts eagerly - doesn't wait for dynamic group to become running in order to involve devices * adds trottling for involving targes into dynamic groups * small style refactoring Signed-off-by: Marinov Avgustin skipImplicitLockForTags = List.of("skip-implicit-lock", "skip_implicit_lock", "SKIP_IMPLICIT_LOCK", "SKIP-IMPLICIT-LOCK"); + + /** + * The minimum period (in milli-seconds) on which dynamic rollouts should make attempt to involve + * new targets + */ + private long dynamicRolloutsMinInvolvePeriodMS = 60_000; } \ No newline at end of file diff --git a/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/JpaRolloutExecutor.java b/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/JpaRolloutExecutor.java index 2bb8d9537..012d53377 100644 --- a/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/JpaRolloutExecutor.java +++ b/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/JpaRolloutExecutor.java @@ -13,9 +13,11 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Proxy; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -24,6 +26,7 @@ import jakarta.persistence.EntityManager; import lombok.extern.slf4j.Slf4j; import org.eclipse.hawkbit.repository.DeploymentManagement; import org.eclipse.hawkbit.repository.QuotaManagement; +import org.eclipse.hawkbit.repository.RepositoryProperties; import org.eclipse.hawkbit.repository.RolloutApprovalStrategy; import org.eclipse.hawkbit.repository.RolloutExecutor; import org.eclipse.hawkbit.repository.RolloutGroupManagement; @@ -99,51 +102,55 @@ public class JpaRolloutExecutor implements RolloutExecutor { private static final List DOWNLOAD_ONLY_ACTION_TERMINATION_STATUSES = Arrays.asList(Status.ERROR, Status.FINISHED, Status.CANCELED, Status.DOWNLOADED); - private final RolloutTargetGroupRepository rolloutTargetGroupRepository; - private final EntityManager entityManager; - private final RolloutRepository rolloutRepository; private final ActionRepository actionRepository; private final RolloutGroupRepository rolloutGroupRepository; - private final AfterTransactionCommitExecutor afterCommit; - private final TenantAware tenantAware; - private final RolloutGroupManagement rolloutGroupManagement; - private final QuotaManagement quotaManagement; - private final DeploymentManagement deploymentManagement; + private final RolloutTargetGroupRepository rolloutTargetGroupRepository; + private final RolloutRepository rolloutRepository; + private final TargetManagement targetManagement; - private final EventPublisherHolder eventPublisherHolder; - private final PlatformTransactionManager txManager; - private final RolloutApprovalStrategy rolloutApprovalStrategy; - private final RolloutGroupEvaluationManager evaluationManager; + private final DeploymentManagement deploymentManagement; + private final RolloutGroupManagement rolloutGroupManagement; private final RolloutManagement rolloutManagement; - - /** - * Constructor - */ - public JpaRolloutExecutor(final RolloutTargetGroupRepository rolloutTargetGroupRepository, - final EntityManager entityManager, final RolloutRepository rolloutRepository, + private final QuotaManagement quotaManagement; + + private final RolloutGroupEvaluationManager evaluationManager; + private final RolloutApprovalStrategy rolloutApprovalStrategy; + + private final EntityManager entityManager; + private final PlatformTransactionManager txManager; + private final AfterTransactionCommitExecutor afterCommit; + private final EventPublisherHolder eventPublisherHolder; + + private final TenantAware tenantAware; + private final RepositoryProperties repositoryProperties; + + public JpaRolloutExecutor( final ActionRepository actionRepository, final RolloutGroupRepository rolloutGroupRepository, - final AfterTransactionCommitExecutor afterCommit, final TenantAware tenantAware, - final RolloutGroupManagement rolloutGroupManagement, final QuotaManagement quotaManagement, - final DeploymentManagement deploymentManagement, final TargetManagement targetManagement, - final EventPublisherHolder eventPublisherHolder, final PlatformTransactionManager txManager, - final RolloutApprovalStrategy rolloutApprovalStrategy, - final RolloutGroupEvaluationManager evaluationManager, final RolloutManagement rolloutManagement) { - this.rolloutTargetGroupRepository = rolloutTargetGroupRepository; - this.entityManager = entityManager; - this.rolloutRepository = rolloutRepository; + final RolloutTargetGroupRepository rolloutTargetGroupRepository, + final RolloutRepository rolloutRepository, final TargetManagement targetManagement, + final DeploymentManagement deploymentManagement, final RolloutGroupManagement rolloutGroupManagement, + final RolloutManagement rolloutManagement, final QuotaManagement quotaManagement, + final RolloutGroupEvaluationManager evaluationManager, final RolloutApprovalStrategy rolloutApprovalStrategy, + final EntityManager entityManager, final PlatformTransactionManager txManager, + final AfterTransactionCommitExecutor afterCommit, final EventPublisherHolder eventPublisherHolder, + final TenantAware tenantAware, final RepositoryProperties repositoryProperties) { this.actionRepository = actionRepository; this.rolloutGroupRepository = rolloutGroupRepository; - this.afterCommit = afterCommit; - this.tenantAware = tenantAware; - this.rolloutGroupManagement = rolloutGroupManagement; - this.quotaManagement = quotaManagement; - this.deploymentManagement = deploymentManagement; + this.rolloutTargetGroupRepository = rolloutTargetGroupRepository; + this.rolloutRepository = rolloutRepository; this.targetManagement = targetManagement; - this.eventPublisherHolder = eventPublisherHolder; - this.txManager = txManager; - this.rolloutApprovalStrategy = rolloutApprovalStrategy; - this.evaluationManager = evaluationManager; + this.deploymentManagement = deploymentManagement; + this.rolloutGroupManagement = rolloutGroupManagement; this.rolloutManagement = rolloutManagement; + this.quotaManagement = quotaManagement; + this.evaluationManager = evaluationManager; + this.rolloutApprovalStrategy = rolloutApprovalStrategy; + this.entityManager = entityManager; + this.txManager = txManager; + this.afterCommit = afterCommit; + this.eventPublisherHolder = eventPublisherHolder; + this.tenantAware = tenantAware; + this.repositoryProperties = repositoryProperties; } @Override @@ -427,7 +434,7 @@ public class JpaRolloutExecutor implements RolloutExecutor { // fakes getTotalTargets count to match expected for the last dynamic group // so the evaluation to use total targets to properly - private RolloutGroup evalProxy(final RolloutGroup group, final List rolloutGroups) { + private RolloutGroup evalProxy(final RolloutGroup group) { if (group.isDynamic()) { final int expected = Math.max((int)group.getTargetPercentage(), 1); return (RolloutGroup) Proxy.newProxyInstance( @@ -457,7 +464,7 @@ public class JpaRolloutExecutor implements RolloutExecutor { } final RolloutGroup evalProxy = rolloutGroup == rolloutGroups.get(rolloutGroups.size() - 1) ? - evalProxy(rolloutGroup, rolloutGroups) : rolloutGroup; + evalProxy(rolloutGroup) : rolloutGroup; // error state check, do we need to stop the whole // rollout because of error? final boolean isError = checkErrorState(rollout, evalProxy); @@ -529,7 +536,7 @@ public class JpaRolloutExecutor implements RolloutExecutor { } } - private boolean checkSuccessCondition(final Rollout rollout, final RolloutGroup rolloutGroup, final RolloutGroup evalProxy, + private void checkSuccessCondition(final Rollout rollout, final RolloutGroup rolloutGroup, final RolloutGroup evalProxy, final RolloutGroupSuccessCondition successCondition) { log.trace("Checking finish condition {} on rolloutgroup {}", successCondition, rolloutGroup); try { @@ -541,11 +548,9 @@ public class JpaRolloutExecutor implements RolloutExecutor { } else { log.debug("Rolloutgroup {} is still running", rolloutGroup); } - return isFinished; } catch (final EvaluatorNotConfiguredException e) { log.error("Something bad happened when accessing the finish condition or success action bean {}", successCondition.name(), e); - return false; } } @@ -675,32 +680,32 @@ public class JpaRolloutExecutor implements RolloutExecutor { }); } + private final Map lastDynamicGroupFill = new ConcurrentHashMap<>(); // return if group change is made private boolean fillDynamicRolloutGroupsWithTargets(final JpaRollout rollout) { + final AtomicLong lastFill = lastDynamicGroupFill.computeIfAbsent(rollout.getId(), id -> new AtomicLong(0)); + final long now = System.currentTimeMillis(); + if (now - lastFill.get() < repositoryProperties.getDynamicRolloutsMinInvolvePeriodMS()) { + // too early to make another dynamic involvement attempt + return false; + } else { + lastFill.set(now); + } + RolloutHelper.verifyRolloutInStatus(rollout, RolloutStatus.RUNNING); final List rolloutGroups = rollout.getRolloutGroups(); final JpaRolloutGroup group = (JpaRolloutGroup)rolloutGroups.get(rolloutGroups.size() - 1); - if (group.getStatus() == RolloutGroupStatus.FINISHED) { - createDynamicGroup(rollout, group, rolloutGroups.size(), RolloutGroupStatus.RUNNING); - return true; - } else if (group.getStatus() != RolloutGroupStatus.RUNNING) { - return false; - } - - // expected as last full group final long expectedInGroup = Math.max((int)group.getTargetPercentage(), 1); - final long currentlyInGroup = group.getTotalTargets(); - if (currentlyInGroup >= expectedInGroup) { - // the last one is filled. create new and start filling it + if (currentlyInGroup >= expectedInGroup || group.getStatus() == RolloutGroupStatus.FINISHED) { + // the last one is full. create new and start filling it on the next iteration createDynamicGroup(rollout, group, rolloutGroups.size(), RolloutGroupStatus.SCHEDULED); return true; } - // there are more to be filled for that group - // do this until there are more matching + // there are more to be filled for the last group do this until there are more matching try { long targetsLeftToAdd = expectedInGroup - currentlyInGroup; final String groupTargetFilter = RolloutHelper.getGroupTargetFilter( @@ -725,12 +730,6 @@ public class JpaRolloutExecutor implements RolloutExecutor { if (newActions > 0) { updateTotalTargetCount(group, group.getTotalTargets() + newActions); - if (targetsLeftToAdd == 0) { - // this is filled create a new one in scheduled state - createDynamicGroup(rollout, group, rolloutGroups.size(), RolloutGroupStatus.SCHEDULED); - return true; - } - // TODO - try to return false and proceed with handleRunningRollout // the problem is that OptimisticLockException is thrown in that case return true; @@ -800,7 +799,7 @@ public class JpaRolloutExecutor implements RolloutExecutor { final ActionType actionType = rollout.getActionType(); final long forceTime = rollout.getForcedTime(); final List newActions = createActions(targets.getContent(), distributionSet, actionType, forceTime, rollout, group); - if (!newActions.isEmpty()) { + if (!newActions.isEmpty() && group.getStatus() == RolloutGroupStatus.RUNNING) { deploymentManagement.startScheduledActions(newActions); } @@ -906,17 +905,14 @@ public class JpaRolloutExecutor implements RolloutExecutor { } /** - * Enforces the quota defining the maximum number of {@link Action}s per - * {@link Target}. + * Enforces the quota defining the maximum number of {@link Action}s per {@link Target}. * - * @param target - * The target - * @param requested - * number of actions to check + * @param target the target + * @param requested number of actions to check */ private void assertActionsPerTargetQuota(final Target target, final int requested) { final int quota = quotaManagement.getMaxActionsPerTarget(); QuotaHelper.assertAssignmentQuota(target.getId(), requested, quota, Action.class, Target.class, actionRepository::countByTargetId); } -} +} \ No newline at end of file diff --git a/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/RepositoryApplicationConfiguration.java b/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/RepositoryApplicationConfiguration.java index d0f127eba..89d0ba309 100644 --- a/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/RepositoryApplicationConfiguration.java +++ b/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/RepositoryApplicationConfiguration.java @@ -739,19 +739,20 @@ public class RepositoryApplicationConfiguration extends JpaBaseConfiguration { @Bean @ConditionalOnMissingBean - RolloutExecutor rolloutExecutor(final RolloutTargetGroupRepository rolloutTargetGroupRepository, - final EntityManager entityManager, final RolloutRepository rolloutRepository, + RolloutExecutor rolloutExecutor( final ActionRepository actionRepository, final RolloutGroupRepository rolloutGroupRepository, - final AfterTransactionCommitExecutor afterCommit, final TenantAware tenantAware, - final RolloutGroupManagement rolloutGroupManagement, final QuotaManagement quotaManagement, - final DeploymentManagement deploymentManagement, final TargetManagement targetManagement, - final EventPublisherHolder eventPublisherHolder, final PlatformTransactionManager txManager, - final RolloutApprovalStrategy rolloutApprovalStrategy, - final RolloutGroupEvaluationManager evaluationManager, final RolloutManagement rolloutManagement) { - return new JpaRolloutExecutor(rolloutTargetGroupRepository, entityManager, rolloutRepository, actionRepository, - rolloutGroupRepository, afterCommit, tenantAware, rolloutGroupManagement, quotaManagement, - deploymentManagement, targetManagement, eventPublisherHolder, txManager, rolloutApprovalStrategy, - evaluationManager, rolloutManagement); + final RolloutTargetGroupRepository rolloutTargetGroupRepository, + final RolloutRepository rolloutRepository, final TargetManagement targetManagement, + final DeploymentManagement deploymentManagement, final RolloutGroupManagement rolloutGroupManagement, + final RolloutManagement rolloutManagement, final QuotaManagement quotaManagement, + final RolloutGroupEvaluationManager evaluationManager, final RolloutApprovalStrategy rolloutApprovalStrategy, + final EntityManager entityManager, final PlatformTransactionManager txManager, + final AfterTransactionCommitExecutor afterCommit, final EventPublisherHolder eventPublisherHolder, + final TenantAware tenantAware, final RepositoryProperties repositoryProperties) { + return new JpaRolloutExecutor(actionRepository, rolloutGroupRepository, rolloutTargetGroupRepository, + rolloutRepository, targetManagement, deploymentManagement, rolloutGroupManagement, rolloutManagement, + quotaManagement, evaluationManager, rolloutApprovalStrategy, entityManager, txManager, afterCommit, + eventPublisherHolder, tenantAware, repositoryProperties); } @Bean diff --git a/hawkbit-repository/hawkbit-repository-jpa/src/test/java/org/eclipse/hawkbit/repository/jpa/management/RolloutManagementFlowTest.java b/hawkbit-repository/hawkbit-repository-jpa/src/test/java/org/eclipse/hawkbit/repository/jpa/management/RolloutManagementFlowTest.java index f8704ece6..1f5fafc47 100644 --- a/hawkbit-repository/hawkbit-repository-jpa/src/test/java/org/eclipse/hawkbit/repository/jpa/management/RolloutManagementFlowTest.java +++ b/hawkbit-repository/hawkbit-repository-jpa/src/test/java/org/eclipse/hawkbit/repository/jpa/management/RolloutManagementFlowTest.java @@ -24,9 +24,11 @@ import org.eclipse.hawkbit.repository.model.RolloutGroup; import org.eclipse.hawkbit.repository.model.RolloutGroup.RolloutGroupStatus; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.springframework.context.annotation.PropertySource; import org.springframework.data.domain.Page; import org.springframework.data.domain.Sort; import org.springframework.data.domain.Sort.Direction; +import org.springframework.test.context.TestPropertySource; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; @@ -38,6 +40,7 @@ import static org.assertj.core.api.Assertions.assertThat; */ @Feature("Component Tests - Repository") @Story("Rollout Management (Flow)") +@TestPropertySource(properties = { "hawkbit.server.repository.dynamicRolloutsMinInvolvePeriodMS=-1" }) class RolloutManagementFlowTest extends AbstractJpaIntegrationTest { @BeforeEach @@ -131,7 +134,8 @@ class RolloutManagementFlowTest extends AbstractJpaIntegrationTest { // fill first and create second testdataFactory.createTargets(targetPrefix, amountGroups * 3 + 2, 2); - rolloutHandler.handleAll(); // fill first dynamic group and create a new dynamic2 + rolloutHandler.handleAll(); // fill first dynamic group + rolloutHandler.handleAll(); // and create a new dynamic2 assertRollout(rollout, true, RolloutStatus.RUNNING, amountGroups + 2, amountGroups * 3 + 3); assertGroup(dynamic1, true, RolloutGroupStatus.RUNNING, 3); groups = rolloutGroupManagement.findByRollout( @@ -142,11 +146,11 @@ class RolloutManagementFlowTest extends AbstractJpaIntegrationTest { // create scheduled actions for the dynamic2 rolloutHandler.handleAll(); - assertRollout(rollout, true, RolloutStatus.RUNNING, amountGroups + 2, amountGroups * 3 + 3); + assertRollout(rollout, true, RolloutStatus.RUNNING, amountGroups + 2, amountGroups * 3 + 4); assertGroup(dynamic1, true, RolloutGroupStatus.RUNNING, 3); - assertGroup(dynamic2, true, RolloutGroupStatus.SCHEDULED, 0); + assertGroup(dynamic2, true, RolloutGroupStatus.SCHEDULED, 1); assertAndGetRunning(rollout, 4); // one from the last static group and 3 from the first dynamic - assertScheduled(rollout, 0); + assertScheduled(rollout, 1); // executes last from static and dynamic1 without 1 target assertAndGetRunning(rollout, 4)// one from the last static and 6 for the first dynamic @@ -158,15 +162,12 @@ class RolloutManagementFlowTest extends AbstractJpaIntegrationTest { assertAndGetRunning(rollout, 1); // remains on in the first dynamic rolloutHandler.handleAll(); - assertRollout(rollout, true, RolloutStatus.RUNNING, amountGroups + 2, amountGroups * 3 + 3); + assertRollout(rollout, true, RolloutStatus.RUNNING, amountGroups + 2, amountGroups * 3 + 4); assertGroup(groups.get(amountGroups - 1), false, RolloutGroupStatus.FINISHED, 3); assertGroup(dynamic1, true, RolloutGroupStatus.RUNNING, 3); - assertGroup(dynamic2, true, RolloutGroupStatus.RUNNING, 0); - - rolloutHandler.handleAll(); // add 1 action to now running second dynamic - assertRollout(rollout, true, RolloutStatus.RUNNING, amountGroups + 2, amountGroups * 3 + 4); - assertAndGetRunning(rollout, 2); + // first dynamic threshold is reached, second is started assertGroup(dynamic2, true, RolloutGroupStatus.RUNNING, 1); + assertAndGetRunning(rollout, 2); testdataFactory.createTargets(targetPrefix, amountGroups * 3 + 4, 1); rolloutManagement.pauseRollout(rollout.getId()); @@ -184,6 +185,7 @@ class RolloutManagementFlowTest extends AbstractJpaIntegrationTest { // finish the second dynamic group testdataFactory.createTargets(targetPrefix, amountGroups * 3 + 5, 1); rolloutHandler.handleAll(); + rolloutHandler.handleAll(); // create next assertRollout(rollout, true, RolloutStatus.RUNNING, amountGroups + 3, amountGroups * 3 + 6); assertAndGetRunning(rollout, 4); // one from the dynamic1 and 3 from the dynamic2 assertGroup(dynamic2, true, RolloutGroupStatus.RUNNING, 3); // assign the target created when paused @@ -249,7 +251,8 @@ class RolloutManagementFlowTest extends AbstractJpaIntegrationTest { // fill first (2) and create fill partially the second (+2 new) testdataFactory.createTargets(targetPrefix, amountGroups * 3 + 4, 4); - rolloutHandler.handleAll(); // fill first dynamic group and create a new dynamic2 + rolloutHandler.handleAll(); // fill first dynamic group + rolloutHandler.handleAll(); // and create a new dynamic2 assertRollout(rollout, true, RolloutStatus.RUNNING, amountGroups + 2, amountGroups * 3 + 6); assertGroup(dynamic1, true, RolloutGroupStatus.RUNNING, 6); groups = rolloutGroupManagement.findByRollout( @@ -260,11 +263,11 @@ class RolloutManagementFlowTest extends AbstractJpaIntegrationTest { // create scheduled actions for the dynamic2 rolloutHandler.handleAll(); - assertRollout(rollout, true, RolloutStatus.RUNNING, amountGroups + 2, amountGroups * 3 + 6); + assertRollout(rollout, true, RolloutStatus.RUNNING, amountGroups + 2, amountGroups * 3 + 8); assertGroup(dynamic1, true, RolloutGroupStatus.RUNNING, 6); - assertGroup(dynamic2, true, RolloutGroupStatus.SCHEDULED, 0); + assertGroup(dynamic2, true, RolloutGroupStatus.SCHEDULED, 2); assertAndGetRunning(rollout, 7); // one from the last static group and 6 from the first dynamic - assertScheduled(rollout, 0); + assertScheduled(rollout, 2); // executes last from static and dynamic1 without 1 target assertAndGetRunning(rollout, 7)// one from the last static and 6 for the first dynamic @@ -276,15 +279,12 @@ class RolloutManagementFlowTest extends AbstractJpaIntegrationTest { assertAndGetRunning(rollout, 1); // remains on in the first dynamic rolloutHandler.handleAll(); - assertRollout(rollout, true, RolloutStatus.RUNNING, amountGroups + 2, amountGroups * 3 + 6); + assertRollout(rollout, true, RolloutStatus.RUNNING, amountGroups + 2, amountGroups * 3 + 8); assertGroup(groups.get(amountGroups - 1), false, RolloutGroupStatus.FINISHED, 3); assertGroup(dynamic1, true, RolloutGroupStatus.RUNNING, 6); - assertGroup(dynamic2, true, RolloutGroupStatus.RUNNING, 0); - - rolloutHandler.handleAll(); // add 2 action to now running second dynamic - assertRollout(rollout, true, RolloutStatus.RUNNING, amountGroups + 2, amountGroups * 3 + 8); - assertAndGetRunning(rollout, 3); + // first dynamic threshold is reached, second is started assertGroup(dynamic2, true, RolloutGroupStatus.RUNNING, 2); + assertAndGetRunning(rollout, 3); testdataFactory.createTargets(targetPrefix, amountGroups * 3 + 8, 2); rolloutManagement.pauseRollout(rollout.getId()); @@ -339,7 +339,8 @@ class RolloutManagementFlowTest extends AbstractJpaIntegrationTest { // fill first (2) and create fill partially the second (+2 new) testdataFactory.createTargets(targetPrefix, 4, 4); - rolloutHandler.handleAll(); // fill first dynamic group and create a new dynamic2 + rolloutHandler.handleAll(); // fill first dynamic group + rolloutHandler.handleAll(); // and create a new dynamic2 assertRollout(rollout, true, RolloutStatus.RUNNING, 2, 6); assertGroup(dynamic1, true, RolloutGroupStatus.RUNNING, 6); groups = rolloutGroupManagement.findByRollout( @@ -350,25 +351,22 @@ class RolloutManagementFlowTest extends AbstractJpaIntegrationTest { // create scheduled actions for the dynamic2 rolloutHandler.handleAll(); - assertRollout(rollout, true, RolloutStatus.RUNNING, 2, 6); + assertRollout(rollout, true, RolloutStatus.RUNNING, 2, 8); assertGroup(dynamic1, true, RolloutGroupStatus.RUNNING, 6); - assertGroup(dynamic2, true, RolloutGroupStatus.SCHEDULED, 0); + assertGroup(dynamic2, true, RolloutGroupStatus.SCHEDULED, 2); assertAndGetRunning(rollout, 6); // 6 from the first dynamic - assertScheduled(rollout, 0); + assertScheduled(rollout, 2); // executes dynamic1 without 1 target executeWithoutOneTargetFromAGroup(dynamic1, rollout, 6); assertAndGetRunning(rollout, 1); // remains on in the first dynamic rolloutHandler.handleAll(); - assertRollout(rollout, true, RolloutStatus.RUNNING, 2, 6); + assertRollout(rollout, true, RolloutStatus.RUNNING, 2, 8); assertGroup(dynamic1, true, RolloutGroupStatus.RUNNING, 6); - assertGroup(dynamic2, true, RolloutGroupStatus.RUNNING, 0); - - rolloutHandler.handleAll(); // add 2 action to now running second dynamic - assertRollout(rollout, true, RolloutStatus.RUNNING, 2, 8); - assertAndGetRunning(rollout, 3); + // first dynamic threshold is reached, second is started assertGroup(dynamic2, true, RolloutGroupStatus.RUNNING, 2); + assertAndGetRunning(rollout, 3); testdataFactory.createTargets(targetPrefix, 8, 2); rolloutManagement.pauseRollout(rollout.getId()); @@ -387,6 +385,8 @@ class RolloutManagementFlowTest extends AbstractJpaIntegrationTest { private void executeStaticWithoutOneTargetFromTheLastGroupAndHandleAll( final List groups, final Rollout rollout, final int amountGroups) { + // create dynamic group if needed + rolloutHandler.handleAll(); // execute groups (without on of the last) assertThat(refresh(groups.get(0)).getStatus()).isEqualTo(RolloutGroupStatus.RUNNING); for (int i = 0; i < amountGroups; i++) { diff --git a/hawkbit-rest/hawkbit-mgmt-resource/src/test/java/org/eclipse/hawkbit/mgmt/rest/resource/MgmtRolloutResourceTest.java b/hawkbit-rest/hawkbit-mgmt-resource/src/test/java/org/eclipse/hawkbit/mgmt/rest/resource/MgmtRolloutResourceTest.java index 2ffec13e3..29f44b066 100644 --- a/hawkbit-rest/hawkbit-mgmt-resource/src/test/java/org/eclipse/hawkbit/mgmt/rest/resource/MgmtRolloutResourceTest.java +++ b/hawkbit-rest/hawkbit-mgmt-resource/src/test/java/org/eclipse/hawkbit/mgmt/rest/resource/MgmtRolloutResourceTest.java @@ -52,7 +52,6 @@ import org.eclipse.hawkbit.repository.model.RolloutGroup.RolloutGroupSuccessCond import org.eclipse.hawkbit.repository.model.RolloutGroupConditionBuilder; import org.eclipse.hawkbit.repository.model.RolloutGroupConditions; import org.eclipse.hawkbit.repository.model.Target; -import org.eclipse.hawkbit.repository.model.TargetUpdateStatus; import org.eclipse.hawkbit.repository.test.util.RolloutTestApprovalStrategy; import org.eclipse.hawkbit.repository.test.util.SecurityContextSwitch; import org.eclipse.hawkbit.repository.test.util.WithUser; @@ -70,6 +69,7 @@ import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort.Direction; import org.springframework.hateoas.MediaTypes; import org.springframework.http.MediaType; +import org.springframework.test.context.TestPropertySource; import org.springframework.test.web.servlet.ResultMatcher; import io.qameta.allure.Description; @@ -82,6 +82,7 @@ import io.qameta.allure.Story; */ @Feature("Component Tests - Management API") @Story("Rollout Resource") +@TestPropertySource(locations = "classpath:/mgmt-test.properties", properties = { "hawkbit.server.repository.dynamicRolloutsMinInvolvePeriodMS=-1" }) class MgmtRolloutResourceTest extends AbstractManagementApiIntegrationTest { private static final String HREF_ROLLOUT_PREFIX = "http://localhost/rest/v1/rollouts/"; @@ -256,25 +257,29 @@ class MgmtRolloutResourceTest extends AbstractManagementApiIntegrationTest { .andExpect(jsonPath("content[0].totalGroups", equalTo(2))); final int amountOfDynamicTargets = 2; - List additionalTargets = testdataFactory.createTargets(amountOfDynamicTargets, "rollout-dynamic-addition-", "rollout"); + testdataFactory.createTargets(amountOfDynamicTargets, "rollout-dynamic-addition-", "rollout"); rolloutHandler.handleAll(); final List groups = rolloutGroupManagement.findByRollout(PAGE, rollout.getId()).getContent(); groups.forEach(group -> { - if (!group.getName().contains("-dynamic")) { - rolloutGroupManagement.findTargetsOfRolloutGroup(PAGE, group.getId()).forEach(target -> deploymentManagement.findActionsByTarget(target.getControllerId(), PAGE).forEach(action -> { + if (!group.isDynamic()) { + rolloutGroupManagement.findTargetsOfRolloutGroup(PAGE, group.getId()) + .forEach(target -> deploymentManagement.findActionsByTarget(target.getControllerId(), PAGE) + .forEach(action -> { + deploymentManagement.cancelAction(action.getId()); + awaitActionStatus(action.getId(), Status.CANCELING); - deploymentManagement.cancelAction(action.getId()); - awaitActionStatus(action.getId(), Status.CANCELING); - - deploymentManagement.forceQuitAction(action.getId()); - awaitActionStatus(action.getId(), Status.CANCELED); - })); + deploymentManagement.forceQuitAction(action.getId()); + awaitActionStatus(action.getId(), Status.CANCELED); + })); } }); - awaitPendingDeviceInRollout(additionalTargets.get(0).getControllerId()); + // process as much as needed to start first dynamic + for (int i = 0; i < 5; i++) { + rolloutHandler.handleAll(); + } mvc.perform(get("/rest/v1/rollouts?representation=full").accept(MediaType.APPLICATION_JSON)) .andDo(MockMvcResultPrinter.print()).andExpect(status().isOk()) @@ -285,16 +290,16 @@ class MgmtRolloutResourceTest extends AbstractManagementApiIntegrationTest { .andExpect(jsonPath("content[0].status", equalTo("running"))) .andExpect(jsonPath("content[0].targetFilterQuery", equalTo("name==rollout*"))) .andExpect(jsonPath("content[0].distributionSetId", equalTo(dsA.getId().intValue()))) - .andExpect(jsonPath("content[0].totalTargets", equalTo(11))) + .andExpect(jsonPath("content[0].totalTargets", equalTo(amountTargets + amountOfDynamicTargets))) .andExpect(jsonPath("content[0].totalTargetsPerStatus").exists()) .andExpect(jsonPath("content[0].totalTargetsPerStatus.running", equalTo(1))) .andExpect(jsonPath("content[0].totalTargetsPerStatus.notstarted", equalTo(0))) - .andExpect(jsonPath("content[0].totalTargetsPerStatus.scheduled", equalTo(0))) + .andExpect(jsonPath("content[0].totalTargetsPerStatus.scheduled", equalTo(1))) .andExpect(jsonPath("content[0].totalTargetsPerStatus.cancelled", equalTo(10))) .andExpect(jsonPath("content[0].totalTargetsPerStatus.finished", equalTo(0))) .andExpect(jsonPath("content[0].totalTargetsPerStatus.error", equalTo(0))) .andExpect(jsonPath("content[0].deleted", equalTo(false))) - .andExpect(jsonPath("content[0].totalGroups", equalTo(3))) + .andExpect(jsonPath("content[0].totalGroups", equalTo(4))) .andExpect(jsonPath("content[0]._links.self.href", startsWith(HREF_ROLLOUT_PREFIX))); } @@ -1323,22 +1328,6 @@ class MgmtRolloutResourceTest extends AbstractManagementApiIntegrationTest { .getStatus().equals(status)); } - private void awaitPendingDeviceInRollout(final String controllerId) { - Awaitility.await().atMost(Duration.ofMinutes(1)).pollInterval(Duration.ofMillis(100)).with() - .until(() -> { - Optional maybeTarget = SecurityContextSwitch.runAsPrivileged(() -> { - rolloutHandler.handleAll(); - List targets = targetManagement.findByRsql(PAGE, "controllerId==" + controllerId).getContent(); - if (targets.size() == 1) { - return Optional.of(targets.get(0)); - } else { - return Optional.empty(); - } - }); - return maybeTarget.isPresent() && maybeTarget.get().getUpdateStatus().equals(TargetUpdateStatus.PENDING); - }); - } - @Test @Description("Deletion of a rollout") void deleteRollout() throws Exception {