Introduce Pause Success Action (#2867)

* Introduce Pause Success Action Signed-off-by: vasilchev <vasil.ilchev@bosch.com> * Instead of overriding SuccessAction, trigger next group from resume rollout Fix Rollout Mgmt Resource to accept new Pause Action Signed-off-by: vasilchev <vasil.ilchev@bosch.com> * Review findings Signed-off-by: vasilchev <vasil.ilchev@bosch.com> * Remove unused import --------- Signed-off-by: vasilchev <vasil.ilchev@bosch.com>
2026-01-13 11:20:21 +02:00
parent c8dd5c2fe5
commit 0083d5538a
12 changed files with 299 additions and 87 deletions
--- a/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/JpaRepositoryConfiguration.java
+++ b/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/JpaRepositoryConfiguration.java
@@ -70,7 +70,8 @@ import org.eclipse.hawkbit.repository.jpa.repository.SoftwareModuleRepository;
 import org.eclipse.hawkbit.repository.jpa.repository.SoftwareModuleTypeRepository;
 import org.eclipse.hawkbit.repository.jpa.repository.TargetRepository;
 import org.eclipse.hawkbit.repository.jpa.repository.TargetTypeRepository;
-import org.eclipse.hawkbit.repository.jpa.rollout.condition.PauseRolloutGroupAction;
+import org.eclipse.hawkbit.repository.jpa.rollout.condition.PauseRolloutGroupErrorAction;
+import org.eclipse.hawkbit.repository.jpa.rollout.condition.PauseRolloutGroupSuccessAction;
 import org.eclipse.hawkbit.repository.jpa.rollout.condition.RolloutGroupActionEvaluator;
 import org.eclipse.hawkbit.repository.jpa.rollout.condition.RolloutGroupConditionEvaluator;
 import org.eclipse.hawkbit.repository.jpa.rollout.condition.RolloutGroupEvaluationManager;
@@ -213,9 +214,9 @@ public class JpaRepositoryConfiguration {

    @Bean
    @ConditionalOnMissingBean
-    PauseRolloutGroupAction pauseRolloutGroupAction(
+    PauseRolloutGroupErrorAction pauseRolloutGroupErrorAction(
            final RolloutManagement rolloutManagement, final RolloutGroupRepository rolloutGroupRepository) {
-        return new PauseRolloutGroupAction(rolloutManagement, rolloutGroupRepository);
+        return new PauseRolloutGroupErrorAction(rolloutManagement, rolloutGroupRepository);
    }

    @Bean
@@ -225,6 +226,13 @@ public class JpaRepositoryConfiguration {
        return new StartNextGroupRolloutGroupSuccessAction(rolloutGroupRepository, deploymentManagement);
    }

+    @Bean
+    @ConditionalOnMissingBean
+    PauseRolloutGroupSuccessAction pauseRolloutGroupSuccessAction(final RolloutManagement rolloutManagement,
+            final RolloutGroupRepository rolloutGroupRepository) {
+        return new PauseRolloutGroupSuccessAction(rolloutManagement, rolloutGroupRepository);
+    }
+
    @Bean
    @ConditionalOnMissingBean
    ThresholdRolloutGroupErrorCondition thresholdRolloutGroupErrorCondition(final ActionRepository actionRepository) {
--- a/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/management/JpaRolloutManagement.java
+++ b/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/management/JpaRolloutManagement.java
@@ -66,6 +66,7 @@ import org.eclipse.hawkbit.repository.jpa.repository.ActionStatusRepository;
 import org.eclipse.hawkbit.repository.jpa.repository.RolloutGroupRepository;
 import org.eclipse.hawkbit.repository.jpa.repository.RolloutRepository;
 import org.eclipse.hawkbit.repository.jpa.repository.TargetRepository;
+import org.eclipse.hawkbit.repository.jpa.rollout.condition.RolloutGroupEvaluationManager;
 import org.eclipse.hawkbit.repository.jpa.rollout.condition.StartNextGroupRolloutGroupSuccessAction;
 import org.eclipse.hawkbit.repository.jpa.specifications.ActionSpecifications;
 import org.eclipse.hawkbit.repository.jpa.specifications.RolloutSpecification;
@@ -86,8 +87,10 @@ import org.eclipse.hawkbit.repository.model.TotalTargetCountActionStatus;
 import org.eclipse.hawkbit.repository.model.TotalTargetCountStatus;
 import org.eclipse.hawkbit.repository.qfields.RolloutFields;
 import org.eclipse.hawkbit.utils.ObjectCopyUtil;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
+import org.springframework.context.annotation.Lazy;
 import org.springframework.dao.ConcurrencyFailureException;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
@@ -118,6 +121,9 @@ public class JpaRolloutManagement implements RolloutManagement {
            RolloutStatus.CREATING, RolloutStatus.READY, RolloutStatus.WAITING_FOR_APPROVAL, RolloutStatus.STARTING, RolloutStatus.RUNNING,
            RolloutStatus.PAUSED, RolloutStatus.APPROVAL_DENIED);

+    private static final Comparator<RolloutGroup> ROLLOUT_GROUP_DESC_COMP = Comparator.comparingLong(RolloutGroup::getId).reversed();
+
+    private RolloutGroupEvaluationManager rolloutGroupEvaluationManager;
    @Value("${hawkbit.repository.jpa.management.rollout.max.actions.per.transaction:5000}")
    private int maxActions;

@@ -164,6 +170,13 @@ public class JpaRolloutManagement implements RolloutManagement {
                quotaManagement, this::isMultiAssignmentsEnabled, this::isConfirmationFlowEnabled, repositoryProperties, null);
    }

+    @Autowired
+    @Lazy
+    private void setRolloutGroupEvaluationManager(
+            final RolloutGroupEvaluationManager rolloutGroupEvaluationManager) {
+        this.rolloutGroupEvaluationManager = rolloutGroupEvaluationManager;
+    }
+
    public static String createRolloutLockKey(final String tenant) {
        return tenant + "-rollout";
    }
@@ -348,10 +361,34 @@ public class JpaRolloutManagement implements RolloutManagement {
            throw new RolloutIllegalStateException("Rollout can only be resumed in state paused but current state is " +
                    rollout.getStatus().name().toLowerCase());
        }
+        final List<RolloutGroup> allStartedGroups = rollout.getRolloutGroups().stream()
+                .filter(g -> RolloutGroupStatus.SCHEDULED != g.getStatus()).toList();
+        if (!allStartedGroups.isEmpty()) {
+            final RolloutGroup lastStartedGroup = allStartedGroups.get(allStartedGroups.size() - 1);
+            if (shouldStartNextGroupOnResume(rollout, lastStartedGroup)) {
+                startNextRolloutGroupAction.exec(rollout, lastStartedGroup);
+            }
+        }
        rollout.setStatus(RolloutStatus.RUNNING);
        rolloutRepository.save(rollout);
    }

+    /**
+     * Check if on resume of a paused rollout the next group shall be started directly.
+     * Cases where we need to manually start the next group:
+     *  - last running group is in error state and there is still some old group in running state, only running groups would be evaluated which would leave Rollout in running state but no trigger new group
+     *  - last running group has success action to PAUSE and the success condition is fulfilled
+     * @param rollout
+     * @param lastStartedGroup
+     * @return true if next group shall be started directly on resume, false otherwise
+     */
+    private boolean shouldStartNextGroupOnResume(final JpaRollout rollout, final RolloutGroup lastStartedGroup) {
+        return lastStartedGroup.getStatus().equals(RolloutGroupStatus.ERROR) ||
+                (lastStartedGroup.getSuccessAction() == RolloutGroup.RolloutGroupSuccessAction.PAUSE &&
+                        rolloutGroupEvaluationManager.getSuccessConditionEvaluator(lastStartedGroup.getSuccessCondition())
+                                .eval(rollout, lastStartedGroup, lastStartedGroup.getSuccessConditionExp()));
+    }
+
    @Override
    @Transactional
    @Retryable(retryFor = { ConcurrencyFailureException.class }, maxAttempts = Constants.TX_RT_MAX,
@@ -482,13 +519,15 @@ public class JpaRolloutManagement implements RolloutManagement {
            throw new RolloutIllegalStateException("Rollout does not have any groups left to be triggered");
        }

-        final RolloutGroup latestRunning = groups.stream()
-                .sorted(Comparator.comparingLong(RolloutGroup::getId).reversed())
-                .filter(g -> RolloutGroupStatus.RUNNING.equals(g.getStatus()))
-                .findFirst()
-                .orElseThrow(() -> new RolloutIllegalStateException("No group is running"));
-
-        startNextRolloutGroupAction.exec(rollout, latestRunning);
+        final List<JpaRolloutGroup> startedRolloutGroups = rollout.getRolloutGroups().stream()
+                .filter(group -> group.getStatus() != RolloutGroupStatus.SCHEDULED)
+                .sorted(ROLLOUT_GROUP_DESC_COMP)
+                .map(JpaRolloutGroup.class::cast)
+                .toList();
+        if (startedRolloutGroups.isEmpty()) {
+            throw new RolloutIllegalStateException("Cannot find any started rollout group to trigger next from");
+        }
+        startNextRolloutGroupAction.exec(rollout, startedRolloutGroups.get(0));
    }

    @Override
--- a/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/rollout/condition/AbstractPauseRolloutGroupAction.java
+++ b/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/rollout/condition/AbstractPauseRolloutGroupAction.java
@@ -1,5 +1,5 @@
 /**
- * Copyright (c) 2015 Bosch Software Innovations GmbH and others
+ * Copyright (c) 2025 Contributors to the Eclipse Foundation
 *
 * This program and the accompanying materials are made
 * available under the terms of the Eclipse Public License 2.0
@@ -12,38 +12,27 @@ package org.eclipse.hawkbit.repository.jpa.rollout.condition;
 import static org.eclipse.hawkbit.context.AccessContext.asSystem;

 import org.eclipse.hawkbit.repository.RolloutManagement;
-import org.eclipse.hawkbit.repository.jpa.model.JpaRolloutGroup;
 import org.eclipse.hawkbit.repository.jpa.repository.RolloutGroupRepository;
 import org.eclipse.hawkbit.repository.model.Rollout;
 import org.eclipse.hawkbit.repository.model.RolloutGroup;
-import org.eclipse.hawkbit.repository.model.RolloutGroup.RolloutGroupStatus;

 /**
- * Error action evaluator which pauses the whole {@link Rollout} and sets the
- * current {@link RolloutGroup} to error.
+ * Abstract class for pausing a rollout group.
+ *
+ * @param <T> the type of the action
 */
-public class PauseRolloutGroupAction implements RolloutGroupActionEvaluator<RolloutGroup.RolloutGroupErrorAction> {
+public abstract class AbstractPauseRolloutGroupAction<T extends Enum<T>> implements RolloutGroupActionEvaluator<T> {

-    private final RolloutManagement rolloutManagement;
-    private final RolloutGroupRepository rolloutGroupRepository;
+    protected final RolloutManagement rolloutManagement;
+    protected final RolloutGroupRepository rolloutGroupRepository;

-    public PauseRolloutGroupAction(final RolloutManagement rolloutManagement,
+    protected AbstractPauseRolloutGroupAction(final RolloutManagement rolloutManagement,
            final RolloutGroupRepository rolloutGroupRepository) {
        this.rolloutManagement = rolloutManagement;
        this.rolloutGroupRepository = rolloutGroupRepository;
    }

-    @Override
-    public RolloutGroup.RolloutGroupErrorAction getAction() {
-        return RolloutGroup.RolloutGroupErrorAction.PAUSE;
-    }
-
-    @Override
-    public void exec(final Rollout rollout, final RolloutGroup rolloutG) {
-        final JpaRolloutGroup rolloutGroup = (JpaRolloutGroup) rolloutG;
-
-        rolloutGroup.setStatus(RolloutGroupStatus.ERROR);
-        rolloutGroupRepository.save(rolloutGroup);
+    public void exec(final Rollout rollout, final RolloutGroup rolloutGroup) {
        // Refresh latest rollout state in order to avoid cases when
        // previous group have matched error condition and paused the rollout
        // and this one tries to pause the rollout too but throws an exception
@@ -56,3 +45,4 @@ public class PauseRolloutGroupAction implements RolloutGroupActionEvaluator<Roll
        }
    }
 }
+
--- a/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/rollout/condition/PauseRolloutGroupErrorAction.java
+++ b/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/rollout/condition/PauseRolloutGroupErrorAction.java
@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) 2025 Contributors to the Eclipse Foundation
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipse.hawkbit.repository.jpa.rollout.condition;
+
+import org.eclipse.hawkbit.repository.RolloutManagement;
+import org.eclipse.hawkbit.repository.jpa.model.JpaRolloutGroup;
+import org.eclipse.hawkbit.repository.jpa.repository.RolloutGroupRepository;
+import org.eclipse.hawkbit.repository.model.Rollout;
+import org.eclipse.hawkbit.repository.model.RolloutGroup;
+import org.eclipse.hawkbit.repository.model.RolloutGroup.RolloutGroupStatus;
+
+/**
+ * Error action evaluator which pauses the whole {@link Rollout} and sets the
+ * current {@link RolloutGroup} to error.
+ */
+public class PauseRolloutGroupErrorAction extends AbstractPauseRolloutGroupAction<RolloutGroup.RolloutGroupErrorAction> {
+
+    public PauseRolloutGroupErrorAction(final RolloutManagement rolloutManagement,
+            final RolloutGroupRepository rolloutGroupRepository) {
+        super(rolloutManagement, rolloutGroupRepository);
+    }
+
+    @Override
+    public RolloutGroup.RolloutGroupErrorAction getAction() {
+        return RolloutGroup.RolloutGroupErrorAction.PAUSE;
+    }
+
+    @Override
+    public void exec(final Rollout rollout, final RolloutGroup rolloutG) {
+        // set rollout group status to error
+        final JpaRolloutGroup rolloutGroup = (JpaRolloutGroup) rolloutG;
+        rolloutGroup.setStatus(RolloutGroupStatus.ERROR);
+        rolloutGroupRepository.save(rolloutGroup);
+        // pause the rollout
+        super.exec(rollout, rolloutGroup);
+    }
+}
+
--- a/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/rollout/condition/PauseRolloutGroupSuccessAction.java
+++ b/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/rollout/condition/PauseRolloutGroupSuccessAction.java
@@ -0,0 +1,43 @@
+/**
+ * Copyright (c) 2025 Contributors to the Eclipse Foundation
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipse.hawkbit.repository.jpa.rollout.condition;
+
+import org.eclipse.hawkbit.repository.RolloutManagement;
+import org.eclipse.hawkbit.repository.jpa.repository.RolloutGroupRepository;
+import org.eclipse.hawkbit.repository.model.Rollout;
+import org.eclipse.hawkbit.repository.model.RolloutGroup;
+
+/**
+ * Success action evaluator which pauses the whole {@link Rollout}.
+ */
+public class PauseRolloutGroupSuccessAction
+        extends AbstractPauseRolloutGroupAction<RolloutGroup.RolloutGroupSuccessAction> {
+
+    public PauseRolloutGroupSuccessAction(final RolloutManagement rolloutManagement,
+            final RolloutGroupRepository rolloutGroupRepository) {
+        super(rolloutManagement, rolloutGroupRepository);
+    }
+
+    @Override
+    public RolloutGroup.RolloutGroupSuccessAction getAction() {
+        return RolloutGroup.RolloutGroupSuccessAction.PAUSE;
+    }
+
+    @Override
+    public void exec(final Rollout rollout, final RolloutGroup rolloutGroup) {
+        if (!rolloutGroupRepository
+                .findByParentIdAndStatus(rolloutGroup.getId(), RolloutGroup.RolloutGroupStatus.SCHEDULED).isEmpty()) {
+            // if there are still scheduled child groups, do pause the rollout, otherwise just let it in running state
+            super.exec(rollout, rolloutGroup);
+        }
+    }
+}
+
+
--- a/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/scheduler/JpaRolloutExecutor.java
+++ b/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/scheduler/JpaRolloutExecutor.java
@@ -111,7 +111,6 @@ public class JpaRolloutExecutor implements RolloutExecutor {
     */
    private static final List<Status> DOWNLOAD_ONLY_ACTION_TERMINATION_STATUSES =
            List.of(Status.ERROR, Status.FINISHED, Status.CANCELED, Status.DOWNLOADED);
-    private static final Comparator<RolloutGroup> DESC_COMP = Comparator.comparingLong(RolloutGroup::getId).reversed();
    private static final String TRANSACTION_ASSIGNING_TARGETS_TO_ROLLOUT_GROUP_FAILED = "Transaction assigning Targets to RolloutGroup failed";

    private final ActionRepository actionRepository;
@@ -353,11 +352,10 @@ public class JpaRolloutExecutor implements RolloutExecutor {
                .filter(group -> group.getStatus() == RolloutGroupStatus.RUNNING)
                .map(JpaRolloutGroup.class::cast)
                .toList();
-
        if (runningGroups.isEmpty()) {
            // no running rollouts, probably there was an error somewhere at the latest group. And the latest group has
            // been switched from running into error state. So we need to find the latest group which
-            executeLatestRolloutGroup(rollout);
+            asSystem(() -> rolloutManagement.triggerNextGroup(rollout.getId()));
        } else {
            log.debug("Rollout {} has {} running groups", rollout.getId(), runningGroups.size());
            executeRunningGroups(rollout, runningGroups, rollout.getRolloutGroups().get(rollout.getRolloutGroups().size() - 1));
@@ -410,18 +408,6 @@ public class JpaRolloutExecutor implements RolloutExecutor {
        return groupsActiveLeft == 0;
    }

-    private void executeLatestRolloutGroup(final JpaRollout rollout) {
-        final List<JpaRolloutGroup> latestRolloutGroup = rollout.getRolloutGroups().stream()
-                .filter(group -> group.getStatus() != RolloutGroupStatus.SCHEDULED)
-                .sorted(DESC_COMP)
-                .map(JpaRolloutGroup.class::cast)
-                .toList();
-        if (latestRolloutGroup.isEmpty()) {
-            return;
-        }
-        executeRolloutGroupSuccessAction(rollout, latestRolloutGroup.get(0));
-    }
-
    // fakes getTotalTargets count to match expected for the last dynamic group
    // so the evaluation to use total targets to properly
    private RolloutGroup evalProxy(final RolloutGroup group) {
@@ -530,7 +516,7 @@ public class JpaRolloutExecutor implements RolloutExecutor {
                    .eval(rollout, evalProxy, rolloutGroup.getSuccessConditionExp());
            if (isFinished) {
                log.debug("Rollout group {} is finished, starting next group", rolloutGroup);
-                executeRolloutGroupSuccessAction(rollout, rolloutGroup);
+                evaluationManager.getSuccessActionEvaluator(rolloutGroup.getSuccessAction()).exec(rollout, rolloutGroup);
            } else {
                log.debug("Rollout group {} is still running", rolloutGroup);
            }
@@ -539,10 +525,6 @@ public class JpaRolloutExecutor implements RolloutExecutor {
        }
    }

-    private void executeRolloutGroupSuccessAction(final Rollout rollout, final RolloutGroup rolloutGroup) {
-        evaluationManager.getSuccessActionEvaluator(rolloutGroup.getSuccessAction()).exec(rollout, rolloutGroup);
-    }
-
    private void startFirstRolloutGroup(final JpaRollout rollout) {
        log.debug("startFirstRolloutGroup called for rollout {}", rollout.getId());