lingwei.kong/hawkbit
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Create Rollout UI with Groups definition (#369)

Browse Source 
* Rollout creation UI with groups definition

Signed-off-by: Dominik Herbst <dominik.herbst@bosch-si.com>

* Improved code quality

Signed-off-by: Dominik Herbst <dominik.herbst@bosch-si.com>

* Added a legend to the Rollouts Creation UI's PieChart

Signed-off-by: Dominik Herbst <dominik.herbst@bosch-si.com>

* Updated color palette for pie chart. Using RolloutGroupCreate for validation. Displaying pie chart for group editing.

Signed-off-by: Dominik Herbst <dominik.herbst@bosch-si.com>

* Optimized ComboBoxBuilder

Signed-off-by: Dominik Herbst <dominik.herbst@bosch-si.com>

* Removed not necessary RPC interfaces.

Signed-off-by: Dominik Herbst <dominik.herbst@bosch-si.com>

* Fix Rollout verification count query. Improved UI style. Added dependency management for d3.

Signed-off-by: Dominik Herbst <dominik.herbst@bosch-si.com>

* Auto start and scheduled start of Rollouts

Signed-off-by: Dominik Herbst <dominik.herbst@bosch-si.com>

* Optimized RSQL Utility to avoid multiple joins on the same table.

Signed-off-by: Dominik Herbst <dominik.herbst@bosch-si.com>

* Fixed RSQL Utility to avoid multiple joins on the same table to only work with OR nodes. Optimized validation queries for rollout group creation.

Signed-off-by: Dominik Herbst <dominik.herbst@bosch-si.com>

* Asynchronous handling of groups validation in the create rollouts UI.

Signed-off-by: Dominik Herbst <dominik.herbst@bosch-si.com>

* Added loading indicator to create rollout UI.

Signed-off-by: Dominik Herbst <dominik.herbst@bosch-si.com>

* Avoid parallel validation for a user session.

Signed-off-by: Dominik Herbst <dominik.herbst@bosch-si.com>

* Fix issues after merge of master into branch.

Signed-off-by: Dominik Herbst <dominik.herbst@bosch-si.com>

* Improve code quality

Signed-off-by: Dominik Herbst <dominik.herbst@bosch-si.com>

* Refreshing the total target count after validation on the rollouts creation UI.

Signed-off-by: Dominik Herbst <dominik.herbst@bosch-si.com>

* Remove unused RolloutUpdate method. Optimization of the RolloutHelper.
Comments for RolloutGroupsValidation and RSQLUtility with explanation.

Signed-off-by: Dominik Herbst <dominik.herbst@bosch-si.com>
This commit is contained in:
Dominik Herbst
2017-01-26 09:37:20 +01:00
committed by Kai Zimmermann
parent 430bf632cf
commit 833b0795e3
45 changed files with 2864 additions and 369 deletions
Download Patch File Download Diff File Expand all files Collapse all files

212
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/JpaRolloutManagement.java
View File

@@ -8,11 +8,13 @@
*/
package org.eclipse.hawkbit.repository.jpa;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
import javax.persistence.EntityNotFoundException;
@@ -20,6 +22,7 @@ import javax.validation.ConstraintDeclarationException;
import org.apache.commons.lang3.StringUtils;
import org.eclipse.hawkbit.repository.DeploymentManagement;
import org.eclipse.hawkbit.repository.DistributionSetManagement;
import org.eclipse.hawkbit.repository.RolloutFields;
import org.eclipse.hawkbit.repository.RolloutGroupManagement;
import org.eclipse.hawkbit.repository.RolloutManagement;
@@ -36,7 +39,6 @@ import org.eclipse.hawkbit.repository.jpa.executor.AfterTransactionCommitExecuto
import org.eclipse.hawkbit.repository.jpa.model.JpaAction;
import org.eclipse.hawkbit.repository.jpa.model.JpaRollout;
import org.eclipse.hawkbit.repository.jpa.model.JpaRolloutGroup;
import org.eclipse.hawkbit.repository.jpa.model.JpaRollout_;
import org.eclipse.hawkbit.repository.jpa.model.RolloutTargetGroup;
import org.eclipse.hawkbit.repository.jpa.rollout.condition.RolloutGroupActionEvaluator;
import org.eclipse.hawkbit.repository.jpa.rollout.condition.RolloutGroupConditionEvaluator;
@@ -52,6 +54,7 @@ import org.eclipse.hawkbit.repository.model.RolloutGroup.RolloutGroupErrorCondit
import org.eclipse.hawkbit.repository.model.RolloutGroup.RolloutGroupStatus;
import org.eclipse.hawkbit.repository.model.RolloutGroup.RolloutGroupSuccessCondition;
import org.eclipse.hawkbit.repository.model.RolloutGroupConditions;
import org.eclipse.hawkbit.repository.model.RolloutGroupsValidation;
import org.eclipse.hawkbit.repository.model.Target;
import org.eclipse.hawkbit.repository.model.TotalTargetCountActionStatus;
import org.eclipse.hawkbit.repository.model.TotalTargetCountStatus;
@@ -63,12 +66,13 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Slice;
import org.springframework.data.jpa.domain.Specification;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.AsyncResult;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.TransactionException;
@@ -78,6 +82,7 @@ import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.DefaultTransactionDefinition;
import org.springframework.transaction.support.TransactionCallback;
import org.springframework.transaction.support.TransactionTemplate;
import org.springframework.util.concurrent.ListenableFuture;
import org.springframework.validation.annotation.Validated;
/**
@@ -89,8 +94,7 @@ public class JpaRolloutManagement implements RolloutManagement {
private static final Logger LOGGER = LoggerFactory.getLogger(RolloutManagement.class);
/**
* Maximum amount of targets that are assigned to a Rollout Group in one
* transaction.
* Max amount of targets that are handled in one transaction.
*/
private static final int TRANSACTION_TARGETS = 1000;
@@ -112,6 +116,9 @@ public class JpaRolloutManagement implements RolloutManagement {
@Autowired
private RolloutGroupManagement rolloutGroupManagement;
@Autowired
private DistributionSetManagement distributionSetManagement;
@Autowired
private ActionRepository actionRepository;
@@ -135,15 +142,7 @@ public class JpaRolloutManagement implements RolloutManagement {
@Override
public Page<Rollout> findAll(final Pageable pageable) {
return convertPage(rolloutRepository.findAll(pageable), pageable);
}
private static Page<Rollout> convertPage(final Page<JpaRollout> findAll, final Pageable pageable) {
return new PageImpl<>(Collections.unmodifiableList(findAll.getContent()), pageable, findAll.getTotalElements());
}
private static Slice<Rollout> convertPage(final Slice<JpaRollout> findAll, final Pageable pageable) {
return new PageImpl<>(Collections.unmodifiableList(findAll.getContent()), pageable, 0);
return RolloutHelper.convertPage(rolloutRepository.findAll(pageable), pageable);
}
@Override
@@ -153,7 +152,7 @@ public class JpaRolloutManagement implements RolloutManagement {
virtualPropertyReplacer);
final Page<JpaRollout> findAll = rolloutRepository.findAll(specification, pageable);
return convertPage(findAll, pageable);
return RolloutHelper.convertPage(findAll, pageable);
}
@Override
@@ -246,7 +245,8 @@ public class JpaRolloutManagement implements RolloutManagement {
.collect(Collectors.toList());
groups.forEach(RolloutHelper::verifyRolloutGroupHasConditions);
verifyRolloutGroupTargetCounts(groups, savedRollout);
RolloutHelper.verifyRemainingTargets(
calculateRemainingTargets(groups, savedRollout.getTargetFilterQuery(), savedRollout.getCreatedAt()));
// Persisting the groups
RolloutGroup lastSavedGroup = null;
@@ -401,22 +401,51 @@ public class JpaRolloutManagement implements RolloutManagement {
targets.forEach(target -> rolloutTargetGroupRepository.save(new RolloutTargetGroup(group, target)));
}
private void verifyRolloutGroupTargetCounts(final List<RolloutGroup> groups, final JpaRollout rollout) {
final String baseFilter = RolloutHelper.getTargetFilterQuery(rollout);
private long calculateRemainingTargets(final List<RolloutGroup> groups, final String targetFilter,
final Long createdAt) {
final String baseFilter = RolloutHelper.getTargetFilterQuery(targetFilter, createdAt);
final long totalTargets = targetManagement.countTargetByTargetFilterQuery(baseFilter);
if (totalTargets == 0) {
throw new ConstraintDeclarationException("Rollout target filter does not match any targets");
}
long targetCount = totalTargets;
RolloutGroupsValidation validation = validateTargetsInGroups(groups, baseFilter, totalTargets);
return totalTargets - validation.getTargetsInGroups();
}
@Override
@Async
public ListenableFuture<RolloutGroupsValidation> validateTargetsInGroups(final List<RolloutGroupCreate> groups,
final String targetFilter, final Long createdAt) {
final String baseFilter = RolloutHelper.getTargetFilterQuery(targetFilter, createdAt);
final long totalTargets = targetManagement.countTargetByTargetFilterQuery(baseFilter);
if (totalTargets == 0) {
throw new ConstraintDeclarationException("Rollout target filter does not match any targets");
}
return new AsyncResult<>(validateTargetsInGroups(
groups.stream().map(RolloutGroupCreate::build).collect(Collectors.toList()), baseFilter, totalTargets));
}
private RolloutGroupsValidation validateTargetsInGroups(final List<RolloutGroup> groups, final String baseFilter,
final long totalTargets) {
final List<Long> groupTargetCounts = new ArrayList<>(groups.size());
final Map<String, Long> targetFilterCounts = groups.stream()
.map(group -> RolloutHelper.getGroupTargetFilter(baseFilter, group)).distinct().collect(Collectors
.toMap(Function.identity(), filter -> targetManagement.countTargetByTargetFilterQuery(filter)));
long unusedTargetsCount = 0;
for (int i = 0; i < groups.size(); i++) {
final RolloutGroup group = groups.get(i);
String groupTargetFilter = RolloutHelper.getGroupTargetFilter(baseFilter, group);
RolloutHelper.verifyRolloutGroupTargetPercentage(group.getTargetPercentage());
final long targetsInGroupFilter = countTargetsOfGroup(baseFilter, totalTargets, group);
final long overlappingTargets = countOverlappingTargetsWithPreviousGroups(baseFilter, groups, group, i);
final long targetsInGroupFilter = targetFilterCounts.get(groupTargetFilter);
final long overlappingTargets = countOverlappingTargetsWithPreviousGroups(baseFilter, groups, group, i,
targetFilterCounts);
final long realTargetsInGroup;
// Assume that targets which were not used in the previous groups
@@ -430,37 +459,31 @@ public class JpaRolloutManagement implements RolloutManagement {
final long reducedTargetsInGroup = Math
.round(group.getTargetPercentage() / 100 * (double) realTargetsInGroup);
targetCount -= reducedTargetsInGroup;
groupTargetCounts.add(reducedTargetsInGroup);
unusedTargetsCount += realTargetsInGroup - reducedTargetsInGroup;
}
RolloutHelper.verifyRemainingTargets(targetCount);
}
private long countTargetsOfGroup(final String baseFilter, final long baseFilterCount, final RolloutGroup group) {
if (StringUtils.isEmpty(group.getTargetFilterQuery())) {
return baseFilterCount;
} else {
return targetManagement.countTargetByTargetFilterQuery(baseFilter + ";" + group.getTargetFilterQuery());
}
return new RolloutGroupsValidation(totalTargets, groupTargetCounts);
}
private long countOverlappingTargetsWithPreviousGroups(final String baseFilter, final List<RolloutGroup> groups,
final RolloutGroup group, final int groupIndex) {
final RolloutGroup group, final int groupIndex, final Map<String, Long> targetFilterCounts) {
// there can't be overlapping targets in the first group
if (groupIndex == 0) {
return 0;
}
final List<RolloutGroup> previousGroups = groups.subList(0, groupIndex);
String overlappingTargetsFilter = RolloutHelper.getOverlappingWithGroupsTargetFilter(previousGroups, group);
if (StringUtils.isEmpty(overlappingTargetsFilter)) {
overlappingTargetsFilter = baseFilter;
String overlappingTargetsFilter = RolloutHelper.getOverlappingWithGroupsTargetFilter(baseFilter, previousGroups,
group);
if (targetFilterCounts.containsKey(overlappingTargetsFilter)) {
return targetFilterCounts.get(overlappingTargetsFilter);
} else {
overlappingTargetsFilter = baseFilter + ";" + overlappingTargetsFilter;
final long overlappingTargets = targetManagement.countTargetByTargetFilterQuery(overlappingTargetsFilter);
targetFilterCounts.put(overlappingTargetsFilter, overlappingTargets);
return overlappingTargets;
}
return targetManagement.countTargetByTargetFilterQuery(overlappingTargetsFilter);
}
@Override
@@ -469,8 +492,9 @@ public class JpaRolloutManagement implements RolloutManagement {
public Rollout startRollout(final Long rolloutId) {
final JpaRollout rollout = Optional.ofNullable(rolloutRepository.findOne(rolloutId))
.orElseThrow(() -> new EntityNotFoundException("Rollout with id " + rolloutId + " not found."));
checkIfRolloutCanStarted(rollout, rollout);
RolloutHelper.checkIfRolloutCanStarted(rollout, rollout);
rollout.setStatus(RolloutStatus.STARTING);
rollout.setLastCheck(0);
return rolloutRepository.save(rollout);
}
@@ -649,19 +673,11 @@ public class JpaRolloutManagement implements RolloutManagement {
@Transactional(propagation = Propagation.REQUIRES_NEW, isolation = Isolation.READ_UNCOMMITTED)
@Modifying
public void checkRunningRollouts(final long delayBetweenChecks) {
final long lastCheck = System.currentTimeMillis();
final int updated = rolloutRepository.updateLastCheck(lastCheck, delayBetweenChecks, RolloutStatus.RUNNING);
if (updated == 0) {
// nothing to check, maybe another instance already checked in
// between
LOGGER.debug("No rolloutcheck necessary for current scheduled check {}, next check at {}", lastCheck,
lastCheck + delayBetweenChecks);
final List<JpaRollout> rolloutsToCheck = getRolloutsToCheckForStatus(delayBetweenChecks, RolloutStatus.RUNNING);
if (rolloutsToCheck.isEmpty()) {
return;
}
final List<JpaRollout> rolloutsToCheck = rolloutRepository.findByLastCheckAndStatus(lastCheck,
RolloutStatus.RUNNING);
LOGGER.info("Found {} running rollouts to check", rolloutsToCheck.size());
for (final JpaRollout rollout : rolloutsToCheck) {
@@ -721,10 +737,9 @@ public class JpaRolloutManagement implements RolloutManagement {
final long updatedTargetCount = jpaRollout.getTotalTargets()
- (rolloutGroup.getTotalTargets() - countTargetsOfRolloutGroup);
jpaRollout.setTotalTargets(updatedTargetCount);
final JpaRolloutGroup jpaRolloutGroup = rolloutGroup;
jpaRolloutGroup.setTotalTargets((int) countTargetsOfRolloutGroup);
rolloutGroup.setTotalTargets((int) countTargetsOfRolloutGroup);
rolloutRepository.save(jpaRollout);
rolloutGroupRepository.save(jpaRolloutGroup);
rolloutGroupRepository.save(rolloutGroup);
}
private long countTargetsFrom(final JpaRolloutGroup rolloutGroup) {
@@ -811,18 +826,12 @@ public class JpaRolloutManagement implements RolloutManagement {
@Transactional(propagation = Propagation.REQUIRES_NEW, isolation = Isolation.READ_UNCOMMITTED)
@Modifying
public void checkCreatingRollouts(final long delayBetweenChecks) {
final long lastCheck = System.currentTimeMillis();
final int updated = rolloutRepository.updateLastCheck(lastCheck, delayBetweenChecks, RolloutStatus.CREATING);
if (updated == 0) {
// nothing to check, maybe another instance already checked in
// between
LOGGER.debug("No rollouts creating check necessary for current scheduled check {}, next check at {}",
lastCheck, lastCheck + delayBetweenChecks);
final List<Long> rolloutsToCheck = getRolloutsToCheckForStatus(delayBetweenChecks, RolloutStatus.CREATING)
.stream().map(Rollout::getId).collect(Collectors.toList());
if (rolloutsToCheck.isEmpty()) {
return;
}
final List<Long> rolloutsToCheck = rolloutRepository.findByLastCheckAndStatus(lastCheck, RolloutStatus.CREATING)
.stream().map(Rollout::getId).collect(Collectors.toList());
LOGGER.info("Found {} creating rollouts to check", rolloutsToCheck.size());
rolloutsToCheck.forEach(this::fillRolloutGroupsWithTargets);
@@ -833,18 +842,12 @@ public class JpaRolloutManagement implements RolloutManagement {
@Transactional(propagation = Propagation.REQUIRES_NEW, isolation = Isolation.READ_UNCOMMITTED)
@Modifying
public void checkStartingRollouts(final long delayBetweenChecks) {
final long lastCheck = System.currentTimeMillis();
final int updated = rolloutRepository.updateLastCheck(lastCheck, delayBetweenChecks, RolloutStatus.STARTING);
if (updated == 0) {
// nothing to check, maybe another instance already checked in
// between
LOGGER.debug("No rollouts starting check necessary for current scheduled check {}, next check at {}",
lastCheck, lastCheck + delayBetweenChecks);
final List<JpaRollout> rolloutsToCheck = getRolloutsToCheckForStatus(delayBetweenChecks,
RolloutStatus.STARTING);
if (rolloutsToCheck.isEmpty()) {
return;
}
final List<JpaRollout> rolloutsToCheck = rolloutRepository.findByLastCheckAndStatus(lastCheck,
RolloutStatus.STARTING);
LOGGER.info("Found {} starting rollouts to check", rolloutsToCheck.size());
rolloutsToCheck.forEach(rollout -> {
@@ -855,6 +858,42 @@ public class JpaRolloutManagement implements RolloutManagement {
}
@Override
@Transactional(propagation = Propagation.REQUIRES_NEW, isolation = Isolation.READ_UNCOMMITTED)
@Modifying
public void checkReadyRollouts(final long delayBetweenChecks) {
final List<JpaRollout> rolloutsToCheck = getRolloutsToCheckForStatus(delayBetweenChecks, RolloutStatus.READY);
if (rolloutsToCheck.isEmpty()) {
return;
}
LOGGER.info("Found {} ready rollouts to check", rolloutsToCheck.size());
final long now = System.currentTimeMillis();
rolloutsToCheck.forEach(rollout -> {
if (rollout.getStartAt() != null && rollout.getStartAt() <= now) {
startRollout(rollout.getId());
}
});
}
private List<JpaRollout> getRolloutsToCheckForStatus(final long delayBetweenChecks, final RolloutStatus status) {
final long lastCheck = System.currentTimeMillis();
final int updated = rolloutRepository.updateLastCheck(lastCheck, delayBetweenChecks, status);
if (updated == 0) {
// nothing to check, maybe another instance already checked in
// between
LOGGER.debug("No rollouts starting check necessary for current scheduled check {}, next check at {}",
lastCheck, lastCheck + delayBetweenChecks);
return Collections.emptyList();
}
return rolloutRepository.findByLastCheckAndStatus(lastCheck, status);
}
@Override
public Long countRolloutsAll() {
return rolloutRepository.count();
@@ -862,25 +901,15 @@ public class JpaRolloutManagement implements RolloutManagement {
@Override
public Long countRolloutsAllByFilters(final String searchText) {
return rolloutRepository.count(likeNameOrDescription(searchText));
}
private static Specification<JpaRollout> likeNameOrDescription(final String searchText) {
return (rolloutRoot, query, criteriaBuilder) -> {
final String searchTextToLower = searchText.toLowerCase();
return criteriaBuilder.or(
criteriaBuilder.like(criteriaBuilder.lower(rolloutRoot.get(JpaRollout_.name)), searchTextToLower),
criteriaBuilder.like(criteriaBuilder.lower(rolloutRoot.get(JpaRollout_.description)),
searchTextToLower));
};
return rolloutRepository.count(RolloutHelper.likeNameOrDescription(searchText));
}
@Override
public Slice<Rollout> findRolloutWithDetailedStatusByFilters(final Pageable pageable, final String searchText) {
final Specification<JpaRollout> specs = likeNameOrDescription(searchText);
final Specification<JpaRollout> specs = RolloutHelper.likeNameOrDescription(searchText);
final Slice<JpaRollout> findAll = criteriaNoCountDao.findAll(specs, pageable, JpaRollout.class);
setRolloutStatusDetails(findAll);
return convertPage(findAll, pageable);
return RolloutHelper.convertPage(findAll, pageable);
}
@Override
@@ -898,6 +927,16 @@ public class JpaRolloutManagement implements RolloutManagement {
update.getName().ifPresent(rollout::setName);
update.getDescription().ifPresent(rollout::setDescription);
update.getActionType().ifPresent(rollout::setActionType);
update.getForcedTime().ifPresent(rollout::setForcedTime);
update.getStartAt().ifPresent(rollout::setStartAt);
update.getSet().ifPresent(setId -> {
final DistributionSet set = distributionSetManagement.findDistributionSetById(setId);
if (set == null) {
throw new EntityNotFoundException("Distribution set cannot be set as it does not exists" + setId);
}
rollout.setDistributionSet(set);
});
return rolloutRepository.save(rollout);
}
@@ -906,7 +945,7 @@ public class JpaRolloutManagement implements RolloutManagement {
public Page<Rollout> findAllRolloutsWithDetailedStatus(final Pageable pageable) {
final Page<JpaRollout> rollouts = rolloutRepository.findAll(pageable);
setRolloutStatusDetails(rollouts);
return convertPage(rollouts, pageable);
return RolloutHelper.convertPage(rollouts, pageable);
}
@@ -939,13 +978,6 @@ public class JpaRolloutManagement implements RolloutManagement {
}
}
private static void checkIfRolloutCanStarted(final Rollout rollout, final Rollout mergedRollout) {
if (!(RolloutStatus.READY.equals(mergedRollout.getStatus()))) {
throw new RolloutIllegalStateException("Rollout can only be started in state ready but current state is "
+ rollout.getStatus().name().toLowerCase());
}
}
@Override
public float getFinishedPercentForRunningGroup(final Long rolloutId, final Long rolloutGroupId) {
final RolloutGroup rolloutGroup = Optional.ofNullable(rolloutGroupRepository.findOne(rolloutGroupId))

9
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/JpaTargetFilterQueryManagement.java
View File

@@ -123,6 +123,15 @@ public class JpaTargetFilterQueryManagement implements TargetFilterQueryManageme
return convertPage(findTargetFilterQueryByCriteriaAPI(pageable, specList), pageable);
}
@Override
public Page<TargetFilterQuery> findTargetFilterQueryByQuery(final Pageable pageable, final String query) {
List<Specification<JpaTargetFilterQuery>> specList = Collections.emptyList();
if (!Strings.isNullOrEmpty(query)) {
specList = Collections.singletonList(TargetFilterQuerySpecification.equalsQuery(query));
}
return convertPage(findTargetFilterQueryByCriteriaAPI(pageable, specList), pageable);
}
@Override
public Page<TargetFilterQuery> findTargetFilterQueryByAutoAssignDS(final Pageable pageable, final Long setId,
final String rsqlFilter) {

5
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/JpaTargetManagement.java
View File

@@ -680,7 +680,10 @@ public class JpaTargetManagement implements TargetManagement {
public Long countTargetByTargetFilterQuery(final String targetFilterQuery) {
final Specification<JpaTarget> specs = RSQLUtility.parse(targetFilterQuery, TargetFields.class,
virtualPropertyReplacer);
return targetRepository.count(specs);
return targetRepository.count((root, query, cb) -> {
query.distinct(true);
return specs.toPredicate(root, query, cb);
});
}
private List<Object[]> getTargetIdNameResultSet(final Pageable pageRequest, final CriteriaBuilder cb,

119
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/RolloutHelper.java
View File

@@ -8,6 +8,8 @@
*/
package org.eclipse.hawkbit.repository.jpa;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
@@ -16,10 +18,17 @@ import org.eclipse.hawkbit.repository.builder.RolloutGroupCreate;
import org.eclipse.hawkbit.repository.exception.ConstraintViolationException;
import org.eclipse.hawkbit.repository.exception.RolloutIllegalStateException;
import org.eclipse.hawkbit.repository.jpa.builder.JpaRolloutGroupCreate;
import org.eclipse.hawkbit.repository.jpa.model.JpaRollout;
import org.eclipse.hawkbit.repository.jpa.model.JpaRolloutGroup;
import org.eclipse.hawkbit.repository.jpa.model.JpaRollout_;
import org.eclipse.hawkbit.repository.model.Rollout;
import org.eclipse.hawkbit.repository.model.RolloutGroup;
import org.eclipse.hawkbit.repository.model.RolloutGroupConditions;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Slice;
import org.springframework.data.jpa.domain.Specification;
/**
* A collection of static helper methods for the {@link JpaRolloutManagement}
@@ -69,7 +78,7 @@ final class RolloutHelper {
* In case the given group is missing conditions or actions, they will be
* set from the supplied default conditions.
*
* @param group
* @param create
* group to check
* @param conditions
* default conditions and actions
@@ -144,10 +153,22 @@ final class RolloutHelper {
* @return resulting target filter query
*/
static String getTargetFilterQuery(final Rollout rollout) {
if (rollout.getCreatedAt() != null) {
return rollout.getTargetFilterQuery() + ";createdat=le=" + rollout.getCreatedAt().toString();
return getTargetFilterQuery(rollout.getTargetFilterQuery(), rollout.getCreatedAt());
}
/**
* @param targetFilter
* the target filter tp be extended
* @param createdAt
* timestamp
* @return a target filter query that only matches targets that were created
* after the provided timestamp.
*/
static String getTargetFilterQuery(final String targetFilter, final Long createdAt) {
if (createdAt != null) {
return targetFilter + ";createdat=le=" + createdAt.toString();
}
return rollout.getTargetFilterQuery();
return targetFilter;
}
/**
@@ -215,30 +236,69 @@ final class RolloutHelper {
if (groups.stream().anyMatch(group -> StringUtils.isEmpty(group.getTargetFilterQuery()))) {
return "";
}
return groups.stream().map(RolloutGroup::getTargetFilterQuery).collect(Collectors.joining(","));
return "(" + groups.stream().map(RolloutGroup::getTargetFilterQuery).distinct().sorted()
.collect(Collectors.joining("),(")) + ")";
}
/**
* Creates an RSQL Filter that matches all targets that are in the provided
* group and in the provided groups.
*
* @param baseFilter
* the base filter from the rollout
* @param groups
* the rollout groups
* @param group
* the group
* the target group
* @return RSQL string without base filter of the Rollout. Can be an empty
* string.
*/
static String getOverlappingWithGroupsTargetFilter(final List<RolloutGroup> groups, final RolloutGroup group) {
static String getOverlappingWithGroupsTargetFilter(final String baseFilter, final List<RolloutGroup> groups,
final RolloutGroup group) {
final String groupFilter = group.getTargetFilterQuery();
// when any previous group has the same filter as the target group the
// overlap is 100%
if (isTargetFilterInGroups(groupFilter, groups)) {
return concatAndTargetFilters(baseFilter, groupFilter);
}
final String previousGroupFilters = getAllGroupsTargetFilter(groups);
if (StringUtils.isNotEmpty(previousGroupFilters) && StringUtils.isNotEmpty(group.getTargetFilterQuery())) {
return group.getTargetFilterQuery() + ";(" + previousGroupFilters + ")";
} else if (StringUtils.isNotEmpty(previousGroupFilters)) {
return "(" + previousGroupFilters + ")";
} else if (StringUtils.isNotEmpty(group.getTargetFilterQuery())) {
return group.getTargetFilterQuery();
if (StringUtils.isNotEmpty(previousGroupFilters)) {
if (StringUtils.isNotEmpty(groupFilter)) {
return concatAndTargetFilters(baseFilter, groupFilter, previousGroupFilters);
} else {
return concatAndTargetFilters(baseFilter, previousGroupFilters);
}
}
if (StringUtils.isNotEmpty(groupFilter)) {
return concatAndTargetFilters(baseFilter, groupFilter);
} else {
return "";
return baseFilter;
}
}
private static boolean isTargetFilterInGroups(final String groupFilter, final List<RolloutGroup> groups) {
return StringUtils.isNotEmpty(groupFilter)
&& groups.stream().anyMatch(prevGroup -> StringUtils.isNotEmpty(prevGroup.getTargetFilterQuery())
&& prevGroup.getTargetFilterQuery().equals(groupFilter));
}
private static String concatAndTargetFilters(String... filters) {
return "(" + Arrays.stream(filters).collect(Collectors.joining(");(")) + ")";
}
/**
* @param baseFilter
* the base filter from the rollout
* @param group
* group for which the filter string should be created
* @return the final target filter query for a rollout group
*/
static String getGroupTargetFilter(final String baseFilter, final RolloutGroup group) {
if (StringUtils.isEmpty(group.getTargetFilterQuery())) {
return baseFilter;
} else {
return concatAndTargetFilters(baseFilter, group.getTargetFilterQuery());
}
}
@@ -258,4 +318,35 @@ final class RolloutHelper {
}
}
/**
* @param searchText
* search string
* @return criteria specification with a query for name or description of a
* rollout
*/
static Specification<JpaRollout> likeNameOrDescription(final String searchText) {
return (rolloutRoot, query, criteriaBuilder) -> {
final String searchTextToLower = searchText.toLowerCase();
return criteriaBuilder.or(
criteriaBuilder.like(criteriaBuilder.lower(rolloutRoot.get(JpaRollout_.name)), searchTextToLower),
criteriaBuilder.like(criteriaBuilder.lower(rolloutRoot.get(JpaRollout_.description)),
searchTextToLower));
};
}
static void checkIfRolloutCanStarted(final Rollout rollout, final Rollout mergedRollout) {
if (!(Rollout.RolloutStatus.READY.equals(mergedRollout.getStatus()))) {
throw new RolloutIllegalStateException("Rollout can only be started in state ready but current state is "
+ rollout.getStatus().name().toLowerCase());
}
}
static Page<Rollout> convertPage(final Page<JpaRollout> findAll, final Pageable pageable) {
return new PageImpl<>(Collections.unmodifiableList(findAll.getContent()), pageable, findAll.getTotalElements());
}
static Slice<Rollout> convertPage(final Slice<JpaRollout> findAll, final Pageable pageable) {
return new PageImpl<>(Collections.unmodifiableList(findAll.getContent()), pageable, 0);
}
}

1
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/builder/JpaRolloutCreate.java
View File

@@ -30,6 +30,7 @@ public class JpaRolloutCreate extends AbstractRolloutUpdateCreate<RolloutCreate>
rollout.setDescription(description);
rollout.setDistributionSet(findDistributionSetAndThrowExceptionIfNotFound(set));
rollout.setTargetFilterQuery(targetFilterQuery);
rollout.setStartAt(startAt);
if (actionType != null) {
rollout.setActionType(actionType);

12
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/model/JpaRollout.java
View File

@@ -87,6 +87,9 @@ public class JpaRollout extends AbstractJpaNamedEntity implements Rollout, Event
@Column(name = "rollout_groups_created")
private int rolloutGroupsCreated;
@Column(name = "start_at")
private Long startAt;
@Transient
private transient TotalTargetCountStatus totalTargetCountStatus;
@@ -134,6 +137,15 @@ public class JpaRollout extends AbstractJpaNamedEntity implements Rollout, Event
this.lastCheck = lastCheck;
}
@Override
public Long getStartAt() {
return startAt;
}
public void setStartAt(Long startAt) {
this.startAt = startAt;
}
@Override
public ActionType getActionType() {
return actionType;

39
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/rollout/RolloutScheduler.java
View File

@@ -25,8 +25,7 @@ import org.springframework.stereotype.Component;
/**
* Scheduler to schedule the
* {@link RolloutManagement#checkRunningRollouts(long)}. The delay between the
* checks be be configured using the property
* {@link #PROP_SCHEDULER_DELAY_PLACEHOLDER}.
* checks be be configured using the properties from {@link RolloutProperties}.
*/
@Component
// don't active the rollout scheduler in test, otherwise it is hard to test
@@ -155,4 +154,40 @@ public class RolloutScheduler {
return null;
});
}
/**
* Scheduler method called by the spring-async mechanism. Retrieves all
* tenants from the {@link SystemManagement#findTenants()} and runs for each
* tenant the {@link RolloutManagement#checkReadyRollouts(long)} in the
* {@link SystemSecurityContext}. Used to auto start Rollouts as soon as
* their startAt time is reached.
*/
@Scheduled(initialDelayString = RolloutProperties.PROP_READY_SCHEDULER_DELAY_PLACEHOLDER, fixedDelayString = RolloutProperties.PROP_READY_SCHEDULER_DELAY_PLACEHOLDER)
public void readyRolloutScheduler() {
if (!rolloutProperties.getReadyScheduler().isEnabled()) {
return;
}
LOGGER.debug("rollout ready schedule checker has been triggered.");
// run this code in system code privileged to have the necessary
// permission to query and create entities.
systemSecurityContext.runAsSystem(() -> {
// workaround eclipselink that is currently not possible to
// execute a query without multitenancy if MultiTenant
// annotation is used.
// https://bugs.eclipse.org/bugs/show_bug.cgi?id=355458. So
// iterate through all tenants and execute the rollout check for
// each tenant seperately.
final List<String> tenants = systemManagement.findTenants();
LOGGER.info("Checking ready rollouts for {} tenants", tenants.size());
for (final String tenant : tenants) {
tenantAware.runAsTenant(tenant, () -> {
final long fixedDelay = rolloutProperties.getReadyScheduler().getFixedDelay();
rolloutManagement.checkReadyRollouts(fixedDelay);
return null;
});
}
return null;
});
}
}

83
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/rsql/RSQLUtility.java
View File

@@ -13,15 +13,23 @@ import static org.eclipse.hawkbit.repository.FieldNameProvider.SUB_ATTRIBUTE_SEP
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Expression;
import javax.persistence.criteria.From;
import javax.persistence.criteria.Join;
import javax.persistence.criteria.JoinType;
import javax.persistence.criteria.MapJoin;
import javax.persistence.criteria.Path;
import javax.persistence.criteria.PluralJoin;
import javax.persistence.criteria.Predicate;
import javax.persistence.criteria.Root;
@@ -157,7 +165,6 @@ public final class RSQLUtility {
@Override
public Predicate toPredicate(final Root<T> root, final CriteriaQuery<?> query, final CriteriaBuilder cb) {
final Node rootNode = parseRsql(rsql);
final JpqQueryRSQLVisitor<A, T> jpqQueryRSQLVisitor = new JpqQueryRSQLVisitor<>(root, cb, enumType,
@@ -192,6 +199,9 @@ public final class RSQLUtility {
private final CriteriaBuilder cb;
private final Class<A> enumType;
private final VirtualPropertyReplacer virtualPropertyReplacer;
private int level;
private boolean isOrLevel;
private final Map<Integer, Set<Join<Object, Object>>> joinsInLevel = new HashMap<>(3);
private final SimpleTypeConverter simpleTypeConverter;
@@ -204,9 +214,41 @@ public final class RSQLUtility {
simpleTypeConverter = new SimpleTypeConverter();
}
private void beginLevel(boolean isOr) {
level++;
isOrLevel = isOr;
joinsInLevel.put(level, new HashSet<>(2));
}
private void endLevel() {
joinsInLevel.remove(level);
level--;
isOrLevel = false;
}
private Set<Join<Object, Object>> getCurrentJoins() {
if(level > 0) {
return joinsInLevel.get(level);
}
return Collections.emptySet();
}
private Optional<Join<Object, Object>> findCurrentJoinOfType(final Class<?> type) {
return getCurrentJoins().stream()
.filter(j -> type.equals(j.getJavaType())).findFirst();
}
private void addCurrentJoin(Join<Object, Object> join) {
if(level > 0) {
getCurrentJoins().add(join);
}
}
@Override
public List<Predicate> visit(final AndNode node, final String param) {
beginLevel(false);
final List<Predicate> childs = acceptChilds(node);
endLevel();
if (!childs.isEmpty()) {
return toSingleList(cb.and(childs.toArray(new Predicate[childs.size()])));
}
@@ -215,7 +257,9 @@ public final class RSQLUtility {
@Override
public List<Predicate> visit(final OrNode node, final String param) {
beginLevel(true);
final List<Predicate> childs = acceptChilds(node);
endLevel();
if (!childs.isEmpty()) {
return toSingleList(cb.or(childs.toArray(new Predicate[childs.size()])));
}
@@ -282,12 +326,28 @@ public final class RSQLUtility {
new Exception());
}
/**
* Resolves the Path for a field in the persistence layer and joins the
* required models. This operation is part of a tree traversal through
* an RSQL expression. It creates for every field that is not part of
* the root model a join to the foreign model. This behavior is
* optimized when several joins happen directly under an OR node in the
* traversed tree. The same foreign model is only joined once.
*
* Example: tags.name==M;(tags.name==A,tags.name==B,tags.name==C) This
* example joins the tags model only twice, because for the OR node in
* brackets only one join is used.
*
* @param enumField
* field from a FieldNameProvider to resolve on the
* persistence layer
* @param finalProperty
* dot notated field path
* @return the Path for a field
*/
private Path<Object> getFieldPath(final A enumField, final String finalProperty) {
Path<Object> fieldPath = null;
final String[] split = finalProperty.split("\\" + SUB_ATTRIBUTE_SEPERATOR);
if (split.length == 0) {
return root.get(split[0]);
}
for (int i = 0; i < split.length; i++) {
final boolean isMapKeyField = enumField.isMap() && i == (split.length - 1);
@@ -297,6 +357,21 @@ public final class RSQLUtility {
final String fieldNameSplit = split[i];
fieldPath = (fieldPath != null) ? fieldPath.get(fieldNameSplit) : root.get(fieldNameSplit);
if (fieldPath instanceof PluralJoin) {
final Join<Object, ?> join = (Join<Object, ?>) fieldPath;
final From<?, Object> joinParent = join.getParent();
Optional<Join<Object, Object>> currentJoinOfType = findCurrentJoinOfType(join.getJavaType());
if(currentJoinOfType.isPresent() && isOrLevel) {
// remove the additional join and use the existing one
joinParent.getJoins().remove(join);
fieldPath = currentJoinOfType.get();
} else {
Join<Object, Object> newJoin = joinParent.join(fieldNameSplit, JoinType.LEFT);
addCurrentJoin(newJoin);
fieldPath = newJoin;
}
}
}
return fieldPath;
}

13
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/specifications/TargetFilterQuerySpecification.java
View File

@@ -24,6 +24,19 @@ public final class TargetFilterQuerySpecification {
// utility class
}
/**
* {@link Specification} for retrieving {@link JpaTargetFilterQuery}s based
* on is {@link JpaTargetFilterQuery#getQuery()}.
*
* @param queryValue
* the query of the filter
* @return the {@link JpaTargetFilterQuery} {@link Specification}
*/
public static Specification<JpaTargetFilterQuery> equalsQuery(final String queryValue) {
return (targetFilterQueryRoot, query, cb) -> cb.equal(targetFilterQueryRoot.get(JpaTargetFilterQuery_.query),
queryValue);
}
/**
* {@link Specification} for retrieving {@link JpaTargetFilterQuery}s based
* on is {@link JpaTargetFilterQuery#getName()}.

2
hawkbit-repository/hawkbit-repository-jpa/src/main/resources/db/migration/H2/V1_10_1__rollout_auto_start__H2.sql Normal file
View File

@@ -0,0 +1,2 @@
ALTER TABLE sp_rollout
ADD COLUMN start_at BIGINT;

2
hawkbit-repository/hawkbit-repository-jpa/src/main/resources/db/migration/MYSQL/V1_10_1__rollout_auto_start__MYSQL.sql Normal file
View File

@@ -0,0 +1,2 @@
ALTER TABLE sp_rollout
ADD COLUMN start_at BIGINT;

57
hawkbit-repository/hawkbit-repository-jpa/src/test/java/org/eclipse/hawkbit/repository/jpa/RolloutManagementTest.java
View File

@@ -1096,6 +1096,12 @@ public class RolloutManagementTest extends AbstractJpaIntegrationTest {
"controllerId==" + targetPrefixName + "-*", distributionSet, successCondition, errorCondition);
assertThat(myRollout.getStatus()).isEqualTo(RolloutStatus.READY);
rolloutManagement.checkReadyRollouts(0);
myRollout = rolloutManagement.findRolloutById(myRollout.getId());
assertThat(myRollout.getStatus()).isEqualTo(RolloutStatus.READY);
rolloutManagement.startRollout(myRollout.getId());
// Run here, because scheduler is disabled during tests
@@ -1114,6 +1120,57 @@ public class RolloutManagementTest extends AbstractJpaIntegrationTest {
validateRolloutActionStatus(myRollout.getId(), expectedTargetCountStatus);
}
@Test
@Description("Verify the creation and the automatic start of a rollout.")
public void createAndAutoStartRollout() throws Exception {
final int amountTargetsForRollout = 500;
final int amountGroups = 5;
final String successCondition = "50";
final String errorCondition = "80";
final String rolloutName = "rolloutTest8";
final String targetPrefixName = rolloutName;
final DistributionSet distributionSet = testdataFactory.createDistributionSet("dsFor" + rolloutName);
testdataFactory.createTargets(amountTargetsForRollout, targetPrefixName + "-", targetPrefixName);
Rollout myRollout = createRolloutByVariables(rolloutName, "desc", amountGroups,
"controllerId==" + targetPrefixName + "-*", distributionSet, successCondition, errorCondition);
assertThat(myRollout.getStatus()).isEqualTo(RolloutStatus.READY);
// schedule rollout auto start into the future
rolloutManagement.updateRollout(
entityFactory.rollout().update(myRollout.getId()).startAt(System.currentTimeMillis() + 60000));
rolloutManagement.checkReadyRollouts(0);
// rollout should not have been started
myRollout = rolloutManagement.findRolloutById(myRollout.getId());
assertThat(myRollout.getStatus()).isEqualTo(RolloutStatus.READY);
// schedule to now
rolloutManagement.updateRollout(
entityFactory.rollout().update(myRollout.getId()).startAt(System.currentTimeMillis()));
rolloutManagement.checkReadyRollouts(0);
myRollout = rolloutManagement.findRolloutById(myRollout.getId());
assertThat(myRollout.getStatus()).isEqualTo(RolloutStatus.STARTING);
// Run here, because scheduler is disabled during tests
rolloutManagement.checkStartingRollouts(0);
final SuccessConditionRolloutStatus conditionRolloutTargetCount = new SuccessConditionRolloutStatus(
RolloutStatus.RUNNING);
assertThat(MultipleInvokeHelper.doWithTimeout(new RolloutStatusCallable(myRollout.getId()),
conditionRolloutTargetCount, 15000, 500)).as("Rollout status").isNotNull();
myRollout = rolloutManagement.findRolloutById(myRollout.getId());
assertThat(myRollout.getStatus()).isEqualTo(RolloutStatus.RUNNING);
final Map<TotalTargetCountStatus.Status, Long> expectedTargetCountStatus = createInitStatusMap();
expectedTargetCountStatus.put(TotalTargetCountStatus.Status.RUNNING, 100L);
expectedTargetCountStatus.put(TotalTargetCountStatus.Status.SCHEDULED, 400L);
validateRolloutActionStatus(myRollout.getId(), expectedTargetCountStatus);
}
@Test
@Description("Verify the creation of a Rollout with a groups definition.")
public void createRolloutWithGroupDefinition() throws Exception {

29
hawkbit-repository/hawkbit-repository-jpa/src/test/java/org/eclipse/hawkbit/repository/jpa/TargetManagementTest.java
View File

@@ -21,7 +21,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import javax.validation.ConstraintViolationException;
@@ -49,6 +48,7 @@ import org.eclipse.hawkbit.repository.test.matcher.ExpectEvents;
import org.eclipse.hawkbit.repository.test.util.WithSpringAuthorityRule;
import org.eclipse.hawkbit.repository.test.util.WithUser;
import org.junit.Test;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import com.google.common.collect.Iterables;
@@ -495,7 +495,7 @@ public class TargetManagementTest extends AbstractJpaIntegrationTest {
firstList = firstList.stream()
.map(t -> targetManagement.updateTarget(
entityFactory.target().update(t.getControllerId()).name(t.getName().concat("\tchanged"))))
.collect(Collectors.toList());
.collect(toList());
// verify that all entries are found
_founds: for (final Target foundTarget : allFound) {
@@ -706,14 +706,14 @@ public class TargetManagementTest extends AbstractJpaIntegrationTest {
toggleTagAssignment(targAs, targTagA);
assertThat(targetManagement.findTargetsByControllerIDsWithTags(
targAs.stream().map(Target::getControllerId).collect(Collectors.toList()))).as("Target count is wrong")
targAs.stream().map(Target::getControllerId).collect(toList()))).as("Target count is wrong")
.hasSize(25);
// no lazy loading exception and tag correctly assigned
assertThat(targetManagement
.findTargetsByControllerIDsWithTags(
targAs.stream().map(Target::getControllerId).collect(Collectors.toList()))
.stream().map(target -> target.getTags().contains(targTagA)).collect(Collectors.toList()))
targAs.stream().map(Target::getControllerId).collect(toList()))
.stream().map(target -> target.getTags().contains(targTagA)).collect(toList()))
.as("Tags not correctly assigned").containsOnly(true);
}
@@ -755,4 +755,23 @@ public class TargetManagementTest extends AbstractJpaIntegrationTest {
});
}
@Test
@Description("Test that RSQL filter finds targets with tags or specific ids.")
public void findTargetsWithTagOrId() {
final String rsqlFilter = "tag==Targ-A-Tag,id==target-id-B-00001,id==target-id-B-00008";
final TargetTag targTagA = tagManagement.createTargetTag(entityFactory.tag().create().name("Targ-A-Tag"));
final List<String> targAs = testdataFactory.createTargets(25, "target-id-A", "first description").stream()
.map(Target::getControllerId).collect(toList());
targetManagement.toggleTagAssignment(targAs, targTagA.getName());
testdataFactory.createTargets(25, "target-id-B", "first description");
Page<Target> foundTargets = targetManagement.findTargetsAll(rsqlFilter, new PageRequest(0, 100));
assertThat(targetManagement.findTargetsAll(new PageRequest(0, 100)).getNumberOfElements()).as("Total targets")
.isEqualTo(50);
assertThat(foundTargets.getTotalElements()).as("Targets in RSQL filter").isEqualTo(27L);
}
}