lingwei.kong/hawkbit
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Target poll DB performance improvements (#613)

Browse Source 
* Async update on last target query.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* 10 seconds

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Completed batch and optimzed schema.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Cleanup and test.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* readibility.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Batch update.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Sonar issue fixed.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>
This commit is contained in:
Kai Zimmermann
2017-12-18 13:50:25 +01:00
committed by GitHub
parent 80d9f1b8fc
commit 7d0bf3a162
22 changed files with 513 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
hawkbit-repository/hawkbit-repository-api/src/main/java/org/eclipse/hawkbit/repository/RepositoryProperties.java
View File

@@ -8,6 +8,8 @@
*/
package org.eclipse.hawkbit.repository;
import java.util.concurrent.TimeUnit;
import org.eclipse.hawkbit.repository.event.remote.TargetPollEvent;
import org.eclipse.hawkbit.repository.model.ActionStatus;
import org.springframework.boot.context.properties.ConfigurationProperties;
@@ -34,6 +36,45 @@ public class RepositoryProperties {
*/
private boolean publishTargetPollEvent = true;
/**
* Maximum number of poll operations queued before flush.
*/
private int pollPersistenceQueueSize = 10_000;
/**
* Maximum time before queue is flushed in {@link TimeUnit#MILLISECONDS}.
*/
private long pollPersistenceFlushTime = TimeUnit.SECONDS.toMillis(10);
/**
* Set to true to persist polls immediately.
*/
private boolean eagerPollPersistence;
public boolean isEagerPollPersistence() {
return eagerPollPersistence;
}
public void setEagerPollPersistence(final boolean eagerPollPersistence) {
this.eagerPollPersistence = eagerPollPersistence;
}
public long getPollPersistenceFlushTime() {
return pollPersistenceFlushTime;
}
public void setPollPersistenceFlushTime(final long pollPersistenceFlushTime) {
this.pollPersistenceFlushTime = pollPersistenceFlushTime;
}
public int getPollPersistenceQueueSize() {
return pollPersistenceQueueSize;
}
public void setPollPersistenceQueueSize(final int pollPersistenceQueueSize) {
this.pollPersistenceQueueSize = pollPersistenceQueueSize;
}
public boolean isRejectActionStatusForClosedAction() {
return rejectActionStatusForClosedAction;
}

5
hawkbit-repository/hawkbit-repository-api/src/main/java/org/eclipse/hawkbit/repository/event/remote/TargetPollEvent.java
View File

@@ -26,6 +26,11 @@ public class TargetPollEvent extends RemoteTenantAwareEvent {
// for serialization libs like jackson
}
public TargetPollEvent(final String controllerId, final String tenant, final String applicationId) {
super(controllerId, tenant, applicationId);
this.controllerId = controllerId;
}
public TargetPollEvent(final Target target, final String applicationId) {
super(target.getControllerId(), target.getTenant(), applicationId);
this.controllerId = target.getControllerId();

2
hawkbit-repository/hawkbit-repository-api/src/main/java/org/eclipse/hawkbit/repository/model/ActionStatus.java
View File

@@ -25,7 +25,7 @@ public interface ActionStatus extends TenantAwareBaseEntity {
* @return time in {@link TimeUnit#MILLISECONDS} when the status was
* reported.
*/
Long getOccurredAt();
long getOccurredAt();
/**
* @return {@link Action} this {@link ActionStatus} belongs to.

224
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/JpaControllerManagement.java
View File

@@ -14,9 +14,15 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Root;
@@ -51,6 +57,7 @@ import org.eclipse.hawkbit.repository.model.SoftwareModuleMetadata;
import org.eclipse.hawkbit.repository.model.Target;
import org.eclipse.hawkbit.repository.model.TargetUpdateStatus;
import org.eclipse.hawkbit.security.SystemSecurityContext;
import org.eclipse.hawkbit.tenancy.TenantAware;
import org.eclipse.hawkbit.tenancy.configuration.TenantConfigurationProperties.TenantConfigurationKey;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,10 +73,20 @@ import org.springframework.data.domain.Sort.Direction;
import org.springframework.data.jpa.domain.Specification;
import org.springframework.retry.annotation.Backoff;
import org.springframework.retry.annotation.Retryable;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.annotation.Isolation;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.DefaultTransactionDefinition;
import org.springframework.transaction.support.TransactionCallback;
import org.springframework.transaction.support.TransactionTemplate;
import org.springframework.validation.annotation.Validated;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
/**
* JPA based {@link ControllerManagement} implementation.
*
@@ -79,6 +96,8 @@ import org.springframework.validation.annotation.Validated;
public class JpaControllerManagement implements ControllerManagement {
private static final Logger LOG = LoggerFactory.getLogger(ControllerManagement.class);
private final BlockingDeque<TargetPoll> queue;
@Autowired
private EntityManager entityManager;
@@ -97,9 +116,6 @@ public class JpaControllerManagement implements ControllerManagement {
@Autowired
private QuotaManagement quotaManagement;
@Autowired
private RepositoryProperties repositoryProperties;
@Autowired
private TenantConfigurationManagement tenantConfigurationManagement;
@@ -121,6 +137,38 @@ public class JpaControllerManagement implements ControllerManagement {
@Autowired
private SoftwareModuleMetadataRepository softwareModuleMetadataRepository;
@Autowired
private PlatformTransactionManager txManager;
@Autowired
private TenantAware tenantAware;
private final RepositoryProperties repositoryProperties;
JpaControllerManagement(final ScheduledExecutorService executorService,
final RepositoryProperties repositoryProperties) {
if (!repositoryProperties.isEagerPollPersistence()) {
executorService.scheduleWithFixedDelay(this::flushUpdateQueue,
repositoryProperties.getPollPersistenceFlushTime(),
repositoryProperties.getPollPersistenceFlushTime(), TimeUnit.MILLISECONDS);
queue = new LinkedBlockingDeque<>(repositoryProperties.getPollPersistenceQueueSize());
} else {
queue = null;
}
this.repositoryProperties = repositoryProperties;
}
private <T> T runInNewTransaction(final String transactionName, final TransactionCallback<T> action) {
final DefaultTransactionDefinition def = new DefaultTransactionDefinition();
def.setName(transactionName);
def.setReadOnly(false);
def.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
return new TransactionTemplate(txManager, def).execute(action);
}
@Override
public String getPollingTime() {
return systemSecurityContext.runAsSystem(() -> tenantConfigurationManagement
@@ -214,19 +262,121 @@ public class JpaControllerManagement implements ControllerManagement {
return updateTargetStatus(target, address);
}
/**
* Flush the update queue by means to persisting
* {@link Target#getLastTargetQuery()}.
*/
private void flushUpdateQueue() {
LOG.debug("Run flushUpdateQueue.");
final int size = queue.size();
if (size <= 0) {
return;
}
LOG.debug("{} events in flushUpdateQueue.", size);
final Set<TargetPoll> events = Sets.newHashSetWithExpectedSize(queue.size());
final int drained = queue.drainTo(events);
if (drained <= 0) {
return;
}
try {
events.stream().collect(Collectors.groupingBy(TargetPoll::getTenant)).forEach((tenant, polls) -> {
final TransactionCallback<Void> createTransaction = status -> updateLastTargetQueries(tenant, polls);
tenantAware.runAsTenant(tenant, () -> runInNewTransaction("flushUpdateQueue", createTransaction));
});
} catch (final RuntimeException ex) {
LOG.error("Failed to persist UpdateQueue content.", ex);
return;
}
LOG.debug("{} events persisted.", drained);
}
private Void updateLastTargetQueries(final String tenant, final List<TargetPoll> polls) {
LOG.debug("Persist {} targetqueries.", polls.size());
final List<List<String>> pollChunks = Lists.partition(
polls.stream().map(TargetPoll::getControllerId).collect(Collectors.toList()),
Constants.MAX_ENTRIES_IN_STATEMENT);
pollChunks.forEach(chunk -> {
setLastTargetQuery(tenant, System.currentTimeMillis(), chunk);
chunk.forEach(controllerId -> afterCommit.afterCommit(() -> eventPublisher
.publishEvent(new TargetPollEvent(controllerId, tenant, applicationContext.getId()))));
});
return null;
}
/**
* Sets {@link Target#getLastTargetQuery()} by native SQL in order to avoid
* raising opt lock revision as this update is not mission critical and in
* fact only written by {@link ControllerManagement}, i.e. the target
* itself.
*/
private void setLastTargetQuery(final String tenant, final long currentTimeMillis, final List<String> chunk) {
final Map<String, String> paramMapping = Maps.newHashMapWithExpectedSize(chunk.size());
for (int i = 0; i < chunk.size(); i++) {
paramMapping.put("cid" + i, chunk.get(i));
}
final Query updateQuery = entityManager.createNativeQuery(
"UPDATE sp_target t SET t.last_target_query = #last_target_query WHERE t.controller_id IN ("
+ formatQueryInStatementParams(paramMapping.keySet()) + ") AND t.tenant = #tenant");
paramMapping.entrySet().forEach(entry -> updateQuery.setParameter(entry.getKey(), entry.getValue()));
updateQuery.setParameter("last_target_query", currentTimeMillis);
updateQuery.setParameter("tenant", tenant);
final int updated = updateQuery.executeUpdate();
if (updated < chunk.size()) {
LOG.error("Targets polls could not be applied completely ({} instead of {}).", updated, chunk.size());
}
}
private static String formatQueryInStatementParams(final Collection<String> paramNames) {
return "#" + Joiner.on(",#").join(paramNames);
}
/**
* Stores target directly to DB in case either {@link Target#getAddress()}
* or {@link Target#getUpdateStatus()} changes or the buffer queue is full.
*
*/
private Target updateTargetStatus(final JpaTarget toUpdate, final URI address) {
boolean storeEager = isStoreEager(toUpdate, address);
if (TargetUpdateStatus.UNKNOWN.equals(toUpdate.getUpdateStatus())) {
toUpdate.setUpdateStatus(TargetUpdateStatus.REGISTERED);
storeEager = true;
}
toUpdate.setAddress(address.toString());
toUpdate.setLastTargetQuery(System.currentTimeMillis());
if (storeEager || !queue.offer(new TargetPoll(toUpdate))) {
toUpdate.setAddress(address.toString());
toUpdate.setLastTargetQuery(System.currentTimeMillis());
afterCommit.afterCommit(
() -> eventPublisher.publishEvent(new TargetPollEvent(toUpdate, applicationContext.getId())));
afterCommit.afterCommit(
() -> eventPublisher.publishEvent(new TargetPollEvent(toUpdate, applicationContext.getId())));
return targetRepository.save(toUpdate);
return targetRepository.save(toUpdate);
}
return toUpdate;
}
private boolean isStoreEager(final JpaTarget toUpdate, final URI address) {
if (repositoryProperties.isEagerPollPersistence()) {
return true;
} else if (toUpdate.getAddress() == null) {
return true;
} else {
return !toUpdate.getAddress().equals(address);
}
}
@Override
@@ -540,4 +690,62 @@ public class JpaControllerManagement implements ControllerManagement {
.getContent().stream().collect(Collectors.groupingBy(o -> (Long) o[0],
Collectors.mapping(o -> (SoftwareModuleMetadata) o[1], Collectors.toList())));
}
private static class TargetPoll {
private final String tenant;
private final String controllerId;
TargetPoll(final Target target) {
this.tenant = target.getTenant();
this.controllerId = target.getControllerId();
}
public String getTenant() {
return tenant;
}
public String getControllerId() {
return controllerId;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((controllerId == null) ? 0 : controllerId.hashCode());
result = prime * result + ((tenant == null) ? 0 : tenant.hashCode());
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
final TargetPoll other = (TargetPoll) obj;
if (controllerId == null) {
if (other.controllerId != null) {
return false;
}
} else if (!controllerId.equals(other.controllerId)) {
return false;
}
if (tenant == null) {
if (other.tenant != null) {
return false;
}
} else if (!tenant.equals(other.tenant)) {
return false;
}
return true;
}
}
}

6
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/RepositoryApplicationConfiguration.java
View File

@@ -9,6 +9,7 @@
package org.eclipse.hawkbit.repository.jpa;
import java.util.Map;
import java.util.concurrent.ScheduledExecutorService;
import javax.persistence.EntityManager;
import javax.sql.DataSource;
@@ -554,8 +555,9 @@ public class RepositoryApplicationConfiguration extends JpaBaseConfiguration {
*/
@Bean
@ConditionalOnMissingBean
ControllerManagement controllerManagement() {
return new JpaControllerManagement();
ControllerManagement controllerManagement(final ScheduledExecutorService executorService,
final RepositoryProperties repositoryProperties) {
return new JpaControllerManagement(executorService, repositoryProperties);
}
@Bean

1
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/TargetRepository.java
View File

@@ -36,7 +36,6 @@ import org.springframework.transaction.annotation.Transactional;
*/
@Transactional(readOnly = true)
public interface TargetRepository extends BaseEntityRepository<JpaTarget, Long>, JpaSpecificationExecutor<JpaTarget> {
/**
* Sets {@link JpaTarget#getAssignedDistributionSet()}.
*

25
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/model/JpaAction.java
View File

@@ -14,8 +14,6 @@ import java.util.List;
import javax.persistence.Column;
import javax.persistence.ConstraintMode;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.FetchType;
import javax.persistence.ForeignKey;
import javax.persistence.Index;
@@ -40,6 +38,9 @@ import org.eclipse.hawkbit.repository.model.RolloutGroup;
import org.eclipse.hawkbit.repository.model.Target;
import org.eclipse.hawkbit.repository.model.helper.EventPublisherHolder;
import org.eclipse.persistence.annotations.CascadeOnDelete;
import org.eclipse.persistence.annotations.ConversionValue;
import org.eclipse.persistence.annotations.Convert;
import org.eclipse.persistence.annotations.ObjectTypeConverter;
import org.eclipse.persistence.descriptors.DescriptorEvent;
/**
@@ -71,8 +72,12 @@ public class JpaAction extends AbstractJpaTenantAwareBaseEntity implements Actio
@Column(name = "active")
private boolean active;
@Column(name = "action_type", nullable = false, length = 16)
@Enumerated(EnumType.STRING)
@Column(name = "action_type", nullable = false)
@ObjectTypeConverter(name = "actionType", objectType = Action.ActionType.class, dataType = Integer.class, conversionValues = {
@ConversionValue(objectValue = "FORCED", dataValue = "0"),
@ConversionValue(objectValue = "SOFT", dataValue = "1"),
@ConversionValue(objectValue = "TIMEFORCED", dataValue = "2") })
@Convert("actionType")
@NotNull
private ActionType actionType;
@@ -80,6 +85,18 @@ public class JpaAction extends AbstractJpaTenantAwareBaseEntity implements Actio
private long forcedTime;
@Column(name = "status", nullable = false)
@ObjectTypeConverter(name = "status", objectType = Action.Status.class, dataType = Integer.class, conversionValues = {
@ConversionValue(objectValue = "FINISHED", dataValue = "0"),
@ConversionValue(objectValue = "ERROR", dataValue = "1"),
@ConversionValue(objectValue = "WARNING", dataValue = "2"),
@ConversionValue(objectValue = "RUNNING", dataValue = "3"),
@ConversionValue(objectValue = "CANCELED", dataValue = "4"),
@ConversionValue(objectValue = "CANCELING", dataValue = "5"),
@ConversionValue(objectValue = "RETRIEVED", dataValue = "6"),
@ConversionValue(objectValue = "DOWNLOAD", dataValue = "7"),
@ConversionValue(objectValue = "SCHEDULED", dataValue = "8"),
@ConversionValue(objectValue = "CANCEL_REJECTED", dataValue = "9") })
@Convert("status")
@NotNull
private Status status;

27
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/model/JpaActionStatus.java
View File

@@ -31,6 +31,9 @@ import org.eclipse.hawkbit.repository.model.Action;
import org.eclipse.hawkbit.repository.model.Action.Status;
import org.eclipse.hawkbit.repository.model.ActionStatus;
import org.eclipse.persistence.annotations.CascadeOnDelete;
import org.eclipse.persistence.annotations.ConversionValue;
import org.eclipse.persistence.annotations.Convert;
import org.eclipse.persistence.annotations.ObjectTypeConverter;
import com.google.common.base.Splitter;
@@ -50,8 +53,8 @@ public class JpaActionStatus extends AbstractJpaTenantAwareBaseEntity implements
private static final long serialVersionUID = 1L;
@Column(name = "target_occurred_at")
private Long occurredAt;
@Column(name = "target_occurred_at", nullable = false, updatable = false)
private long occurredAt;
@ManyToOne(fetch = FetchType.LAZY, optional = false)
@JoinColumn(name = "action", nullable = false, updatable = false, foreignKey = @ForeignKey(value = ConstraintMode.CONSTRAINT, name = "fk_act_stat_action"))
@@ -59,6 +62,18 @@ public class JpaActionStatus extends AbstractJpaTenantAwareBaseEntity implements
private JpaAction action;
@Column(name = "status", nullable = false, updatable = false)
@ObjectTypeConverter(name = "status", objectType = Action.Status.class, dataType = Integer.class, conversionValues = {
@ConversionValue(objectValue = "FINISHED", dataValue = "0"),
@ConversionValue(objectValue = "ERROR", dataValue = "1"),
@ConversionValue(objectValue = "WARNING", dataValue = "2"),
@ConversionValue(objectValue = "RUNNING", dataValue = "3"),
@ConversionValue(objectValue = "CANCELED", dataValue = "4"),
@ConversionValue(objectValue = "CANCELING", dataValue = "5"),
@ConversionValue(objectValue = "RETRIEVED", dataValue = "6"),
@ConversionValue(objectValue = "DOWNLOAD", dataValue = "7"),
@ConversionValue(objectValue = "SCHEDULED", dataValue = "8"),
@ConversionValue(objectValue = "CANCEL_REJECTED", dataValue = "9") })
@Convert("status")
@NotNull
private Status status;
@@ -66,7 +81,7 @@ public class JpaActionStatus extends AbstractJpaTenantAwareBaseEntity implements
@ElementCollection(fetch = FetchType.LAZY, targetClass = String.class)
@CollectionTable(name = "sp_action_status_messages", joinColumns = @JoinColumn(name = "action_status_id", foreignKey = @ForeignKey(value = ConstraintMode.CONSTRAINT, name = "fk_stat_msg_act_stat")), indexes = {
@Index(name = "sp_idx_action_status_msgs_01", columnList = "action_status_id") })
@Column(name = "detail_message", length = 512)
@Column(name = "detail_message", length = MESSAGE_ENTRY_LENGTH, nullable = false, updatable = false)
private List<String> messages;
/**
@@ -97,7 +112,7 @@ public class JpaActionStatus extends AbstractJpaTenantAwareBaseEntity implements
* @param message
* the message which should be added to this action status
*/
public JpaActionStatus(final JpaAction action, final Status status, final Long occurredAt, final String message) {
public JpaActionStatus(final JpaAction action, final Status status, final long occurredAt, final String message) {
this.action = action;
this.status = status;
this.occurredAt = occurredAt;
@@ -125,11 +140,11 @@ public class JpaActionStatus extends AbstractJpaTenantAwareBaseEntity implements
}
@Override
public Long getOccurredAt() {
public long getOccurredAt() {
return occurredAt;
}
public void setOccurredAt(final Long occurredAt) {
public void setOccurredAt(final long occurredAt) {
this.occurredAt = occurredAt;
}

8
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/model/JpaArtifact.java
View File

@@ -41,13 +41,13 @@ public class JpaArtifact extends AbstractJpaTenantAwareBaseEntity implements Art
private static final long serialVersionUID = 1L;
@Column(name = "sha1_hash", length = 40, nullable = false, updatable = false)
@Size(max = 40)
@NotEmpty
@Size(min = 1, max = 40)
@NotNull
private String sha1Hash;
@Column(name = "provided_file_name", length = 256)
@Size(max = 256)
@NotEmpty
@Size(min = 1, max = 256)
@NotNull
private String filename;
@ManyToOne(optional = false, cascade = { CascadeType.PERSIST }, fetch = FetchType.LAZY)

43
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/model/JpaRollout.java
View File

@@ -15,8 +15,6 @@ import java.util.stream.Collectors;
import javax.persistence.Column;
import javax.persistence.ConstraintMode;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.FetchType;
import javax.persistence.ForeignKey;
import javax.persistence.JoinColumn;
@@ -30,10 +28,12 @@ import javax.validation.constraints.Size;
import org.eclipse.hawkbit.repository.event.remote.RolloutDeletedEvent;
import org.eclipse.hawkbit.repository.event.remote.entity.RolloutUpdatedEvent;
import org.eclipse.hawkbit.repository.model.Action;
import org.eclipse.hawkbit.repository.model.Action.ActionType;
import org.eclipse.hawkbit.repository.model.DistributionSet;
import org.eclipse.hawkbit.repository.model.Rollout;
import org.eclipse.hawkbit.repository.model.RolloutGroup;
import org.eclipse.hawkbit.repository.model.TargetFilterQuery;
import org.eclipse.hawkbit.repository.model.TotalTargetCountStatus;
import org.eclipse.hawkbit.repository.model.helper.EventPublisherHolder;
import org.eclipse.persistence.annotations.CascadeOnDelete;
@@ -44,7 +44,6 @@ import org.eclipse.persistence.descriptors.DescriptorEvent;
import org.eclipse.persistence.queries.UpdateObjectQuery;
import org.eclipse.persistence.sessions.changesets.DirectToFieldChangeRecord;
import org.eclipse.persistence.sessions.changesets.ObjectChangeSet;
import org.hibernate.validator.constraints.NotEmpty;
/**
* JPA implementation of a {@link Rollout}.
@@ -56,18 +55,6 @@ import org.hibernate.validator.constraints.NotEmpty;
// exception squid:S2160 - BaseEntity equals/hashcode is handling correctly for
// sub entities
@SuppressWarnings("squid:S2160")
@ObjectTypeConverter(name = "rolloutstatus", objectType = Rollout.RolloutStatus.class, dataType = Integer.class, conversionValues = {
@ConversionValue(objectValue = "CREATING", dataValue = "0"),
@ConversionValue(objectValue = "READY", dataValue = "1"),
@ConversionValue(objectValue = "PAUSED", dataValue = "2"),
@ConversionValue(objectValue = "STARTING", dataValue = "3"),
@ConversionValue(objectValue = "STOPPED", dataValue = "4"),
@ConversionValue(objectValue = "RUNNING", dataValue = "5"),
@ConversionValue(objectValue = "FINISHED", dataValue = "6"),
@ConversionValue(objectValue = "ERROR_CREATING", dataValue = "7"),
@ConversionValue(objectValue = "ERROR_STARTING", dataValue = "8"),
@ConversionValue(objectValue = "DELETING", dataValue = "9"),
@ConversionValue(objectValue = "DELETED", dataValue = "10") })
public class JpaRollout extends AbstractJpaNamedEntity implements Rollout, EventAwareEntity {
private static final long serialVersionUID = 1L;
@@ -78,9 +65,9 @@ public class JpaRollout extends AbstractJpaNamedEntity implements Rollout, Event
@OneToMany(targetEntity = JpaRolloutGroup.class, fetch = FetchType.LAZY, mappedBy = "rollout")
private List<JpaRolloutGroup> rolloutGroups;
@Column(name = "target_filter", length = 1024, nullable = false)
@Size(max = 1024)
@NotEmpty
@Column(name = "target_filter", length = TargetFilterQuery.QUERY_MAX_SIZE, nullable = false)
@Size(min = 1, max = TargetFilterQuery.QUERY_MAX_SIZE)
@NotNull
private String targetFilterQuery;
@ManyToOne(fetch = FetchType.LAZY, optional = false)
@@ -89,6 +76,18 @@ public class JpaRollout extends AbstractJpaNamedEntity implements Rollout, Event
private JpaDistributionSet distributionSet;
@Column(name = "status", nullable = false)
@ObjectTypeConverter(name = "rolloutstatus", objectType = Rollout.RolloutStatus.class, dataType = Integer.class, conversionValues = {
@ConversionValue(objectValue = "CREATING", dataValue = "0"),
@ConversionValue(objectValue = "READY", dataValue = "1"),
@ConversionValue(objectValue = "PAUSED", dataValue = "2"),
@ConversionValue(objectValue = "STARTING", dataValue = "3"),
@ConversionValue(objectValue = "STOPPED", dataValue = "4"),
@ConversionValue(objectValue = "RUNNING", dataValue = "5"),
@ConversionValue(objectValue = "FINISHED", dataValue = "6"),
@ConversionValue(objectValue = "ERROR_CREATING", dataValue = "7"),
@ConversionValue(objectValue = "ERROR_STARTING", dataValue = "8"),
@ConversionValue(objectValue = "DELETING", dataValue = "9"),
@ConversionValue(objectValue = "DELETED", dataValue = "10") })
@Convert("rolloutstatus")
@NotNull
private RolloutStatus status = RolloutStatus.CREATING;
@@ -96,8 +95,12 @@ public class JpaRollout extends AbstractJpaNamedEntity implements Rollout, Event
@Column(name = "last_check")
private long lastCheck;
@Column(name = "action_type", nullable = false, length = 16)
@Enumerated(EnumType.STRING)
@Column(name = "action_type", nullable = false)
@ObjectTypeConverter(name = "actionType", objectType = Action.ActionType.class, dataType = Integer.class, conversionValues = {
@ConversionValue(objectValue = "FORCED", dataValue = "0"),
@ConversionValue(objectValue = "SOFT", dataValue = "1"),
@ConversionValue(objectValue = "TIMEFORCED", dataValue = "2") })
@Convert("actionType")
@NotNull
private ActionType actionType = ActionType.FORCED;

14
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/model/JpaRolloutGroup.java
View File

@@ -48,13 +48,6 @@ import org.eclipse.persistence.descriptors.DescriptorEvent;
// exception squid:S2160 - BaseEntity equals/hashcode is handling correctly for
// sub entities
@SuppressWarnings("squid:S2160")
@ObjectTypeConverter(name = "rolloutgroupstatus", objectType = RolloutGroup.RolloutGroupStatus.class, dataType = Integer.class, conversionValues = {
@ConversionValue(objectValue = "READY", dataValue = "0"),
@ConversionValue(objectValue = "SCHEDULED", dataValue = "1"),
@ConversionValue(objectValue = "FINISHED", dataValue = "2"),
@ConversionValue(objectValue = "ERROR", dataValue = "3"),
@ConversionValue(objectValue = "RUNNING", dataValue = "4"),
@ConversionValue(objectValue = "CREATING", dataValue = "5") })
public class JpaRolloutGroup extends AbstractJpaNamedEntity implements RolloutGroup, EventAwareEntity {
private static final long serialVersionUID = 1L;
@@ -64,6 +57,13 @@ public class JpaRolloutGroup extends AbstractJpaNamedEntity implements RolloutGr
private JpaRollout rollout;
@Column(name = "status", nullable = false)
@ObjectTypeConverter(name = "rolloutgroupstatus", objectType = RolloutGroup.RolloutGroupStatus.class, dataType = Integer.class, conversionValues = {
@ConversionValue(objectValue = "READY", dataValue = "0"),
@ConversionValue(objectValue = "SCHEDULED", dataValue = "1"),
@ConversionValue(objectValue = "FINISHED", dataValue = "2"),
@ConversionValue(objectValue = "ERROR", dataValue = "3"),
@ConversionValue(objectValue = "RUNNING", dataValue = "4"),
@ConversionValue(objectValue = "CREATING", dataValue = "5") })
@Convert("rolloutgroupstatus")
private RolloutGroupStatus status = RolloutGroupStatus.CREATING;

21
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/model/JpaTarget.java
View File

@@ -28,8 +28,6 @@ import javax.persistence.Column;
import javax.persistence.ConstraintMode;
import javax.persistence.ElementCollection;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.FetchType;
import javax.persistence.ForeignKey;
import javax.persistence.Index;
@@ -63,6 +61,9 @@ import org.eclipse.hawkbit.repository.model.helper.TenantConfigurationManagement
import org.eclipse.hawkbit.tenancy.configuration.DurationHelper;
import org.eclipse.hawkbit.tenancy.configuration.TenantConfigurationProperties.TenantConfigurationKey;
import org.eclipse.persistence.annotations.CascadeOnDelete;
import org.eclipse.persistence.annotations.ConversionValue;
import org.eclipse.persistence.annotations.Convert;
import org.eclipse.persistence.annotations.ObjectTypeConverter;
import org.eclipse.persistence.descriptors.DescriptorEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -87,10 +88,10 @@ public class JpaTarget extends AbstractJpaNamedEntity implements Target, EventAw
private static final Logger LOG = LoggerFactory.getLogger(JpaTarget.class);
private static final List<String> TARGET_UPDATE_EVENT_IGNORE_FIELDS = Arrays.asList("lastTargetQuery",
"lastTargetQuery", "address", "optLockRevision", "lastModifiedAt", "lastModifiedBy");
private static final List<String> TARGET_UPDATE_EVENT_IGNORE_FIELDS = Arrays.asList("lastTargetQuery", "address",
"optLockRevision", "lastModifiedAt", "lastModifiedBy");
@Column(name = "controller_id", length = Target.CONTROLLER_ID_MAX_SIZE)
@Column(name = "controller_id", length = Target.CONTROLLER_ID_MAX_SIZE, updatable = false, nullable = false)
@Size(min = 1, max = Target.CONTROLLER_ID_MAX_SIZE)
@NotNull
@Pattern(regexp = "[.\\S]*", message = "has whitespaces which are not allowed")
@@ -130,8 +131,14 @@ public class JpaTarget extends AbstractJpaNamedEntity implements Target, EventAw
@Column(name = "install_date")
private Long installationDate;
@Column(name = "update_status", nullable = false, length = 16)
@Enumerated(EnumType.STRING)
@Column(name = "update_status", nullable = false)
@ObjectTypeConverter(name = "updateStatus", objectType = TargetUpdateStatus.class, dataType = Integer.class, conversionValues = {
@ConversionValue(objectValue = "UNKNOWN", dataValue = "0"),
@ConversionValue(objectValue = "IN_SYNC", dataValue = "1"),
@ConversionValue(objectValue = "PENDING", dataValue = "2"),
@ConversionValue(objectValue = "ERROR", dataValue = "3"),
@ConversionValue(objectValue = "REGISTERED", dataValue = "4") })
@Convert("updateStatus")
@NotNull
private TargetUpdateStatus updateStatus = TargetUpdateStatus.UNKNOWN;

6
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/model/JpaTenantMetaData.java
View File

@@ -19,12 +19,12 @@ import javax.persistence.JoinColumn;
import javax.persistence.OneToOne;
import javax.persistence.Table;
import javax.persistence.UniqueConstraint;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.Size;
import org.eclipse.hawkbit.repository.model.DistributionSetType;
import org.eclipse.hawkbit.repository.model.TenantAwareBaseEntity;
import org.eclipse.hawkbit.repository.model.TenantMetaData;
import org.hibernate.validator.constraints.NotEmpty;
/**
* Tenant entity with meta data that is configured globally for the entire
@@ -45,8 +45,8 @@ public class JpaTenantMetaData extends AbstractJpaBaseEntity implements TenantMe
private static final long serialVersionUID = 1L;
@Column(name = "tenant", nullable = false, updatable = false, length = 40)
@Size(max = 40)
@NotEmpty
@Size(min = 1, max = 40)
@NotNull
private String tenant;
@OneToOne(fetch = FetchType.LAZY)

25
hawkbit-repository/hawkbit-repository-jpa/src/main/resources/db/migration/H2/V1_12_0__action_performance___H2.sql Normal file
View File

@@ -0,0 +1,25 @@
ALTER TABLE sp_action ADD COLUMN action_type_new integer not null;
UPDATE sp_action SET action_type_new =
CASE WHEN (action_type = 'SOFT') THEN 1
WHEN (action_type = 'TIMEFORCED') THEN 2
ELSE 0 END;
ALTER TABLE sp_action DROP COLUMN action_type;
ALTER TABLE sp_action CHANGE COLUMN action_type_new action_type integer;
ALTER TABLE sp_rollout ADD COLUMN action_type_new integer not null;
UPDATE sp_rollout SET action_type_new =
CASE WHEN (action_type = 'SOFT') THEN 1
WHEN (action_type = 'TIMEFORCED') THEN 2
ELSE 0 END;
ALTER TABLE sp_rollout DROP COLUMN action_type;
ALTER TABLE sp_rollout CHANGE COLUMN action_type_new action_type integer;
ALTER TABLE sp_target ADD COLUMN update_status_new integer not null;
UPDATE sp_target SET update_status_new =
CASE WHEN (update_status = 'IN_SYNC') THEN 1
WHEN (update_status = 'PENDING') THEN 2
WHEN (update_status = 'ERROR') THEN 3
WHEN (update_status = 'REGISTERED') THEN 4
ELSE 0 END;
ALTER TABLE sp_target DROP COLUMN update_status;
ALTER TABLE sp_target CHANGE COLUMN update_status_new update_status integer;

12
hawkbit-repository/hawkbit-repository-jpa/src/main/resources/db/migration/H2/V1_12_1__missing_non_null___H2.sql Normal file
View File

@@ -0,0 +1,12 @@
ALTER TABLE sp_action_status_messages CHANGE COLUMN detail_message detail_message varchar(512) not null;
ALTER TABLE sp_action CHANGE COLUMN distribution_set distribution_set bigint not null;
ALTER TABLE sp_action CHANGE COLUMN target target bigint not null;
ALTER TABLE sp_action CHANGE COLUMN status status integer not null;
ALTER TABLE sp_action_status CHANGE COLUMN target_occurred_at target_occurred_at bigint not null;
ALTER TABLE sp_action_status CHANGE COLUMN status status integer not null;
ALTER TABLE sp_rollout CHANGE COLUMN distribution_set distribution_set bigint not null;
ALTER TABLE sp_rollout CHANGE COLUMN status status integer not null;
ALTER TABLE sp_rolloutgroup CHANGE COLUMN rollout rollout bigint not null;
ALTER TABLE sp_rolloutgroup CHANGE COLUMN status status integer not null;
ALTER TABLE sp_artifact CHANGE COLUMN sha1_hash sha1_hash varchar(40) not null;
ALTER TABLE sp_target CHANGE COLUMN controller_id controller_id varchar(64) not null;

25
hawkbit-repository/hawkbit-repository-jpa/src/main/resources/db/migration/MYSQL/V1_12_0__action_performance___MYSQL.sql Normal file
View File

@@ -0,0 +1,25 @@
ALTER TABLE sp_action ADD COLUMN action_type_new integer not null;
UPDATE sp_action SET action_type_new =
CASE WHEN (action_type = 'SOFT') THEN 1
WHEN (action_type = 'TIMEFORCED') THEN 2
ELSE 0 END;
ALTER TABLE sp_action DROP COLUMN action_type;
ALTER TABLE sp_action CHANGE COLUMN action_type_new action_type integer;
ALTER TABLE sp_rollout ADD COLUMN action_type_new integer not null;
UPDATE sp_rollout SET action_type_new =
CASE WHEN (action_type = 'SOFT') THEN 1
WHEN (action_type = 'TIMEFORCED') THEN 2
ELSE 0 END;
ALTER TABLE sp_rollout DROP COLUMN action_type;
ALTER TABLE sp_rollout CHANGE COLUMN action_type_new action_type integer;
ALTER TABLE sp_target ADD COLUMN update_status_new integer not null;
UPDATE sp_target SET update_status_new =
CASE WHEN (update_status = 'IN_SYNC') THEN 1
WHEN (update_status = 'PENDING') THEN 2
WHEN (update_status = 'ERROR') THEN 3
WHEN (update_status = 'REGISTERED') THEN 4
ELSE 0 END;
ALTER TABLE sp_target DROP COLUMN update_status;
ALTER TABLE sp_target CHANGE COLUMN update_status_new update_status integer;

12
hawkbit-repository/hawkbit-repository-jpa/src/main/resources/db/migration/MYSQL/V1_12_1__missing_non_null___MYSQL.sql Normal file
View File

@@ -0,0 +1,12 @@
ALTER TABLE sp_action_status_messages CHANGE COLUMN detail_message detail_message varchar(512) not null;
ALTER TABLE sp_action CHANGE COLUMN distribution_set distribution_set bigint not null;
ALTER TABLE sp_action CHANGE COLUMN target target bigint not null;
ALTER TABLE sp_action CHANGE COLUMN status status integer not null;
ALTER TABLE sp_action_status CHANGE COLUMN target_occurred_at target_occurred_at bigint not null;
ALTER TABLE sp_action_status CHANGE COLUMN status status integer not null;
ALTER TABLE sp_rollout CHANGE COLUMN distribution_set distribution_set bigint not null;
ALTER TABLE sp_rollout CHANGE COLUMN status status integer not null;
ALTER TABLE sp_rolloutgroup CHANGE COLUMN rollout rollout bigint not null;
ALTER TABLE sp_rolloutgroup CHANGE COLUMN status status integer not null;
ALTER TABLE sp_artifact CHANGE COLUMN sha1_hash sha1_hash varchar(40) not null;
ALTER TABLE sp_target CHANGE COLUMN controller_id controller_id varchar(64) not null;

56
hawkbit-repository/hawkbit-repository-jpa/src/test/java/org/eclipse/hawkbit/repository/jpa/LazyControllerManagementTest.java Normal file
View File

@@ -0,0 +1,56 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.repository.jpa;
import static org.assertj.core.api.Assertions.assertThat;
import java.util.concurrent.TimeUnit;
import org.eclipse.hawkbit.repository.RepositoryProperties;
import org.eclipse.hawkbit.repository.event.remote.TargetPollEvent;
import org.eclipse.hawkbit.repository.event.remote.entity.TargetCreatedEvent;
import org.eclipse.hawkbit.repository.model.Target;
import org.eclipse.hawkbit.repository.test.matcher.Expect;
import org.eclipse.hawkbit.repository.test.matcher.ExpectEvents;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.TestPropertySource;
import ru.yandex.qatools.allure.annotations.Description;
import ru.yandex.qatools.allure.annotations.Features;
import ru.yandex.qatools.allure.annotations.Stories;
@Features("Component Tests - Repository")
@Stories("Controller Management")
@TestPropertySource(locations = "classpath:/jpa-test.properties", properties = {
"hawkbit.server.repository.eagerPollPersistence=false",
"hawkbit.server.repository.pollPersistenceFlushTime=1000" })
public class LazyControllerManagementTest extends AbstractJpaIntegrationTest {
@Autowired
private RepositoryProperties repositoryProperties;
@Test
@Description("Verfies that lazy target poll update is executed as specified.")
@ExpectEvents({ @Expect(type = TargetCreatedEvent.class, count = 1),
@Expect(type = TargetPollEvent.class, count = 2) })
public void lazyFindOrRegisterTargetIfItDoesNotexist() throws InterruptedException {
final Target target = controllerManagement.findOrRegisterTargetIfItDoesNotexist("AA", LOCALHOST);
assertThat(target).as("target should not be null").isNotNull();
TimeUnit.MILLISECONDS.sleep(10);
controllerManagement.findOrRegisterTargetIfItDoesNotexist("AA", LOCALHOST);
TimeUnit.MILLISECONDS.sleep(repositoryProperties.getPollPersistenceFlushTime() + 1);
final Target updated = targetManagement.get(target.getId()).get();
assertThat(updated.getOptLockRevision()).isEqualTo(target.getOptLockRevision());
assertThat(updated.getLastTargetQuery()).isGreaterThan(target.getLastTargetQuery());
}
}

10
hawkbit-repository/hawkbit-repository-test/src/main/java/org/eclipse/hawkbit/repository/test/TestConfiguration.java
View File

@@ -10,6 +10,7 @@ package org.eclipse.hawkbit.repository.test;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import org.eclipse.hawkbit.ControllerPollProperties;
import org.eclipse.hawkbit.HawkbitServerProperties;
@@ -60,9 +61,12 @@ import org.springframework.integration.support.locks.LockRegistry;
import org.springframework.messaging.converter.MessageConverter;
import org.springframework.scheduling.annotation.AsyncConfigurer;
import org.springframework.security.concurrent.DelegatingSecurityContextExecutorService;
import org.springframework.security.concurrent.DelegatingSecurityContextScheduledExecutorService;
import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
import org.springframework.util.AntPathMatcher;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
* Spring context configuration required for Dev.Environment.
*/
@@ -186,6 +190,12 @@ public class TestConfiguration implements AsyncConfigurer {
return new SimpleAsyncUncaughtExceptionHandler();
}
@Bean
public ScheduledExecutorService scheduledExecutorService() {
return new DelegatingSecurityContextScheduledExecutorService(Executors.newScheduledThreadPool(1,
new ThreadFactoryBuilder().setNameFormat("central-scheduled-executor-pool-%d").build()));
}
/**
*
* @return returns a VirtualPropertyReplacer

16
hawkbit-repository/hawkbit-repository-test/src/main/java/org/eclipse/hawkbit/repository/test/util/AbstractIntegrationTest.java
View File

@@ -21,7 +21,6 @@ import java.util.stream.Collectors;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.eclipse.hawkbit.artifact.repository.ArtifactFilesystemProperties;
import org.eclipse.hawkbit.artifact.repository.ArtifactRepository;
import org.eclipse.hawkbit.cache.TenantAwareCacheManager;
import org.eclipse.hawkbit.repository.ArtifactManagement;
@@ -176,9 +175,6 @@ public abstract class AbstractIntegrationTest {
@Autowired
protected SystemSecurityContext systemSecurityContext;
@Autowired
private ArtifactFilesystemProperties artifactFilesystemProperties;
@Autowired
protected ArtifactRepository binaryArtifactRepository;
@@ -271,18 +267,18 @@ public abstract class AbstractIntegrationTest {
osType = securityRule
.runAsPrivileged(() -> testdataFactory.findOrCreateSoftwareModuleType(TestdataFactory.SM_TYPE_OS));
osType = securityRule.runAsPrivileged(() -> softwareModuleTypeManagement.update(
entityFactory.softwareModuleType().update(osType.getId()).description(description)));
osType = securityRule.runAsPrivileged(() -> softwareModuleTypeManagement
.update(entityFactory.softwareModuleType().update(osType.getId()).description(description)));
appType = securityRule.runAsPrivileged(
() -> testdataFactory.findOrCreateSoftwareModuleType(TestdataFactory.SM_TYPE_APP, Integer.MAX_VALUE));
appType = securityRule.runAsPrivileged(() -> softwareModuleTypeManagement.update(
entityFactory.softwareModuleType().update(appType.getId()).description(description)));
appType = securityRule.runAsPrivileged(() -> softwareModuleTypeManagement
.update(entityFactory.softwareModuleType().update(appType.getId()).description(description)));
runtimeType = securityRule
.runAsPrivileged(() -> testdataFactory.findOrCreateSoftwareModuleType(TestdataFactory.SM_TYPE_RT));
runtimeType = securityRule.runAsPrivileged(() -> softwareModuleTypeManagement.update(
entityFactory.softwareModuleType().update(runtimeType.getId()).description(description)));
runtimeType = securityRule.runAsPrivileged(() -> softwareModuleTypeManagement
.update(entityFactory.softwareModuleType().update(runtimeType.getId()).description(description)));
standardDsType = securityRule.runAsPrivileged(() -> testdataFactory.findOrCreateDefaultTestDsType());
}

4
hawkbit-repository/hawkbit-repository-test/src/main/resources/hawkbit-test-defaults.properties
View File

@@ -23,6 +23,10 @@ spring.jpa.properties.eclipselink.logging.parameters=true
## JPA Repository - START
spring.datasource.url=jdbc:h2:mem:sp-db;DB_CLOSE_ON_EXIT=FALSE
## JPA Repository - END
# Enforce persistence of targetpolls for test predictability.
hawkbit.server.repository.eagerPollPersistence=true
# Default properties for test that can be overridden during test run - END
# Properties that are managed by autoconfigure module at runtime and not available during test - START