Added artifact download traffic statistics.

Signed-off-by: Kai Zimmermann <kai.zimmermann@bosch-si.com>
This commit is contained in:
Kai Zimmermann
2016-07-05 17:41:08 +02:00
parent eff798393d
commit 227b93c527
16 changed files with 147 additions and 36 deletions

View File

@@ -31,7 +31,7 @@ public class DownloadIdCacheAutoConfiguration {
private CacheManager cacheManager;
/**
* Bean for the downlod id cache.
* Bean for the download id cache.
*
* @return the cache
*/

View File

@@ -56,7 +56,7 @@ public class EventDistributorTest {
@Test
public void distributeDistributedEventSendsToRedis() {
final DownloadProgressEvent event = new DownloadProgressEvent("tenant", 123L, 10);
final DownloadProgressEvent event = new DownloadProgressEvent("tenant", 123L, 10, 100L);
underTest.distribute(event);
// origin node ID should be set by distributing the event
@@ -67,7 +67,7 @@ public class EventDistributorTest {
@Test
public void dontDistributeDistributedEventIfSameNode() {
final String knownNodeId = EventDistributor.getNodeId();
final DownloadProgressEvent event = new DownloadProgressEvent("tenant", 123L, 10);
final DownloadProgressEvent event = new DownloadProgressEvent("tenant", 123L, 10, 100L);
event.setNodeId(knownNodeId);
// test
@@ -79,7 +79,7 @@ public class EventDistributorTest {
@Test
public void handleDistributedMessageFromRedis() {
final DownloadProgressEvent event = new DownloadProgressEvent("tenant", 123L, 10);
final DownloadProgressEvent event = new DownloadProgressEvent("tenant", 123L, 10, 100L);
final String knownChannel = "someChannel";
underTest.handleMessage(event, knownChannel);
@@ -90,7 +90,7 @@ public class EventDistributorTest {
@Test
public void handleDistributedMessageFilteredIfSameNodeId() {
final DownloadProgressEvent event = new DownloadProgressEvent("tenant", 123L, 10);
final DownloadProgressEvent event = new DownloadProgressEvent("tenant", 123L, 10, 100L);
final String knownChannel = "someChannel";
event.setOriginNodeId(EventDistributor.getNodeId());

View File

@@ -21,6 +21,7 @@ public class DownloadProgressEvent extends AbstractDistributedEvent {
private final Long statusId;
private final int progressPercent;
private final long shippedBytes;
/**
* Constructor.
@@ -32,13 +33,15 @@ public class DownloadProgressEvent extends AbstractDistributedEvent {
* @param progressPercent
* number (1-100)
*/
public DownloadProgressEvent(final String tenant, final Long statusId, final int progressPercent) {
public DownloadProgressEvent(final String tenant, final Long statusId, final int progressPercent,
final long shippedBytes) {
// the revision of the DownloadProgressEvent is just equal the
// progressPercentage due the
// percentage is going from 0 to 100.
super(statusId, tenant);
this.statusId = statusId;
this.progressPercent = progressPercent;
this.shippedBytes = shippedBytes;
}
/**
@@ -54,4 +57,12 @@ public class DownloadProgressEvent extends AbstractDistributedEvent {
public int getProgressPercent() {
return progressPercent;
}
/**
* @return the shippedBytes
*/
public long getShippedBytes() {
return shippedBytes;
}
}

View File

@@ -29,11 +29,11 @@ import org.apache.commons.lang3.RandomUtils;
import org.eclipse.hawkbit.eventbus.event.DownloadProgressEvent;
import org.eclipse.hawkbit.repository.model.Action;
import org.eclipse.hawkbit.repository.model.Action.Status;
import org.eclipse.hawkbit.repository.test.util.WithUser;
import org.eclipse.hawkbit.repository.model.Artifact;
import org.eclipse.hawkbit.repository.model.DistributionSet;
import org.eclipse.hawkbit.repository.model.LocalArtifact;
import org.eclipse.hawkbit.repository.model.Target;
import org.eclipse.hawkbit.repository.test.util.WithUser;
import org.eclipse.hawkbit.rest.AbstractRestIntegrationTestWithMongoDB;
import org.junit.Test;
import org.slf4j.LoggerFactory;
@@ -59,11 +59,14 @@ import ru.yandex.qatools.allure.annotations.Stories;
@Stories("Artifact Download Resource")
public class DdiArtifactDownloadTest extends AbstractRestIntegrationTestWithMongoDB {
private static final int ARTIFACT_SIZE = 5 * 1024 * 1024;
public DdiArtifactDownloadTest() {
LOG = LoggerFactory.getLogger(DdiArtifactDownloadTest.class);
}
private volatile int downLoadProgress = 0;
private volatile long shippedBytes = 0;
@Autowired
private EventBus eventBus;
@@ -236,6 +239,8 @@ public class DdiArtifactDownloadTest extends AbstractRestIntegrationTestWithMong
@Description("Tests valid downloads through the artifact resource by identifying the artifact not by ID but file name.")
public void downloadArtifactThroughFileName() throws Exception {
downLoadProgress = 1;
shippedBytes = 0;
tenantStatsManagement.resetTrafficStatsOfTenant();
eventBus.register(this);
assertThat(softwareManagement.findSoftwareModulesAll(pageReq)).hasSize(0);
@@ -249,7 +254,7 @@ public class DdiArtifactDownloadTest extends AbstractRestIntegrationTestWithMong
final DistributionSet ds = testdataFactory.createDistributionSet("");
// create artifact
final byte random[] = RandomUtils.nextBytes(5 * 1024 * 1024);
final byte random[] = RandomUtils.nextBytes(ARTIFACT_SIZE);
final LocalArtifact artifact = artifactManagement.createLocalArtifact(new ByteArrayInputStream(random),
ds.findFirstModuleByType(osType).getId(), "file1", false);
@@ -276,6 +281,8 @@ public class DdiArtifactDownloadTest extends AbstractRestIntegrationTestWithMong
// download complete
assertThat(downLoadProgress).isEqualTo(10);
assertThat(shippedBytes).isEqualTo(ARTIFACT_SIZE)
.isEqualTo(tenantStatsManagement.getStatsOfTenant().getOverallArtifactTrafficInBytes());
}
@Test
@@ -313,6 +320,8 @@ public class DdiArtifactDownloadTest extends AbstractRestIntegrationTestWithMong
+ "anonymous as authorization is notpossible, e.g. chekc if the controller has the artifact assigned.")
public void downloadArtifactByNameFailsIfNotAuthenticated() throws Exception {
downLoadProgress = 1;
shippedBytes = 0;
tenantStatsManagement.resetTrafficStatsOfTenant();
eventBus.register(this);
assertThat(softwareManagement.findSoftwareModulesAll(pageReq)).hasSize(0);
@@ -327,7 +336,7 @@ public class DdiArtifactDownloadTest extends AbstractRestIntegrationTestWithMong
final DistributionSet ds = testdataFactory.createDistributionSet("");
// create artifact
final byte random[] = RandomUtils.nextBytes(5 * 1024);
final byte random[] = RandomUtils.nextBytes(ARTIFACT_SIZE);
final Artifact artifact = artifactManagement.createLocalArtifact(new ByteArrayInputStream(random),
ds.findFirstModuleByType(osType).getId(), "file1.tar.bz2", false);
@@ -335,6 +344,10 @@ public class DdiArtifactDownloadTest extends AbstractRestIntegrationTestWithMong
deploymentManagement.assignDistributionSet(ds, targets);
mvc.perform(get("/controller/artifacts/v1/filename/{filename}", "file1.tar.bz2"))
.andExpect(status().isNotFound());
assertThat(downLoadProgress).isEqualTo(1);
assertThat(shippedBytes).isEqualTo(0)
.isEqualTo(tenantStatsManagement.getStatsOfTenant().getOverallArtifactTrafficInBytes());
}
@Test
@@ -342,6 +355,8 @@ public class DdiArtifactDownloadTest extends AbstractRestIntegrationTestWithMong
@Description("Ensures that an authenticated and named controller is permitted to download.")
public void downloadArtifactByNameByNamedController() throws Exception {
downLoadProgress = 1;
shippedBytes = 0;
tenantStatsManagement.resetTrafficStatsOfTenant();
eventBus.register(this);
assertThat(softwareManagement.findSoftwareModulesAll(pageReq)).hasSize(0);
@@ -356,7 +371,7 @@ public class DdiArtifactDownloadTest extends AbstractRestIntegrationTestWithMong
final DistributionSet ds = testdataFactory.createDistributionSet("");
// create artifact
final byte random[] = RandomUtils.nextBytes(5 * 1024 * 1024);
final byte random[] = RandomUtils.nextBytes(ARTIFACT_SIZE);
final Artifact artifact = artifactManagement.createLocalArtifact(new ByteArrayInputStream(random),
ds.findFirstModuleByType(osType).getId(), "file1", false);
@@ -389,6 +404,8 @@ public class DdiArtifactDownloadTest extends AbstractRestIntegrationTestWithMong
// download complete
assertThat(downLoadProgress).isEqualTo(10);
assertThat(shippedBytes).isEqualTo(ARTIFACT_SIZE)
.isEqualTo(tenantStatsManagement.getStatsOfTenant().getOverallArtifactTrafficInBytes());
}
@Test
@@ -550,5 +567,6 @@ public class DdiArtifactDownloadTest extends AbstractRestIntegrationTestWithMong
@Subscribe
public void listen(final DownloadProgressEvent event) {
downLoadProgress++;
shippedBytes += event.getShippedBytes();
}
}

View File

@@ -65,9 +65,11 @@ public interface ControllerManagement {
* the ID of the {@link ActionStatus}
* @param progressPercent
* the progress in percentage which must be between 0-100
* @param shippedBytes
* since last event
*/
@PreAuthorize(SpringEvalExpressions.IS_CONTROLLER)
void downloadProgressPercent(long statusId, int progressPercent);
void downloadProgressPercent(long statusId, int progressPercent, long shippedBytes);
/**
* Simple addition of a new {@link ActionStatus} entry to the {@link Action}

View File

@@ -63,13 +63,18 @@ public interface SystemManagement {
/**
* @return {@link TenantMetaData} of {@link TenantAware#getCurrentTenant()}
*/
// @PreAuthorize(SpringEvalExpressions.HAS_AUTH_READ_REPOSITORY +
// SpringEvalExpressions.HAS_AUTH_OR
// + SpringEvalExpressions.HAS_AUTH_READ_TARGET +
// SpringEvalExpressions.HAS_AUTH_OR
// + SpringEvalExpressions.HAS_AUTH_TENANT_CONFIGURATION)
TenantMetaData getTenantMetadata();
/**
* Returns {@link TenantMetaData} of given and current tenant. Creates for
* new tenants also two {@link SoftwareModuleType} (os and app) and
* {@link RepositoryConstants#DEFAULT_DS_TYPES_IN_TENANT} {@link DistributionSetType}s
* (os and os_app).
* {@link RepositoryConstants#DEFAULT_DS_TYPES_IN_TENANT}
* {@link DistributionSetType}s (os and os_app).
*
* DISCLAIMER: this variant is used during initial login (where the tenant
* is not yet in the session). Please user {@link #getTenantMetadata()} for
@@ -79,6 +84,7 @@ public interface SystemManagement {
* to retrieve data for
* @return {@link TenantMetaData} of given tenant
*/
// @PreAuthorize(SpringEvalExpressions.IS_SYSTEM_CODE)
TenantMetaData getTenantMetadata(@NotNull String tenant);
/**
@@ -88,6 +94,7 @@ public interface SystemManagement {
* to update
* @return updated {@link TenantMetaData} entity
*/
// @PreAuthorize(SpringEvalExpressions.HAS_AUTH_TENANT_CONFIGURATION)
TenantMetaData updateTenantMetadata(@NotNull TenantMetaData metaData);
}

View File

@@ -16,19 +16,23 @@ import org.springframework.security.access.prepost.PreAuthorize;
* Management service for statistics of a single tenant.
*
*/
@FunctionalInterface
public interface TenantStatsManagement {
/**
* Service for stats of a single tenant. Opens a new transaction and as a
* result can an be used for multiple tenants, i.e. to allow in one session
* to collect data of all tenants in the system.
* Service for stats of a single tenant.
*
* @param tenant
* to collect for
* @return collected statistics
*/
@PreAuthorize(SpringEvalExpressions.HAS_AUTH_SYSTEM_ADMIN)
TenantUsage getStatsOfTenant(String tenant);
@PreAuthorize(SpringEvalExpressions.HAS_AUTH_READ_REPOSITORY + SpringEvalExpressions.HAS_AUTH_OR
+ SpringEvalExpressions.HAS_AUTH_READ_TARGET + SpringEvalExpressions.HAS_AUTH_OR
+ SpringEvalExpressions.HAS_AUTH_TENANT_CONFIGURATION)
TenantUsage getStatsOfTenant();
/**
* Resets {@link TenantUsage#getOverallArtifactTrafficInBytes()} to zero.
*
*/
@PreAuthorize(SpringEvalExpressions.HAS_AUTH_TENANT_CONFIGURATION)
void resetTrafficStatsOfTenant();
}

View File

@@ -19,6 +19,7 @@ public class TenantUsage {
private long artifacts;
private long actions;
private long overallArtifactVolumeInBytes;
private long overallArtifactTrafficInBytes;
/**
* Constructor.
@@ -105,12 +106,28 @@ public class TenantUsage {
return this;
}
/**
* @return the overallArtifactTrafficInBytes
*/
public long getOverallArtifactTrafficInBytes() {
return overallArtifactTrafficInBytes;
}
/**
* @param overallArtifactTrafficInBytes
* the overallArtifactTrafficInBytes to set
*/
public void setOverallArtifactTrafficInBytes(final long overallArtifactTrafficInBytes) {
this.overallArtifactTrafficInBytes = overallArtifactTrafficInBytes;
}
@Override
public int hashCode() { // NOSONAR - as this is generated code
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (int) (actions ^ (actions >>> 32));
result = prime * result + (int) (artifacts ^ (artifacts >>> 32));
result = prime * result + (int) (overallArtifactTrafficInBytes ^ (overallArtifactTrafficInBytes >>> 32));
result = prime * result + (int) (overallArtifactVolumeInBytes ^ (overallArtifactVolumeInBytes >>> 32));
result = prime * result + (int) (targets ^ (targets >>> 32));
result = prime * result + ((tenantName == null) ? 0 : tenantName.hashCode());
@@ -118,8 +135,7 @@ public class TenantUsage {
}
@Override
public boolean equals(final Object obj) { // NOSONAR - as this is generated
// code
public boolean equals(final Object obj) {
if (this == obj) {
return true;
}
@@ -136,6 +152,9 @@ public class TenantUsage {
if (artifacts != other.artifacts) {
return false;
}
if (overallArtifactTrafficInBytes != other.overallArtifactTrafficInBytes) {
return false;
}
if (overallArtifactVolumeInBytes != other.overallArtifactVolumeInBytes) {
return false;
}
@@ -154,8 +173,9 @@ public class TenantUsage {
@Override
public String toString() {
return "SystemUsage [tenantName=" + tenantName + ", targets=" + targets + ", artifacts=" + artifacts
+ ", actions=" + actions + ", overallArtifactVolumeInBytes=" + overallArtifactVolumeInBytes + "]";
return "TenantUsage [tenantName=" + tenantName + ", targets=" + targets + ", artifacts=" + artifacts
+ ", actions=" + actions + ", overallArtifactVolumeInBytes=" + overallArtifactVolumeInBytes
+ ", overallArtifactTrafficInBytes=" + overallArtifactTrafficInBytes + "]";
}
}

View File

@@ -54,6 +54,7 @@ import org.eclipse.hawkbit.repository.jpa.model.helper.TenantConfigurationManage
import org.eclipse.hawkbit.security.SecurityTokenGenerator;
import org.eclipse.hawkbit.security.SystemSecurityContext;
import org.eclipse.hawkbit.tenancy.TenantAware;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.orm.jpa.JpaBaseConfiguration;
@@ -71,6 +72,8 @@ import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.annotation.EnableTransactionManagement;
import org.springframework.validation.beanvalidation.MethodValidationPostProcessor;
import com.google.common.eventbus.EventBus;
/**
* General configuration for hawkBit's Repository.
*
@@ -85,6 +88,9 @@ import org.springframework.validation.beanvalidation.MethodValidationPostProcess
@EnableConfigurationProperties(RepositoryProperties.class)
@EnableScheduling
public class RepositoryApplicationConfiguration extends JpaBaseConfiguration {
@Autowired
private EventBus eventBus;
/**
* @return the {@link SystemSecurityContext} singleton bean which make it
* accessible in beans which cannot access the service directly,
@@ -249,7 +255,9 @@ public class RepositoryApplicationConfiguration extends JpaBaseConfiguration {
@Bean
@ConditionalOnMissingBean
public TenantStatsManagement tenantStatsManagement() {
return new JpaTenantStatsManagement();
final TenantStatsManagement mgmt = new JpaTenantStatsManagement();
eventBus.register(mgmt);
return mgmt;
}
/**

View File

@@ -469,8 +469,8 @@ public class JpaControllerManagement implements ControllerManagement {
}
@Override
public void downloadProgressPercent(final long statusId, final int progressPercent) {
cacheWriteNotify.downloadProgressPercent(statusId, progressPercent);
public void downloadProgressPercent(final long statusId, final int progressPercent, final long shippedBytes) {
cacheWriteNotify.downloadProgressPercent(statusId, progressPercent, shippedBytes);
}
}

View File

@@ -147,7 +147,7 @@ public class JpaSystemManagement implements CurrentTenantCacheKeyGenerator, Syst
final List<String> tenants = findTenants();
tenants.forEach(tenant -> tenantAware.runAsTenant(tenant, () -> {
report.addTenantData(systemStatsManagement.getStatsOfTenant(tenant));
report.addTenantData(systemStatsManagement.getStatsOfTenant());
return null;
}));
}

View File

@@ -8,17 +8,23 @@
*/
package org.eclipse.hawkbit.repository.jpa;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import org.eclipse.hawkbit.eventbus.event.DownloadProgressEvent;
import org.eclipse.hawkbit.repository.TenantStatsManagement;
import org.eclipse.hawkbit.repository.report.model.TenantUsage;
import org.eclipse.hawkbit.tenancy.TenantAware;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Isolation;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.validation.annotation.Validated;
import com.google.common.eventbus.Subscribe;
/**
* Management service for statistics of a single tenant.
*
@@ -35,9 +41,16 @@ public class JpaTenantStatsManagement implements TenantStatsManagement {
@Autowired
private ActionRepository actionRepository;
@Autowired
private TenantAware tenantAware;
private final Map<String, AtomicLong> traffic = new ConcurrentHashMap<>();
@Override
@Transactional(propagation = Propagation.REQUIRES_NEW, isolation = Isolation.READ_UNCOMMITTED)
public TenantUsage getStatsOfTenant(final String tenant) {
public TenantUsage getStatsOfTenant() {
final String tenant = tenantAware.getCurrentTenant();
final TenantUsage result = new TenantUsage(tenant);
result.setTargets(targetRepository.count());
@@ -51,9 +64,26 @@ public class JpaTenantStatsManagement implements TenantStatsManagement {
}
result.setActions(actionRepository.count());
if (traffic.containsKey(tenant)) {
result.setOverallArtifactTrafficInBytes(traffic.get(tenant).get());
}
return result;
}
@Override
public void resetTrafficStatsOfTenant() {
traffic.remove(tenantAware.getCurrentTenant());
}
@Subscribe
public void listen(final DownloadProgressEvent event) {
if (traffic.containsKey(event.getTenant())) {
traffic.get(event.getTenant()).addAndGet(event.getShippedBytes());
} else {
traffic.put(event.getTenant(), new AtomicLong(event.getShippedBytes()));
}
}
}

View File

@@ -54,8 +54,10 @@ public class CacheWriteNotify {
* the ID of the {@link ActionStatus}
* @param progressPercent
* the progress in percentage which must be between 0-100
* @param shippedBytes
* since last event
*/
public void downloadProgressPercent(final long statusId, final int progressPercent) {
public void downloadProgressPercent(final long statusId, final int progressPercent, final long shippedBytes) {
final Cache cache = cacheManager.getCache(Action.class.getName());
final String cacheKey = CacheKeys.entitySpecificCacheKey(String.valueOf(statusId),
@@ -69,7 +71,8 @@ public class CacheWriteNotify {
cache.evict(cacheKey);
}
eventBus.post(new DownloadProgressEvent(tenantAware.getCurrentTenant(), statusId, progressPercent));
eventBus.post(
new DownloadProgressEvent(tenantAware.getCurrentTenant(), statusId, progressPercent, shippedBytes));
}
/**

View File

@@ -64,7 +64,7 @@ public class CacheWriteNotifyTest {
when(cacheManagerMock.getCache(Action.class.getName())).thenReturn(cacheMock);
when(tenantAwareMock.getCurrentTenant()).thenReturn("default");
underTest.downloadProgressPercent(knownStatusId, knownPercentage);
underTest.downloadProgressPercent(knownStatusId, knownPercentage, 100L);
verify(cacheManagerMock).getCache(eq(Action.class.getName()));
verify(cacheMock).put(knownStatusId + "." + CacheKeys.DOWNLOAD_PROGRESS_PERCENT, knownPercentage);

View File

@@ -23,6 +23,7 @@ import org.eclipse.hawkbit.repository.TagManagement;
import org.eclipse.hawkbit.repository.TargetFilterQueryManagement;
import org.eclipse.hawkbit.repository.TargetManagement;
import org.eclipse.hawkbit.repository.TenantConfigurationManagement;
import org.eclipse.hawkbit.repository.TenantStatsManagement;
import org.eclipse.hawkbit.repository.model.DistributionSetType;
import org.eclipse.hawkbit.repository.model.SoftwareModuleType;
import org.eclipse.hawkbit.security.DosFilter;
@@ -93,6 +94,9 @@ public abstract class AbstractIntegrationTest implements EnvironmentAware {
@Autowired
protected TargetManagement targetManagement;
@Autowired
protected TenantStatsManagement tenantStatsManagement;
@Autowired
protected TargetFilterQueryManagement targetFilterQueryManagement;

View File

@@ -293,6 +293,7 @@ public final class RestResourceConversionHelper {
long toRead = length;
boolean toContinue = true;
long shippedSinceLastEvent = 0;
while (toContinue) {
final int r = from.read(buf);
@@ -304,9 +305,11 @@ public final class RestResourceConversionHelper {
if (toRead > 0) {
to.write(buf, 0, r);
total += r;
shippedSinceLastEvent += r;
} else {
to.write(buf, 0, (int) toRead + r);
total += toRead + r;
shippedSinceLastEvent += toRead + r;
toContinue = false;
}
@@ -316,7 +319,8 @@ public final class RestResourceConversionHelper {
// every 10 percent an event
if (newPercent == 100 || newPercent > progressPercent + 10) {
progressPercent = newPercent;
controllerManagement.downloadProgressPercent(statusId, progressPercent);
controllerManagement.downloadProgressPercent(statusId, progressPercent, shippedSinceLastEvent);
shippedSinceLastEvent = 0;
}
}
}