Feature Max Artifact Storage quota (#739)

* Enforcement of artifact storage quota
* Added REST integration tests for artifact upload
* Fix test config
* Fix failing test cases
* Fix PR review findings

Signed-off-by: Stefan Behl <stefan.behl@bosch-si.com>
This commit is contained in:
Stefan Behl
2018-10-05 13:52:46 +02:00
committed by GitHub
parent 124ef54a78
commit 086408e8f8
10 changed files with 249 additions and 118 deletions

View File

@@ -92,8 +92,13 @@ public interface QuotaManagement {
int getMaxActionsPerTarget(); int getMaxActionsPerTarget();
/** /**
* @return the maximum size of software artifacts in bytes * @return the maximum size of artifacts in bytes
*/ */
long getMaxArtifactSize(); long getMaxArtifactSize();
/**
* @return the accumulated maximum size of all artifacts in bytes
*/
long getMaxArtifactStorage();
} }

View File

@@ -98,4 +98,9 @@ public class PropertiesQuotaManagement implements QuotaManagement {
return securityProperties.getDos().getMaxArtifactSize(); return securityProperties.getDos().getMaxArtifactSize();
} }
@Override
public long getMaxArtifactStorage() {
return securityProperties.getDos().getMaxArtifactStorage();
}
} }

View File

@@ -54,6 +54,8 @@ public class JpaArtifactManagement implements ArtifactManagement {
private static final String MAX_ARTIFACT_SIZE_EXCEEDED = "Quota exceeded: The artifact '%s' (%s bytes) which has been uploaded for software module '%s' exceeds the maximum artifact size of %s bytes."; private static final String MAX_ARTIFACT_SIZE_EXCEEDED = "Quota exceeded: The artifact '%s' (%s bytes) which has been uploaded for software module '%s' exceeds the maximum artifact size of %s bytes.";
private static final String MAX_ARTIFACT_SIZE_TOTAL_EXCEEDED = "Quota exceeded: The artifact '%s' (%s bytes) cannot be uploaded. The maximum total artifact storage of %s bytes would be exceeded.";
private final LocalArtifactRepository localArtifactRepository; private final LocalArtifactRepository localArtifactRepository;
private final SoftwareModuleRepository softwareModuleRepository; private final SoftwareModuleRepository softwareModuleRepository;
@@ -104,6 +106,7 @@ public class JpaArtifactManagement implements ArtifactManagement {
assertArtifactQuota(moduleId, 1); assertArtifactQuota(moduleId, 1);
assertMaxArtifactSizeQuota(filename, moduleId, filesize); assertMaxArtifactSizeQuota(filename, moduleId, filesize);
assertMaxArtifactStorageQuota(filename, filesize);
try { try {
result = artifactRepository.store(tenantAware.getCurrentTenant(), stream, filename, contentType, result = artifactRepository.store(tenantAware.getCurrentTenant(), stream, filename, contentType,
@@ -141,6 +144,21 @@ public class JpaArtifactManagement implements ArtifactManagement {
} }
} }
private void assertMaxArtifactStorageQuota(final String filename, final long artifactSize) {
final long maxArtifactSizeTotal = quotaManagement.getMaxArtifactStorage();
if (maxArtifactSizeTotal <= 0) {
return;
}
final Long currentlyUsed = localArtifactRepository.getSumOfUndeletedArtifactSize().orElse(0L);
if (currentlyUsed + artifactSize > maxArtifactSizeTotal) {
final String msg = String.format(MAX_ARTIFACT_SIZE_TOTAL_EXCEEDED, filename, artifactSize,
maxArtifactSizeTotal);
LOG.warn(msg);
throw new QuotaExceededException(msg);
}
}
@Override @Override
@Transactional @Transactional
@Retryable(include = { @Retryable(include = {

View File

@@ -149,7 +149,8 @@ public class RepositoryApplicationConfiguration extends JpaBaseConfiguration {
} }
@Bean @Bean
PropertiesQuotaManagement staticQuotaManagement(final HawkbitSecurityProperties securityProperties) { @ConditionalOnMissingBean
QuotaManagement staticQuotaManagement(final HawkbitSecurityProperties securityProperties) {
return new PropertiesQuotaManagement(securityProperties); return new PropertiesQuotaManagement(securityProperties);
} }

View File

@@ -104,30 +104,30 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest {
softwareModuleRepository.save(new JpaSoftwareModule(osType, "name 3", "version 3", null, null)); softwareModuleRepository.save(new JpaSoftwareModule(osType, "name 3", "version 3", null, null));
final int artifactSize = 5 * 1024; final int artifactSize = 5 * 1024;
final byte random[] = RandomStringUtils.random(artifactSize).getBytes(); final byte[] randomBytes = randomBytes(artifactSize);
try (final InputStream inputStream1 = new ByteArrayInputStream(random); try (final InputStream inputStream1 = new ByteArrayInputStream(randomBytes);
final InputStream inputStream2 = new ByteArrayInputStream(random); final InputStream inputStream2 = new ByteArrayInputStream(randomBytes);
final InputStream inputStream3 = new ByteArrayInputStream(random); final InputStream inputStream3 = new ByteArrayInputStream(randomBytes);
final InputStream inputStream4 = new ByteArrayInputStream(random);) { final InputStream inputStream4 = new ByteArrayInputStream(randomBytes);) {
final Artifact result = artifactManagement.create(inputStream1, sm.getId(), "file1", false, artifactSize); final Artifact artifact1 = createArtifactForSoftwareModule("file1", sm.getId(), artifactSize, inputStream1);
artifactManagement.create(inputStream2, sm.getId(), "file11", false, artifactSize); createArtifactForSoftwareModule("file11", sm.getId(), artifactSize, inputStream2);
artifactManagement.create(inputStream3, sm.getId(), "file12", false, artifactSize); createArtifactForSoftwareModule("file12", sm.getId(), artifactSize, inputStream3);
final Artifact result2 = artifactManagement.create(inputStream4, sm2.getId(), "file2", false, artifactSize); final Artifact artifact2 = createArtifactForSoftwareModule("file2", sm2.getId(), artifactSize, inputStream4);
assertThat(result).isInstanceOf(Artifact.class); assertThat(artifact1).isInstanceOf(Artifact.class);
assertThat(result.getSoftwareModule().getId()).isEqualTo(sm.getId()); assertThat(artifact1.getSoftwareModule().getId()).isEqualTo(sm.getId());
assertThat(result2.getSoftwareModule().getId()).isEqualTo(sm2.getId()); assertThat(artifact2.getSoftwareModule().getId()).isEqualTo(sm2.getId());
assertThat(((JpaArtifact) result).getFilename()).isEqualTo("file1"); assertThat(((JpaArtifact) artifact1).getFilename()).isEqualTo("file1");
assertThat(((JpaArtifact) result).getSha1Hash()).isNotNull(); assertThat(((JpaArtifact) artifact1).getSha1Hash()).isNotNull();
assertThat(result).isNotEqualTo(result2); assertThat(artifact1).isNotEqualTo(artifact2);
assertThat(((JpaArtifact) result).getSha1Hash()).isEqualTo(((JpaArtifact) result2).getSha1Hash()); assertThat(((JpaArtifact) artifact1).getSha1Hash()).isEqualTo(((JpaArtifact) artifact2).getSha1Hash());
assertThat(artifactManagement.getByFilename("file1").get().getSha1Hash()) assertThat(artifactManagement.getByFilename("file1").get().getSha1Hash())
.isEqualTo(HashGeneratorUtils.generateSHA1(random)); .isEqualTo(HashGeneratorUtils.generateSHA1(randomBytes));
assertThat(artifactManagement.getByFilename("file1").get().getMd5Hash()) assertThat(artifactManagement.getByFilename("file1").get().getMd5Hash())
.isEqualTo(HashGeneratorUtils.generateMD5(random)); .isEqualTo(HashGeneratorUtils.generateMD5(randomBytes));
assertThat(artifactRepository.findAll()).hasSize(4); assertThat(artifactRepository.findAll()).hasSize(4);
assertThat(softwareModuleRepository.findAll()).hasSize(3); assertThat(softwareModuleRepository.findAll()).hasSize(3);
@@ -142,43 +142,73 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest {
public void createArtifactsUntilQuotaIsExceeded() throws NoSuchAlgorithmException, IOException { public void createArtifactsUntilQuotaIsExceeded() throws NoSuchAlgorithmException, IOException {
// create a software module // create a software module
final JpaSoftwareModule sm1 = softwareModuleRepository final long smId = softwareModuleRepository.save(new JpaSoftwareModule(osType, "sm1", "1.0", null, null))
.save(new JpaSoftwareModule(osType, "sm1", "1.0", null, null)); .getId();
// now create artifacts for this module until the quota is exceeded // now create artifacts for this module until the quota is exceeded
final long maxArtifacts = quotaManagement.getMaxArtifactsPerSoftwareModule(); final long maxArtifacts = quotaManagement.getMaxArtifactsPerSoftwareModule();
final List<Long> artifactIds = Lists.newArrayList(); final List<Long> artifactIds = Lists.newArrayList();
final int artifactSize = 5 * 1024; final int artifactSize = 5 * 1024;
for (int i = 0; i < maxArtifacts; ++i) { for (int i = 0; i < maxArtifacts; ++i) {
final byte random[] = RandomStringUtils.random(artifactSize).getBytes(); artifactIds.add(createArtifactForSoftwareModule("file" + i, smId, artifactSize).getId());
try (final InputStream inputStream = new ByteArrayInputStream(random)) {
artifactIds.add(
artifactManagement.create(inputStream, sm1.getId(), "file" + i, false, artifactSize).getId());
} }
} assertThat(artifactRepository.findBySoftwareModuleId(PAGE, smId).getTotalElements()).isEqualTo(maxArtifacts);
assertThat(artifactRepository.findBySoftwareModuleId(PAGE, sm1.getId()).getTotalElements())
.isEqualTo(maxArtifacts);
// create one mode to trigger the quota exceeded error // create one mode to trigger the quota exceeded error
assertThatExceptionOfType(QuotaExceededException.class).isThrownBy(() -> { assertThatExceptionOfType(QuotaExceededException.class)
final byte random[] = RandomStringUtils.random(artifactSize).getBytes(); .isThrownBy(() -> createArtifactForSoftwareModule("file" + maxArtifacts, smId, artifactSize));
try (final InputStream inputStream = new ByteArrayInputStream(random)) {
artifactManagement.create(inputStream, sm1.getId(), "file" + maxArtifacts, false, artifactSize);
}
});
// delete one of the artifacts // delete one of the artifacts
artifactManagement.delete(artifactIds.get(0)); artifactManagement.delete(artifactIds.get(0));
assertThat(artifactRepository.findBySoftwareModuleId(PAGE, sm1.getId()).getTotalElements()) assertThat(artifactRepository.findBySoftwareModuleId(PAGE, smId).getTotalElements())
.isEqualTo(maxArtifacts - 1); .isEqualTo(maxArtifacts - 1);
// now we should be able to create an artifact again // now we should be able to create an artifact again
final byte random[] = RandomStringUtils.random(artifactSize).getBytes(); createArtifactForSoftwareModule("fileXYZ", smId, artifactSize);
try (final InputStream inputStream = new ByteArrayInputStream(random)) { assertThat(artifactRepository.findBySoftwareModuleId(PAGE, smId).getTotalElements()).isEqualTo(maxArtifacts);
artifactManagement.create(inputStream, sm1.getId(), "fileXYZ", false, artifactSize);
assertThat(artifactRepository.findBySoftwareModuleId(PAGE, sm1.getId()).getTotalElements())
.isEqualTo(maxArtifacts);
} }
@Test
@Description("Verifies that the quota specifying the maximum artifact storage is enforced (across software modules).")
public void createArtifactsUntilStorageQuotaIsExceeded() throws NoSuchAlgorithmException, IOException {
// create as many small artifacts as possible w/o violating the storage
// quota
final long maxBytes = quotaManagement.getMaxArtifactStorage();
final List<Long> artifactIds = Lists.newArrayList();
// choose an artifact size which does not violate the max file size
final int artifactSize = Math.toIntExact(quotaManagement.getMaxArtifactSize() / 10);
final int numArtifacts = Math.toIntExact(maxBytes / artifactSize);
for (int i = 0; i < numArtifacts; ++i) {
final JpaSoftwareModule sm = softwareModuleRepository
.save(new JpaSoftwareModule(osType, "smd" + i, "1.0", null, null));
artifactIds.add(createArtifactForSoftwareModule("file" + i, sm.getId(), artifactSize).getId());
}
// upload one more artifact to trigger the quota exceeded error
final JpaSoftwareModule sm = softwareModuleRepository
.save(new JpaSoftwareModule(osType, "smd" + numArtifacts, "1.0", null, null));
assertThatExceptionOfType(QuotaExceededException.class)
.isThrownBy(() -> createArtifactForSoftwareModule("file" + numArtifacts, sm.getId(), artifactSize));
// delete one of the artifacts
artifactManagement.delete(artifactIds.get(0));
// now we should be able to create an artifact again
createArtifactForSoftwareModule("fileXYZ", sm.getId(), artifactSize);
}
@Test
@Description("Verifies that the quota specifying the maximum artifact storage is enforced (across software modules).")
public void createArtifactWhichExceedsMaxStorage() throws NoSuchAlgorithmException, IOException {
// create one artifact which exceeds the storage quota at once
final long maxBytes = quotaManagement.getMaxArtifactStorage();
final JpaSoftwareModule sm = softwareModuleRepository
.save(new JpaSoftwareModule(osType, "smd345", "1.0", null, null));
assertThatExceptionOfType(QuotaExceededException.class).isThrownBy(
() -> createArtifactForSoftwareModule("file345", sm.getId(), Math.toIntExact(maxBytes) + 128));
} }
@Test @Test
@@ -191,12 +221,8 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest {
// create an artifact that exceeds the configured quota // create an artifact that exceeds the configured quota
final long maxSize = quotaManagement.getMaxArtifactSize(); final long maxSize = quotaManagement.getMaxArtifactSize();
final int artifactSize = Math.toIntExact(maxSize) + 8;
final byte random[] = RandomStringUtils.random(artifactSize).getBytes();
try (final InputStream inputStream = new ByteArrayInputStream(random)) {
assertThatExceptionOfType(QuotaExceededException.class) assertThatExceptionOfType(QuotaExceededException.class)
.isThrownBy(() -> artifactManagement.create(inputStream, sm1.getId(), "file", false, artifactSize)); .isThrownBy(() -> createArtifactForSoftwareModule("file", sm1.getId(), Math.toIntExact(maxSize) + 8));
}
} }
@Test @Test
@@ -205,16 +231,13 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest {
final JpaSoftwareModule sm = softwareModuleRepository final JpaSoftwareModule sm = softwareModuleRepository
.save(new JpaSoftwareModule(osType, "name 1", "version 1", null, null)); .save(new JpaSoftwareModule(osType, "name 1", "version 1", null, null));
final int artifactSize = 5 * 1024;
final byte random[] = RandomStringUtils.random(artifactSize).getBytes(); createArtifactForSoftwareModule("file1", sm.getId(), 5 * 1024);
try (final InputStream inputStream = new ByteArrayInputStream(random)) {
artifactManagement.create(inputStream, sm.getId(), "file1", false, artifactSize);
assertThat(artifactRepository.findAll()).hasSize(1); assertThat(artifactRepository.findAll()).hasSize(1);
softwareModuleRepository.deleteAll(); softwareModuleRepository.deleteAll();
assertThat(artifactRepository.findAll()).hasSize(0); assertThat(artifactRepository.findAll()).hasSize(0);
} }
}
/** /**
* Test method for * Test method for
@@ -239,32 +262,32 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest {
try (final InputStream inputStream1 = new RandomGeneratedInputStream(artifactSize); try (final InputStream inputStream1 = new RandomGeneratedInputStream(artifactSize);
final InputStream inputStream2 = new RandomGeneratedInputStream(artifactSize)) { final InputStream inputStream2 = new RandomGeneratedInputStream(artifactSize)) {
final Artifact result = artifactManagement.create(inputStream1, sm.getId(), "file1", false, artifactSize); final Artifact artifact1 = createArtifactForSoftwareModule("file1", sm.getId(), artifactSize, inputStream1);
final Artifact result2 = artifactManagement.create(inputStream2, sm2.getId(), "file2", false, artifactSize); final Artifact artifact2 = createArtifactForSoftwareModule("file2", sm2.getId(), artifactSize, inputStream2);
assertThat(artifactRepository.findAll()).hasSize(2); assertThat(artifactRepository.findAll()).hasSize(2);
assertThat(result.getId()).isNotNull(); assertThat(artifact1.getId()).isNotNull();
assertThat(result2.getId()).isNotNull(); assertThat(artifact2.getId()).isNotNull();
assertThat(((JpaArtifact) result).getSha1Hash()).isNotEqualTo(((JpaArtifact) result2).getSha1Hash()); assertThat(((JpaArtifact) artifact1).getSha1Hash()).isNotEqualTo(((JpaArtifact) artifact2).getSha1Hash());
assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result.getSha1Hash())) assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact1.getSha1Hash()))
.isNotNull(); .isNotNull();
assertThat( assertThat(
binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result2.getSha1Hash())) binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact2.getSha1Hash()))
.isNotNull(); .isNotNull();
artifactManagement.delete(result.getId()); artifactManagement.delete(artifact1.getId());
assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result.getSha1Hash())) assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact1.getSha1Hash()))
.isNull(); .isNull();
assertThat( assertThat(
binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result2.getSha1Hash())) binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact2.getSha1Hash()))
.isNotNull(); .isNotNull();
artifactManagement.delete(result2.getId()); artifactManagement.delete(artifact2.getId());
assertThat( assertThat(
binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result2.getSha1Hash())) binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact2.getSha1Hash()))
.isNull(); .isNull();
assertThat(artifactRepository.findAll()).hasSize(0); assertThat(artifactRepository.findAll()).hasSize(0);
@@ -282,26 +305,26 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest {
.save(new JpaSoftwareModule(osType, "name 2", "version 2", null, null)); .save(new JpaSoftwareModule(osType, "name 2", "version 2", null, null));
final int artifactSize = 5 * 1024; final int artifactSize = 5 * 1024;
final byte random[] = RandomStringUtils.random(artifactSize).getBytes(); final byte[] randomBytes = randomBytes(artifactSize);
try (final InputStream inputStream1 = new ByteArrayInputStream(random); try (final InputStream inputStream1 = new ByteArrayInputStream(randomBytes);
final InputStream inputStream2 = new ByteArrayInputStream(random)) { final InputStream inputStream2 = new ByteArrayInputStream(randomBytes)) {
final Artifact result = artifactManagement.create(inputStream1, sm.getId(), "file1", false, artifactSize); final Artifact artifact1 = createArtifactForSoftwareModule("file1", sm.getId(), artifactSize, inputStream1);
final Artifact result2 = artifactManagement.create(inputStream2, sm2.getId(), "file2", false, artifactSize); final Artifact artifact2 = createArtifactForSoftwareModule("file2", sm2.getId(), artifactSize, inputStream2);
assertThat(artifactRepository.findAll()).hasSize(2); assertThat(artifactRepository.findAll()).hasSize(2);
assertThat(result.getId()).isNotNull(); assertThat(artifact1.getId()).isNotNull();
assertThat(result2.getId()).isNotNull(); assertThat(artifact2.getId()).isNotNull();
assertThat(((JpaArtifact) result).getSha1Hash()).isEqualTo(((JpaArtifact) result2).getSha1Hash()); assertThat(((JpaArtifact) artifact1).getSha1Hash()).isEqualTo(((JpaArtifact) artifact2).getSha1Hash());
assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result.getSha1Hash())) assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact1.getSha1Hash()))
.isNotNull(); .isNotNull();
artifactManagement.delete(result.getId()); artifactManagement.delete(artifact1.getId());
assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result.getSha1Hash())) assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact1.getSha1Hash()))
.isNotNull(); .isNotNull();
artifactManagement.delete(result2.getId()); artifactManagement.delete(artifact2.getId());
assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result.getSha1Hash())) assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact1.getSha1Hash()))
.isNull(); .isNull();
} }
} }
@@ -311,9 +334,9 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest {
public void findArtifact() throws NoSuchAlgorithmException, IOException { public void findArtifact() throws NoSuchAlgorithmException, IOException {
final int artifactSize = 5 * 1024; final int artifactSize = 5 * 1024;
try (final InputStream inputStream = new RandomGeneratedInputStream(artifactSize)) { try (final InputStream inputStream = new RandomGeneratedInputStream(artifactSize)) {
final Artifact result = artifactManagement.create(inputStream, final Artifact artifact = createArtifactForSoftwareModule("file1",
testdataFactory.createSoftwareModuleOs().getId(), "file1", false, artifactSize); testdataFactory.createSoftwareModuleOs().getId(), artifactSize, inputStream);
assertThat(artifactManagement.get(result.getId()).get()).isEqualTo(result); assertThat(artifactManagement.get(artifact.getId()).get()).isEqualTo(artifact);
} }
} }
@@ -321,14 +344,14 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest {
@Description("Loads an artifact binary based on given ID.") @Description("Loads an artifact binary based on given ID.")
public void loadStreamOfArtifact() throws NoSuchAlgorithmException, IOException { public void loadStreamOfArtifact() throws NoSuchAlgorithmException, IOException {
final int artifactSize = 5 * 1024; final int artifactSize = 5 * 1024;
final byte random[] = RandomStringUtils.random(artifactSize).getBytes(); final byte[] randomBytes = randomBytes(artifactSize);
try (final InputStream input = new ByteArrayInputStream(random)) { try (final InputStream input = new ByteArrayInputStream(randomBytes)) {
final Artifact result = artifactManagement.create(input, testdataFactory.createSoftwareModuleOs().getId(), final Artifact artifact = createArtifactForSoftwareModule("file1",
"file1", false, artifactSize); testdataFactory.createSoftwareModuleOs().getId(), artifactSize, input);
try (final InputStream inputStream = artifactManagement.loadArtifactBinary(result.getSha1Hash()).get() try (final InputStream inputStream = artifactManagement.loadArtifactBinary(artifact.getSha1Hash()).get()
.getFileInputStream()) { .getFileInputStream()) {
assertTrue("The stored binary matches the given binary", assertTrue("The stored binary matches the given binary",
IOUtils.contentEquals(new ByteArrayInputStream(random), inputStream)); IOUtils.contentEquals(new ByteArrayInputStream(randomBytes), inputStream));
} }
} }
} }
@@ -353,7 +376,7 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest {
final int artifactSize = 5 * 1024; final int artifactSize = 5 * 1024;
try (final InputStream input = new RandomGeneratedInputStream(artifactSize)) { try (final InputStream input = new RandomGeneratedInputStream(artifactSize)) {
artifactManagement.create(input, sm.getId(), "file1", false, artifactSize); createArtifactForSoftwareModule("file1", sm.getId(), artifactSize, input);
assertThat(artifactManagement.findBySoftwareModule(PAGE, sm.getId())).hasSize(1); assertThat(artifactManagement.findBySoftwareModule(PAGE, sm.getId())).hasSize(1);
} }
} }
@@ -368,10 +391,27 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest {
final int artifactSize = 5 * 1024; final int artifactSize = 5 * 1024;
try (final InputStream inputStream1 = new RandomGeneratedInputStream(artifactSize); try (final InputStream inputStream1 = new RandomGeneratedInputStream(artifactSize);
final InputStream inputStream2 = new RandomGeneratedInputStream(artifactSize)) { final InputStream inputStream2 = new RandomGeneratedInputStream(artifactSize)) {
artifactManagement.create(inputStream1, sm.getId(), "file1", false, artifactSize); createArtifactForSoftwareModule("file1", sm.getId(), artifactSize, inputStream1);
artifactManagement.create(inputStream2, sm.getId(), "file2", false, artifactSize); createArtifactForSoftwareModule("file2", sm.getId(), artifactSize, inputStream2);
assertThat(artifactManagement.getByFilenameAndSoftwareModule("file1", sm.getId())).isPresent(); assertThat(artifactManagement.getByFilenameAndSoftwareModule("file1", sm.getId())).isPresent();
} }
} }
private Artifact createArtifactForSoftwareModule(final String filename, final long moduleId, final int artifactSize)
throws IOException {
final byte[] randomBytes = randomBytes(artifactSize);
try (final InputStream inputStream = new ByteArrayInputStream(randomBytes)) {
return createArtifactForSoftwareModule(filename, moduleId, artifactSize, inputStream);
}
}
private Artifact createArtifactForSoftwareModule(final String filename, final long moduleId, final int artifactSize,
final InputStream inputStream) throws IOException {
return artifactManagement.create(inputStream, moduleId, filename, false, artifactSize);
}
private static byte[] randomBytes(final int len) {
return RandomStringUtils.randomAlphanumeric(len).getBytes();
}
} }

View File

@@ -280,7 +280,7 @@ public class SoftwareModuleManagementTest extends AbstractJpaIntegrationTest {
assertThat(softwareModuleManagement.get(unassignedModule.getId())).isNotPresent(); assertThat(softwareModuleManagement.get(unassignedModule.getId())).isNotPresent();
// verify: binary data of artifact is deleted // verify: binary data of artifact is deleted
assertArtfiactNull(artifact1, artifact2); assertArtifactNull(artifact1, artifact2);
// verify: meta data of artifact is deleted // verify: meta data of artifact is deleted
assertThat(artifactRepository.findOne(artifact1.getId())).isNull(); assertThat(artifactRepository.findOne(artifact1.getId())).isNull();
@@ -311,7 +311,7 @@ public class SoftwareModuleManagementTest extends AbstractJpaIntegrationTest {
final Iterator<Artifact> artifactsIt = assignedModule.getArtifacts().iterator(); final Iterator<Artifact> artifactsIt = assignedModule.getArtifacts().iterator();
final Artifact artifact1 = artifactsIt.next(); final Artifact artifact1 = artifactsIt.next();
final Artifact artifact2 = artifactsIt.next(); final Artifact artifact2 = artifactsIt.next();
assertArtfiactNull(artifact1, artifact2); assertArtifactNull(artifact1, artifact2);
// verify: artifact meta data is still available // verify: artifact meta data is still available
assertThat(artifactRepository.findOne(artifact1.getId())).isNotNull(); assertThat(artifactRepository.findOne(artifact1.getId())).isNotNull();
@@ -351,7 +351,7 @@ public class SoftwareModuleManagementTest extends AbstractJpaIntegrationTest {
final Iterator<Artifact> artifactsIt = assignedModule.getArtifacts().iterator(); final Iterator<Artifact> artifactsIt = assignedModule.getArtifacts().iterator();
final Artifact artifact1 = artifactsIt.next(); final Artifact artifact1 = artifactsIt.next();
final Artifact artifact2 = artifactsIt.next(); final Artifact artifact2 = artifactsIt.next();
assertArtfiactNull(artifact1, artifact2); assertArtifactNull(artifact1, artifact2);
// verify: artifact meta data is still available // verify: artifact meta data is still available
assertThat(artifactRepository.findOne(artifact1.getId())).isNotNull(); assertThat(artifactRepository.findOne(artifact1.getId())).isNotNull();
@@ -392,7 +392,7 @@ public class SoftwareModuleManagementTest extends AbstractJpaIntegrationTest {
assertThat(softwareModuleManagement.get(moduleY.getId())).isPresent(); assertThat(softwareModuleManagement.get(moduleY.getId())).isPresent();
// verify: binary data of artifact is not deleted // verify: binary data of artifact is not deleted
assertArtfiactNotNull(artifactY); assertArtifactNotNull(artifactY);
// verify: meta data of artifactX is deleted // verify: meta data of artifactX is deleted
assertThat(artifactRepository.findOne(artifactX.getId())).isNull(); assertThat(artifactRepository.findOne(artifactX.getId())).isNull();
@@ -451,7 +451,7 @@ public class SoftwareModuleManagementTest extends AbstractJpaIntegrationTest {
assertThat(softwareModuleRepository.findAll()).hasSize(2); assertThat(softwareModuleRepository.findAll()).hasSize(2);
// verify: binary data of artifact is deleted // verify: binary data of artifact is deleted
assertArtfiactNull(artifactX, artifactY); assertArtifactNull(artifactX, artifactY);
// verify: meta data of artifactX and artifactY is not deleted // verify: meta data of artifactX and artifactY is not deleted
assertThat(artifactRepository.findOne(artifactY.getId())).isNotNull(); assertThat(artifactRepository.findOne(artifactY.getId())).isNotNull();
@@ -480,14 +480,14 @@ public class SoftwareModuleManagementTest extends AbstractJpaIntegrationTest {
assertThat(artifacts).hasSize(numberArtifacts); assertThat(artifacts).hasSize(numberArtifacts);
if (numberArtifacts != 0) { if (numberArtifacts != 0) {
assertArtfiactNotNull(artifacts.toArray(new Artifact[artifacts.size()])); assertArtifactNotNull(artifacts.toArray(new Artifact[artifacts.size()]));
} }
artifacts.forEach(artifact -> assertThat(artifactRepository.findOne(artifact.getId())).isNotNull()); artifacts.forEach(artifact -> assertThat(artifactRepository.findOne(artifact.getId())).isNotNull());
return softwareModule; return softwareModule;
} }
private void assertArtfiactNotNull(final Artifact... results) { private void assertArtifactNotNull(final Artifact... results) {
assertThat(artifactRepository.findAll()).hasSize(results.length); assertThat(artifactRepository.findAll()).hasSize(results.length);
for (final Artifact result : results) { for (final Artifact result : results) {
assertThat(result.getId()).isNotNull(); assertThat(result.getId()).isNotNull();
@@ -496,7 +496,7 @@ public class SoftwareModuleManagementTest extends AbstractJpaIntegrationTest {
} }
} }
private void assertArtfiactNull(final Artifact... results) { private void assertArtifactNull(final Artifact... results) {
for (final Artifact result : results) { for (final Artifact result : results) {
assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result.getSha1Hash())) assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result.getSha1Hash()))
.isNull(); .isNull();

View File

@@ -18,7 +18,8 @@ hawkbit.server.security.dos.maxSoftwareModuleTypesPerDistributionSetType=10
hawkbit.server.security.dos.maxSoftwareModulesPerDistributionSet=10 hawkbit.server.security.dos.maxSoftwareModulesPerDistributionSet=10
hawkbit.server.security.dos.maxArtifactsPerSoftwareModule=10 hawkbit.server.security.dos.maxArtifactsPerSoftwareModule=10
hawkbit.server.security.dos.maxTargetsPerRolloutGroup=1000 hawkbit.server.security.dos.maxTargetsPerRolloutGroup=1000
hawkbit.server.security.dos.maxArtifactSize=1000000 hawkbit.server.security.dos.maxArtifactSize=600000
hawkbit.server.security.dos.maxArtifactStorage=1000000
# Quota - END # Quota - END
# Debug utility functions - START # Debug utility functions - START

View File

@@ -29,7 +29,6 @@ import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.Random;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomStringUtils;
@@ -70,7 +69,8 @@ import ru.yandex.qatools.allure.annotations.Stories;
*/ */
@Features("Component Tests - Management API") @Features("Component Tests - Management API")
@Stories("Software Module Resource") @Stories("Software Module Resource")
@TestPropertySource(properties = { "hawkbit.server.security.dos.maxArtifactSize=100000" }) @TestPropertySource(properties = { "hawkbit.server.security.dos.maxArtifactSize=100000",
"hawkbit.server.security.dos.maxArtifactStorage=500000" })
public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegrationTest { public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegrationTest {
@Before @Before
@@ -157,7 +157,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
final SoftwareModule sm = testdataFactory.createSoftwareModuleOs(); final SoftwareModule sm = testdataFactory.createSoftwareModuleOs();
// create test file // create test file
final byte random[] = RandomStringUtils.random(5 * 1024).getBytes(); final byte random[] = randomBytes(5 * 1024);
final String md5sum = HashGeneratorUtils.generateMD5(random); final String md5sum = HashGeneratorUtils.generateMD5(random);
final String sha1sum = HashGeneratorUtils.generateSHA1(random); final String sha1sum = HashGeneratorUtils.generateSHA1(random);
final MockMultipartFile file = new MockMultipartFile("file", "origFilename", null, random); final MockMultipartFile file = new MockMultipartFile("file", "origFilename", null, random);
@@ -195,8 +195,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
final long maxSize = quotaManagement.getMaxArtifactSize(); final long maxSize = quotaManagement.getMaxArtifactSize();
// create a file which exceeds the configured maximum size // create a file which exceeds the configured maximum size
final byte[] randomBytes = new byte[Math.toIntExact(maxSize) + 1024]; final byte[] randomBytes = randomBytes(Math.toIntExact(maxSize) + 1024);
new Random().nextBytes(randomBytes);
final MockMultipartFile file = new MockMultipartFile("file", "origFilename", null, randomBytes); final MockMultipartFile file = new MockMultipartFile("file", "origFilename", null, randomBytes);
@@ -251,7 +250,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
public void duplicateUploadArtifact() throws Exception { public void duplicateUploadArtifact() throws Exception {
final SoftwareModule sm = testdataFactory.createSoftwareModuleOs(); final SoftwareModule sm = testdataFactory.createSoftwareModuleOs();
final byte random[] = RandomStringUtils.random(5 * 1024).getBytes(); final byte random[] = randomBytes(5 * 1024);
final String md5sum = HashGeneratorUtils.generateMD5(random); final String md5sum = HashGeneratorUtils.generateMD5(random);
final String sha1sum = HashGeneratorUtils.generateSHA1(random); final String sha1sum = HashGeneratorUtils.generateSHA1(random);
final MockMultipartFile file = new MockMultipartFile("file", "orig", null, random); final MockMultipartFile file = new MockMultipartFile("file", "orig", null, random);
@@ -274,7 +273,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
assertThat(artifactManagement.count()).isEqualTo(0); assertThat(artifactManagement.count()).isEqualTo(0);
// create test file // create test file
final byte random[] = RandomStringUtils.random(5 * 1024).getBytes(); final byte random[] = randomBytes(5 * 1024);
final MockMultipartFile file = new MockMultipartFile("file", "origFilename", null, random); final MockMultipartFile file = new MockMultipartFile("file", "origFilename", null, random);
// upload // upload
@@ -299,7 +298,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
assertThat(artifactManagement.count()).isEqualTo(0); assertThat(artifactManagement.count()).isEqualTo(0);
// create test file // create test file
final byte random[] = RandomStringUtils.random(5 * 1024).getBytes(); final byte random[] = randomBytes(5 * 1024);
final String md5sum = HashGeneratorUtils.generateMD5(random); final String md5sum = HashGeneratorUtils.generateMD5(random);
final String sha1sum = HashGeneratorUtils.generateSHA1(random); final String sha1sum = HashGeneratorUtils.generateSHA1(random);
final MockMultipartFile file = new MockMultipartFile("file", "origFilename", null, random); final MockMultipartFile file = new MockMultipartFile("file", "origFilename", null, random);
@@ -343,7 +342,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
for (int i = 0; i < maxArtifacts; ++i) { for (int i = 0; i < maxArtifacts; ++i) {
// create test file // create test file
final byte random[] = RandomStringUtils.random(5 * 1024).getBytes(); final byte random[] = randomBytes(5 * 1024);
final String md5sum = HashGeneratorUtils.generateMD5(random); final String md5sum = HashGeneratorUtils.generateMD5(random);
final String sha1sum = HashGeneratorUtils.generateSHA1(random); final String sha1sum = HashGeneratorUtils.generateSHA1(random);
final MockMultipartFile file = new MockMultipartFile("file", "origFilename" + i, null, random); final MockMultipartFile file = new MockMultipartFile("file", "origFilename" + i, null, random);
@@ -360,7 +359,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
} }
// upload one more file to cause the quota to be exceeded // upload one more file to cause the quota to be exceeded
final byte random[] = RandomStringUtils.random(5 * 1024).getBytes(); final byte random[] = randomBytes(5 * 1024);
HashGeneratorUtils.generateMD5(random); HashGeneratorUtils.generateMD5(random);
HashGeneratorUtils.generateSHA1(random); HashGeneratorUtils.generateSHA1(random);
final MockMultipartFile file = new MockMultipartFile("file", "origFilename_final", null, random); final MockMultipartFile file = new MockMultipartFile("file", "origFilename_final", null, random);
@@ -374,13 +373,58 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
} }
@Test
@Description("Verifies that artifacts can only be added as long as the artifact storage quota is not exceeded.")
public void uploadArtifactsUntilStorageQuotaExceeded() throws Exception {
final long storageLimit = quotaManagement.getMaxArtifactStorage();
// choose an artifact size which does not violate the max file size
final int artifactSize = Math.toIntExact(quotaManagement.getMaxArtifactSize() / 10);
final int numArtifacts = Math.toIntExact(storageLimit / artifactSize);
for (int i = 0; i < numArtifacts; ++i) {
// create test file
final byte random[] = randomBytes(artifactSize);
final String md5sum = HashGeneratorUtils.generateMD5(random);
final String sha1sum = HashGeneratorUtils.generateSHA1(random);
final MockMultipartFile file = new MockMultipartFile("file", "origFilename" + i, null, random);
// upload
final SoftwareModule sm = testdataFactory.createSoftwareModuleOs("sm" + i);
mvc.perform(fileUpload("/rest/v1/softwaremodules/{smId}/artifacts", sm.getId()).file(file)
.accept(MediaType.APPLICATION_JSON)).andDo(MockMvcResultPrinter.print())
.andExpect(status().isCreated())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8_VALUE))
.andExpect(jsonPath("$.hashes.md5", equalTo(md5sum)))
.andExpect(jsonPath("$.hashes.sha1", equalTo(sha1sum)))
.andExpect(jsonPath("$.size", equalTo(random.length)))
.andExpect(jsonPath("$.providedFilename", equalTo("origFilename" + i))).andReturn();
}
// upload one more file to cause the quota to be exceeded
final byte random[] = randomBytes(artifactSize);
HashGeneratorUtils.generateMD5(random);
HashGeneratorUtils.generateSHA1(random);
final MockMultipartFile file = new MockMultipartFile("file", "origFilename_final", null, random);
// upload
final SoftwareModule sm = testdataFactory.createSoftwareModuleOs("sm" + numArtifacts);
mvc.perform(fileUpload("/rest/v1/softwaremodules/{smId}/artifacts", sm.getId()).file(file)
.accept(MediaType.APPLICATION_JSON)).andDo(MockMvcResultPrinter.print())
.andExpect(status().isForbidden())
.andExpect(jsonPath("$.exceptionClass", equalTo(QuotaExceededException.class.getName())))
.andExpect(jsonPath("$.errorCode", equalTo(SpServerError.SP_QUOTA_EXCEEDED.getKey())));
}
@Test @Test
@Description("Tests binary download of an artifact including verfication that the downloaded binary is consistent and that the etag header is as expected identical to the SHA1 hash of the file.") @Description("Tests binary download of an artifact including verfication that the downloaded binary is consistent and that the etag header is as expected identical to the SHA1 hash of the file.")
public void downloadArtifact() throws Exception { public void downloadArtifact() throws Exception {
final SoftwareModule sm = testdataFactory.createSoftwareModuleOs(); final SoftwareModule sm = testdataFactory.createSoftwareModuleOs();
final int artifactSize = 5 * 1024; final int artifactSize = 5 * 1024;
final byte random[] = RandomStringUtils.random(artifactSize).getBytes(); final byte random[] = randomBytes(artifactSize);
final Artifact artifact = artifactManagement.create(new ByteArrayInputStream(random), sm.getId(), "file1", final Artifact artifact = artifactManagement.create(new ByteArrayInputStream(random), sm.getId(), "file1",
false, artifactSize); false, artifactSize);
@@ -412,7 +456,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
final SoftwareModule sm = testdataFactory.createSoftwareModuleOs(); final SoftwareModule sm = testdataFactory.createSoftwareModuleOs();
final int artifactSize = 5 * 1024; final int artifactSize = 5 * 1024;
final byte random[] = RandomStringUtils.random(artifactSize).getBytes(); final byte random[] = randomBytes(artifactSize);
final Artifact artifact = artifactManagement.create(new ByteArrayInputStream(random), sm.getId(), "file1", final Artifact artifact = artifactManagement.create(new ByteArrayInputStream(random), sm.getId(), "file1",
false, artifactSize); false, artifactSize);
@@ -439,7 +483,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
final SoftwareModule sm = testdataFactory.createSoftwareModuleOs(); final SoftwareModule sm = testdataFactory.createSoftwareModuleOs();
final int artifactSize = 5 * 1024; final int artifactSize = 5 * 1024;
final byte random[] = RandomStringUtils.random(artifactSize).getBytes(); final byte random[] = randomBytes(artifactSize);
final Artifact artifact = artifactManagement.create(new ByteArrayInputStream(random), sm.getId(), "file1", final Artifact artifact = artifactManagement.create(new ByteArrayInputStream(random), sm.getId(), "file1",
false, artifactSize); false, artifactSize);
@@ -470,7 +514,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
public void invalidRequestsOnArtifactResource() throws Exception { public void invalidRequestsOnArtifactResource() throws Exception {
final int artifactSize = 5 * 1024; final int artifactSize = 5 * 1024;
final byte random[] = RandomStringUtils.random(artifactSize).getBytes(); final byte random[] = randomBytes(artifactSize);
final MockMultipartFile file = new MockMultipartFile("file", "orig", null, random); final MockMultipartFile file = new MockMultipartFile("file", "orig", null, random);
final SoftwareModule sm = testdataFactory.createSoftwareModuleOs(); final SoftwareModule sm = testdataFactory.createSoftwareModuleOs();
@@ -1021,4 +1065,8 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
} }
} }
private static byte[] randomBytes(final int len) {
return RandomStringUtils.randomAlphanumeric(len).getBytes();
}
} }

View File

@@ -162,9 +162,14 @@ public class HawkbitSecurityProperties {
private int maxTargetsPerAutoAssignment = 5000; private int maxTargetsPerAutoAssignment = 5000;
/** /**
* Maximum size of artifacts in bytes. * Maximum size of artifacts in bytes. Defaults to 1 GB.
*/ */
private long maxArtifactSize = 1_000_000_000; private long maxArtifactSize = 1_073_741_824;
/**
* Maximum size of all artifacts in bytes. Defaults to 20 GB.
*/
private long maxArtifactStorage = 21_474_836_480L;
private final Filter filter = new Filter(); private final Filter filter = new Filter();
private final Filter uiFilter = new Filter(); private final Filter uiFilter = new Filter();
@@ -290,6 +295,14 @@ public class HawkbitSecurityProperties {
return maxArtifactSize; return maxArtifactSize;
} }
public long getMaxArtifactStorage() {
return maxArtifactStorage;
}
public void setMaxArtifactStorage(final long maxArtifactStorage) {
this.maxArtifactStorage = maxArtifactStorage;
}
/** /**
* Configuration for hawkBits DOS prevention filter. This is usually an * Configuration for hawkBits DOS prevention filter. This is usually an
* infrastructure topic (e.g. Web Application Firewall (WAF)) but might * infrastructure topic (e.g. Web Application Firewall (WAF)) but might

View File

@@ -15,7 +15,6 @@ import java.util.regex.Pattern;
import javax.servlet.http.Cookie; import javax.servlet.http.Cookie;
import com.vaadin.shared.ui.label.ContentMode;
import org.eclipse.hawkbit.im.authentication.MultitenancyIndicator; import org.eclipse.hawkbit.im.authentication.MultitenancyIndicator;
import org.eclipse.hawkbit.im.authentication.TenantUserPasswordAuthenticationToken; import org.eclipse.hawkbit.im.authentication.TenantUserPasswordAuthenticationToken;
import org.eclipse.hawkbit.ui.AbstractHawkbitUI; import org.eclipse.hawkbit.ui.AbstractHawkbitUI;
@@ -48,6 +47,7 @@ import com.vaadin.server.VaadinRequest;
import com.vaadin.server.VaadinService; import com.vaadin.server.VaadinService;
import com.vaadin.server.WebBrowser; import com.vaadin.server.WebBrowser;
import com.vaadin.shared.Position; import com.vaadin.shared.Position;
import com.vaadin.shared.ui.label.ContentMode;
import com.vaadin.ui.Alignment; import com.vaadin.ui.Alignment;
import com.vaadin.ui.Button; import com.vaadin.ui.Button;
import com.vaadin.ui.Component; import com.vaadin.ui.Component;