From 086408e8f8193c40a5721e4f133ad9db5d89a0de Mon Sep 17 00:00:00 2001 From: Stefan Behl Date: Fri, 5 Oct 2018 13:52:46 +0200 Subject: [PATCH] Feature Max Artifact Storage quota (#739) * Enforcement of artifact storage quota * Added REST integration tests for artifact upload * Fix test config * Fix failing test cases * Fix PR review findings Signed-off-by: Stefan Behl --- .../hawkbit/repository/QuotaManagement.java | 7 +- .../repository/PropertiesQuotaManagement.java | 5 + .../repository/jpa/JpaArtifactManagement.java | 18 ++ .../RepositoryApplicationConfiguration.java | 3 +- .../jpa/ArtifactManagementTest.java | 220 +++++++++++------- .../jpa/SoftwareModuleManagementTest.java | 16 +- .../src/test/resources/jpa-test.properties | 3 +- .../MgmtSoftwareModuleResourceTest.java | 76 ++++-- .../security/HawkbitSecurityProperties.java | 17 +- .../ui/login/AbstractHawkbitLoginUI.java | 2 +- 10 files changed, 249 insertions(+), 118 deletions(-) diff --git a/hawkbit-repository/hawkbit-repository-api/src/main/java/org/eclipse/hawkbit/repository/QuotaManagement.java b/hawkbit-repository/hawkbit-repository-api/src/main/java/org/eclipse/hawkbit/repository/QuotaManagement.java index c35265a4c..1d3ee5eec 100644 --- a/hawkbit-repository/hawkbit-repository-api/src/main/java/org/eclipse/hawkbit/repository/QuotaManagement.java +++ b/hawkbit-repository/hawkbit-repository-api/src/main/java/org/eclipse/hawkbit/repository/QuotaManagement.java @@ -92,8 +92,13 @@ public interface QuotaManagement { int getMaxActionsPerTarget(); /** - * @return the maximum size of software artifacts in bytes + * @return the maximum size of artifacts in bytes */ long getMaxArtifactSize(); + /** + * @return the accumulated maximum size of all artifacts in bytes + */ + long getMaxArtifactStorage(); + } diff --git a/hawkbit-repository/hawkbit-repository-core/src/main/java/org/eclipse/hawkbit/repository/PropertiesQuotaManagement.java b/hawkbit-repository/hawkbit-repository-core/src/main/java/org/eclipse/hawkbit/repository/PropertiesQuotaManagement.java index 00517dadd..40f6f1927 100644 --- a/hawkbit-repository/hawkbit-repository-core/src/main/java/org/eclipse/hawkbit/repository/PropertiesQuotaManagement.java +++ b/hawkbit-repository/hawkbit-repository-core/src/main/java/org/eclipse/hawkbit/repository/PropertiesQuotaManagement.java @@ -98,4 +98,9 @@ public class PropertiesQuotaManagement implements QuotaManagement { return securityProperties.getDos().getMaxArtifactSize(); } + @Override + public long getMaxArtifactStorage() { + return securityProperties.getDos().getMaxArtifactStorage(); + } + } diff --git a/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/JpaArtifactManagement.java b/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/JpaArtifactManagement.java index a53d7ebfb..ce32098c2 100644 --- a/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/JpaArtifactManagement.java +++ b/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/JpaArtifactManagement.java @@ -54,6 +54,8 @@ public class JpaArtifactManagement implements ArtifactManagement { private static final String MAX_ARTIFACT_SIZE_EXCEEDED = "Quota exceeded: The artifact '%s' (%s bytes) which has been uploaded for software module '%s' exceeds the maximum artifact size of %s bytes."; + private static final String MAX_ARTIFACT_SIZE_TOTAL_EXCEEDED = "Quota exceeded: The artifact '%s' (%s bytes) cannot be uploaded. The maximum total artifact storage of %s bytes would be exceeded."; + private final LocalArtifactRepository localArtifactRepository; private final SoftwareModuleRepository softwareModuleRepository; @@ -104,6 +106,7 @@ public class JpaArtifactManagement implements ArtifactManagement { assertArtifactQuota(moduleId, 1); assertMaxArtifactSizeQuota(filename, moduleId, filesize); + assertMaxArtifactStorageQuota(filename, filesize); try { result = artifactRepository.store(tenantAware.getCurrentTenant(), stream, filename, contentType, @@ -141,6 +144,21 @@ public class JpaArtifactManagement implements ArtifactManagement { } } + private void assertMaxArtifactStorageQuota(final String filename, final long artifactSize) { + final long maxArtifactSizeTotal = quotaManagement.getMaxArtifactStorage(); + if (maxArtifactSizeTotal <= 0) { + return; + } + + final Long currentlyUsed = localArtifactRepository.getSumOfUndeletedArtifactSize().orElse(0L); + if (currentlyUsed + artifactSize > maxArtifactSizeTotal) { + final String msg = String.format(MAX_ARTIFACT_SIZE_TOTAL_EXCEEDED, filename, artifactSize, + maxArtifactSizeTotal); + LOG.warn(msg); + throw new QuotaExceededException(msg); + } + } + @Override @Transactional @Retryable(include = { diff --git a/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/RepositoryApplicationConfiguration.java b/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/RepositoryApplicationConfiguration.java index 5ee71d9a4..57c6657e7 100644 --- a/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/RepositoryApplicationConfiguration.java +++ b/hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/RepositoryApplicationConfiguration.java @@ -149,7 +149,8 @@ public class RepositoryApplicationConfiguration extends JpaBaseConfiguration { } @Bean - PropertiesQuotaManagement staticQuotaManagement(final HawkbitSecurityProperties securityProperties) { + @ConditionalOnMissingBean + QuotaManagement staticQuotaManagement(final HawkbitSecurityProperties securityProperties) { return new PropertiesQuotaManagement(securityProperties); } diff --git a/hawkbit-repository/hawkbit-repository-jpa/src/test/java/org/eclipse/hawkbit/repository/jpa/ArtifactManagementTest.java b/hawkbit-repository/hawkbit-repository-jpa/src/test/java/org/eclipse/hawkbit/repository/jpa/ArtifactManagementTest.java index 112f7388f..628e82a6f 100644 --- a/hawkbit-repository/hawkbit-repository-jpa/src/test/java/org/eclipse/hawkbit/repository/jpa/ArtifactManagementTest.java +++ b/hawkbit-repository/hawkbit-repository-jpa/src/test/java/org/eclipse/hawkbit/repository/jpa/ArtifactManagementTest.java @@ -104,30 +104,30 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest { softwareModuleRepository.save(new JpaSoftwareModule(osType, "name 3", "version 3", null, null)); final int artifactSize = 5 * 1024; - final byte random[] = RandomStringUtils.random(artifactSize).getBytes(); + final byte[] randomBytes = randomBytes(artifactSize); - try (final InputStream inputStream1 = new ByteArrayInputStream(random); - final InputStream inputStream2 = new ByteArrayInputStream(random); - final InputStream inputStream3 = new ByteArrayInputStream(random); - final InputStream inputStream4 = new ByteArrayInputStream(random);) { + try (final InputStream inputStream1 = new ByteArrayInputStream(randomBytes); + final InputStream inputStream2 = new ByteArrayInputStream(randomBytes); + final InputStream inputStream3 = new ByteArrayInputStream(randomBytes); + final InputStream inputStream4 = new ByteArrayInputStream(randomBytes);) { - final Artifact result = artifactManagement.create(inputStream1, sm.getId(), "file1", false, artifactSize); - artifactManagement.create(inputStream2, sm.getId(), "file11", false, artifactSize); - artifactManagement.create(inputStream3, sm.getId(), "file12", false, artifactSize); - final Artifact result2 = artifactManagement.create(inputStream4, sm2.getId(), "file2", false, artifactSize); + final Artifact artifact1 = createArtifactForSoftwareModule("file1", sm.getId(), artifactSize, inputStream1); + createArtifactForSoftwareModule("file11", sm.getId(), artifactSize, inputStream2); + createArtifactForSoftwareModule("file12", sm.getId(), artifactSize, inputStream3); + final Artifact artifact2 = createArtifactForSoftwareModule("file2", sm2.getId(), artifactSize, inputStream4); - assertThat(result).isInstanceOf(Artifact.class); - assertThat(result.getSoftwareModule().getId()).isEqualTo(sm.getId()); - assertThat(result2.getSoftwareModule().getId()).isEqualTo(sm2.getId()); - assertThat(((JpaArtifact) result).getFilename()).isEqualTo("file1"); - assertThat(((JpaArtifact) result).getSha1Hash()).isNotNull(); - assertThat(result).isNotEqualTo(result2); - assertThat(((JpaArtifact) result).getSha1Hash()).isEqualTo(((JpaArtifact) result2).getSha1Hash()); + assertThat(artifact1).isInstanceOf(Artifact.class); + assertThat(artifact1.getSoftwareModule().getId()).isEqualTo(sm.getId()); + assertThat(artifact2.getSoftwareModule().getId()).isEqualTo(sm2.getId()); + assertThat(((JpaArtifact) artifact1).getFilename()).isEqualTo("file1"); + assertThat(((JpaArtifact) artifact1).getSha1Hash()).isNotNull(); + assertThat(artifact1).isNotEqualTo(artifact2); + assertThat(((JpaArtifact) artifact1).getSha1Hash()).isEqualTo(((JpaArtifact) artifact2).getSha1Hash()); assertThat(artifactManagement.getByFilename("file1").get().getSha1Hash()) - .isEqualTo(HashGeneratorUtils.generateSHA1(random)); + .isEqualTo(HashGeneratorUtils.generateSHA1(randomBytes)); assertThat(artifactManagement.getByFilename("file1").get().getMd5Hash()) - .isEqualTo(HashGeneratorUtils.generateMD5(random)); + .isEqualTo(HashGeneratorUtils.generateMD5(randomBytes)); assertThat(artifactRepository.findAll()).hasSize(4); assertThat(softwareModuleRepository.findAll()).hasSize(3); @@ -142,43 +142,73 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest { public void createArtifactsUntilQuotaIsExceeded() throws NoSuchAlgorithmException, IOException { // create a software module - final JpaSoftwareModule sm1 = softwareModuleRepository - .save(new JpaSoftwareModule(osType, "sm1", "1.0", null, null)); + final long smId = softwareModuleRepository.save(new JpaSoftwareModule(osType, "sm1", "1.0", null, null)) + .getId(); // now create artifacts for this module until the quota is exceeded final long maxArtifacts = quotaManagement.getMaxArtifactsPerSoftwareModule(); final List artifactIds = Lists.newArrayList(); final int artifactSize = 5 * 1024; for (int i = 0; i < maxArtifacts; ++i) { - final byte random[] = RandomStringUtils.random(artifactSize).getBytes(); - try (final InputStream inputStream = new ByteArrayInputStream(random)) { - artifactIds.add( - artifactManagement.create(inputStream, sm1.getId(), "file" + i, false, artifactSize).getId()); - } + artifactIds.add(createArtifactForSoftwareModule("file" + i, smId, artifactSize).getId()); } - assertThat(artifactRepository.findBySoftwareModuleId(PAGE, sm1.getId()).getTotalElements()) - .isEqualTo(maxArtifacts); + assertThat(artifactRepository.findBySoftwareModuleId(PAGE, smId).getTotalElements()).isEqualTo(maxArtifacts); // create one mode to trigger the quota exceeded error - assertThatExceptionOfType(QuotaExceededException.class).isThrownBy(() -> { - final byte random[] = RandomStringUtils.random(artifactSize).getBytes(); - try (final InputStream inputStream = new ByteArrayInputStream(random)) { - artifactManagement.create(inputStream, sm1.getId(), "file" + maxArtifacts, false, artifactSize); - } - }); + assertThatExceptionOfType(QuotaExceededException.class) + .isThrownBy(() -> createArtifactForSoftwareModule("file" + maxArtifacts, smId, artifactSize)); // delete one of the artifacts artifactManagement.delete(artifactIds.get(0)); - assertThat(artifactRepository.findBySoftwareModuleId(PAGE, sm1.getId()).getTotalElements()) + assertThat(artifactRepository.findBySoftwareModuleId(PAGE, smId).getTotalElements()) .isEqualTo(maxArtifacts - 1); // now we should be able to create an artifact again - final byte random[] = RandomStringUtils.random(artifactSize).getBytes(); - try (final InputStream inputStream = new ByteArrayInputStream(random)) { - artifactManagement.create(inputStream, sm1.getId(), "fileXYZ", false, artifactSize); - assertThat(artifactRepository.findBySoftwareModuleId(PAGE, sm1.getId()).getTotalElements()) - .isEqualTo(maxArtifacts); + createArtifactForSoftwareModule("fileXYZ", smId, artifactSize); + assertThat(artifactRepository.findBySoftwareModuleId(PAGE, smId).getTotalElements()).isEqualTo(maxArtifacts); + } + + @Test + @Description("Verifies that the quota specifying the maximum artifact storage is enforced (across software modules).") + public void createArtifactsUntilStorageQuotaIsExceeded() throws NoSuchAlgorithmException, IOException { + + // create as many small artifacts as possible w/o violating the storage + // quota + final long maxBytes = quotaManagement.getMaxArtifactStorage(); + final List artifactIds = Lists.newArrayList(); + + // choose an artifact size which does not violate the max file size + final int artifactSize = Math.toIntExact(quotaManagement.getMaxArtifactSize() / 10); + final int numArtifacts = Math.toIntExact(maxBytes / artifactSize); + for (int i = 0; i < numArtifacts; ++i) { + final JpaSoftwareModule sm = softwareModuleRepository + .save(new JpaSoftwareModule(osType, "smd" + i, "1.0", null, null)); + artifactIds.add(createArtifactForSoftwareModule("file" + i, sm.getId(), artifactSize).getId()); } + + // upload one more artifact to trigger the quota exceeded error + final JpaSoftwareModule sm = softwareModuleRepository + .save(new JpaSoftwareModule(osType, "smd" + numArtifacts, "1.0", null, null)); + assertThatExceptionOfType(QuotaExceededException.class) + .isThrownBy(() -> createArtifactForSoftwareModule("file" + numArtifacts, sm.getId(), artifactSize)); + + // delete one of the artifacts + artifactManagement.delete(artifactIds.get(0)); + + // now we should be able to create an artifact again + createArtifactForSoftwareModule("fileXYZ", sm.getId(), artifactSize); + } + + @Test + @Description("Verifies that the quota specifying the maximum artifact storage is enforced (across software modules).") + public void createArtifactWhichExceedsMaxStorage() throws NoSuchAlgorithmException, IOException { + + // create one artifact which exceeds the storage quota at once + final long maxBytes = quotaManagement.getMaxArtifactStorage(); + final JpaSoftwareModule sm = softwareModuleRepository + .save(new JpaSoftwareModule(osType, "smd345", "1.0", null, null)); + assertThatExceptionOfType(QuotaExceededException.class).isThrownBy( + () -> createArtifactForSoftwareModule("file345", sm.getId(), Math.toIntExact(maxBytes) + 128)); } @Test @@ -191,12 +221,8 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest { // create an artifact that exceeds the configured quota final long maxSize = quotaManagement.getMaxArtifactSize(); - final int artifactSize = Math.toIntExact(maxSize) + 8; - final byte random[] = RandomStringUtils.random(artifactSize).getBytes(); - try (final InputStream inputStream = new ByteArrayInputStream(random)) { - assertThatExceptionOfType(QuotaExceededException.class) - .isThrownBy(() -> artifactManagement.create(inputStream, sm1.getId(), "file", false, artifactSize)); - } + assertThatExceptionOfType(QuotaExceededException.class) + .isThrownBy(() -> createArtifactForSoftwareModule("file", sm1.getId(), Math.toIntExact(maxSize) + 8)); } @Test @@ -205,15 +231,12 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest { final JpaSoftwareModule sm = softwareModuleRepository .save(new JpaSoftwareModule(osType, "name 1", "version 1", null, null)); - final int artifactSize = 5 * 1024; - final byte random[] = RandomStringUtils.random(artifactSize).getBytes(); - try (final InputStream inputStream = new ByteArrayInputStream(random)) { - artifactManagement.create(inputStream, sm.getId(), "file1", false, artifactSize); - assertThat(artifactRepository.findAll()).hasSize(1); - softwareModuleRepository.deleteAll(); - assertThat(artifactRepository.findAll()).hasSize(0); - } + createArtifactForSoftwareModule("file1", sm.getId(), 5 * 1024); + assertThat(artifactRepository.findAll()).hasSize(1); + + softwareModuleRepository.deleteAll(); + assertThat(artifactRepository.findAll()).hasSize(0); } /** @@ -239,32 +262,32 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest { try (final InputStream inputStream1 = new RandomGeneratedInputStream(artifactSize); final InputStream inputStream2 = new RandomGeneratedInputStream(artifactSize)) { - final Artifact result = artifactManagement.create(inputStream1, sm.getId(), "file1", false, artifactSize); - final Artifact result2 = artifactManagement.create(inputStream2, sm2.getId(), "file2", false, artifactSize); + final Artifact artifact1 = createArtifactForSoftwareModule("file1", sm.getId(), artifactSize, inputStream1); + final Artifact artifact2 = createArtifactForSoftwareModule("file2", sm2.getId(), artifactSize, inputStream2); assertThat(artifactRepository.findAll()).hasSize(2); - assertThat(result.getId()).isNotNull(); - assertThat(result2.getId()).isNotNull(); - assertThat(((JpaArtifact) result).getSha1Hash()).isNotEqualTo(((JpaArtifact) result2).getSha1Hash()); + assertThat(artifact1.getId()).isNotNull(); + assertThat(artifact2.getId()).isNotNull(); + assertThat(((JpaArtifact) artifact1).getSha1Hash()).isNotEqualTo(((JpaArtifact) artifact2).getSha1Hash()); - assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result.getSha1Hash())) + assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact1.getSha1Hash())) .isNotNull(); assertThat( - binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result2.getSha1Hash())) + binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact2.getSha1Hash())) .isNotNull(); - artifactManagement.delete(result.getId()); + artifactManagement.delete(artifact1.getId()); - assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result.getSha1Hash())) + assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact1.getSha1Hash())) .isNull(); assertThat( - binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result2.getSha1Hash())) + binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact2.getSha1Hash())) .isNotNull(); - artifactManagement.delete(result2.getId()); + artifactManagement.delete(artifact2.getId()); assertThat( - binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result2.getSha1Hash())) + binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact2.getSha1Hash())) .isNull(); assertThat(artifactRepository.findAll()).hasSize(0); @@ -282,26 +305,26 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest { .save(new JpaSoftwareModule(osType, "name 2", "version 2", null, null)); final int artifactSize = 5 * 1024; - final byte random[] = RandomStringUtils.random(artifactSize).getBytes(); + final byte[] randomBytes = randomBytes(artifactSize); - try (final InputStream inputStream1 = new ByteArrayInputStream(random); - final InputStream inputStream2 = new ByteArrayInputStream(random)) { - final Artifact result = artifactManagement.create(inputStream1, sm.getId(), "file1", false, artifactSize); - final Artifact result2 = artifactManagement.create(inputStream2, sm2.getId(), "file2", false, artifactSize); + try (final InputStream inputStream1 = new ByteArrayInputStream(randomBytes); + final InputStream inputStream2 = new ByteArrayInputStream(randomBytes)) { + final Artifact artifact1 = createArtifactForSoftwareModule("file1", sm.getId(), artifactSize, inputStream1); + final Artifact artifact2 = createArtifactForSoftwareModule("file2", sm2.getId(), artifactSize, inputStream2); assertThat(artifactRepository.findAll()).hasSize(2); - assertThat(result.getId()).isNotNull(); - assertThat(result2.getId()).isNotNull(); - assertThat(((JpaArtifact) result).getSha1Hash()).isEqualTo(((JpaArtifact) result2).getSha1Hash()); + assertThat(artifact1.getId()).isNotNull(); + assertThat(artifact2.getId()).isNotNull(); + assertThat(((JpaArtifact) artifact1).getSha1Hash()).isEqualTo(((JpaArtifact) artifact2).getSha1Hash()); - assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result.getSha1Hash())) + assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact1.getSha1Hash())) .isNotNull(); - artifactManagement.delete(result.getId()); - assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result.getSha1Hash())) + artifactManagement.delete(artifact1.getId()); + assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact1.getSha1Hash())) .isNotNull(); - artifactManagement.delete(result2.getId()); - assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result.getSha1Hash())) + artifactManagement.delete(artifact2.getId()); + assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact1.getSha1Hash())) .isNull(); } } @@ -311,9 +334,9 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest { public void findArtifact() throws NoSuchAlgorithmException, IOException { final int artifactSize = 5 * 1024; try (final InputStream inputStream = new RandomGeneratedInputStream(artifactSize)) { - final Artifact result = artifactManagement.create(inputStream, - testdataFactory.createSoftwareModuleOs().getId(), "file1", false, artifactSize); - assertThat(artifactManagement.get(result.getId()).get()).isEqualTo(result); + final Artifact artifact = createArtifactForSoftwareModule("file1", + testdataFactory.createSoftwareModuleOs().getId(), artifactSize, inputStream); + assertThat(artifactManagement.get(artifact.getId()).get()).isEqualTo(artifact); } } @@ -321,14 +344,14 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest { @Description("Loads an artifact binary based on given ID.") public void loadStreamOfArtifact() throws NoSuchAlgorithmException, IOException { final int artifactSize = 5 * 1024; - final byte random[] = RandomStringUtils.random(artifactSize).getBytes(); - try (final InputStream input = new ByteArrayInputStream(random)) { - final Artifact result = artifactManagement.create(input, testdataFactory.createSoftwareModuleOs().getId(), - "file1", false, artifactSize); - try (final InputStream inputStream = artifactManagement.loadArtifactBinary(result.getSha1Hash()).get() + final byte[] randomBytes = randomBytes(artifactSize); + try (final InputStream input = new ByteArrayInputStream(randomBytes)) { + final Artifact artifact = createArtifactForSoftwareModule("file1", + testdataFactory.createSoftwareModuleOs().getId(), artifactSize, input); + try (final InputStream inputStream = artifactManagement.loadArtifactBinary(artifact.getSha1Hash()).get() .getFileInputStream()) { assertTrue("The stored binary matches the given binary", - IOUtils.contentEquals(new ByteArrayInputStream(random), inputStream)); + IOUtils.contentEquals(new ByteArrayInputStream(randomBytes), inputStream)); } } } @@ -353,7 +376,7 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest { final int artifactSize = 5 * 1024; try (final InputStream input = new RandomGeneratedInputStream(artifactSize)) { - artifactManagement.create(input, sm.getId(), "file1", false, artifactSize); + createArtifactForSoftwareModule("file1", sm.getId(), artifactSize, input); assertThat(artifactManagement.findBySoftwareModule(PAGE, sm.getId())).hasSize(1); } } @@ -368,10 +391,27 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest { final int artifactSize = 5 * 1024; try (final InputStream inputStream1 = new RandomGeneratedInputStream(artifactSize); final InputStream inputStream2 = new RandomGeneratedInputStream(artifactSize)) { - artifactManagement.create(inputStream1, sm.getId(), "file1", false, artifactSize); - artifactManagement.create(inputStream2, sm.getId(), "file2", false, artifactSize); + createArtifactForSoftwareModule("file1", sm.getId(), artifactSize, inputStream1); + createArtifactForSoftwareModule("file2", sm.getId(), artifactSize, inputStream2); assertThat(artifactManagement.getByFilenameAndSoftwareModule("file1", sm.getId())).isPresent(); } } + + private Artifact createArtifactForSoftwareModule(final String filename, final long moduleId, final int artifactSize) + throws IOException { + final byte[] randomBytes = randomBytes(artifactSize); + try (final InputStream inputStream = new ByteArrayInputStream(randomBytes)) { + return createArtifactForSoftwareModule(filename, moduleId, artifactSize, inputStream); + } + } + + private Artifact createArtifactForSoftwareModule(final String filename, final long moduleId, final int artifactSize, + final InputStream inputStream) throws IOException { + return artifactManagement.create(inputStream, moduleId, filename, false, artifactSize); + } + + private static byte[] randomBytes(final int len) { + return RandomStringUtils.randomAlphanumeric(len).getBytes(); + } } diff --git a/hawkbit-repository/hawkbit-repository-jpa/src/test/java/org/eclipse/hawkbit/repository/jpa/SoftwareModuleManagementTest.java b/hawkbit-repository/hawkbit-repository-jpa/src/test/java/org/eclipse/hawkbit/repository/jpa/SoftwareModuleManagementTest.java index e95e767d1..06afdec43 100644 --- a/hawkbit-repository/hawkbit-repository-jpa/src/test/java/org/eclipse/hawkbit/repository/jpa/SoftwareModuleManagementTest.java +++ b/hawkbit-repository/hawkbit-repository-jpa/src/test/java/org/eclipse/hawkbit/repository/jpa/SoftwareModuleManagementTest.java @@ -280,7 +280,7 @@ public class SoftwareModuleManagementTest extends AbstractJpaIntegrationTest { assertThat(softwareModuleManagement.get(unassignedModule.getId())).isNotPresent(); // verify: binary data of artifact is deleted - assertArtfiactNull(artifact1, artifact2); + assertArtifactNull(artifact1, artifact2); // verify: meta data of artifact is deleted assertThat(artifactRepository.findOne(artifact1.getId())).isNull(); @@ -311,7 +311,7 @@ public class SoftwareModuleManagementTest extends AbstractJpaIntegrationTest { final Iterator artifactsIt = assignedModule.getArtifacts().iterator(); final Artifact artifact1 = artifactsIt.next(); final Artifact artifact2 = artifactsIt.next(); - assertArtfiactNull(artifact1, artifact2); + assertArtifactNull(artifact1, artifact2); // verify: artifact meta data is still available assertThat(artifactRepository.findOne(artifact1.getId())).isNotNull(); @@ -351,7 +351,7 @@ public class SoftwareModuleManagementTest extends AbstractJpaIntegrationTest { final Iterator artifactsIt = assignedModule.getArtifacts().iterator(); final Artifact artifact1 = artifactsIt.next(); final Artifact artifact2 = artifactsIt.next(); - assertArtfiactNull(artifact1, artifact2); + assertArtifactNull(artifact1, artifact2); // verify: artifact meta data is still available assertThat(artifactRepository.findOne(artifact1.getId())).isNotNull(); @@ -392,7 +392,7 @@ public class SoftwareModuleManagementTest extends AbstractJpaIntegrationTest { assertThat(softwareModuleManagement.get(moduleY.getId())).isPresent(); // verify: binary data of artifact is not deleted - assertArtfiactNotNull(artifactY); + assertArtifactNotNull(artifactY); // verify: meta data of artifactX is deleted assertThat(artifactRepository.findOne(artifactX.getId())).isNull(); @@ -451,7 +451,7 @@ public class SoftwareModuleManagementTest extends AbstractJpaIntegrationTest { assertThat(softwareModuleRepository.findAll()).hasSize(2); // verify: binary data of artifact is deleted - assertArtfiactNull(artifactX, artifactY); + assertArtifactNull(artifactX, artifactY); // verify: meta data of artifactX and artifactY is not deleted assertThat(artifactRepository.findOne(artifactY.getId())).isNotNull(); @@ -480,14 +480,14 @@ public class SoftwareModuleManagementTest extends AbstractJpaIntegrationTest { assertThat(artifacts).hasSize(numberArtifacts); if (numberArtifacts != 0) { - assertArtfiactNotNull(artifacts.toArray(new Artifact[artifacts.size()])); + assertArtifactNotNull(artifacts.toArray(new Artifact[artifacts.size()])); } artifacts.forEach(artifact -> assertThat(artifactRepository.findOne(artifact.getId())).isNotNull()); return softwareModule; } - private void assertArtfiactNotNull(final Artifact... results) { + private void assertArtifactNotNull(final Artifact... results) { assertThat(artifactRepository.findAll()).hasSize(results.length); for (final Artifact result : results) { assertThat(result.getId()).isNotNull(); @@ -496,7 +496,7 @@ public class SoftwareModuleManagementTest extends AbstractJpaIntegrationTest { } } - private void assertArtfiactNull(final Artifact... results) { + private void assertArtifactNull(final Artifact... results) { for (final Artifact result : results) { assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result.getSha1Hash())) .isNull(); diff --git a/hawkbit-repository/hawkbit-repository-jpa/src/test/resources/jpa-test.properties b/hawkbit-repository/hawkbit-repository-jpa/src/test/resources/jpa-test.properties index 1910b49b1..fd4873072 100644 --- a/hawkbit-repository/hawkbit-repository-jpa/src/test/resources/jpa-test.properties +++ b/hawkbit-repository/hawkbit-repository-jpa/src/test/resources/jpa-test.properties @@ -18,7 +18,8 @@ hawkbit.server.security.dos.maxSoftwareModuleTypesPerDistributionSetType=10 hawkbit.server.security.dos.maxSoftwareModulesPerDistributionSet=10 hawkbit.server.security.dos.maxArtifactsPerSoftwareModule=10 hawkbit.server.security.dos.maxTargetsPerRolloutGroup=1000 -hawkbit.server.security.dos.maxArtifactSize=1000000 +hawkbit.server.security.dos.maxArtifactSize=600000 +hawkbit.server.security.dos.maxArtifactStorage=1000000 # Quota - END # Debug utility functions - START diff --git a/hawkbit-rest/hawkbit-mgmt-resource/src/test/java/org/eclipse/hawkbit/mgmt/rest/resource/MgmtSoftwareModuleResourceTest.java b/hawkbit-rest/hawkbit-mgmt-resource/src/test/java/org/eclipse/hawkbit/mgmt/rest/resource/MgmtSoftwareModuleResourceTest.java index 47b2f74ad..4720c64c1 100644 --- a/hawkbit-rest/hawkbit-mgmt-resource/src/test/java/org/eclipse/hawkbit/mgmt/rest/resource/MgmtSoftwareModuleResourceTest.java +++ b/hawkbit-rest/hawkbit-mgmt-resource/src/test/java/org/eclipse/hawkbit/mgmt/rest/resource/MgmtSoftwareModuleResourceTest.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.io.InputStream; import java.util.Arrays; import java.util.List; -import java.util.Random; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.RandomStringUtils; @@ -70,7 +69,8 @@ import ru.yandex.qatools.allure.annotations.Stories; */ @Features("Component Tests - Management API") @Stories("Software Module Resource") -@TestPropertySource(properties = { "hawkbit.server.security.dos.maxArtifactSize=100000" }) +@TestPropertySource(properties = { "hawkbit.server.security.dos.maxArtifactSize=100000", + "hawkbit.server.security.dos.maxArtifactStorage=500000" }) public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegrationTest { @Before @@ -157,7 +157,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra final SoftwareModule sm = testdataFactory.createSoftwareModuleOs(); // create test file - final byte random[] = RandomStringUtils.random(5 * 1024).getBytes(); + final byte random[] = randomBytes(5 * 1024); final String md5sum = HashGeneratorUtils.generateMD5(random); final String sha1sum = HashGeneratorUtils.generateSHA1(random); final MockMultipartFile file = new MockMultipartFile("file", "origFilename", null, random); @@ -195,8 +195,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra final long maxSize = quotaManagement.getMaxArtifactSize(); // create a file which exceeds the configured maximum size - final byte[] randomBytes = new byte[Math.toIntExact(maxSize) + 1024]; - new Random().nextBytes(randomBytes); + final byte[] randomBytes = randomBytes(Math.toIntExact(maxSize) + 1024); final MockMultipartFile file = new MockMultipartFile("file", "origFilename", null, randomBytes); @@ -251,7 +250,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra public void duplicateUploadArtifact() throws Exception { final SoftwareModule sm = testdataFactory.createSoftwareModuleOs(); - final byte random[] = RandomStringUtils.random(5 * 1024).getBytes(); + final byte random[] = randomBytes(5 * 1024); final String md5sum = HashGeneratorUtils.generateMD5(random); final String sha1sum = HashGeneratorUtils.generateSHA1(random); final MockMultipartFile file = new MockMultipartFile("file", "orig", null, random); @@ -274,7 +273,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra assertThat(artifactManagement.count()).isEqualTo(0); // create test file - final byte random[] = RandomStringUtils.random(5 * 1024).getBytes(); + final byte random[] = randomBytes(5 * 1024); final MockMultipartFile file = new MockMultipartFile("file", "origFilename", null, random); // upload @@ -299,7 +298,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra assertThat(artifactManagement.count()).isEqualTo(0); // create test file - final byte random[] = RandomStringUtils.random(5 * 1024).getBytes(); + final byte random[] = randomBytes(5 * 1024); final String md5sum = HashGeneratorUtils.generateMD5(random); final String sha1sum = HashGeneratorUtils.generateSHA1(random); final MockMultipartFile file = new MockMultipartFile("file", "origFilename", null, random); @@ -343,7 +342,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra for (int i = 0; i < maxArtifacts; ++i) { // create test file - final byte random[] = RandomStringUtils.random(5 * 1024).getBytes(); + final byte random[] = randomBytes(5 * 1024); final String md5sum = HashGeneratorUtils.generateMD5(random); final String sha1sum = HashGeneratorUtils.generateSHA1(random); final MockMultipartFile file = new MockMultipartFile("file", "origFilename" + i, null, random); @@ -360,7 +359,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra } // upload one more file to cause the quota to be exceeded - final byte random[] = RandomStringUtils.random(5 * 1024).getBytes(); + final byte random[] = randomBytes(5 * 1024); HashGeneratorUtils.generateMD5(random); HashGeneratorUtils.generateSHA1(random); final MockMultipartFile file = new MockMultipartFile("file", "origFilename_final", null, random); @@ -374,13 +373,58 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra } + @Test + @Description("Verifies that artifacts can only be added as long as the artifact storage quota is not exceeded.") + public void uploadArtifactsUntilStorageQuotaExceeded() throws Exception { + + final long storageLimit = quotaManagement.getMaxArtifactStorage(); + + // choose an artifact size which does not violate the max file size + final int artifactSize = Math.toIntExact(quotaManagement.getMaxArtifactSize() / 10); + final int numArtifacts = Math.toIntExact(storageLimit / artifactSize); + + for (int i = 0; i < numArtifacts; ++i) { + // create test file + final byte random[] = randomBytes(artifactSize); + final String md5sum = HashGeneratorUtils.generateMD5(random); + final String sha1sum = HashGeneratorUtils.generateSHA1(random); + final MockMultipartFile file = new MockMultipartFile("file", "origFilename" + i, null, random); + + // upload + final SoftwareModule sm = testdataFactory.createSoftwareModuleOs("sm" + i); + mvc.perform(fileUpload("/rest/v1/softwaremodules/{smId}/artifacts", sm.getId()).file(file) + .accept(MediaType.APPLICATION_JSON)).andDo(MockMvcResultPrinter.print()) + .andExpect(status().isCreated()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8_VALUE)) + .andExpect(jsonPath("$.hashes.md5", equalTo(md5sum))) + .andExpect(jsonPath("$.hashes.sha1", equalTo(sha1sum))) + .andExpect(jsonPath("$.size", equalTo(random.length))) + .andExpect(jsonPath("$.providedFilename", equalTo("origFilename" + i))).andReturn(); + } + + // upload one more file to cause the quota to be exceeded + final byte random[] = randomBytes(artifactSize); + HashGeneratorUtils.generateMD5(random); + HashGeneratorUtils.generateSHA1(random); + final MockMultipartFile file = new MockMultipartFile("file", "origFilename_final", null, random); + + // upload + final SoftwareModule sm = testdataFactory.createSoftwareModuleOs("sm" + numArtifacts); + mvc.perform(fileUpload("/rest/v1/softwaremodules/{smId}/artifacts", sm.getId()).file(file) + .accept(MediaType.APPLICATION_JSON)).andDo(MockMvcResultPrinter.print()) + .andExpect(status().isForbidden()) + .andExpect(jsonPath("$.exceptionClass", equalTo(QuotaExceededException.class.getName()))) + .andExpect(jsonPath("$.errorCode", equalTo(SpServerError.SP_QUOTA_EXCEEDED.getKey()))); + + } + @Test @Description("Tests binary download of an artifact including verfication that the downloaded binary is consistent and that the etag header is as expected identical to the SHA1 hash of the file.") public void downloadArtifact() throws Exception { final SoftwareModule sm = testdataFactory.createSoftwareModuleOs(); final int artifactSize = 5 * 1024; - final byte random[] = RandomStringUtils.random(artifactSize).getBytes(); + final byte random[] = randomBytes(artifactSize); final Artifact artifact = artifactManagement.create(new ByteArrayInputStream(random), sm.getId(), "file1", false, artifactSize); @@ -412,7 +456,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra final SoftwareModule sm = testdataFactory.createSoftwareModuleOs(); final int artifactSize = 5 * 1024; - final byte random[] = RandomStringUtils.random(artifactSize).getBytes(); + final byte random[] = randomBytes(artifactSize); final Artifact artifact = artifactManagement.create(new ByteArrayInputStream(random), sm.getId(), "file1", false, artifactSize); @@ -439,7 +483,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra final SoftwareModule sm = testdataFactory.createSoftwareModuleOs(); final int artifactSize = 5 * 1024; - final byte random[] = RandomStringUtils.random(artifactSize).getBytes(); + final byte random[] = randomBytes(artifactSize); final Artifact artifact = artifactManagement.create(new ByteArrayInputStream(random), sm.getId(), "file1", false, artifactSize); @@ -470,7 +514,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra public void invalidRequestsOnArtifactResource() throws Exception { final int artifactSize = 5 * 1024; - final byte random[] = RandomStringUtils.random(artifactSize).getBytes(); + final byte random[] = randomBytes(artifactSize); final MockMultipartFile file = new MockMultipartFile("file", "orig", null, random); final SoftwareModule sm = testdataFactory.createSoftwareModuleOs(); @@ -1021,4 +1065,8 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra } } + private static byte[] randomBytes(final int len) { + return RandomStringUtils.randomAlphanumeric(len).getBytes(); + } + } diff --git a/hawkbit-security-core/src/main/java/org/eclipse/hawkbit/security/HawkbitSecurityProperties.java b/hawkbit-security-core/src/main/java/org/eclipse/hawkbit/security/HawkbitSecurityProperties.java index ef4ae7c7d..d1094caf3 100644 --- a/hawkbit-security-core/src/main/java/org/eclipse/hawkbit/security/HawkbitSecurityProperties.java +++ b/hawkbit-security-core/src/main/java/org/eclipse/hawkbit/security/HawkbitSecurityProperties.java @@ -162,9 +162,14 @@ public class HawkbitSecurityProperties { private int maxTargetsPerAutoAssignment = 5000; /** - * Maximum size of artifacts in bytes. + * Maximum size of artifacts in bytes. Defaults to 1 GB. */ - private long maxArtifactSize = 1_000_000_000; + private long maxArtifactSize = 1_073_741_824; + + /** + * Maximum size of all artifacts in bytes. Defaults to 20 GB. + */ + private long maxArtifactStorage = 21_474_836_480L; private final Filter filter = new Filter(); private final Filter uiFilter = new Filter(); @@ -290,6 +295,14 @@ public class HawkbitSecurityProperties { return maxArtifactSize; } + public long getMaxArtifactStorage() { + return maxArtifactStorage; + } + + public void setMaxArtifactStorage(final long maxArtifactStorage) { + this.maxArtifactStorage = maxArtifactStorage; + } + /** * Configuration for hawkBits DOS prevention filter. This is usually an * infrastructure topic (e.g. Web Application Firewall (WAF)) but might diff --git a/hawkbit-ui/src/main/java/org/eclipse/hawkbit/ui/login/AbstractHawkbitLoginUI.java b/hawkbit-ui/src/main/java/org/eclipse/hawkbit/ui/login/AbstractHawkbitLoginUI.java index d0d25d97e..096ecba7a 100644 --- a/hawkbit-ui/src/main/java/org/eclipse/hawkbit/ui/login/AbstractHawkbitLoginUI.java +++ b/hawkbit-ui/src/main/java/org/eclipse/hawkbit/ui/login/AbstractHawkbitLoginUI.java @@ -15,7 +15,6 @@ import java.util.regex.Pattern; import javax.servlet.http.Cookie; -import com.vaadin.shared.ui.label.ContentMode; import org.eclipse.hawkbit.im.authentication.MultitenancyIndicator; import org.eclipse.hawkbit.im.authentication.TenantUserPasswordAuthenticationToken; import org.eclipse.hawkbit.ui.AbstractHawkbitUI; @@ -48,6 +47,7 @@ import com.vaadin.server.VaadinRequest; import com.vaadin.server.VaadinService; import com.vaadin.server.WebBrowser; import com.vaadin.shared.Position; +import com.vaadin.shared.ui.label.ContentMode; import com.vaadin.ui.Alignment; import com.vaadin.ui.Button; import com.vaadin.ui.Component;