Feature Max Artifact Storage quota (#739)
* Enforcement of artifact storage quota * Added REST integration tests for artifact upload * Fix test config * Fix failing test cases * Fix PR review findings Signed-off-by: Stefan Behl <stefan.behl@bosch-si.com>
This commit is contained in:
Stefan Behl
@@ -92,8 +92,13 @@ public interface QuotaManagement {
|
||||
int getMaxActionsPerTarget();
|
||||
|
||||
/**
|
||||
* @return the maximum size of software artifacts in bytes
|
||||
* @return the maximum size of artifacts in bytes
|
||||
*/
|
||||
long getMaxArtifactSize();
|
||||
|
||||
/**
|
||||
* @return the accumulated maximum size of all artifacts in bytes
|
||||
*/
|
||||
long getMaxArtifactStorage();
|
||||
|
||||
}
|
||||
|
||||
@@ -98,4 +98,9 @@ public class PropertiesQuotaManagement implements QuotaManagement {
|
||||
return securityProperties.getDos().getMaxArtifactSize();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMaxArtifactStorage() {
|
||||
return securityProperties.getDos().getMaxArtifactStorage();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -54,6 +54,8 @@ public class JpaArtifactManagement implements ArtifactManagement {
|
||||
|
||||
private static final String MAX_ARTIFACT_SIZE_EXCEEDED = "Quota exceeded: The artifact '%s' (%s bytes) which has been uploaded for software module '%s' exceeds the maximum artifact size of %s bytes.";
|
||||
|
||||
private static final String MAX_ARTIFACT_SIZE_TOTAL_EXCEEDED = "Quota exceeded: The artifact '%s' (%s bytes) cannot be uploaded. The maximum total artifact storage of %s bytes would be exceeded.";
|
||||
|
||||
private final LocalArtifactRepository localArtifactRepository;
|
||||
|
||||
private final SoftwareModuleRepository softwareModuleRepository;
|
||||
@@ -104,6 +106,7 @@ public class JpaArtifactManagement implements ArtifactManagement {
|
||||
|
||||
assertArtifactQuota(moduleId, 1);
|
||||
assertMaxArtifactSizeQuota(filename, moduleId, filesize);
|
||||
assertMaxArtifactStorageQuota(filename, filesize);
|
||||
|
||||
try {
|
||||
result = artifactRepository.store(tenantAware.getCurrentTenant(), stream, filename, contentType,
|
||||
@@ -141,6 +144,21 @@ public class JpaArtifactManagement implements ArtifactManagement {
|
||||
}
|
||||
}
|
||||
|
||||
private void assertMaxArtifactStorageQuota(final String filename, final long artifactSize) {
|
||||
final long maxArtifactSizeTotal = quotaManagement.getMaxArtifactStorage();
|
||||
if (maxArtifactSizeTotal <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
final Long currentlyUsed = localArtifactRepository.getSumOfUndeletedArtifactSize().orElse(0L);
|
||||
if (currentlyUsed + artifactSize > maxArtifactSizeTotal) {
|
||||
final String msg = String.format(MAX_ARTIFACT_SIZE_TOTAL_EXCEEDED, filename, artifactSize,
|
||||
maxArtifactSizeTotal);
|
||||
LOG.warn(msg);
|
||||
throw new QuotaExceededException(msg);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@Transactional
|
||||
@Retryable(include = {
|
||||
|
||||
@@ -149,7 +149,8 @@ public class RepositoryApplicationConfiguration extends JpaBaseConfiguration {
|
||||
}
|
||||
|
||||
@Bean
|
||||
PropertiesQuotaManagement staticQuotaManagement(final HawkbitSecurityProperties securityProperties) {
|
||||
@ConditionalOnMissingBean
|
||||
QuotaManagement staticQuotaManagement(final HawkbitSecurityProperties securityProperties) {
|
||||
return new PropertiesQuotaManagement(securityProperties);
|
||||
}
|
||||
|
||||
|
||||
@@ -104,30 +104,30 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest {
|
||||
softwareModuleRepository.save(new JpaSoftwareModule(osType, "name 3", "version 3", null, null));
|
||||
|
||||
final int artifactSize = 5 * 1024;
|
||||
final byte random[] = RandomStringUtils.random(artifactSize).getBytes();
|
||||
final byte[] randomBytes = randomBytes(artifactSize);
|
||||
|
||||
try (final InputStream inputStream1 = new ByteArrayInputStream(random);
|
||||
final InputStream inputStream2 = new ByteArrayInputStream(random);
|
||||
final InputStream inputStream3 = new ByteArrayInputStream(random);
|
||||
final InputStream inputStream4 = new ByteArrayInputStream(random);) {
|
||||
try (final InputStream inputStream1 = new ByteArrayInputStream(randomBytes);
|
||||
final InputStream inputStream2 = new ByteArrayInputStream(randomBytes);
|
||||
final InputStream inputStream3 = new ByteArrayInputStream(randomBytes);
|
||||
final InputStream inputStream4 = new ByteArrayInputStream(randomBytes);) {
|
||||
|
||||
final Artifact result = artifactManagement.create(inputStream1, sm.getId(), "file1", false, artifactSize);
|
||||
artifactManagement.create(inputStream2, sm.getId(), "file11", false, artifactSize);
|
||||
artifactManagement.create(inputStream3, sm.getId(), "file12", false, artifactSize);
|
||||
final Artifact result2 = artifactManagement.create(inputStream4, sm2.getId(), "file2", false, artifactSize);
|
||||
final Artifact artifact1 = createArtifactForSoftwareModule("file1", sm.getId(), artifactSize, inputStream1);
|
||||
createArtifactForSoftwareModule("file11", sm.getId(), artifactSize, inputStream2);
|
||||
createArtifactForSoftwareModule("file12", sm.getId(), artifactSize, inputStream3);
|
||||
final Artifact artifact2 = createArtifactForSoftwareModule("file2", sm2.getId(), artifactSize, inputStream4);
|
||||
|
||||
assertThat(result).isInstanceOf(Artifact.class);
|
||||
assertThat(result.getSoftwareModule().getId()).isEqualTo(sm.getId());
|
||||
assertThat(result2.getSoftwareModule().getId()).isEqualTo(sm2.getId());
|
||||
assertThat(((JpaArtifact) result).getFilename()).isEqualTo("file1");
|
||||
assertThat(((JpaArtifact) result).getSha1Hash()).isNotNull();
|
||||
assertThat(result).isNotEqualTo(result2);
|
||||
assertThat(((JpaArtifact) result).getSha1Hash()).isEqualTo(((JpaArtifact) result2).getSha1Hash());
|
||||
assertThat(artifact1).isInstanceOf(Artifact.class);
|
||||
assertThat(artifact1.getSoftwareModule().getId()).isEqualTo(sm.getId());
|
||||
assertThat(artifact2.getSoftwareModule().getId()).isEqualTo(sm2.getId());
|
||||
assertThat(((JpaArtifact) artifact1).getFilename()).isEqualTo("file1");
|
||||
assertThat(((JpaArtifact) artifact1).getSha1Hash()).isNotNull();
|
||||
assertThat(artifact1).isNotEqualTo(artifact2);
|
||||
assertThat(((JpaArtifact) artifact1).getSha1Hash()).isEqualTo(((JpaArtifact) artifact2).getSha1Hash());
|
||||
|
||||
assertThat(artifactManagement.getByFilename("file1").get().getSha1Hash())
|
||||
.isEqualTo(HashGeneratorUtils.generateSHA1(random));
|
||||
.isEqualTo(HashGeneratorUtils.generateSHA1(randomBytes));
|
||||
assertThat(artifactManagement.getByFilename("file1").get().getMd5Hash())
|
||||
.isEqualTo(HashGeneratorUtils.generateMD5(random));
|
||||
.isEqualTo(HashGeneratorUtils.generateMD5(randomBytes));
|
||||
|
||||
assertThat(artifactRepository.findAll()).hasSize(4);
|
||||
assertThat(softwareModuleRepository.findAll()).hasSize(3);
|
||||
@@ -142,43 +142,73 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest {
|
||||
public void createArtifactsUntilQuotaIsExceeded() throws NoSuchAlgorithmException, IOException {
|
||||
|
||||
// create a software module
|
||||
final JpaSoftwareModule sm1 = softwareModuleRepository
|
||||
.save(new JpaSoftwareModule(osType, "sm1", "1.0", null, null));
|
||||
final long smId = softwareModuleRepository.save(new JpaSoftwareModule(osType, "sm1", "1.0", null, null))
|
||||
.getId();
|
||||
|
||||
// now create artifacts for this module until the quota is exceeded
|
||||
final long maxArtifacts = quotaManagement.getMaxArtifactsPerSoftwareModule();
|
||||
final List<Long> artifactIds = Lists.newArrayList();
|
||||
final int artifactSize = 5 * 1024;
|
||||
for (int i = 0; i < maxArtifacts; ++i) {
|
||||
final byte random[] = RandomStringUtils.random(artifactSize).getBytes();
|
||||
try (final InputStream inputStream = new ByteArrayInputStream(random)) {
|
||||
artifactIds.add(
|
||||
artifactManagement.create(inputStream, sm1.getId(), "file" + i, false, artifactSize).getId());
|
||||
artifactIds.add(createArtifactForSoftwareModule("file" + i, smId, artifactSize).getId());
|
||||
}
|
||||
}
|
||||
assertThat(artifactRepository.findBySoftwareModuleId(PAGE, sm1.getId()).getTotalElements())
|
||||
.isEqualTo(maxArtifacts);
|
||||
assertThat(artifactRepository.findBySoftwareModuleId(PAGE, smId).getTotalElements()).isEqualTo(maxArtifacts);
|
||||
|
||||
// create one mode to trigger the quota exceeded error
|
||||
assertThatExceptionOfType(QuotaExceededException.class).isThrownBy(() -> {
|
||||
final byte random[] = RandomStringUtils.random(artifactSize).getBytes();
|
||||
try (final InputStream inputStream = new ByteArrayInputStream(random)) {
|
||||
artifactManagement.create(inputStream, sm1.getId(), "file" + maxArtifacts, false, artifactSize);
|
||||
}
|
||||
});
|
||||
assertThatExceptionOfType(QuotaExceededException.class)
|
||||
.isThrownBy(() -> createArtifactForSoftwareModule("file" + maxArtifacts, smId, artifactSize));
|
||||
|
||||
// delete one of the artifacts
|
||||
artifactManagement.delete(artifactIds.get(0));
|
||||
assertThat(artifactRepository.findBySoftwareModuleId(PAGE, sm1.getId()).getTotalElements())
|
||||
assertThat(artifactRepository.findBySoftwareModuleId(PAGE, smId).getTotalElements())
|
||||
.isEqualTo(maxArtifacts - 1);
|
||||
|
||||
// now we should be able to create an artifact again
|
||||
final byte random[] = RandomStringUtils.random(artifactSize).getBytes();
|
||||
try (final InputStream inputStream = new ByteArrayInputStream(random)) {
|
||||
artifactManagement.create(inputStream, sm1.getId(), "fileXYZ", false, artifactSize);
|
||||
assertThat(artifactRepository.findBySoftwareModuleId(PAGE, sm1.getId()).getTotalElements())
|
||||
.isEqualTo(maxArtifacts);
|
||||
createArtifactForSoftwareModule("fileXYZ", smId, artifactSize);
|
||||
assertThat(artifactRepository.findBySoftwareModuleId(PAGE, smId).getTotalElements()).isEqualTo(maxArtifacts);
|
||||
}
|
||||
|
||||
@Test
|
||||
@Description("Verifies that the quota specifying the maximum artifact storage is enforced (across software modules).")
|
||||
public void createArtifactsUntilStorageQuotaIsExceeded() throws NoSuchAlgorithmException, IOException {
|
||||
|
||||
// create as many small artifacts as possible w/o violating the storage
|
||||
// quota
|
||||
final long maxBytes = quotaManagement.getMaxArtifactStorage();
|
||||
final List<Long> artifactIds = Lists.newArrayList();
|
||||
|
||||
// choose an artifact size which does not violate the max file size
|
||||
final int artifactSize = Math.toIntExact(quotaManagement.getMaxArtifactSize() / 10);
|
||||
final int numArtifacts = Math.toIntExact(maxBytes / artifactSize);
|
||||
for (int i = 0; i < numArtifacts; ++i) {
|
||||
final JpaSoftwareModule sm = softwareModuleRepository
|
||||
.save(new JpaSoftwareModule(osType, "smd" + i, "1.0", null, null));
|
||||
artifactIds.add(createArtifactForSoftwareModule("file" + i, sm.getId(), artifactSize).getId());
|
||||
}
|
||||
|
||||
// upload one more artifact to trigger the quota exceeded error
|
||||
final JpaSoftwareModule sm = softwareModuleRepository
|
||||
.save(new JpaSoftwareModule(osType, "smd" + numArtifacts, "1.0", null, null));
|
||||
assertThatExceptionOfType(QuotaExceededException.class)
|
||||
.isThrownBy(() -> createArtifactForSoftwareModule("file" + numArtifacts, sm.getId(), artifactSize));
|
||||
|
||||
// delete one of the artifacts
|
||||
artifactManagement.delete(artifactIds.get(0));
|
||||
|
||||
// now we should be able to create an artifact again
|
||||
createArtifactForSoftwareModule("fileXYZ", sm.getId(), artifactSize);
|
||||
}
|
||||
|
||||
@Test
|
||||
@Description("Verifies that the quota specifying the maximum artifact storage is enforced (across software modules).")
|
||||
public void createArtifactWhichExceedsMaxStorage() throws NoSuchAlgorithmException, IOException {
|
||||
|
||||
// create one artifact which exceeds the storage quota at once
|
||||
final long maxBytes = quotaManagement.getMaxArtifactStorage();
|
||||
final JpaSoftwareModule sm = softwareModuleRepository
|
||||
.save(new JpaSoftwareModule(osType, "smd345", "1.0", null, null));
|
||||
assertThatExceptionOfType(QuotaExceededException.class).isThrownBy(
|
||||
() -> createArtifactForSoftwareModule("file345", sm.getId(), Math.toIntExact(maxBytes) + 128));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -191,12 +221,8 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest {
|
||||
|
||||
// create an artifact that exceeds the configured quota
|
||||
final long maxSize = quotaManagement.getMaxArtifactSize();
|
||||
final int artifactSize = Math.toIntExact(maxSize) + 8;
|
||||
final byte random[] = RandomStringUtils.random(artifactSize).getBytes();
|
||||
try (final InputStream inputStream = new ByteArrayInputStream(random)) {
|
||||
assertThatExceptionOfType(QuotaExceededException.class)
|
||||
.isThrownBy(() -> artifactManagement.create(inputStream, sm1.getId(), "file", false, artifactSize));
|
||||
}
|
||||
.isThrownBy(() -> createArtifactForSoftwareModule("file", sm1.getId(), Math.toIntExact(maxSize) + 8));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -205,16 +231,13 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest {
|
||||
|
||||
final JpaSoftwareModule sm = softwareModuleRepository
|
||||
.save(new JpaSoftwareModule(osType, "name 1", "version 1", null, null));
|
||||
final int artifactSize = 5 * 1024;
|
||||
final byte random[] = RandomStringUtils.random(artifactSize).getBytes();
|
||||
try (final InputStream inputStream = new ByteArrayInputStream(random)) {
|
||||
artifactManagement.create(inputStream, sm.getId(), "file1", false, artifactSize);
|
||||
|
||||
createArtifactForSoftwareModule("file1", sm.getId(), 5 * 1024);
|
||||
assertThat(artifactRepository.findAll()).hasSize(1);
|
||||
|
||||
softwareModuleRepository.deleteAll();
|
||||
assertThat(artifactRepository.findAll()).hasSize(0);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Test method for
|
||||
@@ -239,32 +262,32 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest {
|
||||
try (final InputStream inputStream1 = new RandomGeneratedInputStream(artifactSize);
|
||||
final InputStream inputStream2 = new RandomGeneratedInputStream(artifactSize)) {
|
||||
|
||||
final Artifact result = artifactManagement.create(inputStream1, sm.getId(), "file1", false, artifactSize);
|
||||
final Artifact result2 = artifactManagement.create(inputStream2, sm2.getId(), "file2", false, artifactSize);
|
||||
final Artifact artifact1 = createArtifactForSoftwareModule("file1", sm.getId(), artifactSize, inputStream1);
|
||||
final Artifact artifact2 = createArtifactForSoftwareModule("file2", sm2.getId(), artifactSize, inputStream2);
|
||||
|
||||
assertThat(artifactRepository.findAll()).hasSize(2);
|
||||
|
||||
assertThat(result.getId()).isNotNull();
|
||||
assertThat(result2.getId()).isNotNull();
|
||||
assertThat(((JpaArtifact) result).getSha1Hash()).isNotEqualTo(((JpaArtifact) result2).getSha1Hash());
|
||||
assertThat(artifact1.getId()).isNotNull();
|
||||
assertThat(artifact2.getId()).isNotNull();
|
||||
assertThat(((JpaArtifact) artifact1).getSha1Hash()).isNotEqualTo(((JpaArtifact) artifact2).getSha1Hash());
|
||||
|
||||
assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result.getSha1Hash()))
|
||||
assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact1.getSha1Hash()))
|
||||
.isNotNull();
|
||||
assertThat(
|
||||
binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result2.getSha1Hash()))
|
||||
binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact2.getSha1Hash()))
|
||||
.isNotNull();
|
||||
|
||||
artifactManagement.delete(result.getId());
|
||||
artifactManagement.delete(artifact1.getId());
|
||||
|
||||
assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result.getSha1Hash()))
|
||||
assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact1.getSha1Hash()))
|
||||
.isNull();
|
||||
assertThat(
|
||||
binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result2.getSha1Hash()))
|
||||
binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact2.getSha1Hash()))
|
||||
.isNotNull();
|
||||
|
||||
artifactManagement.delete(result2.getId());
|
||||
artifactManagement.delete(artifact2.getId());
|
||||
assertThat(
|
||||
binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result2.getSha1Hash()))
|
||||
binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact2.getSha1Hash()))
|
||||
.isNull();
|
||||
|
||||
assertThat(artifactRepository.findAll()).hasSize(0);
|
||||
@@ -282,26 +305,26 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest {
|
||||
.save(new JpaSoftwareModule(osType, "name 2", "version 2", null, null));
|
||||
|
||||
final int artifactSize = 5 * 1024;
|
||||
final byte random[] = RandomStringUtils.random(artifactSize).getBytes();
|
||||
final byte[] randomBytes = randomBytes(artifactSize);
|
||||
|
||||
try (final InputStream inputStream1 = new ByteArrayInputStream(random);
|
||||
final InputStream inputStream2 = new ByteArrayInputStream(random)) {
|
||||
final Artifact result = artifactManagement.create(inputStream1, sm.getId(), "file1", false, artifactSize);
|
||||
final Artifact result2 = artifactManagement.create(inputStream2, sm2.getId(), "file2", false, artifactSize);
|
||||
try (final InputStream inputStream1 = new ByteArrayInputStream(randomBytes);
|
||||
final InputStream inputStream2 = new ByteArrayInputStream(randomBytes)) {
|
||||
final Artifact artifact1 = createArtifactForSoftwareModule("file1", sm.getId(), artifactSize, inputStream1);
|
||||
final Artifact artifact2 = createArtifactForSoftwareModule("file2", sm2.getId(), artifactSize, inputStream2);
|
||||
|
||||
assertThat(artifactRepository.findAll()).hasSize(2);
|
||||
assertThat(result.getId()).isNotNull();
|
||||
assertThat(result2.getId()).isNotNull();
|
||||
assertThat(((JpaArtifact) result).getSha1Hash()).isEqualTo(((JpaArtifact) result2).getSha1Hash());
|
||||
assertThat(artifact1.getId()).isNotNull();
|
||||
assertThat(artifact2.getId()).isNotNull();
|
||||
assertThat(((JpaArtifact) artifact1).getSha1Hash()).isEqualTo(((JpaArtifact) artifact2).getSha1Hash());
|
||||
|
||||
assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result.getSha1Hash()))
|
||||
assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact1.getSha1Hash()))
|
||||
.isNotNull();
|
||||
artifactManagement.delete(result.getId());
|
||||
assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result.getSha1Hash()))
|
||||
artifactManagement.delete(artifact1.getId());
|
||||
assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact1.getSha1Hash()))
|
||||
.isNotNull();
|
||||
|
||||
artifactManagement.delete(result2.getId());
|
||||
assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result.getSha1Hash()))
|
||||
artifactManagement.delete(artifact2.getId());
|
||||
assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), artifact1.getSha1Hash()))
|
||||
.isNull();
|
||||
}
|
||||
}
|
||||
@@ -311,9 +334,9 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest {
|
||||
public void findArtifact() throws NoSuchAlgorithmException, IOException {
|
||||
final int artifactSize = 5 * 1024;
|
||||
try (final InputStream inputStream = new RandomGeneratedInputStream(artifactSize)) {
|
||||
final Artifact result = artifactManagement.create(inputStream,
|
||||
testdataFactory.createSoftwareModuleOs().getId(), "file1", false, artifactSize);
|
||||
assertThat(artifactManagement.get(result.getId()).get()).isEqualTo(result);
|
||||
final Artifact artifact = createArtifactForSoftwareModule("file1",
|
||||
testdataFactory.createSoftwareModuleOs().getId(), artifactSize, inputStream);
|
||||
assertThat(artifactManagement.get(artifact.getId()).get()).isEqualTo(artifact);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -321,14 +344,14 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest {
|
||||
@Description("Loads an artifact binary based on given ID.")
|
||||
public void loadStreamOfArtifact() throws NoSuchAlgorithmException, IOException {
|
||||
final int artifactSize = 5 * 1024;
|
||||
final byte random[] = RandomStringUtils.random(artifactSize).getBytes();
|
||||
try (final InputStream input = new ByteArrayInputStream(random)) {
|
||||
final Artifact result = artifactManagement.create(input, testdataFactory.createSoftwareModuleOs().getId(),
|
||||
"file1", false, artifactSize);
|
||||
try (final InputStream inputStream = artifactManagement.loadArtifactBinary(result.getSha1Hash()).get()
|
||||
final byte[] randomBytes = randomBytes(artifactSize);
|
||||
try (final InputStream input = new ByteArrayInputStream(randomBytes)) {
|
||||
final Artifact artifact = createArtifactForSoftwareModule("file1",
|
||||
testdataFactory.createSoftwareModuleOs().getId(), artifactSize, input);
|
||||
try (final InputStream inputStream = artifactManagement.loadArtifactBinary(artifact.getSha1Hash()).get()
|
||||
.getFileInputStream()) {
|
||||
assertTrue("The stored binary matches the given binary",
|
||||
IOUtils.contentEquals(new ByteArrayInputStream(random), inputStream));
|
||||
IOUtils.contentEquals(new ByteArrayInputStream(randomBytes), inputStream));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -353,7 +376,7 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest {
|
||||
|
||||
final int artifactSize = 5 * 1024;
|
||||
try (final InputStream input = new RandomGeneratedInputStream(artifactSize)) {
|
||||
artifactManagement.create(input, sm.getId(), "file1", false, artifactSize);
|
||||
createArtifactForSoftwareModule("file1", sm.getId(), artifactSize, input);
|
||||
assertThat(artifactManagement.findBySoftwareModule(PAGE, sm.getId())).hasSize(1);
|
||||
}
|
||||
}
|
||||
@@ -368,10 +391,27 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTest {
|
||||
final int artifactSize = 5 * 1024;
|
||||
try (final InputStream inputStream1 = new RandomGeneratedInputStream(artifactSize);
|
||||
final InputStream inputStream2 = new RandomGeneratedInputStream(artifactSize)) {
|
||||
artifactManagement.create(inputStream1, sm.getId(), "file1", false, artifactSize);
|
||||
artifactManagement.create(inputStream2, sm.getId(), "file2", false, artifactSize);
|
||||
createArtifactForSoftwareModule("file1", sm.getId(), artifactSize, inputStream1);
|
||||
createArtifactForSoftwareModule("file2", sm.getId(), artifactSize, inputStream2);
|
||||
assertThat(artifactManagement.getByFilenameAndSoftwareModule("file1", sm.getId())).isPresent();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private Artifact createArtifactForSoftwareModule(final String filename, final long moduleId, final int artifactSize)
|
||||
throws IOException {
|
||||
final byte[] randomBytes = randomBytes(artifactSize);
|
||||
try (final InputStream inputStream = new ByteArrayInputStream(randomBytes)) {
|
||||
return createArtifactForSoftwareModule(filename, moduleId, artifactSize, inputStream);
|
||||
}
|
||||
}
|
||||
|
||||
private Artifact createArtifactForSoftwareModule(final String filename, final long moduleId, final int artifactSize,
|
||||
final InputStream inputStream) throws IOException {
|
||||
return artifactManagement.create(inputStream, moduleId, filename, false, artifactSize);
|
||||
}
|
||||
|
||||
private static byte[] randomBytes(final int len) {
|
||||
return RandomStringUtils.randomAlphanumeric(len).getBytes();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -280,7 +280,7 @@ public class SoftwareModuleManagementTest extends AbstractJpaIntegrationTest {
|
||||
assertThat(softwareModuleManagement.get(unassignedModule.getId())).isNotPresent();
|
||||
|
||||
// verify: binary data of artifact is deleted
|
||||
assertArtfiactNull(artifact1, artifact2);
|
||||
assertArtifactNull(artifact1, artifact2);
|
||||
|
||||
// verify: meta data of artifact is deleted
|
||||
assertThat(artifactRepository.findOne(artifact1.getId())).isNull();
|
||||
@@ -311,7 +311,7 @@ public class SoftwareModuleManagementTest extends AbstractJpaIntegrationTest {
|
||||
final Iterator<Artifact> artifactsIt = assignedModule.getArtifacts().iterator();
|
||||
final Artifact artifact1 = artifactsIt.next();
|
||||
final Artifact artifact2 = artifactsIt.next();
|
||||
assertArtfiactNull(artifact1, artifact2);
|
||||
assertArtifactNull(artifact1, artifact2);
|
||||
|
||||
// verify: artifact meta data is still available
|
||||
assertThat(artifactRepository.findOne(artifact1.getId())).isNotNull();
|
||||
@@ -351,7 +351,7 @@ public class SoftwareModuleManagementTest extends AbstractJpaIntegrationTest {
|
||||
final Iterator<Artifact> artifactsIt = assignedModule.getArtifacts().iterator();
|
||||
final Artifact artifact1 = artifactsIt.next();
|
||||
final Artifact artifact2 = artifactsIt.next();
|
||||
assertArtfiactNull(artifact1, artifact2);
|
||||
assertArtifactNull(artifact1, artifact2);
|
||||
|
||||
// verify: artifact meta data is still available
|
||||
assertThat(artifactRepository.findOne(artifact1.getId())).isNotNull();
|
||||
@@ -392,7 +392,7 @@ public class SoftwareModuleManagementTest extends AbstractJpaIntegrationTest {
|
||||
assertThat(softwareModuleManagement.get(moduleY.getId())).isPresent();
|
||||
|
||||
// verify: binary data of artifact is not deleted
|
||||
assertArtfiactNotNull(artifactY);
|
||||
assertArtifactNotNull(artifactY);
|
||||
|
||||
// verify: meta data of artifactX is deleted
|
||||
assertThat(artifactRepository.findOne(artifactX.getId())).isNull();
|
||||
@@ -451,7 +451,7 @@ public class SoftwareModuleManagementTest extends AbstractJpaIntegrationTest {
|
||||
assertThat(softwareModuleRepository.findAll()).hasSize(2);
|
||||
|
||||
// verify: binary data of artifact is deleted
|
||||
assertArtfiactNull(artifactX, artifactY);
|
||||
assertArtifactNull(artifactX, artifactY);
|
||||
|
||||
// verify: meta data of artifactX and artifactY is not deleted
|
||||
assertThat(artifactRepository.findOne(artifactY.getId())).isNotNull();
|
||||
@@ -480,14 +480,14 @@ public class SoftwareModuleManagementTest extends AbstractJpaIntegrationTest {
|
||||
|
||||
assertThat(artifacts).hasSize(numberArtifacts);
|
||||
if (numberArtifacts != 0) {
|
||||
assertArtfiactNotNull(artifacts.toArray(new Artifact[artifacts.size()]));
|
||||
assertArtifactNotNull(artifacts.toArray(new Artifact[artifacts.size()]));
|
||||
}
|
||||
|
||||
artifacts.forEach(artifact -> assertThat(artifactRepository.findOne(artifact.getId())).isNotNull());
|
||||
return softwareModule;
|
||||
}
|
||||
|
||||
private void assertArtfiactNotNull(final Artifact... results) {
|
||||
private void assertArtifactNotNull(final Artifact... results) {
|
||||
assertThat(artifactRepository.findAll()).hasSize(results.length);
|
||||
for (final Artifact result : results) {
|
||||
assertThat(result.getId()).isNotNull();
|
||||
@@ -496,7 +496,7 @@ public class SoftwareModuleManagementTest extends AbstractJpaIntegrationTest {
|
||||
}
|
||||
}
|
||||
|
||||
private void assertArtfiactNull(final Artifact... results) {
|
||||
private void assertArtifactNull(final Artifact... results) {
|
||||
for (final Artifact result : results) {
|
||||
assertThat(binaryArtifactRepository.getArtifactBySha1(tenantAware.getCurrentTenant(), result.getSha1Hash()))
|
||||
.isNull();
|
||||
|
||||
@@ -18,7 +18,8 @@ hawkbit.server.security.dos.maxSoftwareModuleTypesPerDistributionSetType=10
|
||||
hawkbit.server.security.dos.maxSoftwareModulesPerDistributionSet=10
|
||||
hawkbit.server.security.dos.maxArtifactsPerSoftwareModule=10
|
||||
hawkbit.server.security.dos.maxTargetsPerRolloutGroup=1000
|
||||
hawkbit.server.security.dos.maxArtifactSize=1000000
|
||||
hawkbit.server.security.dos.maxArtifactSize=600000
|
||||
hawkbit.server.security.dos.maxArtifactStorage=1000000
|
||||
# Quota - END
|
||||
|
||||
# Debug utility functions - START
|
||||
|
||||
@@ -29,7 +29,6 @@ import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Random;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.lang3.RandomStringUtils;
|
||||
@@ -70,7 +69,8 @@ import ru.yandex.qatools.allure.annotations.Stories;
|
||||
*/
|
||||
@Features("Component Tests - Management API")
|
||||
@Stories("Software Module Resource")
|
||||
@TestPropertySource(properties = { "hawkbit.server.security.dos.maxArtifactSize=100000" })
|
||||
@TestPropertySource(properties = { "hawkbit.server.security.dos.maxArtifactSize=100000",
|
||||
"hawkbit.server.security.dos.maxArtifactStorage=500000" })
|
||||
public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegrationTest {
|
||||
|
||||
@Before
|
||||
@@ -157,7 +157,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
|
||||
final SoftwareModule sm = testdataFactory.createSoftwareModuleOs();
|
||||
|
||||
// create test file
|
||||
final byte random[] = RandomStringUtils.random(5 * 1024).getBytes();
|
||||
final byte random[] = randomBytes(5 * 1024);
|
||||
final String md5sum = HashGeneratorUtils.generateMD5(random);
|
||||
final String sha1sum = HashGeneratorUtils.generateSHA1(random);
|
||||
final MockMultipartFile file = new MockMultipartFile("file", "origFilename", null, random);
|
||||
@@ -195,8 +195,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
|
||||
final long maxSize = quotaManagement.getMaxArtifactSize();
|
||||
|
||||
// create a file which exceeds the configured maximum size
|
||||
final byte[] randomBytes = new byte[Math.toIntExact(maxSize) + 1024];
|
||||
new Random().nextBytes(randomBytes);
|
||||
final byte[] randomBytes = randomBytes(Math.toIntExact(maxSize) + 1024);
|
||||
|
||||
final MockMultipartFile file = new MockMultipartFile("file", "origFilename", null, randomBytes);
|
||||
|
||||
@@ -251,7 +250,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
|
||||
public void duplicateUploadArtifact() throws Exception {
|
||||
final SoftwareModule sm = testdataFactory.createSoftwareModuleOs();
|
||||
|
||||
final byte random[] = RandomStringUtils.random(5 * 1024).getBytes();
|
||||
final byte random[] = randomBytes(5 * 1024);
|
||||
final String md5sum = HashGeneratorUtils.generateMD5(random);
|
||||
final String sha1sum = HashGeneratorUtils.generateSHA1(random);
|
||||
final MockMultipartFile file = new MockMultipartFile("file", "orig", null, random);
|
||||
@@ -274,7 +273,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
|
||||
assertThat(artifactManagement.count()).isEqualTo(0);
|
||||
|
||||
// create test file
|
||||
final byte random[] = RandomStringUtils.random(5 * 1024).getBytes();
|
||||
final byte random[] = randomBytes(5 * 1024);
|
||||
final MockMultipartFile file = new MockMultipartFile("file", "origFilename", null, random);
|
||||
|
||||
// upload
|
||||
@@ -299,7 +298,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
|
||||
assertThat(artifactManagement.count()).isEqualTo(0);
|
||||
|
||||
// create test file
|
||||
final byte random[] = RandomStringUtils.random(5 * 1024).getBytes();
|
||||
final byte random[] = randomBytes(5 * 1024);
|
||||
final String md5sum = HashGeneratorUtils.generateMD5(random);
|
||||
final String sha1sum = HashGeneratorUtils.generateSHA1(random);
|
||||
final MockMultipartFile file = new MockMultipartFile("file", "origFilename", null, random);
|
||||
@@ -343,7 +342,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
|
||||
|
||||
for (int i = 0; i < maxArtifacts; ++i) {
|
||||
// create test file
|
||||
final byte random[] = RandomStringUtils.random(5 * 1024).getBytes();
|
||||
final byte random[] = randomBytes(5 * 1024);
|
||||
final String md5sum = HashGeneratorUtils.generateMD5(random);
|
||||
final String sha1sum = HashGeneratorUtils.generateSHA1(random);
|
||||
final MockMultipartFile file = new MockMultipartFile("file", "origFilename" + i, null, random);
|
||||
@@ -360,7 +359,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
|
||||
}
|
||||
|
||||
// upload one more file to cause the quota to be exceeded
|
||||
final byte random[] = RandomStringUtils.random(5 * 1024).getBytes();
|
||||
final byte random[] = randomBytes(5 * 1024);
|
||||
HashGeneratorUtils.generateMD5(random);
|
||||
HashGeneratorUtils.generateSHA1(random);
|
||||
final MockMultipartFile file = new MockMultipartFile("file", "origFilename_final", null, random);
|
||||
@@ -374,13 +373,58 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
@Description("Verifies that artifacts can only be added as long as the artifact storage quota is not exceeded.")
|
||||
public void uploadArtifactsUntilStorageQuotaExceeded() throws Exception {
|
||||
|
||||
final long storageLimit = quotaManagement.getMaxArtifactStorage();
|
||||
|
||||
// choose an artifact size which does not violate the max file size
|
||||
final int artifactSize = Math.toIntExact(quotaManagement.getMaxArtifactSize() / 10);
|
||||
final int numArtifacts = Math.toIntExact(storageLimit / artifactSize);
|
||||
|
||||
for (int i = 0; i < numArtifacts; ++i) {
|
||||
// create test file
|
||||
final byte random[] = randomBytes(artifactSize);
|
||||
final String md5sum = HashGeneratorUtils.generateMD5(random);
|
||||
final String sha1sum = HashGeneratorUtils.generateSHA1(random);
|
||||
final MockMultipartFile file = new MockMultipartFile("file", "origFilename" + i, null, random);
|
||||
|
||||
// upload
|
||||
final SoftwareModule sm = testdataFactory.createSoftwareModuleOs("sm" + i);
|
||||
mvc.perform(fileUpload("/rest/v1/softwaremodules/{smId}/artifacts", sm.getId()).file(file)
|
||||
.accept(MediaType.APPLICATION_JSON)).andDo(MockMvcResultPrinter.print())
|
||||
.andExpect(status().isCreated())
|
||||
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8_VALUE))
|
||||
.andExpect(jsonPath("$.hashes.md5", equalTo(md5sum)))
|
||||
.andExpect(jsonPath("$.hashes.sha1", equalTo(sha1sum)))
|
||||
.andExpect(jsonPath("$.size", equalTo(random.length)))
|
||||
.andExpect(jsonPath("$.providedFilename", equalTo("origFilename" + i))).andReturn();
|
||||
}
|
||||
|
||||
// upload one more file to cause the quota to be exceeded
|
||||
final byte random[] = randomBytes(artifactSize);
|
||||
HashGeneratorUtils.generateMD5(random);
|
||||
HashGeneratorUtils.generateSHA1(random);
|
||||
final MockMultipartFile file = new MockMultipartFile("file", "origFilename_final", null, random);
|
||||
|
||||
// upload
|
||||
final SoftwareModule sm = testdataFactory.createSoftwareModuleOs("sm" + numArtifacts);
|
||||
mvc.perform(fileUpload("/rest/v1/softwaremodules/{smId}/artifacts", sm.getId()).file(file)
|
||||
.accept(MediaType.APPLICATION_JSON)).andDo(MockMvcResultPrinter.print())
|
||||
.andExpect(status().isForbidden())
|
||||
.andExpect(jsonPath("$.exceptionClass", equalTo(QuotaExceededException.class.getName())))
|
||||
.andExpect(jsonPath("$.errorCode", equalTo(SpServerError.SP_QUOTA_EXCEEDED.getKey())));
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
@Description("Tests binary download of an artifact including verfication that the downloaded binary is consistent and that the etag header is as expected identical to the SHA1 hash of the file.")
|
||||
public void downloadArtifact() throws Exception {
|
||||
final SoftwareModule sm = testdataFactory.createSoftwareModuleOs();
|
||||
|
||||
final int artifactSize = 5 * 1024;
|
||||
final byte random[] = RandomStringUtils.random(artifactSize).getBytes();
|
||||
final byte random[] = randomBytes(artifactSize);
|
||||
|
||||
final Artifact artifact = artifactManagement.create(new ByteArrayInputStream(random), sm.getId(), "file1",
|
||||
false, artifactSize);
|
||||
@@ -412,7 +456,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
|
||||
final SoftwareModule sm = testdataFactory.createSoftwareModuleOs();
|
||||
|
||||
final int artifactSize = 5 * 1024;
|
||||
final byte random[] = RandomStringUtils.random(artifactSize).getBytes();
|
||||
final byte random[] = randomBytes(artifactSize);
|
||||
|
||||
final Artifact artifact = artifactManagement.create(new ByteArrayInputStream(random), sm.getId(), "file1",
|
||||
false, artifactSize);
|
||||
@@ -439,7 +483,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
|
||||
final SoftwareModule sm = testdataFactory.createSoftwareModuleOs();
|
||||
|
||||
final int artifactSize = 5 * 1024;
|
||||
final byte random[] = RandomStringUtils.random(artifactSize).getBytes();
|
||||
final byte random[] = randomBytes(artifactSize);
|
||||
|
||||
final Artifact artifact = artifactManagement.create(new ByteArrayInputStream(random), sm.getId(), "file1",
|
||||
false, artifactSize);
|
||||
@@ -470,7 +514,7 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
|
||||
public void invalidRequestsOnArtifactResource() throws Exception {
|
||||
|
||||
final int artifactSize = 5 * 1024;
|
||||
final byte random[] = RandomStringUtils.random(artifactSize).getBytes();
|
||||
final byte random[] = randomBytes(artifactSize);
|
||||
final MockMultipartFile file = new MockMultipartFile("file", "orig", null, random);
|
||||
|
||||
final SoftwareModule sm = testdataFactory.createSoftwareModuleOs();
|
||||
@@ -1021,4 +1065,8 @@ public class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegra
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] randomBytes(final int len) {
|
||||
return RandomStringUtils.randomAlphanumeric(len).getBytes();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -162,9 +162,14 @@ public class HawkbitSecurityProperties {
|
||||
private int maxTargetsPerAutoAssignment = 5000;
|
||||
|
||||
/**
|
||||
* Maximum size of artifacts in bytes.
|
||||
* Maximum size of artifacts in bytes. Defaults to 1 GB.
|
||||
*/
|
||||
private long maxArtifactSize = 1_000_000_000;
|
||||
private long maxArtifactSize = 1_073_741_824;
|
||||
|
||||
/**
|
||||
* Maximum size of all artifacts in bytes. Defaults to 20 GB.
|
||||
*/
|
||||
private long maxArtifactStorage = 21_474_836_480L;
|
||||
|
||||
private final Filter filter = new Filter();
|
||||
private final Filter uiFilter = new Filter();
|
||||
@@ -290,6 +295,14 @@ public class HawkbitSecurityProperties {
|
||||
return maxArtifactSize;
|
||||
}
|
||||
|
||||
public long getMaxArtifactStorage() {
|
||||
return maxArtifactStorage;
|
||||
}
|
||||
|
||||
public void setMaxArtifactStorage(final long maxArtifactStorage) {
|
||||
this.maxArtifactStorage = maxArtifactStorage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Configuration for hawkBits DOS prevention filter. This is usually an
|
||||
* infrastructure topic (e.g. Web Application Firewall (WAF)) but might
|
||||
|
||||
@@ -15,7 +15,6 @@ import java.util.regex.Pattern;
|
||||
|
||||
import javax.servlet.http.Cookie;
|
||||
|
||||
import com.vaadin.shared.ui.label.ContentMode;
|
||||
import org.eclipse.hawkbit.im.authentication.MultitenancyIndicator;
|
||||
import org.eclipse.hawkbit.im.authentication.TenantUserPasswordAuthenticationToken;
|
||||
import org.eclipse.hawkbit.ui.AbstractHawkbitUI;
|
||||
@@ -48,6 +47,7 @@ import com.vaadin.server.VaadinRequest;
|
||||
import com.vaadin.server.VaadinService;
|
||||
import com.vaadin.server.WebBrowser;
|
||||
import com.vaadin.shared.Position;
|
||||
import com.vaadin.shared.ui.label.ContentMode;
|
||||
import com.vaadin.ui.Alignment;
|
||||
import com.vaadin.ui.Button;
|
||||
import com.vaadin.ui.Component;
|
||||
|
||||
Reference in New Issue
Block a user