lingwei.kong/hawkbit
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Fix upload quota check and provide better error message (#893)

Browse Source 
* Adjusted upload quota check to include file size and show proper error message

Signed-off-by: Alexander Dobler <alexander.dobler3@bosch-si.com>

* Fixed failing upload quota tests

Signed-off-by: Alexander Dobler <alexander.dobler3@bosch-si.com>

* Moved quota check to stream, fixed review findings

Signed-off-by: Alexander Dobler <alexander.dobler3@bosch-si.com>

* Added missing license header to QuotaInputStream

Signed-off-by: Alexander Dobler <alexander.dobler3@bosch-si.com>

* Reworked uploadLock, ensured error messages may be translated, review fixes

Signed-off-by: Alexander Dobler <alexander.dobler3@bosch-si.com>

* Added local artifactrepo to gitignore

Signed-off-by: Alexander Dobler <alexander.dobler3@bosch-si.com>

* Fixed sonar issues and assignment quota message

Signed-off-by: Alexander Dobler <alexander.dobler3@bosch-si.com>

* PR review fixes

Signed-off-by: Alexander Dobler <alexander.dobler3@bosch-si.com>

* Split quota exceptions, PR fixes

Signed-off-by: Alexander Dobler <alexander.dobler3@bosch-si.com>

* Removed left over conversion method in quota helper

Signed-off-by: Alexander Dobler <alexander.dobler3@bosch-si.com>

* Made conversion helper class final

Signed-off-by: Alexander Dobler <alexander.dobler3@bosch-si.com>
This commit is contained in:
Alexander Dobler
2019-10-30 11:24:33 +01:00
committed by Dominic Schabel
parent 9b8ab40c55
commit 09f2d8a481
43 changed files with 477 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/JpaArtifactManagement.java
View File

@@ -8,6 +8,8 @@
*/
package org.eclipse.hawkbit.repository.jpa;
import java.io.IOException;
import java.io.InputStream;
import java.util.Optional;
import org.eclipse.hawkbit.artifact.repository.ArtifactRepository;
@@ -23,11 +25,11 @@ import org.eclipse.hawkbit.repository.exception.EntityAlreadyExistsException;
import org.eclipse.hawkbit.repository.exception.EntityNotFoundException;
import org.eclipse.hawkbit.repository.exception.InvalidMD5HashException;
import org.eclipse.hawkbit.repository.exception.InvalidSHA1HashException;
import org.eclipse.hawkbit.repository.exception.QuotaExceededException;
import org.eclipse.hawkbit.repository.jpa.configuration.Constants;
import org.eclipse.hawkbit.repository.jpa.model.JpaArtifact;
import org.eclipse.hawkbit.repository.jpa.model.JpaSoftwareModule;
import org.eclipse.hawkbit.repository.jpa.utils.QuotaHelper;
import org.eclipse.hawkbit.repository.jpa.utils.FileSizeAndStorageQuotaCheckingInputStream;
import org.eclipse.hawkbit.repository.model.Artifact;
import org.eclipse.hawkbit.repository.model.ArtifactUpload;
import org.eclipse.hawkbit.repository.model.SoftwareModule;
@@ -53,10 +55,6 @@ public class JpaArtifactManagement implements ArtifactManagement {
private static final Logger LOG = LoggerFactory.getLogger(JpaArtifactManagement.class);
private static final String MAX_ARTIFACT_SIZE_EXCEEDED = "Quota exceeded: The artifact '%s' (%s bytes) which has been uploaded for software module '%s' exceeds the maximum artifact size of %s bytes.";
private static final String MAX_ARTIFACT_SIZE_TOTAL_EXCEEDED = "Quota exceeded: The artifact '%s' (%s bytes) cannot be uploaded. The maximum total artifact storage of %s bytes would be exceeded.";
private final LocalArtifactRepository localArtifactRepository;
private final SoftwareModuleRepository softwareModuleRepository;
@@ -105,8 +103,6 @@ public class JpaArtifactManagement implements ArtifactManagement {
final Artifact existing = checkForExistingArtifact(filename, artifactUpload.overrideExisting(), softwareModule);
assertArtifactQuota(moduleId, 1);
assertMaxArtifactSizeQuota(filename, moduleId, artifactUpload.getFilesize());
assertMaxArtifactStorageQuota(filename, artifactUpload.getFilesize());
return getOrCreateArtifact(artifactUpload)
.map(artifact -> storeArtifactMetadata(softwareModule, filename, artifact, existing)).orElse(null);
@@ -125,11 +121,12 @@ public class JpaArtifactManagement implements ArtifactManagement {
}
private AbstractDbArtifact storeArtifact(final ArtifactUpload artifactUpload) {
try {
return artifactRepository.store(tenantAware.getCurrentTenant(), artifactUpload.getInputStream(),
try(final InputStream quotaStream = wrapInQuotaStream(artifactUpload.getInputStream())) {
return artifactRepository.store(tenantAware.getCurrentTenant(), quotaStream,
artifactUpload.getFilename(), artifactUpload.getContentType(),
new DbArtifactHash(artifactUpload.getProvidedSha1Sum(), artifactUpload.getProvidedMd5Sum(), artifactUpload.getProvidedSha256Sum()));
} catch (final ArtifactStoreException e) {
new DbArtifactHash(artifactUpload.getProvidedSha1Sum(), artifactUpload.getProvidedMd5Sum(),
artifactUpload.getProvidedSha256Sum()));
} catch (final ArtifactStoreException | IOException e) {
throw new ArtifactUploadFailedException(e);
} catch (final HashNotMatchException e) {
if (e.getHashFunction().equals(HashNotMatchException.SHA1)) {
@@ -145,31 +142,13 @@ public class JpaArtifactManagement implements ArtifactManagement {
Artifact.class, SoftwareModule.class, localArtifactRepository::countBySoftwareModuleId);
}
private void assertMaxArtifactSizeQuota(final String filename, final long id, final long artifactSize) {
private InputStream wrapInQuotaStream(final InputStream in) {
final long maxArtifactSize = quotaManagement.getMaxArtifactSize();
if (maxArtifactSize <= 0) {
return;
}
if (artifactSize > maxArtifactSize) {
final String msg = String.format(MAX_ARTIFACT_SIZE_EXCEEDED, filename, artifactSize, id, maxArtifactSize);
LOG.warn(msg);
throw new QuotaExceededException(msg);
}
}
private void assertMaxArtifactStorageQuota(final String filename, final long artifactSize) {
final long currentlyUsed = localArtifactRepository.getSumOfUndeletedArtifactSize().orElse(0L);
final long maxArtifactSizeTotal = quotaManagement.getMaxArtifactStorage();
if (maxArtifactSizeTotal <= 0) {
return;
}
final Long currentlyUsed = localArtifactRepository.getSumOfUndeletedArtifactSize().orElse(0L);
if (currentlyUsed + artifactSize > maxArtifactSizeTotal) {
final String msg = String.format(MAX_ARTIFACT_SIZE_TOTAL_EXCEEDED, filename, artifactSize,
maxArtifactSizeTotal);
LOG.warn(msg);
throw new QuotaExceededException(msg);
}
return new FileSizeAndStorageQuotaCheckingInputStream(in, maxArtifactSize, maxArtifactSizeTotal - currentlyUsed);
}
@Override

4
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/JpaDistributionSetTypeManagement.java
View File

@@ -23,7 +23,7 @@ import org.eclipse.hawkbit.repository.builder.DistributionSetTypeUpdate;
import org.eclipse.hawkbit.repository.builder.GenericDistributionSetTypeUpdate;
import org.eclipse.hawkbit.repository.exception.EntityNotFoundException;
import org.eclipse.hawkbit.repository.exception.EntityReadOnlyException;
import org.eclipse.hawkbit.repository.exception.QuotaExceededException;
import org.eclipse.hawkbit.repository.exception.AssignmentQuotaExceededException;
import org.eclipse.hawkbit.repository.jpa.builder.JpaDistributionSetTypeCreate;
import org.eclipse.hawkbit.repository.jpa.configuration.Constants;
import org.eclipse.hawkbit.repository.jpa.model.JpaDistributionSetType;
@@ -163,7 +163,7 @@ public class JpaDistributionSetTypeManagement implements DistributionSetTypeMana
* @param requested
* number of software module types to check
*
* @throws QuotaExceededException
* @throws AssignmentQuotaExceededException
* if the software module type quota is exceeded
*/
private void assertSoftwareModuleTypeQuota(final long id, final int requested) {

81
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/utils/FileSizeAndStorageQuotaCheckingInputStream.java Normal file
View File

@@ -0,0 +1,81 @@
/**
* Copyright (c) 2019 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.repository.jpa.utils;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import org.eclipse.hawkbit.repository.exception.FileSizeQuotaExceededException;
import org.eclipse.hawkbit.repository.exception.StorageQuotaExceededException;
/**
* A FilterInputStream that ensures file size and storage quotas are enforced. It check during read operations if the
* quota will be exceeded and throws an QuotaExceededException if this happens.
*/
public class FileSizeAndStorageQuotaCheckingInputStream extends FilterInputStream {
private final long quota;
private final long sizeLimit;
private long size;
/**
* Creates a <code>QuotaInputStream</code> using the input stream in and a
* limiting quota
*
* @param in
* Inner InputStream that read operations will be forwarded to
* @param sizeLimit
* Quota file size limit in byte
* @param storageLeft
* Storage left until quota is reached
*/
public FileSizeAndStorageQuotaCheckingInputStream(final InputStream in, final long sizeLimit, final long storageLeft) {
super(in);
// only limit to lower bound to avoid two checks
this.quota = Math.min(sizeLimit, storageLeft);
this.sizeLimit = sizeLimit;
}
@Override
public int read() throws IOException {
final int read = super.read();
checkQuotaAndUpdateSize(read);
return read;
}
@Override
public int read(final byte[] b) throws IOException {
final int read = super.read(b);
checkQuotaAndUpdateSize(read);
return read;
}
@Override
public int read(final byte[] b, final int off, final int len) throws IOException {
final int read = super.read(b, off, len);
checkQuotaAndUpdateSize(read);
return read;
}
private void checkQuotaAndUpdateSize(final int read) {
if ((size + read) > quota) {
// pick exception based on quota type
if (quota == sizeLimit) {
throw new FileSizeQuotaExceededException(quota);
} else {
throw new StorageQuotaExceededException(quota);
}
} else if (read >= 0) {
size += read;
}
}
}

21
hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/utils/QuotaHelper.java
View File

@@ -12,12 +12,12 @@ import java.util.function.ToLongFunction;
import javax.validation.constraints.NotNull;
import org.eclipse.hawkbit.repository.exception.QuotaExceededException;
import org.eclipse.hawkbit.repository.exception.AssignmentQuotaExceededException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Helper class to check assignment quotas.
* Helper class to check quotas.
*/
public final class QuotaHelper {
@@ -26,6 +26,8 @@ public final class QuotaHelper {
*/
private static final Logger LOG = LoggerFactory.getLogger(QuotaHelper.class);
private static final String MAX_ASSIGNMENT_QUOTA_EXCEEDED = "Quota exceeded: Cannot assign %s entities at once. The maximum is %s.";
private QuotaHelper() {
// no need to instantiate this class
}
@@ -44,7 +46,7 @@ public final class QuotaHelper {
* @param parentType
* The type of the parent entity.
*
* @throws QuotaExceededException
* @throws AssignmentQuotaExceededException
* if the assignment operation would cause the quota to be
* exceeded
*/
@@ -72,7 +74,7 @@ public final class QuotaHelper {
* Function to count the entities that are currently assigned to
* the parent entity.
*
* @throws QuotaExceededException
* @throws AssignmentQuotaExceededException
* if the assignment operation would cause the quota to be
* exceeded
*/
@@ -100,7 +102,7 @@ public final class QuotaHelper {
* Function to count the entities that are currently assigned to
* the parent entity.
*
* @throws QuotaExceededException
* @throws AssignmentQuotaExceededException
* if the assignment operation would cause the quota to be
* exceeded
*/
@@ -117,7 +119,7 @@ public final class QuotaHelper {
final String parentIdStr = parentId != null ? String.valueOf(parentId) : "<new>";
LOG.warn("Cannot assign {} {} entities to {} '{}' because of the configured quota limit {}.", requested,
type, parentType, parentIdStr, limit);
throw new QuotaExceededException(type, parentType, parentId, requested, limit);
throw new AssignmentQuotaExceededException(type, parentType, parentId, requested, limit);
}
if (parentId != null && countFct != null) {
@@ -126,7 +128,7 @@ public final class QuotaHelper {
LOG.warn(
"Cannot assign {} {} entities to {} '{}' because of the configured quota limit {}. Currently, there are {} {} entities assigned.",
requested, type, parentType, parentId, limit, currentCount, type);
throw new QuotaExceededException(type, parentType, parentId, requested, limit);
throw new AssignmentQuotaExceededException(type, parentType, parentId, requested, limit);
}
}
}
@@ -142,10 +144,9 @@ public final class QuotaHelper {
*/
public static void assertAssignmentRequestSizeQuota(final long requested, final long limit) {
if (requested > limit) {
final String message = String.format(
"Quota exceeded: Cannot assign %s entities at once. The maximum is %s.", requested, limit);
final String message = String.format(MAX_ASSIGNMENT_QUOTA_EXCEEDED, requested, limit);
LOG.warn(message);
throw new QuotaExceededException(message);
throw new AssignmentQuotaExceededException(message);
}
}
}