lingwei.kong/hawkbit
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Cleanup file streaming utilities (#559)

Browse Source 
* Cleanup file streaming.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Added missing comments.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Fix typo.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Split utility class.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Dependency cleanup.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Add missing dependency,

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Remove repository api dependency from rest core.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Fix build and sonar issues.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Remove custom ConstraintViolationException

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* RequestMapping should be public.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Fix errors.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Removed dead code.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Not null

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Fix nullpointer.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Code cleanup.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>
This commit is contained in:
Kai Zimmermann
2017-07-19 12:43:07 +02:00
committed by GitHub
parent 66feae2756
commit 09b24fa97d
59 changed files with 911 additions and 908 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
hawkbit-rest-core/pom.xml
View File

@@ -21,22 +21,34 @@
<dependencies>
<dependency>
<groupId>org.eclipse.hawkbit</groupId>
<artifactId>hawkbit-repository-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.hawkbit</groupId>
<artifactId>hawkbit-repository-jpa</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.hawkbit</groupId>
<artifactId>hawkbit-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.hateoas</groupId>
<artifactId>spring-hateoas</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-commons</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
@@ -52,23 +64,29 @@
</dependency>
<!-- Test -->
<dependency>
<dependency>
<groupId>org.eclipse.hawkbit</groupId>
<artifactId>hawkbit-repository-test</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
</dependency>
<dependency>
<groupId>org.eclipse.hawkbit</groupId>
<artifactId>hawkbit-repository-jpa</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>ru.yandex.qatools.allure</groupId>
<artifactId>allure-junit-adaptor</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<scope>test</scope>

3
hawkbit-rest-core/src/main/java/org/eclipse/hawkbit/rest/data/SortDirection.java
View File

@@ -12,9 +12,6 @@ import org.eclipse.hawkbit.rest.exception.SortParameterUnsupportedDirectionExcep
/**
* A definition of possible sorting direction.
*
*
*
*/
public enum SortDirection {
/**

30
hawkbit-rest-core/src/main/java/org/eclipse/hawkbit/rest/exception/MultiPartFileUploadException.java Normal file
View File

@@ -0,0 +1,30 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.rest.exception;
import org.eclipse.hawkbit.exception.AbstractServerRtException;
import org.eclipse.hawkbit.exception.SpServerError;
/**
* Thrown if a multi part exception occurred.
*
*/
public final class MultiPartFileUploadException extends AbstractServerRtException {
private static final long serialVersionUID = 1L;
/**
* @param cause
* for the exception
*/
public MultiPartFileUploadException(final Throwable cause) {
super(cause.getMessage(), SpServerError.SP_ARTIFACT_UPLOAD_FAILED, cause);
}
}

41
hawkbit-rest-core/src/main/java/org/eclipse/hawkbit/rest/exception/ResponseExceptionHandler.java
View File

@@ -11,14 +11,15 @@ package org.eclipse.hawkbit.rest.exception;
import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import javax.servlet.http.HttpServletRequest;
import javax.validation.ConstraintViolationException;
import javax.validation.ValidationException;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.eclipse.hawkbit.exception.AbstractServerRtException;
import org.eclipse.hawkbit.exception.SpServerError;
import org.eclipse.hawkbit.repository.exception.MultiPartFileUploadException;
import org.eclipse.hawkbit.rest.json.model.ExceptionInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,6 +42,8 @@ public class ResponseExceptionHandler {
private static final Map<SpServerError, HttpStatus> ERROR_TO_HTTP_STATUS = new EnumMap<>(SpServerError.class);
private static final HttpStatus DEFAULT_RESPONSE_STATUS = HttpStatus.INTERNAL_SERVER_ERROR;
private static final String MESSAGE_FORMATTER_SEPARATOR = " ";
static {
ERROR_TO_HTTP_STATUS.put(SpServerError.SP_REPO_ENTITY_NOT_EXISTS, HttpStatus.NOT_FOUND);
ERROR_TO_HTTP_STATUS.put(SpServerError.SP_REPO_ENTITY_ALRREADY_EXISTS, HttpStatus.CONFLICT);
@@ -138,12 +141,40 @@ public class ResponseExceptionHandler {
*/
@ExceptionHandler(ConstraintViolationException.class)
public ResponseEntity<ExceptionInfo> handleConstraintViolationException(final HttpServletRequest request,
final Exception ex) {
final ConstraintViolationException ex) {
logRequest(request, ex);
final ExceptionInfo response = createExceptionInfo(
new org.eclipse.hawkbit.repository.exception.ConstraintViolationException(
(ConstraintViolationException) ex));
final ExceptionInfo response = new ExceptionInfo();
response.setMessage(ex.getConstraintViolations().stream().map(
violation -> violation.getPropertyPath() + MESSAGE_FORMATTER_SEPARATOR + violation.getMessage() + ".")
.collect(Collectors.joining(MESSAGE_FORMATTER_SEPARATOR)));
response.setExceptionClass(ex.getClass().getName());
response.setErrorCode(SpServerError.SP_REPO_CONSTRAINT_VIOLATION.getKey());
return new ResponseEntity<>(response, HttpStatus.BAD_REQUEST);
}
/**
* Method for handling exception of type ValidationException which is thrown
* in case the request is rejected due to invalid requests. Called by the
* Spring-Framework for exception handling.
*
* @param request
* the Http request
* @param ex
* the exception which occurred
* @return the entity to be responded containing the exception information
* as entity.
*/
@ExceptionHandler(ValidationException.class)
public ResponseEntity<ExceptionInfo> handleValidationException(final HttpServletRequest request,
final ValidationException ex) {
logRequest(request, ex);
final ExceptionInfo response = new ExceptionInfo();
response.setMessage(ex.getMessage());
response.setExceptionClass(ex.getClass().getName());
response.setErrorCode(SpServerError.SP_REPO_CONSTRAINT_VIOLATION.getKey());
return new ResponseEntity<>(response, HttpStatus.BAD_REQUEST);
}

112
hawkbit-rest-core/src/main/java/org/eclipse/hawkbit/rest/util/ByteRange.java
View File

@@ -1,112 +0,0 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.rest.util;
/**
* Byte range for resume download operations.
*
*
*
*
*
*/
public class ByteRange {
public static final String MULTIPART_BOUNDARY = "THIS_STRING_SEPARATES_MULTIPART";
private final long start;
private final long end;
private final long length;
private final long total;
/**
* Construct a byte range.
*
* @param start
* Start of the byte range.
* @param end
* End of the byte range.
* @param total
* Total length of the byte source.
*/
public ByteRange(final long start, final long end, final long total) {
this.start = start;
this.end = end;
length = end - start + 1;
this.total = total;
}
/**
* @return the start
*/
public long getStart() {
return start;
}
/**
* @return the end
*/
public long getEnd() {
return end;
}
/**
* @return the length
*/
public long getLength() {
return length;
}
/**
* @return the total
*/
public long getTotal() {
return total;
}
@Override
// NOSONAR - as this is generated
@SuppressWarnings("squid:S864")
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (int) (end ^ end >>> 32);
result = prime * result + (int) (length ^ length >>> 32);
result = prime * result + (int) (start ^ start >>> 32);
result = prime * result + (int) (total ^ total >>> 32);
return result;
}
@Override
public boolean equals(final Object obj) { // NOSONAR - as this is generated
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
final ByteRange other = (ByteRange) obj;
if (end != other.end) {
return false;
}
if (length != other.length) {
return false;
}
if (start != other.start) {
return false;
}
if (total != other.total) {
return false;
}
return true;
}
}

20
hawkbit-rest-core/src/main/java/org/eclipse/hawkbit/rest/util/FileSteamingFailedException.java → hawkbit-rest-core/src/main/java/org/eclipse/hawkbit/rest/util/FileStreamingFailedException.java
View File

@@ -14,7 +14,7 @@ import org.eclipse.hawkbit.exception.SpServerError;
/**
* Thrown if artifact content streaming to client failed.
*/
public final class FileSteamingFailedException extends AbstractServerRtException {
public final class FileStreamingFailedException extends AbstractServerRtException {
private static final long serialVersionUID = 1L;
@@ -22,7 +22,7 @@ public final class FileSteamingFailedException extends AbstractServerRtException
* Creates a new FileUploadFailedException with
* {@link SpServerError#SP_REST_BODY_NOT_READABLE} error.
*/
public FileSteamingFailedException() {
public FileStreamingFailedException() {
super(SpServerError.SP_ARTIFACT_LOAD_FAILED);
}
@@ -32,7 +32,7 @@ public final class FileSteamingFailedException extends AbstractServerRtException
* @param cause
* for the exception
*/
public FileSteamingFailedException(final Throwable cause) {
public FileStreamingFailedException(final Throwable cause) {
super(SpServerError.SP_ARTIFACT_LOAD_FAILED, cause);
}
@@ -42,7 +42,19 @@ public final class FileSteamingFailedException extends AbstractServerRtException
* @param message
* of the error
*/
public FileSteamingFailedException(final String message) {
public FileStreamingFailedException(final String message) {
super(message, SpServerError.SP_ARTIFACT_LOAD_FAILED);
}
/**
* Constructor with error string and cause.
*
* @param message
* of the error
* @param cause
* for the exception
*/
public FileStreamingFailedException(final String message, final Throwable cause) {
super(message, SpServerError.SP_ARTIFACT_LOAD_FAILED, cause);
}
}

29
hawkbit-rest-core/src/main/java/org/eclipse/hawkbit/rest/util/FileStreamingProgressListener.java Normal file
View File

@@ -0,0 +1,29 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.rest.util;
/**
* Listener for progress on artifact file streaming.
*
*/
@FunctionalInterface
public interface FileStreamingProgressListener {
/**
* Called multiple times during streaming.
*
* @param requestedBytes
* requested bytes of the request
* @param shippedBytesSinceLast
* since the last report
* @param shippedBytesOverall
* during the request
*/
void progress(long requestedBytes, long shippedBytesSinceLast, long shippedBytesOverall);
}

445
hawkbit-rest-core/src/main/java/org/eclipse/hawkbit/rest/util/FileStreamingUtil.java Normal file
View File

@@ -0,0 +1,445 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.rest.util;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.math.RoundingMode;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.eclipse.hawkbit.artifact.repository.model.AbstractDbArtifact;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import com.google.common.base.Preconditions;
import com.google.common.io.ByteStreams;
import com.google.common.math.DoubleMath;
/**
* Utility class for artifact file streaming.
*/
public final class FileStreamingUtil {
private static final Logger LOG = LoggerFactory.getLogger(FileStreamingUtil.class);
/**
* File suffix for MDH hash download (see Linux md5sum).
*/
public static final String ARTIFACT_MD5_DWNL_SUFFIX = ".MD5SUM";
private static final int BUFFER_SIZE = 0x2000; // 8k
private FileStreamingUtil() {
}
/**
* Write a md5 file response.
*
* @param response
* the response
* @param md5Hash
* of the artifact
* @param filename
* as provided by the client
* @return the response
* @throws IOException
* cannot write output stream
*/
public static ResponseEntity<Void> writeMD5FileResponse(final HttpServletResponse response, final String md5Hash,
final String filename) throws IOException {
if (md5Hash == null) {
return ResponseEntity.notFound().build();
}
final StringBuilder builder = new StringBuilder();
builder.append(md5Hash);
builder.append(" ");
builder.append(filename);
final byte[] content = builder.toString().getBytes(StandardCharsets.US_ASCII);
final StringBuilder header = new StringBuilder().append("attachment;filename=").append(filename)
.append(ARTIFACT_MD5_DWNL_SUFFIX);
response.setContentLength(content.length);
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, header.toString());
response.getOutputStream().write(content);
return ResponseEntity.ok().build();
}
/**
* <p>
* Write response with target relation and publishes events concerning the
* download progress based on given update action status.
* </p>
*
* <p>
* The request supports RFC7233 range requests.
* </p>
*
* @param artifact
* the artifact
* @param filename
* to be written to the client response
* @param lastModified
* unix timestamp of the artifact
* @param response
* to be sent back to the requesting client
* @param request
* from the client
* @param progressListener
* to write progress updates to
*
* @return http response
*
* @see <a href="https://tools.ietf.org/html/rfc7233">https://tools.ietf.org
* /html/rfc7233</a>
*
* @throws FileStreamingFailedException
* if streaming fails
*/
public static ResponseEntity<InputStream> writeFileResponse(final AbstractDbArtifact artifact,
final String filename, final Long lastModified, final HttpServletResponse response,
final HttpServletRequest request, final FileStreamingProgressListener progressListener) {
ResponseEntity<InputStream> result;
final String etag = artifact.getHashes().getSha1();
final long length = artifact.getSize();
response.reset();
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + filename);
response.setHeader(HttpHeaders.ETAG, etag);
response.setHeader(HttpHeaders.ACCEPT_RANGES, "bytes");
if (lastModified != null) {
response.setDateHeader(HttpHeaders.LAST_MODIFIED, lastModified);
}
response.setContentType(MediaType.APPLICATION_OCTET_STREAM_VALUE);
response.setBufferSize(BUFFER_SIZE);
final ByteRange full = new ByteRange(0, length - 1, length);
final List<ByteRange> ranges = new ArrayList<>();
// Validate and process Range and If-Range headers.
final String range = request.getHeader("Range");
if (lastModified != null && range != null) {
LOG.debug("range header for filename ({}) is: {}", filename, range);
// Range header matches"bytes=n-n,n-n,n-n..."
if (!range.matches("^bytes=\\d*-\\d*(,\\d*-\\d*)*$")) {
response.setHeader(HttpHeaders.CONTENT_RANGE, "bytes */" + length);
LOG.debug("range header for filename ({}) is not satisfiable: ", filename);
return new ResponseEntity<>(HttpStatus.REQUESTED_RANGE_NOT_SATISFIABLE);
}
// RFC: if the representation is unchanged, send me the part(s) that
// I am requesting in
// Range; otherwise, send me the entire representation.
checkForShortcut(request, etag, lastModified, full, ranges);
// it seems there are valid ranges
result = extractRange(response, length, ranges, range);
// return if range extraction turned out to be invalid
if (result != null) {
return result;
}
}
// full request - no range
if (ranges.isEmpty() || ranges.get(0).equals(full)) {
LOG.debug("filename ({}) results into a full request: ", filename);
result = handleFullFileRequest(artifact, filename, response, progressListener, full);
}
// standard range request
else if (ranges.size() == 1) {
LOG.debug("filename ({}) results into a standard range request: ", filename);
result = handleStandardRangeRequest(artifact, filename, response, progressListener, ranges);
}
// multipart range request
else {
LOG.debug("filename ({}) results into a multipart range request: ", filename);
result = handleMultipartRangeRequest(artifact, filename, response, progressListener, ranges);
}
return result;
}
private static ResponseEntity<InputStream> handleFullFileRequest(final AbstractDbArtifact artifact,
final String filename, final HttpServletResponse response,
final FileStreamingProgressListener progressListener, final ByteRange full) {
final ByteRange r = full;
response.setHeader(HttpHeaders.CONTENT_RANGE, "bytes " + r.getStart() + "-" + r.getEnd() + "/" + r.getTotal());
response.setContentLengthLong(r.getLength());
try (InputStream from = artifact.getFileInputStream()) {
final ServletOutputStream to = response.getOutputStream();
copyStreams(from, to, progressListener, r.getStart(), r.getLength(), filename);
} catch (final IOException e) {
throw new FileStreamingFailedException("fullfileRequest " + filename, e);
}
return ResponseEntity.ok().build();
}
private static ResponseEntity<InputStream> extractRange(final HttpServletResponse response, final long length,
final List<ByteRange> ranges, final String range) {
if (ranges.isEmpty()) {
for (final String part : range.substring(6).split(",")) {
long start = sublong(part, 0, part.indexOf('-'));
long end = sublong(part, part.indexOf('-') + 1, part.length());
if (start == -1) {
start = length - end;
end = length - 1;
} else if (end == -1 || end > length - 1) {
end = length - 1;
}
// Check if Range is syntactically valid. If not, then return
// 416.
if (start > end) {
response.setHeader(HttpHeaders.CONTENT_RANGE, "bytes */" + length);
return new ResponseEntity<>(HttpStatus.REQUESTED_RANGE_NOT_SATISFIABLE);
}
// Add range.
ranges.add(new ByteRange(start, end, length));
}
}
return null;
}
private static long sublong(final String value, final int beginIndex, final int endIndex) {
final String substring = value.substring(beginIndex, endIndex);
return substring.length() > 0 ? Long.parseLong(substring) : -1;
}
private static void checkForShortcut(final HttpServletRequest request, final String etag, final long lastModified,
final ByteRange full, final List<ByteRange> ranges) {
final String ifRange = request.getHeader(HttpHeaders.IF_RANGE);
if (ifRange != null && !ifRange.equals(etag)) {
try {
final long ifRangeTime = request.getDateHeader(HttpHeaders.IF_RANGE);
if (ifRangeTime != -1 && ifRangeTime + 1000 < lastModified) {
ranges.add(full);
}
} catch (final IllegalArgumentException ignore) {
LOG.info("Invalid if-range header field", ignore);
ranges.add(full);
}
}
}
private static ResponseEntity<InputStream> handleMultipartRangeRequest(final AbstractDbArtifact artifact,
final String filename, final HttpServletResponse response,
final FileStreamingProgressListener progressListener, final List<ByteRange> ranges) {
response.setContentType("multipart/byteranges; boundary=" + ByteRange.MULTIPART_BOUNDARY);
response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
try {
final ServletOutputStream to = response.getOutputStream();
for (final ByteRange r : ranges) {
try (InputStream from = artifact.getFileInputStream()) {
// Add multipart boundary and header fields for every range.
to.println();
to.println("--" + ByteRange.MULTIPART_BOUNDARY);
to.println(HttpHeaders.CONTENT_RANGE + ": bytes " + r.getStart() + "-" + r.getEnd() + "/"
+ r.getTotal());
// Copy single part range of multi part range.
copyStreams(from, to, progressListener, r.getStart(), r.getLength(), filename);
}
}
// End with final multipart boundary.
to.println();
to.print("--" + ByteRange.MULTIPART_BOUNDARY + "--");
} catch (final IOException e) {
throw new FileStreamingFailedException("multipartRangeRequest " + filename, e);
}
return ResponseEntity.status(HttpStatus.PARTIAL_CONTENT).build();
}
private static ResponseEntity<InputStream> handleStandardRangeRequest(final AbstractDbArtifact artifact,
final String filename, final HttpServletResponse response,
final FileStreamingProgressListener progressListener, final List<ByteRange> ranges) {
final ByteRange r = ranges.get(0);
response.setHeader(HttpHeaders.CONTENT_RANGE, "bytes " + r.getStart() + "-" + r.getEnd() + "/" + r.getTotal());
response.setContentLengthLong(r.getLength());
response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
try (InputStream from = artifact.getFileInputStream()) {
final ServletOutputStream to = response.getOutputStream();
copyStreams(from, to, progressListener, r.getStart(), r.getLength(), filename);
} catch (final IOException e) {
LOG.error("standardRangeRequest of file ({}) failed!", filename, e);
throw new FileStreamingFailedException(filename);
}
return ResponseEntity.status(HttpStatus.PARTIAL_CONTENT).build();
}
private static long copyStreams(final InputStream from, final OutputStream to,
final FileStreamingProgressListener progressListener, final long start, final long length,
final String filename) throws IOException {
final long startMillis = System.currentTimeMillis();
LOG.trace("Start of copy-streams of file {} from {} to {}", filename, start, length);
Preconditions.checkNotNull(from);
Preconditions.checkNotNull(to);
final byte[] buf = new byte[BUFFER_SIZE];
long total = 0;
int progressPercent = 1;
ByteStreams.skipFully(from, start);
long toRead = length;
boolean toContinue = true;
long shippedSinceLastEvent = 0;
while (toContinue) {
final int r = from.read(buf);
if (r == -1) {
break;
}
toRead -= r;
if (toRead > 0) {
to.write(buf, 0, r);
total += r;
shippedSinceLastEvent += r;
} else {
to.write(buf, 0, (int) toRead + r);
total += toRead + r;
shippedSinceLastEvent += toRead + r;
toContinue = false;
}
if (progressListener != null) {
final int newPercent = DoubleMath.roundToInt(total * 100.0 / length, RoundingMode.DOWN);
// every 10 percent an event
if (newPercent == 100 || newPercent > progressPercent + 10) {
progressPercent = newPercent;
progressListener.progress(length, shippedSinceLastEvent, total);
shippedSinceLastEvent = 0;
}
}
}
final long totalTime = System.currentTimeMillis() - startMillis;
if (total < length) {
throw new FileStreamingFailedException(filename + ": " + (length - total)
+ " bytes could not be written to client, total time on write: !" + totalTime + " ms");
}
LOG.trace("Finished copy-stream of file {} with length {} in {} ms", filename, length, totalTime);
return total;
}
private static final class ByteRange {
private static final String MULTIPART_BOUNDARY = "THIS_STRING_SEPARATES_MULTIPART";
private final long start;
private final long end;
private final long length;
private final long total;
private ByteRange(final long start, final long end, final long total) {
this.start = start;
this.end = end;
length = end - start + 1;
this.total = total;
}
private long getStart() {
return start;
}
private long getEnd() {
return end;
}
private long getLength() {
return length;
}
private long getTotal() {
return total;
}
@Override
// Generated code
@SuppressWarnings("squid:S864")
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (int) (end ^ (end >>> 32));
result = prime * result + (int) (length ^ (length >>> 32));
result = prime * result + (int) (start ^ (start >>> 32));
result = prime * result + (int) (total ^ (total >>> 32));
return result;
}
@Override
// Generated code
@SuppressWarnings("squid:S1126")
public boolean equals(final Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
final ByteRange other = (ByteRange) obj;
if (end != other.end) {
return false;
}
if (length != other.length) {
return false;
}
if (start != other.start) {
return false;
}
if (total != other.total) {
return false;
}
return true;
}
}
}

38
hawkbit-rest-core/src/main/java/org/eclipse/hawkbit/rest/util/HttpUtil.java Normal file
View File

@@ -0,0 +1,38 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.rest.util;
import java.util.Arrays;
/**
* Utility class for the Rest Source API.
*/
public final class HttpUtil {
private HttpUtil() {
}
/**
* Checks given CSV string for defined match value or wildcard.
*
* @param matchHeader
* to search through
* @param toMatch
* to search for
*
* @return <code>true</code> if string matches.
*/
public static boolean matchesHttpHeader(final String matchHeader, final String toMatch) {
final String[] matchValues = matchHeader.split("\\s*,\\s*");
Arrays.sort(matchValues);
return Arrays.binarySearch(matchValues, toMatch) > -1 || Arrays.binarySearch(matchValues, "*") > -1;
}
}

352
hawkbit-rest-core/src/main/java/org/eclipse/hawkbit/rest/util/RestResourceConversionHelper.java
View File

@@ -1,352 +0,0 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.rest.util;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.math.RoundingMode;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.eclipse.hawkbit.artifact.repository.model.DbArtifact;
import org.eclipse.hawkbit.repository.ControllerManagement;
import org.eclipse.hawkbit.repository.model.ActionStatus;
import org.eclipse.hawkbit.repository.model.Artifact;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import com.google.common.base.Preconditions;
import com.google.common.math.DoubleMath;
import com.google.common.net.HttpHeaders;
/**
* Utility class for the Rest Source API.
*/
public final class RestResourceConversionHelper {
private static final Logger LOG = LoggerFactory.getLogger(RestResourceConversionHelper.class);
private static final int BUFFER_SIZE = 4096;
private RestResourceConversionHelper() {
}
/**
* Write response without target relation.
*
* @param artifact
* the artifact
* @param servletResponse
* to be sent back to the requesting client
* @param request
* from the client
* @param file
* to be write to the client response
*
* @return http code
*/
public static ResponseEntity<InputStream> writeFileResponse(final Artifact artifact,
final HttpServletResponse servletResponse, final HttpServletRequest request, final DbArtifact file) {
return writeFileResponse(artifact, servletResponse, request, file, null, null);
}
/**
* <p>
* Write response with target relation and publishes events concerning the
* download progress based on given update action status.
* </p>
*
* <p>
* The request supports RFC7233 range requests.
* </p>
*
* @param artifact
* the artifact
* @param response
* to be sent back to the requesting client
* @param request
* from the client
* @param file
* to be write to the client response
* @param controllerManagement
* to write progress updates to
* @param statusId
* of the {@link ActionStatus}
*
* @return http code
*
* @see <a href="https://tools.ietf.org/html/rfc7233">https://tools.ietf.org
* /html/rfc7233</a>
*/
public static ResponseEntity<InputStream> writeFileResponse(final Artifact artifact,
final HttpServletResponse response, final HttpServletRequest request, final DbArtifact file,
final ControllerManagement controllerManagement, final Long statusId) {
ResponseEntity<InputStream> result;
final String etag = artifact.getSha1Hash();
final Long lastModified = artifact.getLastModifiedAt() != null ? artifact.getLastModifiedAt()
: artifact.getCreatedAt();
final long length = file.getSize();
response.reset();
response.setBufferSize(BUFFER_SIZE);
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + artifact.getFilename());
response.setHeader(HttpHeaders.ETAG, etag);
response.setHeader(HttpHeaders.ACCEPT_RANGES, "bytes");
response.setDateHeader(HttpHeaders.LAST_MODIFIED, lastModified);
response.setContentType(MediaType.APPLICATION_OCTET_STREAM_VALUE);
final ByteRange full = new ByteRange(0, length - 1, length);
final List<ByteRange> ranges = new ArrayList<>();
// Validate and process Range and If-Range headers.
final String range = request.getHeader("Range");
if (range != null) {
LOG.debug("range header for filename ({}) is: {}", artifact.getFilename(), range);
// Range header matches"bytes=n-n,n-n,n-n..."
if (!range.matches("^bytes=\\d*-\\d*(,\\d*-\\d*)*$")) {
response.setHeader(HttpHeaders.CONTENT_RANGE, "bytes */" + length);
LOG.debug("range header for filename ({}) is not satisfiable: ", artifact.getFilename());
return new ResponseEntity<>(HttpStatus.REQUESTED_RANGE_NOT_SATISFIABLE);
}
// RFC: if the representation is unchanged, send me the part(s) that
// I am requesting in
// Range; otherwise, send me the entire representation.
checkForShortcut(request, etag, lastModified, full, ranges);
// it seems there are valid ranges
result = extractRange(response, length, ranges, range);
// return if range extraction turned out to be invalid
if (result != null) {
return result;
}
}
// full request - no range
if (ranges.isEmpty() || ranges.get(0).equals(full)) {
LOG.debug("filename ({}) results into a full request: ", artifact.getFilename());
handleFullFileRequest(artifact, response, file, controllerManagement, statusId, full);
result = ResponseEntity.ok().build();
}
// standard range request
else if (ranges.size() == 1) {
LOG.debug("filename ({}) results into a standard range request: ", artifact.getFilename());
handleStandardRangeRequest(artifact, response, file, controllerManagement, statusId, ranges);
result = new ResponseEntity<>(HttpStatus.PARTIAL_CONTENT);
}
// multipart range request
else {
LOG.debug("filename ({}) results into a multipart range request: ", artifact.getFilename());
handleMultipartRangeRequest(artifact, response, file, controllerManagement, statusId, ranges);
result = new ResponseEntity<>(HttpStatus.PARTIAL_CONTENT);
}
return result;
}
private static void handleFullFileRequest(final Artifact artifact, final HttpServletResponse response,
final DbArtifact file, final ControllerManagement controllerManagement, final Long statusId,
final ByteRange full) {
final ByteRange r = full;
response.setHeader(HttpHeaders.CONTENT_RANGE, "bytes " + r.getStart() + "-" + r.getEnd() + "/" + r.getTotal());
response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(r.getLength()));
try (InputStream inputStream = file.getFileInputStream()) {
copyStreams(inputStream, response.getOutputStream(), controllerManagement, statusId, r.getStart(),
r.getLength());
} catch (final IOException e) {
LOG.error("fullfileRequest of file ({}) failed!", artifact.getFilename(), e);
throw new FileSteamingFailedException(artifact.getFilename());
}
}
private static ResponseEntity<InputStream> extractRange(final HttpServletResponse response, final long length,
final List<ByteRange> ranges, final String range) {
if (ranges.isEmpty()) {
for (final String part : range.substring(6).split(",")) {
long start = sublong(part, 0, part.indexOf('-'));
long end = sublong(part, part.indexOf('-') + 1, part.length());
if (start == -1) {
start = length - end;
end = length - 1;
} else if (end == -1 || end > length - 1) {
end = length - 1;
}
// Check if Range is syntactically valid. If not, then return
// 416.
if (start > end) {
response.setHeader(HttpHeaders.CONTENT_RANGE, "bytes */" + length);
return new ResponseEntity<>(HttpStatus.REQUESTED_RANGE_NOT_SATISFIABLE);
}
// Add range.
ranges.add(new ByteRange(start, end, length));
}
}
return null;
}
private static long sublong(final String value, final int beginIndex, final int endIndex) {
final String substring = value.substring(beginIndex, endIndex);
return substring.length() > 0 ? Long.parseLong(substring) : -1;
}
private static void checkForShortcut(final HttpServletRequest request, final String etag, final long lastModified,
final ByteRange full, final List<ByteRange> ranges) {
final String ifRange = request.getHeader(HttpHeaders.IF_RANGE);
if (ifRange != null && !ifRange.equals(etag)) {
try {
final long ifRangeTime = request.getDateHeader(HttpHeaders.IF_RANGE);
if (ifRangeTime != -1 && ifRangeTime + 1000 < lastModified) {
ranges.add(full);
}
} catch (final IllegalArgumentException ignore) {
LOG.info("Invalid if-range header field", ignore);
ranges.add(full);
}
}
}
private static void handleMultipartRangeRequest(final Artifact artifact, final HttpServletResponse response,
final DbArtifact file, final ControllerManagement controllerManagement, final Long statusId,
final List<ByteRange> ranges) {
response.setContentType("multipart/byteranges; boundary=" + ByteRange.MULTIPART_BOUNDARY);
response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
for (final ByteRange r : ranges) {
try (InputStream inputStream = file.getFileInputStream()) {
// Add multipart boundary and header fields for every range.
response.getOutputStream().println();
response.getOutputStream().println("--" + ByteRange.MULTIPART_BOUNDARY);
response.getOutputStream()
.println("Content-Range: bytes " + r.getStart() + "-" + r.getEnd() + "/" + r.getTotal());
// Copy single part range of multi part range.
copyStreams(inputStream, response.getOutputStream(), controllerManagement, statusId, r.getStart(),
r.getLength());
} catch (final IOException e) {
throwFileStreamingFailedException(artifact, e);
}
}
try {
// End with final multipart boundary.
response.getOutputStream().println();
response.getOutputStream().print("--" + ByteRange.MULTIPART_BOUNDARY + "--");
} catch (final IOException e) {
throwFileStreamingFailedException(artifact, e);
}
}
private static void throwFileStreamingFailedException(final Artifact artifact, final IOException e) {
LOG.error("multipartRangeRequest of file ({}) failed!", artifact.getFilename(), e);
throw new FileSteamingFailedException(artifact.getFilename());
}
private static void handleStandardRangeRequest(final Artifact artifact, final HttpServletResponse response,
final DbArtifact file, final ControllerManagement controllerManagement, final Long statusId,
final List<ByteRange> ranges) {
final ByteRange r = ranges.get(0);
response.setHeader(HttpHeaders.CONTENT_RANGE, "bytes " + r.getStart() + "-" + r.getEnd() + "/" + r.getTotal());
response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(r.getLength()));
response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
try (InputStream inputStream = file.getFileInputStream()) {
copyStreams(inputStream, response.getOutputStream(), controllerManagement, statusId, r.getStart(),
r.getLength());
} catch (final IOException e) {
LOG.error("standardRangeRequest of file ({}) failed!", artifact.getFilename(), e);
throw new FileSteamingFailedException(artifact.getFilename());
}
}
private static long copyStreams(final InputStream from, final OutputStream to,
final ControllerManagement controllerManagement, final Long statusId, final long start, final long length)
throws IOException {
Preconditions.checkNotNull(from);
Preconditions.checkNotNull(to);
final byte[] buf = new byte[BUFFER_SIZE];
long total = 0;
int progressPercent = 1;
// skipp until start is reached
long skipped = 0;
do {
skipped += from.skip(start);
} while (skipped < start);
long toRead = length;
boolean toContinue = true;
long shippedSinceLastEvent = 0;
while (toContinue) {
final int r = from.read(buf);
if (r == -1) {
break;
}
toRead -= r;
if (toRead > 0) {
to.write(buf, 0, r);
total += r;
shippedSinceLastEvent += r;
} else {
to.write(buf, 0, (int) toRead + r);
total += toRead + r;
shippedSinceLastEvent += toRead + r;
toContinue = false;
}
if (controllerManagement != null) {
final int newPercent = DoubleMath.roundToInt(total * 100.0 / length, RoundingMode.DOWN);
// every 10 percent an event
if (newPercent == 100 || newPercent > progressPercent + 10) {
progressPercent = newPercent;
controllerManagement.downloadProgress(statusId, length, shippedSinceLastEvent, total);
shippedSinceLastEvent = 0;
}
}
}
return total;
}
/**
* Checks given CSV string for defined match value or * wildcard.
*
* @param matchHeader
* to search through
* @param toMatch
* to search for
*
* @return <code>true</code> if string matches.
*/
public static boolean matchesHttpHeader(final String matchHeader, final String toMatch) {
final String[] matchValues = matchHeader.split("\\s*,\\s*");
Arrays.sort(matchValues);
return Arrays.binarySearch(matchValues, toMatch) > -1 || Arrays.binarySearch(matchValues, "*") > -1;
}
}