lingwei.kong/hawkbit
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[#2918] Refactor FileStreamingUtil (#2921)

Browse Source 
Signed-off-by: Avgustin Marinov <Avgustin.Marinov@bosch.com>
This commit is contained in:
Avgustin Marinov
2026-02-13 17:11:43 +02:00
committed by GitHub
parent ef3900a31c
commit 520b887b70
6 changed files with 360 additions and 400 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
hawkbit-ddi/hawkbit-ddi-resource/src/main/java/org/eclipse/hawkbit/ddi/rest/resource/DdiRootController.java
View File

@@ -182,12 +182,11 @@ public class DdiRootController implements DdiRootControllerRestApi {
? logDownload(RequestResponseContextHolder.getHttpServletRequest(), target, module.getId()) ? logDownload(RequestResponseContextHolder.getHttpServletRequest(), target, module.getId())
: null; // range request - could have too many - so doesn't check action, don't log action status, and don't publish events : null; // range request - could have too many - so doesn't check action, don't log action status, and don't publish events
result = FileStreamingUtil.writeFileResponse(file, artifact.getFilename(), artifact.getCreatedAt(), result = FileStreamingUtil.writeFileResponse(file, artifact.getFilename(), artifact.getCreatedAt(),
RequestResponseContextHolder.getHttpServletResponse(), RequestResponseContextHolder.getHttpServletRequest(), RequestResponseContextHolder.getHttpServletResponse(),
RequestResponseContextHolder.getHttpServletRequest(),
(length, shippedSinceLastEvent, total) -> { (length, shippedSinceLastEvent, total) -> {
if (actionStatus != null) { if (actionStatus != null) {
eventPublisher.publishEvent(new DownloadProgressEvent( eventPublisher.publishEvent(
AccessContext.tenant(), actionStatus.getId(), shippedSinceLastEvent)); new DownloadProgressEvent(AccessContext.tenant(), actionStatus.getId(), shippedSinceLastEvent));
} }
}); });
} }

263
hawkbit-ddi/hawkbit-ddi-resource/src/test/java/org/eclipse/hawkbit/ddi/rest/resource/DdiArtifactDownloadTest.java
View File

@@ -10,7 +10,19 @@
package org.eclipse.hawkbit.ddi.rest.resource; package org.eclipse.hawkbit.ddi.rest.resource;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.eclipse.hawkbit.context.AccessContext.tenant;
import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.springframework.http.HttpHeaders.ACCEPT_RANGES;
import static org.springframework.http.HttpHeaders.CONTENT_DISPOSITION;
import static org.springframework.http.HttpHeaders.CONTENT_LENGTH;
import static org.springframework.http.HttpHeaders.CONTENT_RANGE;
import static org.springframework.http.HttpHeaders.CONTENT_TYPE;
import static org.springframework.http.HttpHeaders.ETAG;
import static org.springframework.http.HttpHeaders.LAST_MODIFIED;
import static org.springframework.http.HttpHeaders.RANGE;
import static org.springframework.http.HttpStatus.PARTIAL_CONTENT;
import static org.springframework.http.HttpStatus.REQUESTED_RANGE_NOT_SATISFIABLE;
import static org.springframework.http.MediaType.APPLICATION_OCTET_STREAM;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
@@ -30,7 +42,6 @@ import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.TimeZone; import java.util.TimeZone;
import org.eclipse.hawkbit.context.AccessContext;
import org.eclipse.hawkbit.ddi.rest.resource.DdiArtifactDownloadTest.DownloadTestConfiguration; import org.eclipse.hawkbit.ddi.rest.resource.DdiArtifactDownloadTest.DownloadTestConfiguration;
import org.eclipse.hawkbit.repository.event.remote.DownloadProgressEvent; import org.eclipse.hawkbit.repository.event.remote.DownloadProgressEvent;
import org.eclipse.hawkbit.repository.model.Artifact; import org.eclipse.hawkbit.repository.model.Artifact;
@@ -39,6 +50,8 @@ import org.eclipse.hawkbit.repository.model.DistributionSet;
import org.eclipse.hawkbit.repository.model.Target; import org.eclipse.hawkbit.repository.model.Target;
import org.eclipse.hawkbit.repository.test.util.TestdataFactory; import org.eclipse.hawkbit.repository.test.util.TestdataFactory;
import org.eclipse.hawkbit.repository.test.util.WithUser; import org.eclipse.hawkbit.repository.test.util.WithUser;
import org.hamcrest.BaseMatcher;
import org.hamcrest.Description;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest;
@@ -46,7 +59,7 @@ import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
import org.springframework.context.event.EventListener; import org.springframework.context.event.EventListener;
import org.springframework.http.HttpHeaders; import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType; import org.springframework.http.HttpStatus;
import org.springframework.test.web.servlet.MvcResult; import org.springframework.test.web.servlet.MvcResult;
/** /**
@@ -58,6 +71,8 @@ import org.springframework.test.web.servlet.MvcResult;
@SpringBootTest(classes = { DownloadTestConfiguration.class }) @SpringBootTest(classes = { DownloadTestConfiguration.class })
class DdiArtifactDownloadTest extends AbstractDDiApiIntegrationTest { class DdiArtifactDownloadTest extends AbstractDDiApiIntegrationTest {
private static final String DOWNLOAD_FN = "/{tenant}/controller/v1/{controllerId}/softwaremodules/{smId}/artifacts/{filename}";
private static int downloadProgress = 0; private static int downloadProgress = 0;
private static long shippedBytes = 0; private static long shippedBytes = 0;
@@ -76,86 +91,65 @@ class DdiArtifactDownloadTest extends AbstractDDiApiIntegrationTest {
// create target // create target
final Target target = testdataFactory.createTarget(); final Target target = testdataFactory.createTarget();
final List<Target> targets = Collections.singletonList(target); final List<Target> targets = Collections.singletonList(target);
// create ds // create ds
final DistributionSet ds = testdataFactory.createDistributionSet(""); final DistributionSet ds = testdataFactory.createDistributionSet("");
// create artifact // create artifact
final int artifactSize = 5 * 1024; final int artifactSize = 5 * 1024;
final byte[] random = nextBytes(artifactSize); final byte[] random = nextBytes(artifactSize);
final Artifact artifact = artifactManagement.create(new ArtifactUpload( final Artifact artifact = artifactManagement.create(new ArtifactUpload(
new ByteArrayInputStream(random), null, artifactSize, null, new ByteArrayInputStream(random),
findFirstModuleByType(ds, osType).orElseThrow().getId(), "file1", false)); null, artifactSize, null, findFirstModuleByType(ds, osType).orElseThrow().getId(), "file1", false));
assignDistributionSet(ds, targets); assignDistributionSet(ds, targets);
// no artifact available // no artifact available
mvc.perform(get("/controller/v1/{controllerId}/softwaremodules/{softwareModuleId}/artifacts/123455", mvc.perform(get(DOWNLOAD_FN, tenant(), target.getControllerId(), getOsModule(ds), "123455"))
target.getControllerId(), getOsModule(ds)))
.andExpect(status().isNotFound()); .andExpect(status().isNotFound());
mvc.perform(get("/controller/v1/{controllerId}/softwaremodules/{softwareModuleId}/artifacts/123455.MD5SUM", mvc.perform(get(DOWNLOAD_FN + ".MD5SUM", tenant(), target.getControllerId(), getOsModule(ds), "123455"))
target.getControllerId(), getOsModule(ds)))
.andExpect(status().isNotFound()); .andExpect(status().isNotFound());
// SM does not exist // SM does not exist
mvc.perform(get("/controller/v1/{controllerId}/softwaremodules/1234567890/artifacts/{filename}", mvc.perform(get(DOWNLOAD_FN, tenant(), target.getControllerId(), "1234567890", artifact.getFilename()))
target.getControllerId(), artifact.getFilename()))
.andExpect(status().isNotFound()); .andExpect(status().isNotFound());
mvc.perform(get("/controller/v1/{controllerId}/softwaremodules/1234567890/artifacts/{filename}.MD5SUM", mvc.perform(get(DOWNLOAD_FN + ".MD5SUM", tenant(), target.getControllerId(), "1234567890", artifact.getFilename()))
target.getControllerId(), artifact.getFilename()))
.andExpect(status().isNotFound()); .andExpect(status().isNotFound());
// test now consistent data to test allowed methods // test now consistent data to test allowed methods
mvc.perform(get("/{tenant}/controller/v1/{controllerId}/softwaremodules/{smId}/artifacts/{filename}", mvc.perform(get(DOWNLOAD_FN, tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename())
AccessContext.tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename())
.header(HttpHeaders.IF_MATCH, artifact.getSha1Hash())) .header(HttpHeaders.IF_MATCH, artifact.getSha1Hash()))
.andExpect(status().isOk()); .andExpect(status().isOk());
mvc.perform(get("/{tenant}/controller/v1/{controllerId}/softwaremodules/{smId}/artifacts/{filename}.MD5SUM", mvc.perform(get(DOWNLOAD_FN + ".MD5SUM",
AccessContext.tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename())) tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename()))
.andExpect(status().isOk()); .andExpect(status().isOk());
// test failed If-match // test failed If-match
mvc.perform(get("/{tenant}/controller/v1/{controllerId}/softwaremodules/{smId}/artifacts/{filename}", mvc.perform(get(DOWNLOAD_FN, tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename())
AccessContext.tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename())
.header(HttpHeaders.IF_MATCH, "fsjkhgjfdhg")) .header(HttpHeaders.IF_MATCH, "fsjkhgjfdhg"))
.andExpect(status().isPreconditionFailed()); .andExpect(status().isPreconditionFailed());
// test invalid range // test invalid range
mvc.perform(get("/{tenant}/controller/v1/{controllerId}/softwaremodules/{smId}/artifacts/{filename}", mvc.perform(get(DOWNLOAD_FN, tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename())
AccessContext.tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename()) .header(RANGE, "bytes=1-10,hdsfjksdh"))
.header("Range", "bytes=1-10,hdsfjksdh")) .andExpect(header().string(CONTENT_RANGE, "bytes */" + 5 * 1024))
.andExpect(header().string("Content-Range", "bytes */" + 5 * 1024))
.andExpect(status().isRequestedRangeNotSatisfiable()); .andExpect(status().isRequestedRangeNotSatisfiable());
mvc.perform(get("/{tenant}/controller/v1/{controllerId}/softwaremodules/{smId}/artifacts/{filename}", mvc.perform(get(DOWNLOAD_FN, tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename())
AccessContext.tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename()) .header(RANGE, "bytes=100-10"))
.header("Range", "bytes=100-10")) .andExpect(header().string(CONTENT_RANGE, "bytes */" + 5 * 1024))
.andExpect(header().string("Content-Range", "bytes */" + 5 * 1024))
.andExpect(status().isRequestedRangeNotSatisfiable()); .andExpect(status().isRequestedRangeNotSatisfiable());
// not allowed methods // not allowed methods
mvc.perform(put("/{tenant}/controller/v1/{controllerId}/softwaremodules/{smId}/artifacts/{filename}", mvc.perform(put(DOWNLOAD_FN, tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename()))
AccessContext.tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename()))
.andExpect(status().isMethodNotAllowed()); .andExpect(status().isMethodNotAllowed());
mvc.perform(delete(DOWNLOAD_FN, tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename()))
mvc.perform(delete("/{tenant}/controller/v1/{controllerId}/softwaremodules/{smId}/artifacts/{filename}",
AccessContext.tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename()))
.andExpect(status().isMethodNotAllowed()); .andExpect(status().isMethodNotAllowed());
mvc.perform(post(DOWNLOAD_FN, tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename()))
mvc.perform(post("/{tenant}/controller/v1/{controllerId}/softwaremodules/{smId}/artifacts/{filename}",
AccessContext.tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename()))
.andExpect(status().isMethodNotAllowed()); .andExpect(status().isMethodNotAllowed());
mvc.perform(put(DOWNLOAD_FN + ".MD5SUM", tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename()))
mvc.perform(put("/{tenant}/controller/v1/{controllerId}/softwaremodules/{smId}/artifacts/{filename}.MD5SUM",
AccessContext.tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename()))
.andExpect(status().isMethodNotAllowed()); .andExpect(status().isMethodNotAllowed());
mvc.perform(delete(DOWNLOAD_FN + ".MD5SUM", tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename()))
mvc.perform(delete("/{tenant}/controller/v1/{controllerId}/softwaremodules/{smId}/artifacts/{filename}.MD5SUM",
AccessContext.tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename()))
.andExpect(status().isMethodNotAllowed()); .andExpect(status().isMethodNotAllowed());
mvc.perform(post(DOWNLOAD_FN + ".MD5SUM", tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename()))
mvc.perform(post("/{tenant}/controller/v1/{controllerId}/softwaremodules/{smId}/artifacts/{filename}.MD5SUM",
AccessContext.tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename()))
.andExpect(status().isMethodNotAllowed()); .andExpect(status().isMethodNotAllowed());
} }
@@ -174,32 +168,26 @@ class DdiArtifactDownloadTest extends AbstractDDiApiIntegrationTest {
// create target // create target
final Target target = testdataFactory.createTarget(); final Target target = testdataFactory.createTarget();
final List<Target> targets = Collections.singletonList(target); final List<Target> targets = Collections.singletonList(target);
// create ds // create ds
final DistributionSet ds = testdataFactory.createDistributionSet(""); final DistributionSet ds = testdataFactory.createDistributionSet("");
// create artifact // create artifact
final int artifactSize = (int) quotaManagement.getMaxArtifactSize(); final int artifactSize = (int) quotaManagement.getMaxArtifactSize();
final byte[] random = nextBytes(artifactSize); final byte[] random = nextBytes(artifactSize);
final Artifact artifact = artifactManagement.create(new ArtifactUpload( final Artifact artifact = artifactManagement.create(new ArtifactUpload(
new ByteArrayInputStream(random), null, artifactSize, null, new ByteArrayInputStream(random),
findFirstModuleByType(ds, osType).orElseThrow().getId(), "file1", false)); null, artifactSize, null, findFirstModuleByType(ds, osType).orElseThrow().getId(), "file1", false));
// download fails as artifact is not yet assigned // download fails as artifact is not yet assigned
mvc.perform(get("/controller/v1/{controllerId}/softwaremodules/{softwareModuleId}/artifacts/{filename}", mvc.perform(get(DOWNLOAD_FN, tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename()))
target.getControllerId(), getOsModule(ds), artifact.getFilename()))
.andExpect(status().isNotFound()); .andExpect(status().isNotFound());
// now assign and download successful // now assign and download successful
assignDistributionSet(ds, targets); assignDistributionSet(ds, targets);
final MvcResult result = mvc.perform(get( final MvcResult result = mvc.perform(get(DOWNLOAD_FN, tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename()))
"/{tenant}/controller/v1/{controllerId}/softwaremodules/{softwareModuleId}/artifacts/{filename}",
AccessContext.tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename()))
.andExpect(status().isOk()) .andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_OCTET_STREAM)) .andExpect(content().contentType(APPLICATION_OCTET_STREAM))
.andExpect(header().string("Accept-Ranges", "bytes")) .andExpect(header().string(ACCEPT_RANGES, "bytes"))
.andExpect(header().string("Last-Modified", dateFormat.format(new Date(artifact.getCreatedAt())))) .andExpect(header().string(LAST_MODIFIED, dateFormat.format(new Date(artifact.getCreatedAt()))))
.andExpect(header().string("Content-Disposition", "attachment;filename=" + artifact.getFilename())) .andExpect(header().string(CONTENT_DISPOSITION, "attachment;filename=" + artifact.getFilename()))
.andReturn(); .andReturn();
assertArrayEquals(result.getResponse().getContentAsByteArray(), random, "The same file that was uploaded is expected when downloaded"); assertArrayEquals(result.getResponse().getContentAsByteArray(), random, "The same file that was uploaded is expected when downloaded");
@@ -215,29 +203,24 @@ class DdiArtifactDownloadTest extends AbstractDDiApiIntegrationTest {
* Tests valid MD5SUm file downloads through the artifact resource by identifying the artifact by ID. * Tests valid MD5SUm file downloads through the artifact resource by identifying the artifact by ID.
*/ */
@Test @Test
void downloadMd5sumThroughControllerApi() throws Exception { void downloadMd5SumThroughControllerApi() throws Exception {
// create target // create target
final Target target = testdataFactory.createTarget(); final Target target = testdataFactory.createTarget();
// create ds // create ds
final DistributionSet ds = testdataFactory.createDistributionSet(""); final DistributionSet ds = testdataFactory.createDistributionSet("");
// create artifact // create artifact
final int artifactSize = 5 * 1024; final int artifactSize = 5 * 1024;
final byte[] random = nextBytes(artifactSize); final byte[] random = nextBytes(artifactSize);
final Artifact artifact = artifactManagement.create(new ArtifactUpload( final Artifact artifact = artifactManagement.create(new ArtifactUpload(
new ByteArrayInputStream(random), null, artifactSize, null, new ByteArrayInputStream(random), null, artifactSize, null, getOsModule(ds), "file1", false));
getOsModule(ds), "file1", false));
assignDistributionSet(ds, target); assignDistributionSet(ds, target);
// download // download
final MvcResult result = mvc.perform(get( final MvcResult result = mvc.perform(
"/{tenant}/controller/v1/{controllerId}/softwaremodules/{softwareModuleId}/artifacts/{filename}.MD5SUM", get(DOWNLOAD_FN + ".MD5SUM", tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename()))
AccessContext.tenant(), target.getControllerId(), getOsModule(ds), artifact.getFilename()))
.andExpect(status().isOk()) .andExpect(status().isOk())
.andExpect(header().string("Content-Disposition", .andExpect(header().string(CONTENT_DISPOSITION, "attachment;filename=" + artifact.getFilename() + ".MD5SUM"))
"attachment;filename=" + artifact.getFilename() + ".MD5SUM"))
.andReturn(); .andReturn();
assertThat(result.getResponse().getContentAsByteArray()) assertThat(result.getResponse().getContentAsByteArray())
@@ -253,124 +236,80 @@ class DdiArtifactDownloadTest extends AbstractDDiApiIntegrationTest {
// create target // create target
final Target target = testdataFactory.createTarget(); final Target target = testdataFactory.createTarget();
final List<Target> targets = Collections.singletonList(target); final List<Target> targets = Collections.singletonList(target);
// create ds // create ds
final DistributionSet ds = testdataFactory.createDistributionSet(""); final DistributionSet ds = testdataFactory.createDistributionSet("");
final int resultLength = (int) quotaManagement.getMaxArtifactSize();
// create artifact // create artifact
final int resultLength = (int) quotaManagement.getMaxArtifactSize();
final byte[] random = nextBytes(resultLength); final byte[] random = nextBytes(resultLength);
final Artifact artifact = artifactManagement.create(new ArtifactUpload( final Artifact artifact = artifactManagement.create(new ArtifactUpload(
new ByteArrayInputStream(random), null, resultLength, null, new ByteArrayInputStream(random), null, resultLength, null, getOsModule(ds), "file1", false));
getOsModule(ds), "file1", false));
assertThat(random).hasSize(resultLength); assertThat(random).hasSize(resultLength);
// now assign and download successful // now assign and download successful
assignDistributionSet(ds, targets); assignDistributionSet(ds, targets);
final int range = resultLength / 50; final int range = resultLength / 50;
// full file download with standard range request // full file download with standard range request
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
for (int i = 0; i < resultLength / range; i++) { for (int i = 0; i < resultLength / range; i++) {
final String rangeString = i * range + "-" + ((i + 1) * range - 1); final String rangeString = (i * range) + "-" + ((i + 1) * range - 1);
final MvcResult result = getRange(target, ds, artifact, rangeString, PARTIAL_CONTENT, range, rangeString + "/" + resultLength);
final MvcResult result = mvc.perform(get(
"/{tenant}/controller/v1/{controllerId}/softwaremodules/{softwareModuleId}/artifacts/{filename}",
AccessContext.tenant(), target.getControllerId(), getOsModule(ds), "file1").header("Range",
"bytes=" + rangeString))
.andExpect(status().isPartialContent())
.andExpect(header().string("ETag", artifact.getSha1Hash()))
.andExpect(content().contentType(MediaType.APPLICATION_OCTET_STREAM))
.andExpect(header().string("Accept-Ranges", "bytes"))
.andExpect(header().string("Last-Modified", dateFormat.format(new Date(artifact.getCreatedAt()))))
.andExpect(header().longValue("Content-Length", range))
.andExpect(header().string("Content-Range", "bytes " + rangeString + "/" + resultLength))
.andExpect(header().string("Content-Disposition", "attachment;filename=file1"))
.andReturn();
outputStream.write(result.getResponse().getContentAsByteArray()); outputStream.write(result.getResponse().getContentAsByteArray());
} }
assertThat(outputStream.toByteArray()).isEqualTo(random); assertThat(outputStream.toByteArray()).isEqualTo(random);
// return last 1000 Bytes // return last 1000 Bytes
MvcResult result = mvc.perform( MvcResult result = getRange(
get("/{tenant}/controller/v1/{controllerId}/softwaremodules/{softwareModuleId}/artifacts/{filename}", target, ds, artifact, "-1000",
AccessContext.tenant(), target.getControllerId(), getOsModule(ds), "file1") PARTIAL_CONTENT, 1000, (resultLength - 1000) + "-" + (resultLength - 1) + "/" + resultLength);
.header("Range", "bytes=-1000")) assertThat(result.getResponse().getContentAsByteArray()).isEqualTo(Arrays.copyOfRange(random, resultLength - 1000, resultLength));
.andExpect(status().isPartialContent())
.andExpect(header().string("ETag", artifact.getSha1Hash()))
.andExpect(content().contentType(MediaType.APPLICATION_OCTET_STREAM))
.andExpect(header().string("Accept-Ranges", "bytes"))
.andExpect(header().string("Last-Modified", dateFormat.format(new Date(artifact.getCreatedAt()))))
.andExpect(header().longValue("Content-Length", 1000))
.andExpect(header().string("Content-Range",
"bytes " + (resultLength - 1000) + "-" + (resultLength - 1) + "/" + resultLength))
.andExpect(header().string("Content-Disposition", "attachment;filename=file1"))
.andReturn();
assertThat(result.getResponse().getContentAsByteArray())
.isEqualTo(Arrays.copyOfRange(random, resultLength - 1000, resultLength));
// skip first 1000 Bytes and return the rest // skip first 1000 Bytes and return the rest
result = mvc.perform( result = getRange(
get("/{tenant}/controller/v1/{controllerId}/softwaremodules/{softwareModuleId}/artifacts/{filename}", target, ds, artifact, "1000-",
AccessContext.tenant(), target.getControllerId(), getOsModule(ds), "file1") PARTIAL_CONTENT, resultLength - 1000, 1000 + "-" + (resultLength - 1) + "/" + resultLength);
.header("Range", "bytes=1000-")) assertThat(result.getResponse().getContentAsByteArray()).isEqualTo(Arrays.copyOfRange(random, 1000, resultLength));
.andExpect(status().isPartialContent())
.andExpect(header().string("ETag", artifact.getSha1Hash()))
.andExpect(content().contentType(MediaType.APPLICATION_OCTET_STREAM))
.andExpect(header().string("Accept-Ranges", "bytes"))
.andExpect(header().string("Last-Modified", dateFormat.format(new Date(artifact.getCreatedAt()))))
.andExpect(header().longValue("Content-Length", resultLength - 1000))
.andExpect(header().string("Content-Range", "bytes " + 1000 + "-" + (resultLength - 1) + "/" + resultLength))
.andExpect(header().string("Content-Disposition", "attachment;filename=file1"))
.andReturn();
assertThat(result.getResponse().getContentAsByteArray())
.isEqualTo(Arrays.copyOfRange(random, 1000, resultLength));
// Start download from file end fails // Start download from file end fails
mvc.perform( getRange(target, ds, artifact, random.length + "-", REQUESTED_RANGE_NOT_SATISFIABLE, null, "*/" + resultLength);
get("/{tenant}/controller/v1/{controllerId}/softwaremodules/{softwareModuleId}/artifacts/{filename}",
AccessContext.tenant(), target.getControllerId(), getOsModule(ds), "file1")
.header("Range", "bytes=" + random.length + "-"))
.andExpect(status().isRequestedRangeNotSatisfiable())
.andExpect(content().contentType(MediaType.APPLICATION_OCTET_STREAM))
.andExpect(header().string("Accept-Ranges", "bytes"))
.andExpect(header().string("Last-Modified", dateFormat.format(new Date(artifact.getCreatedAt()))))
.andExpect(header().string("Content-Range", "bytes */" + random.length))
.andExpect(header().string("Content-Disposition", "attachment;filename=file1"));
// multipart download - first 20 bytes in 2 parts // multipart download - first 20 bytes in 2 parts
result = mvc.perform( result = getRange(target, ds, artifact, "0-9,10-19", PARTIAL_CONTENT, null, null);
get("/{tenant}/controller/v1/{controllerId}/softwaremodules/{softwareModuleId}/artifacts/{filename}",
AccessContext.tenant(), target.getControllerId(), getOsModule(ds), "file1")
.header("Range", "bytes=0-9,10-19"))
.andExpect(status().isPartialContent())
.andExpect(header().string("ETag", artifact.getSha1Hash()))
.andExpect(content().contentType("multipart/byteranges; boundary=THIS_STRING_SEPARATES_MULTIPART"))
.andExpect(header().string("Accept-Ranges", "bytes"))
.andExpect(header().string("Last-Modified", dateFormat.format(new Date(artifact.getCreatedAt()))))
.andExpect(header().string("Content-Disposition", "attachment;filename=file1"))
.andReturn();
outputStream.reset(); outputStream.reset();
outputStream.write("--THIS_STRING_SEPARATES_MULTIPART\r\n".getBytes(StandardCharsets.ISO_8859_1));
outputStream.write("\r\n--THIS_STRING_SEPARATES_MULTIPART\r\n".getBytes(StandardCharsets.ISO_8859_1)); outputStream.write(("Content-Type: application/octet-stream\r\n").getBytes(StandardCharsets.ISO_8859_1));
outputStream.write(("Content-Range: bytes 0-9/" + resultLength + "\r\n").getBytes(StandardCharsets.ISO_8859_1)); outputStream.write(("Content-Range: bytes 0-9/" + resultLength + "\r\n\r\n").getBytes(StandardCharsets.ISO_8859_1));
outputStream.write(Arrays.copyOfRange(random, 0, 10)); outputStream.write(Arrays.copyOfRange(random, 0, 10));
outputStream.write("\r\n--THIS_STRING_SEPARATES_MULTIPART\r\n".getBytes(StandardCharsets.ISO_8859_1)); outputStream.write("\r\n--THIS_STRING_SEPARATES_MULTIPART\r\n".getBytes(StandardCharsets.ISO_8859_1));
outputStream outputStream.write(("Content-Type: application/octet-stream\r\n").getBytes(StandardCharsets.ISO_8859_1));
.write(("Content-Range: bytes 10-19/" + resultLength + "\r\n").getBytes(StandardCharsets.ISO_8859_1)); outputStream.write(("Content-Range: bytes 10-19/" + resultLength + "\r\n\r\n").getBytes(StandardCharsets.ISO_8859_1));
outputStream.write(Arrays.copyOfRange(random, 10, 20)); outputStream.write(Arrays.copyOfRange(random, 10, 20));
outputStream.write("\r\n--THIS_STRING_SEPARATES_MULTIPART--".getBytes(StandardCharsets.ISO_8859_1)); outputStream.write("\r\n--THIS_STRING_SEPARATES_MULTIPART--".getBytes(StandardCharsets.ISO_8859_1));
assertThat(result.getResponse().getContentAsByteArray()).isEqualTo(outputStream.toByteArray()); assertThat(result.getResponse().getContentAsByteArray()).isEqualTo(outputStream.toByteArray());
}
private MvcResult getRange(
final Target target, final DistributionSet ds, final Artifact artifact, final String range,
final HttpStatus expectStatus, final Integer expectedContentLength, final String expectContentRange) throws Exception {
return mvc.perform(get(DOWNLOAD_FN, tenant(), target.getControllerId(), getOsModule(ds), "file1").header(RANGE, "bytes=" + range))
.andExpect(status().is(expectStatus.value()))
.andExpect(header().string(
CONTENT_TYPE,
range.contains(",")
? "multipart/byteranges; boundary=THIS_STRING_SEPARATES_MULTIPART"
: APPLICATION_OCTET_STREAM.toString()))
.andExpect(header().string(
ETAG, new HeaderMatcher(expectStatus.value() == 419 ? null : ('"' + artifact.getSha1Hash() + '"'))))
.andExpect(header().string(ACCEPT_RANGES, "bytes"))
.andExpect(header().string(LAST_MODIFIED, dateFormat.format(new Date(artifact.getCreatedAt()))))
.andExpect(header().string(
CONTENT_LENGTH, new HeaderMatcher(expectedContentLength == null ? null : String.valueOf(expectedContentLength))))
.andExpect(header().string(
CONTENT_RANGE, new HeaderMatcher(expectContentRange == null ? null : ("bytes " + expectContentRange))))
.andExpect(header().string(CONTENT_DISPOSITION, "attachment;filename=file1"))
.andReturn();
} }
@Configuration @Configuration
@@ -380,7 +319,25 @@ class DdiArtifactDownloadTest extends AbstractDDiApiIntegrationTest {
Listener cancelEventHandlerStubBean() { Listener cancelEventHandlerStubBean() {
return new Listener(); return new Listener();
} }
}
private static class HeaderMatcher extends BaseMatcher<String> {
private final String expectedValue;
private HeaderMatcher(final String expectedValue) {
this.expectedValue = expectedValue;
}
@Override
public boolean matches(final Object actual) {
return expectedValue == null ? actual == null : expectedValue.equals(String.valueOf(actual));
}
@Override
public void describeTo(final Description description) {
description.appendValue(expectedValue);
}
} }
private static class Listener { private static class Listener {

2
hawkbit-mgmt/hawkbit-mgmt-resource/src/main/java/org/eclipse/hawkbit/mgmt/rest/resource/MgmtDownloadArtifactResource.java
View File

@@ -69,6 +69,6 @@ public class MgmtDownloadArtifactResource implements MgmtDownloadArtifactRestApi
} }
return FileStreamingUtil.writeFileResponse(file, artifact.getFilename(), artifact.getCreatedAt(), return FileStreamingUtil.writeFileResponse(file, artifact.getFilename(), artifact.getCreatedAt(),
RequestResponseContextHolder.getHttpServletResponse(), request, null); request, RequestResponseContextHolder.getHttpServletResponse(), null);
} }
} }

9
hawkbit-mgmt/hawkbit-mgmt-resource/src/test/java/org/eclipse/hawkbit/mgmt/rest/resource/MgmtSoftwareModuleResourceTest.java
View File

@@ -1537,16 +1537,13 @@ class MgmtSoftwareModuleResourceTest extends AbstractManagementApiIntegrationTes
.as("wrong metadata of the filename").isEqualTo("origFilename"); .as("wrong metadata of the filename").isEqualTo("origFilename");
} }
private void downloadAndVerify(final SoftwareModule sm, final byte[] random, final Artifact artifact) private void downloadAndVerify(final SoftwareModule sm, final byte[] random, final Artifact artifact) throws Exception {
throws Exception {
final MvcResult result = mvc final MvcResult result = mvc
.perform( .perform(get("/rest/v1/softwaremodules/{smId}/artifacts/{artId}/download", sm.getId(), artifact.getId()))
get("/rest/v1/softwaremodules/{smId}/artifacts/{artId}/download", sm.getId(), artifact.getId()))
.andExpect(status().isOk()) .andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_OCTET_STREAM)) .andExpect(content().contentType(MediaType.APPLICATION_OCTET_STREAM))
.andExpect(header().string("ETag", artifact.getSha1Hash())) .andExpect(header().string("ETag", '"' + artifact.getSha1Hash() + '"'))
.andReturn(); .andReturn();
assertArrayEquals(result.getResponse().getContentAsByteArray(), random, "Wrong response content"); assertArrayEquals(result.getResponse().getContentAsByteArray(), random, "Wrong response content");
} }

446
hawkbit-rest/hawkbit-rest-core/src/main/java/org/eclipse/hawkbit/rest/util/FileStreamingUtil.java
View File

@@ -9,10 +9,21 @@
*/ */
package org.eclipse.hawkbit.rest.util; package org.eclipse.hawkbit.rest.util;
import static org.springframework.http.HttpHeaders.ACCEPT_RANGES;
import static org.springframework.http.HttpHeaders.CONTENT_DISPOSITION;
import static org.springframework.http.HttpHeaders.CONTENT_RANGE;
import static org.springframework.http.HttpHeaders.CONTENT_TYPE;
import static org.springframework.http.HttpHeaders.ETAG;
import static org.springframework.http.HttpHeaders.IF_RANGE;
import static org.springframework.http.HttpHeaders.LAST_MODIFIED;
import static org.springframework.http.HttpHeaders.RANGE;
import static org.springframework.http.MediaType.APPLICATION_OCTET_STREAM_VALUE;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.OutputStream; import java.io.OutputStream;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@@ -24,17 +35,20 @@ import jakarta.servlet.http.HttpServletResponse;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import lombok.Value;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.eclipse.hawkbit.artifact.model.ArtifactStream; import org.eclipse.hawkbit.artifact.model.ArtifactStream;
import org.eclipse.hawkbit.rest.exception.FileStreamingFailedException; import org.eclipse.hawkbit.rest.exception.FileStreamingFailedException;
import org.springframework.http.HttpHeaders; import org.jspecify.annotations.NonNull;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
/** /**
* Utility class for artifact file streaming. * Utility class for artifact file streaming supporting RFC-7233 range requests.
* <p/>
*
* @see <a href="https://datatracker.ietf.org/doc/html/rfc7233">RFC-7233</a>
*/ */
@NoArgsConstructor(access = AccessLevel.PRIVATE) @NoArgsConstructor(access = AccessLevel.PRIVATE)
@Slf4j @Slf4j
@@ -44,93 +58,77 @@ public final class FileStreamingUtil {
/** /**
* <p> * <p>
* Write response with target relation and publishes events concerning the * Write response with target relation and publishes events concerning the download progress based on given update action status.
* download progress based on given update action status.
* </p>
* <p>
* The request supports RFC7233 range requests.
* </p> * </p>
* *
* @param artifact the artifact * @param artifact the artifact
* @param filename to be written to the client response * @param filename to be written to the client response
* @param lastModified unix timestamp of the artifact * @param lastModified unix timestamp of the artifact
* @param response to be sent back to the requesting client
* @param request from the client * @param request from the client
* @param response to be sent back to the requesting client
* @param progressListener to write progress updates to * @param progressListener to write progress updates to
* @return http response * @return response entity containing the input stream of the artifact file (or ranges requested)
* @throws FileStreamingFailedException if streaming fails * @throws FileStreamingFailedException if streaming fails
* @see <a href="https://tools.ietf.org/html/rfc7233">https://tools.ietf.org/html/rfc7233</a>
*/ */
@SuppressWarnings("java:S3776") // not so complex - linear logic at one place
public static ResponseEntity<InputStream> writeFileResponse( public static ResponseEntity<InputStream> writeFileResponse(
final ArtifactStream artifact, final String filename, final ArtifactStream artifact, final String filename, final long lastModified,
final long lastModified, final HttpServletResponse response, final HttpServletRequest request, final HttpServletRequest request, final HttpServletResponse response,
final FileStreamingProgressListener progressListener) { final FileStreamingProgressListener progressListener) {
ResponseEntity<InputStream> result;
final String etag = artifact.getSha1Hash();
final long length = artifact.getSize();
resetResponseExceptHeaders(response); resetResponseExceptHeaders(response);
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + filename); response.setHeader(CONTENT_DISPOSITION, "attachment;filename=" + encodeFilename(filename));
response.setHeader(HttpHeaders.ETAG, etag);
response.setHeader(HttpHeaders.ACCEPT_RANGES, "bytes");
// set the x-content-type options header to prevent browsers from doing
// MIME-sniffing when downloading an artifact, as this could cause a
// security vulnerability
response.setHeader("X-Content-Type-Options", "nosniff");
if (lastModified > 0) { if (lastModified > 0) {
response.setDateHeader(HttpHeaders.LAST_MODIFIED, lastModified); response.setDateHeader(LAST_MODIFIED, lastModified);
} }
final String etag = '"' + artifact.getSha1Hash() + '"'; // ETag header value should be quoted as per RFC-7232, section 2.3.
response.setContentType(MediaType.APPLICATION_OCTET_STREAM_VALUE); response.setHeader(ETAG, etag);
response.setHeader(ACCEPT_RANGES, "bytes"); // range-unit -> bytes
response.setContentType(APPLICATION_OCTET_STREAM_VALUE);
// set the x-content-type options header to prevent browsers from doing MIME-sniffing when downloading an artifact,
// as this could cause a security vulnerability
response.setHeader("X-Content-Type-Options", "nosniff");
response.setBufferSize(BUFFER_SIZE); response.setBufferSize(BUFFER_SIZE);
final ByteRange full = new ByteRange(0, length - 1, length); final long length = artifact.getSize();
final List<ByteRange> ranges = new ArrayList<>(); // Validate and process Range (only, by spec, for GET methods) and If-Range headers.
final String rangeHeader = "GET".equalsIgnoreCase(request.getMethod()) ? request.getHeader(RANGE) : null;
final List<Range> ranges;
if (rangeHeader != null) {
log.debug("range header for filename ({}) is: {}", filename, rangeHeader);
// Validate and process Range and If-Range headers. if (checkIfRangeSendAll(request, etag, lastModified)) {
final String range = request.getHeader("Range"); ranges = List.of();
if (lastModified > 0 && range != null) { } else {
log.debug("range header for filename ({}) is: {}", filename, range); // it seems there are valid ranges
try {
// Range header matches"bytes=n-n,n-n,n-n..." ranges = Range.of(rangeHeader, length);
if (!range.matches("^bytes=\\d*-\\d*(,\\d*-\\d*)*+$")) { } catch (final IllegalArgumentException e) {
response.setHeader(HttpHeaders.CONTENT_RANGE, "bytes */" + length); log.debug("range header ({}) for filename ({}) is not satisfiable: {}", rangeHeader, filename, e.getMessage());
log.debug("range header for filename ({}) is not satisfiable: ", filename); response.setHeader(CONTENT_RANGE, "bytes */" + length);
return new ResponseEntity<>(HttpStatus.REQUESTED_RANGE_NOT_SATISFIABLE); return new ResponseEntity<>(HttpStatus.REQUESTED_RANGE_NOT_SATISFIABLE);
} }
// RFC: if the representation is unchanged, send me the part(s) that I am requesting in
// Range; otherwise, send me the entire representation.
checkForShortcut(request, etag, lastModified, full, ranges);
// it seems there are valid ranges
result = extractRange(response, length, ranges, range);
// return if range extraction turned out to be invalid
if (result != null) {
return result;
} }
} else {
ranges = List.of();
} }
try (final InputStream inputStream = artifact) { try (final InputStream inputStream = artifact) {
// full request - no range // full request - no range
if (ranges.isEmpty() || ranges.get(0).equals(full)) { if (ranges.isEmpty()) {
log.debug("filename ({}) results into a full request: ", filename); log.debug("filename ({}) results into a full request: ", filename);
result = handleFullFileRequest(inputStream, filename, response, progressListener, full); return handleFullFileRequest(inputStream, filename, length, response, progressListener);
} else if (ranges.size() == 1) { // standard range request } else if (ranges.size() == 1) { // standard range request
log.debug("filename ({}) results into a standard range request: ", filename); log.debug("filename ({}) results into a single range request: ", filename);
result = handleStandardRangeRequest(inputStream, filename, response, progressListener, ranges.get(0)); return handleSingleRangeRequest(inputStream, ranges.get(0), filename, response, progressListener);
} else { // multipart range request } else { // multipart range request
log.debug("filename ({}) results into a multipart range request: ", filename); log.debug("filename ({}) results into a multipart range request: ", filename);
result = handleMultipartRangeRequest(inputStream, filename, response, progressListener, ranges); return handleMultipartRangeRequest(inputStream, ranges, filename, response, progressListener);
} }
} catch (final IOException e) { } catch (final IOException e) {
log.error("streaming of file ({}) failed!", filename, e); log.error("streaming of file ({}) failed!", filename, e);
throw new FileStreamingFailedException(filename, e); throw new FileStreamingFailedException(filename, e);
} }
return result;
} }
private static void resetResponseExceptHeaders(final HttpServletResponse response) { private static void resetResponseExceptHeaders(final HttpServletResponse response) {
@@ -139,126 +137,53 @@ public final class FileStreamingUtil {
for (final String header : response.getHeaderNames()) { for (final String header : response.getHeaderNames()) {
storedHeaders.put(header, response.getHeader(header)); storedHeaders.put(header, response.getHeader(header));
} }
// resetting the response is needed only partially. Headers set before e.b. by // resetting the response is needed only partially. Headers set before e.b. by the CORS security config needs to be persisted.
// the CORS security config needs to be persisted.
response.reset(); response.reset();
// restore headers again // restore headers again
storedHeaders.forEach(response::addHeader); storedHeaders.forEach(response::addHeader);
} }
// RFC: "if the representation is unchanged, send me the part(s) that I am requesting in Range; otherwise, send me the entire representation."
private static boolean checkIfRangeSendAll(final HttpServletRequest request, final String etag, final long lastModified) {
final String ifRange = request.getHeader(IF_RANGE);
if (ifRange != null && !ifRange.equals(etag)) {
try {
final long ifRangeTime = request.getDateHeader(IF_RANGE);
if (ifRangeTime != -1 && ifRangeTime < lastModified) {
return true;
}
} catch (final IllegalArgumentException e) {
log.info("Invalid if-range header field", e);
return true;
}
}
return false;
}
private static ResponseEntity<InputStream> handleFullFileRequest( private static ResponseEntity<InputStream> handleFullFileRequest(
final InputStream inputStream, final String filename, final HttpServletResponse response, final InputStream inputStream, final String filename, final long length, final HttpServletResponse response,
final FileStreamingProgressListener progressListener, final ByteRange full) { final FileStreamingProgressListener progressListener) {
response.setHeader(HttpHeaders.CONTENT_RANGE, "bytes " + full.getStart() + "-" + full.getEnd() + "/" + full.getTotal()); response.setContentLengthLong(length);
response.setContentLengthLong(full.getLength());
try { try {
final ServletOutputStream to = response.getOutputStream(); final ServletOutputStream to = response.getOutputStream();
copyStreams(inputStream, to, progressListener, full.getStart(), full.getLength(), filename); copyStreams(inputStream, 0, length, filename, to, progressListener);
} catch (final IOException e) { } catch (final IOException e) {
throw new FileStreamingFailedException("fullfileRequest " + filename, e); throw new FileStreamingFailedException("fullFileRequest " + filename, e);
} }
return ResponseEntity.ok().build(); return ResponseEntity.ok().build();
} }
private static ResponseEntity<InputStream> extractRange(final HttpServletResponse response, final long length, private static ResponseEntity<InputStream> handleSingleRangeRequest(
final List<ByteRange> ranges, final String range) { final InputStream inputStream, final Range range, final String filename, final HttpServletResponse response,
final FileStreamingProgressListener progressListener) {
if (ranges.isEmpty()) { response.setHeader(CONTENT_RANGE, range.contentRange());
for (final String part : range.substring(6).split(",")) { response.setContentLengthLong(range.getPartLen());
long start = sublong(part, 0, part.indexOf('-'));
long end = sublong(part, part.indexOf('-') + 1, part.length());
if (start == -1) {
start = length - end;
end = length - 1;
} else if (end == -1 || end > length - 1) {
end = length - 1;
}
// Check if Range is syntactically valid. If not, then return
// 416.
if (start > end) {
response.setHeader(HttpHeaders.CONTENT_RANGE, "bytes */" + length);
return new ResponseEntity<>(HttpStatus.REQUESTED_RANGE_NOT_SATISFIABLE);
}
// Add range.
ranges.add(new ByteRange(start, end, length));
}
}
return null;
}
private static long sublong(final String value, final int beginIndex, final int endIndex) {
final String substring = value.substring(beginIndex, endIndex);
return substring.isEmpty() ? -1 : Long.parseLong(substring);
}
private static void checkForShortcut(final HttpServletRequest request, final String etag, final long lastModified,
final ByteRange full, final List<ByteRange> ranges) {
final String ifRange = request.getHeader(HttpHeaders.IF_RANGE);
if (ifRange != null && !ifRange.equals(etag)) {
try {
final long ifRangeTime = request.getDateHeader(HttpHeaders.IF_RANGE);
if (ifRangeTime != -1 && ifRangeTime + 1000 < lastModified) {
ranges.add(full);
}
} catch (final IllegalArgumentException ignored) {
log.info("Invalid if-range header field", ignored);
ranges.add(full);
}
}
}
private static ResponseEntity<InputStream> handleMultipartRangeRequest(
final InputStream inputStream, final String filename, final HttpServletResponse response,
final FileStreamingProgressListener progressListener, final List<ByteRange> ranges) {
ranges.sort((r1, r2) -> Long.compare(r1.getStart(), r2.getStart()));
response.setContentType("multipart/byteranges; boundary=" + ByteRange.MULTIPART_BOUNDARY);
response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
try { try {
final ServletOutputStream to = response.getOutputStream(); copyStreams(inputStream, range.getStart(), range.getPartLen(), filename, response.getOutputStream(), progressListener);
long streamPos = 0;
for (final ByteRange range : ranges) {
if (streamPos > range.getStart()) {
throw new FileStreamingFailedException("Ranges are overlapping or not in order");
}
// Add multipart boundary and header fields for every range.
to.println();
to.println("--" + ByteRange.MULTIPART_BOUNDARY);
to.println(HttpHeaders.CONTENT_RANGE + ": bytes " + range.getStart() + "-" + range.getEnd() + "/" + range.getTotal());
// Copy single part range of multipart range.
copyStreams(inputStream, to, progressListener, range.getStart() - streamPos, range.getLength(), filename);
streamPos = range.getStart() + range.getLength();
}
// End with final multipart boundary.
to.println();
to.print("--" + ByteRange.MULTIPART_BOUNDARY + "--");
} catch (final IOException e) {
throw new FileStreamingFailedException("multipartRangeRequest " + filename, e);
}
return ResponseEntity.status(HttpStatus.PARTIAL_CONTENT).build();
}
private static ResponseEntity<InputStream> handleStandardRangeRequest(
final InputStream inputStream, final String filename, final HttpServletResponse response,
final FileStreamingProgressListener progressListener, final ByteRange range) {
response.setHeader(HttpHeaders.CONTENT_RANGE, "bytes " + range.getStart() + "-" + range.getEnd() + "/" + range.getTotal());
response.setContentLengthLong(range.getLength());
response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
try {
final ServletOutputStream to = response.getOutputStream();
copyStreams(inputStream, to, progressListener, range.getStart(), range.getLength(), filename);
} catch (final IOException e) { } catch (final IOException e) {
log.error("standardRangeRequest of file ({}) failed!", filename, e); log.error("standardRangeRequest of file ({}) failed!", filename, e);
throw new FileStreamingFailedException(filename); throw new FileStreamingFailedException(filename);
@@ -267,11 +192,65 @@ public final class FileStreamingUtil {
return ResponseEntity.status(HttpStatus.PARTIAL_CONTENT).build(); return ResponseEntity.status(HttpStatus.PARTIAL_CONTENT).build();
} }
private static long copyStreams( private static final String CRLF = "\r\n";
final InputStream from, final OutputStream to, private static final String DASH_DASH = "--";
final FileStreamingProgressListener progressListener, private static final String BOUNDARY = "THIS_STRING_SEPARATES_MULTIPART"; // boundary := 0*69<bchars> bcharsnospace
final long start, final long length, private static final String CONTENT_TYPE_MULTIPART_BYTE_RANGES_AND_BOUNDARY = "multipart/byteranges; boundary=" + BOUNDARY;
final String filename) throws IOException { private static final String DASH_BOUNDARY = DASH_DASH + BOUNDARY; // dash-boundary := "--" boundary
private static final String DELIMITER = CRLF + DASH_BOUNDARY; // delimiter := CRLF dash-boundary
private static final String CLOSE_DELIMITER = DELIMITER + DASH_DASH; // close-delimiter := delimiter "--"
// follows the RFC-2046 -> https://datatracker.ietf.org/doc/html/rfc2046#section-5.1
private static ResponseEntity<InputStream> handleMultipartRangeRequest(
final InputStream inputStream, final List<Range> ranges, final String filename, final HttpServletResponse response,
final FileStreamingProgressListener progressListener) {
// add headers
response.setContentType(CONTENT_TYPE_MULTIPART_BYTE_RANGES_AND_BOUNDARY);
response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
// write multipart-body as defined in RFC-2046 (we use transport-padding is empty as per RFC-2046, don't send the optional preamble and epilogue):
// multipart-body :=
// dash-boundary CRLF body-part // first body-part (range, headers + content)
// *encapsulation // next ranges
// close-delimiter
// encapsulation := delimiter CRLF body-part
// body-part := MIME-part-headers [CRLF *OCTET]
try {
// println of ServletOutputStream appends CRLF, which is required for separating multipart boundaries and header fields.
final ServletOutputStream to = response.getOutputStream();
long streamPos = 0;
for (int i = 0; i < ranges.size(); i++) { // dash-boundary CRLF body-part or encapsulation (delimiter CRLF body-part)
final Range range = ranges.get(i);
// write body-part prefix - first body-part is prefixed with dash-boundary, the following ones with delimiter
to.println(i == 0 ? DASH_BOUNDARY : DELIMITER);
// write body-part
// * write MIME-part-headers
// If the selected representation would have had a Content-Type header field in a 200 (OK) response,
// the server SHOULD generate that same Content-Type field in the header area of each body part.
to.print(CONTENT_TYPE);
to.print(": ");
to.println(APPLICATION_OCTET_STREAM_VALUE);
to.print(CONTENT_RANGE);
to.print(": ");
to.println(range.contentRange());
// * write [CRLF *OCTET]
to.println();
copyStreams(inputStream, range.getStart() - streamPos, range.getPartLen(), filename, to, progressListener);
// update stream position
streamPos = range.getStart() + range.getPartLen();
}
// write close-delimiter
to.print(CLOSE_DELIMITER);
} catch (final IOException e) {
throw new FileStreamingFailedException("multipartRangeRequest " + filename, e);
}
return ResponseEntity.status(HttpStatus.PARTIAL_CONTENT).build();
}
private static void copyStreams(
final InputStream from, final long start, final long length, final String filename,
final OutputStream to, final FileStreamingProgressListener progressListener) throws IOException {
final long startMillis = System.currentTimeMillis(); final long startMillis = System.currentTimeMillis();
log.trace("Start of copy-streams of file {} from {} to {}", filename, start, length); log.trace("Start of copy-streams of file {} from {} to {}", filename, start, length);
@@ -318,15 +297,45 @@ public final class FileStreamingUtil {
} }
final long totalTime = System.currentTimeMillis() - startMillis; final long totalTime = System.currentTimeMillis() - startMillis;
if (total < length) { if (total < length) {
throw new FileStreamingFailedException( throw new FileStreamingFailedException(
filename + ": " + (length - total) + " bytes could not be written to client, total time on write: !" + totalTime + " ms"); filename + ": " + (length - total) + " bytes could not be written to client, total time on write: !" + totalTime + " ms");
} }
log.trace("Finished copy-stream of file {} with length {} in {} ms", filename, length, totalTime); log.trace("Finished copy-stream of file {} with length {} in {} ms", filename, length, totalTime);
}
return total; // tspecials := "(" / ")" / "<" / ">" / "@" / "," / ";" / ":" / "\" / <"> / "/" / "[" / "]" / "?" / "="
private static final String TSPECIALS = "()<>@,;:\\\"/[]?=";
// Encodes filename for Content-Disposition header according to RFC-2183 (https://datatracker.ietf.org/doc/html/rfc2183)
private static String encodeFilename(String filename) {
if (filename.length() > 78) {
filename = filename.substring(0, 78); // RFC-2183: parameter values longer than 78 characters should be truncated to 78 characters
}
// Check if filename contains any tspecials, and only if so, quotes
for (int i = 0; i < filename.length(); i++) {
if (TSPECIALS.indexOf(filename.charAt(i)) != -1) {
return quotedString(filename);
}
}
return filename;
}
private static @NonNull String quotedString(final String filename) {
// RFC-2183: A short parameter value containing only ASCII characters, but including `tspecials' characters, SHOULD be represented as `quoted-string'
// RFC-822: quoted-string = <"> *(qtext/quoted-pair) <">, quoted-pair = "\" CHAR - we need to escape " and \ inside the quoted string
final StringBuilder quoted = new StringBuilder("\"");
for (int i = 0; i < filename.length(); i++) {
final char c = filename.charAt(i);
// Escape backslash and quote characters
if (c == '"' || c == '\\') {
quoted.append('\\');
}
quoted.append(c);
}
quoted.append('"');
return quoted.toString();
} }
/** /**
@@ -345,78 +354,73 @@ public final class FileStreamingUtil {
void progress(long requestedBytes, long shippedBytesSinceLast, long shippedBytesOverall); void progress(long requestedBytes, long shippedBytesSinceLast, long shippedBytesOverall);
} }
private static final class ByteRange { @Value
private static class Range {
private static final String MULTIPART_BOUNDARY = "THIS_STRING_SEPARATES_MULTIPART"; // byte-content-range = bytes-unit SP ( byte-range-resp / unsatisfied-range )
// byte-range-resp = byte-range "/" ( complete-length / "*" )
// byte-range = first-byte-pos "-" last-byte-pos
private static final String CONTENT_RANGE_FORMAT = "bytes %d-%d/%d";
private final long start; long start; // first-byte-pos
private final long end; long end; // last-byte-pos
private final long length; long completeLen; // complete-length
private final long total; long partLen; // partial, range length (end - start + 1)
private ByteRange(final long start, final long end, final long total) { private Range(final long start, final long end, final long completeLen) {
this.start = start; this.start = start;
this.end = end; this.end = end;
length = end - start + 1; this.completeLen = completeLen;
this.total = total; partLen = end - start + 1;
} }
@Override // throws IllegalArgumentException if the header doesn't conform the expected format and constraints (like non-overlapping)
// Generated code // return validated, ordered and non-overlapping ranges
@SuppressWarnings("squid:S864") private static List<Range> of(final String rangeHeader, final long length) {
public int hashCode() { // Range header matches"bytes=n-n,n-n,n-n..."
final int prime = 31; if (!rangeHeader.matches("^bytes=\\d*-\\d*(,\\d*-\\d*)*+$")) {
int result = 1; throw new IllegalArgumentException("Doesn't match pattern");
result = prime * result + (int) (end ^ (end >>> 32));
result = prime * result + (int) (length ^ (length >>> 32));
result = prime * result + (int) (start ^ (start >>> 32));
result = prime * result + (int) (total ^ (total >>> 32));
return result;
} }
@Override final List<Range> ranges = new ArrayList<>();
// Generated code // parse range header
@SuppressWarnings("squid:S1126") for (final String part : rangeHeader.substring(6).split(",")) {
public boolean equals(final Object obj) { final int index = part.indexOf('-');
if (this == obj) { final long start;
return true; final long end;
if (index == 0) { // -n, means last n bytes
start = length - Long.parseLong(part.substring(index + 1));
end = length - 1;
} else {
start = Long.parseLong(part.substring(0, index));
end = index == part.length() - 1 ? length - 1 : Math.min(Long.parseLong(part.substring(index + 1)), length - 1);
} }
if (obj == null) { // Check if Range is syntactically valid. If not, then return 416.
return false; if (start > end) {
throw new IllegalArgumentException("Start bigger then end");
} }
if (getClass() != obj.getClass()) { // Add range.
return false; ranges.add(new Range(start, end, length));
} }
final ByteRange other = (ByteRange) obj; // ranges must not be empty
if (end != other.end) { if (ranges.isEmpty()) {
return false; throw new IllegalArgumentException("Empty range list");
} }
if (length != other.length) { // order ranges by start position
return false; ranges.sort(Comparator.comparingLong(Range::getStart));
// validate ranges, we don't allow overlapping, as this would make the streaming logic more complex and computational expensive.
long streamPos = 0;
for (final Range range : ranges) {
if (streamPos > range.getStart()) {
throw new IllegalArgumentException("Ranges are overlapping or not in order");
} }
if (start != other.start) { streamPos = range.getStart() + range.getPartLen();
return false;
} }
if (total != other.total) { return ranges;
return false;
}
return true;
} }
private long getStart() { private String contentRange() {
return start; return String.format(CONTENT_RANGE_FORMAT, start, end, completeLen);
}
private long getEnd() {
return end;
}
private long getLength() {
return length;
}
private long getTotal() {
return total;
} }
} }
} }

11
hawkbit-rest/hawkbit-rest-core/src/test/java/org/eclipse/hawkbit/rest/util/FileStreamingUtilTest.java
View File

@@ -53,19 +53,21 @@ class FileStreamingUtilTest {
Mockito.when(servletResponse.getOutputStream()).thenReturn(outputStream); Mockito.when(servletResponse.getOutputStream()).thenReturn(outputStream);
final HttpServletRequest servletRequest = Mockito.mock(HttpServletRequest.class); final HttpServletRequest servletRequest = Mockito.mock(HttpServletRequest.class);
Mockito.when(servletRequest.getMethod()).thenReturn("GET");
Mockito.when(servletRequest.getHeader("Range")).thenReturn("bytes=0-10,11-15,16-"); Mockito.when(servletRequest.getHeader("Range")).thenReturn("bytes=0-10,11-15,16-");
long lastModified = System.currentTimeMillis(); long lastModified = System.currentTimeMillis();
final ResponseEntity<InputStream> responseEntity = FileStreamingUtil.writeFileResponse( final ResponseEntity<InputStream> responseEntity = FileStreamingUtil.writeFileResponse(
TEST_ARTIFACT.get(), "test.file", lastModified, servletResponse, servletRequest, null); TEST_ARTIFACT.get(), "test.file", lastModified, servletRequest, servletResponse, null);
assertThat(responseEntity).isNotNull(); assertThat(responseEntity).isNotNull();
verify(servletResponse).setDateHeader(HttpHeaders.LAST_MODIFIED, lastModified); verify(servletResponse).setDateHeader(HttpHeaders.LAST_MODIFIED, lastModified);
final ArgumentCaptor<String> stringCaptor = ArgumentCaptor.forClass(String.class); final ArgumentCaptor<String> stringCaptor = ArgumentCaptor.forClass(String.class);
final ArgumentCaptor<Integer> lenCaptor = ArgumentCaptor.forClass(Integer.class); final ArgumentCaptor<Integer> lenCaptor = ArgumentCaptor.forClass(Integer.class);
verify(outputStream).print(stringCaptor.capture()); verify(outputStream, times(3 * 4 + 1)).print(stringCaptor.capture());
assertThat(stringCaptor.getValue()).contains("--THIS_STRING_SEPARATES_MULTIPART--"); verify(outputStream, times(3 * 3)).println(stringCaptor.capture());
verify(outputStream, times(3)).println();
verify(outputStream, times(3)).write(any(), anyInt(), lenCaptor.capture()); verify(outputStream, times(3)).write(any(), anyInt(), lenCaptor.capture());
assertThat(lenCaptor.getAllValues()).containsExactly(11, 5, 39); // Range lengths assertThat(lenCaptor.getAllValues()).containsExactly(11, 5, 39); // Range lengths
} }
@@ -78,10 +80,11 @@ class FileStreamingUtilTest {
Mockito.when(servletResponse.getOutputStream()).thenReturn(outputStream); Mockito.when(servletResponse.getOutputStream()).thenReturn(outputStream);
final HttpServletRequest servletRequest = Mockito.mock(HttpServletRequest.class); final HttpServletRequest servletRequest = Mockito.mock(HttpServletRequest.class);
Mockito.when(servletRequest.getMethod()).thenReturn("GET");
Mockito.when(servletRequest.getHeader("Range")).thenReturn("bytes=0-10***,9-15,16-"); Mockito.when(servletRequest.getHeader("Range")).thenReturn("bytes=0-10***,9-15,16-");
final ResponseEntity<InputStream> responseEntity = FileStreamingUtil.writeFileResponse( final ResponseEntity<InputStream> responseEntity = FileStreamingUtil.writeFileResponse(
TEST_ARTIFACT.get(), "test.file", lastModified, servletResponse, servletRequest, null); TEST_ARTIFACT.get(), "test.file", lastModified, servletRequest, servletResponse, null);
assertThat(responseEntity.getStatusCode()).isEqualTo(HttpStatus.REQUESTED_RANGE_NOT_SATISFIABLE); assertThat(responseEntity.getStatusCode()).isEqualTo(HttpStatus.REQUESTED_RANGE_NOT_SATISFIABLE);
verify(outputStream, times(0)).print(anyString()); verify(outputStream, times(0)).print(anyString());