Simple UI: Streaming upload (#2254)

thus not loading whole artifact into memory

Signed-off-by: Avgustin Marinov <Avgustin.Marinov@bosch.com>

hawkbit-rest-core/src/main/java/org/eclipse/hawkbit/rest/RestConfiguration.java

@@ -159,6 +159,7 @@ public class RestConfiguration {
         @ExceptionHandler(AbstractServerRtException.class)
         public ResponseEntity<ExceptionInfo> handleSpServerRtExceptions(final HttpServletRequest request, final Exception ex) {
             logRequest(request, ex);
+
             final ExceptionInfo response = createExceptionInfo(ex);
             final HttpStatus responseStatus;
             if (ex instanceof AbstractServerRtException abstractServerRtException) {
@@ -172,8 +173,7 @@ public class RestConfiguration {
         /**
          * Method for handling exception of type {@link FileStreamingFailedException} which is thrown in case the streaming of a file failed
          * due to an internal server error. As the streaming of the file has already begun, no JSON response but only the ResponseStatus 500
-         * is returned.
+         * is returned. Called by the Spring-Framework for exception handling.
-         * Called by the Spring-Framework for exception handling.
          *
          * @param request the Http request
          * @param ex the exception which occurred
@@ -182,6 +182,7 @@ public class RestConfiguration {
         @ExceptionHandler(FileStreamingFailedException.class)
         public ResponseEntity<Object> handleFileStreamingFailedException(final HttpServletRequest request, final Exception ex) {
             logRequest(request, ex);
+
             log.warn("File streaming failed: {}", ex.getMessage());
             return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR);
         }
@@ -201,22 +202,22 @@ public class RestConfiguration {
                 IllegalArgumentException.class })
         public ResponseEntity<ExceptionInfo> handleExceptionCausedByIncorrectRequestBody(final HttpServletRequest request, final Exception ex) {
             logRequest(request, ex);
+
             final ExceptionInfo response = createExceptionInfo(new MessageNotReadableException());
             return new ResponseEntity<>(response, HttpStatus.BAD_REQUEST);
         }
 
         /**
          * Method for handling exception of type ConstraintViolationException which is thrown in case the request is rejected due to a
-         * constraint violation.
+         * constraint violation. Called by the Spring-Framework for exception handling.
-         * Called by the Spring-Framework for exception handling.
          *
          * @param request the Http request
          * @param ex the exception which occurred
          * @return the entity to be responded containing the exception information as entity.
          */
         @ExceptionHandler(ConstraintViolationException.class)
-        public ResponseEntity<ExceptionInfo> handleConstraintViolationException(final HttpServletRequest request,
+        public ResponseEntity<ExceptionInfo> handleConstraintViolationException(
-                final ConstraintViolationException ex) {
+                final HttpServletRequest request, final ConstraintViolationException ex) {
             logRequest(request, ex);
 
             final ExceptionInfo response = new ExceptionInfo();
@@ -251,8 +252,7 @@ public class RestConfiguration {
 
         /**
          * Method for handling exception of type {@link MultipartException} which is thrown in case the request body is not well-formed and
-         * cannot be deserialized.
+         * cannot be deserialized. Called by the Spring-Framework for exception handling.
-         * Called by the Spring-Framework for exception handling.
          *
          * @param request the Http request
          * @param ex the exception which occurred
@@ -266,8 +266,7 @@ public class RestConfiguration {
             final Throwable responseCause = throwables.get(throwables.size() - 1);
 
             if (ObjectUtils.isEmpty(responseCause.getMessage())) {
-                log.warn("Request {} lead to MultipartException without root cause message:\n{}", request.getRequestURL(),
+                log.warn("Request {} lead to MultipartException without root cause message:\n{}", request.getRequestURL(), ex.getStackTrace());
-                        ex.getStackTrace());
             }
 
             return new ResponseEntity<>(createExceptionInfo(new MultiPartFileUploadException(responseCause)), HttpStatus.BAD_REQUEST);
@@ -277,6 +276,10 @@ public class RestConfiguration {
             return ERROR_TO_HTTP_STATUS.getOrDefault(error, DEFAULT_RESPONSE_STATUS);
         }
 
+        // enable certain level of debug with
+        //  -> logging.level.org.eclipse.hawkbit.rest.RestConfiguration=DEBUG
+        // or for more detailed log
+        //  -> logging.level.org.eclipse.hawkbit.rest.RestConfiguration=TRACE
         private void logRequest(final HttpServletRequest request, final Exception ex) {
             if (log.isTraceEnabled()) {
                 log.trace("Handling exception {} of request {}", ex.getClass().getName(), request.getRequestURL(), ex);
@@ -295,7 +298,6 @@ public class RestConfiguration {
             }
             return response;
         }
-
     }
 
     /**
@@ -309,7 +311,7 @@ public class RestConfiguration {
         private final String[] excludeAntPaths;
         private final AntPathMatcher antMatcher = new AntPathMatcher();
 
-         public ExcludePathAwareShallowETagFilter(final String... excludeAntPaths) {
+        public ExcludePathAwareShallowETagFilter(final String... excludeAntPaths) {
             this.excludeAntPaths = excludeAntPaths;
         }
 

hawkbit-sdk/hawkbit-sdk-commons/src/main/java/org/eclipse/hawkbit/sdk/HawkbitClient.java

@@ -9,21 +9,44 @@
  */
 package org.eclipse.hawkbit.sdk;
 
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Proxy;
+import java.net.HttpURLConnection;
+import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.util.Base64;
+import java.util.List;
 import java.util.Objects;
+import java.util.UUID;
 import java.util.function.BiFunction;
+import java.util.stream.Stream;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import feign.Client;
 import feign.Contract;
 import feign.Feign;
 import feign.RequestInterceptor;
+import feign.RequestTemplate;
 import feign.codec.Decoder;
 import feign.codec.Encoder;
 import feign.codec.ErrorDecoder;
 import lombok.Builder;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.http.HttpStatusCode;
+import org.springframework.http.ResponseEntity;
 import org.springframework.util.ObjectUtils;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RequestPart;
+import org.springframework.web.multipart.MultipartFile;
 
 @Slf4j
 @Builder
@@ -104,15 +127,145 @@ public class HawkbitClient {
     }
 
     private <T> T service(final Class<T> serviceType, final Tenant tenant, final Controller controller) {
+        final T service = service0(serviceType, tenant, controller);
+        if (serviceType.isInterface() // proxy only interfaces
+                && Stream.of(serviceType.getDeclaredMethods()) // and has MultipartFile argument
+                .anyMatch(method -> method.getAnnotation(PostMapping.class) != null
+                        && List.of(method.getParameterTypes()).contains(MultipartFile.class))) {
+            // doesn't use feign client since it doesn't (?) support streaming - loading all in memory which could lead to OOM
+            // https://github.com/OpenFeign/feign-form/issues/121 (?)
+            return proxy(serviceType, service, tenant, controller);
+        } else { // default
+            return service;
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private <T> T proxy(final Class<T> serviceType, final T service, final Tenant tenant, final Controller controller) {
+        final ObjectMapper objectMapper = new ObjectMapper();
+        return (T) Proxy.newProxyInstance(service.getClass().getClassLoader(), new Class<?>[] { serviceType }, (proxy, method, args) -> {
+            try {
+                final Class<?>[] parameterTypes = method.getParameterTypes();
+                if (method.getDeclaringClass() == serviceType && List.of(parameterTypes).contains(MultipartFile.class)) {
+                    return processMultipartFormData(method, args, tenant, controller, parameterTypes, objectMapper);
+                } else {
+                    return method.invoke(service, args);
+                }
+            } catch (final InvocationTargetException e) {
+                throw e.getTargetException() == null ? e : e.getTargetException();
+            }
+        });
+    }
+
+    private static final String CRLF = "\r\n";
+
+    private Object processMultipartFormData(
+            final Method method, final Object[] args,
+            final Tenant tenant, final Controller controller,
+            final Class<?>[] parameterTypes, final ObjectMapper objectMapper) throws IOException {
+        final PostMapping postMapping = method.getAnnotation(PostMapping.class);
+        final Annotation[][] parametersAnnotations = method.getParameterAnnotations();
+        // build path - replace @PathVariables
+        String path = postMapping.value()[0];
+        for (int i = 0; i < args.length; i++) {
+            final PathVariable pathVariable = getAnnotation(PathVariable.class, parametersAnnotations[i]);
+            if (pathVariable != null) {
+                path = path.replace("{" + pathVariable.value() + "}", args[i].toString());
+            }
+        }
+
+        final HttpURLConnection conn = (HttpURLConnection) new URL(
+                (controller == null ? hawkBitServer.getMgmtUrl() : hawkBitServer.getDdiUrl()) + path).openConnection();
+        conn.setRequestMethod("POST");
+
+        // deal with authentication - only from headers1
+        final RequestTemplate requestTemplate = new RequestTemplate();
+        requestInterceptorFn.apply(tenant, controller).apply(requestTemplate);
+        requestTemplate.headers().forEach((k, v) -> v.forEach(e -> conn.setRequestProperty(k, e)));
+
+        final String boundary = UUID.randomUUID().toString().replace("-", "");
+        conn.setRequestProperty("content-type", "multipart/form-data;boundary=" + boundary);
+        // consumes what the method produces
+        final String[] consumes = postMapping.produces();
+        if (!ObjectUtils.isEmpty(consumes)) {
+            conn.setRequestProperty("accept", String.join(",", consumes));
+        }
+
+        conn.setDoOutput(true);
+        conn.setDoInput(true);
+
+        try (final OutputStream out = new BufferedOutputStream(conn.getOutputStream())) {
+            for (int i = 0; i < args.length; i++) {
+                final Class<?> type = parameterTypes[i];
+                if (MultipartFile.class.isAssignableFrom(type)) {
+                    final MultipartFile multipartFile = (MultipartFile) args[i];
+                    if (multipartFile != null) {
+                        writeMultipartFile(multipartFile, out, boundary, parametersAnnotations[i]);
+                    }
+                } else {
+                    writeSimpleFormData(args[i], out, boundary, parametersAnnotations[i]);
+                }
+            }
+            out.write(("--" + boundary + "--\r\n").getBytes(StandardCharsets.UTF_8));
+        }
+        return method.getReturnType() == ResponseEntity.class
+                ? new ResponseEntity<Object>(
+                    objectMapper.readValue(
+                            conn.getInputStream(),
+                            (Class<?>) ((ParameterizedType) method.getGenericReturnType()).getActualTypeArguments()[0]),
+                    HttpStatusCode.valueOf(conn.getResponseCode()))
+                : objectMapper.readValue(conn.getInputStream(), method.getReturnType());
+    }
+
+    private void writeMultipartFile(
+            final MultipartFile multipartFile, final OutputStream out, final String boundary, final Annotation[] parametersAnnotations)
+            throws IOException {
+        final String name = Objects.requireNonNull(
+                        getAnnotation(RequestPart.class, parametersAnnotations), "MultipartFile shall have RequestPart annotation")
+                .value();
+        try (final InputStream in = multipartFile.getInputStream()) {
+            out.write(("--" + boundary + CRLF +
+                    "Content-Disposition: form-data; name=\"" + name + "\"; filename=\"" + multipartFile.getName() + "\"\r\n" +
+                    "Content-Type: " + multipartFile.getContentType() + "\r\n\r\n"
+            ).getBytes(StandardCharsets.UTF_8));
+            final byte[] buff = new byte[8096];
+            for (int read; (read = in.read(buff)) != -1; ) {
+                out.write(buff, 0, read);
+            }
+            out.write(CRLF.getBytes(StandardCharsets.UTF_8));
+        }
+    }
+
+    private void writeSimpleFormData(
+            final Object arg, final OutputStream out, final String boundary, final Annotation[] parameterAnnotations) throws IOException {
+        if (arg != null) {
+            final RequestParam requestParam = getAnnotation(RequestParam.class, parameterAnnotations);
+            if (requestParam != null) {
+                out.write(("--" + boundary + CRLF +
+                        "Content-Disposition: form-data; name=\"" + requestParam.value() + "\"\r\n\r\n" +
+                        arg + CRLF
+                ).getBytes(StandardCharsets.UTF_8));
+            }
+        } // otherwise default
+    }
+
+    @SuppressWarnings("unchecked")
+    private <T extends Annotation> T getAnnotation(final Class<T> annotationClass, final Annotation[] annotations) {
+        for (final Annotation annotation : annotations) {
+            if (annotation.annotationType().equals(annotationClass)) {
+                return (T) annotation;
+            }
+        }
+        return null;
+    }
+
+    private <T> T service0(final Class<T> serviceType, final Tenant tenant, final Controller controller) {
         return Feign.builder().client(client)
                 .encoder(encoder)
                 .decoder(decoder)
                 .errorDecoder(errorDecoder)
                 .contract(contract)
                 .requestInterceptor(requestInterceptorFn.apply(tenant, controller))
-                .target(serviceType,
+                .target(serviceType, controller == null ? hawkBitServer.getMgmtUrl() : hawkBitServer.getDdiUrl());
-                        controller == null ?
-                                hawkBitServer.getMgmtUrl() :
-                                hawkBitServer.getDdiUrl());
     }
 }

hawkbit-simple-ui/.gitignore

@@ -1 +1,2 @@
-frontend
+frontend
+bundles

hawkbit-simple-ui/pom.xml

@@ -103,6 +103,10 @@
                         <phase>compile</phase>
                     </execution>
                 </executions>
+                <configuration>
+                    <forceProductionBuild>true</forceProductionBuild>
+                    <frontendHotdeploy>false</frontendHotdeploy>
+                </configuration>
             </plugin>
         </plugins>
     </build>

hawkbit-simple-ui/src/main/java/org/eclipse/hawkbit/ui/simple/view/DistributionSetView.java

@@ -142,8 +142,7 @@ public class DistributionSetView extends TableView<MgmtDistributionSet, Long> {
             return Filter.filter(
                     Map.of(
                             "name", name.getOptionalValue(),
-                            "type", type.getSelectedItems().stream().map(MgmtDistributionSetType::getKey)
+                            "type", type.getSelectedItems().stream().map(MgmtDistributionSetType::getKey).toList(),
-                                    .toList(),
                             "tag", tag.getSelectedItems()));
         }
     }

hawkbit-simple-ui/src/main/java/org/eclipse/hawkbit/ui/simple/view/SoftwareModuleView.java

@@ -45,6 +45,7 @@ import com.vaadin.flow.component.upload.receivers.FileBuffer;
 import com.vaadin.flow.data.renderer.ComponentRenderer;
 import com.vaadin.flow.router.PageTitle;
 import com.vaadin.flow.router.Route;
+import lombok.extern.slf4j.Slf4j;
 import org.eclipse.hawkbit.mgmt.json.model.PagedList;
 import org.eclipse.hawkbit.mgmt.json.model.artifact.MgmtArtifact;
 import org.eclipse.hawkbit.mgmt.json.model.softwaremodule.MgmtSoftwareModule;
@@ -63,6 +64,7 @@ import org.springframework.web.multipart.MultipartFile;
 @Route(value = "software_modules", layout = MainLayout.class)
 @RolesAllowed({ "SOFTWARE_MODULE_READ" })
 @Uses(Icon.class)
+@Slf4j
 public class SoftwareModuleView extends TableView<MgmtSoftwareModule, Long> {
 
     @Autowired
@@ -307,9 +309,11 @@ public class SoftwareModuleView extends TableView<MgmtSoftwareModule, Long> {
             uploadBtn.setDropAllowed(true);
             uploadBtn.addSucceededListener(e -> {
                 final MgmtArtifact artifact = hawkbitClient.getSoftwareModuleRestApi()
-                        .uploadArtifact(softwareModuleId,
+                        .uploadArtifact(
-                                new MultipartFileImpl(fileBuffer, e.getContentLength(), e.getMIMEType()), fileBuffer.getFileName(), null, null,
+                                softwareModuleId,
-                                null).getBody();
+                                new MultipartFileImpl(fileBuffer, e.getContentLength(), e.getMIMEType()),
+                                fileBuffer.getFileName(), null, null, null)
+                        .getBody();
                 artifacts.add(artifact);
                 artifactGrid.refreshGrid(false);
             });
@@ -367,6 +371,7 @@ public class SoftwareModuleView extends TableView<MgmtSoftwareModule, Long> {
 
             @Override
             public byte[] getBytes() throws IOException {
+                log.warn("Multipart file getBytes() is called. Whole input stream is loaded into the memory!");
                 try (final InputStream is = getInputStream()) {
                     return is.readAllBytes();
                 }

hawkbit-simple-ui/src/main/resources/application.properties

@@ -16,8 +16,9 @@ logging.level.org.springframework.boot.actuate.audit.listener.AuditListener=WARN
 # logging pattern
 logging.pattern.console=%clr(%d{${logging.pattern.dateformat:yyyy-MM-dd'T'HH:mm:ss.SSSXXX}}){faint} %clr(${logging.pattern.level:%5p}) %clr(${PID:}){magenta} %clr(---){faint} %clr([${spring.application.name}] [%X{tenant}:%X{user}] [%15.15t]){faint} %clr(${logging.pattern.correlation:}){faint}%clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${logging.exception-conversion-word:%wEx}
 
-### Vaadin start ###`
+### Vaadin start ###
 # build with mvn vaadin:build-frontend to enable / disable
+vaadin.productionMode=true
 vaadin.frontend.hotdeploy=false
 logging.level.org.atmosphere=warn
 spring.mustache.check-template-location=false