Artifact modules moved in new hawkbit-artifact parent (#2012)

Signed-off-by: Avgustin Marinov <Avgustin.Marinov@bosch.com>
2024-11-12 09:40:44 +02:00
parent aa4c753ffe
commit 32acb44e31
36 changed files with 100 additions and 10 deletions
--- a/hawkbit-artifact/hawkbit-artifact-repository-filesystem/README.md
+++ b/hawkbit-artifact/hawkbit-artifact-repository-filesystem/README.md
@@ -0,0 +1,18 @@
+# Eclipse.IoT hawkBit - Artifact Repository File System
+
+This module contains the implementation of the `ArtifactRepository` based on the file-system.
+It's a very convenient and easy implementation of storing the artifact binaries into the file-system based on the SHA-1
+hash naming.
+
+Due the limit of many file-systems of files within one directory, the files
+are stored in different sub-directories based on the last four digits of the
+SHA1-hash `/basepath/[two digit sha1]/[two digit sha1/sha1-hash-filename]`.
+
+# Compile
+
+#### Build hawkbit-artifact-repository-filesystem
+
+```
+$ cd hawkbit/hawkbit-artifact-repository-filesystem
+$ mvn clean install
+```
--- a/hawkbit-artifact/hawkbit-artifact-repository-filesystem/pom.xml
+++ b/hawkbit-artifact/hawkbit-artifact-repository-filesystem/pom.xml
@@ -0,0 +1,44 @@
+<!--
+
+    Copyright (c) 2015 Bosch Software Innovations GmbH and others
+
+    This program and the accompanying materials are made
+    available under the terms of the Eclipse Public License 2.0
+    which is available at https://www.eclipse.org/legal/epl-2.0/
+
+    SPDX-License-Identifier: EPL-2.0
+
+-->
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.eclipse.hawkbit</groupId>
+        <artifactId>hawkbit-artifact-parent</artifactId>
+        <version>${revision}</version>
+    </parent>
+
+    <artifactId>hawkbit-artifact-repository-filesystem</artifactId>
+    <name>hawkBit :: Artifact Repository :: Filesystem</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.eclipse.hawkbit</groupId>
+            <artifactId>hawkbit-artifact-api</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-autoconfigure</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+        </dependency>
+    </dependencies>
+</project>
--- a/hawkbit-artifact/hawkbit-artifact-repository-filesystem/src/main/java/org/eclipse/hawkbit/artifact/repository/ArtifactFileNotFoundException.java
+++ b/hawkbit-artifact/hawkbit-artifact-repository-filesystem/src/main/java/org/eclipse/hawkbit/artifact/repository/ArtifactFileNotFoundException.java
@@ -0,0 +1,25 @@
+/**
+ * Copyright (c) 2022 Bosch.IO GmbH and others
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipse.hawkbit.artifact.repository;
+
+/**
+ * Exception thrown in case that the artifact could not be read.
+ */
+public class ArtifactFileNotFoundException extends RuntimeException {
+
+    /**
+     * Creates the Exception from it's cause
+     *
+     * @param cause the original exception
+     */
+    public ArtifactFileNotFoundException(final Exception cause) {
+        super(cause);
+    }
+}
--- a/hawkbit-artifact/hawkbit-artifact-repository-filesystem/src/main/java/org/eclipse/hawkbit/artifact/repository/ArtifactFilesystem.java
+++ b/hawkbit-artifact/hawkbit-artifact-repository-filesystem/src/main/java/org/eclipse/hawkbit/artifact/repository/ArtifactFilesystem.java
@@ -0,0 +1,50 @@
+/**
+ * Copyright (c) 2015 Bosch Software Innovations GmbH and others
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipse.hawkbit.artifact.repository;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.util.Objects;
+
+import jakarta.validation.constraints.NotNull;
+
+import org.eclipse.hawkbit.artifact.repository.model.AbstractDbArtifact;
+import org.eclipse.hawkbit.artifact.repository.model.DbArtifactHash;
+
+/**
+ * {@link AbstractDbArtifact} implementation which dynamically creates a
+ * {@link FileInputStream} on calling {@link #getFileInputStream()}.
+ */
+public class ArtifactFilesystem extends AbstractDbArtifact {
+
+    private final File file;
+
+    public ArtifactFilesystem(@NotNull final File file, @NotNull final String artifactId,
+            @NotNull final DbArtifactHash hashes, final Long size,
+            final String contentType) {
+        super(artifactId, hashes, size, contentType);
+        this.file = Objects.requireNonNull(file, "Artifact file may not be null");
+    }
+
+    @Override
+    // suppress warning, this InputStream needs to be closed by the caller, this
+    // cannot be closed in this method
+    @SuppressWarnings("squid:S2095")
+    public InputStream getFileInputStream() {
+        try {
+            return new BufferedInputStream(new FileInputStream(file));
+        } catch (final FileNotFoundException e) {
+            throw new ArtifactFileNotFoundException(e);
+        }
+    }
+}
--- a/hawkbit-artifact/hawkbit-artifact-repository-filesystem/src/main/java/org/eclipse/hawkbit/artifact/repository/ArtifactFilesystemConfiguration.java
+++ b/hawkbit-artifact/hawkbit-artifact-repository-filesystem/src/main/java/org/eclipse/hawkbit/artifact/repository/ArtifactFilesystemConfiguration.java
@@ -0,0 +1,33 @@
+/**
+ * Copyright (c) 2015 Bosch Software Innovations GmbH and others
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipse.hawkbit.artifact.repository;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * Configuration for the {@link ArtifactFilesystemRepository}.
+ */
+@Configuration
+@EnableConfigurationProperties(ArtifactFilesystemProperties.class)
+public class ArtifactFilesystemConfiguration {
+
+    /**
+     * @param artifactFilesystemProperties the artifact file system properties
+     * @return Default {@link ArtifactRepository} implementation.
+     */
+    @Bean
+    @ConditionalOnMissingBean
+    public ArtifactRepository artifactRepository(final ArtifactFilesystemProperties artifactFilesystemProperties) {
+        return new ArtifactFilesystemRepository(artifactFilesystemProperties);
+    }
+}
--- a/hawkbit-artifact/hawkbit-artifact-repository-filesystem/src/main/java/org/eclipse/hawkbit/artifact/repository/ArtifactFilesystemProperties.java
+++ b/hawkbit-artifact/hawkbit-artifact-repository-filesystem/src/main/java/org/eclipse/hawkbit/artifact/repository/ArtifactFilesystemProperties.java
@@ -0,0 +1,27 @@
+/**
+ * Copyright (c) 2015 Bosch Software Innovations GmbH and others
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipse.hawkbit.artifact.repository;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+/**
+ * Configuration properties for the file-system repository, e.g. the base-path
+ * to store the files.
+ */
+@Data
+@ConfigurationProperties("org.eclipse.hawkbit.repository.file")
+public class ArtifactFilesystemProperties {
+
+    /**
+     * The base-path of the directory to store the artifacts.
+     */
+    private String path = "./artifactrepo";
+}
--- a/hawkbit-artifact/hawkbit-artifact-repository-filesystem/src/main/java/org/eclipse/hawkbit/artifact/repository/ArtifactFilesystemRepository.java
+++ b/hawkbit-artifact/hawkbit-artifact-repository-filesystem/src/main/java/org/eclipse/hawkbit/artifact/repository/ArtifactFilesystemRepository.java
@@ -0,0 +1,114 @@
+/**
+ * Copyright (c) 2015 Bosch Software Innovations GmbH and others
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipse.hawkbit.artifact.repository;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.apache.commons.io.FileUtils;
+import org.eclipse.hawkbit.artifact.repository.model.AbstractDbArtifact;
+import org.eclipse.hawkbit.artifact.repository.model.DbArtifactHash;
+import org.springframework.validation.annotation.Validated;
+
+/**
+ * Implementation of the {@link ArtifactRepository} to store artifacts on the
+ * file-system. The files are stored by their SHA1 hash of the artifact binary.
+ * Duplicate files with the same SHA1 hash will only stored once.
+ *
+ * All files are stored flat in one base directory configured in the
+ * {@link ArtifactFilesystemProperties#getPath()}.
+ *
+ * Due the limit of many file-systems of files within one directory, the files
+ * are stored in different sub-directories based on the last four digits of the
+ * SHA1-hash {@code (/basepath/[two digit sha1]/[two digit sha1])}.
+ */
+@Validated
+public class ArtifactFilesystemRepository extends AbstractArtifactRepository {
+
+    private final ArtifactFilesystemProperties artifactResourceProperties;
+
+    /**
+     * Constructor.
+     *
+     * @param artifactResourceProperties the properties which holds the necessary configuration for the
+     *         file-system repository
+     */
+    public ArtifactFilesystemRepository(final ArtifactFilesystemProperties artifactResourceProperties) {
+        this.artifactResourceProperties = artifactResourceProperties;
+    }
+
+    @Override
+    public void deleteBySha1(final String tenant, final String sha1Hash) {
+        FileUtils.deleteQuietly(getFile(tenant, sha1Hash));
+    }
+
+    @Override
+    public ArtifactFilesystem getArtifactBySha1(final String tenant, final String sha1) {
+        final File file = getFile(tenant, sha1);
+        if (!file.exists()) {
+            return null;
+        }
+
+        return new ArtifactFilesystem(file, sha1, new DbArtifactHash(sha1, null, null), file.length(), null);
+    }
+
+    @Override
+    public void deleteByTenant(final String tenant) {
+        FileUtils.deleteQuietly(Paths.get(artifactResourceProperties.getPath(), sanitizeTenant(tenant)).toFile());
+    }
+
+    @Override
+    public boolean existsByTenantAndSha1(final String tenant, final String sha1) {
+        return getFile(tenant, sha1).exists();
+    }
+
+    @Override
+    protected AbstractDbArtifact store(final String tenant, final DbArtifactHash base16Hashes, final String contentType, final String tempFile)
+            throws IOException {
+        final File file = new File(tempFile);
+        return renameFileToSHA1Naming(
+                tenant, file,
+                new ArtifactFilesystem(file, base16Hashes.getSha1(), base16Hashes, file.length(), contentType));
+    }
+
+    private ArtifactFilesystem renameFileToSHA1Naming(final String tenant, final File file,
+            final AbstractDbArtifact artifact) throws IOException {
+        final File fileSHA1Naming = getFile(tenant, artifact.getHashes().getSha1());
+        if (fileSHA1Naming.exists()) {
+            FileUtils.deleteQuietly(file);
+        } else {
+            Files.move(file.toPath(), fileSHA1Naming.toPath());
+        }
+
+        return new ArtifactFilesystem(
+                fileSHA1Naming, artifact.getArtifactId(), artifact.getHashes(), artifact.getSize(), artifact.getContentType());
+    }
+
+    private File getFile(final String tenant, final String sha1) {
+        // ensure that the sha1 is not a path traversal attack
+        if (sha1.indexOf('/') >= 0 || sha1.indexOf('\\') >= 0) {
+            throw new IllegalArgumentException("Invalid sha1 hash: " + sha1);
+        }
+
+        final File aritfactDirectory = getSha1DirectoryPath(tenant, sha1).toFile();
+        aritfactDirectory.mkdirs();
+        return new File(aritfactDirectory, sha1);
+    }
+
+    private Path getSha1DirectoryPath(final String tenant, final String sha1) {
+        final int length = sha1.length();
+        final String folder1 = sha1.substring(length - 4, length - 2);
+        final String folder2 = sha1.substring(length - 2, length);
+        return Paths.get(artifactResourceProperties.getPath(), sanitizeTenant(tenant), folder1, folder2);
+    }
+}
--- a/hawkbit-artifact/hawkbit-artifact-repository-filesystem/src/test/java/org/eclipse/hawkbit/artifact/repository/ArtifactFilesystemRepositoryTest.java
+++ b/hawkbit-artifact/hawkbit-artifact-repository-filesystem/src/test/java/org/eclipse/hawkbit/artifact/repository/ArtifactFilesystemRepositoryTest.java
@@ -0,0 +1,135 @@
+/**
+ * Copyright (c) 2015 Bosch Software Innovations GmbH and others
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipse.hawkbit.artifact.repository;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.Random;
+
+import io.qameta.allure.Description;
+import io.qameta.allure.Feature;
+import io.qameta.allure.Story;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.assertj.core.api.Assertions;
+import org.eclipse.hawkbit.artifact.repository.model.AbstractDbArtifact;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+@Slf4j
+@Feature("Unit Tests - Artifact File System Repository")
+@Story("Test storing artifact binaries in the file-system")
+class ArtifactFilesystemRepositoryTest {
+
+    private static final String TENANT = "test_tenant";
+
+    private static ArtifactFilesystemProperties artifactResourceProperties;
+    private static ArtifactFilesystemRepository artifactFilesystemRepository;
+
+    @BeforeAll
+    static void setup() throws IOException {
+        artifactResourceProperties = new ArtifactFilesystemProperties();
+        artifactResourceProperties.setPath(Files.createTempDirectory(null).toString());
+
+        artifactFilesystemRepository = new ArtifactFilesystemRepository(artifactResourceProperties);
+    }
+
+    @AfterAll
+    static void afterClass() {
+        if (new File(artifactResourceProperties.getPath()).exists()) {
+            try {
+                FileUtils.deleteDirectory(new File(artifactResourceProperties.getPath()));
+            } catch (final IOException | IllegalArgumentException e) {
+                log.warn("Cannot delete file-directory", e);
+            }
+        }
+    }
+
+    @Test
+    @Description("Verifies that an artifact can be successfully stored in the file-system repository")
+    void storeSuccessfully() throws IOException {
+        final byte[] fileContent = randomBytes();
+        final AbstractDbArtifact artifact = storeRandomArtifact(fileContent);
+
+        final byte[] readContent = new byte[fileContent.length];
+        IOUtils.read(artifact.getFileInputStream(), readContent);
+        assertThat(readContent).isEqualTo(fileContent);
+    }
+
+    @Test
+    @Description("Verifies that an artifact can be successfully stored in the file-system repository")
+    void getStoredArtifactBasedOnSHA1Hash() {
+        final byte[] fileContent = randomBytes();
+        final AbstractDbArtifact artifact = storeRandomArtifact(fileContent);
+
+        assertThat(artifactFilesystemRepository.getArtifactBySha1(TENANT, artifact.getHashes().getSha1())).isNotNull();
+    }
+
+    @Test
+    @Description("Verifies that an artifact can be deleted in the file-system repository")
+    void deleteStoredArtifactBySHA1Hash() {
+        final AbstractDbArtifact artifact = storeRandomArtifact(randomBytes());
+        artifactFilesystemRepository.deleteBySha1(TENANT, artifact.getHashes().getSha1());
+
+        assertThat(artifactFilesystemRepository.getArtifactBySha1(TENANT, artifact.getHashes().getSha1())).isNull();
+    }
+
+    @Test
+    @Description("Verifies that all artifacts of a tenant can be deleted in the file-system repository")
+    void deleteStoredArtifactOfTenant() {
+        final AbstractDbArtifact artifact = storeRandomArtifact(randomBytes());
+        artifactFilesystemRepository.deleteByTenant(TENANT);
+
+        assertThat(artifactFilesystemRepository.getArtifactBySha1(TENANT, artifact.getHashes().getSha1())).isNull();
+    }
+
+    @Test
+    @Description("Verfies that an artifact which does not exists is deleted quietly in the file-system repository")
+    void deleteArtifactWhichDoesNotExistsBySHA1HashWithoutException() {
+        try {
+            artifactFilesystemRepository.deleteBySha1(TENANT, "sha1HashWhichDoesNotExists");
+        } catch (final Exception e) {
+            Assertions.fail("did not expect an exception while deleting a file which does not exists");
+        }
+
+        final AbstractDbArtifact artifact = storeRandomArtifact(randomBytes());
+        try {
+            artifactFilesystemRepository.deleteBySha1("tenantWhichDoesNotExist", artifact.getHashes().getSha1());
+        } catch (final Exception e) {
+            Assertions.fail("did not expect an exception while deleting a file which does not exists");
+        }
+    }
+
+    private static byte[] randomBytes() {
+        final byte[] randomBytes = new byte[20];
+        new Random().nextBytes(randomBytes);
+        return randomBytes;
+    }
+
+    private AbstractDbArtifact storeRandomArtifact(final byte[] fileContent) {
+        final ByteArrayInputStream inputStream = new ByteArrayInputStream(fileContent);
+        try {
+            return artifactFilesystemRepository.store(TENANT, inputStream, "filename.tmp", "application/txt", null);
+        } finally {
+            try {
+                inputStream.close();
+            } catch (final IOException e) {
+                // do nothing
+                // still return the artifact
+            }
+        }
+    }
+}
--- a/hawkbit-artifact/hawkbit-artifact-repository-filesystem/src/test/java/org/eclipse/hawkbit/artifact/repository/ArtifactFilesystemTest.java
+++ b/hawkbit-artifact/hawkbit-artifact-repository-filesystem/src/test/java/org/eclipse/hawkbit/artifact/repository/ArtifactFilesystemTest.java
@@ -0,0 +1,57 @@
+/**
+ * Copyright (c) 2015 Bosch Software Innovations GmbH and others
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipse.hawkbit.artifact.repository;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.Files;
+
+import io.qameta.allure.Description;
+import io.qameta.allure.Feature;
+import io.qameta.allure.Story;
+import org.apache.commons.io.IOUtils;
+import org.assertj.core.api.Assertions;
+import org.eclipse.hawkbit.artifact.repository.model.DbArtifactHash;
+import org.junit.jupiter.api.Test;
+
+@Feature("Unit Tests - Artifact File System Repository")
+@Story("Test storing artifact binaries in the file-system")
+class ArtifactFilesystemTest {
+
+    @Test
+    @Description("Verifies that an exception is thrown on opening an InputStream when file does not exists")
+    void getInputStreamOfNonExistingFileThrowsException() {
+        final File file = new File("fileWhichTotalDoesNotExists");
+        final ArtifactFilesystem underTest = new ArtifactFilesystem(
+                file, "fileWhichTotalDoesNotExists",
+                new DbArtifactHash("1", "2", "3"), 0L, null);
+        try {
+            underTest.getFileInputStream();
+            Assertions.fail("Expected a FileNotFoundException because file does not exists");
+        } catch (final RuntimeException e) {
+            assertThat(e.getCause()).isInstanceOf(FileNotFoundException.class);
+        }
+    }
+
+    @Test
+    @Description("Verifies that an InputStream can be opened if file exists")
+    void getInputStreamOfExistingFile() throws IOException {
+        final File createTempFile = Files.createTempFile(ArtifactFilesystemTest.class.getSimpleName(), "").toFile();
+        createTempFile.deleteOnExit();
+
+        final ArtifactFilesystem underTest = new ArtifactFilesystem(
+                createTempFile, ArtifactFilesystemTest.class.getSimpleName(), new DbArtifactHash("1", "2", "3"), 0L, null);
+        final byte[] buffer = new byte[1024];
+        assertThat(IOUtils.read(underTest.getFileInputStream(), buffer)).isEqualTo(0);
+    }
+}