Add filesystem artifact repository implementation (#336)

Signed-off-by: Michael Hirsch <michael.hirsch@bosch-si.com>

hawkbit-artifact-repository-filesystem/.gitignore

@@ -0,0 +1 @@
+/artifactrepo/*

hawkbit-artifact-repository-filesystem/README.md

@@ -0,0 +1,17 @@
+# Eclipse.IoT hawkBit - Artifact Repository File System
+
+This module contains the implementation of the `ArtifactRepository` based on the file-system.
+It's a very convenient and easy implementation of storing the artifact binaries into the file-system based on the SHA-1 hash naming.
+
+Due the limit of many file-systems of files within one directory, the files
+are stored in different sub-directories based on the last four digits of the
+SHA1-hash `/basepath/[two digit sha1]/[two digit sha1/sha1-hash-filename]`. 
+
+# Compile
+
+#### Build hawkbit-artifact-repository-filesystem
+
+```
+$ cd hawkbit/hawkbit-artifact-repository-filesystem
+$ mvn clean install
+```

hawkbit-artifact-repository-filesystem/pom.xml

@@ -0,0 +1,69 @@
+<!--
+
+    Copyright (c) 2015 Bosch Software Innovations GmbH and others.
+
+    All rights reserved. This program and the accompanying materials
+    are made available under the terms of the Eclipse Public License v1.0
+    which accompanies this distribution, and is available at
+    http://www.eclipse.org/legal/epl-v10.html
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+   <modelVersion>4.0.0</modelVersion>
+   <parent>
+      <groupId>org.eclipse.hawkbit</groupId>
+      <artifactId>hawkbit-parent</artifactId>
+      <version>0.2.0-SNAPSHOT</version>
+   </parent>
+   <artifactId>hawkbit-artifact-repository-filesystem</artifactId>
+
+   <dependencies>
+      <dependency>
+         <groupId>org.eclipse.hawkbit</groupId>
+         <artifactId>hawkbit-core</artifactId>
+         <version>${project.version}</version>
+      </dependency>
+      <dependency>
+         <groupId>com.google.guava</groupId>
+         <artifactId>guava</artifactId>
+      </dependency>
+      <dependency>
+         <groupId>org.springframework</groupId>
+         <artifactId>spring-core</artifactId>
+      </dependency>
+      <dependency>
+         <groupId>org.springframework.boot</groupId>
+         <artifactId>spring-boot-autoconfigure</artifactId>
+      </dependency>
+      <dependency>
+         <groupId>commons-io</groupId>
+         <artifactId>commons-io</artifactId>
+      </dependency>
+      <dependency>
+         <groupId>org.easytesting</groupId>
+         <artifactId>fest-assert-core</artifactId>
+         <scope>test</scope>
+      </dependency>
+      <dependency>
+         <groupId>org.easytesting</groupId>
+         <artifactId>fest-assert</artifactId>
+         <scope>test</scope>
+      </dependency>
+      <dependency>
+         <groupId>org.mockito</groupId>
+         <artifactId>mockito-core</artifactId>
+         <scope>test</scope>
+      </dependency>
+      <dependency>
+         <groupId>ru.yandex.qatools.allure</groupId>
+         <artifactId>allure-junit-adaptor</artifactId>
+         <scope>test</scope>
+      </dependency>
+      <dependency>
+         <groupId>org.springframework.boot</groupId>
+         <artifactId>spring-boot-configuration-processor</artifactId>
+         <optional>true</optional>
+      </dependency>
+   </dependencies>
+
+</project>

hawkbit-artifact-repository-filesystem/src/main/java/org/eclipse/hawkbit/artifact/repository/ArtifactFilesystem.java

@@ -0,0 +1,44 @@
+/**
+ * Copyright (c) 2015 Bosch Software Innovations GmbH and others.
+ *
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.eclipse.hawkbit.artifact.repository;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+
+import org.eclipse.hawkbit.artifact.repository.model.DbArtifact;
+
+import com.google.common.base.Throwables;
+
+/**
+ * A {@link DbArtifact} implementation which dynamically creates a
+ * {@link FileInputStream} on calling {@link #getFileInputStream()}.
+ */
+public class ArtifactFilesystem extends DbArtifact {
+
+    private final File file;
+
+    ArtifactFilesystem(final File file) {
+        this.file = file;
+    }
+
+    @Override
+    // suppress warning, this InputStream needs to be closed by the caller, this
+    // cannot be closed in this method
+    @SuppressWarnings("squid:S2095")
+    public InputStream getFileInputStream() {
+        try {
+            return new BufferedInputStream(new FileInputStream(file));
+        } catch (final FileNotFoundException e) {
+            throw Throwables.propagate(e);
+        }
+    }
+}

hawkbit-artifact-repository-filesystem/src/main/java/org/eclipse/hawkbit/artifact/repository/ArtifactFilesystemProperties.java

@@ -0,0 +1,32 @@
+/**
+ * Copyright (c) 2015 Bosch Software Innovations GmbH and others.
+ *
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.eclipse.hawkbit.artifact.repository;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+/**
+ * Configuration properties for the file-system repository, e.g. the base-path
+ * to store the files.
+ */
+@ConfigurationProperties("org.eclipse.hawkbit.repository.file")
+public class ArtifactFilesystemProperties {
+
+    /**
+     * The base-path of the directory to store the artifacts.
+     */
+    private String path = "./artifactrepo";
+
+    public String getPath() {
+        return path;
+    }
+
+    public void setPath(final String path) {
+        this.path = path;
+    }
+}

hawkbit-artifact-repository-filesystem/src/main/java/org/eclipse/hawkbit/artifact/repository/ArtifactFilesystemRepository.java

@@ -0,0 +1,188 @@
+/**
+ * Copyright (c) 2015 Bosch Software Innovations GmbH and others.
+ *
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.eclipse.hawkbit.artifact.repository;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.DigestOutputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.eclipse.hawkbit.artifact.repository.model.DbArtifact;
+import org.eclipse.hawkbit.artifact.repository.model.DbArtifactHash;
+
+import com.google.common.base.Splitter;
+import com.google.common.io.BaseEncoding;
+import com.google.common.io.ByteStreams;
+
+/**
+ * Implementation of the {@link ArtifactRepository} to store artifacts on the
+ * file-system. The files are stored by their SHA1 hash of the artifact binary.
+ * Duplicate files with the same SHA1 hash will only stored once.
+ * 
+ * All files are stored flat in one base directory configured in the
+ * {@link ArtifactFilesystemProperties#getPath()}.
+ * 
+ * Due the limit of many file-systems of files within one directory, the files
+ * are stored in different sub-directories based on the last four digits of the
+ * SHA1-hash {@code (/basepath/[two digit sha1]/[two digit sha1])}.
+ */
+public class ArtifactFilesystemRepository implements ArtifactRepository {
+
+    private static final String TEMP_FILE_PREFIX = "tmp";
+    private static final String TEMP_FILE_SUFFIX = "artifactrepo";
+    private final ArtifactFilesystemProperties artifactResourceProperties;
+
+    /**
+     * Constructor.
+     * 
+     * @param artifactResourceProperties
+     *            the properties which holds the necessary configuration for the
+     *            file-system repository
+     */
+    public ArtifactFilesystemRepository(final ArtifactFilesystemProperties artifactResourceProperties) {
+        this.artifactResourceProperties = artifactResourceProperties;
+    }
+
+    @Override
+    public ArtifactFilesystem store(final InputStream content, final String filename, final String contentType) {
+        return store(content, filename, contentType, null);
+    }
+
+    @Override
+    // suppress warning, of not strong enough hashing algorithm, SHA-1 and MD5
+    // is not used security related
+    @SuppressWarnings("squid:S2070")
+    public ArtifactFilesystem store(final InputStream content, final String filename, final String contentType,
+            final DbArtifactHash hash) {
+
+        final MessageDigest mdSHA1;
+        final MessageDigest mdMD5;
+        try {
+            mdSHA1 = MessageDigest.getInstance("SHA1");
+            mdMD5 = MessageDigest.getInstance("MD5");
+        } catch (final NoSuchAlgorithmException e) {
+            throw new ArtifactStoreException(e.getMessage(), e);
+        }
+
+        final File file = createTempFile();
+        final DbArtifact artifact = store(content, contentType, hash, mdSHA1, mdMD5, file);
+        return renameFileToSHA1Naming(file, artifact);
+    }
+
+    @Override
+    public void deleteBySha1(final String sha1Hash) {
+        FileUtils.deleteQuietly(getFile(sha1Hash));
+    }
+
+    @Override
+    public ArtifactFilesystem getArtifactBySha1(final String sha1) {
+        final File file = getFile(sha1);
+        if (!file.exists()) {
+            return null;
+        }
+        final ArtifactFilesystem artifact = new ArtifactFilesystem(file);
+        artifact.setArtifactId(sha1);
+        artifact.setHashes(new DbArtifactHash(sha1, null));
+        artifact.setSize(file.length());
+        return artifact;
+    }
+
+    private DbArtifact store(final InputStream content, final String contentType, final DbArtifactHash hash,
+            final MessageDigest mdSHA1, final MessageDigest mdMD5, final File file) {
+        final DbArtifact artifact = new DbArtifact();
+        try (final DigestOutputStream outputstream = openFileOutputStream(file, mdSHA1, mdMD5)) {
+            final long artifactSize = ByteStreams.copy(content, outputstream);
+            outputstream.flush();
+            final String sha1Hash = BaseEncoding.base16().lowerCase().encode(mdSHA1.digest());
+            final String md5Hash = BaseEncoding.base16().lowerCase().encode(mdMD5.digest());
+            artifact.setArtifactId(sha1Hash);
+            artifact.setSize(artifactSize);
+            artifact.setContentType(contentType);
+            artifact.setHashes(new DbArtifactHash(sha1Hash, md5Hash));
+            checkHashes(artifact, hash);
+        } catch (final IOException e) {
+            throw new ArtifactStoreException(e.getMessage(), e);
+        } catch (final HashNotMatchException e) {
+            file.delete();
+            throw e;
+        }
+        return artifact;
+    }
+
+    private ArtifactFilesystem renameFileToSHA1Naming(final File file, final DbArtifact artifact) {
+        final File fileSHA1Naming = getFile(artifact.getHashes().getSha1());
+        final ArtifactFilesystem fileSystemArtifact = new ArtifactFilesystem(fileSHA1Naming);
+        if (fileSHA1Naming.exists()) {
+            FileUtils.deleteQuietly(file);
+        } else if (!file.renameTo(fileSHA1Naming)) {
+            throw new ArtifactStoreException("Could not store the file " + fileSHA1Naming);
+        }
+
+        file.delete();
+        fileSystemArtifact.setArtifactId(artifact.getArtifactId());
+        fileSystemArtifact.setContentType(artifact.getContentType());
+        fileSystemArtifact.setHashes(artifact.getHashes());
+        fileSystemArtifact.setSize(artifact.getSize());
+        return fileSystemArtifact;
+    }
+
+    private DbArtifact checkHashes(final DbArtifact artifact, final DbArtifactHash hash) {
+        if (hash == null) {
+            return artifact;
+        }
+        if (hash.getSha1() != null && !artifact.getHashes().getSha1().equals(hash.getSha1())) {
+            throw new HashNotMatchException("The given sh1 hash " + hash.getSha1()
+                    + " does not match with the calcualted sha1 hash " + artifact.getHashes().getSha1(),
+                    HashNotMatchException.SHA1);
+        }
+        if (hash.getMd5() != null && !artifact.getHashes().getMd5().equals(hash.getMd5())) {
+            throw new HashNotMatchException("The given md5 hash " + hash.getMd5()
+                    + " does not match with the calcualted md5 hash " + artifact.getHashes().getMd5(),
+                    HashNotMatchException.MD5);
+        }
+        return artifact;
+    }
+
+    private File createTempFile() {
+        try {
+            return File.createTempFile(TEMP_FILE_PREFIX, TEMP_FILE_SUFFIX);
+        } catch (final IOException e) {
+            throw new ArtifactStoreException("Cannot create tempfile", e);
+        }
+    }
+
+    private File getFile(final String sha1) {
+        final File aritfactDirectory = getSha1DirectoryPath(sha1).toFile();
+        aritfactDirectory.mkdirs();
+        return new File(aritfactDirectory, sha1);
+    }
+
+    private Path getSha1DirectoryPath(final String sha1) {
+        final int length = sha1.length();
+        final List<String> folders = Splitter.fixedLength(2).splitToList(sha1.substring(length - 4, length));
+        final String folder1 = folders.get(0);
+        final String folder2 = folders.get(1);
+        return Paths.get(artifactResourceProperties.getPath(), folder1, folder2);
+    }
+
+    private DigestOutputStream openFileOutputStream(final File file, final MessageDigest mdSHA1,
+            final MessageDigest mdMD5) throws FileNotFoundException {
+        return new DigestOutputStream(
+                new DigestOutputStream(new BufferedOutputStream(new FileOutputStream(file)), mdMD5), mdSHA1);
+    }
+}

hawkbit-artifact-repository-filesystem/src/test/java/org/eclipse/hawkbit/artifact/repository/ArtifactFilesystemRepositoryTest.java

@@ -0,0 +1,93 @@
+/**
+ * Copyright (c) 2015 Bosch Software Innovations GmbH and others.
+ *
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.eclipse.hawkbit.artifact.repository;
+
+import static org.fest.assertions.Assertions.assertThat;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.commons.io.IOUtils;
+import org.eclipse.hawkbit.artifact.repository.model.DbArtifact;
+import org.fest.assertions.api.Assertions;
+import org.junit.Test;
+
+import ru.yandex.qatools.allure.annotations.Description;
+import ru.yandex.qatools.allure.annotations.Features;
+import ru.yandex.qatools.allure.annotations.Stories;
+
+@Features("Unit Tests - Artifact File System Repository")
+@Stories("Test storing artifact binaries in the file-system")
+public class ArtifactFilesystemRepositoryTest {
+
+    private final ArtifactFilesystemProperties artifactResourceProperties = new ArtifactFilesystemProperties();
+
+    private final ArtifactFilesystemRepository artifactFilesystemRepository = new ArtifactFilesystemRepository(
+            artifactResourceProperties);
+
+    @Test
+    @Description("Verfies that an artifact can be successfully stored in the file-system repository")
+    public void storeSuccessfully() throws IOException {
+        final byte[] fileContent = randomBytes();
+        final ArtifactFilesystem artifact = storeRandomArtifact(fileContent);
+
+        final byte[] readContent = new byte[fileContent.length];
+        IOUtils.read(artifact.getFileInputStream(), readContent);
+
+        assertThat(readContent).isEqualTo(fileContent);
+    }
+
+    @Test
+    @Description("Verfies that an artifact can be successfully stored in the file-system repository")
+    public void getStoredArtifactBasedOnSHA1Hash() {
+
+        final byte[] fileContent = randomBytes();
+        final ArtifactFilesystem artifact = storeRandomArtifact(fileContent);
+
+        final DbArtifact artifactBySha1 = artifactFilesystemRepository
+                .getArtifactBySha1(artifact.getHashes().getSha1());
+        assertThat(artifactBySha1).isNotNull();
+    }
+
+    @Test
+    @Description("Verfies that an artifact can be deleted in the file-system repository")
+    public void deleteStoredArtifactBySHA1Hash() {
+        final ArtifactFilesystem artifact = storeRandomArtifact(randomBytes());
+
+        artifactFilesystemRepository.deleteBySha1(artifact.getHashes().getSha1());
+
+        assertThat(artifactFilesystemRepository.getArtifactBySha1(artifact.getHashes().getSha1())).isNull();
+    }
+
+    @Test
+    @Description("Verfies that an artifact which does not exists is deleted quietly in the file-system repository")
+    public void deleteArtifactWhichDoesNotExistsBySHA1HashWithoutException() {
+        try {
+            artifactFilesystemRepository.deleteBySha1("sha1HashWhichDoesNotExists");
+        } catch (final Exception e) {
+            Assertions.fail("did not expect an exception while deleting a file which does not exists");
+        }
+    }
+
+    private ArtifactFilesystem storeRandomArtifact(final byte[] fileContent) {
+        final String fileName = "filename.tmp";
+        final ByteArrayInputStream inputStream = new ByteArrayInputStream(fileContent);
+        final ArtifactFilesystem store = artifactFilesystemRepository.store(inputStream, fileName, "application/txt");
+        return store;
+    }
+
+    private static byte[] randomBytes() {
+        final byte[] randomBytes = new byte[20];
+        final Random ran = new Random();
+        ran.nextBytes(randomBytes);
+        return randomBytes;
+    }
+
+}

hawkbit-artifact-repository-filesystem/src/test/java/org/eclipse/hawkbit/artifact/repository/ArtifactFilesystemTest.java

@@ -0,0 +1,52 @@
+/**
+ * Copyright (c) 2015 Bosch Software Innovations GmbH and others.
+ *
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.eclipse.hawkbit.artifact.repository;
+
+import static org.fest.assertions.Assertions.assertThat;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import org.apache.commons.io.IOUtils;
+import org.fest.assertions.api.Assertions;
+import org.junit.Test;
+
+import ru.yandex.qatools.allure.annotations.Description;
+import ru.yandex.qatools.allure.annotations.Features;
+import ru.yandex.qatools.allure.annotations.Stories;
+
+@Features("Unit Tests - Artifact File System Repository")
+@Stories("Test storing artifact binaries in the file-system")
+public class ArtifactFilesystemTest {
+
+    @Test
+    @Description("Verifies that an exception is thrown on opening an InputStream when file does not exists")
+    public void getInputStreamOfNonExistingFileThrowsException() {
+        final File file = new File("fileWhichTotalDoesNotExists");
+        final ArtifactFilesystem underTest = new ArtifactFilesystem(file);
+        try {
+            underTest.getFileInputStream();
+            Assertions.fail("Expected a FileNotFoundException because file does not exists");
+        } catch (final RuntimeException e) {
+            assertThat(e.getCause()).isInstanceOf(FileNotFoundException.class);
+        }
+    }
+
+    @Test
+    @Description("Verifies that an InputStream can be opened if file exists")
+    public void getInputStreamOfExistingFile() throws IOException {
+        final File createTempFile = File.createTempFile(ArtifactFilesystemTest.class.getSimpleName(), "");
+        createTempFile.deleteOnExit();
+
+        final ArtifactFilesystem underTest = new ArtifactFilesystem(createTempFile);
+        final byte[] buffer = new byte[1024];
+        IOUtils.read(underTest.getFileInputStream(), buffer);
+    }
+}