lingwei.kong/hawkbit
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Add filesystem artifact repository implementation (#336)

Browse Source 
Signed-off-by: Michael Hirsch <michael.hirsch@bosch-si.com>
This commit is contained in:
Michael Hirsch
2016-11-14 11:23:50 +01:00
committed by Kai Zimmermann
parent 9b42c8cf57
commit 8be49a1184
48 changed files with 682 additions and 425 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
hawkbit-artifact-repository-filesystem/src/main/java/org/eclipse/hawkbit/artifact/repository/ArtifactFilesystem.java Normal file
View File

@@ -0,0 +1,44 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.artifact.repository;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import org.eclipse.hawkbit.artifact.repository.model.DbArtifact;
import com.google.common.base.Throwables;
/**
* A {@link DbArtifact} implementation which dynamically creates a
* {@link FileInputStream} on calling {@link #getFileInputStream()}.
*/
public class ArtifactFilesystem extends DbArtifact {
private final File file;
ArtifactFilesystem(final File file) {
this.file = file;
}
@Override
// suppress warning, this InputStream needs to be closed by the caller, this
// cannot be closed in this method
@SuppressWarnings("squid:S2095")
public InputStream getFileInputStream() {
try {
return new BufferedInputStream(new FileInputStream(file));
} catch (final FileNotFoundException e) {
throw Throwables.propagate(e);
}
}
}

32
hawkbit-artifact-repository-filesystem/src/main/java/org/eclipse/hawkbit/artifact/repository/ArtifactFilesystemProperties.java Normal file
View File

@@ -0,0 +1,32 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.artifact.repository;
import org.springframework.boot.context.properties.ConfigurationProperties;
/**
* Configuration properties for the file-system repository, e.g. the base-path
* to store the files.
*/
@ConfigurationProperties("org.eclipse.hawkbit.repository.file")
public class ArtifactFilesystemProperties {
/**
* The base-path of the directory to store the artifacts.
*/
private String path = "./artifactrepo";
public String getPath() {
return path;
}
public void setPath(final String path) {
this.path = path;
}
}

188
hawkbit-artifact-repository-filesystem/src/main/java/org/eclipse/hawkbit/artifact/repository/ArtifactFilesystemRepository.java Normal file
View File

@@ -0,0 +1,188 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.artifact.repository;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.DigestOutputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.eclipse.hawkbit.artifact.repository.model.DbArtifact;
import org.eclipse.hawkbit.artifact.repository.model.DbArtifactHash;
import com.google.common.base.Splitter;
import com.google.common.io.BaseEncoding;
import com.google.common.io.ByteStreams;
/**
* Implementation of the {@link ArtifactRepository} to store artifacts on the
* file-system. The files are stored by their SHA1 hash of the artifact binary.
* Duplicate files with the same SHA1 hash will only stored once.
*
* All files are stored flat in one base directory configured in the
* {@link ArtifactFilesystemProperties#getPath()}.
*
* Due the limit of many file-systems of files within one directory, the files
* are stored in different sub-directories based on the last four digits of the
* SHA1-hash {@code (/basepath/[two digit sha1]/[two digit sha1])}.
*/
public class ArtifactFilesystemRepository implements ArtifactRepository {
private static final String TEMP_FILE_PREFIX = "tmp";
private static final String TEMP_FILE_SUFFIX = "artifactrepo";
private final ArtifactFilesystemProperties artifactResourceProperties;
/**
* Constructor.
*
* @param artifactResourceProperties
* the properties which holds the necessary configuration for the
* file-system repository
*/
public ArtifactFilesystemRepository(final ArtifactFilesystemProperties artifactResourceProperties) {
this.artifactResourceProperties = artifactResourceProperties;
}
@Override
public ArtifactFilesystem store(final InputStream content, final String filename, final String contentType) {
return store(content, filename, contentType, null);
}
@Override
// suppress warning, of not strong enough hashing algorithm, SHA-1 and MD5
// is not used security related
@SuppressWarnings("squid:S2070")
public ArtifactFilesystem store(final InputStream content, final String filename, final String contentType,
final DbArtifactHash hash) {
final MessageDigest mdSHA1;
final MessageDigest mdMD5;
try {
mdSHA1 = MessageDigest.getInstance("SHA1");
mdMD5 = MessageDigest.getInstance("MD5");
} catch (final NoSuchAlgorithmException e) {
throw new ArtifactStoreException(e.getMessage(), e);
}
final File file = createTempFile();
final DbArtifact artifact = store(content, contentType, hash, mdSHA1, mdMD5, file);
return renameFileToSHA1Naming(file, artifact);
}
@Override
public void deleteBySha1(final String sha1Hash) {
FileUtils.deleteQuietly(getFile(sha1Hash));
}
@Override
public ArtifactFilesystem getArtifactBySha1(final String sha1) {
final File file = getFile(sha1);
if (!file.exists()) {
return null;
}
final ArtifactFilesystem artifact = new ArtifactFilesystem(file);
artifact.setArtifactId(sha1);
artifact.setHashes(new DbArtifactHash(sha1, null));
artifact.setSize(file.length());
return artifact;
}
private DbArtifact store(final InputStream content, final String contentType, final DbArtifactHash hash,
final MessageDigest mdSHA1, final MessageDigest mdMD5, final File file) {
final DbArtifact artifact = new DbArtifact();
try (final DigestOutputStream outputstream = openFileOutputStream(file, mdSHA1, mdMD5)) {
final long artifactSize = ByteStreams.copy(content, outputstream);
outputstream.flush();
final String sha1Hash = BaseEncoding.base16().lowerCase().encode(mdSHA1.digest());
final String md5Hash = BaseEncoding.base16().lowerCase().encode(mdMD5.digest());
artifact.setArtifactId(sha1Hash);
artifact.setSize(artifactSize);
artifact.setContentType(contentType);
artifact.setHashes(new DbArtifactHash(sha1Hash, md5Hash));
checkHashes(artifact, hash);
} catch (final IOException e) {
throw new ArtifactStoreException(e.getMessage(), e);
} catch (final HashNotMatchException e) {
file.delete();
throw e;
}
return artifact;
}
private ArtifactFilesystem renameFileToSHA1Naming(final File file, final DbArtifact artifact) {
final File fileSHA1Naming = getFile(artifact.getHashes().getSha1());
final ArtifactFilesystem fileSystemArtifact = new ArtifactFilesystem(fileSHA1Naming);
if (fileSHA1Naming.exists()) {
FileUtils.deleteQuietly(file);
} else if (!file.renameTo(fileSHA1Naming)) {
throw new ArtifactStoreException("Could not store the file " + fileSHA1Naming);
}
file.delete();
fileSystemArtifact.setArtifactId(artifact.getArtifactId());
fileSystemArtifact.setContentType(artifact.getContentType());
fileSystemArtifact.setHashes(artifact.getHashes());
fileSystemArtifact.setSize(artifact.getSize());
return fileSystemArtifact;
}
private DbArtifact checkHashes(final DbArtifact artifact, final DbArtifactHash hash) {
if (hash == null) {
return artifact;
}
if (hash.getSha1() != null && !artifact.getHashes().getSha1().equals(hash.getSha1())) {
throw new HashNotMatchException("The given sh1 hash " + hash.getSha1()
+ " does not match with the calcualted sha1 hash " + artifact.getHashes().getSha1(),
HashNotMatchException.SHA1);
}
if (hash.getMd5() != null && !artifact.getHashes().getMd5().equals(hash.getMd5())) {
throw new HashNotMatchException("The given md5 hash " + hash.getMd5()
+ " does not match with the calcualted md5 hash " + artifact.getHashes().getMd5(),
HashNotMatchException.MD5);
}
return artifact;
}
private File createTempFile() {
try {
return File.createTempFile(TEMP_FILE_PREFIX, TEMP_FILE_SUFFIX);
} catch (final IOException e) {
throw new ArtifactStoreException("Cannot create tempfile", e);
}
}
private File getFile(final String sha1) {
final File aritfactDirectory = getSha1DirectoryPath(sha1).toFile();
aritfactDirectory.mkdirs();
return new File(aritfactDirectory, sha1);
}
private Path getSha1DirectoryPath(final String sha1) {
final int length = sha1.length();
final List<String> folders = Splitter.fixedLength(2).splitToList(sha1.substring(length - 4, length));
final String folder1 = folders.get(0);
final String folder2 = folders.get(1);
return Paths.get(artifactResourceProperties.getPath(), folder1, folder2);
}
private DigestOutputStream openFileOutputStream(final File file, final MessageDigest mdSHA1,
final MessageDigest mdMD5) throws FileNotFoundException {
return new DigestOutputStream(
new DigestOutputStream(new BufferedOutputStream(new FileOutputStream(file)), mdMD5), mdSHA1);
}
}

93
hawkbit-artifact-repository-filesystem/src/test/java/org/eclipse/hawkbit/artifact/repository/ArtifactFilesystemRepositoryTest.java Normal file
View File

@@ -0,0 +1,93 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.artifact.repository;
import static org.fest.assertions.Assertions.assertThat;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.Random;
import org.apache.commons.io.IOUtils;
import org.eclipse.hawkbit.artifact.repository.model.DbArtifact;
import org.fest.assertions.api.Assertions;
import org.junit.Test;
import ru.yandex.qatools.allure.annotations.Description;
import ru.yandex.qatools.allure.annotations.Features;
import ru.yandex.qatools.allure.annotations.Stories;
@Features("Unit Tests - Artifact File System Repository")
@Stories("Test storing artifact binaries in the file-system")
public class ArtifactFilesystemRepositoryTest {
private final ArtifactFilesystemProperties artifactResourceProperties = new ArtifactFilesystemProperties();
private final ArtifactFilesystemRepository artifactFilesystemRepository = new ArtifactFilesystemRepository(
artifactResourceProperties);
@Test
@Description("Verfies that an artifact can be successfully stored in the file-system repository")
public void storeSuccessfully() throws IOException {
final byte[] fileContent = randomBytes();
final ArtifactFilesystem artifact = storeRandomArtifact(fileContent);
final byte[] readContent = new byte[fileContent.length];
IOUtils.read(artifact.getFileInputStream(), readContent);
assertThat(readContent).isEqualTo(fileContent);
}
@Test
@Description("Verfies that an artifact can be successfully stored in the file-system repository")
public void getStoredArtifactBasedOnSHA1Hash() {
final byte[] fileContent = randomBytes();
final ArtifactFilesystem artifact = storeRandomArtifact(fileContent);
final DbArtifact artifactBySha1 = artifactFilesystemRepository
.getArtifactBySha1(artifact.getHashes().getSha1());
assertThat(artifactBySha1).isNotNull();
}
@Test
@Description("Verfies that an artifact can be deleted in the file-system repository")
public void deleteStoredArtifactBySHA1Hash() {
final ArtifactFilesystem artifact = storeRandomArtifact(randomBytes());
artifactFilesystemRepository.deleteBySha1(artifact.getHashes().getSha1());
assertThat(artifactFilesystemRepository.getArtifactBySha1(artifact.getHashes().getSha1())).isNull();
}
@Test
@Description("Verfies that an artifact which does not exists is deleted quietly in the file-system repository")
public void deleteArtifactWhichDoesNotExistsBySHA1HashWithoutException() {
try {
artifactFilesystemRepository.deleteBySha1("sha1HashWhichDoesNotExists");
} catch (final Exception e) {
Assertions.fail("did not expect an exception while deleting a file which does not exists");
}
}
private ArtifactFilesystem storeRandomArtifact(final byte[] fileContent) {
final String fileName = "filename.tmp";
final ByteArrayInputStream inputStream = new ByteArrayInputStream(fileContent);
final ArtifactFilesystem store = artifactFilesystemRepository.store(inputStream, fileName, "application/txt");
return store;
}
private static byte[] randomBytes() {
final byte[] randomBytes = new byte[20];
final Random ran = new Random();
ran.nextBytes(randomBytes);
return randomBytes;
}
}

52
hawkbit-artifact-repository-filesystem/src/test/java/org/eclipse/hawkbit/artifact/repository/ArtifactFilesystemTest.java Normal file
View File

@@ -0,0 +1,52 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.artifact.repository;
import static org.fest.assertions.Assertions.assertThat;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import org.apache.commons.io.IOUtils;
import org.fest.assertions.api.Assertions;
import org.junit.Test;
import ru.yandex.qatools.allure.annotations.Description;
import ru.yandex.qatools.allure.annotations.Features;
import ru.yandex.qatools.allure.annotations.Stories;
@Features("Unit Tests - Artifact File System Repository")
@Stories("Test storing artifact binaries in the file-system")
public class ArtifactFilesystemTest {
@Test
@Description("Verifies that an exception is thrown on opening an InputStream when file does not exists")
public void getInputStreamOfNonExistingFileThrowsException() {
final File file = new File("fileWhichTotalDoesNotExists");
final ArtifactFilesystem underTest = new ArtifactFilesystem(file);
try {
underTest.getFileInputStream();
Assertions.fail("Expected a FileNotFoundException because file does not exists");
} catch (final RuntimeException e) {
assertThat(e.getCause()).isInstanceOf(FileNotFoundException.class);
}
}
@Test
@Description("Verifies that an InputStream can be opened if file exists")
public void getInputStreamOfExistingFile() throws IOException {
final File createTempFile = File.createTempFile(ArtifactFilesystemTest.class.getSimpleName(), "");
createTempFile.deleteOnExit();
final ArtifactFilesystem underTest = new ArtifactFilesystem(createTempFile);
final byte[] buffer = new byte[1024];
IOUtils.read(underTest.getFileInputStream(), buffer);
}
}