lingwei.kong/hawkbit
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Add filesystem artifact repository implementation (#336)

Browse Source 
Signed-off-by: Michael Hirsch <michael.hirsch@bosch-si.com>
This commit is contained in:
Michael Hirsch
2016-11-14 11:23:50 +01:00
committed by Kai Zimmermann
parent 9b42c8cf57
commit 8be49a1184
48 changed files with 682 additions and 425 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
extensions/hawkbit-extension-artifact-repository-mongo/README.md Normal file
View File

@@ -0,0 +1,23 @@
# Eclipse.IoT hawkBit - Artifact Repository MongoDB
HawkBit Artifact Repository is a library for storing binary artifacts and metadata into MongoDB.
## Using Artifact Repository MongoDB Extension
The module contains a spring-boot autoconfiguration for easily integration into spring-boot projects.
For using this extension in the hawkbit-example-application you just need to add the maven dependency.
```
<dependency>
<groupId>org.eclipse.hawkbit</groupId>
<artifactId>hawkbit-extension-artifact-repository-mongo</artifactId>
<version>${project.version}</version>
</dependency>
```
If you do not have a mongoDB running you can use the the flapdoodle project to download and start an mongoDB on demand
```
<dependency>
<groupId>de.flapdoodle.embed</groupId>
<artifactId>de.flapdoodle.embed.mongo</artifactId>
<version>${flapdoodle.version}</version>
</dependency>
```

81
extensions/hawkbit-extension-artifact-repository-mongo/pom.xml Normal file
View File

@@ -0,0 +1,81 @@
<!--
Copyright (c) 2015 Bosch Software Innovations GmbH and others.
All rights reserved. This program and the accompanying materials
are made available under the terms of the Eclipse Public License v1.0
which accompanies this distribution, and is available at
http://www.eclipse.org/legal/epl-v10.html
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.eclipse.hawkbit</groupId>
<version>0.2.0-SNAPSHOT</version>
<artifactId>hawkbit-parent</artifactId>
</parent>
<artifactId>hawkbit-extension-artifact-repository-mongo</artifactId>
<name>hawkBit :: Artifact Repository Mongo</name>
<dependencies>
<dependency>
<groupId>org.eclipse.hawkbit</groupId>
<artifactId>hawkbit-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-mongodb</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<!-- TEST -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.easytesting</groupId>
<artifactId>fest-assert-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>ru.yandex.qatools.allure</groupId>
<artifactId>allure-junit-adaptor</artifactId>
<scope>test</scope>
</dependency>
<!-- MongoDB -->
<dependency>
<groupId>de.flapdoodle.embed</groupId>
<artifactId>de.flapdoodle.embed.mongo</artifactId>
<scope>test</scope>
</dependency>
<!-- Mockito -->
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

45
extensions/hawkbit-extension-artifact-repository-mongo/src/main/java/org/eclipse/hawkbit/artifact/repository/GridFsArtifact.java Normal file
View File

@@ -0,0 +1,45 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.artifact.repository;
import java.io.InputStream;
import org.eclipse.hawkbit.artifact.repository.model.DbArtifact;
import com.mongodb.gridfs.GridFSDBFile;
import com.mongodb.gridfs.GridFSFile;
/**
* A wrapper object for the {@link DbArtifact} object which returns the
* {@link InputStream} directly from {@link GridFSDBFile#getInputStream()} which
* retrieves when calling {@link #getFileInputStream()} always a new
* {@link InputStream} and not the same.
*
*
*
*/
public class GridFsArtifact extends DbArtifact {
private final GridFSFile dbFile;
/**
* @param dbFile
*/
public GridFsArtifact(final GridFSFile dbFile) {
this.dbFile = dbFile;
}
@Override
public InputStream getFileInputStream() {
if (dbFile instanceof GridFSDBFile) {
return ((GridFSDBFile) dbFile).getInputStream();
}
return null;
}
}

244
extensions/hawkbit-extension-artifact-repository-mongo/src/main/java/org/eclipse/hawkbit/artifact/repository/MongoDBArtifactStore.java Normal file
View File

@@ -0,0 +1,244 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.artifact.repository;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.security.DigestOutputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.List;
import java.util.stream.Collectors;
import org.eclipse.hawkbit.artifact.repository.model.DbArtifact;
import org.eclipse.hawkbit.artifact.repository.model.DbArtifactHash;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.gridfs.GridFsOperations;
import com.google.common.io.BaseEncoding;
import com.google.common.io.ByteStreams;
import com.mongodb.BasicDBObject;
import com.mongodb.MongoClientException;
import com.mongodb.MongoException;
import com.mongodb.gridfs.GridFSDBFile;
import com.mongodb.gridfs.GridFSFile;
/**
* The file management which looks up all the file in the file tore.
*
*/
public class MongoDBArtifactStore implements ArtifactRepository {
private static final Logger LOGGER = LoggerFactory.getLogger(MongoDBArtifactStore.class);
/**
* The mongoDB field which holds the filename of the file to download.
* hawkBit update-server uses the SHA hash as a filename and lookup in the
* mongoDB.
*/
private static final String FILENAME = "filename";
/**
* The mongoDB field which automatically calculated by the mongoDB.
*/
private static final String MD5 = "md5";
/**
* The mongoDB field which holds the SHA1 hash, stored in the meta data
* object.
*/
private static final String SHA1 = "sha1";
private static final String ID = "_id";
@Autowired
private GridFsOperations gridFs;
MongoTemplate mongoTemplate;
/**
* Retrieves a {@link GridFSDBFile} from the store by it's SHA1 hash.
*
* @param sha1Hash
* the sha1-hash of the file to lookup.
*
* @return The DbArtifact object or {@code null} if no file exists.
*/
@Override
public DbArtifact getArtifactBySha1(final String sha1Hash) {
return map(gridFs.findOne(new Query().addCriteria(Criteria.where(FILENAME).is(sha1Hash))));
}
/**
* Retrieves a {@link GridFSDBFile} from the store by it's MD5 hash.
*
* @param md5Hash
* the md5-hash of the file to lookup.
* @return The gridfs file object or {@code null} if no file exists.
*/
public DbArtifact getArtifactByMd5(final String md5Hash) {
return map(gridFs.findOne(new Query().addCriteria(Criteria.where(MD5).is(md5Hash))));
}
@Override
public DbArtifact store(final InputStream content, final String filename, final String contentType) {
return store(content, filename, contentType, null);
}
@Override
public DbArtifact store(final InputStream content, final String filename, final String contentType,
final DbArtifactHash hash) {
File tempFile = null;
try {
LOGGER.debug("storing file {} of content {}", filename, contentType);
tempFile = File.createTempFile("uploadFile", null);
try (final BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(tempFile))) {
try (BufferedInputStream bis = new BufferedInputStream(content)) {
return store(bis, contentType, bos, tempFile, hash);
}
}
} catch (final IOException | MongoException e1) {
throw new ArtifactStoreException(e1.getMessage(), e1);
} finally {
if (tempFile != null && !tempFile.delete()) {
LOGGER.error("Could not delete temporary file: {}", tempFile);
}
}
}
@Override
public void deleteBySha1(final String sha1Hash) {
try {
deleteArtifact(gridFs.findOne(new Query().addCriteria(Criteria.where(FILENAME).is(sha1Hash))));
} catch (final MongoException e) {
throw new ArtifactStoreException(e.getMessage(), e);
}
}
private void deleteArtifact(final GridFSDBFile dbFile) {
if (dbFile != null) {
try {
gridFs.delete(new Query().addCriteria(Criteria.where(ID).is(dbFile.getId())));
} catch (final MongoClientException e) {
throw new ArtifactStoreException(e.getMessage(), e);
}
}
}
private DbArtifact store(final InputStream content, final String contentType, final OutputStream os,
final File tempFile, final DbArtifactHash hash) {
final GridFsArtifact storedArtifact;
try {
final String sha1Hash = computeSHA1Hash(content, os, hash != null ? hash.getSha1() : null);
// upload if it does not exist already, check if file exists, not
// tenant specific.
final GridFSDBFile result = gridFs.findOne(new Query().addCriteria(Criteria.where(FILENAME).is(sha1Hash)));
if (null == result) {
try (FileInputStream inputStream = new FileInputStream(tempFile)) {
final BasicDBObject metadata = new BasicDBObject();
metadata.put(SHA1, sha1Hash);
storedArtifact = map(gridFs.store(inputStream, sha1Hash, contentType, metadata));
}
} else {
LOGGER.info("file with sha1 hash {} already exists in database, increase reference counter", sha1Hash);
result.save();
storedArtifact = map(result);
}
} catch (final NoSuchAlgorithmException | IOException e) {
throw new ArtifactStoreException(e.getMessage(), e);
}
if (hash != null && hash.getMd5() != null
&& !storedArtifact.getHashes().getMd5().equalsIgnoreCase(hash.getMd5())) {
throw new HashNotMatchException("The given md5 hash " + hash.getMd5()
+ " not matching the calculated md5 hash " + storedArtifact.getHashes().getMd5(),
HashNotMatchException.MD5);
}
return storedArtifact;
}
private static String computeSHA1Hash(final InputStream stream, final OutputStream os, final String providedSHA1Sum)
throws NoSuchAlgorithmException, IOException {
String sha1Hash;
// compute digest
// Exception squid:S2070 - not used for hashing sensitive
// data
@SuppressWarnings("squid:S2070")
final MessageDigest md = MessageDigest.getInstance("SHA-1");
try (final DigestOutputStream dos = new DigestOutputStream(os, md)) {
ByteStreams.copy(stream, dos);
}
sha1Hash = BaseEncoding.base16().lowerCase().encode(md.digest());
if (providedSHA1Sum != null && !providedSHA1Sum.equalsIgnoreCase(sha1Hash)) {
throw new HashNotMatchException(
"The given sha1 hash " + providedSHA1Sum + " not matching the calculated sha1 hash " + sha1Hash,
HashNotMatchException.SHA1);
}
return sha1Hash;
}
/**
* Maps a list of {@link GridFSDBFile} to paged list of {@link DbArtifact}s.
*
* @param tenant
* the tenant
* @param dbFiles
* the list of mongoDB gridFs files.
* @return a paged list of artifacts mapped from the given dbFiles
*/
private List<DbArtifact> map(final List<GridFSDBFile> dbFiles) {
return dbFiles.stream().map(MongoDBArtifactStore::map).collect(Collectors.toList());
}
/**
* Retrieves a list of {@link GridFSDBFile} from the store by all ids.
*
* @param ids
* the ids of the files to lookup.
* @return list of artfiacts
*/
public List<DbArtifact> getArtifactsByIds(final List<String> ids) {
return map(gridFs.find(new Query().addCriteria(Criteria.where(ID).in(ids))));
}
/**
* Maps a single {@link GridFSFile} to {@link DbArtifact}.
*
* @param tenant
* the tenant
* @param dbFile
* the mongoDB gridFs file.
* @return a mapped artifact from the given dbFile
*/
private static GridFsArtifact map(final GridFSFile fsFile) {
if (fsFile == null) {
return null;
}
final GridFsArtifact artifact = new GridFsArtifact(fsFile);
artifact.setArtifactId(fsFile.getId().toString());
artifact.setSize(fsFile.getLength());
artifact.setContentType(fsFile.getContentType());
artifact.setHashes(new DbArtifactHash(fsFile.getFilename(), fsFile.getMD5()));
return artifact;
}
}

28
extensions/hawkbit-extension-artifact-repository-mongo/src/main/java/org/eclipse/hawkbit/artifact/repository/MongoDBArtifactStoreAutoConfiguration.java Normal file
View File

@@ -0,0 +1,28 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.artifact.repository;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* Auto configuration for the {@link MongoDBArtifactStore}.
*/
@Configuration
public class MongoDBArtifactStoreAutoConfiguration {
/**
* @return Default {@link ArtifactRepository} implementation.
*/
@Bean
public ArtifactRepository artifactRepository() {
return new MongoDBArtifactStore();
}
}

3
extensions/hawkbit-extension-artifact-repository-mongo/src/main/resources/META-INF/spring.factories Normal file
View File

@@ -0,0 +1,3 @@
# Auto Configure
org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
org.eclipse.hawkbit.artifact.repository.MongoDBArtifactStoreAutoConfiguration

17
extensions/hawkbit-extension-artifact-repository-mongo/src/test/java/org/eclipse/hawkbit/artifact/TestConfiguration.java Normal file
View File

@@ -0,0 +1,17 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.artifact;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.context.annotation.ComponentScan;
@ComponentScan
@EnableAutoConfiguration
public class TestConfiguration {
}

84
extensions/hawkbit-extension-artifact-repository-mongo/src/test/java/org/eclipse/hawkbit/artifact/repository/MongoDBArtifactStoreTest.java Normal file
View File

@@ -0,0 +1,84 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.artifact.repository;
import static org.fest.assertions.api.Assertions.assertThat;
import java.io.ByteArrayInputStream;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Random;
import org.eclipse.hawkbit.artifact.TestConfiguration;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.SpringApplicationConfiguration;
import org.springframework.data.mongodb.gridfs.GridFsOperations;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import com.google.common.io.BaseEncoding;
import ru.yandex.qatools.allure.annotations.Description;
import ru.yandex.qatools.allure.annotations.Features;
import ru.yandex.qatools.allure.annotations.Stories;
@Features("Component Tests - Repository")
@Stories("Artifact Store MongoDB")
@RunWith(SpringJUnit4ClassRunner.class)
@SpringApplicationConfiguration(classes = { MongoDBArtifactStoreAutoConfiguration.class, TestConfiguration.class })
@TestPropertySource(properties = { "spring.data.mongodb.port=0", "spring.mongodb.embedded.version=3.2.7" })
public class MongoDBArtifactStoreTest {
@Autowired
private MongoDBArtifactStore artifactStoreUnderTest;
@Autowired
private GridFsOperations gridFs;
@Test
@Description("Ensures that search by SHA1 hash (which is used by hawkBit as artifact ID) finds the expected results.")
public void findArtifactBySHA1Hash() throws NoSuchAlgorithmException {
final int filelengthBytes = 128;
final String filename = "testfile.json";
final String contentType = "application/json";
final DigestInputStream digestInputStream = digestInputStream(generateInputStream(filelengthBytes), "SHA-1");
artifactStoreUnderTest.store(digestInputStream, filename, contentType);
assertThat(artifactStoreUnderTest.getArtifactBySha1(
BaseEncoding.base16().lowerCase().encode(digestInputStream.getMessageDigest().digest()))).isNotNull();
}
@Test
@Description("Ensures that search by MD5 hash finds the expected results.")
public void findArtifactByMD5Hash() throws NoSuchAlgorithmException {
final int filelengthBytes = 128;
final String filename = "testfile.json";
final String contentType = "application/json";
final DigestInputStream digestInputStream = digestInputStream(generateInputStream(filelengthBytes), "MD5");
artifactStoreUnderTest.store(digestInputStream, filename, contentType);
assertThat(artifactStoreUnderTest.getArtifactByMd5(
BaseEncoding.base16().lowerCase().encode(digestInputStream.getMessageDigest().digest()))).isNotNull();
}
private static ByteArrayInputStream generateInputStream(final int length) {
final byte[] bytes = new byte[length];
new Random().nextBytes(bytes);
return new ByteArrayInputStream(bytes);
}
private static DigestInputStream digestInputStream(final ByteArrayInputStream stream, final String digest)
throws NoSuchAlgorithmException {
return new DigestInputStream(stream, MessageDigest.getInstance(digest));
}
}

1
extensions/pom.xml
View File

@@ -22,6 +22,7 @@
<modules>
<module>hawkbit-extension-uaa</module>
<module>hawkbit-extension-artifact-repository-mongo</module>
</modules>
</project>