lingwei.kong/hawkbit
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Feature s3 repository extension (#366)

Browse Source 
Signed-off-by: Michael Hirsch <michael.hirsch@bosch-si.com>
This commit is contained in:
Michael Hirsch
2016-12-07 09:52:43 +01:00
committed by Kai Zimmermann
parent 11fa4469dd
commit edbadc4002
10 changed files with 737 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
extensions/hawkbit-extension-artifact-repository-mongo/src/main/java/org/eclipse/hawkbit/artifact/repository/MongoDBArtifactStoreAutoConfiguration.java
View File

@@ -8,7 +8,7 @@
*/
package org.eclipse.hawkbit.artifact.repository;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@@ -16,6 +16,7 @@ import org.springframework.context.annotation.Configuration;
* Auto configuration for the {@link MongoDBArtifactStore}.
*/
@Configuration
@ConditionalOnProperty(prefix = "org.eclipse.hawkbit.artifact.repository.mongo", name = "enabled", matchIfMissing = true)
public class MongoDBArtifactStoreAutoConfiguration {
/**

33
extensions/hawkbit-extension-artifact-repository-s3/README.md Normal file
View File

@@ -0,0 +1,33 @@
# Eclipse hawkBit - Artifact Repository AWS S3
HawkBit Artifact Repository is a library for storing binary artifacts and metadata into the AWS S3 service.
## Using Artifact Repository S3 Extension
The module contains a spring-boot autoconfiguration for easily integration into spring-boot projects.
For using this extension in the hawkbit-example-application you just need to add the maven dependency.
```
<dependency>
<groupId>org.eclipse.hawkbit</groupId>
<artifactId>hawkbit-extension-artifact-repository-s3</artifactId>
<version>${project.version}</version>
</dependency>
```
## Configuration of the S3 Extension
#### Bucket
All files are stored in a bucket configured via property `org.eclipse.hawkbit.repository.s3.bucketName` (see S3RepositoryProperties).
The name of the object stored in the S3 bucket is the SHA1-hash of the binary file.
#### S3 Credentials
The extension is using the `DefaultAWSCredentialsProviderChain` class which looks for credentials in this order:
1. Environment variables (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY)
2. Java system properties (aws.accessKeyId and aws.secretKey)
3. Default credential profile file (~/.aws/credentials)
4. Amazon ECS container credentials
5. Instance profile credentials
For more information check the [Amazon credentials guide](http://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html).
You can exchange the credentials provider by overwriting the `AWSCredentialsProvider` bean (see S3RepositoryAutoConfiguration).

78
extensions/hawkbit-extension-artifact-repository-s3/pom.xml Normal file
View File

@@ -0,0 +1,78 @@
<!--
Copyright (c) 2015 Bosch Software Innovations GmbH and others.
All rights reserved. This program and the accompanying materials
are made available under the terms of the Eclipse Public License v1.0
which accompanies this distribution, and is available at
http://www.eclipse.org/legal/epl-v10.html
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.eclipse.hawkbit</groupId>
<artifactId>hawkbit-extensions-parent</artifactId>
<version>0.2.0-SNAPSHOT</version>
</parent>
<artifactId>hawkbit-extension-artifact-repository-s3</artifactId>
<name>hawkBit :: S3 Repository</name>
<dependencies>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-core</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.hawkbit</groupId>
<artifactId>hawkbit-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-core</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
<dependency>
<groupId>org.easytesting</groupId>
<artifactId>fest-assert-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.easytesting</groupId>
<artifactId>fest-assert</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>ru.yandex.qatools.allure</groupId>
<artifactId>allure-junit-adaptor</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
</dependencies>
</project>

43
extensions/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java Normal file
View File

@@ -0,0 +1,43 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.artifact.repository;
import java.io.InputStream;
import org.eclipse.hawkbit.artifact.repository.model.DbArtifact;
import com.amazonaws.services.s3.AmazonS3;
/**
* An {@link DbArtifact} implementation which retrieves the {@link InputStream}
* from the {@link AmazonS3} client.
*/
public class S3Artifact extends DbArtifact {
private final AmazonS3 amazonS3;
private final S3RepositoryProperties s3Properties;
private final String sha1;
S3Artifact(final AmazonS3 amazonS3, final S3RepositoryProperties s3Properties, final String sha1) {
this.amazonS3 = amazonS3;
this.s3Properties = s3Properties;
this.sha1 = sha1;
}
@Override
public InputStream getFileInputStream() {
return amazonS3.getObject(s3Properties.getBucketName(), sha1).getObjectContent();
}
@Override
public String toString() {
return "S3Artifact [sha1=" + sha1 + ", getArtifactId()=" + getArtifactId() + ", getHashes()=" + getHashes()
+ ", getSize()=" + getSize() + ", getContentType()=" + getContentType() + "]";
}
}

221
extensions/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Repository.java Normal file
View File

@@ -0,0 +1,221 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.artifact.repository;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.DigestOutputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import org.eclipse.hawkbit.artifact.repository.model.DbArtifact;
import org.eclipse.hawkbit.artifact.repository.model.DbArtifactHash;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.amazonaws.AmazonClientException;
import com.amazonaws.RequestClientOptions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.Headers;
import com.amazonaws.services.s3.model.DeleteObjectRequest;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3Object;
import com.google.common.io.BaseEncoding;
import com.google.common.io.ByteStreams;
/**
* An {@link ArtifactRepository} implementation for the AWS S3 service. All
* binaries are stored in single bucket using the configured name
* {@link S3RepositoryProperties#getBucketName()}.
*
* From the AWS S3 documentation:
* <p>
* There is no limit to the number of objects that can be stored in a bucket and
* no difference in performance whether you use many buckets or just a few. You
* can store all of your objects in a single bucket, or you can organize them
* across several buckets.
* </p>
*/
public class S3Repository implements ArtifactRepository {
private static final Logger LOG = LoggerFactory.getLogger(S3Repository.class);
private static final String TEMP_FILE_PREFIX = "tmp";
private static final String TEMP_FILE_SUFFIX = "artifactrepo";
private final AmazonS3 amazonS3;
private final S3RepositoryProperties s3Properties;
/**
* Constructor.
*
* @param amazonS3
* the amazonS3 client to use
* @param s3Properties
* the properties which e.g. holds the name of the bucket to
* store in
*/
public S3Repository(final AmazonS3 amazonS3, final S3RepositoryProperties s3Properties) {
this.amazonS3 = amazonS3;
this.s3Properties = s3Properties;
}
@Override
public DbArtifact store(final InputStream content, final String filename, final String contentType) {
return store(content, filename, contentType, null);
}
@Override
// suppress warning, of not strong enough hashing algorithm, SHA-1 and MD5
// is not used security related
@SuppressWarnings("squid:S2070")
public DbArtifact store(final InputStream content, final String filename, final String contentType,
final DbArtifactHash hash) {
final MessageDigest mdSHA1;
final MessageDigest mdMD5;
try {
mdSHA1 = MessageDigest.getInstance("SHA1");
mdMD5 = MessageDigest.getInstance("MD5");
} catch (final NoSuchAlgorithmException e) {
throw new ArtifactStoreException(e.getMessage(), e);
}
LOG.debug("Creating temporary file to store the inputstream to it");
final File file = createTempFile();
LOG.debug("Calculating sha1 and md5 hashes");
try (final DigestOutputStream outputstream = openFileOutputStream(file, mdSHA1, mdMD5)) {
ByteStreams.copy(content, outputstream);
outputstream.flush();
final String sha1Hash16 = BaseEncoding.base16().lowerCase().encode(mdSHA1.digest());
final String md5Hash16 = BaseEncoding.base16().lowerCase().encode(mdMD5.digest());
return store(sha1Hash16, md5Hash16, contentType, file, hash);
} catch (final IOException e) {
throw new ArtifactStoreException(e.getMessage(), e);
} finally {
file.delete();
}
}
private DbArtifact store(final String sha1Hash16, final String mdMD5Hash16, final String contentType,
final File file, final DbArtifactHash hash) {
final S3Artifact s3Artifact = createS3Artifact(sha1Hash16, mdMD5Hash16, contentType, file);
checkHashes(s3Artifact, hash);
LOG.info("Storing file {} with length {} to AWS S3 bucket {} as SHA1 {}", file.getName(), file.length(),
s3Properties.getBucketName(), sha1Hash16);
if (exists(sha1Hash16)) {
LOG.debug("Artifact {} already exists on S3 bucket {}, don't need to upload twice", sha1Hash16,
s3Properties.getBucketName());
return s3Artifact;
}
try (final InputStream inputStream = new BufferedInputStream(new FileInputStream(file),
RequestClientOptions.DEFAULT_STREAM_BUFFER_SIZE)) {
final ObjectMetadata objectMetadata = createObjectMetadata(mdMD5Hash16, contentType, file);
amazonS3.putObject(s3Properties.getBucketName(), sha1Hash16, inputStream, objectMetadata);
return s3Artifact;
} catch (final IOException | AmazonClientException e) {
throw new ArtifactStoreException(e.getMessage(), e);
}
}
private S3Artifact createS3Artifact(final String sha1Hash16, final String mdMD5Hash16, final String contentType,
final File file) {
final S3Artifact s3Artifact = new S3Artifact(amazonS3, s3Properties, sha1Hash16);
s3Artifact.setContentType(contentType);
s3Artifact.setArtifactId(sha1Hash16);
s3Artifact.setSize(file.length());
s3Artifact.setContentType(contentType);
s3Artifact.setHashes(new DbArtifactHash(sha1Hash16, mdMD5Hash16));
return s3Artifact;
}
private ObjectMetadata createObjectMetadata(final String mdMD5Hash16, final String contentType, final File file) {
final ObjectMetadata objectMetadata = new ObjectMetadata();
final String mdMD5Hash64 = BaseEncoding.base64().encode(BaseEncoding.base16().lowerCase().decode(mdMD5Hash16));
objectMetadata.setContentMD5(mdMD5Hash64);
objectMetadata.setContentType(contentType);
objectMetadata.setContentLength(file.length());
objectMetadata.setHeader("x-amz-meta-md5chksum", mdMD5Hash64);
if (s3Properties.isServerSideEncryption()) {
objectMetadata.setHeader(Headers.SERVER_SIDE_ENCRYPTION, s3Properties.getServerSideEncryptionAlgorithm());
}
return objectMetadata;
}
@Override
public void deleteBySha1(final String sha1Hash) {
LOG.info("Deleting S3 object from bucket {} and key {}", s3Properties.getBucketName(), sha1Hash);
amazonS3.deleteObject(new DeleteObjectRequest(s3Properties.getBucketName(), sha1Hash));
}
@Override
public DbArtifact getArtifactBySha1(final String sha1) {
LOG.info("Retrieving S3 object from bucket {} and key {}", s3Properties.getBucketName(), sha1);
final S3Object s3Object = amazonS3.getObject(s3Properties.getBucketName(), sha1);
if (s3Object == null) {
return null;
}
final ObjectMetadata s3ObjectMetadata = s3Object.getObjectMetadata();
final S3Artifact s3Artifact = new S3Artifact(amazonS3, s3Properties, sha1);
s3Artifact.setArtifactId(sha1);
s3Artifact.setSize(s3ObjectMetadata.getContentLength());
// the MD5Content is stored in the ETag
s3Artifact.setHashes(new DbArtifactHash(sha1,
BaseEncoding.base16().lowerCase().encode(BaseEncoding.base64().decode(s3ObjectMetadata.getETag()))));
s3Artifact.setContentType(s3ObjectMetadata.getContentType());
return s3Artifact;
}
private static void checkHashes(final DbArtifact artifact, final DbArtifactHash hash) {
if (hash == null) {
return;
}
if (hash.getSha1() != null && !artifact.getHashes().getSha1().equals(hash.getSha1())) {
throw new HashNotMatchException("The given sha1 hash " + hash.getSha1()
+ " does not match with the calcualted sha1 hash " + artifact.getHashes().getSha1(),
HashNotMatchException.SHA1);
}
if (hash.getMd5() != null && !artifact.getHashes().getMd5().equals(hash.getMd5())) {
throw new HashNotMatchException("The given md5 hash " + hash.getMd5()
+ " does not match with the calcualted md5 hash " + artifact.getHashes().getMd5(),
HashNotMatchException.MD5);
}
}
private static File createTempFile() {
try {
return File.createTempFile(TEMP_FILE_PREFIX, TEMP_FILE_SUFFIX);
} catch (final IOException e) {
throw new ArtifactStoreException("Cannot create tempfile", e);
}
}
private static DigestOutputStream openFileOutputStream(final File file, final MessageDigest mdSHA1,
final MessageDigest mdMD5) throws FileNotFoundException {
return new DigestOutputStream(
new DigestOutputStream(new BufferedOutputStream(new FileOutputStream(file)), mdMD5), mdSHA1);
}
private boolean exists(final String sha1) {
return amazonS3.doesObjectExist(s3Properties.getBucketName(), sha1);
}
}

83
extensions/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryAutoConfiguration.java Normal file
View File

@@ -0,0 +1,83 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.artifact.repository;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
/**
* The Spring auto-configuration to register the necessary beans for the S3
* artifact repository implementation.
*/
@Configuration
@ConditionalOnProperty(prefix = "org.eclipse.hawkbit.artifact.repository.s3", name = "enabled", matchIfMissing = true)
@EnableConfigurationProperties(S3RepositoryProperties.class)
public class S3RepositoryAutoConfiguration {
/**
* The {@link DefaultAWSCredentialsProviderChain} looks for credentials in
* this order:
*
* <pre>
* 1. Environment Variables (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY)
* 2. Java System Properties (aws.accessKeyId and aws.secretKey)
* 3. The default credential profiles file (~/.aws/credentials)
* 4. Amazon ECS container credentials
* 5. Instance profile credentials
* </pre>
*
* @return the {@link DefaultAWSCredentialsProviderChain} if no other
* {@link AWSCredentialsProvider} bean is registered.
*/
@Bean
@ConditionalOnMissingBean
public AWSCredentialsProvider awsCredentialsProvider() {
return new DefaultAWSCredentialsProviderChain();
}
/**
* The default AmazonS3 client configuration, which declares the
* configuration for managing connection behavior to s3.
*
* @return the default {@link ClientConfiguration} bean with the default
* client configuration
*/
@Bean
@ConditionalOnMissingBean
public ClientConfiguration awsClientConfiguration() {
return new ClientConfiguration();
}
/**
* @return the {@link AmazonS3Client} if no other {@link AmazonS3} bean is
* registered.
*/
@Bean
@ConditionalOnMissingBean
public AmazonS3 amazonS3() {
return new AmazonS3Client(awsCredentialsProvider(), awsClientConfiguration());
}
/**
* @return AWS S3 repository {@link ArtifactRepository} implementation.
*/
@Bean
public ArtifactRepository artifactRepository(final S3RepositoryProperties s3Properties) {
return new S3Repository(amazonS3(), s3Properties);
}
}

49
extensions/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryProperties.java Normal file
View File

@@ -0,0 +1,49 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.artifact.repository;
import org.springframework.boot.context.properties.ConfigurationProperties;
import com.amazonaws.services.s3.model.SSEAlgorithm;
/**
* The AWS S3 configuration properties for the S3 artifact repository
* implementation.
*/
@ConfigurationProperties("org.eclipse.hawkbit.repository.s3")
public class S3RepositoryProperties {
private String bucketName = "artifactrepository";
private boolean serverSideEncryption = false;
private String serverSideEncryptionAlgorithm = SSEAlgorithm.AES256.getAlgorithm();
public String getBucketName() {
return bucketName;
}
public void setBucketName(final String bucketName) {
this.bucketName = bucketName;
}
public boolean isServerSideEncryption() {
return serverSideEncryption;
}
public void setServerSideEncryption(final boolean serverSideEncryption) {
this.serverSideEncryption = serverSideEncryption;
}
public String getServerSideEncryptionAlgorithm() {
return serverSideEncryptionAlgorithm;
}
public void setServerSideEncryptionAlgorithm(final String serverSideEncryptionAlgorithm) {
this.serverSideEncryptionAlgorithm = serverSideEncryptionAlgorithm;
}
}

3
extensions/hawkbit-extension-artifact-repository-s3/src/main/resources/META-INF/spring.factories Normal file
View File

@@ -0,0 +1,3 @@
# Auto Configure
org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
org.eclipse.hawkbit.artifact.repository.S3RepositoryAutoConfiguration

224
extensions/hawkbit-extension-artifact-repository-s3/src/test/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryTest.java Normal file
View File

@@ -0,0 +1,224 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.artifact.repository;
import static org.fest.assertions.Assertions.assertThat;
import static org.fest.assertions.api.Assertions.fail;
import static org.mockito.Matchers.anyString;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.when;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.security.DigestOutputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Random;
import org.eclipse.hawkbit.artifact.repository.model.DbArtifact;
import org.eclipse.hawkbit.artifact.repository.model.DbArtifactHash;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.runners.MockitoJUnitRunner;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3Object;
import com.google.common.io.BaseEncoding;
import com.google.common.io.ByteStreams;
import ru.yandex.qatools.allure.annotations.Description;
import ru.yandex.qatools.allure.annotations.Features;
import ru.yandex.qatools.allure.annotations.Stories;
/**
* Test class for the {@link S3Repository}.
*/
@RunWith(MockitoJUnitRunner.class)
@Features("Unit Tests - S3 Repository")
@Stories("S3 Artifact Repository")
public class S3RepositoryTest {
@Mock
private AmazonS3 amazonS3Mock;
@Mock
private S3Object s3ObjectMock;
@Mock
private ObjectMetadata s3ObjectMetadataMock;
@Captor
private ArgumentCaptor<ObjectMetadata> objectMetaDataCaptor;
@Captor
private ArgumentCaptor<InputStream> inputStreamCaptor;
private final S3RepositoryProperties s3Properties = new S3RepositoryProperties();
private S3Repository s3RepositoryUnderTest;
@Before
public void before() {
s3RepositoryUnderTest = new S3Repository(amazonS3Mock, s3Properties);
}
@Test
@Description("Verifies that the amazonS3 client is called to put the object to S3 with the correct inputstream and meta-data")
public void storeInputStreamCallAmazonS3Client() throws IOException, NoSuchAlgorithmException {
final byte[] rndBytes = randomBytes();
final String knownSHA1 = getSha1OfBytes(rndBytes);
final String knownContentType = "application/octet-stream";
// test
storeRandomBytes(rndBytes, knownContentType);
// verify
Mockito.verify(amazonS3Mock).putObject(eq(s3Properties.getBucketName()), eq(knownSHA1),
inputStreamCaptor.capture(), objectMetaDataCaptor.capture());
final ObjectMetadata recordedObjectMetadata = objectMetaDataCaptor.getValue();
assertThat(recordedObjectMetadata.getContentType()).isEqualTo(knownContentType);
assertThat(recordedObjectMetadata.getContentMD5()).isNotNull();
assertThat(recordedObjectMetadata.getContentLength()).isEqualTo(rndBytes.length);
}
@Test
@Description("Verifies that the amazonS3 client is called to retrieve the correct artifact from S3 and the mapping to the DBArtifact is correct")
public void getArtifactBySHA1Hash() {
final String knownSHA1Hash = "da39a3ee5e6b4b0d3255bfef95601890afd80709";
final long knownContentLength = 100;
final String knownContentType = "application/octet-stream";
final String knownMd5 = "098f6bcd4621d373cade4e832627b4f6";
final String knownMdBase16 = BaseEncoding.base16().lowerCase().encode(knownMd5.getBytes());
final String knownMd5Base64 = BaseEncoding.base64().encode(knownMd5.getBytes());
when(amazonS3Mock.getObject(anyString(), anyString())).thenReturn(s3ObjectMock);
when(s3ObjectMock.getObjectMetadata()).thenReturn(s3ObjectMetadataMock);
when(s3ObjectMetadataMock.getContentLength()).thenReturn(knownContentLength);
when(s3ObjectMetadataMock.getETag()).thenReturn(knownMd5Base64);
when(s3ObjectMetadataMock.getContentType()).thenReturn(knownContentType);
// test
final DbArtifact artifactBySha1 = s3RepositoryUnderTest.getArtifactBySha1(knownSHA1Hash);
// verify
assertThat(artifactBySha1.getArtifactId()).isEqualTo(knownSHA1Hash);
assertThat(artifactBySha1.getContentType()).isEqualTo(knownContentType);
assertThat(artifactBySha1.getSize()).isEqualTo(knownContentLength);
assertThat(artifactBySha1.getHashes().getSha1()).isEqualTo(knownSHA1Hash);
assertThat(artifactBySha1.getHashes().getMd5()).isEqualTo(knownMdBase16);
}
@Test
@Description("Verifies that the amazonS3 client is not called to put the object to S3 due the artifact already exists on S3")
public void artifactIsNotUploadedIfAlreadyExists() throws NoSuchAlgorithmException, IOException {
final byte[] rndBytes = randomBytes();
final String knownSHA1 = getSha1OfBytes(rndBytes);
final String knownContentType = "application/octet-stream";
when(amazonS3Mock.doesObjectExist(s3Properties.getBucketName(), knownSHA1)).thenReturn(true);
// test
storeRandomBytes(rndBytes, knownContentType);
// verify
Mockito.verify(amazonS3Mock, never()).putObject(eq(s3Properties.getBucketName()), eq(knownSHA1),
inputStreamCaptor.capture(), objectMetaDataCaptor.capture());
}
@Test
@Description("Verifies that null is returned if the given hash does not exists on S3")
public void getArtifactBySha1ReturnsNullIfFileDoesNotExists() {
final String knownSHA1Hash = "0815";
when(amazonS3Mock.getObject(s3Properties.getBucketName(), knownSHA1Hash)).thenReturn(null);
// test
final DbArtifact artifactBySha1NotExists = s3RepositoryUnderTest.getArtifactBySha1(knownSHA1Hash);
// verify
assertThat(artifactBySha1NotExists).isNull();
}
@Test
@Description("Verifies that given SHA1 hash are checked and if not match will throw exception")
public void sha1HashValuesAreNotTheSameThrowsException() throws IOException {
final byte[] rndBytes = randomBytes();
final String knownContentType = "application/octet-stream";
final String wrongSHA1Hash = "wrong";
final String wrongMD5 = "wrong";
// test
try {
storeRandomBytes(rndBytes, knownContentType, new DbArtifactHash(wrongSHA1Hash, wrongMD5));
fail("Expected an HashNotMatchException, but didn't throw");
} catch (final HashNotMatchException e) {
assertThat(e.getHashFunction()).isEqualTo(HashNotMatchException.SHA1);
}
}
@Test
@Description("Verifies that given MD5 hash are checked and if not match will throw exception")
public void md5HashValuesAreNotTheSameThrowsException() throws IOException, NoSuchAlgorithmException {
final byte[] rndBytes = randomBytes();
final String knownContentType = "application/octet-stream";
final String knownSHA1 = getSha1OfBytes(rndBytes);
final String wrongMD5 = "wrong";
// test
try {
storeRandomBytes(rndBytes, knownContentType, new DbArtifactHash(knownSHA1, wrongMD5));
fail("Expected an HashNotMatchException, but didn't throw");
} catch (final HashNotMatchException e) {
assertThat(e.getHashFunction()).isEqualTo(HashNotMatchException.MD5);
}
}
private void storeRandomBytes(final byte[] rndBytes, final String contentType)
throws IOException, NoSuchAlgorithmException {
storeRandomBytes(rndBytes, contentType, null);
}
private void storeRandomBytes(final byte[] rndBytes, final String contentType, final DbArtifactHash hashes)
throws IOException {
final String knownFileName = "randomBytes";
try (InputStream content = new BufferedInputStream(new ByteArrayInputStream(rndBytes))) {
s3RepositoryUnderTest.store(content, knownFileName, contentType, hashes);
}
}
private static String getSha1OfBytes(final byte[] bytes) throws IOException, NoSuchAlgorithmException {
final MessageDigest messageDigest = MessageDigest.getInstance("SHA1");
try (InputStream input = new ByteArrayInputStream(bytes);
OutputStream output = new DigestOutputStream(new ByteArrayOutputStream(), messageDigest)) {
ByteStreams.copy(input, output);
return BaseEncoding.base16().lowerCase().encode(messageDigest.digest());
}
}
private static byte[] randomBytes() {
final byte[] randomBytes = new byte[20];
final Random ran = new Random();
ran.nextBytes(randomBytes);
return randomBytes;
}
}

1
extensions/pom.xml
View File

@@ -23,6 +23,7 @@
<modules>
<module>hawkbit-extension-uaa</module>
<module>hawkbit-extension-artifact-repository-mongo</module>
<module>hawkbit-extension-artifact-repository-s3</module>
</modules>
</project>