From edbadc4002efced42b699acdbf659299fca41296 Mon Sep 17 00:00:00 2001 From: Michael Hirsch Date: Wed, 7 Dec 2016 09:52:43 +0100 Subject: [PATCH] Feature s3 repository extension (#366) Signed-off-by: Michael Hirsch --- ...MongoDBArtifactStoreAutoConfiguration.java | 3 +- .../README.md | 33 +++ .../pom.xml | 78 ++++++ .../artifact/repository/S3Artifact.java | 43 ++++ .../artifact/repository/S3Repository.java | 221 +++++++++++++++++ .../S3RepositoryAutoConfiguration.java | 83 +++++++ .../repository/S3RepositoryProperties.java | 49 ++++ .../main/resources/META-INF/spring.factories | 3 + .../artifact/repository/S3RepositoryTest.java | 224 ++++++++++++++++++ extensions/pom.xml | 1 + 10 files changed, 737 insertions(+), 1 deletion(-) create mode 100644 extensions/hawkbit-extension-artifact-repository-s3/README.md create mode 100644 extensions/hawkbit-extension-artifact-repository-s3/pom.xml create mode 100644 extensions/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java create mode 100644 extensions/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Repository.java create mode 100644 extensions/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryAutoConfiguration.java create mode 100644 extensions/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryProperties.java create mode 100644 extensions/hawkbit-extension-artifact-repository-s3/src/main/resources/META-INF/spring.factories create mode 100644 extensions/hawkbit-extension-artifact-repository-s3/src/test/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryTest.java diff --git a/extensions/hawkbit-extension-artifact-repository-mongo/src/main/java/org/eclipse/hawkbit/artifact/repository/MongoDBArtifactStoreAutoConfiguration.java b/extensions/hawkbit-extension-artifact-repository-mongo/src/main/java/org/eclipse/hawkbit/artifact/repository/MongoDBArtifactStoreAutoConfiguration.java index 9014ca1c8..b204f34a1 100644 --- a/extensions/hawkbit-extension-artifact-repository-mongo/src/main/java/org/eclipse/hawkbit/artifact/repository/MongoDBArtifactStoreAutoConfiguration.java +++ b/extensions/hawkbit-extension-artifact-repository-mongo/src/main/java/org/eclipse/hawkbit/artifact/repository/MongoDBArtifactStoreAutoConfiguration.java @@ -8,7 +8,7 @@ */ package org.eclipse.hawkbit.artifact.repository; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -16,6 +16,7 @@ import org.springframework.context.annotation.Configuration; * Auto configuration for the {@link MongoDBArtifactStore}. */ @Configuration +@ConditionalOnProperty(prefix = "org.eclipse.hawkbit.artifact.repository.mongo", name = "enabled", matchIfMissing = true) public class MongoDBArtifactStoreAutoConfiguration { /** diff --git a/extensions/hawkbit-extension-artifact-repository-s3/README.md b/extensions/hawkbit-extension-artifact-repository-s3/README.md new file mode 100644 index 000000000..ee4e20cad --- /dev/null +++ b/extensions/hawkbit-extension-artifact-repository-s3/README.md @@ -0,0 +1,33 @@ +# Eclipse hawkBit - Artifact Repository AWS S3 +HawkBit Artifact Repository is a library for storing binary artifacts and metadata into the AWS S3 service. + + +## Using Artifact Repository S3 Extension +The module contains a spring-boot autoconfiguration for easily integration into spring-boot projects. +For using this extension in the hawkbit-example-application you just need to add the maven dependency. +``` + + org.eclipse.hawkbit + hawkbit-extension-artifact-repository-s3 + ${project.version} + +``` + +## Configuration of the S3 Extension + +#### Bucket +All files are stored in a bucket configured via property `org.eclipse.hawkbit.repository.s3.bucketName` (see S3RepositoryProperties). +The name of the object stored in the S3 bucket is the SHA1-hash of the binary file. + +#### S3 Credentials +The extension is using the `DefaultAWSCredentialsProviderChain` class which looks for credentials in this order: + +1. Environment variables (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY) +2. Java system properties (aws.accessKeyId and aws.secretKey) +3. Default credential profile file (~/.aws/credentials) +4. Amazon ECS container credentials +5. Instance profile credentials + +For more information check the [Amazon credentials guide](http://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html). + +You can exchange the credentials provider by overwriting the `AWSCredentialsProvider` bean (see S3RepositoryAutoConfiguration). diff --git a/extensions/hawkbit-extension-artifact-repository-s3/pom.xml b/extensions/hawkbit-extension-artifact-repository-s3/pom.xml new file mode 100644 index 000000000..7fae4f098 --- /dev/null +++ b/extensions/hawkbit-extension-artifact-repository-s3/pom.xml @@ -0,0 +1,78 @@ + + + 4.0.0 + + org.eclipse.hawkbit + hawkbit-extensions-parent + 0.2.0-SNAPSHOT + + hawkbit-extension-artifact-repository-s3 + hawkBit :: S3 Repository + + + + + com.amazonaws + aws-java-sdk-s3 + + + com.amazonaws + aws-java-sdk-core + + + org.eclipse.hawkbit + hawkbit-core + ${project.version} + + + com.google.guava + guava + + + org.springframework + spring-core + + + org.springframework.boot + spring-boot-autoconfigure + + + commons-io + commons-io + + + org.easytesting + fest-assert-core + test + + + org.easytesting + fest-assert + test + + + org.mockito + mockito-core + test + + + ru.yandex.qatools.allure + allure-junit-adaptor + test + + + org.springframework.boot + spring-boot-configuration-processor + true + + + \ No newline at end of file diff --git a/extensions/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java b/extensions/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java new file mode 100644 index 000000000..30603722b --- /dev/null +++ b/extensions/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java @@ -0,0 +1,43 @@ +/** + * Copyright (c) 2015 Bosch Software Innovations GmbH and others. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + */ +package org.eclipse.hawkbit.artifact.repository; + +import java.io.InputStream; + +import org.eclipse.hawkbit.artifact.repository.model.DbArtifact; + +import com.amazonaws.services.s3.AmazonS3; + +/** + * An {@link DbArtifact} implementation which retrieves the {@link InputStream} + * from the {@link AmazonS3} client. + */ +public class S3Artifact extends DbArtifact { + + private final AmazonS3 amazonS3; + private final S3RepositoryProperties s3Properties; + private final String sha1; + + S3Artifact(final AmazonS3 amazonS3, final S3RepositoryProperties s3Properties, final String sha1) { + this.amazonS3 = amazonS3; + this.s3Properties = s3Properties; + this.sha1 = sha1; + } + + @Override + public InputStream getFileInputStream() { + return amazonS3.getObject(s3Properties.getBucketName(), sha1).getObjectContent(); + } + + @Override + public String toString() { + return "S3Artifact [sha1=" + sha1 + ", getArtifactId()=" + getArtifactId() + ", getHashes()=" + getHashes() + + ", getSize()=" + getSize() + ", getContentType()=" + getContentType() + "]"; + } +} diff --git a/extensions/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Repository.java b/extensions/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Repository.java new file mode 100644 index 000000000..59c5583c1 --- /dev/null +++ b/extensions/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Repository.java @@ -0,0 +1,221 @@ +/** + * Copyright (c) 2015 Bosch Software Innovations GmbH and others. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + */ +package org.eclipse.hawkbit.artifact.repository; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.DigestOutputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +import org.eclipse.hawkbit.artifact.repository.model.DbArtifact; +import org.eclipse.hawkbit.artifact.repository.model.DbArtifactHash; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.amazonaws.AmazonClientException; +import com.amazonaws.RequestClientOptions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.Headers; +import com.amazonaws.services.s3.model.DeleteObjectRequest; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.S3Object; +import com.google.common.io.BaseEncoding; +import com.google.common.io.ByteStreams; + +/** + * An {@link ArtifactRepository} implementation for the AWS S3 service. All + * binaries are stored in single bucket using the configured name + * {@link S3RepositoryProperties#getBucketName()}. + * + * From the AWS S3 documentation: + *

+ * There is no limit to the number of objects that can be stored in a bucket and + * no difference in performance whether you use many buckets or just a few. You + * can store all of your objects in a single bucket, or you can organize them + * across several buckets. + *

+ */ +public class S3Repository implements ArtifactRepository { + + private static final Logger LOG = LoggerFactory.getLogger(S3Repository.class); + + private static final String TEMP_FILE_PREFIX = "tmp"; + private static final String TEMP_FILE_SUFFIX = "artifactrepo"; + + private final AmazonS3 amazonS3; + private final S3RepositoryProperties s3Properties; + + /** + * Constructor. + * + * @param amazonS3 + * the amazonS3 client to use + * @param s3Properties + * the properties which e.g. holds the name of the bucket to + * store in + */ + public S3Repository(final AmazonS3 amazonS3, final S3RepositoryProperties s3Properties) { + this.amazonS3 = amazonS3; + this.s3Properties = s3Properties; + } + + @Override + public DbArtifact store(final InputStream content, final String filename, final String contentType) { + return store(content, filename, contentType, null); + } + + @Override + // suppress warning, of not strong enough hashing algorithm, SHA-1 and MD5 + // is not used security related + @SuppressWarnings("squid:S2070") + public DbArtifact store(final InputStream content, final String filename, final String contentType, + final DbArtifactHash hash) { + final MessageDigest mdSHA1; + final MessageDigest mdMD5; + try { + mdSHA1 = MessageDigest.getInstance("SHA1"); + mdMD5 = MessageDigest.getInstance("MD5"); + } catch (final NoSuchAlgorithmException e) { + throw new ArtifactStoreException(e.getMessage(), e); + } + + LOG.debug("Creating temporary file to store the inputstream to it"); + + final File file = createTempFile(); + LOG.debug("Calculating sha1 and md5 hashes"); + try (final DigestOutputStream outputstream = openFileOutputStream(file, mdSHA1, mdMD5)) { + ByteStreams.copy(content, outputstream); + outputstream.flush(); + final String sha1Hash16 = BaseEncoding.base16().lowerCase().encode(mdSHA1.digest()); + final String md5Hash16 = BaseEncoding.base16().lowerCase().encode(mdMD5.digest()); + + return store(sha1Hash16, md5Hash16, contentType, file, hash); + } catch (final IOException e) { + throw new ArtifactStoreException(e.getMessage(), e); + } finally { + file.delete(); + } + } + + private DbArtifact store(final String sha1Hash16, final String mdMD5Hash16, final String contentType, + final File file, final DbArtifactHash hash) { + final S3Artifact s3Artifact = createS3Artifact(sha1Hash16, mdMD5Hash16, contentType, file); + checkHashes(s3Artifact, hash); + + LOG.info("Storing file {} with length {} to AWS S3 bucket {} as SHA1 {}", file.getName(), file.length(), + s3Properties.getBucketName(), sha1Hash16); + + if (exists(sha1Hash16)) { + LOG.debug("Artifact {} already exists on S3 bucket {}, don't need to upload twice", sha1Hash16, + s3Properties.getBucketName()); + return s3Artifact; + } + + try (final InputStream inputStream = new BufferedInputStream(new FileInputStream(file), + RequestClientOptions.DEFAULT_STREAM_BUFFER_SIZE)) { + final ObjectMetadata objectMetadata = createObjectMetadata(mdMD5Hash16, contentType, file); + amazonS3.putObject(s3Properties.getBucketName(), sha1Hash16, inputStream, objectMetadata); + + return s3Artifact; + } catch (final IOException | AmazonClientException e) { + throw new ArtifactStoreException(e.getMessage(), e); + } + } + + private S3Artifact createS3Artifact(final String sha1Hash16, final String mdMD5Hash16, final String contentType, + final File file) { + final S3Artifact s3Artifact = new S3Artifact(amazonS3, s3Properties, sha1Hash16); + s3Artifact.setContentType(contentType); + s3Artifact.setArtifactId(sha1Hash16); + s3Artifact.setSize(file.length()); + s3Artifact.setContentType(contentType); + s3Artifact.setHashes(new DbArtifactHash(sha1Hash16, mdMD5Hash16)); + return s3Artifact; + } + + private ObjectMetadata createObjectMetadata(final String mdMD5Hash16, final String contentType, final File file) { + final ObjectMetadata objectMetadata = new ObjectMetadata(); + final String mdMD5Hash64 = BaseEncoding.base64().encode(BaseEncoding.base16().lowerCase().decode(mdMD5Hash16)); + objectMetadata.setContentMD5(mdMD5Hash64); + objectMetadata.setContentType(contentType); + objectMetadata.setContentLength(file.length()); + objectMetadata.setHeader("x-amz-meta-md5chksum", mdMD5Hash64); + if (s3Properties.isServerSideEncryption()) { + objectMetadata.setHeader(Headers.SERVER_SIDE_ENCRYPTION, s3Properties.getServerSideEncryptionAlgorithm()); + } + return objectMetadata; + } + + @Override + public void deleteBySha1(final String sha1Hash) { + LOG.info("Deleting S3 object from bucket {} and key {}", s3Properties.getBucketName(), sha1Hash); + amazonS3.deleteObject(new DeleteObjectRequest(s3Properties.getBucketName(), sha1Hash)); + } + + @Override + public DbArtifact getArtifactBySha1(final String sha1) { + LOG.info("Retrieving S3 object from bucket {} and key {}", s3Properties.getBucketName(), sha1); + final S3Object s3Object = amazonS3.getObject(s3Properties.getBucketName(), sha1); + if (s3Object == null) { + return null; + } + + final ObjectMetadata s3ObjectMetadata = s3Object.getObjectMetadata(); + + final S3Artifact s3Artifact = new S3Artifact(amazonS3, s3Properties, sha1); + s3Artifact.setArtifactId(sha1); + s3Artifact.setSize(s3ObjectMetadata.getContentLength()); + // the MD5Content is stored in the ETag + s3Artifact.setHashes(new DbArtifactHash(sha1, + BaseEncoding.base16().lowerCase().encode(BaseEncoding.base64().decode(s3ObjectMetadata.getETag())))); + s3Artifact.setContentType(s3ObjectMetadata.getContentType()); + return s3Artifact; + } + + private static void checkHashes(final DbArtifact artifact, final DbArtifactHash hash) { + if (hash == null) { + return; + } + if (hash.getSha1() != null && !artifact.getHashes().getSha1().equals(hash.getSha1())) { + throw new HashNotMatchException("The given sha1 hash " + hash.getSha1() + + " does not match with the calcualted sha1 hash " + artifact.getHashes().getSha1(), + HashNotMatchException.SHA1); + } + if (hash.getMd5() != null && !artifact.getHashes().getMd5().equals(hash.getMd5())) { + throw new HashNotMatchException("The given md5 hash " + hash.getMd5() + + " does not match with the calcualted md5 hash " + artifact.getHashes().getMd5(), + HashNotMatchException.MD5); + } + } + + private static File createTempFile() { + try { + return File.createTempFile(TEMP_FILE_PREFIX, TEMP_FILE_SUFFIX); + } catch (final IOException e) { + throw new ArtifactStoreException("Cannot create tempfile", e); + } + } + + private static DigestOutputStream openFileOutputStream(final File file, final MessageDigest mdSHA1, + final MessageDigest mdMD5) throws FileNotFoundException { + return new DigestOutputStream( + new DigestOutputStream(new BufferedOutputStream(new FileOutputStream(file)), mdMD5), mdSHA1); + } + + private boolean exists(final String sha1) { + return amazonS3.doesObjectExist(s3Properties.getBucketName(), sha1); + } +} diff --git a/extensions/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryAutoConfiguration.java b/extensions/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryAutoConfiguration.java new file mode 100644 index 000000000..a4296c2ba --- /dev/null +++ b/extensions/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryAutoConfiguration.java @@ -0,0 +1,83 @@ +/** + * Copyright (c) 2015 Bosch Software Innovations GmbH and others. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + */ +package org.eclipse.hawkbit.artifact.repository; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; + +/** + * The Spring auto-configuration to register the necessary beans for the S3 + * artifact repository implementation. + */ +@Configuration +@ConditionalOnProperty(prefix = "org.eclipse.hawkbit.artifact.repository.s3", name = "enabled", matchIfMissing = true) +@EnableConfigurationProperties(S3RepositoryProperties.class) +public class S3RepositoryAutoConfiguration { + + /** + * The {@link DefaultAWSCredentialsProviderChain} looks for credentials in + * this order: + * + * 
+     * 1. Environment Variables (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY)
+     * 2. Java System Properties (aws.accessKeyId and aws.secretKey)
+     * 3. The default credential profiles file (~/.aws/credentials)
+     * 4. Amazon ECS container credentials
+     * 5. Instance profile credentials
+     *
+ * + * @return the {@link DefaultAWSCredentialsProviderChain} if no other + * {@link AWSCredentialsProvider} bean is registered. + */ + @Bean + @ConditionalOnMissingBean + public AWSCredentialsProvider awsCredentialsProvider() { + return new DefaultAWSCredentialsProviderChain(); + } + + /** + * The default AmazonS3 client configuration, which declares the + * configuration for managing connection behavior to s3. + * + * @return the default {@link ClientConfiguration} bean with the default + * client configuration + */ + @Bean + @ConditionalOnMissingBean + public ClientConfiguration awsClientConfiguration() { + return new ClientConfiguration(); + } + + /** + * @return the {@link AmazonS3Client} if no other {@link AmazonS3} bean is + * registered. + */ + @Bean + @ConditionalOnMissingBean + public AmazonS3 amazonS3() { + return new AmazonS3Client(awsCredentialsProvider(), awsClientConfiguration()); + } + + /** + * @return AWS S3 repository {@link ArtifactRepository} implementation. + */ + @Bean + public ArtifactRepository artifactRepository(final S3RepositoryProperties s3Properties) { + return new S3Repository(amazonS3(), s3Properties); + } +} diff --git a/extensions/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryProperties.java b/extensions/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryProperties.java new file mode 100644 index 000000000..df6bbd880 --- /dev/null +++ b/extensions/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryProperties.java @@ -0,0 +1,49 @@ +/** + * Copyright (c) 2015 Bosch Software Innovations GmbH and others. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + */ +package org.eclipse.hawkbit.artifact.repository; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +import com.amazonaws.services.s3.model.SSEAlgorithm; + +/** + * The AWS S3 configuration properties for the S3 artifact repository + * implementation. + */ +@ConfigurationProperties("org.eclipse.hawkbit.repository.s3") +public class S3RepositoryProperties { + + private String bucketName = "artifactrepository"; + private boolean serverSideEncryption = false; + private String serverSideEncryptionAlgorithm = SSEAlgorithm.AES256.getAlgorithm(); + + public String getBucketName() { + return bucketName; + } + + public void setBucketName(final String bucketName) { + this.bucketName = bucketName; + } + + public boolean isServerSideEncryption() { + return serverSideEncryption; + } + + public void setServerSideEncryption(final boolean serverSideEncryption) { + this.serverSideEncryption = serverSideEncryption; + } + + public String getServerSideEncryptionAlgorithm() { + return serverSideEncryptionAlgorithm; + } + + public void setServerSideEncryptionAlgorithm(final String serverSideEncryptionAlgorithm) { + this.serverSideEncryptionAlgorithm = serverSideEncryptionAlgorithm; + } +} diff --git a/extensions/hawkbit-extension-artifact-repository-s3/src/main/resources/META-INF/spring.factories b/extensions/hawkbit-extension-artifact-repository-s3/src/main/resources/META-INF/spring.factories new file mode 100644 index 000000000..3736f86fd --- /dev/null +++ b/extensions/hawkbit-extension-artifact-repository-s3/src/main/resources/META-INF/spring.factories @@ -0,0 +1,3 @@ +# Auto Configure +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +org.eclipse.hawkbit.artifact.repository.S3RepositoryAutoConfiguration diff --git a/extensions/hawkbit-extension-artifact-repository-s3/src/test/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryTest.java b/extensions/hawkbit-extension-artifact-repository-s3/src/test/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryTest.java new file mode 100644 index 000000000..20f2976bf --- /dev/null +++ b/extensions/hawkbit-extension-artifact-repository-s3/src/test/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryTest.java @@ -0,0 +1,224 @@ +/** + * Copyright (c) 2015 Bosch Software Innovations GmbH and others. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + */ +package org.eclipse.hawkbit.artifact.repository; + +import static org.fest.assertions.Assertions.assertThat; +import static org.fest.assertions.api.Assertions.fail; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.when; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.security.DigestOutputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Random; + +import org.eclipse.hawkbit.artifact.repository.model.DbArtifact; +import org.eclipse.hawkbit.artifact.repository.model.DbArtifactHash; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.S3Object; +import com.google.common.io.BaseEncoding; +import com.google.common.io.ByteStreams; + +import ru.yandex.qatools.allure.annotations.Description; +import ru.yandex.qatools.allure.annotations.Features; +import ru.yandex.qatools.allure.annotations.Stories; + +/** + * Test class for the {@link S3Repository}. + */ +@RunWith(MockitoJUnitRunner.class) +@Features("Unit Tests - S3 Repository") +@Stories("S3 Artifact Repository") +public class S3RepositoryTest { + + @Mock + private AmazonS3 amazonS3Mock; + + @Mock + private S3Object s3ObjectMock; + + @Mock + private ObjectMetadata s3ObjectMetadataMock; + + @Captor + private ArgumentCaptor objectMetaDataCaptor; + + @Captor + private ArgumentCaptor inputStreamCaptor; + + private final S3RepositoryProperties s3Properties = new S3RepositoryProperties(); + private S3Repository s3RepositoryUnderTest; + + @Before + public void before() { + s3RepositoryUnderTest = new S3Repository(amazonS3Mock, s3Properties); + } + + @Test + @Description("Verifies that the amazonS3 client is called to put the object to S3 with the correct inputstream and meta-data") + public void storeInputStreamCallAmazonS3Client() throws IOException, NoSuchAlgorithmException { + final byte[] rndBytes = randomBytes(); + final String knownSHA1 = getSha1OfBytes(rndBytes); + final String knownContentType = "application/octet-stream"; + + // test + storeRandomBytes(rndBytes, knownContentType); + + // verify + Mockito.verify(amazonS3Mock).putObject(eq(s3Properties.getBucketName()), eq(knownSHA1), + inputStreamCaptor.capture(), objectMetaDataCaptor.capture()); + + final ObjectMetadata recordedObjectMetadata = objectMetaDataCaptor.getValue(); + assertThat(recordedObjectMetadata.getContentType()).isEqualTo(knownContentType); + assertThat(recordedObjectMetadata.getContentMD5()).isNotNull(); + assertThat(recordedObjectMetadata.getContentLength()).isEqualTo(rndBytes.length); + } + + @Test + @Description("Verifies that the amazonS3 client is called to retrieve the correct artifact from S3 and the mapping to the DBArtifact is correct") + public void getArtifactBySHA1Hash() { + final String knownSHA1Hash = "da39a3ee5e6b4b0d3255bfef95601890afd80709"; + final long knownContentLength = 100; + final String knownContentType = "application/octet-stream"; + final String knownMd5 = "098f6bcd4621d373cade4e832627b4f6"; + final String knownMdBase16 = BaseEncoding.base16().lowerCase().encode(knownMd5.getBytes()); + final String knownMd5Base64 = BaseEncoding.base64().encode(knownMd5.getBytes()); + + when(amazonS3Mock.getObject(anyString(), anyString())).thenReturn(s3ObjectMock); + when(s3ObjectMock.getObjectMetadata()).thenReturn(s3ObjectMetadataMock); + when(s3ObjectMetadataMock.getContentLength()).thenReturn(knownContentLength); + when(s3ObjectMetadataMock.getETag()).thenReturn(knownMd5Base64); + when(s3ObjectMetadataMock.getContentType()).thenReturn(knownContentType); + + // test + final DbArtifact artifactBySha1 = s3RepositoryUnderTest.getArtifactBySha1(knownSHA1Hash); + + // verify + assertThat(artifactBySha1.getArtifactId()).isEqualTo(knownSHA1Hash); + assertThat(artifactBySha1.getContentType()).isEqualTo(knownContentType); + assertThat(artifactBySha1.getSize()).isEqualTo(knownContentLength); + assertThat(artifactBySha1.getHashes().getSha1()).isEqualTo(knownSHA1Hash); + assertThat(artifactBySha1.getHashes().getMd5()).isEqualTo(knownMdBase16); + } + + @Test + @Description("Verifies that the amazonS3 client is not called to put the object to S3 due the artifact already exists on S3") + public void artifactIsNotUploadedIfAlreadyExists() throws NoSuchAlgorithmException, IOException { + final byte[] rndBytes = randomBytes(); + final String knownSHA1 = getSha1OfBytes(rndBytes); + final String knownContentType = "application/octet-stream"; + + when(amazonS3Mock.doesObjectExist(s3Properties.getBucketName(), knownSHA1)).thenReturn(true); + + // test + storeRandomBytes(rndBytes, knownContentType); + + // verify + Mockito.verify(amazonS3Mock, never()).putObject(eq(s3Properties.getBucketName()), eq(knownSHA1), + inputStreamCaptor.capture(), objectMetaDataCaptor.capture()); + + } + + @Test + @Description("Verifies that null is returned if the given hash does not exists on S3") + public void getArtifactBySha1ReturnsNullIfFileDoesNotExists() { + final String knownSHA1Hash = "0815"; + when(amazonS3Mock.getObject(s3Properties.getBucketName(), knownSHA1Hash)).thenReturn(null); + + // test + final DbArtifact artifactBySha1NotExists = s3RepositoryUnderTest.getArtifactBySha1(knownSHA1Hash); + + // verify + assertThat(artifactBySha1NotExists).isNull(); + } + + @Test + @Description("Verifies that given SHA1 hash are checked and if not match will throw exception") + public void sha1HashValuesAreNotTheSameThrowsException() throws IOException { + + final byte[] rndBytes = randomBytes(); + final String knownContentType = "application/octet-stream"; + final String wrongSHA1Hash = "wrong"; + final String wrongMD5 = "wrong"; + + // test + try { + storeRandomBytes(rndBytes, knownContentType, new DbArtifactHash(wrongSHA1Hash, wrongMD5)); + fail("Expected an HashNotMatchException, but didn't throw"); + } catch (final HashNotMatchException e) { + assertThat(e.getHashFunction()).isEqualTo(HashNotMatchException.SHA1); + } + } + + @Test + @Description("Verifies that given MD5 hash are checked and if not match will throw exception") + public void md5HashValuesAreNotTheSameThrowsException() throws IOException, NoSuchAlgorithmException { + + final byte[] rndBytes = randomBytes(); + final String knownContentType = "application/octet-stream"; + final String knownSHA1 = getSha1OfBytes(rndBytes); + final String wrongMD5 = "wrong"; + + // test + try { + storeRandomBytes(rndBytes, knownContentType, new DbArtifactHash(knownSHA1, wrongMD5)); + fail("Expected an HashNotMatchException, but didn't throw"); + } catch (final HashNotMatchException e) { + assertThat(e.getHashFunction()).isEqualTo(HashNotMatchException.MD5); + } + } + + private void storeRandomBytes(final byte[] rndBytes, final String contentType) + throws IOException, NoSuchAlgorithmException { + storeRandomBytes(rndBytes, contentType, null); + } + + private void storeRandomBytes(final byte[] rndBytes, final String contentType, final DbArtifactHash hashes) + throws IOException { + final String knownFileName = "randomBytes"; + try (InputStream content = new BufferedInputStream(new ByteArrayInputStream(rndBytes))) { + s3RepositoryUnderTest.store(content, knownFileName, contentType, hashes); + } + } + + private static String getSha1OfBytes(final byte[] bytes) throws IOException, NoSuchAlgorithmException { + final MessageDigest messageDigest = MessageDigest.getInstance("SHA1"); + + try (InputStream input = new ByteArrayInputStream(bytes); + OutputStream output = new DigestOutputStream(new ByteArrayOutputStream(), messageDigest)) { + ByteStreams.copy(input, output); + return BaseEncoding.base16().lowerCase().encode(messageDigest.digest()); + } + } + + private static byte[] randomBytes() { + final byte[] randomBytes = new byte[20]; + final Random ran = new Random(); + ran.nextBytes(randomBytes); + return randomBytes; + } +} diff --git a/extensions/pom.xml b/extensions/pom.xml index 160f7287c..341ba6a9e 100644 --- a/extensions/pom.xml +++ b/extensions/pom.xml @@ -23,6 +23,7 @@ hawkbit-extension-uaa hawkbit-extension-artifact-repository-mongo + hawkbit-extension-artifact-repository-s3