lingwei.kong/hawkbit
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

ArtifactRepository tenant aware. (#539)

Browse Source 
* ArtifactRepository tenant aware.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* No need to have this protected. Updated event to boot > 1.3

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Remove conditional.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Remove Debug log.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Cleanup

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Missing validation and readability.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Fix test after change.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Fix computation is DosFilter

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Fix session state on RESTful APIs.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Performance improvement controllermanagement

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Added cross tenant test.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>

* Typos.

Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>
This commit is contained in:
Kai Zimmermann
2017-06-14 19:07:52 +02:00
committed by GitHub
parent 787042ce85
commit 8d17d21259
27 changed files with 340 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
extensions/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java
View File

@@ -22,22 +22,22 @@ public class S3Artifact extends DbArtifact {
private final AmazonS3 amazonS3;
private final S3RepositoryProperties s3Properties;
private final String sha1;
private final String key;
S3Artifact(final AmazonS3 amazonS3, final S3RepositoryProperties s3Properties, final String sha1) {
S3Artifact(final AmazonS3 amazonS3, final S3RepositoryProperties s3Properties, final String key) {
this.amazonS3 = amazonS3;
this.s3Properties = s3Properties;
this.sha1 = sha1;
this.key = key;
}
@Override
public InputStream getFileInputStream() {
return amazonS3.getObject(s3Properties.getBucketName(), sha1).getObjectContent();
return amazonS3.getObject(s3Properties.getBucketName(), key).getObjectContent();
}
@Override
public String toString() {
return "S3Artifact [sha1=" + sha1 + ", getArtifactId()=" + getArtifactId() + ", getHashes()=" + getHashes()
return "S3Artifact [key=" + key + ", getArtifactId()=" + getArtifactId() + ", getHashes()=" + getHashes()
+ ", getSize()=" + getSize() + ", getContentType()=" + getContentType() + "]";
}
}

86
extensions/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Repository.java
View File

@@ -24,14 +24,17 @@ import org.eclipse.hawkbit.artifact.repository.model.DbArtifact;
import org.eclipse.hawkbit.artifact.repository.model.DbArtifactHash;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.validation.annotation.Validated;
import com.amazonaws.AmazonClientException;
import com.amazonaws.RequestClientOptions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.Headers;
import com.amazonaws.services.s3.model.DeleteObjectRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.google.common.io.BaseEncoding;
import com.google.common.io.ByteStreams;
@@ -48,6 +51,7 @@ import com.google.common.io.ByteStreams;
* across several buckets.
* </p>
*/
@Validated
public class S3Repository implements ArtifactRepository {
private static final Logger LOG = LoggerFactory.getLogger(S3Repository.class);
@@ -73,16 +77,17 @@ public class S3Repository implements ArtifactRepository {
}
@Override
public DbArtifact store(final InputStream content, final String filename, final String contentType) {
return store(content, filename, contentType, null);
public DbArtifact store(final String tenant, final InputStream content, final String filename,
final String contentType) {
return store(tenant, content, filename, contentType, null);
}
@Override
// suppress warning, of not strong enough hashing algorithm, SHA-1 and MD5
// is not used security related
@SuppressWarnings("squid:S2070")
public DbArtifact store(final InputStream content, final String filename, final String contentType,
final DbArtifactHash hash) {
public DbArtifact store(final String tenant, final InputStream content, final String filename,
final String contentType, final DbArtifactHash hash) {
final MessageDigest mdSHA1;
final MessageDigest mdMD5;
try {
@@ -102,7 +107,7 @@ public class S3Repository implements ArtifactRepository {
final String sha1Hash16 = BaseEncoding.base16().lowerCase().encode(mdSHA1.digest());
final String md5Hash16 = BaseEncoding.base16().lowerCase().encode(mdMD5.digest());
return store(sha1Hash16, md5Hash16, contentType, file, hash);
return store(tenant, sha1Hash16, md5Hash16, contentType, file, hash);
} catch (final IOException e) {
throw new ArtifactStoreException(e.getMessage(), e);
} finally {
@@ -112,15 +117,16 @@ public class S3Repository implements ArtifactRepository {
}
}
private DbArtifact store(final String sha1Hash16, final String mdMD5Hash16, final String contentType,
final File file, final DbArtifactHash hash) {
final S3Artifact s3Artifact = createS3Artifact(sha1Hash16, mdMD5Hash16, contentType, file);
private DbArtifact store(final String tenant, final String sha1Hash16, final String mdMD5Hash16,
final String contentType, final File file, final DbArtifactHash hash) {
final S3Artifact s3Artifact = createS3Artifact(tenant, sha1Hash16, mdMD5Hash16, contentType, file);
checkHashes(s3Artifact, hash);
final String key = objectKey(tenant, sha1Hash16);
LOG.info("Storing file {} with length {} to AWS S3 bucket {} as SHA1 {}", file.getName(), file.length(),
s3Properties.getBucketName(), sha1Hash16);
s3Properties.getBucketName(), key);
if (exists(sha1Hash16)) {
if (exists(key)) {
LOG.debug("Artifact {} already exists on S3 bucket {}, don't need to upload twice", sha1Hash16,
s3Properties.getBucketName());
return s3Artifact;
@@ -129,7 +135,7 @@ public class S3Repository implements ArtifactRepository {
try (final InputStream inputStream = new BufferedInputStream(new FileInputStream(file),
RequestClientOptions.DEFAULT_STREAM_BUFFER_SIZE)) {
final ObjectMetadata objectMetadata = createObjectMetadata(mdMD5Hash16, contentType, file);
amazonS3.putObject(s3Properties.getBucketName(), sha1Hash16, inputStream, objectMetadata);
amazonS3.putObject(s3Properties.getBucketName(), key, inputStream, objectMetadata);
return s3Artifact;
} catch (final IOException | AmazonClientException e) {
@@ -137,14 +143,15 @@ public class S3Repository implements ArtifactRepository {
}
}
private S3Artifact createS3Artifact(final String sha1Hash16, final String mdMD5Hash16, final String contentType,
final File file) {
final S3Artifact s3Artifact = new S3Artifact(amazonS3, s3Properties, sha1Hash16);
private S3Artifact createS3Artifact(final String tenant, final String sha1Hash, final String mdMD5Hash16,
final String contentType, final File file) {
final String key = objectKey(tenant, sha1Hash);
final S3Artifact s3Artifact = new S3Artifact(amazonS3, s3Properties, key);
s3Artifact.setContentType(contentType);
s3Artifact.setArtifactId(sha1Hash16);
s3Artifact.setArtifactId(sha1Hash);
s3Artifact.setSize(file.length());
s3Artifact.setContentType(contentType);
s3Artifact.setHashes(new DbArtifactHash(sha1Hash16, mdMD5Hash16));
s3Artifact.setHashes(new DbArtifactHash(sha1Hash, mdMD5Hash16));
return s3Artifact;
}
@@ -162,26 +169,34 @@ public class S3Repository implements ArtifactRepository {
}
@Override
public void deleteBySha1(final String sha1Hash) {
LOG.info("Deleting S3 object from bucket {} and key {}", s3Properties.getBucketName(), sha1Hash);
amazonS3.deleteObject(new DeleteObjectRequest(s3Properties.getBucketName(), sha1Hash));
public void deleteBySha1(final String tenant, final String sha1Hash) {
final String key = objectKey(tenant, sha1Hash);
LOG.info("Deleting S3 object from bucket {} and key {}", s3Properties.getBucketName(), key);
amazonS3.deleteObject(new DeleteObjectRequest(s3Properties.getBucketName(), key));
}
private static String objectKey(final String tenant, final String sha1Hash) {
return sanitizeTenant(tenant) + "/" + sha1Hash;
}
@Override
public DbArtifact getArtifactBySha1(final String sha1) {
LOG.info("Retrieving S3 object from bucket {} and key {}", s3Properties.getBucketName(), sha1);
final S3Object s3Object = amazonS3.getObject(s3Properties.getBucketName(), sha1);
public DbArtifact getArtifactBySha1(final String tenant, final String sha1Hash) {
final String key = objectKey(tenant, sha1Hash);
LOG.info("Retrieving S3 object from bucket {} and key {}", s3Properties.getBucketName(), key);
final S3Object s3Object = amazonS3.getObject(s3Properties.getBucketName(), key);
if (s3Object == null) {
return null;
}
final ObjectMetadata s3ObjectMetadata = s3Object.getObjectMetadata();
final S3Artifact s3Artifact = new S3Artifact(amazonS3, s3Properties, sha1);
s3Artifact.setArtifactId(sha1);
final S3Artifact s3Artifact = new S3Artifact(amazonS3, s3Properties, key);
s3Artifact.setArtifactId(sha1Hash);
s3Artifact.setSize(s3ObjectMetadata.getContentLength());
// the MD5Content is stored in the ETag
s3Artifact.setHashes(new DbArtifactHash(sha1,
s3Artifact.setHashes(new DbArtifactHash(sha1Hash,
BaseEncoding.base16().lowerCase().encode(BaseEncoding.base64().decode(s3ObjectMetadata.getETag()))));
s3Artifact.setContentType(s3ObjectMetadata.getContentType());
return s3Artifact;
@@ -220,4 +235,25 @@ public class S3Repository implements ArtifactRepository {
private boolean exists(final String sha1) {
return amazonS3.doesObjectExist(s3Properties.getBucketName(), sha1);
}
@Override
public void deleteByTenant(final String tenant) {
final String folder = sanitizeTenant(tenant);
LOG.info("Deleting S3 object folder (tenant) from bucket {} and key {}", s3Properties.getBucketName(), folder);
// Delete artifacts
ObjectListing objects = amazonS3.listObjects(s3Properties.getBucketName(), folder + "/");
do {
for (final S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
amazonS3.deleteObject(s3Properties.getBucketName(), objectSummary.getKey());
}
objects = amazonS3.listNextBatchOfObjects(objects);
} while (objects.isTruncated());
}
private static String sanitizeTenant(final String tenant) {
return tenant.trim().toUpperCase();
}
}

12
extensions/hawkbit-extension-artifact-repository-s3/src/test/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryTest.java
View File

@@ -54,6 +54,7 @@ import ru.yandex.qatools.allure.annotations.Stories;
@Features("Unit Tests - S3 Repository")
@Stories("S3 Artifact Repository")
public class S3RepositoryTest {
private static final String TENANT = "test_tenant";
@Mock
private AmazonS3 amazonS3Mock;
@@ -89,8 +90,9 @@ public class S3RepositoryTest {
storeRandomBytes(rndBytes, knownContentType);
// verify
Mockito.verify(amazonS3Mock).putObject(eq(s3Properties.getBucketName()), eq(knownSHA1),
inputStreamCaptor.capture(), objectMetaDataCaptor.capture());
Mockito.verify(amazonS3Mock).putObject(eq(s3Properties.getBucketName()),
eq(TENANT.toUpperCase() + "/" + knownSHA1), inputStreamCaptor.capture(),
objectMetaDataCaptor.capture());
final ObjectMetadata recordedObjectMetadata = objectMetaDataCaptor.getValue();
assertThat(recordedObjectMetadata.getContentType()).isEqualTo(knownContentType);
@@ -115,7 +117,7 @@ public class S3RepositoryTest {
when(s3ObjectMetadataMock.getContentType()).thenReturn(knownContentType);
// test
final DbArtifact artifactBySha1 = s3RepositoryUnderTest.getArtifactBySha1(knownSHA1Hash);
final DbArtifact artifactBySha1 = s3RepositoryUnderTest.getArtifactBySha1(TENANT, knownSHA1Hash);
// verify
assertThat(artifactBySha1.getArtifactId()).isEqualTo(knownSHA1Hash);
@@ -150,7 +152,7 @@ public class S3RepositoryTest {
when(amazonS3Mock.getObject(s3Properties.getBucketName(), knownSHA1Hash)).thenReturn(null);
// test
final DbArtifact artifactBySha1NotExists = s3RepositoryUnderTest.getArtifactBySha1(knownSHA1Hash);
final DbArtifact artifactBySha1NotExists = s3RepositoryUnderTest.getArtifactBySha1(TENANT, knownSHA1Hash);
// verify
assertThat(artifactBySha1NotExists).isNull();
@@ -201,7 +203,7 @@ public class S3RepositoryTest {
throws IOException {
final String knownFileName = "randomBytes";
try (InputStream content = new BufferedInputStream(new ByteArrayInputStream(rndBytes))) {
s3RepositoryUnderTest.store(content, knownFileName, contentType, hashes);
s3RepositoryUnderTest.store(TENANT, content, knownFileName, contentType, hashes);
}
}