ArtifactRepository tenant aware. (#539)
* ArtifactRepository tenant aware. Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com> * No need to have this protected. Updated event to boot > 1.3 Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com> * Remove conditional. Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com> * Remove Debug log. Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com> * Cleanup Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com> * Missing validation and readability. Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com> * Fix test after change. Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com> * Fix computation is DosFilter Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com> * Fix session state on RESTful APIs. Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com> * Performance improvement controllermanagement Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com> * Added cross tenant test. Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com> * Typos. Signed-off-by: kaizimmerm <kai.zimmermann@bosch-si.com>
This commit is contained in:
@@ -54,6 +54,11 @@
|
||||
<artifactId>allure-junit-adaptor</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-logging</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- MongoDB -->
|
||||
<dependency>
|
||||
<groupId>de.flapdoodle.embed</groupId>
|
||||
|
||||
@@ -24,13 +24,13 @@ import java.util.stream.Collectors;
|
||||
|
||||
import org.eclipse.hawkbit.artifact.repository.model.DbArtifact;
|
||||
import org.eclipse.hawkbit.artifact.repository.model.DbArtifactHash;
|
||||
import org.eclipse.hawkbit.tenancy.TenantAware;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.data.mongodb.core.MongoTemplate;
|
||||
import org.springframework.data.mongodb.core.query.Criteria;
|
||||
import org.springframework.data.mongodb.core.query.Query;
|
||||
import org.springframework.data.mongodb.gridfs.GridFsOperations;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
|
||||
import com.google.common.io.BaseEncoding;
|
||||
import com.google.common.io.ByteStreams;
|
||||
@@ -44,6 +44,7 @@ import com.mongodb.gridfs.GridFSFile;
|
||||
* The file management which looks up all the file in the file tore.
|
||||
*
|
||||
*/
|
||||
@Validated
|
||||
public class MongoDBArtifactStore implements ArtifactRepository {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(MongoDBArtifactStore.class);
|
||||
@@ -56,9 +57,14 @@ public class MongoDBArtifactStore implements ArtifactRepository {
|
||||
private static final String FILENAME = "filename";
|
||||
|
||||
/**
|
||||
* The mongoDB field which automatically calculated by the mongoDB.
|
||||
* The mongoDB field which holds the tenant of the file to download.
|
||||
*/
|
||||
private static final String MD5 = "md5";
|
||||
private static final String TENANT = "tenant";
|
||||
|
||||
/**
|
||||
* Query by {@link TenantAware} field.
|
||||
*/
|
||||
private static final String TENANT_QUERY = "metadata." + TENANT;
|
||||
|
||||
/**
|
||||
* The mongoDB field which holds the SHA1 hash, stored in the meta data
|
||||
@@ -68,10 +74,11 @@ public class MongoDBArtifactStore implements ArtifactRepository {
|
||||
|
||||
private static final String ID = "_id";
|
||||
|
||||
@Autowired
|
||||
private GridFsOperations gridFs;
|
||||
private final GridFsOperations gridFs;
|
||||
|
||||
MongoTemplate mongoTemplate;
|
||||
MongoDBArtifactStore(final GridFsOperations gridFs) {
|
||||
this.gridFs = gridFs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves a {@link GridFSDBFile} from the store by it's SHA1 hash.
|
||||
@@ -82,36 +89,28 @@ public class MongoDBArtifactStore implements ArtifactRepository {
|
||||
* @return The DbArtifact object or {@code null} if no file exists.
|
||||
*/
|
||||
@Override
|
||||
public DbArtifact getArtifactBySha1(final String sha1Hash) {
|
||||
return map(gridFs.findOne(new Query().addCriteria(Criteria.where(FILENAME).is(sha1Hash))));
|
||||
}
|
||||
public DbArtifact getArtifactBySha1(final String tenant, final String sha1Hash) {
|
||||
|
||||
/**
|
||||
* Retrieves a {@link GridFSDBFile} from the store by it's MD5 hash.
|
||||
*
|
||||
* @param md5Hash
|
||||
* the md5-hash of the file to lookup.
|
||||
* @return The gridfs file object or {@code null} if no file exists.
|
||||
*/
|
||||
public DbArtifact getArtifactByMd5(final String md5Hash) {
|
||||
return map(gridFs.findOne(new Query().addCriteria(Criteria.where(MD5).is(md5Hash))));
|
||||
return map(gridFs.findOne(new Query()
|
||||
.addCriteria(Criteria.where(FILENAME).is(sha1Hash).and(TENANT_QUERY).is(sanitizeTenant(tenant)))));
|
||||
}
|
||||
|
||||
@Override
|
||||
public DbArtifact store(final InputStream content, final String filename, final String contentType) {
|
||||
return store(content, filename, contentType, null);
|
||||
public DbArtifact store(final String tenant, final InputStream content, final String filename,
|
||||
final String contentType) {
|
||||
return store(tenant, content, filename, contentType, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DbArtifact store(final InputStream content, final String filename, final String contentType,
|
||||
final DbArtifactHash hash) {
|
||||
public DbArtifact store(final String tenant, final InputStream content, final String filename,
|
||||
final String contentType, final DbArtifactHash hash) {
|
||||
File tempFile = null;
|
||||
try {
|
||||
LOGGER.debug("storing file {} of content {}", filename, contentType);
|
||||
tempFile = File.createTempFile("uploadFile", null);
|
||||
try (final BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(tempFile))) {
|
||||
try (BufferedInputStream bis = new BufferedInputStream(content)) {
|
||||
return store(bis, contentType, bos, tempFile, hash);
|
||||
return store(tenant, bis, contentType, bos, tempFile, hash);
|
||||
}
|
||||
}
|
||||
} catch (final IOException | MongoException e1) {
|
||||
@@ -123,10 +122,15 @@ public class MongoDBArtifactStore implements ArtifactRepository {
|
||||
}
|
||||
}
|
||||
|
||||
private static String sanitizeTenant(final String tenant) {
|
||||
return tenant.trim().toUpperCase();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteBySha1(final String sha1Hash) {
|
||||
public void deleteBySha1(final String tenant, final String sha1Hash) {
|
||||
try {
|
||||
deleteArtifact(gridFs.findOne(new Query().addCriteria(Criteria.where(FILENAME).is(sha1Hash))));
|
||||
deleteArtifact(gridFs.findOne(new Query()
|
||||
.addCriteria(Criteria.where(FILENAME).is(sha1Hash).and(TENANT_QUERY).is(sanitizeTenant(tenant)))));
|
||||
} catch (final MongoException e) {
|
||||
throw new ArtifactStoreException(e.getMessage(), e);
|
||||
}
|
||||
@@ -143,18 +147,21 @@ public class MongoDBArtifactStore implements ArtifactRepository {
|
||||
|
||||
}
|
||||
|
||||
private DbArtifact store(final InputStream content, final String contentType, final OutputStream os,
|
||||
private DbArtifact store(final String t, final InputStream content, final String contentType, final OutputStream os,
|
||||
final File tempFile, final DbArtifactHash hash) {
|
||||
final GridFsArtifact storedArtifact;
|
||||
final String tenant = sanitizeTenant(t);
|
||||
try {
|
||||
final String sha1Hash = computeSHA1Hash(content, os, hash != null ? hash.getSha1() : null);
|
||||
// upload if it does not exist already, check if file exists, not
|
||||
// tenant specific.
|
||||
final GridFSDBFile result = gridFs.findOne(new Query().addCriteria(Criteria.where(FILENAME).is(sha1Hash)));
|
||||
if (null == result) {
|
||||
final GridFSDBFile result = gridFs.findOne(
|
||||
new Query().addCriteria(Criteria.where(FILENAME).is(sha1Hash).and(TENANT_QUERY).is(tenant)));
|
||||
if (result == null) {
|
||||
try (FileInputStream inputStream = new FileInputStream(tempFile)) {
|
||||
final BasicDBObject metadata = new BasicDBObject();
|
||||
metadata.put(SHA1, sha1Hash);
|
||||
metadata.put(TENANT, tenant);
|
||||
storedArtifact = map(gridFs.store(inputStream, sha1Hash, contentType, metadata));
|
||||
}
|
||||
} else {
|
||||
@@ -244,4 +251,13 @@ public class MongoDBArtifactStore implements ArtifactRepository {
|
||||
artifact.setHashes(new DbArtifactHash(fsFile.getFilename(), fsFile.getMD5()));
|
||||
return artifact;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteByTenant(final String tenant) {
|
||||
try {
|
||||
gridFs.delete(new Query().addCriteria(Criteria.where(TENANT_QUERY).is(sanitizeTenant(tenant))));
|
||||
} catch (final MongoClientException e) {
|
||||
throw new ArtifactStoreException(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.PropertySource;
|
||||
import org.springframework.data.mongodb.gridfs.GridFsOperations;
|
||||
|
||||
/**
|
||||
* Auto configuration for the {@link MongoDBArtifactStore}.
|
||||
@@ -25,7 +26,7 @@ public class MongoDBArtifactStoreAutoConfiguration {
|
||||
* @return Default {@link ArtifactRepository} implementation.
|
||||
*/
|
||||
@Bean
|
||||
public ArtifactRepository artifactRepository() {
|
||||
return new MongoDBArtifactStore();
|
||||
ArtifactRepository artifactRepository(final GridFsOperations gridFs) {
|
||||
return new MongoDBArtifactStore(gridFs);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,54 +20,75 @@ import org.eclipse.hawkbit.artifact.TestConfiguration;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.SpringApplicationConfiguration;
|
||||
import org.springframework.data.mongodb.gridfs.GridFsOperations;
|
||||
import org.springframework.test.context.TestPropertySource;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
|
||||
|
||||
import com.google.common.io.BaseEncoding;
|
||||
|
||||
import ru.yandex.qatools.allure.annotations.Description;
|
||||
import ru.yandex.qatools.allure.annotations.Features;
|
||||
import ru.yandex.qatools.allure.annotations.Step;
|
||||
import ru.yandex.qatools.allure.annotations.Stories;
|
||||
|
||||
@Features("Component Tests - Repository")
|
||||
@Stories("Artifact Store MongoDB")
|
||||
@RunWith(SpringJUnit4ClassRunner.class)
|
||||
@SpringApplicationConfiguration(classes = { MongoDBArtifactStoreAutoConfiguration.class, TestConfiguration.class })
|
||||
@TestPropertySource(properties = { "spring.data.mongodb.port=0", "spring.mongodb.embedded.version=3.2.7" })
|
||||
@SpringBootTest(classes = { MongoDBArtifactStoreAutoConfiguration.class, TestConfiguration.class }, properties = {
|
||||
"spring.data.mongodb.port=0", "spring.mongodb.embedded.version=3.4.4" })
|
||||
public class MongoDBArtifactStoreTest {
|
||||
private static final String TENANT = "test_tenant";
|
||||
private static final String TENANT2 = "test_tenant2";
|
||||
|
||||
@Autowired
|
||||
private MongoDBArtifactStore artifactStoreUnderTest;
|
||||
|
||||
@Autowired
|
||||
private GridFsOperations gridFs;
|
||||
|
||||
@Test
|
||||
@Description("Ensures that search by SHA1 hash (which is used by hawkBit as artifact ID) finds the expected results.")
|
||||
public void findArtifactBySHA1Hash() throws NoSuchAlgorithmException {
|
||||
|
||||
final String sha1 = storeRandomArifactAndVerify(TENANT);
|
||||
final String sha2 = storeRandomArifactAndVerify(TENANT2);
|
||||
|
||||
assertThat(artifactStoreUnderTest.getArtifactBySha1(TENANT2, sha1)).isNull();
|
||||
assertThat(artifactStoreUnderTest.getArtifactBySha1(TENANT, sha2)).isNull();
|
||||
}
|
||||
|
||||
@Step
|
||||
private String storeRandomArifactAndVerify(final String tenant) throws NoSuchAlgorithmException {
|
||||
final int filelengthBytes = 128;
|
||||
final String filename = "testfile.json";
|
||||
final String contentType = "application/json";
|
||||
|
||||
final DigestInputStream digestInputStream = digestInputStream(generateInputStream(filelengthBytes), "SHA-1");
|
||||
artifactStoreUnderTest.store(digestInputStream, filename, contentType);
|
||||
assertThat(artifactStoreUnderTest.getArtifactBySha1(
|
||||
BaseEncoding.base16().lowerCase().encode(digestInputStream.getMessageDigest().digest()))).isNotNull();
|
||||
artifactStoreUnderTest.store(tenant, digestInputStream, filename, contentType);
|
||||
|
||||
final String sha1 = BaseEncoding.base16().lowerCase().encode(digestInputStream.getMessageDigest().digest());
|
||||
assertThat(artifactStoreUnderTest.getArtifactBySha1(tenant, sha1)).isNotNull();
|
||||
return sha1;
|
||||
}
|
||||
|
||||
@Test
|
||||
@Description("Ensures that search by MD5 hash finds the expected results.")
|
||||
public void findArtifactByMD5Hash() throws NoSuchAlgorithmException {
|
||||
final int filelengthBytes = 128;
|
||||
final String filename = "testfile.json";
|
||||
final String contentType = "application/json";
|
||||
@Description("Deletes file from repository identified by SHA1 hash as filename.")
|
||||
public void deleteArtifactBySHA1Hash() throws NoSuchAlgorithmException {
|
||||
|
||||
final DigestInputStream digestInputStream = digestInputStream(generateInputStream(filelengthBytes), "MD5");
|
||||
artifactStoreUnderTest.store(digestInputStream, filename, contentType);
|
||||
assertThat(artifactStoreUnderTest.getArtifactByMd5(
|
||||
BaseEncoding.base16().lowerCase().encode(digestInputStream.getMessageDigest().digest()))).isNotNull();
|
||||
final String sha1 = storeRandomArifactAndVerify(TENANT);
|
||||
|
||||
artifactStoreUnderTest.deleteBySha1(TENANT, sha1);
|
||||
assertThat(artifactStoreUnderTest.getArtifactBySha1(TENANT, sha1)).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
@Description("Verfies that all data of a tenant is erased if repository is asked to do so. "
|
||||
+ "Data of other tenants is not affected.")
|
||||
public void deleteTenant() throws NoSuchAlgorithmException {
|
||||
|
||||
final String shaDeleted = storeRandomArifactAndVerify(TENANT);
|
||||
final String shaUndeleted = storeRandomArifactAndVerify("another_tenant");
|
||||
|
||||
artifactStoreUnderTest.deleteByTenant("tenant_that_does_not_exist");
|
||||
artifactStoreUnderTest.deleteByTenant(TENANT);
|
||||
assertThat(artifactStoreUnderTest.getArtifactBySha1(TENANT, shaDeleted)).isNull();
|
||||
assertThat(artifactStoreUnderTest.getArtifactBySha1("another_tenant", shaUndeleted)).isNotNull();
|
||||
}
|
||||
|
||||
private static ByteArrayInputStream generateInputStream(final int length) {
|
||||
|
||||
@@ -22,22 +22,22 @@ public class S3Artifact extends DbArtifact {
|
||||
|
||||
private final AmazonS3 amazonS3;
|
||||
private final S3RepositoryProperties s3Properties;
|
||||
private final String sha1;
|
||||
private final String key;
|
||||
|
||||
S3Artifact(final AmazonS3 amazonS3, final S3RepositoryProperties s3Properties, final String sha1) {
|
||||
S3Artifact(final AmazonS3 amazonS3, final S3RepositoryProperties s3Properties, final String key) {
|
||||
this.amazonS3 = amazonS3;
|
||||
this.s3Properties = s3Properties;
|
||||
this.sha1 = sha1;
|
||||
this.key = key;
|
||||
}
|
||||
|
||||
@Override
|
||||
public InputStream getFileInputStream() {
|
||||
return amazonS3.getObject(s3Properties.getBucketName(), sha1).getObjectContent();
|
||||
return amazonS3.getObject(s3Properties.getBucketName(), key).getObjectContent();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "S3Artifact [sha1=" + sha1 + ", getArtifactId()=" + getArtifactId() + ", getHashes()=" + getHashes()
|
||||
return "S3Artifact [key=" + key + ", getArtifactId()=" + getArtifactId() + ", getHashes()=" + getHashes()
|
||||
+ ", getSize()=" + getSize() + ", getContentType()=" + getContentType() + "]";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,14 +24,17 @@ import org.eclipse.hawkbit.artifact.repository.model.DbArtifact;
|
||||
import org.eclipse.hawkbit.artifact.repository.model.DbArtifactHash;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
|
||||
import com.amazonaws.AmazonClientException;
|
||||
import com.amazonaws.RequestClientOptions;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.Headers;
|
||||
import com.amazonaws.services.s3.model.DeleteObjectRequest;
|
||||
import com.amazonaws.services.s3.model.ObjectListing;
|
||||
import com.amazonaws.services.s3.model.ObjectMetadata;
|
||||
import com.amazonaws.services.s3.model.S3Object;
|
||||
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||
import com.google.common.io.BaseEncoding;
|
||||
import com.google.common.io.ByteStreams;
|
||||
|
||||
@@ -48,6 +51,7 @@ import com.google.common.io.ByteStreams;
|
||||
* across several buckets.
|
||||
* </p>
|
||||
*/
|
||||
@Validated
|
||||
public class S3Repository implements ArtifactRepository {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(S3Repository.class);
|
||||
@@ -73,16 +77,17 @@ public class S3Repository implements ArtifactRepository {
|
||||
}
|
||||
|
||||
@Override
|
||||
public DbArtifact store(final InputStream content, final String filename, final String contentType) {
|
||||
return store(content, filename, contentType, null);
|
||||
public DbArtifact store(final String tenant, final InputStream content, final String filename,
|
||||
final String contentType) {
|
||||
return store(tenant, content, filename, contentType, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
// suppress warning, of not strong enough hashing algorithm, SHA-1 and MD5
|
||||
// is not used security related
|
||||
@SuppressWarnings("squid:S2070")
|
||||
public DbArtifact store(final InputStream content, final String filename, final String contentType,
|
||||
final DbArtifactHash hash) {
|
||||
public DbArtifact store(final String tenant, final InputStream content, final String filename,
|
||||
final String contentType, final DbArtifactHash hash) {
|
||||
final MessageDigest mdSHA1;
|
||||
final MessageDigest mdMD5;
|
||||
try {
|
||||
@@ -102,7 +107,7 @@ public class S3Repository implements ArtifactRepository {
|
||||
final String sha1Hash16 = BaseEncoding.base16().lowerCase().encode(mdSHA1.digest());
|
||||
final String md5Hash16 = BaseEncoding.base16().lowerCase().encode(mdMD5.digest());
|
||||
|
||||
return store(sha1Hash16, md5Hash16, contentType, file, hash);
|
||||
return store(tenant, sha1Hash16, md5Hash16, contentType, file, hash);
|
||||
} catch (final IOException e) {
|
||||
throw new ArtifactStoreException(e.getMessage(), e);
|
||||
} finally {
|
||||
@@ -112,15 +117,16 @@ public class S3Repository implements ArtifactRepository {
|
||||
}
|
||||
}
|
||||
|
||||
private DbArtifact store(final String sha1Hash16, final String mdMD5Hash16, final String contentType,
|
||||
final File file, final DbArtifactHash hash) {
|
||||
final S3Artifact s3Artifact = createS3Artifact(sha1Hash16, mdMD5Hash16, contentType, file);
|
||||
private DbArtifact store(final String tenant, final String sha1Hash16, final String mdMD5Hash16,
|
||||
final String contentType, final File file, final DbArtifactHash hash) {
|
||||
final S3Artifact s3Artifact = createS3Artifact(tenant, sha1Hash16, mdMD5Hash16, contentType, file);
|
||||
checkHashes(s3Artifact, hash);
|
||||
final String key = objectKey(tenant, sha1Hash16);
|
||||
|
||||
LOG.info("Storing file {} with length {} to AWS S3 bucket {} as SHA1 {}", file.getName(), file.length(),
|
||||
s3Properties.getBucketName(), sha1Hash16);
|
||||
s3Properties.getBucketName(), key);
|
||||
|
||||
if (exists(sha1Hash16)) {
|
||||
if (exists(key)) {
|
||||
LOG.debug("Artifact {} already exists on S3 bucket {}, don't need to upload twice", sha1Hash16,
|
||||
s3Properties.getBucketName());
|
||||
return s3Artifact;
|
||||
@@ -129,7 +135,7 @@ public class S3Repository implements ArtifactRepository {
|
||||
try (final InputStream inputStream = new BufferedInputStream(new FileInputStream(file),
|
||||
RequestClientOptions.DEFAULT_STREAM_BUFFER_SIZE)) {
|
||||
final ObjectMetadata objectMetadata = createObjectMetadata(mdMD5Hash16, contentType, file);
|
||||
amazonS3.putObject(s3Properties.getBucketName(), sha1Hash16, inputStream, objectMetadata);
|
||||
amazonS3.putObject(s3Properties.getBucketName(), key, inputStream, objectMetadata);
|
||||
|
||||
return s3Artifact;
|
||||
} catch (final IOException | AmazonClientException e) {
|
||||
@@ -137,14 +143,15 @@ public class S3Repository implements ArtifactRepository {
|
||||
}
|
||||
}
|
||||
|
||||
private S3Artifact createS3Artifact(final String sha1Hash16, final String mdMD5Hash16, final String contentType,
|
||||
final File file) {
|
||||
final S3Artifact s3Artifact = new S3Artifact(amazonS3, s3Properties, sha1Hash16);
|
||||
private S3Artifact createS3Artifact(final String tenant, final String sha1Hash, final String mdMD5Hash16,
|
||||
final String contentType, final File file) {
|
||||
final String key = objectKey(tenant, sha1Hash);
|
||||
final S3Artifact s3Artifact = new S3Artifact(amazonS3, s3Properties, key);
|
||||
s3Artifact.setContentType(contentType);
|
||||
s3Artifact.setArtifactId(sha1Hash16);
|
||||
s3Artifact.setArtifactId(sha1Hash);
|
||||
s3Artifact.setSize(file.length());
|
||||
s3Artifact.setContentType(contentType);
|
||||
s3Artifact.setHashes(new DbArtifactHash(sha1Hash16, mdMD5Hash16));
|
||||
s3Artifact.setHashes(new DbArtifactHash(sha1Hash, mdMD5Hash16));
|
||||
return s3Artifact;
|
||||
}
|
||||
|
||||
@@ -162,26 +169,34 @@ public class S3Repository implements ArtifactRepository {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteBySha1(final String sha1Hash) {
|
||||
LOG.info("Deleting S3 object from bucket {} and key {}", s3Properties.getBucketName(), sha1Hash);
|
||||
amazonS3.deleteObject(new DeleteObjectRequest(s3Properties.getBucketName(), sha1Hash));
|
||||
public void deleteBySha1(final String tenant, final String sha1Hash) {
|
||||
final String key = objectKey(tenant, sha1Hash);
|
||||
|
||||
LOG.info("Deleting S3 object from bucket {} and key {}", s3Properties.getBucketName(), key);
|
||||
amazonS3.deleteObject(new DeleteObjectRequest(s3Properties.getBucketName(), key));
|
||||
}
|
||||
|
||||
private static String objectKey(final String tenant, final String sha1Hash) {
|
||||
return sanitizeTenant(tenant) + "/" + sha1Hash;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DbArtifact getArtifactBySha1(final String sha1) {
|
||||
LOG.info("Retrieving S3 object from bucket {} and key {}", s3Properties.getBucketName(), sha1);
|
||||
final S3Object s3Object = amazonS3.getObject(s3Properties.getBucketName(), sha1);
|
||||
public DbArtifact getArtifactBySha1(final String tenant, final String sha1Hash) {
|
||||
final String key = objectKey(tenant, sha1Hash);
|
||||
|
||||
LOG.info("Retrieving S3 object from bucket {} and key {}", s3Properties.getBucketName(), key);
|
||||
final S3Object s3Object = amazonS3.getObject(s3Properties.getBucketName(), key);
|
||||
if (s3Object == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
final ObjectMetadata s3ObjectMetadata = s3Object.getObjectMetadata();
|
||||
|
||||
final S3Artifact s3Artifact = new S3Artifact(amazonS3, s3Properties, sha1);
|
||||
s3Artifact.setArtifactId(sha1);
|
||||
final S3Artifact s3Artifact = new S3Artifact(amazonS3, s3Properties, key);
|
||||
s3Artifact.setArtifactId(sha1Hash);
|
||||
s3Artifact.setSize(s3ObjectMetadata.getContentLength());
|
||||
// the MD5Content is stored in the ETag
|
||||
s3Artifact.setHashes(new DbArtifactHash(sha1,
|
||||
s3Artifact.setHashes(new DbArtifactHash(sha1Hash,
|
||||
BaseEncoding.base16().lowerCase().encode(BaseEncoding.base64().decode(s3ObjectMetadata.getETag()))));
|
||||
s3Artifact.setContentType(s3ObjectMetadata.getContentType());
|
||||
return s3Artifact;
|
||||
@@ -220,4 +235,25 @@ public class S3Repository implements ArtifactRepository {
|
||||
private boolean exists(final String sha1) {
|
||||
return amazonS3.doesObjectExist(s3Properties.getBucketName(), sha1);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteByTenant(final String tenant) {
|
||||
final String folder = sanitizeTenant(tenant);
|
||||
|
||||
LOG.info("Deleting S3 object folder (tenant) from bucket {} and key {}", s3Properties.getBucketName(), folder);
|
||||
|
||||
// Delete artifacts
|
||||
ObjectListing objects = amazonS3.listObjects(s3Properties.getBucketName(), folder + "/");
|
||||
do {
|
||||
for (final S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
|
||||
amazonS3.deleteObject(s3Properties.getBucketName(), objectSummary.getKey());
|
||||
}
|
||||
objects = amazonS3.listNextBatchOfObjects(objects);
|
||||
} while (objects.isTruncated());
|
||||
|
||||
}
|
||||
|
||||
private static String sanitizeTenant(final String tenant) {
|
||||
return tenant.trim().toUpperCase();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,6 +54,7 @@ import ru.yandex.qatools.allure.annotations.Stories;
|
||||
@Features("Unit Tests - S3 Repository")
|
||||
@Stories("S3 Artifact Repository")
|
||||
public class S3RepositoryTest {
|
||||
private static final String TENANT = "test_tenant";
|
||||
|
||||
@Mock
|
||||
private AmazonS3 amazonS3Mock;
|
||||
@@ -89,8 +90,9 @@ public class S3RepositoryTest {
|
||||
storeRandomBytes(rndBytes, knownContentType);
|
||||
|
||||
// verify
|
||||
Mockito.verify(amazonS3Mock).putObject(eq(s3Properties.getBucketName()), eq(knownSHA1),
|
||||
inputStreamCaptor.capture(), objectMetaDataCaptor.capture());
|
||||
Mockito.verify(amazonS3Mock).putObject(eq(s3Properties.getBucketName()),
|
||||
eq(TENANT.toUpperCase() + "/" + knownSHA1), inputStreamCaptor.capture(),
|
||||
objectMetaDataCaptor.capture());
|
||||
|
||||
final ObjectMetadata recordedObjectMetadata = objectMetaDataCaptor.getValue();
|
||||
assertThat(recordedObjectMetadata.getContentType()).isEqualTo(knownContentType);
|
||||
@@ -115,7 +117,7 @@ public class S3RepositoryTest {
|
||||
when(s3ObjectMetadataMock.getContentType()).thenReturn(knownContentType);
|
||||
|
||||
// test
|
||||
final DbArtifact artifactBySha1 = s3RepositoryUnderTest.getArtifactBySha1(knownSHA1Hash);
|
||||
final DbArtifact artifactBySha1 = s3RepositoryUnderTest.getArtifactBySha1(TENANT, knownSHA1Hash);
|
||||
|
||||
// verify
|
||||
assertThat(artifactBySha1.getArtifactId()).isEqualTo(knownSHA1Hash);
|
||||
@@ -150,7 +152,7 @@ public class S3RepositoryTest {
|
||||
when(amazonS3Mock.getObject(s3Properties.getBucketName(), knownSHA1Hash)).thenReturn(null);
|
||||
|
||||
// test
|
||||
final DbArtifact artifactBySha1NotExists = s3RepositoryUnderTest.getArtifactBySha1(knownSHA1Hash);
|
||||
final DbArtifact artifactBySha1NotExists = s3RepositoryUnderTest.getArtifactBySha1(TENANT, knownSHA1Hash);
|
||||
|
||||
// verify
|
||||
assertThat(artifactBySha1NotExists).isNull();
|
||||
@@ -201,7 +203,7 @@ public class S3RepositoryTest {
|
||||
throws IOException {
|
||||
final String knownFileName = "randomBytes";
|
||||
try (InputStream content = new BufferedInputStream(new ByteArrayInputStream(rndBytes))) {
|
||||
s3RepositoryUnderTest.store(content, knownFileName, contentType, hashes);
|
||||
s3RepositoryUnderTest.store(TENANT, content, knownFileName, contentType, hashes);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user