Add filesystem artifact repository implementation (#336)

Signed-off-by: Michael Hirsch <michael.hirsch@bosch-si.com>
This commit is contained in:
Michael Hirsch
2016-11-14 11:23:50 +01:00
committed by Kai Zimmermann
parent 9b42c8cf57
commit 8be49a1184
48 changed files with 682 additions and 425 deletions

View File

@@ -49,7 +49,7 @@
</dependency>
<dependency>
<groupId>org.eclipse.hawkbit</groupId>
<artifactId>hawkbit-artifact-repository-mongo</artifactId>
<artifactId>hawkbit-artifact-repository-filesystem</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>

View File

@@ -20,12 +20,10 @@ import javax.persistence.Table;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.Size;
import org.eclipse.hawkbit.artifact.repository.model.DbArtifact;
import org.eclipse.hawkbit.repository.model.Artifact;
import org.eclipse.hawkbit.repository.model.SoftwareModule;
import com.mongodb.gridfs.GridFS;
import com.mongodb.gridfs.GridFSFile;
/**
* JPA implementation of {@link LocalArtifact}.
*
@@ -73,9 +71,9 @@ public class JpaArtifact extends AbstractJpaTenantAwareBaseEntity implements Art
* Constructs artifact.
*
* @param gridFsFileName
* that is the link to the {@link GridFS} entity.
* that is the link to the {@link DbArtifact} entity.
* @param filename
* that is used by {@link GridFSFile} store.
* that is used by {@link DbArtifact} store.
* @param softwareModule
* of this artifact
*/

View File

@@ -15,7 +15,6 @@ import org.eclipse.hawkbit.cache.TenantAwareCacheManager;
import org.eclipse.hawkbit.repository.test.util.AbstractIntegrationTest;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.SpringApplicationConfiguration;
import org.springframework.data.mongodb.gridfs.GridFsOperations;
@SpringApplicationConfiguration(classes = { org.eclipse.hawkbit.RepositoryApplicationConfiguration.class })
public abstract class AbstractJpaIntegrationTest extends AbstractIntegrationTest {
@@ -62,9 +61,6 @@ public abstract class AbstractJpaIntegrationTest extends AbstractIntegrationTest
@Autowired
protected TargetInfoRepository targetInfoRepository;
@Autowired
protected GridFsOperations operations;
@Autowired
protected RolloutGroupRepository rolloutGroupRepository;

View File

@@ -1,73 +0,0 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.repository.jpa;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import org.eclipse.hawkbit.cache.TenantAwareCacheManager;
import org.eclipse.hawkbit.repository.test.util.AbstractIntegrationTest;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.SpringApplicationConfiguration;
@SpringApplicationConfiguration(classes = { org.eclipse.hawkbit.RepositoryApplicationConfiguration.class })
public abstract class AbstractJpaIntegrationTestWithMongoDB extends AbstractIntegrationTest {
@PersistenceContext
protected EntityManager entityManager;
@Autowired
protected TargetRepository targetRepository;
@Autowired
protected ActionRepository actionRepository;
@Autowired
protected DistributionSetRepository distributionSetRepository;
@Autowired
protected SoftwareModuleRepository softwareModuleRepository;
@Autowired
protected TenantMetaDataRepository tenantMetaDataRepository;
@Autowired
protected DistributionSetTypeRepository distributionSetTypeRepository;
@Autowired
protected SoftwareModuleTypeRepository softwareModuleTypeRepository;
@Autowired
protected TargetTagRepository targetTagRepository;
@Autowired
protected DistributionSetTagRepository distributionSetTagRepository;
@Autowired
protected SoftwareModuleMetadataRepository softwareModuleMetadataRepository;
@Autowired
protected ActionStatusRepository actionStatusRepository;
@Autowired
protected LocalArtifactRepository artifactRepository;
@Autowired
protected TargetInfoRepository targetInfoRepository;
@Autowired
protected RolloutGroupRepository rolloutGroupRepository;
@Autowired
protected RolloutRepository rolloutRepository;
@Autowired
protected TenantAwareCacheManager cacheManager;
}

View File

@@ -1,73 +0,0 @@
/**
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*/
package org.eclipse.hawkbit.repository.jpa;
import static org.fest.assertions.api.Assertions.assertThat;
import static org.junit.Assert.fail;
import java.io.IOException;
import java.net.UnknownHostException;
import org.eclipse.hawkbit.repository.exception.ArtifactDeleteFailedException;
import org.eclipse.hawkbit.repository.exception.ArtifactUploadFailedException;
import org.eclipse.hawkbit.repository.jpa.model.JpaSoftwareModule;
import org.eclipse.hawkbit.repository.model.Artifact;
import org.junit.After;
import org.junit.Test;
import ru.yandex.qatools.allure.annotations.Description;
import ru.yandex.qatools.allure.annotations.Features;
import ru.yandex.qatools.allure.annotations.Stories;
@Features("Component Tests - Repository")
@Stories("Artifact Management")
public class ArtifactManagementFailedMongoDBTest extends AbstractJpaIntegrationTestWithMongoDB {
@Test
@Description("Trys and fails to delete or create local artifact with a down mongodb and checks if expected ArtifactDeleteFailedException is thrown.")
public void deleteArtifactsWithNoMongoDb() throws UnknownHostException, IOException {
// ensure baseline
assertThat(artifactRepository.findAll()).isEmpty();
// prepare test
JpaSoftwareModule sm = new JpaSoftwareModule(softwareManagement.findSoftwareModuleTypeByKey("os"), "name 1",
"version 1", null, null);
sm = softwareModuleRepository.save(sm);
final Artifact result = artifactManagement.createArtifact(new RandomGeneratedInputStream(5 * 1024), sm.getId(),
"file1", false);
assertThat(artifactRepository.findAll()).hasSize(1);
mongodExecutable.stop();
try {
artifactManagement.deleteArtifact(result.getId());
fail("deletion should have failed");
} catch (final ArtifactDeleteFailedException e) {
}
try {
artifactManagement.createArtifact(new RandomGeneratedInputStream(5 * 1024), sm.getId(), "file2", false);
fail("Should not have worked with MongoDb down.");
} catch (final ArtifactUploadFailedException e) {
}
assertThat(artifactRepository.findAll()).hasSize(1);
}
@Override
@After
public void cleanCurrentCollection() {
// no need to clean, is stopped already
}
}

View File

@@ -14,6 +14,7 @@ import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.NoSuchAlgorithmException;
import org.apache.commons.io.IOUtils;
@@ -28,8 +29,6 @@ import org.eclipse.hawkbit.repository.model.SoftwareModule;
import org.eclipse.hawkbit.repository.test.util.HashGeneratorUtils;
import org.eclipse.hawkbit.repository.test.util.WithUser;
import org.junit.Test;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import ru.yandex.qatools.allure.annotations.Description;
import ru.yandex.qatools.allure.annotations.Features;
@@ -44,7 +43,7 @@ import ru.yandex.qatools.allure.annotations.Stories;
*/
@Features("Component Tests - Repository")
@Stories("Artifact Management")
public class ArtifactManagementTest extends AbstractJpaIntegrationTestWithMongoDB {
public class ArtifactManagementTest extends AbstractJpaIntegrationTest {
/**
* Test method for
@@ -151,25 +150,17 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTestWithMongoD
assertThat(result2.getId()).isNotNull();
assertThat(((JpaArtifact) result).getGridFsFileName())
.isNotEqualTo(((JpaArtifact) result2).getGridFsFileName());
assertThat(operations.findOne(
new Query().addCriteria(Criteria.where("filename").is(((JpaArtifact) result).getGridFsFileName()))))
.isNotNull();
assertThat(operations.findOne(
new Query().addCriteria(Criteria.where("filename").is(((JpaArtifact) result2).getGridFsFileName()))))
.isNotNull();
assertThat(binaryArtifactRepository.getArtifactBySha1(((JpaArtifact) result).getGridFsFileName())).isNotNull();
assertThat(binaryArtifactRepository.getArtifactBySha1(((JpaArtifact) result2).getGridFsFileName())).isNotNull();
artifactManagement.deleteArtifact(result.getId());
assertThat(operations.findOne(
new Query().addCriteria(Criteria.where("filename").is(((JpaArtifact) result).getGridFsFileName()))))
.isNull();
assertThat(operations.findOne(
new Query().addCriteria(Criteria.where("filename").is(((JpaArtifact) result2).getGridFsFileName()))))
.isNotNull();
assertThat(binaryArtifactRepository.getArtifactBySha1(((JpaArtifact) result).getGridFsFileName())).isNull();
assertThat(binaryArtifactRepository.getArtifactBySha1(((JpaArtifact) result2).getGridFsFileName())).isNotNull();
artifactManagement.deleteArtifact(result2.getId());
assertThat(operations.findOne(
new Query().addCriteria(Criteria.where("filename").is(((JpaArtifact) result2).getGridFsFileName()))))
.isNull();
assertThat(binaryArtifactRepository.getArtifactBySha1(((JpaArtifact) result2).getGridFsFileName())).isNull();
assertThat(artifactRepository.findAll()).hasSize(0);
}
@@ -198,18 +189,12 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTestWithMongoD
assertThat(result2.getId()).isNotNull();
assertThat(((JpaArtifact) result).getGridFsFileName()).isEqualTo(((JpaArtifact) result2).getGridFsFileName());
assertThat(operations.findOne(
new Query().addCriteria(Criteria.where("filename").is(((JpaArtifact) result).getGridFsFileName()))))
.isNotNull();
assertThat(binaryArtifactRepository.getArtifactBySha1(((JpaArtifact) result).getGridFsFileName())).isNotNull();
artifactManagement.deleteArtifact(result.getId());
assertThat(operations.findOne(
new Query().addCriteria(Criteria.where("filename").is(((JpaArtifact) result).getGridFsFileName()))))
.isNotNull();
assertThat(binaryArtifactRepository.getArtifactBySha1(((JpaArtifact) result).getGridFsFileName())).isNotNull();
artifactManagement.deleteArtifact(result2.getId());
assertThat(operations.findOne(
new Query().addCriteria(Criteria.where("filename").is(((JpaArtifact) result).getGridFsFileName()))))
.isNull();
assertThat(binaryArtifactRepository.getArtifactBySha1(((JpaArtifact) result).getGridFsFileName())).isNull();
}
/**
@@ -253,8 +238,10 @@ public class ArtifactManagementTest extends AbstractJpaIntegrationTestWithMongoD
final Artifact result = artifactManagement.createArtifact(new ByteArrayInputStream(random), sm.getId(), "file1",
false);
assertTrue("The stored binary matches the given binary", IOUtils.contentEquals(new ByteArrayInputStream(random),
artifactManagement.loadArtifactBinary(result).getFileInputStream()));
try (InputStream fileInputStream = artifactManagement.loadArtifactBinary(result).getFileInputStream()) {
assertTrue("The stored binary matches the given binary",
IOUtils.contentEquals(new ByteArrayInputStream(random), fileInputStream));
}
}
@Test

View File

@@ -45,8 +45,6 @@ import org.eclipse.hawkbit.repository.test.util.WithUser;
import org.junit.Test;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -57,7 +55,7 @@ import ru.yandex.qatools.allure.annotations.Stories;
@Features("Component Tests - Repository")
@Stories("Software Management")
public class SoftwareManagementTest extends AbstractJpaIntegrationTestWithMongoDB {
public class SoftwareManagementTest extends AbstractJpaIntegrationTest {
@Test
@Description("Try to update non updatable fields results in repository doing nothing.")
@@ -387,7 +385,7 @@ public class SoftwareManagementTest extends AbstractJpaIntegrationTestWithMongoD
}
@Test
@Description("Deletes an artifact, which is assigned to a Distribution Set")
@Description("Deletes an artifact, which is assigned to a DistributionSet")
public void softDeleteOfAssignedArtifact() {
// Init DistributionSet
@@ -464,12 +462,9 @@ public class SoftwareManagementTest extends AbstractJpaIntegrationTestWithMongoD
}
@Test
@Description("Delete an softwaremodule with an artifact, which is also used by another softwaremodule.")
@Description("Delete an softwaremodule with an artifact, which is alsoused by another softwaremodule.")
public void deleteSoftwareModulesWithSharedArtifact() throws IOException {
// Precondition: Make sure MongoDB is Empty
assertThat(operations.find(new Query())).hasSize(0);
// Init artifact binary data, target and DistributionSets
final byte[] source = RandomUtils.nextBytes(1024);
@@ -489,9 +484,6 @@ public class SoftwareManagementTest extends AbstractJpaIntegrationTestWithMongoD
moduleY = softwareManagement.findSoftwareModuleById(moduleY.getId());
final Artifact artifactY = moduleY.getArtifacts().iterator().next();
// verify: that only one entry was created in mongoDB
assertThat(operations.find(new Query())).hasSize(1);
// [STEP5]: Delete SoftwareModuleX
softwareManagement.deleteSoftwareModule(moduleX.getId());
@@ -515,9 +507,6 @@ public class SoftwareManagementTest extends AbstractJpaIntegrationTestWithMongoD
@Description("Delete two assigned softwaremodules which share an artifact.")
public void deleteMultipleSoftwareModulesWhichShareAnArtifact() throws IOException {
// Precondition: Make sure MongoDB is Empty
assertThat(operations.find(new Query())).hasSize(0);
// Init artifact binary data, target and DistributionSets
final byte[] source = RandomUtils.nextBytes(1024);
final Target target = targetManagement.createTarget(new JpaTarget("test123"));
@@ -540,9 +529,6 @@ public class SoftwareManagementTest extends AbstractJpaIntegrationTestWithMongoD
moduleY = softwareManagement.findSoftwareModuleById(moduleY.getId());
final Artifact artifactY = moduleY.getArtifacts().iterator().next();
// verify: that only one entry was created in mongoDB
assertThat(operations.find(new Query())).hasSize(1);
// [STEP3]: Assign SoftwareModuleX to DistributionSetX and to target
distributionSetManagement.assignSoftwareModules(disSetX, Sets.newHashSet(moduleX));
deploymentManagement.assignDistributionSet(disSetX, Lists.newArrayList(target));
@@ -611,17 +597,14 @@ public class SoftwareManagementTest extends AbstractJpaIntegrationTestWithMongoD
assertThat(artifactRepository.findAll()).hasSize(results.length);
for (final Artifact result : results) {
assertThat(result.getId()).isNotNull();
assertThat(operations.findOne(
new Query().addCriteria(Criteria.where("filename").is(((JpaArtifact) result).getGridFsFileName()))))
.isNotNull();
assertThat(binaryArtifactRepository.getArtifactBySha1(((JpaArtifact) result).getGridFsFileName()))
.isNotNull();
}
}
private void assertArtfiactNull(final Artifact... results) {
for (final Artifact result : results) {
assertThat(operations.findOne(
new Query().addCriteria(Criteria.where("filename").is(((JpaArtifact) result).getGridFsFileName()))))
.isNull();
assertThat(binaryArtifactRepository.getArtifactBySha1(((JpaArtifact) result).getGridFsFileName())).isNull();
}
}

View File

@@ -28,7 +28,7 @@ import ru.yandex.qatools.allure.annotations.Stories;
@Features("Component Tests - Repository")
@Stories("System Management")
public class SystemManagementTest extends AbstractJpaIntegrationTestWithMongoDB {
public class SystemManagementTest extends AbstractJpaIntegrationTest {
@Test
@Description("Ensures that findTenants returns all tenants and not only restricted to the tenant which currently is logged in")

View File

@@ -30,7 +30,7 @@ import ru.yandex.qatools.allure.annotations.Stories;
@Features("Component Tests - Repository")
@Stories("Tenant Configuration Management")
public class TenantConfigurationManagementTest extends AbstractJpaIntegrationTestWithMongoDB {
public class TenantConfigurationManagementTest extends AbstractJpaIntegrationTest {
@Test
@Description("Tests that tenant specific configuration can be persisted and in case the tenant does not have specific configuration the default from environment is used instead.")

View File

@@ -31,7 +31,7 @@
</dependency>
<dependency>
<groupId>org.eclipse.hawkbit</groupId>
<artifactId>hawkbit-artifact-repository-mongo</artifactId>
<artifactId>hawkbit-artifact-repository-filesystem</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
@@ -43,8 +43,8 @@
<artifactId>spring-context-support</artifactId>
</dependency>
<dependency>
<groupId>de.flapdoodle.embed</groupId>
<artifactId>de.flapdoodle.embed.mongo</artifactId>
<groupId>org.springframework</groupId>
<artifactId>spring-tx</artifactId>
</dependency>
<dependency>
<groupId>net._01001111</groupId>

View File

@@ -13,6 +13,9 @@ import java.util.concurrent.Executors;
import org.eclipse.hawkbit.api.ArtifactUrlHandlerProperties;
import org.eclipse.hawkbit.api.PropertyBasedArtifactUrlHandler;
import org.eclipse.hawkbit.artifact.repository.ArtifactFilesystemProperties;
import org.eclipse.hawkbit.artifact.repository.ArtifactFilesystemRepository;
import org.eclipse.hawkbit.artifact.repository.ArtifactRepository;
import org.eclipse.hawkbit.cache.DefaultDownloadIdCache;
import org.eclipse.hawkbit.cache.DownloadIdCache;
import org.eclipse.hawkbit.cache.TenantAwareCacheManager;
@@ -53,7 +56,6 @@ import org.springframework.security.concurrent.DelegatingSecurityContextExecutor
import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
import org.springframework.util.AntPathMatcher;
import com.mongodb.MongoClientOptions;
/**
* Spring context configuration required for Dev.Environment.
@@ -64,10 +66,16 @@ import com.mongodb.MongoClientOptions;
@Configuration
@EnableGlobalMethodSecurity(prePostEnabled = true, mode = AdviceMode.PROXY, proxyTargetClass = false, securedEnabled = true)
@EnableConfigurationProperties({ HawkbitServerProperties.class, DdiSecurityProperties.class,
ArtifactUrlHandlerProperties.class })
ArtifactUrlHandlerProperties.class, ArtifactFilesystemProperties.class })
@Profile("test")
@EnableAutoConfiguration
public class TestConfiguration implements AsyncConfigurer {
@Bean
public ArtifactRepository artifactRepository(final ArtifactFilesystemProperties artifactFilesystemProperties) {
return new ArtifactFilesystemRepository(artifactFilesystemProperties);
}
@Bean
public TestRepositoryManagement testRepositoryManagement(final SystemSecurityContext systemSecurityContext,
final SystemManagement systemManagement) {
@@ -85,13 +93,6 @@ public class TestConfiguration implements AsyncConfigurer {
return new PropertyBasedArtifactUrlHandler(urlHandlerProperties);
}
@Bean
public MongoClientOptions options() {
return MongoClientOptions.builder().connectTimeout(500).maxWaitTime(500).connectionsPerHost(2)
.serverSelectionTimeout(500).build();
}
@Bean
public TenantAware tenantAware() {
return new SecurityContextTenantAware();

View File

@@ -12,8 +12,15 @@ import static org.eclipse.hawkbit.im.authentication.SpPermission.SpringEvalExpre
import static org.eclipse.hawkbit.im.authentication.SpPermission.SpringEvalExpressions.SYSTEM_ROLE;
import static org.junit.rules.RuleChain.outerRule;
import java.io.File;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
import org.eclipse.hawkbit.ExcludePathAwareShallowETagFilter;
import org.eclipse.hawkbit.TestConfiguration;
import org.eclipse.hawkbit.artifact.repository.ArtifactFilesystemProperties;
import org.eclipse.hawkbit.artifact.repository.ArtifactRepository;
import org.eclipse.hawkbit.cache.TenantAwareCacheManager;
import org.eclipse.hawkbit.repository.ArtifactManagement;
import org.eclipse.hawkbit.repository.ControllerManagement;
import org.eclipse.hawkbit.repository.DeploymentManagement;
@@ -54,13 +61,10 @@ import org.springframework.core.env.Environment;
import org.springframework.data.auditing.AuditingHandler;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.gridfs.GridFsOperations;
import org.springframework.hateoas.MediaTypes;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.annotation.DirtiesContext.ClassMode;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.context.web.WebAppConfiguration;
import org.springframework.test.web.servlet.MockMvc;
@@ -68,8 +72,6 @@ import org.springframework.test.web.servlet.setup.DefaultMockMvcBuilder;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.web.context.WebApplicationContext;
import de.flapdoodle.embed.mongo.MongodExecutable;
@RunWith(SpringJUnit4ClassRunner.class)
@WebAppConfiguration
@ActiveProfiles({ "test" })
@@ -80,7 +82,6 @@ import de.flapdoodle.embed.mongo.MongodExecutable;
// refreshed we e.g. get two instances of CacheManager which leads to very
// strange test failures.
@DirtiesContext(classMode = ClassMode.AFTER_CLASS)
@TestPropertySource(properties = { "spring.data.mongodb.port=0", "spring.mongodb.embedded.version=3.2.7" })
public abstract class AbstractIntegrationTest implements EnvironmentAware {
private final static Logger LOG = LoggerFactory.getLogger(AbstractIntegrationTest.class);
@@ -154,6 +155,15 @@ public abstract class AbstractIntegrationTest implements EnvironmentAware {
@Autowired
protected SystemSecurityContext systemSecurityContext;
@Autowired
private ArtifactFilesystemProperties artifactFilesystemProperties;
@Autowired
protected ArtifactRepository binaryArtifactRepository;
@Autowired
protected TenantAwareCacheManager cacheManager;
protected MockMvc mvc;
protected SoftwareModuleType osType;
@@ -165,12 +175,6 @@ public abstract class AbstractIntegrationTest implements EnvironmentAware {
@Autowired
protected TestdataFactory testdataFactory;
@Autowired
protected GridFsOperations operations;
@Autowired
protected MongodExecutable mongodExecutable;
@Autowired
protected ServiceMatcher serviceMatcher;
@@ -232,8 +236,8 @@ public abstract class AbstractIntegrationTest implements EnvironmentAware {
}
@After
public void cleanCurrentCollection() {
operations.delete(new Query());
public void cleanCurrentCollection() throws IOException {
FileUtils.deleteDirectory(new File(artifactFilesystemProperties.getPath()));
}
protected DefaultMockMvcBuilder createMvcWebAppContext() {