Initial check in accordance with Parallel IP
This commit is contained in:
Kai Zimmermann
@@ -0,0 +1,280 @@
|
||||
/**
|
||||
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
|
||||
*
|
||||
* All rights reserved. This program and the accompanying materials
|
||||
* are made available under the terms of the Eclipse Public License v1.0
|
||||
* which accompanies this distribution, and is available at
|
||||
* http://www.eclipse.org/legal/epl-v10.html
|
||||
*/
|
||||
package org.eclipse.hawkbit.artifact.repository;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.security.DigestOutputStream;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.eclipse.hawkbit.artifact.repository.model.DbArtifact;
|
||||
import org.eclipse.hawkbit.artifact.repository.model.DbArtifactHash;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.data.mongodb.core.MongoTemplate;
|
||||
import org.springframework.data.mongodb.core.query.Criteria;
|
||||
import org.springframework.data.mongodb.core.query.Query;
|
||||
import org.springframework.data.mongodb.gridfs.GridFsOperations;
|
||||
|
||||
import com.google.common.io.BaseEncoding;
|
||||
import com.google.common.io.ByteStreams;
|
||||
import com.mongodb.BasicDBObject;
|
||||
import com.mongodb.MongoClientException;
|
||||
import com.mongodb.MongoException;
|
||||
import com.mongodb.gridfs.GridFSDBFile;
|
||||
import com.mongodb.gridfs.GridFSFile;
|
||||
|
||||
/**
|
||||
* The file management which looks up all the file in the filestore.
|
||||
*
|
||||
*
|
||||
*
|
||||
*
|
||||
*/
|
||||
public class ArtifactStore implements ArtifactRepository {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(ArtifactStore.class);
|
||||
|
||||
/**
|
||||
* The mongoDB field which holds the filename of the file to download. SP
|
||||
* Server uses the SHA hash as a filename and lookup in the mongoDB.
|
||||
*/
|
||||
private static final String FILENAME = "filename";
|
||||
|
||||
/**
|
||||
* The mongoDB field which automatically calculated by the mongoDB.
|
||||
*/
|
||||
private static final String MD5 = "md5";
|
||||
|
||||
/**
|
||||
* The mongoDB field which holds the SHA1 hash, stored in the metadata
|
||||
* object.
|
||||
*/
|
||||
private static final String SHA1 = "sha1";
|
||||
|
||||
private static final String ID = "_id";
|
||||
|
||||
@Autowired
|
||||
private GridFsOperations gridFs;
|
||||
|
||||
MongoTemplate mongoTemplate;
|
||||
|
||||
/**
|
||||
* Retrieves a {@link GridFSDBFile} from the store by it's SHA1 hash.
|
||||
*
|
||||
* @param tenant
|
||||
* the tenant to retrieve the artifacts from, ignore case.
|
||||
* @param sha1Hash
|
||||
* the sha1-hash of the file to lookup.
|
||||
* @return The gridfs file object or {@code null} if no file exists.
|
||||
*/
|
||||
@Override
|
||||
public DbArtifact getArtifactBySha1(final String sha1Hash) {
|
||||
return map(gridFs.findOne(new Query().addCriteria(Criteria.where(FILENAME).is(sha1Hash))));
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves a {@link GridFSDBFile} from the store by it's MD5 hash.
|
||||
*
|
||||
* @param tenant
|
||||
* the tenant to retrieve the artifacts from, ignore case.
|
||||
* @param md5Hash
|
||||
* the md5-hash of the file to lookup.
|
||||
* @return The gridfs file object or {@code null} if no file exists.
|
||||
*/
|
||||
public DbArtifact getArtifactByMd5(final String md5Hash) {
|
||||
return map(gridFs.findOne(new Query().addCriteria(Criteria.where(MD5).is(md5Hash))));
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves a {@link GridFSDBFile} from the store by it's object id.
|
||||
*
|
||||
* @param tenant
|
||||
* the tenant to retrieve the artifacts from, ignore case.
|
||||
* @param id
|
||||
* the id of the file to lookup.
|
||||
* @return The gridfs file object or {@code null} if no file exists.
|
||||
*/
|
||||
@Override
|
||||
public DbArtifact getArtifactById(final String id) {
|
||||
return map(gridFs.findOne(new Query().addCriteria(Criteria.where(ID).is(id))));
|
||||
}
|
||||
|
||||
@Override
|
||||
public DbArtifact store(final InputStream content, final String filename, final String contentType) {
|
||||
return store(content, filename, contentType, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DbArtifact store(final InputStream content, final String filename, final String contentType,
|
||||
final DbArtifactHash hash) {
|
||||
File tempFile = null;
|
||||
try {
|
||||
LOGGER.debug("storing file {} of content {}", filename, contentType);
|
||||
tempFile = File.createTempFile("uploadFile", null);
|
||||
try (final FileOutputStream os = new FileOutputStream(tempFile)) {
|
||||
return store(content, contentType, os, tempFile, hash);
|
||||
}
|
||||
} catch (final IOException | MongoException e1) {
|
||||
throw new ArtifactStoreException(e1.getMessage(), e1);
|
||||
} finally {
|
||||
if (tempFile != null && !tempFile.delete()) {
|
||||
LOGGER.error("Could not delete temporary file: {}", tempFile);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteById(final String artifactId) {
|
||||
try {
|
||||
deleteArtifact(gridFs.findOne(new Query().addCriteria(Criteria.where(ID).is(artifactId))));
|
||||
} catch (final MongoException e) {
|
||||
throw new ArtifactStoreException(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteBySha1(final String sha1Hash) {
|
||||
try {
|
||||
deleteArtifact(gridFs.findOne(new Query().addCriteria(Criteria.where(FILENAME).is(sha1Hash))));
|
||||
} catch (final MongoException e) {
|
||||
throw new ArtifactStoreException(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
private void deleteArtifact(final GridFSDBFile dbFile) {
|
||||
if (dbFile != null) {
|
||||
try {
|
||||
gridFs.delete(new Query().addCriteria(Criteria.where(ID).is(dbFile.getId())));
|
||||
} catch (final MongoClientException e) {
|
||||
throw new ArtifactStoreException(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private DbArtifact store(final InputStream content, final String contentType, final FileOutputStream os,
|
||||
final File tempFile, final DbArtifactHash hash) {
|
||||
final GridFsArtifact storedArtifact;
|
||||
try {
|
||||
final String sha1Hash = computeSHA1Hash(content, os, hash != null ? hash.getSha1() : null);
|
||||
// upload if it does not exist already, check if file exists, not
|
||||
// tenant specific.
|
||||
final GridFSDBFile result = gridFs.findOne(new Query().addCriteria(Criteria.where(FILENAME).is(sha1Hash)));
|
||||
if (null == result) {
|
||||
try (FileInputStream inputStream = new FileInputStream(tempFile)) {
|
||||
final BasicDBObject metadata = new BasicDBObject();
|
||||
metadata.put(SHA1, sha1Hash);
|
||||
storedArtifact = map(gridFs.store(inputStream, sha1Hash, contentType, metadata));
|
||||
}
|
||||
} else {
|
||||
LOGGER.info("file with sha1 hash {} already exists in database, increase reference counter", sha1Hash);
|
||||
result.save();
|
||||
storedArtifact = map(result);
|
||||
}
|
||||
} catch (final NoSuchAlgorithmException | IOException e) {
|
||||
throw new ArtifactStoreException(e.getMessage(), e);
|
||||
}
|
||||
|
||||
if (hash != null && hash.getMd5() != null && !storedArtifact.getHashes().getMd5().equals(hash.getMd5())) {
|
||||
throw new HashNotMatchException("The given md5 hash " + hash.getMd5()
|
||||
+ " not matching the calculated md5 hash " + storedArtifact.getHashes().getMd5(),
|
||||
HashNotMatchException.MD5);
|
||||
}
|
||||
|
||||
return storedArtifact;
|
||||
|
||||
}
|
||||
|
||||
private String computeSHA1Hash(final InputStream stream, final FileOutputStream os, final String providedSHA1Sum)
|
||||
throws NoSuchAlgorithmException, IOException {
|
||||
String sha1Hash;
|
||||
// compute digest
|
||||
final MessageDigest md = MessageDigest.getInstance("SHA-1");
|
||||
final DigestOutputStream dos = new DigestOutputStream(os, md);
|
||||
ByteStreams.copy(stream, dos);
|
||||
dos.close();
|
||||
sha1Hash = BaseEncoding.base16().lowerCase().encode(md.digest());
|
||||
if (providedSHA1Sum != null && !providedSHA1Sum.equalsIgnoreCase(sha1Hash)) {
|
||||
throw new HashNotMatchException(
|
||||
"The given sha1 hash " + providedSHA1Sum + " not matching the calculated sha1 hash " + sha1Hash,
|
||||
HashNotMatchException.SHA1);
|
||||
}
|
||||
return sha1Hash;
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps a list of {@link GridFSDBFile} to paged list of {@link DbArtifact}s.
|
||||
*
|
||||
* @param tenant
|
||||
* the tenant
|
||||
* @param dbFiles
|
||||
* the list of mongoDB gridFs files.
|
||||
* @return a paged list of artifacts mapped from the given dbFiles
|
||||
*/
|
||||
private List<DbArtifact> map(final List<GridFSDBFile> dbFiles) {
|
||||
final List<DbArtifact> collect = dbFiles.stream().map(dbFile -> map(dbFile)).collect(Collectors.toList());
|
||||
return collect;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves a list of {@link GridFSDBFile} from the store by all SHA1
|
||||
* hashes.
|
||||
*
|
||||
* @param tenant
|
||||
* the tenant to retrieve the artifacts from, ignore case.
|
||||
* @param sha1Hashes
|
||||
* the sha1-hashes of the files to lookup.
|
||||
* @return list of artfiacts
|
||||
*/
|
||||
public List<DbArtifact> getArtifactsBySha1(final List<String> sha1Hashes) {
|
||||
return map(gridFs.find(new Query().addCriteria(Criteria.where(SHA1).in(sha1Hashes))));
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves a list of {@link GridFSDBFile} from the store by all ids.
|
||||
*
|
||||
* @param tenant
|
||||
* the tenant to retrieve the artifacts from, ignore case.
|
||||
* @param ids
|
||||
* the ids of the files to lookup.
|
||||
* @return list of artfiacts
|
||||
*/
|
||||
public List<DbArtifact> getArtifactsByIds(final List<String> ids) {
|
||||
return map(gridFs.find(new Query().addCriteria(Criteria.where(ID).in(ids))));
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps a single {@link GridFSFile} to {@link DbArtifact}.
|
||||
*
|
||||
* @param tenant
|
||||
* the tenant
|
||||
* @param dbFile
|
||||
* the mongoDB gridFs file.
|
||||
* @return a mapped artifact from the given dbFile
|
||||
*/
|
||||
private GridFsArtifact map(final GridFSFile fsFile) {
|
||||
if (fsFile == null) {
|
||||
return null;
|
||||
}
|
||||
final GridFsArtifact artifact = new GridFsArtifact(fsFile);
|
||||
artifact.setArtifactId(fsFile.getId().toString());
|
||||
artifact.setSize(fsFile.getLength());
|
||||
artifact.setContentType(fsFile.getContentType());
|
||||
artifact.setHashes(new DbArtifactHash(fsFile.getFilename(), fsFile.getMD5()));
|
||||
return artifact;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
/**
|
||||
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
|
||||
*
|
||||
* All rights reserved. This program and the accompanying materials
|
||||
* are made available under the terms of the Eclipse Public License v1.0
|
||||
* which accompanies this distribution, and is available at
|
||||
* http://www.eclipse.org/legal/epl-v10.html
|
||||
*/
|
||||
package org.eclipse.hawkbit.artifact.repository;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.ComponentScan;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.Import;
|
||||
|
||||
/**
|
||||
* Auto configuration for the {@link ArtifactStore}.
|
||||
*
|
||||
*
|
||||
*
|
||||
*/
|
||||
@Configuration
|
||||
@ComponentScan
|
||||
@ConditionalOnMissingBean(value = ArtifactRepository.class)
|
||||
@Import(value = MongoConfiguration.class)
|
||||
public class ArtifactStoreAutoConfiguration {
|
||||
|
||||
/**
|
||||
* @return Default {@link ArtifactRepository} implementation.
|
||||
*/
|
||||
@Bean
|
||||
public ArtifactRepository artifactRepository() {
|
||||
return new ArtifactStore();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
|
||||
*
|
||||
* All rights reserved. This program and the accompanying materials
|
||||
* are made available under the terms of the Eclipse Public License v1.0
|
||||
* which accompanies this distribution, and is available at
|
||||
* http://www.eclipse.org/legal/epl-v10.html
|
||||
*/
|
||||
package org.eclipse.hawkbit.artifact.repository;
|
||||
|
||||
import java.io.InputStream;
|
||||
|
||||
import org.eclipse.hawkbit.artifact.repository.model.DbArtifact;
|
||||
|
||||
import com.mongodb.gridfs.GridFSDBFile;
|
||||
import com.mongodb.gridfs.GridFSFile;
|
||||
|
||||
/**
|
||||
* A wrapper object for the {@link DbArtifact} object which returns the
|
||||
* {@link InputStream} directly from {@link GridFSDBFile#getInputStream()} which
|
||||
* retrieves when calling {@link #getFileInputStream()} always a new
|
||||
* {@link InputStream} and not the same.
|
||||
*
|
||||
*
|
||||
*
|
||||
*/
|
||||
public class GridFsArtifact extends DbArtifact {
|
||||
|
||||
private final GridFSFile dbFile;
|
||||
|
||||
/**
|
||||
* @param dbFile
|
||||
*/
|
||||
public GridFsArtifact(final GridFSFile dbFile) {
|
||||
this.dbFile = dbFile;
|
||||
}
|
||||
|
||||
@Override
|
||||
public InputStream getFileInputStream() {
|
||||
if (dbFile instanceof GridFSDBFile) {
|
||||
return ((GridFSDBFile) dbFile).getInputStream();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
/**
|
||||
* Copyright (c) 2015 Bosch Software Innovations GmbH and others.
|
||||
*
|
||||
* All rights reserved. This program and the accompanying materials
|
||||
* are made available under the terms of the Eclipse Public License v1.0
|
||||
* which accompanies this distribution, and is available at
|
||||
* http://www.eclipse.org/legal/epl-v10.html
|
||||
*/
|
||||
package org.eclipse.hawkbit.artifact.repository;
|
||||
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.Arrays;
|
||||
|
||||
import javax.annotation.PreDestroy;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||
import org.springframework.boot.autoconfigure.mongo.MongoProperties;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.Profile;
|
||||
import org.springframework.data.mongodb.config.AbstractMongoConfiguration;
|
||||
|
||||
import com.mongodb.Mongo;
|
||||
import com.mongodb.MongoClient;
|
||||
import com.mongodb.MongoClientOptions;
|
||||
import com.mongodb.MongoClientOptions.Builder;
|
||||
import com.mongodb.MongoClientURI;
|
||||
import com.mongodb.ServerAddress;
|
||||
|
||||
/**
|
||||
* {@link AbstractMongoConfiguration} that uses {@link MongoClientURI} even when
|
||||
* port is configured for NON {@link Cloud} use cases.
|
||||
*
|
||||
*/
|
||||
@Configuration
|
||||
@EnableConfigurationProperties(MongoProperties.class)
|
||||
@ConditionalOnClass(Mongo.class)
|
||||
@ConditionalOnMissingBean(type = "org.springframework.data.mongodb.MongoDbFactory")
|
||||
@Profile({ "!cloud" })
|
||||
public class MongoConfiguration extends AbstractMongoConfiguration {
|
||||
private static final Logger LOG = LoggerFactory.getLogger(MongoConfiguration.class);
|
||||
|
||||
@Autowired
|
||||
private MongoProperties properties;
|
||||
|
||||
@Autowired(required = false)
|
||||
private MongoClientOptions options;
|
||||
|
||||
private Mongo mongo;
|
||||
|
||||
@Override
|
||||
public String getDatabaseName() {
|
||||
return properties.getMongoClientDatabase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes mongo client when destroyd.
|
||||
*/
|
||||
@PreDestroy
|
||||
public void close() {
|
||||
if (this.mongo != null) {
|
||||
this.mongo.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public Mongo mongo() throws UnknownHostException {
|
||||
final MongoClientURI uri = new MongoClientURI(properties.getUri(), createBuilderOutOfOptions(options));
|
||||
|
||||
if (properties.getPort() != null) {
|
||||
LOG.debug("Create mongo by properties (host: {}, port: {})", uri.getHosts().get(0), properties.getPort());
|
||||
this.mongo = new MongoClient(Arrays.asList(new ServerAddress(uri.getHosts().get(0), properties.getPort())),
|
||||
uri.getOptions());
|
||||
} else {
|
||||
LOG.debug("Create mongo by URI : {}", uri);
|
||||
this.mongo = new MongoClient(uri);
|
||||
}
|
||||
|
||||
return this.mongo;
|
||||
}
|
||||
|
||||
/*
|
||||
* Creates {@link MongoClientOptions} builder out of existing options as the
|
||||
* {@link MongoClientURI} expects a builder.
|
||||
*
|
||||
* Based on MongoProperties#builder method.
|
||||
*/
|
||||
private Builder createBuilderOutOfOptions(final MongoClientOptions options) {
|
||||
final Builder builder = MongoClientOptions.builder();
|
||||
if (options != null) {
|
||||
builder.alwaysUseMBeans(options.isAlwaysUseMBeans());
|
||||
builder.connectionsPerHost(options.getConnectionsPerHost());
|
||||
builder.connectTimeout(options.getConnectTimeout());
|
||||
builder.cursorFinalizerEnabled(options.isCursorFinalizerEnabled());
|
||||
builder.dbDecoderFactory(options.getDbDecoderFactory());
|
||||
builder.dbEncoderFactory(options.getDbEncoderFactory());
|
||||
builder.description(options.getDescription());
|
||||
builder.maxWaitTime(options.getMaxWaitTime());
|
||||
builder.readPreference(options.getReadPreference());
|
||||
builder.serverSelectionTimeout(options.getServerSelectionTimeout());
|
||||
builder.socketFactory(options.getSocketFactory());
|
||||
builder.socketKeepAlive(options.isSocketKeepAlive());
|
||||
builder.socketTimeout(options.getSocketTimeout());
|
||||
builder.threadsAllowedToBlockForConnectionMultiplier(options
|
||||
.getThreadsAllowedToBlockForConnectionMultiplier());
|
||||
builder.writeConcern(options.getWriteConcern());
|
||||
}
|
||||
return builder;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
# Auto Configure
|
||||
org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
|
||||
org.eclipse.hawkbit.artifact.repository.ArtifactStoreAutoConfiguration
|
||||
Reference in New Issue
Block a user