lingwei.kong/hawkbit
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

20250828 cleanup (#2639)

Browse Source 
* Cleanup

* Refactor artifact management
This commit is contained in:
Avgustin Marinov
2025-09-02 16:08:14 +03:00
committed by GitHub
parent 4f0a8893c7
commit 2a636328a0
305 changed files with 2253 additions and 4566 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
hawkbit-artifact/hawkbit-artifact-fs/src/main/java/org/eclipse/hawkbit/artifact/fs/FileArtifactProperties.java Normal file
View File

@@ -0,0 +1,26 @@
/**
* Copyright (c) 2025 Contributors to the Eclipse Foundation
*
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*/
package org.eclipse.hawkbit.artifact.fs;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
/**
* Configuration properties for the file-system repository, e.g. the base-path to store the files.
*/
@Data
@ConfigurationProperties("org.eclipse.hawkbit.repository.file")
public class FileArtifactProperties {
/**
* The base-path of the directory to store the artifacts.
*/
private String path = "./artifactrepo";
}

114
hawkbit-artifact/hawkbit-artifact-fs/src/main/java/org/eclipse/hawkbit/artifact/fs/FileArtifactStorage.java Normal file
View File

@@ -0,0 +1,114 @@
/**
* Copyright (c) 2025 Contributors to the Eclipse Foundation
*
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*/
package org.eclipse.hawkbit.artifact.fs;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import org.apache.commons.io.FileUtils;
import org.eclipse.hawkbit.artifact.AbstractArtifactStorage;
import org.eclipse.hawkbit.artifact.ArtifactStorage;
import org.eclipse.hawkbit.artifact.exception.ArtifactBinaryNotFoundException;
import org.eclipse.hawkbit.artifact.exception.ArtifactStoreException;
import org.eclipse.hawkbit.artifact.model.ArtifactHashes;
import org.eclipse.hawkbit.artifact.model.StoredArtifactInfo;
import org.springframework.validation.annotation.Validated;
/**
* Implementation of the {@link ArtifactStorage} to store artifacts on the file-system. The files are stored by their SHA1 hash of the
* artifact binary. Duplicate files with the same SHA1 hash will only be stored once.
* <p/>
* All files are stored flat in one base directory configured in the {@link FileArtifactProperties#getPath()}.
* <p/>
* Due to the limit of many file-systems of files within one directory, the files are stored in different subdirectories based on the last four
* digits of the SHA1-hash {@code (/basepath/[two digit sha1]/[two digit sha1])}.
*/
@Validated
public class FileArtifactStorage extends AbstractArtifactStorage {
private final FileArtifactProperties artifactResourceProperties;
public FileArtifactStorage(final FileArtifactProperties artifactResourceProperties) {
this.artifactResourceProperties = artifactResourceProperties;
}
@Override
public void deleteBySha1(final String tenant, final String sha1) {
FileUtils.deleteQuietly(getFile(tenant, sha1));
}
@Override
public InputStream getBySha1(final String tenant, final String sha1) {
final File file = getFile(tenant, sha1);
if (!file.exists()) {
throw new ArtifactBinaryNotFoundException(sha1);
}
try {
return new BufferedInputStream(new FileInputStream(file));
} catch (final FileNotFoundException e) {
throw new ArtifactBinaryNotFoundException(sha1);
}
}
@Override
public void deleteByTenant(final String tenant) {
FileUtils.deleteQuietly(Paths.get(artifactResourceProperties.getPath(), sanitizeTenant(tenant)).toFile());
}
@Override
public boolean existsBySha1(final String tenant, final String sha1) {
return getFile(tenant, sha1).exists();
}
@Override
protected StoredArtifactInfo store(final String tenant, final ArtifactHashes base16Hashes, final String contentType, final String tempFile)
throws IOException {
final File file = new File(tempFile);
final File fileSHA1Naming = getFile(tenant, base16Hashes.sha1());
if (fileSHA1Naming.exists()) {
FileUtils.deleteQuietly(file);
} else {
Files.move(file.toPath(), fileSHA1Naming.toPath());
}
return new StoredArtifactInfo(contentType, fileSHA1Naming.length(), base16Hashes);
}
private File getFile(final String tenant, final String sha1) {
// ensure that the sha1 is not a path traversal attack
if (sha1.indexOf('/') >= 0 || sha1.indexOf('\\') >= 0) {
throw new IllegalArgumentException("Invalid SHA-1 hash: " + sha1);
}
final File artifactDirectory = getSha1DirectoryPath(tenant, sha1).toFile();
if (!artifactDirectory.isDirectory()) {
if (artifactDirectory.isFile()) {
throw new ArtifactStoreException(artifactDirectory + " is a file, but a directory is required.");
} else if (!artifactDirectory.mkdirs()) {
throw new ArtifactStoreException("Fail to create directories: " + artifactDirectory);
}
}
return new File(artifactDirectory, sha1);
}
private Path getSha1DirectoryPath(final String tenant, final String sha1) {
final int length = sha1.length();
final String folder1 = sha1.substring(length - 4, length - 2);
final String folder2 = sha1.substring(length - 2, length);
return Paths.get(artifactResourceProperties.getPath(), sanitizeTenant(tenant), folder1, folder2);
}
}

36
hawkbit-artifact/hawkbit-artifact-fs/src/main/java/org/eclipse/hawkbit/autoconfigure/artifact/fs/FileArtifactStorageConfiguration.java Normal file
View File

@@ -0,0 +1,36 @@
/**
* Copyright (c) 2025 Contributors to the Eclipse Foundation
*
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*/
package org.eclipse.hawkbit.autoconfigure.artifact.fs;
import org.eclipse.hawkbit.artifact.ArtifactStorage;
import org.eclipse.hawkbit.artifact.fs.FileArtifactProperties;
import org.eclipse.hawkbit.artifact.fs.FileArtifactStorage;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* Configuration for the {@link FileArtifactStorage}.
*/
@Configuration
@EnableConfigurationProperties(FileArtifactProperties.class)
public class FileArtifactStorageConfiguration {
/**
* @param artifactFilesystemProperties the artifact file system properties
* @return Default {@link ArtifactStorage} implementation.
*/
@Bean
@ConditionalOnMissingBean
public ArtifactStorage artifactRepository(final FileArtifactProperties artifactFilesystemProperties) {
return new FileArtifactStorage(artifactFilesystemProperties);
}
}

1
hawkbit-artifact/hawkbit-artifact-fs/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports Normal file
View File

@@ -0,0 +1 @@
org.eclipse.hawkbit.autoconfigure.artifact.fs.FileArtifactStorageConfiguration

142
hawkbit-artifact/hawkbit-artifact-fs/src/test/java/org/eclipse/hawkbit/artifact/fs/FileArtifactStorageTest.java Normal file
View File

@@ -0,0 +1,142 @@
/**
* Copyright (c) 2025 Contributors to the Eclipse Foundation
*
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*/
package org.eclipse.hawkbit.artifact.fs;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.util.Random;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.assertj.core.api.Assertions;
import org.eclipse.hawkbit.artifact.AbstractArtifactStorage;
import org.eclipse.hawkbit.artifact.exception.ArtifactBinaryNotFoundException;
import org.eclipse.hawkbit.artifact.model.StoredArtifactInfo;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
/**
* Feature: Unit Tests - Artifact File System Repository<br/>
* Story: Test storing artifact binaries in the file-system
*/
@Slf4j
class FileArtifactStorageTest {
private static final String TENANT = "test_tenant";
private static FileArtifactProperties artifactResourceProperties;
private static FileArtifactStorage artifactFilesystemRepository;
@BeforeAll
static void setup() {
artifactResourceProperties = new FileArtifactProperties();
artifactResourceProperties.setPath(AbstractArtifactStorage.createTempFile(true).toString());
artifactFilesystemRepository = new FileArtifactStorage(artifactResourceProperties);
}
@AfterAll
static void afterClass() {
if (new File(artifactResourceProperties.getPath()).exists()) {
try {
FileUtils.deleteDirectory(new File(artifactResourceProperties.getPath()));
} catch (final IOException | IllegalArgumentException e) {
log.warn("Cannot delete file-directory", e);
}
}
}
/**
* Verifies that an artifact can be successfully stored in the file-system repository
*/
@Test
void storeSuccessfully() throws IOException {
final byte[] fileContent = randomBytes();
final StoredArtifactInfo artifact = storeRandomArtifact(fileContent);
final byte[] readContent = new byte[fileContent.length];
IOUtils.read(artifactFilesystemRepository.getBySha1(TENANT, artifact.getHashes().sha1()), readContent);
assertThat(readContent).isEqualTo(fileContent);
}
/**
* Verifies that an artifact can be successfully stored in the file-system repository
*/
@Test
void getStoredArtifactBasedOnSHA1Hash() throws IOException {
final byte[] fileContent = randomBytes();
final StoredArtifactInfo artifact = storeRandomArtifact(fileContent);
assertThat(artifactFilesystemRepository.getBySha1(TENANT, artifact.getHashes().sha1())).isNotNull();
}
/**
* Verifies that an artifact can be deleted in the file-system repository
*/
@Test
void deleteStoredArtifactBySHA1Hash() throws IOException {
final StoredArtifactInfo artifact = storeRandomArtifact(randomBytes());
artifactFilesystemRepository.deleteBySha1(TENANT, artifact.getHashes().sha1());
final String sha1Hash = artifact.getHashes().sha1();
assertThatExceptionOfType(ArtifactBinaryNotFoundException.class)
.isThrownBy(() -> artifactFilesystemRepository.getBySha1(TENANT, sha1Hash));
}
/**
* Verifies that all artifacts of a tenant can be deleted in the file-system repository
*/
@Test
void deleteStoredArtifactOfTenant() throws IOException {
final StoredArtifactInfo artifact = storeRandomArtifact(randomBytes());
artifactFilesystemRepository.deleteByTenant(TENANT);
final String sha1Hash = artifact.getHashes().sha1();
assertThatExceptionOfType(ArtifactBinaryNotFoundException.class)
.isThrownBy(() -> artifactFilesystemRepository.getBySha1(TENANT, sha1Hash));
}
/**
* Verifies that an artifact which does not exists is deleted quietly in the file-system repository
*/
@Test
void deleteArtifactWhichDoesNotExistsBySHA1HashWithoutException() throws IOException {
try {
artifactFilesystemRepository.deleteBySha1(TENANT, "sha1HashWhichDoesNotExists");
} catch (final Exception e) {
Assertions.fail("did not expect an exception while deleting a file which does not exists");
}
final StoredArtifactInfo artifact = storeRandomArtifact(randomBytes());
try {
artifactFilesystemRepository.deleteBySha1("tenantWhichDoesNotExist", artifact.getHashes().sha1());
} catch (final Exception e) {
Assertions.fail("did not expect an exception while deleting a file which does not exists");
}
}
private static byte[] randomBytes() {
final byte[] randomBytes = new byte[20];
new Random().nextBytes(randomBytes);
return randomBytes;
}
private StoredArtifactInfo storeRandomArtifact(final byte[] fileContent) throws IOException {
try (final ByteArrayInputStream inputStream = new ByteArrayInputStream(fileContent)) {
return artifactFilesystemRepository.store(TENANT, inputStream, "filename.tmp", "application/txt", null);
}
}
}