Openapi restdoc generation v2 (#1442)

* OpenApi restdoc generation v2

Signed-off-by: Denislav Prinov <denislav.prinov@bosch.com>

* Attach json and yaml artifacts only if -DskipTests is not provided

Signed-off-by: Denislav Prinov <denislav.prinov@bosch.com>

* Add missing header

Signed-off-by: Denislav Prinov <denislav.prinov@bosch.com>

* Add license header

Signed-off-by: Denislav Prinov <denislav.prinov@bosch.com>

---------

Signed-off-by: Denislav Prinov <denislav.prinov@bosch.com>

docs/README.md

@@ -4,10 +4,9 @@ theme. Compiling the documentation is not included within the regular Maven buil
 
 ## Prerequisites
 1. **Install Hugo**: see [installing Hugo](https://gohugo.io/getting-started/installing/) documentation on how to install Hugo.
-2. **Install JQ**: see [installing jq](https://jqlang.github.io/jq/download/) documentation on how to install jq.
+2. **Install NODE.js and npm** see [installing Node.js and npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) documentation on how to install Node.js and npm
-3. **Install NODE.js and npm** see [installing Node.js and npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) documentation on how to install Node.js and npm
+3. **Install Redocly CLI** see [installing Redocly CLI](https://redocly.com/docs/cli/installation/) documentation on how to install Redocly CLI
-4. **Install Redocly CLI** see [installing Redocly CLI](https://redocly.com/docs/cli/installation/) documentation on how to install Redocly CLI
+4. **Install hawkBit**: run `mvn install` in the parent directory to generate the latest REST docs for hawkBit.
-5. **Install hawkBit**: run `mvn install` in the parent directory to generate the latest REST docs for hawkBit.
 
 
 ## Build and Serve documentation

docs/build-htmls.sh

@@ -24,7 +24,7 @@ else
 fi
 
 # Execute the npx command
-npx @redocly/cli build-docs ${CURRENT_DIR}/content/rest-api/mgmt.json -o ${CURRENT_DIR}/content/rest-api/mgmt.html
+npx @redocly/cli build-docs ${CURRENT_DIR}/content/rest-api/mgmt.yaml -o ${CURRENT_DIR}/content/rest-api/mgmt.html
 
 if [ $? != 0 ]; then
     echo "[ERROR] Failed to execute the Redoc CLI command form MGMT API."
@@ -34,7 +34,7 @@ else
 fi
 
 # Execute the npx command
-npx @redocly/cli build-docs ${CURRENT_DIR}/content/rest-api/ddi.json -o ${CURRENT_DIR}/content/rest-api/ddi.html
+npx @redocly/cli build-docs ${CURRENT_DIR}/content/rest-api/ddi.yaml -o ${CURRENT_DIR}/content/rest-api/ddi.html
 
 if [ $? != 0 ]; then
     echo "[ERROR] Failed to execute the Redoc CLI command form DDI API."

docs/pom.xml

@@ -72,11 +72,21 @@
                            <groupId>org.eclipse.hawkbit</groupId>
                            <artifactId>hawkbit-update-server</artifactId>
                            <version>${project.version}</version>
-                           <classifier>openapi</classifier>
+                           <classifier>mgmt-openapi</classifier>
-                           <type>json</type>
+                           <type>yaml</type>
                            <overWrite>true</overWrite>
                            <outputDirectory>${basedir}/content/rest-api</outputDirectory>
-                           <destFileName>openapi.json</destFileName>
+                           <destFileName>mgmt.yaml</destFileName>
+                        </artifactItem>
+                        <artifactItem>
+                           <groupId>org.eclipse.hawkbit</groupId>
+                           <artifactId>hawkbit-update-server</artifactId>
+                           <version>${project.version}</version>
+                           <classifier>ddi-openapi</classifier>
+                           <type>yaml</type>
+                           <overWrite>true</overWrite>
+                           <outputDirectory>${basedir}/content/rest-api</outputDirectory>
+                           <destFileName>ddi.yaml</destFileName>
                         </artifactItem>
                      </artifactItems>
                   </configuration>
@@ -88,21 +98,6 @@
             <artifactId>exec-maven-plugin</artifactId>
             <version>${exec-maven-plugin.version}</version>
             <executions>
-               <execution>
-                  <id>split-docs</id>
-                  <goals>
-                     <goal>exec</goal>
-                  </goals>
-                  <phase>install</phase>
-                  <configuration>
-                     <executable>${shell}</executable>
-                     <workingDirectory>${project.basedir}</workingDirectory>
-                     <arguments>
-                        <argument>${shell.option}</argument>
-                        <argument>split-doc.${batch.ext}</argument>
-                     </arguments>
-                  </configuration>
-               </execution>
                <execution>
                   <id>build-htmls</id>
                   <goals>

docs/split-doc.sh

@@ -1,52 +0,0 @@
-#
-# Copyright (c) 2018 Bosch Software Innovations GmbH and others
-#
-# This program and the accompanying materials are made
-# available under the terms of the Eclipse Public License 2.0
-# which is available at https://www.eclipse.org/legal/epl-2.0/
-#
-# SPDX-License-Identifier: EPL-2.0
-#
-
-#!/bin/bash
-
-CURRENT_DIR=$(pwd)
-input_file=${CURRENT_DIR}/content/rest-api/openapi.json
-mgmt_file=${CURRENT_DIR}/content/rest-api/mgmt.json
-ddi_file=${CURRENT_DIR}/content/rest-api/ddi.json
-
-jq '
-  .paths |= with_entries(
-    select(
-      reduce .value[] as $item (
-        false;
-        . or ($item.tags? | index("DDI Root Controller")) == null
-      )
-    )
-  )
-  | .tags |= map(select(.name | contains("DDI") | not))
-  | .components.schemas = (.components.schemas | with_entries(select(.key | startswith("Ddi") | not)))
-' "$input_file" > "$mgmt_file"
-
-jq '
-  .paths |= with_entries(
-    select(
-      reduce .value[] as $item (
-        false;
-        . or ($item.tags? | index("DDI Root Controller")) != null
-      )
-    )
-  )
-  | .tags |= map(select(.name | contains("DDI")))
-  | .components.schemas = (
-        .components.schemas
-        | with_entries(
-            select(
-                (.key | startswith("Ddi"))
-                or (.key | . == "Link")
-                or (.key | . == "ExceptionInfo")
-            )
-        )
-    )
-' "$input_file" > "$ddi_file"
-

hawkbit-runtime/hawkbit-update-server/pom.xml

@@ -32,65 +32,63 @@
                 </plugins>
             </build>
         </profile>
+        <profile>
+            <id>attach-artifacts-profile</id>
+            <!-- This profile activates when -DskipTests is not provided -->
+            <activation>
+                <property>
+                    <name>!skipTests</name>
+                </property>
+            </activation>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.codehaus.mojo</groupId>
+                        <artifactId>build-helper-maven-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <id>attach-artifacts</id>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>attach-artifact</goal>
+                                </goals>
+                                <configuration>
+                                    <artifacts>
+                                        <artifact>
+                                            <file>${project.build.directory}/rest-api/mgmt-openapi.json</file>
+                                            <type>json</type>
+                                            <classifier>mgmt-openapi</classifier>
+                                        </artifact>
+                                        <artifact>
+                                            <file>${project.build.directory}/rest-api/mgmt-openapi.yaml</file>
+                                            <type>yaml</type>
+                                            <classifier>mgmt-openapi</classifier>
+                                        </artifact>
+                                        <artifact>
+                                            <file>${project.build.directory}/rest-api/ddi-openapi.json</file>
+                                            <type>json</type>
+                                            <classifier>ddi-openapi</classifier>
+                                        </artifact>
+                                        <artifact>
+                                            <file>${project.build.directory}/rest-api/ddi-openapi.yaml</file>
+                                            <type>yaml</type>
+                                            <classifier>ddi-openapi</classifier>
+                                        </artifact>
+                                    </artifacts>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
     </profiles>
     <build>
         <plugins>
-            <!-- Generating the rest doc during integration tests -->
-            <plugin>
-                <groupId>org.springdoc</groupId>
-                <artifactId>springdoc-openapi-maven-plugin</artifactId>
-                <version>1.4</version>
-                <executions>
-                    <!-- Execution for JSON -->
-                    <execution>
-                        <id>generate-json</id>
-                        <phase>integration-test</phase>
-                        <goals>
-                            <goal>generate</goal>
-                        </goals>
-                        <configuration>
-                            <outputFileName>openapi.json</outputFileName>
-                            <apiDocsUrl>http://localhost:8080/v3/api-docs</apiDocsUrl>
-                            <attachArtifact>true</attachArtifact>
-                        </configuration>
-                    </execution>
-                    <!-- Execution for YAML -->
-                    <execution>
-                        <id>generate-yaml</id>
-                        <phase>integration-test</phase>
-                        <goals>
-                            <goal>generate</goal>
-                        </goals>
-                        <configuration>
-                            <outputFileName>openapi.yaml</outputFileName>
-                            <apiDocsUrl>http://localhost:8080/v3/api-docs.yaml</apiDocsUrl>
-                            <attachArtifact>true</attachArtifact>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
             <plugin>
                 <groupId>org.springframework.boot</groupId>
                 <artifactId>spring-boot-maven-plugin</artifactId>
-                <configuration>
-                    <jvmArguments>-Dspring.application.admin.enabled=true</jvmArguments>
-                    <mainClass>org.eclipse.hawkbit.app.Start</mainClass>
-                </configuration>
                 <executions>
-                    <execution>
-                        <phase>pre-integration-test</phase>
-                        <id>pre-api-docs-generation</id>
-                        <goals>
-                            <goal>start</goal>
-                        </goals>
-                    </execution>
-                    <execution>
-                        <id>post-api-docs-generation</id>
-                        <phase>post-integration-test</phase>
-                        <goals>
-                            <goal>stop</goal>
-                        </goals>
-                    </execution>
                     <execution>
                         <goals>
                             <goal>repackage</goal>

hawkbit-runtime/hawkbit-update-server/src/test/java/org/eclipse/hawkbit/app/RestApiDocTest.java

@@ -0,0 +1,154 @@
+/**
+ * Copyright (c) 2023 Bosch.IO GmbH and others
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+
+package org.eclipse.hawkbit.app;
+
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.dataformat.yaml.YAMLMapper;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.web.client.TestRestTemplate;
+import org.springframework.boot.test.web.server.LocalServerPort;
+import org.springframework.http.ResponseEntity;
+import org.springframework.test.context.ActiveProfiles;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@ActiveProfiles({"maven"})
+class RestApiDocTest {
+    private static final String MANAGEMENT_PREFIX = "mgmt-openapi";
+    private static final String DDI_PREFIX = "ddi-openapi";
+    private static final String TARGET_DIRECTORY = "target/rest-api/";
+
+    @LocalServerPort
+    private int port;
+
+    @Autowired
+    private TestRestTemplate restTemplate;
+
+    private final ObjectMapper objectMapper = new ObjectMapper();
+
+    @Test
+    void openapiJson() throws IOException {
+        ResponseEntity<String> response =
+                restTemplate.getForEntity("http://localhost:" + port + "/v3/api-docs", String.class);
+        String openapiDoc = response.getBody();
+        assertThat(openapiDoc).isNotNull();
+        splitDocumentation(openapiDoc);
+    }
+
+    private void splitDocumentation(String json) throws IOException {
+        processDocumentation(json, true);
+        processDocumentation(json, false);
+    }
+
+    private void processDocumentation(String json, boolean isMgmt) throws IOException {
+        JsonNode rootNode = objectMapper.readTree(json);
+        updateJsonNodeForApi(rootNode, isMgmt);
+        saveDocumentation(rootNode, isMgmt);
+    }
+
+    private void updateJsonNodeForApi(JsonNode rootNode, boolean isMgmt) {
+        removeTags(rootNode, isMgmt);
+        removePaths(rootNode, isMgmt);
+        removeComponents(rootNode, isMgmt);
+    }
+
+    private void removeTags(JsonNode rootNode, boolean isMgmt) {
+        ArrayNode tagsNode = (ArrayNode) rootNode.get("tags");
+        ArrayNode modifiedTagsNode = objectMapper.createArrayNode();
+
+        for (JsonNode tagNode : tagsNode) {
+            String tagName = tagNode.get("name").asText();
+            if (isMgmt != tagName.startsWith("DDI")) {
+                modifiedTagsNode.add(tagNode);
+            }
+        }
+
+        ((ObjectNode) rootNode).set("tags", modifiedTagsNode);
+    }
+    private void removePaths(JsonNode rootNode, boolean isMgmt) {
+        ObjectNode pathsNode = (ObjectNode) rootNode.get("paths");
+        List<String> fieldsToRemove = new ArrayList<>();
+        pathsNode.fieldNames().forEachRemaining(fieldName -> {
+            JsonNode pathNode = pathsNode.get(fieldName);
+            pathNode.fieldNames().forEachRemaining(path -> {
+                JsonNode methodNode = pathNode.get(path);
+                JsonNode tagsNode = methodNode.get("tags");
+                if (tagsNode != null) {
+                    for (JsonNode tagNode : tagsNode) {
+                        String tag = tagNode.asText();
+                        if (isMgmt == tag.startsWith("DDI")) {
+                            fieldsToRemove.add(fieldName);
+                            break;
+                        }
+                    }
+                }
+            });
+        });
+        fieldsToRemove.forEach(pathsNode::remove);
+    }
+
+    private void removeComponents(JsonNode rootNode, boolean isMgmt) {
+        ObjectNode schemasNode = (ObjectNode) rootNode.get("components").get("schemas");
+
+        List<String> fieldsToRemove = new ArrayList<>();
+        schemasNode.fieldNames().forEachRemaining(fieldName -> {
+            if (shouldDeleteComponent(fieldName, isMgmt)) {
+                fieldsToRemove.add(fieldName);
+            }
+        });
+        fieldsToRemove.forEach(schemasNode::remove);
+    }
+
+    private boolean shouldDeleteComponent(String fieldName, boolean isMgmt) {
+        if (isMgmt) {
+            return fieldName.startsWith("Ddi");
+        }
+        return !(fieldName.startsWith("Ddi") || fieldName.equals("Link") || fieldName.equals("ExceptionInfo"));
+
+    }
+
+    private void saveDocumentation(JsonNode rootNode, boolean isMgmt) throws IOException {
+        String prefix = isMgmt ? MANAGEMENT_PREFIX : DDI_PREFIX;
+        saveAsJson(rootNode, prefix);
+        saveAsYaml(rootNode, prefix);
+    }
+
+    private void saveAsJson(JsonNode rootNode, String prefix) throws IOException {
+        Path targetPath = getTargetPath(prefix, ".json");
+        Files.writeString(targetPath, objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(rootNode));
+    }
+
+    private void saveAsYaml(JsonNode rootNode, String prefix) throws IOException {
+        YAMLMapper yamlMapper = new YAMLMapper();
+        Path targetPath = getTargetPath(prefix, ".yaml");
+        Files.writeString(targetPath, yamlMapper.writeValueAsString(rootNode));
+    }
+
+    private Path getTargetPath(String prefix, String extension) throws IOException {
+        Path targetPath = Paths.get(TARGET_DIRECTORY + prefix + extension);
+        Files.createDirectories(targetPath.getParent());
+        return targetPath;
+    }
+}
+