diff --git a/.gitignore b/.gitignore index cf3154a..7f5329a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ bin .idea .vscode /api.yaml -*.zip \ No newline at end of file +*.zip +apis \ No newline at end of file diff --git a/src/main/java/cz/trask/migration/ApiSync.java b/src/main/java/cz/trask/migration/ApiSync.java index 98ca83b..5e1ab1f 100644 --- a/src/main/java/cz/trask/migration/ApiSync.java +++ b/src/main/java/cz/trask/migration/ApiSync.java @@ -5,6 +5,7 @@ import org.apache.logging.log4j.Logger; import cz.trask.migration.impl.v32.Wso2AppsToApicurio; import cz.trask.migration.impl.v32.Wso2v32ToApicurio; +import cz.trask.migration.impl.v32.Wso2v32ToApicurioFromDir; import cz.trask.migration.impl.v45.ExportApisToWso2FromV32; import cz.trask.migration.impl.v45.ExportAppsToWso2FromV32; import cz.trask.migration.model.StartParameters; @@ -27,6 +28,10 @@ public class ApiSync { log.info("wso2ApisToApicurio command selected."); Wso2v32ToApicurio imp = new Wso2v32ToApicurio(); imp.process(); + } else if (sp.getCommand().equalsIgnoreCase("wso2ApiFilesToApicurio")) { + log.info("wso2ApiFilesToApicurio command selected."); + Wso2v32ToApicurioFromDir imp = new Wso2v32ToApicurioFromDir(); + imp.process(); } else if (sp.getCommand().equalsIgnoreCase("apicurioApisToWso2")) { log.info("apicurioApisToWso2 command selected."); ExportApisToWso2FromV32 exp = new ExportApisToWso2FromV32(); diff --git a/src/main/java/cz/trask/migration/impl/v32/Wso2v32ToApicurioFromDir.java b/src/main/java/cz/trask/migration/impl/v32/Wso2v32ToApicurioFromDir.java new file mode 100644 index 0000000..c909269 --- /dev/null +++ b/src/main/java/cz/trask/migration/impl/v32/Wso2v32ToApicurioFromDir.java @@ -0,0 +1,331 @@ +package cz.trask.migration.impl.v32; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import cz.trask.migration.AbstractProcess; +import cz.trask.migration.model.APIInfo; +import cz.trask.migration.model.FileType; +import cz.trask.migration.model.ZipEntryData; +import cz.trask.migration.model.v32.ApiDefinition32; +import cz.trask.migration.model.v32.Subscriptions; +import cz.trask.migration.model.v32.Subscriptions.ApplicationInfo; +import cz.trask.migration.model.v32.Subscriptions.Subscription; +import io.apicurio.registry.rest.client.exception.VersionAlreadyExistsException; +import io.apicurio.registry.rest.v2.beans.ArtifactMetaData; +import io.apicurio.registry.rest.v2.beans.ArtifactReference; +import io.apicurio.registry.rest.v2.beans.VersionSearchResults; +import io.apicurio.registry.types.RuleType; +import lombok.extern.log4j.Log4j2; + +@Log4j2 +public class Wso2v32ToApicurioFromDir extends AbstractProcess { + + private final AtomicInteger apiCounter = new AtomicInteger(1); + + /** + * Main entry point for the import process. + * + * @throws RuntimeException if any error occurs + */ + public void process() { + try { + log.info("Starting API import to Apicurio from WSO2 Directory..."); + + File[] apiFiles = new File(config.getSource().getWso2ApisDir()) + .listFiles((dir, name) -> name.endsWith(".zip")); + if (apiFiles == null || apiFiles.length == 0) { + log.warn("No API zip files found in directory: {}", config.getSource().getWso2ApisDir()); + return; + } + + log.info("Found {} APIs", apiFiles.length); + + int maxThreads = config.getMaxThreads(); + ExecutorService executor = Executors.newFixedThreadPool(maxThreads); + + for (File api : apiFiles) { + final int index = apiCounter.getAndIncrement(); + executor.submit(() -> processApi(api, index, apiFiles.length)); + } + + executor.shutdown(); + if (!executor.awaitTermination(10, TimeUnit.MINUTES)) { + log.warn("Timeout waiting for API import tasks to finish"); + } + log.info("Finished processing APIs."); + } catch (Exception e) { + log.error("Error while exporting APIs.", e); + throw new RuntimeException("Export failed", e); + } + } + + /** + * Process a single API – fetches the data, creates or updates the corresponding + * artifact in Apicurio. + */ + private void processApi(File apiFile, int index, int total) { + long start = System.currentTimeMillis(); + + try { + log.info("Processing API {} of {}", index, total); + + List zipEntries = ZipUtils.extractFilesFromZip(Files.readAllBytes(apiFile.toPath())); + + String swagger = null; + String apiDefStr = null; + + for (ZipEntryData e : zipEntries) { + if (swagger != null && apiDefStr != null) { + break; + } + if (e.getType().toString().equals(FileType.OPENAPI.toString())) { + log.debug("Found main API swagger file: {}", e.getName()); + swagger = new String(e.getContent()); + } else if (e.getType().toString().equals(FileType.APIDEF.toString())) { + log.debug("Found main API definition file: {}", e.getName()); + apiDefStr = new String(e.getContent()); + } + } + + ApiDefinition32 apiDef = mapper.readValue(apiDefStr, ApiDefinition32.class); + Map swaggerMap = mapper.readValue(swagger, Map.class); + + APIInfo api = new APIInfo(); + api.setName(apiDef.getId().getApiName()); + api.setVersion(apiDef.getId().getVersion()); + api.setContext(apiDef.getContext()); + api.setDescription((String) ((Map) swaggerMap.get("info")).get("description")); + api.setType(apiDef.getType()); + + System.out.println("Context: " + apiDef.getContext()); + + @SuppressWarnings("unchecked") + List tagsList = (List) apiDef.getTags(); + + Map props = new LinkedHashMap<>(); + props.put("version", api.getVersion()); + props.put("status", apiDef.getStatus()); + props.put(PARAM_SOURCE_APIM, VERSION_32); + // addSubscriptionsToProps(props, subs); + // addEndpointsToProps(props, apiMap); + addTagsToProps(props, tagsList); + + String baseDesc = api.getDescription() != null ? api.getDescription() : ""; + String pubUrl = config.getPatterns().getPublisherUrlPattern().replace("{API_ID}", + api.getId()); + String devPortUrl = config.getPatterns().getDevPortalUrlPattern().replace("{API_ID}", + api.getId()); + + String fullDesc = baseDesc + " ***** PUBLISHER URL ***** " + pubUrl + " ***** DEVPORTAL URL ***** " + + devPortUrl; + + ObjectNode swaggerObj = mapperYaml.valueToTree(swaggerMap); + // updateSwagger(swaggerObj, apiMap, fullDesc); + + String group = config.getApicurio().getDefaultApiGroup(); + String mainArtifactId = api.getName() + api.getContext(); + + VersionSearchResults existingArtifacts; + try { + existingArtifacts = client.listArtifactVersions(group, mainArtifactId, 0, + Integer.MAX_VALUE); + } catch (Exception e) { + log.debug("No API {} exists – will create it", api.getContext()); + existingArtifacts = null; + } + + if (existingArtifacts == null) { + // Create new artifact + List references = createReferencesFromZip(zipEntries, + api); + // addSubscriptionsToReferences(references, subs, api); + + ArtifactMetaData meta = client.createArtifact(group, mainArtifactId, + api.getVersion(), null, null, null, + api.getName(), fullDesc, null, null, null, + new ByteArrayInputStream(swaggerObj.toString().getBytes()), references); + + setArtifactMetaData(meta, props); + // Create the three required rules + createRule(meta, "NONE", RuleType.COMPATIBILITY); + createRule(meta, "NONE", RuleType.VALIDITY); + createRule(meta, "NONE", RuleType.INTEGRITY); + + } else { + // Artifact exists – check if the version exists + boolean versionExists = false; + try { + client.getArtifactVersionMetaData(group, mainArtifactId, api.getVersion()); + versionExists = true; + } catch (Exception e) { + // Version missing – will create it below + } + + List references = createReferencesFromZip(zipEntries, + api); + // addSubscriptionsToReferences(references, subs, api); + + if (!versionExists) { + ArtifactMetaData meta = client.updateArtifact(group, mainArtifactId, + api.getVersion(), + api.getName(), fullDesc, new ByteArrayInputStream(swaggerObj.toString().getBytes()), + references); + setArtifactMetaData(meta, props); + } else { + // Version already exists – no action needed + log.warn("API {} with version {} already exists. Skipping import.", + api.getContext(), + api.getVersion()); + } + } + + log.info("Successfully imported API '{}' ({}). Took {} ms", api.getName(), + api.getVersion(), + System.currentTimeMillis() - start); + } catch (IOException e) { + log.error("IO error while importing API file {}: {}", apiFile.getName(), e.getMessage(), e); + } catch (VersionAlreadyExistsException e) { + log.warn("API version already exists for file: {}. Skipping.", apiFile.getName(), e.getMessage(), e); + } catch (Exception e) { + log.error("Cannot export API '{}': {}", apiFile.getName(), e.getMessage(), e); + } + } + + /* --------------------------------------------------------------------- */ + /* Helper methods */ + /* --------------------------------------------------------------------- */ + + private void updateSwagger(ObjectNode swagger, Map apiMap, String description) { + // Update "info.description" + ObjectNode info = (ObjectNode) swagger.get("info"); + if (info != null) { + info.put("description", description); + } + + // Build "servers" array + ArrayNode servers = mapper.createArrayNode(); + + List> endpoints = (List>) apiMap.get("endpointURLs"); + if (endpoints != null) { + for (Map env : endpoints) { + Map urls = (Map) env.get("URLs"); + if (urls == null || urls.isEmpty()) + continue; + + ObjectNode server = mapper.createObjectNode(); + urls.forEach((k, v) -> { + if (v != null && !v.isBlank()) { + if (k.equals("https") || k.equals("wss")) { + server.put("url", v); + } + } + }); + server.put("description", "Gateway: " + env.getOrDefault("environmentName", "")); + servers.add(server); + } + } + + // Replace "servers" node + swagger.set("servers", servers); + } + + private void addSubscriptionsToProps(Map props, Subscriptions subs) { + if (subs == null || subs.getList() == null || subs.getList().isEmpty()) + return; + int i = 1; + for (Subscription sub : subs.getList()) { + ApplicationInfo appInfo = sub.getApplicationInfo(); + if (appInfo == null) + continue; + props.put("subscription" + i, appInfo.getName() + " (Owner: " + appInfo.getSubscriber() + ")"); + i++; + } + } + + private void addEndpointsToProps(Map props, Map apiMap) { + if (apiMap == null || !apiMap.containsKey("endpointURLs")) + return; + @SuppressWarnings("unchecked") + List> envs = (List>) apiMap.get("endpointURLs"); + for (Map env : envs) { + @SuppressWarnings("unchecked") + Map urls = (Map) env.get("URLs"); + if (urls == null) + continue; + urls.forEach((k, v) -> { + if (v != null) + props.put(k + " Endpoint", v); + }); + } + } + + private void addTagsToProps(Map props, List tags) { + if (tags != null && !tags.isEmpty()) { + props.put("tags", String.join(", ", tags)); + } + } + + private List createReferencesFromZip(List zipEntries, APIInfo api) + throws IOException { + + List references = new ArrayList<>(); + for (ZipEntryData entry : zipEntries) { + String artifactId = api.getName() + "/" + api.getVersion() + "/" + entry.getName(); + + try (ByteArrayInputStream is = new ByteArrayInputStream(entry.getContent())) { + ArtifactMetaData meta = client.createArtifactWithVersion(entry.getType().toString(), artifactId, + api.getVersion(), is); + Map props = new LinkedHashMap<>(); + props.put(PARAM_SOURCE_APIM, VERSION_32); + setArtifactMetaData(meta, props); + } + + ArtifactReference ref = new ArtifactReference(); + ref.setName(entry.getName()); + ref.setGroupId(entry.getType().toString()); + ref.setArtifactId(artifactId); + ref.setVersion(api.getVersion()); + references.add(ref); + } + return references; + } + + private void addSubscriptionsToReferences(List references, Subscriptions subs, APIInfo api) + throws Exception { + if (subs == null || subs.getList() == null || subs.getList().isEmpty()) + return; + + String artifactId = api.getName() + "/" + api.getVersion() + "/" + ARTIFACT_NAME_SUBSCRIPTIONS; + + byte[] subsBytes = mapperYaml.writeValueAsBytes(subs); + + try (ByteArrayInputStream is = new ByteArrayInputStream(subsBytes)) { + ArtifactMetaData meta = client.createArtifactWithVersion(ARTIFACT_GROUP_SUBSCRIPTIONS, artifactId, + api.getVersion(), is); + Map props = new LinkedHashMap<>(); + props.put(PARAM_SOURCE_APIM, VERSION_32); + setArtifactMetaData(meta, props); + } + + ArtifactReference ref = new ArtifactReference(); + ref.setName(ARTIFACT_NAME_SUBSCRIPTIONS); + ref.setGroupId(ARTIFACT_GROUP_SUBSCRIPTIONS); + ref.setArtifactId(artifactId); + ref.setVersion(api.getVersion()); + references.add(ref); + } +} diff --git a/src/main/java/cz/trask/migration/impl/v32/ZipUtils.java b/src/main/java/cz/trask/migration/impl/v32/ZipUtils.java index 664c8a7..476e64f 100644 --- a/src/main/java/cz/trask/migration/impl/v32/ZipUtils.java +++ b/src/main/java/cz/trask/migration/impl/v32/ZipUtils.java @@ -39,9 +39,11 @@ public class ZipUtils { private static FileType determineFileType(String fileName) { String lowerFileName = fileName.toLowerCase(); - if (lowerFileName.endsWith("/meta-information/api.yaml")) { + if (lowerFileName.endsWith("/meta-information/api.yaml") + || lowerFileName.endsWith("/meta-information/api.json")) { return FileType.APIDEF; - } else if (lowerFileName.endsWith("/meta-information/swagger.yaml")) { + } else if (lowerFileName.endsWith("/meta-information/swagger.yaml") + || lowerFileName.endsWith("/meta-information/swagger.json")) { return FileType.OPENAPI; } else if (lowerFileName.endsWith(".wsdl")) { return FileType.WSDL; diff --git a/src/main/java/cz/trask/migration/model/ApplicationConfig.java b/src/main/java/cz/trask/migration/model/ApplicationConfig.java index 5e9b565..7ca8a6f 100644 --- a/src/main/java/cz/trask/migration/model/ApplicationConfig.java +++ b/src/main/java/cz/trask/migration/model/ApplicationConfig.java @@ -62,5 +62,8 @@ public class ApplicationConfig { private String publisherTokenUrl; @JsonProperty("wso2_user") private String wso2User; + @JsonProperty("wso2_apis_dir") + private String wso2ApisDir; + } -} +} \ No newline at end of file diff --git a/src/main/resources/apicurio-migrator.yaml b/src/main/resources/apicurio-migrator.yaml index 8ecaca0..4a21cc7 100644 --- a/src/main/resources/apicurio-migrator.yaml +++ b/src/main/resources/apicurio-migrator.yaml @@ -5,6 +5,7 @@ source: devportal_api_url: https://localhost:9444/api/am/store publisher_token_url: https://localhost:9444/oauth2/token wso2_user: YWRtaW46YWRtaW4= + wso2_apis_dir: apis target: registration_api_url: https://localhost:9443/client-registration/v0.17/register