diff --git a/pom.xml b/pom.xml index 9e54627ab8af5213e2bf1bcf9275cfd3a56ed188..f68d734ae15aea27f6f77c00c0b6827711772daa 100644 --- a/pom.xml +++ b/pom.xml @@ -119,6 +119,11 @@ <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-oauth2-client</artifactId> </dependency> + <dependency> + <groupId>com.jayway.jsonpath</groupId> + <artifactId>json-path</artifactId> + <version>2.4.0</version> + </dependency> </dependencies> <build> diff --git a/src/main/java/com/smartharvester/config/AsynConfig.java b/src/main/java/com/smartharvester/config/AsynConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..705221e10568a557357d8e19e4c4f1d9caa6d5a8 --- /dev/null +++ b/src/main/java/com/smartharvester/config/AsynConfig.java @@ -0,0 +1,29 @@ +package com.smartharvester.config; + +import java.util.concurrent.Executor; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +@Configuration +@EnableAsync +public class AsynConfig { + + @Bean + public Executor asyncExecutor() { + return new ThreadPoolTaskExecutor(); + } + + @Bean + public Executor executor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(5); + executor.setMaxPoolSize(5); + executor.setQueueCapacity(500); + executor.initialize(); + return executor; + } + +} diff --git a/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java b/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java index 01265fbf5c0fa70cb4935b7a1d6779c35c937682..863482fce762b476ac3bcad805ea97bada268146 100644 --- a/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java +++ b/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java @@ -54,26 +54,23 @@ public class SmartHarvesterMappingController { } @PostMapping("/publish") - public ResponseEntity<?> map(@RequestBody MappingRequest mappîngRequest, - @RequestParam("catalogId") String catalogId, @RequestParam String fdpUrl) { - LOGGER.info("init mapping"); + public ResponseEntity<MappingResponse> map(@RequestBody MappingRequest mappingRequest, + @RequestParam("catalogId") String catalogId, @RequestParam String fdpUrl, @RequestParam boolean isJsonpath) { ResponseEntity<?> responseEntity = null; - - List<String> urls = mappîngRequest.getUrls(); - List<Path> paths = mappîngRequest.getPaths(); - String fdpToken = mappîngRequest.getFdpToken(); - - List<String> publishedUrl = new ArrayList<String>(); - List<String> notPublishedUrl = new ArrayList<String>(); + List<String> urls = mappingRequest.getUrls(); + List<Path> paths = mappingRequest.getPaths(); + String fdpToken = mappingRequest.getFdpToken(); + List<String> publishedUrl = new ArrayList<>(); + List<String> notPublishedUrl = new ArrayList<>(); List<Path> distributionPaths = paths.stream().filter(path -> path.getDcatClass().equals("dcat:distribution")) .collect(Collectors.toList()); for (String url : urls) { String datasetId = null; - List<String> properties = new ArrayList<String>(); + List<String> properties = new ArrayList<>(); - properties = this.mappingService.getValues(url, paths); + properties = (isJsonpath) ? this.mappingService.getValuesFromJsonpath(url, paths) : this.mappingService.getValues(url, paths); String dataset = this.mappingService.getDatasetString(catalogId, properties.get(0), fdpUrl); String locationDataset = null; try { @@ -82,7 +79,7 @@ public class SmartHarvesterMappingController { datasetId = locationDataset.substring(locationDataset.indexOf("dataset/") + 8); HttpStatus statusCode = responseEntity.getStatusCode(); - if (distributionPaths.size() > 0) { + if (!distributionPaths.isEmpty()) { String distribution = this.mappingService.getDistributionString(datasetId, properties.get(1), fdpUrl); try { diff --git a/src/main/java/com/smartharvester/controller/SmartHarvesterPublicationController.java b/src/main/java/com/smartharvester/controller/SmartHarvesterPublicationController.java index 1fc19d9f7c87d1cf054f2338f706247d810edd1b..1291af29f34e7386c25d2a790d95a10cf01bc56f 100644 --- a/src/main/java/com/smartharvester/controller/SmartHarvesterPublicationController.java +++ b/src/main/java/com/smartharvester/controller/SmartHarvesterPublicationController.java @@ -1,26 +1,13 @@ package com.smartharvester.controller; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.mongodb.client.FindIterable; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoDatabase; -import com.mongodb.client.MongoIterable; import com.smartharvester.dao.CatalogDaoRepository; import com.smartharvester.dao.OpenApiDaoRepository; -import com.smartharvester.dao.UserDaoRepository; -import com.smartharvester.exception.ResourceNotFoundException; import com.smartharvester.model.login.response.MessageResponse; import com.smartharvester.model.openapi.OpenApi; -import com.smartharvester.model.user.Catalog; -import com.smartharvester.model.user.SmartHarvesterUser; -import com.smartharvester.service.UserDaoSevice; import io.swagger.v3.oas.annotations.tags.Tag; -import org.bson.Document; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; diff --git a/src/main/java/com/smartharvester/model/mapping/request/MappingRequest.java b/src/main/java/com/smartharvester/model/mapping/request/MappingRequest.java index d622fcab51286eb1f3d7a217fcd073fda6053f92..b785750bb9d4f5237b4dc97469f30580d47194f6 100644 --- a/src/main/java/com/smartharvester/model/mapping/request/MappingRequest.java +++ b/src/main/java/com/smartharvester/model/mapping/request/MappingRequest.java @@ -34,6 +34,7 @@ public class MappingRequest { public void setFdpToken(String fdpToken) { this.fdpToken = fdpToken; } + diff --git a/src/main/java/com/smartharvester/service/MappingService.java b/src/main/java/com/smartharvester/service/MappingService.java index 36f3eb34950041aeae81636e409e2de61e0a6328..c0509b704809ea26f87c06e7453f6af27118bee6 100644 --- a/src/main/java/com/smartharvester/service/MappingService.java +++ b/src/main/java/com/smartharvester/service/MappingService.java @@ -31,6 +31,9 @@ import org.springframework.stereotype.Service; import org.springframework.web.client.HttpClientErrorException.BadRequest; import org.springframework.web.client.RequestCallback; import org.springframework.web.client.RestTemplate; + +import com.jayway.jsonpath.DocumentContext; +import com.jayway.jsonpath.JsonPath; import com.smartharvester.model.mapping.Path; import com.smartharvester.model.mapping.request.PublishedRequest; @@ -94,7 +97,7 @@ public class MappingService { } public static String write(String indentifier, String value) { - value = value.replaceAll("[\\r\\n]+", " "); + value = value.replaceAll("[\\r\\n]+", " ").replaceAll("\"", "\\\\\""); String property = ""; switch (indentifier) { @@ -112,6 +115,7 @@ public class MappingService { break; case "dct:publisher": property += indentifier + "[ a foaf:Agent; foaf:name \"" + value + "\"];\n"; + break; default: property += indentifier + " \"" + value + "\";\n"; break; @@ -124,7 +128,64 @@ public class MappingService { return card.endsWith("n"); } - public List<String> getValues(String urlRepo, List<Path> paths) { + public List<String> getValuesFromJsonpath(String urlRepo, List<Path> paths) { + List<String> result = new ArrayList<String>(); + String datasetProperties = ""; + String distributionProperties = ""; + JSONObject json = null; + + try { + json = getJson(urlRepo); + } catch (JSONException e1) { + e1.printStackTrace(); + } + + if (paths.stream().filter(e -> e.getDcatClass().equals("dcat:dataset")) + .filter(e -> e.getProperty().equals("dct:hasVersion")).collect(Collectors.toList()).isEmpty()) { + datasetProperties += "dct:hasVersion \"null\";\n"; + } + if (paths.stream().filter(e -> e.getDcatClass().equals("dcat:dataset")) + .filter(e -> e.getProperty().equals("dct:publisher")).collect(Collectors.toList()).isEmpty()) { + datasetProperties += "dct:publisher [ a foaf:Agent; foaf:name \"undefined\"];\n"; + } + distributionProperties += "dct:hasVersion \"null\";\ndct:publisher [ a foaf:Agent; foaf:name \"undefined\"];\n"; + + for (Path path: paths) { + DocumentContext jsonContext = JsonPath.parse(json.toString()); + Object value = jsonContext.read(path.getPath()); + if (value instanceof String) { + String valueString = (String) value; + if (path.getDcatClass().equals("dcat:dataset")) { + datasetProperties += write(path.getProperty(), valueString); + } else if (path.getDcatClass().equals("dcat:distribution")) { + distributionProperties += write(path.getProperty(), valueString); + } + } else if (value instanceof Integer){ + int valueInt= (Integer) value; + if (path.getDcatClass().equals("dcat:dataset")) { + datasetProperties += write(path.getProperty(), String.valueOf(valueInt)); + } else if (path.getDcatClass().equals("dcat:distribution")) { + distributionProperties += write(path.getProperty(), String.valueOf(valueInt)); + } + } else if (value instanceof List<?>) { + @SuppressWarnings("unchecked") + List<String> valueList = (List<String>) value; + for(String element: valueList) { + if (path.getDcatClass().equals("dcat:dataset")) { + datasetProperties += write(path.getProperty(), element); + } else if (path.getDcatClass().equals("dcat:distribution")) { + distributionProperties += write(path.getProperty(), element); + } + } + } + } + result.add(datasetProperties); + result.add(distributionProperties); + + return result; + } + + public List<String> getValues(String urlRepo, List<Path> paths) { List<String> result = new ArrayList<String>(); String datasetProperties = ""; @@ -138,146 +199,146 @@ public class MappingService { } if (paths.stream().filter(e -> e.getDcatClass().equals("dcat:dataset")) - .filter(e -> e.getProperty().equals("dct:hasVersion")).collect(Collectors.toList()).size() == 0) { + .filter(e -> e.getProperty().equals("dct:hasVersion")).collect(Collectors.toList()).isEmpty()) { datasetProperties += "dct:hasVersion \"null\";\n"; } if (paths.stream().filter(e -> e.getDcatClass().equals("dcat:dataset")) - .filter(e -> e.getProperty().equals("dct:publisher")).collect(Collectors.toList()).size() == 0) { + .filter(e -> e.getProperty().equals("dct:publisher")).collect(Collectors.toList()).isEmpty()) { datasetProperties += "dct:publisher [ a foaf:Agent; foaf:name \"undefined\"];\n"; } distributionProperties += "dct:hasVersion \"null\";\ndct:publisher [ a foaf:Agent; foaf:name \"undefined\"];\n"; for (Path path : paths) { try { - String[] array = path.getPath().split(" : "); - List<String> list = new ArrayList<String>(Arrays.asList(array)); - - JSONObject jsonT = json; - if (list.size() == 1) { - if (jsonT.optJSONArray(list.get(0)) != null) { - JSONArray jsonA = jsonT.optJSONArray(list.get(0)); - if (isReplicable(path.getCard())) { - for (int j = 0; j < jsonA.length(); j++) { + String[] array = path.getPath().split(" : "); + List<String> list = new ArrayList<>(Arrays.asList(array)); + + JSONObject jsonT = json; + if (list.size() == 1) { + if (jsonT.optJSONArray(list.get(0)) != null) { + JSONArray jsonA = jsonT.optJSONArray(list.get(0)); + if (isReplicable(path.getCard())) { + for (int j = 0; j < jsonA.length(); j++) { + if (path.getDcatClass().equals("dcat:dataset")) { + datasetProperties += write(path.getProperty(), jsonA.getString(j)); + } else if (path.getDcatClass().equals("dcat:distribution")) { + distributionProperties += write(path.getProperty(), jsonA.getString(j)); + } + } + } else { if (path.getDcatClass().equals("dcat:dataset")) { - datasetProperties += write(path.getProperty(), jsonA.getString(j)); + datasetProperties += write(path.getProperty(), jsonA.getString(0)); } else if (path.getDcatClass().equals("dcat:distribution")) { - distributionProperties += write(path.getProperty(), jsonA.getString(j)); + distributionProperties += write(path.getProperty(), jsonA.getString(0)); } } + } else { if (path.getDcatClass().equals("dcat:dataset")) { - datasetProperties += write(path.getProperty(), jsonA.getString(0)); + datasetProperties += write(path.getProperty(), jsonT.getString(list.get(0))); } else if (path.getDcatClass().equals("dcat:distribution")) { - distributionProperties += write(path.getProperty(), jsonA.getString(0)); + distributionProperties += write(path.getProperty(), jsonT.getString(list.get(0))); } } - } else { - if (path.getDcatClass().equals("dcat:dataset")) { - datasetProperties += write(path.getProperty(), jsonT.getString(list.get(0))); - } else if (path.getDcatClass().equals("dcat:distribution")) { - distributionProperties += write(path.getProperty(), jsonT.getString(list.get(0))); - } - } - } else { - for (int i = 0; i < list.size(); i++) { - if (i < list.size() - 1) { - if (jsonT.optJSONObject(list.get(i)) != null) { - jsonT = jsonT.optJSONObject(list.get(i)); - } else if (jsonT.optJSONArray(list.get(i)) != null) { - JSONArray jsonA = jsonT.optJSONArray(list.get(i)); - - if (i < list.size() - 2) { - jsonT = jsonA.getJSONObject(Integer.parseInt(list.get(i + 1))); - - list.remove(i + 1); - } - } - } else { - if (jsonT.optJSONArray(list.get(i)) != null) { - if (!jsonT.has(list.get(i))) { - if (path.getDcatClass().equals("dcat:dataset")) { - datasetProperties += write(path.getProperty(), "undefined"); - } else if (path.getDcatClass().equals("dcat:distribution")) { - distributionProperties += write(path.getProperty(), "undefined"); - } - } else { + for (int i = 0; i < list.size(); i++) { + if (i < list.size() - 1) { + if (jsonT.optJSONObject(list.get(i)) != null) { + jsonT = jsonT.optJSONObject(list.get(i)); + } else if (jsonT.optJSONArray(list.get(i)) != null) { JSONArray jsonA = jsonT.optJSONArray(list.get(i)); - if (isReplicable(path.getCard())) { - for (int j = 0; j < jsonA.length(); j++) { - if (path.getDcatClass().equals("dcat:dataset")) { - datasetProperties += write(path.getProperty(), jsonA.optString(j)); - } else if (path.getDcatClass().equals("dcat:distribution")) { - distributionProperties += write(path.getProperty(), jsonA.optString(j)); - } - } - break; - } else { - if (path.getDcatClass().equals("dcat:dataset")) { - datasetProperties += write(path.getProperty(), jsonA.optString(0)); - } else if (path.getDcatClass().equals("dcat:distribution")) { - distributionProperties += write(path.getProperty(), jsonA.optString(0)); - } - } - break; - } - - } else if (jsonT.optJSONArray(list.get(list.size() - 2)) != null) { + if (i < list.size() - 2) { + jsonT = jsonA.getJSONObject(Integer.parseInt(list.get(i + 1))); - JSONArray jsonA = jsonT.optJSONArray(list.get(list.size() - 2)); - if (path.getDcatClass().equals("dcat:dataset")) { - datasetProperties += write(path.getProperty(), - jsonA.optString(Integer.parseInt(list.get(i)))); - } else if (path.getDcatClass().equals("dcat:distribution")) { - distributionProperties += write(path.getProperty(), - jsonA.optString(Integer.parseInt(list.get(i)))); - } - break; - } else if (jsonT.optString(list.get(i)) != null && jsonT.has(list.get(i))) { - if (path.getDcatClass().equals("dcat:dataset")) { - datasetProperties += write(path.getProperty(), jsonT.optString(list.get(i))); - } else if (path.getDcatClass().equals("dcat:distribution")) { - distributionProperties += write(path.getProperty(), jsonT.optString(list.get(i))); + list.remove(i + 1); + } } - break; - } else if (jsonT instanceof JSONObject) { - Iterator keys = jsonT.keys(); - while (keys.hasNext()) { - String key = keys.next().toString(); - if (jsonT.optJSONArray(key) != null) { - JSONArray jsonA = jsonT.optJSONArray(key); + } else { + if (jsonT.optJSONArray(list.get(i)) != null) { + if (!jsonT.has(list.get(i))) { + if (path.getDcatClass().equals("dcat:dataset")) { + datasetProperties += write(path.getProperty(), "undefined"); + } else if (path.getDcatClass().equals("dcat:distribution")) { + distributionProperties += write(path.getProperty(), "undefined"); + } + } else { + JSONArray jsonA = jsonT.optJSONArray(list.get(i)); if (isReplicable(path.getCard())) { for (int j = 0; j < jsonA.length(); j++) { - String value = jsonA.getJSONObject(j).getString(list.get(i)); if (path.getDcatClass().equals("dcat:dataset")) { - datasetProperties += write(path.getProperty(), value); + datasetProperties += write(path.getProperty(), jsonA.optString(j)); } else if (path.getDcatClass().equals("dcat:distribution")) { - distributionProperties += write(path.getProperty(), value); + distributionProperties += write(path.getProperty(), jsonA.optString(j)); } } + break; } else { - String value = jsonA.getJSONObject(0).getString(list.get(i)); if (path.getDcatClass().equals("dcat:dataset")) { - datasetProperties += write(path.getProperty(), value); + datasetProperties += write(path.getProperty(), jsonA.optString(0)); } else if (path.getDcatClass().equals("dcat:distribution")) { - distributionProperties += write(path.getProperty(), value); + distributionProperties += write(path.getProperty(), jsonA.optString(0)); } } + break; + } + + } else if (jsonT.optJSONArray(list.get(list.size() - 2)) != null) { + + JSONArray jsonA = jsonT.optJSONArray(list.get(list.size() - 2)); + if (path.getDcatClass().equals("dcat:dataset")) { + datasetProperties += write(path.getProperty(), + jsonA.optString(Integer.parseInt(list.get(i)))); + } else if (path.getDcatClass().equals("dcat:distribution")) { + distributionProperties += write(path.getProperty(), + jsonA.optString(Integer.parseInt(list.get(i)))); + } + break; + } else if (jsonT.optString(list.get(i)) != null && jsonT.has(list.get(i))) { + if (path.getDcatClass().equals("dcat:dataset")) { + datasetProperties += write(path.getProperty(), jsonT.optString(list.get(i))); + } else if (path.getDcatClass().equals("dcat:distribution")) { + distributionProperties += write(path.getProperty(), jsonT.optString(list.get(i))); + } + break; + } else if (jsonT instanceof JSONObject) { + Iterator keys = jsonT.keys(); + while (keys.hasNext()) { + String key = keys.next().toString(); + if (jsonT.optJSONArray(key) != null) { + JSONArray jsonA = jsonT.optJSONArray(key); + if (isReplicable(path.getCard())) { + for (int j = 0; j < jsonA.length(); j++) { + String value = jsonA.getJSONObject(j).getString(list.get(i)); + if (path.getDcatClass().equals("dcat:dataset")) { + datasetProperties += write(path.getProperty(), value); + } else if (path.getDcatClass().equals("dcat:distribution")) { + distributionProperties += write(path.getProperty(), value); + } + } + } else { + String value = jsonA.getJSONObject(0).getString(list.get(i)); + if (path.getDcatClass().equals("dcat:dataset")) { + datasetProperties += write(path.getProperty(), value); + } else if (path.getDcatClass().equals("dcat:distribution")) { + distributionProperties += write(path.getProperty(), value); + } + } + + } } } } - } + } + } + } catch (JSONException e1) { + if (path.getDcatClass().equals("dcat:dataset")) { + datasetProperties += write(path.getProperty(), "undefined"); + } else if (path.getDcatClass().equals("dcat:distribution")) { + distributionProperties += write(path.getProperty(), "undefined"); } } - } catch (JSONException e1) { - if (path.getDcatClass().equals("dcat:dataset")) { - datasetProperties += write(path.getProperty(), "undefined"); - } else if (path.getDcatClass().equals("dcat:distribution")) { - distributionProperties += write(path.getProperty(), "undefined"); - } - } } @@ -311,15 +372,16 @@ public class MappingService { return this.restTemplate.exchange(fdpUrl + path, HttpMethod.POST, entity, String.class); } - + public void draftToPublished(String fdpToken, String url) { HttpHeaders headers = new HttpHeaders(); - + headers.setBearerAuth(fdpToken); headers.setContentType(MediaType.APPLICATION_JSON); - - HttpEntity<PublishedRequest> entity = new HttpEntity<PublishedRequest>(new PublishedRequest("PUBLISHED"), headers); - + + HttpEntity<PublishedRequest> entity = new HttpEntity<PublishedRequest>(new PublishedRequest("PUBLISHED"), + headers); + this.restTemplate.exchange(url + "/meta/state", HttpMethod.PUT, entity, Void.class); }