From 658d7ce885e1b6a0e5aceaccf917eafd5cbaa4aa Mon Sep 17 00:00:00 2001
From: Baptiste Toulemonde <toulemonde@cines.fr>
Date: Tue, 1 Mar 2022 09:29:37 +0100
Subject: [PATCH] fix publish distrib to fdp
---
.../com/smartharvester/config/AsynConfig.java | 7 +-
.../SmartHarvesterMappingController.java | 40 +-
.../CustomAsyncExceptionHandler.java | 26 +
.../service/MappingService.java | 723 ++++++++++--------
.../util/SmartHarvesterRequestUtil.java | 6 +-
5 files changed, 451 insertions(+), 351 deletions(-)
create mode 100644 src/main/java/com/smartharvester/exception/CustomAsyncExceptionHandler.java
diff --git a/src/main/java/com/smartharvester/config/AsynConfig.java b/src/main/java/com/smartharvester/config/AsynConfig.java
index 705221e..920ab49 100644
--- a/src/main/java/com/smartharvester/config/AsynConfig.java
+++ b/src/main/java/com/smartharvester/config/AsynConfig.java
@@ -1,15 +1,19 @@
package com.smartharvester.config;
+import java.util.concurrent.Callable;
import java.util.concurrent.Executor;
+import java.util.concurrent.Future;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
+import org.springframework.core.task.AsyncTaskExecutor;
+import org.springframework.scheduling.annotation.AsyncConfigurer;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
@Configuration
@EnableAsync
-public class AsynConfig {
+public class AsynConfig implements AsyncConfigurer {
@Bean
public Executor asyncExecutor() {
@@ -27,3 +31,4 @@ public class AsynConfig {
}
}
+
diff --git a/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java b/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java
index 779fc38..7aba48b 100644
--- a/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java
+++ b/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java
@@ -1,23 +1,20 @@
package com.smartharvester.controller;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
+
+import org.json.JSONException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.CrossOrigin;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.bind.annotation.*;
import org.springframework.web.client.HttpStatusCodeException;
import com.smartharvester.model.mapping.Path;
@@ -54,7 +51,7 @@ public class SmartHarvesterMappingController {
@PostMapping("/publish")
public ResponseEntity<MappingResponse> map(@RequestBody MappingRequest mappingRequest,
- @RequestParam("catalogId") String catalogId, @RequestParam String fdpUrl, @RequestParam boolean isJsonpath) throws ExecutionException, InterruptedException {
+ @RequestParam("catalogId") String catalogId, @RequestParam String fdpUrl, @RequestParam boolean isJsonpath) {
ResponseEntity<?> responseEntity;
List<String> urls = mappingRequest.getUrls();
List<Path> paths = mappingRequest.getPaths();
@@ -67,28 +64,26 @@ public class SmartHarvesterMappingController {
for (String url : urls) {
String datasetId;
- List<String> properties;
-
- properties = (isJsonpath) ? this.mappingService.getValuesFromJsonpath(url, paths).get() : this.mappingService.getValues(url, paths).get();
- String dataset = this.mappingService.getDatasetString(catalogId, properties.get(0), fdpUrl);
String locationDataset;
try {
+ String dataset = this.mappingService.buildDataset(url, paths, isJsonpath, catalogId, fdpUrl);
responseEntity = this.mappingService.asyncPostToFdp("/dataset", fdpUrl, dataset, fdpToken).get();
locationDataset = Objects.requireNonNull(responseEntity.getHeaders().getLocation()).toString();
datasetId = locationDataset.substring(locationDataset.indexOf("dataset/") + 8);
HttpStatus statusCode = responseEntity.getStatusCode();
if (!distributionPaths.isEmpty()) {
- String distribution = this.mappingService.getDistributionString(datasetId, properties.get(1),
- fdpUrl);
- try {
- ResponseEntity<String> distributionResponse = this.mappingService.asyncPostToFdp("/distribution", fdpUrl, distribution, fdpToken).get();
+ List<String> distributions = this.mappingService.buildDistribution(url, paths, isJsonpath, datasetId, fdpUrl);
+ for (String distribution: distributions) {
+ try {
+ ResponseEntity<String> distributionResponse = this.mappingService.asyncPostToFdp("/distribution", fdpUrl, distribution, fdpToken).get();
String locationDistribution = Objects.requireNonNull(Objects.requireNonNull(distributionResponse.getHeaders().getLocation()).toString());
- if (distributionResponse.getStatusCode().value() == 201) {
- this.mappingService.draftToPublished(fdpToken, locationDistribution);
+ if (distributionResponse.getStatusCode().value() == 201) {
+ this.mappingService.draftToPublished(fdpToken, locationDistribution);
+ }
+ } catch (ExecutionException | InterruptedException e) {
+ LOGGER.warn(e.getMessage());
}
- } catch (HttpStatusCodeException e) {
- LOGGER.warn(e.getMessage());
}
}
if (statusCode.value() == 201) {
@@ -97,10 +92,13 @@ public class SmartHarvesterMappingController {
} else {
notPublishedUrl.add(url);
}
- } catch (Exception e) {
+ } catch (JSONException | ExecutionException | IllegalStateException e ) {
notPublishedUrl.add(url + " => " + e.getMessage());
LOGGER.warn(e.getMessage());
+ } catch (InterruptedException e) {
Thread.currentThread().interrupt();
+ notPublishedUrl.add(url + " => " + e.getMessage());
+ LOGGER.warn(e.getMessage());
}
}
diff --git a/src/main/java/com/smartharvester/exception/CustomAsyncExceptionHandler.java b/src/main/java/com/smartharvester/exception/CustomAsyncExceptionHandler.java
new file mode 100644
index 0000000..ece5df1
--- /dev/null
+++ b/src/main/java/com/smartharvester/exception/CustomAsyncExceptionHandler.java
@@ -0,0 +1,26 @@
+package com.smartharvester.exception;
+
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.aop.interceptor.AsyncUncaughtExceptionHandler;
+
+import java.lang.reflect.Method;
+
+public class CustomAsyncExceptionHandler implements AsyncUncaughtExceptionHandler {
+
+ public static final Logger LOGGER = LoggerFactory.getLogger(CustomAsyncExceptionHandler.class);
+
+ @Override
+ public void handleUncaughtException(Throwable ex, Method method, Object... params) {
+ LOGGER.warn(ex.getMessage());
+ LOGGER.warn(method.getName());
+ for (Object param: params) {
+ LOGGER.warn((String) param);
+ }
+ }
+
+ public AsyncUncaughtExceptionHandler getAsyncUncaughtExceptionHandler() {
+ return new CustomAsyncExceptionHandler();
+ }
+}
diff --git a/src/main/java/com/smartharvester/service/MappingService.java b/src/main/java/com/smartharvester/service/MappingService.java
index 5b40586..2cc51c2 100644
--- a/src/main/java/com/smartharvester/service/MappingService.java
+++ b/src/main/java/com/smartharvester/service/MappingService.java
@@ -1,14 +1,14 @@
package com.smartharvester.service;
-import com.jayway.jsonpath.DocumentContext;
-import com.jayway.jsonpath.JsonPath;
-import com.jayway.jsonpath.PathNotFoundException;
+import com.jayway.jsonpath.*;
import com.smartharvester.model.mapping.Path;
import com.smartharvester.model.mapping.request.PublishedRequest;
import com.smartharvester.util.SmartHarvesterRequestUtil;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.*;
import org.springframework.scheduling.annotation.Async;
@@ -16,334 +16,403 @@ import org.springframework.stereotype.Service;
import org.springframework.web.client.HttpClientErrorException.BadRequest;
import org.springframework.web.client.RestTemplate;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
+import java.util.*;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
@Service
public class MappingService {
- @Autowired
- private RestTemplate restTemplate;
-
- @Autowired
- SmartHarvesterRequestUtil requestUtil;
-
- public String getDatasetString(String catId, String properties, String fdpUrl) {
- return "@prefix dcat: <http://www.w3.org/ns/dcat#>.\n" + "@prefix dct: <http://purl.org/dc/terms/>.\n"
- + "@prefix adms: <http://www.w3.org/ns/adms#>.\n" + "@prefix dqv: <http://www.w3.org/ns/dqv#> .\n"
- + "@prefix geodcat: <http://data.europa.eu/930/>.\n" + "@prefix prov: <http://www.w3.org/ns/prov#>.\n"
- + "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n"
- + "@prefix language: <http://id.loc.gov/vocabulary/iso639-1/>.\n" + "@prefix s: <" + fdpUrl + "/>.\n"
- + "@prefix c: <" + fdpUrl + "/catalog/>.\n" + "s:new \n" + "a dcat:Dataset, dcat:Resource;\n"
- + "dct:isPartOf c:" + catId + ";\n" + properties + ".";
- }
-
- public String getDistributionString(String datasetId, String properties, String fdpUrl) {
- return "@prefix dcat: <http://www.w3.org/ns/dcat#>. \n"
- + "@prefix dct: <http://purl.org/dc/terms/>.\n" + "@prefix adms: <http://www.w3.org/ns/adms#> .\n"
- + "@prefix dqv: <http://www.w3.org/ns/dqv#> .\n" + "@prefix geodcat: <http://data.europa.eu/930/>. \n"
- + "@prefix prov: <http://www.w3.org/ns/prov#>.\n"
- + "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n"
- + "@prefix skos: <http://www.w3.org/2004/02/skos/core#>.\n"
- + "@prefix spdx: <http://spdx.org/rdf/terms#>.\n" + "@prefix foaf: <http://xmlns.com/foaf/0.1/>.\n"
- + "@prefix odrl: <http://www.w3.org/ns/odrl/2/>.\n"
- + "@prefix cnt: <http://www.w3.org/2011/content#>.\n"
- + "@prefix language: <http://id.loc.gov/vocabulary/iso639-1/>.\n" + "@prefix s: <" + fdpUrl + "/>.\n"
- + "@prefix c: <" + fdpUrl + "/dataset/>.\n" + "s:new \n" + "a dcat:Distribution, dcat:Resource;\n"
- + "dct:isPartOf c:" + datasetId + ";\n" + properties + ".";
- }
-
-
-
- public static String write(String indentifier, String value) {
- value = value.replaceAll("[\\r\\n]+", " ").replaceAll("\"", "\\\\\"");
- String property = "";
-
- switch (indentifier) {
- case "dcat:landingPage":
- case "dcat:theme":
- case "dcat:contactPoint":
- case "dcat:accessURL":
- case "dcat:downloadURL":
- case "dct:language":
- property += indentifier + " <" + value + ">;\n";
- break;
- case "dct:issued":
- case "dct:modified":
- property += indentifier + " \"" + value + "\"^^xsd:dateTime;\n";
- break;
- case "dct:publisher":
- property += indentifier + "[ a foaf:Agent; foaf:name \"" + value + "\"];\n";
- break;
- default:
- property += indentifier + " \"" + value + "\";\n";
- break;
- }
-
- return property;
- }
-
- public static boolean isReplicable(String card) {
- return card.endsWith("n");
- }
-
- @Async
- public CompletableFuture<List<String>> getValuesFromJsonpath(String urlRepo, List<Path> paths) {
- List<String> result = new ArrayList<>();
- StringBuilder datasetProperties = new StringBuilder();
- StringBuilder distributionProperties = new StringBuilder();
- JSONObject json = null;
-
- try {
- json = this.requestUtil.getJson(urlRepo);
- } catch (JSONException e1) {
- e1.printStackTrace();
- }
-
- if (paths.stream().filter(e -> e.getDcatClass().equals("dcat:dataset")).noneMatch(e -> e.getProperty().equals("dct:hasVersion"))) {
- datasetProperties.append("dct:hasVersion \"null\";\n");
- }
- if (paths.stream().filter(e -> e.getDcatClass().equals("dcat:dataset")).noneMatch(e -> e.getProperty().equals("dct:publisher"))) {
- datasetProperties.append("dct:publisher [ a foaf:Agent; foaf:name \"undefined\"];\n");
- }
- distributionProperties.append("dct:hasVersion \"null\";\ndct:publisher [ a foaf:Agent; foaf:name \"undefined\"];\n");
-
- for (Path path: paths) {
- assert json != null;
- DocumentContext jsonContext = JsonPath.parse(json.toString());
- Object value;
- try {
- value = jsonContext.read(path.getPath());
- } catch (PathNotFoundException e) {
- value = "undefined";
- }
-
- if (value instanceof String) {
- String valueString = (String) value;
- if (path.getDcatClass().equals("dcat:dataset")) {
- datasetProperties.append(write(path.getProperty(), valueString));
- } else if (path.getDcatClass().equals("dcat:distribution")) {
- distributionProperties.append(write(path.getProperty(), valueString));
- }
- } else if (value instanceof Integer){
- int valueInt= (Integer) value;
- if (path.getDcatClass().equals("dcat:dataset")) {
- datasetProperties.append(write(path.getProperty(), String.valueOf(valueInt)));
- } else if (path.getDcatClass().equals("dcat:distribution")) {
- distributionProperties.append(write(path.getProperty(), String.valueOf(valueInt)));
- }
- } else if (value instanceof List<?>) {
- @SuppressWarnings("unchecked")
- List<String> valueList = (List<String>) value;
- for(String element: valueList) {
- if (path.getDcatClass().equals("dcat:dataset")) {
- datasetProperties.append(write(path.getProperty(), element));
- } else if (path.getDcatClass().equals("dcat:distribution")) {
- distributionProperties.append(write(path.getProperty(), element));
- }
- }
- }
- }
- result.add(datasetProperties.toString());
- result.add(distributionProperties.toString());
-
- return CompletableFuture.completedFuture(result);
- }
-
- @Async
- public CompletableFuture<List<String>> getValues(String urlRepo, List<Path> paths) {
-
- List<String> result = new ArrayList<>();
- StringBuilder datasetProperties = new StringBuilder();
- StringBuilder distributionProperties = new StringBuilder();
- JSONObject json = null;
-
- try {
- json = this.requestUtil.getJson(urlRepo);
- } catch (JSONException e1) {
- e1.printStackTrace();
- }
-
- if (paths.stream().filter(e -> e.getDcatClass().equals("dcat:dataset")).noneMatch(e -> e.getProperty().equals("dct:hasVersion"))) {
- datasetProperties.append("dct:hasVersion \"null\";\n");
- }
- if (paths.stream().filter(e -> e.getDcatClass().equals("dcat:dataset")).noneMatch(e -> e.getProperty().equals("dct:publisher"))) {
- datasetProperties.append("dct:publisher [ a foaf:Agent; foaf:name \"undefined\"];\n");
- }
- distributionProperties.append("dct:hasVersion \"null\";\ndct:publisher [ a foaf:Agent; foaf:name \"undefined\"];\n");
- for (Path path : paths) {
- try {
- String[] array = path.getPath().split(" : ");
- List<String> list = new ArrayList<>(Arrays.asList(array));
-
- JSONObject jsonT = json;
- if (list.size() == 1) {
- assert jsonT != null;
- if (jsonT.optJSONArray(list.get(0)) != null) {
- JSONArray jsonA = jsonT.optJSONArray(list.get(0));
- if (isReplicable(path.getCard())) {
- for (int j = 0; j < jsonA.length(); j++) {
- if (path.getDcatClass().equals("dcat:dataset")) {
- datasetProperties.append(write(path.getProperty(), jsonA.getString(j)));
- } else if (path.getDcatClass().equals("dcat:distribution")) {
- distributionProperties.append(write(path.getProperty(), jsonA.getString(j)));
- }
- }
- } else {
- if (path.getDcatClass().equals("dcat:dataset")) {
- datasetProperties.append(write(path.getProperty(), jsonA.getString(0)));
- } else if (path.getDcatClass().equals("dcat:distribution")) {
- distributionProperties.append(write(path.getProperty(), jsonA.getString(0)));
- }
- }
-
- } else {
- if (path.getDcatClass().equals("dcat:dataset")) {
- datasetProperties.append(write(path.getProperty(), jsonT.getString(list.get(0))));
- } else if (path.getDcatClass().equals("dcat:distribution")) {
- distributionProperties.append(write(path.getProperty(), jsonT.getString(list.get(0))));
- }
- }
- } else {
- for (int i = 0; i < list.size(); i++) {
- if (i < list.size() - 1) {
- assert jsonT != null;
- if (jsonT.optJSONObject(list.get(i)) != null) {
- jsonT = jsonT.optJSONObject(list.get(i));
- } else if (jsonT.optJSONArray(list.get(i)) != null) {
- JSONArray jsonA = jsonT.optJSONArray(list.get(i));
-
- if (i < list.size() - 2) {
- jsonT = jsonA.getJSONObject(Integer.parseInt(list.get(i + 1)));
-
- list.remove(i + 1);
- }
- }
- } else {
- if (jsonT.optJSONArray(list.get(i)) != null) {
- if (!jsonT.has(list.get(i))) {
- if (path.getDcatClass().equals("dcat:dataset")) {
- datasetProperties.append(write(path.getProperty(), "undefined"));
- } else if (path.getDcatClass().equals("dcat:distribution")) {
- distributionProperties.append(write(path.getProperty(), "undefined"));
- }
- } else {
- JSONArray jsonA = jsonT.optJSONArray(list.get(i));
- if (isReplicable(path.getCard())) {
- for (int j = 0; j < jsonA.length(); j++) {
- if (path.getDcatClass().equals("dcat:dataset")) {
- datasetProperties.append(write(path.getProperty(), jsonA.optString(j)));
- } else if (path.getDcatClass().equals("dcat:distribution")) {
- distributionProperties.append(write(path.getProperty(), jsonA.optString(j)));
- }
- }
- break;
- } else {
- if (path.getDcatClass().equals("dcat:dataset")) {
- datasetProperties.append(write(path.getProperty(), jsonA.optString(0)));
- } else if (path.getDcatClass().equals("dcat:distribution")) {
- distributionProperties.append(write(path.getProperty(), jsonA.optString(0)));
- }
- }
-
- break;
- }
-
- } else if (jsonT.optJSONArray(list.get(list.size() - 2)) != null) {
-
- JSONArray jsonA = jsonT.optJSONArray(list.get(list.size() - 2));
- if (path.getDcatClass().equals("dcat:dataset")) {
- datasetProperties.append(write(path.getProperty(),
- jsonA.optString(Integer.parseInt(list.get(i)))));
- } else if (path.getDcatClass().equals("dcat:distribution")) {
- distributionProperties.append(write(path.getProperty(),
- jsonA.optString(Integer.parseInt(list.get(i)))));
- }
- break;
- } else if (jsonT.optString(list.get(i)) != null && jsonT.has(list.get(i))) {
- if (path.getDcatClass().equals("dcat:dataset")) {
- datasetProperties.append(write(path.getProperty(), jsonT.optString(list.get(i))));
- } else if (path.getDcatClass().equals("dcat:distribution")) {
- distributionProperties.append(write(path.getProperty(), jsonT.optString(list.get(i))));
- }
- break;
- } else {
- var keys = jsonT.keys();
- while (keys.hasNext()) {
- String key = keys.next().toString();
- if (jsonT.optJSONArray(key) != null) {
- JSONArray jsonA = jsonT.optJSONArray(key);
- if (isReplicable(path.getCard())) {
- for (int j = 0; j < jsonA.length(); j++) {
- String value = jsonA.getJSONObject(j).getString(list.get(i));
- if (path.getDcatClass().equals("dcat:dataset")) {
- datasetProperties.append(write(path.getProperty(), value));
- } else if (path.getDcatClass().equals("dcat:distribution")) {
- distributionProperties.append(write(path.getProperty(), value));
- }
- }
- } else {
- String value = jsonA.getJSONObject(0).getString(list.get(i));
- if (path.getDcatClass().equals("dcat:dataset")) {
- datasetProperties.append(write(path.getProperty(), value));
- } else if (path.getDcatClass().equals("dcat:distribution")) {
- distributionProperties.append(write(path.getProperty(), value));
- }
- }
-
- }
- }
- }
- }
-
- }
- }
- } catch (JSONException e1) {
- if (path.getDcatClass().equals("dcat:dataset")) {
- datasetProperties.append(write(path.getProperty(), "undefined"));
- } else if (path.getDcatClass().equals("dcat:distribution")) {
- distributionProperties.append(write(path.getProperty(), "undefined"));
- }
- }
-
- }
-
- result.add(datasetProperties.toString());
- result.add(distributionProperties.toString());
-
- return CompletableFuture.completedFuture(result);
-
- }
-
-
- @Async
- public CompletableFuture<ResponseEntity<String>> asyncPostToFdp(String path, String fdpUrl, String dataset, String fdpToken)
- throws BadRequest {
-
- HttpHeaders headers = new HttpHeaders();
- headers.add("Accept", "text/turtle;charset=UTF-8");
- headers.add("Content-Type", "text/turtle;charset=UTF-8");
- headers.setBearerAuth(fdpToken);
-
- HttpEntity<String> entity = new HttpEntity<>(dataset, headers);
-
- return CompletableFuture.completedFuture(this.restTemplate.exchange(fdpUrl + path, HttpMethod.POST, entity, String.class));
-
- }
-
- @Async
- public void draftToPublished(String fdpToken, String url) {
- HttpHeaders headers = new HttpHeaders();
-
- headers.setBearerAuth(fdpToken);
- headers.setContentType(MediaType.APPLICATION_JSON);
-
- HttpEntity<PublishedRequest> entity = new HttpEntity<>(new PublishedRequest("PUBLISHED"),
- headers);
-
- this.restTemplate.exchange(url + "/meta/state", HttpMethod.PUT, entity, Void.class);
- }
+ @Autowired
+ private RestTemplate restTemplate;
+
+ @Autowired
+ SmartHarvesterRequestUtil requestUtil;
+
+ public static final Logger LOGGER = LoggerFactory.getLogger(MappingService.class);
+
+ public StringBuilder getDatasetString(String catId, String fdpUrl) {
+ return new StringBuilder("@prefix dcat: <http://www.w3.org/ns/dcat#>.\n" + "@prefix dct: <http://purl.org/dc/terms/>.\n" + "@prefix adms: <http://www.w3.org/ns/adms#>.\n" + "@prefix dqv: <http://www.w3.org/ns/dqv#> .\n" + "@prefix geodcat: <http://data.europa.eu/930/>.\n" + "@prefix prov: <http://www.w3.org/ns/prov#>.\n" + "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n" + "@prefix language: <http://id.loc.gov/vocabulary/iso639-1/>.\n" + "@prefix s: <" + fdpUrl + "/>.\n" + "@prefix c: <" + fdpUrl + "/catalog/>.\n" + "s:new \n" + "a dcat:Dataset, dcat:Resource;\n" + "dct:isPartOf c:" + catId + ";\n");
+ }
+
+ public StringBuilder getDistributionString(String datasetId, String fdpUrl) {
+ return new StringBuilder("@prefix dcat: <http://www.w3.org/ns/dcat#>. \n" + "@prefix dct: <http://purl.org/dc/terms/>.\n" + "@prefix adms: <http://www.w3.org/ns/adms#> .\n" + "@prefix dqv: <http://www.w3.org/ns/dqv#> .\n" + "@prefix geodcat: <http://data.europa.eu/930/>. \n" + "@prefix prov: <http://www.w3.org/ns/prov#>.\n" + "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n" + "@prefix skos: <http://www.w3.org/2004/02/skos/core#>.\n" + "@prefix spdx: <http://spdx.org/rdf/terms#>.\n" + "@prefix foaf: <http://xmlns.com/foaf/0.1/>.\n" + "@prefix odrl: <http://www.w3.org/ns/odrl/2/>.\n" + "@prefix cnt: <http://www.w3.org/2011/content#>.\n" + "@prefix language: <http://id.loc.gov/vocabulary/iso639-1/>.\n" + "@prefix s: <" + fdpUrl + "/>.\n" + "@prefix c: <" + fdpUrl + "/dataset/>.\n" + "s:new \n" + "a dcat:Distribution, dcat:Resource;\n" + "dct:isPartOf c:" + datasetId + ";\n");
+ }
+
+ public String buildDataset(String urlRepo, List<Path> paths, boolean isJsonpath, String catId, String fdpUrl) throws JSONException, ExecutionException, InterruptedException {
+ Map<String, List<Map<String, List<String>>>> dcatPropertiesMap = this.mapToDcat(urlRepo, paths, isJsonpath);
+ Map<String, List<String>> datasetMap = dcatPropertiesMap.get("dataset").get(0);
+
+ datasetMap.putIfAbsent("dct:hasVersion", List.of("undefined"));
+
+
+ datasetMap.putIfAbsent("dct:publisher", List.of("undefined"));
+
+
+ StringBuilder datasetString = this.getDatasetString(catId, fdpUrl);
+ for (Map.Entry<String, List<String>> entry : datasetMap.entrySet()) {
+ if (entry.getValue().isEmpty()) {
+ entry.setValue(List.of("undefined"));
+ }
+ if (isReplicable(entry.getKey(), paths) && (entry.getValue().size() > 1)) {
+ for (String value : entry.getValue()) {
+ datasetString.append(this.write(entry.getKey(), value));
+ }
+ } else {
+ datasetString.append(this.write(entry.getKey(), entry.getValue().get(0)));
+ }
+
+ }
+ datasetString.append(".");
+ LOGGER.info(datasetString.toString());
+ return datasetString.toString();
+ }
+
+ public List<String> buildDistribution(String urlRepo, List<Path> paths, boolean isJsonpath, String datasetId, String fdpUrl) throws JSONException, ExecutionException, InterruptedException {
+ Map<String, List<Map<String, List<String>>>> dcatPropertiesMap = this.mapToDcat(urlRepo, paths, isJsonpath);
+ List<Map<String, List<String>>> distributionList = dcatPropertiesMap.get("distribution");
+ List<String> distributionStrings = new ArrayList<>();
+
+ if (!distributionList.get(0).entrySet().stream().map(v -> v.getValue()).allMatch(v -> v.get(0).equals("undefined"))) {
+ for (Map<String, List<String>> distributionMap : distributionList) {
+ distributionMap.putIfAbsent("dct:hasVersion", List.of("undefined"));
+
+ distributionMap.putIfAbsent("dct:publisher", List.of("undefined"));
+
+
+ StringBuilder datasetString = this.getDistributionString(datasetId, fdpUrl);
+ for (Map.Entry<String, List<String>> entry : distributionMap.entrySet()) {
+ if (isReplicable(entry.getKey(), paths)) {
+ for (String value : entry.getValue()) {
+ datasetString.append(this.write(entry.getKey(), value));
+ }
+ } else {
+ datasetString.append(this.write(entry.getKey(), entry.getValue().get(0)));
+ }
+
+ }
+ datasetString.append(".");
+ distributionStrings.add(datasetString.toString());
+ }
+ for (String distribString : distributionStrings) {
+ LOGGER.info(distribString);
+ }
+ }
+
+
+ return distributionStrings;
+ }
+
+ public String write(String indentifier, String value) {
+ value = value.replaceAll("[\\r\\n]+", " ").replaceAll("\"", "\\\\\"");
+ String property = "";
+
+ switch (indentifier) {
+ case "dcat:landingPage":
+ case "dcat:theme":
+ case "dcat:contactPoint":
+ case "dcat:accessURL":
+ case "dcat:downloadURL":
+ case "dct:language":
+ if (value.startsWith("http")) {
+ property += indentifier + " <" + value + ">;\n";
+ }
+ break;
+ case "dct:issued":
+ case "dct:modified":
+ property += indentifier + " \"" + value + "\"^^xsd:dateTime;\n";
+ break;
+ case "dct:publisher":
+ property += indentifier + "[ a foaf:Agent; foaf:name \"" + value + "\"];\n";
+ break;
+ default:
+ property += indentifier + " \"" + value + "\";\n";
+ break;
+ }
+
+ return property;
+ }
+
+ public static boolean isReplicable(String dcatProperty, List<Path> paths) {
+ String card = paths.stream().filter(e -> e.getProperty().equals(dcatProperty)).map(e -> e.getCard()).findFirst().orElse("1..1");
+ return card.endsWith("n");
+ }
+
+ @Async
+ public CompletableFuture<JSONObject> getJson(String url) throws JSONException {
+ return this.requestUtil.getJson(url);
+ }
+
+ @Async
+ public CompletableFuture<List<Map<String, List<String>>>> getAllValuesFomCustomPath(JSONObject json, List<Path> paths) {
+ Map<String, List<String>> propertyValueMap = new HashMap<>();
+ List<Map<String, List<String>>> propertyValueList = new ArrayList<>();
+
+ for (Path path : paths) {
+ try {
+ String[] array = path.getPath().split(" : ");
+ List<String> list = new ArrayList<>(Arrays.asList(array));
+
+ JSONObject jsonT = json;
+ if (list.size() == 1) {
+ assert jsonT != null;
+ if (jsonT.optJSONArray(list.get(0)) != null) {
+ JSONArray jsonA = jsonT.optJSONArray(list.get(0));
+ List<String> valueList = new ArrayList<>();
+ for (int j = 0; j < jsonA.length(); j++) {
+ valueList.add(jsonA.getString(j));
+ }
+ if (propertyValueMap.containsKey(path.getProperty())) {
+ List<String> values = propertyValueMap.get(path.getProperty());
+ values.addAll(valueList);
+ propertyValueMap.replace(path.getProperty(), values);
+ } else {
+ propertyValueMap.put(path.getProperty(), valueList);
+ }
+ } else {
+ if (propertyValueMap.containsKey(path.getProperty())) {
+ List<String> values = propertyValueMap.get(path.getProperty());
+ values.add(jsonT.getString(list.get(0)));
+ propertyValueMap.replace(path.getProperty(), values);
+ } else {
+ propertyValueMap.put(path.getProperty(), List.of(jsonT.getString(list.get(0))));
+ }
+ }
+ } else {
+ for (int i = 0; i < list.size(); i++) {
+ if (i < list.size() - 1) {
+ assert jsonT != null;
+ if (jsonT.optJSONObject(list.get(i)) != null) {
+ jsonT = jsonT.optJSONObject(list.get(i));
+ } else if (jsonT.optJSONArray(list.get(i)) != null) {
+ JSONArray jsonA = jsonT.optJSONArray(list.get(i));
+
+ if (i < list.size() - 2) {
+ jsonT = jsonA.getJSONObject(Integer.parseInt(list.get(i + 1)));
+
+ list.remove(i + 1);
+ }
+ }
+ } else {
+ if (jsonT.optJSONArray(list.get(i)) != null) {
+ if (!jsonT.has(list.get(i))) {
+ if (propertyValueMap.containsKey(path.getProperty())) {
+ List<String> values = propertyValueMap.get(path.getProperty());
+ values.add("undefined");
+ propertyValueMap.replace(path.getProperty(), values);
+ } else {
+ propertyValueMap.put(path.getProperty(), List.of("undefined"));
+ }
+ } else {
+ JSONArray jsonA = jsonT.optJSONArray(list.get(i));
+ List<String> valueList = new ArrayList<>();
+ for (int j = 0; j < jsonA.length(); j++) {
+ valueList.add(jsonA.getString(j));
+ }
+ if (propertyValueMap.containsKey(path.getProperty())) {
+ List<String> values = propertyValueMap.get(path.getProperty());
+ values.addAll(valueList);
+ propertyValueMap.replace(path.getProperty(), values);
+ } else {
+ propertyValueMap.put(path.getProperty(), valueList);
+ }
+ break;
+
+ }
+ } else if (jsonT.optJSONArray(list.get(list.size() - 2)) != null) {
+
+ JSONArray jsonA = jsonT.optJSONArray(list.get(list.size() - 2));
+ String value = jsonA.optString(Integer.parseInt(list.get(i)));
+ if (propertyValueMap.containsKey(path.getProperty())) {
+ List<String> values = propertyValueMap.get(path.getProperty());
+ values.add(value);
+ propertyValueMap.replace(path.getProperty(), values);
+ } else {
+ propertyValueMap.put(path.getProperty(), List.of(value));
+ }
+ break;
+ } else if (jsonT.optString(list.get(i)) != null && jsonT.has(list.get(i))) {
+ String value = jsonT.optString(list.get(i));
+ if (propertyValueMap.containsKey(path.getProperty())) {
+ List<String> values = propertyValueMap.get(path.getProperty());
+ values.add(value);
+ propertyValueMap.replace(path.getProperty(), values);
+ } else {
+ propertyValueMap.put(path.getProperty(), List.of(value));
+ }
+ break;
+ } else {
+ var keys = jsonT.keys();
+ while (keys.hasNext()) {
+ String key = keys.next().toString();
+ if (jsonT.optJSONArray(key) != null) {
+ JSONArray jsonA = jsonT.optJSONArray(key);
+ for (int j = 0; j < jsonA.length(); j++) {
+ String value = jsonA.getJSONObject(j).getString(list.get(i));
+ if (propertyValueMap.containsKey(path.getProperty())) {
+ List<String> values = propertyValueMap.get(path.getProperty());
+ values.add(value);
+ propertyValueMap.replace(path.getProperty(), values);
+ } else {
+ propertyValueMap.put(path.getProperty(), List.of(value));
+ }
+ }
+
+ }
+ }
+ }
+ }
+
+ }
+ }
+ } catch (JSONException e1) {
+ if (propertyValueMap.containsKey(path.getProperty())) {
+ List<String> values = propertyValueMap.get(path.getProperty());
+ values.add("undefined");
+ propertyValueMap.replace(path.getProperty(), values);
+ } else {
+ propertyValueMap.put(path.getProperty(), List.of("undefined"));
+ }
+ LOGGER.warn(e1.getMessage());
+ }
+ }
+ propertyValueList.add(propertyValueMap);
+ return CompletableFuture.completedFuture(propertyValueList);
+ }
+
+ @Async
+ public CompletableFuture<List<Map<String, List<String>>>> getAllValuesFomJsonpath(JSONObject json, List<Path> paths) {
+ Map<String, List<String>> propertyValueMap = new HashMap<>();
+ List<Map<String, List<String>>> propertyValueList = new ArrayList<>();
+
+ for (Path path : paths) {
+ DocumentContext jsonContext = JsonPath.parse(json.toString());
+ Object value;
+ try {
+ value = jsonContext.read(path.getPath());
+ } catch (PathNotFoundException e) {
+ LOGGER.warn(e.getMessage());
+ value = "undefined";
+ }
+
+ if (value instanceof String) {
+ String valueString = (String) value;
+ if (propertyValueMap.containsKey(path.getProperty())) {
+ List<String> values = propertyValueMap.get(path.getProperty());
+ values.add(valueString);
+ propertyValueMap.replace(path.getProperty(), values);
+ } else {
+ propertyValueMap.put(path.getProperty(), List.of(valueString));
+ }
+ } else if (value instanceof Integer || value instanceof Long) {
+ String valueInt = "" + value;
+ if (propertyValueMap.containsKey(path.getProperty())) {
+ List<String> values = propertyValueMap.get(path.getProperty());
+ values.add(String.valueOf(valueInt));
+ propertyValueMap.replace(path.getProperty(), values);
+ } else {
+ propertyValueMap.put(path.getProperty(), List.of(String.valueOf(valueInt)));
+ }
+ } else if (value instanceof net.minidev.json.JSONArray) {
+ List<String> valueList = (List<String>) value;
+ if (propertyValueMap.containsKey(path.getProperty())) {
+ List<String> values = propertyValueMap.get(path.getProperty());
+ values.addAll(valueList);
+ propertyValueMap.replace(path.getProperty(), values);
+ } else {
+ propertyValueMap.put(path.getProperty(), valueList);
+ }
+ }
+ }
+ propertyValueList.add(propertyValueMap);
+ return CompletableFuture.completedFuture(propertyValueList);
+ }
+
+ public Map<String, List<Map<String, List<String>>>> mapToDcat(String urlRepo, List<Path> paths, boolean isJsonpath) throws JSONException, ExecutionException, InterruptedException {
+ JSONObject jsonDataset = this.getJson(urlRepo).get();
+ boolean distributionPathIsPresent = paths.stream().anyMatch(e -> e.getProperty().equals("dcat:distribution"));
+
+ List<Path> datasetPaths = paths.stream().filter(e -> e.getDcatClass().equals("dcat:dataset")).collect(Collectors.toList());
+ List<Path> distributionPaths = paths.stream().filter(e -> e.getDcatClass().equals("dcat:distribution")).collect(Collectors.toList());
+
+ List<Map<String, List<String>>> datasetValues = (isJsonpath) ? this.getAllValuesFomJsonpath(jsonDataset, datasetPaths).get() : this.getAllValuesFomCustomPath(jsonDataset, datasetPaths).get();
+ List<Map<String, List<String>>> distributionValues = new ArrayList<>();
+ if (distributionPathIsPresent) {
+
+ String distributionPath = paths.stream().filter(e -> e.getProperty().equals("dcat:distribution") && e.getDcatClass().equals("dcat:distribution")).map(Path::getPath).collect(Collectors.toList()).get(0);
+ distributionPaths.remove(0);
+
+ try {
+ JSONArray jsonDistribution = new JSONArray();
+ if (!isJsonpath) {
+ List<String> parentDistributionPaths = List.of(distributionPath.split(" : "));
+ JSONObject jsonT = jsonDataset;
+ for (int i = 0; i < parentDistributionPaths.size(); i++) {
+
+ if (i < parentDistributionPaths.size() - 1) {
+ jsonT = jsonT.getJSONObject(parentDistributionPaths.get(i));
+ } else {
+ jsonDistribution = jsonT.getJSONArray(parentDistributionPaths.get(i));
+ }
+ }
+ } else {
+ DocumentContext jsonContext = JsonPath.parse(jsonDataset.toString());
+ net.minidev.json.JSONArray read = jsonContext.read(distributionPath);
+
+ for (Object dist : read) {
+ jsonDistribution.put(new JSONObject((LinkedHashMap) dist));
+ }
+
+ }
+
+
+ for (int i = 0; i < jsonDistribution.length(); i++) {
+ if ((isJsonpath)) {
+ distributionValues.addAll(this.getAllValuesFomJsonpath((JSONObject) jsonDistribution.get(i), distributionPaths).get());
+ } else {
+ distributionValues.addAll(this.getAllValuesFomCustomPath((JSONObject) jsonDistribution.get(i), distributionPaths).get());
+ }
+ }
+ } catch (PathNotFoundException e) {
+ LOGGER.error(e.getMessage());
+ }
+ } else {
+ distributionValues = (isJsonpath) ? this.getAllValuesFomJsonpath(jsonDataset, distributionPaths).get() : this.getAllValuesFomCustomPath(jsonDataset, distributionPaths).get();
+ }
+
+ Map<String, List<Map<String, List<String>>>> valuesMap = new HashMap<>();
+ valuesMap.put("dataset", datasetValues);
+ valuesMap.put("distribution", distributionValues);
+ return valuesMap;
+ }
+
+
+ @Async
+ public CompletableFuture<ResponseEntity<String>> asyncPostToFdp(String path, String fdpUrl, String dataset, String fdpToken) throws BadRequest {
+
+ HttpHeaders headers = new HttpHeaders();
+ headers.add("Accept", "text/turtle;charset=UTF-8");
+ headers.add("Content-Type", "text/turtle;charset=UTF-8");
+ headers.setBearerAuth(fdpToken);
+
+ HttpEntity<String> entity = new HttpEntity<>(dataset, headers);
+
+ return CompletableFuture.completedFuture(this.restTemplate.exchange(fdpUrl + path, HttpMethod.POST, entity, String.class));
+
+ }
+
+ @Async
+ public void draftToPublished(String fdpToken, String url) {
+ HttpHeaders headers = new HttpHeaders();
+
+ headers.setBearerAuth(fdpToken);
+ headers.setContentType(MediaType.APPLICATION_JSON);
+
+ HttpEntity<PublishedRequest> entity = new HttpEntity<>(new PublishedRequest("PUBLISHED"), headers);
+
+ this.restTemplate.exchange(url + "/meta/state", HttpMethod.PUT, entity, Void.class);
+ }
}
diff --git a/src/main/java/com/smartharvester/util/SmartHarvesterRequestUtil.java b/src/main/java/com/smartharvester/util/SmartHarvesterRequestUtil.java
index 8aab62e..0c589b9 100644
--- a/src/main/java/com/smartharvester/util/SmartHarvesterRequestUtil.java
+++ b/src/main/java/com/smartharvester/util/SmartHarvesterRequestUtil.java
@@ -24,7 +24,9 @@ import org.springframework.web.client.RestTemplate;
@Component
public class SmartHarvesterRequestUtil {
- public JSONObject getJson(String urlRepo) throws JSONException {
+
+ @Async
+ public CompletableFuture<JSONObject> getJson(String urlRepo) throws JSONException {
URL url;
HttpURLConnection urlConnection = null;
String result = null;
@@ -45,7 +47,7 @@ public class SmartHarvesterRequestUtil {
json = new JSONObject(result);
- return json;
+ return CompletableFuture.completedFuture(json);
}
private String readStream(InputStream input) throws IOException {
--
GitLab