Skip to content
Snippets Groups Projects
Commit 658d7ce8 authored by Baptiste Toulemonde's avatar Baptiste Toulemonde
Browse files

fix publish distrib to fdp

parent a5d2bf46
No related branches found
No related tags found
1 merge request!17Feature/json path
Hide whitespace changes
Inline Side-by-side
Showing
with 451 additions and 351 deletions
src/main/java/com/smartharvester/config/AsynConfig.java
+ 6
− 1
View file @ 658d7ce8
package com.smartharvester.config;
import java.util.concurrent.Callable;
import java.util.concurrent.Executor;
import java.util.concurrent.Future;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.AsyncTaskExecutor;
import org.springframework.scheduling.annotation.AsyncConfigurer;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
@Configuration
@EnableAsync
public class AsynConfig {
public class AsynConfig implements AsyncConfigurer {
@Bean
public Executor asyncExecutor() {
......@@ -27,3 +31,4 @@ public class AsynConfig {
}
}
src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java
+ 19
− 21
View file @ 658d7ce8
package com.smartharvester.controller;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import org.json.JSONException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.client.HttpStatusCodeException;
import com.smartharvester.model.mapping.Path;
......@@ -54,7 +51,7 @@ public class SmartHarvesterMappingController {
@PostMapping("/publish")
public ResponseEntity<MappingResponse> map(@RequestBody MappingRequest mappingRequest,
@RequestParam("catalogId") String catalogId, @RequestParam String fdpUrl, @RequestParam boolean isJsonpath) throws ExecutionException, InterruptedException {
@RequestParam("catalogId") String catalogId, @RequestParam String fdpUrl, @RequestParam boolean isJsonpath) {
ResponseEntity<?> responseEntity;
List<String> urls = mappingRequest.getUrls();
List<Path> paths = mappingRequest.getPaths();
......@@ -67,28 +64,26 @@ public class SmartHarvesterMappingController {
for (String url : urls) {
String datasetId;
List<String> properties;
properties = (isJsonpath) ? this.mappingService.getValuesFromJsonpath(url, paths).get() : this.mappingService.getValues(url, paths).get();
String dataset = this.mappingService.getDatasetString(catalogId, properties.get(0), fdpUrl);
String locationDataset;
try {
String dataset = this.mappingService.buildDataset(url, paths, isJsonpath, catalogId, fdpUrl);
responseEntity = this.mappingService.asyncPostToFdp("/dataset", fdpUrl, dataset, fdpToken).get();
locationDataset = Objects.requireNonNull(responseEntity.getHeaders().getLocation()).toString();
datasetId = locationDataset.substring(locationDataset.indexOf("dataset/") + 8);
HttpStatus statusCode = responseEntity.getStatusCode();
if (!distributionPaths.isEmpty()) {
String distribution = this.mappingService.getDistributionString(datasetId, properties.get(1),
fdpUrl);
try {
ResponseEntity<String> distributionResponse = this.mappingService.asyncPostToFdp("/distribution", fdpUrl, distribution, fdpToken).get();
List<String> distributions = this.mappingService.buildDistribution(url, paths, isJsonpath, datasetId, fdpUrl);
for (String distribution: distributions) {
try {
ResponseEntity<String> distributionResponse = this.mappingService.asyncPostToFdp("/distribution", fdpUrl, distribution, fdpToken).get();
String locationDistribution = Objects.requireNonNull(Objects.requireNonNull(distributionResponse.getHeaders().getLocation()).toString());
if (distributionResponse.getStatusCode().value() == 201) {
this.mappingService.draftToPublished(fdpToken, locationDistribution);
if (distributionResponse.getStatusCode().value() == 201) {
this.mappingService.draftToPublished(fdpToken, locationDistribution);
}
} catch (ExecutionException | InterruptedException e) {
LOGGER.warn(e.getMessage());
}
} catch (HttpStatusCodeException e) {
LOGGER.warn(e.getMessage());
}
}
if (statusCode.value() == 201) {
......@@ -97,10 +92,13 @@ public class SmartHarvesterMappingController {
} else {
notPublishedUrl.add(url);
}
} catch (Exception e) {
} catch (JSONException | ExecutionException | IllegalStateException e ) {
notPublishedUrl.add(url + " => " + e.getMessage());
LOGGER.warn(e.getMessage());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
notPublishedUrl.add(url + " => " + e.getMessage());
LOGGER.warn(e.getMessage());
}
}
......
src/main/java/com/smartharvester/exception/CustomAsyncExceptionHandler.java 0 → 100644
+ 26
− 0
View file @ 658d7ce8
package com.smartharvester.exception;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.aop.interceptor.AsyncUncaughtExceptionHandler;
import java.lang.reflect.Method;
public class CustomAsyncExceptionHandler implements AsyncUncaughtExceptionHandler {
public static final Logger LOGGER = LoggerFactory.getLogger(CustomAsyncExceptionHandler.class);
@Override
public void handleUncaughtException(Throwable ex, Method method, Object... params) {
LOGGER.warn(ex.getMessage());
LOGGER.warn(method.getName());
for (Object param: params) {
LOGGER.warn((String) param);
}
}
public AsyncUncaughtExceptionHandler getAsyncUncaughtExceptionHandler() {
return new CustomAsyncExceptionHandler();
}
}
src/main/java/com/smartharvester/service/MappingService.java
+ 396
− 327
View file @ 658d7ce8
package com.smartharvester.service;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.PathNotFoundException;
import com.jayway.jsonpath.*;
import com.smartharvester.model.mapping.Path;
import com.smartharvester.model.mapping.request.PublishedRequest;
import com.smartharvester.util.SmartHarvesterRequestUtil;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.*;
import org.springframework.scheduling.annotation.Async;
......@@ -16,334 +16,403 @@ import org.springframework.stereotype.Service;
import org.springframework.web.client.HttpClientErrorException.BadRequest;
import org.springframework.web.client.RestTemplate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Service
public class MappingService {
@Autowired
private RestTemplate restTemplate;
@Autowired
SmartHarvesterRequestUtil requestUtil;
public String getDatasetString(String catId, String properties, String fdpUrl) {
return "@prefix dcat: <http://www.w3.org/ns/dcat#>.\n" + "@prefix dct: <http://purl.org/dc/terms/>.\n"
+ "@prefix adms: <http://www.w3.org/ns/adms#>.\n" + "@prefix dqv: <http://www.w3.org/ns/dqv#> .\n"
+ "@prefix geodcat: <http://data.europa.eu/930/>.\n" + "@prefix prov: <http://www.w3.org/ns/prov#>.\n"
+ "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n"
+ "@prefix language: <http://id.loc.gov/vocabulary/iso639-1/>.\n" + "@prefix s: <" + fdpUrl + "/>.\n"
+ "@prefix c: <" + fdpUrl + "/catalog/>.\n" + "s:new \n" + "a dcat:Dataset, dcat:Resource;\n"
+ "dct:isPartOf c:" + catId + ";\n" + properties + ".";
}
public String getDistributionString(String datasetId, String properties, String fdpUrl) {
return "@prefix dcat: <http://www.w3.org/ns/dcat#>. \n"
+ "@prefix dct: <http://purl.org/dc/terms/>.\n" + "@prefix adms: <http://www.w3.org/ns/adms#> .\n"
+ "@prefix dqv: <http://www.w3.org/ns/dqv#> .\n" + "@prefix geodcat: <http://data.europa.eu/930/>. \n"
+ "@prefix prov: <http://www.w3.org/ns/prov#>.\n"
+ "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n"
+ "@prefix skos: <http://www.w3.org/2004/02/skos/core#>.\n"
+ "@prefix spdx: <http://spdx.org/rdf/terms#>.\n" + "@prefix foaf: <http://xmlns.com/foaf/0.1/>.\n"
+ "@prefix odrl: <http://www.w3.org/ns/odrl/2/>.\n"
+ "@prefix cnt: <http://www.w3.org/2011/content#>.\n"
+ "@prefix language: <http://id.loc.gov/vocabulary/iso639-1/>.\n" + "@prefix s: <" + fdpUrl + "/>.\n"
+ "@prefix c: <" + fdpUrl + "/dataset/>.\n" + "s:new \n" + "a dcat:Distribution, dcat:Resource;\n"
+ "dct:isPartOf c:" + datasetId + ";\n" + properties + ".";
}
public static String write(String indentifier, String value) {
value = value.replaceAll("[\\r\\n]+", " ").replaceAll("\"", "\\\\\"");
String property = "";
switch (indentifier) {
case "dcat:landingPage":
case "dcat:theme":
case "dcat:contactPoint":
case "dcat:accessURL":
case "dcat:downloadURL":
case "dct:language":
property += indentifier + " <" + value + ">;\n";
break;
case "dct:issued":
case "dct:modified":
property += indentifier + " \"" + value + "\"^^xsd:dateTime;\n";
break;
case "dct:publisher":
property += indentifier + "[ a foaf:Agent; foaf:name \"" + value + "\"];\n";
break;
default:
property += indentifier + " \"" + value + "\";\n";
break;
}
return property;
}
public static boolean isReplicable(String card) {
return card.endsWith("n");
}
@Async
public CompletableFuture<List<String>> getValuesFromJsonpath(String urlRepo, List<Path> paths) {
List<String> result = new ArrayList<>();
StringBuilder datasetProperties = new StringBuilder();
StringBuilder distributionProperties = new StringBuilder();
JSONObject json = null;
try {
json = this.requestUtil.getJson(urlRepo);
} catch (JSONException e1) {
e1.printStackTrace();
}
if (paths.stream().filter(e -> e.getDcatClass().equals("dcat:dataset")).noneMatch(e -> e.getProperty().equals("dct:hasVersion"))) {
datasetProperties.append("dct:hasVersion \"null\";\n");
}
if (paths.stream().filter(e -> e.getDcatClass().equals("dcat:dataset")).noneMatch(e -> e.getProperty().equals("dct:publisher"))) {
datasetProperties.append("dct:publisher [ a foaf:Agent; foaf:name \"undefined\"];\n");
}
distributionProperties.append("dct:hasVersion \"null\";\ndct:publisher [ a foaf:Agent; foaf:name \"undefined\"];\n");
for (Path path: paths) {
assert json != null;
DocumentContext jsonContext = JsonPath.parse(json.toString());
Object value;
try {
value = jsonContext.read(path.getPath());
} catch (PathNotFoundException e) {
value = "undefined";
}
if (value instanceof String) {
String valueString = (String) value;
if (path.getDcatClass().equals("dcat:dataset")) {
datasetProperties.append(write(path.getProperty(), valueString));
} else if (path.getDcatClass().equals("dcat:distribution")) {
distributionProperties.append(write(path.getProperty(), valueString));
}
} else if (value instanceof Integer){
int valueInt= (Integer) value;
if (path.getDcatClass().equals("dcat:dataset")) {
datasetProperties.append(write(path.getProperty(), String.valueOf(valueInt)));
} else if (path.getDcatClass().equals("dcat:distribution")) {
distributionProperties.append(write(path.getProperty(), String.valueOf(valueInt)));
}
} else if (value instanceof List<?>) {
@SuppressWarnings("unchecked")
List<String> valueList = (List<String>) value;
for(String element: valueList) {
if (path.getDcatClass().equals("dcat:dataset")) {
datasetProperties.append(write(path.getProperty(), element));
} else if (path.getDcatClass().equals("dcat:distribution")) {
distributionProperties.append(write(path.getProperty(), element));
}
}
}
}
result.add(datasetProperties.toString());
result.add(distributionProperties.toString());
return CompletableFuture.completedFuture(result);
}
@Async
public CompletableFuture<List<String>> getValues(String urlRepo, List<Path> paths) {
List<String> result = new ArrayList<>();
StringBuilder datasetProperties = new StringBuilder();
StringBuilder distributionProperties = new StringBuilder();
JSONObject json = null;
try {
json = this.requestUtil.getJson(urlRepo);
} catch (JSONException e1) {
e1.printStackTrace();
}
if (paths.stream().filter(e -> e.getDcatClass().equals("dcat:dataset")).noneMatch(e -> e.getProperty().equals("dct:hasVersion"))) {
datasetProperties.append("dct:hasVersion \"null\";\n");
}
if (paths.stream().filter(e -> e.getDcatClass().equals("dcat:dataset")).noneMatch(e -> e.getProperty().equals("dct:publisher"))) {
datasetProperties.append("dct:publisher [ a foaf:Agent; foaf:name \"undefined\"];\n");
}
distributionProperties.append("dct:hasVersion \"null\";\ndct:publisher [ a foaf:Agent; foaf:name \"undefined\"];\n");
for (Path path : paths) {
try {
String[] array = path.getPath().split(" : ");
List<String> list = new ArrayList<>(Arrays.asList(array));
JSONObject jsonT = json;
if (list.size() == 1) {
assert jsonT != null;
if (jsonT.optJSONArray(list.get(0)) != null) {
JSONArray jsonA = jsonT.optJSONArray(list.get(0));
if (isReplicable(path.getCard())) {
for (int j = 0; j < jsonA.length(); j++) {
if (path.getDcatClass().equals("dcat:dataset")) {
datasetProperties.append(write(path.getProperty(), jsonA.getString(j)));
} else if (path.getDcatClass().equals("dcat:distribution")) {
distributionProperties.append(write(path.getProperty(), jsonA.getString(j)));
}
}
} else {
if (path.getDcatClass().equals("dcat:dataset")) {
datasetProperties.append(write(path.getProperty(), jsonA.getString(0)));
} else if (path.getDcatClass().equals("dcat:distribution")) {
distributionProperties.append(write(path.getProperty(), jsonA.getString(0)));
}
}
} else {
if (path.getDcatClass().equals("dcat:dataset")) {
datasetProperties.append(write(path.getProperty(), jsonT.getString(list.get(0))));
} else if (path.getDcatClass().equals("dcat:distribution")) {
distributionProperties.append(write(path.getProperty(), jsonT.getString(list.get(0))));
}
}
} else {
for (int i = 0; i < list.size(); i++) {
if (i < list.size() - 1) {
assert jsonT != null;
if (jsonT.optJSONObject(list.get(i)) != null) {
jsonT = jsonT.optJSONObject(list.get(i));
} else if (jsonT.optJSONArray(list.get(i)) != null) {
JSONArray jsonA = jsonT.optJSONArray(list.get(i));
if (i < list.size() - 2) {
jsonT = jsonA.getJSONObject(Integer.parseInt(list.get(i + 1)));
list.remove(i + 1);
}
}
} else {
if (jsonT.optJSONArray(list.get(i)) != null) {
if (!jsonT.has(list.get(i))) {
if (path.getDcatClass().equals("dcat:dataset")) {
datasetProperties.append(write(path.getProperty(), "undefined"));
} else if (path.getDcatClass().equals("dcat:distribution")) {
distributionProperties.append(write(path.getProperty(), "undefined"));
}
} else {
JSONArray jsonA = jsonT.optJSONArray(list.get(i));
if (isReplicable(path.getCard())) {
for (int j = 0; j < jsonA.length(); j++) {
if (path.getDcatClass().equals("dcat:dataset")) {
datasetProperties.append(write(path.getProperty(), jsonA.optString(j)));
} else if (path.getDcatClass().equals("dcat:distribution")) {
distributionProperties.append(write(path.getProperty(), jsonA.optString(j)));
}
}
break;
} else {
if (path.getDcatClass().equals("dcat:dataset")) {
datasetProperties.append(write(path.getProperty(), jsonA.optString(0)));
} else if (path.getDcatClass().equals("dcat:distribution")) {
distributionProperties.append(write(path.getProperty(), jsonA.optString(0)));
}
}
break;
}
} else if (jsonT.optJSONArray(list.get(list.size() - 2)) != null) {
JSONArray jsonA = jsonT.optJSONArray(list.get(list.size() - 2));
if (path.getDcatClass().equals("dcat:dataset")) {
datasetProperties.append(write(path.getProperty(),
jsonA.optString(Integer.parseInt(list.get(i)))));
} else if (path.getDcatClass().equals("dcat:distribution")) {
distributionProperties.append(write(path.getProperty(),
jsonA.optString(Integer.parseInt(list.get(i)))));
}
break;
} else if (jsonT.optString(list.get(i)) != null && jsonT.has(list.get(i))) {
if (path.getDcatClass().equals("dcat:dataset")) {
datasetProperties.append(write(path.getProperty(), jsonT.optString(list.get(i))));
} else if (path.getDcatClass().equals("dcat:distribution")) {
distributionProperties.append(write(path.getProperty(), jsonT.optString(list.get(i))));
}
break;
} else {
var keys = jsonT.keys();
while (keys.hasNext()) {
String key = keys.next().toString();
if (jsonT.optJSONArray(key) != null) {
JSONArray jsonA = jsonT.optJSONArray(key);
if (isReplicable(path.getCard())) {
for (int j = 0; j < jsonA.length(); j++) {
String value = jsonA.getJSONObject(j).getString(list.get(i));
if (path.getDcatClass().equals("dcat:dataset")) {
datasetProperties.append(write(path.getProperty(), value));
} else if (path.getDcatClass().equals("dcat:distribution")) {
distributionProperties.append(write(path.getProperty(), value));
}
}
} else {
String value = jsonA.getJSONObject(0).getString(list.get(i));
if (path.getDcatClass().equals("dcat:dataset")) {
datasetProperties.append(write(path.getProperty(), value));
} else if (path.getDcatClass().equals("dcat:distribution")) {
distributionProperties.append(write(path.getProperty(), value));
}
}
}
}
}
}
}
}
} catch (JSONException e1) {
if (path.getDcatClass().equals("dcat:dataset")) {
datasetProperties.append(write(path.getProperty(), "undefined"));
} else if (path.getDcatClass().equals("dcat:distribution")) {
distributionProperties.append(write(path.getProperty(), "undefined"));
}
}
}
result.add(datasetProperties.toString());
result.add(distributionProperties.toString());
return CompletableFuture.completedFuture(result);
}
@Async
public CompletableFuture<ResponseEntity<String>> asyncPostToFdp(String path, String fdpUrl, String dataset, String fdpToken)
throws BadRequest {
HttpHeaders headers = new HttpHeaders();
headers.add("Accept", "text/turtle;charset=UTF-8");
headers.add("Content-Type", "text/turtle;charset=UTF-8");
headers.setBearerAuth(fdpToken);
HttpEntity<String> entity = new HttpEntity<>(dataset, headers);
return CompletableFuture.completedFuture(this.restTemplate.exchange(fdpUrl + path, HttpMethod.POST, entity, String.class));
}
@Async
public void draftToPublished(String fdpToken, String url) {
HttpHeaders headers = new HttpHeaders();
headers.setBearerAuth(fdpToken);
headers.setContentType(MediaType.APPLICATION_JSON);
HttpEntity<PublishedRequest> entity = new HttpEntity<>(new PublishedRequest("PUBLISHED"),
headers);
this.restTemplate.exchange(url + "/meta/state", HttpMethod.PUT, entity, Void.class);
}
@Autowired
private RestTemplate restTemplate;
@Autowired
SmartHarvesterRequestUtil requestUtil;
public static final Logger LOGGER = LoggerFactory.getLogger(MappingService.class);
public StringBuilder getDatasetString(String catId, String fdpUrl) {
return new StringBuilder("@prefix dcat: <http://www.w3.org/ns/dcat#>.\n" + "@prefix dct: <http://purl.org/dc/terms/>.\n" + "@prefix adms: <http://www.w3.org/ns/adms#>.\n" + "@prefix dqv: <http://www.w3.org/ns/dqv#> .\n" + "@prefix geodcat: <http://data.europa.eu/930/>.\n" + "@prefix prov: <http://www.w3.org/ns/prov#>.\n" + "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n" + "@prefix language: <http://id.loc.gov/vocabulary/iso639-1/>.\n" + "@prefix s: <" + fdpUrl + "/>.\n" + "@prefix c: <" + fdpUrl + "/catalog/>.\n" + "s:new \n" + "a dcat:Dataset, dcat:Resource;\n" + "dct:isPartOf c:" + catId + ";\n");
}
public StringBuilder getDistributionString(String datasetId, String fdpUrl) {
return new StringBuilder("@prefix dcat: <http://www.w3.org/ns/dcat#>. \n" + "@prefix dct: <http://purl.org/dc/terms/>.\n" + "@prefix adms: <http://www.w3.org/ns/adms#> .\n" + "@prefix dqv: <http://www.w3.org/ns/dqv#> .\n" + "@prefix geodcat: <http://data.europa.eu/930/>. \n" + "@prefix prov: <http://www.w3.org/ns/prov#>.\n" + "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n" + "@prefix skos: <http://www.w3.org/2004/02/skos/core#>.\n" + "@prefix spdx: <http://spdx.org/rdf/terms#>.\n" + "@prefix foaf: <http://xmlns.com/foaf/0.1/>.\n" + "@prefix odrl: <http://www.w3.org/ns/odrl/2/>.\n" + "@prefix cnt: <http://www.w3.org/2011/content#>.\n" + "@prefix language: <http://id.loc.gov/vocabulary/iso639-1/>.\n" + "@prefix s: <" + fdpUrl + "/>.\n" + "@prefix c: <" + fdpUrl + "/dataset/>.\n" + "s:new \n" + "a dcat:Distribution, dcat:Resource;\n" + "dct:isPartOf c:" + datasetId + ";\n");
}
public String buildDataset(String urlRepo, List<Path> paths, boolean isJsonpath, String catId, String fdpUrl) throws JSONException, ExecutionException, InterruptedException {
Map<String, List<Map<String, List<String>>>> dcatPropertiesMap = this.mapToDcat(urlRepo, paths, isJsonpath);
Map<String, List<String>> datasetMap = dcatPropertiesMap.get("dataset").get(0);
datasetMap.putIfAbsent("dct:hasVersion", List.of("undefined"));
datasetMap.putIfAbsent("dct:publisher", List.of("undefined"));
StringBuilder datasetString = this.getDatasetString(catId, fdpUrl);
for (Map.Entry<String, List<String>> entry : datasetMap.entrySet()) {
if (entry.getValue().isEmpty()) {
entry.setValue(List.of("undefined"));
}
if (isReplicable(entry.getKey(), paths) && (entry.getValue().size() > 1)) {
for (String value : entry.getValue()) {
datasetString.append(this.write(entry.getKey(), value));
}
} else {
datasetString.append(this.write(entry.getKey(), entry.getValue().get(0)));
}
}
datasetString.append(".");
LOGGER.info(datasetString.toString());
return datasetString.toString();
}
public List<String> buildDistribution(String urlRepo, List<Path> paths, boolean isJsonpath, String datasetId, String fdpUrl) throws JSONException, ExecutionException, InterruptedException {
Map<String, List<Map<String, List<String>>>> dcatPropertiesMap = this.mapToDcat(urlRepo, paths, isJsonpath);
List<Map<String, List<String>>> distributionList = dcatPropertiesMap.get("distribution");
List<String> distributionStrings = new ArrayList<>();
if (!distributionList.get(0).entrySet().stream().map(v -> v.getValue()).allMatch(v -> v.get(0).equals("undefined"))) {
for (Map<String, List<String>> distributionMap : distributionList) {
distributionMap.putIfAbsent("dct:hasVersion", List.of("undefined"));
distributionMap.putIfAbsent("dct:publisher", List.of("undefined"));
StringBuilder datasetString = this.getDistributionString(datasetId, fdpUrl);
for (Map.Entry<String, List<String>> entry : distributionMap.entrySet()) {
if (isReplicable(entry.getKey(), paths)) {
for (String value : entry.getValue()) {
datasetString.append(this.write(entry.getKey(), value));
}
} else {
datasetString.append(this.write(entry.getKey(), entry.getValue().get(0)));
}
}
datasetString.append(".");
distributionStrings.add(datasetString.toString());
}
for (String distribString : distributionStrings) {
LOGGER.info(distribString);
}
}
return distributionStrings;
}
public String write(String indentifier, String value) {
value = value.replaceAll("[\\r\\n]+", " ").replaceAll("\"", "\\\\\"");
String property = "";
switch (indentifier) {
case "dcat:landingPage":
case "dcat:theme":
case "dcat:contactPoint":
case "dcat:accessURL":
case "dcat:downloadURL":
case "dct:language":
if (value.startsWith("http")) {
property += indentifier + " <" + value + ">;\n";
}
break;
case "dct:issued":
case "dct:modified":
property += indentifier + " \"" + value + "\"^^xsd:dateTime;\n";
break;
case "dct:publisher":
property += indentifier + "[ a foaf:Agent; foaf:name \"" + value + "\"];\n";
break;
default:
property += indentifier + " \"" + value + "\";\n";
break;
}
return property;
}
public static boolean isReplicable(String dcatProperty, List<Path> paths) {
String card = paths.stream().filter(e -> e.getProperty().equals(dcatProperty)).map(e -> e.getCard()).findFirst().orElse("1..1");
return card.endsWith("n");
}
@Async
public CompletableFuture<JSONObject> getJson(String url) throws JSONException {
return this.requestUtil.getJson(url);
}
@Async
public CompletableFuture<List<Map<String, List<String>>>> getAllValuesFomCustomPath(JSONObject json, List<Path> paths) {
Map<String, List<String>> propertyValueMap = new HashMap<>();
List<Map<String, List<String>>> propertyValueList = new ArrayList<>();
for (Path path : paths) {
try {
String[] array = path.getPath().split(" : ");
List<String> list = new ArrayList<>(Arrays.asList(array));
JSONObject jsonT = json;
if (list.size() == 1) {
assert jsonT != null;
if (jsonT.optJSONArray(list.get(0)) != null) {
JSONArray jsonA = jsonT.optJSONArray(list.get(0));
List<String> valueList = new ArrayList<>();
for (int j = 0; j < jsonA.length(); j++) {
valueList.add(jsonA.getString(j));
}
if (propertyValueMap.containsKey(path.getProperty())) {
List<String> values = propertyValueMap.get(path.getProperty());
values.addAll(valueList);
propertyValueMap.replace(path.getProperty(), values);
} else {
propertyValueMap.put(path.getProperty(), valueList);
}
} else {
if (propertyValueMap.containsKey(path.getProperty())) {
List<String> values = propertyValueMap.get(path.getProperty());
values.add(jsonT.getString(list.get(0)));
propertyValueMap.replace(path.getProperty(), values);
} else {
propertyValueMap.put(path.getProperty(), List.of(jsonT.getString(list.get(0))));
}
}
} else {
for (int i = 0; i < list.size(); i++) {
if (i < list.size() - 1) {
assert jsonT != null;
if (jsonT.optJSONObject(list.get(i)) != null) {
jsonT = jsonT.optJSONObject(list.get(i));
} else if (jsonT.optJSONArray(list.get(i)) != null) {
JSONArray jsonA = jsonT.optJSONArray(list.get(i));
if (i < list.size() - 2) {
jsonT = jsonA.getJSONObject(Integer.parseInt(list.get(i + 1)));
list.remove(i + 1);
}
}
} else {
if (jsonT.optJSONArray(list.get(i)) != null) {
if (!jsonT.has(list.get(i))) {
if (propertyValueMap.containsKey(path.getProperty())) {
List<String> values = propertyValueMap.get(path.getProperty());
values.add("undefined");
propertyValueMap.replace(path.getProperty(), values);
} else {
propertyValueMap.put(path.getProperty(), List.of("undefined"));
}
} else {
JSONArray jsonA = jsonT.optJSONArray(list.get(i));
List<String> valueList = new ArrayList<>();
for (int j = 0; j < jsonA.length(); j++) {
valueList.add(jsonA.getString(j));
}
if (propertyValueMap.containsKey(path.getProperty())) {
List<String> values = propertyValueMap.get(path.getProperty());
values.addAll(valueList);
propertyValueMap.replace(path.getProperty(), values);
} else {
propertyValueMap.put(path.getProperty(), valueList);
}
break;
}
} else if (jsonT.optJSONArray(list.get(list.size() - 2)) != null) {
JSONArray jsonA = jsonT.optJSONArray(list.get(list.size() - 2));
String value = jsonA.optString(Integer.parseInt(list.get(i)));
if (propertyValueMap.containsKey(path.getProperty())) {
List<String> values = propertyValueMap.get(path.getProperty());
values.add(value);
propertyValueMap.replace(path.getProperty(), values);
} else {
propertyValueMap.put(path.getProperty(), List.of(value));
}
break;
} else if (jsonT.optString(list.get(i)) != null && jsonT.has(list.get(i))) {
String value = jsonT.optString(list.get(i));
if (propertyValueMap.containsKey(path.getProperty())) {
List<String> values = propertyValueMap.get(path.getProperty());
values.add(value);
propertyValueMap.replace(path.getProperty(), values);
} else {
propertyValueMap.put(path.getProperty(), List.of(value));
}
break;
} else {
var keys = jsonT.keys();
while (keys.hasNext()) {
String key = keys.next().toString();
if (jsonT.optJSONArray(key) != null) {
JSONArray jsonA = jsonT.optJSONArray(key);
for (int j = 0; j < jsonA.length(); j++) {
String value = jsonA.getJSONObject(j).getString(list.get(i));
if (propertyValueMap.containsKey(path.getProperty())) {
List<String> values = propertyValueMap.get(path.getProperty());
values.add(value);
propertyValueMap.replace(path.getProperty(), values);
} else {
propertyValueMap.put(path.getProperty(), List.of(value));
}
}
}
}
}
}
}
}
} catch (JSONException e1) {
if (propertyValueMap.containsKey(path.getProperty())) {
List<String> values = propertyValueMap.get(path.getProperty());
values.add("undefined");
propertyValueMap.replace(path.getProperty(), values);
} else {
propertyValueMap.put(path.getProperty(), List.of("undefined"));
}
LOGGER.warn(e1.getMessage());
}
}
propertyValueList.add(propertyValueMap);
return CompletableFuture.completedFuture(propertyValueList);
}
@Async
public CompletableFuture<List<Map<String, List<String>>>> getAllValuesFomJsonpath(JSONObject json, List<Path> paths) {
Map<String, List<String>> propertyValueMap = new HashMap<>();
List<Map<String, List<String>>> propertyValueList = new ArrayList<>();
for (Path path : paths) {
DocumentContext jsonContext = JsonPath.parse(json.toString());
Object value;
try {
value = jsonContext.read(path.getPath());
} catch (PathNotFoundException e) {
LOGGER.warn(e.getMessage());
value = "undefined";
}
if (value instanceof String) {
String valueString = (String) value;
if (propertyValueMap.containsKey(path.getProperty())) {
List<String> values = propertyValueMap.get(path.getProperty());
values.add(valueString);
propertyValueMap.replace(path.getProperty(), values);
} else {
propertyValueMap.put(path.getProperty(), List.of(valueString));
}
} else if (value instanceof Integer || value instanceof Long) {
String valueInt = "" + value;
if (propertyValueMap.containsKey(path.getProperty())) {
List<String> values = propertyValueMap.get(path.getProperty());
values.add(String.valueOf(valueInt));
propertyValueMap.replace(path.getProperty(), values);
} else {
propertyValueMap.put(path.getProperty(), List.of(String.valueOf(valueInt)));
}
} else if (value instanceof net.minidev.json.JSONArray) {
List<String> valueList = (List<String>) value;
if (propertyValueMap.containsKey(path.getProperty())) {
List<String> values = propertyValueMap.get(path.getProperty());
values.addAll(valueList);
propertyValueMap.replace(path.getProperty(), values);
} else {
propertyValueMap.put(path.getProperty(), valueList);
}
}
}
propertyValueList.add(propertyValueMap);
return CompletableFuture.completedFuture(propertyValueList);
}
public Map<String, List<Map<String, List<String>>>> mapToDcat(String urlRepo, List<Path> paths, boolean isJsonpath) throws JSONException, ExecutionException, InterruptedException {
JSONObject jsonDataset = this.getJson(urlRepo).get();
boolean distributionPathIsPresent = paths.stream().anyMatch(e -> e.getProperty().equals("dcat:distribution"));
List<Path> datasetPaths = paths.stream().filter(e -> e.getDcatClass().equals("dcat:dataset")).collect(Collectors.toList());
List<Path> distributionPaths = paths.stream().filter(e -> e.getDcatClass().equals("dcat:distribution")).collect(Collectors.toList());
List<Map<String, List<String>>> datasetValues = (isJsonpath) ? this.getAllValuesFomJsonpath(jsonDataset, datasetPaths).get() : this.getAllValuesFomCustomPath(jsonDataset, datasetPaths).get();
List<Map<String, List<String>>> distributionValues = new ArrayList<>();
if (distributionPathIsPresent) {
String distributionPath = paths.stream().filter(e -> e.getProperty().equals("dcat:distribution") && e.getDcatClass().equals("dcat:distribution")).map(Path::getPath).collect(Collectors.toList()).get(0);
distributionPaths.remove(0);
try {
JSONArray jsonDistribution = new JSONArray();
if (!isJsonpath) {
List<String> parentDistributionPaths = List.of(distributionPath.split(" : "));
JSONObject jsonT = jsonDataset;
for (int i = 0; i < parentDistributionPaths.size(); i++) {
if (i < parentDistributionPaths.size() - 1) {
jsonT = jsonT.getJSONObject(parentDistributionPaths.get(i));
} else {
jsonDistribution = jsonT.getJSONArray(parentDistributionPaths.get(i));
}
}
} else {
DocumentContext jsonContext = JsonPath.parse(jsonDataset.toString());
net.minidev.json.JSONArray read = jsonContext.read(distributionPath);
for (Object dist : read) {
jsonDistribution.put(new JSONObject((LinkedHashMap) dist));
}
}
for (int i = 0; i < jsonDistribution.length(); i++) {
if ((isJsonpath)) {
distributionValues.addAll(this.getAllValuesFomJsonpath((JSONObject) jsonDistribution.get(i), distributionPaths).get());
} else {
distributionValues.addAll(this.getAllValuesFomCustomPath((JSONObject) jsonDistribution.get(i), distributionPaths).get());
}
}
} catch (PathNotFoundException e) {
LOGGER.error(e.getMessage());
}
} else {
distributionValues = (isJsonpath) ? this.getAllValuesFomJsonpath(jsonDataset, distributionPaths).get() : this.getAllValuesFomCustomPath(jsonDataset, distributionPaths).get();
}
Map<String, List<Map<String, List<String>>>> valuesMap = new HashMap<>();
valuesMap.put("dataset", datasetValues);
valuesMap.put("distribution", distributionValues);
return valuesMap;
}
@Async
public CompletableFuture<ResponseEntity<String>> asyncPostToFdp(String path, String fdpUrl, String dataset, String fdpToken) throws BadRequest {
HttpHeaders headers = new HttpHeaders();
headers.add("Accept", "text/turtle;charset=UTF-8");
headers.add("Content-Type", "text/turtle;charset=UTF-8");
headers.setBearerAuth(fdpToken);
HttpEntity<String> entity = new HttpEntity<>(dataset, headers);
return CompletableFuture.completedFuture(this.restTemplate.exchange(fdpUrl + path, HttpMethod.POST, entity, String.class));
}
@Async
public void draftToPublished(String fdpToken, String url) {
HttpHeaders headers = new HttpHeaders();
headers.setBearerAuth(fdpToken);
headers.setContentType(MediaType.APPLICATION_JSON);
HttpEntity<PublishedRequest> entity = new HttpEntity<>(new PublishedRequest("PUBLISHED"), headers);
this.restTemplate.exchange(url + "/meta/state", HttpMethod.PUT, entity, Void.class);
}
}
src/main/java/com/smartharvester/util/SmartHarvesterRequestUtil.java
+ 4
− 2
View file @ 658d7ce8
......@@ -24,7 +24,9 @@ import org.springframework.web.client.RestTemplate;
@Component
public class SmartHarvesterRequestUtil {
public JSONObject getJson(String urlRepo) throws JSONException {
@Async
public CompletableFuture<JSONObject> getJson(String urlRepo) throws JSONException {
URL url;
HttpURLConnection urlConnection = null;
String result = null;
......@@ -45,7 +47,7 @@ public class SmartHarvesterRequestUtil {
json = new JSONObject(result);
return json;
return CompletableFuture.completedFuture(json);
}
private String readStream(InputStream input) throws IOException {
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment