From 1390741ac2793416efa915bcb6d38c4f1568727a Mon Sep 17 00:00:00 2001
From: Baptiste Toulemonde <toulemonde@cines.fr>
Date: Wed, 28 Sep 2022 09:57:32 +0200
Subject: [PATCH] fix edit dataset
---
.../SmartHarvesterCatalogController.java | 15 +-
.../mapping/response/KeywordResponse.java | 26 +-
.../service/ElasticSearchService.java | 70 ++---
.../service/MappingService.java | 278 ++++++++++++------
4 files changed, 238 insertions(+), 151 deletions(-)
diff --git a/src/main/java/com/smartharvester/controller/SmartHarvesterCatalogController.java b/src/main/java/com/smartharvester/controller/SmartHarvesterCatalogController.java
index 4ea77e9..bac486e 100644
--- a/src/main/java/com/smartharvester/controller/SmartHarvesterCatalogController.java
+++ b/src/main/java/com/smartharvester/controller/SmartHarvesterCatalogController.java
@@ -3,6 +3,7 @@ package com.smartharvester.controller;
import com.smartharvester.dao.CatalogDaoRepository;
import com.smartharvester.dao.OpenApiDaoRepository;
import com.smartharvester.model.login.response.MessageResponse;
+import com.smartharvester.model.mapping.response.KeywordResponse;
import com.smartharvester.model.user.Catalog;
import com.smartharvester.service.MappingService;
import io.swagger.v3.oas.annotations.tags.Tag;
@@ -60,9 +61,19 @@ public class SmartHarvesterCatalogController {
}
@GetMapping("/catalog/{catalogId}")
- public ResponseEntity<?> getDatasetDetailsByCatalog(@PathVariable("catalogId") String catId, @RequestParam("baseUrl") String url, @RequestParam("token") String token) {
+ public ResponseEntity<?> getCatalogDetails(@PathVariable("catalogId") String catId, @RequestParam("baseUrl") String url, @RequestParam("token") String token) {
try {
- return this.mappingService.getDatasetDetailsByCatalog(catId, url, token);
+ List<KeywordResponse> catalogDetails = this.mappingService.getCatalogDetails(catId, url, token);
+ return ResponseEntity.ok(catalogDetails);
+ } catch (Exception e) {
+ return ResponseEntity.badRequest().body(e.getMessage());
+ }
+ }
+
+ @GetMapping("/dataset/{datasetId}")
+ public ResponseEntity<?> getDatasetDetails(@PathVariable("datasetId") String datasetId, @RequestParam("baseUrl") String url, @RequestParam("token") String token) {
+ try {
+ return ResponseEntity.ok(this.mappingService.getDatasetDetails(datasetId, url, token, true));
} catch (Exception e) {
return ResponseEntity.badRequest().body(e.getMessage());
}
diff --git a/src/main/java/com/smartharvester/model/mapping/response/KeywordResponse.java b/src/main/java/com/smartharvester/model/mapping/response/KeywordResponse.java
index c5a1fb2..1c12d24 100644
--- a/src/main/java/com/smartharvester/model/mapping/response/KeywordResponse.java
+++ b/src/main/java/com/smartharvester/model/mapping/response/KeywordResponse.java
@@ -3,16 +3,16 @@ package com.smartharvester.model.mapping.response;
import com.smartharvester.model.elasticSearch.ElasticSearchResponse;
import java.util.List;
+import java.util.Map;
public class KeywordResponse {
private String id;
private String title;
+ private String description;
private String url;
- private List<String> keywords;
-
- private ElasticSearchResponse concepts;
+ private Map<String, ElasticSearchResponse> keywords;
private List<String> conceptIri;
@@ -24,22 +24,14 @@ public class KeywordResponse {
this.id = id;
}
- public List<String> getKeywords() {
+ public Map<String, ElasticSearchResponse> getKeywords() {
return keywords;
}
- public void setKeywords(List<String> keywords) {
+ public void setKeywords(Map<String, ElasticSearchResponse> keywords) {
this.keywords = keywords;
}
- public ElasticSearchResponse getConcepts() {
- return concepts;
- }
-
- public void setConcepts(ElasticSearchResponse concepts) {
- this.concepts = concepts;
- }
-
public String getUrl() {
return url;
}
@@ -63,4 +55,12 @@ public class KeywordResponse {
public void setConceptIri(List<String> conceptIri) {
this.conceptIri = conceptIri;
}
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
}
diff --git a/src/main/java/com/smartharvester/service/ElasticSearchService.java b/src/main/java/com/smartharvester/service/ElasticSearchService.java
index decfead..0937725 100644
--- a/src/main/java/com/smartharvester/service/ElasticSearchService.java
+++ b/src/main/java/com/smartharvester/service/ElasticSearchService.java
@@ -1,6 +1,7 @@
package com.smartharvester.service;
import com.smartharvester.model.elasticSearch.ElasticSearchResponse;
+import com.smartharvester.model.elasticSearch.Result;
import com.smartharvester.model.mapping.Path;
import com.smartharvester.model.mapping.request.KeywordRequest;
import com.smartharvester.model.mapping.response.KeywordResponse;
@@ -28,6 +29,8 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Set;
+import java.util.function.Function;
import java.util.stream.Collectors;
@Service
@@ -70,7 +73,6 @@ public class ElasticSearchService {
builder = UriComponentsBuilder.fromUriString(url)
.queryParam("search1", URLEncoder.encode(search1, StandardCharsets.UTF_8));
- assert builder != null;
return this.restTemplate.exchange(builder.build(true).toUri(), HttpMethod.GET, entity, ElasticSearchResponse.class);
}
@@ -87,15 +89,12 @@ public class ElasticSearchService {
try {
assert builder != null;
response = this.restTemplate.exchange(builder.build(true).toUri(), HttpMethod.GET, entity, ElasticSearchResponse.class);
- Thread.sleep(200);
} catch (HttpClientErrorException e) {
if (e.getRawStatusCode() == 429) {
response = this.getSearchRequest(search1, search2);
} else {
LOGGER.error(e.getMessage());
}
- } catch (InterruptedException e) {
- LOGGER.error(e.getMessage());
}
return response;
}
@@ -103,7 +102,7 @@ public class ElasticSearchService {
public List<KeywordResponse> getKeywords(KeywordRequest req, String catalogId, boolean isJsonPath) {
OpenApi openApi = this.openApiService.getOpenApiByUUDI(catalogId);
List<String> urls = new ArrayList<>();
- req.getIds().forEach(id -> urls.addAll(this.openApiService.getUrls(this.openApiService.getOpenApiByUUDI(catalogId), id)));
+ req.getIds().forEach(id -> urls.addAll(this.openApiService.getUrls(openApi, id)));
List<Path> paths = req.getPaths().stream()
.filter(path -> path.getProperty().equals("dct:identifier") ||
path.getProperty().equals("dcat:keyword") ||
@@ -135,7 +134,11 @@ public class ElasticSearchService {
keywordResponse.setUrl(url);
Assert.assertNotNull(e.get("dct:title"));
keywordResponse.setTitle(e.get("dct:title").get(0));
- keywordResponse.setKeywords(e.get("dcat:keyword"));
+ List<String> keywords = e.get("dcat:keyword");
+ Set<String> keywordsSet = keywords.stream().collect(Collectors.toSet());
+ Map<String, ElasticSearchResponse> collect = keywordsSet.stream()
+ .collect(Collectors.toMap(Function.identity(), keyword -> new ElasticSearchResponse()));
+ keywordResponse.setKeywords(collect);
result.add(keywordResponse);
});
});
@@ -145,59 +148,32 @@ public class ElasticSearchService {
public List<KeywordResponse> getConceptsByKeywords(KeywordRequest req, String catalogId, boolean isJsonPath) {
List<KeywordResponse> keywordResponses = this.getKeywords(req, catalogId, isJsonPath);
keywordResponses.forEach(k -> {
- List<String> keywords = k.getKeywords();
+ Set<String> keywords = k.getKeywords().keySet();
buildListConcepts(k, keywords);
});
return keywordResponses;
}
- public void buildListConcepts(KeywordResponse k, List<String> keywords) {
+ public void buildListConcepts(KeywordResponse k, Set<String> keywords) {
if (!keywords.isEmpty()) {
- if (keywords.size() == 1 ) {
- if (isValid(keywords.get(0))) {
- extractConcepts(keywords.get(0), "", k);
+ keywords.parallelStream().forEach(keyword -> {
+ if (isValid(keyword)) {
+ extractConcepts(keyword, k);
}
- } else if(keywords.size() == 2) {
- if (isValid(keywords.get(0)) && isValid(keywords.get(1))) {
- extractConcepts(keywords.get(0), keywords.get(1), k);
- }
- } else {
- getRecursively(k, keywords, 0);
-
-
- }
+ });
}
}
- private void getRecursively(KeywordResponse k, List<String> keywords, int i) {
-
- if ( i + 2 < keywords.size()) {
- if (isValid(keywords.get(i)) && isValid(keywords.get(i + 1))) {
- extractConcepts(keywords.get(i), keywords.get(i + 1), k);
- } else if (isValid(keywords.get(i))){
- extractConcepts(keywords.get(i), keywords.get(i + 2), k);
- } else if (isValid(keywords.get(i + 1))) {
- extractConcepts(keywords.get(i + 2), keywords.get(i + 1), k);
- } else {
- this.getRecursively( k, keywords, ++i);
- }
- } else if (i + 1 < keywords.size()) {
- if (isValid(keywords.get(i)) && isValid(keywords.get(i + 1))) {
- extractConcepts(keywords.get(i), keywords.get(i + 1), k);
- } else if (isValid(keywords.get(i))){
- extractConcepts(keywords.get(i), "", k);
- } else if (isValid(keywords.get(i + 1))) {
- extractConcepts(keywords.get(i + 1), "", k);
- }
- }
- }
- private void extractConcepts(String keyword1, String keyword2, KeywordResponse k) {
- ResponseEntity<ElasticSearchResponse> searchRequest = this.getSearchRequest(keyword1, keyword2);
+ private void extractConcepts(String keyword, KeywordResponse k) {
+ ResponseEntity<ElasticSearchResponse> searchRequest = this.getSearchRequest(keyword, "");
if (searchRequest.getStatusCodeValue() == 200 && Objects.requireNonNull(searchRequest.getBody()).getCount() != 0) {
- k.setConcepts(searchRequest.getBody());
- } else {
- k.setConcepts(null);
+ ElasticSearchResponse body = searchRequest.getBody();
+ List<Result> results = body.getResults().stream().limit(20).collect(Collectors.toList());
+ body.setResults(results);
+ Map<String, ElasticSearchResponse> keywords = k.getKeywords();
+ keywords.replace(keyword, body);
+ k.setKeywords(keywords);
}
}
diff --git a/src/main/java/com/smartharvester/service/MappingService.java b/src/main/java/com/smartharvester/service/MappingService.java
index de3403f..0e29424 100644
--- a/src/main/java/com/smartharvester/service/MappingService.java
+++ b/src/main/java/com/smartharvester/service/MappingService.java
@@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.SerializationFeature;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.PathNotFoundException;
+import com.smartharvester.model.elasticSearch.ElasticSearchResponse;
import com.smartharvester.model.mapping.Path;
import com.smartharvester.model.mapping.request.MappingRequest;
import com.smartharvester.model.mapping.request.PublishedRequest;
@@ -15,6 +16,7 @@ import com.smartharvester.util.SmartHarvesterRequestUtil;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
+import org.junit.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -27,6 +29,7 @@ import org.springframework.http.ResponseEntity;
import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
+import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.HttpClientErrorException.BadRequest;
import org.springframework.web.client.RestTemplate;
@@ -36,6 +39,7 @@ import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
@@ -43,6 +47,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
+import java.util.function.Function;
import java.util.stream.Collectors;
@Service
@@ -63,19 +68,38 @@ public class MappingService {
public static final Logger LOGGER = LoggerFactory.getLogger(MappingService.class);
public StringBuilder getDatasetString(String catId, String fdpUrl) {
- return new StringBuilder("@prefix dcat: <http://www.w3.org/ns/dcat#>.\n" + "@prefix dct: <http://purl.org/dc/terms/>.\n" + "@prefix adms: <http://www.w3.org/ns/adms#>.\n" + "@prefix dqv: <http://www.w3.org/ns/dqv#> .\n" + "@prefix geodcat: <http://data.europa.eu/930/>.\n" + "@prefix prov: <http://www.w3.org/ns/prov#>.\n" + "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n" + "@prefix language: <http://id.loc.gov/vocabulary/iso639-1/>.\n" + "@prefix s: <" + fdpUrl + "/>.\n" + "@prefix c: <" + fdpUrl + "/catalog/>.\n" + "s:new \n" + "a dcat:Dataset, dcat:Resource;\n" + "dct:isPartOf c:" + catId + ";\n");
+ return new StringBuilder(
+ "@prefix dcat: <http://www.w3.org/ns/dcat#>.\n" + "@prefix dct: <http://purl.org/dc/terms/>.\n" +
+ "@prefix adms: <http://www.w3.org/ns/adms#>.\n" + "@prefix dqv: <http://www.w3.org/ns/dqv#> .\n" +
+ "@prefix geodcat: <http://data.europa.eu/930/>.\n" + "@prefix prov: <http://www.w3.org/ns/prov#>.\n" +
+ "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n" +
+ "@prefix language: <http://id.loc.gov/vocabulary/iso639-1/>.\n" + "@prefix s: <" + fdpUrl + "/>.\n" +
+ "@prefix c: <" + fdpUrl + "/catalog/>.\n" + "s:new \n" + "a dcat:Dataset, dcat:Resource;\n" +
+ "dct:isPartOf c:" + catId + ";\n");
}
public StringBuilder getDistributionString(String datasetId, String fdpUrl) {
- return new StringBuilder("@prefix dcat: <http://www.w3.org/ns/dcat#>. \n" + "@prefix dct: <http://purl.org/dc/terms/>.\n" + "@prefix adms: <http://www.w3.org/ns/adms#> .\n" + "@prefix dqv: <http://www.w3.org/ns/dqv#> .\n" + "@prefix geodcat: <http://data.europa.eu/930/>. \n" + "@prefix prov: <http://www.w3.org/ns/prov#>.\n" + "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n" + "@prefix skos: <http://www.w3.org/2004/02/skos/core#>.\n" + "@prefix spdx: <http://spdx.org/rdf/terms#>.\n" + "@prefix foaf: <http://xmlns.com/foaf/0.1/>.\n" + "@prefix odrl: <http://www.w3.org/ns/odrl/2/>.\n" + "@prefix cnt: <http://www.w3.org/2011/content#>.\n" + "@prefix language: <http://id.loc.gov/vocabulary/iso639-1/>.\n" + "@prefix s: <" + fdpUrl + "/>.\n" + "@prefix c: <" + fdpUrl + "/dataset/>.\n" + "s:new \n" + "a dcat:Distribution, dcat:Resource;\n" + "dct:isPartOf c:" + datasetId + ";\n");
+ return new StringBuilder(
+ "@prefix dcat: <http://www.w3.org/ns/dcat#>. \n" + "@prefix dct: <http://purl.org/dc/terms/>.\n" +
+ "@prefix adms: <http://www.w3.org/ns/adms#> .\n" + "@prefix dqv: <http://www.w3.org/ns/dqv#> .\n" +
+ "@prefix geodcat: <http://data.europa.eu/930/>. \n" + "@prefix prov: <http://www.w3.org/ns/prov#>.\n" +
+ "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n" +
+ "@prefix skos: <http://www.w3.org/2004/02/skos/core#>.\n" +
+ "@prefix spdx: <http://spdx.org/rdf/terms#>.\n" + "@prefix foaf: <http://xmlns.com/foaf/0.1/>.\n" +
+ "@prefix odrl: <http://www.w3.org/ns/odrl/2/>.\n" +
+ "@prefix cnt: <http://www.w3.org/2011/content#>.\n" +
+ "@prefix language: <http://id.loc.gov/vocabulary/iso639-1/>.\n" + "@prefix s: <" + fdpUrl + "/>.\n" +
+ "@prefix c: <" + fdpUrl + "/dataset/>.\n" + "s:new \n" + "a dcat:Distribution, dcat:Resource;\n" +
+ "dct:isPartOf c:" + datasetId + ";\n");
}
- public Object getDatasetByUrl (URI url, Map<String, String> header) throws Exception {
+ public Object getDatasetByUrl(URI url, Map<String, String> header) throws Exception {
return this.requestUtil.getItemByURl(url, header);
}
- public String buildDataset(URI urlRepo, List<Path> paths, boolean isJsonpath, String catId, String fdpUrl, List<String> iris) throws Exception {
+ public String buildDataset(URI urlRepo, List<Path> paths, boolean isJsonpath, String catId, String fdpUrl,
+ List<String> iris) throws Exception {
Map<String, List<Map<String, List<String>>>> dcatPropertiesMap = this.mapToDcat(urlRepo, paths, isJsonpath);
Map<String, List<String>> datasetMap = dcatPropertiesMap.get("dataset").get(0);
@@ -100,20 +124,22 @@ public class MappingService {
}
if (!iris.isEmpty()) {
- iris.forEach(iri -> datasetString.append(this.write("dcat:theme", iri) ));
+ iris.forEach(iri -> datasetString.append(this.write("dcat:theme", iri)));
}
datasetString.append(".");
LOGGER.info(datasetString.toString());
return datasetString.toString();
}
- public List<String> buildDistribution(URI urlRepo, List<Path> paths, boolean isJsonpath, String datasetId, String fdpUrl) throws Exception {
+ public List<String> buildDistribution(URI urlRepo, List<Path> paths, boolean isJsonpath, String datasetId,
+ String fdpUrl) throws Exception {
Map<String, List<Map<String, List<String>>>> dcatPropertiesMap = this.mapToDcat(urlRepo, paths, isJsonpath);
List<Map<String, List<String>>> distributionList = dcatPropertiesMap.get("distribution");
List<String> distributionStrings = new ArrayList<>();
if (!distributionList.isEmpty()) {
- if (!distributionList.get(0).entrySet().stream().map(v -> v.getValue()).allMatch(v -> v.get(0).equals("undefined")) ) {
+ if (!distributionList.get(0).entrySet().stream().map(v -> v.getValue())
+ .allMatch(v -> v.get(0).equals("undefined"))) {
for (Map<String, List<String>> distributionMap : distributionList) {
distributionMap.putIfAbsent("dct:hasVersion", List.of("undefined"));
@@ -190,17 +216,19 @@ public class MappingService {
}
public static boolean isReplicable(String dcatProperty, List<Path> paths) {
- String card = paths.stream().filter(e -> e.getProperty().equals(dcatProperty)).map(e -> e.getCard()).findFirst().orElse("1..1");
+ String card = paths.stream().filter(e -> e.getProperty().equals(dcatProperty)).map(e -> e.getCard()).findFirst()
+ .orElse("1..1");
return card.endsWith("n");
}
@Async
public CompletableFuture<JSONObject> getJson(URI url) throws Exception {
- return this.requestUtil.httpRequest(url, null, 0);
+ return this.requestUtil.httpRequest(url, null, 0);
}
@Async
- public CompletableFuture<List<Map<String, List<String>>>> getAllValuesFomCustomPath(JSONObject json, List<Path> paths) {
+ public CompletableFuture<List<Map<String, List<String>>>> getAllValuesFomCustomPath(JSONObject json,
+ List<Path> paths) {
Map<String, List<String>> propertyValueMap = new HashMap<>();
List<Map<String, List<String>>> propertyValueList = new ArrayList<>();
@@ -343,7 +371,8 @@ public class MappingService {
}
@Async
- public CompletableFuture<List<Map<String, List<String>>>> getAllValuesFomJsonpath(JSONObject json, List<Path> paths) {
+ public CompletableFuture<List<Map<String, List<String>>>> getAllValuesFomJsonpath(JSONObject json,
+ List<Path> paths) {
Map<String, List<String>> propertyValueMap = new HashMap<>();
List<Map<String, List<String>>> propertyValueList = new ArrayList<>();
@@ -390,18 +419,25 @@ public class MappingService {
return CompletableFuture.completedFuture(propertyValueList);
}
- public Map<String, List<Map<String, List<String>>>> mapToDcat(URI urlRepo, List<Path> paths, boolean isJsonpath) throws Exception {
+ public Map<String, List<Map<String, List<String>>>> mapToDcat(URI urlRepo, List<Path> paths, boolean isJsonpath)
+ throws Exception {
JSONObject jsonDataset = this.getJson(urlRepo).get();
boolean distributionPathIsPresent = paths.stream().anyMatch(e -> e.getProperty().equals("dcat:distribution"));
- List<Path> datasetPaths = paths.stream().filter(e -> e.getDcatClass().equals("dcat:dataset")).collect(Collectors.toList());
- List<Path> distributionPaths = paths.stream().filter(e -> e.getDcatClass().equals("dcat:distribution")).collect(Collectors.toList());
+ List<Path> datasetPaths =
+ paths.stream().filter(e -> e.getDcatClass().equals("dcat:dataset")).collect(Collectors.toList());
+ List<Path> distributionPaths =
+ paths.stream().filter(e -> e.getDcatClass().equals("dcat:distribution")).collect(Collectors.toList());
- List<Map<String, List<String>>> datasetValues = (isJsonpath) ? this.getAllValuesFomJsonpath(jsonDataset, datasetPaths).get() : this.getAllValuesFomCustomPath(jsonDataset, datasetPaths).get();
+ List<Map<String, List<String>>> datasetValues = (isJsonpath) ?
+ this.getAllValuesFomJsonpath(jsonDataset, datasetPaths).get() :
+ this.getAllValuesFomCustomPath(jsonDataset, datasetPaths).get();
List<Map<String, List<String>>> distributionValues = new ArrayList<>();
if (distributionPathIsPresent) {
- String distributionPath = paths.stream().filter(e -> e.getProperty().equals("dcat:distribution") && e.getDcatClass().equals("dcat:distribution")).map(Path::getPath).collect(Collectors.toList()).get(0);
+ String distributionPath = paths.stream().filter(
+ e -> e.getProperty().equals("dcat:distribution") && e.getDcatClass().equals("dcat:distribution"))
+ .map(Path::getPath).collect(Collectors.toList()).get(0);
distributionPaths.remove(0);
try {
@@ -430,16 +466,22 @@ public class MappingService {
for (int i = 0; i < jsonDistribution.length(); i++) {
if ((isJsonpath)) {
- distributionValues.addAll(this.getAllValuesFomJsonpath((JSONObject) jsonDistribution.get(i), distributionPaths).get());
+ distributionValues.addAll(
+ this.getAllValuesFomJsonpath((JSONObject) jsonDistribution.get(i), distributionPaths)
+ .get());
} else {
- distributionValues.addAll(this.getAllValuesFomCustomPath((JSONObject) jsonDistribution.get(i), distributionPaths).get());
+ distributionValues.addAll(
+ this.getAllValuesFomCustomPath((JSONObject) jsonDistribution.get(i), distributionPaths)
+ .get());
}
}
} catch (PathNotFoundException e) {
LOGGER.error(e.getMessage());
}
} else {
- distributionValues = (isJsonpath) ? this.getAllValuesFomJsonpath(jsonDataset, distributionPaths).get() : this.getAllValuesFomCustomPath(jsonDataset, distributionPaths).get();
+ distributionValues = (isJsonpath) ?
+ this.getAllValuesFomJsonpath(jsonDataset, distributionPaths).get() :
+ this.getAllValuesFomCustomPath(jsonDataset, distributionPaths).get();
}
Map<String, List<Map<String, List<String>>>> valuesMap = new HashMap<>();
@@ -450,7 +492,8 @@ public class MappingService {
@Async
- public CompletableFuture<ResponseEntity<String>> asyncPostToFdp(String path, String fdpUrl, String dataset, String fdpToken) throws BadRequest {
+ public CompletableFuture<ResponseEntity<String>> asyncPostToFdp(String path, String fdpUrl, String dataset,
+ String fdpToken) throws BadRequest {
RestTemplate restTemplate1 = new RestTemplate();
HttpHeaders headers = new HttpHeaders();
headers.add("Accept", "text/turtle;charset=UTF-8");
@@ -459,7 +502,8 @@ public class MappingService {
HttpEntity<String> entity = new HttpEntity<>(dataset, headers);
- return CompletableFuture.completedFuture(restTemplate1.exchange(fdpUrl + path, HttpMethod.POST, entity, String.class));
+ return CompletableFuture.completedFuture(
+ restTemplate1.exchange(fdpUrl + path, HttpMethod.POST, entity, String.class));
}
@@ -497,11 +541,13 @@ public class MappingService {
List<String> distributions = this.buildDistribution(new URI(url), paths, isJsonpath, datasetId,
fdpUrl);
- for (String distribution: distributions) {
+ for (String distribution : distributions) {
try {
- ResponseEntity<String> distributionResponse = this.asyncPostToFdp("/distribution", fdpUrl.replace(":8080", ""), distribution,
- fdpToken).get();
- String locationDistribution = Objects.requireNonNull(Objects.requireNonNull(distributionResponse.getHeaders().getLocation()).toString());
+ ResponseEntity<String> distributionResponse =
+ this.asyncPostToFdp("/distribution", fdpUrl.replace(":8080", ""), distribution,
+ fdpToken).get();
+ String locationDistribution = Objects.requireNonNull(
+ Objects.requireNonNull(distributionResponse.getHeaders().getLocation()).toString());
if (distributionResponse.getStatusCode().value() == 201) {
this.draftToPublished(fdpToken, locationDistribution.replace(":8080", ""));
}
@@ -516,7 +562,7 @@ public class MappingService {
} else {
notPublishedUrl.add(url);
}
- } catch (JSONException | ExecutionException | IllegalStateException e ) {
+ } catch (JSONException | ExecutionException | IllegalStateException e) {
notPublishedUrl.add(url + " => " + e.getMessage());
LOGGER.warn(e.getMessage());
} catch (Exception e) {
@@ -526,8 +572,8 @@ public class MappingService {
}
}
- public ResponseEntity<?> getDatasetDetailsByCatalog(String catId, String url, String fdpToken)
- throws JsonProcessingException, JSONException {
+ public List<KeywordResponse> getCatalogDetails(String catId, String url, String fdpToken)
+ throws JsonProcessingException, JSONException, HttpClientErrorException {
HttpHeaders headers = new HttpHeaders();
headers.add("Accept", "application/ld+json");
headers.setBearerAuth(fdpToken);
@@ -535,48 +581,86 @@ public class MappingService {
ResponseEntity<Object> response =
this.restTemplate.exchange(url + "/catalog/" + catId, HttpMethod.GET, entity, Object.class);
- if (response.getStatusCodeValue() == 200) {
+
ObjectMapper mapper = new ObjectMapper();
JSONArray body = new JSONArray(mapper.writeValueAsString(response.getBody()));
- List<String> datasets = getValuesFromJsonLd(body.getJSONObject(1), "http://www.w3.org/ns/dcat#dataset", "@id");
+ List<String> datasets =
+ getValuesFromJsonLd(body.getJSONObject(1), "http://www.w3.org/ns/dcat#dataset", "@id");
- List<KeywordResponse> keywordResponses = new ArrayList<>();
- datasets.parallelStream().forEach(datasetUrl -> {
- KeywordResponse keywordResponse = new KeywordResponse();
- keywordResponse.setUrl(datasetUrl);
+ List<KeywordResponse> keywordResponses = new ArrayList<>();
+ datasets.stream().forEach(datasetUrl -> {
+ KeywordResponse keywordResponse =
+ getDatasetDetails(datasetUrl.substring(datasetUrl.indexOf("dataset/") + 8), url, fdpToken, false);
+ keywordResponses.add(keywordResponse);
+ });
+ return keywordResponses.stream().sorted(Comparator.comparing(KeywordResponse::getTitle)).collect(
+ Collectors.toList());
+ }
- ResponseEntity<Object> datasetResponse =
- this.restTemplate.exchange(datasetUrl.replace(":8080", ""), HttpMethod.GET, entity, Object.class);
- if (datasetResponse.getStatusCodeValue() == 200) {
+ public KeywordResponse getDatasetDetails(String datasetId, String url, String fdpToken, boolean getConcepts) {
+ String datasetUrl = url + "/dataset/" + datasetId;
+ HttpHeaders headers = new HttpHeaders();
+ headers.add("Accept", "application/ld+json");
+ headers.setBearerAuth(fdpToken);
+ HttpEntity entity = new HttpEntity<>(headers);
+ KeywordResponse keywordResponse = new KeywordResponse();
+ keywordResponse.setUrl(datasetUrl);
+ keywordResponse.setId(datasetUrl.substring(datasetUrl.indexOf("dataset/") + 8));
- JSONArray datasetBodyArray = null;
- JSONObject datasetBody = null;
- try {
- datasetBodyArray = new JSONArray(mapper.writeValueAsString(datasetResponse.getBody()));
- datasetBody = datasetBodyArray.getJSONObject(1);
- } catch (Exception e) {
- LOGGER.error(e.getMessage());
+ ResponseEntity<Object> datasetResponse =
+ this.restTemplate.exchange(datasetUrl.replace(":8080", ""), HttpMethod.GET, entity, Object.class);
+ if (datasetResponse.getStatusCodeValue() == 200) {
+
+ JSONArray datasetBodyArray = null;
+ JSONObject datasetBody = null;
+ try {
+ ObjectMapper mapper = new ObjectMapper();
+ datasetBodyArray = new JSONArray(mapper.writeValueAsString(datasetResponse.getBody()));
+ int index = 0;
+ int max = 0;
+ for (int i = 0; i < datasetBodyArray.length(); i++) {
+ if (null != datasetBodyArray.optJSONObject(i)) {
+ if (datasetBodyArray.getJSONObject(i).length() > max) {
+ max = datasetBodyArray.getJSONObject(i).length();
+ index = i;
+ }
}
+ }
+ datasetBody = datasetBodyArray.getJSONObject(index);
+ } catch (Exception e) {
+ LOGGER.error(e.getMessage());
+ }
- List<String> keywords = getValuesFromJsonLd(datasetBody, "http://www.w3.org/ns/dcat#keyword", "@value");
- keywordResponse.setKeywords(keywords);
- this.esService.buildListConcepts(keywordResponse, keywords);
+ List<String> keywords = getValuesFromJsonLd(datasetBody, "http://www.w3.org/ns/dcat#keyword", "@value");
+ Assert.assertNotNull(keywords);
+ Map<String, ElasticSearchResponse> collect = keywords.stream()
+ .collect(Collectors.toMap(Function.identity(), keyword -> new ElasticSearchResponse()));
+ keywordResponse.setKeywords(collect);
- List<String> themes = getValuesFromJsonLd(datasetBody, "http://www.w3.org/ns/dcat#theme", "@id");
- keywordResponse.setConceptIri(themes);
- if (!getValuesFromJsonLd(datasetBody, "http://purl.org/dc/terms/title", "@value").isEmpty()) {
- String title = getValuesFromJsonLd(datasetBody, "http://purl.org/dc/terms/title","@value").get(0);
- keywordResponse.setTitle(title);
- }
- }
- keywordResponses.add(keywordResponse);
- });
- return ResponseEntity.ok().body(keywordResponses);
+ List<String> themes = getValuesFromJsonLd(datasetBody, "http://www.w3.org/ns/dcat#theme", "@id");
+ keywordResponse.setConceptIri(themes);
+ if (!getValuesFromJsonLd(datasetBody, "http://purl.org/dc/terms/title", "@value").isEmpty()) {
+ String title = getValuesFromJsonLd(datasetBody, "http://purl.org/dc/terms/title", "@value").get(0);
+ keywordResponse.setTitle(title);
+ } else {
+ keywordResponse.setTitle("undefined");
+ }
+ if (!getValuesFromJsonLd(datasetBody, "http://purl.org/dc/terms/description", "@value").isEmpty()) {
+ String description =
+ getValuesFromJsonLd(datasetBody, "http://purl.org/dc/terms/description", "@value").get(0);
+ keywordResponse.setDescription(description);
+ } else {
+ keywordResponse.setDescription("undefined");
+ }
+ if (getConcepts) {
+ this.esService.buildListConcepts(keywordResponse, collect.keySet());
+
+ }
}
- return ResponseEntity.status(response.getStatusCode()).body(response.getBody());
+ return keywordResponse;
}
private List<String> getValuesFromJsonLd(JSONObject datasetBody, String property, String key) {
@@ -609,50 +693,66 @@ public class MappingService {
ObjectMapper mapper = new ObjectMapper();
try {
JSONArray body = new JSONArray(mapper.writeValueAsString(response.getBody()));
- concept.getIris().parallelStream().forEach(iri -> {
- if (null != body.optJSONObject(1)) {
- if (null != body.optJSONObject(1).optJSONArray("http://www.w3.org/ns/dcat#theme")) {
- try {
- body.getJSONObject(1)
- .getJSONArray("http://www.w3.org/ns/dcat#theme")
- .put(new JSONObject().put("@id", iri));
- } catch (JSONException e) {
- notPublishedUrl.add(concept.getUrl() + " => " + e.getMessage());
- LOGGER.error(e.getMessage());
- }
- } else {
+ int index = 0;
+ int max = 0;
+ for (int i = 0; i < body.length(); i++) {
+ if (null != body.optJSONObject(i)) {
+ if (body.getJSONObject(i).length() > max) {
+ max = body.getJSONObject(i).length();
+ index = i;
+ }
+ }
+ }
+ if (concept.getIris().size() > 0) {
+ int finalIndex = index;
+ concept.getIris().forEach(iri -> {
+ if (null != body.optJSONObject(finalIndex)) {
try {
- body.getJSONObject(1)
- .put("http://www.w3.org/ns/dcat#theme", new JSONArray());
- body.getJSONObject(1)
- .getJSONArray("http://www.w3.org/ns/dcat#theme")
- .put(new JSONObject().put("@id", iri));
+ if (concept.getIris().indexOf(iri) == 0) {
+ body.getJSONObject(finalIndex).remove("http://www.w3.org/ns/dcat#theme");
+ body.getJSONObject(finalIndex)
+ .put("http://www.w3.org/ns/dcat#theme", new JSONArray());
+ }
+ if (null != body.getJSONObject(finalIndex).optJSONArray("http://www.w3.org/ns/dcat#theme")) {
+ body.getJSONObject(finalIndex)
+ .getJSONArray("http://www.w3.org/ns/dcat#theme")
+ .put(new JSONObject().put("@id", iri));
+ }
} catch (JSONException e) {
notPublishedUrl.add(concept.getUrl() + " => " + e.getMessage());
LOGGER.error(e.getMessage());
}
}
- final ObjectMapper objectMapper = new ObjectMapper();
- List<Object> list = null;
+ });
+ } else {
+ if (null != body.optJSONObject(index)) {
try {
- list = objectMapper.readValue(body.toString(), new TypeReference<List<Object>>() {
- });
- } catch (JsonProcessingException e) {
+ body.getJSONObject(index).remove("http://www.w3.org/ns/dcat#theme");
+ } catch (JSONException e) {
+ notPublishedUrl.add(concept.getUrl() + " => " + e.getMessage());
LOGGER.error(e.getMessage());
}
+ }
+ }
+ final ObjectMapper objectMapper = new ObjectMapper();
+ List<Object> list = null;
+ try {
+ list = objectMapper.readValue(body.toString(), new TypeReference<List<Object>>() {
+ });
+ } catch (JsonProcessingException e) {
+ LOGGER.error(e.getMessage());
+ }
- HttpEntity entityPut = new HttpEntity(list, headers);
- ResponseEntity<Object> exchange =
- this.restTemplate.exchange(concept.getUrl().replace(":8080", ""), HttpMethod.PUT,
- entityPut, Object.class);
- if (exchange.getStatusCodeValue() == 200) {
- publishedUrl.add(concept.getUrl());
- }
- }
- });
+ HttpEntity entityPut = new HttpEntity(list, headers);
+ ResponseEntity<Object> exchange =
+ this.restTemplate.exchange(concept.getUrl().replace(":8080", ""), HttpMethod.PUT,
+ entityPut, Object.class);
+ if (exchange.getStatusCodeValue() == 200) {
+ publishedUrl.add(concept.getUrl());
+ }
} catch (Exception e) {
LOGGER.error(e.getMessage());
notPublishedUrl.add(concept.getUrl() + " => " + e.getMessage());
--
GitLab