From 98078cf7c7557b7022b85f2be474e27be22f732e Mon Sep 17 00:00:00 2001
From: Baptiste Toulemonde <toulemonde@cines.fr>
Date: Fri, 7 Oct 2022 13:25:45 +0200
Subject: [PATCH 1/3] wip
---
.../SmartHarvesterMappingController.java | 3 +-
.../model/mapping/request/Concepts.java | 6 ++--
.../model/mapping/request/Iri.java | 32 +++++++++++++++++++
.../service/MappingService.java | 31 +++++++++++++++---
src/main/resources/application-dev.properties | 2 +-
5 files changed, 65 insertions(+), 9 deletions(-)
create mode 100644 src/main/java/com/smartharvester/model/mapping/request/Iri.java
diff --git a/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java b/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java
index 854d045..42f3c97 100644
--- a/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java
+++ b/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java
@@ -3,6 +3,7 @@ package com.smartharvester.controller;
import com.smartharvester.model.mapping.Path;
import com.smartharvester.model.mapping.request.Concepts;
+import com.smartharvester.model.mapping.request.Iri;
import com.smartharvester.model.mapping.request.MappingRequest;
import com.smartharvester.model.mapping.response.MappingResponse;
import com.smartharvester.service.MappingFromIsoService;
@@ -70,7 +71,7 @@ public class SmartHarvesterMappingController {
List<Path> distributionPaths = paths.stream().filter(path -> path.getDcatClass().equals("dcat:distribution"))
.collect(Collectors.toList());
- Map<String, List<String>> urls = new HashMap<>();
+ Map<String, List<Iri>> urls = new HashMap<>();
concepts.parallelStream().forEach(
concept -> urls.put(
this.openApiService.getUrls(this.openApiService.getOpenApiByUUDI(catalogId), concept.getId()).get(0),
diff --git a/src/main/java/com/smartharvester/model/mapping/request/Concepts.java b/src/main/java/com/smartharvester/model/mapping/request/Concepts.java
index 7e90861..1f93005 100644
--- a/src/main/java/com/smartharvester/model/mapping/request/Concepts.java
+++ b/src/main/java/com/smartharvester/model/mapping/request/Concepts.java
@@ -6,7 +6,7 @@ public class Concepts {
private String id;
private String url;
- private List<String> iris;
+ private List<Iri> iris;
public String getId() {
return id;
@@ -16,11 +16,11 @@ public class Concepts {
this.id = id;
}
- public List<String> getIris() {
+ public List<Iri> getIris() {
return iris;
}
- public void setIris(List<String> iris) {
+ public void setIris(List<Iri> iris) {
this.iris = iris;
}
diff --git a/src/main/java/com/smartharvester/model/mapping/request/Iri.java b/src/main/java/com/smartharvester/model/mapping/request/Iri.java
new file mode 100644
index 0000000..5cd16d2
--- /dev/null
+++ b/src/main/java/com/smartharvester/model/mapping/request/Iri.java
@@ -0,0 +1,32 @@
+package com.smartharvester.model.mapping.request;
+
+public class Iri {
+
+ private String iri;
+ private String label;
+ private String originalKeyword;
+
+ public String getIri() {
+ return iri;
+ }
+
+ public void setIri(String iri) {
+ this.iri = iri;
+ }
+
+ public String getLabel() {
+ return label;
+ }
+
+ public void setLabel(String label) {
+ this.label = label;
+ }
+
+ public String getOriginalKeyword() {
+ return originalKeyword;
+ }
+
+ public void setOriginalKeyword(String originalKeyword) {
+ this.originalKeyword = originalKeyword;
+ }
+}
diff --git a/src/main/java/com/smartharvester/service/MappingService.java b/src/main/java/com/smartharvester/service/MappingService.java
index 0e29424..d4c9074 100644
--- a/src/main/java/com/smartharvester/service/MappingService.java
+++ b/src/main/java/com/smartharvester/service/MappingService.java
@@ -9,6 +9,7 @@ import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.PathNotFoundException;
import com.smartharvester.model.elasticSearch.ElasticSearchResponse;
import com.smartharvester.model.mapping.Path;
+import com.smartharvester.model.mapping.request.Iri;
import com.smartharvester.model.mapping.request.MappingRequest;
import com.smartharvester.model.mapping.request.PublishedRequest;
import com.smartharvester.model.mapping.response.KeywordResponse;
@@ -99,7 +100,7 @@ public class MappingService {
public String buildDataset(URI urlRepo, List<Path> paths, boolean isJsonpath, String catId, String fdpUrl,
- List<String> iris) throws Exception {
+ List<Iri> iris) throws Exception {
Map<String, List<Map<String, List<String>>>> dcatPropertiesMap = this.mapToDcat(urlRepo, paths, isJsonpath);
Map<String, List<String>> datasetMap = dcatPropertiesMap.get("dataset").get(0);
@@ -109,7 +110,29 @@ public class MappingService {
datasetMap.putIfAbsent("dct:publisher", List.of("undefined"));
+
StringBuilder datasetString = this.getDatasetString(catId, fdpUrl);
+
+ if (!iris.isEmpty()) {
+ iris.forEach(iri -> {
+ datasetString.append(this.write("dcat:theme", iri.getIri()));
+ if (null !=iri.getLabel()) {
+ datasetString.append("<" + iri.getIri() + "> <http://www.w3.org/2000/01/rdf-schema#label>\n\"" + iri.getLabel() + "\".");
+ if (datasetMap.containsKey("dcat:keyword")) {
+ List<String> keywords = datasetMap.get("dcat:keyword");
+ if (keywords.stream().noneMatch(e -> e.equals(iri.getLabel()))) {
+ keywords.add(iri.getLabel());
+ }
+
+ datasetMap.replace("dcat:keyword", keywords);
+ }
+ }
+
+
+ });
+
+
+ }
for (Map.Entry<String, List<String>> entry : datasetMap.entrySet()) {
if (entry.getValue().isEmpty()) {
entry.setValue(List.of("undefined"));
@@ -124,7 +147,7 @@ public class MappingService {
}
if (!iris.isEmpty()) {
- iris.forEach(iri -> datasetString.append(this.write("dcat:theme", iri)));
+ iris.forEach(iri -> datasetString.append(this.write("dcat:theme", iri.getIri())));
}
datasetString.append(".");
LOGGER.info(datasetString.toString());
@@ -522,11 +545,11 @@ public class MappingService {
public void mapAndPublish(String catalogId, String fdpUrl, boolean isJsonpath, List<Path> paths, String fdpToken,
List<String> publishedUrl, List<String> notPublishedUrl, List<Path> distributionPaths,
- Map.Entry<String, List<String>> entry) {
+ Map.Entry<String, List<Iri>> entry) {
ResponseEntity<String> responseEntity;
String datasetId;
String url = entry.getKey();
- List<String> iris = entry.getValue();
+ List<Iri> iris = entry.getValue();
String locationDataset;
try {
diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties
index 1df8462..8dfb349 100644
--- a/src/main/resources/application-dev.properties
+++ b/src/main/resources/application-dev.properties
@@ -44,7 +44,7 @@ spring.security.oauth2.client.registration.oidc.client-secret=cLONCJ8MccdHwobCEM
spring.security.oauth2.client.registration.oidc.scope=address,phone,openid,email,profile
spring.security.oauth2.client.registration.oidc.redirect-uri=http://localhost:4200/callback
-elasticsearch.baseURL=https://localhost:4242/eosc/api
+elasticsearch.baseURL=https://90.147.189.237:4242/eosc/api
elasticsearch.token=uHiOaPyo6IjKa7lV6qDHWYz3lgcf8bG2
elasticsearch.minChar=3
elasticsearch.maxChar=30
--
GitLab
From bf92a8253f4f8d27c07f33759199510fab78f4f2 Mon Sep 17 00:00:00 2001
From: Baptiste Toulemonde <toulemonde@cines.fr>
Date: Fri, 7 Oct 2022 14:01:14 +0200
Subject: [PATCH 2/3] wip
---
.../com/smartharvester/service/MappingService.java | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/src/main/java/com/smartharvester/service/MappingService.java b/src/main/java/com/smartharvester/service/MappingService.java
index d4c9074..d085289 100644
--- a/src/main/java/com/smartharvester/service/MappingService.java
+++ b/src/main/java/com/smartharvester/service/MappingService.java
@@ -115,20 +115,20 @@ public class MappingService {
if (!iris.isEmpty()) {
iris.forEach(iri -> {
- datasetString.append(this.write("dcat:theme", iri.getIri()));
+ //datasetString.append("dcat:theme <" + iri.getIri() + ">" + " <http://www.w3.org/2000/01/rdf-schema#label> \"" + iri.getLabel() + "\".\n" );
if (null !=iri.getLabel()) {
- datasetString.append("<" + iri.getIri() + "> <http://www.w3.org/2000/01/rdf-schema#label>\n\"" + iri.getLabel() + "\".");
if (datasetMap.containsKey("dcat:keyword")) {
List<String> keywords = datasetMap.get("dcat:keyword");
if (keywords.stream().noneMatch(e -> e.equals(iri.getLabel()))) {
keywords.add(iri.getLabel());
}
-
datasetMap.replace("dcat:keyword", keywords);
+ } else {
+ List<String> keywords = new ArrayList<>();
+ keywords.add(iri.getLabel());
+ datasetMap.put("dcat:keyword", keywords);
}
}
-
-
});
--
GitLab
From f2115257131d292b539c12173e9c47742818a776 Mon Sep 17 00:00:00 2001
From: Baptiste Toulemonde <toulemonde@cines.fr>
Date: Fri, 7 Oct 2022 15:35:50 +0200
Subject: [PATCH 3/3] add keyword
---
.../service/MappingService.java | 56 ++++++++++++++-----
1 file changed, 43 insertions(+), 13 deletions(-)
diff --git a/src/main/java/com/smartharvester/service/MappingService.java b/src/main/java/com/smartharvester/service/MappingService.java
index d085289..9b396e3 100644
--- a/src/main/java/com/smartharvester/service/MappingService.java
+++ b/src/main/java/com/smartharvester/service/MappingService.java
@@ -115,8 +115,8 @@ public class MappingService {
if (!iris.isEmpty()) {
iris.forEach(iri -> {
- //datasetString.append("dcat:theme <" + iri.getIri() + ">" + " <http://www.w3.org/2000/01/rdf-schema#label> \"" + iri.getLabel() + "\".\n" );
- if (null !=iri.getLabel()) {
+ datasetString.append(this.write("dcat:theme", iri.getIri()));
+ if (null != iri.getLabel()) {
if (datasetMap.containsKey("dcat:keyword")) {
List<String> keywords = datasetMap.get("dcat:keyword");
if (keywords.stream().noneMatch(e -> e.equals(iri.getLabel()))) {
@@ -134,7 +134,7 @@ public class MappingService {
}
for (Map.Entry<String, List<String>> entry : datasetMap.entrySet()) {
- if (entry.getValue().isEmpty()) {
+ if (entry.getValue().isEmpty() && !entry.getKey().equals("dcat:keyword")) {
entry.setValue(List.of("undefined"));
}
if (isReplicable(entry.getKey(), paths) && (entry.getValue().size() > 1)) {
@@ -605,11 +605,11 @@ public class MappingService {
ResponseEntity<Object> response =
this.restTemplate.exchange(url + "/catalog/" + catId, HttpMethod.GET, entity, Object.class);
- ObjectMapper mapper = new ObjectMapper();
- JSONArray body = new JSONArray(mapper.writeValueAsString(response.getBody()));
+ ObjectMapper mapper = new ObjectMapper();
+ JSONArray body = new JSONArray(mapper.writeValueAsString(response.getBody()));
- List<String> datasets =
- getValuesFromJsonLd(body.getJSONObject(1), "http://www.w3.org/ns/dcat#dataset", "@id");
+ List<String> datasets =
+ getValuesFromJsonLd(body.getJSONObject(1), "http://www.w3.org/ns/dcat#dataset", "@id");
List<KeywordResponse> keywordResponses = new ArrayList<>();
datasets.stream().forEach(datasetUrl -> {
@@ -731,16 +731,46 @@ public class MappingService {
concept.getIris().forEach(iri -> {
if (null != body.optJSONObject(finalIndex)) {
try {
- if (concept.getIris().indexOf(iri) == 0) {
+ if (concept.getIris().indexOf(iri.getIri()) == 0) {
body.getJSONObject(finalIndex).remove("http://www.w3.org/ns/dcat#theme");
body.getJSONObject(finalIndex)
.put("http://www.w3.org/ns/dcat#theme", new JSONArray());
}
- if (null != body.getJSONObject(finalIndex).optJSONArray("http://www.w3.org/ns/dcat#theme")) {
- body.getJSONObject(finalIndex)
- .getJSONArray("http://www.w3.org/ns/dcat#theme")
- .put(new JSONObject().put("@id", iri));
- }
+ if (null != body.getJSONObject(finalIndex)
+ .optJSONArray("http://www.w3.org/ns/dcat#theme")) {
+ body.getJSONObject(finalIndex)
+ .getJSONArray("http://www.w3.org/ns/dcat#theme")
+ .put(new JSONObject().put("@id", iri.getIri()));
+ }
+ if (null != iri.getLabel()) {
+
+
+ if (null != body.getJSONObject(finalIndex)
+ .optJSONArray("http://www.w3.org/ns/dcat#keyword")) {
+ JSONArray keywordsArray = body.getJSONObject(finalIndex)
+ .getJSONArray("http://www.w3.org/ns/dcat#keyword");
+ boolean isPresent = false;
+ for (int i = 0; i < keywordsArray.length(); i++) {
+ if (keywordsArray.optJSONObject(i) != null
+ && keywordsArray.optJSONObject(i).getString("@value")
+ .equals(iri.getLabel())) {
+ isPresent = true;
+ }
+ }
+ if (!isPresent) {
+ body.getJSONObject(finalIndex)
+ .getJSONArray("http://www.w3.org/ns/dcat#keyword")
+ .put(new JSONObject().put("@value", iri.getLabel()));
+ }
+ } else {
+ body.getJSONObject(finalIndex)
+ .put("http://www.w3.org/ns/dcat#keyword", new JSONArray());
+ body.getJSONObject(finalIndex)
+ .getJSONArray("http://www.w3.org/ns/dcat#keyword")
+ .put(new JSONObject().put("@value", iri.getLabel()));
+
+ }
+ }
} catch (JSONException e) {
notPublishedUrl.add(concept.getUrl() + " => " + e.getMessage());
LOGGER.error(e.getMessage());
--
GitLab