From 4981a42fcdb7f83e87f1b8d44b20b229b279cd45 Mon Sep 17 00:00:00 2001
From: Baptiste Toulemonde <toulemonde@cines.fr>
Date: Fri, 9 Sep 2022 16:44:00 +0200
Subject: [PATCH] feature to get the concepts by keywords
---
pom.xml | 29 ++-
.../SmartHarvesterSecurityConfiguration.java | 1 +
.../controller/ElacticSearchController.java | 52 +++++
.../SmartHarvesterMappingController.java | 21 +-
.../controller/SparqlController.java | 33 +++
.../model/elasticSearch/Document.java | 131 ++++++++++++
.../elasticSearch/ElasticSearchResponse.java | 57 ++++++
.../model/elasticSearch/Result.java | 30 +++
.../model/elasticSearch/Source.java | 29 +++
.../model/mapping/request/KeywordRequest.java | 25 ---
.../mapping/response/KeywordResponse.java | 48 +++++
.../service/ElasticSearchService.java | 193 ++++++++++++++++++
.../service/MappingService.java | 18 +-
.../service/OpenApiServiceImpl.java | 2 +
.../smartharvester/service/SparqlService.java | 39 ++++
.../util/SmartHarvesterRequestUtil.java | 31 ++-
src/main/resources/application-dev.properties | 7 +-
17 files changed, 700 insertions(+), 46 deletions(-)
create mode 100644 src/main/java/com/smartharvester/controller/ElacticSearchController.java
create mode 100644 src/main/java/com/smartharvester/controller/SparqlController.java
create mode 100644 src/main/java/com/smartharvester/model/elasticSearch/Document.java
create mode 100644 src/main/java/com/smartharvester/model/elasticSearch/ElasticSearchResponse.java
create mode 100644 src/main/java/com/smartharvester/model/elasticSearch/Result.java
create mode 100644 src/main/java/com/smartharvester/model/elasticSearch/Source.java
delete mode 100644 src/main/java/com/smartharvester/model/mapping/request/KeywordRequest.java
create mode 100644 src/main/java/com/smartharvester/model/mapping/response/KeywordResponse.java
create mode 100644 src/main/java/com/smartharvester/service/ElasticSearchService.java
create mode 100644 src/main/java/com/smartharvester/service/SparqlService.java
diff --git a/pom.xml b/pom.xml
index 15ac448..9b08e2e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -18,6 +18,7 @@
<properties>
<java.version>11</java.version>
<spring.framework.version>5.2.9.RELEASE</spring.framework.version>
+ <rdf4j.version>3.7.2</rdf4j.version>
</properties>
@@ -50,7 +51,33 @@
<artifactId>spring-cloud-commons</artifactId>
<version>2.2.6.RELEASE</version>
</dependency>
-
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-devtools</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.rdf4j</groupId>
+ <artifactId>rdf4j-runtime</artifactId>
+ <version>${rdf4j.version}</version>
+ <type>pom</type>
+ <exclusions>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.rdf4j</groupId>
+ <artifactId>rdf4j-rio-api</artifactId>
+ <version>${rdf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.rdf4j</groupId>
+ <artifactId>rdf4j-sail-nativerdf</artifactId>
+ <version>${rdf4j.version}</version>
+ </dependency>
<dependency>
<groupId>io.jsonwebtoken</groupId>
<artifactId>jjwt</artifactId>
diff --git a/src/main/java/com/smartharvester/config/SmartHarvesterSecurityConfiguration.java b/src/main/java/com/smartharvester/config/SmartHarvesterSecurityConfiguration.java
index ea881c0..71787c9 100644
--- a/src/main/java/com/smartharvester/config/SmartHarvesterSecurityConfiguration.java
+++ b/src/main/java/com/smartharvester/config/SmartHarvesterSecurityConfiguration.java
@@ -99,6 +99,7 @@ public class SmartHarvesterSecurityConfiguration extends WebSecurityConfigurerAd
private void successHandler(HttpServletRequest request, HttpServletResponse response, Authentication authentication)
throws IOException {
String token = this.tokenStorage.generateToken(authentication);
+ LOGGER.info("Auth token is - " + token);
response.getWriter().write(mapper.writeValueAsString(Collections.singletonMap("accessToken", token)));
}
diff --git a/src/main/java/com/smartharvester/controller/ElacticSearchController.java b/src/main/java/com/smartharvester/controller/ElacticSearchController.java
new file mode 100644
index 0000000..d4c8934
--- /dev/null
+++ b/src/main/java/com/smartharvester/controller/ElacticSearchController.java
@@ -0,0 +1,52 @@
+package com.smartharvester.controller;
+
+import com.smartharvester.model.elasticSearch.ElasticSearchResponse;
+import com.smartharvester.model.mapping.request.MappingRequest;
+import com.smartharvester.model.mapping.response.KeywordResponse;
+import com.smartharvester.service.ElasticSearchService;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.CrossOrigin;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.List;
+
+@CrossOrigin(origins = "*")
+@RequestMapping("/harvester/api/es")
+@Tag(name = "Harvest", description = "request to eleasticsearch index")
+@RestController
+public class ElacticSearchController {
+
+ @Autowired
+ private ElasticSearchService service;
+
+ @GetMapping("/autocomplete")
+ public ResponseEntity<?> getAutocompleteRequest(@RequestParam("search1") String search1){
+ ResponseEntity<ElasticSearchResponse> autocompleteRequest = this.service.getAutocompleteRequest(search1);
+
+ switch (autocompleteRequest.getStatusCodeValue()) {
+ case 401:
+ return ResponseEntity.badRequest().body("bad password");
+ case 500:
+ return ResponseEntity.badRequest().body("connection with elasticSearch index is down...");
+ case 200:
+ return ResponseEntity.ok(autocompleteRequest.getBody());
+ default:
+ return ResponseEntity.badRequest().body("Something wrong happened");
+ }
+ }
+
+
+ @PostMapping("/keywords")
+ public ResponseEntity<List<KeywordResponse>> getKeywords (@RequestBody MappingRequest req, @RequestParam("catalogId") String catalogId, @RequestParam("isJsonPath") boolean isJsonPath) {
+ List<KeywordResponse> keywords = this.service.getConceptsByKeywords(req, catalogId, isJsonPath);
+
+ return ResponseEntity.ok(keywords);
+ }
+}
diff --git a/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java b/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java
index d393cb9..54fc752 100644
--- a/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java
+++ b/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java
@@ -2,7 +2,6 @@ package com.smartharvester.controller;
import com.smartharvester.model.mapping.Path;
-import com.smartharvester.model.mapping.request.KeywordRequest;
import com.smartharvester.model.mapping.request.MappingRequest;
import com.smartharvester.model.mapping.response.MappingResponse;
import com.smartharvester.service.MappingFromIsoService;
@@ -15,7 +14,13 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.*;
+import org.springframework.web.bind.annotation.CrossOrigin;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
import java.net.URI;
import java.util.ArrayList;
@@ -26,7 +31,7 @@ import java.util.stream.Collectors;
@CrossOrigin(origins = "*")
@RequestMapping("/harvester/api/transform")
-@Tag(name = "Harvest", description = "transform xml iso 19115 to RDF")
+@Tag(name = "Harvest", description = "mapping to RDF")
@RestController
public class SmartHarvesterMappingController {
@@ -113,15 +118,5 @@ public class SmartHarvesterMappingController {
}
- @PostMapping("/keywords")
- public ResponseEntity<List<String>> getKeywords (@RequestBody KeywordRequest req) {
- List<String> keywords = new ArrayList<>();
- if (req.isJSonPath()) {
-
- } else {
-
- }
- return ResponseEntity.ok(keywords);
- }
}
diff --git a/src/main/java/com/smartharvester/controller/SparqlController.java b/src/main/java/com/smartharvester/controller/SparqlController.java
new file mode 100644
index 0000000..d767b88
--- /dev/null
+++ b/src/main/java/com/smartharvester/controller/SparqlController.java
@@ -0,0 +1,33 @@
+package com.smartharvester.controller;
+
+import com.smartharvester.service.SparqlService;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.CrossOrigin;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+@CrossOrigin(origins = "*")
+@RequestMapping("/harvester/api/sparql")
+@Tag(name = "Harvest", description = "sparq request to blazegraph database")
+@RestController
+public class SparqlController {
+
+ @Autowired
+ private SparqlService service;
+
+ @PostMapping
+ public ResponseEntity<?> getSparql(@RequestBody String body, @RequestParam(value = "url") String url) {
+ ResponseEntity<Object> jsonObjectResponseEntity = this.service.postSparqlRequest(body, url);
+ if (jsonObjectResponseEntity.getStatusCode() == HttpStatus.OK) {
+ return ResponseEntity.ok(jsonObjectResponseEntity.getBody());
+ } else {
+ return ResponseEntity.badRequest().body(jsonObjectResponseEntity.getBody());
+ }
+ }
+}
diff --git a/src/main/java/com/smartharvester/model/elasticSearch/Document.java b/src/main/java/com/smartharvester/model/elasticSearch/Document.java
new file mode 100644
index 0000000..d902ca6
--- /dev/null
+++ b/src/main/java/com/smartharvester/model/elasticSearch/Document.java
@@ -0,0 +1,131 @@
+package com.smartharvester.model.elasticSearch;
+
+import com.fasterxml.jackson.annotation.JsonAlias;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+
+public class Document {
+
+ private String iri;
+
+ @JsonProperty("resourceIri")
+ @JsonAlias("resource iri")
+ private String resourceIri;
+
+ @JsonProperty("resourceAcronym")
+ @JsonAlias("resource acronym")
+ private String resourceAcronym;
+
+ @JsonProperty("resourceReusingAcronyms")
+ @JsonAlias("resource_reusing_acronyms")
+ private List<String> resourceReusingAcronyms;
+
+ @JsonProperty("resourceDate")
+ @JsonAlias("resource date")
+ private String resourceDate;
+
+ @JsonProperty("resourceName")
+ @JsonAlias("resource name")
+ private String resourceName;
+
+ private List<String> synonyms;
+
+ @JsonProperty("resource version")
+ private String resourceVersion;
+
+ private List<String> domains;
+
+ private String label;
+
+ @JsonProperty("resources_reusing")
+ private List<String> resourcesReusing;
+
+ public String getIri() {
+ return iri;
+ }
+
+ public void setIri(String iri) {
+ this.iri = iri;
+ }
+
+ public String getResourceIri() {
+ return resourceIri;
+ }
+
+ public void setResourceIri(String resourceIri) {
+ this.resourceIri = resourceIri;
+ }
+
+ public String getResourceAcronym() {
+ return resourceAcronym;
+ }
+
+ public void setResourceAcronym(String resourceAcronym) {
+ this.resourceAcronym = resourceAcronym;
+ }
+
+ public List<String> getResourceReusingAcronyms() {
+ return resourceReusingAcronyms;
+ }
+
+ public void setResourceReusingAcronyms(List<String> resourceReusingAcronyms) {
+ this.resourceReusingAcronyms = resourceReusingAcronyms;
+ }
+
+ public String getResourceDate() {
+ return resourceDate;
+ }
+
+ public void setResourceDate(String resourceDate) {
+ this.resourceDate = resourceDate;
+ }
+
+ public String getResourceName() {
+ return resourceName;
+ }
+
+ public void setResourceName(String resourceName) {
+ this.resourceName = resourceName;
+ }
+
+ public List<String> getSynonyms() {
+ return synonyms;
+ }
+
+ public void setSynonyms(List<String> synonyms) {
+ this.synonyms = synonyms;
+ }
+
+ public String getResourceVersion() {
+ return resourceVersion;
+ }
+
+ public void setResourceVersion(String resourceVersion) {
+ this.resourceVersion = resourceVersion;
+ }
+
+ public List<String> getDomains() {
+ return domains;
+ }
+
+ public void setDomains(List<String> domains) {
+ this.domains = domains;
+ }
+
+ public String getLabel() {
+ return label;
+ }
+
+ public void setLabel(String label) {
+ this.label = label;
+ }
+
+ public List<String> getResourcesReusing() {
+ return resourcesReusing;
+ }
+
+ public void setResourcesReusing(List<String> resourcesReusing) {
+ this.resourcesReusing = resourcesReusing;
+ }
+}
diff --git a/src/main/java/com/smartharvester/model/elasticSearch/ElasticSearchResponse.java b/src/main/java/com/smartharvester/model/elasticSearch/ElasticSearchResponse.java
new file mode 100644
index 0000000..883fb32
--- /dev/null
+++ b/src/main/java/com/smartharvester/model/elasticSearch/ElasticSearchResponse.java
@@ -0,0 +1,57 @@
+package com.smartharvester.model.elasticSearch;
+
+import com.fasterxml.jackson.annotation.JsonAlias;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.List;
+
+public class ElasticSearchResponse {
+
+ private String search1;
+ private String search2;
+ private int count;
+ private float best;
+ @JsonProperty("results")
+ @JsonAlias("result")
+ private List<Result> results;
+
+ public String getSearch1() {
+ return search1;
+ }
+
+ public void setSearch1(String search1) {
+ this.search1 = search1;
+ }
+
+ public String getSearch2() {
+ return search2;
+ }
+
+ public void setSearch2(String search2) {
+ this.search2 = search2;
+ }
+
+ public int getCount() {
+ return count;
+ }
+
+ public void setCount(int count) {
+ this.count = count;
+ }
+
+ public float getBest() {
+ return best;
+ }
+
+ public void setBest(float best) {
+ this.best = best;
+ }
+
+ public List<Result> getResults() {
+ return results;
+ }
+
+ public void setResults(List<Result> results) {
+ this.results = results;
+ }
+}
diff --git a/src/main/java/com/smartharvester/model/elasticSearch/Result.java b/src/main/java/com/smartharvester/model/elasticSearch/Result.java
new file mode 100644
index 0000000..0027953
--- /dev/null
+++ b/src/main/java/com/smartharvester/model/elasticSearch/Result.java
@@ -0,0 +1,30 @@
+package com.smartharvester.model.elasticSearch;
+
+import com.fasterxml.jackson.annotation.JsonAlias;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class Result {
+ @JsonProperty("score")
+ @JsonAlias("_score")
+ private float score;
+
+ @JsonProperty("source")
+ @JsonAlias("_source")
+ private Source source;
+
+ public float getScore() {
+ return score;
+ }
+
+ public void setScore(float score) {
+ this.score = score;
+ }
+
+ public Source getSource() {
+ return source;
+ }
+
+ public void setSource(Source source) {
+ this.source = source;
+ }
+}
diff --git a/src/main/java/com/smartharvester/model/elasticSearch/Source.java b/src/main/java/com/smartharvester/model/elasticSearch/Source.java
new file mode 100644
index 0000000..1dc7dcb
--- /dev/null
+++ b/src/main/java/com/smartharvester/model/elasticSearch/Source.java
@@ -0,0 +1,29 @@
+package com.smartharvester.model.elasticSearch;
+
+import com.fasterxml.jackson.annotation.JsonAlias;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class Source {
+
+ @JsonProperty("timestamp")
+ @JsonAlias("@timestamp")
+ private String timestamp;
+
+ private Document document;
+
+ public String getTimestamp() {
+ return timestamp;
+ }
+
+ public void setTimestamp(String timestamp) {
+ this.timestamp = timestamp;
+ }
+
+ public Document getDocument() {
+ return document;
+ }
+
+ public void setDocument(Document document) {
+ this.document = document;
+ }
+}
diff --git a/src/main/java/com/smartharvester/model/mapping/request/KeywordRequest.java b/src/main/java/com/smartharvester/model/mapping/request/KeywordRequest.java
deleted file mode 100644
index 0d85f24..0000000
--- a/src/main/java/com/smartharvester/model/mapping/request/KeywordRequest.java
+++ /dev/null
@@ -1,25 +0,0 @@
-package com.smartharvester.model.mapping.request;
-
-import java.util.List;
-
-public class KeywordRequest {
-
- private boolean isJSonPath;
- private List<String> keywordPaths;
-
- public boolean isJSonPath() {
- return isJSonPath;
- }
-
- public void setJSonPath(boolean JSonPath) {
- isJSonPath = JSonPath;
- }
-
- public List<String> getKeywordPaths() {
- return keywordPaths;
- }
-
- public void setKeywordPaths(List<String> keywordPaths) {
- this.keywordPaths = keywordPaths;
- }
-}
diff --git a/src/main/java/com/smartharvester/model/mapping/response/KeywordResponse.java b/src/main/java/com/smartharvester/model/mapping/response/KeywordResponse.java
new file mode 100644
index 0000000..1d8ef62
--- /dev/null
+++ b/src/main/java/com/smartharvester/model/mapping/response/KeywordResponse.java
@@ -0,0 +1,48 @@
+package com.smartharvester.model.mapping.response;
+
+import com.smartharvester.model.elasticSearch.ElasticSearchResponse;
+
+import java.util.List;
+
+public class KeywordResponse {
+
+ private String id;
+
+ private String url;
+ private List<String> keywords;
+
+ private ElasticSearchResponse concepts;
+
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public List<String> getKeywords() {
+ return keywords;
+ }
+
+ public void setKeywords(List<String> keywords) {
+ this.keywords = keywords;
+ }
+
+ public ElasticSearchResponse getConcepts() {
+ return concepts;
+ }
+
+ public void setConcepts(ElasticSearchResponse concepts) {
+ this.concepts = concepts;
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public void setUrl(String url) {
+ this.url = url;
+ }
+}
diff --git a/src/main/java/com/smartharvester/service/ElasticSearchService.java b/src/main/java/com/smartharvester/service/ElasticSearchService.java
new file mode 100644
index 0000000..a865770
--- /dev/null
+++ b/src/main/java/com/smartharvester/service/ElasticSearchService.java
@@ -0,0 +1,193 @@
+package com.smartharvester.service;
+
+import com.smartharvester.model.elasticSearch.ElasticSearchResponse;
+import com.smartharvester.model.mapping.Path;
+import com.smartharvester.model.mapping.request.MappingRequest;
+import com.smartharvester.model.mapping.response.KeywordResponse;
+import com.smartharvester.model.openapi.OpenApi;
+import org.json.JSONObject;
+import org.junit.Assert;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Service;
+import org.springframework.web.client.HttpClientErrorException;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import java.net.URI;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+@Service
+public class ElasticSearchService {
+
+ @Autowired
+ private RestTemplate restTemplate;
+
+ @Autowired
+ private MappingService mappingService;
+ @Autowired
+ private OpenApiServiceImpl openApiService;
+
+ @Value("${elasticsearch.baseURL}")
+ private String baseUrl;
+
+ @Value("${elasticsearch.token}")
+ private String token;
+
+ @Value("${elasticsearch.minChar}")
+ private int minChar;
+
+ @Value("${elasticsearch.maxChar}")
+ private int maxChar;
+
+ public static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchService.class);
+
+ private HttpEntity getHttpEntity() {
+ HttpHeaders headers = new HttpHeaders();
+ headers.add("x-key-token", this.token);
+ return new HttpEntity<>(headers);
+ }
+ public ResponseEntity<ElasticSearchResponse> getAutocompleteRequest( String search1) {
+ HttpEntity entity = getHttpEntity();
+ UriComponentsBuilder builder = null;
+ String url = this.baseUrl + "/search-typing";
+ builder = UriComponentsBuilder.fromUriString(url)
+ .queryParam("search1", URLEncoder.encode(search1, StandardCharsets.UTF_8));
+
+ assert builder != null;
+ return this.restTemplate.exchange(builder.build(true).toUri(), HttpMethod.GET, entity, ElasticSearchResponse.class);
+ }
+
+
+
+ public ResponseEntity<ElasticSearchResponse> getSearchRequest(String search1, String search2) {
+ HttpEntity entity = this.getHttpEntity();
+ UriComponentsBuilder builder = null;
+ String url = this.baseUrl + "/search";
+ builder = UriComponentsBuilder.fromUriString(url)
+ .queryParam("search1", URLEncoder.encode(search1, StandardCharsets.UTF_8))
+ .queryParam("search2", URLEncoder.encode((search2), StandardCharsets.UTF_8));
+ ResponseEntity response = null;
+ try {
+ assert builder != null;
+ response = this.restTemplate.exchange(builder.build(true).toUri(), HttpMethod.GET, entity, ElasticSearchResponse.class);
+ Thread.sleep(200);
+ } catch (HttpClientErrorException e) {
+ if (e.getRawStatusCode() == 429) {
+ response = this.getSearchRequest(search1, search2);
+ } else {
+ LOGGER.error(e.getMessage());
+ }
+ } catch (InterruptedException e) {
+ LOGGER.error(e.getMessage());
+ }
+ return response;
+ }
+
+ public List<KeywordResponse> getKeywords(MappingRequest req, String catalogId, boolean isJsonPath) {
+ OpenApi openApi = this.openApiService.getOpenApiByUUDI(catalogId);
+ List<String> urls = new ArrayList<>();
+ req.getIds().parallelStream().forEach(id -> urls.addAll(this.openApiService.getUrls(this.openApiService.getOpenApiByUUDI(catalogId), id)));
+ List<Path> paths = req.getPaths().stream()
+ .filter(path -> path.getProperty().equals("dct:identifier") || path.getProperty().equals("dcat:keyword"))
+ .collect(Collectors.toList());
+ List<KeywordResponse> result = new ArrayList<>();
+ urls.parallelStream().forEach(url -> {
+ JSONObject json = new JSONObject();
+ List<Map<String, List<String>>> retrievedFromJson = new ArrayList<>();
+ try {
+ json = this.mappingService.getJson(new URI(url)).get();
+ } catch (Exception e) {
+ LOGGER.error(e.getMessage());
+ }
+ try {
+ if (isJsonPath) {
+ retrievedFromJson = this.mappingService.getAllValuesFomJsonpath(json, paths).get();
+ } else {
+ retrievedFromJson = this.mappingService.getAllValuesFomCustomPath(json, paths).get();
+ }
+ } catch (Exception e) {
+ LOGGER.error(e.getMessage());
+ }
+
+ retrievedFromJson.forEach(e -> {
+ KeywordResponse keywordResponse = new KeywordResponse();
+ Assert.assertNotNull(e.get("dct:identifier"));
+ keywordResponse.setId(e.get("dct:identifier").get(0));
+ keywordResponse.setUrl(url);
+ keywordResponse.setKeywords(e.get("dcat:keyword"));
+ result.add(keywordResponse);
+ });
+ });
+ return result;
+ }
+
+ public List<KeywordResponse> getConceptsByKeywords(MappingRequest req, String catalogId, boolean isJsonPath) {
+ List<KeywordResponse> keywordResponses = this.getKeywords(req, catalogId, isJsonPath);
+ keywordResponses.forEach(k -> {
+ List<String> keywords = k.getKeywords();
+ if (!keywords.isEmpty()) {
+ if (keywords.size() == 1 ) {
+ if (isValid(keywords.get(0))) {
+ extractConcepts(keywords.get(0), "null", k);
+ }
+ } else if(keywords.size() == 2) {
+ if (isValid(keywords.get(0)) && isValid(keywords.get(1))) {
+ extractConcepts(keywords.get(0), keywords.get(1), k);
+ }
+ } else {
+ getRecursively(k, keywords, 0);
+
+
+ }
+ }
+ });
+ return keywordResponses;
+ }
+
+ private void getRecursively(KeywordResponse k, List<String> keywords, int i) {
+
+ if ( i + 2 < keywords.size()) {
+ if (isValid(keywords.get(i)) && isValid(keywords.get(i + 1))) {
+ extractConcepts(keywords.get(i), keywords.get(i + 1), k);
+ } else if (isValid(keywords.get(i))){
+ extractConcepts(keywords.get(i), keywords.get(i + 2), k);
+ } else if (isValid(keywords.get(i + 1))) {
+ extractConcepts(keywords.get(i + 2), keywords.get(i + 1), k);
+ } else {
+ this.getRecursively( k, keywords, ++i);
+ }
+ } else if (i + 1 < keywords.size()) {
+ if (isValid(keywords.get(i)) && isValid(keywords.get(i + 1))) {
+ extractConcepts(keywords.get(i), keywords.get(i + 1), k);
+ } else if (isValid(keywords.get(i))){
+ extractConcepts(keywords.get(i), "null", k);
+ } else if (isValid(keywords.get(i + 1))) {
+ extractConcepts("null", keywords.get(i + 1), k);
+ }
+ }
+ }
+
+ private void extractConcepts(String keyword1, String keyword2, KeywordResponse k) {
+ ResponseEntity<ElasticSearchResponse> searchRequest = this.getSearchRequest(keyword1, keyword2);
+ if (searchRequest.getStatusCodeValue() == 200 && Objects.requireNonNull(searchRequest.getBody()).getCount() != 0) {
+ k.setConcepts(searchRequest.getBody());
+ }
+ }
+
+ private Boolean isValid(String keyword) {
+ return keyword.length() >= minChar && keyword.length() <= maxChar;
+ }
+}
diff --git a/src/main/java/com/smartharvester/service/MappingService.java b/src/main/java/com/smartharvester/service/MappingService.java
index 6ef10d4..a72f43c 100644
--- a/src/main/java/com/smartharvester/service/MappingService.java
+++ b/src/main/java/com/smartharvester/service/MappingService.java
@@ -12,7 +12,11 @@ import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.http.*;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.web.client.HttpClientErrorException.BadRequest;
@@ -22,7 +26,12 @@ import java.net.URI;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
@@ -35,6 +44,9 @@ public class MappingService {
@Autowired
SmartHarvesterRequestUtil requestUtil;
+ @Autowired
+ OpenApiServiceImpl openApiService;
+
public static final Logger LOGGER = LoggerFactory.getLogger(MappingService.class);
public StringBuilder getDatasetString(String catId, String fdpUrl) {
@@ -447,4 +459,6 @@ public class MappingService {
this.restTemplate.exchange(url + "/meta/state", HttpMethod.PUT, entity, Void.class);
}
+
+
}
diff --git a/src/main/java/com/smartharvester/service/OpenApiServiceImpl.java b/src/main/java/com/smartharvester/service/OpenApiServiceImpl.java
index 98d95ba..01766d0 100644
--- a/src/main/java/com/smartharvester/service/OpenApiServiceImpl.java
+++ b/src/main/java/com/smartharvester/service/OpenApiServiceImpl.java
@@ -6,10 +6,12 @@ import com.smartharvester.model.openapi.OpenApiPathItem;
import com.smartharvester.model.openapi.OpenApiPathItemParameter;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
import java.util.*;
import java.util.stream.Collectors;
+@Service
public class OpenApiServiceImpl implements OpenApiService {
@Autowired
diff --git a/src/main/java/com/smartharvester/service/SparqlService.java b/src/main/java/com/smartharvester/service/SparqlService.java
new file mode 100644
index 0000000..6b56322
--- /dev/null
+++ b/src/main/java/com/smartharvester/service/SparqlService.java
@@ -0,0 +1,39 @@
+package com.smartharvester.service;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Service;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+
+@Service
+public class SparqlService {
+
+ @Autowired
+ private RestTemplate restTemplate;
+
+
+ public ResponseEntity<Object> postSparqlRequest(String body, String url) {
+ HttpHeaders httpHeaders = new HttpHeaders();
+ //httpHeaders.addAll("Content-Type", List.of("application/sparql-query"));
+ httpHeaders.add("Accept", "application/sparql-results+json");
+ UriComponentsBuilder builder = null;
+ try {
+ builder = UriComponentsBuilder.fromUriString(url)
+ .queryParam("query", URLEncoder.encode(body, "UTF-8"));
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+
+ HttpEntity<String> httpEntity = new HttpEntity<>(httpHeaders);
+ ResponseEntity<Object> response =
+ this.restTemplate.exchange(builder.build(true).toUri(), HttpMethod.GET, httpEntity, Object.class);
+ return response;
+ }
+}
diff --git a/src/main/java/com/smartharvester/util/SmartHarvesterRequestUtil.java b/src/main/java/com/smartharvester/util/SmartHarvesterRequestUtil.java
index cc8e0cf..68c9041 100644
--- a/src/main/java/com/smartharvester/util/SmartHarvesterRequestUtil.java
+++ b/src/main/java/com/smartharvester/util/SmartHarvesterRequestUtil.java
@@ -3,8 +3,15 @@ package com.smartharvester.util;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.json.JSONException;
import org.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.http.*;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;
import org.springframework.web.client.HttpClientErrorException;
@@ -13,7 +20,11 @@ import org.springframework.web.client.ResourceAccessException;
import org.springframework.web.client.RestTemplate;
import javax.net.ssl.HttpsURLConnection;
-import java.io.*;
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
@@ -26,6 +37,8 @@ public class SmartHarvesterRequestUtil {
@Autowired
private RestTemplate restTemplate;
+ public static final Logger LOGGER = LoggerFactory.getLogger(SmartHarvesterRequestUtil.class);
+
@Async
public CompletableFuture<JSONObject> getJson(String urlRepo) throws JSONException {
URL url;
@@ -134,9 +147,19 @@ public class SmartHarvesterRequestUtil {
httpHeaders.add(header.getKey(), header.getValue());
}
- HttpEntity<String> httpEntity = new HttpEntity<>("parameters", httpHeaders);
+ HttpEntity<String> httpEntity = new HttpEntity<>( httpHeaders);
CompletableFuture<JSONObject> jsonObjectCompletableFuture;
- ResponseEntity<Object> response = this.restTemplate.exchange(url, HttpMethod.GET, httpEntity, Object.class);
+ ResponseEntity<Object> response = null;
+
+ try {
+ response = this.restTemplate.exchange(url, HttpMethod.GET, httpEntity, Object.class);
+ } catch (HttpClientErrorException e) {
+ if (e.getRawStatusCode() == 429) {
+ response = (ResponseEntity<Object>) this.getItemByURl(url, headers);
+ } else {
+ LOGGER.error(e.getMessage());
+ }
+ }
return response.getBody();
}
diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties
index d82af29..670e0f8 100644
--- a/src/main/resources/application-dev.properties
+++ b/src/main/resources/application-dev.properties
@@ -7,7 +7,7 @@ server.port=8080
# 3. MongoDB configuration.
#spring.data.mongodb.uri=mongodb://admin:adminadmin@90.147.167.103:27017/fdp?authSource=admin
-spring.data.mongodb.uri=mongodb://10.6.10.97:27017/fdp?authSource=admin
+spring.data.mongodb.uri=mongodb://10.6.10.97:27010/fdp?authSource=admin
# 4. Logging configuration.
logging.level.com.assignment.springboot.mongo=DEBUG
@@ -43,3 +43,8 @@ spring.security.oauth2.client.registration.oidc.client-id=a03a6ac2-acfe-4916-9d0
spring.security.oauth2.client.registration.oidc.client-secret=cLONCJ8MccdHwobCEMSl_sYDJGKpmBxH16SyiRIBx8XeoDa2ZLwzTvF_aVoEeOt3h2sNbZqltRqhfHKeI3g7Dw
spring.security.oauth2.client.registration.oidc.scope=address,phone,openid,email,profile
spring.security.oauth2.client.registration.oidc.redirect-uri=http://localhost:4200/callback
+
+elasticsearch.baseURL=https://localhost:4242/eosc/api
+elasticsearch.token=uHiOaPyo6IjKa7lV6qDHWYz3lgcf8bG2
+elasticsearch.minChar=4
+elasticsearch.maxChar=30
--
GitLab