diff --git a/pom.xml b/pom.xml
index f68d734ae15aea27f6f77c00c0b6827711772daa..15ac448b310e929cb1beb8dfb3d994173af86b86 100644
--- a/pom.xml
+++ b/pom.xml
@@ -45,6 +45,11 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-webflux</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.springframework.cloud</groupId>
+ <artifactId>spring-cloud-commons</artifactId>
+ <version>2.2.6.RELEASE</version>
+ </dependency>
<dependency>
<groupId>io.jsonwebtoken</groupId>
diff --git a/src/main/java/com/smartharvester/SmartHarvesterApplication.java b/src/main/java/com/smartharvester/SmartHarvesterApplication.java
index ac4cceb2064f2d2ff3992e1df7e1a7b644eff299..058400044acf7d2072075e1d948e5a004c340d41 100644
--- a/src/main/java/com/smartharvester/SmartHarvesterApplication.java
+++ b/src/main/java/com/smartharvester/SmartHarvesterApplication.java
@@ -1,12 +1,31 @@
package com.smartharvester;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.web.client.RestTemplateBuilder;
+import org.springframework.cloud.client.loadbalancer.LoadBalanced;
import org.springframework.context.annotation.Bean;
+import org.springframework.http.MediaType;
+import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
+import org.springframework.http.converter.HttpMessageConverter;
+import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.reactive.function.client.WebClient;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
@SpringBootApplication
public class SmartHarvesterApplication extends WebSecurityConfigurerAdapter {
@@ -14,11 +33,36 @@ public class SmartHarvesterApplication extends WebSecurityConfigurerAdapter {
SpringApplication.run(SmartHarvesterApplication.class, args);
}
-
+
+ @LoadBalanced
@Bean
- public RestTemplate restTemplate() {
- return new RestTemplate();
+ public RestTemplate restTemplate(RestTemplateBuilder builder) throws NoSuchAlgorithmException, KeyManagementException {
+ TrustManager[] trustAllCerts = new TrustManager[]{
+ new X509TrustManager() {
+ public java.security.cert.X509Certificate[] getAcceptedIssuers() {
+ return new X509Certificate[0];
+ }
+
+ public void checkClientTrusted(
+ java.security.cert.X509Certificate[] certs, String authType) {
+ }
+
+ public void checkServerTrusted(
+ java.security.cert.X509Certificate[] certs, String authType) {
+ }
+ }
+ };
+ SSLContext sslContext = SSLContext.getInstance("SSL");
+ sslContext.init(null, trustAllCerts, new java.security.SecureRandom());
+ CloseableHttpClient httpClient = HttpClients.custom()
+ .setSSLContext(sslContext)
+ .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE)
+ .build();
+ HttpComponentsClientHttpRequestFactory customRequestFactory = new HttpComponentsClientHttpRequestFactory();
+ customRequestFactory.setHttpClient(httpClient);
+ return builder.requestFactory(() -> customRequestFactory).build();
}
-
+
+
}
diff --git a/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java b/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java
index 00bb9d68bce153425f8546dbc6abba387a505931..c797e571c60043333fdae35f9988c0299e63d0bc 100644
--- a/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java
+++ b/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java
@@ -96,7 +96,7 @@ public class SmartHarvesterMappingController {
} catch (JSONException | ExecutionException | IllegalStateException e ) {
notPublishedUrl.add(url + " => " + e.getMessage());
LOGGER.warn(e.getMessage());
- } catch (InterruptedException e) {
+ } catch (Exception e) {
Thread.currentThread().interrupt();
notPublishedUrl.add(url + " => " + e.getMessage());
LOGGER.warn(e.getMessage());
diff --git a/src/main/java/com/smartharvester/controller/SmartHarvesterPublicationController.java b/src/main/java/com/smartharvester/controller/SmartHarvesterPublicationController.java
index 1291af29f34e7386c25d2a790d95a10cf01bc56f..8f5ae5ec973ad03a3dab2860b7c6436d86027b60 100644
--- a/src/main/java/com/smartharvester/controller/SmartHarvesterPublicationController.java
+++ b/src/main/java/com/smartharvester/controller/SmartHarvesterPublicationController.java
@@ -2,9 +2,11 @@ package com.smartharvester.controller;
import com.smartharvester.dao.CatalogDaoRepository;
import com.smartharvester.dao.OpenApiDaoRepository;
+import com.smartharvester.model.RequestItem;
import com.smartharvester.model.login.response.MessageResponse;
import com.smartharvester.model.openapi.OpenApi;
+import com.smartharvester.service.MappingService;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.springframework.beans.factory.annotation.Autowired;
@@ -12,6 +14,7 @@ import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
+import java.net.URL;
import java.util.*;
import javax.validation.Valid;
@@ -23,6 +26,8 @@ import javax.validation.Valid;
public class SmartHarvesterPublicationController {
+ @Autowired
+ private MappingService mappingService;
@Autowired
private OpenApiDaoRepository openApiDaoRepository;
@@ -41,6 +46,19 @@ public class SmartHarvesterPublicationController {
.map(openApi -> ResponseEntity.ok().body(openApi)).orElseGet(() -> ResponseEntity.notFound().build());
}
+ @PostMapping("/item")
+ public ResponseEntity<?> getItemByUrl( @RequestBody RequestItem body) {
+ try {
+
+
+ Object datasetByUrl = this.mappingService.getDatasetByUrl(body.getUrl().toString(), body.getHeader());
+ return ResponseEntity.ok(datasetByUrl);
+ } catch (Exception e) {
+ return ResponseEntity.badRequest().body(e.getMessage());
+ }
+ }
+
+
@PostMapping("/publish")
public ResponseEntity<?> registerUser(@Valid @RequestBody OpenApi openApi) {
@@ -62,4 +80,4 @@ public class SmartHarvesterPublicationController {
}
-}
\ No newline at end of file
+}
diff --git a/src/main/java/com/smartharvester/model/RequestItem.java b/src/main/java/com/smartharvester/model/RequestItem.java
new file mode 100644
index 0000000000000000000000000000000000000000..78ba22bed8b12e2dfe3f5125615e7cd113fdf250
--- /dev/null
+++ b/src/main/java/com/smartharvester/model/RequestItem.java
@@ -0,0 +1,27 @@
+package com.smartharvester.model;
+
+
+import java.net.URL;
+import java.util.Map;
+
+public class RequestItem {
+
+ private Map<String, String> header;
+ private URL url;
+
+ public Map<String, String> getHeader() {
+ return header;
+ }
+
+ public void setHeader(Map<String, String> header) {
+ this.header = header;
+ }
+
+ public URL getUrl() {
+ return url;
+ }
+
+ public void setUrl(URL url) {
+ this.url = url;
+ }
+}
diff --git a/src/main/java/com/smartharvester/service/MappingService.java b/src/main/java/com/smartharvester/service/MappingService.java
index f8ba5b3094ce5566a66cbe6cc086abebe4418078..34c70b23107b827b8b6e094b1d5461570cce5298 100644
--- a/src/main/java/com/smartharvester/service/MappingService.java
+++ b/src/main/java/com/smartharvester/service/MappingService.java
@@ -16,6 +16,7 @@ import org.springframework.stereotype.Service;
import org.springframework.web.client.HttpClientErrorException.BadRequest;
import org.springframework.web.client.RestTemplate;
+import java.net.URLDecoder;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
@@ -44,7 +45,12 @@ public class MappingService {
return new StringBuilder("@prefix dcat: <http://www.w3.org/ns/dcat#>. \n" + "@prefix dct: <http://purl.org/dc/terms/>.\n" + "@prefix adms: <http://www.w3.org/ns/adms#> .\n" + "@prefix dqv: <http://www.w3.org/ns/dqv#> .\n" + "@prefix geodcat: <http://data.europa.eu/930/>. \n" + "@prefix prov: <http://www.w3.org/ns/prov#>.\n" + "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n" + "@prefix skos: <http://www.w3.org/2004/02/skos/core#>.\n" + "@prefix spdx: <http://spdx.org/rdf/terms#>.\n" + "@prefix foaf: <http://xmlns.com/foaf/0.1/>.\n" + "@prefix odrl: <http://www.w3.org/ns/odrl/2/>.\n" + "@prefix cnt: <http://www.w3.org/2011/content#>.\n" + "@prefix language: <http://id.loc.gov/vocabulary/iso639-1/>.\n" + "@prefix s: <" + fdpUrl + "/>.\n" + "@prefix c: <" + fdpUrl + "/dataset/>.\n" + "s:new \n" + "a dcat:Distribution, dcat:Resource;\n" + "dct:isPartOf c:" + datasetId + ";\n");
}
- public String buildDataset(String urlRepo, List<Path> paths, boolean isJsonpath, String catId, String fdpUrl) throws JSONException, ExecutionException, InterruptedException {
+ public Object getDatasetByUrl (String url, Map<String, String> header) throws Exception {
+ return this.requestUtil.getItemByURl( URLDecoder.decode(url, "UTF-8"), header);
+ }
+
+
+ public String buildDataset(String urlRepo, List<Path> paths, boolean isJsonpath, String catId, String fdpUrl) throws Exception {
Map<String, List<Map<String, List<String>>>> dcatPropertiesMap = this.mapToDcat(urlRepo, paths, isJsonpath);
Map<String, List<String>> datasetMap = dcatPropertiesMap.get("dataset").get(0);
@@ -73,7 +79,7 @@ public class MappingService {
return datasetString.toString();
}
- public List<String> buildDistribution(String urlRepo, List<Path> paths, boolean isJsonpath, String datasetId, String fdpUrl) throws JSONException, ExecutionException, InterruptedException {
+ public List<String> buildDistribution(String urlRepo, List<Path> paths, boolean isJsonpath, String datasetId, String fdpUrl) throws Exception {
Map<String, List<Map<String, List<String>>>> dcatPropertiesMap = this.mapToDcat(urlRepo, paths, isJsonpath);
List<Map<String, List<String>>> distributionList = dcatPropertiesMap.get("distribution");
List<String> distributionStrings = new ArrayList<>();
@@ -161,8 +167,8 @@ public class MappingService {
}
@Async
- public CompletableFuture<JSONObject> getJson(String url) throws JSONException {
- return this.requestUtil.getJson(url);
+ public CompletableFuture<JSONObject> getJson(String url) throws Exception {
+ return this.requestUtil.httpRequest(URLDecoder.decode(url, "UTF-8"), null, 0);
}
@Async
@@ -356,7 +362,7 @@ public class MappingService {
return CompletableFuture.completedFuture(propertyValueList);
}
- public Map<String, List<Map<String, List<String>>>> mapToDcat(String urlRepo, List<Path> paths, boolean isJsonpath) throws JSONException, ExecutionException, InterruptedException {
+ public Map<String, List<Map<String, List<String>>>> mapToDcat(String urlRepo, List<Path> paths, boolean isJsonpath) throws Exception {
JSONObject jsonDataset = this.getJson(urlRepo).get();
boolean distributionPathIsPresent = paths.stream().anyMatch(e -> e.getProperty().equals("dcat:distribution"));
diff --git a/src/main/java/com/smartharvester/util/SmartHarvesterRequestUtil.java b/src/main/java/com/smartharvester/util/SmartHarvesterRequestUtil.java
index 0c589b9c8b75005247ab4d961956b1395e10a18e..14bdcc7a1e2d6b7f4c4aa0b9100c1f668158e630 100644
--- a/src/main/java/com/smartharvester/util/SmartHarvesterRequestUtil.java
+++ b/src/main/java/com/smartharvester/util/SmartHarvesterRequestUtil.java
@@ -10,20 +10,25 @@ import java.net.URL;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import javax.net.ssl.HttpsURLConnection;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.json.JSONException;
import org.json.JSONObject;
-import org.springframework.http.HttpEntity;
-import org.springframework.http.HttpHeaders;
-import org.springframework.http.HttpMethod;
-import org.springframework.http.MediaType;
-import org.springframework.http.ResponseEntity;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.*;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;
+import org.springframework.web.client.HttpClientErrorException;
+import org.springframework.web.client.HttpServerErrorException;
+import org.springframework.web.client.ResourceAccessException;
import org.springframework.web.client.RestTemplate;
+import org.springframework.web.server.ServerErrorException;
@Component
public class SmartHarvesterRequestUtil {
+ @Autowired
+ private RestTemplate restTemplate;
@Async
public CompletableFuture<JSONObject> getJson(String urlRepo) throws JSONException {
@@ -62,7 +67,6 @@ public class SmartHarvesterRequestUtil {
@Async
public CompletableFuture<ResponseEntity<Object>> httpRequest(String url, Map<String, String> headers) {
- RestTemplate restTemplate = new RestTemplate();
HttpHeaders httpHeaders = new HttpHeaders();
httpHeaders.setContentType(MediaType.APPLICATION_JSON);
for (Map.Entry<String, String> header: headers.entrySet()) {
@@ -72,6 +76,75 @@ public class SmartHarvesterRequestUtil {
ResponseEntity<Object> response = restTemplate.exchange(url, HttpMethod.GET, httpEntity, Object.class);
return CompletableFuture.completedFuture(response);
}
+ @Async
+ public CompletableFuture<JSONObject> httpRequest(String url, Map<String, String> headers, int i) throws Exception {
+ HttpHeaders httpHeaders = new HttpHeaders();
+ httpHeaders.setContentType(MediaType.APPLICATION_JSON);
+ if (null != headers) {
+ for (Map.Entry<String, String> header : headers.entrySet()) {
+ httpHeaders.add(header.getKey(), header.getValue());
+ }
+ }
+
+ HttpEntity<String> httpEntity = new HttpEntity<>("parameters", httpHeaders);
+ CompletableFuture<JSONObject> jsonObjectCompletableFuture;
+ try {
+ ResponseEntity<Object> response = this.restTemplate.exchange(url, HttpMethod.GET, httpEntity, Object.class);
+ Thread.sleep(1000);
+
+
+ JSONObject json = null;
+ if (response.getStatusCode().is2xxSuccessful()) {
+ ObjectMapper mapper = new ObjectMapper();
+ json = new JSONObject(mapper.writeValueAsString(response.getBody()));
+ }
+
+ jsonObjectCompletableFuture = CompletableFuture.completedFuture(json);
+ } catch (HttpClientErrorException e) {
+ if ((e.getStatusCode() == HttpStatus.TOO_MANY_REQUESTS ||
+ e.getStatusCode().is5xxServerError() || e.getStatusCode() == HttpStatus.GATEWAY_TIMEOUT) && i <= 5) {
+ ++i;
+ jsonObjectCompletableFuture = this.httpRequest(url, headers, i);
+ } else {
+ throw new HttpServerErrorException(e.getStatusCode());
+ }
+ } catch (HttpServerErrorException e) {
+
+ if (i <= 3) {
+ ++i;
+ jsonObjectCompletableFuture = this.httpRequest(url, headers, i);
+ } else {
+ throw new HttpServerErrorException(e.getStatusCode());
+ }
+ } catch (ResourceAccessException e) {
+ if (i <= 3) {
+ ++i;
+ jsonObjectCompletableFuture = this.httpRequest(url, headers, i);
+ } else {
+ throw new HttpServerErrorException(HttpStatus.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+
+ return jsonObjectCompletableFuture;
+ }
+
+
+ public Object getItemByURl(String url, Map<String, String> headers) throws Exception {
+
+ HttpHeaders httpHeaders = new HttpHeaders();
+ httpHeaders.setContentType(MediaType.APPLICATION_JSON);
+ for (Map.Entry<String, String> header : headers.entrySet()) {
+ httpHeaders.add(header.getKey(), header.getValue());
+ }
+
+ HttpEntity<String> httpEntity = new HttpEntity<>("parameters", httpHeaders);
+ CompletableFuture<JSONObject> jsonObjectCompletableFuture;
+ ResponseEntity<Object> response = this.restTemplate.exchange(url, HttpMethod.GET, httpEntity, Object.class);
+
+ return response.getBody();
+ }
+
}