Skip to content
Snippets Groups Projects
Commit 4b052cfe authored by Baptiste Toulemonde's avatar Baptiste Toulemonde
Browse files

Merge branch 'feature/getValues' into 'master'

Feature/get values

See merge request !11
parents dc9fa982 52878aa4
No related branches found
No related tags found
1 merge request!11Feature/get values
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" <project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<parent> <parent>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId> <artifactId>spring-boot-starter-parent</artifactId>
<version>2.3.3.RELEASE</version> <version>2.3.3.RELEASE</version>
<relativePath/> <!-- lookup parent from repository --> <relativePath /> <!-- lookup parent from repository -->
</parent> </parent>
<groupId>com.smartharvester</groupId> <groupId>com.smartharvester</groupId>
<artifactId>smart-harvester</artifactId> <artifactId>smart-harvester</artifactId>
...@@ -28,7 +29,7 @@ ...@@ -28,7 +29,7 @@
<artifactId>spring-boot-starter-web</artifactId> <artifactId>spring-boot-starter-web</artifactId>
</dependency> </dependency>
<!-- Mongo Java Driver + Spring data MongoDB--> <!-- Mongo Java Driver + Spring data MongoDB -->
<dependency> <dependency>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-mongodb</artifactId> <artifactId>spring-boot-starter-data-mongodb</artifactId>
...@@ -40,13 +41,18 @@ ...@@ -40,13 +41,18 @@
<artifactId>spring-boot-starter-security</artifactId> <artifactId>spring-boot-starter-security</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-webflux</artifactId>
</dependency>
<dependency> <dependency>
<groupId>io.jsonwebtoken</groupId> <groupId>io.jsonwebtoken</groupId>
<artifactId>jjwt</artifactId> <artifactId>jjwt</artifactId>
<version>0.9.1</version> <version>0.9.1</version>
</dependency> </dependency>
<!--Gson--> <!--Gson -->
<dependency> <dependency>
<groupId>com.google.code.gson</groupId> <groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId> <artifactId>gson</artifactId>
...@@ -60,7 +66,7 @@ ...@@ -60,7 +66,7 @@
<version>1.1.1</version> <version>1.1.1</version>
</dependency> </dependency>
<!-- Swagger openapi--> <!-- Swagger openapi -->
<dependency> <dependency>
<groupId>org.springdoc</groupId> <groupId>org.springdoc</groupId>
<artifactId>springdoc-openapi-ui</artifactId> <artifactId>springdoc-openapi-ui</artifactId>
...@@ -117,15 +123,15 @@ ...@@ -117,15 +123,15 @@
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId> <artifactId>spring-boot-maven-plugin</artifactId>
</plugin> </plugin>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId> <artifactId>maven-compiler-plugin</artifactId>
<configuration> <configuration>
<source>11</source> <source>11</source>
<target>11</target> <target>11</target>
</configuration> </configuration>
</plugin> </plugin>
</plugins> </plugins>
</build> </build>
</project> </project>
...@@ -2,7 +2,10 @@ package com.smartharvester; ...@@ -2,7 +2,10 @@ package com.smartharvester;
import org.springframework.boot.SpringApplication; import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Bean;
import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.reactive.function.client.WebClient;
@SpringBootApplication @SpringBootApplication
public class SmartHarvesterApplication extends WebSecurityConfigurerAdapter { public class SmartHarvesterApplication extends WebSecurityConfigurerAdapter {
...@@ -11,6 +14,11 @@ public class SmartHarvesterApplication extends WebSecurityConfigurerAdapter { ...@@ -11,6 +14,11 @@ public class SmartHarvesterApplication extends WebSecurityConfigurerAdapter {
SpringApplication.run(SmartHarvesterApplication.class, args); SpringApplication.run(SmartHarvesterApplication.class, args);
} }
@Bean
public RestTemplate restTemplate() {
return new RestTemplate();
}
} }
package com.smartharvester.controller;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import org.json.JSONException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.client.HttpStatusCodeException;
import com.smartharvester.model.mapping.Path;
import com.smartharvester.model.mapping.request.MappingRequest;
import com.smartharvester.model.mapping.response.MappingResponse;
import com.smartharvester.service.MappingFromIsoService;
import com.smartharvester.service.MappingService;
import io.swagger.v3.oas.annotations.tags.Tag;
import reactor.core.publisher.Flux;
@CrossOrigin(origins = "*")
@RequestMapping("/harvester/api/transform")
@Tag(name = "Harvest", description = "transform xml iso 19115 to RDF")
@RestController
public class SmartHarvesterMappingController {
public static final Logger LOGGER = LoggerFactory.getLogger(SmartHarvesterMappingController.class);
@Autowired
private MappingFromIsoService mappingFromIsoService;
@Autowired
private MappingService mappingService;
@GetMapping
public ResponseEntity<String> transformXmlToRdf(@RequestParam(value = "url") String url,
@RequestParam(value = "catalogId") String catalogId) {
try {
return ResponseEntity.ok().body(this.mappingFromIsoService.mapfromCWStoRDF(url, catalogId));
} catch (Exception e) {
return ResponseEntity.badRequest().body("ERROR " + e.getMessage());
}
}
@PostMapping("/publish")
public ResponseEntity<?> map(@RequestBody MappingRequest mappîngRequest,
@RequestParam("catalogId") String catalogId, @RequestParam String fdpUrl) {
LOGGER.info("init mapping");
ResponseEntity<?> responseEntity = null;
List<String> urls = mappîngRequest.getUrls();
List<Path> paths = mappîngRequest.getPaths();
String fdpToken = mappîngRequest.getFdpToken();
List<String> publishedUrl = new ArrayList<String>();
List<String> notPublishedUrl = new ArrayList<String>();
List<Path> distributionPaths = paths.stream().filter(path -> path.getDcatClass().equals("dcat:distribution"))
.collect(Collectors.toList());
try {
for (String url : urls) {
String datasetId = null;
List<String> properties = this.mappingService.getValues(url, paths);
String dataset = this.mappingService.getDatasetString(catalogId, properties.get(0), fdpUrl);
try {
responseEntity = this.mappingService.asyncPostToFdp("/dataset", fdpUrl, dataset, fdpToken);
String location = responseEntity.getHeaders().getLocation().toString();
datasetId = location.substring(location.indexOf("dataset/") + 8);
HttpStatus statusCode = responseEntity.getStatusCode();
if (distributionPaths.size() > 0) {
String distribution = this.mappingService.getDistributionString(datasetId, properties.get(1),
fdpUrl);
try {
this.mappingService.asyncPostToFdp("/distribution", fdpUrl, distribution, fdpToken);
} catch (HttpStatusCodeException e) {
LOGGER.warn(e.getMessage());
}
}
if (statusCode.value() == 201) {
publishedUrl.add(url);
} else {
notPublishedUrl.add(url);
}
} catch (HttpStatusCodeException e) {
notPublishedUrl.add(url + " => " + e.getMessage());
LOGGER.warn(e.getMessage());
}
}
LOGGER.info("mapping closed");
return ResponseEntity.ok(new MappingResponse(publishedUrl, notPublishedUrl));
} catch (JSONException e) {
LOGGER.warn(e.getMessage());
LOGGER.info("mapping closed");
return ResponseEntity.badRequest().body(e.getMessage());
}
}
}
package com.smartharvester.controller;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import com.smartharvester.service.MappingFromIsoService;
import io.swagger.v3.oas.annotations.tags.Tag;
@CrossOrigin(origins = "*")
@RequestMapping("/harvester/api/transform")
@Tag(name = "Harvest",description = "transform xml iso 19115 to RDF")
@RestController
public class SmartHarvesterTransformerController {
@Autowired
private MappingFromIsoService mappingService;
@GetMapping
public ResponseEntity<String> transformXmlToRdf(@RequestParam (value="url") String url, @RequestParam (value="catalogId") String catalogId) {
try {
return ResponseEntity.ok().body(this.mappingService.mapfromCWStoRDF(url, catalogId));
} catch (Exception e) {
return ResponseEntity.badRequest().body("ERROR " + e.getMessage());
}
}
}
package com.smartharvester.model.mapping;
public class Path {
private String property;
private String path;
private String card;
private String dcatClass;
public String getCard() {
return card;
}
public void setCard(String card) {
this.card = card;
}
public String getProperty() {
return property;
}
public void setProperty(String property) {
this.property = property;
}
public String getPath() {
return path;
}
public void setPath(String path) {
this.path = path;
}
public String getDcatClass() {
return dcatClass;
}
public void setDcatClass(String dcatClass) {
this.dcatClass = dcatClass;
}
}
package com.smartharvester.model.mapping.request;
import java.util.List;
import com.smartharvester.model.mapping.Path;
public class MappingRequest {
private List<String> urls;
private List<Path> paths;
private String fdpToken;
public List<String> getUrls() {
return urls;
}
public void setUrls(List<String> urls) {
this.urls = urls;
}
public List<Path> getPaths() {
return paths;
}
public void setPaths(List<Path> paths) {
this.paths = paths;
}
public String getFdpToken() {
return fdpToken;
}
public void setFdpToken(String fdpToken) {
this.fdpToken = fdpToken;
}
}
package com.smartharvester.model.mapping.response;
import java.util.List;
public class MappingResponse {
private List<String> publishedUrl;
private List<String> notPublishedUrl;
public MappingResponse(List<String> publishedUrl, List<String> notPublishedUrl) {
super();
this.publishedUrl = publishedUrl;
this.notPublishedUrl = notPublishedUrl;
}
public List<String> getPublishedUrl() {
return publishedUrl;
}
public void setPublishedUrl(List<String> publishedUrl) {
this.publishedUrl = publishedUrl;
}
public List<String> getNotPublishedUrl() {
return notPublishedUrl;
}
public void setNotPublishedUrl(List<String> notPublishedUrl) {
this.notPublishedUrl = notPublishedUrl;
}
}
package com.smartharvester.service;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import javax.net.ssl.HttpsURLConnection;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.web.client.HttpClientErrorException.BadRequest;
import org.springframework.web.client.RequestCallback;
import org.springframework.web.client.RestTemplate;
import com.smartharvester.model.mapping.Path;
@Service
public class MappingService {
@Autowired
private RestTemplate restTemplate;
public String getDatasetString(String catId, String properties, String fdpUrl) {
String dataset = "@prefix dcat: <http://www.w3.org/ns/dcat#>.\n" + "@prefix dct: <http://purl.org/dc/terms/>.\n"
+ "@prefix adms: <http://www.w3.org/ns/adms#>.\n" + "@prefix dqv: <http://www.w3.org/ns/dqv#> .\n"
+ "@prefix geodcat: <http://data.europa.eu/930/>.\n" + "@prefix prov: <http://www.w3.org/ns/prov#>.\n"
+ "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n"
+ "@prefix language: <http://id.loc.gov/vocabulary/iso639-1/>.\n" + "@prefix s: <" + fdpUrl + "/>.\n"
+ "@prefix c: <" + fdpUrl + "/catalog/>.\n" + "s:new \n" + "a dcat:Dataset, dcat:Resource;\n"
+ "dct:isPartOf c:" + catId + ";\n" + properties + ".";
return dataset;
}
public String getDistributionString(String datasetId, String properties, String fdpUrl) {
String distribution = "@prefix dcat: <http://www.w3.org/ns/dcat#>. \n"
+ "@prefix dct: <http://purl.org/dc/terms/>.\n" + "@prefix adms: <http://www.w3.org/ns/adms#> .\n"
+ "@prefix dqv: <http://www.w3.org/ns/dqv#> .\n" + "@prefix geodcat: <http://data.europa.eu/930/>. \n"
+ "@prefix prov: <http://www.w3.org/ns/prov#>.\n"
+ "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n"
+ "@prefix skos: <http://www.w3.org/2004/02/skos/core#>.\n"
+ "@prefix spdx: <http://spdx.org/rdf/terms#>.\n" + "@prefix foaf: <http://xmlns.com/foaf/0.1/>.\n"
+ "@prefix odrl: <http://www.w3.org/ns/odrl/2/>.\n"
+ "@prefix cnt: <http://www.w3.org/2011/content#>.\n"
+ "@prefix language: <http://id.loc.gov/vocabulary/iso639-1/>.\n" + "@prefix s: <" + fdpUrl + "/>.\n"
+ "@prefix c: <" + fdpUrl + "/dataset/>.\n" + "s:new \n" + "a dcat:Distribution, dcat:Resource;\n"
+ "dct:isPartOf c:" + datasetId + ";\n" + properties + ".";
return distribution;
}
public static JSONObject getJson(String urlRepo) throws JSONException {
URL url = null;
HttpURLConnection urlConnection = null;
String result = null;
try {
url = new URL(urlRepo);
urlConnection = (HttpsURLConnection) url.openConnection();
InputStream input = new BufferedInputStream(urlConnection.getInputStream());
result = readStream(input);
} catch (MalformedURLException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
if (urlConnection != null)
urlConnection.disconnect();
}
JSONObject json = null;
json = new JSONObject(result);
return json;
}
public static String write(String indentifier, String value) {
value = value.replaceAll("[\\r\\n]+", " ");
String property = "";
switch (indentifier) {
case "dcat:landingPage":
case "dcat:theme":
case "dcat:contactPoint":
case "dcat:accessURL":
case "dcat:downloadURL":
case "dct:language":
property += indentifier + " <" + value + ">;\n";
break;
case "dct:issued":
case "dct:modified":
property += indentifier + " \"" + value + "\"^^xsd:dateTime;\n";
break;
default:
property += indentifier + " \"" + value + "\";\n";
break;
}
return property;
}
public static boolean isReplicable(String card) {
return card.endsWith("n");
}
public List<String> getValues(String urlRepo, List<Path> paths)
throws JSONException {
List<String> result = new ArrayList<String>();
String datasetProperties = "";
String distributionProperties = "";
JSONObject json = null;
json = getJson(urlRepo);
if (paths.stream().filter(e -> e.getDcatClass().equals("dcat:dataset")).filter(e -> e.getProperty().equals("dct:hasVersion")).collect(Collectors.toList()).size() == 0) {
datasetProperties += "dct:hasVersion \"null\";\n";
}
distributionProperties += "dct:hasVersion \"null\";\n";
for (Path path : paths) {
String[] array = path.getPath().split(" : ");
List<String> list = new ArrayList<String>(Arrays.asList(array));
JSONObject jsonT = json;
if (list.size() == 1) {
if (jsonT.optJSONArray(list.get(0)) != null) {
JSONArray jsonA = jsonT.getJSONArray(list.get(0));
if (isReplicable(path.getCard())) {
for (int j = 0; j < jsonA.length(); j++) {
if (path.getDcatClass().equals("dcat:dataset")) {
datasetProperties += write(path.getProperty(), jsonA.getString(j));
} else if (path.getDcatClass().equals("dcat:distribution")) {
distributionProperties += write(path.getProperty(), jsonA.getString(j));
}
}
} else {
if (path.getDcatClass().equals("dcat:dataset")) {
datasetProperties += write(path.getProperty(), jsonA.getString(0));
} else if (path.getDcatClass().equals("dcat:distribution")) {
distributionProperties += write(path.getProperty(), jsonA.getString(0));
}
}
} else {
if (path.getDcatClass().equals("dcat:dataset")) {
datasetProperties += write(path.getProperty(), jsonT.getString(list.get(0)));
} else if (path.getDcatClass().equals("dcat:distribution")) {
distributionProperties += write(path.getProperty(), jsonT.getString(list.get(0)));
}
}
} else {
for (int i = 0; i < list.size(); i++) {
if (i < list.size() - 1) {
if (jsonT.optJSONObject(list.get(i)) != null) {
jsonT = jsonT.getJSONObject(list.get(i));
} else if (jsonT.optJSONArray(list.get(i)) != null) {
JSONArray jsonA = jsonT.getJSONArray(list.get(i));
if (i < list.size() - 2) {
jsonT = jsonA.getJSONObject(Integer.parseInt(list.get(i + 1)));
list.remove(i + 1);
}
}
} else {
if (jsonT.optJSONArray(list.get(i)) != null) {
JSONArray jsonA = jsonT.getJSONArray(list.get(i));
if (isReplicable(path.getCard())) {
for (int j = 0; j < jsonA.length(); j++) {
if (path.getDcatClass().equals("dcat:dataset")) {
datasetProperties += write(path.getProperty(), jsonA.getString(j));
} else if (path.getDcatClass().equals("dcat:distribution")) {
distributionProperties += write(path.getProperty(), jsonA.getString(j));
}
}
} else {
if (path.getDcatClass().equals("dcat:dataset")) {
datasetProperties += write(path.getProperty(), jsonA.getString(0));
} else if (path.getDcatClass().equals("dcat:distribution")) {
distributionProperties += write(path.getProperty(), jsonA.getString(0));
}
}
break;
} else if (jsonT.optJSONArray(list.get(list.size() - 2)) != null) {
JSONArray jsonA = jsonT.getJSONArray(list.get(list.size() - 2));
if (path.getDcatClass().equals("dcat:dataset")) {
datasetProperties += write(path.getProperty(), jsonA.getString(Integer.parseInt(list.get(i))));
} else if (path.getDcatClass().equals("dcat:distribution")) {
distributionProperties += write(path.getProperty(), jsonA.getString(Integer.parseInt(list.get(i))));
}
break;
} else if (jsonT.optString(list.get(i)) != null && jsonT.has(list.get(i))) {
if (path.getDcatClass().equals("dcat:dataset")) {
datasetProperties += write(path.getProperty(), jsonT.getString(list.get(i)));
} else if (path.getDcatClass().equals("dcat:distribution")) {
distributionProperties += write(path.getProperty(), jsonT.getString(list.get(i)));
}
break;
} else if (jsonT instanceof JSONObject) {
Iterator keys = jsonT.keys();
while (keys.hasNext()) {
String key = keys.next().toString();
if (jsonT.optJSONArray(key) != null) {
JSONArray jsonA = jsonT.getJSONArray(key);
if (isReplicable(path.getCard())) {
for (int j = 0; j < jsonA.length(); j++) {
String value = jsonA.getJSONObject(j).getString(list.get(i));
if (path.getDcatClass().equals("dcat:dataset")) {
datasetProperties += write(path.getProperty(), value);
} else if (path.getDcatClass().equals("dcat:distribution")) {
distributionProperties += write(path.getProperty(), value);
}
}
} else {
String value = jsonA.getJSONObject(0).getString(list.get(i));
if (path.getDcatClass().equals("dcat:dataset")) {
datasetProperties += write(path.getProperty(), value);
} else if (path.getDcatClass().equals("dcat:distribution")) {
distributionProperties += write(path.getProperty(), value);
}
}
}
}
}
}
}
}
}
result.add(datasetProperties);
result.add(distributionProperties);
return result;
}
private static String readStream(InputStream input) throws IOException {
StringBuilder sb = new StringBuilder();
BufferedReader reader = new BufferedReader(new InputStreamReader(input), 1000);
for (String line = reader.readLine(); line != null; line = reader.readLine()) {
sb.append(line);
}
input.close();
return sb.toString();
}
public ResponseEntity<String> asyncPostToFdp(String path, String fdpUrl, String dataset, String fdpToken)
throws BadRequest {
HttpHeaders headers = new HttpHeaders();
headers.add("Accept", "text/turtle;charset=UTF-8");
headers.add("Content-Type", "text/turtle;charset=UTF-8");
headers.setBearerAuth(fdpToken);
HttpEntity<String> entity = new HttpEntity<String>(dataset, headers);
return this.restTemplate.exchange(fdpUrl + path, HttpMethod.POST, entity, String.class);
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment