Skip to content
Snippets Groups Projects
Commit 9b0ceca4 authored by Baptiste Toulemonde's avatar Baptiste Toulemonde
Browse files

fix after functional review

parent 8d7cb45f
No related branches found
No related tags found
1 merge request!18Feature/semantic enrichment
Hide whitespace changes
Inline Side-by-side
Showing
with 85 additions and 53 deletions
src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java
+ 4
− 49
View file @ 9b0ceca4
......@@ -9,11 +9,9 @@ import com.smartharvester.service.MappingFromIsoService;
import com.smartharvester.service.MappingService;
import com.smartharvester.service.OpenApiServiceImpl;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.json.JSONException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.GetMapping;
......@@ -23,13 +21,10 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.net.URI;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
@CrossOrigin(origins = "*")
......@@ -79,54 +74,14 @@ public class SmartHarvesterMappingController {
)
);
for (Map.Entry<String, List<String>> entry : urls.entrySet()) {
String datasetId;
String url = entry.getKey();
List<String> iris = entry.getValue();
urls.entrySet().parallelStream().forEach(entry -> {
this.mappingService.mapAndPublish(catalogId, fdpUrl, isJsonpath, paths, fdpToken, publishedUrl, notPublishedUrl, distributionPaths,
entry);
});
String locationDataset;
try {
String dataset = this.mappingService.buildDataset(new URI(url), paths, isJsonpath, catalogId, fdpUrl, iris);
responseEntity = this.mappingService.asyncPostToFdp("/dataset", fdpUrl.replace(":8080", ""), dataset, fdpToken).get();
locationDataset = Objects.requireNonNull(responseEntity.getHeaders().getLocation()).toString();
datasetId = locationDataset.substring(locationDataset.indexOf("dataset/") + 8);
HttpStatus statusCode = responseEntity.getStatusCode();
if (!distributionPaths.isEmpty()) {
List<String> distributions = this.mappingService.buildDistribution(new URI(url), paths, isJsonpath, datasetId, fdpUrl);
for (String distribution: distributions) {
try {
ResponseEntity<String> distributionResponse = this.mappingService.asyncPostToFdp("/distribution", fdpUrl.replace(":8080", ""), distribution, fdpToken).get();
String locationDistribution = Objects.requireNonNull(Objects.requireNonNull(distributionResponse.getHeaders().getLocation()).toString());
if (distributionResponse.getStatusCode().value() == 201) {
this.mappingService.draftToPublished(fdpToken, locationDistribution.replace(":8080", ""));
}
} catch (ExecutionException | InterruptedException e) {
LOGGER.warn(e.getMessage());
}
}
}
if (statusCode.value() == 201) {
publishedUrl.add(url);
this.mappingService.draftToPublished(fdpToken, locationDataset.replace(":8080", ""));
} else {
notPublishedUrl.add(url);
}
} catch (JSONException | ExecutionException | IllegalStateException e ) {
notPublishedUrl.add(url + " => " + e.getMessage());
LOGGER.warn(e.getMessage());
} catch (Exception e) {
Thread.currentThread().interrupt();
notPublishedUrl.add(url + " => " + e.getMessage());
LOGGER.warn(e.getMessage());
}
}
LOGGER.info("mapping closed");
return ResponseEntity.ok(new MappingResponse(publishedUrl, notPublishedUrl));
}
}
src/main/java/com/smartharvester/model/elasticSearch/Document.java
+ 10
− 0
View file @ 9b0ceca4
......@@ -41,6 +41,8 @@ public class Document {
@JsonProperty("resources_reusing")
private List<String> resourcesReusing;
private String description;
public String getIri() {
return iri;
}
......@@ -128,4 +130,12 @@ public class Document {
public void setResourcesReusing(List<String> resourcesReusing) {
this.resourcesReusing = resourcesReusing;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
}
src/main/java/com/smartharvester/model/mapping/response/KeywordResponse.java
+ 9
− 0
View file @ 9b0ceca4
......@@ -8,6 +8,7 @@ public class KeywordResponse {
private String id;
private String title;
private String url;
private List<String> keywords;
......@@ -45,4 +46,12 @@ public class KeywordResponse {
public void setUrl(String url) {
this.url = url;
}
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
}
src/main/java/com/smartharvester/service/ElasticSearchService.java
+ 9
− 3
View file @ 9b0ceca4
......@@ -101,10 +101,12 @@ public class ElasticSearchService {
List<String> urls = new ArrayList<>();
req.getIds().forEach(id -> urls.addAll(this.openApiService.getUrls(this.openApiService.getOpenApiByUUDI(catalogId), id)));
List<Path> paths = req.getPaths().stream()
.filter(path -> path.getProperty().equals("dct:identifier") || path.getProperty().equals("dcat:keyword"))
.filter(path -> path.getProperty().equals("dct:identifier") ||
path.getProperty().equals("dcat:keyword") ||
path.getProperty().equals("dct:title"))
.collect(Collectors.toList());
List<KeywordResponse> result = new ArrayList<>();
urls.forEach(url -> {
urls.parallelStream().forEach(url -> {
JSONObject json = new JSONObject();
List<Map<String, List<String>>> retrievedFromJson = new ArrayList<>();
try {
......@@ -122,11 +124,13 @@ public class ElasticSearchService {
LOGGER.error(e.getMessage());
}
retrievedFromJson.forEach(e -> {
retrievedFromJson.parallelStream().forEach(e -> {
KeywordResponse keywordResponse = new KeywordResponse();
Assert.assertNotNull(e.get("dct:identifier"));
keywordResponse.setId(e.get("dct:identifier").get(0));
keywordResponse.setUrl(url);
Assert.assertNotNull(e.get("dct:title"));
keywordResponse.setTitle(e.get("dct:title").get(0));
keywordResponse.setKeywords(e.get("dcat:keyword"));
result.add(keywordResponse);
});
......@@ -184,6 +188,8 @@ public class ElasticSearchService {
ResponseEntity<ElasticSearchResponse> searchRequest = this.getSearchRequest(keyword1, keyword2);
if (searchRequest.getStatusCodeValue() == 200 && Objects.requireNonNull(searchRequest.getBody()).getCount() != 0) {
k.setConcepts(searchRequest.getBody());
} else {
k.setConcepts(null);
}
}
......
src/main/java/com/smartharvester/service/MappingService.java
+ 52
− 0
View file @ 9b0ceca4
......@@ -15,6 +15,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.scheduling.annotation.Async;
......@@ -32,7 +33,9 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
@Service
......@@ -463,5 +466,54 @@ public class MappingService {
}
public void mapAndPublish(String catalogId, String fdpUrl, boolean isJsonpath, List<Path> paths, String fdpToken,
List<String> publishedUrl, List<String> notPublishedUrl, List<Path> distributionPaths,
Map.Entry<String, List<String>> entry) {
ResponseEntity<String> responseEntity;
String datasetId;
String url = entry.getKey();
List<String> iris = entry.getValue();
String locationDataset;
try {
String dataset = this.buildDataset(new URI(url), paths, isJsonpath, catalogId, fdpUrl, iris);
responseEntity = this.asyncPostToFdp("/dataset", fdpUrl.replace(":8080", ""), dataset,
fdpToken).get();
locationDataset = Objects.requireNonNull(responseEntity.getHeaders().getLocation()).toString();
datasetId = locationDataset.substring(locationDataset.indexOf("dataset/") + 8);
HttpStatus statusCode = responseEntity.getStatusCode();
if (!distributionPaths.isEmpty()) {
List<String> distributions = this.buildDistribution(new URI(url), paths, isJsonpath, datasetId,
fdpUrl);
for (String distribution: distributions) {
try {
ResponseEntity<String> distributionResponse = this.asyncPostToFdp("/distribution", fdpUrl.replace(":8080", ""), distribution,
fdpToken).get();
String locationDistribution = Objects.requireNonNull(Objects.requireNonNull(distributionResponse.getHeaders().getLocation()).toString());
if (distributionResponse.getStatusCode().value() == 201) {
this.draftToPublished(fdpToken, locationDistribution.replace(":8080", ""));
}
} catch (ExecutionException | InterruptedException e) {
LOGGER.warn(e.getMessage());
}
}
}
if (statusCode.value() == 201) {
publishedUrl.add(url);
this.draftToPublished(fdpToken, locationDataset.replace(":8080", ""));
} else {
notPublishedUrl.add(url);
}
} catch (JSONException | ExecutionException | IllegalStateException e ) {
notPublishedUrl.add(url + " => " + e.getMessage());
LOGGER.warn(e.getMessage());
} catch (Exception e) {
Thread.currentThread().interrupt();
notPublishedUrl.add(url + " => " + e.getMessage());
LOGGER.warn(e.getMessage());
}
}
}
src/main/resources/application-dev.properties
+ 1
− 1
View file @ 9b0ceca4
......@@ -47,4 +47,4 @@ spring.security.oauth2.client.registration.oidc.redirect-uri=http://localhost:42
elasticsearch.baseURL=https://localhost:4242/eosc/api
elasticsearch.token=uHiOaPyo6IjKa7lV6qDHWYz3lgcf8bG2
elasticsearch.minChar=4
elasticsearch.maxChar=30
elasticsearch.maxChar=10
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment