diff --git a/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java b/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java index a4df2ad1732e6461e8bd515c6205235c7479aae6..339d7b604c46306d2425e55922a51263f303f8ef 100644 --- a/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java +++ b/src/main/java/com/smartharvester/controller/SmartHarvesterMappingController.java @@ -9,11 +9,9 @@ import com.smartharvester.service.MappingFromIsoService; import com.smartharvester.service.MappingService; import com.smartharvester.service.OpenApiServiceImpl; import io.swagger.v3.oas.annotations.tags.Tag; -import org.json.JSONException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.CrossOrigin; import org.springframework.web.bind.annotation.GetMapping; @@ -23,13 +21,10 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import java.net.URI; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; @CrossOrigin(origins = "*") @@ -79,54 +74,14 @@ public class SmartHarvesterMappingController { ) ); - for (Map.Entry<String, List<String>> entry : urls.entrySet()) { - String datasetId; - String url = entry.getKey(); - List<String> iris = entry.getValue(); + urls.entrySet().parallelStream().forEach(entry -> { + this.mappingService.mapAndPublish(catalogId, fdpUrl, isJsonpath, paths, fdpToken, publishedUrl, notPublishedUrl, distributionPaths, + entry); + }); - String locationDataset; - try { - String dataset = this.mappingService.buildDataset(new URI(url), paths, isJsonpath, catalogId, fdpUrl, iris); - responseEntity = this.mappingService.asyncPostToFdp("/dataset", fdpUrl.replace(":8080", ""), dataset, fdpToken).get(); - locationDataset = Objects.requireNonNull(responseEntity.getHeaders().getLocation()).toString(); - datasetId = locationDataset.substring(locationDataset.indexOf("dataset/") + 8); - - HttpStatus statusCode = responseEntity.getStatusCode(); - if (!distributionPaths.isEmpty()) { - List<String> distributions = this.mappingService.buildDistribution(new URI(url), paths, isJsonpath, datasetId, fdpUrl); - - for (String distribution: distributions) { - try { - ResponseEntity<String> distributionResponse = this.mappingService.asyncPostToFdp("/distribution", fdpUrl.replace(":8080", ""), distribution, fdpToken).get(); - String locationDistribution = Objects.requireNonNull(Objects.requireNonNull(distributionResponse.getHeaders().getLocation()).toString()); - if (distributionResponse.getStatusCode().value() == 201) { - this.mappingService.draftToPublished(fdpToken, locationDistribution.replace(":8080", "")); - } - } catch (ExecutionException | InterruptedException e) { - LOGGER.warn(e.getMessage()); - } - } - } - if (statusCode.value() == 201) { - publishedUrl.add(url); - this.mappingService.draftToPublished(fdpToken, locationDataset.replace(":8080", "")); - } else { - notPublishedUrl.add(url); - } - } catch (JSONException | ExecutionException | IllegalStateException e ) { - notPublishedUrl.add(url + " => " + e.getMessage()); - LOGGER.warn(e.getMessage()); - } catch (Exception e) { - Thread.currentThread().interrupt(); - notPublishedUrl.add(url + " => " + e.getMessage()); - LOGGER.warn(e.getMessage()); - } - - } LOGGER.info("mapping closed"); return ResponseEntity.ok(new MappingResponse(publishedUrl, notPublishedUrl)); } - } diff --git a/src/main/java/com/smartharvester/model/elasticSearch/Document.java b/src/main/java/com/smartharvester/model/elasticSearch/Document.java index d902ca624c16ed652b2f2d1551f7cdc697d0439d..2af4c5f3be5500b8bd0a8642756128f44ced809b 100644 --- a/src/main/java/com/smartharvester/model/elasticSearch/Document.java +++ b/src/main/java/com/smartharvester/model/elasticSearch/Document.java @@ -41,6 +41,8 @@ public class Document { @JsonProperty("resources_reusing") private List<String> resourcesReusing; + private String description; + public String getIri() { return iri; } @@ -128,4 +130,12 @@ public class Document { public void setResourcesReusing(List<String> resourcesReusing) { this.resourcesReusing = resourcesReusing; } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } } diff --git a/src/main/java/com/smartharvester/model/mapping/response/KeywordResponse.java b/src/main/java/com/smartharvester/model/mapping/response/KeywordResponse.java index 1d8ef62001757a597f9afccc77611e7707955111..5956a8aa5c909df45a66a2bd62d1280c7374e8e5 100644 --- a/src/main/java/com/smartharvester/model/mapping/response/KeywordResponse.java +++ b/src/main/java/com/smartharvester/model/mapping/response/KeywordResponse.java @@ -8,6 +8,7 @@ public class KeywordResponse { private String id; + private String title; private String url; private List<String> keywords; @@ -45,4 +46,12 @@ public class KeywordResponse { public void setUrl(String url) { this.url = url; } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } } diff --git a/src/main/java/com/smartharvester/service/ElasticSearchService.java b/src/main/java/com/smartharvester/service/ElasticSearchService.java index 78bd19c1176d139fbe53a6465a01838d5878e32c..216c3b95b076e19291fa2acb287595c5b24ad1f7 100644 --- a/src/main/java/com/smartharvester/service/ElasticSearchService.java +++ b/src/main/java/com/smartharvester/service/ElasticSearchService.java @@ -101,10 +101,12 @@ public class ElasticSearchService { List<String> urls = new ArrayList<>(); req.getIds().forEach(id -> urls.addAll(this.openApiService.getUrls(this.openApiService.getOpenApiByUUDI(catalogId), id))); List<Path> paths = req.getPaths().stream() - .filter(path -> path.getProperty().equals("dct:identifier") || path.getProperty().equals("dcat:keyword")) + .filter(path -> path.getProperty().equals("dct:identifier") || + path.getProperty().equals("dcat:keyword") || + path.getProperty().equals("dct:title")) .collect(Collectors.toList()); List<KeywordResponse> result = new ArrayList<>(); - urls.forEach(url -> { + urls.parallelStream().forEach(url -> { JSONObject json = new JSONObject(); List<Map<String, List<String>>> retrievedFromJson = new ArrayList<>(); try { @@ -122,11 +124,13 @@ public class ElasticSearchService { LOGGER.error(e.getMessage()); } - retrievedFromJson.forEach(e -> { + retrievedFromJson.parallelStream().forEach(e -> { KeywordResponse keywordResponse = new KeywordResponse(); Assert.assertNotNull(e.get("dct:identifier")); keywordResponse.setId(e.get("dct:identifier").get(0)); keywordResponse.setUrl(url); + Assert.assertNotNull(e.get("dct:title")); + keywordResponse.setTitle(e.get("dct:title").get(0)); keywordResponse.setKeywords(e.get("dcat:keyword")); result.add(keywordResponse); }); @@ -184,6 +188,8 @@ public class ElasticSearchService { ResponseEntity<ElasticSearchResponse> searchRequest = this.getSearchRequest(keyword1, keyword2); if (searchRequest.getStatusCodeValue() == 200 && Objects.requireNonNull(searchRequest.getBody()).getCount() != 0) { k.setConcepts(searchRequest.getBody()); + } else { + k.setConcepts(null); } } diff --git a/src/main/java/com/smartharvester/service/MappingService.java b/src/main/java/com/smartharvester/service/MappingService.java index 621a6834debb71f2356f7d304e73236888ddf32e..36ff6749f1bf4efc76d25014396c688fb14f757a 100644 --- a/src/main/java/com/smartharvester/service/MappingService.java +++ b/src/main/java/com/smartharvester/service/MappingService.java @@ -15,6 +15,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.scheduling.annotation.Async; @@ -32,7 +33,9 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; @Service @@ -463,5 +466,54 @@ public class MappingService { } + public void mapAndPublish(String catalogId, String fdpUrl, boolean isJsonpath, List<Path> paths, String fdpToken, + List<String> publishedUrl, List<String> notPublishedUrl, List<Path> distributionPaths, + Map.Entry<String, List<String>> entry) { + ResponseEntity<String> responseEntity; + String datasetId; + String url = entry.getKey(); + List<String> iris = entry.getValue(); + + String locationDataset; + try { + String dataset = this.buildDataset(new URI(url), paths, isJsonpath, catalogId, fdpUrl, iris); + responseEntity = this.asyncPostToFdp("/dataset", fdpUrl.replace(":8080", ""), dataset, + fdpToken).get(); + locationDataset = Objects.requireNonNull(responseEntity.getHeaders().getLocation()).toString(); + datasetId = locationDataset.substring(locationDataset.indexOf("dataset/") + 8); + + HttpStatus statusCode = responseEntity.getStatusCode(); + if (!distributionPaths.isEmpty()) { + List<String> distributions = this.buildDistribution(new URI(url), paths, isJsonpath, datasetId, + fdpUrl); + + for (String distribution: distributions) { + try { + ResponseEntity<String> distributionResponse = this.asyncPostToFdp("/distribution", fdpUrl.replace(":8080", ""), distribution, + fdpToken).get(); + String locationDistribution = Objects.requireNonNull(Objects.requireNonNull(distributionResponse.getHeaders().getLocation()).toString()); + if (distributionResponse.getStatusCode().value() == 201) { + this.draftToPublished(fdpToken, locationDistribution.replace(":8080", "")); + } + } catch (ExecutionException | InterruptedException e) { + LOGGER.warn(e.getMessage()); + } + } + } + if (statusCode.value() == 201) { + publishedUrl.add(url); + this.draftToPublished(fdpToken, locationDataset.replace(":8080", "")); + } else { + notPublishedUrl.add(url); + } + } catch (JSONException | ExecutionException | IllegalStateException e ) { + notPublishedUrl.add(url + " => " + e.getMessage()); + LOGGER.warn(e.getMessage()); + } catch (Exception e) { + Thread.currentThread().interrupt(); + notPublishedUrl.add(url + " => " + e.getMessage()); + LOGGER.warn(e.getMessage()); + } + } } diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties index 670e0f8e22710e0c77e65b0e6a921453834e47e6..9b31ca4d63cdd9dae536a57eda2b767de6dbf279 100644 --- a/src/main/resources/application-dev.properties +++ b/src/main/resources/application-dev.properties @@ -47,4 +47,4 @@ spring.security.oauth2.client.registration.oidc.redirect-uri=http://localhost:42 elasticsearch.baseURL=https://localhost:4242/eosc/api elasticsearch.token=uHiOaPyo6IjKa7lV6qDHWYz3lgcf8bG2 elasticsearch.minChar=4 -elasticsearch.maxChar=30 +elasticsearch.maxChar=10