From afbaf000d78ad05aa9bf9ad3df820174f1e0fa25 Mon Sep 17 00:00:00 2001 From: Baptiste Toulemonde <toulemonde@cines.fr> Date: Fri, 28 Jan 2022 12:08:26 +0100 Subject: [PATCH] add methods to get urls and ids --- pom.xml | 4 + .../controller/SmartHarvesterController.java | 55 +++-- .../service/MappingService.java | 41 +--- .../service/OpenApiServiceImpl.java | 194 +++++++++++++----- .../util/SmartHarvesterGetJsonUtil.java | 103 ++++++++++ 5 files changed, 294 insertions(+), 103 deletions(-) create mode 100644 src/main/java/com/smartharvester/util/SmartHarvesterGetJsonUtil.java diff --git a/pom.xml b/pom.xml index 1568ab7..6e248e3 100644 --- a/pom.xml +++ b/pom.xml @@ -119,6 +119,10 @@ <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-oauth2-client</artifactId> </dependency> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-devtools</artifactId> + </dependency> </dependencies> <build> diff --git a/src/main/java/com/smartharvester/controller/SmartHarvesterController.java b/src/main/java/com/smartharvester/controller/SmartHarvesterController.java index 65d1e32..9bca4eb 100644 --- a/src/main/java/com/smartharvester/controller/SmartHarvesterController.java +++ b/src/main/java/com/smartharvester/controller/SmartHarvesterController.java @@ -1,16 +1,11 @@ package com.smartharvester.controller; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.mongodb.client.FindIterable; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoDatabase; -import com.mongodb.client.MongoIterable; +import com.fasterxml.jackson.databind.JsonMappingException; import com.smartharvester.dao.OpenApiDaoRepository; import com.smartharvester.exception.ResourceNotFoundException; import com.smartharvester.model.openapi.OpenApi; -import com.smartharvester.service.OpenApiService; -import io.swagger.annotations.Api; +import com.smartharvester.service.OpenApiServiceImpl; import io.swagger.v3.oas.annotations.tags.Tag; import io.swagger.v3.oas.models.Operation; import io.swagger.v3.parser.OpenAPIV3Parser; @@ -19,22 +14,21 @@ import org.apache.http.HttpHost; import org.apache.http.entity.ContentType; import org.apache.http.nio.entity.NStringEntity; import org.apache.http.util.EntityUtils; -import org.bson.Document; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; - import java.io.*; import java.net.HttpURLConnection; +import java.net.URISyntaxException; import java.net.URL; import java.util.*; @@ -54,8 +48,7 @@ public class SmartHarvesterController { private OpenApiDaoRepository repository; @Autowired - @Qualifier(value = "openApiService") - OpenApiService openApiService; + OpenApiServiceImpl openApiService; private final Logger logger = LoggerFactory.getLogger(this.getClass()); @@ -66,20 +59,44 @@ public class SmartHarvesterController { @RequestMapping(value = "/all", method = RequestMethod.GET) public Collection<OpenApi> getAll() { logger.debug("Getting all OpenApi from database..."); - List<OpenApi> response = openApiService.getAllOpenApi(); + List<OpenApi> response = this.openApiService.getAllOpenApi(); return response; } /** - * Method to fetch all openAPIs from mongodb. + * Method to fetch url by uuid * @return */ - @RequestMapping(value = "/allurls", method = RequestMethod.GET) - public Collection<String> getAllUrl() { - logger.debug("Getting all OpenApi from database..."); - Collection<String> response = openApiService.getAllUrl(); + @GetMapping("/url/{uuid}") + public ResponseEntity<String> getSearchUrl(@PathVariable ("uuid") String uuid) { + OpenApi openApi = this.openApiService.getOpenApiByUUDI(uuid); + String response = this.openApiService.getUrlById(openApi, 0); - return response; + return ResponseEntity.ok().body(response); + } + + @GetMapping("/ids/{uuid}") + public ResponseEntity<?> getIdList(@PathVariable ("uuid") String uuid) { + OpenApi openApi = this.openApiService.getOpenApiByUUDI(uuid); + try { + List<String> datasetList = this.openApiService.getDatasetList(openApi); + return ResponseEntity.ok().body(datasetList); + } catch (URISyntaxException | JSONException | JsonProcessingException e) { + return ResponseEntity.badRequest().body(e.getMessage()); + } + } + + @GetMapping("/urls/{uuid}") + public ResponseEntity<?> getDatasetUrls(@PathVariable ("uuid") String uuid) { + OpenApi openApi = this.openApiService.getOpenApiByUUDI(uuid); + List<String> datasetList; + try { + List<String> datasetUrl = this.openApiService.getDatasetUrl(openApi, 1); + return ResponseEntity.ok().body(datasetUrl); + } catch (JsonProcessingException | URISyntaxException | JSONException e) { + return ResponseEntity.badRequest().body(e.getMessage()); + } + } /** diff --git a/src/main/java/com/smartharvester/service/MappingService.java b/src/main/java/com/smartharvester/service/MappingService.java index 36f3eb3..cc3f10e 100644 --- a/src/main/java/com/smartharvester/service/MappingService.java +++ b/src/main/java/com/smartharvester/service/MappingService.java @@ -33,12 +33,15 @@ import org.springframework.web.client.RequestCallback; import org.springframework.web.client.RestTemplate; import com.smartharvester.model.mapping.Path; import com.smartharvester.model.mapping.request.PublishedRequest; +import com.smartharvester.util.SmartHarvesterGetJsonUtil; @Service public class MappingService { @Autowired private RestTemplate restTemplate; + + private SmartHarvesterGetJsonUtil getJsonUtil; public String getDatasetString(String catId, String properties, String fdpUrl) { String dataset = "@prefix dcat: <http://www.w3.org/ns/dcat#>.\n" + "@prefix dct: <http://purl.org/dc/terms/>.\n" @@ -67,31 +70,7 @@ public class MappingService { return distribution; } - public static JSONObject getJson(String urlRepo) throws JSONException { - URL url = null; - HttpURLConnection urlConnection = null; - String result = null; - - try { - url = new URL(urlRepo); - urlConnection = (HttpsURLConnection) url.openConnection(); - InputStream input = new BufferedInputStream(urlConnection.getInputStream()); - result = readStream(input); - } catch (MalformedURLException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } finally { - if (urlConnection != null) - urlConnection.disconnect(); - } - - JSONObject json = null; - - json = new JSONObject(result); - - return json; - } + public static String write(String indentifier, String value) { value = value.replaceAll("[\\r\\n]+", " "); @@ -132,7 +111,7 @@ public class MappingService { JSONObject json = null; try { - json = getJson(urlRepo); + json = this.getJsonUtil.getJson(urlRepo); } catch (JSONException e1) { e1.printStackTrace(); } @@ -288,15 +267,7 @@ public class MappingService { } - private static String readStream(InputStream input) throws IOException { - StringBuilder sb = new StringBuilder(); - BufferedReader reader = new BufferedReader(new InputStreamReader(input), 1000); - for (String line = reader.readLine(); line != null; line = reader.readLine()) { - sb.append(line); - } - input.close(); - return sb.toString(); - } + public ResponseEntity<String> asyncPostToFdp(String path, String fdpUrl, String dataset, String fdpToken) throws BadRequest { diff --git a/src/main/java/com/smartharvester/service/OpenApiServiceImpl.java b/src/main/java/com/smartharvester/service/OpenApiServiceImpl.java index c921ed2..959419e 100644 --- a/src/main/java/com/smartharvester/service/OpenApiServiceImpl.java +++ b/src/main/java/com/smartharvester/service/OpenApiServiceImpl.java @@ -1,63 +1,159 @@ package com.smartharvester.service; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.smartharvester.dao.OpenApiDaoRepository; import com.smartharvester.model.openapi.OpenApi; import com.smartharvester.model.openapi.OpenApiPathItem; import com.smartharvester.model.openapi.OpenApiPathItemParameter; +import com.smartharvester.model.openapi.OpenApiResponseContent; +import com.smartharvester.util.SmartHarvesterGetJsonUtil; + import org.apache.commons.lang3.StringUtils; +import org.json.JSONException; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import java.net.URISyntaxException; import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Service +public class OpenApiServiceImpl { + + @Autowired + OpenApiDaoRepository openApiDaoRepository; + + @Autowired + SmartHarvesterGetJsonUtil getJsonUtil; + + public List<OpenApi> getAllOpenApi() { + return openApiDaoRepository.findAll(); + } + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + public String getUrlById(OpenApi openApi, int requestNumber) { + String path = (String) openApi.getPaths().keySet().toArray()[0]; + String servers = openApi.getServers().get(0).getUrl(); + Map<String, String> queryparameters = this.getQueryparameters(openApi,0); + Map<String, String> pathparameters = this.getPathparameters(openApi, 0); + String listOfParams = "?"; + String pathParam = ""; + for (String param : queryparameters.keySet()) { + listOfParams = listOfParams.concat(param + "=" + queryparameters.get(param) + "&"); + } + for (String pathvalue : pathparameters.values()) { + pathParam += "/" + pathvalue; + } + + listOfParams = StringUtils.substring(listOfParams, 0, listOfParams.length() - 1); + + return servers + path + pathParam + listOfParams; + + } + + public List<String> getDatasetUrl(OpenApi openApi, int requestNumber) throws JsonProcessingException, URISyntaxException, JSONException { + String path = (String) openApi.getPaths().keySet().toArray()[1]; + String servers = openApi.getServers().get(0).getUrl(); + List<String> ids = this.getDatasetList(openApi); + List<OpenApiPathItemParameter> parameters = this.getParameters(openApi, 1); + List<String> urls = new ArrayList<String>(); + for (String id : ids) { + Map<String, String> mapQueryParameters = new HashMap<String, String>(); + Map<String, String> mapPathParameters = new HashMap<String, String>(); + for(OpenApiPathItemParameter param: parameters) { + if (param.getRequired()) { + if(param.getIn().equals("query")) { + mapQueryParameters.put(param.getName(), id); + } else if (param.getIn().equals("path")){ + mapPathParameters.put(param.getName(), id); + } + } else { + if(param.getIn().equals("query")) { + mapQueryParameters.put(param.getName(), param.getSchema().getDefaultValue()); + } else if (param.getIn().equals("path")){ + mapPathParameters.put(param.getName(), param.getSchema().getDefaultValue()); + } + } + } + String listOfParams = "?"; + String pathParam = ""; + for(Map.Entry<String, String> entry: mapQueryParameters.entrySet()) { + listOfParams = listOfParams.concat(entry.getKey() + "=" + entry.getValue() + "&"); + } + for(Map.Entry<String, String> entry: mapPathParameters.entrySet()) { + pathParam += "/" + entry.getValue();; + } + + listOfParams = StringUtils.substring(listOfParams, 0, listOfParams.length() - 1); + urls.add(servers + path + pathParam + listOfParams); + } + + return urls; + } + + public List<OpenApiPathItemParameter> getParameters(OpenApi openApi, int requestNumber) { + String path = (String) openApi.getPaths().keySet().toArray()[requestNumber]; + Map<String, OpenApiPathItem> map = openApi.getPaths().get(path); + return map.get("get").getParameters(); + } + + public Map<String, String> getQueryparameters(OpenApi openApi, int requestNumber) { + return this.getParameters(openApi, requestNumber).stream().filter(param -> param.getIn().equals("query")) + .collect(Collectors.toMap(OpenApiPathItemParameter::getName, e -> e.getSchema().getDefaultValue())); + } + + public Map<String, String> getPathparameters(OpenApi openApi, int requestNumber) { + return this.getParameters(openApi, requestNumber).stream().filter(param -> param.getIn().equals("path")) + .collect(Collectors.toMap(OpenApiPathItemParameter::getName, e -> e.getSchema().getDefaultValue())); + } + + public Map<String, String> getHeaderyparameters(OpenApi openApi, int requestNumber) { + return this.getParameters(openApi, requestNumber).stream().filter(param -> param.getIn().equals("header")) + .collect(Collectors.toMap(OpenApiPathItemParameter::getName, e -> e.getSchema().getDefaultValue())); + } + + public Map<String, String> getCookieparameters(OpenApi openApi, int requestNumber) { + return this.getParameters(openApi, requestNumber).stream().filter(param -> param.getIn().equals("cookie")) + .collect(Collectors.toMap(OpenApiPathItemParameter::getName, e -> e.getSchema().getDefaultValue())); + } + + public List<String> getPathToids(OpenApi openApi) { + String path = (String) openApi.getPaths().keySet().toArray()[0]; + String pathsToIds = openApi.getPaths().get(path).get("get").getResponses().get("200").getContent() + .get("application/json").getExample(); + return Arrays.stream(pathsToIds.split("#")).collect(Collectors.toList()); + } + + public List<String> getDatasetList(OpenApi openApi) + throws URISyntaxException, JSONException, JsonProcessingException { + String url = this.getUrlById(openApi, 0); + Map<String, String> headers = this.getHeaderyparameters(openApi, 0); + Object body = this.getJsonUtil.httpRequest(url, headers).getBody(); + ObjectMapper mapper = new ObjectMapper(); + JSONObject json = new JSONObject(mapper.writeValueAsString(body)); + List<String> pathToids = this.getPathToids(openApi); + List<String> idList = new ArrayList<String>(); + this.getJsonUtil.getListFromJson(json, pathToids, idList); + + return idList; + } + + public OpenApi getOpenApiByUUDI(String uui) { + return openApiDaoRepository.findByUUID(uui); + } -public class OpenApiServiceImpl implements OpenApiService { - - @Autowired - OpenApiDaoRepository openApiDaoRepository; - - @Override - public List<OpenApi> getAllOpenApi() { - return openApiDaoRepository.findAll(); - } - - - @Override - public List<String> getAllUrl() { - List<OpenApi> listOfOpenApis = this.getAllOpenApi(); - List<String> allUrl = new ArrayList<>(); - if (!listOfOpenApis.isEmpty()) { - for (OpenApi api : listOfOpenApis) { - String servers = api.getServers().get(0).getUrl(); - String path = (String) api.getPaths().keySet().toArray()[0]; - String listOfParams = "?"; - for (Map.Entry<String, Map<String, OpenApiPathItem>> apiPathItem: api.getPaths().entrySet()) { - OpenApiPathItem mapOpenApiPathItem = (OpenApiPathItem) apiPathItem.getValue().values().toArray()[0]; - List<OpenApiPathItemParameter> paramsList= mapOpenApiPathItem.getParameters(); - for (OpenApiPathItemParameter parameter : paramsList) { - listOfParams = listOfParams.concat(parameter.getName() + "=" + parameter.getSchema().getDefaultValue() + "&"); - } - } - listOfParams = StringUtils.substring(listOfParams, 0, listOfParams.length() - 1); - allUrl.add(servers + path + listOfParams); - } - - } - return allUrl; - } - - @Override - public OpenApi getOpenApiByUUDI(String uui) { - return openApiDaoRepository.findByUUID(uui); - } - - @Override - public List<OpenApi> getOpenApiByTitle(String titleDescription) { - return openApiDaoRepository.findByTitleDescription(titleDescription); - } - - @Override - public void createOpenApi(OpenApi openApi) { - openApiDaoRepository.save(openApi); - } + public List<OpenApi> getOpenApiByTitle(String titleDescription) { + return openApiDaoRepository.findByTitleDescription(titleDescription); + } + public void createOpenApi(OpenApi openApi) { + openApiDaoRepository.save(openApi); + } } diff --git a/src/main/java/com/smartharvester/util/SmartHarvesterGetJsonUtil.java b/src/main/java/com/smartharvester/util/SmartHarvesterGetJsonUtil.java new file mode 100644 index 0000000..401fad3 --- /dev/null +++ b/src/main/java/com/smartharvester/util/SmartHarvesterGetJsonUtil.java @@ -0,0 +1,103 @@ +package com.smartharvester.util; + +import java.io.BufferedInputStream; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.List; +import java.util.Map; + +import javax.net.ssl.HttpsURLConnection; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; + +@Component +public class SmartHarvesterGetJsonUtil { + + public JSONObject getJson(String urlRepo) throws JSONException { + URL url = null; + HttpURLConnection urlConnection = null; + String result = null; + + try { + url = new URL(urlRepo); + urlConnection = (HttpsURLConnection) url.openConnection(); + InputStream input = new BufferedInputStream(urlConnection.getInputStream()); + result = this.readStream(input); + } catch (MalformedURLException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } finally { + if (urlConnection != null) + urlConnection.disconnect(); + } + + JSONObject json = null; + + json = new JSONObject(result); + + return json; + } + + private String readStream(InputStream input) throws IOException { + StringBuilder sb = new StringBuilder(); + BufferedReader reader = new BufferedReader(new InputStreamReader(input), 1000); + for (String line = reader.readLine(); line != null; line = reader.readLine()) { + sb.append(line); + } + input.close(); + return sb.toString(); + } + + public ResponseEntity<Object> httpRequest(String url, Map<String, String> headers) throws URISyntaxException { + RestTemplate restTemplate = new RestTemplate(); + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.setContentType(MediaType.APPLICATION_JSON); + ResponseEntity<Object> response = restTemplate.getForEntity(url, Object.class, httpHeaders); + return response; + } + + public void getListFromJson(JSONObject json, List<String> idPath, List<String> idList) throws JSONException { + if (idPath.size() == 0) { + return; + } + + String currentpath = idPath.get(0); + + if (!json.has(currentpath) || json.opt(currentpath) == null) { + return; + } + + if (json.optJSONObject(currentpath) != null) { + idPath.remove(0); + this.getListFromJson(json.getJSONObject(currentpath), idPath, idList); + + } else if (json.optJSONArray(currentpath) != null ) { + JSONArray jsonArray = json.getJSONArray(currentpath); + idPath.remove(0); + for (int i = 0; i < jsonArray.length(); i++) { + + this.getListFromJson(jsonArray.getJSONObject(i), idPath, idList); + + } + } else { + idList.add(json.getString(currentpath)); + } + + } + +} -- GitLab