From afbaf000d78ad05aa9bf9ad3df820174f1e0fa25 Mon Sep 17 00:00:00 2001
From: Baptiste Toulemonde <toulemonde@cines.fr>
Date: Fri, 28 Jan 2022 12:08:26 +0100
Subject: [PATCH] add methods to get urls and ids

---
 pom.xml                                       |   4 +
 .../controller/SmartHarvesterController.java  |  55 +++--
 .../service/MappingService.java               |  41 +---
 .../service/OpenApiServiceImpl.java           | 194 +++++++++++++-----
 .../util/SmartHarvesterGetJsonUtil.java       | 103 ++++++++++
 5 files changed, 294 insertions(+), 103 deletions(-)
 create mode 100644 src/main/java/com/smartharvester/util/SmartHarvesterGetJsonUtil.java

diff --git a/pom.xml b/pom.xml
index 1568ab7..6e248e3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -119,6 +119,10 @@
 			<groupId>org.springframework.boot</groupId>
 			<artifactId>spring-boot-starter-oauth2-client</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-devtools</artifactId>
+		</dependency>
 	</dependencies>
 
 	<build>
diff --git a/src/main/java/com/smartharvester/controller/SmartHarvesterController.java b/src/main/java/com/smartharvester/controller/SmartHarvesterController.java
index 65d1e32..9bca4eb 100644
--- a/src/main/java/com/smartharvester/controller/SmartHarvesterController.java
+++ b/src/main/java/com/smartharvester/controller/SmartHarvesterController.java
@@ -1,16 +1,11 @@
 package com.smartharvester.controller;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.mongodb.client.FindIterable;
-import com.mongodb.client.MongoCollection;
-import com.mongodb.client.MongoDatabase;
-import com.mongodb.client.MongoIterable;
+import com.fasterxml.jackson.databind.JsonMappingException;
 import com.smartharvester.dao.OpenApiDaoRepository;
 import com.smartharvester.exception.ResourceNotFoundException;
 import com.smartharvester.model.openapi.OpenApi;
-import com.smartharvester.service.OpenApiService;
-import io.swagger.annotations.Api;
+import com.smartharvester.service.OpenApiServiceImpl;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import io.swagger.v3.oas.models.Operation;
 import io.swagger.v3.parser.OpenAPIV3Parser;
@@ -19,22 +14,21 @@ import org.apache.http.HttpHost;
 import org.apache.http.entity.ContentType;
 import org.apache.http.nio.entity.NStringEntity;
 import org.apache.http.util.EntityUtils;
-import org.bson.Document;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.json.JSONArray;
+import org.json.JSONException;
 import org.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.data.mongodb.core.MongoTemplate;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
-
 import java.io.*;
 import java.net.HttpURLConnection;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.*;
 
@@ -54,8 +48,7 @@ public class SmartHarvesterController {
     private OpenApiDaoRepository repository;
 
     @Autowired
-    @Qualifier(value = "openApiService")
-    OpenApiService openApiService;
+    OpenApiServiceImpl openApiService;
 
     private final Logger logger = LoggerFactory.getLogger(this.getClass());
 
@@ -66,20 +59,44 @@ public class SmartHarvesterController {
     @RequestMapping(value = "/all", method = RequestMethod.GET)
     public Collection<OpenApi> getAll() {
         logger.debug("Getting all OpenApi from database...");
-        List<OpenApi> response = openApiService.getAllOpenApi();
+        List<OpenApi> response = this.openApiService.getAllOpenApi();
         return response;
     }
 
     /**
-     * Method to fetch all openAPIs from mongodb.
+     * Method to fetch url by uuid
      * @return
      */
-    @RequestMapping(value = "/allurls", method = RequestMethod.GET)
-    public Collection<String> getAllUrl() {
-        logger.debug("Getting all OpenApi from database...");
-        Collection<String> response = openApiService.getAllUrl();
+    @GetMapping("/url/{uuid}")
+    public  ResponseEntity<String>  getSearchUrl(@PathVariable ("uuid") String uuid) {
+        OpenApi openApi = this.openApiService.getOpenApiByUUDI(uuid);
+        String response = this.openApiService.getUrlById(openApi, 0);
 
-        return response;
+        return ResponseEntity.ok().body(response);
+    }
+    
+    @GetMapping("/ids/{uuid}")
+    public ResponseEntity<?> getIdList(@PathVariable ("uuid") String uuid)  {
+        OpenApi openApi = this.openApiService.getOpenApiByUUDI(uuid);
+        try {
+			List<String> datasetList = this.openApiService.getDatasetList(openApi);
+			return ResponseEntity.ok().body(datasetList);
+		} catch (URISyntaxException | JSONException | JsonProcessingException e) {
+			return ResponseEntity.badRequest().body(e.getMessage());
+		}
+    }
+    
+    @GetMapping("/urls/{uuid}")
+    public  ResponseEntity<?> getDatasetUrls(@PathVariable ("uuid") String uuid) {
+    	OpenApi openApi = this.openApiService.getOpenApiByUUDI(uuid);
+    	List<String> datasetList;
+		try {
+			List<String> datasetUrl = this.openApiService.getDatasetUrl(openApi, 1);
+			return ResponseEntity.ok().body(datasetUrl);
+		} catch (JsonProcessingException | URISyntaxException | JSONException e) {
+			return ResponseEntity.badRequest().body(e.getMessage());
+		}
+    	
     }
 
     /**
diff --git a/src/main/java/com/smartharvester/service/MappingService.java b/src/main/java/com/smartharvester/service/MappingService.java
index 36f3eb3..cc3f10e 100644
--- a/src/main/java/com/smartharvester/service/MappingService.java
+++ b/src/main/java/com/smartharvester/service/MappingService.java
@@ -33,12 +33,15 @@ import org.springframework.web.client.RequestCallback;
 import org.springframework.web.client.RestTemplate;
 import com.smartharvester.model.mapping.Path;
 import com.smartharvester.model.mapping.request.PublishedRequest;
+import com.smartharvester.util.SmartHarvesterGetJsonUtil;
 
 @Service
 public class MappingService {
 
 	@Autowired
 	private RestTemplate restTemplate;
+	
+	private SmartHarvesterGetJsonUtil getJsonUtil;
 
 	public String getDatasetString(String catId, String properties, String fdpUrl) {
 		String dataset = "@prefix dcat: <http://www.w3.org/ns/dcat#>.\n" + "@prefix dct: <http://purl.org/dc/terms/>.\n"
@@ -67,31 +70,7 @@ public class MappingService {
 		return distribution;
 	}
 
-	public static JSONObject getJson(String urlRepo) throws JSONException {
-		URL url = null;
-		HttpURLConnection urlConnection = null;
-		String result = null;
-
-		try {
-			url = new URL(urlRepo);
-			urlConnection = (HttpsURLConnection) url.openConnection();
-			InputStream input = new BufferedInputStream(urlConnection.getInputStream());
-			result = readStream(input);
-		} catch (MalformedURLException e) {
-			e.printStackTrace();
-		} catch (IOException e) {
-			e.printStackTrace();
-		} finally {
-			if (urlConnection != null)
-				urlConnection.disconnect();
-		}
-
-		JSONObject json = null;
-
-		json = new JSONObject(result);
-
-		return json;
-	}
+	
 
 	public static String write(String indentifier, String value) {
 		value = value.replaceAll("[\\r\\n]+", " ");
@@ -132,7 +111,7 @@ public class MappingService {
 		JSONObject json = null;
 
 		try {
-			json = getJson(urlRepo);
+			json = this.getJsonUtil.getJson(urlRepo);
 		} catch (JSONException e1) {
 			e1.printStackTrace();
 		}
@@ -288,15 +267,7 @@ public class MappingService {
 
 	}
 
-	private static String readStream(InputStream input) throws IOException {
-		StringBuilder sb = new StringBuilder();
-		BufferedReader reader = new BufferedReader(new InputStreamReader(input), 1000);
-		for (String line = reader.readLine(); line != null; line = reader.readLine()) {
-			sb.append(line);
-		}
-		input.close();
-		return sb.toString();
-	}
+	
 
 	public ResponseEntity<String> asyncPostToFdp(String path, String fdpUrl, String dataset, String fdpToken)
 			throws BadRequest {
diff --git a/src/main/java/com/smartharvester/service/OpenApiServiceImpl.java b/src/main/java/com/smartharvester/service/OpenApiServiceImpl.java
index c921ed2..959419e 100644
--- a/src/main/java/com/smartharvester/service/OpenApiServiceImpl.java
+++ b/src/main/java/com/smartharvester/service/OpenApiServiceImpl.java
@@ -1,63 +1,159 @@
 package com.smartharvester.service;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.smartharvester.dao.OpenApiDaoRepository;
 import com.smartharvester.model.openapi.OpenApi;
 import com.smartharvester.model.openapi.OpenApiPathItem;
 import com.smartharvester.model.openapi.OpenApiPathItemParameter;
+import com.smartharvester.model.openapi.OpenApiResponseContent;
+import com.smartharvester.util.SmartHarvesterGetJsonUtil;
+
 import org.apache.commons.lang3.StringUtils;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
 
+import java.net.URISyntaxException;
 import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+@Service
+public class OpenApiServiceImpl {
+
+	@Autowired
+	OpenApiDaoRepository openApiDaoRepository;
+
+	@Autowired
+	SmartHarvesterGetJsonUtil getJsonUtil;
+
+	public List<OpenApi> getAllOpenApi() {
+		return openApiDaoRepository.findAll();
+	}
+
+	private final Logger logger = LoggerFactory.getLogger(this.getClass());
+
+	public String getUrlById(OpenApi openApi, int requestNumber) {
+		String path = (String) openApi.getPaths().keySet().toArray()[0];
+		String servers = openApi.getServers().get(0).getUrl();
+		Map<String, String> queryparameters = this.getQueryparameters(openApi,0);
+		Map<String, String> pathparameters = this.getPathparameters(openApi, 0);
+		String listOfParams = "?";
+		String pathParam = "";
+		for (String param : queryparameters.keySet()) {
+			listOfParams = listOfParams.concat(param + "=" + queryparameters.get(param) + "&");
+		}
+		for (String pathvalue : pathparameters.values()) {
+			pathParam += "/" + pathvalue;
+		}
+
+		listOfParams = StringUtils.substring(listOfParams, 0, listOfParams.length() - 1);
+
+		return servers + path + pathParam + listOfParams;
+
+	}
+
+	public List<String> getDatasetUrl(OpenApi openApi, int requestNumber) throws JsonProcessingException, URISyntaxException, JSONException {
+		String path = (String) openApi.getPaths().keySet().toArray()[1];
+		String servers = openApi.getServers().get(0).getUrl();
+		List<String> ids = this.getDatasetList(openApi);
+		List<OpenApiPathItemParameter> parameters = this.getParameters(openApi, 1);
+		List<String> urls = new ArrayList<String>();
+		for (String id : ids) {
+			Map<String, String> mapQueryParameters = new HashMap<String, String>();
+			Map<String, String> mapPathParameters = new HashMap<String, String>();
+			for(OpenApiPathItemParameter param: parameters) {
+				if (param.getRequired()) {
+					if(param.getIn().equals("query")) {
+						mapQueryParameters.put(param.getName(), id);
+					} else if (param.getIn().equals("path")){
+						mapPathParameters.put(param.getName(), id);
+					}
+				} else {
+					if(param.getIn().equals("query")) {
+						mapQueryParameters.put(param.getName(), param.getSchema().getDefaultValue());
+					} else if (param.getIn().equals("path")){
+						mapPathParameters.put(param.getName(), param.getSchema().getDefaultValue());
+					}
+				}
+			}
+			String listOfParams = "?";
+			String pathParam = "";
+			for(Map.Entry<String, String> entry: mapQueryParameters.entrySet()) {
+				listOfParams = listOfParams.concat(entry.getKey() + "=" + entry.getValue() + "&");
+			}
+			for(Map.Entry<String, String> entry: mapPathParameters.entrySet()) {
+				pathParam += "/" + entry.getValue();;
+			}
+			
+			listOfParams = StringUtils.substring(listOfParams, 0, listOfParams.length() - 1);
+			urls.add(servers + path + pathParam + listOfParams);
+		}
+		
+		return urls;
+	}
+
+	public List<OpenApiPathItemParameter> getParameters(OpenApi openApi, int requestNumber) {
+		String path = (String) openApi.getPaths().keySet().toArray()[requestNumber];
+		Map<String, OpenApiPathItem> map = openApi.getPaths().get(path);
+		return map.get("get").getParameters();
+	}
+
+	public Map<String, String> getQueryparameters(OpenApi openApi, int requestNumber) {
+		return this.getParameters(openApi, requestNumber).stream().filter(param -> param.getIn().equals("query"))
+				.collect(Collectors.toMap(OpenApiPathItemParameter::getName, e -> e.getSchema().getDefaultValue()));
+	}
+
+	public Map<String, String> getPathparameters(OpenApi openApi, int requestNumber) {
+		return this.getParameters(openApi, requestNumber).stream().filter(param -> param.getIn().equals("path"))
+				.collect(Collectors.toMap(OpenApiPathItemParameter::getName, e -> e.getSchema().getDefaultValue()));
+	}
+
+	public Map<String, String> getHeaderyparameters(OpenApi openApi, int requestNumber) {
+		return this.getParameters(openApi, requestNumber).stream().filter(param -> param.getIn().equals("header"))
+				.collect(Collectors.toMap(OpenApiPathItemParameter::getName, e -> e.getSchema().getDefaultValue()));
+	}
+
+	public Map<String, String> getCookieparameters(OpenApi openApi, int requestNumber) {
+		return this.getParameters(openApi, requestNumber).stream().filter(param -> param.getIn().equals("cookie"))
+				.collect(Collectors.toMap(OpenApiPathItemParameter::getName, e -> e.getSchema().getDefaultValue()));
+	}
+
+	public List<String> getPathToids(OpenApi openApi) {
+		String path = (String) openApi.getPaths().keySet().toArray()[0];
+		String pathsToIds = openApi.getPaths().get(path).get("get").getResponses().get("200").getContent()
+				.get("application/json").getExample();
+		return Arrays.stream(pathsToIds.split("#")).collect(Collectors.toList());
+	}
+
+	public List<String> getDatasetList(OpenApi openApi)
+			throws URISyntaxException, JSONException, JsonProcessingException {
+		String url = this.getUrlById(openApi, 0);
+		Map<String, String> headers = this.getHeaderyparameters(openApi, 0);
+		Object body = this.getJsonUtil.httpRequest(url, headers).getBody();
+		ObjectMapper mapper = new ObjectMapper();
+		JSONObject json = new JSONObject(mapper.writeValueAsString(body));
+		List<String> pathToids = this.getPathToids(openApi);
+		List<String> idList = new ArrayList<String>();
+		this.getJsonUtil.getListFromJson(json, pathToids, idList);
+
+		return idList;
+	}
+
+	public OpenApi getOpenApiByUUDI(String uui) {
+		return openApiDaoRepository.findByUUID(uui);
+	}
 
-public class OpenApiServiceImpl implements OpenApiService {
-
-    @Autowired
-    OpenApiDaoRepository openApiDaoRepository;
-
-    @Override
-    public List<OpenApi> getAllOpenApi() {
-        return openApiDaoRepository.findAll();
-    }
-
-
-    @Override
-    public List<String> getAllUrl() {
-        List<OpenApi> listOfOpenApis = this.getAllOpenApi();
-        List<String> allUrl = new ArrayList<>();
-        if (!listOfOpenApis.isEmpty()) {
-            for (OpenApi api : listOfOpenApis) {
-                String servers = api.getServers().get(0).getUrl();
-                String path = (String) api.getPaths().keySet().toArray()[0];
-                String listOfParams = "?";
-                for (Map.Entry<String, Map<String, OpenApiPathItem>> apiPathItem: api.getPaths().entrySet()) {
-                    OpenApiPathItem mapOpenApiPathItem = (OpenApiPathItem) apiPathItem.getValue().values().toArray()[0];
-                    List<OpenApiPathItemParameter> paramsList= mapOpenApiPathItem.getParameters();
-                    for (OpenApiPathItemParameter parameter : paramsList) {
-                        listOfParams = listOfParams.concat(parameter.getName() + "=" + parameter.getSchema().getDefaultValue() + "&");
-                    }
-                }
-                listOfParams = StringUtils.substring(listOfParams, 0, listOfParams.length() - 1);
-                allUrl.add(servers + path + listOfParams);
-            }
-
-        }
-        return allUrl;
-    }
-
-    @Override
-    public OpenApi getOpenApiByUUDI(String uui) {
-        return openApiDaoRepository.findByUUID(uui);
-    }
-
-    @Override
-    public List<OpenApi> getOpenApiByTitle(String titleDescription) {
-        return openApiDaoRepository.findByTitleDescription(titleDescription);
-    }
-
-    @Override
-    public void createOpenApi(OpenApi openApi) {
-        openApiDaoRepository.save(openApi);
-    }
+	public List<OpenApi> getOpenApiByTitle(String titleDescription) {
+		return openApiDaoRepository.findByTitleDescription(titleDescription);
+	}
 
+	public void createOpenApi(OpenApi openApi) {
+		openApiDaoRepository.save(openApi);
+	}
 
 }
diff --git a/src/main/java/com/smartharvester/util/SmartHarvesterGetJsonUtil.java b/src/main/java/com/smartharvester/util/SmartHarvesterGetJsonUtil.java
new file mode 100644
index 0000000..401fad3
--- /dev/null
+++ b/src/main/java/com/smartharvester/util/SmartHarvesterGetJsonUtil.java
@@ -0,0 +1,103 @@
+package com.smartharvester.util;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.List;
+import java.util.Map;
+
+import javax.net.ssl.HttpsURLConnection;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Component;
+import org.springframework.web.client.RestTemplate;
+
+@Component
+public class SmartHarvesterGetJsonUtil {
+
+	public JSONObject getJson(String urlRepo) throws JSONException {
+		URL url = null;
+		HttpURLConnection urlConnection = null;
+		String result = null;
+
+		try {
+			url = new URL(urlRepo);
+			urlConnection = (HttpsURLConnection) url.openConnection();
+			InputStream input = new BufferedInputStream(urlConnection.getInputStream());
+			result = this.readStream(input);
+		} catch (MalformedURLException e) {
+			e.printStackTrace();
+		} catch (IOException e) {
+			e.printStackTrace();
+		} finally {
+			if (urlConnection != null)
+				urlConnection.disconnect();
+		}
+
+		JSONObject json = null;
+
+		json = new JSONObject(result);
+
+		return json;
+	}
+
+	private String readStream(InputStream input) throws IOException {
+		StringBuilder sb = new StringBuilder();
+		BufferedReader reader = new BufferedReader(new InputStreamReader(input), 1000);
+		for (String line = reader.readLine(); line != null; line = reader.readLine()) {
+			sb.append(line);
+		}
+		input.close();
+		return sb.toString();
+	}
+
+	public ResponseEntity<Object> httpRequest(String url, Map<String, String> headers) throws URISyntaxException {
+		RestTemplate restTemplate = new RestTemplate();
+		HttpHeaders httpHeaders = new HttpHeaders();
+		httpHeaders.setContentType(MediaType.APPLICATION_JSON);
+		ResponseEntity<Object> response = restTemplate.getForEntity(url, Object.class, httpHeaders);
+		return response;
+	}
+
+	public void getListFromJson(JSONObject json, List<String> idPath, List<String> idList) throws JSONException {
+		if (idPath.size() == 0) {
+			return;
+		}
+
+		String currentpath = idPath.get(0);
+
+		if (!json.has(currentpath) || json.opt(currentpath) == null) {
+			return;
+		}
+
+		 if (json.optJSONObject(currentpath) != null) {
+			idPath.remove(0);
+			this.getListFromJson(json.getJSONObject(currentpath), idPath, idList);
+			
+		} else if (json.optJSONArray(currentpath) != null ) {
+			JSONArray jsonArray = json.getJSONArray(currentpath);
+			idPath.remove(0);
+			for (int i = 0; i < jsonArray.length(); i++) {
+				
+					this.getListFromJson(jsonArray.getJSONObject(i), idPath, idList);
+				
+			}
+		} else {
+			idList.add(json.getString(currentpath)); 
+		}
+		
+	}
+
+}
-- 
GitLab