HTTP-150 Add Proxy to HttpClient for Lookup Source

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## [Unreleased]
 
+-   Added option to define a proxy for the lookup source
+
 -   Added support for generic json and URL query creator 
 
 -   Retries support for source table:

README.md

@@ -555,6 +555,7 @@ be requested if the current time is later than the cached token expiry time minu
 | gid.connector.http.source.lookup.retry-strategy.exponential-delay.initial-backoff    | optional | Exponential-delay initial delay. Default 1 second.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
 | gid.connector.http.source.lookup.retry-strategy.exponential-delay.max-backoff        | optional | Exponential-delay maximum delay. Default 1 minute. Use with `lookup.max-retries` parameter.                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
 | gid.connector.http.source.lookup.retry-strategy.exponential-delay.backoff-multiplier | optional | Exponential-delay multiplier. Default value 1.5                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
+| gid.connector.http.source.lookup.proxy                                               | optional | Specify a proxy that will be used for the lookup source. It has to be in format hostname:port (e.g., proxy:8080).                                                                                                                                                                                                                                                                                                                                                                                                                                                |
 | gid.connector.http.request.query-param-fields                                        | optional | Used for the `GenericJsonAndUrlQueryCreator` query creator. The names of the fields that will be mapped to query parameters. The parameters are separated by semicolons, such as `param1;param2`.                                                                                                                                                                                                                                                                                                                                                                |                                                                                                                                                                                                                                                                                                                                                               
 | gid.connector.http.request.body-fields                                               | optional | Used for the `GenericJsonAndUrlQueryCreator` query creator. The names of the fields that will be mapped to the body. The parameters are separated by semicolons, such as `param1;param2`.                                                                                                                                                                                                                                                                                                                                                                        |                                                                                                                                                                                                                                                                                                                                                                         |
 | gid.connector.http.request.url-map                                                   | optional | Used for the `GenericJsonAndUrlQueryCreator` query creator. The map of insert names to column names used as url segments. Parses a string as a map of strings. For example if there are table columns called `customerId` and `orderId`, then specifying value `customerId:cid1,orderID:oid` and a url of https://myendpoint/customers/{cid}/orders/{oid} will mean that the url used for the lookup query will dynamically pickup the values for `customerId`, `orderId` and use them in the url. The expected format of the map is: `key1:value1,key2:value2`. |

src/main/java/com/getindata/connectors/http/internal/config/HttpConnectorConfigConstants.java

@@ -88,6 +88,9 @@ public final class HttpConnectorConfigConstants {
     public static final String SOURCE_CONNECTION_TIMEOUT =
         SOURCE_LOOKUP_PREFIX + "connection.timeout";
 
+    public static final String SOURCE_PROXY =
+            SOURCE_LOOKUP_PREFIX + "proxy";
+
     public static final String SINK_HTTP_TIMEOUT_SECONDS =
         GID_CONNECTOR_HTTP + "sink.request.timeout";
 

src/main/java/com/getindata/connectors/http/internal/table/lookup/HttpLookupConnectorOptions.java

@@ -82,6 +82,12 @@ public class HttpLookupConnectorOptions {
                     .noDefaultValue()
                     .withDescription("Http client connection timeout.");
 
+    public static final ConfigOption<String> SOURCE_LOOKUP_PROXY =
+            ConfigOptions.key(SOURCE_PROXY)
+                    .stringType()
+                    .noDefaultValue()
+                    .withDescription("Http client proxy, separated by ':'.");
+
     public static final ConfigOption<String> SOURCE_LOOKUP_RETRY_STRATEGY =
             ConfigOptions.key(SOURCE_RETRY_STRATEGY_TYPE)
                     .stringType()

src/main/java/com/getindata/connectors/http/internal/table/lookup/HttpLookupTableSourceFactory.java

@@ -134,6 +134,7 @@ public Set<ConfigOption<?>> optionalOptions() {
             SOURCE_LOOKUP_HTTP_RETRY_CODES,
             SOURCE_LOOKUP_HTTP_IGNORED_RESPONSE_CODES,
 
+            SOURCE_LOOKUP_PROXY,
             SOURCE_LOOKUP_CONNECTION_TIMEOUT        // TODO: add request timeout from properties
         );
     }

src/main/java/com/getindata/connectors/http/internal/utils/JavaNetHttpClientFactory.java

@@ -1,5 +1,7 @@
 package com.getindata.connectors.http.internal.utils;
 
+import java.net.InetSocketAddress;
+import java.net.ProxySelector;
 import java.net.http.HttpClient;
 import java.net.http.HttpClient.Redirect;
 import java.security.NoSuchAlgorithmException;
@@ -45,6 +47,13 @@ public static HttpClient createClient(HttpLookupConfig options) {
                 .getOptional(HttpLookupConnectorOptions.SOURCE_LOOKUP_CONNECTION_TIMEOUT)
                 .ifPresent(clientBuilder::connectTimeout);
 
+        options.getReadableConfig()
+                .getOptional(HttpLookupConnectorOptions.SOURCE_LOOKUP_PROXY)
+                .ifPresent(proxy -> {
+                    ProxyConfig proxyConfig = new ProxyConfig(proxy);
+                    clientBuilder.proxy(ProxySelector.of(new InetSocketAddress(proxyConfig.getHost(), proxyConfig.getPort())));
+                });
+
         return clientBuilder.build();
     }
 

src/main/java/com/getindata/connectors/http/internal/utils/ProxyConfig.java

@@ -0,0 +1,19 @@
+package com.getindata.connectors.http.internal.utils;
+
+import lombok.Getter;
+
+@Getter
+public class ProxyConfig {
+
+    private final String host;
+
+    private final int port;
+
+    public ProxyConfig(String proxyString) {
+        String host = proxyString.substring(0, proxyString.lastIndexOf(':'));
+        int port = Integer.parseInt(proxyString.substring(proxyString.lastIndexOf(':') + 1));
+        this.host = host;
+        this.port = port;
+    }
+
+}

src/test/java/com/getindata/connectors/http/internal/utils/ConfigUtilsTest.java

@@ -49,6 +49,24 @@ public void shouldConvertNoProperty() {
         assertThat(mappedProperties).isEmpty();
     }
 
+    @Test
+    public void shouldSeparateProxyWithoutProtocol() {
+        String proxy = "proxy:8080";
+
+        ProxyConfig proxyConfig = new ProxyConfig(proxy);
+        assertThat(proxyConfig.getHost().equals("proxy"));
+        assertThat(proxyConfig.getPort() == 8080);
+    }
+
+    @Test
+    public void shouldSeparateProxyWithProtocol() {
+        String proxy = "https://proxy:80";
+
+        ProxyConfig proxyConfig = new ProxyConfig(proxy);
+        assertThat(proxyConfig.getHost().equals("https://proxy"));
+        assertThat(proxyConfig.getPort() == 80);
+    }
+
     @Test
     public void shouldHandleInvalidPropertyType() {