Smoke tests

CHANGELOG.md

@@ -17,6 +17,8 @@
   - [Deploy] Production configuration for peerage to use DNS discovery - [#29](https://github.com/Accenture/reactive-interaction-gateway/pull/29)
   - [Rig] Module for auto-discovery, using `Peerage` library - [#29](https://github.com/Accenture/reactive-interaction-gateway/pull/29)
   - [Deploy] Kubernetes deployment configuration file - [#29](https://github.com/Accenture/reactive-interaction-gateway/pull/29)
+  - [Misc] Smoke tests setup and test cases for API Proxy and Kafka + Phoenix messaging - [#42](https://github.com/Accenture/reactive-interaction-gateway/pull/42)
+  - [Outbound] Kafka consumer ready check utility function - [#42](https://github.com/Accenture/reactive-interaction-gateway/pull/42)
 
 - Fixed
   - [Inbound] Make presence channel respect `JWT_USER_FIELD` setting (currently hardcoded to "username")

CONTRIBUTING.md

@@ -22,9 +22,12 @@ the community chimes in with helpful advice if you have questions.
 
 You can
 - generate documentation with `mix docs`,
-- run tests with `mix test` and see the test coverage with
+- run unit tests with `mix test` and see the test coverage with
   `mix coveralls.html`, which generates a file at `doc/excoveralls.html`,
 - run the linter with `mix credo --strict`.
+- run smoke tests with `docker-compose -f smoke_tests.docker-compose.yml up -d --build`
+  - to see smoke tests logs run `docker logs -f rig`
+  - to re-run smoke tests without re-creating entire environment run `docker-compose -f smoke_tests.docker-compose.yml up --no-deps --build rig`
 
 We follow the [standard GitHub workflow](https://guides.github.com/introduction/flow/).
 Before submitting a PR,

apps/rig_auth/mix.exs

@@ -40,7 +40,7 @@ defmodule RigAuth.Mixfile do
       {:rig, in_umbrella: true},
       {:confex, "~> 3.3"},
       {:cowboy, "~> 1.0"},
-      {:httpoison, "~> 0.13.0"},
+      {:httpoison, "~> 1.0.0"},
       {:joken, "~> 1.4"},
       {:phoenix, "~> 1.3.0"},
       {:plug, "~> 1.4"},

apps/rig_inbound_gateway/config/test.exs

@@ -32,7 +32,7 @@ config :rig, RigInboundGatewayWeb.Presence.Channel,
   privileged_roles: ["support"]
 
 config :rig, RigInboundGateway.Proxy,
-  config_file: "proxy/proxy.test.json"
+  config_file: {:system, "PROXY_CONFIG_FILE", "proxy/proxy.test.json"}
 
 config :rig, RigInboundGatewayWeb.Proxy.Controller,
   rig_proxy: RigInboundGateway.ProxyMock

apps/rig_inbound_gateway/mix.exs

@@ -49,7 +49,7 @@ defmodule RigInboundGateway.Mixfile do
       {:phoenix, "~> 1.3.0"},
       {:phoenix_pubsub, "~> 1.0"},
       {:cowboy, "~> 1.0"},
-      {:httpoison, "~> 0.13.0"},
+      {:httpoison, "~> 1.0.0"},
       {:bypass, "~> 0.8.1", only: :test},
       {:poison, "~> 2.0 or ~> 3.0"},
       {:timex, "~> 3.1.22"},

apps/rig_inbound_gateway/priv/proxy/proxy.smoke_test.json

@@ -0,0 +1,73 @@
+[
+  {
+    "id": "smoke-service",
+    "name": "smoke-service",
+    "auth_type": "none",
+    "auth": {
+      "use_header": false,
+      "header_name": "",
+      "use_query": false,
+      "query_name": ""
+    },
+    "versioned": false,
+    "version_data": {
+      "default": {
+        "endpoints": [
+          {
+            "id": "get",
+            "path": "/api",
+            "method": "GET",
+            "not_secured": true
+          },
+          {
+            "id": "post",
+            "path": "/api",
+            "method": "POST",
+            "not_secured": true
+          },
+          {
+            "id": "put",
+            "path": "/api",
+            "method": "PUT",
+            "not_secured": true
+          },
+          {
+            "id": "patch",
+            "path": "/api",
+            "method": "PATCH",
+            "not_secured": true
+          },
+          {
+            "id": "delete",
+            "path": "/api",
+            "method": "DELETE",
+            "not_secured": true
+          },
+          {
+            "id": "head",
+            "path": "/api",
+            "method": "HEAD",
+            "not_secured": true
+          },
+          {
+            "id": "options",
+            "path": "/api",
+            "method": "OPTIONS",
+            "not_secured": true
+          },
+          {
+            "id": "post-kafka-produce",
+            "path": "/kafka/produce",
+            "method": "POST",
+            "not_secured": true
+          }
+        ]
+      }
+    },
+    "proxy": {
+      "use_env": true,
+      "target_url": "API_HOST",
+      "port": 8000
+    }
+  }
+]

apps/rig_inbound_gateway/test/rig/api_proxy/router_test.exs

@@ -225,6 +225,75 @@ defmodule RigInboundGateway.ApiProxy.RouterTest do
     assert conn.resp_body =~ "{\"status\":\"ok\"}"
   end
 
+  @tag :smoke
+  test "GET request should be correctly proxied to external service" do
+    conn = call(Router, build_conn(:get, "/api"))
+    assert conn.status == 200
+    assert conn.resp_body =~ "{\"msg\":\"GET\"}"
+  end
+
+  @tag :smoke
+  test "POST request should be correctly proxied to external service" do
+    conn = call(Router, build_conn(:post, "/api"))
+    assert conn.status == 200
+    assert conn.resp_body =~ "{\"msg\":\"POST\"}"
+  end
+
+  @tag :smoke
+  test "PUT request should be correctly proxied to external service" do
+    conn = call(Router, build_conn(:put, "/api"))
+    assert conn.status == 200
+    assert conn.resp_body =~ "{\"msg\":\"PUT\"}"
+  end
+
+  @tag :smoke
+  test "PATCH request should be correctly proxied to external service" do
+    conn = call(Router, build_conn(:patch, "/api"))
+    assert conn.status == 200
+    assert conn.resp_body =~ "{\"msg\":\"PATCH\"}"
+  end
+
+  @tag :smoke
+  test "DELETE request should be correctly proxied to external service" do
+    conn = call(Router, build_conn(:delete, "/api"))
+    assert conn.status == 200
+    assert conn.resp_body =~ "{\"msg\":\"DELETE\"}"
+  end
+
+  @tag :smoke
+  test "HEAD request should be correctly proxied to external service" do
+    conn = call(Router, build_conn(:head, "/api"))
+    assert conn.status == 200
+    assert conn.resp_body =~ ""
+  end
+
+  @tag :smoke
+  test "OPTIONS request should be correctly proxied to external service" do
+    conn = call(Router, build_conn(:options, "/api"))
+    assert conn.status == 200
+    assert conn.resp_body =~ "{\"msg\":\"OPTIONS\"}"
+  end
+
+  @tag :smoke
+  test "POST request with file body should be correctly proxied to external service" do
+    upload = %Plug.Upload{
+      path: __DIR__ <> "/upload_example.txt",
+      filename: "upload_example.txt",
+      content_type: "plain/text",
+    }
+
+    conn = call(Router, build_conn(:post, "/api", %{"qqfile" => upload}))
+    assert conn.status == 200
+    assert conn.resp_body =~ "{\"msg\":\"POST FILE\"}"
+  end
+
+  @tag :smoke
+  test "GET request with queries should be correctly proxied to external service" do
+    conn = call(Router, build_conn(:get, "/api", %{"foo" => %{"bar" => "baz"}}))
+    assert conn.status == 200
+    assert conn.resp_body =~ "{\"msg\":\"GET QUERY\"}"
+  end
+
   defp construct_request_with_jwt(method, url, query \\ %{}) do
     jwt = generate_jwt()
     build_conn(method, url, query)

apps/rig_inbound_gateway/test/rig_web/presence/channel_test.exs

@@ -3,6 +3,15 @@ defmodule RigInboundGatewayWeb.Presence.ChannelTest do
   use ExUnit.Case, async: true
   import ExUnit.CaptureLog
   use RigInboundGatewayWeb.ChannelCase
+  use RigInboundGatewayWeb.ConnCase
+
+  setup do
+    conn =
+      build_conn()
+      |> put_req_header("content-type", "application/json")
+
+    {:ok, conn: conn}
+  end
 
   test "a user connecting to her own topic works" do
     assert {:ok, _response, sock} = subscribe_and_join_user(
@@ -52,4 +61,39 @@ defmodule RigInboundGatewayWeb.Presence.ChannelTest do
     end
     assert capture_log(fun) =~ "unauthorized"
   end
+
+  @tag :smoke
+  test "subscribed user should receive message", %{conn: conn} do
+    {:ok, _msg} = RigOutboundGateway.Kafka.GroupSubscriber.wait_for_consumer_ready("rig", 0)
+    assert {:ok, _response, sock} = subscribe_and_join_user(
+      "testuser",
+      [@customer_role],
+      "user:testuser"
+    )
+
+    body = ~s({"user":"testuser","foo":"bar"})
+    produce_kafka_message(conn, body)
+
+    expected_event = "message"
+    expected_payload = %{"user" => "testuser", "foo" => "bar"}
+    assert_broadcast ^expected_event, ^expected_payload, 3000
+
+    leave sock
+  end
+
+  @tag :smoke
+  test "not subscribed user shouldn't receive message", %{conn: conn} do
+    {:ok, _msg} = RigOutboundGateway.Kafka.GroupSubscriber.wait_for_consumer_ready("rig", 0)
+    body = ~s({"user":"testuser","foo":"bar"})
+    produce_kafka_message(conn, body)
+
+    expected_event = "message"
+    expected_payload = %{"user" => "testuser", "foo" => "bar"}
+    refute_broadcast ^expected_event, ^expected_payload, 2000
+  end
+
+  defp produce_kafka_message(conn, body) do
+    conn = post conn, "/kafka/produce", body
+    assert conn.status == 200
+  end
 end

apps/rig_inbound_gateway/test/test_helper.exs

@@ -1,2 +1,4 @@
 ExUnit.start()
+# Exclude all smoke tests from running by default
+ExUnit.configure exclude: [smoke: true]
 Application.ensure_all_started(:bypass)

apps/rig_outbound_gateway/lib/rig_outbound_gateway/kafka/group_subscriber.ex

@@ -99,13 +99,39 @@ defmodule RigOutboundGateway.Kafka.GroupSubscriber do
 
     0..(n_partitions - 1)
     |> Enum.reduce(%{}, fn partition, acc ->
-         handler_pid = spawn_for_partition.(partition)
-         Map.put(acc, "#{topic}-#{partition}", handler_pid)
-       end)
+      handler_pid = spawn_for_partition.(partition)
+      Map.put(acc, "#{topic}-#{partition}", handler_pid)
+    end)
     |> Map.merge(spawn_message_handlers(brod_client_id, remaining_topics))
   end
 
   defp spawn_message_handlers(_brod_client_id, []) do
     %{}
   end
+
+  @doc """
+  Periodically checks if Kafka consumer is ready for given topic and partition.
+  If it exceedes maximum timeout returns :error, otherwise stops and returns :ok as
+  soon as possible.
+  """
+  @spec wait_for_consumer_ready(String.t(), non_neg_integer, non_neg_integer) ::
+          {:ok | :error, String.t()}
+  def wait_for_consumer_ready(topic, partition, timeout \\ 10_000)
+
+  def wait_for_consumer_ready(topic, partition, timeout) when timeout <= 0 do
+    {:error, "Consumer ready check for topic: #{topic} and partition: #{partition} timeouted."}
+  end
+
+  def wait_for_consumer_ready(topic, partition, timeout) do
+    conf = config()
+
+    case :brod.get_consumer(conf.brod_client_id, topic, partition) do
+      {:ok, _pid} ->
+        {:ok, "Consumer for topic: #{topic} and partition: #{partition} is ready."}
+
+      _ ->
+        :timer.sleep(1_000)
+        wait_for_consumer_ready(topic, partition, timeout - 1_000)
+    end
+  end
 end

apps/rig_outbound_gateway/test/kafka/group_subscriber_test.exs

@@ -0,0 +1,29 @@
+defmodule RigOutboundGateway.Kafka.GroupSubscriberTest do
+  @moduledoc false
+  use ExUnit.Case, async: true
+  alias RigOutboundGateway.Kafka.GroupSubscriber
+
+  @tag :smoke
+  test "kafka consumer connection should be correctly established" do
+    assert GroupSubscriber.wait_for_consumer_ready("rig", 0) == {
+      :ok,
+      "Consumer for topic: rig and partition: 0 is ready."
+    }
+  end
+
+  @tag :smoke
+  test "kafka consumer connection should fail with wrong topic" do
+    assert GroupSubscriber.wait_for_consumer_ready("bad", 0, 5000) == {
+      :error,
+      "Consumer ready check for topic: bad and partition: 0 timeouted."
+    }
+  end
+
+  @tag :smoke
+  test "kafka consumer connection should fail with wrong partition" do
+    assert GroupSubscriber.wait_for_consumer_ready("rig", 1, 5000) == {
+      :error,
+      "Consumer ready check for topic: rig and partition: 1 timeouted."
+    }
+  end
+end

apps/rig_outbound_gateway/test/test_helper.exs

@@ -1 +1,3 @@
 ExUnit.start()
+# Exclude all smoke tests from running by default
+ExUnit.configure exclude: [smoke: true]

examples/channels-example/kafka.docker-compose.yml

@@ -8,7 +8,7 @@ services:
 
   kafka:
     container_name: kafka
-    image: wurstmeister/kafka:0.10.0.1-2
+    image: wurstmeister/kafka:1.0.0
     environment:
       - KAFKA_ADVERTISED_HOST_NAME=localhost
       - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
@@ -19,4 +19,4 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     depends_on:
-      - zookeeper
+      - zookeeper

mix.lock

@@ -20,8 +20,8 @@
   "file_system": {:hex, :file_system, "0.2.2", "7f1e9de4746f4eb8a4ca8f2fbab582d84a4e40fa394cce7bfcb068b988625b06", [], [], "hexpm"},
   "fs": {:hex, :fs, "0.9.2", "ed17036c26c3f70ac49781ed9220a50c36775c6ca2cf8182d123b6566e49ec59", [:rebar], [], "hexpm"},
   "gettext": {:hex, :gettext, "0.13.1", "5e0daf4e7636d771c4c71ad5f3f53ba09a9ae5c250e1ab9c42ba9edccc476263", [:mix], [], "hexpm"},
-  "hackney": {:hex, :hackney, "1.10.1", "c38d0ca52ea80254936a32c45bb7eb414e7a96a521b4ce76d00a69753b157f21", [:rebar3], [{:certifi, "2.0.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "5.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},
-  "httpoison": {:hex, :httpoison, "0.13.0", "bfaf44d9f133a6599886720f3937a7699466d23bb0cd7a88b6ba011f53c6f562", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"},
+  "hackney": {:hex, :hackney, "1.11.0", "4951ee019df102492dabba66a09e305f61919a8a183a7860236c0fde586134b6", [:rebar3], [{:certifi, "2.0.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "5.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},
+  "httpoison": {:hex, :httpoison, "1.0.0", "1f02f827148d945d40b24f0b0a89afe40bfe037171a6cf70f2486976d86921cd", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"},
   "idna": {:hex, :idna, "5.1.0", "d72b4effeb324ad5da3cab1767cb16b17939004e789d8c0ad5b70f3cea20c89a", [:rebar3], [{:unicode_util_compat, "0.3.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"},
   "joken": {:hex, :joken, "1.5.0", "42a0953e80bd933fc98a0874e156771f78bf0e92abe6c3a9c22feb6da28efb0b", [:mix], [{:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}, {:poison, "~> 1.5 or ~> 2.0 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"},
   "jose": {:hex, :jose, "1.8.4", "7946d1e5c03a76ac9ef42a6e6a20001d35987afd68c2107bcd8f01a84e75aa73", [:mix, :rebar3], [{:base64url, "~> 0.0.1", [hex: :base64url, repo: "hexpm", optional: false]}], "hexpm"},