use toxiproxy for network failure testing

Author: bvogelzang

.travis.yml

@@ -14,9 +14,9 @@ rvm:
   - 2.7.0
 before_install:
   - docker info
+  - docker-compose up -d
   - sudo ./test/bin/install-openssl.sh
   - sudo ./test/bin/install-freetds.sh
-  - sudo ./test/bin/setup.sh
 install:
   - gem install bundler
   - bundle --version

README.md

@@ -423,17 +423,20 @@ First, clone the repo using the command line or your Git GUI of choice.
 $ git clone [email protected]:rails-sqlserver/tiny_tds.git
 ```
 
-After that, the quickest way to get setup for development is to use [Docker](https://www.docker.com/). Assuming you have [downloaded docker](https://www.docker.com/products/docker) for your platform and you have , you can run our test setup script.
+After that, the quickest way to get setup for development is to use [Docker](https://www.docker.com/). Assuming you have [downloaded docker](https://www.docker.com/products/docker) for your platform, you can use [docker-compose](https://docs.docker.com/compose/install/) to run the necessary containers for testing.
 
 ```shell
-$ ./test/bin/setup.sh
+$ docker-compose up -d
 ```
 
-This will download our SQL Server for Linux Docker image based from [microsoft/mssql-server-linux/](https://hub.docker.com/r/microsoft/mssql-server-linux/). Our image already has the `[tinytdstest]` DB and `tinytds` users created. Basically, it does the following.
+This will download our SQL Server for Linux Docker image based from [microsoft/mssql-server-linux/](https://hub.docker.com/r/microsoft/mssql-server-linux/). Our image already has the `[tinytdstest]` DB and `tinytds` users created. This will also download a [toxiproxy](https://github.com/shopify/toxiproxy) Docker image which we can use to simulate network failures for tests. Basically, it does the following.
 
 ```shell
+$ docker network create main-network
 $ docker pull metaskills/mssql-server-linux-tinytds
-$ docker run -p 1433:1433 -d metaskills/mssql-server-linux-tinytds
+$ docker run -p 1433:1433 -d --name sqlserver --network main-network metaskills/mssql-server-linux-tinytds
+$ docker pull shopify/toxiproxy
+$ docker run -p 8474:8474 -p 1234:1234 -d --name toxiproxy --network main-network shopify/toxiproxy
 ```
 
 If you are using your own database. Make sure to run these SQL commands as SA to get the test database and user installed.

docker-compose.yml

@@ -0,0 +1,22 @@
+version: '3'
+
+networks:
+  main-network:
+
+services:
+  mssql:
+    image: metaskills/mssql-server-linux-tinytds:2017-GA
+    container_name: sqlserver
+    ports:
+      - "1433:1433"
+    networks:
+      - main-network
+
+  toxiproxy:
+    image: shopify/toxiproxy
+    container_name: toxiproxy
+    ports:
+      - "8474:8474"
+      - "1234:1234"
+    networks:
+      - main-network

test/client_test.rb

@@ -68,6 +68,9 @@ class ClientTest < TinyTds::TestCase
   end
 
   describe 'With in-valid options' do
+    before(:all) do
+      init_toxiproxy
+    end
 
     it 'raises an argument error when no :host given and :dataserver is blank' do
       assert_raises(ArgumentError) { new_connection :dataserver => nil, :host => nil }
@@ -129,27 +132,40 @@ class ClientTest < TinyTds::TestCase
       end
     end
 
-    it 'raises TinyTds exception with sql batch timeout due to network failure' do
+    it 'raises TinyTds exception with tcp socket network failure' do
       skip if ENV['CI'] && ENV['APPVEYOR_BUILD_FOLDER'] # only CI using docker
       begin
-        client = new_connection timeout: 2
+        client = new_connection timeout: 1, port: 1234
         assert_client_works(client)
-        docker_container('pause', wait_for: 1)
-        action = lambda { client.execute('SELECT 1 as [one]').each }
-        assert_raise_tinytds_error(action) do |e|
-          assert_equal 20003, e.db_error_number
-          assert_equal 6, e.severity
-          assert_match %r{timed out}i, e.message, 'ignore if non-english test run'
+        action = lambda { client.execute("waitfor delay '00:00:02'").do }
+        # Delay the TCP socket from closing until delay has elapsed.
+        Toxiproxy[:sqlserver_test].toxic(:slow_close, delay: 500).apply do
+          assert_raise_tinytds_error(action) do |e|
+            assert_equal 20003, e.db_error_number
+            assert_equal 6, e.severity
+            assert_match %r{timed out}i, e.message, 'ignore if non-english test run'
+          end
         end
       ensure
-        docker_container('unpause', wait_for: 1)
-        action = lambda { client.execute('SELECT 1 as [one]').each }
-        assert_raise_tinytds_error(action) do |e|
-          assert_equal 20047, e.db_error_number
-          assert_includes [1,9], e.severity
-          assert_match %r{dead or not enabled}i, e.message, 'ignore if non-english test run'
+        assert_new_connections_work
+      end
+    end
+
+    it 'raises TinyTds exception with dead connection network failure' do
+      skip if ENV['CI'] && ENV['APPVEYOR_BUILD_FOLDER'] # only CI using docker
+      begin
+        client = new_connection timeout: 1, port: 1234
+        assert_client_works(client)
+        action = lambda { client.execute("waitfor delay '00:00:02'").do }
+        # Stops all data from getting through, and closes the connection after timeout
+        Toxiproxy[:sqlserver_test].toxic(:timeout, timeout: 500).apply do
+          assert_raise_tinytds_error(action) do |e|
+            assert_equal 20047, e.db_error_number
+            assert_includes [1,9], e.severity
+            assert_match %r{dead or not enabled}i, e.message, 'ignore if non-english test run'
+          end
         end
-        close_client(client)
+      ensure
         assert_new_connections_work
       end
     end

test/test_helper.rb

@@ -2,6 +2,7 @@
 require 'bundler' ; Bundler.require :development, :test
 require 'tiny_tds'
 require 'minitest/autorun'
+require 'toxiproxy'
 
 TINYTDS_SCHEMAS = ['sqlserver_2000', 'sqlserver_2005', 'sqlserver_2008', 'sqlserver_2014', 'sqlserver_azure', 'sybase_ase'].freeze
 
@@ -212,9 +213,25 @@ def rollback_transaction(client)
       client.execute("ROLLBACK TRANSACTION").do
     end
 
-    def docker_container(cmd, wait_for: 0)
-      system("docker #{cmd} $(docker ps --format '{{.Names}}' --filter 'ancestor=metaskills/mssql-server-linux-tinytds:2017-GA') > /dev/null")
-      sleep(wait_for) if wait_for > 0
+    def init_toxiproxy
+      return if ENV['APPVEYOR_BUILD_FOLDER'] # only for CI using docker
+
+      # In order for toxiproxy to work for local docker instances of mssql, the containers must be on the same network
+      # and the host used below must match the mssql container name so toxiproxy knows where to proxy to.
+      # localhost from the perspective of toxiproxy's container is its own container an *not* the mssql container it needs to proxy to.
+      # docker-compose.yml handles this automatically for us. In instances where someone is using their own local mssql container they'll
+      # need to set up the networks manually and set TINYTDS_UNIT_HOST to their mssql container name
+      # For anything other than localhost just use the environment config
+      env_host = ENV['TINYTDS_UNIT_HOST_TEST'] || ENV['TINYTDS_UNIT_HOST'] || 'localhost'
+      host = ['localhost', '127.0.0.1', '0.0.0.0'].include?(env_host) ? 'sqlserver' : env_host
+      port = ENV['TINYTDS_UNIT_PORT'] || 1433
+      Toxiproxy.populate([
+        {
+          name: "sqlserver_test",
+          listen: "0.0.0.0:1234",
+          upstream: "#{host}:#{port}"
+        }
+      ])
     end
   end
 end

tiny_tds.gemspec

@@ -26,4 +26,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'rake-compiler-dock', '~> 1.0'
   s.add_development_dependency 'minitest', '~> 5.6'
   s.add_development_dependency 'connection_pool', '~> 2.2'
+  s.add_development_dependency 'toxiproxy'
 end