HA connection, handling failover with sentinel

lib/redic/client.rb

@@ -13,7 +13,12 @@ def initialize(url, timeout)
     end
 
     def read
-      @connection.read
+      ret = @connection.read
+      if ret.is_a? RuntimeError and ret.to_s.start_with? "READONLY"
+        raise ret 
+      else
+        return ret
+      end
     end
 
     def write(command)

lib/redic_ha.rb

@@ -0,0 +1,81 @@
+require_relative "redic/client"
+require_relative "redic"
+
+class RedicHA
+  attr :sentinel_url
+  attr :master_name
+  attr :client
+  attr :timeout
+  attr :db
+  attr :queue
+
+  def initialize(master_name, sentinel_url = "redis://127.0.0.1:26379", db = 0, timeout = 10_000_000)
+    @sentinel_url = sentinel_url
+    @master_name = master_name
+    @timeout = timeout
+    @db = db 
+    @client = Redic::Client.new(get_master_url, timeout)
+    @queue = []
+  end
+
+  def call(*args)
+    exec_with_retry do
+      @client.connect do
+        @client.write(args)
+        @client.read
+      end
+    end
+  end
+
+  def queue(*args)
+    @queue << args
+  end
+
+  def commit
+    exec_with_retry do 
+      @client.connect do
+        @queue.each do |args|
+          @client.write(args)
+        end
+
+        @queue.map do
+          @client.read
+        end
+      end
+    end
+    ensure
+      @queue.clear
+  end
+
+  def timeout
+    @client.timeout
+  end
+
+  def url
+    @sentinel_url
+  end
+
+  private
+
+    def get_master_url
+      sentinel = Redic.new self.sentinel_url
+      master_ip, master_port = sentinel.call "SENTINEL", "get-master-addr-by-name", master_name
+      return "redis://#{master_ip}:#{master_port}/#{@db}"
+    end
+
+    def exec_with_retry(&block)
+      retries = 20
+      begin
+        block.call
+      rescue Errno::ECONNREFUSED, RuntimeError => e
+        if retries >= 0
+          retries -= 1
+          sleep 0.5
+          @client = Redic::Client.new(get_master_url, @timeout)
+          retry
+        else
+          raise e
+        end
+      end
+    end
+end

tests/redic_ha_test.rb

@@ -0,0 +1,25 @@
+require "cutest"
+require_relative "../lib/redic_ha"
+require_relative "../lib/redic"
+
+prepare do
+  RedicHA.new(ENV['MASTER_NAME'], ENV['SENTINEL_URL']).call("FLUSHDB")
+end
+
+setup do
+  RedicHA.new(ENV['MASTER_NAME'], ENV['SENTINEL_URL'])
+end
+
+test "test_failover_handling" do |c|
+  10.times do
+    assert_equal "OK", c.call(:set, :foo, :bar)
+  end
+
+  # Simulate failover
+  c2 = Redic.new ENV['SENTINEL_URL']
+  c2.call :sentinel, :failover, :test
+
+  10.times do
+    assert_equal "OK", c.call(:set, :foo, :bar)
+  end
+end

tests/redic_test.rb

@@ -1,3 +1,5 @@
+# coding=utf-8
+
 require "cutest"
 require_relative "../lib/redic"