Merge pull request #125 from mfontanini/events

Event implementation

Author: mfontanini

.travis.yml

@@ -7,6 +7,7 @@ compiler:
   - clang
 
 env:
+  - RDKAFKA_VERSION=v0.9.4
   - RDKAFKA_VERSION=v0.11.5
 
 os:

include/cppkafka/configuration.h

@@ -42,6 +42,7 @@
 #include "clonable_ptr.h"
 #include "configuration_base.h"
 #include "macros.h"
+#include "event.h"
 
 namespace cppkafka {
 
@@ -78,6 +79,7 @@ class CPPKAFKA_API Configuration : public ConfigurationBase<Configuration> {
                                            const std::string& message)>;
     using StatsCallback = std::function<void(KafkaHandleBase& handle, const std::string& json)>;
     using SocketCallback = std::function<int(int domain, int type, int protocol)>;
+    using BackgroundEventCallback = std::function<void(KafkaHandleBase& handle, Event)>;
 
     using ConfigurationBase<Configuration>::set;
     using ConfigurationBase<Configuration>::get;
@@ -142,6 +144,13 @@ class CPPKAFKA_API Configuration : public ConfigurationBase<Configuration> {
      */
     Configuration& set_socket_callback(SocketCallback callback);
 
+#if RD_KAFKA_VERSION >= RD_KAFKA_ADMIN_API_SUPPORT_VERSION
+    /**
+     * Sets the background event callback (invokes rd_kafka_conf_set_background_event_cb)
+     */
+    Configuration& set_background_event_callback(BackgroundEventCallback callback);
+#endif
+
     /** 
      * Sets the default topic configuration
      */
@@ -204,6 +213,11 @@ class CPPKAFKA_API Configuration : public ConfigurationBase<Configuration> {
      */
     const SocketCallback& get_socket_callback() const;
 
+    /**
+     * Gets the background event callback
+     */
+    const BackgroundEventCallback& get_background_event_callback() const;
+
     /**
      * Gets the default topic configuration
      */
@@ -229,6 +243,7 @@ class CPPKAFKA_API Configuration : public ConfigurationBase<Configuration> {
     LogCallback log_callback_;
     StatsCallback stats_callback_;
     SocketCallback socket_callback_;
+    BackgroundEventCallback background_event_callback_;
 };
 
 } // cppkafka

include/cppkafka/consumer.h

@@ -461,7 +461,6 @@ class CPPKAFKA_API Consumer : public KafkaHandleBase {
 private:
     static void rebalance_proxy(rd_kafka_t *handle, rd_kafka_resp_err_t error,
                                 rd_kafka_topic_partition_list_t *partitions, void *opaque);
-    static Queue get_queue(rd_kafka_queue_t* handle);
     void close();
     void commit(const Message& msg, bool async);
     void commit(const TopicPartitionList* topic_partitions, bool async);
@@ -485,7 +484,7 @@ std::vector<Message, Allocator> Consumer::poll_batch(size_t max_batch_size,
                                                      const Allocator& alloc) {
     std::vector<rd_kafka_message_t*> raw_messages(max_batch_size);
     // Note that this will leak the queue when using rdkafka < 0.11.5 (see get_queue comment)
-    Queue queue(get_queue(rd_kafka_queue_get_consumer(get_handle())));
+    Queue queue = Queue::make_queue(rd_kafka_queue_get_consumer(get_handle()));
     ssize_t result = rd_kafka_consume_batch_queue(queue.get_handle(),
                                                   timeout.count(),
                                                   raw_messages.data(),

include/cppkafka/cppkafka.h

@@ -37,6 +37,7 @@
 #include <cppkafka/configuration_option.h>
 #include <cppkafka/consumer.h>
 #include <cppkafka/error.h>
+#include <cppkafka/event.h>
 #include <cppkafka/exceptions.h>
 #include <cppkafka/group_information.h>
 #include <cppkafka/header.h>

include/cppkafka/event.h

@@ -0,0 +1,180 @@
+/*
+ * Copyright (c) 2018, Matias Fontanini
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following disclaimer
+ *   in the documentation and/or other materials provided with the
+ *   distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef CPPKAFKA_EVENT_H
+#define CPPKAFKA_EVENT_H
+
+#include <memory>
+#include <string>
+#include <vector>
+#include "error.h"
+#include "message.h"
+#include "topic_partition.h"
+#include "topic_partition_list.h"
+
+namespace cppkafka {
+
+class Event {
+public:
+    /**
+     * Construct an Event from a rdkafka event handle and take ownership of it
+     *
+     * /param handle The handle to construct this event from
+     */
+    Event(rd_kafka_event_t* handle);
+
+    /**
+     * Returns the name of this event
+     */
+    std::string get_name() const;
+
+    /**
+     * Returns the type of this event
+     */
+    rd_kafka_event_type_t get_type() const;
+
+    /**
+     * \brief Gets the next message contained in this event.
+     *
+     * This call is only valid if the event type is one of:
+     * * RD_KAFKA_EVENT_FETCH
+     * * RD_KAFKA_EVENT_DR
+     *
+     * \note The returned message's lifetime *is tied to this Event*. That is, if the event
+     * is free'd so will the contents of the message.
+     */
+    Message get_next_message() const;
+
+    /**
+     * \brief Gets all messages in this event (if any)
+     *
+     * This call is only valid if the event type is one of:
+     * * RD_KAFKA_EVENT_FETCH
+     * * RD_KAFKA_EVENT_DR
+     *
+     * \note The returned messages' lifetime *is tied to this Event*. That is, if the event
+     * is free'd so will the contents of the messages.
+     *
+     * \return A vector containing 0 or more messages
+     */
+    std::vector<Message> get_messages();
+
+    /**
+     * \brief Gets all messages in this event (if any)
+     *
+     * This call is only valid if the event type is one of:
+     * * RD_KAFKA_EVENT_FETCH
+     * * RD_KAFKA_EVENT_DR
+     *
+     * \param allocator The allocator to use on the output vector
+     *
+     * \note The returned messages' lifetime *is tied to this Event*. That is, if the event
+     * is free'd so will the contents of the messages.
+     *
+     * \return A vector containing 0 or more messages
+     */
+    template <typename Allocator>
+    std::vector<Message, Allocator> get_messages(const Allocator allocator);
+
+    /**
+     * \brief Gets the number of messages contained in this event
+     *
+     * This call is only valid if the event type is one of:
+     * * RD_KAFKA_EVENT_FETCH
+     * * RD_KAFKA_EVENT_DR
+     */
+    size_t get_message_count() const;
+
+    /**
+     * \brief Returns the error in this event
+     */
+    Error get_error() const;
+
+    /**
+     * Gets the opaque pointer in this event
+     */
+    void* get_opaque() const;
+
+#if RD_KAFKA_VERSION >= RD_KAFKA_EVENT_STATS_SUPPORT_VERSION
+    /**
+     * \brief Gets the stats in this event
+     *
+     * This call is only valid if the event type is RD_KAFKA_EVENT_STATS
+     */
+    std::string get_stats() const {
+        return rd_kafka_event_stats(handle_.get());
+    }
+#endif
+
+    /**
+     * \brief Gets the topic/partition for this event
+     *
+     * This call is only valid if the event type is RD_KAFKA_EVENT_ERROR
+     */
+    TopicPartition get_topic_partition() const;
+
+    /**
+     * \brief Gets the list of topic/partitions in this event
+     *
+     * This call is only valid if the event type is one of:
+     * * RD_KAFKA_EVENT_REBALANCE
+     * * RD_KAFKA_EVENT_OFFSET_COMMIT
+     */
+    TopicPartitionList get_topic_partition_list() const;
+
+    /**
+     * Check whether this event is valid
+     *
+     * /return true iff this event has a valid (non-null) handle inside
+     */
+    operator bool() const;
+private:
+    using HandlePtr = std::unique_ptr<rd_kafka_event_t, decltype(&rd_kafka_event_destroy)>;
+
+    HandlePtr handle_;
+};
+
+template <typename Allocator>
+std::vector<Message, Allocator> Event::get_messages(const Allocator allocator) {
+    const size_t total_messages = get_message_count();
+    std::vector<const rd_kafka_message_t*> raw_messages(total_messages);
+    const auto messages_read = rd_kafka_event_message_array(handle_.get(),
+                                                            raw_messages.data(),
+                                                            total_messages);
+    std::vector<Message, Allocator> output(allocator);
+    output.reserve(messages_read);
+    for (auto message : raw_messages) {
+        output.emplace_back(Message::make_non_owning(const_cast<rd_kafka_message_t*>(message)));
+    }
+    return output;
+}
+
+} // cppkafka
+
+#endif // CPPKAFKA_EVENT_H

include/cppkafka/kafka_handle_base.h

@@ -46,6 +46,7 @@
 #include "configuration.h"
 #include "macros.h"
 #include "logging.h"
+#include "queue.h"
 
 namespace cppkafka {
 
@@ -239,6 +240,19 @@ class CPPKAFKA_API KafkaHandleBase {
      */ 
     const Configuration& get_configuration() const;
 
+#if RD_KAFKA_VERSION >= RD_KAFKA_ADMIN_API_SUPPORT_VERSION
+    /**
+     * \brief Gets the background queue
+     *
+     * This translates into a call to rd_kafka_queue_get_background
+     *
+     * \return The background queue
+     */
+    Queue get_background_queue() const {
+        return Queue::make_queue(rd_kafka_queue_get_background(handle_.get()));
+    }
+#endif
+
     /**
      * \brief Gets the length of the out queue 
      *

include/cppkafka/macros.h

@@ -46,5 +46,8 @@
 // See: https://github.com/edenhill/librdkafka/issues/1792
 #define RD_KAFKA_QUEUE_REFCOUNT_BUG_VERSION 0x000b0500 //v0.11.5.00
 #define RD_KAFKA_HEADERS_SUPPORT_VERSION 0x000b0402 //v0.11.4.02
+#define RD_KAFKA_ADMIN_API_SUPPORT_VERSION 0x000b0500 //v0.11.5.00
+#define RD_KAFKA_MESSAGE_LATENCY_SUPPORT_VERSION 0x000b0000 //v0.11.0.00
+#define RD_KAFKA_EVENT_STATS_SUPPORT_VERSION 0x000b0000 //v0.11.0.00
 
 #endif // CPPKAFKA_MACROS_H

include/cppkafka/message.h

@@ -177,13 +177,15 @@ class CPPKAFKA_API Message {
      */
     boost::optional<MessageTimestamp> get_timestamp() const;
 
+#if RD_KAFKA_VERSION >= RD_KAFKA_MESSAGE_LATENCY_SUPPORT_VERSION
     /**
      * \brief Gets the message latency in microseconds as measured from the produce() call.
      */
     std::chrono::microseconds get_latency() const {
         assert(handle_);
         return std::chrono::microseconds(rd_kafka_message_latency(handle_.get()));
     }
+#endif
 
     /**
      * \brief Indicates whether this message is valid (not null)

include/cppkafka/queue.h

@@ -31,6 +31,7 @@
 #include <memory>
 #include <boost/optional.hpp>
 #include <librdkafka/rdkafka.h>
+#include "event.h"
 #include "macros.h"
 #include "message.h"
 
@@ -51,7 +52,18 @@ class CPPKAFKA_API Queue {
      * \param handle The handle to be used
      */
     static Queue make_non_owning(rd_kafka_queue_t* handle);
-    
+
+    /**
+     * \brieef Creates a Queue object out of a handle.
+     *
+     * This will check what the rdkafka version is and will return either an owned
+     * queue handle or a non owned one, depending on whether the current version
+     * is >= RD_KAFKA_QUEUE_REFCOUNT_BUG_VERSION (see macros.h)
+     *
+     * \param handle The handle to be used
+     */
+    static Queue make_queue(rd_kafka_queue_t* handle);
+
     /**
      * \brief Constructs an empty queue
      *
@@ -130,7 +142,7 @@ class CPPKAFKA_API Queue {
      * \return A message
      */
     Message consume(std::chrono::milliseconds timeout) const;
-    
+
     /**
      * \brief Consumes a batch of messages from this queue
      *
@@ -188,7 +200,23 @@ class CPPKAFKA_API Queue {
      */
     std::vector<Message> consume_batch(size_t max_batch_size,
                                        std::chrono::milliseconds timeout) const;
-    
+
+    /**
+     * \brief Extracts the next message in this Queue
+     *
+     * /return The latest event, if any
+     */
+    Event next_event() const;
+
+    /**
+     * \brief Extracts the next message in this Queue
+     *
+     * \param timeout The amount of time to wait for this operation to complete
+     *
+     * /return The latest event, if any
+     */
+    Event next_event(std::chrono::milliseconds timeout) const;
+
     /**
      * Indicates whether this queue is valid (not null)
      */

src/CMakeLists.txt

@@ -14,6 +14,7 @@ set(SOURCES
     metadata.cpp
     group_information.cpp
     error.cpp
+    event.cpp
 
     kafka_handle_base.cpp
     producer.cpp