Skip to content

Commit eb774bc

Browse files
author
accelerated
committed
Thread safe buffered producer
1 parent 46c396f commit eb774bc

6 files changed

Lines changed: 233 additions & 63 deletions

File tree

CMakeLists.txt

Lines changed: 37 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,18 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib)
2424
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib)
2525

2626
# Build output checks
27+
option(CPPKAFKA_CMAKE_VERBOSE "Generate verbose output." OFF)
2728
option(CPPKAFKA_BUILD_SHARED "Build cppkafka as a shared library." ON)
2829
option(CPPKAFKA_DISABLE_TESTS "Disable build of cppkafka tests." OFF)
30+
option(CPPKAFKA_DISABLE_EXAMPLES "Disable build of cppkafka examples." OFF)
31+
option(CPPKAFKA_BOOST_STATIC_LIBS "Link with Boost static libraries." ON)
32+
option(CPPKAFKA_BOOST_USE_MULTITHREADED "Use Boost multithreaded libraries." ON)
33+
34+
# Disable output from find_package macro
35+
if (NOT CPPKAFKA_CMAKE_VERBOSE)
36+
set(FIND_PACKAGE_QUIET QUIET)
37+
endif()
38+
2939
if(CPPKAFKA_BUILD_SHARED)
3040
message(STATUS "Build will generate a shared library. "
3141
"Use CPPKAFKA_BUILD_SHARED=0 to perform a static build")
@@ -37,16 +47,36 @@ else()
3747
endif()
3848

3949
# Look for Boost (just need boost.optional headers here)
40-
find_package(Boost REQUIRED)
41-
find_package(RdKafka REQUIRED)
50+
find_package(Boost REQUIRED ${FIND_PACKAGE_QUIET})
51+
find_package(RdKafka REQUIRED ${FIND_PACKAGE_QUIET})
52+
53+
if (Boost_FOUND)
54+
find_package(Boost COMPONENTS thread program_options ${FIND_PACKAGE_QUIET})
55+
set(Boost_USE_STATIC_LIBS ${CPPKAFKA_BOOST_STATIC_LIBS})
56+
set(Boost_USE_MULTITHREADED ${CPPKAFKA_BOOST_USE_MULTITHREADED})
57+
include_directories(${Boost_INCLUDE_DIRS})
58+
link_directories(${Boost_LIBRARY_DIRS})
59+
if (CPPKAFKA_CMAKE_VERBOSE)
60+
message(STATUS "Boost include dir: ${Boost_INCLUDE_DIRS}")
61+
message(STATUS "Boost library dir: ${Boost_LIBRARY_DIRS}")
62+
message(STATUS "Boost use static libs: ${Boost_USE_STATIC_LIBS}")
63+
message(STATUS "Boost is multi-threaded: ${CPPKAFKA_BOOST_USE_MULTITHREADED}")
64+
message(STATUS "Boost libraries: ${Boost_LIBRARIES}")
65+
endif()
66+
endif()
4267

4368
add_subdirectory(src)
4469
add_subdirectory(include)
4570

46-
add_subdirectory(examples)
71+
# Examples target
72+
if (NOT CPPKAFKA_DISABLE_EXAMPLES AND Boost_PROGRAM_OPTIONS_FOUND)
73+
add_subdirectory(examples)
74+
else()
75+
message(STATUS "Disabling examples")
76+
endif()
4777

4878
# Add a target to generate API documentation using Doxygen
49-
find_package(Doxygen QUIET)
79+
find_package(Doxygen ${FIND_PACKAGE_QUIET})
5080
if(DOXYGEN_FOUND)
5181
configure_file(
5282
${CMAKE_CURRENT_SOURCE_DIR}/docs/Doxyfile.in
@@ -61,16 +91,17 @@ if(DOXYGEN_FOUND)
6191
)
6292
endif(DOXYGEN_FOUND)
6393

64-
if(NOT CPPKAFKA_DISABLE_TESTS)
94+
if(NOT CPPKAFKA_DISABLE_TESTS AND Boost_THREAD_FOUND)
6595
set(CATCH_ROOT ${CMAKE_SOURCE_DIR}/third_party/Catch2)
6696
if(EXISTS ${CATCH_ROOT}/CMakeLists.txt)
6797
set(CATCH_INCLUDE ${CATCH_ROOT}/single_include)
68-
6998
enable_testing()
7099
add_subdirectory(tests)
71100
else()
72101
message(STATUS "Disabling tests because submodule Catch2 isn't checked out")
73102
endif()
103+
else()
104+
message(STATUS "Disabling tests")
74105
endif()
75106

76107
if(NOT TARGET uninstall)

README.md

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -55,25 +55,32 @@ In order to compile _cppkafka_ you need:
5555
* _CMake_
5656
* A compiler with good C++11 support (e.g. gcc >= 4.8). This was tested successfully on
5757
_g++ 4.8.3_.
58-
* The boost library. _cppkafka_ only requires boost.optional, which is a header only library,
59-
so this doesn't add any additional runtime dependencies.
58+
* The boost library.
6059

6160
Now, in order to build, just run:
6261

6362
```Shell
6463
mkdir build
6564
cd build
66-
cmake ..
65+
cmake <OPTIONS> ..
6766
make
6867
```
6968

7069
## CMake options
7170

72-
If you have installed _librdkafka_ on a non standard directory, you can use the
73-
`RDKAFKA_ROOT_DIR` cmake parameter when configuring the project:
74-
71+
The following cmake options can be specified:
72+
* `RDKAFKA_ROOT_DIR` : Specify a different librdkafka install directory.
73+
* `BOOST_ROOT` : Specify a different Boost install directory.
74+
* `CPPKAFKA_CMAKE_VERBOSE` : Generate verbose output. Default is `OFF`.
75+
* `CPPKAFKA_BUILD_SHARED` : Build cppkafka as a shared library. Default is `ON`.
76+
* `CPPKAFKA_DISABLE_TESTS` : Disable build of cppkafka tests. Default is `OFF`.
77+
* `CPPKAFKA_DISABLE_EXAMPLES` : Disable build of cppkafka examples. Default is `OFF`.
78+
* `CPPKAFKA_BOOST_STATIC_LIBS` : Link with Boost static libraries. Default is `ON`.
79+
* `CPPKAFKA_BOOST_USE_MULTITHREADED` : Use Boost multi-threaded libraries. Default is `ON`.
80+
81+
Example:
7582
```Shell
76-
cmake .. -DRDKAFKA_ROOT_DIR=/some/other/dir
83+
cmake -DRDKAFKA_ROOT_DIR=/some/other/dir -DCPPKAFKA_BUILD_SHARED=OFF ...
7784
```
7885

7986
Note that the `RDKAFKA_ROOT_DIR` must contain the following structure:
@@ -86,13 +93,6 @@ ${RDKAFKA_ROOT_DIR}/
8693
+ lib/librdkafka.a
8794
```
8895

89-
By default, a shared library will be built. If you want to perform a static build,
90-
use the `CPPKAFKA_BUILD_SHARED` parameter:
91-
92-
```Shell
93-
cmake .. -DCPPKAFKA_BUILD_SHARED=0
94-
```
95-
9696
# Using
9797

9898
If you want to use _cppkafka_, you'll need to link your application with:
@@ -108,4 +108,3 @@ _Doxygen_ to be installed. The documentation will be written in html format at
108108

109109
Make sure to check the [wiki](https://github.com/mfontanini/cppkafka/wiki) which includes
110110
some documentation about the project and some of its features.
111-

examples/CMakeLists.txt

Lines changed: 15 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,15 @@
1-
find_package(Boost COMPONENTS program_options)
2-
3-
if (Boost_PROGRAM_OPTIONS_FOUND)
4-
link_libraries(${Boost_LIBRARIES} cppkafka ${RDKAFKA_LIBRARY})
5-
6-
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../include)
7-
include_directories(SYSTEM ${Boost_INCLUDE_DIRS} ${RDKAFKA_INCLUDE_DIR})
8-
9-
add_custom_target(examples)
10-
macro(create_example example_name)
11-
add_executable(${example_name} EXCLUDE_FROM_ALL "${example_name}.cpp")
12-
add_dependencies(examples ${example_name})
13-
endmacro()
14-
15-
create_example(kafka_producer)
16-
create_example(kafka_consumer)
17-
create_example(kafka_consumer_dispatcher)
18-
create_example(metadata)
19-
create_example(consumers_information)
20-
else()
21-
message(STATUS "Disabling examples since boost.program_options was not found")
22-
endif()
1+
link_libraries(cppkafka ${RDKAFKA_LIBRARY} ${Boost_LIBRARIES} pthread)
2+
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../include)
3+
include_directories(SYSTEM ${RDKAFKA_INCLUDE_DIR})
4+
5+
add_custom_target(examples)
6+
macro(create_example example_name)
7+
add_executable(${example_name} EXCLUDE_FROM_ALL "${example_name}.cpp")
8+
add_dependencies(examples ${example_name})
9+
endmacro()
10+
11+
create_example(kafka_producer)
12+
create_example(kafka_consumer)
13+
create_example(kafka_consumer_dispatcher)
14+
create_example(metadata)
15+
create_example(consumers_information)

include/cppkafka/utils/buffered_producer.h

Lines changed: 71 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@
3737
#include <unordered_set>
3838
#include <unordered_map>
3939
#include <map>
40+
#include <boost/thread/mutex.hpp>
41+
#include <boost/thread/shared_mutex.hpp>
42+
#include <atomic>
4043
#include <boost/optional.hpp>
4144
#include "../producer.h"
4245
#include "../message.h"
@@ -126,6 +129,29 @@ class CPPKAFKA_API BufferedProducer {
126129
* Clears any buffered messages
127130
*/
128131
void clear();
132+
133+
/**
134+
* \brief Returns the current size of the buffer
135+
*
136+
* \return Number of unsent messages in the buffer
137+
*/
138+
size_t get_buffer_size() const;
139+
140+
/**
141+
* \brief Returns the total number of messages ack-ed by the broker
142+
*
143+
* \return The total number of messages since the beginning or since the last roll-over
144+
*
145+
* \remark Call get_rollover_count() to get the number of times the counter has rolled over
146+
*/
147+
size_t get_total_messages_acked() const;
148+
149+
/**
150+
* \brief Roll-over counter for get_total_messages_acked
151+
*
152+
* \return The number of rolls
153+
*/
154+
uint16_t get_rollover_count() const;
129155

130156
/**
131157
* Gets the Producer object
@@ -152,6 +178,7 @@ class CPPKAFKA_API BufferedProducer {
152178
* \param callback The callback to be set
153179
*/
154180
void set_produce_failure_callback(ProduceFailureCallback callback);
181+
155182
private:
156183
using QueueType = std::queue<Builder>;
157184

@@ -163,9 +190,12 @@ class CPPKAFKA_API BufferedProducer {
163190

164191
Producer producer_;
165192
QueueType messages_;
193+
mutable boost::mutex exclusive_access_;
194+
mutable boost::shared_mutex shared_access_;
166195
ProduceFailureCallback produce_failure_callback_;
167-
size_t expected_acks_{0};
168-
size_t messages_acked_{0};
196+
std::atomic_ulong expected_acks_{0};
197+
std::atomic_ullong total_messages_acked_{0};
198+
std::atomic_ushort rollover_counter_{0};
169199
};
170200

171201
template <typename BufferType>
@@ -187,22 +217,25 @@ void BufferedProducer<BufferType>::add_message(Builder builder) {
187217
template <typename BufferType>
188218
void BufferedProducer<BufferType>::produce(const MessageBuilder& builder) {
189219
produce_message(builder);
190-
expected_acks_++;
191220
}
192221

193222
template <typename BufferType>
194223
void BufferedProducer<BufferType>::flush() {
195-
while (!messages_.empty()) {
196-
produce_message(messages_.front());
197-
messages_.pop();
224+
{
225+
boost::shared_lock<boost::shared_mutex> grant(shared_access_);
226+
size_t num_messages = messages_.size();
227+
while (num_messages--) {
228+
produce_message(messages_.front());
229+
boost::lock_guard<boost::mutex> require(exclusive_access_);
230+
messages_.pop();
231+
}
198232
}
199-
200233
wait_for_acks();
201234
}
202235

203236
template <typename BufferType>
204237
void BufferedProducer<BufferType>::wait_for_acks() {
205-
while (messages_acked_ < expected_acks_) {
238+
while (expected_acks_ > 0) {
206239
try {
207240
producer_.flush();
208241
}
@@ -216,22 +249,20 @@ void BufferedProducer<BufferType>::wait_for_acks() {
216249
}
217250
}
218251
}
219-
expected_acks_ = 0;
220-
messages_acked_ = 0;
221252
}
222253

223254
template <typename BufferType>
224255
void BufferedProducer<BufferType>::clear() {
256+
boost::unique_lock<boost::shared_mutex> restrict(shared_access_);
225257
QueueType tmp;
226258
std::swap(tmp, messages_);
227-
expected_acks_ = 0;
228-
messages_acked_ = 0;
229259
}
230260

231261
template <typename BufferType>
232262
template <typename BuilderType>
233263
void BufferedProducer<BufferType>::do_add_message(BuilderType&& builder) {
234-
expected_acks_++;
264+
boost::shared_lock<boost::shared_mutex> grant(shared_access_);
265+
boost::lock_guard<boost::mutex> require(exclusive_access_);
235266
messages_.push(std::move(builder));
236267
}
237268

@@ -245,6 +276,21 @@ const Producer& BufferedProducer<BufferType>::get_producer() const {
245276
return producer_;
246277
}
247278

279+
template <typename BufferType>
280+
size_t BufferedProducer<BufferType>::get_buffer_size() const {
281+
return messages_.size();
282+
}
283+
284+
template <typename BufferType>
285+
size_t BufferedProducer<BufferType>::get_total_messages_acked() const {
286+
return total_messages_acked_;
287+
}
288+
289+
template <typename BufferType>
290+
uint16_t BufferedProducer<BufferType>::get_rollover_count() const {
291+
return rollover_counter_;
292+
}
293+
248294
template <typename BufferType>
249295
typename BufferedProducer<BufferType>::Builder
250296
BufferedProducer<BufferType>::make_builder(std::string topic) {
@@ -275,6 +321,8 @@ void BufferedProducer<BufferType>::produce_message(const MessageBuilder& builder
275321
}
276322
}
277323
}
324+
// Sent successfully
325+
++expected_acks_;
278326
}
279327

280328
template <typename BufferType>
@@ -287,6 +335,10 @@ Configuration BufferedProducer<BufferType>::prepare_configuration(Configuration
287335

288336
template <typename BufferType>
289337
void BufferedProducer<BufferType>::on_delivery_report(const Message& message) {
338+
// Decrement the expected acks
339+
--expected_acks_;
340+
assert(expected_acks_ != (unsigned long)-1); // Prevent underflow
341+
290342
// We should produce this message again if it has an error and we either don't have a
291343
// produce failure callback or we have one but it returns true
292344
bool should_produce = message.get_error() &&
@@ -305,9 +357,12 @@ void BufferedProducer<BufferType>::on_delivery_report(const Message& message) {
305357
produce_message(builder);
306358
return;
307359
}
308-
// If production was successful or the produce failure callback returned false, then
309-
// let's consider it to be acked
310-
messages_acked_++;
360+
361+
// Increment the total successful transmissions
362+
++total_messages_acked_;
363+
if (total_messages_acked_ == 0) {
364+
++rollover_counter_;
365+
}
311366
}
312367

313368
} // cppkafka

tests/CMakeLists.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../include/)
22
include_directories(SYSTEM ${CATCH_INCLUDE})
3-
include_directories(SYSTEM ${Boost_INCLUDE_DIRS} ${RDKAFKA_INCLUDE_DIR})
3+
include_directories(SYSTEM ${RDKAFKA_INCLUDE_DIR})
44

55
set(KAFKA_TEST_INSTANCE "kafka-vm:9092"
66
CACHE STRING "The kafka instance to which to connect to run tests")
77
add_custom_target(tests)
88

99
include_directories(${CMAKE_CURRENT_SOURCE_DIR})
1010
add_library(cppkafka-test EXCLUDE_FROM_ALL test_utils.cpp)
11-
target_link_libraries(cppkafka-test cppkafka ${RDKAFKA_LIBRARY} pthread)
11+
target_link_libraries(cppkafka-test cppkafka ${RDKAFKA_LIBRARY} ${Boost_LIBRARIES} pthread)
1212

1313
add_definitions("-DKAFKA_TEST_INSTANCE=\"${KAFKA_TEST_INSTANCE}\"")
1414

0 commit comments

Comments
 (0)