diff --git a/mongodb-replicaset/README.md b/mongodb-replicaset/README.md new file mode 100644 index 0000000000..aaac673b58 --- /dev/null +++ b/mongodb-replicaset/README.md @@ -0,0 +1,64 @@ +# Using Debezium with Multi-Node MongoDB Replica Set + +This demo shows how the Debezium connector for MongoDB supports failover to secondary nodes elected as new primaries in a MongoDB replica set. +It's based on the services as defined in the [Debezium Tutorial](http://debezium.io/docs/tutorial/). + +Start the topology as defined in http://debezium.io/docs/tutorial/ (using a replica set with three nodes): + + $ export DEBEZIUM_VERSION=0.9 + $ docker-compose up + +Initialize MongoDB replica set and insert some test data: + + $ docker-compose exec mongodb-1 bash -c '/usr/local/bin/init-inventory-replicaset.sh "mongodb-1:27017 mongodb-2:27017 mongodb-3:27017"' + +Start MongoDB connector: + + $ curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @register-mongodb-replicaset.json + +Consume messages from the topic created for the "customers" collection: + + $ docker-compose exec kafka /kafka/bin/kafka-console-consumer.sh \ + --bootstrap-server kafka:9092 \ + --from-beginning \ + --property print.key=true \ + --topic dbserver1.inventory.customers + +Shut down the current primary node: + + $ docker-compose stop mongodb-1 + +Find out which MongoDB node is the new primary one: + + $ docker-compose exec mongodb-2 bash -c 'mongo inventory --eval "rs.status()"' + +Modify records in the database via MongoDB client, connecting to new primary +(assuming this is _mongodb-2_ in the following): + + $ docker-compose exec mongodb-2 bash -c 'mongo inventory' + + rs0:PRIMARY> db.customers.insert([ + { \_id : NumberLong("1005"), first_name : 'Bob', last_name : 'Hopper', email : 'thebob@example.com' } + ]); + +The connector should have automatically failed over to the new primary, so the new record should show up in the topic inspected above. + +Step down as primary, which will make the third node the new primary: + + rs0:PRIMARY> rs.stepDown(120); + rs0:PRIMARY> exit; + +Modify records in the database via MongoDB client, connecting to new primary +(assuming this is _mongodb-3_ in the following): + + $ docker-compose exec mongodb-3 bash -c 'mongo inventory' + + rs0:PRIMARY> db.customers.insert([ + { \_id : NumberLong("1006"), first_name : 'Bob', last_name : 'Hopper', email : 'thebob@example.com' } + ]); + +Again the connector should have automatically failed over to the new primary, so the new record should show up in the topic inspected above. + +Shut down the cluster: + + docker-compose down diff --git a/mongodb-replicaset/docker-compose.yaml b/mongodb-replicaset/docker-compose.yaml new file mode 100644 index 0000000000..cbc20beac5 --- /dev/null +++ b/mongodb-replicaset/docker-compose.yaml @@ -0,0 +1,43 @@ +version: '2' +services: + zookeeper: + image: debezium/zookeeper:${DEBEZIUM_VERSION} + ports: + - 2181:2181 + - 2888:2888 + - 3888:3888 + kafka: + image: debezium/kafka:${DEBEZIUM_VERSION} + ports: + - 9092:9092 + links: + - zookeeper + environment: + - ZOOKEEPER_CONNECT=zookeeper:2181 + mongodb-1: + image: debezium/example-mongodb:${DEBEZIUM_VERSION} + ports: + - 27017:27017 + mongodb-2: + image: debezium/example-mongodb:${DEBEZIUM_VERSION} + ports: + - 27018:27017 + mongodb-3: + image: debezium/example-mongodb:${DEBEZIUM_VERSION} + ports: + - 27019:27017 + connect: + image: debezium/connect:${DEBEZIUM_VERSION} + ports: + - 8083:8083 + links: + - kafka + - mongodb-1 + - mongodb-2 + - mongodb-3 + environment: + - BOOTSTRAP_SERVERS=kafka:9092 + - GROUP_ID=1 + - CONFIG_STORAGE_TOPIC=my_connect_configs + - OFFSET_STORAGE_TOPIC=my_connect_offsets + - STATUS_STORAGE_TOPIC=my_connect_statuses diff --git a/mongodb-replicaset/register-mongodb-replicaset.json b/mongodb-replicaset/register-mongodb-replicaset.json new file mode 100644 index 0000000000..fd6737da86 --- /dev/null +++ b/mongodb-replicaset/register-mongodb-replicaset.json @@ -0,0 +1,11 @@ +{ + "name": "inventory-connector", + "config": { + "connector.class" : "io.debezium.connector.mongodb.MongoDbConnector", + "tasks.max" : "1", + "mongodb.hosts" : "rs0/mongodb-1:27017", + "mongodb.name" : "dbserver1", + "database.whitelist" : "inventory", + "database.history.kafka.bootstrap.servers" : "kafka:9092" + } +} diff --git a/tutorial/README.md b/tutorial/README.md index 8b11b56b4f..76ba03830a 100644 --- a/tutorial/README.md +++ b/tutorial/README.md @@ -116,7 +116,7 @@ docker-compose -f docker-compose-mongodb.yaml exec kafka /kafka/bin/kafka-consol docker-compose -f docker-compose-mongodb.yaml exec mongodb bash -c 'mongo -u $MONGODB_USER -p $MONGODB_PASSWORD --authenticationDatabase admin inventory' db.customers.insert([ - { _id : 1005, first_name : 'Bob', last_name : 'Hopper', email : 'thebob@example.com' } + { _id : NumberLong("1005"), first_name : 'Bob', last_name : 'Hopper', email : 'thebob@example.com' } ]); # Shut down the cluster