Skip to content

Commit 2e06d6b

Browse files
hkulekcihkulekci
committed
docker integrated
1 parent 1a10c9a commit 2e06d6b

16 files changed

Lines changed: 255 additions & 6 deletions

File tree

‎.docker/elasticsearch/Dockerfile‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
FROM docker.elastic.co/elasticsearch/elasticsearch:5.6.2
2+
3+
RUN bin/elasticsearch-plugin install analysis-icu --batch
4+
RUN bin/elasticsearch-plugin install ingest-attachment --batch
5+
RUN bin/elasticsearch-plugin remove x-pack --purge
6+
7+
ENV ES_JAVA_OPTS "-Xms1g -Xmx1g"
8+
9+
EXPOSE 9200:9200 9300:9300

‎.docker/elasticsearch/config/elasticsearch.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
network.host: 0.0.0.0

‎.docker/elasticsearch/config/logging.yml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# you can override this using by setting a system property, for example -Des.logger.level=DEBUG
2+
es.logger.level: INFO
3+
rootLogger: ${es.logger.level}, console
4+
logger:
5+
# log action execution errors for easier debugging
6+
action: DEBUG
7+
# reduce the logging for aws, too much is logged under the default INFO
8+
com.amazonaws: WARN
9+
10+
appender:
11+
console:
12+
type: console
13+
layout:
14+
type: consolePattern
15+
conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"

‎.docker/kibana/Dockerfile‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
FROM docker.elastic.co/kibana/kibana:5.6.2
2+
3+
ENV ELASTICSEARCH_URL "http://elasticsearch:9200"
4+
5+
EXPOSE 5601:5601

‎.docker/logstash/Dockerfile‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
FROM docker.elastic.co/logstash/logstash:5.6.2
2+
3+
WORKDIR /usr/share/logstash
4+
RUN bin/logstash-plugin install logstash-input-jdbc
5+
RUN bin/logstash-plugin remove x-pack
6+
RUN bin/logstash-plugin install logstash-filter-json_encode
7+
RUN bin/logstash-plugin install logstash-filter-json
8+
RUN bin/logstash-plugin install logstash-filter-mutate
9+
10+
COPY conf/logstash.conf /etc/logstash/conf.d/logstash.conf
11+
COPY conf/template.json /etc/logstash/conf.d/template.json
12+
COPY connector/* /usr/share/logstash/
13+
14+
# https://github.com/elastic/logstash-docker/issues/45
15+
RUN sed -i '/xpack/d' /usr/share/logstash/config/logstash.yml
16+
17+
EXPOSE 5044:5044
18+
CMD ["logstash", "-f", "/etc/logstash/conf.d/logstash.conf", "--config.reload.automatic", "--debug", "--verbose"]

‎.docker/logstash/conf/logstash.conf‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
input {
2+
jdbc {
3+
jdbc_driver_library => "mysql-connector-java-5.1.36-bin.jar"
4+
jdbc_driver_class => "com.mysql.jdbc.Driver"
5+
jdbc_connection_string => "jdbc:mysql://163.172.108.190:8989/node_es_example"
6+
jdbc_user => "root"
7+
jdbc_password => "root"
8+
schedule => "* * * * *"
9+
statement => "CALL fetchDataForElastic(:sql_last_value);"
10+
}
11+
}
12+
13+
filter {
14+
json {
15+
source => "categories"
16+
target => "categories"
17+
}
18+
mutate { remove_field => [ "@version", "@timestamp" ] }
19+
}
20+
21+
output {
22+
stdout
23+
{
24+
codec => rubydebug
25+
}
26+
27+
elasticsearch
28+
{
29+
hosts => ["elasticsearch:9200"]
30+
index => "products"
31+
document_type => "product"
32+
document_id => "%{id}"
33+
}
34+
}

‎.docker/logstash/conf/template.json‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
2+
{
3+
"template": "myindex*",
4+
"settings": {
5+
"index.refresh_interval": "5s"
6+
},
7+
"mappings": {
8+
"myindex": {
9+
"dynamic_templates": [
10+
{
11+
"message_field": {
12+
"mapping": {
13+
"fielddata": {
14+
"format": "disabled"
15+
},
16+
"omit_norms": true,
17+
"type": "text"
18+
},
19+
"match_mapping_type": "string",
20+
"match": "message"
21+
}
22+
},
23+
{
24+
"string_fields": {
25+
"match_mapping_type": "string",
26+
"match": "*",
27+
"mapping": {
28+
"fielddata": {
29+
"format": "disabled"
30+
},
31+
"omit_norms": true,
32+
"type": "text",
33+
"fields": {
34+
"raw": {
35+
"ignore_above": 256,
36+
"type": "keyword"
37+
}
38+
}
39+
}
40+
}
41+
}
42+
],
43+
"_all": {
44+
"enabled": true
45+
}
46+
}
47+
},
48+
"aliases": {}
49+
}

‎.docker/logstash/connector/mysql-connector-java-5.1.36-bin.jar‎

949 KB
Binary file not shown.

‎.docker/mysql/mysqld.cnf‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Copyright (c) 2014, 2016, Oracle and/or its affiliates. All rights reserved.
2+
#
3+
# This program is free software; you can redistribute it and/or modify
4+
# it under the terms of the GNU General Public License as published by
5+
# the Free Software Foundation; version 2 of the License.
6+
#
7+
# This program is distributed in the hope that it will be useful,
8+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
9+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10+
# GNU General Public License for more details.
11+
#
12+
# You should have received a copy of the GNU General Public License
13+
# along with this program; if not, write to the Free Software
14+
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
15+
16+
#
17+
# The MySQL Server configuration file.
18+
#
19+
# For explanations see
20+
# http://dev.mysql.com/doc/mysql/en/server-system-variables.html
21+
22+
[mysqld]
23+
port = 33060
24+
pid-file = /var/run/mysqld/mysqld.pid
25+
socket = /var/run/mysqld/mysqld.sock
26+
datadir = /var/lib/mysql
27+
#log-error = /var/log/mysql/error.log
28+
# By default we only accept connections from localhost
29+
bind-address = 0.0.0.0
30+
# Disabling symbolic-links is recommended to prevent assorted security risks
31+
symbolic-links=0

‎README.md‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,31 @@
1-
## Node Elasticsearch Example
1+
# Node Elasticsearch Example
2+
3+
## Starting Services
4+
5+
```
6+
docker-compose up elasticsearch kibana
7+
docker-compose up mysql
8+
docker-compose up app
9+
```
210

311
## Database Structure and Initialization
412

5-
Please check `data/sample_data.sql` for our sample data.
13+
Connect the mysql container with following configuration:
14+
15+
```
16+
Host: mysql
17+
User : root
18+
Password : 123456
19+
Port : 33060
20+
```
21+
22+
And import `data/sample_data.sql` file for our sample data.
623

724

825
## Elastic Integration
926

10-
First of all create your `products` index and `product` type.
27+
Then use Kibana Console interface to be able to create your index. First of all
28+
create your `products` index and `product` type.
1129

1230
```
1331
DELETE products

0 commit comments

Comments
 (0)