init project

Author: dimmaryanto93

.gitignore

@@ -0,0 +1,41 @@
+# Created by .ignore support plugin (hsz.mobi)
+### VisualStudioCode template
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+### JetBrains template
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/
+
+# CMake
+cmake-build-*/
+
+# File-based project format
+*.iws
+*.iml
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# filesystems
+.DS_Store

README.md

@@ -0,0 +1 @@
+# Belajar logstash input jdbc plugin

docker-compose.yml

@@ -0,0 +1,69 @@
+version: '3.6'
+services:
+  mysql:
+    image: mysql:5.7
+    ports: 
+      - 3306:3306
+    environment:
+      - MYSQL_ROOT_PASSWORD=admin
+      - MYSQL_DATABASE=example
+      - MYSQL_USER=admin
+      - MYSQL_PASSWORD=admin
+    volumes:
+      - mysql_data:/var/lib/mysql
+    networks: 
+      - logstash_network
+  flyway:
+    image: boxfuse/flyway
+    command: -url=jdbc:mysql://mysql:3306/example?useSSL=false -schemas=example -user=admin -password=admin -driver=com.mysql.jdbc.Driver -connectRetries=60  -cleanOnValidationError=true -baselineOnMigrate=true migrate
+    environment:
+      - FLYWAY_EDITION=community
+    volumes:
+      - ./flyway/sql/:/flyway/sql/
+    networks: 
+      - logstash_network
+    depends_on: 
+      - mysql
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.5.4
+    ports:
+      - 9300:9300
+      - 9200:9200
+    environment:
+      - transport.host=127.0.0.1
+      - cluster.name=docker-cluster
+      - discovery.type=single-node
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    volumes:
+      - elasticsearch_data:/usr/share/elasticsearch/data
+    networks:
+      - logstash_network
+  kibana:
+    image: docker.elastic.co/kibana/kibana:6.5.4
+    ports:
+      - 5601:5601
+    networks:
+      - logstash_network
+    depends_on: 
+      - elasticsearch
+  logstash:
+    build: ./logstash/
+    image: dimmaryanto93/logstash-jdbc-plugin:1.1
+    ports: 
+      - 9600:9600
+      - 5044:5044
+    networks:
+      - logstash_network
+    volumes: 
+      - ./logstash/jdbc-input-plugin.conf:/usr/share/logstash/pipeline/logstash.conf
+    depends_on: 
+      - elasticsearch
+      - kibana
+      - flyway
+      - mysql
+volumes:
+  elasticsearch_data:
+  filebeat_data:
+  mysql_data:
+networks:
+  logstash_network:

flyway/sql/V1__default-sechema.sql

@@ -0,0 +1,31 @@
+create table requirement
+(
+    id varchar(64) not null primary key,
+    name varchar(100) not null,
+    registration_nomor varchar(100) not null
+)
+engine = InnoDB;
+
+create table test_script
+(
+    id varchar(64) not null primary key,
+    file_name varchar(100) not null,
+    requirement_id varchar(64) not null
+)
+engine = InnoDB;
+
+alter table test_script
+ add constraint fk_script_requirement_id foreign key (requirement_id)
+ REFERENCES requirement(id) on update CASCADE on DELETE CASCADE;
+
+create table design_step
+(
+    id varchar(64) not null primary key,
+    step_name varchar(100) not null,
+    test_script_id varchar(64) not null
+)
+engine = InnoDB;
+
+alter table design_step
+ add CONSTRAINT fk_design_script_id FOREIGN key (test_script_id)
+ REFERENCES test_script(id) on update CASCADE on DELETE CASCADE;

logstash/Dockerfile

@@ -0,0 +1,3 @@
+FROM docker.elastic.co/logstash/logstash:6.5.4
+RUN /usr/share/logstash/bin/logstash-plugin install logstash-input-jdbc
+COPY ./mysql-connector-java-5.1.46.jar /usr/share/logstash/mysql-connector-java.jar

logstash/jdbc-input-plugin.conf

@@ -0,0 +1,22 @@
+# Logstash jdbc input -> Elasticsearch pipeline.
+input {
+    jdbc {
+        jdbc_driver_library => "mysql-connector-java.jar"
+        jdbc_driver_class => "com.mysql.jdbc.Driver"
+        jdbc_connection_string => "jdbc:mysql://mysql:3306/example"
+        jdbc_user => "admin"
+        jdbc_password => "admin"
+        schedule => "* * * * *"
+        statement => "SELECT * from requirement"
+    }
+}
+
+output {
+    elasticsearch {
+        hosts => ["http://elasticsearch:9200"]
+        index => "test_script"
+        document_id => "%{id}"
+    }
+    stdout { codec => json_lines }
+
+}