docker-compose.yml

---
version: '3.3'
services:


  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_STACK_VERSION}
    hostname: elasticsearch
    container_name: elasticsearch
    environment:
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - "discovery.type=single-node"
      - "xpack.monitoring.collection.enabled=true"
    ports:
      - "127.0.0.1:9200:9200"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9200/_cat/health"]
      interval: 30s
      timeout: 10s
      retries: 5
    networks:
      - stack


  kibana:
    image: docker.elastic.co/kibana/kibana:${ELASTIC_STACK_VERSION}
    hostname: kibana
    container_name: kibana
    ports:
      - "127.0.0.1:5601:5601"
    networks:
      - stack
    depends_on:
      - elasticsearch


  logstash:
    image: docker.elastic.co/logstash/logstash:${ELASTIC_STACK_VERSION}
    hostname: logstash
    container_name: logstash
    environment:
      - "xpack.monitoring.enabled=true"
    ports:
      - "127.0.0.1:4560:4560"
      - "127.0.0.1:5044:5044"
    volumes:
      # Provide a pipeline configuration for Logstash with a bind-mounted file
      - ./docker-compose/logstash.conf:/usr/share/logstash/pipeline/logstash.conf
    networks:
      - stack
    depends_on:
      - elasticsearch


  # Filebeat sending data to Logstash
  filebeat_for_logstash:
    image: docker.elastic.co/beats/filebeat:${ELASTIC_STACK_VERSION}
    hostname: filebeat_for_logstash
    container_name: filebeat_for_logstash
    volumes:
      # Bind-mount the logs/ directory, so Filebeat can read its files
      - "./logs-docker/:/mnt/logs/:ro"
      # Bind-mount a custom configuration file
      - "./docker-compose/filebeat-logstash.yml:/usr/share/filebeat/filebeat.yml:ro"
      # Bind-mount the config directory so it can be dynamically loaded
      - "./docker-compose/filebeat-logstash/config/:/usr/share/filebeat/config/:ro"
      # Bind-mount the registry file to avoid data duplication between restarts
      - "./docker-compose/filebeat-logstash/registry/:/usr/share/filebeat/data/"
    command: filebeat -e
    networks:
      - stack
    depends_on:
      - elasticsearch
      - logstash


  # Filebeat receiving syslog data and sending it to Elasticsearch
  filebeat_syslog:
    image: docker.elastic.co/beats/filebeat:${ELASTIC_STACK_VERSION}
    hostname: filebeat_syslog
    container_name: filebeat_syslog
    command: filebeat -e
    volumes:
      # Bind-mount a custom configuration file
      - "./docker-compose/filebeat-syslog.yml:/usr/share/filebeat/filebeat.yml:ro"
      # Bind-mount the registry file to avoid data duplication between restarts
      - "./docker-compose/filebeat-syslog/registry/:/usr/share/filebeat/data/"
    networks:
      - stack
    depends_on:
      - elasticsearch


  # Filebeat sending data to Elasticsearch
  filebeat_for_elasticsearch:
    image: docker.elastic.co/beats/filebeat:${ELASTIC_STACK_VERSION}
    hostname: filebeat_for_elasticsearch
    container_name: filebeat_for_elasticsearch
    volumes:
      # Bind-mount the logs/ directory, so Filebeat can read its files
      - "./logs-docker/:/mnt/logs/:ro"
      # Bind-mount a custom configuration file
      - "./docker-compose/filebeat-elasticsearch.yml:/usr/share/filebeat/filebeat.yml:ro"
      # Bind-mount the config directory so it can be dynamically loaded
      - "./docker-compose/filebeat-elasticsearch/config/:/usr/share/filebeat/config/:ro"
      # Bind-mount the registry file to avoid data duplication between restarts
      - "./docker-compose/filebeat-elasticsearch/registry/:/usr/share/filebeat/data/"
    command: filebeat -e
    networks:
      - stack
    depends_on:
      - elasticsearch
      - kibana


  # Filebeat collecting from a Docker container and sending to Elasticsearch
  filebeat_docker_for_elasticsearch:
    image: docker.elastic.co/beats/filebeat:${ELASTIC_STACK_VERSION}
    hostname: filebeat_docker_for_elasticsearch
    container_name: filebeat_docker_for_elasticsearch
    user: root #To read the docker socket
    volumes:
      # Bind-mount the Docker log directory from the Java container, so Filebeat can read its files
      - "/var/lib/docker/containers:/var/lib/docker/containers:ro"
      # Bind-mount a custom configuration file
      - "./docker-compose/filebeat-docker.yml:/usr/share/filebeat/filebeat.yml:ro"
      # Bind-mount the registry file to avoid data duplication between restarts
      - "./docker-compose/filebeat-docker/registry/:/usr/share/filebeat/data/"
      # Bind-mount the Filebeat log file (not going to system out to avoid a logging loop)
      - "./docker-compose/filebeat-docker/logs/:/usr/share/filebeat/logs/"
      # Bind-mount the Docker daemon to enable add_docker_metadata from within the container
      - "/var/run/docker.sock:/var/run/docker.sock:ro"
    command: filebeat
    networks:
      - stack
    depends_on:
      - elasticsearch
      - kibana


  # Metricbeat to optionally collect metrics from the entire setup; not required for logging itself
  metricbeat:
    hostname: metricbeat
    container_name: metricbeat
    user: root #To read the docker socket
    image: docker.elastic.co/beats/metricbeat:${ELASTIC_STACK_VERSION}
    volumes:
      # Bind-mount a custom configuration file
      - "./docker-compose/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro"
      # Bind-mount the config directory so it can be dynamically loaded
      - "./docker-compose/metricbeat/config/:/usr/share/metricbeat/config/:ro"
      # Monitor the Docker host rather than the Metricbeat container; these are used by the system module
      - "/proc:/hostfs/proc:ro"
      - "/sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro"
      - "/:/hostfs:ro"
      # Bind-mount the Docker daemon to enable add_docker_metadata from within the container
      - "/var/run/docker.sock:/var/run/docker.sock:ro"
    command: metricbeat -e
    restart: on-failure
    networks:
      - stack
    depends_on:
      - elasticsearch
      - kibana


  # Metricbeat to optionally collect metrics from the entire setup; not required for logging itself
  heartbeat:
    hostname: heartbeat
    container_name: heartbeat
    user: root #To read the docker socket
    image: docker.elastic.co/beats/heartbeat:${ELASTIC_STACK_VERSION}
    volumes:
      # Bind-mount a custom configuration file
      - "./docker-compose/heartbeat.yml:/usr/share/heartbeat/heartbeat.yml:ro"
      # Bind-mount the Docker daemon to enable add_docker_metadata from within the container
      - "/var/run/docker.sock:/var/run/docker.sock:ro"
    command: heartbeat -e
    networks:
      - stack
    depends_on:
      - elasticsearch
      - kibana
      - logstash


  #Short lived container to configure the stack once Kibana and Elasticsearch are available
  configure_stack:
    image: docker.elastic.co/beats/filebeat:${ELASTIC_STACK_VERSION}
    hostname: configure_stack
    container_name: configure_stack
    volumes:
      - "./docker-compose/setup.sh:/usr/local/bin/setup.sh:ro"
      - "./docker-compose/setup_ingest-pipeline_parse-java.json:/usr/local/bin/setup_ingest-pipeline_parse-java.json:ro"
    command: /usr/local/bin/setup.sh
    networks:
      - stack
    depends_on:
      - elasticsearch
      - kibana


  #The Java app to be monitored
  java_app:
    hostname: java_app
    container_name: java_app
    build:
      dockerfile: $PWD/dockerfile_java
      context: $PWD
      labels:
        - "app=fizzbuzz"
        - "environment=production"
        - "co.elastic.logs/multiline.pattern=^\\["
        - "co.elastic.logs/multiline.negate=true"
        - "co.elastic.logs/multiline.match=after"
    image: java-logging:1.0.0
    volumes:
      # Bind-mount the log folder to the host so a sidecar can collect the logs
      - "./logs-docker/:/app/logs/"
    networks:
      - stack
    depends_on:
      - elasticsearch
      - logstash

  # The syslog app sending messages continuously
  syslog_sender:
    image: syslog-sender:1.0.0
    hostname: syslog_sender
    container_name: syslog_sender
    build:
      dockerfile: $PWD/dockerfile_syslog_sender
      context: $PWD
    networks:
      - stack
    depends_on:
      - elasticsearch
      - filebeat_syslog


networks:
  stack: {}