Configurations - mata-elang-stable/MataElang-Platform GitHub Wiki

services:
  snort:
    environment:
      - NETWORK_INTERFACE=eth1

  snort-parser:
    image: mataelang/snort3-parser:1.1
    environment:
      - MQTT_HOST=172.16.2.30
      - MQTT_PORT=1883
      - MQTT_USERNAME=mataelang
      - MQTT_PASSWORD=mataelang
      - MAX_PCAP_FILES=5
      - SENSOR_ID=<machine-id>
      - MQTT_TOPIC=mataelang/sensor/v3/<sensor-id>
      - SNORT_ALERT_FILE_PATH=/var/log/snort/alert_json.txt

---------------------------------------------------------------------------
-- 1. configure defaults
---------------------------------------------------------------------------

-- HOME_NET and EXTERNAL_NET must be set now
-- setup the network addresses you are protecting
HOME_NET = 'any'

-- set up the external network addresses.
-- (leave as "any" in most situations)
EXTERNAL_NET = 'any'

---------------------------------------------------------------------------
-- 7. configure outputs
---------------------------------------------------------------------------

alert_json = {
    file = true,
    limit = 100,
}

log_pcap = {
    limit = 100, -- in MBytes
}

# Your Snort oinkcode is required for snort/talos Subscription, Light_SPD, and Registered rulesets
oinkcode = <your-oinkcode>

FROM mataelang/snort-base:3.1.47.0

source-directory /etc/network/interfaces.d

auto eth1
iface eth1 inet manual
  address 0.0.0.0/0
  up ip link set eth1 promisc on
  down ip link set eth1 promisc off

auto eth2
iface eth2 inet static
  address 172.16.2.10/24

services:
  mosquitto:
    image: eclipse-mosquitto:2.0.15
    ports:
      - 1883:1883

listener 1883
allow_anonymous false
persistence true
persistence_location /mosquitto/data/
autosave_interval 60
queue_qos0_messages true
max_queued_messages 100000
max_inflight_messages 10000

connection_messages true
log_dest stdout

persistent_client_expiration 1h

password_file /mosquitto/config/password_file

services:
  mqtt-source:
    image: mataelang/kafka-mqtt-source:1.1
    container_name: mqtt-source
    environment:
      MQTT_HOST: 172.17.0.1
      MQTT_PORT: 1883
      MQTT_USERNAME: mataelang
      MQTT_PASSWORD: mataelang
      MQTT_TOPIC: mataelang/sensor/v3/+
      KAFKA_BOOSTRAP_SERVERS: kafka:9092
      KAFKA_PRODUCE_TOPIC: sensor_events
    deploy:
      resources:
        limits:
          cpus: '0.5'
          memory: 64M
        reservations:
          cpus: '0.25'
          memory: 32M

  zookeeper:
    image: confluentinc/cp-zookeeper:7.3.0
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    volumes:
      - zookeeper_data:/var/lib/zookeeper/data
      - zookeeper_log:/var/lib/zookeeper/log
    deploy:
      mode: replicated
      replicas: 1
      restart_policy:
        condition: on-failure
      resources:
        limits:
          cpus: '0.5'
          memory: 512M
        reservations:
          cpus: '0.25'
          memory: 256M

  kafka:
    image: confluentinc/cp-kafka:7.3.0
    depends_on:
      - zookeeper
    ports:
      - target: 9093
        published: 9093
        protocol: tcp
        mode: host
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,EXTERNAL://172.17.0.1:9093
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_LOG_SEGMENT_BYTES: 536870912   # 512MB
      KAFKA_LOG_RETENTION_BYTES: 536870912 # 512MB
      KAFKA_LOG_RETENTION_MS: 86400000     # 1 day
    volumes:
      - kafka_data:/var/lib/kafka/data
    deploy:
      mode: replicated
      replicas: 1
      resources:
        limits:
          cpus: '0.5'
          memory: 2G
        reservations:
          cpus: '0.25'
          memory: 1G

  control-center:
    image: provectuslabs/kafka-ui
    container_name: control-center
    depends_on:
      - zookeeper
      - kafka
      - mqtt-source
    ports:
      - "9021:8080"
    environment:
      KAFKA_CLUSTERS_0_NAME: MataElangKafkaCluster
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9092
      KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper:2181
    deploy:
      resources:
        limits:
          cpus: '0.50'
          memory: 768M
        reservations:
          cpus: '0.25'
          memory: 384M

### Append to the end of the file.
export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64
export HADOOP_HOME=/usr/local/hadoop
export PATH=$PATH:$HADOOP_HOME/sbin:$HADOOP_HOME/bin

### Line 55: Change JAVA_HOME.
# export JAVA_HOME=
export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
  <property>
    <name>fs.defaultFS</name>
    <value>hdfs://172.17.0.1:9000</value>
  </property>
  <property>
    <name>io.file.buffre.size</name>
    <value>65536</value>
  </property>
</configuration>

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
  <property>
    <name>dfs.namenode.name.dir</name>
    <value>/home/ubuntu/hadoop/dfs/name</value>
  </property>
  <property>
    <name>dfs.datanode.data.dir</name>
    <value>/home/ubuntu/hadoop/dfs/data</value>
  </property>
  <property>
    <name>dfs.replication</name>
    <value>1</value>
  </property>
  <property>
    <name>dfs.namenode.rpc-bind-host</name>
    <value>0.0.0.0</value>
  </property>
</configuration>

HADOOP_USER_NAME=ubuntu
SPARK_EVENTLOG_DIR=hdfs://172.17.0.1:9000/user/ubuntu/spark/spark-events
SPARK_APP_JAR_PATH=hdfs://172.17.0.1:9000/user/ubuntu/kaspacore/files/kaspacore.jar
SPARK_HISTORY_OPTS="-Dspark.history.fs.logDirectory=hdfs://172.17.0.1:9000/user/ubuntu/spark/spark-events"

SPARK_MASTER=spark://spark-master:7077
SPARK_CHECKPOINT_PATH=hdfs://172.17.0.1:9000/user/ubuntu/kafka-checkpoint
TIMEZONE=UTC

KAFKA_BOOTSTRAP_SERVERS=172.17.0.1:9093
KAFKA_INPUT_STARTING_OFFSETS=latest

SENSOR_STREAM_INPUT_TOPIC=sensor_events
SENSOR_STREAM_OUTPUT_TOPIC=sensor_events_with_geoip

MAXMIND_DB_PATH=hdfs://172.17.0.1:9000/user/ubuntu/kaspacore/files/GeoLite2-City.mmdb
MAXMIND_DB_FILENAME=GeoLite2-City.mmdb

# Worker
spark.worker.cleanup.enabled=true
spark.worker.cleanup.interval=1800
spark.worker.cleanup.appDataTtl=14400

# History Server
spark.history.ui.port=18080
spark.history.retainedApplications=10
spark.history.fs.update.interval=10s
spark.history.fs.cleaner.enabled=true
spark.history.fs.cleaner.interval=1d
spark.history.fs.cleaner.maxAge=7d

# App Configuration
spark.master=spark://spark-master:7077
spark.eventLog.enabled=true

log4j.rootLogger=ERROR, console

# set the log level for these components
log4j.logger.com.test=DEBUG
log4j.logger.org=ERROR
log4j.logger.org.apache.spark=ERROR
log4j.logger.org.spark-project=ERROR
log4j.logger.org.apache.hadoop=ERROR
log4j.logger.io.netty=ERROR
log4j.logger.org.apache.zookeeper=ERROR

# add a ConsoleAppender to the logger stdout to write to the console
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.layout=org.apache.log4j.PatternLayout
# use a simple message format
log4j.appender.console.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n

### Append to the end of the file.
vm.max_map_count=262144

input {
   kafka {
     bootstrap_servers => "172.17.0.1:9093"
     topics => "event_all_10s"
   }
 }
filter {
	json {
		source => "message"
		remove_field => ["message", "event"]	
	}
}
 output {
   opensearch {
     hosts => ["https://opensearch-node1:9200", "https://opensearch-node2:9200"]
     index => "event-all-10s-%{+YYYY.MM.dd}"
     user => "admin"
     password => "admin"
     ssl => true
     ssl_certificate_verification => false
   }
 }

### Line 117: Set the IP or DNS names of the Zabbix server
#Server=127.0.0.1
Server=<ZABBIX_SERVER_IP_OR_NAME (e.g. 172.16.2.110)>,172.16.238.0/24

### Line 182: Set the hostname of the Zabbix agent.
#Hostname=Zabbix server
Hostname=<HOSTNAME_OF_AGENT (e.g. zabbix)>