osquery Filebeat Elasticsearch Kibana - lyonwang/TechNotes GitHub Wiki

osquery Installation

Windows

• Install Chocolaty
• Install osquery

choco install osquery

• Prepare C:\ProgramData\osquery\osquery.conf from C:\ProgramData\osquery\osquery.example.conf

{
  // Configure the daemon below:
  "options": {
    // Select the osquery config plugin.
    "config_plugin": "filesystem",

    // Select the osquery logging plugin.
    "logger_plugin": "filesystem",

    // The log directory stores info, warning, and errors.
    // If the daemon uses the 'filesystem' logging retriever then the log_dir
    // will also contain the query results.
    //"logger_path": "/var/log/osquery",

    // Set 'disable_logging' to true to prevent writing any info, warning, error
    // logs. If a logging plugin is selected it will still write query results.
    //"disable_logging": "false",

    // Splay the scheduled interval for queries.
    // This is very helpful to prevent system performance impact when scheduling
    // large numbers of queries that run a smaller or similar intervals.
    //"schedule_splay_percent": "10",

    // A filesystem path for disk-based backing storage used for events and
    // query results differentials. See also 'use_in_memory_database'.
    //"database_path": "/var/osquery/osquery.db",

    // Comma-delimited list of table names to be disabled.
    // This allows osquery to be launched without certain tables.
    //"disable_tables": "foo_bar,time",

    "utc": "true"
  },

  // Define a schedule of queries:
  "schedule": {
    // This is a simple example query that outputs basic system information.
    "system_info": {
      // The exact query to run.
      "query": "SELECT hostname, cpu_brand, physical_memory FROM system_info;",
      // The interval in seconds to run this query, not an exact interval.
      "interval": 3600
    },
    // My log
    "sample_query": {
      // The exact query to run.
      "query": "SELECT DISTINCT process.name, listening.port, process.pid FROM processes AS process JOIN listening_ports AS listening ON process.pid = listening.pid WHERE listening.address = '0.0.0.0';",
      // The interval in seconds to run this query, not an exact interval.
      "interval": 10
    }
  },

  // Decorators are normal queries that append data to every query.
  "decorators": {
    "load": [
      "SELECT uuid AS host_uuid FROM system_info;",
      "SELECT user AS username FROM logged_in_users ORDER BY time DESC LIMIT 1;"
    ]
  },

  // Add default osquery packs or install your own.
  //
  // There are several 'default' packs installed with 'make install' or via
  // packages and/or Homebrew.
  //
  // Linux:        /usr/share/osquery/packs
  // OS X:         /var/osquery/packs
  // Homebrew:     /usr/local/share/osquery/packs
  // make install: {PREFIX}/share/osquery/packs
  //
  "packs": {
    // "osquery-monitoring": "/usr/share/osquery/packs/osquery-monitoring.conf",
    // "incident-response": "/usr/share/osquery/packs/incident-response.conf",
    // "it-compliance": "/usr/share/osquery/packs/it-compliance.conf",
    // "osx-attacks": "/usr/share/osquery/packs/osx-attacks.conf",
    // "vuln-management": "/usr/share/osquery/packs/vuln-management.conf",
    // "hardware-monitoring": "/usr/share/osquery/packs/hardware-monitoring.conf",
    // "ossec-rootkit": "/usr/share/osquery/packs/ossec-rootkit.conf"
  }
}

• Install osquery windows service and Start service

powershell C:\ProgramData\osquery\manage-osqueryd.ps1 -install
powershell C:\ProgramData\osquery\manage-osqueryd.ps1 -start

• Stop filebeat windows service add prospector and index

- input_type: log
 
  # Paths that should be crawled and fetched. Glob based paths.
  paths:
    - C:\ProgramData\osquery\log\osqueryd.results.log
    #- c:\programdata\elasticsearch\logs\*
 
  # Exclude lines. A list of regular expressions to match. It drops the lines that are
  # matching any regular expression from the list.
  #exclude_lines: ["^DBG"]
 
  # Include lines. A list of regular expressions to match. It exports the lines that are
  # matching any regular expression from the list.
  #include_lines: ["^ERR", "^WARN"]
 
  # Exclude files. A list of regular expressions to match. Filebeat drops the files that
  # are matching any regular expression from the list. By default, no files are dropped.
  #exclude_files: [".gz$"]
 
  # Optional additional fields. These field can be freely picked
  # to add additional information to the crawled log files for filtering
  fields:
    application: osquery
    host: UAT-Zeus-02
    type: osquery_json
    codec: json     
  #  review: 1
  json.message_key: log
  fields_under_root: true

#-------------------------- Elasticsearch output ------------------------------
output.elasticsearch:
  # Array of hosts to connect to.
  hosts: ["172.20.5.61:9200", "172.20.5.62:9200"]
  index: "others-%{+yyyy.MM.dd}"
  indices:
    - index: "osquery-%{+yyyy.MM.dd}"
      when.contains:
        type: "osquery_json"

• Restart Filebeat windows service