parsing

geoip processor ingest pipeline

Use the GeoLite2-ASN.mmdb file with the GeoIP processor to get ASN info.

remove empty fields

Shamelessly stolen from elastic discuss

using painless possibly in an ingest pipeline:

- script:
   description: Drops null/empty values recursively
   lang: painless
   source: |
     boolean drop(Object o) {
       if (o == null || o == "") {
         return true;
       } else if (o instanceof Map) {
         ((Map) o).values().removeIf(v -> drop(v));
         return (((Map) o).size() == 0);
       } else if (o instanceof List) {
         ((List) o).removeIf(v -> drop(v));
         return (((List) o).length == 0);
       }
       return false;
     }
     drop(ctx);

in logstash with ruby:

filter {
 ruby {
    init => "
    def remove_empty_fields(event)
      _find_empty_fields(event.to_hash) { |path| event.remove('[' + path.join('][') + ']') }
    end

    def _find_empty_fields(event, path = [], &blk)
      event.each do |k, v|
        curpath = path + [k]
        case v
        when nil
          yield curpath
        when ''
          yield curpath
        when []
          yield curpath
        when {}
          yield curpath
        when Hash
          _find_empty_fields(v, curpath, &blk)
        end
      end
    end
    "
    code => "remove_empty_fields(event)"
 }
}