lmd-ml.json

{
    "attributes": {
        "id": "lmd-ml",
        "title": "Lateral Movement Detection",
        "description": "Detects lateral movement activity by identifying malicious file transfers and suspicious RDP sessions in an environment.",
        "type": "lmd",
        "logo": {
            "icon": "machineLearningApp"
        },
        "query": {
            "bool": {
                "should": [
                    {
                        "term": {
                            "event.category": "file"
                        }
                    },
                    {
                        "bool": {
                            "filter": [
                                {
                                    "exists": {
                                        "field": "session.start_time"
                                    }
                                }
                            ]
                        }
                    }
                ],
                "must_not": {
                    "terms": {
                        "_tier": [
                            "data_frozen",
                            "data_cold"
                        ]
                    }
                }
            }
        },
        "jobs": [
            {
                "id": "lmd_high_count_remote_file_transfer",
                "config": {
                    "groups": [
                        "security",
                        "lateral_movement"
                    ],
                    "description": "Detects unusually high file transfers to a remote host in the network.",
                    "analysis_config": {
                        "bucket_span": "1h",
                        "detectors": [
                            {
                                "detector_description": "high_count by \"event.action\" partitionfield=\"host.name\"",
                                "function": "high_count",
                                "by_field_name": "event.action",
                                "partition_field_name": "host.name",
                                "detector_index": 0
                            }
                        ],
                        "influencers": [
                            "host.name",
                            "user.name",
                            "process.name"
                        ]
                    },
                    "data_description": {
                        "time_field": "@timestamp",
                        "time_format": "epoch_ms"
                    },
                    "custom_settings": {
                        "created_by": "ml-module-lmd"
                    }
                }
            },
            {
                "id": "lmd_high_file_size_remote_file_transfer",
                "config": {
                    "groups": [
                        "security",
                        "lateral_movement"
                    ],
                    "description": "Detects unusually high size of files shared with a remote host in the network.",
                    "analysis_config": {
                        "bucket_span": "1h",
                        "detectors": [
                            {
                                "detector_description": "high_sum(\"file.size\") partitionfield=\"host.name\"",
                                "function": "high_sum",
                                "field_name": "file.size",
                                "partition_field_name": "host.name",
                                "detector_index": 0
                            }
                        ],
                        "influencers": [
                            "host.name",
                            "user.name",
                            "process.name"
                        ]
                    },
                    "data_description": {
                        "time_field": "@timestamp",
                        "time_format": "epoch_ms"
                    },
                    "custom_settings": {
                        "created_by": "ml-module-lmd"
                    }
                }
            },
            {
                "id": "lmd_rare_file_extension_remote_transfer",
                "config": {
                    "groups": [
                        "security",
                        "lateral_movement"
                    ],
                    "description": "Detects rare file extensions shared with a remote host in the network.",
                    "analysis_config": {
                        "bucket_span": "15m",
                        "detectors": [
                            {
                                "detector_description": "rare by \"file.extension\" partitionfield=\"host.name\"",
                                "function": "rare",
                                "by_field_name": "file.extension",
                                "partition_field_name": "host.name",
                                "detector_index": 0
                            }
                        ],
                        "influencers": [
                            "host.name",
                            "user.name",
                            "file.name"
                        ]
                    },
                    "data_description": {
                        "time_field": "@timestamp",
                        "time_format": "epoch_ms"
                    },
                    "custom_settings": {
                        "created_by": "ml-module-lmd"
                    }
                }
            },
            {
                "id": "lmd_rare_file_path_remote_transfer",
                "config": {
                    "groups": [
                        "security",
                        "lateral_movement"
                    ],
                    "description": "Detects unusual folders and directories on which a file is transferred.",
                    "analysis_config": {
                        "bucket_span": "15m",
                        "detectors": [
                            {
                                "detector_description": "rare by file_directory partitionfield=\"host.name\"",
                                "function": "rare",
                                "by_field_name": "file_directory",
                                "partition_field_name": "host.name",
                                "detector_index": 0
                            }
                        ],
                        "influencers": [
                            "host.name",
                            "user.name",
                            "file.path"
                        ]
                    },
                    "data_description": {
                        "time_field": "@timestamp",
                        "time_format": "epoch_ms"
                    },
                    "custom_settings": {
                        "created_by": "ml-module-lmd"
                    }
                }
            },
            {
                "id": "lmd_high_mean_rdp_session_duration",
                "config": {
                    "groups": [
                        "security",
                        "lateral_movement"
                    ],
                    "description": "Detects unusually high mean of RDP session duration.",
                    "analysis_config": {
                        "bucket_span": "6h",
                        "detectors": [
                            {
                                "detector_description": "high_mean(session.duration) partitionfield=\"source.ip\"",
                                "function": "high_mean",
                                "field_name": "session.duration",
                                "partition_field_name": "source.ip",
                                "detector_index": 0
                            },
                            {
                                "detector_description": "high_mean(session.duration) partitionfield=\"destination.ip\"",
                                "function": "high_mean",
                                "field_name": "session.duration",
                                "partition_field_name": "destination.ip",
                                "detector_index": 1
                            }
                        ],
                        "influencers": [
                            "host.name",
                            "user.name",
                            "source.ip",
                            "destination.ip"
                        ]
                    },
                    "data_description": {
                        "time_field": "session.start_time",
                        "time_format": "epoch_ms"
                    },
                    "custom_settings": {
                        "created_by": "ml-module-lmd"
                    }
                }
            },
            {
                "id": "lmd_high_var_rdp_session_duration",
                "config": {
                    "groups": [
                        "security",
                        "lateral_movement"
                    ],
                    "description": "Detects unusually high variance in RDP session duration.",
                    "analysis_config": {
                        "bucket_span": "6h",
                        "detectors": [
                            {
                                "detector_description": "high_varp(session.duration) partitionfield=\"source.ip\"",
                                "function": "high_varp",
                                "field_name": "session.duration",
                                "partition_field_name": "source.ip",
                                "detector_index": 0
                            },
                            {
                                "detector_description": "high_varp(session.duration) partitionfield=\"destination.ip\"",
                                "function": "high_varp",
                                "field_name": "session.duration",
                                "partition_field_name": "destination.ip",
                                "detector_index": 1
                            }
                        ],
                        "influencers": [
                            "host.name",
                            "user.name",
                            "source.ip",
                            "destination.ip"
                        ]
                    },
                    "data_description": {
                        "time_field": "session.start_time",
                        "time_format": "epoch_ms"
                    },
                    "custom_settings": {
                        "created_by": "ml-module-lmd"
                    }
                }
            },
            {
                "id": "lmd_high_sum_rdp_number_of_processes",
                "config": {
                    "groups": [
                        "security",
                        "lateral_movement"
                    ],
                    "description": "Detects unusually high number of processes started in a single RDP session.",
                    "analysis_config": {
                        "bucket_span": "6h",
                        "detectors": [
                            {
                                "detector_description": "high_sum(number_processes_per_session) partitionfield=\"source.ip\"",
                                "function": "high_sum",
                                "field_name": "number_processes_per_session",
                                "partition_field_name": "source.ip",
                                "detector_index": 0
                            },
                            {
                                "detector_description": "high_sum(number_processes_per_session) partitionfield=\"destination.ip\"",
                                "function": "high_sum",
                                "field_name": "number_processes_per_session",
                                "partition_field_name": "destination.ip",
                                "detector_index": 1
                            }
                        ],
                        "influencers": [
                            "host.name",
                            "user.name",
                            "source.ip",
                            "destination.ip"
                        ]
                    },
                    "data_description": {
                        "time_field": "session.start_time",
                        "time_format": "epoch_ms"
                    },
                    "custom_settings": {
                        "created_by": "ml-module-lmd"
                    }
                }
            },
            {
                "id": "lmd_unusual_time_weekday_rdp_session_start",
                "config": {
                    "groups": [
                        "security",
                        "lateral_movement"
                    ],
                    "description": "Detects an RDP session started at an usual time or weekday.",
                    "analysis_config": {
                        "bucket_span": "6h",
                        "detectors": [
                            {
                                "detector_description": "time_of_day partitionfield=\"source.ip\"",
                                "function": "time_of_day",
                                "partition_field_name": "source.ip",
                                "detector_index": 0
                            },
                            {
                                "detector_description": "time_of_week partitionfield=\"source.ip\"",
                                "function": "time_of_week",
                                "partition_field_name": "source.ip",
                                "detector_index": 1
                            }
                        ],
                        "influencers": [
                            "host.name",
                            "user.name",
                            "destination.ip",
                            "source.ip"
                        ]
                    },
                    "data_description": {
                        "time_field": "session.start_time",
                        "time_format": "epoch_ms"
                    },
                    "custom_settings": {
                        "created_by": "ml-module-lmd"
                    }
                }
            },
            {
                "id": "lmd_high_rdp_distinct_count_source_ip_for_destination",
                "config": {
                    "groups": [
                        "security",
                        "lateral_movement"
                    ],
                    "description": "Detects a high count of source IPs making an RDP connection with a single destination IP.",
                    "analysis_config": {
                        "bucket_span": "6h",
                        "detectors": [
                            {
                                "detector_description": "high_distinct_count(\"source.ip\") partitionfield=\"destination.ip\"",
                                "function": "high_distinct_count",
                                "field_name": "source.ip",
                                "partition_field_name": "destination.ip",
                                "detector_index": 0
                            }
                        ],
                        "influencers": [
                            "host.name",
                            "user.name",
                            "destination.ip"
                        ]
                    },
                    "data_description": {
                        "time_field": "session.start_time",
                        "time_format": "epoch_ms"
                    },
                    "custom_settings": {
                        "created_by": "ml-module-lmd"
                    }
                }
            },
            {
                "id": "lmd_high_rdp_distinct_count_destination_ip_for_source",
                "config": {
                    "groups": [
                        "security",
                        "lateral_movement"
                    ],
                    "description": "Detects a high count of destination IPs establishing an RDP connection with a single source IP.",
                    "analysis_config": {
                        "bucket_span": "6h",
                        "detectors": [
                            {
                                "detector_description": "high_distinct_count(\"destination.ip\") partitionfield=\"source.ip\"",
                                "function": "high_distinct_count",
                                "field_name": "destination.ip",
                                "partition_field_name": "source.ip",
                                "detector_index": 0
                            }
                        ],
                        "influencers": [
                            "host.name",
                            "user.name",
                            "source.ip"
                        ]
                    },
                    "data_description": {
                        "time_field": "session.start_time",
                        "time_format": "epoch_ms"
                    },
                    "custom_settings": {
                        "created_by": "ml-module-lmd"
                    }
                }
            },
            {
                "id": "lmd_high_mean_rdp_process_args",
                "config": {
                    "groups": [
                        "security",
                        "lateral_movement"
                    ],
                    "description": "Detects unusually high number of process arguments in an RDP session.",
                    "analysis_config": {
                        "bucket_span": "6h",
                        "detectors": [
                            {
                                "detector_description": "high_mean(total_length_process_args) partitionfield=\"source.ip\"",
                                "function": "high_mean",
                                "field_name": "total_length_process_args",
                                "partition_field_name": "source.ip",
                                "detector_index": 0
                            },
                            {
                                "detector_description": "high_mean(total_length_process_args) partitionfield=\"destination.ip\"",
                                "function": "high_mean",
                                "field_name": "total_length_process_args",
                                "partition_field_name": "destination.ip",
                                "detector_index": 1
                            }
                        ],
                        "influencers": [
                            "host.name",
                            "user.name",
                            "source.ip",
                            "destination.ip"
                        ]
                    },
                    "data_description": {
                        "time_field": "session.start_time",
                        "time_format": "epoch_ms"
                    },
                    "custom_settings": {
                        "created_by": "ml-module-lmd"
                    }
                }
            }
        ],
        "datafeeds": [
            {
                "id": "datafeed-lmd_high_count_remote_file_transfer",
                "job_id": "lmd_high_count_remote_file_transfer",
                "config": {
                    "indices": [
                        "INDEX_PATTERN_NAME"
                    ],
                    "job_id": "lmd_high_count_remote_file_transfer",
                    "query": {
                        "bool": {
                            "must_not": [
                                {
                                    "terms": {
                                        "user.name": [
                                            "SYSTEM",
                                            "root"
                                        ]
                                    }
                                }
                            ],
                            "filter": [
                                {
                                    "term": {
                                        "event.category": "file"
                                    }
                                },
                                {
                                    "term": {
                                        "event.action": "creation"
                                    }
                                },
                                {
                                    "terms": {
                                        "process.name": [
                                            "System",
                                            "scp",
                                            "sshd",
                                            "smbd",
                                            "vsftpd",
                                            "sftp-server"
                                        ]
                                    }
                                },
                                {
                                    "exists": {
                                        "field": "process.name"
                                    }
                                },
                                {
                                    "exists": {
                                        "field": "file.name"
                                    }
                                }
                            ]
                        }
                    }
                }
            },
            {
                "id": "datafeed-lmd_high_file_size_remote_file_transfer",
                "job_id": "lmd_high_file_size_remote_file_transfer",
                "config": {
                    "indices": [
                        "INDEX_PATTERN_NAME"
                    ],
                    "job_id": "lmd_high_file_size_remote_file_transfer",
                    "query": {
                        "bool": {
                            "must_not": [
                                {
                                    "terms": {
                                        "user.name": [
                                            "SYSTEM",
                                            "root"
                                        ]
                                    }
                                }
                            ],
                            "filter": [
                                {
                                    "term": {
                                        "event.category": "file"
                                    }
                                },
                                {
                                    "term": {
                                        "event.action": "creation"
                                    }
                                },
                                {
                                    "terms": {
                                        "process.name": [
                                            "System",
                                            "scp",
                                            "sshd",
                                            "smbd",
                                            "vsftpd",
                                            "sftp-server"
                                        ]
                                    }
                                },
                                {
                                    "exists": {
                                        "field": "process.name"
                                    }
                                },
                                {
                                    "exists": {
                                        "field": "file.name"
                                    }
                                }
                            ]
                        }
                    }
                }
            },
            {
                "id": "datafeed-lmd_rare_file_extension_remote_transfer",
                "job_id": "lmd_rare_file_extension_remote_transfer",
                "config": {
                    "indices": [
                        "INDEX_PATTERN_NAME"
                    ],
                    "job_id": "lmd_rare_file_extension_remote_transfer",
                    "query": {
                        "bool": {
                            "must_not": [
                                {
                                    "terms": {
                                        "user.name": [
                                            "SYSTEM",
                                            "root"
                                        ]
                                    }
                                }
                            ],
                            "filter": [
                                {
                                    "term": {
                                        "event.category": "file"
                                    }
                                },
                                {
                                    "term": {
                                        "event.action": "creation"
                                    }
                                },
                                {
                                    "terms": {
                                        "process.name": [
                                            "System",
                                            "scp",
                                            "sshd",
                                            "smbd",
                                            "vsftpd",
                                            "sftp-server"
                                        ]
                                    }
                                },
                                {
                                    "exists": {
                                        "field": "process.name"
                                    }
                                },
                                {
                                    "exists": {
                                        "field": "file.name"
                                    }
                                }
                            ]
                        }
                    }
                }
            },
            {
                "id": "datafeed-lmd_high_mean_rdp_session_duration",
                "job_id": "lmd_high_mean_rdp_session_duration",
                "config": {
                    "indices": [
                        "INDEX_PATTERN_NAME"
                    ],
                    "job_id": "lmd_high_mean_rdp_session_duration",
                    "query": {
                        "bool": {
                            "filter": [
                                {
                                    "exists": {
                                        "field": "source.ip"
                                    }
                                },
                                {
                                    "exists": {
                                        "field": "destination.ip"
                                    }
                                },
                                {
                                    "exists": {
                                        "field": "session.start_time"
                                    }
                                }
                            ]
                        }
                    }
                }
            },
            {
                "id": "datafeed-lmd_high_var_rdp_session_duration",
                "job_id": "lmd_high_var_rdp_session_duration",
                "config": {
                    "indices": [
                        "INDEX_PATTERN_NAME"
                    ],
                    "job_id": "lmd_high_var_rdp_session_duration",
                    "query": {
                        "bool": {
                            "filter": [
                                {
                                    "exists": {
                                        "field": "source.ip"
                                    }
                                },
                                {
                                    "exists": {
                                        "field": "destination.ip"
                                    }
                                },
                                {
                                    "exists": {
                                        "field": "session.start_time"
                                    }
                                }
                            ]
                        }
                    }
                }
            },
            {
                "id": "datafeed-lmd_high_sum_rdp_number_of_processes",
                "job_id": "lmd_high_sum_rdp_number_of_processes",
                "config": {
                    "indices": [
                        "INDEX_PATTERN_NAME"
                    ],
                    "job_id": "lmd_high_sum_rdp_number_of_processes",
                    "query": {
                        "bool": {
                            "filter": [
                                {
                                    "exists": {
                                        "field": "source.ip"
                                    }
                                },
                                {
                                    "exists": {
                                        "field": "destination.ip"
                                    }
                                },
                                {
                                    "exists": {
                                        "field": "session.start_time"
                                    }
                                }
                            ]
                        }
                    }
                }
            },
            {
                "id": "datafeed-lmd_unusual_time_weekday_rdp_session_start",
                "job_id": "lmd_unusual_time_weekday_rdp_session_start",
                "config": {
                    "indices": [
                        "INDEX_PATTERN_NAME"
                    ],
                    "job_id": "lmd_unusual_time_weekday_rdp_session_start",
                    "query": {
                        "bool": {
                            "filter": [
                                {
                                    "exists": {
                                        "field": "source.ip"
                                    }
                                },
                                {
                                    "exists": {
                                        "field": "destination.ip"
                                    }
                                },
                                {
                                    "exists": {
                                        "field": "session.start_time"
                                    }
                                }
                            ]
                        }
                    }
                }
            },
            {
                "id": "datafeed-lmd_high_rdp_distinct_count_source_ip_for_destination",
                "job_id": "lmd_high_rdp_distinct_count_source_ip_for_destination",
                "config": {
                    "indices": [
                        "INDEX_PATTERN_NAME"
                    ],
                    "job_id": "lmd_high_rdp_distinct_count_source_ip_for_destination",
                    "query": {
                        "bool": {
                            "filter": [
                                {
                                    "exists": {
                                        "field": "source.ip"
                                    }
                                },
                                {
                                    "exists": {
                                        "field": "destination.ip"
                                    }
                                },
                                {
                                    "exists": {
                                        "field": "session.start_time"
                                    }
                                }
                            ]
                        }
                    }
                }
            },
            {
                "id": "datafeed-lmd_high_rdp_distinct_count_destination_ip_for_source",
                "job_id": "lmd_high_rdp_distinct_count_destination_ip_for_source",
                "config": {
                    "indices": [
                        "INDEX_PATTERN_NAME"
                    ],
                    "job_id": "lmd_high_rdp_distinct_count_destination_ip_for_source",
                    "query": {
                        "bool": {
                            "filter": [
                                {
                                    "exists": {
                                        "field": "source.ip"
                                    }
                                },
                                {
                                    "exists": {
                                        "field": "destination.ip"
                                    }
                                },
                                {
                                    "exists": {
                                        "field": "session.start_time"
                                    }
                                }
                            ]
                        }
                    }
                }
            },
            {
                "id": "datafeed-lmd_high_mean_rdp_process_args",
                "job_id": "lmd_high_mean_rdp_process_args",
                "config": {
                    "indices": [
                        "INDEX_PATTERN_NAME"
                    ],
                    "job_id": "lmd_high_mean_rdp_process_args",
                    "query": {
                        "bool": {
                            "filter": [
                                {
                                    "exists": {
                                        "field": "source.ip"
                                    }
                                },
                                {
                                    "exists": {
                                        "field": "destination.ip"
                                    }
                                },
                                {
                                    "exists": {
                                        "field": "session.start_time"
                                    }
                                }
                            ]
                        }
                    }
                }
            },
            {
                "id": "datafeed-lmd_rare_file_path_remote_transfer",
                "job_id": "lmd_rare_file_path_remote_transfer",
                "config": {
                    "indices": [
                        "INDEX_PATTERN_NAME"
                    ],
                    "job_id": "lmd_rare_file_path_remote_transfer",
                    "query": {
                        "bool": {
                            "must_not": [
                                {
                                    "terms": {
                                        "user.name": [
                                            "SYSTEM",
                                            "root"
                                        ]
                                    }
                                }
                            ],
                            "filter": [
                                {
                                    "term": {
                                        "event.category": "file"
                                    }
                                },
                                {
                                    "term": {
                                        "event.action": "creation"
                                    }
                                },
                                {
                                    "terms": {
                                        "process.name": [
                                            "System",
                                            "scp",
                                            "sshd",
                                            "smbd",
                                            "vsftpd",
                                            "sftp-server"
                                        ]
                                    }
                                },
                                {
                                    "exists": {
                                        "field": "process.name"
                                    }
                                },
                                {
                                    "exists": {
                                        "field": "file.name"
                                    }
                                }
                            ]
                        }
                    },
                    "runtime_mappings": {
                        "file_directory": {
                            "type": "keyword",
                            "script": {
                                "source": "def st=new String();\r\ndef st1=new String(); \r\nif(doc.containsKey('file.path') && doc['file.path'].size() != 0 )\r\n{st=doc['file.path'].value.replace('/','\\\\').splitOnToken('\\\\')[-1]; \r\nst1=doc['file.path'].value.replace(st,\"\");\r\n emit((st1));} else { emit('None');}"
                            }
                        }
                    }
                }
            }
        ]
    },
    "id": "lmd-ml",
    "migrationVersion": {
        "search": "7.16.0"
    },
    "references": [],
    "type": "ml-module"
}