せっかくManticoreSearchを使える状況にしたので、備忘録的に。。。
0. 環境
Python：3.9.17
FastAPI: 0.99.0
uvicorn: 0.22.0
urllib3: 1.26.6
※urllib3はSSLエラーが出るため、あえてダウングレードしています
（根本解決としてはNGだとおもいますが、取り急ぎ動作確認優先の為）
1. モジュールインストール
下記サイトに従いモジュールをインストール
https://github.com/manticoresoftware/manticoresearch-python
pip3 install manticoresearch
pip3 list #確認
Package Version
----------------- --------
manticoresearch 3.3.0
2. main.py作成
ではFastAPIでリクエストを受け付けできるようAPIを作成します。
import json
import time
import manticoresearch
from manticoresearch import *
from manticoresearch.rest import ApiException
from datetime import datetime, timedelta
from fastapi import FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
configuration = manticoresearch.Configuration(
host = "http://xx.xx.xx.xx:9308" # ManticoresearchサーバのIP
)
app = FastAPI()
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
@app.get("/manticoreapi")
def search():
with manticoresearch.ApiClient(configuration) as api_client:
api_instance = manticoresearch.SearchApi(api_client)
search_request = SearchRequest()
search_request.index='XXXXXXXXX' # テーブル(index)名を指定
api_response = api_instance.search(search_request)
results = api_response.hits.hits
return {"data": results }
3. FastAPI起動
では起動しておきましょう。
uvicorn main:app --host 0.0.0.0 --reload
4. Logstash設定
今回はNW機器のPaloaltoのログを送信するようのFilterを作成します。
vim /etc/logstash/conf.d/palo.conf
input {
stdin {} #とりあえず、標準入力から受付
}
filter {
csv {
source => "message"
columns => [
"FUTURE_USE", "ReceiveTime", "SerialNumber", "Type", "Threat_ContentType", "FUTURE_USE",
"GeneratedTime", "SourceIP", "DestinationIP", "NATSourceIP", "NATDestinationIP", "RuleName",
"SourceUser", "DestinationUser", "Application", "VirtualSystem", "SourceZone", "DestinationZone",
"InboundInterface", "OutboundInterface", "LogAction", "FUTURE_USE", "SessionID",
"RepeatCount", "SourcePort", "DestinationPort", "NATSourcePort", "NATDestinationPort", "Flags",
"Protocol", "Action", "Bytes", "BytesSent", "BytesReceived", "Packets", "StartTime", "ElapsedTime",
"Category", "FUTURE_USE", "SequenceNumber", "ActionFlags", "SourceLocation",
"DestinationLocation", "FUTURE_USE", "PacketsSent", "PacketsReceived", "SessionEndReason",
"DeviceGroupHierarchyLevel1", "DeviceGroupHierarchyLevel2", "DeviceGroupHierarchyLevel3",
"DeviceGroupHierarchyLevel4", "VirtualSystemName", "DeviceName", "ActionSource", "SourceVMUUID",
"DestinationVMUUID", "TunnelID_IMSI", "MonitorTag_IMEI", "ParentSessionID", "ParentStartTime",
"TunnelType", "SCTPAssociationID", "SCTPChunks", "SCTPChunksSent", "SCTPChunksReceived"
]
}
mutate {
remove_field => [ "message" ]
}
}
output {
stdout {
codec => rubydebug
}
elasticsearch {
index => "XXXXXXXXXX" #index(テーブル)名を指定
hosts => ["http://XX.XX.XX.XX:9308"] # ManticoresearchサーバのIP指定
ilm_enabled => false
manage_template => false
}
}
:wq
5. テスト用ログ
下記サイトのtraffic Logを使わせていただきました。
https://help.sumologic.com/docs/integrations/cloud-security-monitoring-analytics/palo-alto-firewall-10/
ただし、そのままだと文字列をIntegerに変える際エラーがでます。一旦下記のようにすべて文字列として認識させます。（このあたりはプロジェクト側への改善要求になるんでしょうね。）
★7/2追記
どうやらすでにMasterへのPushが計画されているようです。（反映結果が待ち遠しい）
https://github.com/manticoresoftware/manticoresearch/discussions/1048
#★Logstash貼り付け用 Log
"Oct 09 10:19:15 SumPunFw07.sumotest.com 1","2019/10/09 10:19:15","001234567890002","TRAFFIC","drop","2304","2019/10/09 10:19:15","209.118.103.150","160.177.222.249","0.0.0.0","0.0.0.0","InternalServer","","","not-applicable","vsys1","inside","z1-FW-Transit","ethernet1/2","","All traffic","2019/10/09 10:19:15","0","1","63712","443","0","0","0x0","udp","deny","60","60","0","1","2019/10/09 10:19:15","0","any","0","0123456789","0x0","Netherlands","10.0.0.0-10.255.255.255","0","1","0","policy-deny","0","0","0","0","","SumPunFw07","from-policy","","","0","","0","","N/A","0","0","0","0","1202585d-b4d5-5b4c-aaa2-d80d77ba456e","0"
6. Logstash起動＋データ貼り付け
では、Logstash起動しコンソールに貼り付けます。
/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/palo.conf
// 省略
[INFO ] 2023-07-02 18:10:07.984 [[main]-pipeline-manager] javapipeline - Pipeline started {"pipeline.id"=>"main"}
The stdin plugin is now waiting for input:
[INFO ] 2023-07-02 18:10:08.003 [Agent thread] agent - Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
"Oct 09 10:19:15 SumPunFw07.sumotest.com 1","2019/10/09 10:19:15","001234567890002","TRAFFIC","drop","2304","2019/10/09 10:19:15","209.118.103.150","160.177.222.249","0.0.0.0","0.0.0.0","InternalServer","","","not-applicable","vsys1","inside","z1-FW-Transit","ethernet1/2","","All traffic","2019/10/09 10:19:15","0","1","63712","443","0","0","0x0","udp","deny","60","60","0","1","2019/10/09 10:19:15","0","any","0","0123456789","0x0","Netherlands","10.0.0.0-10.255.255.255","0","1","0","policy-deny","0","0","0","0","","SumPunFw07","from-policy","","","0","","0","","N/A","0","0","0","0","1202585d-b4d5-5b4c-aaa2-d80d77ba456e","0"
{
"SourceZone" => "inside",
"NATDestinationPort" => "0",
"Category" => "any",
"RepeatCount" => "1",
"DestinationLocation" => "10.0.0.0-10.255.255.255",
"ActionSource" => "from-policy",
"ParentSessionID" => "0",
"column67" => "0",
"BytesReceived" => "0",
"ElapsedTime" => "0",
"column66" => "1202585d-b4d5-5b4c-aaa2-d80d77ba456e",
"@timestamp" => 2023-07-02T09:10:46.427Z,
"SequenceNumber" => "0123456789",
"SCTPChunks" => "0",
"NATSourceIP" => "0.0.0.0",
"NATSourcePort" => "0",
"LogAction" => "All traffic",
"SourceVMUUID" => "",
"DeviceName" => "SumPunFw07",
"SCTPChunksReceived" => "0",
"SessionEndReason" => "policy-deny",
"SourcePort" => "63712",
"ParentStartTime" => "",
"ActionFlags" => "0x0",
"SourceUser" => "",
"DeviceGroupHierarchyLevel3" => "0",
"SerialNumber" => "001234567890002",
"SourceIP" => "209.118.103.150",
"DestinationIP" => "160.177.222.249",
"OutboundInterface" => "",
"Protocol" => "udp",
"FUTURE_USE" => "0",
"Threat_ContentType" => "drop",
"VirtualSystem" => "vsys1",
"VirtualSystemName" => "",
"PacketsSent" => "1",
"PacketsReceived" => "0",
"GeneratedTime" => "2019/10/09 10:19:15",
"SCTPChunksSent" => "0",
"Type" => "TRAFFIC",
"Bytes" => "60",
"DeviceGroupHierarchyLevel4" => "0",
"host" => "manticore-logstash",
"DeviceGroupHierarchyLevel1" => "0",
"TunnelType" => "N/A",
"SessionID" => "0",
"DestinationUser" => "",
"StartTime" => "2019/10/09 10:19:15",
"DeviceGroupHierarchyLevel2" => "0",
"ReceiveTime" => "2019/10/09 10:19:15",
"Application" => "not-applicable",
"@version" => "1",
"DestinationZone" => "z1-FW-Transit",
"InboundInterface" => "ethernet1/2",
"Flags" => "0x0",
"Action" => "deny",
"NATDestinationIP" => "0.0.0.0",
"SourceLocation" => "Netherlands",
"DestinationVMUUID" => "",
"TunnelID_IMSI" => "0",
"SCTPAssociationID" => "0",
"Packets" => "1",
"DestinationPort" => "443",
"MonitorTag_IMEI" => "",
"BytesSent" => "60",
"RuleName" => "InternalServer"
}
7. FastAPIエンドポイントを叩く
ではURLをブラウザからたたきます。
下記のように表示されていれば、無事データ取得完了ですね。
あとはReactとかでブラウザからデータ表示できるようにするとかすれば、KibanaみたいなUIも構成できそうですね。