{"id":1810,"date":"2020-12-20T12:55:23","date_gmt":"2020-12-20T11:55:23","guid":{"rendered":"https:\/\/blog.mhasin.eu\/?p=1810"},"modified":"2021-02-10T07:56:44","modified_gmt":"2021-02-10T06:56:44","slug":"nfstream-ubuntu-monitor-send-to-elasticsearch","status":"publish","type":"post","link":"https:\/\/blog.mhasin.eu\/?p=1810","title":{"rendered":"nfstream ubuntu monitor"},"content":{"rendered":"\n<p>Ubuntu optimize:<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>ethtool -G ens161 rx 4096 tx 4096\nip link set ens161 txqueuelen 100000\nsysctl -w net.core.rmem_max=$((1024*1024*16))<\/code><\/pre>\n\n\n\n<p>sysctl<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>fs.aio-max-nr = 524288\nfs.file-max = 611160\nkernel.msgmax = 131072\nkernel.msgmnb = 131072\nkernel.panic = 15\nkernel.pid_max = 65536\nkernel.printk = 4 4 1 7\nkernel.sysrq = 0\nnet.core.default_qdisc = fq\nnet.core.netdev_max_backlog = 262144\nnet.core.optmem_max = 16777216\nnet.core.somaxconn = 65535\nnet.core.wmem_max = 16777216\nnet.ipv4.conf.all.accept_redirects = 0\nnet.ipv4.conf.all.log_martians = 1\nnet.ipv4.conf.all.rp_filter = 1\nnet.ipv4.conf.all.secure_redirects = 0\nnet.ipv4.conf.all.send_redirects = 0\nnet.ipv4.conf.default.accept_redirects = 0\nnet.ipv4.conf.default.accept_source_route = 0\nnet.ipv4.conf.default.rp_filter = 1\nnet.ipv4.conf.default.secure_redirects = 0\nnet.ipv4.conf.default.send_redirects = 0\nnet.ipv4.ip_forward = 0\nnet.ipv4.ip_local_port_range = 1024 65535\nnet.ipv4.tcp_congestion_control = bbr\nnet.ipv4.tcp_fin_timeout = 10\nnet.ipv4.tcp_keepalive_intvl = 10\nnet.ipv4.tcp_keepalive_probes = 5\nnet.ipv4.tcp_keepalive_time = 60\nnet.ipv4.tcp_low_latency = 1\nnet.ipv4.tcp_max_orphans = 10000\nnet.ipv4.tcp_max_syn_backlog = 65000\nnet.ipv4.tcp_max_tw_buckets = 1440000\nnet.ipv4.tcp_moderate_rcvbuf = 1\nnet.ipv4.tcp_no_metrics_save = 1\nnet.ipv4.tcp_notsent_lowat = 16384\nnet.ipv4.tcp_rfc1337 = 1\nnet.ipv4.tcp_rmem = 4096 87380 16777216\nnet.ipv4.tcp_sack = 0\nnet.ipv4.tcp_slow_start_after_idle = 0\nnet.ipv4.tcp_synack_retries = 2\nnet.ipv4.tcp_syncookies = 1\nnet.ipv4.tcp_syn_retries = 2\nnet.ipv4.tcp_timestamps = 0\nnet.ipv4.tcp_tw_reuse = 1\nnet.ipv4.tcp_window_scaling = 0\nnet.ipv4.tcp_wmem = 4096 65536 16777216\nnet.ipv6.conf.all.disable_ipv6 = 1\nnet.ipv6.conf.default.disable_ipv6 = 1\nnet.ipv6.conf.lo.disable_ipv6 = 1\nvm.dirty_background_ratio = 2\nvm.dirty_ratio = 60\nvm.max_map_count = 262144\nvm.overcommit_memory = 1\nvm.swappiness = 1<\/code><\/pre>\n\n\n\n<p>NFSTREAM:<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>from nfstream import NFStreamer\nmy_capture_streamer = NFStreamer(source=args.nic, statistical_analysis=True, splt_analysis=1 )\n\nfor flow in my_capture_streamer:\n                NFEntry={\n                        \"epoch_first_seen\":flow.bidirectional_first_seen_ms,\n                        \"epoch_last_seen\":flow.bidirectional_last_seen_ms,\n                        \"timedate_first_seen\":datetime_to_int(epoch_to_time(flow.bidirectional_first_seen_ms)),\n                        \"timedate_last_seen\":datetime_to_int(epoch_to_time(flow.bidirectional_last_seen_ms)),\n                        \"time_duration\":flow.bidirectional_duration_ms\/1000,\n                        \"version\":flow.ip_version,\n                        \"protocol\":flow.protocol,\n                        \"src_ip\":flow.src_ip,\n                        \"src_port\":flow.src_port,\n                        \"dst_ip\":flow.dst_ip,\n                        \"dst_port\":flow.dst_port,\n                        \"dst_mac\":flow.dst_mac,\n                        \"vlan_id\":flow.vlan_id,\n                        \"src_mac\":flow.src_mac,\n                        \"whois_destination_name\": whois_cache.get(flow.dst_ip)&#91;0],\n                        \"whois_destination_cidr\": whois_cache.get(flow.dst_ip)&#91;1],\n                        #\"process_name\":''.join(find_process),\n                        #\"process_pid\":find_pid&#91;0],\n                        #\"process_name\":''.join(retrieve_process_info(int(find_pid&#91;0]))&#91;0]),\n                        #\"process_exe\":''.join(retrieve_process_info(int(find_pid&#91;0]))&#91;1]),\n                        #\"process_cmd_line\":' '.join(retrieve_process_info(int(find_pid&#91;0]))&#91;2])&#91;0:255],\n                        #\"process_parent_pid\":retrieve_process_info(int(find_pid&#91;0]))&#91;3],\n                        #\"process_parent_name\":''.join(retrieve_process_info(int(find_pid&#91;0]))&#91;4]),\n                        #\"process_create_time\":retrieve_process_info(int(find_pid&#91;0]))&#91;5],\n                        \"total_packets\":flow.bidirectional_packets,\n                        \"total_bytes\":flow.bidirectional_bytes,\n                        \"src2dst_packets\":flow.src2dst_packets,\n !\/#@5                  \"src2dst_bytes\":flow.src2dst_bytes,\n                        \"dst2src_packets\":flow.dst2src_packets,\n                        \"dst2src_bytes\":flow.dst2src_bytes,\n                        \"application_name\":flow.application_name,\n                        \"category_name\":flow.application_category_name,\n                        \"auth0_signal_badip\":auth0_signal(flow.dst_ip)\n                }\n\n                \n                print(NFEntry)\n<\/code><\/pre>\n\n\n\n<p>Ref: https:\/\/github.com\/nfstream\/nfstream<\/p>\n\n\n\n<p>API DOC: https:\/\/www.nfstream.org\/docs\/api<\/p>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/www.nfstream.org\/resources\/architecture_nfstream.png\" alt=\"https:\/\/www.nfstream.org\/resources\/architecture_nfstream.png\"\/><\/figure>\n<div class=\"pdf24Plugin-cp\"> \t<form name=\"pdf24Form0\" method=\"post\" action=\"https:\/\/doc2pdf.pdf24.org\/wordpress.php\" target=\"pdf24PopWin\" onsubmit=\"var pdf24Win = window.open('about:blank', 'pdf24PopWin', 'resizable=yes,scrollbars=yes,width=600,height=250,left='+(screen.width\/2-300)+',top='+(screen.height\/3-125)+''); pdf24Win.focus(); if(typeof pdf24OnCreatePDF === 'function'){void(pdf24OnCreatePDF(this,pdf24Win));}\"> \t\t<input type=\"hidden\" name=\"blogCharset\" value=\"Cw1x07UAAA==\" \/><input type=\"hidden\" name=\"blogPosts\" value=\"MwQA\" \/><input type=\"hidden\" name=\"blogUrl\" value=\"yygpKSi20tdPyslP18vNSCzOzNNLLQUA\" \/><input type=\"hidden\" name=\"blogName\" value=\"c\/LxdwcA\" \/><input type=\"hidden\" name=\"blogValueEncoding\" value=\"gzdeflate base64\" \/><input type=\"hidden\" name=\"postId_0\" value=\"M7QwNAAA\" \/><input type=\"hidden\" name=\"postTitle_0\" value=\"y0srLilKTcxVKE0qzSspVcjNz8ssyS8CAA==\" \/><input type=\"hidden\" name=\"postLink_0\" value=\"yygpKSi20tdPyslP18vNSCzOzNNLLdW3L7A1tDA0AAA=\" \/><input type=\"hidden\" name=\"postAuthor_0\" value=\"y00syfcwNAIA\" \/><input type=\"hidden\" name=\"postDateTime_0\" value=\"MzIwMtA1NNI1MlAwNLICIiNjAA==\" \/><input type=\"hidden\" name=\"postContent_0\" value=\"pVhtk9o2EP7uX6FynQY6OcC8XujBNJMmnX5o2knTjxmPkNdGxZZ8kgxHO\/3vXdkGDDaGa8hM7rT77ItWq0fLOY\/J4s9lKkxKZGJ4zP+G2WMvWTj4eUwUEBZRreetbXK\/jCRb3zPpQ2vxaH8swKyMlBG5\/5mA0O7EJeqZjPpvJsTkPx2ekIiLNdFg9hDz\/JRCChEI4vbtx9E7zQx62RIBpsukgq6KIfZi+jz\/tt12+4PR9\/l\/k07nsZeFxhwVFFkucgc3px3oLuXyHt3fC0XmZDwYDR4eHBQHPAIrR+HEdd1J31mDEhB1Yx3mYnfo9qeDslgsK+KECs6sdHyQcN8r\/I7Hw8lBrLgwa5SO8J9Lpns5bkg9objvHCriQ0DTyHhPPtfWd\/B01OEvPmxsAG9J2TqSIQIGk4E7Gh1BeLxFUW1mk+l0OnAnR7WWqGFSiCLH8VG1vWDHk80IESLo0ijqUsYgMZ4CnytgRh\/SP4VhcuhLGU6Fhbg1EJV4eBAG1AW9BpYquBpJg\/AbQUVJb8n8DKplqhhmIFMDDeimjewxN2zmCL28IY6hpNpS5Vfk2P808hKpcItUhDZhe5tKp5whDUs8DBeCNlwK+6tReLfnZLlUp6iACw+pAnD3ma9T7RogoRHfgIe9vYkaEYmSS7A7GV9C2Di2I89cRHLrRdSAYLvT2lqlvQdSJauiwzKOqQD0TpQuy2RcCzJbb5myNWTVxttU4woJRWEinmKbZRpUkxHSi8EozrSn6QbqAAYP1tgt0ayek+HD6BSiAuYOh9OqraVJyx6Wch+mw4d+zQW1OI0bPe2LTGqrqA1eRo8G2KUe9yOoge0EmmPj4S6ywxpU9EzKda5zK7rLhvZsMXyc6GpQrLyCVNfUa8uFb\/PGpuYirFpuSzXJyLZSk8mRI5BM6TLCVkVpOdTk9OZdgUWyitjEKFNmlzVZiExh7y7Fq5XV4aDdi7DBUWZ7Lqb2HuKLfORw1MgNKCbjmBtsp1iq3T6I3tIk4QJ0Vv261\/Hjhz8+f3r\/9tfb3\/VAyZiIQBsFNCY8ttxB0E22BuXEO4\/RxFje0oUMgx8B7Zwf51SFuotP4WuCx2w4EotlIipotNNczz+rFFCV4Kt2kLmk4zjIZCTA5iRckJpYM4ecfT5+eI9stZv\/U9HsPy1IJFshdSmN9A0gWjMbobvkOZ0i5WFqR7UX69fXnGEFG3wdtM2u7CXwLYGUU7MCq\/CMtCzazuPhwgrb1xLvdG6JV8r+xeHKe7sazfPTrMvry7RXoqeepeoGZ3gF9NENvmyFoMEEHxgjmYwKm\/2ywUIrhpe4wOeLK2h7O0p4u2yw8LFyB\/\/54gq65H+\/vGIRU1YywFVTUSMqkPgLfLG6suOj\/2LVgN+uJNeeb6cKkR+0wDvcmpFcwShbQTcE0y4VpPPd3Rv3h\/6XF7ll3Fe3uHUb3N7ZhmHIpUWSr151\/5JctHHi8b1C1bnFPMkqmllx\/+puLoXNX858TsrUXASybW\/omevOvmI3JQfPXxHDvS0Gi30P32cbiPzPSIMvxa5mg\/H4puolVNlJKiv+S6MNXxLh645pdGMJ8bUz+QT88u00FaxlpLHfCGg22daycqG76mO5M1DvIdM088ggo7OTLM6kaP9N7+7HcYN9OYMTWTNHDjKqPol9Jr3Bvhz7RNZgi9NahGNQiQkz83NxgwdEQYgT4CXzE31TJqlZ9T3Nw+y4qG9fpbLshDorbv51KqKKIPvzRrsYzjpO3XD6CYIZWRmT6FmvF3KzSpc4U8e9\/fR5+OU4vC7e\/v4L+em3d0e77Xbb3eO6UoU9XzLdowk\/GAU8TGuGXh7T0E69PA6JD5gdfqOYt6j9RtMieJrz1sUQCvIhF+MotuIGG9\/OqAdMIsIWoZH5Shc9LFee\/ML5Dw==\" \/> \t\t<a href=\"https:\/\/www.pdf24.org\" target=\"_blank\" title=\"www.pdf24.org\" rel=\"nofollow\"><img src=\"https:\/\/blog.mhasin.eu\/wp-content\/plugins\/pdf24-post-to-pdf\/img\/pdf_32x32.png\" alt=\"\" border=\"0\" height=\"32\" \/><\/a> \t\t<span class=\"pdf24Plugin-cp-space\">&nbsp;&nbsp;<\/span> \t\t<span class=\"pdf24Plugin-cp-text\">Send article as PDF<\/span> \t\t<span class=\"pdf24Plugin-cp-space\">&nbsp;&nbsp;<\/span> \t\t<input class=\"pdf24Plugin-cp-input\" style=\"margin: 0px;\" type=\"text\" name=\"sendEmailTo\" placeholder=\"Enter email address\" \/> \t\t<input class=\"pdf24Plugin-cp-submit\" style=\"margin: 0px;\" type=\"submit\" value=\"Send\" \/> \t<\/form> <\/div>","protected":false},"excerpt":{"rendered":"Ubuntu optimize: sysctl NFSTREAM: Ref: https:\/\/github.com\/nfstream\/nfstream API DOC: https:\/\/www.nfstream.org\/docs\/api &nbsp;&nbsp; Send article as PDF &nbsp;&nbsp;\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"arc_restricted_post":false,"footnotes":""},"categories":[8],"tags":[],"class_list":["post-1810","post","type-post","status-publish","format-standard","hentry","category-monitoring"],"_links":{"self":[{"href":"https:\/\/blog.mhasin.eu\/index.php?rest_route=\/wp\/v2\/posts\/1810","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/blog.mhasin.eu\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/blog.mhasin.eu\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/blog.mhasin.eu\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/blog.mhasin.eu\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1810"}],"version-history":[{"count":5,"href":"https:\/\/blog.mhasin.eu\/index.php?rest_route=\/wp\/v2\/posts\/1810\/revisions"}],"predecessor-version":[{"id":1875,"href":"https:\/\/blog.mhasin.eu\/index.php?rest_route=\/wp\/v2\/posts\/1810\/revisions\/1875"}],"wp:attachment":[{"href":"https:\/\/blog.mhasin.eu\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1810"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/blog.mhasin.eu\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1810"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/blog.mhasin.eu\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1810"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}