diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ccb9333 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*.pyc +__pycache__ +Dshell.egg-info +build/ +.idea/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..df6549e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +FROM python:3-alpine as builder + +COPY . /src + +WORKDIR /src + +ARG OUI_SRC="http://standards-oui.ieee.org/oui/oui.txt" + +ENV VIRTUAL_ENV="/opt/venv" + +RUN apk add cargo curl g++ gcc rust libpcap-dev libffi-dev \ + && python3 -m venv "${VIRTUAL_ENV}" \ + && curl --location --silent --output "/src/dshell/data/oui.txt" "${OUI_SRC}" + +ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" + +RUN pip install --upgrade pip wheel && pip install . + +FROM python:3-alpine + +ENV VIRTUAL_ENV="/opt/venv" + +COPY --from=builder "${VIRTUAL_ENV}/" "${VIRTUAL_ENV}/" + +RUN apk add --no-cache bash libstdc++ libpcap + +VOLUME ["/data"] + +WORKDIR "/data" + +ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" + +ENTRYPOINT ["dshell"] diff --git a/Dshell-Training-Pack-0.1.tar.gz b/Dshell-Training-Pack-0.1.tar.gz new file mode 100644 index 0000000..5841d34 Binary files /dev/null and b/Dshell-Training-Pack-0.1.tar.gz differ diff --git a/Dshell_Developer_Guide.pdf b/Dshell_Developer_Guide.pdf new file mode 100644 index 0000000..dd5cb4e Binary files /dev/null and b/Dshell_Developer_Guide.pdf differ diff --git a/Dshell_User_Guide.pdf b/Dshell_User_Guide.pdf new file mode 100644 index 0000000..21badf1 Binary files /dev/null and b/Dshell_User_Guide.pdf differ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c10e25f --- /dev/null +++ b/LICENSE @@ -0,0 +1,8 @@ +© (2020) United States Government, as represented by the Secretary of the Army. All rights reserved. + +ICF Incorporated, L.L.C. contributed to the development of Dshell (Python 3). + +Because the project utilizes code licensed from contributors and other third parties, it therefore is licensed under the MIT License. http://opensource.org/licenses/mit-license.php. Under that license, permission is granted free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the conditions that any appropriate copyright notices and this permission notice are included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/LICENSE.txt b/LICENSE.txt deleted file mode 100644 index b398bc9..0000000 --- a/LICENSE.txt +++ /dev/null @@ -1,5 +0,0 @@ -This project constitutes a work of the United States Government and is not subject to domestic copyright protection under 17 USC § 105. - -However, because the project utilizes code licensed from contributors and other third parties, it therefore is licensed under the MIT License. http://opensource.org/licenses/mit-license.php. Under that license, permission is granted free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the conditions that any appropriate copyright notices and this permission notice are included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Makefile b/Makefile deleted file mode 100755 index daa5cc5..0000000 --- a/Makefile +++ /dev/null @@ -1,39 +0,0 @@ -default: all - -all: rc dshell - -dshell: rc initpy pydoc - -rc: - # Generating .dshellrc and dshell files - python $(PWD)/bin/generate-dshellrc.py $(PWD) - chmod 755 $(PWD)/dshell - chmod 755 $(PWD)/dshell-decode - chmod 755 $(PWD)/bin/decode.py - ln -s $(PWD)/bin/decode.py $(PWD)/bin/decode - -initpy: - find $(PWD)/decoders -type d -not -path \*.svn\* -print -exec touch {}/__init__.py \; - -pydoc: - (cd $(PWD)/doc && ./generate-doc.sh $(PWD) ) - -clean: clean_pyc - -distclean: clean clean_py clean_pydoc clean_rc - -clean_rc: - rm -fv $(PWD)/dshell - rm -fv $(PWD)/dshell-decode - rm -fv $(PWD)/.dshellrc - rm -fv $(PWD)/bin/decode - -clean_py: - find $(PWD)/decoders -name '__init__.py' -exec rm -v {} \; - -clean_pyc: - find $(PWD)/decoders -name '*.pyc' -exec rm -v {} \; - find $(PWD)/lib -name '*.pyc' -exec rm -v {} \; - -clean_pydoc: - find $(PWD)/doc -name '*.htm*' -exec rm -v {} \; diff --git a/README b/README new file mode 100644 index 0000000..29ea177 --- /dev/null +++ b/README @@ -0,0 +1,196 @@ +# Dshell +An extensible network forensic analysis framework. Enables rapid development of plugins to support the dissection of network packet captures. + +Key features: +* Deep packet analysis using specialized plugins +* Robust stream reassembly +* IPv4 and IPv6 support +* Multiple user-selectable output formats and the ability to create custom output handlers +* Chainable plugins +* Parallel processing option to divide the handling of data source into separate Python processes +* Enables development of external plugin packs to share and install new externally developed plugins without overlapping the core Dshell plugin directories + +## Guides +* [Dshell User Guide](Dshell_User_Guide.pdf) + * A guide to installation as well as both basic and advanced analysis with examples + * Helps new and experienced end users with using and understanding the decoder-shell (Dshell) framework +* [Dshell Developer Guide](Dshell_Developer_Guide.pdf) + * A guide to plugin development with basic examples, as well as core function and class definitions, and an overview of data flow + * Helps end users develop new, custom Dshell plugins as well as modify existing plugins + +## Requirements +* Linux (developed on Ubuntu 20.04 LTS) +* Python 3 (developed with Python 3.8.10) +* [pypacker](https://gitlab.com/mike01/pypacker) +* [pcapy-ng](https://github.com/stamparm/pcapy-ng/) +* [pyOpenSSL](https://github.com/pyca/pyopenssl) +* [geoip2](https://github.com/maxmind/GeoIP2-python) + * [MaxMind GeoIP2 data sets](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data) + * Used to map IP addresses to country codes + * See Installation section for configuration + +## Optional +* [oui.txt](http://standards-oui.ieee.org/oui.txt) + * used by some plugins that handle MAC addresses + * place in <dshell>/data/ +* [elasticsearch](https://www.elastic.co/guide/en/elasticsearch/client/python-api/current/index.html) + * used in the elasticout output module + * only necessary if planning to use elasticsearch to store output +* [pyJA3](https://github.com/salesforce/ja3/tree/master/python) + * used in the tls plugin + +## Installation + +1. Install Dshell with pip + * `python3 -m pip install Dshell/` OR `python3 -m pip install ` +2. Configure geoip2 by placing the MaxMind GeoLite2 data set files (GeoLite2-ASN.mmdb, GeoLite2-City.mmdb, GeoLite2-Country.mmdb) in [...]/site-packages/dshell/data/GeoIP/ +3. Run `dshell`. This should drop you into a `Dshell> ` prompt. + +## Basic Usage + +* `decode -l` + * This will list all available plugins, alongside basic information about them +* `decode -h` + * Show generic command-line flags available to most plugins, such as the color blind friendly mode for all color output +* `decode -p ` + * Display information about a plugin, including available command line flags +* `decode -p ` + * Run the selected plugin on a pcap or pcapng file +* `decode -p + ` + * Chain two (or more) plugins together and run them on a pcap file +* `decode -p -i ` + * Run the selected plugin live on an interface (may require superuser privileges) + +## Usage Examples +Showing DNS lookups in [sample traffic](http://wiki.wireshark.org/SampleCaptures#General_.2F_Unsorted) + +``` +Dshell> decode -p dns ~/pcap/dns.cap | sort +[DNS] 2005-03-30 03:47:46 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 4146, TXT? google.com., TXT: b'\x0fv=spf1 ptr ?all' ** +[DNS] 2005-03-30 03:47:50 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 63343, MX? google.com., MX: b'\x00(\x05smtp4\xc0\x0c', MX: b'\x00\n\x05smtp5\xc0\x0c', MX: b'\x00\n\x05smtp6\xc0\x0c', MX: b'\x00\n\x05smtp1\xc0\x0c', MX: b'\x00\n\x05smtp2\xc0\x0c', MX: b'\x00(\x05smtp3\xc0\x0c' ** +[DNS] 2005-03-30 03:47:59 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 18849, LOC? google.com. ** +[DNS] 2005-03-30 03:48:07 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 39867, PTR? 104.9.192.66.in-addr.arpa., PTR: 66-192-9-104.gen.twtelecom.net. ** +[DNS] 2005-03-30 03:49:18 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 30144, A? www.netbsd.org., A: 204.152.190.12 (ttl 82159s) ** +[DNS] 2005-03-30 03:49:35 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 61652, AAAA? www.netbsd.org., AAAA: 2001:4f8:4:7:2e0:81ff:fe52:9a6b (ttl 86400s) ** +[DNS] 2005-03-30 03:50:35 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 32569, AAAA? www.netbsd.org., AAAA: 2001:4f8:4:7:2e0:81ff:fe52:9a6b (ttl 86340s) ** +[DNS] 2005-03-30 03:50:44 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 36275, AAAA? www.google.com., CNAME: 'www.l.google.com.' ** +[DNS] 2005-03-30 03:50:54 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 56482, AAAA? www.l.google.com. ** +[DNS] 2005-03-30 03:51:35 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 48159, AAAA? www.example.com. ** +[DNS] 2005-03-30 03:51:46 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 9837, AAAA? www.example.notginh., NXDOMAIN ** +[DNS] 2005-03-30 03:52:17 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 65251, AAAA: 2001:4f8:0:2::d (ttl 600s), A: 204.152.184.88 (ttl 600s) ** +[DNS] 2005-03-30 03:52:17 192.168.170.8:32796 -- 192.168.170.20:53 ** ID: 23123, PTR? 1.0.0.127.in-addr.arpa., PTR: localhost. ** +[DNS] 2005-03-30 03:52:17 192.168.170.8:32797 -- 192.168.170.20:53 ** ID: 8330, NS: b'\x06ns-ext\x04nrt1\xc0\x0c', NS: b'\x06ns-ext\x04sth1\xc0\x0c', NS: b'\x06ns-ext\xc0\x0c', NS: b'\x06ns-ext\x04lga1\xc0\x0c' ** +[DNS] 2005-03-30 03:52:17 192.168.170.56:1707 -- 217.13.4.24:53 ** ID: 12910, SRV? _ldap._tcp.Default-First-Site-Name._sites.dc._msdcs.utelsystems.local., NXDOMAIN ** +[DNS] 2005-03-30 03:52:17 192.168.170.56:1708 -- 217.13.4.24:53 ** ID: 61793, SRV? _ldap._tcp.dc._msdcs.utelsystems.local., NXDOMAIN ** +[DNS] 2005-03-30 03:52:17 192.168.170.56:1709 -- 217.13.4.24:53 ** ID: 33633, SRV? _ldap._tcp.05b5292b-34b8-4fb7-85a3-8beef5fd2069.domains._msdcs.utelsystems.local., NXDOMAIN ** +[DNS] 2005-03-30 03:52:17 192.168.170.56:1710 -- 217.13.4.24:53 ** ID: 53344, A? GRIMM.utelsystems.local., NXDOMAIN ** +[DNS] 2005-03-30 03:52:25 192.168.170.56:1711 -- 217.13.4.24:53 ** ID: 30307, A? GRIMM.utelsystems.local., NXDOMAIN ** +``` + +Following and reassembling a stream in [sample traffic](http://wiki.wireshark.org/SampleCaptures#General_.2F_Unsorted) + +``` +Dshell> decode -p followstream ~/pcap/v6-http.cap +Connection 1 (TCP) +Start: 2007-08-05 15:16:44.189851 +End: 2007-08-05 15:16:44.219460 +2001:6f8:102d:0:2d0:9ff:fee3:e8de: 59201 -> 2001:6f8:900:7c0::2: 80 (300 bytes) +2001:6f8:900:7c0::2: 80 -> 2001:6f8:102d:0:2d0:9ff:fee3:e8de: 59201 (2379 bytes) + +GET / HTTP/1.0 +Host: cl-1985.ham-01.de.sixxs.net +Accept: text/html, text/plain, text/css, text/sgml, */*;q=0.01 +Accept-Encoding: gzip, bzip2 +Accept-Language: en +User-Agent: Lynx/2.8.6rel.2 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.8b + + + +HTTP/1.1 200 OK +Date: Sun, 05 Aug 2007 19:16:44 GMT +Server: Apache +Content-Length: 2121 +Connection: close +Content-Type: text/html + + + + + Index of / + + +

Index of /

+
Icon  Name                    Last modified      Size  Description
[DIR] 202-vorbereitung/       06-Jul-2007 14:31    -   
+[   ] Efficient_Video_on_d..> 19-Dec-2006 03:17  291K  
+[   ] Welcome Stranger!!!     28-Dec-2006 03:46    0   
+[TXT] barschel.htm            31-Jul-2007 02:21   44K  
+[DIR] bnd/                    30-Dec-2006 08:59    -   
+[DIR] cia/                    28-Jun-2007 00:04    -   
+[   ] cisco_ccna_640-801_c..> 28-Dec-2006 03:48  236K  
+[DIR] doc/                    19-Sep-2006 01:43    -   
+[DIR] freenetproto/           06-Dec-2006 09:00    -   
+[DIR] korrupt/                03-Jul-2007 11:57    -   
+[DIR] mp3_technosets/         04-Jul-2007 08:56    -   
+[TXT] neues_von_rainald_go..> 21-Mar-2007 23:27   31K  
+[TXT] neues_von_rainald_go..> 21-Mar-2007 23:29   36K  
+[   ] pruef.pdf               28-Dec-2006 07:48   88K  
+
+ +``` + +Chaining plugins to view flow data for a specific country code in [sample traffic](http://wiki.wireshark.org/SampleCaptures#General_.2F_Unsorted) (note: TCP handshakes are not included in the packet count) + +``` +Dshell> decode -p country+netflow --country_code=JP ~/pcap/SkypeIRC.cap +2006-08-25 15:32:20.766761 192.168.1.2 -> 202.232.205.123 (-- -> JP) UDP 60583 33438 1 0 64 0 0.0000s +2006-08-25 15:32:20.634046 192.168.1.2 -> 202.232.205.123 (-- -> JP) UDP 60583 33435 1 0 64 0 0.0000s +2006-08-25 15:32:20.747503 192.168.1.2 -> 202.232.205.123 (-- -> JP) UDP 60583 33437 1 0 64 0 0.0000s +2006-08-25 15:32:20.651501 192.168.1.2 -> 202.232.205.123 (-- -> JP) UDP 60583 33436 1 0 64 0 0.0000s +``` + +Collecting DNS traffic from several files and storing it in a new pcap file. + +``` +Dshell> decode -p dns+pcapwriter --pcapwriter_outfile=test.pcap ~/pcap/*.cap > /dev/null +Dshell> tcpdump -nnr test.pcap | head +reading from file test.pcap, link-type EN10MB (Ethernet) +15:36:08.670569 IP 192.168.1.2.2131 > 192.168.1.1.53: 40209+ A? ui.skype.com. (30) +15:36:08.670687 IP 192.168.1.2.2131 > 192.168.1.1.53: 40210+ AAAA? ui.skype.com. (30) +15:36:08.674022 IP 192.168.1.1.53 > 192.168.1.2.2131: 40209- 1/0/0 A 212.72.49.131 (46) +15:36:09.011208 IP 192.168.1.1.53 > 192.168.1.2.2131: 40210 0/1/0 (94) +15:36:10.171350 IP 192.168.1.2.2131 > 192.168.1.1.53: 40210+ AAAA? ui.skype.com. (30) +15:36:10.961350 IP 192.168.1.1.53 > 192.168.1.2.2131: 40210* 0/1/0 (85) +15:36:10.961608 IP 192.168.1.2.2131 > 192.168.1.1.53: 40211+ AAAA? ui.skype.com. (30) +15:36:11.294333 IP 192.168.1.1.53 > 192.168.1.2.2131: 40211 0/1/0 (94) +15:32:21.664798 IP 192.168.1.2.2130 > 192.168.1.1.53: 39862+ A? ui.skype.com. (30) +15:32:21.664913 IP 192.168.1.2.2130 > 192.168.1.1.53: 39863+ AAAA? ui.skype.com. (30) +``` + +Collecting TFTP data and converting alerts to JSON format using [sample traffic](https://wiki.wireshark.org/SampleCaptures#TFTP) + +``` +Dshell> decode -p tftp -O jsonout ~/pcap/tftp_*.pcap +{"ts": 1367411051.972852, "sip": "192.168.0.253", "sport": 50618, "dip": "192.168.0.10", "dport": 3445, "readwrite": "read", "filename": "rfc1350.txt", "plugin": "tftp", "pcapfile": "/home/pcap/tftp_rrq.pcap", "data": "read rfc1350.txt (24599 bytes) "} +{"ts": 1367053679.45274, "sip": "192.168.0.1", "sport": 57509, "dip": "192.168.0.13", "dport": 2087, "readwrite": "write", "filename": "rfc1350.txt", "plugin": "tftp", "pcapfile": "/home/pcap/tftp_wrq.pcap", "data": "write rfc1350.txt (24599 bytes) "} +``` + +Running a plugin within a separate Python script using [sample traffic](https://wiki.wireshark.org/SampleCaptures#TFTP) + +``` +# Import required Dshell libraries +import dshell.decode as decode +import dshell.plugins.tftp.tftp as tftp + +# Instantiate plugin +plugin = tftp.DshellPlugin() +# Define plugin-specific arguments, if needed +dargs = {plugin: {"rip": True, "outdir": "/tmp/"}} +# Add plugin(s) to plugin chain +decode.plugin_chain = [plugin] +# Run decode main function with all other arguments +decode.main( + debug=True, + files=["/home/user/pcap/tftp_rrq.pcap", "/home/user/pcap/tftp_wrq.pcap"], + plugin_args=dargs +) +``` diff --git a/README.md b/README.md index 7d1087c..e1e0f6f 100644 --- a/README.md +++ b/README.md @@ -1,77 +1,101 @@ # Dshell - -An extensible network forensic analysis framework. Enables rapid development of plugins to support the dissection of network packet captures. +An extensible network forensic analysis framework. Enables rapid development of plugins to support the dissection of network packet captures. Key features: - - -* Robust stream reassembly +* Deep packet analysis using specialized plugins +* Robust stream reassembly * IPv4 and IPv6 support -* Custom output handlers -* Chainable decoders - -## Prerequisites - -* Linux (developed on Ubuntu 12.04) -* Python 2.7 -* [pygeoip](https://github.com/appliedsec/pygeoip), GNU Lesser GPL - * [MaxMind GeoIP Legacy datasets](http://dev.maxmind.com/geoip/legacy/geolite/) -* [PyCrypto](https://pypi.python.org/pypi/pycrypto), custom license -* [dpkt](https://code.google.com/p/dpkt/), New BSD License -* [IPy](https://github.com/haypo/python-ipy), BSD 2-Clause License -* [pypcap](https://code.google.com/p/pypcap/), New BSD License +* Multiple user-selectable output formats and the ability to create custom output handlers +* Chainable plugins +* Parallel processing option to divide the handling of data source into separate Python processes +* Enables development of external plugin packs to share and install new externally developed plugins without overlapping the core Dshell plugin directories + +## Guides +* [Dshell User Guide](Dshell_User_Guide.pdf) + * A guide to installation as well as both basic and advanced analysis with examples + * Helps new and experienced end users with using and understanding the decoder-shell (Dshell) framework +* [Dshell Developer Guide](Dshell_Developer_Guide.pdf) + * A guide to plugin development with basic examples, as well as core function and class definitions, and an overview of data flow + * Helps end users develop new, custom Dshell plugins as well as modify existing plugins + +## Requirements +* Linux (developed on Ubuntu 20.04 LTS) +* Python 3 (developed with Python 3.8.10) +* [pypacker](https://gitlab.com/mike01/pypacker) +* [pcapy-ng](https://github.com/stamparm/pcapy-ng/) +* [pyOpenSSL](https://github.com/pyca/pyopenssl) +* [geoip2](https://github.com/maxmind/GeoIP2-python) + * [MaxMind GeoIP2 data sets](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data) + * Used to map IP addresses to country codes + * See Installation section for configuration + +## Optional +* [oui.txt](http://standards-oui.ieee.org/oui.txt) + * used by some plugins that handle MAC addresses + * place in <dshell>/data/ +* [elasticsearch](https://www.elastic.co/guide/en/elasticsearch/client/python-api/current/index.html) + * used in the elasticout output module + * only necessary if planning to use elasticsearch to store output +* [pyJA3](https://github.com/salesforce/ja3/tree/master/python) + * used in the tls plugin ## Installation -1. Install all of the necessary Python modules listed above. Many of them are available via pip and/or apt-get. Pygeoip is not yet available as a package and must be installed with pip or manually. All except dpkt are available with pip. - - 1. sudo apt-get install python-crypto python-dpkt python-ipy python-pypcap +1. Install Dshell with pip + * `python3 -m pip install Dshell/` OR `python3 -m pip install ` +2. Configure geoip2 by placing the MaxMind GeoLite2 data set files (GeoLite2-ASN.mmdb, GeoLite2-City.mmdb, GeoLite2-Country.mmdb) in [...]/site-packages/dshell/data/GeoIP/ +3. Run `dshell`. This should drop you into a `Dshell> ` prompt. - 2. sudo pip install pygeoip - -2. Configure pygeoip by moving the MaxMind data files (GeoIP.dat, GeoIPv6.dat, GeoIPASNum.dat, GeoIPASNumv6.dat) to /share/GeoIP/ - -2. Run `make`. This will build Dshell. - -3. Run `./dshell`. This is Dshell. If you get a Dshell> prompt, you're good to go! - -## Basic usage +## Basic Usage * `decode -l` - * This will list all available decoders alongside basic information about them + * This will list all available plugins, alongside basic information about them * `decode -h` - * Show generic command-line flags available to most decoders -* `decode -d ` - * Display information about a decoder, including available command-line flags -* `decode -d ` - * Run the selected decoder on a pcap file + * Show generic command-line flags available to most plugins, such as the color blind friendly mode for all color output +* `decode -p ` + * Display information about a plugin, including available command line flags +* `decode -p ` + * Run the selected plugin on a pcap or pcapng file +* `decode -p + ` + * Chain two (or more) plugins together and run them on a pcap file +* `decode -p -i ` + * Run the selected plugin live on an interface (may require superuser privileges) ## Usage Examples - Showing DNS lookups in [sample traffic](http://wiki.wireshark.org/SampleCaptures#General_.2F_Unsorted) ``` -Dshell> decode -d dns ~/pcap/dns.cap -dns 2005-03-30 03:47:46 192.168.170.8:32795 -> 192.168.170.20:53 ** 39867 PTR? 66.192.9.104 / PTR: 66-192-9-104.gen.twtelecom.net ** -dns 2005-03-30 03:47:46 192.168.170.8:32795 -> 192.168.170.20:53 ** 30144 A? www.netbsd.org / A: 204.152.190.12 (ttl 82159s) ** -dns 2005-03-30 03:47:46 192.168.170.8:32795 -> 192.168.170.20:53 ** 61652 AAAA? www.netbsd.org / AAAA: 2001:4f8:4:7:2e0:81ff:fe52:9a6b (ttl 86400s) ** -dns 2005-03-30 03:47:46 192.168.170.8:32795 -> 192.168.170.20:53 ** 32569 AAAA? www.netbsd.org / AAAA: 2001:4f8:4:7:2e0:81ff:fe52:9a6b (ttl 86340s) ** -dns 2005-03-30 03:47:46 192.168.170.8:32795 -> 192.168.170.20:53 ** 36275 AAAA? www.google.com / CNAME: www.l.google.com ** -dns 2005-03-30 03:47:46 192.168.170.8:32795 -> 192.168.170.20:53 ** 9837 AAAA? www.example.notginh / NXDOMAIN ** -dns 2005-03-30 03:52:17 192.168.170.8:32796 <- 192.168.170.20:53 ** 23123 PTR? 127.0.0.1 / PTR: localhost ** -dns 2005-03-30 03:52:25 192.168.170.56:1711 <- 217.13.4.24:53 ** 30307 A? GRIMM.utelsystems.local / NXDOMAIN ** -dns 2005-03-30 03:52:17 192.168.170.56:1710 <- 217.13.4.24:53 ** 53344 A? GRIMM.utelsystems.local / NXDOMAIN ** +Dshell> decode -p dns ~/pcap/dns.cap | sort +[DNS] 2005-03-30 03:47:46 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 4146, TXT? google.com., TXT: b'\x0fv=spf1 ptr ?all' ** +[DNS] 2005-03-30 03:47:50 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 63343, MX? google.com., MX: b'\x00(\x05smtp4\xc0\x0c', MX: b'\x00\n\x05smtp5\xc0\x0c', MX: b'\x00\n\x05smtp6\xc0\x0c', MX: b'\x00\n\x05smtp1\xc0\x0c', MX: b'\x00\n\x05smtp2\xc0\x0c', MX: b'\x00(\x05smtp3\xc0\x0c' ** +[DNS] 2005-03-30 03:47:59 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 18849, LOC? google.com. ** +[DNS] 2005-03-30 03:48:07 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 39867, PTR? 104.9.192.66.in-addr.arpa., PTR: 66-192-9-104.gen.twtelecom.net. ** +[DNS] 2005-03-30 03:49:18 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 30144, A? www.netbsd.org., A: 204.152.190.12 (ttl 82159s) ** +[DNS] 2005-03-30 03:49:35 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 61652, AAAA? www.netbsd.org., AAAA: 2001:4f8:4:7:2e0:81ff:fe52:9a6b (ttl 86400s) ** +[DNS] 2005-03-30 03:50:35 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 32569, AAAA? www.netbsd.org., AAAA: 2001:4f8:4:7:2e0:81ff:fe52:9a6b (ttl 86340s) ** +[DNS] 2005-03-30 03:50:44 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 36275, AAAA? www.google.com., CNAME: 'www.l.google.com.' ** +[DNS] 2005-03-30 03:50:54 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 56482, AAAA? www.l.google.com. ** +[DNS] 2005-03-30 03:51:35 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 48159, AAAA? www.example.com. ** +[DNS] 2005-03-30 03:51:46 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 9837, AAAA? www.example.notginh., NXDOMAIN ** +[DNS] 2005-03-30 03:52:17 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 65251, AAAA: 2001:4f8:0:2::d (ttl 600s), A: 204.152.184.88 (ttl 600s) ** +[DNS] 2005-03-30 03:52:17 192.168.170.8:32796 -- 192.168.170.20:53 ** ID: 23123, PTR? 1.0.0.127.in-addr.arpa., PTR: localhost. ** +[DNS] 2005-03-30 03:52:17 192.168.170.8:32797 -- 192.168.170.20:53 ** ID: 8330, NS: b'\x06ns-ext\x04nrt1\xc0\x0c', NS: b'\x06ns-ext\x04sth1\xc0\x0c', NS: b'\x06ns-ext\xc0\x0c', NS: b'\x06ns-ext\x04lga1\xc0\x0c' ** +[DNS] 2005-03-30 03:52:17 192.168.170.56:1707 -- 217.13.4.24:53 ** ID: 12910, SRV? _ldap._tcp.Default-First-Site-Name._sites.dc._msdcs.utelsystems.local., NXDOMAIN ** +[DNS] 2005-03-30 03:52:17 192.168.170.56:1708 -- 217.13.4.24:53 ** ID: 61793, SRV? _ldap._tcp.dc._msdcs.utelsystems.local., NXDOMAIN ** +[DNS] 2005-03-30 03:52:17 192.168.170.56:1709 -- 217.13.4.24:53 ** ID: 33633, SRV? _ldap._tcp.05b5292b-34b8-4fb7-85a3-8beef5fd2069.domains._msdcs.utelsystems.local., NXDOMAIN ** +[DNS] 2005-03-30 03:52:17 192.168.170.56:1710 -- 217.13.4.24:53 ** ID: 53344, A? GRIMM.utelsystems.local., NXDOMAIN ** +[DNS] 2005-03-30 03:52:25 192.168.170.56:1711 -- 217.13.4.24:53 ** ID: 30307, A? GRIMM.utelsystems.local., NXDOMAIN ** ``` Following and reassembling a stream in [sample traffic](http://wiki.wireshark.org/SampleCaptures#General_.2F_Unsorted) ``` -Dshell> decode -d followstream ~/pcap/v6-http.cap +Dshell> decode -p followstream ~/pcap/v6-http.cap Connection 1 (TCP) -Start: 2007-08-05 19:16:44.189852 UTC - End: 2007-08-05 19:16:44.204687 UTC -2001:6f8:102d:0:2d0:9ff:fee3:e8de:59201 -> 2001:6f8:900:7c0::2:80 (240 bytes) -2001:6f8:900:7c0::2:80 -> 2001:6f8:102d:0:2d0:9ff:fee3:e8de:59201 (2259 bytes) +Start: 2007-08-05 15:16:44.189851 +End: 2007-08-05 15:16:44.219460 +2001:6f8:102d:0:2d0:9ff:fee3:e8de: 59201 -> 2001:6f8:900:7c0::2: 80 (300 bytes) +2001:6f8:900:7c0::2: 80 -> 2001:6f8:102d:0:2d0:9ff:fee3:e8de: 59201 (2379 bytes) GET / HTTP/1.0 Host: cl-1985.ham-01.de.sixxs.net @@ -80,6 +104,8 @@ Accept-Encoding: gzip, bzip2 Accept-Language: en User-Agent: Lynx/2.8.6rel.2 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.8b + + HTTP/1.1 200 OK Date: Sun, 05 Aug 2007 19:16:44 GMT Server: Apache @@ -112,31 +138,59 @@ Content-Type: text/html ``` -Chaining decoders to view flow data for a specific country code in [sample traffic](http://wiki.wireshark.org/SampleCaptures#General_.2F_Unsorted) (note: TCP handshakes are not included in the packet count) +Chaining plugins to view flow data for a specific country code in [sample traffic](http://wiki.wireshark.org/SampleCaptures#General_.2F_Unsorted) (note: TCP handshakes are not included in the packet count) + +``` +Dshell> decode -p country+netflow --country_code=JP ~/pcap/SkypeIRC.cap +2006-08-25 15:32:20.766761 192.168.1.2 -> 202.232.205.123 (-- -> JP) UDP 60583 33438 1 0 64 0 0.0000s +2006-08-25 15:32:20.634046 192.168.1.2 -> 202.232.205.123 (-- -> JP) UDP 60583 33435 1 0 64 0 0.0000s +2006-08-25 15:32:20.747503 192.168.1.2 -> 202.232.205.123 (-- -> JP) UDP 60583 33437 1 0 64 0 0.0000s +2006-08-25 15:32:20.651501 192.168.1.2 -> 202.232.205.123 (-- -> JP) UDP 60583 33436 1 0 64 0 0.0000s +``` + +Collecting DNS traffic from several files and storing it in a new pcap file. + +``` +Dshell> decode -p dns+pcapwriter --pcapwriter_outfile=test.pcap ~/pcap/*.cap > /dev/null +Dshell> tcpdump -nnr test.pcap | head +reading from file test.pcap, link-type EN10MB (Ethernet) +15:36:08.670569 IP 192.168.1.2.2131 > 192.168.1.1.53: 40209+ A? ui.skype.com. (30) +15:36:08.670687 IP 192.168.1.2.2131 > 192.168.1.1.53: 40210+ AAAA? ui.skype.com. (30) +15:36:08.674022 IP 192.168.1.1.53 > 192.168.1.2.2131: 40209- 1/0/0 A 212.72.49.131 (46) +15:36:09.011208 IP 192.168.1.1.53 > 192.168.1.2.2131: 40210 0/1/0 (94) +15:36:10.171350 IP 192.168.1.2.2131 > 192.168.1.1.53: 40210+ AAAA? ui.skype.com. (30) +15:36:10.961350 IP 192.168.1.1.53 > 192.168.1.2.2131: 40210* 0/1/0 (85) +15:36:10.961608 IP 192.168.1.2.2131 > 192.168.1.1.53: 40211+ AAAA? ui.skype.com. (30) +15:36:11.294333 IP 192.168.1.1.53 > 192.168.1.2.2131: 40211 0/1/0 (94) +15:32:21.664798 IP 192.168.1.2.2130 > 192.168.1.1.53: 39862+ A? ui.skype.com. (30) +15:32:21.664913 IP 192.168.1.2.2130 > 192.168.1.1.53: 39863+ AAAA? ui.skype.com. (30) +``` + +Collecting TFTP data and converting alerts to JSON format using [sample traffic](https://wiki.wireshark.org/SampleCaptures#TFTP) ``` -Dshell> decode -d country+netflow --country_code=JP ~/pcap/SkypeIRC.cap -2006-08-25 19:32:20.651502 192.168.1.2 -> 202.232.205.123 (-- -> JP) UDP 60583 33436 1 0 36 0 0.0000s -2006-08-25 19:32:20.766761 192.168.1.2 -> 202.232.205.123 (-- -> JP) UDP 60583 33438 1 0 36 0 0.0000s -2006-08-25 19:32:20.634046 192.168.1.2 -> 202.232.205.123 (-- -> JP) UDP 60583 33435 1 0 36 0 0.0000s -2006-08-25 19:32:20.747503 192.168.1.2 -> 202.232.205.123 (-- -> JP) UDP 60583 33437 1 0 36 0 0.0000s +Dshell> decode -p tftp -O jsonout ~/pcap/tftp_*.pcap +{"ts": 1367411051.972852, "sip": "192.168.0.253", "sport": 50618, "dip": "192.168.0.10", "dport": 3445, "readwrite": "read", "filename": "rfc1350.txt", "plugin": "tftp", "pcapfile": "/home/pcap/tftp_rrq.pcap", "data": "read rfc1350.txt (24599 bytes) "} +{"ts": 1367053679.45274, "sip": "192.168.0.1", "sport": 57509, "dip": "192.168.0.13", "dport": 2087, "readwrite": "write", "filename": "rfc1350.txt", "plugin": "tftp", "pcapfile": "/home/pcap/tftp_wrq.pcap", "data": "write rfc1350.txt (24599 bytes) "} ``` -Collecting netflow data for [sample traffic](http://wiki.wireshark.org/SampleCaptures#General_.2F_Unsorted) with vlan headers, then tracking the connection to a specific IP address +Running a plugin within a separate Python script using [sample traffic](https://wiki.wireshark.org/SampleCaptures#TFTP) ``` -Dshell> decode -d netflow ~/pcap/vlan.cap -1999-11-05 18:20:43.170500 131.151.20.254 -> 255.255.255.255 (US -> --) UDP 520 520 1 0 24 0 0.0000s -1999-11-05 18:20:42.063074 131.151.32.71 -> 131.151.32.255 (US -> US) UDP 138 138 1 0 201 0 0.0000s -1999-11-05 18:20:43.096540 131.151.1.254 -> 255.255.255.255 (US -> --) UDP 520 520 1 0 24 0 0.0000s -1999-11-05 18:20:43.079765 131.151.5.254 -> 255.255.255.255 (US -> --) UDP 520 520 1 0 24 0 0.0000s -1999-11-05 18:20:41.521798 131.151.104.96 -> 131.151.107.255 (US -> US) UDP 137 137 3 0 150 0 1.5020s -1999-11-05 18:20:43.087010 131.151.6.254 -> 255.255.255.255 (US -> --) UDP 520 520 1 0 24 0 0.0000s -1999-11-05 18:20:43.368210 131.151.111.254 -> 255.255.255.255 (US -> --) UDP 520 520 1 0 24 0 0.0000s -1999-11-05 18:20:43.250410 131.151.32.254 -> 255.255.255.255 (US -> --) UDP 520 520 1 0 24 0 0.0000s -1999-11-05 18:20:43.115330 131.151.10.254 -> 255.255.255.255 (US -> --) UDP 520 520 1 0 24 0 0.0000s -1999-11-05 18:20:43.375145 131.151.115.254 -> 255.255.255.255 (US -> --) UDP 520 520 1 0 24 0 0.0000s -1999-11-05 18:20:43.363348 131.151.107.254 -> 255.255.255.255 (US -> --) UDP 520 520 1 0 24 0 0.0000s -1999-11-05 18:20:40.112031 131.151.5.55 -> 131.151.5.255 (US -> US) UDP 138 138 1 0 201 0 0.0000s -1999-11-05 18:20:43.183825 131.151.32.79 -> 131.151.32.255 (US -> US) UDP 138 138 1 0 201 0 0.0000s +# Import required Dshell libraries +import dshell.decode as decode +import dshell.plugins.tftp.tftp as tftp + +# Instantiate plugin +plugin = tftp.DshellPlugin() +# Define plugin-specific arguments, if needed +dargs = {plugin: {"rip": True, "outdir": "/tmp/"}} +# Add plugin(s) to plugin chain +decode.plugin_chain = [plugin] +# Run decode main function with all other arguments +decode.main( + debug=True, + files=["/home/user/pcap/tftp_rrq.pcap", "/home/user/pcap/tftp_wrq.pcap"], + plugin_args=dargs +) ``` diff --git a/bin/decode.py b/bin/decode.py deleted file mode 100755 index 561352d..0000000 --- a/bin/decode.py +++ /dev/null @@ -1,765 +0,0 @@ -#!/usr/bin/env python - -import sys,os,glob,copy -import optparse -import logging,traceback -import gzip,zipfile,tempfile -import output,util -import dshell -try: import pcap -except: - pcap=None - print 'pcap not available: decoders requiring pcap are not usable' - -def import_module(name=None,silent=False,search={}): - try: - #we will first check search[name] for the module - #else split foo.bar:baz to get from foo.bar import baz - #else split dotted path to perform a 'from foo import bar' operation - try: module=search[name] #a -> from search[a] import a - except: - module,name=name.split('.')[:-1],name.split('.')[-1] #a.b.c from a.b import c - if module: module='.'.join(module) - else: module=name - path=None - if os.path.sep in module: #was a full path to a decoder given? - path,module=os.path.dirname(module),os.path.basename(module) - #print module,name - if path: sys.path.append(path) - obj=__import__(module,fromlist=[name]) - if path: sys.path.remove(path) - if 'dObj' in dir(obj) or 'obj' in dir(obj): return obj - elif name in dir(obj): - obj = getattr(obj, name) - if 'dObj' in dir(obj) or 'obj' in dir(obj): return obj - except Exception, err: - if not silent: sys.stderr.write( "Error '%s' loading module %s\n" % (str(err),module)) - return False - -def setDecoderPath(decoder_path): - '''set the base decoder path, - add it to sys.path for importing, - and walk it to return all subpaths''' - paths=[] - paths.append(decoder_path) #append base path first - # walk decoder directories an add to sys.path - for root, dirs, files in os.walk(decoder_path): - [dirs.remove(d) for d in dirs if d.startswith('.')] #skip hidden dirs like .svn - for d in sorted(dirs): paths.append(os.path.join(root,d)) - return paths #return the paths we found - -def getDecoders(decoder_paths): - ''' find all decoders and map decoder to import.path.decoder - expect common prefix to start with basepath''' - import_base=os.path.commonprefix(decoder_paths).split(os.path.sep)[:-1] #keep last part as base - decoders={} - for path in decoder_paths: - # split path and trim off part before base - import_path=path.split(os.path.sep)[len(import_base):] - for f in glob.iglob("%s/*.py" % path): - name=os.path.splitext(os.path.basename(f))[0] - if name != '__init__': #skip package stubs - #build topdir.path...module name from topdir/dir.../file - decoders[name]='.'.join(import_path+[name]) - return decoders - -def printDecoders(decoder_map,silent=True): - '''Print list of decoders with additional info''' - dList = [] - FS=' %-40s %-30s %-10s %s %1s %s' - for name,module in sorted(decoder_map.iteritems()): - try: - try: decoder=import_module(module,silent).dObj - except Exception, e: - print "Exception loading module '%s': %s" % (module, e) - continue - # get the type of decoder it is - dtype = 'RAW' - if 'IP' in dir(decoder): dtype = 'IP ' - if 'UDP' in dir(decoder): dtype = 'UDP' - if 'TCP' in dir(decoder): dtype = 'TCP' - dList.append( FS % (module,decoder.name, - decoder.author, - dtype,'+' if decoder.chainable else '', - decoder.description)) - except: pass - - print FS %('module','name','author',' ',' ','desc') - print FS %('-'*40,'-'*30,'-'*10,'---','-','-'*50) - for d in sorted(dList): print d - -def readInFilter(fname): - '''Read in a BPF filter provided by a command line argument''' - filter = '' - tmpfd = open(fname,'r') - for line in tmpfd: - if '#' in line: - line = line.split('#')[0]+'\n' # keep \n for visual output sanity - filter += line - tmpfd.close() - - return filter - -def decode_live(out,options,decoder,decoder_args,decoder_options): - - #set decoder options - initDecoderOptions(decoder,out,options,decoder_args,decoder_options) - - if 'preModule' in dir(decoder): - decoder.preModule() - - decoder.input_file = options.interface # give the interface name to the decoder - stats=None - if options.verbose: log('Attempting to listen on %s' % options.interface) - try: - - if not pcap: raise NotImplementedError("raw capture support not implemented") - decoder.capture=pcap.pcap(options.interface,65535,True) - if decoder.filter: decoder.capture.setfilter(decoder.filter) - while not options.count or decoder.count -1: - (path,wildcard) = os.path.split(file_path) - - # If just file is specified (no path) - if len(path) == 0: - inputs.extend(glob.glob(wildcard)) - - # If there is a path, but recursion not specified, - # then just add matching files from specified dir - elif not len(path) == 0 and not options.recursive: - inputs.extend(glob.glob(file_path)) - - # Otherwise, recursion specified and there is a directory. - # Recurse directory and add files - else: - addFilesFromDirectory(inputs,path,wildcard) - - # Just a normal file, append to list of inputs - else: - inputs.append(file_path) - - if options.parallel or options.threaded: - import multiprocessing - procs=[] - q=multiprocessing.Queue() - kwargs=options.__dict__.copy() #put parsed base options in kwargs - kwargs.update(config=None,outfile=None,queue=q) #pass the q, - #do not pass the config file or outfile because we handled that here - for d in decoder_options: #put pre-parsed decoder options in kwargs - for k,v in decoder_options[d].items(): kwargs[d+'_'+k]=v - - #check here to see if we are running in parallel-file mode - if options.parallel and len(inputs)>1: - for f in inputs: - #create a child process for each input file - procs.append(multiprocessing.Process(target=main,kwargs=kwargs,args=[f])) - runChildProcs(procs,q,out,numprocs=options.numprocs) - - #check here to see if we are running decoders multithreaded - elif options.threaded and len(options.decoder)>1: - for d in options.decoder: - #create a child for each decoder - kwargs.update(decoder=d) - procs.append(multiprocessing.Process(target=main,kwargs=kwargs,args=inputs)) - runChildProcs(procs,q,out,numprocs=options.numprocs) - - #fall through to here (single threaded or child process) - else: - # - # Here is where we use the decoder(s) to process the pcap - # - - temporaryFiles = [] # used when uncompressing files - - for module in decoders.keys(): - decoder = decoders[module] - initDecoderOptions(decoder,out,options,decoder_args,decoder_options) - - # If the decoder has a preModule function, will execute it now - decoder.preModule() - - - for input_file in inputs: - # Decoder-specific options may be seen as input files - # Skip anything starts with "--" - if input_file[:2] == '--': - continue - - # Recursive directory processing is handled elsewhere, - # so we should only be dealing with files at this point. - if os.path.isdir(input_file): continue - - log('+Processing file %s' % input_file) - - # assume the input_file is not compressed - # Allows the processing of .pcap files that are compressed with - # gzip, bzip2, or zip. Writes uncompressed file to a - # NamedTemporaryFile and unlinks the file once it is no longer - # needed. Might consider using mkstemp() since this implementation - # requires Python >= 2.6. - try: - exts = ['.gz','.bz2','.zip'] - if os.path.splitext(input_file)[1] not in exts: - pcapfile=input_file - - else: - # we have a compressed file - tmpfile = expandCompressedFile(input_file,options.verbose,options.tmpdir) - temporaryFiles.append(tmpfile) - pcapfile=tmpfile - except: - if options.verbose: sys.stderr.write('+Error processing file %s' % (input_file)) - continue - - # give the decoder access to the input filename - # motivation: run a decoder against a large number of pcap - # files and have the decoder print the filename - # so you can go straight to the pcap file for - # further analysis - decoder.input_file = input_file - - # Check to see if the decoder has a preFile function - # This will be called before the decoder processes each - # input file - decoder.preFile() - - try: - if not pcap: raise NotImplementedError("pcap support not implemented") - decoder.capture=pcap.pcap(pcapfile) - if decoder.filter: decoder.capture.setfilter(decoder.filter) - while not options.count or decoder.count2 and sys.argv[2]=='with_bash_completion': - outfd.write(''' - - -if [ `echo $BASH_VERSION | cut -d'.' -f1` -ge '4' ]; then -if [ -f ~/.bash_aliases ]; then -. ~/.bash_aliases -fi - -if [ -f /etc/bash_completion ]; then -. /etc/bash_completion -fi - -find_decoder() -{ -local IFS="+" -for (( i=0; i<${#COMP_WORDS[@]}; i++ )); -do - if [ "${COMP_WORDS[$i]}" == '-d' ] ; then - decoders=(${COMP_WORDS[$i+1]}) - fi -done -} - -get_decoders() -{ - decoders=$(for x in `find $DECODERPATH -iname '*.py' | grep -v '__init__'`; do basename ${x} .py; done) -} - -_decode() -{ -local dashdashcommands=' --ebpf --output --outfile --logfile' - -local cur prev xspec decoders -COMPREPLY=() -cur=`_get_cword` -_expand || return 0 -prev="${COMP_WORDS[COMP_CWORD-1]}" - -case "${cur}" in ---*) - find_decoder - local options="" -# if [ -n "$decoders" ]; then -# for decoder in "${decoders[@]}" -# do -# options+=`/usr/bin/python $BINPATH/gen_decoder_options.py $decoder` -# options+=" " -# done -# fi - - options+=$dashdashcommands - COMPREPLY=( $(compgen -W "${options}" -- ${cur}) ) - return 0 - ;; - -*+*) - get_decoders - firstdecoder=${cur%+*}"+" - COMPREPLY=( $(compgen -W "${decoders}" -P $firstdecoder -- ${cur//*+}) ) - return 0 - ;; - -esac - -xspec="*.@(cap|pcap)" -xspec="!"$xspec -case "${prev}" in --d) - get_decoders - COMPREPLY=( $(compgen -W "${decoders[0]}" -- ${cur}) ) - return 0 - ;; - ---output) - local outputs=$(for x in `find $DSHELL/lib/output -iname '*.py' | grep -v 'output.py'`; do basename ${x} .py; done) - - COMPREPLY=( $(compgen -W "${outputs}" -- ${cur}) ) - return 0 - ;; - --F | -o | --outfile | -L | --logfile) - xspec= - ;; - -esac - -COMPREPLY=( $( compgen -f -X "$xspec" -- "$cur" ) \ -$( compgen -d -- "$cur" ) ) -} -complete -F _decode -o filenames decode -complete -F _decode -o filenames decode.py -fi -''') - outfd.close() - - #dshell text - outfd = open('dshell','w') - outfd.write('#!/bin/bash\n') - outfd.write('/bin/bash --rcfile %s/.dshellrc\n' % (cwd)) - outfd.close() - - #dshell-decode text - outfd = open('dshell-decode','w') - outfd.write('#!/bin/bash\n') - outfd.write('source %s/.dshellrc\n' % (cwd)) - outfd.write('decode "$@"') - outfd.close() diff --git a/bin/pcapanon.py b/bin/pcapanon.py deleted file mode 100755 index 29d1b12..0000000 --- a/bin/pcapanon.py +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env python -''' -Created on Feb 6, 2012 - -@author: tparker -''' - -import sys,dpkt,struct,pcap,socket,time -from Crypto.Random import random -from Crypto.Hash import SHA -from output import PCAPWriter -from util import getopts - -def hashaddr(addr,*extra): - #hash key+address plus any extra data (ports if flow) - global key,ip_range,ip_mask - sha=SHA.new(key+addr) - for e in extra: sha.update(str(extra)) - #take len(addr) octets of digest as address, to int, mask, or with range, back to octets - return inttoip( ( iptoint(sha.digest()[0:len(addr)]) & ip_mask) | ip_range ) - -def mangleMAC(addr): - global zero_mac - if zero_mac: return "\x00\x00\x00\x00\x00\x00" - if addr in emap: return emap[addr] - haddr=None - if addr == "\x00\x00\x00\x00\x00\x00": haddr=addr #return null MAC - if ord(addr[0])&0x01: haddr=addr #mac&0x800000000000 == broadcast addr, do not touch - if not haddr: - haddr=hashaddr(addr) - haddr=chr(ord(haddr[0])&0xfc|0x2)+haddr[1:6] #return hash bytes with first byte set to xxxxxx10 (LAA unicast) - emap[addr]=haddr - return haddr - -def mangleIP(addr,*ports): #addr,extra=our_port,other_port - global exclude,exclude_port,anon_all,by_flow - haddr=None - intip=iptoint(addr) - if len(addr)==4 and intip>=0xE0000000: haddr=addr #pass multicast 224.x.x.x and higher - ip=iptoa(addr) - #pass 127.x.x.x, IANA reserved, and autoconfig ranges - if not anon_all and (ip.startswith('127.') \ - or ip.startswith('10.') \ - or ip.startswith('172.16.') \ - or ip.startswith('192.168.') \ - or ip.startswith('169.254.')): haddr=addr - #pass ips matching exclude - for x in exclude: - if ip.startswith(x): haddr=addr - if ports and ports[0] in exclude_port: haddr=addr #if our port is exclude - if not haddr: - if by_flow: haddr=hashaddr(addr,*ports) #use ports if by flow, else just use ip - else: haddr=hashaddr(addr) - return haddr - -def mangleIPs(src,dst,sport,dport): - if by_flow: #if by flow, hash addresses with s/d ports - if (src,sport,dst,dport) in ipmap: src,dst=ipmap[(src,sport,dst,dport)] - elif (dst,dport,src,sport) in ipmap: dst,src=ipmap[(dst,dport,src,sport)] #make sure reverse flow maps same - else: src,dst=ipmap.setdefault((src,sport,dst,dport),(mangleIP(src,sport,dport),mangleIP(dst,dport,sport))) - else: - if src in ipmap: src=ipmap[src] - else: src=ipmap.setdefault(src,mangleIP(src,sport)) - if dst in ipmap: dst=ipmap[dst] - else: dst=ipmap.setdefault(dst,mangleIP(dst,dport)) - return src,dst - -def mactoa(addr): - return ':'.join(['%02x'%b for b in struct.unpack('6B',addr)]) - -def iptoa(addr): - if len(addr) is 16: return socket.inet_ntop(socket.AF_INET6,addr) - else: return socket.inet_ntop(socket.AF_INET,addr) - -def iptoint(addr): - if len(addr) is 16: #ipv6 to long - ip=struct.unpack('!IIII',addr) - return ip[0]<<96|ip[1]<<64|ip[2]<<32|ip[3] - else: return struct.unpack('!I',addr)[0] #ip to int - -def inttoip(l): - if l>0xffffffff: #ipv6 - return struct.pack('!IIII',l>>96,l>>64&0xffffffff,l>>32&0xffffffff,l&0xffffffff) - else: return struct.pack('!I',l) - -def pcap_handler(ts,pktdata): - global init_ts,start_ts,replace_ts,by_flow,anon_mac,zero_mac - if not init_ts: init_ts=ts - if replace_ts: ts=start_ts+(ts-init_ts) #replace timestamps - try: - pkt=dpkt.ethernet.Ethernet(pktdata) - if anon_mac or zero_mac: - pkt.src=mangleMAC(pkt.src) - pkt.dst=mangleMAC(pkt.dst) - if pkt.type==dpkt.ethernet.ETH_TYPE_IP: - try: sport,dport=pkt.data.data.sport,pkt.data.data.dport #TCP or UDP? - except: sport=dport=None #nope - pkt.data.src,pkt.data.dst=mangleIPs(pkt.data.src,pkt.data.dst,sport,dport) - pktdata=str(pkt) - except Exception,e: print e - out.write(len(pktdata),pktdata,ts) - -if __name__ == '__main__': - - global key,init_ts,start_ts,replace_ts,by_flow,anon_mac,zero_mac,exclude,exclude_port,anon_all,ip_range,ip_mask - opts,args=getopts(sys.argv[1:],'i:aezftx:p:rk:',['ip=','all','ether','zero','flow','ts','exclude=','random','key=','port='],['-x','--exclude','-p','--port']) - - if '-r' in opts or '--random' in opts: key=random.long_to_bytes(random.getrandbits(64),8) - else: key='' - key=opts.get('-k',opts.get('--key',key)) - - ip_range=opts.get('-i',opts.get('--ip','0.0.0.0')) - ip_mask=0 #bitmask for hashed address - ipr='' - for o in map(int,ip_range.split('.')): - ipr+=chr(o) - ip_mask<<=8 #shift by 8 bits - if not o: ip_mask|=0xff #set octet mask to 0xff if ip_range octet is zero - ip_range=iptoint(ipr) #convert to int value for hash&mask|ip_range - - replace_ts='-t' in opts or '--ts' in opts - by_flow='-f' in opts or '--flow' in opts - anon_mac='-e' in opts or '--ether' in opts - zero_mac='-z' in opts or '--zero' in opts - anon_all='-a' in opts or '--all' in opts - - start_ts=time.time() - init_ts=None - - exclude=opts.get('-x',[]) - exclude.extend(opts.get('--exclude',[])) - - exclude_port=map(int,opts.get('-p',[])) - exclude_port.extend(map(int,opts.get('--port',[]))) - - emap={} - ipmap={} - - if len(args)<2: - print "usage: pcapanon.py [options] > mapping.csv\nOptions:\n\t[-i/--ip range]\n\t[-r/--random | -k/--key 'salt' ]\n\t[-a/--all] [-t/--ts] [-f/--flow]\n\t[-e/--ether | -z/--zero]\n\t[-x/--exclude pattern...]\n\t[-p/--port list...]" - print "Will anonymize all non-reserved IPs to be in range specified by -i/--ip option," - print "\tnonzero range octets are copied to anonymized address,\n\t(default range is 0.0.0.0 for fully random IPs)" - print "CSV output maps original to anonymized addresses" - print "By default anonymization will use a straight SHA1 hash of the address" - print "\t***this is crackable as mapping is always the same***".upper() - print "Use -r/--random to generate a random salt (cannot easily reverse without knowing map)" - print "\tor use -k/--key 'salt' (will generate same mapping given same salt)," - print "-f/--flows will anonymize by flow (per source:port<->dest:port tuples)" - print "-a/--all will also anonymize reserved IPs" - print "-x/--exclude will leave IPs starting with pattern unchanged" - print "-p/--port port will leave IP unchanged if port is in list" - print "-t/--ts will replace timestamp of first packet with time pcapanon was run,\n\tsubsequent packets will preserve delta from initial ts" - print "-e/--ether will also anonymize non-broadcast MAC addresses" - print "-z/--zero will zero all MAC addresses" - sys.exit(0) - - out=PCAPWriter(args[-1]) - print '#file, packets' - for f in args[0:-1]: - p=0 - cap=pcap.pcap(f) - while cap.dispatch(1,pcap_handler): p+=1 #process whole file - del cap - print '%s,%s'%(f,p) - out.close() - - print "#type,is-anonymized, original, anonymized" - for ia,oa in sorted(emap.items()): print 'ether,%d, %s, %s'%(int(not ia==oa),mactoa(ia),mactoa(oa)) - for ia,oa in sorted(ipmap.items()): - if by_flow: - sip,sp,dip,dp=ia - osip,odip=oa - print "flow,%d, %s:%s,%s:%s, %s:%s,%s:%s"%(int(sip!=osip or dip!=odip),iptoa(sip),sp,iptoa(dip),dp,iptoa(osip),sp,iptoa(odip),dp) - else: print 'ip,%d, %s, %s'%(int(ia!=oa),iptoa(ia),iptoa(oa)) diff --git a/decoders/dns/dns-asn.py b/decoders/dns/dns-asn.py deleted file mode 100644 index c545658..0000000 --- a/decoders/dns/dns-asn.py +++ /dev/null @@ -1,65 +0,0 @@ -import dshell,dpkt,socket -from dnsdecoder import DNSDecoder - -class DshellDecoder(DNSDecoder): - def __init__(self): - DNSDecoder.__init__(self, - name = 'dns-asn', - description = 'identify AS of DNS A/AAAA record responses', - filter = '(port 53)', - author = 'bg', - cleanupinterval=10, - maxblobs=2, - ) - - def decode_q(self,dns): - queried = "" - if dns.qd[0].type == dpkt.dns.DNS_A: - queried = queried + "A? %s" % (dns.qd[0].name) - if dns.qd[0].type == dpkt.dns.DNS_AAAA: - queried = queried + "AAAA? %s" % (dns.qd[0].name) - return queried - - def DNSHandler(self,conn,request,response,**kwargs): - anstext='' - queried='' - id=None - for dns in request,response: - if dns is None: continue - id=dns.id - #DNS Question, update connection info with query - if dns.qr==dpkt.dns.DNS_Q: conn.info(query=self.decode_q(dns)) - - # DNS Answer with data and no errors - elif (dns.qr == dpkt.dns.DNS_A and dns.rcode == dpkt.dns.DNS_RCODE_NOERR and len(dns.an)>0): - - queried=self.decode_q(dns) - - answers = [] - for an in dns.an: - if an.type == dpkt.dns.DNS_A: - try: - cc = self.getASN( socket.inet_ntoa(an.ip) ) - answers.append('A: %s (%s) (ttl %s)' % (socket.inet_ntoa(an.ip), cc, an.ttl) ) - except: continue - elif an.type == dpkt.dns.DNS_AAAA: - try: - cc = self.getASN( socket.inet_ntop(socket.AF_INET6, an.ip6) ) - answers.append('AAAA: %s (%s) (ttl %s)' % (socket.inet_ntop(socket.AF_INET6, an.ip6), cc, an.ttl) ) - except: continue - else: - # un-handled type - continue - if queried != '': - anstext =", ".join(answers) - - if anstext: #did we get an answer? - self.alert(str(id)+' '+queried+' / '+anstext,**conn.info(response=anstext)) - - - -if __name__=='__main__': - dObj = DshellDecoder() - print dObj -else: - dObj = DshellDecoder() diff --git a/decoders/dns/dns-cc.py b/decoders/dns/dns-cc.py deleted file mode 100644 index e2154f7..0000000 --- a/decoders/dns/dns-cc.py +++ /dev/null @@ -1,71 +0,0 @@ -import dshell,dpkt,socket -from dnsdecoder import DNSDecoder - -class DshellDecoder(DNSDecoder): - def __init__(self): - DNSDecoder.__init__(self, - name = 'dns-cc', - description = 'identify country code of DNS A/AAAA record responses', - filter = '(port 53)', - author = 'bg', - cleanupinterval=10, - maxblobs=2, - optiondict={'foreign':{'action':'store_true','help':'report responses in foreign countries'}, - 'code':{'type':'string', 'help':'filter on a specific country code (ex. US)'}} - ) - - def decode_q(self,dns): - queried = "" - if dns.qd[0].type == dpkt.dns.DNS_A: - queried = queried + "A? %s" % (dns.qd[0].name) - if dns.qd[0].type == dpkt.dns.DNS_AAAA: - queried = queried + "AAAA? %s" % (dns.qd[0].name) - return queried - - def DNSHandler(self,conn,request,response,**kwargs): - anstext='' - queried='' - id=None - for dns in request,response: - if dns is None: continue - id=dns.id - #DNS Question, update connection info with query - if dns.qr==dpkt.dns.DNS_Q: conn.info(query=self.decode_q(dns)) - - # DNS Answer with data and no errors - elif (dns.qr == dpkt.dns.DNS_A and dns.rcode == dpkt.dns.DNS_RCODE_NOERR and len(dns.an)>0): - - queried=self.decode_q(dns) - - answers = [] - for an in dns.an: - if an.type == dpkt.dns.DNS_A: - try: - cc = self.getGeoIP( socket.inet_ntoa(an.ip) ) - if self.foreign and ( cc == 'US' or cc == '--' ): continue - elif self.code != None and cc != self.code: continue - answers.append('A: %s (%s) (ttl %ss)' % (socket.inet_ntoa(an.ip), cc, an.ttl) ) - except: continue - elif an.type == dpkt.dns.DNS_AAAA: - try: - cc = self.getGeoIP( socket.inet_ntop(socket.AF_INET6, an.ip6) ) - if self.foreign and ( cc == 'US' or cc == '--' ): continue - elif self.code != None and cc != self.code: continue - answers.append('AAAA: %s (%s) (ttl %ss)' % (socket.inet_ntop(socket.AF_INET6, an.ip6), cc, an.ttl) ) - except: continue - else: - # un-handled type - continue - if queried != '': - anstext =", ".join(answers) - - if anstext: #did we get an answer? - self.alert(str(id)+' '+queried+' / '+anstext,**conn.info(response=anstext)) - - - -if __name__=='__main__': - dObj = DshellDecoder() - print dObj -else: - dObj = DshellDecoder() diff --git a/decoders/dns/dns.py b/decoders/dns/dns.py deleted file mode 100644 index f097782..0000000 --- a/decoders/dns/dns.py +++ /dev/null @@ -1,98 +0,0 @@ -import dpkt,socket -from dnsdecoder import DNSDecoder - -class DshellDecoder(DNSDecoder): - def __init__(self): - DNSDecoder.__init__(self, - name = 'dns', - description = 'extract and summarize DNS queries/responses (defaults: A,AAAA,CNAME,PTR records)', - filter = '(udp and port 53)', - author = 'bg/twp', - optiondict={'show_noanswer':{'action':'store_true','help':'report unanswered queries alongside other queries'}, - 'show_norequest':{'action':'store_true','help':'report unsolicited responses alongside other responses'}, - 'only_noanswer':{'action':'store_true','help':'report only unanswered queries'}, - 'only_norequest':{'action':'store_true','help':'report only unsolicited responses'}, - 'showall':{'action':'store_true','help':'show all answered queries/responses'}} - ) - - def decode_q(self,dns): - queried = "" - if dns.qd[0].type == dpkt.dns.DNS_A: - queried = queried + "A? %s" % (dns.qd[0].name) - if dns.qd[0].type == dpkt.dns.DNS_CNAME: - queried = queried + "CNAME? %s" % (dns.qd[0].name) - if dns.qd[0].type == dpkt.dns.DNS_AAAA: - queried = queried + "AAAA? %s" % (dns.qd[0].name) - if dns.qd[0].type == dpkt.dns.DNS_PTR: - if dns.qd[0].name.endswith('.in-addr.arpa'): - query_name = '.'.join(reversed(dns.qd[0].name.split('.in-addr.arpa')[0].split('.'))) - else: - query_name = dns.qd[0].name - queried = queried + "PTR? %s" % (query_name) - - if not self.showall: return queried - - if dns.qd[0].type == dpkt.dns.DNS_NS: - queried = queried + "NS? %s" % (dns.qd[0].name) - if dns.qd[0].type == dpkt.dns.DNS_MX: - queried = queried + "MX? %s" % (dns.qd[0].name) - if dns.qd[0].type == dpkt.dns.DNS_TXT: - queried = queried + "TXT? %s" % (dns.qd[0].name) - if dns.qd[0].type == dpkt.dns.DNS_SRV: - queried = queried + "SRV? %s" % (dns.qd[0].name) - - return queried - - def DNSHandler(self,conn,request,response,**kwargs): - if self.only_norequest and request is not None: return - if not self.show_norequest and request is None: return - anstext='' - queried='' - id=None - for dns in request,response: - if dns is None: continue - id=dns.id - #DNS Question, update connection info with query - if dns.qr==dpkt.dns.DNS_Q: conn.info(query=self.decode_q(dns)) - - # DNS Answer with data and no errors - elif (dns.qr == dpkt.dns.DNS_A and dns.rcode == dpkt.dns.DNS_RCODE_NOERR and len(dns.an)>0): - - queried=self.decode_q(dns) - - answers = [] - for an in dns.an: - if an.type == dpkt.dns.DNS_A: - try: answers.append('A: %s (ttl %ss)' % (socket.inet_ntoa(an.ip), str(an.ttl))) - except: continue - elif an.type == dpkt.dns.DNS_AAAA: - try: answers.append('AAAA: %s (ttl %ss)' % (socket.inet_ntop(socket.AF_INET6,an.ip6),str(an.ttl))) - except: continue - elif an.type == dpkt.dns.DNS_CNAME: answers.append('CNAME: '+an.cname) - elif an.type == dpkt.dns.DNS_PTR: answers.append('PTR: '+an.ptrname) - elif an.type == dpkt.dns.DNS_NS: answers.append('NS: '+an.nsname) - elif an.type == dpkt.dns.DNS_MX: answers.append('MX: '+an.mxname) - elif an.type == dpkt.dns.DNS_TXT: answers.append('TXT: '+' '.join(an.text)) - elif an.type == dpkt.dns.DNS_SRV: answers.append('SRV: '+an.srvname) - else: - # un-handled type - continue - if queried != '': - anstext =", ".join(answers) - - #NXDOMAIN in response - elif dns.qr == dpkt.dns.DNS_A and dns.rcode == dpkt.dns.DNS_RCODE_NXDOMAIN: - queried = self.decode_q(dns) #decode query part - - if queried != '': anstext='NXDOMAIN' - - if anstext and not self.only_noanswer and not self.only_norequest: #did we get an answer? - self.alert(str(id)+' '+queried+' / '+anstext,**conn.info(response=anstext)) - elif not anstext and (self.show_noanswer or self.only_noanswer): - self.alert(str(id)+' '+conn.query+' / (no answer)',**conn.info()) - -if __name__=='__main__': - dObj = DshellDecoder() - print dObj -else: - dObj = DshellDecoder() diff --git a/decoders/dns/innuendo-dns.py b/decoders/dns/innuendo-dns.py deleted file mode 100644 index 2810570..0000000 --- a/decoders/dns/innuendo-dns.py +++ /dev/null @@ -1,84 +0,0 @@ -import dpkt -from dnsdecoder import DNSDecoder -import base64 - -class DshellDecoder(DNSDecoder): - """ - Proof-of-concept Dshell decoder to detect INNUENDO DNS Channel - - Based on the short marketing video [http://vimeo.com/115206626] the INNUENDO - DNS Channel relies on DNS to communicate with an authoritative name server. - The name server will respond with a base64 encoded TXT answer. This decoder - will analyze DNS TXT queries and responses to determine if it matches the - network traffic described in the video. There are multiple assumptions (*very - poor*) in this detection plugin but serves as a proof-of-concept detector. This - detector has not been tested against authentic INNUENDO DNS Channel traffic. - - Usage: decode -d innuendo-dns *.pcap - - """ - - def __init__(self): - DNSDecoder.__init__(self, - name = 'innuendo-dns', - description = 'proof-of-concept detector for INNUENDO DNS channel', - filter = '(port 53)', - author = 'primalsec', - ) - self.whitelist = [] # probably be necessary to whitelist A/V domains - - def in_whitelist(self, domain): - # add logic - return False - - def decrypt_payload(payload): pass - - def DNSHandler(self,conn,request,response,**kwargs): - query = '' - answers = [] - - for dns in request,response: - - if dns is None: continue - - id = dns.id - - #DNS Question, extract query name if it is a TXT record request - if dns.qr==dpkt.dns.DNS_Q and dns.qd[0].type == dpkt.dns.DNS_TXT: - query = dns.qd[0].name - - # DNS Answer with data and no errors - elif (dns.qr == dpkt.dns.DNS_A and dns.rcode == dpkt.dns.DNS_RCODE_NOERR and len(dns.an)>0): - - for an in dns.an: - if an.type == dpkt.dns.DNS_TXT: - answers.append(an.text[0]) - - if query != '' and len(answers)>0: - # add check here to see if the second level domain and top level domain are not in a white list - if self.in_whitelist(query): return - - # assumption: INNUENDO will use the lowest level domain for C2 - # example: AAAABBBBCCCC.foo.bar.com -> AAAABBBBCCCC is the INNUENDO data - subdomain = query.split('.')[0] - - if subdomain.isupper(): # weak test based on video observation *very poor assumption* - # check each answer in the TXT response - for answer in answers: - try: - # INNUENDO DNS channel base64 encodes the response, check to see if - # it contains a valid base64 string *poor assumption* - dummy = base64.b64decode( answer ) - - self.alert('INNUENDO DNS Channel', query ,'/',answer,**conn.info()) - - # here would be a good place to decrypt the payload (if you have the keys) - # decrypt_payload( answer ) - except: pass - - -if __name__=='__main__': - dObj = DshellDecoder() - print dObj -else: - dObj = DshellDecoder() diff --git a/decoders/dns/reservedips.py b/decoders/dns/reservedips.py deleted file mode 100644 index 655152f..0000000 --- a/decoders/dns/reservedips.py +++ /dev/null @@ -1,106 +0,0 @@ -import dshell, dpkt, socket -from dnsdecoder import DNSDecoder -import IPy - -class DshellDecoder(DNSDecoder): - def __init__(self): - DNSDecoder.__init__(self, - name = 'reservedips', - description = 'identify DNS resolutions that fall into reserved ip space', - filter = '(port 53)', - author = 'bg', - cleanupinterval=10, - maxblobs=2, - ) - - # source: https://en.wikipedia.org/wiki/Reserved_IP_addresses - nets = [ '0.0.0.0/8', # Used for broadcast messages to the current ("this") network as specified by RFC 1700, page 4. - '10.0.0.0/8', # Used for local communications within a private network as specified by RFC 1918. - '100.64.0.0/10', #Used for communications between a service provider and its subscribers when using a Carrier-grade NAT, as specified by RFC 6598. - '127.0.0.0/8', # Used for loopback addresses to the local host, as specified by RFC 990. - '169.254.0.0/16', # Used for autoconfiguration between two hosts on a single link when no IP address is otherwise specified - '172.16.0.0/12', # Used for local communications within a private network as specified by RFC 1918 - '192.0.0.0/29', # Used for the DS-Lite transition mechanism as specified by RFC 6333 - '192.0.2.0/24', # Assigned as "TEST-NET" in RFC 5737 for use solely in documentation and example source code and should not be used publicly - '192.88.99.0/24', # Used by 6to4 anycast relays as specified by RFC 3068 - '192.168.0.0/16', # Used for local communications within a private network as specified by RFC 1918 - '198.18.0.0/15', # Used for testing of inter-network communications between two separate subnets as specified in RFC 2544 - '198.51.100.0/24', # Assigned as "TEST-NET-2" in RFC 5737 for use solely in documentation and example source code and should not be used publicly - '203.0.113.0/24', # Assigned as "TEST-NET-3" in RFC 5737 for use solely in documentation and example source code and should not be used publicly - '224.0.0.0/4', # Reserved for multicast assignments as specified in RFC 5771 - '240.0.0.0/4', # Reserved for future use, as specified by RFC 6890 - '255.255.255.255/32', # Reserved for the "limited broadcast" destination address, as specified by RFC 6890 - - '::/128', # Unspecified address - '::1/128', # loopback address to the local host. - '::ffff:0:0/96', # IPv4 mapped addresses - '100::/64', # Discard Prefix RFC 6666 - '64:ff9b::/96', # IPv4/IPv6 translation (RFC 6052) - '2001::/32', # Teredo tunneling - '2001:10::/28', # Overlay Routable Cryptographic Hash Identifiers (ORCHID) - '2001:db8::/32', # Addresses used in documentation - '2002::/16', # 6to4 - 'fc00::/7', # Unique local address - 'fe80::/10', # Link-local address - 'ff00::/8', # Multicast - ] - - self.reservednets= [] - for net in nets: - self.reservednets.append(IPy.IP(net)) - self.domains = [] # list for known domains - - def inReservedSpace(self,ipaddress): - for net in self.reservednets: - if ipaddress in net: return True - return False - - def decode_q(self,dns): - queried = "" - if dns.qd[0].type == dpkt.dns.DNS_A: - queried = queried + "A? %s" % (dns.qd[0].name) - if dns.qd[0].type == dpkt.dns.DNS_AAAA: - queried = queried + "AAAA? %s" % (dns.qd[0].name) - return queried - - def DNSHandler(self,conn,request,response,**kwargs): - anstext='' - queried='' - id=None - for dns in request,response: - if dns is None: continue - id=dns.id - #DNS Question, update connection info with query - if dns.qr==dpkt.dns.DNS_Q: conn.info(query=self.decode_q(dns)) - - # DNS Answer with data and no errors - elif (dns.qr == dpkt.dns.DNS_A and dns.rcode == dpkt.dns.DNS_RCODE_NOERR and len(dns.an)>0): - - queried=self.decode_q(dns) - - answers = [] - for an in dns.an: - if an.type == dpkt.dns.DNS_A: - try: - if self.inReservedSpace(socket.inet_ntoa(an.ip)): answers.append('A: '+socket.inet_ntoa(an.ip)+' (ttl '+str(an.ttl)+'s)') - except: continue - elif an.type == dpkt.dns.DNS_AAAA: - try: - if self.inReservedSpace(socket.inet_ntop(socket.AF_INET6,an.ip6)): answers.append('AAAA: '+socket.inet_ntop(socket.AF_INET6,an.ip6)+' (ttl '+str(an.ttl)+'s)') - except: continue - else: - # un-handled type - continue - if queried != '': - anstext =", ".join(answers) - - if anstext: #did we get an answer? - self.alert(str(id)+' '+queried+' / '+anstext,**conn.info(response=anstext)) - - - -if __name__=='__main__': - dObj = DshellDecoder() - print dObj -else: - dObj = DshellDecoder() diff --git a/decoders/filter/country.py b/decoders/filter/country.py deleted file mode 100644 index 46498e7..0000000 --- a/decoders/filter/country.py +++ /dev/null @@ -1,107 +0,0 @@ -''' -@author: tparker -''' - -import dshell,util -import netflowout - -class DshellDecoder(dshell.TCPDecoder): - '''activity tracker ''' - def __init__(self,**kwargs): - ''' - Constructor - ''' - self.sessions={} - self.alerts=False - self.file=None - dshell.TCPDecoder.__init__(self, - name = 'country', - description = 'filter connections on geolocation (country code)', - longdescription=""" -country: filter connections on geolocation (country code) - -Chainable decoder to filter TCP/UDP streams on geolocation data. If no -downstream (+) decoders are specified, netflow data will be printed to -the screen. - -Mandatory option: - - --country_code: specify (2 character) country code to filter on - -Default behavior: - - If either the client or server IP address matches the specified country, - the stream will be included. - -Modifier options: - - --country_neither: Include only streams where neither the client nor the - server IP address matches the specified country. - - --country_both: Include only streams where both the client AND the server - IP addresses match the specified country. - - --country_notboth: Include streams where the specified country is NOT BOTH - the client and server IP. Streams where it is one or - the other may be included. - - -Example: - - decode -d country traffic.pcap -W USonly.pcap --country_code US - decode -d country+followstream traffic.pcap --country_code US --country_notboth -""", - filter="ip or ip6", - author='twp', - optiondict={ - 'code':{'type':'string','help':'two-char country code'}, - 'neither':{'action':'store_true','help':'neither (client/server) is in specified country'}, - 'both':{'action':'store_true','help':'both (client/server) ARE in specified country'}, - 'notboth':{'action':'store_true','help':'specified country is not both client and server'}, - 'alerts':{'action':'store_true'}} ) - '''instantiate an decoder that will call back to us once the IP decoding is done''' - self.__decoder=dshell.IPDecoder() - self.out=netflowout.NetflowOutput() - self.chainable=True - - def decode(self,*args): - if len(args) is 3: pktlen,pktdata,ts=args #orig_len,packet,ts format (pylibpcap) - else: #ts,pktdata (pypcap) - ts,pktdata=args - pktlen=len(pktdata) - '''do normal decoder stack to track session ''' - dshell.TCPDecoder.decode(self,pktlen,pktdata,ts) - '''our hook to decode the ip/ip6 addrs, then dump the addrs and raw packet to our callback''' - self.__decoder.IPHandler=self.__callback #set private decoder to our callback - self.__decoder.decode(pktlen,pktdata,ts,raw=pktdata) - - def __callback(self,addr,pkt,ts,raw=None,**kw): - '''substitute IPhandler for forwarding packets to subdecoders''' - if addr in self.sessions or (addr[1],addr[0]) in self.sessions: #if we are not passing this session, drop the packet - if self.subDecoder: self.subDecoder.decode(len(raw),str(raw),ts) #make it look like a capture - else: self.dump(raw,ts) - - def connectionInitHandler(self,conn): - '''see if we have a country match and if so, flag this session for forwarding or dumping''' - m=self.__countryTest(conn) - if m: self.sessions[conn.addr]=m - - def __countryTest(self, conn): - # If no country code specified, pass all traffic through - if self.code == None or not len(self.code): return True - #check criteria - if self.neither and conn.clientcountrycode != self.code and conn.servercountrycode != self.code: return 'neither '+self.code - if self.both and conn.clientcountrycode == self.code and conn.servercountrycode == self.code: return 'both '+self.code - if self.notboth and (conn.clientcountrycode != self.code or conn.servercountrycode != self.code): return 'not both '+self.code - if conn.clientcountrycode == self.code: return 'client '+self.code - if conn.servercountrycode == self.code: return 'server '+self.code - #no match - return None - - def connectionHandler(self,conn): - if conn.addr in self.sessions and self.alerts: self.alert(self.sessions[conn.addr],**conn.info()) - - def connectionCloseHandler(self,conn): - if conn.addr in self.sessions: del self.sessions[conn.addr] - -dObj = DshellDecoder() diff --git a/decoders/filter/snort.py b/decoders/filter/snort.py deleted file mode 100644 index 021e29a..0000000 --- a/decoders/filter/snort.py +++ /dev/null @@ -1,153 +0,0 @@ -import dshell - -class DshellDecoder(dshell.IPDecoder): - def __init__(self): - dshell.IPDecoder.__init__(self, - name = 'snort', - description = 'filter packets by snort rule', - longdescription="""Chainable decoder to filter TCP/UDP streams by snort rule -rule is parsed by dshell, a limited number of options are supported: - currently supported rule options: - content - nocase - depth - offset - within - distance - -Mandatory option: - ---snort_rule: snort rule to filter by - -or - --snort_conf: snort.conf formatted file to read for multiple rules - -Modifier options: - ---snort_all: Pass only if all rules pass ---snort_none: Pass only if no rules pass ---snort_alert: Alert if rule matches? - -Example: -decode -d snort+followstream traffic.pcap --snort_rule 'alert tcp any any -> any any (content:"....."; nocase; depth .... )' - -""", - filter = 'ip or ip6', - author = 'twp', - optiondict={ 'rule':{'type':'string','help':'snort rule to filter packets'}, - 'conf':{'type':'string','help':'snort.conf file to read'}, - 'alerts':{'action':'store_true','help':'alert if rule matched' }, - 'none':{'action':'store_true','help':'pass if NO rules matched' }, - 'all':{'action':'store_true','help':'all rules must match to pass' } - } - ) - self.chainable=True - - def preModule(self): - rules=[] - if self.conf: # - fh=file(self.conf) - rules=[r for r in (r.strip() for r in fh.readlines()) if len(r)] - fh.close() - else: - if not self.rule or not len(self.rule): self.warn("No rule specified (--%s_rule)" % self.name) - else: rules=[self.rule] - self.rules=[] - for r in rules: - try: self.rules.append( ( self.parseSnortRule(r) ) ) - except Exception,e: - self.error('bad snort rule "%s": %s'%(r,e)) - if self._DEBUG: self._exc(e) - if self.subDecoder: self.subDecoder.ignore_handshake=True #we filter individual packets so session-based subdecoders will need this set - dshell.IPDecoder.preModule(self) - - def rawHandler(self,pktlen,pkt,ts,**kwargs): - kwargs['raw']=pkt #put the raw frame in the kwargs - return dshell.IPDecoder.rawHandler(self,pktlen,pkt,ts,**kwargs) #continue decoding - - def IPHandler(self,addr,pkt,ts,**kwargs): - '''check packets using filterfn here''' - raw=str(kwargs['raw']) #get the raw frame for forwarding if we match - p=dshell.Packet(self,addr,pkt=str(pkt),ts=ts,**kwargs) - a=[] - match=None - for r,msg in self.rules: - if r(p): #if this rule matched - match=True - if msg: a.append(msg) #append rule message to alerts - if self.none or not self.all: break #unless matching all, one match does it - else: #last rule did not match - match=False - if self.all: break #stop once no match if all - - #all rules processed, match = state of last rule match - if (match is not None) and ((match and not self.none) or (self.none and not match)): - self.decodedbytes+=len(str(pkt)) - self.count+=1 - if self.alerts: self.alert(*a,**p.info()) - if self.subDecoder: self.subDecoder.decode(len(raw),raw,ts) #decode or dump packet - else: self.dump(len(raw),raw,ts) - - def parseSnortRule(self,ruletext): - '''returns a lambda function that can be used to filter traffic and the alert message - this function will expect a Packet() object and return True or False''' - KEYWORDS=('msg','content') #rule start, signal when we process all seen keywords - msg='' - f=[] - rule=ruletext.split(' ',7) - (a,proto,sip,sp,arrow,dip,dp)=rule[:7] - if len(rule)>7: rule=rule[7] - else: rule=None - if a != 'alert': raise Exception('Must be alert rule') - f.append('p.proto == "'+proto.upper()+'"') - if sip != 'any': f.append('p.sip == "'+sip+'"') - if dip != 'any': f.append('p.dip == "'+dip+'"') - if sp != 'any': f.append('p.sport == '+sp) - if dp != 'any': f.append('p.dport == '+dp) - f=['(' + (' and '.join(f)) + ')' ] #create header condition - if rule: rule=rule.strip('()').split(';') #split between () and split on ; - last=None #no last match - while rule: - try: k,v=rule.pop(0).strip().split(':',1) - except: continue - if k.lower() == 'content': #reset content match - content=v.strip().strip('"') - if content.startswith('|') and content.endswith('|'): #hex bytes? - content=''.join( '\\x'+c for c in content.strip('|').split() ) - nocase=depth=offset=distance=within=None - while rule: - r=rule[0].strip() - if ':' in r: k,v=r.split(':',1) - else: k,v=r,None - k=k.lower() - if k in KEYWORDS: break #next rule part - elif k == 'nocase': nocase=True - elif k == 'depth': depth=int(v) - elif k == 'offset': offset=int(v) - elif k == 'distance': distance=int(v) - elif k == 'within': within=int(v) - rule.pop(0) #remove this keyword:valuea - #add coerce to lower if nocase? - if nocase: nocase='.lower()' - else: nocase='' - st,end=offset,depth #start,end offsets of find(), maybe number or result of another find() - if last: #if we have a last content match, use the distance/within kws - #within means this match has to be within X from previous+distance, so use previous as offset and within as depth - if within: st,end=last,last+'+'+str(within) #set to last match and X from last match - #distance means the next match must be AT LEAST X from the last - if distance: st=last+'+'+str(distance)# set start to last match+distance - #else use the offset/depth values as given - last='p.pkt'+nocase+'.find('+"'"+content+"'"+nocase+','+str(st)+','+str(end)+') != -1' - if k.lower() == 'msg': msg=v.strip().strip('"')#get alert message - if last: f.append('('+last+')') - f=' and '.join(f) - self.debug('%s\t%s\t"%s"'%(ruletext,f,msg)) - return eval('lambda(p): '+f),msg # return fn and msg - - -if __name__=='__main__': - dObj = DshellDecoder() - print dObj -else: - dObj = DshellDecoder() diff --git a/decoders/filter/track.py b/decoders/filter/track.py deleted file mode 100644 index cda1a77..0000000 --- a/decoders/filter/track.py +++ /dev/null @@ -1,115 +0,0 @@ -''' -@author: tparker -''' - -import dshell,util - -class DshellDecoder(dshell.TCPDecoder): - '''activity tracker ''' - def __init__(self,**kwargs): - ''' - Constructor - ''' - self.sources=[] - self.targets=[] - self.sessions={} - self.alerts=False - self.file=None - dshell.TCPDecoder.__init__(self, - name='track', - description='tracked activity recorder', - longdescription='''captures all traffic to/from target while a specific connection to the target is up - specify target(s) ip and/or port as --track_target=ip:port,ip... - --track_source=ip,ip.. can be used to limit to specified sources - --track_alerts will turn on alerts for session start/end''', - filter="ip", - author='twp', - optiondict={'target':{'action':'append'}, - 'source':{'action':'append'}, - 'alerts':{'action':'store_true'}} ) - self.chainable=True - - '''instantiate an IPDecoder and replace the IPHandler - to decode the ip/ip6 addr and then pass the packet - to _IPHandler, which will write the packet if in addr is in session''' - self.__decoder=dshell.IPDecoder() - - - def preModule(self): - '''parse the source and target lists''' - if self.target: - for tstr in self.target: - targets=util.strtok(tstr, as_list=True)[0] - for t in targets: - try: - parts=t.split(':') - if len(parts)==2: ip,port=parts #IP:port - else: ip,port=t,None #IPv6 addr - except: ip,port=t,None #IP - if ip=='': ip=None # :port - self.targets.append((ip,port)) - if self.source: - for sstr in self.source: - sources=util.strtok(sstr, as_list=True)[0] - for ip in sources: - self.sources.append(ip) - dshell.TCPDecoder.preModule(self) - - def decode(self,*args): - if len(args) is 3: pktlen,pktdata,ts=args #orig_len,packet,ts format (pylibpcap) - else: #ts,pktdata (pypcap) - ts,pktdata=args - pktlen=len(pktdata) - '''do normal decoder stack to track session ''' - dshell.TCPDecoder.decode(self,pktlen,pktdata,ts) - '''our hook to decode the ip/ip6 addrs, then dump the addrs and raw packet - to our session check routine''' - self.__decoder.IPHandler=self.__callback #set private decoder to our callback - self.__decoder.decode(pktlen,pktdata,ts,raw=pktdata) - - def __callback(self,addr,pkt,ts,raw=None,**kw): - '''check to see if this packet is to/from an IP in a session, - if so write it. the packet will be passed in the 'raw' kwarg''' - if addr[0][0] in self.sessions: ip=addr[0][0] #source ip - elif addr[1][0] in self.sessions: ip=addr[1][0] #dest ip - else: return #not tracked - for s in self.sessions[ip].values(): - s.sessionpackets+=1 - s.sessionbytes+=len(raw) #actual captured data len - #dump the packet or sub-decode it - if self.subDecoder: self.subDecoder.decode(len(raw),str(raw),ts) #make it look like a capture - else: self.dump(raw,ts) - - def connectionInitHandler(self,conn): - '''see if dest ip and/or port is in target list and (if a source list) - source ip is in source list - if so, put the connection in the tracked-session list by dest ip - if a new connection to the target comes in from an allowed source, - the existing connection will still be tracked''' - ((sip,sport),(dip,dport))=conn.addr - sport,dport=str(sport),str(dport) - if ((dip,dport) in self.targets) or ((dip,None) in self.targets) or ((None,dport) in self.targets): - if not self.sources or (sip in self.sources): - s=self.sessions.setdefault(dip,{}) - s[conn.addr]=conn - if self.alerts: self.alert('session started',**conn.info()) - conn.info(sessionpackets=0,sessionbytes=0) - - def connectionHandler(self,conn): - '''if a connection to a tracked-session host, alert and write if no subdecoder''' - if self.alerts: - if conn.serverip in self.sessions: self.alert('inbound',**conn.info()) - if conn.clientip in self.sessions: self.alert('outbound',**conn.info()) - if conn.serverip in self.sessions or conn.clientip in self.sessions: - if not self.subDecoder: self.write(conn) - - def connectionCloseHandler(self,conn): - '''close the tracked session if the initiating connection is closing - make sure the conn in the session list matches, - as we may have had more incoming connections to the same ip during the session''' - if conn.serverip in self.sessions and conn.addr in self.sessions[conn.serverip]: - if self.alerts: self.alert('session ended',**conn.info()) - del self.sessions[conn.serverip][conn.addr] - if not self.sessions[conn.serverip]: del self.sessions[conn.serverip] - -dObj = DshellDecoder() diff --git a/decoders/flows/large-flows.py b/decoders/flows/large-flows.py deleted file mode 100644 index 23789a2..0000000 --- a/decoders/flows/large-flows.py +++ /dev/null @@ -1,31 +0,0 @@ -import dshell -import netflowout - -class DshellDecoder(dshell.TCPDecoder): - def __init__(self): - dshell.TCPDecoder.__init__(self, - name = 'large-flows', - description = 'display netflows that have at least 1MB transferred', - filter = 'tcp', - author = 'bg', - optiondict={'size': {'type':'float','default':1,'help':'number of megabytes transferred'}} - ) - self.out = netflowout.NetflowOutput() - self.min = 1048576 # 1MB - - def preModule(self): - if self.size <= 0: - self.warn("Cannot have a size that's less than or equal to zero. (size: %s)" % (self.size)) - self.size = 1 - self.min = 104857 * self.size - self.debug("Input: %s, Final size: %s bytes" % (self.size, self.min)) - - def connectionHandler(self,conn): - if (conn.clientbytes + conn.serverbytes) >= self.min: self.alert(**conn.info()) - - -if __name__=='__main__': - dObj = DshellDecoder() - print dObj -else: - dObj = DshellDecoder() diff --git a/decoders/flows/long-flows.py b/decoders/flows/long-flows.py deleted file mode 100644 index 44a644f..0000000 --- a/decoders/flows/long-flows.py +++ /dev/null @@ -1,26 +0,0 @@ -import dshell -import netflowout - -class DshellDecoder(dshell.TCPDecoder): - def __init__(self): - self.len=5 - dshell.TCPDecoder.__init__(self, - name = 'long-flows', - description = 'display netflows that have a duration of at least 5mins', - filter = '(tcp or udp)', - author = 'bg', - optiondict={ - 'len':{'type':'int','default':5,'help':'set minimum connection time to alert on, in minutes [default: 5 mins]'}, - } - ) - self.out=netflowout.NetflowOutput() - - def connectionHandler(self,conn): - if (conn.endtime - conn.starttime) >= (60*self.len): self.alert(**conn.info()) - - -if __name__=='__main__': - dObj = DshellDecoder() - print dObj -else: - dObj = DshellDecoder() diff --git a/decoders/flows/netflow.py b/decoders/flows/netflow.py deleted file mode 100644 index 5f8c6f6..0000000 --- a/decoders/flows/netflow.py +++ /dev/null @@ -1,33 +0,0 @@ -import dshell -import netflowout - -class DshellDecoder(dshell.TCPDecoder): - def __init__(self): - dshell.TCPDecoder.__init__(self, - name = 'netflow', - description = 'generate netflow information from pcap', - longdescription = 'generate netflow information from pcap', - filter = '(tcp or udp)', - author = 'bg', - optiondict={'group':dict()} #grouping for output module - ) - self.out=netflowout.NetflowOutput() - - def preModule(self): - #pass grouping to output module - if self.group: self.out.group=self.group.split(',') - dshell.TCPDecoder.preModule(self) - - def connectionHandler(self,conn): - self.alert(**conn.info()) - - def postModule(self): - self.out.close() #write flow groups if grouping - dshell.TCPDecoder.postModule(self) - - -if __name__=='__main__': - dObj = DshellDecoder() - print dObj -else: - dObj = DshellDecoder() diff --git a/decoders/http/httpdump.py b/decoders/http/httpdump.py deleted file mode 100644 index 684122c..0000000 --- a/decoders/http/httpdump.py +++ /dev/null @@ -1,128 +0,0 @@ -import dshell -import util -import hashlib, urllib, re - -from httpdecoder import HTTPDecoder - -class DshellDecoder(HTTPDecoder): - def __init__(self): - HTTPDecoder.__init__(self, - name='httpdump', - description='Dump useful information about HTTP sessions', - filter='tcp and (port 80 or port 8080 or port 8000)', - filterfn=lambda ((sip,sp),(dip,dp)): sp in (80, 8000, 8080) or dp in (80, 8000, 8080), - author='amm', - optiondict={ - 'maxurilen':{'type':'int','default':30,'help':'Truncate URLs longer than max len. Set to 0 for no truncating. (default: 30)'}, - 'maxpost':{'type':'int','default':1000,'help':'Truncate POST body longer than max chars. Set to 0 for no truncating. (default: 1000)'}, - 'showcontent':{'action':'store_true','help':'Display response BODY.'}, - 'showhtml':{'action':'store_true','help':'Display response BODY only if HTML.'}, - 'urlfilter':{'type':'string','default':None,'help':'Filter to URLs matching this regex'}, - }, - ) - self.output='colorout' - self.gunzip=False # Disable auto-gunzip as we want to indicate content that was compressed in the output - - def HTTPHandler(self,conn,request,response,requesttime,responsetime): - host = '' - loc = '' - uri = '' - lastmodified = '' - - #request_time, request, response = self.httpDict[conn.addr] - - # extract method,uri,host from response - host = util.getHeader(request, 'host') - if host == '': - host = conn.serverip - - try: - status = response.status - except: - status = '' - try: - reason = response.reason - except: - reason = '' - - if self.urlfilter: - if not re.search(self.urlfilter, host+request.uri): return - - if '?' in request.uri: - [uri_location, uri_data] = request.uri.split('?',1) - else: - uri_location = request.uri - uri_data = '' - - if self.maxurilen > 0 and len(uri_location)> self.maxurilen: - uri_location = uri_location[:self.maxurilen]+'[truncated]' - else: - uri_location = uri_location - - - if response == None: - response_message = "%s (%s) %s%s" % (request.method, 'NO RESPONSE', host, uri_location) - else: - response_message = "%s (%s) %s%s (%s)" % (request.method, response.status, host, uri_location, util.getHeader(response, 'content-type')) - urlParams = util.URLDataToParameterDict(uri_data) - postParams = util.URLDataToParameterDict(request.body) - - clientCookies = self._parseCookies(util.getHeader(request, 'cookie')) - serverCookies = self._parseCookies(util.getHeader(response, 'set-cookie')) - - self.alert(response_message, - urlParams=urlParams, postParams=postParams, clientCookies=clientCookies, serverCookies=serverCookies, - **conn.info() - ) - - referer = util.getHeader(request, 'referer') - if len(referer): self.out.write(' Referer: %s\n' % referer) - - if clientCookies: - self.out.write(' Client Transmitted Cookies:\n', direction='cs') - for key in clientCookies: - self.out.write(' %s -> %s\n' % (util.printableUnicode(key),util.printableUnicode(clientCookies[key])), direction='cs') - if serverCookies: - self.out.write(' Server Set Cookies:\n', direction='sc') - for key in serverCookies: - self.out.write(' %s -> %s\n' % (util.printableUnicode(key),util.printableUnicode(serverCookies[key])), direction='sc') - - if urlParams: - self.out.write(' URLParameters:\n', direction='cs') - for key in urlParams: - self.out.write(' %s -> %s\n' % (util.printableUnicode(key),util.printableUnicode(urlParams[key])), direction='cs') - if postParams: - self.out.write(' POSTParameters:\n', direction='cs') - for key in postParams: - self.out.write(' %s -> %s\n' % (util.printableUnicode(key),util.printableUnicode(postParams[key])), direction='cs') - elif len(request.body): - self.out.write(' POST Body:\n', direction='cs') - if len(request.body) > self.maxpost and self.maxpost > 0: - self.out.write('%s[truncated]\n' % util.printableUnicode(request.body[:self.maxpost]), direction='cs') - else: - self.out.write(util.printableUnicode(request.body) + u"\n", direction='cs') - - if self.showcontent or self.showhtml: - - if self.showhtml and 'html' not in util.getHeader(response, 'content-type'): return - - if 'gzip' in util.getHeader(response, 'content-encoding'): - content = self.decompressGzipContent(response.body) - if content == None: content = '(gunzip failed)\n' + response.body - else: content = '(gzip encoded)\n' + content - else: content = response.body - - self.out.write("Body Content:\n", direction='sc') - self.out.write(util.printableUnicode(content) + u"\n", direction='sc') - - def _parseCookies(self, data): - p,kwp=util.strtok(data,sep='; ') - return dict((urllib.unquote(k),urllib.unquote(kwp[k]))for k in kwp.keys()) - - - -if __name__=='__main__': - dObj = DshellDecoder() - print dObj -else: - dObj = DshellDecoder() diff --git a/decoders/http/rip-http.py b/decoders/http/rip-http.py deleted file mode 100644 index cc75317..0000000 --- a/decoders/http/rip-http.py +++ /dev/null @@ -1,168 +0,0 @@ -import dshell,re -import datetime, sys, string - -#import any other modules here -# import binascii -import re, os, hashlib, util - -#we extend this -from httpdecoder import HTTPDecoder - -class DshellDecoder(HTTPDecoder): - def __init__(self): - HTTPDecoder.__init__(self, - name = 'rip-http', - description = 'rip files from HTTP traffic', - filter = 'tcp and port 80', - author = 'bg/twp', - optiondict = {'append_conn':{'action':'store_true','help':'append sourceip-destip to filename'}, - 'append_ts':{'action':'store_true','help':'append timestamp to filename'}, - 'direction':{'help':'cs=only capture client POST, sc=only capture server GET response'}, - 'content_filter':{'help':'regex MIME type filter for files to save'}, - 'name_filter':{'help':'regex filename filter for files to save'}} - ) - - def preModule(self): - if self.content_filter: self.content_filter=re.compile(self.content_filter) - if self.name_filter: self.name_filter=re.compile(self.name_filter) - HTTPDecoder.preModule(self) - self.openfiles = {} # dict of httpfile objects, indexed by url - - def splitstrip(self,data,sep,strip=' '): - return [lpart.strip(strip) for lpart in data.split(sep)] - - def POSTHandler(self,postdata): - next_line_is_data=False - for l in postdata.split("\r\n"): - if next_line_is_data: break - if l=='': - next_line_is_data=True #\r\n\r\n before data - continue - try: - k,v=self.splitstrip(l,':') - if k=='Content-Type': contenttype=v - if k=='Content-Disposition': - cdparts=self.splitstrip(v,';') - for cdpart in cdparts: - try: - k,v=self.splitstrip(cdpart,'=','"') - if k=='filename': filename=v - except: pass - except: pass - return contenttype,filename,l - - - def HTTPHandler(self,conn,request,response,requesttime,responsetime): - self.debug ('%s %s'%(repr(request),repr(response))) - if (not self.direction or self.direction=='cs') and request and request.method=='POST' and request.body: - contenttype,filename,data=self.POSTHandler(request.body) - if not self.content_filter or self.content_filter.search(contenttype): - if not self.name_filter or self.name_filter.search(filename): - if self.append_conn: filename+='_%s-%s'%(conn.clientip,conn.serverip) - if self.append_ts: filename+='_%d'%(conn.ts) - self.debug(filename) - f=open(filename,'w') - f.write(data) - f.close() - elif (not self.direction or self.direction=='sc') and response and response.status[0]=='2': - if not self.content_filter or self.content_filter.search(response.headers['content-type']): - # Calculate URL - host = util.getHeader(request, 'host') - if host == '': - host = conn.serverip - url = host + request.uri - # File already open - if url in self.openfiles: - self.debug("Adding response section to %s" % url) - (s,e) = self.openfiles[url].handleresponse(response) - self.write(" --> Range: %d - %d\n" % (s,e)) - # New file - else: - filename=request.uri.split('?')[0].split('/')[-1] - self.debug("New file with URL: %s" % url) - if not self.name_filter or self.name_filter.search(filename): - if self.append_conn: filename+='_%s-%s'%(conn.serverip,conn.clientip) - if self.append_ts: filename+='_%d'%(conn.ts) - if not len(filename): filename ='%s-%s_index.html'%(conn.serverip,conn.clientip) - while os.path.exists(filename): filename += '_' - self.alert("New file: %s (%s)" % (filename, url), conn.info()) - self.openfiles[url] = httpfile(filename) - (s,e) = self.openfiles[url].handleresponse(response) - self.write(" --> Range: %d - %d\n" % (s,e)) - if self.openfiles[url].done(): - self.alert("File done: %s (%s)" % (self.openfiles[url].filename, url), conn.info()) - del self.openfiles[url] - -class httpfile: - - def __init__(self,filename): - self.complete = False - self.size = 0 # Expected size in bytes of full file transfer - self.ranges = [] # List of tuples indicating byte chunks already received and written to disk - self.filename = filename - self.fh = open(filename,'w') - - def __del__(self): - self.fh.close() - if not self.done(): - print "Incomplete file: %s" % self.filename - try: os.rename(self.filename, self.filename+"_INCOMPLETE") - except: pass - ls = 0 - le = 0 - for s,e in self.ranges: - if s>le+1: - print "Missing bytes between %d and %d" % (le, s) - ls,le = s,e - - - def handleresponse(self,response): - # Check for Content Range - range_start = 0 - range_end = len(response.body)-1 - if 'content-range' in response.headers: - m = re.search('bytes (\d+)-(\d+)/(\d+|\*)', response.headers['content-range']) - if m: - range_start = int(m.group(1)) - range_end = int(m.group(2)) - if len(response.body) < (range_end-range_start+1): range_end=range_start+len(response.body)-1 - try: - if int(m.group(3)) > self.size: self.size = int(m.group(3)) - except: pass - elif 'content-length' in response.headers: - try: - if int(response.headers['content-length']) > self.size: self.size = int(response.headers['content-length']) - except: pass - # Update range tracking - self.ranges.append((range_start, range_end)) - # Write part of file - self.fh.seek(range_start) - self.fh.write(response.body) - return (range_start, range_end) - - def done(self): - self.checkranges() - return self.complete - - def checkranges(self): - self.ranges.sort() - current_start = 0 - current_end = 0 - foundgap = False - #print self.ranges - for s,e in self.ranges: - if s <= current_end+1: current_end = e - else: - foundgap = True - current_start = s - current_end = e - if not foundgap: - if (current_end+1) >= self.size: - self.complete = True - return foundgap - -if __name__=='__main__': - dObj = DshellDecoder() - print dObj -else: - dObj = DshellDecoder() diff --git a/decoders/http/web.py b/decoders/http/web.py deleted file mode 100644 index 86975d6..0000000 --- a/decoders/http/web.py +++ /dev/null @@ -1,128 +0,0 @@ -import dshell, dfile -import util -import hashlib - -from httpdecoder import HTTPDecoder - -class DshellDecoder(HTTPDecoder): - def __init__(self): - HTTPDecoder.__init__(self, - name='web', - description='Improved version of web that tracks server response', - filter='tcp and (port 80 or port 8080 or port 8000)', - filterfn=lambda ((sip,sp),(dip,dp)): sp in (80, 8000, 8080) or dp in (80, 8000, 8080), - author='bg,twp', - optiondict={ - 'maxurilen':{'type':'int','default':30,'help':'Truncate URLs longer than max len. Set to 0 for no truncating. (default: 30)'}, - 'md5':{'action':'store_true','help':'calculate MD5 for each response. Available in CSV output.'} - }, - ) - self.gunzip = False # Not interested in response body - - def HTTPHandler(self,conn,request,response,requesttime,responsetime): - host = '' - loc = '' - lastmodified = '' - - #request_time, request, response = self.httpDict[conn.addr] - - # extract method,uri,host from response - host = util.getHeader(request, 'host') - if host == '': - host = conn.serverip - - try: - status = response.status - except: - status = '' - try: - reason = response.reason - except: - reason = '' - - loc = '' - if status[:2] == '30': - loc = util.getHeader(response, 'location') - if len(loc): - loc = '-> '+loc - - lastmodified = util.HTTPlastmodified(response) - referer = util.getHeader(request, 'referer') - useragent = util.getHeader(request, 'user-agent') - via = util.getHeader(request, 'via') - - try: - responsesize = len(response.body.rstrip('\0')) - except: - responsesize = 0 - - - if self.md5: - md5 = self._bodyMD5(response) - else: - md5 = '' - - # File objects - try: - if len(response.body) > 0: responsefile = dfile.dfile(name=request.uri,data=response.body) - else: responsefile = '' - except: responsefile = '' - if request.method == 'POST' and len(request.body): - ulcontenttype,ulfilename,uldata=self.POSTHandler(request.body) - uploadfile=dfile.dfile(name=ulfilename,data=uldata) - else: uploadfile = None - - requestInfo = '%s %s%s HTTP/%s' % (request.method, - host, - request.uri[:self.maxurilen]+'[truncated]' if self.maxurilen > 0 and len(request.uri)> self.maxurilen else request.uri, - request.version) - if response: responseInfo = '%s %s %s %s' % (status,reason,loc,lastmodified) - else: responseInfo='' - - self.alert( "%-80s // %s" % (requestInfo,responseInfo),referer=referer,useragent=useragent,request=requestInfo,response=responseInfo,request_time=requesttime,response_time=responsetime,request_method=request.method,host=host,uri=request.uri,status=status,reason=reason,lastmodified=lastmodified,md5=md5,responsesize=responsesize,contenttype=util.getHeader(response, 'content-type'),responsefile=responsefile,uploadfile=uploadfile,via=via,**conn.info()) - if self.out.sessionwriter: - self.write(request.data,direction='cs') - if response: self.write(response.body,direction='sc') - - - # MD5sum(hex) of the body portion of the response - def _bodyMD5(self, response): - try: - if len(response.body) > 0: - return hashlib.md5(response.body.rstrip('\0')).hexdigest() - else: - return '' - except: - return '' - - def POSTHandler(self,postdata): - next_line_is_data=False - contenttype = '' - filename = '' - for l in postdata.split("\r\n"): - if next_line_is_data: break - if l=='': - next_line_is_data=True #\r\n\r\n before data - continue - try: - k,v=self.splitstrip(l,':') - if k=='Content-Type': contenttype=v - if k=='Content-Disposition': - cdparts=self.splitstrip(v,';') - for cdpart in cdparts: - try: - k,v=self.splitstrip(cdpart,'=','"') - if k=='filename': filename=v - except: pass - except: pass - return contenttype,filename,l - - def splitstrip(self,data,sep,strip=' '): - return [lpart.strip(strip) for lpart in data.split(sep)] - - -if __name__=='__main__': - dObj = DshellDecoder() - print dObj -else: - dObj = DshellDecoder() diff --git a/decoders/misc/followstream.py b/decoders/misc/followstream.py deleted file mode 100644 index 40ff267..0000000 --- a/decoders/misc/followstream.py +++ /dev/null @@ -1,101 +0,0 @@ -import dshell, util -import colorout -#from impacket.ImpactDecoder import EthDecoder -import datetime, sys -import traceback -import logging - -#import any other modules here -import cgi - -class DshellDecoder(dshell.TCPDecoder): - - def __init__(self): - dshell.TCPDecoder.__init__(self, - name='followstream', - description='Generates color-coded Screen/HTML output similar to Wireshark Follow Stream', - longdescription=""" -Generates color-coded Screen/HTML output similar to Wireshark Follow Stream. - -Output by default uses the "colorout" output class. This will send TTY -color-formatted text to stdout (the screen) if available. If output -is directed to a file (-o or --outfile), the output will be in HTML format. - -Note that the default bpf filter is to view all tcp traffic. The decoder -can also process UDP traffic, or it can be limited to specific streams -with --bpf/--ebpf. - -Useful options: - - --followstream_hex -- generates output in hex mode - --followstream_time -- includes timestamp for each blob/transmission - -Example: - - decode -d followstream --ebpf 'port 80' mypcap.pcap --followstream_time - decode -d followstream --ebpf 'port 80' mypcap.pcap -o file.html --followstream_time - -""", - filter="tcp", - author='amm', - optiondict={ - 'hex':{'action':'store_true','help':'two-column hex/ascii output'}, - 'time':{'action':'store_true','help':'include timestamp for each blob'}, - 'encoding':{'type':'string','help':'attempt to interpret text as encoded with specified schema'}, - } - ) - self.out = colorout.ColorOutput() - - def __errorHandler(self, blob, expected, offset, caller): - # Custom error handler that is called when data in a blob is missing or overlapping - if offset > expected: # data is missing - self.data_missing_message += "[%d missing bytes]" % (offset-expected) - elif offset < expected: # data is overlapping - self.data_missing_message += "[%d overlapping bytes]" % (offset-expected) - return True - - def preModule(self): - self.connectionCount = 0 - # Reset the color mode, in case a file is specified - self.out.setColorMode() - # Used to indicate when data is missing or overlapping - self.data_missing_message = '' - # overwrite the output module's default error handler - self.out.errorH = self.__errorHandler - - def connectionHandler(self,connection): - - try: - - # Skip Connections with no data transferred - if connection.clientbytes + connection.serverbytes < 1: - return - - # Update Connection Counter - self.connectionCount += 1 - - # Connection Header Information - self.out.write("Connection %d (%s)\n" % (self.connectionCount, str(connection.proto)), formatTag='H1') - self.out.write("Start: %s UTC\n End: %s UTC\n" % (datetime.datetime.utcfromtimestamp(connection.starttime), datetime.datetime.utcfromtimestamp(connection.endtime)), formatTag='H2') - self.out.write("%s:%s -> %s:%s (%d bytes)\n" % (connection.clientip, connection.clientport, connection.serverip, connection.serverport, connection.clientbytes), formatTag="H2", direction="cs") - self.out.write("%s:%s -> %s:%s (%d bytes)\n\n" % (connection.serverip, connection.serverport, connection.clientip, connection.clientport, connection.serverbytes), formatTag="H2", direction="sc") - - self.out.write(connection, hex=self.hex, time=self.time, encoding=self.encoding) - if self.data_missing_message: self.out.write(self.data_missing_message+"\n", level=logging.WARNING, time=self.time) - self.data_missing_message = '' - - # Line break before next session - self.out.write("\n\n") - - except KeyboardInterrupt: - raise - except: - print 'Error in connectionHandler: ', sys.exc_info()[1] - traceback.print_exc(file=sys.stdout) - - -if __name__=='__main__': - dObj = DshellDecoder() - print dObj -else: - dObj = DshellDecoder() diff --git a/decoders/misc/merge.py b/decoders/misc/merge.py deleted file mode 100644 index 8c2dbc6..0000000 --- a/decoders/misc/merge.py +++ /dev/null @@ -1,28 +0,0 @@ -import dshell -import dpkt - -class DshellDecoder(dshell.Decoder): - """ - merge.py - merge all pcap in to a single file - - Example: decode -d merge *.pcap -W merged.pcap - """ - def __init__(self): - dshell.Decoder.__init__(self, - name = 'merge', - description = 'dump all packets to single file', - longdescription ="""Example: decode -d merge *.pcap -W merged.pcap""", - author = 'bg/twp' - ) - self.chainable=True - - def rawHandler(self,pktlen,pkt,ts,**kw): - if self.subDecoder: return self.subDecoder.rawHandler(pktlen,str(pkt),ts,**kw) - else: return self.dump(pktlen,pkt,ts) - - -if __name__=='__main__': - dObj = DshellDecoder() - print dObj -else: - dObj = DshellDecoder() diff --git a/decoders/misc/synrst.py b/decoders/misc/synrst.py deleted file mode 100644 index 4c1c69b..0000000 --- a/decoders/misc/synrst.py +++ /dev/null @@ -1,35 +0,0 @@ -import dshell, dpkt - -class DshellDecoder(dshell.IPDecoder): - """ - Simple TCP syn/rst filter (ipv4) only - """ - def __init__(self): - dshell.IPDecoder.__init__(self, - name = 'synrst', - description = 'detect failed attempts to connect (SYN followed by a RST/ACK)', - filter = "tcp[13]=2 or tcp[13]=20", - author = 'bg' - ) - self.tracker = {} # key = (srcip,srcport,seqnum,dstip,dstport) - - def packetHandler(self,ip=None): - tcp = dpkt.ip.IP(ip.pkt).data - - if tcp.flags & 2: # check for SYN flag - seqnum = tcp.seq - key = '%s:%s:%d:%s:%s' % (ip.sip,ip.sport,seqnum,ip.dip,ip.dport) - self.tracker[key] = '' - elif tcp.flags & 20: # check for RST/ACK flags - acknum = tcp.ack - 1 - tmpkey = '%s:%s:%d:%s:%s' % (ip.dip,ip.dport,acknum,ip.sip,ip.sport) - if self.tracker.__contains__(tmpkey): - self.alert('Failed connection',**ip.info()) - del self.tracker[tmpkey] - - -if __name__=='__main__': - dObj = DshellDecoder() - print dObj -else: - dObj = DshellDecoder() diff --git a/decoders/misc/writer.py b/decoders/misc/writer.py deleted file mode 100644 index 656b644..0000000 --- a/decoders/misc/writer.py +++ /dev/null @@ -1,44 +0,0 @@ -''' -Created on Jan 13, 2012 - -@author: tparker -''' - -import dshell,dpkt -from output import PCAPWriter - -class DshellDecoder(dshell.Decoder): - ''' - session writer - chain to a decoder to end the chain if the decoder does not output session or packets on its own - if chained to a packet-based decoder, writes all packets to pcap file, can be used to convert or concatenate files - if chained to a connection-based decoder, writes selected streams to session file - ''' - def __init__(self,**kwargs): - ''' - Constructor - ''' - self.file=None - dshell.Decoder.__init__(self, - name='writer', - description='pcap/session writer', - author='twp', - raw=True, - optiondict=dict( filename=dict(default='%(clientip)s:%(clientport)s-%(serverip)s:%(serverport)s-%(direction)s.txt'), - ) - ) - - def rawHandler(self,pktlen,pkt,ts): - self.decodedbytes+=pktlen - self.count+=1 - self.dump(pktlen,pkt,ts) #pktlen may be wrong if we stripped vlan - - def IPHandler(self,addr,ip,ts,pkttype=None,**kw): - self.decodedbytes+=len(ip.data) - self.count+=1 - #if we are passed in IP data vs layer-2 frames, we need to encapsulate them - self.dump(dpkt.ethernet.Ethernet(data=str(ip),pkttype=type),ts=ts) - - def connectionHandler(self,conn): - self.write(conn) - -dObj = DshellDecoder() diff --git a/decoders/misc/xor.py b/decoders/misc/xor.py deleted file mode 100644 index 0d747dd..0000000 --- a/decoders/misc/xor.py +++ /dev/null @@ -1,101 +0,0 @@ -import dshell, util -import struct - -class DshellDecoder(dshell.TCPDecoder): - def __init__(self): - self.xorconn={} # required to track each individual connection - dshell.TCPDecoder.__init__(self, - name='xor', - description='XOR an entire stream with a given single byte key', - filter="tcp", - author='twp', - optiondict={ - 'key':{'type':'str','default':'0xff','help':'xor key [default 255]'}, - 'cskey':{'type':'str','default':None,'help':'c->s xor key [default None]'}, - 'sckey':{'type':'str','default':None,'help':'s->c xor key [default None]'}, - 'resync':{'action':'store_true','help':'resync if the key is seen in the stream'}, - } - ) - self.chainable=True # sets chainable to true and requires connectionInitHandler() and connectionCloseHandler() - - def preModule(self,*args,**kwargs): - dshell.TCPDecoder.preModule(self, *args, **kwargs) - #twp handle hex keys - self.key=self.makeKey(self.key) - if self.cskey: self.cskey=self.makeKey(self.cskey) - if self.sckey: self.sckey=self.makeKey(self.sckey) - - def makeKey(self,key): - if key.startswith('"'): return key[1:-1] - if key.startswith('0x'): - k,key='',key[2:] - for i in xrange(0,len(key),2): k+=chr(int(key[i:i+2],16)) - return k - else: return struct.pack('I', int(key)) - - # - # connectionInitHandler is required as this module (and all other chainable modules) will have to track all - # each connection independently of dshell.TCPDecoder - # - def connectionInitHandler(self,conn): - # need to set up a custom connection tracker to handle - self.xorconn[conn.addr]=dshell.Connection(self,conn.addr,conn.ts) - #self.xorconn[conn.addr]=conn - - # - # Each blob will be xor'ed and the "newblob" data will be added to the connection - # we are individually tracking - # - def blobHandler(self,conn,blob): - k=0 #key index - # create new data (ie. pkt data) - # with appropriate key - data,newdata=blob.data(),'' - self.debug('IN '+util.hexPlusAscii(blob.data())) - if self.cskey != None and blob.direction == 'cs': key=self.cskey - elif self.sckey != None and blob.direction == 'sc': key=self.sckey - else: key=self.key - for i in xrange(len(data)): - if self.resync and data[i:i+len(key)]==key: k=0 #resync if the key is seen - newdata+=chr(ord(data[i])^ord(key[k])) #xor this byte with the aligned byte from the key - k = (k+1) % len(key) #move key position - # update our connection object with the new data - newblob = self.xorconn[conn.addr].update(conn.endtime,blob.direction,newdata) - self.debug('OUT '+repr(self.key)+' '+util.hexPlusAscii(newdata)) - # if there is another decoder we want to pass this data too - if newblob and 'blobHandler' in dir(self.subDecoder): - self.subDecoder.blobHandler(self.xorconn[conn.addr],newblob) # pass to the subDecoder's blobHandler() - - # - # The connection has finished without errors, then we pass the entire connection to the subDecoder's - # connectionHandler() - # - def connectionHandler(self,conn): - if conn.addr in self.xorconn: - self.xorconn[conn.addr].proto = conn.proto - if 'connectionHandler' in dir(self.subDecoder): - self.subDecoder.connectionHandler(self.xorconn[conn.addr]) - else: - self.write(self.xorconn[conn.addr]) - - # - # connectionCloseHandler is called when: - # - a connection finishes w/o errors (no data loss) - # - a connection finishes w errors - # - # If the connection exists in our custom connection tracker (self.xorconn), - # we will have to pass it to the subDecoder's connectionCloseHandler - # - # - def connectionCloseHandler(self,conn): - if conn.addr in self.xorconn: - if 'connectionCloseHandler' in dir(self.subDecoder): - self.subDecoder.connectionCloseHandler(self.xorconn[conn.addr]) - del self.xorconn[conn.addr] - - -if __name__=='__main__': - dObj = DshellDecoder() - print dObj -else: - dObj = DshellDecoder() diff --git a/decoders/protocol/ether.py b/decoders/protocol/ether.py deleted file mode 100644 index b5e34db..0000000 --- a/decoders/protocol/ether.py +++ /dev/null @@ -1,24 +0,0 @@ -import dshell,util,dpkt,datetime,binascii - -class DshellDecoder(dshell.Decoder): - - def __init__(self): - dshell.Decoder.__init__(self, - name = 'ether', - description = 'raw ethernet capture decoder', - filter = '', - author = 'twp',asdatetime=True - ) - - def rawHandler(self,dlen,data,ts,**kw): - if self.verbose: self.log("%.06f %d\n%s"%(ts,dlen,util.hexPlusAscii(str(data)))) - eth=dpkt.ethernet.Ethernet(str(data)) - src=binascii.hexlify(eth.src) - dst=binascii.hexlify(eth.dst) - self.alert('%6x->%6x %4x len %d'%(long(src,16),long(dst,16),eth.type,len(eth.data)),type=eth.type,bytes=len(eth.data),src=src,dst=dst,ts=ts) - -if __name__=='__main__': - dObj = DshellDecoder() - print dObj -else: - dObj = DshellDecoder() diff --git a/decoders/protocol/ip.py b/decoders/protocol/ip.py deleted file mode 100644 index cdd9e94..0000000 --- a/decoders/protocol/ip.py +++ /dev/null @@ -1,23 +0,0 @@ -import dshell,util,dpkt,traceback - -class DshellDecoder(dshell.IP6Decoder): - - _PROTO_MAP={dpkt.ip.IP_PROTO_TCP:'TCP',17:'UDP'} - - def __init__(self): - dshell.IP6Decoder.__init__(self, - name = 'ip', - description = 'IPv4/IPv6 decoder', - filter = 'ip or ip6', - author = 'twp', - ) - - def packetHandler(self,ip=None,proto=None): - if self.verbose: self.out.log(util.hexPlusAscii(ip.pkt)) - self.alert(**ip.info()) - if self.out.sessionwriter: self.write(ip) - -if __name__=='__main__': - dObj = DshellDecoder() - print dObj -else: dObj = DshellDecoder() diff --git a/decoders/protocol/protocol.py b/decoders/protocol/protocol.py deleted file mode 100644 index cad82dd..0000000 --- a/decoders/protocol/protocol.py +++ /dev/null @@ -1,33 +0,0 @@ -import dshell -import dpkt - -# Build a list of known IP protocols from dpkt -try: PROTOCOL_MAP = dict((v,k[9:]) for k,v in dpkt.ip.__dict__.iteritems() if type(v) == int and k.startswith('IP_PROTO_') and k != 'IP_PROTO_HOPOPTS') -except: PROTOCOL_MAP = {} - -class DshellDecoder(dshell.IPDecoder): - """ - protocol.py - - Identifies non-standard protocols (not tcp, udp or icmp) - - References: - http://www.networksorcery.com/enp/protocol/ip.htm - """ - def __init__(self): - dshell.IPDecoder.__init__(self, - name = 'protocol', - description = 'Identifies non-standard protocols (not tcp, udp or icmp)', - filter = '(ip and not tcp and not udp and not icmp)', - author = 'bg', - ) - - def packetHandler(self,ip): - p = PROTOCOL_MAP.get(ip.proto, ip.proto) - self.alert('PROTOCOL: %s (%d)' % (p, ip.proto), sip=ip.sip, dip=ip.dip, ts=ip.ts) - -if __name__=='__main__': - dObj = DshellDecoder() - print dObj -else: - dObj = DshellDecoder() diff --git a/decoders/templates/PacketDecoder.py b/decoders/templates/PacketDecoder.py deleted file mode 100644 index f58893f..0000000 --- a/decoders/templates/PacketDecoder.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python - -import dshell, output, util - -class DshellDecoder(dshell.IPDecoder): - '''generic packet-level decoder template''' - - def __init__(self,**kwargs): - '''decoder-specific config''' - - '''pairs of 'option':{option-config}''' - self.optiondict={} - - '''bpf filter, for ipV4''' - self.filter='' - '''filter function''' - #self.filterfn= - - '''init superclasses''' - self.__super__().__init__(**kwargs) - - def packetHandler(self,ip): - '''handle as Packet() ojects''' - pass - -#create an instance at load-time -dObj=DshellDecoder() diff --git a/decoders/templates/SessionDecoder.py b/decoders/templates/SessionDecoder.py deleted file mode 100644 index f97579f..0000000 --- a/decoders/templates/SessionDecoder.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python - -import dshell, output, util - -class DshellDecoder(dshell.TCPDecoder): - '''generic session-level decoder template''' - - def __init__(self,**kwargs): - '''decoder-specific config''' - - '''pairs of 'option':{option-config}''' - self.optiondict={} - - '''bpf filter, for ipV4''' - self.filter='' - '''filter function''' - #self.filterfn= - - '''init superclasses''' - self.__super__().__init__(**kwargs) - - def packetHandler(self,udp,data): - '''handle UDP as Packet(),payload data - remove this if you want to make UDP into pseudo-sessions''' - pass - - def connectionInitHandler(self,conn): - '''called when connection starts, before any data''' - pass - - def blobHandler(self,conn,blob): - '''handle session data as soon as reassembly is possible''' - pass - - def connectionHandler(self,conn): - '''handle session once all data is reassembled''' - pass - - def connectionCloseHandler(self,conn): - '''called when connection ends, after data is handled''' - -#create an instance at load-time -dObj=DshellDecoder() diff --git a/dist/Dshell-3.1.3.tar.gz b/dist/Dshell-3.1.3.tar.gz new file mode 100644 index 0000000..b3fc79d Binary files /dev/null and b/dist/Dshell-3.1.3.tar.gz differ diff --git a/doc/generate-doc.sh b/doc/generate-doc.sh deleted file mode 100755 index 878fd98..0000000 --- a/doc/generate-doc.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -d=`pwd` -if [ "$1" ]; then d=$1; fi -source $d/.dshellrc || exit - -for f in $d/lib/*.py $d/lib/output/*.py $d/bin/*.py; do - pydoc -w `basename $f|cut -d. -f1` -done - -for f in `find $d/decoders -name \*.py -not -name __init__.py`; do - pydoc -w $f -done diff --git a/dshell/__init__.py b/dshell/__init__.py new file mode 100644 index 0000000..c241f84 --- /dev/null +++ b/dshell/__init__.py @@ -0,0 +1,14 @@ + +# Expose classes and functions that external users will need to access as the API +from .core import ConnectionPlugin, PacketPlugin, Packet +# TODO: Make decode.process_files()/main() function more API friendly through documentation and unwrapping the kwargs +from .api import get_plugins, get_plugin_information + +from .output.alertout import AlertOutput +from .output.colorout import ColorOutput +from .output.csvout import CSVOutput +from .output.elasticout import ElasticOutput +from .output.htmlout import HTMLOutput +from .output.jsonout import JSONOutput +from .output.netflowout import NetflowOutput +from .output.pcapout import PCAPOutput diff --git a/dshell/api.py b/dshell/api.py new file mode 100644 index 0000000..94307b6 --- /dev/null +++ b/dshell/api.py @@ -0,0 +1,36 @@ +""" +Dshell3 Python API +""" + +import logging +import operator +from importlib import import_module + +# TODO: Move get_plugins() here? +from .dshelllist import get_plugins + + +logger = logging.getLogger(__name__) + + +# TODO: Should this be renamed to "load_plugins()" since it actually imports the modules? +def get_plugin_information() -> dict: + """ + Generates and returns a dictionary of plugins. + :return: dictionary containing plugin name -> plugin module + :raises ImportError: If a plugin could not be imported. + """ + plugin_map = get_plugins() + # Import ALL of the decoders and print info about them before exiting + plugins = {} + for name, module in sorted(plugin_map.items(), key=operator.itemgetter(1)): + try: + module = import_module(module) + if not module.DshellPlugin: + continue + module = module.DshellPlugin() + plugins[name] = module + except Exception as e: + raise ImportError(f"Could not load {repr(module)} with error: {e}") + + return plugins diff --git a/dshell/core.py b/dshell/core.py new file mode 100644 index 0000000..9b57fd9 --- /dev/null +++ b/dshell/core.py @@ -0,0 +1,1980 @@ +""" +The core Dshell library + +This library contains the base level plugins that all others will inherit. + +PacketPlugin contains attributes and functions for plugins that work with +individual packets. + +ConnectionPlugin inherits from PacketPlugin and includes additional functions +for handling reassembled connections. + +It also contains class definitions used by the plugins, including definitions +for Blob, Connection, and Packet. + +""" + +# standard Python imports +import datetime +import heapq +import inspect +import logging +import warnings +from collections import defaultdict +from multiprocessing import Value +from typing import Iterable, List, Tuple, Union + +# Dshell imports +from dshell.output.output import Output +from dshell.dshellgeoip import DshellGeoIP, DshellFailedGeoIP + +# third-party imports +import pcapy +from pypacker import pypacker +from pypacker.layer12 import can, ethernet, ieee80211, linuxcc, ppp, pppoe, radiotap +from pypacker.layer3 import ip, ip6, icmp, icmp6 +from pypacker.layer4 import tcp, udp + + +logger = logging.getLogger(__name__) + +__version__ = "3.2.3" + +class SequenceNumberError(Exception): + """ + Raised when reassembling connections and data is missing or overlapping. + See Blob.reassemble function + """ + pass + + +class DataError(Exception): + """ + Raised when any data being handled just isn't right. + For example, invalid headers in httpplugin.py + """ + pass + + +# Create GeoIP refrence object +try: + geoip = DshellGeoIP() +except FileNotFoundError: + logger.warning( + "Could not find GeoIP data files! Country and ASN lookups will not be possible. Check README for instructions on where to find and install necessary data files.") + geoip = DshellFailedGeoIP() + + +def print_handler_exception(e, plugin, handler): + """ + A convenience function to display an error message when a handler raises + an exception. + + If using --debug, it will print a full traceback. + + Args: + e: the exception object + plugin: the plugin object + handler: name of the handler function + """ + etype = e.__class__.__name__ + logger.error( + "The {!s} for the {!r} plugin raised an exception and failed! ({}: {!s})".format( + handler, plugin.name, etype, e)) + logger.debug(e, exc_info=True) + + +class PacketPlugin(object): + """ + Base level class that plugins will inherit. + + This plugin handles individual packets. To handle reconstructed + connections, use the ConnectionPlugin. + + Attributes: + name: the name of the plugin + description: short description of the plugin (used with decode -l) + longdescription: verbose description of the plugin (used with -h) + bpf: default BPF to apply to traffic entering plugin + compiled_bpf: a compiled BPF for pcapy, usually created in decode.py + vlan_bpf: boolean that tells whether BPF should be compiled with + VLAN support + author: preferably, the initials of the plugin's author + seen_packet_count: number of packets this plugin has seen + handled_packet_count: number of packets this plugin has passed + through a handler function + seen_conn_count: number of connections this plugin has seen + handled_conn_count: number of connections this plugin has passed + through a handler function + out: output module instance + link_layer_type: numeric label for link layer + defrag_ip: rebuild fragmented IP packets (default: True) + """ + + # TODO: Move attributes like name, author, and description to be class attributes instead of instance. + def __init__(self, **kwargs): + self.name = kwargs.get('name', __name__) + self.description = kwargs.get('description', '') + self.longdescription = kwargs.get('longdescription', self.description) + self.bpf = kwargs.get('bpf', '') + self.compiled_bpf = kwargs.get('compiled_bpf', None) + self.vlan_bpf = kwargs.get("vlan_bpf", True) + self.author = kwargs.get('author', '') + self.logger = logging.getLogger(inspect.getmodule(self).__name__) + + # define overall counts as multiprocessing Values for --parallel + self.seen_packet_count = Value('i', 0) + self.handled_packet_count = Value('i', 0) + + # dict of options specific to this plugin in format + # 'optname':{configdict} translates to --pluginname_optname + self.optiondict = kwargs.get('optiondict', {}) + + # queues used by decode.py + # if a handler decides a packet is worth keeping, it is placed in a + # queue and later grabbed by decode.py to pass to subplugins + self._packet_queue = [] + + # self.out holds the output plugin instance + # can be overwritten in decode.py by user selection + self.out = kwargs.get('output', Output()) + + # capture options + # these can be updated with set_link_layer_type function + self.link_layer_type = 1 # assume Ethernet + # rebuild fragmented IP packets + self.defrag_ip = True + + # holder for the pcap file being processing + self.current_pcap_file = None + + # a holder for IP packet fragments when attempting to reassemble them + self._packet_fragments = defaultdict(dict) + + def produce_packets(self) -> Iterable["Packet"]: + """ + Produces packets ready to be processed by the next plugin in the chain. + """ + while self._packet_queue: + yield self._packet_queue.pop(0) + + def flush(self): + """ + Triggers plugin to finish processing any remaining packets that are being held onto. + """ + # By default we don't need to do anything because any consumed packet is placed onto the queue + # right away. + pass + + def purge(self): + """ + When finished with handling a pcap file, calling this will clear all + caches in preparation for next file. + """ + self._packet_queue = [] + self._packet_fragments = defaultdict(dict) + + def write(self, *args, **kwargs): + """ + Sends information to the output formatter, after adding some + additional fields. + """ + if 'plugin' not in kwargs: + kwargs['plugin'] = self.name + if 'pcapfile' not in kwargs: + kwargs['pcapfile'] = self.current_pcap_file + self.out.write(*args, **kwargs) + + def log(self, msg, level=logging.INFO): + """ + Logs msg argument at specified level + (default of INFO is for -v/--verbose output) + + Arguments: + msg: text string to log + level: logging level (default: logging.INFO) + """ + warnings.warn("log() function is deprecated. Please use logging library instead.", DeprecationWarning) + logger.log(level, msg) + + def debug(self, msg): + """ + Logs msg argument at debug level + """ + warnings.warn("debug() function is deprecated. Please use logging library instead.", DeprecationWarning) + logger.debug(msg) + + def warn(self, msg): + """ + Logs msg argument at warning level + """ + warnings.warn("warn() function is deprecated. Please use logging library instead.", DeprecationWarning) + logger.warning(msg) + + def error(self, msg): + """ + Logs msg argument at error level. + """ + warnings.warn("error() function is deprecated. Please use logging library instead.", DeprecationWarning) + logger.warning(msg) + + def __str__(self): + return "<{}: {}>".format("Plugin", self.name) + + def __repr__(self): + return '<{}: {}/{}/{}>'.format("Plugin", self.name, self.bpf, + ','.join([('%s=%s' % (x, str(self.__dict__.get(x)))) for x in self.optiondict])) + + # TODO: Perhaps make bpf a property which auto-triggers this when the property value is set. + def recompile_bpf(self): + """ + Compile the BPF stored in the .bpf attribute + """ + # This function is normally only called by the decode.py script, + # but can also be called by plugins that need to dynamically update + # their filter. + if not self.bpf: + logger.debug("Cannot compile BPF: .bpf attribute not set for plugin {!r}.".format(self.name)) + self.compiled_bpf = None + return + + # Add VLAN wrapper, if necessary + if self.vlan_bpf: + bpf = "({0}) or (vlan and {0})".format(self.bpf) + else: + bpf = self.bpf + logger.debug("Compiling BPF as {!r}".format(bpf)) + + # Compile BPF and handle any expected errors + try: + self.compiled_bpf = pcapy.compile( + self.link_layer_type, 65536, bpf, True, 0xffffffff + ) + except pcapy.PcapError as e: + if str(e).startswith("no VLAN support for data link type"): + logger.error("Cannot use VLAN filters for {!r} plugin. Recommend running with --no-vlan argument.".format(self.name)) + elif str(e) == "syntax error": + raise ValueError("Fatal error when compiling BPF: {!r}".format(bpf)) + else: + raise e + + def ipdefrag(self, packet: 'Packet') -> 'Packet': + """ + IP fragment reassembly + + Store the first seen packet, collect data from followup packets, then + glue it all together and update that first packet with new data + """ + pkt = packet.pkt + ipp = pkt.upper_layer + if isinstance(ipp, ip.IP): # IPv4 + f = self._packet_fragments[(ipp.src, ipp.dst, ipp.id)] + f[ipp.offset] = packet + + if not ipp.flags & 0x1: # If no more fragments (MF) + if len(f) <= 1 and 0 in f: + # If only one unfragmented packet, return that packet + del self._packet_fragments[(ipp.src, ipp.dst, ipp.id)] + return f[0] + elif 0 not in f: + logger.debug(f"Missing first fragment of fragmented packet. Dropping ({packet.sip} -> {packet.dip}: {ipp.id}:{ipp.flags}:{ipp.offset})") + del self._packet_fragments[(ipp.src, ipp.dst, ipp.id)] + return None + fkeys = sorted(f.keys()) + data = b'' + firstpacket = f[fkeys[0]] + for key in fkeys: + data += f[key].pkt.upper_layer.body_bytes + newip = ip.IP(firstpacket.pkt.upper_layer.header_bytes + data) + newip.bin(update_auto_fields=True) # refresh checksum + firstpacket.pkt.upper_layer = newip + del self._packet_fragments[(ipp.src, ipp.dst, ipp.id)] + return Packet( + firstpacket.pkt.__len__, + firstpacket.pkt, + firstpacket.ts, + firstpacket.frame + ) + + elif isinstance(pkt, ip6.IP6): # IPv6 + # TODO handle IPv6 offsets https://en.wikipedia.org/wiki/IPv6_packet#Fragment + return pkt + + def handle_plugin_options(self): + """ + A placeholder. + + This function is called immediately after plugin args are processed + and set in decode.py. A plugin can overwrite this function to perform + actions based on the arg values as soon as they are set, before + decode.py does any further processing (e.g. updating a BPF based on + provided arguments before handling --ebpf and --bpf flags). + """ + pass + + def _premodule(self): + """ + _premodule is called before capture starts or files are read. It will + attempt to call the child plugin's premodule function. + """ + self.premodule() + self.out.setup() + # self.debug('{}'.format(pprint.pformat(self.__dict__))) + self.debug(str(self.__dict__)) + + def premodule(self): + """ + A placeholder. + + A plugin can overwrite this function to perform an action before + capture starts or files are read. + """ + pass + + def _postmodule(self): + """ + _postmodule is called when capture ends. It will attempt to call the + child plugin's postmodule function. It will also print stats if in + debug mode. + """ + self.postmodule() + self.out.close() + logger.info( + f"{self.seen_packet_count.value} seen packets, " + f"{self.handled_packet_count.value} handled packets " + ) + + def postmodule(self): + """ + A placeholder. + + A plugin can overwrite this function to perform an action after + capture ends or all files are processed. + """ + pass + + def _prefile(self, infile=None): + """ + _prefile is called just before an individual file is processed. + Stores the current pcap file string and calls the child plugin's + prefile function. + """ + self.current_pcap_file = infile + self.prefile(infile) + logger.info('working on file "{}"'.format(infile)) + + def prefile(self, infile=None): + """ + A placeholder. + + A plugin will be able to overwrite this function to perform an action + before an individual file is processed. + + Arguments: + infile: filepath or interface that will be processed + """ + pass + + def _postfile(self): + """ + _postfile is called just after an individual file is processed. + It may expand some day, but for now it just calls a child's postfile + function. + """ + self.postfile() + + def postfile(self): + """ + A placeholder. + + A plugin will be able to overwrite this function to perform an action + after an individual file is processed. + """ + pass + + def filter(self, packet) -> bool: + """ + Determines if plugin accepts the packet or it should be filtered out. + + :param packet: dshell.Packet object + :return: + """ + # By default we filter by running the compiled bpf, but a plugin can + # inherit this to do extra stuff if desired. + if not self.compiled_bpf: + return True + return bool(self.compiled_bpf.filter(packet.rawpkt)) + + # NOTE: This was originally called '_packet_handler' + def consume_packet(self, packet: "Packet"): + """ + Filters and defragments packet and then passes the packet along to the packet_handler() + function to determine whether we should pass the packet(s) along to the next plugin. + """ + # First apply filter to packet. + if not self.filter(packet): + return + + with self.seen_packet_count.get_lock(): + self.seen_packet_count.value += 1 + + # Attempt to perform defragmentation + if self.defrag_ip and isinstance(packet.pkt.upper_layer, (ip.IP, ip6.IP6)): + defragpkt = self.ipdefrag(packet) + if not defragpkt: + # we do not yet have all of the packet fragments, so move + # on to next packet for now + return + else: + packet = defragpkt + + # call packet_handler and return its output + # decode.py will continue down the chain if it returns anything + try: + packet_handler_out = self.packet_handler(packet) + except Exception as e: + print_handler_exception(e, self, 'packet_handler') + return + failed_msg = ( + f"The output from {self.name} packet_handler must be of type dshell.Packet or a list of " + f"such objects! Handling connections or chaining from this plugin may not be possible." + ) + if isinstance(packet_handler_out, (list, tuple)): + for phout in packet_handler_out: + if isinstance(phout, Packet): + self._packet_queue.append(phout) + with self.handled_packet_count.get_lock(): + self.handled_packet_count.value += 1 + elif phout: + logger.warning(failed_msg) + elif isinstance(packet_handler_out, Packet): + self._packet_queue.append(packet_handler_out) + with self.handled_packet_count.get_lock(): + self.handled_packet_count.value += 1 + elif packet_handler_out: + logger.warning(failed_msg) + + def packet_handler(self, pkt: "Packet"): + """ + A placeholder. + + Plugins will be able to overwrite this to perform custom activites on + Packet data. + + It should return a Packet object for functions further down the chain + (i.e. connection_handler and/or blob_handler) + + Arguments: + pkt: a Packet object + """ + return pkt + + +class ConnectionPlugin(PacketPlugin): + """ + Base level class that plugins will inherit. + + This plugin reassembles connections from packets. + """ + + # Determines whether to filter out packets based on blobs or to produce packets directly. + # Turning this off if the plugin doesn't mark any blobs as hidden can help improve speed. + # TODO: There is another hacky reason this boolean exists. + # Due to how we modified the blob creation code, the ACK and handshake methods are not + # part of any of the blobs. Therefore, when the produce_packets() function is called, those + # packets are missing if we are only producing the packets within a blob. + blob_filtering = True + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + # similar to packet_queue and raw_packet_queue in superclass + self._connection_queue = [] + # Flag used to determine if we are ready to produce closed connections + # for the next plugin in the chain. + self._production_ready = True + + # dictionary to store packets for connections according to addr() + # NOTE: Only currently unhandled (ie. open) connections are stored here. + self._connection_tracker = {} + + # define overall counts as multiprocessing Values for --parallel + self.seen_conn_count = Value('i', 0) + self.handled_conn_count = Value('i', 0) + + # maximum number of blobs a connection will store before calling + # connection_handler + # it defaults to infinite, but this should be lowered for huge datasets + self.maxblobs = float("inf") # infinite + + # how long do we wait before deciding a connection is "finished" + # time is checked by iterating over cached connections and checking if + # the timestamp of the connection's last packet is older than the + # timestamp of the current packet, minus this value + self.timeout = datetime.timedelta(hours=1) + # The number of packets to process between timeout checks. + self.timeout_frequency = 50 + # The maximum number of open connections allowed at one time. + # If the maximum number of connections is met, the oldest connections + # will be force closed. + self.max_open_connections = 1000 + + def _postmodule(self): + """ + Overwriting _postmodule to add log info about connection counts. + """ + super()._postmodule() + logger.info( + f"{self.seen_conn_count.value} seen connections, " + f"{self.handled_conn_count.value} handled connections" + ) + + def produce_connections(self) -> Iterable["Connection"]: + """ + Produces recently closed connections ready to be passed down to the next plugin in the chain. + """ + # Avoid producing connections if we are still waiting for an older connection to close. + # This helps to ensure connections are produced in the right order.... for the most part. + if not self._production_ready: + return + while self._connection_queue: + # Pop off oldest closed connection. + _, full, connection = heapq.heappop(self._connection_queue) + # Handle connection + success = self._handle_connection(connection, full=full) + if not success or connection.stop: + continue + # Pass along connection to next plugin. + yield connection + self._production_ready = False + + def produce_packets(self) -> Iterable["Packet"]: + """ + Produces packets ready to be processed by the next plugin in the chain. + """ + # Produce connections + for connection in self.produce_connections(): + if self.blob_filtering: + for blob in connection.blobs: + if not blob.hidden: + yield from blob.packets + else: + # TODO: Perhaps have a "hidden" field on the packet itself? + yield from connection.packets + + def consume_packet(self, packet: "Packet"): + # First run super() to handle the individual packets. + super().consume_packet(packet) + + # Now process any produced packets to be processed through connection handler. + for _packet in super().produce_packets(): + self._connection_handler(_packet) + + def flush(self): + """ + Triggers plugin to finish processing any remaining packets that are being held onto. + """ + super().flush() + # Call cleanup_connections() to force close any remaining open connections so they are + # on the queue ready to be passed down the chain. + self._cleanup_connections() + + def _connection_handler(self, packet: "Packet"): + """ + Accepts a single Packet object and tracks the connection it belongs to. + + If it is the first packet in a connection, it creates a new Connection + object and passes it to connection_init_handler. Otherwise, it will + find the existing Connection in self.connection_tracker. + + The Connection will then be passed to connection_handler. + + If a connection changes direction with this packet, blob_handler will + be called. + + Finally, if this packet is a FIN or RST, it will determine if the + connection should close. + """ + # Sort the addr value for consistent dictionary key purposes + connkey = tuple(sorted(packet.addr) + [packet.protocol_num]) + + # If this is a new connection, initialize it and call the init handler + if connkey not in self._connection_tracker: + conn = Connection(packet) + self._connection_tracker[connkey] = conn + try: + self.connection_init_handler(conn) + except Exception as e: + print_handler_exception(e, self, 'connection_init_handler') + return + with self.seen_conn_count.get_lock(): + self.seen_conn_count.value += 1 + else: + conn = self._connection_tracker[connkey] + conn.add_packet(packet) + + # TODO: Do we need this? This flag is set to False when the connection is initialized and not + # set to true until it is closed. + # Is there any scenario where we would want to undo a True handled state? + # # If connection data is about to change, we set it to a "dirty" state + # # for future calls to connection_handler + # if pkt.data: + # conn.handled = False + + if conn.closed: + # Both sides have closed the connection, process blobs (messages) and + # close connection. + for blob in conn.blobs: + self._blob_handler(conn, blob) + self._close_connection(conn, full=True) + + # TODO: Switch to a max_packets option. + # elif len(conn.blobs) > self.maxblobs: + # # Max blobs hit, so we will run connection_handler and decode.py + # # will clear the connection's blob cache + # self._close_connection(conn) + + # Check for and close old connections every so often. + if self.handled_packet_count.value % self.timeout_frequency == 0: + self._timeout_connections(packet.dt) + + def _close_connection(self, conn, full=False): + """ + Runs through some standard actions to close a connection + """ + # Add connection to queue ready to be processed, based on order they were received on the wire. + heapq.heappush(self._connection_queue, (conn.packets[0].frame, full, conn)) + + # Remove connection from tracker once in the queue. + try: + connkey = tuple(sorted(conn.addr) + [conn.protocol_num]) + del self._connection_tracker[connkey] + except KeyError: + pass + + def _handle_connection(self, conn: "Connection", full=False) -> bool: + """ + Handles produced connections. + + :returns: True if connection was handled successfully. + """ + try: + connection_handler_out = self.connection_handler(conn) + except Exception as e: + print_handler_exception(e, self, 'connection_handler') + return False + conn.handled = True + + # TODO: Perhaps connection_handler() just returns a True or False indicating success? + if connection_handler_out and not isinstance(connection_handler_out, Connection): + logger.warning( + "The output from {} connection_handler must be of type dshell.Connection! Chaining plugins from here may not be possible.".format( + self.name)) + connection_handler_out = None + + if not connection_handler_out: + return False + + with self.handled_conn_count.get_lock(): + self.handled_conn_count.value += 1 + + if full: + try: + self.connection_close_handler(conn) + except Exception as e: + print_handler_exception(e, self, 'connection_close_handler') + return True + + def _timeout_connections(self, timestamp: datetime.datetime): + """ + Checks for and force closes connections that have been alive for too long. + It also closes the oldest connections if too many connections are open. + """ + # Force close any connections that have timed out. + # This is based on comparing the time of the current packet, minus + # self.timeout, to each connection's current endtime value. + for conn in list(self._connection_tracker.values()): + if conn.endtime < (timestamp - self.timeout): + self._close_connection(conn) + + # Force close oldest connections if we have too many. + if len(self._connection_tracker) > self.max_open_connections: + connections = sorted(self._connection_tracker.values(), key=lambda conn: conn.endtime, reverse=True) + for conn in connections[self.max_open_connections:]: + self._close_connection(conn) + + # We can produce connections again, now that we have handled lingering old connections. + self._production_ready = True + + def _cleanup_connections(self): + """ + decode.py will often reach the end of packet capture before all of the + connections are closed properly. This function is called at the end + of things to process those dangling connections. + + NOTE: Because the connections did not close cleanly, + connection_close_handler will not be called. + """ + for conn in list(self._connection_tracker.values()): + if not conn.handled: + self._close_connection(conn) + self._production_ready = True + + def purge(self): + """ + When finished with handling a pcap file, calling this will clear all + caches in preparation for next file. + """ + super().purge() + self._connection_queue = [] + self._connection_tracker = {} + self._production_ready = False + + # TODO: Have blobs handled with consumer/producer model just like Packets and Connections? + def _blob_handler(self, conn: "Connection", blob: "Blob"): + """ + Accepts a Connection and a Blob. + + It doesn't really do anything except call the blob_handler and is only + here for consistency and possible future features. + """ + try: + blob_handler_out = self.blob_handler(conn, blob) + except Exception as e: + print_handler_exception(e, self, 'blob_handler') + blob_handler_out = None + if blob_handler_out: + connection, blob = blob_handler_out + if not isinstance(connection, Connection) or not isinstance(blob, Blob): + logger.warning( + "The output from {} blob_handler must be of type (dshell.Connection, dshell.Blob)! Chaining plugins from here may not be possible.".format( + self.name)) + blob_handler_out = None + if not blob_handler_out: + blob.hidden = True + + def blob_handler(self, conn: "Connection", blob: "Blob"): + """ + A placeholder. + + Plugins will be able to overwrite this to perform custom activites on + Blob data. + + It should return a Connection object and a Blob object for functions + further down the chain. + + Args: + conn: Connection object + blob: Blob object + """ + return conn, blob + + def connection_init_handler(self, conn: "Connection"): + """ + A placeholder. + + Plugins will be able to overwrite this to perform custom activites on + a connection it is first seen. + + Args: + conn: Connection object + """ + return + + def connection_handler(self, conn: "Connection"): + """ + A placeholder. + + Plugins will be able to overwrite this to perform custom activites on + Connection data. + + It should return a Connection object for functions further down the chain + + Args: + conn: Connection object + """ + return conn + + def connection_close_handler(self, conn: "Connection"): + """ + A placeholder. + + Plugins will be able to overwrite this to perform custom activites on + a TCP connection when it is cleanly closed with RST or FIN. + + Args: + conn: Connection object + """ + return + + +class Packet(object): + """ + Class for holding data of individual packets + + def __init__(self, plugin, pktlen, pkt, ts): + + Args: + pktlen: length of packet + pkt: pypacker object for the packet + ts: timestamp of packet + + Attributes: + ts: timestamp of packet + dt: datetime of packet + frame: sequential packet number as read from data stream + pkt: pypacker object for the packet + rawpkt: raw bytestring of the packet + pktlen: length of packet + byte_count: length of packet body + sip: source IP + dip: destination IP + sip_bytes: source IP as bytes + dip_bytes: destination IP as bytes + sport: source port + dport: destination port + smac: source MAC + dmac: destination MAC + sipcc: source IP country code + dipcc: dest IP country code + siplat: source IP latitude + diplat: dest IP latitude + siplon: source IP longitude + diplon: dest IP longitude + sipasn: source IP ASN + dipasn: dest IP ASN + protocol: text version of protocol in layer-3 header + protocol_num: numeric version of protocol in layer-3 header + data: data of the packet after TCP layer, or highest layer + sequence_number: TCP sequence number, or None + ack_number: TCP ACK number, or None + tcp_flags: TCP header flags, or None + """ + + IP_PROTOCOL_MAP = dict((v, k[9:]) for k, v in ip.__dict__.items() if + type(v) == int and k.startswith('IP_PROTO_') and k != 'IP_PROTO_HOPOPTS') + + def __init__(self, pktlen, packet: pypacker.Packet, timestamp: int, frame=0): + # TODO: Use full variable names. + self.ts = timestamp + self.dt = datetime.datetime.fromtimestamp(timestamp) + self.frame = frame + self.pkt = packet + self.pktlen = pktlen # TODO: Is this needed? + + self.sip = None + self.dip = None + self.sport = None + self.dport = None + self.smac = None + self.dmac = None + self.sipcc = None + self.dipcc = None + self.siplat = None + self.diplat = None + self.siplon = None + self.diplon = None + self.sipasn = None + self.dipasn = None + self.protocol = None + self.protocol_num = None + self.sequence_number = None + self.ack_number = None + self.tcp_flags = None + + # attribute cache + self._byte_count = None + self._data = None + + # these are the layers Dshell will help parse + # try to find them in the packet and eventually pull out useful data + ethernet_p = None + ieee80211_p = None + ip_p = None + tcp_p = None + udp_p = None + highest_layer = None + for layer in packet: + highest_layer = layer + if ethernet_p is None and isinstance(layer, ethernet.Ethernet): + ethernet_p = layer + elif ieee80211_p is None and isinstance(layer, ieee80211.IEEE80211): + ieee80211_p = layer + elif ip_p is None and isinstance(layer, (ip.IP, ip6.IP6)): + ip_p = layer + try: + if ip_p.flags & 0x1 and ip_p.offset > 0: + # IP fragmentation, break all further layer processing + break + except AttributeError: + # IPv6 does not always have flags header field set + pass + elif tcp_p is None and isinstance(layer, tcp.TCP): + tcp_p = layer + elif udp_p is None and isinstance(layer, udp.UDP): + udp_p = layer + self._highest_layer = highest_layer + self._ethernet_layer = ethernet_p # type: ethernet.Ethernet + self._ieee80211_layer = ieee80211_p # type: ieee80211.IEEE80211 + self._ip_layer = ip_p # type: Union[ip.IP, ip6.IP6] + self._tcp_layer = tcp_p # type: tcp.TCP + self._udp_layer = udp_p # type: udp.UDP + + # attempt to grab MAC addresses + if ethernet_p: + # from Ethernet + self.smac = ethernet_p.src_s + self.dmac = ethernet_p.dst_s + elif ieee80211_p: + # from 802.11 + try: + if ieee80211_p.subtype == ieee80211.M_BEACON: + ieee80211_p2 = ieee80211_p.beacon + elif ieee80211_p.subtype == ieee80211.M_DISASSOC: + ieee80211_p2 = ieee80211_p.disassoc + elif ieee80211_p.subtype == ieee80211.M_AUTH: + ieee80211_p2 = ieee80211_p.auth + elif ieee80211_p.subtype == ieee80211.M_DEAUTH: + ieee80211_p2 = ieee80211_p.deauth + elif ieee80211_p.subtype == ieee80211.M_ACTION: + ieee80211_p2 = ieee80211_p.action + else: + # can't figure out how pypacker stores the other subtypes + raise AttributeError + self.smac = ieee80211_p2.src_s + self.dmac = ieee80211_p2.dst_s + except AttributeError as e: + pass + + # process IP addresses and associated metadata (if applicable) + if ip_p: + # get IP addresses + self.sip = ip_p.src_s + self.dip = ip_p.dst_s + self.sip_bytes = ip_p.src + self.dip_bytes = ip_p.dst + + # get protocols, country codes, and ASNs + self.protocol_num = ip_p.p if isinstance(ip_p, ip.IP) else ip_p.nxt + self.protocol = self.IP_PROTOCOL_MAP.get(self.protocol_num, str(self.protocol_num)) + self.sipcc, self.siplat, self.siplon = geoip.geoip_location_lookup(self.sip) + self.sipasn = geoip.geoip_asn_lookup(self.sip) + self.dipcc, self.diplat, self.diplon = geoip.geoip_location_lookup(self.dip) + self.dipasn = geoip.geoip_asn_lookup(self.dip) + + if tcp_p: + self.sport = tcp_p.sport + self.dport = tcp_p.dport + self.sequence_number = tcp_p.seq + self.ack_number = tcp_p.ack + self.tcp_flags = tcp_p.flags + + elif udp_p: + self.sport = udp_p.sport + self.dport = udp_p.dport + + @property + def addr(self): + """ + A standard representation of the address: + ((self.sip, self.sport), (self.dip, self.dport)) + or + ((self.smac, self.sport), (self.dmac, self.dport)) + """ + # try using IP addresses first + if self.sip or self.dip: + return (self.sip, self.sport), (self.dip, self.dport) + # then try MAC addresses + elif self.smac or self.dmac: + return (self.smac, self.sport), (self.dmac, self.dport) + # if all else fails, return Nones + else: + return (None, None), (None, None) + + @property + def byte_count(self) -> int: + """ + Total number of payload bytes in the packet. + """ + if self._byte_count is None: + self._byte_count = len(self.data) + return self._byte_count + + @property + def packet_tuple(self): + """ + A standard representation of the raw packet tuple: + (self.pktlen, self.rawpkt, self.ts) + """ + return self.pktlen, self.rawpkt, self.ts + + @property + def rawpkt(self): + """ + The raw data that represents the full packet. + """ + return self.pkt.bin() + + @property + def data(self): + """ + Retrieve data bytes from TCP/UDP data layer. Backtracks to data from highest layer. + """ + if self._data is None: + # NOTE: Using cached layers because pypacker's __getitem__ is slow. + # best_layer = self.pkt[tcp.TCP] or self.pkt[udp.UDP] or self.pkt.highest_layer + best_layer = self._tcp_layer or self._udp_layer or self._highest_layer + + # Pypacker doesn't handle Ethernet trailers correctly, so we need to + # do some header calculation in order to determine the true body_bytes size. + ip_layer = self._ip_layer + tcp_layer = self._tcp_layer + if ip_layer and tcp_layer: + if isinstance(ip_layer, ip.IP): # IPv4 + data_size = ip_layer.len - (ip_layer.header_len + tcp_layer.header_len) + self._data = best_layer.body_bytes[:data_size] + else: # IPv6 + # TODO handle extension headers + data_size = ip_layer.dlen - tcp_layer.header_len + self._data = best_layer.body_bytes[:data_size] + else: + self._data = best_layer.body_bytes + + return self._data + + @data.setter + def data(self, data): + """ + Sets data bytes to TCP/UDP data layer. Backtracks to setting data at highest layer. + """ + # NOTE: Using cached layers because pypacker's __getitem__ is slow. + # best_layer = self.pkt[tcp.TCP] or self.pkt[udp.UDP] or self.pkt.highest_layer + best_layer = self._tcp_layer or self._udp_layer or self._highest_layer + + # Pypacker doesn't handle Ethernet trailers correctly, so we need to + # do some header calculation in order to determine the true body_bytes size. + ip_layer = self._ip_layer + tcp_layer = self._tcp_layer + if ip_layer and tcp_layer: + if isinstance(ip_layer, ip.IP): # IPv4 + data_size = ip_layer.len - (ip_layer.header_len + tcp_layer.header_len) + best_layer.body_bytes = data + best_layer.body_bytes[data_size:] + else: # IPv6 + # TODO handle extension headers + data_size = ip_layer.dlen - tcp_layer.header_len + best_layer.body_bytes = data + best_layer.body_bytes[data_size:] + else: + best_layer.body_bytes = data + + self._data = data + # TODO: Rebuild packet object to allow for pypacker to do its thing. + + def __repr__(self): + return "%s %16s :%-5s -> %5s :%-5s (%s -> %s)" % ( + self.dt, self.sip, self.sport, self.dip, self.dport, self.sipcc, self.dipcc) + + def info(self): + """ + Provides a dictionary with information about a packet. Useful for + calls to a plugin's write() function, e.g. self.write(\\*\\*pkt.info()) + """ + d = {k: v for k, v in self.__dict__.items() if not k.startswith('_')} + d['byte_count'] = self.byte_count + d['rawpkt'] = self.pkt.bin() + del d['pkt'] + return d + + +class Connection(object): + """ + Class for holding data about connections + + def __init__(self, plugin, first_packet) + + Args: + first_packet: the first Packet object to initialize connection + + Attributes: + addr: .addr attribute of first packet + sip: source IP + smac: source MAC address + sport: source port + sipcc: country code of source IP + siplat: latitude of source IP + siplon: longitude of source IP + sipasn: ASN of source IP + clientip: same as sip + clientmac: same as smac + clientport: same as sport + clientcc: same as sipcc + clientlat: same as siplat + clientlon: same as siplon + clientasn: same as sipasn + dip: dest IP + dmac: dest MAC address + dport: dest port + dipcc: country code of dest IP + diplat: latitude of dest IP + diplon: longitude of dest IP + dipasn: ASN of dest IP + serverip: same as dip + servermac: same as dmac + serverport: same as dport + servercc: same as dipcc + serverlat: same as diplat + serverlon: same as diplon + serverasn: same as dipasn + protocol: text version of protocol in layer-3 header + protocol_num: numeric version of protocol in layer-3 header + clientpackets: counts of packets from client side + clientbytes: total bytes transferred from client side + serverpackets: counts of packets from server side + serverbytes: total bytes transferred from server side + ts: timestamp of first packet + dt: datetime of first packet + starttime: datetime of first packet + endtime: datetime of last packet + client_state: the TCP state on the client side ("init", + "established", "closed", etc.) + server_state: the TCP state on server side + blobs: list of reassembled half-stream Blobs + stop: if True, stop following connection + handled: used to indicate if a connection was already passed through + a plugin's connection_handler function. Resets when new + data for a connection comes in. + + """ + + # status + # NOTE: Using strings instead of int enum to stay backwards compatible. + INIT = "init" + ESTABLISHED = "established" + FINISHING = "finishing" + CLOSED = "closed" + + def __init__(self, first_packet): + """ + Initializes Connection object + + Args: + first_packet: the first Packet object to initialize connection + """ + self.addr = first_packet.addr + # TODO: Rename these variables to something more verbose like "source_ip" + # I keep getting confused whether the "s" stands for "source" or "server". + self.sip = first_packet.sip + self.smac = first_packet.smac + self.sport = first_packet.sport + self.sipcc = first_packet.sipcc + self.siplat = first_packet.siplat + self.siplon = first_packet.siplon + self.sipasn = first_packet.sipasn + self.clientip = first_packet.sip + self.clientmac = first_packet.smac + self.clientport = first_packet.sport + self.clientcc = first_packet.sipcc + self.clientlat = first_packet.siplat + self.clientlon = first_packet.siplon + self.clientasn = first_packet.sipasn + self.dip = first_packet.dip + self.dmac = first_packet.dmac + self.dport = first_packet.dport + self.dipcc = first_packet.dipcc + self.diplat = first_packet.diplat + self.diplon = first_packet.diplon + self.dipasn = first_packet.dipasn + self.serverip = first_packet.dip + self.servermac = first_packet.dmac + self.serverport = first_packet.dport + self.servercc = first_packet.dipcc + self.serverlat = first_packet.diplat + self.serverlon = first_packet.diplon + self.serverasn = first_packet.dipasn + self.protocol = first_packet.protocol + self.protocol_num = first_packet.protocol_num + self.ts = first_packet.ts + self.dt = first_packet.dt + self.starttime = first_packet.dt + self.endtime = first_packet.dt + self.client_state = None + self.server_state = None + # self.blobs = [] + self.packets = [] # keeps track of packets in connection. + self.stop = False + self.handled = False + + # Cache of created blobs + self._blob_cache = [] + + self.add_packet(first_packet) + + @property + def duration(self): + """ + Total seconds from start_time to end_time. + """ + tdelta = self.endtime - self.starttime + return tdelta.total_seconds() + + @property + def closed(self): + return self.client_state == self.CLOSED and self.server_state == self.CLOSED + + @property + def established(self): + return self.client_state == self.ESTABLISHED and self.server_state == self.ESTABLISHED + + @property + def blobs(self) -> Iterable["Blob"]: + """ + Iterates the blobs (or messages) contained in this tcp connection + + This is dynamically generated on-demand based on the current set of packets in the connection. + """ + if self._blob_cache: + yield from self._blob_cache + + else: + blobs = [] + + for packet in self.packets: + # TODO: skipping packets without data greatly improves speed, but we may want to + # allow them if we support using ack numbers. + if not packet.data: + continue + + # If we see a sequence for an old blob, this is a retransmission. + # Find the blob and add this packet. + # NOTE: There is probably more to it than this, but this seems to work for now. + seq = packet.sequence_number + if seq is not None: + found = False + for blob in blobs: + if blob.sip == packet.sip and seq in blob.sequence_range: + blob.add_packet(packet) + found = True + break + if found: + continue + + # Create a new message if the first or the other direction has started sending data. + if not blobs or (packet.sip != blobs[-1].sip and packet.data): + blobs.append(Blob(self, packet)) + + # Otherwise add packet to last blob. + else: + blobs[-1].add_packet(packet) + + self._blob_cache = blobs + yield from blobs + + def add_packet(self, packet: Packet): + """ + Adds packet to connection. + + :param packet: a Packet object to add to the connection + """ + if packet.sip not in (self.sip, self.dip): + raise ValueError(f"Address {repr(packet.sip)} is not part of connection.") + + self.packets.append(packet) + # A new packet means we might need to recalculate all of the blobs + self._blob_cache = [] + + # Adjust state if packet is part of a startup or shutdown. + if packet.tcp_flags is not None: + # Acknowledging a completed handshake to open connection. + if packet.tcp_flags == (tcp.TH_SYN | tcp.TH_ACK): + self.server_state = self.ESTABLISHED + self.client_state = self.ESTABLISHED + + # Asking to close connection. + elif packet.tcp_flags & (tcp.TH_FIN | tcp.TH_RST): + if packet.sip == self.serverip: + self.server_state = self.FINISHING + else: + self.client_state = self.FINISHING + + # Closing connection acknowledged. + elif packet.tcp_flags & tcp.TH_ACK: + if packet.dip == self.serverip and self.server_state == self.FINISHING: + self.server_state = self.CLOSED + elif packet.dip == self.clientip and self.client_state == self.FINISHING: + self.client_state = self.CLOSED + + if packet.dt > self.endtime: + self.endtime = packet.dt + + def info(self): + """ + Provides a dictionary with information about a connection. Useful for + calls to a plugin's write() function, e.g. self.write(\\*\\*conn.info()) + + Returns: + Dictionary with information + """ + d = {k: v for k, v in self.__dict__.items() if not k.startswith('_')} + + cb, cp, sb, sp = self.bytes_and_counts() + + d['duration'] = self.duration +# d['clientbytes'] = self.clientbytes +# d['clientpackets'] = self.clientpackets +# d['serverbytes'] = self.serverbytes +# d['serverpackets'] = self.serverpackets + d['clientbytes'] = cb + d['clientpackets'] = cp + d['serverbytes'] = sb + d['serverpackets'] = sp + del d['stop'] + del d['handled'] + del d['packets'] + return d + + def _client_packets(self) -> Iterable[Packet]: + for packet in self.packets: + if packet.addr == self.addr: + yield packet + + def _server_packets(self) -> Iterable[Packet]: + for packet in self.packets: + if packet.addr != self.addr: + yield packet + + def bytes_and_counts(self) -> Tuple[int, int, int, int]: + """ + Convenience function to get client and server packet and byte counts + while only iterating over the packet list once. + Returns a tuple of: + (client bytes, client packets, server bytes, server packets) + """ + cbytes, cpkts, sbytes, spkts = 0, 0, 0, 0 + for packet in self.packets: + if packet.addr == self.addr: + # client + cbytes += packet.byte_count + cpkts += bool(packet.byte_count) # only count packets with data + else: + # server + sbytes += packet.byte_count + spkts += bool(packet.byte_count) # only count packets with data + return (cbytes, cpkts, sbytes, spkts) + + @property + def totalbytes(self) -> int: + """ + The total number of bytes from both directions + """ + return sum(packet.byte_count for packet in self.packets) + + @property + def clientbytes(self) -> int: + """ + The total number of bytes from the client. + """ + return sum(packet.byte_count for packet in self._client_packets()) + + @property + def clientpackets(self) -> int: + """ + The total number of packets from the client. + """ + # (Only counting packets with data.) + return sum(bool(packet.byte_count) for packet in self._client_packets()) + + @property + def serverbytes(self) -> int: + """ + The total number of bytes from the server. + """ + return sum(packet.byte_count for packet in self._server_packets()) + + @property + def serverpackets(self) -> int: + """ + The total number of packets from the server. + """ + # (Only counting packets with data.) + return sum(bool(packet.byte_count) for packet in self._server_packets()) + + def __repr__(self): + cb, cp, sb, sp = self.bytes_and_counts() + return '%s %16s -> %16s (%s -> %s) %6s %6s %5d %5d %7d %7d %-.4fs' % ( + self.starttime, + self.clientip, + self.serverip, + self.clientcc, + self.servercc, + self.clientport, + self.serverport, +# self.clientpackets, +# self.serverpackets, +# self.clientbytes, +# self.serverbytes, + cp, + sp, + cb, + sb, + self.duration, + ) + + +# TODO: Rename this "TCPBlob" and then have a more generic "Blob" class it inherits from. +class Blob(object): + """ + Class for holding and reassembling pieces of a connection. + + A Blob holds the packets and reassembled data for traffic moving in one + direction in a connection, before direction changes. + + def __init__(self, first_packet, direction) + + Args: + connection: The Connection object that this Blob comes from. (Used for validating packets.) + first_packet: the first Packet object to initialize Blob + + Attributes: + addr: .addr attribute of the first packet + ts: timestamp of the first packet + starttime: datetime for first packet + endtime: datetime of last packet + sip: source IP + smac: source MAC address + sport: source port + sipcc: country code of source IP + sipasn: ASN of source IP + dip: dest IP + dmac: dest MAC address + dport: dest port + dipcc: country code of dest IP + dipasn: ASN of dest IP + protocol: text version of protocol in layer-3 header + direction: direction of the blob - + 'cs' for client-to-server, 'sc' for server-to-client + ack_sequence_numbers: set of ACK numbers from the receiver for #################################### + collected data packets + packets: list of all packets in the blob + hidden (bool): Used to indicate that a Blob should not be passed to + next plugin. Can theoretically be overruled in, say, a + connection_handler to force a Blob to be passed to next + plugin. + """ + + # max offset before wrap, default is MAXINT32 for TCP sequence numbers + MAX_OFFSET = 0xffffffff + + CLIENT_TO_SERVER = 'cs' + SERVER_TO_CLIENT = 'sc' + + def __init__(self, connection: Connection, first_packet): + self.connection = connection + self.addr = first_packet.addr + self.ts = first_packet.ts + self.starttime = first_packet.ts + self.endtime = first_packet.ts + self.sip = first_packet.sip + self.smac = first_packet.smac + self.sport = first_packet.sport + self.sipcc = first_packet.sipcc + self.sipasn = first_packet.sipasn + self.dip = first_packet.dip + self.dmac = first_packet.dmac + self.dport = first_packet.dport + self.dipcc = first_packet.dipcc + self.dipasn = first_packet.dipasn + self.protocol = first_packet.protocol + # self.ack_sequence_numbers = {} + self.packets = [] + # self.data_packets = [] + self.__data_bytes = b'' + + # Used for data caching + self._data = None + self._segments = None + + # Maps sequence number with packets + self._seq_map = {} + self.seq_max = 0 + self.seq_min = 0 + + # Used to indicate that a Blob should not be passed to next plugin. + # Can theoretically be overruled in, say, a connection_handler to + # force a Blob to be passed to next plugin. + self.hidden = False + + if self.sip == self.connection.clientip and \ + (not self.sport or self.sport == self.connection.clientport): + # packet moving from client to server + self.direction = self.CLIENT_TO_SERVER + else: + # packet moving from server to client + self.direction = self.SERVER_TO_CLIENT + + self.add_packet(first_packet) + + @property + def all_packets(self): + warnings.warn("all_packets has been replaced with packets attribute", DeprecationWarning) + return self.packets + +# @property +# def starttime(self): +# return min(packet.dt for packet in self.packets) + + @property + def start_time(self): + return self.starttime + +# @property +# def endtime(self): +# return max(packet.dt for packet in self.packets) + + @property + def end_time(self): + return self.endtime + + @property + def frames(self) -> List[int]: + """ + The frame identifiers for the packets which contain the message. + """ + return [packet.frame for packet in self.packets] + + def get_packets(self, start, end=None) -> List["Packet"]: + """ + Returns the packets that contain data for the given start offset up to the end offset. + If end offset is not provided, just the packet containing the start offset is provided. + """ + packets = [] + + # TODO: Double check logic on this. + + # If not a TCP connection, return frames that had data. + if self.packets[0].tcp_flags is None: + offset = 0 + for packet in self.packets: + if not packet.data: + continue + + offset += len(packet.data) + if offset > start: + packets.append(packet) + if end is None or offset >= end: + break + + # Otherwise, base offsets on sequence numbers. + else: + initial_seq = None + for seq, packet in self.segments: + if initial_seq is None: + initial_seq = seq + offset = seq - initial_seq + end_offset = offset + len(packet.data) + if end_offset > start: + packets.append(packet) + if end is None or end_offset >= end: + break + + return packets + + def get_frames(self, start, end=None) -> List[int]: + """ + Returns frame identifiers for the packets that contain data for the given start offset + up to the end offset. + If end offset is not provided, just the frame identifier for the packet containing the + start offset is provided. + """ + return [packet.frame for packet in self.get_packets(start, end=end)] + + @property + def sequence_numbers(self) -> List[int]: + """ + The starting sequence numbers found within the packets. + """ + return list(self._seq_map.keys()) + + @property + def sequence_range(self) -> range: + """ + The range of sequence numbers found within the packets. + """ +# sequence_numbers = self.sequence_numbers +# if not sequence_numbers: +# return range(0, 0) +# +# min_seq = min(sequence_numbers) +# max_seq = max(sequence_numbers) +# return range(min_seq, max_seq + len(self._seq_map[max_seq].data)) + if not self._seq_map: + return range(0, 0) + + return range(self.seq_min, self.seq_max + len(self._seq_map[self.seq_max].data)) + + @property + def segments(self) -> List[Tuple[int, "Packet"]]: + """ + List of valid (sequence number, packet) tuples in order by sequence number. + """ + if self._segments is not None: + return self._segments + + segments = [] + # Iterate through segments, ignoring segments that cause overlap in data. + expected_seq = None + prev_packet = None + for seq, packet in sorted(self._seq_map.items()): + if expected_seq is None: + expected_seq = seq + + # If the sequence is greater than or equal to the expected sequence, this segment is valid. + if seq >= expected_seq: + segments.append((seq, packet)) + missing_num_bytes = seq - expected_seq + if missing_num_bytes: + logger.debug( + f"Missing {missing_num_bytes} bytes of data between packets " + f"{prev_packet.frame} and {packet.frame}" + ) + expected_seq += missing_num_bytes + len(packet.data) + prev_packet = packet + + # TODO: Support rollover sequence numbers. + # Otherwise, we have some overlap in data and need to remove the invalid segment/packet + # and ignoring adding it to the segments list. + else: + logger.debug(f"Packet {packet.frame} contains overlapped data. Removing...") + self._remove_packet(packet) + + self._segments = segments # cache for next time. + return segments + + @property + def data(self): + """ + Raw data of tcp message. + """ + # Return cache if set. + if self._data is not None: + return self._data + + # If not a TCP connection, just join packet data as they arrived on the wire. + # TODO: Move this logic to a base class. + if self.packets[0].tcp_flags is None: + return b''.join(packet.data for packet in self.packets) + + # Join packet data based on segment data. + data = bytearray() # using bytearray to improve speed. + initial_seq = None + for seq, packet in self.segments: + if initial_seq is None: + initial_seq = seq + + # Check if we have missing packets. + if seq - initial_seq != len(data): + # buffer data with null bytes + data += b'\x00' * (seq - initial_seq - len(data)) + + data += packet.data + data = bytes(data) + + self._data = data # set cache + return data + + @data.setter + def data(self, data): + """ + Replaces message data with new data. + + WARNING: Currently, data must match original length. + """ + # TODO: Support different amount of bytes by adding packets or padding/removing packets. + orig_len = len(self.data) + if len(data) != orig_len: + raise ValueError( + f'Message data must be of the same length as original. ' + f'Expected {orig_len} bytes, got {len(data)} bytes.') + + # If not a TCP connection, just add data to packets in same order they arrived on wire. + if self.packets[0].tcp_flags is None: + written_bytes = 0 + for packet in self.packets: + packet.data = data[written_bytes : written_bytes + len(packet.data)] + written_bytes += len(packet.data) + # Clear old cache. + self._data = None + return + + # If TCP connection, add data based on sequence numbers. + written_bytes = 0 + initial_seq = None + for seq, packet in self.segments: + if initial_seq is None: + initial_seq = seq + + relative_seq = seq - initial_seq + if relative_seq < written_bytes: + raise RuntimeError( + "Relative sequence is less then written byte count. " + "Sequence numbers have be miss-calculated." + ) + # Skip holes in data. (User should have put padding in these areas) + elif relative_seq != written_bytes: + written_bytes = relative_seq + + packet.data = data[written_bytes:written_bytes + len(packet.data)] + written_bytes += len(packet.data) + + # Clear old cache. + self._data = None + + # TODO: Merge this in with the add_packet() logic, however I am unsure how using acknowledge numbers + # works if we are only looking at one side. + def reassemble(self, allow_padding=True, allow_overlap=True, padding=b'\x00'): + """ + Rebuild the data string from the current list of data packets + For each packet, the TCP sequence number is checked. + + If overlapping or padding is disallowed, it will raise a + SequenceNumberError exception if a respective event occurs. + + Args: + allow_padding (bool): If data is missing and allow_padding = True + (default: True), then the padding argument + will be used to fill the gaps. + allow_overlap (bool): If data is overlapping, the new data is + used if the allow_overlap argument is True + (default). Otherwise, the earliest data is + kept. + padding: Byte character(s) to use to fill in missing data. Used + in conjunction with allow_padding (default: b'\\\\x00') + """ + data = b"" + unacknowledged_data = [] + acknowledged_data = {} + for pkt in self.packets: + if not pkt.sequence_number: + # if there are no sequence numbers (i.e. not TCP), just rebuild + # in chronological order + data += pkt.data + continue + + if pkt.data: + if pkt.sequence_number in acknowledged_data: + continue + unacknowledged_data.append(pkt) + + elif pkt.tcp_flags and pkt.tcp_flags & tcp.TH_ACK: + ackpkt = pkt + for i, datapkt in enumerate(unacknowledged_data): + if (datapkt.ack_number == ackpkt.sequence_number + and ackpkt.ack_number == (datapkt.sequence_number + len(datapkt.data))): + # if the seq/ack numbers align, this is the data packet + # we want + # TODO confirm this logic is correct + acknowledged_data[datapkt.sequence_number] = datapkt.data + unacknowledged_data.pop(i) + break + + if not acknowledged_data and not unacknowledged_data: + # For non-sequential protocols, just return what we have + self.__data_bytes = data + + else: + # Create a list of each segment of the complete data. Use + # acknowledged data first, and then try to fill in the blanks with + # unacknowledged data. + segments = acknowledged_data.copy() + for pkt in reversed(unacknowledged_data): + if pkt.sequence_number in segments: continue + segments[pkt.sequence_number] = pkt.data + + offsets = sorted(segments.keys()) + # iterate over the segments and try to piece them together + # handle any instances of missing or overlapping segments + nextoffset = offsets[0] + startoffset = offsets[0] + for offset in offsets: + if offset > nextoffset: + # data is missing + if allow_padding: + data += padding * (offset - nextoffset) + else: + raise SequenceNumberError("Missing data for sequence number %d %s" % (nextoffset, self.addr)) + elif offset < nextoffset: + # data is overlapping + if not allow_overlap: + raise SequenceNumberError( + "Overlapping data for sequence number %d %s" % (nextoffset, self.addr)) + + nextoffset = (offset + len(segments[offset])) & self.MAX_OFFSET + data = data[:offset - startoffset] + \ + segments[offset] + \ + data[nextoffset - startoffset:] + self.__data_bytes = data + + return data + + # segments = {} + # for pkt in self.data_packets: + # if pkt.sequence_number: + # segments.setdefault(pkt.sequence_number, []).append(pkt.data) + # else: + # # if there are no sequence numbers (i.e. not TCP), just rebuild + # # in chronological order + # data += pkt.data + # + # if not segments: + # # For non-sequential protocols, just return what we have + # self.__data_bytes = data + # return data + # + # offsets = sorted(segments.keys()) + # + # # iterate over the segments and try to piece them together + # # handle any instances of missing or overlapping segments + # nextoffset = offsets[0] + # startoffset = offsets[0] + # for offset in offsets: + # # TODO do we still want to implement custom error handling? + # if offset > nextoffset: + # # data is missing + # if allow_padding: + # data += padding * (offset - nextoffset) + # else: + # raise SequenceNumberError("Missing data for sequence number %d %s" % (nextoffset, self.addr)) + # elif offset < nextoffset: + # # data is overlapping + # if not allow_overlap: + # raise SequenceNumberError("Overlapping data for sequence number %d %s" % (nextoffset, self.addr)) + ## nextoffset = (offset + len(segments[offset][dup])) & self.MAX_OFFSET + ## if nextoffset in self.ack_sequence_numbers: + # if offset in self.ack_sequence_numbers: + # # If the data packet was acknowledged by the receiver, + # # we use the first packet received. + # dup = 0 + # else: + # # If it went unacknowledged, we use the last packet and hope + # # for the best. + # dup = -1 + # print(dup) + # print(offset) + # print(nextoffset) + # print(str(self.ack_sequence_numbers)) + # nextoffset = (offset + len(segments[offset][dup])) & self.MAX_OFFSET + # data = data[:offset - startoffset] + \ + # segments[offset][dup] + \ + # data[nextoffset - startoffset:] + # self.__data_bytes = data + # return data + + def info(self): + """ + Provides a dictionary with information about a blob. Useful for + calls to a plugin's write() function, e.g. self.write(\\*\\*blob.info()) + + Returns: + Dictionary with information + """ + d = {k: v for k, v in self.__dict__.items() if not k.startswith('_')} + del d['hidden'] + del d['packets'] + return d + + + # TODO: Trying to determine if we should do this or take into account acknowledgement numbers + # like originally implemented. + # Perhaps rewrite their assemble to do some work in add_packet()? + # Do we want to ensure all segments are acknowledged or should we avoid that so we can handle + # partial/corrupt pcaps? + def add_packet(self, packet): + """ + Accepts a Packet object and stores it. + + Args: + packet: a Packet object + """ + # Clear old data and segment cache. + self._data = None + self._segments = None + + seq = packet.sequence_number + + # If packet is not TCP just add packet to list. + if seq is None: + self.packets.append(packet) + if packet.ts < self.starttime: self.starttime = packet.ts + if packet.ts > self.endtime: self.endtime = packet.ts + return + + # If this a new sequence number we haven't seen before, add it to the map. + if seq not in self._seq_map: + self._seq_map[seq] = packet + if seq < self.seq_min: self.seq_min = seq + if seq > self.seq_max: self.seq_max = seq + if packet.ts < self.starttime: self.starttime = packet.ts + if packet.ts > self.endtime: self.endtime = packet.ts + self.packets.append(packet) + return + + # Otherwise, if we already have the packet for the given sequence + # then we have a retransmission and will need to determine which packet to keep + # and possibly remove other packets if this packet overlaps them. + orig_packet = self._seq_map[seq] + + # ignore duplicate packet. +# if len(packet.data) <= len(orig_packet.data): + if packet.data == orig_packet.data: + # TODO: should we still handle duplicate packets. + logger.debug(f'Ignoring duplicate packet: {packet.frame}') + return + + # If this packet would create more inconsistencies in our sequence numbers (more holes) + # than the packet to be replaced, then this is most likely an out-of-order packet that the + # sender has ignored, and we should too. + orig_next_seq = seq + len(orig_packet.data) + next_seq = seq + len(packet.data) + if ( +# next_seq < max(self.sequence_numbers) + next_seq < self.seq_max + and orig_packet.data + and next_seq not in self._seq_map + and orig_next_seq in self._seq_map + ): + logger.debug(f'Ignoring out-of-order packet: {packet.frame}') + return + + # Replace packet(s) with retransmitted packet + + # First add the retransmitted packet, replacing the original packet matching the + # sequence number. + logger.debug(f'Replacing packet {orig_packet.frame} with {packet.frame}') + self._seq_map[seq] = packet + self.packets = [packet if p.sequence_number == seq else p for p in self.packets] + if packet.ts < self.starttime: self.starttime = packet.ts + if packet.ts > self.endtime: self.endtime = packet.ts + + # Now remove any packets that contained data that is now part of the retransmitted packet. + packets_to_remove = [] + for seq_, packet_ in self._seq_map.items(): + if 0 < (seq_ - seq) < len(packet.data): + logger.debug(f'Removing packet: {packet_.frame}') + packets_to_remove.append(packet_) + # NOTE: need to remove packets outside the above loop because removing packets affect seq_map + for packet_ in packets_to_remove: + self._remove_packet(packet_) + + def _remove_packet(self, packet): + """ + Removes packet from Blob. (internal use only) + """ + # Clear old data and segment cache. + self._data = None + self._segments = None + + for seq, packet_ in list(self._seq_map.items()): + if packet_ == packet: + del self._seq_map[seq] + if seq == self.seq_max: + self.seq_max = max(self._seq_map.keys()) + + self.packets.remove(packet) diff --git a/dshell/data/GeoIP/readme.txt b/dshell/data/GeoIP/readme.txt new file mode 100644 index 0000000..eb5f048 --- /dev/null +++ b/dshell/data/GeoIP/readme.txt @@ -0,0 +1 @@ +GeoIP data sets go here. diff --git a/dshell/data/dshellrc b/dshell/data/dshellrc new file mode 100644 index 0000000..f428e2e --- /dev/null +++ b/dshell/data/dshellrc @@ -0,0 +1,2 @@ +export PS1="`whoami`@`hostname`:\w Dshell> " +alias decode="python3 -m dshell.decode " diff --git a/dshell/data/empty.pcap b/dshell/data/empty.pcap new file mode 100644 index 0000000..a324304 Binary files /dev/null and b/dshell/data/empty.pcap differ diff --git a/dshell/decode.py b/dshell/decode.py new file mode 100755 index 0000000..8b37be1 --- /dev/null +++ b/dshell/decode.py @@ -0,0 +1,766 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +This is the core script for running plugins. + +It works by grabbing individual packets from a file or interface and feeding +them into a chain of plugins (plugin_chain). Each plugin in the chain +decides if the packet will continue on to the next plugin or just fade away. + +In practice, users generally only use one plugin, so the "chain" will only +have one plugin, which is perfectly fine. The chain exists to allow plugins +to alter or filter packets before passing them to more general plugins. For +example, --plugin=country+netflow would pass packets through the country +plugin, and then the netflow plugin. This would allow filtering traffic by +country code before viewing flow data. + +Many things go into making this chain run smoothly, however. This includes +reading in user arguments, setting filters, opening files/interfaces, etc. All +of this largely takes place in the main() function. +""" + +# standard Python library imports +import bz2 +import faulthandler +import gzip +import multiprocessing +import logging +import operator +import os +import queue +import sys +import tempfile +import zipfile +from collections import OrderedDict +from datetime import timedelta +from getpass import getpass +from glob import glob +from importlib import import_module +from typing import Iterable + +import pcapy +from pypacker.layer12 import ethernet, ppp, pppoe, ieee80211, linuxcc, radiotap, can +from pypacker.layer3 import ip, ip6 + +import dshell.core +from dshell.api import get_plugin_information +from dshell.core import Packet +from dshell.dshelllist import get_plugins, get_output_modules +from dshell.dshellargparse import DshellArgumentParser +from dshell.output.output import QueueOutputWrapper +from dshell.util import get_output_path +from tabulate import tabulate + +logger = logging.getLogger(__name__) + + +# plugin_chain will eventually hold the user-selected plugins that packets +# will trickle through. +plugin_chain = [] + + +def feed_plugin_chain(plugin_index: int, packet: Packet): + """ + Every packet fed into Dshell goes through this function. + Its goal is to pass each packet down the chain of selected plugins. + Each plugin decides whether the packet(s) will proceed to the next + plugin, i.e. act as a filter. + """ + if plugin_index >= len(plugin_chain): + # We are at the end of the chain. + return + + current_plugin = plugin_chain[plugin_index] + + # Pass packet into plugin for processing. + current_plugin.consume_packet(packet) + + # Process produced packets. + for _packet in current_plugin.produce_packets(): + feed_plugin_chain(plugin_index + 1, _packet) + + +def clean_plugin_chain(plugin_index): + """ + This is called at the end of packet capture. + It will go through the plugins and attempt to cleanup any connections + that were not yet closed. + """ + if plugin_index >= len(plugin_chain): + # We are at the end of the chain + return + + current_plugin = plugin_chain[plugin_index] + + # need to flush even if there are no more plugins in the chain to ensure all packets are processed. + current_plugin.flush() + + # Feed plugin chain with lingering packets released by flush. + for _packet in current_plugin.produce_packets(): + feed_plugin_chain(plugin_index + 1, _packet) + + clean_plugin_chain(plugin_index + 1) + + +def decompress_file(filepath, extension, unzipdir): + """ + Attempts to decompress a provided file and write the data to a temporary + file. The list of created temporary files is returned. + """ + filename = os.path.split(filepath)[-1] + openfiles = [] + logger.debug("Attempting to decompress {!r}".format(filepath)) + if extension == '.gz': + f = gzip.open(filepath, 'rb') + openfiles.append(f) + elif extension == '.bz2': + f = bz2.open(filepath, 'rb') + openfiles.append(f) + elif extension == '.zip': + pswd = getpass("Enter password for .zip file {!r} [default: none]: ".format(filepath)) + pswd = pswd.encode() # TODO I'm not sure encoding to utf-8 will work in all cases + try: + z = zipfile.ZipFile(filepath) + for z2 in z.namelist(): + f = z.open(z2, 'r', pswd) + openfiles.append(f) + except (RuntimeError, zipfile.BadZipFile) as e: + logger.error("Could not process .zip file {!r}. {!s}".format(filepath, e)) + return [] + + tempfiles = [] + for openfile in openfiles: + with openfile: + try: + # check if this file is actually something decompressable + openfile.peek(1) + except OSError as e: + logger.error("Could not process compressed file {!r}. {!s}".format(filepath, e)) + continue + with tempfile.NamedTemporaryFile(dir=unzipdir, delete=False, prefix=filename) as tfile: + for piece in openfile: + tfile.write(piece) + tempfiles.append(tfile.name) + return tempfiles + + +def print_plugins(plugins): + """ + Print list of plugins with additional info. + """ + headers = ['module', 'name', 'title', 'type', 'author', 'description'] + rows = [] + for name, module in sorted(plugins.items()): + rows.append([ + module.__module__, + name, + module.name, + module.__class__.__bases__[0].__name__, + module.author, + module.description, + ]) + + print(tabulate(rows, headers=headers)) + + +def main(plugin_args=None, **kwargs): + global plugin_chain + + if not plugin_args: + plugin_args = {} + + # dictionary of all available plugins: {name: module path} + plugin_map = get_plugins() + + # Attempt to catch segfaults caused when certain linktypes (e.g. 204) are + # given to pcapy + faulthandler.enable() + + if not plugin_chain: + logger.error("No plugin selected") + sys.exit(1) + + plugin_chain[0].defrag_ip = kwargs.get("defrag", False) + + # Setup logging + log_format = "%(levelname)s (%(name)s) - %(message)s" + if kwargs.get("verbose", False): + log_level = logging.INFO + elif kwargs.get("debug", False): + log_level = logging.DEBUG + elif kwargs.get("quiet", False): + log_level = logging.CRITICAL + else: + log_level = logging.WARNING + logging.basicConfig(format=log_format, level=log_level) + + # since pypacker handles its own exceptions (loudly), this attempts to keep + # it quiet + logging.getLogger("pypacker").setLevel(logging.CRITICAL) + + if kwargs.get("allcc", False): + # Activate all country code (allcc) mode to display all 3 GeoIP2 country + # codes + dshell.core.geoip.acc = True + + dshell.core.geoip.check_file_dates() + + # If alternate output module is selected, tell each plugin to use that + # instead + if kwargs.get("omodule", None): + try: + # TODO: Create a factory classmethod in the base Output class (e.g. "from_name()") instead. + omodule = import_module("dshell.output."+kwargs["omodule"]) + omodule = omodule.obj + for plugin in plugin_chain: + # TODO: Should we have a single instance of the Output module used by all plugins? + oomodule = omodule() + plugin.out = oomodule + except ImportError as e: + logger.error("Could not import module named '{}'. Use --list-output flag to see available modules".format(kwargs["omodule"])) + sys.exit(1) + + # Check if any user-defined output arguments are provided + if kwargs.get("oargs", None): + oargs = {} + for oarg in kwargs["oargs"]: + if '=' in oarg: + key, val = oarg.split('=', 1) + oargs[key] = val + else: + oargs[oarg] = True + logger.debug("oargs: %s" % oargs) + for plugin in plugin_chain: + plugin.out.set_oargs(**oargs) + + for plugin in plugin_chain: + # If writing to a file, set for each output module here + if kwargs.get("outfile", None): + plugin.out.reset_fh(filename=kwargs["outfile"]) + + # Set nobuffer mode if that's what the user wants + if kwargs.get("nobuffer", False): + plugin.out.nobuffer = True + + # Set color blind friendly mode + if kwargs.get("cbf", False): + plugin.out.cbf = True + + # Set the extra flag for all output modules + if kwargs.get("extra", False): + plugin.out.extra = True + plugin.out.set_format(plugin.out.format) + + # Set some attributes for ConnectionPlugins + if hasattr(plugin, "timeout"): + # Set wait time since last packet arrived in a connection before + # considering connection closed + if t := kwargs.get("conntimeout"): + td = timedelta(seconds=int(t)) + plugin.timeout = td + # Set max number of allowed open connections + if t := kwargs.get("connmax"): + plugin.max_open_connections = int(t) + + # Set the BPF filters + # Each plugin has its own default BPF that will be extended or replaced + # based on --no-vlan, --ebpf, or --bpf arguments. + if kwargs.get("bpf", None): + plugin.bpf = kwargs.get("bpf", "") + continue + if plugin.bpf: + if kwargs.get("ebpf", None): + plugin.bpf = "({}) and ({})".format(plugin.bpf, kwargs.get("ebpf", "")) + else: + if kwargs.get("ebpf", None): + plugin.bpf = kwargs.get("ebpf", "") + if kwargs.get("novlan", False): + plugin.vlan_bpf = False + + # Decide on the inputs to use for pcap + # If --interface is set, ignore all files and listen live on the wire + # Otherwise, use all of the files and globs to open offline pcap. + # Recurse through any directories if the command-line flag is set. + if kwargs.get("interface", None): + inputs = [kwargs.get("interface")] + else: + inputs = [] + inglobs = kwargs.get("files", []) + infiles = [] + for inglob in inglobs: + outglob = glob(inglob) + if not outglob: + logger.warning("Could not find file(s) matching {!r}".format(inglob)) + continue + infiles.extend(outglob) + while len(infiles) > 0: + infile = infiles.pop(0) + if kwargs.get("recursive", False) and os.path.isdir(infile): + morefiles = os.listdir(infile) + for morefile in morefiles: + infiles.append(os.path.join(infile, morefile)) + elif os.path.isfile(infile): + inputs.append(infile) + + # Process plugin-specific options + for plugin in plugin_chain: + for option, args in plugin.optiondict.items(): + if option in plugin_args.get(plugin, {}): + setattr(plugin, option, plugin_args[plugin][option]) + else: + setattr(plugin, option, args.get("default", None)) + plugin.handle_plugin_options() + + + #### Dshell is ready to read pcap! #### + for plugin in plugin_chain: + plugin._premodule() + + # If we are not multiprocessing, simply pass the files for processing + if not kwargs.get("multiprocessing", False): + process_files(inputs, **kwargs) + # If we are multiprocessing, things get more complicated. + else: + # Create an output queue, and wrap the 'write' function of each + # plugins's output module to send calls to the multiprocessing queue + output_queue = multiprocessing.Queue() + output_wrappers = {} + for plugin in plugin_chain: + qo = QueueOutputWrapper(plugin.out, output_queue) + output_wrappers[qo.id] = qo + plugin.out.write = qo.write + + # Create processes to handle each separate input file + processes = [] + for i in inputs: + processes.append( + multiprocessing.Process(target=process_files, args=([i],), kwargs=kwargs) + ) + + # Spawn processes, and keep track of which ones are running + running = [] + max_writes_per_batch = 50 + while processes or running: + if processes and len(running) < kwargs.get("process_max", 4): + # Start a process and move it to the 'running' list + proc = processes.pop(0) + proc.start() + logger.debug("Started process {}".format(proc.pid)) + running.append(proc) + for proc in running: + if not proc.is_alive(): + # Remove finished processes from 'running' list + logger.debug("Ended process {} (exit code: {})".format(proc.pid, proc.exitcode)) + running.remove(proc) + try: + # Process write commands in the output queue. + # Since some plugins write copiously and may block other + # processes from launching, only write up to a maximum number + # before breaking and rechecking the processes. + writes = 0 + while writes < max_writes_per_batch: + wrapper_id, args, kwargs = output_queue.get(True, 1) + owrapper = output_wrappers[wrapper_id] + owrapper.true_write(*args, **kwargs) + writes += 1 + except queue.Empty: + pass + + output_queue.close() + + for plugin in plugin_chain: + plugin._postmodule() + + +# Maps datalink type reported by pcapy to a pypacker packet class. +datalink_map = { + 1: ethernet.Ethernet, + 9: ppp.PPP, + 51: pppoe.PPPoE, + 105: ieee80211.IEEE80211, + 113: linuxcc.LinuxCC, + 127: radiotap.Radiotap, + 204: ppp.PPP, + 227: can.CAN, + 228: ip.IP, + 229: ip6.IP6, +} + + +def read_packets(input: str, interface=False, bpf=None, count=None) -> Iterable[dshell.Packet]: + """ + Yields packets from input pcap file or device. + + :param str input: device or pcap file path + :param bool interface: Whether input is a device. + :param str bpf: Optional bpf filter. + :param int count: Optional max count of packets to read before exiting. + + :yields: packets defined by pypacker. + NOTE: Timestamp and frame id are added to packet for convenience. + """ + + if interface: + # Listen on an interface if the option is set + try: + capture = pcapy.open_live(input, 65536, True, 1) + except pcapy.PcapError as e: + # User probably doesn't have permission to listen on interface + # In any case, print just the error without traceback + logger.error(str(e)) + return + else: + # Otherwise, read from pcap file(s) + try: + capture = pcapy.open_offline(input) + except pcapy.PcapError as e: + logger.error("Could not open '{}': {!s}".format(input, e)) + return + + # TODO: We may want to allow all packets to go through and then allow the plugin to filter + # them out in feed_plugin_chain(). + # That way our frame_id won't be out of sync from skipped packets. + # Try and use the first plugin's BPF as the initial filter + # The BPFs for other plugins will be applied along the chain as needed + try: + if bpf: + capture.setfilter(bpf) + except pcapy.PcapError as e: + if str(e).startswith("no VLAN support for data link type"): + logger.error("Cannot use VLAN filters for {!r}. Recommend running with --no-vlan argument.".format(input)) + return + elif "syntax error" in str(e) or "link layer applied in wrong context" == str(e): + logger.error("Could not compile BPF: {!s} ({!r})".format(e, bpf)) + return + elif "802.11 link-layer types supported only on 802.11" == str(e): + logger.error("BPF incompatible with pcap file: {!s}".format(e)) + return + else: + raise e + + # Set the datalink layer for each plugin, based on the pcapy capture. + # Also compile a pcapy BPF object for each. + datalink = capture.datalink() + for plugin in plugin_chain: + # TODO Find way around libpcap bug that segfaults when certain BPFs + # are used with certain datalink types + # (e.g. datalink=204, bpf="ip") + plugin.link_layer_type = datalink + plugin.recompile_bpf() + + # Get correct pypacker class based on datalink layer. + packet_class = datalink_map.get(datalink, ethernet.Ethernet) + + logger.info(f"Datalink: {datalink} - {packet_class.__name__}") + + # Iterate over the file/interface and yield Packet objects. + frame = 1 # Start with 1 because Wireshark starts with 1. + while True: + try: + header, packet_data = capture.next() + if header is None and not packet_data: + if not interface: + # probably the end of the capture + break + else: + # interface timed out + continue + if count and frame - 1 >= count: + # we've reached the maximum number of packets to process + break + + # Add timestamp and frame id to packet object for convenience. + pktlen = header.getlen() + s, us = header.getts() + ts = s + us / 1000000.0 + + # Wrap packet in dshell's Packet class. + packet = dshell.Packet(pktlen, packet_class(packet_data), ts, frame=frame) + frame += 1 + + yield packet + + # handle SIGINT gracefully, break read loop and allow shutdown + except KeyboardInterrupt: + logger.debug("Caught KeyboardInterrupt or SIGINT. Closing capture.") + break + + except pcapy.PcapError as e: + estr = str(e) + eformat = "Error processing '{i}' - {e}" + if estr.startswith("truncated dump file"): + logger.error(eformat.format(i=input, e=estr)) + logger.debug(e, exc_info=True) + elif estr.startswith("bogus savefile header"): + logger.error(eformat.format(i=input, e=estr)) + logger.debug(e, exc_info=True) + else: + raise + break + + +# TODO: The use of kwargs makes it difficult to understand what arguments the function accept +# and difficult to follow the code flow. +def process_files(inputs, **kwargs): + # Iterate over each of the input files + # For live capture, the "input" would just be the name of the interface + global plugin_chain + interface = kwargs.get("interface", False) + count = kwargs.get("count", None) + # Try and use the first plugin's BPF as the initial filter + # The BPFs for other plugins will be applied along the chain as needed + bpf = plugin_chain[0].bpf + + while len(inputs) > 0: + input0 = inputs.pop(0) + + # Check if file needs to be decompressed by its file extension + extension = os.path.splitext(input0)[-1] + if extension in (".gz", ".bz2", ".zip") and "interface" not in kwargs: + tempfiles = decompress_file(input0, extension, kwargs.get("unzipdir", tempfile.gettempdir())) + inputs = tempfiles + inputs + continue + + for plugin in plugin_chain: + plugin._prefile(input0) + + for packet in read_packets(input0, interface=interface, bpf=bpf, count=count): + feed_plugin_chain(0, packet) + + clean_plugin_chain(0) + for plugin in plugin_chain: + plugin.purge() + plugin._postfile() + + +# TODO: Separate some of this logic outside of this function so we can call +# dshell as a library. +def main_command_line(): + # Since plugin_chain contains the actual plugin instances we have to make sure + # we reset the global plugin_chain so multiple runs don't affect each other. + # (This was necessary to call this function through a python script.) + # TODO: Should plugin_chain be a list of plugin classes instead of instances? + global plugin_chain + plugin_chain = [] + + # dictionary of all available plugins: {name: module path} + plugin_map = get_plugins() + # dictionary of plugins that the user wants to use: {name: object} + active_plugins = OrderedDict() + + # The main argument parser. It will have every command line option + # available and should be used when actually parsing + parser = DshellArgumentParser( + usage="%(prog)s [options] [plugin options] file1 file2 ... fileN", + add_help=False) + parser.add_argument('-c', '--count', type=int, default=0, + help='Number of packets to process') + parser.add_argument('--debug', action="store_true", + help="Show debug messages") + parser.add_argument('-v', '--verbose', action="store_true", + help="Show informational messages") + parser.add_argument('--acc', '--allcc', action="store_true", + help="Show all 3 GeoIP2 country code types (represented_country/registered_country/country)") + parser.add_argument('-d', '-p', '--plugin', dest='plugin', type=str, + action='append', metavar="PLUGIN", + help="Use a specific plugin module. Can be chained with '+'.") + parser.add_argument('--defragment', dest='defrag', action='store_true', + help='Reconnect fragmented IP packets') + parser.add_argument('-h', '-?', '--help', dest='help', + help="Print common command-line flags and exit", action='store_true', + default=False) + parser.add_argument('-i', '--interface', default=None, type=str, + help="Listen live on INTERFACE instead of reading pcap") + parser.add_argument('-l', '--ls', '--list', action="store_true", + help='List all available plugins', dest='list') + parser.add_argument('-r', '--recursive', dest='recursive', action='store_true', + help='Recursively process all PCAP files under input directory') + parser.add_argument('--unzipdir', type=str, metavar="DIRECTORY", + default=tempfile.gettempdir(), + help='Directory to use when decompressing input files (.gz, .bz2, and .zip only)') + parser.add_argument('--conn-timeout', dest="conntimeout", type=int, + metavar="SECONDS", default=3600, + help="Number of seconds to wait after last packet in a connection before closing it (default: 3600)") + parser.add_argument('--conn-max-open', dest='connmax', type=int, + metavar="NUMBER", default=1000, + help="Number of connections to hold in an open state before Dshell begins closing the oldest (default: 1000)") + + multiprocess_group = parser.add_argument_group("multiprocessing arguments") + multiprocess_group.add_argument('-P', '--parallel', dest='multiprocessing', action='store_true', + help='Handle each file in separate parallel processes') + multiprocess_group.add_argument('-n', '--nprocs', type=int, default=4, + metavar='NUMPROCS', dest='process_max', + help='Define max number of parallel processes (default: 4)') + + filter_group = parser.add_argument_group("filter arguments") + filter_group.add_argument('--bpf', default='', type=str, + help="Overwrite all BPFs and use provided input. Use carefully!") + filter_group.add_argument('--ebpf', default='', type=str, metavar="BPF", + help="Extend existing BPFs with provided input for additional filtering. It will transform input into \"() and ()\"") + filter_group.add_argument("--no-vlan", action="store_true", dest="novlan", + help="Ignore packets with VLAN headers") + + output_group = parser.add_argument_group("output arguments") + output_group.add_argument("--lo", "--list-output", action="store_true", + help="List available output modules", + dest="listoutput") + output_group.add_argument("--no-buffer", action="store_true", + help="Do not buffer plugin output", + dest="nobuffer") + output_group.add_argument("--cbf", "--color-blind-friendly", action="store_true", + help="Activate color blind friendly mode, colorout and htmlout output modules use yellow/gold in place of red and different shades of green/yellow/blue are used to help better differentiate between them", + dest="cbf") + output_group.add_argument("-x", "--extra", action="store_true", + help="Appends extra data to all plugin output.") + # TODO Figure out how to make --extra flag play nicely with user-only + # output modules, like jsonout and csvout + output_group.add_argument("-O", "--omodule", type=str, dest="omodule", + metavar="MODULE", + help="Use specified output module for plugins instead of defaults. For example, --omodule=jsonout for JSON output.") + output_group.add_argument("--oarg", type=str, metavar="ARG=VALUE", + dest="oargs", action="append", + help="Supply a specific keyword argument to plugins' output modules. Can be used multiple times for multiple arguments. Not using an equal sign will treat it as a flag and set the value to True. Example: --oarg \"delimiter=:\" --oarg \"timeformat=%%H %%M %%S\"") + output_group.add_argument("-q", "--quiet", action="store_true", + help="Disable logging") + output_group.add_argument("-W", metavar="OUTFILE", dest="outfile", + help="Write to OUTFILE instead of stdout") + + parser.add_argument('files', nargs='*', + help="pcap files or globs to process") + + # A short argument parser, meant to only hold the simplified list of + # arguments for when a plugin is called without a pcap file. + # DO NOT USE for any serious argument parsing. + parser_short = DshellArgumentParser( + usage="%(prog)s [options] [plugin options] file1 file2 ... fileN", + add_help=False) + parser_short.add_argument('-h', '-?', '--help', dest='help', + help="Print common command-line flags and exit", action='store_true', + default=False) + parser.add_argument('--version', action='version', + version="Dshell " + str(dshell.core.__version__)) + parser_short.add_argument('-d', '-p', '--plugin', dest='plugin', type=str, + action='append', metavar="PLUGIN", + help="Use a specific plugin module") + parser_short.add_argument('--ebpf', default='', type=str, metavar="BPF", + help="Extend existing BPFs with provided input for additional filtering. It will transform input into \"() and ()\"") + parser_short.add_argument('-i', '--interface', + help="Listen live on INTERFACE instead of reading pcap") + parser_short.add_argument('-l', '--ls', '--list', action="store_true", + help='List all available plugins', dest='list') + parser_short.add_argument("--lo", "--list-output", action="store_true", + help="List available output modules") + parser_short.add_argument("--cbf", "--color-blind-friendly", action="store_true", + help="Activate color blind friendly mode, colorout and htmlout output modules use yellow/gold in place of red and different shades of green/yellow/blue are used to help better differentiate between them") + # FIXME: Should this duplicate option be removed? + parser_short.add_argument("-o", "--omodule", type=str, metavar="MODULE", + help="Use specified output module for plugins instead of defaults. For example, --omodule=jsonout for JSON output.") + parser_short.add_argument('files', nargs='*', + help="pcap files or globs to process") + + # Start parsing the arguments + # Specifically, we want to grab the desired plugin list + # This will let us add the plugin-specific arguments and reprocess the args + opts, xopts = parser.parse_known_args() + if opts.plugin: + # Multiple plugins can be chained using either multiple instances + # of -d/-p/--plugin or joining them together with + signs. + plugins = '+'.join(opts.plugin) + plugins = plugins.split('+') + # check for invalid plugins + for plugin in plugins: + plugin = plugin.strip() + if not plugin: + # User probably mistyped '++' instead of '+' somewhere. + # Be nice and ignore this minor infraction. + continue + if plugin not in plugin_map: + parser_short.epilog = "ERROR! Invalid plugin provided: '{}'".format(plugin) + parser_short.print_help() + sys.exit(1) + # While we're at it, go ahead and import the plugin modules now + # This can probably be done further down the line, but here is + # just convenient + plugin_module = import_module(plugin_map[plugin]) + # Handle multiple instances of same plugin by appending number to + # end of plugin name. This is used mostly to separate + # plugin-specific arguments from each other + if plugin in active_plugins: + i = 1 + plugin = plugin + str(i) + while plugin in active_plugins: + i += 1 + plugin = plugin[:-(len(str(i-1)))] + str(i) + # Add copy of plugin object to chain and add to argument parsers + # TODO: Use class attributes for class related things like name, description, optionsdict + # This way we don't have to initialize the plugin at this point and fixes a lot of the + # issues that arise that come from dealing with a singleton. + active_plugins[plugin] = plugin_module.DshellPlugin() + plugin_chain.append(active_plugins[plugin]) + parser.add_plugin_arguments(plugin, active_plugins[plugin]) + parser_short.add_plugin_arguments(plugin, active_plugins[plugin]) + opts, xopts = parser.parse_known_args() + + if xopts: + for xopt in xopts: + logger.warning('Could not understand argument {!r}'.format(xopt)) + + if opts.help: + # Just print the full help message and exit + parser.print_help() + print("\n") + for plugin in plugin_chain: + print("############### {}".format(plugin.name)) + print(plugin.longdescription) + print("\n") + print('Default BPF: "{}"'.format(plugin.bpf)) + print("\n") + sys.exit() + + if opts.list: + try: + print_plugins(get_plugin_information()) + except ImportError as e: + logger.error(e, exc_info=opts.debug) + sys.exit() + + if opts.listoutput: + # List available output modules and a brief description + output_map = get_output_modules(get_output_path()) + for modulename in sorted(output_map): + try: + module = import_module("dshell.output."+modulename) + module = module.obj + except Exception as e: + etype = e.__class__.__name__ + logger.debug("Could not load {} module. ({}: {!s})".format(modulename, etype, e)) + else: + print("\t{:<25} {}".format(modulename, module._DESCRIPTION)) + sys.exit() + + if not opts.plugin: + # If a plugin isn't provided, print the short help message + parser_short.epilog = "Select a plugin to use with -d or --plugin" + parser_short.print_help() + sys.exit() + + if not opts.files and not opts.interface: + # If no files are provided, print the short help message + parser_short.epilog = "Include a pcap file to get started. Use --help for more information." + parser_short.print_help() + sys.exit() + + # Process the plugin-specific args and set the attributes within them + plugin_args = {} + for plugin_name, plugin in active_plugins.items(): + plugin_args[plugin] = {} + args_and_attrs = parser.get_plugin_arguments(plugin_name, plugin) + for darg, dattr in args_and_attrs: + value = getattr(opts, darg) + plugin_args[plugin][dattr] = value + + main(plugin_args=plugin_args, **vars(opts)) + + +if __name__ == "__main__": + main_command_line() diff --git a/dshell/dshellargparse.py b/dshell/dshellargparse.py new file mode 100644 index 0000000..afcc43c --- /dev/null +++ b/dshell/dshellargparse.py @@ -0,0 +1,63 @@ +""" +This argument parser is almost identical to the Python standard argparse. +This one adds a function to automatically add plugin-specific arguments. +""" + +import argparse + + +def custom_bytes(value): + """ + Converts value strings for command lines that are suppose to be bytes. + If value startswith "0x", value will be assumed to be a hex string. + Otherwise data will be encoded with utf8 + """ + if isinstance(value, bytes): + return value + if value.startswith("0x"): + try: + return bytes.fromhex(value[2:]) + except ValueError: + pass # Wasn't hex after all, just treat as a utf8 string. + return value.encode("utf8") + + +class DshellArgumentParser(argparse.ArgumentParser): + + def add_plugin_arguments(self, plugin_name, plugin_obj): + """ + add_plugin_arguments(self, plugin_name, plugin_obj) + + Give it the name of the plugin and an instance of the plugin, and + it will automatically create argument entries. + """ + if plugin_obj.optiondict: + group = '{} plugin options'.format(plugin_obj.name) + group = self.add_argument_group(group) + for argname, optargs in plugin_obj.optiondict.items(): + optname = "{}_{}".format(plugin_name, argname) + data_type = optargs.get("type", None) + if data_type and data_type == bytes: + optargs["type"] = custom_bytes + default = optargs.get("default", None) + if default is not None: + optargs["default"] = custom_bytes(default) + group.add_argument("--" + optname, dest=optname, **optargs) + + def get_plugin_arguments(self, plugin_name, plugin_obj): + """ + get_plugin_arguments(self, plugin_name, plugin_obj) + + Returns a list of argument names and the attributes they're associated + with. + + e.g. --country_code for the "country" plugin ties to the "code" attr + in the plugin object. Thus, the return would be + [("country_code", "code"), ...] + """ + args_and_attrs = [] + if plugin_obj.optiondict: + for argname in plugin_obj.optiondict.keys(): + optname = "{}_{}".format(plugin_name, argname) + args_and_attrs.append((optname, argname)) + return args_and_attrs diff --git a/dshell/dshellgeoip.py b/dshell/dshellgeoip.py new file mode 100644 index 0000000..0a61414 --- /dev/null +++ b/dshell/dshellgeoip.py @@ -0,0 +1,157 @@ +# -*- coding: utf-8 -*- +""" +A wrapper around GeoIP2 that provides convenience functions for querying and +collecting GeoIP data +""" + +import datetime +import logging +import os +from collections import OrderedDict + +import geoip2.database +import geoip2.errors + +from dshell.util import get_data_path + + +logger = logging.getLogger(__name__) + + +class DshellGeoIP(object): + MAX_CACHE_SIZE = 5000 + + def __init__(self, acc=False): + self.geodir = os.path.join(get_data_path(), 'GeoIP') + self.geoccfile = os.path.join(self.geodir, 'GeoLite2-City.mmdb') + self.geoasnfile = os.path.join(self.geodir, 'GeoLite2-ASN.mmdb') + self.geoccdb = geoip2.database.Reader(self.geoccfile) + self.geoasndb = geoip2.database.Reader(self.geoasnfile) + self.geo_asn_cache = DshellGeoIPCache(max_cache_size=self.MAX_CACHE_SIZE) + self.geo_loc_cache = DshellGeoIPCache(max_cache_size=self.MAX_CACHE_SIZE) + self.acc = acc + + def check_file_dates(self): + """ + Check the data file age, and log a warning if it's over a year old. + """ + cc_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(self.geoccfile)) + asn_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(self.geoasnfile)) + n = datetime.datetime.now() + year = datetime.timedelta(days=365) + if (n - cc_mtime) > year or (n - asn_mtime) > year: + logger.debug("GeoIP data file(s) over a year old, and possibly outdated.") + + def geoip_country_lookup(self, ip): + """ + Looks up the IP and returns the two-character country code. + """ + location = self.geoip_location_lookup(ip) + return location[0] + + def geoip_asn_lookup(self, ip): + """ + Looks up the IP and returns an ASN string. + Example: + print geoip_asn_lookup("74.125.26.103") + "AS15169 Google LLC" + """ + try: + return self.geo_asn_cache[ip] + except KeyError: + try: + template = "AS{0.autonomous_system_number} {0.autonomous_system_organization}" + asn = template.format(self.geoasndb.asn(ip)) + self.geo_asn_cache[ip] = asn + return asn + except geoip2.errors.AddressNotFoundError: + return None + + def geoip_location_lookup(self, ip): + """ + Looks up the IP and returns a tuple containing country code, latitude, + and longitude. + """ + try: + return self.geo_loc_cache[ip] + except KeyError: + try: + location = self.geoccdb.city(ip) + # Get country code based on order of importance + # 1st: Country that owns an IP address registered in another + # location (e.g. military bases in foreign countries) + # 2nd: Country in which the IP address is registered + # 3rd: Physical country where IP address is located + # https://dev.maxmind.com/geoip/geoip2/whats-new-in-geoip2/#Country_Registered_Country_and_Represented_Country + # Handle flag from plugin optional args to enable all 3 country codes + if self.acc: + try: + cc = "{}/{}/{}".format(location.represented_country.iso_code, + location.registered_country.iso_code, + location.country.iso_code) + cc = cc.replace("None", "--") + + except KeyError: + pass + else: + cc = (location.represented_country.iso_code or + location.registered_country.iso_code or + location.country.iso_code or + '--') + + location = ( + cc, + location.location.latitude, + location.location.longitude + ) + self.geo_loc_cache[ip] = location + return location + except geoip2.errors.AddressNotFoundError: + # Handle flag from plugin optional args to enable all 3 country codes + if self.acc: + location = ("--/--/--", None, None) + else: + location = ("--", None, None) + self.geo_loc_cache[ip] = location + return location + + +class DshellFailedGeoIP(object): + """ + Class used in place of DshellGeoIP if GeoIP database files are not found. + """ + + def __init__(self): + self.geodir = os.path.join(get_data_path(), 'GeoIP') + self.geoccdb = None + self.geoasndb = None + + def check_file_dates(self): + pass + + def geoip_country_lookup(self, ip): + return "??" + + def geoip_asn_lookup(self, ip): + return None + + def geoip_location_lookup(self, ip): + return ("??", None, None) + + +class DshellGeoIPCache(OrderedDict): + """ + A cache for storing recent IP lookups to improve performance. + """ + + def __init__(self, *args, **kwargs): + self.max_cache_size = kwargs.pop("max_cache_size", 500) + OrderedDict.__init__(self, *args, **kwargs) + + def __setitem__(self, key, value): + OrderedDict.__setitem__(self, key, value) + self.check_max_size() + + def check_max_size(self): + while len(self) > self.max_cache_size: + self.popitem(last=False) diff --git a/dshell/dshelllist.py b/dshell/dshelllist.py new file mode 100644 index 0000000..4841311 --- /dev/null +++ b/dshell/dshelllist.py @@ -0,0 +1,60 @@ +""" +A library containing functions for generating lists of important modules. +These are mostly used in decode.py and in unit tests +""" + +import logging +import os +import pkg_resources +from glob import iglob + +from dshell.util import get_plugin_path + + +logger = logging.getLogger(__name__) + + +def get_plugins(): + """ + Generate a list of all available plugin modules, either in the + dshell.plugins directory or external packages + """ + plugins = {} + # List of directories above the plugins directory that we don't care about + import_base = get_plugin_path().split(os.path.sep)[:-1] + + # Walk through the plugin path and find any Python modules that aren't + # __init__.py. These are assumed to be plugin modules and will be + # treated as such. + for root, dirs, files in os.walk(get_plugin_path()): + if '__init__.py' in files: + import_path = root.split(os.path.sep)[len(import_base):] + for f in iglob("{}/*.py".format(root)): + name = os.path.splitext(os.path.basename(f))[0] + if name != '__init__': + if name in plugins and logger: + logger.warning("Duplicate plugin name found: {}".format(name)) + module = '.'.join(["dshell"] + import_path + [name]) + plugins[name] = module + + # Next, try to discover additional plugins installed externally. + # Uses entry points in setup.py files. + for ep_plugin in pkg_resources.iter_entry_points("dshell_plugins"): + if ep_plugin.name in plugins: + logger.warning("Duplicate plugin name found: {}".format(ep_plugin.name)) + plugins[ep_plugin.name] = ep_plugin.module_name + + return plugins + + +def get_output_modules(output_module_path): + """ + Generate a list of all available output modules under an output_module_path + """ + modules = [] + for f in iglob("{}/*.py".format(output_module_path)): + name = os.path.splitext(os.path.basename(f))[0] + if name != '__init__' and name != 'output': + # Ignore __init__ and the base output.py module + modules.append(name) + return modules diff --git a/dshell/output/__init__.py b/dshell/output/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dshell/output/alertout.py b/dshell/output/alertout.py new file mode 100644 index 0000000..e977599 --- /dev/null +++ b/dshell/output/alertout.py @@ -0,0 +1,15 @@ +""" +This output module is used to display single-line alerts + +It inherits nearly everything from the base Output class, and only resets the +_DEFAULT_FORMAT to a more expressive format. +""" + +from dshell.output.output import Output + +class AlertOutput(Output): + "A class that provides a default format for printing a single-line alert" + _DESCRIPTION = "Default format for printing a single-line alert" + _DEFAULT_FORMAT = "[%(plugin)s] %(ts)s %(sip)16s:%(sport)-5s %(dir_arrow)s %(dip)16s:%(dport)-5s ** %(data)s **\n" + +obj = AlertOutput diff --git a/dshell/output/colorout.py b/dshell/output/colorout.py new file mode 100644 index 0000000..76e1cea --- /dev/null +++ b/dshell/output/colorout.py @@ -0,0 +1,95 @@ +""" +Generates packet or reconstructed stream output with ANSI color codes. + +Based on output module originally written by amm +""" + +from dshell.output.output import Output +import dshell.core +import dshell.util + +class ColorOutput(Output): + _DESCRIPTION = "Reconstructed output with ANSI color codes" + _PACKET_FORMAT = """Packet %(counter)s (%(proto)s) +Start: %(ts)s +%(sip)16s:%(sport)6s -> %(dip)16s:%(dport)6s (%(bytes)s bytes) + +%(data)s + +""" + _CONNECTION_FORMAT = """Connection %(counter)s (%(protocol)s) +Start: %(starttime)s +End: %(endtime)s +%(clientip)16s:%(clientport)6s -> %(serverip)16s:%(serverport)6s (%(clientbytes)s bytes) +%(serverip)16s:%(serverport)6s -> %(clientip)16s:%(clientport)6s (%(serverbytes)s bytes) + +%(data)s + +""" + _DEFAULT_FORMAT = _PACKET_FORMAT + _DEFAULT_DELIM = "\n\n" + + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.counter = 1 + self.colors = { + 'cs': '31', # client-to-server is red + 'sc': '32', # server-to-client is green + '--': '34', # everything else is blue + } + self.hexmode = kwargs.get('hex', False) + self.format_is_set = False + + def setup(self): + # activate color blind friendly mode + if self.cbf: + self.colors['cs'] = '33' #client-to-server is yellow + + def write(self, *args, **kwargs): + if not self.format_is_set: + if 'clientip' in kwargs: + self.set_format(self._CONNECTION_FORMAT) + else: + self.set_format(self._PACKET_FORMAT) + self.format_is_set = True + + # a template string for data output + colorformat = "\x1b[%sm%s\x1b[0m" + + # Iterate over the args and try to parse out any raw data strings + rawdata = [] + for arg in args: + if type(arg) == dshell.core.Blob: + if arg.data: + rawdata.append((arg.data, arg.direction)) + elif type(arg) == dshell.core.Connection: + for blob in arg.blobs: + if blob.data: + rawdata.append((blob.data, blob.direction)) + elif type(arg) == dshell.core.Packet: + rawdata.append((arg.pkt.body_bytes, kwargs.get('direction', '--'))) + elif type(arg) == tuple: + rawdata.append(arg) + else: + rawdata.append((arg, kwargs.get('direction', '--'))) + + # Clean up the rawdata into something more presentable + if self.hexmode: + cleanup_func = dshell.util.hex_plus_ascii + else: + cleanup_func = dshell.util.printable_text + for k, v in enumerate(rawdata): + newdata = cleanup_func(v[0]) + rawdata[k] = (newdata, v[1]) + + # Convert the raw data strings into color-coded output + data = [] + for arg in rawdata: + datastring = colorformat % (self.colors.get(arg[1], '0'), arg[0]) + data.append(datastring) + + super().write(counter=self.counter, *data, **kwargs) + self.counter += 1 + +obj = ColorOutput diff --git a/dshell/output/csvout.py b/dshell/output/csvout.py new file mode 100644 index 0000000..0ad2b9c --- /dev/null +++ b/dshell/output/csvout.py @@ -0,0 +1,63 @@ +""" +This output module converts plugin output into a CSV format +""" + +import csv +from dshell.output.output import Output + +class CSVOutput(Output): + """ + Takes specified fields provided to the write function and print them in + a CSV format. + + Delimiter can be set with --oarg delimiter= + + A header row can be printed with --oarg header + + Additional fields can be included with --oarg fields=field1,field2,field3 + For example, MAC address can be included with --oarg fields=smac,dmac + Note: Field names must match the variable names in the plugin + + Additional flow fields for connection can be included with --oarg flows + """ + + # TODO refine plugin to do things like wrap quotes around long strings + + _DEFAULT_FIELDS = ['plugin', 'ts', 'sip', 'sport', 'dip', 'dport', 'data'] + _DEFAULT_FLOW_FIELDS = ['plugin', 'starttime', 'clientip', 'serverip', 'clientcc', 'servercc', 'protocol', 'clientport', 'serverport', 'clientpackets', 'serverpackets', 'clientbytes', 'serverbytes', 'duration', 'data'] + _DEFAULT_DELIM = ',' + _DESCRIPTION = "CSV format output" + + def __init__(self, *args, **kwargs): + self.use_header = False + self.fields = list(self._DEFAULT_FIELDS) + super().__init__(**kwargs) + + def set_format(self, _=None): + "Set the format to a CSV list of fields" + columns = [] + for f in self.fields: + if f: + columns.append(f) + if self.extra: + columns.append("extra") + fmt = self.delimiter.join('%%(%s)r' % f for f in columns) + fmt += "\n" + super().set_format(fmt) + + def set_oargs(self, **kwargs): + self.use_header = kwargs.pop("header", False) + if kwargs.pop("flows", False): + self.fields = list(self._DEFAULT_FLOW_FIELDS) + if exfields := kwargs.pop("fields", None): + for field in exfields.split(','): + self.fields.append(field) + super().set_oargs(**kwargs) + self.set_format() + + def setup(self): + if self.use_header: + self.fh.write(self.delimiter.join([f for f in self.fields]) + "\n") + + +obj = CSVOutput \ No newline at end of file diff --git a/dshell/output/elasticout.py b/dshell/output/elasticout.py new file mode 100644 index 0000000..0eae856 --- /dev/null +++ b/dshell/output/elasticout.py @@ -0,0 +1,74 @@ +""" +This output module converts plugin output into JSON and indexes it into +an Elasticsearch datastore + +NOTE: This module requires the third-party 'elasticsearch' Python module +""" + +import ipaddress +import json + +from elasticsearch import Elasticsearch + +import dshell.output.jsonout + +class ElasticOutput(dshell.output.jsonout.JSONOutput): + """ + Elasticsearch output module + Use with --output=elasticsearchout + + It is recommended that it be run with some options set: + host: server hosting the database (localhost) + port: HTTP port listening (9200) + index: name of index storing results ("dshell") + type: the type for each alert ("alerts") + + Example use: + decode --output=elasticout --oargs="index=dshellalerts" --oargs="type=netflowout" -d netflow ~/pcap/example.pcap + """ + + _DESCRIPTION = "Automatically insert data into an elasticsearch instance" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs.copy()) + + self.options = {} + self.options['host'] = kwargs.get('host', 'localhost') + self.options['port'] = int(kwargs.get('port', 9200)) + self.options['index'] = kwargs.get('index', 'dshell') + self.options['type'] = kwargs.get('type', 'alerts') + + self.es = Elasticsearch([self.options['host']], port=self.options['port']) + + def write(self, *args, **kwargs): + "Converts alert's keyword args to JSON and indexes it into Elasticsearch datastore." + if args and 'data' not in kwargs: + kwargs['data'] = self.delimiter.join(map(str, args)) + + # Elasticsearch can't handle IPv6 (at time of writing) + # Just delete the ints and expand the string notation. + # Hopefully, it will be possible to perform range searches on this + # consistent IP string format. + try: + del kwargs['dipint'] + except KeyError: + pass + try: + del kwargs['sipint'] + except KeyError: + pass + try: + kwargs['dip'] = ipaddress.ip_address(kwargs['dip']).exploded + except KeyError: + pass + try: + kwargs['sip'] = ipaddress.ip_address(kwargs['sip']).exploded + except KeyError: + pass + + jsondata = json.dumps(kwargs, ensure_ascii=self.ensure_ascii, default=self.json_default) +# from pprint import pprint +# pprint(jsondata) + self.es.index(index=self.options['index'], doc_type=self.options['type'], body=jsondata) + +obj = ElasticOutput diff --git a/dshell/output/htmlout.py b/dshell/output/htmlout.py new file mode 100644 index 0000000..0e0a1da --- /dev/null +++ b/dshell/output/htmlout.py @@ -0,0 +1,133 @@ +""" +Generates packet or reconstructed stream output as a HTML page. + +Based on colorout module originally written by amm +""" + +from dshell.output.output import Output +import dshell.util +import dshell.core +from xml.sax.saxutils import escape + +class HTMLOutput(Output): + _DESCRIPTION = "HTML format output" + _PACKET_FORMAT = """

Packet %(counter)s (%(protocol)s)

Start: %(ts)s +%(sip)s:%(sport)s -> %(dip)s:%(dport)s (%(bytes)s bytes) +

+%(data)s +""" + _CONNECTION_FORMAT = """

Connection %(counter)s (%(protocol)s)

Start: %(starttime)s +End: %(endtime)s +%(clientip)s:%(clientport)s -> %(serverip)s:%(serverport)s (%(clientbytes)s bytes) +%(serverip)s:%(serverport)s -> %(clientip)s:%(clientport)s (%(serverbytes)s bytes) +

+%(data)s +""" + _DEFAULT_FORMAT = _PACKET_FORMAT + _DEFAULT_DELIM = "
" + + _HTML_HEADER = """ + + + + Dshell Output + + + +""" + + _HTML_FOOTER = """ + + +""" + + def __init__(self, *args, **kwargs): + "Can be called with an optional 'hex' argument to display output in hex" + super().__init__(*args, **kwargs) + self.counter = 1 + self.colors = { + 'cs': 'red', # client-to-server is red + 'sc': 'green', # server-to-client is green + '--': 'blue', # everything else is blue + } + self.hexmode = kwargs.get('hex', False) + self.format_is_set = False + + def setup(self): + # activate color blind friendly mode + if self.cbf: + self.colors['cs'] = 'gold' # client-to-server is gold (darker yellow) + self.colors['sc'] = 'seagreen' # server-to-client is sea green (lighter green) + self.fh.write(self._HTML_HEADER) + + def write(self, *args, **kwargs): + if not self.format_is_set: + if 'clientip' in kwargs: + self.set_format(self._CONNECTION_FORMAT) + else: + self.set_format(self._PACKET_FORMAT) + self.format_is_set = True + + # a template string for data output + colorformat = '%s' + + # Iterate over the args and try to parse out any raw data strings + rawdata = [] + for arg in args: + if type(arg) == dshell.core.Blob: + if arg.data: + rawdata.append((arg.data, arg.direction)) + elif type(arg) == dshell.core.Connection: + for blob in arg.blobs: + if blob.data: + rawdata.append((blob.data, blob.direction)) + elif type(arg) == dshell.core.Packet: + rawdata.append((arg.pkt.body_bytes, kwargs.get('direction', '--'))) + elif type(arg) == tuple: + rawdata.append(arg) + else: + rawdata.append((arg, kwargs.get('direction', '--'))) + + # Clean up the rawdata into something more presentable + if self.hexmode: + cleanup_func = dshell.util.hex_plus_ascii + else: + cleanup_func = dshell.util.printable_text + for k, v in enumerate(rawdata): + newdata = cleanup_func(v[0]) + newdata = escape(newdata) + rawdata[k] = (newdata, v[1]) + + # Convert the raw data strings into color-coded output + data = [] + for arg in rawdata: + datastring = colorformat % (self.colors.get(arg[1], ''), arg[0]) + data.append(datastring) + + super().write(counter=self.counter, *data, **kwargs) + self.counter += 1 + + def close(self): + self.fh.write(self._HTML_FOOTER) + Output.close(self) + +obj = HTMLOutput diff --git a/dshell/output/jsonout.py b/dshell/output/jsonout.py new file mode 100644 index 0000000..f1e6121 --- /dev/null +++ b/dshell/output/jsonout.py @@ -0,0 +1,48 @@ +""" +This output module converts plugin output into JSON +""" + +from datetime import datetime +import json +from dshell.output.output import Output +from dshell.core import Packet, Blob, Connection + +class JSONOutput(Output): + """ + Converts arguments for every write into JSON + Can be called with ensure_ascii=True to pass flag on to the json module. + """ + _DEFAULT_FORMAT = "%(jsondata)s\n" + _DESCRIPTION = "JSON format output" + + def __init__(self, *args, **kwargs): + self.ensure_ascii = kwargs.get('ensure_ascii', False) + super().__init__(*args, **kwargs) + + def write(self, *args, **kwargs): + if self.extra: + # JSONOutput does not make use of the --extra flag, so disable it + # before printing output + self.extra = False + if args and 'data' not in kwargs: + kwargs['data'] = self.delimiter.join(map(str, args)) + jsondata = json.dumps(kwargs, ensure_ascii=self.ensure_ascii, default=self.json_default) + super().write(jsondata=jsondata) + + def json_default(self, obj): + """ + JSON serializer for objects not serializable by default json code + https://stackoverflow.com/a/22238613 + """ + if isinstance(obj, datetime): + serial = obj.strftime(self.timeformat) + return serial + if isinstance(obj, bytes): + serial = repr(obj) + return serial + if isinstance(obj, (Connection, Blob, Packet)): + serial = obj.info() + return serial + raise TypeError ("Type not serializable ({})".format(str(type(obj)))) + +obj = JSONOutput diff --git a/dshell/output/netflowout.py b/dshell/output/netflowout.py new file mode 100644 index 0000000..f27cc3f --- /dev/null +++ b/dshell/output/netflowout.py @@ -0,0 +1,103 @@ +""" +This output module is used for generating flow-format output +""" + +from dshell.output.output import Output +from datetime import datetime + +class NetflowOutput(Output): + """ + A class for printing connection information for pcap + + Output can be grouped by setting the group flag to a field or fields + separated by a forward-slash + For example: + --output=netflowout --oarg="group=clientip/serverip" + Note: Output when grouping is only generated at the end of analysis + + A header row can be printed before output using --oarg header + """ + + _DESCRIPTION = "Flow (connection overview) format output" + # Define two types of formats: + # Those for plugins handling individual packets (not really helpful) + _PACKET_FORMAT = "%(ts)s %(sip)16s -> %(dip)16s (%(sipcc)s -> %(dipcc)s) %(protocol)5s %(sport)6s %(dport)6s %(bytes)7s %(data)s\n" + _PACKET6_FORMAT = "%(ts)s %(sip)40s -> %(dip)40s (%(sipcc)s -> %(dipcc)s) %(protocol)5s %(sport)6s %(dport)6s %(bytes)7s %(data)s\n" + _PACKET_PRETTY_HEADER = "[start timestamp] [source IP] -> [destination IP] ([source country] -> [destination country]) [protocol] [source port] [destination port] [bytes] [message data]\n" + # And those plugins handling full connections (more useful and common) + _CONNECTION_FORMAT = "%(starttime)s %(clientip)16s -> %(serverip)16s (%(clientcc)s -> %(servercc)s) %(protocol)5s %(clientport)6s %(serverport)6s %(clientpackets)5s %(serverpackets)5s %(clientbytes)7s %(serverbytes)7s %(duration)-.4fs %(data)s\n" + _CONNECTION6_FORMAT = "%(starttime)s %(clientip)40s -> %(serverip)40s (%(clientcc)s -> %(servercc)s) %(protocol)5s %(clientport)6s %(serverport)6s %(clientpackets)5s %(serverpackets)5s %(clientbytes)7s %(serverbytes)7s %(duration)-.4fs %(data)s\n" + _CONNECTION_PRETTY_HEADER = "[start timestamp] [client IP] -> [server IP] ([client country] -> [server country]) [protocol] [client port] [server port] [client packets] [server packets] [client bytes] [server bytes] [duration] [message data]\n" + # TODO decide if IPv6 formats are necessary, and how to switch between them + # and IPv4 formats + # Default to packets since those fields are in both types of object + _DEFAULT_FORMAT = _PACKET_FORMAT + + def __init__(self, *args, **kwargs): + self.group = False + self.group_cache = {} # results will be stored here, if grouping + self.format_is_set = False + self.use_header = False + Output.__init__(self, *args, **kwargs) + + def set_format(self, fmt, pretty_header=_PACKET_PRETTY_HEADER): + if self.use_header: + self.fh.write(str(pretty_header)) + return super().set_format(fmt) + + def set_oargs(self, **kwargs): + # Are we printing the format string as a file header? + self.use_header = kwargs.pop("header", False) + # Are we grouping the results, and by what fields? + if 'group' in kwargs: + self.group = True + groupfields = kwargs.pop('group', '') + self.group_fields = groupfields.split('/') + else: + self.group = False + super().set_oargs(**kwargs) + + def write(self, *args, **kwargs): + # Change output format depending on if we're handling a connection or + # a single packet + if not self.format_is_set: + if "clientip" in kwargs: + self.set_format(self._CONNECTION_FORMAT, self._CONNECTION_PRETTY_HEADER) + else: + self.set_format(self._PACKET_FORMAT, self._PACKET_PRETTY_HEADER) + self.format_is_set = True + + if self.group: + # If grouping, check if the IP tuple is in the cache already. + # If not, check the reverse of the tuple (i.e. opposite direction) + try: + key = tuple([kwargs[g] for g in self.group_fields]) + except KeyError as e: + Output.write(self, *args, **kwargs) + return + if key not in self.group_cache: + rkey = key[::-1] + if rkey in self.group_cache: + key = rkey + else: + self.group_cache[key] = [] + self.group_cache[key].append(kwargs) + else: + # If not grouping, just write out the connection immediately + Output.write(self, *args, **kwargs) + + def close(self): + if self.group: + self.group = False # we're done grouping, so turn it off + for key in self.group_cache.keys(): + # write header by mapping key index with user's group list + self.fh.write(' '.join([ + '%s=%s' % (self.group_fields[i], key[i]) for i in range(len(self.group_fields))]) + + "\n") + for kw in self.group_cache[key]: + self.fh.write("\t") + Output.write(self, **kw) + self.fh.write("\n") + Output.close(self) + +obj = NetflowOutput \ No newline at end of file diff --git a/dshell/output/output.py b/dshell/output/output.py new file mode 100644 index 0000000..60b53e8 --- /dev/null +++ b/dshell/output/output.py @@ -0,0 +1,280 @@ +""" +Generic Dshell output class(es) + +Contains the base-level Output class that other modules inherit from. +""" + +import logging +import os +import re +import sys +from collections import defaultdict +from datetime import datetime +import warnings + + +logger = logging.getLogger(__name__) + + +class Output: + """ + Base-level output class + + Arguments: + format : 'format string' to override default formatstring for output class + timeformat : 'format string' for datetime representation + delimiter : set a delimiter for CSV or similar output + nobuffer : true/false to run flush() after every relevant write + noclobber : set to true to avoid overwriting existing files + fh : existing open file handle + file : filename to write to, assuming fh is not defined + mode : mode to open file, assuming fh is not defined (default 'w') + cbf : activate color blind friendly mode, colorout and htmlout output + modules use yellow/gold in place of red and different shades of + green/yellow/blue are used to help better differentiate between them + """ + _DEFAULT_FORMAT = "%(data)s\n" + _DEFAULT_TIME_FORMAT = "%Y-%m-%d %H:%M:%S" + _DEFAULT_DELIM = ',' + _DESCRIPTION = "Base output class" + + def __init__( + self, file=None, fh=None, mode='w', format=None, timeformat=None, delimiter=None, nobuffer=False, + noclobber=False, extra=None, cbf=False, **unused_kwargs + ): + self.format_fields = [] + self.timeformat = timeformat or self._DEFAULT_TIME_FORMAT + self.delimiter = delimiter or self._DEFAULT_DELIM + self.nobuffer = nobuffer + self.noclobber = noclobber + self.extra = extra + self.mode = mode + self.cbf = cbf + + # Must define attributes even if they are setup in different function. + self.format_fields = None + self.format = None + self.set_format(format or self._DEFAULT_FORMAT) + + # Set the filehandle for any output + if fh: + self.fh = fh + return + + f = file + if f: + if self.noclobber: + f = self._increment_filename(f) + self.fh = open(f, self.mode) + else: + self.fh = sys.stdout + + def reset_fh(self, filename=None, fh=None, mode=None): + """ + Alter the module's open file handle without changing any of the other + settings. Must supply at least a filename or a filehandle (fh). + reset_fh(filename=None, fh=None, mode=None) + """ + if fh: + self.fh = fh + elif filename: + if self.noclobber: + filename = self._increment_filename(filename) + if mode: + self.mode = mode + self.fh = open(filename, mode) + else: + self.fh = open(filename, self.mode) + + def set_oargs(self, format=None, noclobber=None, delimiter=None, timeformat=None, hex=None, **unused_kwargs): + """ + Process the standard oargs from the command line. + """ + if delimiter: + if delimiter == "tab": + self.delimiter = '\t' + else: + self.delimiter = delimiter + if timeformat: + self.timeformat = timeformat + if noclobber: + self.noclobber = noclobber + if hex: + self.hexmode = hex + if format: + self.set_format(format) + + def set_format(self, fmt): + """Set the output format to a new format string""" + # Use a regular expression to identify all fields that the format will + # populate, based on limited printf-style formatting. + # https://docs.python.org/3/library/stdtypes.html#old-string-formatting + regexmatch = "%\((?P.*?)\)[diouxXeEfFgGcrs]" + self.format_fields = re.findall(regexmatch, fmt) + self.format = fmt + + def _increment_filename(self, filename): + """ + Used with the noclobber argument. + Creates a distinct filename by appending a sequence number. + """ + try: + while os.stat(filename): + p = filename.rsplit('-', 1) + try: + p, n = p[0], int(p[1]) + except ValueError: + n = 0 + filename = '-'.join(p + ['%04d' % (int(n) + 1)]) + except OSError: + pass # file not found + return filename + + def setup(self): + """ + Perform any additional setup outside of the standard __init__. + For example, printing header data to the outfile. + """ + pass + + def close(self): + """ + Close output file, assuming it's not stdout + """ + if self.fh not in (sys.stdout, sys.stdout.buffer): + self.fh.close() + + # NOTE: Output modules no longer handles logging. Logging should be done by creating a logger + # at the top of each of the modules. + # If we want to change the destination of the log messages we can create a log handler. + def log(self, msg, level=logging.INFO, *args, **kwargs): + """ + Write a message to the log + Passes all args and kwargs thru to logging, except for 'level' + """ + warnings.warn("Please create and use a logger using the logging module instead", DeprecationWarning) + logger.log(level, msg, *args, **kwargs) + + def convert(self, *args, **kwargs): + """ + Attempts to convert the args/kwargs into the format defined in + self.format and self.timeformat + """ + # Have the keyword arguments default to empty strings, in the event + # of missing keys for string formatting + outdict = defaultdict(str, **kwargs) + outformat = self.format + extras = [] + + # Convert raw timestamps into a datetime object + if 'ts' in outdict: + try: + outdict['ts'] = datetime.fromtimestamp(float(outdict['ts'])) + outdict['ts'] = outdict['ts'].strftime(self.timeformat) + except TypeError: + pass + except KeyError: + pass + except ValueError: + pass + + if "starttime" in outdict and isinstance(outdict["starttime"], datetime): + outdict['starttime'] = outdict['starttime'].strftime(self.timeformat) + if "endtime" in outdict and isinstance(outdict["endtime"], datetime): + outdict['endtime'] = outdict['endtime'].strftime(self.timeformat) + if 'dt' in outdict and isinstance(outdict["dt"], datetime): + outdict['dt'] = outdict['dt'].strftime(self.timeformat) + + # Create directional arrows + if 'dir_arrow' not in outdict: + if outdict.get('direction') == 'cs': + outdict['dir_arrow'] = '->' + elif outdict.get('direction') == 'sc': + outdict['dir_arrow'] = '<-' + else: + outdict['dir_arrow'] = '--' + + # Convert Nones into empty strings. + # If --extra flag used, generate string representing otherwise hidden + # fields. + for key, val in sorted(outdict.items()): + if val is None: + val = '' + outdict[key] = val + if self.extra: + if key not in self.format_fields: + extras.append("%s=%s" % (key, val)) + + # Dump the args into a 'data' field + outdict['data'] = self.delimiter.join(map(str, args)) + + # Create an optional 'extra' field + if self.extra: + if 'extra' not in self.format_fields: + outformat = outformat[:-1] + " [ %(extra)s ]\n" + outdict['extra'] = ', '.join(extras) + + # Convert the output dictionary into a string that is dumped to the + # output location. + output = outformat % outdict + return output + + def write(self, *args, **kwargs): + """ + Primary output function. Should be overwritten by subclasses. + """ + line = self.convert(*args, **kwargs) + try: + self.fh.write(line) + if self.nobuffer: + self.fh.flush() + except BrokenPipeError: + pass + + def alert(self, *args, **kwargs): + """ + DEPRECATED + Use the write function of the AlertOutput class + """ + warnings.warn("Use the write function of the AlertOutput class", DeprecationWarning) + self.write(*args, **kwargs) + + def dump(self, *args, **kwargs): + """ + DEPRECATED + Use the write function of the PCAPOutput class + """ + warnings.warn("Use the write function of the PCAPOutput class", DeprecationWarning) + self.write(*args, **kwargs) + + +class QueueOutputWrapper(object): + """ + Wraps an instance of any other Output-like object to make its + write function more thread safe. + """ + + def __init__(self, oobject, oqueue): + self.__oobject = oobject + self.__owrite = oobject.write + self.queue = oqueue + self.id = str(self.__oobject) + + def true_write(self, *args, **kwargs): + "Calls the wrapped class's write function. Called from decode.py." + self.__owrite(*args, **kwargs) + + def write(self, *args, **kwargs): + """ + Adds a message to the queue indicating that this wrapper is ready to + run its write function + """ + self.queue.put((self.id, args, kwargs)) + + +############################################################################### + +# The "obj" variable is used in decode.py as a standard name for each output +# module's primary class. It technically imports this variable and uses it to +# construct an instance. +obj = Output diff --git a/dshell/output/pcapout.py b/dshell/output/pcapout.py new file mode 100644 index 0000000..96a7af3 --- /dev/null +++ b/dshell/output/pcapout.py @@ -0,0 +1,64 @@ +""" +This output module generates pcap output when given very specific arguments. +""" + +from dshell.output.output import Output +import struct +import sys + +# TODO get this module to work with ConnectionPlugins + +class PCAPOutput(Output): + "Writes data to a pcap file." + _DESCRIPTION = "Writes data to a pcap file (does not work with connection-based plugins)" + + def __init__(self, *args, **kwargs): + super().__init__(*args, mode='wb', **kwargs) + if self.fh == sys.stdout: + # Switch to a stdout that can handle byte output + self.fh = sys.stdout.buffer + # Since we have to wait until the link-layer type is set, we wait + # until the first write() operation before writing the pcap header + self.header_written = False + + def write(self, *args, **kwargs): + """ + Write a packet to the pcap file. + + Arguments: + pktlen : raw packet length + rawpkt : raw packet data string + ts : timestamp + link_layer_type : link-layer type (optional) (default: 1) + (e.g. 1 for Ethernet, 105 for 802.11, etc.) + """ + # The first time write() is called, the pcap header is written. + # This is to allow the plugin enough time to figure out what the + # link-layer type is for the data. + if not self.header_written: + link_layer_type = kwargs.get('link_layer_type', 1) + # write the header: + # magic_number, version_major, version_minor, thiszone, sigfigs, + # snaplen, link-layer type + self.fh.write( + struct.pack('IHHIIII', 0xa1b2c3d4, 2, 4, 0, 0, 65535, link_layer_type)) + self.header_written = True + + # Attempt to fetch the required fields + pktlen = kwargs.get('pktlen', None) + rawpkt = kwargs.get('rawpkt', None) + ts = kwargs.get('ts', None) + if pktlen is None or rawpkt is None or ts is None: + raise TypeError("PCAPOutput.write() requires at least these arguments to write packet data: pktlen, rawpkt, and ts.\n\tIt is possible this plugin is not configured to handle pcap output.") + + self.fh.write( + struct.pack('II', int(ts), int((ts - int(ts)) * 1000000))) + self.fh.write(struct.pack('II', len(rawpkt), pktlen)) + self.fh.write(rawpkt) + + def close(self): + if self.fh == sys.stdout.buffer: + self.fh = sys.stdout + super().close() + +obj = PCAPOutput diff --git a/dshell/plugins/__init__.py b/dshell/plugins/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dshell/plugins/dhcp/__init__.py b/dshell/plugins/dhcp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dshell/plugins/dhcp/dhcp.py b/dshell/plugins/dhcp/dhcp.py new file mode 100644 index 0000000..3bb3755 --- /dev/null +++ b/dshell/plugins/dhcp/dhcp.py @@ -0,0 +1,102 @@ +""" +DHCP Plugin +""" + +import dshell.core +import dshell.util +from dshell.output.alertout import AlertOutput + +from pypacker.layer4 import udp +from pypacker.layer567 import dhcp + +from struct import unpack + +class DshellPlugin(dshell.core.PacketPlugin): + def __init__(self, **kwargs): + super().__init__(name='dhcp', + description='extract client information from DHCP messages', + longdescription=""" +The dhcp plugin will extract the Transaction ID, Hostname, and +Client ID (MAC address) from every UDP DHCP packet found in the given pcap +using port 67. DHCP uses BOOTP as its transport protocol. +BOOTP assigns port 67 for the 'BOOTP server' and port 68 for the 'BOOTP client'. +This filter pulls DHCP Inform packets. + +Examples: + + General usage: + + decode -d dhcp + + This will display the connection info including the timestamp, + the source IP : source port, destination IP : destination port, + Transaction ID, Client Hostname, and the Client MAC address + in a tabular format. + + + Malware Traffic Analysis Exercise Traffic from 2015-03-03 where a user was hit with an Angler exploit kit: + + We want to find out more about the infected machine, and some of this information can be pulled from DHCP traffic + + decode -d dhcp 2015-03-03-traffic-analysis-exercise.pcap + + OUTPUT: +[dhcp] 2015-03-03 14:05:10 172.16.101.196:68 -> 172.16.101.1:67 ** Transaction ID: 0xba5a2cfe Client ID (MAC): 38:2C:4A:3D:EF:01 Hostname: Gregory-PC ** +[dhcp] 2015-03-03 14:08:40 172.16.101.196:68 -> 255.255.255.255:67 ** Transaction ID: 0x6a482406 Client ID (MAC): 38:2C:4A:3D:EF:01 Hostname: Gregory-PC ** +[dhcp] 2015-03-03 14:10:11 172.16.101.196:68 -> 172.16.101.1:67 ** Transaction ID: 0xe74b17fe Client ID (MAC): 38:2C:4A:3D:EF:01 Hostname: Gregory-PC ** +[dhcp] 2015-03-03 14:12:50 172.16.101.196:68 -> 255.255.255.255:67 ** Transaction ID: 0xd62614a0 Client ID (MAC): 38:2C:4A:3D:EF:01 Hostname: Gregory-PC ** +""", + bpf='(udp and port 67)', + output=AlertOutput(label=__name__), + author='dek', + ) + self.mac_address = None + self.client_hostname = None + self.xid = None + + # A packetHandler is used to ensure that every DHCP packet in the traffic is parsed + def packet_handler(self, pkt): + + # iterate through the layers and find the DHCP layer + dhcp_packet = pkt.pkt.upper_layer + while not isinstance(dhcp_packet, dhcp.DHCP): + try: + dhcp_packet = dhcp_packet.upper_layer + except AttributeError: + # There doesn't appear to be a DHCP layer + return + + # Pull the transaction ID from the packet + self.xid = hex(dhcp_packet.xid) + + # if we have a DHCP INFORM PACKET + if dhcp_packet.op == dhcp.DHCP_OP_REQUEST: + for opt in list(dhcp_packet.opts): + try: + option_code = opt.type + msg_value = opt.body_bytes + except AttributeError: + continue + + # if opt is CLIENT_ID (61) + # unpack the msg_value and reformat the MAC address + if option_code == dhcp.DHCP_OPT_CLIENT_ID: + hardware_type, mac = unpack('B6s', msg_value) + mac = mac.hex().upper() + self.mac_address = ':'.join([mac[i:i+2] for i in range(0, len(mac), 2)]) + + # if opt is HOSTNAME (12) + elif option_code == dhcp.DHCP_OPT_HOSTNAME: + self.client_hostname = msg_value.decode('utf-8') + + # Allow for unknown hostnames + if not self.client_hostname: + self.client_hostname = "" + + if self.xid and self.mac_address: + self.write('Transaction ID: {0:<12} Client ID (MAC): {1:<20} Hostname: {2:<}'.format( + self.xid, self.mac_address, self.client_hostname), **pkt.info(), dir_arrow='->') + return pkt + +if __name__ == "__main__": + print(DshellPlugin()) diff --git a/dshell/plugins/dns/__init__.py b/dshell/plugins/dns/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dshell/plugins/dns/dns.py b/dshell/plugins/dns/dns.py new file mode 100644 index 0000000..afb9e0b --- /dev/null +++ b/dshell/plugins/dns/dns.py @@ -0,0 +1,174 @@ +""" +Extracts and summarizes DNS queries and responses. +""" + +import dshell.core +from dshell.plugins import dnsplugin +from dshell.output.alertout import AlertOutput + +from pypacker.pypacker import dns_name_decode +from pypacker.layer567 import dns + +import ipaddress + +RESPONSE_ERRORS = { + dns.DNS_RCODE_FORMERR: "FormErr", + dns.DNS_RCODE_SERVFAIL: "ServFail", + dns.DNS_RCODE_NXDOMAIN: "NXDOMAIN", + dns.DNS_RCODE_NOTIMP: "NotImp", + dns.DNS_RCODE_REFUSED: "Refused", + dns.DNS_RCODE_YXDOMAIN: "YXDp,aom", + dns.DNS_RCODE_YXRRSET: "YXRRSet", + dns.DNS_RCODE_NXRRSET: "NXRRSet", + dns.DNS_RCODE_NOTAUTH: "NotAuth", + dns.DNS_RCODE_NOTZONE: "NotZone", +} + +class DshellPlugin(dnsplugin.DNSPlugin): + + def __init__(self, *args, **kwargs): + super().__init__( + name="DNS", + description="Extract and summarize DNS queries/responses", + longdescription=""" +The DNS plugin extracts and summarizes DNS queries and their responses. If +possible, each query is paired with its response(s). + +Possible anomalies can be found using the --dns_show_noanswer, +--dns_only_noanswer, --dns_show_norequest, or --dns_only_norequest flags +(see --help). + +For example, looking for responses that did not come from a request: + decode -d dns --dns_only_norequest + +Additional information for responses can be seen with --dns_country and +--dns_asn to show country codes and ASNs, respectively. These results can be +piped to grep for filtering results. + +For example, to look for all traffic from Germany: + decode -d dns --dns_country |grep "country: DE" + +To look for non-US traffic, try: + decode -d dns --dns_country |grep "country:" |grep -v "country: US" +""", + author="bg/twp", + bpf="udp and port 53", + output=AlertOutput(label=__name__), + optiondict={'show_noanswer': {'action': 'store_true', 'help': 'report unanswered queries alongside other queries'}, + 'show_norequest': {'action': 'store_true', 'help': 'report unsolicited responses alongside other responses'}, + 'only_noanswer': {'action': 'store_true', 'help': 'report only unanswered queries'}, + 'only_norequest': {'action': 'store_true', 'help': 'report only unsolicited responses'}, + 'country': {'action': 'store_true', 'help': 'show country code for returned IP addresses'}, + 'asn': {'action': 'store_true', 'help': 'show ASN for returned IP addresses'}, + } + ) + + def premodule(self): + if self.only_norequest: + self.show_norequest = True + if self.only_noanswer: + self.show_noanswer = True + + + def dns_handler(self, conn, requests, responses): + if self.only_norequest and requests is not None: + return + if self.only_noanswer and responses is not None: + return + if not self.show_norequest and requests is None: + return + if not self.show_noanswer and responses is None: + return + + msg = [] + + # For simplicity, we focus only on the last request if there's more + # than one. + if requests: + request_pkt = requests[-1] + request = request_pkt.pkt.highest_layer + id = request.id + for query in request.queries: + if query.type == dns.DNS_A: + msg.append("A? {}".format(query.name_s)) + elif query.type == dns.DNS_AAAA: + msg.append("AAAA? {}".format(query.name_s)) + elif query.type == dns.DNS_CNAME: + msg.append("CNAME? {}".format(query.name_s)) + elif query.type == dns.DNS_LOC: + msg.append("LOC? {}".format(query.name_s)) + elif query.type == dns.DNS_MX: + msg.append("MX? {}".format(query.name_s)) + elif query.type == dns.DNS_PTR: + msg.append("PTR? {}".format(query.name_s)) + elif query.type == dns.DNS_SRV: + msg.append("SRV? {}".format(query.name_s)) + elif query.type == dns.DNS_TXT: + msg.append("TXT? {}".format(query.name_s)) + else: + request = None + + if responses: + response_pkt = responses[-1] + for response in responses: + rcode = response.rcode + response = response.pkt.highest_layer + id = response.id + # Check for errors in the response code + err = RESPONSE_ERRORS.get(rcode, None) + if err: + msg.append(err) + continue + # Get the response counts + msg.append("{}/{}/{}".format(response.answers_amount, response.authrr_amount, response.addrr_amount)) + # Parse the answers from the response + for answer in response.answers: + if answer.type == dns.DNS_A or answer.type == dns.DNS_AAAA: + msg_fields = {} + msg_format = "A: {ip} (ttl {ttl}s)" + answer_ip = ipaddress.ip_address(answer.address) + msg_fields['ip'] = str(answer_ip) + msg_fields['ttl'] = str(answer.ttl) + if self.country: + msg_fields['country'] = dshell.core.geoip.geoip_country_lookup(msg_fields['ip']) or '--' + msg_format += " (country: {country})" + if self.asn: + msg_fields['asn'] = dshell.core.geoip.geoip_asn_lookup(msg_fields['ip']) + msg_format += " (ASN: {asn})" + msg.append(msg_format.format(**msg_fields)) + # TODO pypacker doesn't really parse CNAMEs out. We try + # to get what we can manually, but keep checking if + # if it gets officially included in pypacker + elif answer.type == dns.DNS_CNAME: + if request: + cname = dnsplugin.basic_cname_decode(request.queries[0].name, answer.address) + else: + cname = dns_name_decode(answer.address) + msg.append('CNAME: {!r}'.format(cname)) + elif answer.type == dns.DNS_LOC: + msg.append("LOC: {!s}".format(answer.address)) + elif answer.type == dns.DNS_MX: + msg.append('MX: {!s}'.format(answer.address)) + elif answer.type == dns.DNS_NS: + msg.append('NS: {!s}'.format(answer.address)) + elif answer.type == dns.DNS_PTR: + ptr = dns_name_decode(answer.address) + msg.append('PTR: {!s}'.format(ptr)) + elif answer.type == dns.DNS_SRV: + msg.append('SRV: {!s}'.format(answer.address)) + elif answer.type == dns.DNS_TXT: + msg.append('TXT: {!s}'.format(answer.address)) + + else: + msg.append("No response") + + msg.insert(0, "ID: {}".format(id)) + msg = ", ".join(msg) + if request: + self.write(msg, **request_pkt.info()) + elif response: + self.write(msg, **response_pkt.info()) + else: + self.write(msg, **conn.info()) + + return conn, requests, responses diff --git a/dshell/plugins/dns/dnscc.py b/dshell/plugins/dns/dnscc.py new file mode 100644 index 0000000..a58a98f --- /dev/null +++ b/dshell/plugins/dns/dnscc.py @@ -0,0 +1,84 @@ +""" +Identifies DNS queries and finds the country code of the record response. +""" + +import dshell.core +from dshell.plugins import dnsplugin +from dshell.output.alertout import AlertOutput + +from pypacker.pypacker import dns_name_decode +from pypacker.layer567 import dns + +import ipaddress + +class DshellPlugin(dnsplugin.DNSPlugin): + + def __init__(self, *args, **kwargs): + super().__init__( + name="DNS Country Code", + description="identify country code of DNS A/AAAA record responses", + bpf="port 53", + author="bg", + output=AlertOutput(label=__name__), + optiondict={ + 'foreign': { + 'action': 'store_true', + 'help': 'report responses in non-US countries' + }, + 'code': { + 'type': str, + 'help': 'filter on a specific country code (ex. US, DE, JP, etc.)' + } + } + ) + + def dns_handler(self, conn, requests, responses): + "pull out the A/AAAA queries from the last DNS request in a connection" + queries = [] + if requests: + request = requests[-1].pkt.highest_layer + id = request.id + for query in request.queries: + if query.type == dns.DNS_A: + queries.append("A? {}".format(query.name_s)) + elif query.type == dns.DNS_AAAA: + queries.append("AAAA? {}".format(query.name_s)) + queries = ', '.join(queries) + + answers = [] + if responses: + for response in responses: + response = response.pkt.highest_layer + id = response.id + for answer in response.answers: + if answer.type == dns.DNS_A: + ip = ipaddress.ip_address(answer.address).compressed + cc = dshell.core.geoip.geoip_country_lookup(ip) or '--' + if self.foreign and (cc == 'US' or cc == '--'): + continue + elif self.code and cc != self.code: + continue + answers.append("A: {} ({}) (ttl: {}s)".format( + ip, cc, answer.ttl)) + elif answer.type == dns.DNS_AAAA: + ip = ipaddress.ip_address(answer.address).compressed + if ip == '::': + cc = '--' + else: + cc = dshell.core.geoip.geoip_country_lookup(ip) or '--' + if self.foreign and (cc == 'US' or cc == '--'): + continue + elif self.code and cc != self.code: + continue + answers.append("AAAA: {} ({}) (ttl: {}s)".format( + ip, cc, answer.ttl)) + answers = ', '.join(answers) + + if answers: + msg = "ID: {}, {} / {}".format(id, queries, answers) + self.write(msg, queries=queries, answers=answers, **conn.info()) + return conn, requests, responses + else: + return + + diff --git a/dshell/plugins/dns/innuendo-dns.py b/dshell/plugins/dns/innuendo-dns.py new file mode 100644 index 0000000..d7fb508 --- /dev/null +++ b/dshell/plugins/dns/innuendo-dns.py @@ -0,0 +1,85 @@ +""" +Proof-of-concept Dshell plugin to detect INNUENDO DNS Channel + +Based on the short marketing video (http://vimeo.com/115206626) the +INNUENDO DNS Channel relies on DNS to communicate with an authoritative +name server. The name server will respond with a base64 encoded TXT +answer. This plugin will analyze DNS TXT queries and responses to +determine if it matches the network traffic described in the video. +There are multiple assumptions (*very poor*) in this detection plugin +but serves as a proof-of-concept detector. This detector has not been +tested against authentic INNUENDO DNS Channel traffic. +""" + + +from dshell.plugins.dnsplugin import DNSPlugin +from dshell.output.alertout import AlertOutput + +from pypacker.layer567 import dns + +import base64 + +class DshellPlugin(DNSPlugin): + """ + Proof-of-concept Dshell plugin to detect INNUENDO DNS Channel + + Usage: decode -d innuendo *.pcap + """ + + def __init__(self): + super().__init__( + name="innuendo-dns", + description="proof-of-concept detector for INNUENDO DNS channel", + bpf="port 53", + author="primalsec", + output=AlertOutput(label=__name__), + ) + + def dns_handler(self, conn, requests, responses): + response = responses[-1] + + query = None + answers = [] + + if requests: + request = requests[-1].pkt.highest_layer + query = request.queries[-1] + # DNS Question, extract query name if it is a TXT record request + if query.type == dns.DNS_TXT: + query = query.name_s + + if responses: + for response in responses: + rcode = response.rcode + response = response.pkt.highest_layer + # DNS Answer with data and no errors + if rcode == dns.DNS_RCODE_NOERR and response.answers: + for answer in response.answers: + if answer.type == dns.DNS_TXT: + answers.append(answer.address) + + if query and answers: + # assumption: INNUENDO will use the lowest level domain for C2 + # example: AAAABBBBCCCC.foo.bar.com -> AAAABBBBCCCC is the INNUENDO + # data + subdomain = query.split('.', 1)[0] + + # weak test based on video observation *very poor assumption* + if subdomain.isupper(): + # check each answer in the TXT response + for answer in answers: + try: + # INNUENDO DNS channel base64 encodes the response, check to see if + # it contains a valid base64 string *poor assumption* + dummy = base64.b64decode(answer) + + self.write('INNUENDO DNS Channel', query, '/', answer, **conn.info()) + + # here would be a good place to decrypt the payload (if you have the keys) + # decrypt_payload( answer ) + except: + return None + return conn, requests, responses + + return None + diff --git a/dshell/plugins/dns/specialips.py b/dshell/plugins/dns/specialips.py new file mode 100644 index 0000000..a5fc4e0 --- /dev/null +++ b/dshell/plugins/dns/specialips.py @@ -0,0 +1,110 @@ +""" +Identifies DNS resolutions that fall into special IP spaces (i.e. private, +reserved, loopback, multicast, link-local, or unspecified). + +When found, it will print an alert for the request/response pair. The alert +will include the type of special IP in parentheses: + (loopback) + (private) + (reserved) + (multicast) + (link-local) + (unspecified) +""" + +from dshell.plugins import dnsplugin +from dshell.output.alertout import AlertOutput + +from pypacker.layer567 import dns + +import ipaddress + + +class DshellPlugin(dnsplugin.DNSPlugin): + + def __init__(self, *args, **kwargs): + super().__init__( + name="special-ips", + description="identify DNS resolutions that fall into special IP (IPv4 and IPv6) spaces (i.e. private, reserved, loopback, multicast, link-local, or unspecified)", + bpf="port 53", + author="dev195", + output=AlertOutput(label=__name__), + longdescription=""" +Identifies DNS resolutions that fall into special IP spaces (i.e. private, +reserved, loopback, multicast, link-local, or unspecified). + +When found, it will print an alert for the request/response pair. The alert +will include the type of special IP in parentheses: + (loopback) + (private) + (reserved) + (multicast) + (link-local) + (unspecified) + +For example, to look for responses with private IPs: + Dshell> decode -d specialips ~/pcap/SkypeIRC.cap |grep "(private)" + [special-ips] 2006-08-25 15:31:06 192.168.1.2:2128 -- 192.168.1.1:53 ** ID: 12579, A? voyager.home., A: 192.168.1.1 (private) (ttl 10000s) ** + +Finding can also be written to a separate pcap file by chaining: + Dshell> decode -d specialips+pcapwriter --pcapwriter_outfile="special-dns.pcap" ~/pcap/example.pcap +""", + ) + + + def dns_handler(self, conn, requests, responses): + """ + Stores the DNS request, then iterates over responses looking for + special IP addresses. If it finds one, it will print an alert for the + request/response pair. + """ + msg = [] + + if requests: + request_pkt = requests[-1] + request = request_pkt.pkt.highest_layer + id = request.id + for query in request.queries: + if query.type == dns.DNS_A: + msg.append("A? {}".format(query.name_s)) + elif query.type == dns.DNS_AAAA: + msg.append("AAAA? {}".format(query.name_s)) + + + if responses: + keep_responses = False + for response in responses: + response = response.pkt.highest_layer + for answer in response.answers: + if answer.type == dns.DNS_A or answer.type == dns.DNS_AAAA: + answer_ip = ipaddress.ip_address(answer.address) + msg_fields = {} + msg_format = "A: {ip} ({type}) (ttl {ttl}s)" + msg_fields['ip'] = str(answer_ip) + msg_fields['ttl'] = str(answer.ttl) + msg_fields['type'] = '' + if answer_ip.is_loopback: + msg_fields['type'] = 'loopback' + keep_responses = True + elif answer_ip.is_private: + msg_fields['type'] = 'private' + keep_responses = True + elif answer_ip.is_reserved: + msg_fields['type'] = 'reserved' + keep_responses = True + elif answer_ip.is_multicast: + msg_fields['type'] = 'multicast' + keep_responses = True + elif answer_ip.is_link_local: + msg_fields['type'] = 'link-local' + keep_responses = True + elif answer_ip.is_unspecified: + msg_fields['type'] = 'unspecified' + keep_responses = True + msg.append(msg_format.format(**msg_fields)) + if keep_responses: + msg.insert(0, "ID: {}".format(id)) + msg = ", ".join(msg) + self.write(msg, **conn.info()) + return conn, requests, responses + diff --git a/dshell/plugins/dnsplugin.py b/dshell/plugins/dnsplugin.py new file mode 100644 index 0000000..613ddf6 --- /dev/null +++ b/dshell/plugins/dnsplugin.py @@ -0,0 +1,131 @@ +""" +This is a base-level plugin intended to handle DNS lookups and responses + +It inherits from the base ConnectionPlugin and provides a new handler +function: dns_handler(conn, requests, responses) + +It automatically pairs request/response packets by ID and passes them to the +handler for a custom plugin, such as dns.py, to use. +""" + +import logging + +import dshell.core as dshell + +from pypacker.pypacker import dns_name_decode +from pypacker.layer567 import dns + +logger = logging.getLogger(__name__) + + +def basic_cname_decode(request, answer): + """ + DIRTY HACK ALERT + + This function exists to convert DNS CNAME responses into human-readable + strings. pypacker cannot currently convert these, so this one attempts + to do it. However, it is not complete and will only work for the most + common situations (i.e. no pointers, or pointers that only point to the + first request). + + Feed it the bytes (query.name) of the first request and the bytes for the + answer (answer.address) with a CNAME, and it will return the parsed string. + """ + + if b"\xc0" not in answer: + # short-circuit if there is no pointer + return dns_name_decode(answer) + # Get the offset into the question by grabbing the number after \xc0 + # Then, offset the offset by subtracting the query header length (12) + snip_index = answer[answer.index(b"\xc0") + 1] - 12 + # Grab the necessary piece from the request + snip = request[snip_index:] + # Reassemble and return + rebuilt = answer[:answer.index(b"\xc0")] + snip + return dns_name_decode(rebuilt) + + +class DNSPlugin(dshell.ConnectionPlugin): + """ + A base-level plugin that overwrites the connection_handler in + ConnectionPlugin. It provides a new handler function: dns_handler. + """ + + def __init__(self, **kwargs): + dshell.ConnectionPlugin.__init__(self, **kwargs) + + def connection_handler(self, conn): + requests = {} + responses = {} + id_to_blob_map = {} + id_to_packets_map = {} + + for blob in conn.blobs: + for pkt in blob.packets: + packet = pkt.pkt + if not isinstance(packet.highest_layer, dns.DNS): + # First packet is not DNS, so we don't care + blob.hidden = True + break + + dnsp = packet.highest_layer + id_to_blob_map.setdefault(dnsp.id, []).append(blob) + id_to_packets_map.setdefault(dnsp.id, []).append(pkt) + qr_flag = dnsp.flags >> 15 + rcode = dnsp.flags & 15 + setattr(pkt, 'qr', qr_flag) + setattr(pkt, 'rcode', rcode) +# print("{0:016b}".format(dnsp.flags)) + if qr_flag == dns.DNS_Q: + requests.setdefault(dnsp.id, []).append(pkt) + elif qr_flag == dns.DNS_A: + responses.setdefault(dnsp.id, []).append(pkt) + + all_ids = set(list(requests.keys()) + list(responses.keys())) + keep_connection = False + for id in all_ids: + request_list = requests.get(id, None) + response_list = responses.get(id, None) + dns_handler_out = self.dns_handler(conn, requests=request_list, responses=response_list) + if not dns_handler_out: + # remove packets from connections that dns_handler did not like + for blob in id_to_blob_map[id]: + for pkt in id_to_packets_map[id]: + try: + blob.packets.remove(pkt) + except ValueError: + continue + else: + for blob in id_to_blob_map[id]: + blob.hidden = False + try: + if dns_handler_out and not isinstance(dns_handler_out[0], dshell.Connection): + logger.warning("The output from {} dns_handler must be a list with a dshell.Connection as the first element! Chaining plugins from here may not be possible.".format(self.name)) + continue + except TypeError: + logger.warning("The output from {} dns_handler must be a list with a dshell.Connection as the first element! Chaining plugins from here may not be possible.".format(self.name)) + continue + keep_connection = True + if keep_connection: + return conn + + def dns_handler(self, conn, requests, responses): + """ + A placeholder. + + Plugins will be able to overwrite this to perform custom activites + on DNS data. + + It takes in a Connection, a list of requests (or None), and a list of + responses (or None). The requests and responses are not intermixed; + the responses in the list correspond to the requests according to ID. + + It should return a list containing the same types of values that came + in as arguments (i.e. return (conn, requests, responses)). This is + mostly a consistency thing, as only the Connection is passed along to + other plugins. + """ + return (conn, requests, responses) + + +DshellPlugin = None diff --git a/dshell/plugins/filter/__init__.py b/dshell/plugins/filter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dshell/plugins/filter/country.py b/dshell/plugins/filter/country.py new file mode 100644 index 0000000..aa66280 --- /dev/null +++ b/dshell/plugins/filter/country.py @@ -0,0 +1,103 @@ +""" +A filter for connections by IP address country code. Will generally be chained +with other plugins. +""" + +import dshell.core +from dshell.output.netflowout import NetflowOutput + +class DshellPlugin(dshell.core.ConnectionPlugin): + + def __init__(self, *args, **kwargs): + super().__init__( + name="Country Filter", + bpf='ip or ip6', + description="filter connections by IP address country code", + longdescription=""" +country: filter connections on geolocation (country code) + +Mandatory option: + + --country_code: specify (2 character) country code to filter on + +Default behavior: + + If either the client or server IP address matches the specified country, + the stream will be included. + +Modifier options: + + --country_neither: Include only streams where neither the client nor the + server IP address matches the specified country. + + --country_both: Include only streams where both the client AND the server + IP addresses match the specified country. + + --country_notboth: Include streams where the specified country is NOT BOTH + the client and server IP. Streams where it is one or + the other may be included. + + --country_alerts: Show alerts for this plugin (default: false) + + +Example: + + decode -d country+pcapwriter traffic.pcap --pcapwriter_outfile=USonly.pcap --country_code US + decode -d country+followstream traffic.pcap --country_code US --country_notboth +""", + author="tp", + output=NetflowOutput(label=__name__), + optiondict={ + 'code': {'type': str, 'help': 'two-char country code', 'metavar':'CC'}, + 'neither': {'action': 'store_true', 'help': 'neither (client/server) is in specified country'}, + 'both': {'action': 'store_true', 'help': 'both (client/server) ARE in specified country'}, + 'notboth': {'action': 'store_true', 'help': 'specified country is not both client and server'}, + 'alerts': {'action': 'store_true', 'default':False, 'help':'have this filter show alerts for matches'} + }, + ) + + def premodule(self): + # Several of the args are mutually exclusive + # Check if more than one is set, and print a warning if so + if (self.neither + self.both + self.notboth) > 1: + self.logger.warning("Can only use one of these args at a time: 'neither', 'both', or 'notboth'") + + def connection_handler(self, conn): + # If no country code specified, pass all traffic through + if not self.code: + return conn + + if self.neither: + if conn.clientcc != self.code and conn.servercc != self.code: + if self.alerts: self.write('neither', **conn.info()) + return conn + else: + return + + elif self.both: + if conn.clientcc == self.code and conn.servercc == self.code: + if self.alerts: self.write('both', **conn.info()) + return conn + else: + return + + elif self.notboth: + if ((conn.clientcc != self.code and conn.servercc == self.code) + or + (conn.clientcc == self.code and conn.servercc != self.code)): + if self.alerts: self.write('notboth', **conn.info()) + return conn + else: + return + + else: + if conn.clientcc == self.code or conn.servercc == self.code: + if self.alerts: self.write('match', **conn.info()) + return conn + + # no match + return None + + +if __name__ == "__main__": + print(DshellPlugin()) diff --git a/dshell/plugins/filter/track.py b/dshell/plugins/filter/track.py new file mode 100644 index 0000000..59ebc3c --- /dev/null +++ b/dshell/plugins/filter/track.py @@ -0,0 +1,153 @@ +""" +Only follows connections that match user-provided IP addresses and ports. Is +generally chained with other plugins. +""" + +import ipaddress +import sys + +import dshell.core +from dshell.output.alertout import AlertOutput + +class DshellPlugin(dshell.core.ConnectionPlugin): + def __init__(self, **kwargs): + super().__init__( + name="track", + author="twp,dev195", + description="Only follow connections that match user-provided IP addresses and ports", + longdescription="""Only follow connections that match user-provided IP addresses + +IP addresses can be specified with --track_source and --track_target. +Multiple IPs can be used with commas (e.g. --track_source=192.168.1.1,127.0.0.1). +Ports can be included with IP addresses by joining them with a 'p' (e.g. --track_target=192.168.1.1p80,127.0.0.1). +Ports can be used alone with just a 'p' (e.g. --track_target=p53). +CIDR notation is okay (e.g. --track_source=196.168.0.0/16). + +--track_source : used to limit connections by the IP that initiated the connection (usually the client) +--trace_target : used to limit connections by the IP that received the connection (usually the server) +--track_alerts : used to display optional alerts indicating when a connection starts/ends""", + bpf="ip or ip6", + output=AlertOutput(label=__name__), + optiondict={ + "target": { + "default": [], + "action": "append", + "metavar": "IPpPORT"}, + "source": { + "default": [], + "action": "append", + "metavar": "IPpPORT"}, + "alerts": { + "action": "store_true"} + } + ) + self.sources = [] + self.targets = [] + + def __split_ips(self, input): + """ + Used to split --track_target and --track_source arguments into + list-of-lists used in the connection handler + """ + return_val = [] + for piece in input.split(','): + if 'p' in piece: + ip, port = piece.split('p', 1) + try: + port = int(port) + except ValueError as e: + self.error("Could not parse port number in {!r} - {!s}".format(piece, e)) + sys.exit(1) + if 0 < port > 65535: + self.error("Could not parse port number in {!r} - must be in valid port range".format(piece)) + sys.exit(1) + else: + ip, port = piece, None + if '/' in ip: + try: + ip = ipaddress.ip_network(ip) + except ValueError as e: + self.error("Could not parse CIDR netrange - {!s}".format(e)) + sys.exit(1) + elif ip: + try: + ip = ipaddress.ip_address(ip) + except ValueError as e: + self.error("Could not parse IP address - {!s}".format(e)) + sys.exit(1) + else: + ip = None + return_val.append((ip, port)) + return return_val + + def __check_ips(self, masterip, masterport, checkip, checkport): + "Checks IPs and ports for matches against the user-selected values" + # masterip, masterport are the values selected by the user + # checkip, checkport are the values to be checked against masters + ip_okay = False + port_okay = False + + if masterip is None: + ip_okay = True + elif (isinstance(masterip, (ipaddress.IPv4Network, ipaddress.IPv6Network)) + and checkip in masterip): + ip_okay = True + elif (isinstance(masterip, (ipaddress.IPv4Address, ipaddress.IPv6Address)) + and masterip == checkip): + ip_okay = True + + if masterport is None: + port_okay = True + elif masterport == checkport: + port_okay = True + + if port_okay and ip_okay: + return True + else: + return False + + + def premodule(self): + if self.target: + for tstr in self.target: + self.targets.extend(self.__split_ips(tstr)) + if self.source: + for sstr in self.source: + self.sources.extend(self.__split_ips(sstr)) + self.logger.debug("targets: {!s}".format(self.targets)) + self.logger.debug("sources: {!s}".format(self.sources)) + + def connection_handler(self, conn): + if self.targets: + conn_okay = False + for target in self.targets: + targetip = target[0] + targetport = target[1] + serverip = ipaddress.ip_address(conn.serverip) + serverport = conn.serverport + if self.__check_ips(targetip, targetport, serverip, serverport): + conn_okay = True + break + if not conn_okay: + return + + if self.sources: + conn_okay = False + for source in self.sources: + sourceip = source[0] + sourceport = source[1] + clientip = ipaddress.ip_address(conn.clientip) + clientport = conn.clientport + if self.__check_ips(sourceip, sourceport, clientip, clientport): + conn_okay = True + break + if not conn_okay: + return + + if self.alerts: + self.write("matching connection", **conn.info()) + + return conn + +if __name__ == "__main__": + print(DshellPlugin()) diff --git a/dshell/plugins/flows/__init__.py b/dshell/plugins/flows/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dshell/plugins/flows/dataflows.py b/dshell/plugins/flows/dataflows.py new file mode 100644 index 0000000..ea31d51 --- /dev/null +++ b/dshell/plugins/flows/dataflows.py @@ -0,0 +1,36 @@ +""" +Displays netflows that have at least 1 byte transferred, by default. +Bytes threshold can be updated by the user. +""" + +import dshell.core +from dshell.output.netflowout import NetflowOutput + +class DshellPlugin(dshell.core.ConnectionPlugin): + + def __init__(self): + super().__init__( + name="dataflows", + description="Display netflows that have at least 1 byte transferred", + author="amm", + output=NetflowOutput(label=__name__), + optiondict={ + 'size': { + 'type': int, + 'default': 1, + 'metavar': 'SIZE', + 'help': 'number of bytes transferred (default: 1)'} + } + ) + + def premodule(self): + if self.size <= 0: + self.warn("Cannot have a size that's less than or equal to zero (size: {}). Setting to 1.".format(self.size)) + self.size = 1 + + def connection_handler(self, conn): + if conn.clientbytes + conn.serverbytes >= self.size: + self.write(**conn.info()) + return conn + + diff --git a/dshell/plugins/flows/largeflows.py b/dshell/plugins/flows/largeflows.py new file mode 100644 index 0000000..6988a57 --- /dev/null +++ b/dshell/plugins/flows/largeflows.py @@ -0,0 +1,38 @@ +""" +Displays netflows that have at least 1MB transferred, by default. +Megabyte threshold can be updated by the user. +""" + +import dshell.core +from dshell.output.netflowout import NetflowOutput + +class DshellPlugin(dshell.core.ConnectionPlugin): + + def __init__(self): + super().__init__( + name="large-flows", + description="Display netflows that have at least 1MB transferred", + author="bg", + output=NetflowOutput(label=__name__), + optiondict={ + 'size': { + 'type': float, + 'default': 1, + 'metavar': 'SIZE', + 'help': 'number of megabytes transferred (default: 1)'} + } + ) + + def premodule(self): + if self.size <= 0: + self.logger.warning("Cannot have a size that's less than or equal to zero (size: {}). Setting to 1.".format(self.size)) + self.size = 1 + self.min = 1048576 * self.size + self.logger.debug("Input: {}, Final size: {} bytes".format(self.size, self.min)) + + def connection_handler(self, conn): + if conn.clientbytes + conn.serverbytes >= self.min: + self.write(**conn.info()) + return conn + + diff --git a/dshell/plugins/flows/longflows.py b/dshell/plugins/flows/longflows.py new file mode 100644 index 0000000..729b978 --- /dev/null +++ b/dshell/plugins/flows/longflows.py @@ -0,0 +1,38 @@ +""" +Displays netflows that have a duration of at least 5 minutes. +Minute threshold can be updated by the user. +""" + +import dshell.core +from dshell.output.netflowout import NetflowOutput + +class DshellPlugin(dshell.core.ConnectionPlugin): + + def __init__(self): + super().__init__( + name="long-flows", + description="Display netflows that have a duration of at least 5 minutes", + author="bg", + output=NetflowOutput(label=__name__), + optiondict={ + "len": { + "type": float, + "default": 5, + "help": "set minimum connection time to MIN minutes (default: 5)", + "metavar": "MIN", + } + } + ) + + def premodule(self): + if self.len <= 0: + self.logger.warning("Cannot have a time that's less than or equal to zero (size: {}). Setting to 5.".format(self.len)) + self.len = 5 + self.secs = 60 * self.len + + def connection_handler(self, conn): + tdelta = (conn.endtime - conn.starttime).total_seconds() + if tdelta >= self.secs: + self.write(**conn.info()) + return conn + diff --git a/dshell/plugins/flows/netflow.py b/dshell/plugins/flows/netflow.py new file mode 100644 index 0000000..df9a7a8 --- /dev/null +++ b/dshell/plugins/flows/netflow.py @@ -0,0 +1,43 @@ +""" +Collects and displays statistics about connections (a.k.a. flow data) +""" + +import dshell.core +from dshell.output.netflowout import NetflowOutput + +class DshellPlugin(dshell.core.ConnectionPlugin): + def __init__(self, *args, **kwargs): + super().__init__( + name="Netflow", + description="Collects and displays flow statistics about connections", + author="dev195", + bpf="ip or ip6", + output=NetflowOutput(label=__name__), + longdescription=""" +Collect and display flow statistics about connections. + +It will reassemble connections and print one row for each flow keyed by +address four-tuple. Each row, by default, will have the following fields: + +- Start Time : the timestamp of the first packet for a connection +- Client IP : the IP address of the host that initiated the connection +- Server IP : the IP address of the host that receives the connection + (note: client/server designation is based on first packet seen for a connection) +- Client Country : the country code for the client IP address +- Server Country : the country code for the server IP address +- Protocol : the layer-3 protocol of the connection +- Client Port: port number used by client +- Server Port: port number used by server +- Client Packets : number of data-carrying packets from the client +- Server Packets : number of data-carrying packets from the server + (note: packet counts ignore packets without data, e.g. handshakes, ACKs, etc.) +- Client Bytes : total bytes sent by the client +- Server Bytes : total bytes sent by the server +- Duration : time between the first packet and final packet of a connection +- Message Data: extra field not used by this plugin +""" + ) + + def connection_handler(self, conn): + self.write(**conn.info()) + return conn diff --git a/dshell/plugins/flows/reverseflows.py b/dshell/plugins/flows/reverseflows.py new file mode 100644 index 0000000..a68f60d --- /dev/null +++ b/dshell/plugins/flows/reverseflows.py @@ -0,0 +1,84 @@ +""" +Generate an alert when a client transmits more data than the server. + +Additionally, the user can specify a threshold. This means that an alert +will be generated if the client transmits more than three times as much data +as the server. + +The default threshold value is 3.0, meaning that any client transmits +more than three times as much data as the server will generate an alert. + +Examples: +1) decode -d reverse-flow + Generates an alert for client transmissions that are three times + greater than the server transmission. + +2) decode -d reverse-flow --reverse-flow_threshold 61 + Generates an alert for all client transmissions that are 61 times + greater than the server transmission + +3) decode -d reverse-flow --reverse-flow_threshold 61 --reverse-flow_zero + Generates an alert for all client transmissions that are 61 times greater + than the server transmission. +""" + +import dshell.core +from dshell.output.alertout import AlertOutput + +class DshellPlugin(dshell.core.ConnectionPlugin): + + def __init__(self): + super().__init__( + name="reverse-flows", + description="Generate an alert if the client transmits more data than the server", + author="me", + bpf="tcp or udp", + output=AlertOutput(label=__name__), + optiondict={ + 'threshold': {'type':float, 'default':3.0, + 'help':'Alerts if client transmits more than threshold times the data of the server'}, + 'minimum': {'type':int, 'default':0, + 'help':'alert on client transmissions larger than min bytes [default: 0]'}, + 'zero': {'action':'store_true', 'default':False, + 'help':'alert if the server transmits zero bytes [default: false]'}, + }, + longdescription=""" +Generate an alert when a client transmits more data than the server. + +Additionally, the user can specify a threshold. This means that an alert +will be generated if the client transmits more than three times as much data +as the server. + +The default threshold value is 3.0, meaning that any client transmits +more than three times as much data as the server will generate an alert. + +Examples: +1) decode -d reverse-flow + Generates an alert for client transmissions that are three times + greater than the server transmission. + +2) decode -d reverse-flow --reverse-flow_threshold 61 + Generates an alert for all client transmissions that are 61 times + greater than the server transmission + +3) decode -d reverse-flow --reverse-flow_threshold 61 --reverse-flow_zero + Generates an alert for all client transmissions that are 61 times greater + than the server transmission. + """, + ) + + def premodule(self): + if self.threshold < 0: + self.logger.warning("Cannot have a negative threshold. Defaulting to 3.0. (threshold: {0})".format(self.threshold)) + self.threshold = 3.0 + elif not self.threshold: + self.logger.warning("Threshold not set. Displaying all client-server transmissions (threshold: {0})".format(self.threshold)) + + def connection_handler(self, conn): + if conn.clientbytes < self.minimum: + return + + if self.zero or (conn.serverbytes and float(conn.clientbytes)/conn.serverbytes > self.threshold): + self.write('client sent {:>6.2f} more than the server'.format(conn.clientbytes/float(conn.serverbytes)), **conn.info(), dir_arrow="->") + return conn + diff --git a/dshell/plugins/flows/toptalkers.py b/dshell/plugins/flows/toptalkers.py new file mode 100644 index 0000000..b26f2df --- /dev/null +++ b/dshell/plugins/flows/toptalkers.py @@ -0,0 +1,83 @@ +""" +Finds the top-talkers in a file or on an interface based on byte count. +""" + +import dshell.core +from dshell.output.alertout import AlertOutput +from dshell.util import human_readable_filesize + +class DshellPlugin(dshell.core.ConnectionPlugin): + + def __init__(self, *args, **kwargs): + super().__init__( + name="Top Talkers", + description="Find top-talkers based on byte count", + author="dev195", + bpf="tcp or udp", + output=AlertOutput(label=__name__), + optiondict={ + "top_x": { + "type": int, + "default": 20, + "help": "Only display the top X results (default: 20)", + "metavar": "X" + }, + "total": { + "action": "store_true", + "help": "Sum byte counts from both directions instead of separate entries for individual directions" + }, + "h": { + "action": "store_true", + "help": "Print byte counts in human-readable format" + } + }, + longdescription=""" +Finds top 20 connections with largest transferred byte count. + +Can be configured to display an arbitrary Top X list with arguments. + +Does not pass connections down plugin chain. +""" + ) + + def premodule(self): + """ + Initialize a list to hold the top X talkers + Format of each entry: + (bytes, direction, Connection object) + """ + self.top_talkers = [(0, '---', None)] + + def connection_handler(self, conn): + if self.total: + # total up the client and server bytes + self.__process_bytes(conn.clientbytes + conn.serverbytes, '<->', conn) + else: + # otherwise, treat client and server bytes separately + self.__process_bytes(conn.clientbytes, '-->', conn) + self.__process_bytes(conn.serverbytes, '<--', conn) + + def postmodule(self): + "Iterate over the entries in top_talkers list and print them" + for bytecount, direction, conn in self.top_talkers: + if conn is None: + break + if self.h: + byte_display = human_readable_filesize(bytecount) + else: + byte_display = "{} B".format(bytecount) + msg = "client {} server {}".format(direction, byte_display) + self.write(msg, **conn.info(), dir_arrow="->") + + def __process_bytes(self, bytecount, direction, conn): + """ + Check if the bytecount for a connection belongs in top_talkers + If so, insert it into the list and pop off the lowest entry + """ + for i, oldbytecount in enumerate(self.top_talkers): + if bytecount >= oldbytecount[0]: + self.top_talkers.insert(i, (bytecount, direction, conn)) + break + + while len(self.top_talkers) > self.top_x: + self.top_talkers.pop(-1) diff --git a/dshell/plugins/ftp/__init__.py b/dshell/plugins/ftp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dshell/plugins/ftp/ftp.py b/dshell/plugins/ftp/ftp.py new file mode 100644 index 0000000..29328f2 --- /dev/null +++ b/dshell/plugins/ftp/ftp.py @@ -0,0 +1,352 @@ +""" +Goes through TCP connections and tries to find FTP control channels and +associated data channels. Optionally, it will write out any file data it +sees into a separate directory. + +If a data connection is seen, it prints a message indicating the user, pass, +and file requested. If the --ftp_dump flag is set, it also dumps the file into the +--ftp_outdir directory. +""" + +import dshell.core +import dshell.util +from dshell.output.alertout import AlertOutput + +import os +import re +import sys + +# constants for channel type +CTRL_CONN = 0 +DATA_CONN = 1 + +class DshellPlugin(dshell.core.ConnectionPlugin): + + def __init__(self): + super().__init__( + name="ftp", + description="alerts on FTP traffic and, optionally, rips files", + longdescription=""" +Goes through TCP connections and tries to find FTP control channels and +associated data channels. Optionally, it will write out any file data it +sees into a separate directory. + +If a data connection is seen, it prints a message indicating the user, pass, +and file requested. If the --ftp_dump flag is set, it also dumps the file into the +--ftp_outdir directory. +""", + author="amm,dev195", + bpf="tcp", + output=AlertOutput(label=__name__), + optiondict={ + "ports": { + 'help': 'comma-separated list of ports to watch for control connections (default: 21)', + 'metavar': 'PORT,PORT,PORT,[...]', + 'default': '21'}, + "dump": { + 'action': 'store_true', + 'help': 'dump files from stream'}, + "outdir": { + 'help': 'directory to write output files (default: "ftpout")', + 'metavar': 'DIRECTORY', + 'default': 'ftpout'} + } + ) + + def __update_bpf(self): + """ + Dynamically change the BPF to allow processing of data transfer + channels. + """ + dynfilters = [] + for conn, metadata in self.conns.items(): + try: + dynfilters += ["(host %s and host %s)" % metadata["tempippair"]] + except (KeyError, TypeError): + continue + for a, p in self.data_channel_map.keys(): + dynfilters += ["(host %s and port %d)" % (a, p)] + self.bpf = "(%s) and ((%s)%s)" % ( + self.original_bpf, + " or ".join( "port %d" % p for p in self.control_ports ), + " or " + " or ".join(dynfilters) if dynfilters else "" + ) + self.recompile_bpf() + + def premodule(self): + # dictionary containing metadata for connections + self.conns = {} + # dictionary mapping data channels (host, port) to their control channels + self.data_channel_map = {} + # ports used for control channels + self.control_ports = set() + # Original BPF without manipulation + self.original_bpf = self.bpf + # set control ports using user-provided info + for p in self.ports.split(','): + try: + self.control_ports.add(int(p)) + except ValueError as e: + self.error("{!r} is not a valid port. Skipping.".format(p)) + if not self.control_ports: + self.error("Could not find any control ports. At least one must be set for this plugin.") + sys.exit(1) + + # create output directory + # break if it cannot be created + if self.dump and not os.path.exists(self.outdir): + try: + os.makedirs(self.outdir) + except (IOError, OSError) as e: + self.error("Could not create output directory: {!r}: {!s}" + .format(self.outdir, e)) + sys.exit(1) + + def connection_init_handler(self, conn): + # Create metadata containers for any new connections + if conn.serverport in self.control_ports: + self.conns[conn.addr] = { + 'mode': CTRL_CONN, + 'user': '', + 'pass': '', + 'path': [], + 'datachan': None, + 'lastcommand': '', + 'tempippair': None, + 'filedata': None, + 'file': ['', '', ''] + } + elif self.dump and (conn.clientip, conn.clientport) in self.data_channel_map: + self.conns[conn.addr] = { + 'mode': DATA_CONN, + 'ctrlchan': self.data_channel_map[(conn.clientip, conn.clientport)], + 'filedata': None + } + elif self.dump and (conn.serverip, conn.serverport) in self.data_channel_map: + self.conns[conn.addr] = { + 'mode': DATA_CONN, + 'ctrlchan': self.data_channel_map[(conn.serverip, conn.serverport)], + 'filedata': None + } + elif self.dump: + # This is a data connection with an unknown control connection. It + # may be a passive mode transfer without known port info, yet. + self.conns[conn.addr] = { + 'mode': DATA_CONN, + 'ctrlchan': None, + 'filedata': None + } + + def connection_close_handler(self, conn): + # After data channel closes, store file content in control channel's + # 'filedata' field. + # Control channel will write it to disk after it determines the + # filename. + try: + info = self.conns[conn.addr] + except KeyError: + return + + if self.dump and info['mode'] == DATA_CONN: + # find the associated control channel + if info['ctrlchan'] == None: + if (conn.clientip, conn.clientport) in self.data_channel_map: + info['ctrlchan'] = self.data_channel_map[(conn.clientip, conn.clientport)] + if (conn.serverip, conn.serverport) in self.data_channel_map: + info['ctrlchan'] = self.data_channel_map[(conn.serverip, conn.serverport)] + try: + ctrlchan = self.conns[info['ctrlchan']] + except KeyError: + return + # add data to control channel dictionary + for blob in conn.blobs: + if ctrlchan['filedata']: + ctrlchan['filedata'] += blob.data + else: + ctrlchan['filedata'] = blob.data + # update port list and data channel knowledge + if (conn.serverip, conn.serverport) == ctrlchan['datachan']: + del self.data_channel_map[ctrlchan['datachan']] + ctrlchan['datachan'] = None + self.__update_bpf() + if (conn.clientip, conn.clientport) == ctrlchan['datachan']: + del self.data_channel_map[ctrlchan['datachan']] + ctrlchan['datachan'] = None + self.__update_bpf() + del self.conns[conn.addr] + + elif info['mode'] == CTRL_CONN: + # clear control channels if they've been alerted on + if info['file'] == None: + del self.conns[conn.addr] + + def postmodule(self): + for addr, info in self.conns.items(): + if self.dump and 'filedata' in info and info['filedata']: + origname = info['file'][0] + '_' + os.path.join(*info['file'][1:3]) + outname = dshell.util.gen_local_filename(self.outdir, origname) + with open(outname, 'wb') as fh: + fh.write(info['filedata']) + numbytes = len(info['filedata']) + info['filedata'] = None + info['outfile'] = outname + msg = 'User: %s, Pass: %s, %s File: %s (Incomplete: %d bytes written to %s)' % (info['user'], info['pass'], info['file'][0], os.path.join(*info['file'][1:3]), numbytes, os.path.basename(outname)) + self.write(msg, **info) + + + def blob_handler(self, conn, blob): + try: + info = self.conns[conn.addr] + except KeyError: + # connection was not initialized correctly + # set the blob to hidden and move on + blob.hidden = True + return + + if info['mode'] == DATA_CONN: + return conn, blob + + try: + data = blob.data + data = data.decode('ascii') + except UnicodeDecodeError as e: + # Could not convert command data to readable ASCII + blob.hidden = True + return + + if blob.direction == 'cs': + # client-to-server: try and get the command issued + if ' ' not in data.rstrip(): + command = data.rstrip() + param = '' + else: + command, param = data.rstrip().split(' ', 1) + command = command.upper() + info['lastcommand'] = command + + if command == 'USER': + info['user'] = param + + elif command == 'PASS': + info['pass'] = param + + elif command == 'CWD': + info['path'].append(param) + + elif command == 'PASV' or command == 'EPSV': + if self.dump: + # Temporarily store the pair of IP addresses + # to open up the BPF filter until blob_handler processes + # the response with the full IP/Port information. + # Note: Due to the way blob processing works, we don't + # get this information until after the data channel is + # established. + info['tempippair'] = tuple( + sorted((conn.clientip, conn.serverip)) + ) + self.__update_bpf() + + # For file transfers (including LIST), store tuple + # (Direction, Path, Filename) in info['file'] + elif command == 'LIST': + if param == '': + info['file'] = ( + 'RETR', os.path.normpath(os.path.join(*info['path'])) + if len(info['path']) + else '', 'LIST' + ) + else: + info['file'] = ( + 'RETR', os.path.normpath(os.path.join(os.path.join(*info['path']), param)) + if len(info['path']) + else '', 'LIST' + ) + elif command == 'RETR': + info['file'] = ( + 'RETR', os.path.normpath(os.path.join(*info['path'])) + if len(info['path']) + else '', param + ) + elif command == 'STOR': + info['file'] = ( + 'STOR', os.path.normpath(os.path.join(*info['path'])) + if len(info['path']) + else '', param + ) + + # Responses + else: + # Rollback directory change unless 2xx response + if info['lastcommand'] == 'CWD' and data[0] != '2': + info['path'].pop() + # Write out files upon resonse to transfer commands + if info['lastcommand'] in ('LIST', 'RETR', 'STOR'): + if self.dump and info['filedata']: + origname = info['file'][0] + '_' + os.path.join(*info['file'][1:3]) + outname = dshell.util.gen_local_filename(self.outdir, origname) + with open(outname, 'wb') as fh: + fh.write(info['filedata']) + numbytes = len(info['filedata']) + info['filedata'] = None + info['outfile'] = outname + info.update(conn.info()) + msg = 'User: "{}", Pass: "{}", {} File: {} ({:,} bytes written to {})'.format( + info['user'], + info['pass'], + info['file'][0], + os.path.join(*info['file'][1:3]), + numbytes, + os.path.basename(outname) + ) + else: + info.update(conn.info()) + msg = 'User: "{}", Pass: "{}", {} File: {}'.format( + info['user'], + info['pass'], + info['file'][0], + os.path.join(*info['file'][1:3]) + ) + if data[0] not in ('1','2'): + msg += ' ({})'.format(data.rstrip()) + info['ts'] = blob.ts + if (blob.sip == conn.sip): + self.write(msg, **info, dir_arrow="->") + else: + self.write(msg, **info, dir_arrow="<-") + info['file'] = None + + # Handle EPSV mode port setting + if info['lastcommand'] == 'EPSV' and data[0] == '2': + ret = re.findall('\(\|\|\|\d+\|\)', data) + # TODO delimiters other than pipes + if ret: + tport = int(ret[0].split('|')[3]) + info['datachan'] = (conn.serverip, tport) + if self.dump: + self.data_channel_map[(conn.serverip, tport)] = conn.addr + info['tempippair'] = None + self.__update_bpf() + + # Look for ip/port information, assuming PSV response + ret = re.findall('\d+,\d+,\d+,\d+,\d+\,\d+', data) + if len(ret) == 1: + tip, tport = self.calculateTransfer(ret[0]) # transfer ip, transfer port + info['datachan'] = (tip, tport) # Update this control channel's knowledge of currently working data channel + if self.dump: + self.data_channel_map[(tip,tport)] = conn.addr # Update plugin's global datachan knowledge + info['tempippair'] = None + self.__update_bpf() + + return conn, blob + + + def calculateTransfer(self, val): + # calculate passive FTP data port + tmp = val.split(',') + ip = '.'.join(tmp[:4]) + port = int(tmp[4])*256 + int(tmp[5]) + return ip, port + + +if __name__ == "__main__": + print(DshellPlugin()) diff --git a/dshell/plugins/http/__init__.py b/dshell/plugins/http/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dshell/plugins/http/httpdump.py b/dshell/plugins/http/httpdump.py new file mode 100644 index 0000000..71362a5 --- /dev/null +++ b/dshell/plugins/http/httpdump.py @@ -0,0 +1,177 @@ +""" +Presents useful information points for HTTP sessions +""" + +import dshell.core +import dshell.util +from dshell.plugins.httpplugin import HTTPPlugin +from dshell.output.colorout import ColorOutput + +from urllib.parse import parse_qs +from http import cookies + +class DshellPlugin(HTTPPlugin): + def __init__(self): + super().__init__( + name="httpdump", + description="Dump useful information about HTTP sessions", + bpf="tcp and (port 80 or port 8080 or port 8000)", + author="amm", + output=ColorOutput(label=__name__), + optiondict={ + "maxurilen": { + "type": int, + "default": 30, + "metavar": "LENGTH", + "help": "Truncate URLs longer than LENGTH (default: 30). Set to 0 for no truncating."}, + "maxpost": { + "type": int, + "default": 1000, + "metavar": "LENGTH", + "help": "Truncate POST bodies longer than LENGTH characters (default: 1000). Set to 0 for no truncating."}, + "maxcontent": { + "type": int, + "default": 0, + "metavar": "LENGTH", + "help": "Truncate response bodies longer than LENGTH characters (default: no truncating). Set to 0 for no truncating."}, + "showcontent": { + "action": "store_true", + "help": "Display response body"}, + "showhtml": { + "action": "store_true", + "help": "Display only HTML results"}, + "urlfilter": { + "type": str, + "default": None, + "metavar": "REGEX", + "help": "Filter to URLs matching this regular expression"} + } + ) + + def premodule(self): + if self.urlfilter: + import re + self.urlfilter = re.compile(self.urlfilter) + + def http_handler(self, conn, request, response): + host = request.headers.get('host', conn.serverip) + url = host + request.uri + pretty_url = url + + # separate URL-encoded data from the location + if '?' in request.uri: + uri_location, uri_data = request.uri.split('?', 1) + pretty_url = host + uri_location + else: + uri_location, uri_data = request.uri, "" + + # Check if the URL matches a user-defined filter + if self.urlfilter and not self.urlfilter.search(pretty_url): + return + + if self.maxurilen > 0 and len(uri_location) > self.maxurilen: + uri_location = "{}[truncated]".format(uri_location[:self.maxurilen]) + pretty_url = host + uri_location + + # Set the first line of the alert to show some basic metadata + if response == None: + msg = ["{} (NO RESPONSE) {}".format(request.method, pretty_url)] + else: + msg = ["{} ({}) {} ({})".format(request.method, response.status, pretty_url, response.headers.get("content-type", "[no content-type]"))] + + # Determine if there is any POST data from the client and parse + if request and request.method == "POST": + try: + post_params = parse_qs(request.body.decode("utf-8"), keep_blank_values=True) + # If parse_qs only returns a single element with a null + # value, it's probably an eroneous evaluation. Most likely + # base64 encoded payload ending in an '=' character. + if len(post_params) == 1 and list(post_params.values()) == [["\x00"]]: + post_params = request.body + except UnicodeDecodeError: + post_params = request.body + else: + post_params = {} + + # Get some additional useful data + url_params = parse_qs(uri_data, keep_blank_values=True) + referer = request.headers.get("referer", None) + client_cookie = cookies.SimpleCookie(request.headers.get("cookie", "")) + server_cookie = cookies.SimpleCookie(response.headers.get("cookie", "")) + + # Piece together the alert message + if referer: + msg.append("Referer: {}".format(referer)) + + if client_cookie: + msg.append("Client Transmitted Cookies:") + for k, v in client_cookie.items(): + msg.append("\t{} -> {}".format(k, v.value)) + + if server_cookie: + msg.append("Server Set Cookies:") + for k, v in server_cookie.items(): + msg.append("\t{} -> {}".format(k, v.value)) + + if url_params: + msg.append("URL Parameters:") + for k, v in url_params.items(): + msg.append("\t{} -> {}".format(k, v)) + + if post_params: + if isinstance(post_params, dict): + msg.append("POST Parameters:") + for k, v in post_params.items(): + msg.append("\t{} -> {}".format(k, v)) + else: + msg.append("POST Data:") + msg.append(dshell.util.printable_text(str(post_params))) + elif request.body: + msg.append("POST Body:") + request_body = dshell.util.printable_text(request.body) + if self.maxpost > 0 and len(request.body) > self.maxpost: + msg.append("{}[truncated]".format(request_body[:self.maxpost])) + else: + msg.append(request_body) + + if self.showcontent or self.showhtml: + if self.showhtml and 'html' not in response.headers.get('content-type', ''): + return + if 'gzip' in response.headers.get('content-encoding', ''): + # TODO gunzipping + content = '(gzip encoded)\n{}'.format(response.body) + else: + content = response.body + content = dshell.util.printable_text(content) + if self.maxcontent and len(content) > self.maxcontent: + content = "{}[truncated]".format(content[:self.maxcontent]) + msg.append("Body Content:") + msg.append(content) + + # Display the start and end times based on Blob instead of Connection + kwargs = conn.info() + if request: + kwargs['starttime'] = request.blob.starttime + kwargs['clientbytes'] = len(request.blob.data) + else: + kwargs['starttime'] = None + kwargs['clientbytes'] = 0 + if response: + kwargs['endtime'] = response.blob.endtime + kwargs['serverbytes'] = len(response.blob.data) + else: + kwargs['endtime'] = None + kwargs['serverbytes'] = 0 + + if post_params: + kwargs['post_params'] = post_params + if url_params: + kwargs['url_params'] = url_params + if client_cookie: + kwargs['client_cookie'] = client_cookie + if server_cookie: + kwargs['server_cookie'] = server_cookie + + self.write('\n'.join(msg), **kwargs) + + return conn, request, response diff --git a/dshell/plugins/http/joomla.py b/dshell/plugins/http/joomla.py new file mode 100644 index 0000000..e0a8225 --- /dev/null +++ b/dshell/plugins/http/joomla.py @@ -0,0 +1,86 @@ +""" +Detect and dissect malformed HTTP headers targeting Joomla + +https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2015-8562 +""" + +from dshell.plugins.httpplugin import HTTPPlugin +from dshell.output.alertout import AlertOutput + +import re + +class DshellPlugin(HTTPPlugin): + def __init__(self): + super().__init__( + name="Joomla CVE-2015-8562", + author="bg", + description='detect attempts to enumerate MS15-034 vulnerable IIS servers', + bpf='tcp and (port 80 or port 8080 or port 8000)', + output=AlertOutput(label=__name__), + optiondict={ + "raw_payload": { + "action": "store_true", + "help": "return the raw payload (do not attempt to decode chr encoding)", + } + }, + longdescription=''' +Detect and dissect malformed HTTP headers targeting Joomla + +https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2015-8562 + +Usage Examples: +--------------- + +Dshell> decode -d joomla *.pcap +[Joomla CVE-2015-8562] 2015-12-15 20:17:18 192.168.1.119:43865 <- 192.168.1.139:80 ** x-forwarded-for -> system('touch /tmp/2'); ** + +The module assumes the cmd payload is encoded using chr. To turn this off run: + +Dshell> decode -d joomla --joomla_raw_payload *.pcap +[Joomla CVE-2015-8562] 2015-12-15 20:17:18 192.168.1.119:43865 <- 192.168.1.139:80 ** x-forwarded-for -> "eval(chr(115).chr(121).chr(115).chr(116).chr(101).chr(109).chr(40).chr(39).chr(116).chr(111).chr(117).chr(99).chr(104).chr(32).chr(47).chr(116).chr(109).chr(112).chr(47).chr(50).chr(39).chr(41).chr(59)); ** +''', + ) + + # Indicator of (potential) compromise + self.ioc = "JFactory::getConfig();exit" + self.ioc_bytes = bytes(self.ioc, "ascii") + + def attempt_decode(self, cmd): + ptext = '' + for c in re.findall('\d+', cmd): + ptext += chr(int(c)) + return ptext + + def parse_cmd(self, data): + start = data.find('"feed_url";')+11 + end = data.find(self.ioc) + chunk = data[start:end] + + try: + cmd = chunk.split(':')[-1] + if self.raw_payload: + return cmd + + plaintext_cmd = self.attempt_decode(cmd) + return plaintext_cmd + except: + return None + + def http_handler(self, conn, request, response): + if not request: + return + + if self.ioc_bytes not in request.blob.data: + # indicator of (potential) compromise is not here + return + + # there is an attempt to exploit Joomla! + + # The Joomla exploit could be sent any HTTP header field + for hdr, val in request.headers.items(): + if self.ioc in val: + cmd = self.parse_cmd(val) + if cmd: + self.alert('{} -> {}'.format(hdr, cmd), **conn.info()) + return conn, request, response + diff --git a/dshell/plugins/http/ms15-034.py b/dshell/plugins/http/ms15-034.py new file mode 100644 index 0000000..9210603 --- /dev/null +++ b/dshell/plugins/http/ms15-034.py @@ -0,0 +1,64 @@ +""" +Proof-of-concept code to detect attempts to enumerate MS15-034 vulnerable +IIS servers and/or cause a denial of service. Each event will generate an +alert that prints out the HTTP Request method and the range value contained +with the HTTP stream. +""" + +from dshell.plugins.httpplugin import HTTPPlugin +from dshell.output.alertout import AlertOutput + +class DshellPlugin(HTTPPlugin): + def __init__(self): + super().__init__( + name="ms15-034", + author="bg", + description='detect attempts to enumerate MS15-034 vulnerable IIS servers', + bpf='tcp and (port 80 or port 8080 or port 8000)', + output=AlertOutput(label=__name__), + longdescription=''' +Proof-of-concept code to detect attempts to enumerate MS15-034 vulnerable +IIS servers and/or cause a denial of service. Each event will generate an +alert that prints out the HTTP Request method and the range value contained +with the HTTP stream. + +Usage: +decode -d ms15-034 -q *.pcap +decode -d ms15-034 -i -q + +References: +https://technet.microsoft.com/library/security/ms15-034 +https://ma.ttias.be/remote-code-execution-via-http-request-in-iis-on-windows/ +''', + ) + + + def http_handler(self, conn, request, response): + if response == None: + # Denial of Service (no server response) + try: + rangestr = request.headers.get("range", '') + # check range value to reduce false positive rate + if not rangestr.endswith('18446744073709551615'): + return + except: + return + self.write('MS15-034 DoS [Request Method: "{0}" URI: "{1}" Range: "{2}"]'.format(request.method, request.uri, rangestr), conn.info()) + return conn, request, response + + else: + # probing for vulnerable server + try: + rangestr = request.headers.get("range", '') + if not rangestr.endswith('18446744073709551615'): + return + except: + return + + # indication of vulnerable server + if rangestr and (response.status == '416' or \ + response.reason == 'Requested Range Not Satisfiable'): + self.write('MS15-034 Vulnerable Server [Request Method: "{0}" Range: "{1}"]'.format(request.method,rangestr), conn.info()) + return conn, request, response + + diff --git a/dshell/plugins/http/riphttp.py b/dshell/plugins/http/riphttp.py new file mode 100644 index 0000000..1bd90cd --- /dev/null +++ b/dshell/plugins/http/riphttp.py @@ -0,0 +1,206 @@ +""" +Identifies HTTP traffic and reassembles file transfers before writing them to +files. +""" + +import os +import re +import sys + +from dshell.plugins.httpplugin import HTTPPlugin +from dshell.output.alertout import AlertOutput + +class DshellPlugin(HTTPPlugin): + def __init__(self): + super().__init__( + name="rip-http", + author="bg,twp", + bpf="tcp and (port 80 or port 8080 or port 8000)", + description="Rips files from HTTP traffic", + output=AlertOutput(label=__name__), + optiondict={'append_conn': + {'action': 'store_true', + 'help': 'append sourceip-destip to filename'}, + 'append_ts': + {'action': 'store_true', + 'help': 'append timestamp to filename'}, + 'direction': + {'help': 'cs=only capture client POST, sc=only capture server GET response', + 'metavar': '"cs" OR "sc"', + 'default': None}, + 'outdir': + {'help': 'directory to write output files (Default: current directory)', + 'metavar': 'DIRECTORY', + 'default': '.'}, + 'content_filter': + {'help': 'regex MIME type filter for files to save', + 'metavar': 'REGEX'}, + 'name_filter': + {'help': 'regex filename filter for files to save', + 'metavar': 'REGEX'} + } + ) + + def premodule(self): + if self.direction not in ('cs', 'sc', None): + self.logger.warning("Invalid value for direction: {!r}. Argument must be either 'sc' for server-to-client or 'cs' for client-to-server.".format(self.direction)) + sys.exit(1) + + if self.content_filter: + self.content_filter = re.compile(self.content_filter) + if self.name_filter: + self.name_filter = re.compile(self.name_filter) + + self.openfiles = {} + + if not os.path.exists(self.outdir): + try: + os.makedirs(self.outdir) + except (IOError, OSError) as e: + self.error("Could not create output directory: {!r}: {!s}" + .format(self.outdir, e)) + sys.exit(1) + + def http_handler(self, conn, request, response): + if (not self.direction or self.direction == 'cs') and request and request.method == "POST" and request.body: + if not self.content_filter or self.content_filter.search(request.headers.get('content-type', '')): + payload = request + elif (not self.direction or self.direction == 'sc') and response and response.status[0] == '2': + if not self.content_filter or self.content_filter.search(response.headers.get('content-type', '')): + payload = response + else: + payload = None + + if not payload: + # Connection did not match any filters, so get rid of it + return + + host = request.headers.get('host', conn.serverip) + url = host + request.uri + + if url in self.openfiles: + # File is already open, so just insert the new data + s, e = self.openfiles[url].handleresponse(response) + self.logger.debug("{0!r} --> Range: {1} - {2}".format(url, s, e)) + else: + # A new file! + filename = request.uri.split('?', 1)[0].split('/')[-1] + if self.name_filter and self.name_filter.search(filename): + # Filename did not match filter, so get rid of it + return + if not filename: + # Assume index.html if there is no filename + filename = "index.html" + if self.append_conn: + filename += "_{0}-{1}".format(conn.serverip, conn.clientip) + if self.append_ts: + filename += "_{}".format(conn.ts) + while os.path.exists(os.path.join(self.outdir, filename)): + filename += "_" + self.write("New file {} ({})".format(filename, url), **conn.info(), dir_arrow="<-") + self.openfiles[url] = HTTPFile(os.path.join(self.outdir, filename), self) + s, e = self.openfiles[url].handleresponse(payload) + self.logger.debug("{0!r} --> Range: {1} - {2}".format(url, s, e)) + if self.openfiles[url].done(): + self.write("File done {} ({})".format(filename, url), **conn.info(), dir_arrow="<-") + del self.openfiles[url] + + return conn, request, response + + +class HTTPFile(object): + """ + An internal class used to hold metadata for open HTTP files. + Used mostly to reassemble fragmented transfers. + """ + + def __init__(self, filename, plugin_instance): + self.complete = False + # Expected size in bytes of full file transfer + self.size = 0 + # List of tuples indicating byte chunks already received and written to + # disk + self.ranges = [] + self.plugin = plugin_instance + self.filename = filename + try: + self.fh = open(filename, 'wb') + except IOError as e: + self.plugin.error( + "Could not create file {!r}: {!s}".format(filename, e)) + self.fh = None + + def __del__(self): + if self.fh is None: + return + self.fh.close() + if not self.done(): + self.plugin.warning("Incomplete file: {!r}".format(self.filename)) + try: + os.rename(self.filename, self.filename + "_INCOMPLETE") + except: + pass + ls = 0 + le = 0 + for s, e in self.ranges: + if s > le + 1: + self.plugin.warning( + "Missing bytes between {0} and {1}".format(le, s)) + ls, le = s, e + + def handleresponse(self, response): + # Check for Content Range + range_start = 0 + range_end = len(response.body) - 1 + if 'content-range' in response.headers: + m = re.search( + 'bytes (\d+)-(\d+)/(\d+|\*)', response.headers['content-range']) + if m: + range_start = int(m.group(1)) + range_end = int(m.group(2)) + if len(response.body) < (range_end - range_start + 1): + range_end = range_start + len(response.body) - 1 + try: + if int(m.group(3)) > self.size: + self.size = int(m.group(3)) + except: + pass + elif 'content-length' in response.headers: + try: + if int(response.headers['content-length']) > self.size: + self.size = int(response.headers['content-length']) + except: + pass + # Update range tracking + self.ranges.append((range_start, range_end)) + # Write part of file + if self.fh is not None: + self.fh.seek(range_start) + self.fh.write(response.body) + return (range_start, range_end) + + def done(self): + self.checkranges() + return self.complete + + def checkranges(self): + self.ranges.sort() + current_start = 0 + current_end = 0 + foundgap = False + # print self.ranges + for s, e in self.ranges: + if s <= current_end + 1: + current_end = e + else: + foundgap = True + current_start = s + current_end = e + if not foundgap: + if (current_end + 1) >= self.size: + self.complete = True + return foundgap + + +if __name__ == "__main__": + print(DshellPlugin()) diff --git a/dshell/plugins/http/web.py b/dshell/plugins/http/web.py new file mode 100644 index 0000000..b11d0bf --- /dev/null +++ b/dshell/plugins/http/web.py @@ -0,0 +1,76 @@ +""" +Displays basic information for web requests/responses in a connection. +""" + +from dshell.plugins.httpplugin import HTTPPlugin +from dshell.output.alertout import AlertOutput + +from hashlib import md5 + +class DshellPlugin(HTTPPlugin): + def __init__(self): + super().__init__( + name="web", + author="bg,twp", + description="Displays basic information for web requests/responses in a connection", + bpf="tcp and (port 80 or port 8080 or port 8000)", + output=AlertOutput(label=__name__), + optiondict={ + "md5": {"action": "store_true", + "help": "Calculate MD5 for each response."} + }, + ) + + def http_handler(self, conn, request, response): + + if request: + if request.method=="": + # It's impossible to have a properly formed HTTP request without a method + # indicating, the httpplugin is calling http_handler without a full object + return None + # Collect basics about the request, if available + method = request.method + host = request.headers.get("host", "") + uri = request.uri +# useragent = request.headers.get("user-agent", None) +# referer = request.headers.get("referer", None) + version = request.version + else: + method = "(no request)" + host = "" + uri = "" + version = "" + + if response: + if response.status == "" and response.reason == "": + # Another indication of improperly parsed HTTP object in httpplugin + return None + # Collect basics about the response, if available + status = response.status + reason = response.reason + if self.md5: + hash = "(md5: {})".format(md5(response.body).hexdigest()) + else: + hash = "" + else: + status = "(no response)" + reason = "" + hash = "" + + data = "{} {}{} HTTP/{} {} {} {}".format(method, + host, + uri, + version, + status, + reason, + hash) + if not request: + self.write(data, method=method, host=host, uri=uri, version=version, status=status, reason=reason, hash=hash, **response.blob.info()) + elif not response: + self.write(data, method=method, uri=uri, version=version, status=status, reason=reason, hash=hash, **request.headers, **request.blob.info()) + else: + self.write(data, method=method, uri=uri, version=version, status=status, reason=reason, hash=hash, request_headers=request.headers, response_headers=response.headers, **request.blob.info()) + return conn, request, response + +if __name__ == "__main__": + print(DshellPlugin()) diff --git a/dshell/plugins/httpplugin.py b/dshell/plugins/httpplugin.py new file mode 100644 index 0000000..f7b7d19 --- /dev/null +++ b/dshell/plugins/httpplugin.py @@ -0,0 +1,270 @@ +""" +This is a base-level plugin inteded to handle HTTP connections. + +It inherits from the base ConnectionPlugin and provides a new handler +function: http_handler(conn, request, response). + +It automatically pairs requests/responses, parses headers, reassembles bodies, +and collects them into HTTPRequest and HTTPResponse objects that are passed +to the http_handler. +""" + +import logging + +import dshell.core + +from pypacker.layer567 import http + +import gzip +import io + + +logger = logging.getLogger(__name__) + + +def parse_headers(obj, f): + """Return dict of HTTP headers parsed from a file object.""" + # Logic lifted mostly from dpkt's http module + d = {} + while 1: + line = f.readline() + line = line.decode('utf-8') + line = line.strip() + if not line: + break + l = line.split(None, 1) + if not l[0].endswith(':'): + raise dshell.core.DataError("Invalid header {!r}".format(line)) + k = l[0][:-1].lower() + v = len(l) != 1 and l[1] or '' + if k in d: + if not type(d[k]) is list: + d[k] = [d[k]] + d[k].append(v) + else: + d[k] = v + return d + + +def parse_body(obj, f, headers): + """Return HTTP body parsed from a file object, given HTTP header dict.""" + # Logic lifted mostly from dpkt's http module + if headers.get('transfer-encoding', '').lower() == 'chunked': + l = [] + found_end = False + while 1: + try: + sz = f.readline().split(None, 1)[0] + except IndexError: + obj.errors.append(dshell.core.DataError('missing chunk size')) + # FIXME: If this error occurs sz is not available to continue parsing! + # The appropriate exception should be thrown. + raise + n = int(sz, 16) + if n == 0: + found_end = True + buf = f.read(n) + if f.readline().strip(): + break + if n and len(buf) == n: + l.append(buf) + else: + break + if not found_end: + raise dshell.core.DataError('premature end of chunked body') + body = b''.join(l) + elif 'content-length' in headers: + n = int(headers['content-length']) + body = f.read(n) + if len(body) != n: + obj.errors.append(dshell.core.DataError('short body (missing {} bytes)'.format(n - len(body)))) + elif 'content-type' in headers: + body = f.read() + else: + # XXX - need to handle HTTP/0.9 + body = b'' + return body + + +class HTTPRequest(object): + """ + A class for HTTP requests + + Attributes: + blob : the Blob instance of the request + errors : a list of caught exceptions from parsing + method : the method of the request (e.g. GET, PUT, POST, etc.) + uri : the URI being requested (host not included) + version : the HTTP version (e.g. "1.1" for "HTTP/1.1") + headers : a dictionary containing the headers and values + body : bytestring of the reassembled body, after the headers + """ + _methods = ( + 'GET', 'PUT', 'ICY', + 'COPY', 'HEAD', 'LOCK', 'MOVE', 'POLL', 'POST', + 'BCOPY', 'BMOVE', 'MKCOL', 'TRACE', 'LABEL', 'MERGE', + 'DELETE', 'SEARCH', 'UNLOCK', 'REPORT', 'UPDATE', 'NOTIFY', + 'BDELETE', 'CONNECT', 'OPTIONS', 'CHECKIN', + 'PROPFIND', 'CHECKOUT', 'CCM_POST', + 'SUBSCRIBE', 'PROPPATCH', 'BPROPFIND', + 'BPROPPATCH', 'UNCHECKOUT', 'MKACTIVITY', + 'MKWORKSPACE', 'UNSUBSCRIBE', 'RPC_CONNECT', + 'VERSION-CONTROL', + 'BASELINE-CONTROL' + ) + + def __init__(self, blob): + self.errors = [] + self.headers = {} + self.body = b'' + self.blob = blob + data = io.BytesIO(blob.data) + rawline = data.readline() + try: + line = rawline.decode('utf-8') + except UnicodeDecodeError: + line = '' + l = line.strip().split() + if len(l) != 3 or l[0] not in self._methods or not l[2].startswith('HTTP'): + self.errors.append(dshell.core.DataError('invalid HTTP request: {!r}'.format(rawline))) + self.method = '' + self.uri = '' + self.version = '' + return + else: + self.method = l[0] + self.uri = l[1] + self.version = l[2][5:] + self.headers = parse_headers(self, data) + self.body = parse_body(self, data, self.headers) + + +class HTTPResponse(object): + """ + A class for HTTP responses + + Attributes: + blob : the Blob instance of the request + errors : a list of caught exceptions from parsing + version : the HTTP version (e.g. "1.1" for "HTTP/1.1") + status : the status code of the response (e.g. "200" or "304") + reason : the status text of the response (e.g. "OK" or "Not Modified") + headers : a dictionary containing the headers and values + body : bytestring of the reassembled body, after the headers + """ + def __init__(self, blob): + self.errors = [] + self.headers = {} + self.body = b'' + self.blob = blob + data = io.BytesIO(blob.data) + rawline = data.readline() + try: + line = rawline.decode('utf-8') + except UnicodeDecodeError: + line = '' + l = line.strip().split(None, 2) + if len(l) < 2 or not l[0].startswith("HTTP") or not l[1].isdigit(): + self.errors.append(dshell.core.DataError('invalid HTTP response: {!r}'.format(rawline))) + self.version = '' + self.status = '' + self.reason = '' + return + else: + self.version = l[0][5:] + self.status = l[1] + self.reason = l[2] + self.headers = parse_headers(self, data) + self.body = parse_body(self, data, self.headers) + + def decompress_gzip_content(self): + """ + If this response has Content-Encoding set to something with "gzip", + this function will decompress it and store it in the body. + """ + if "gzip" in self.headers.get("content-encoding", ""): + try: + iobody = io.BytesIO(self.body) + except TypeError as e: + # TODO: Why would body ever not be bytes? If it's not bytes, then that means + # we have a bug somewhere in the code and therefore should just allow the + # original exception to be raised. + self.errors.append(dshell.core.DataError("Body was not a byte string ({!s}). Could not decompress.".format(type(self.body)))) + return + try: + self.body = gzip.GzipFile(fileobj=iobody).read() + except OSError as e: + self.errors.append(OSError("Could not gunzip body. {!s}".format(e))) + return + + +class HTTPPlugin(dshell.core.ConnectionPlugin): + + def __init__(self, **kwargs): + super().__init__(**kwargs) + # Use "gunzip" argument to automatically decompress gzipped responses + self.gunzip = kwargs.get("gunzip", False) + + def connection_handler(self, conn): + """ + Goes through each Blob in a Connection, assuming they appear in pairs + of requests and responses, and builds HTTPRequest and HTTPResponse + objects. + + After a response (or only a request at the end of a connection), + http_handler is called. If it returns nothing, the respective blobs + are marked as hidden so they won't be passed to additional plugins. + """ + request = None + response = None + for blob in conn.blobs: + # blob.reassemble(allow_overlap=True, allow_padding=True) + if not blob.data: + continue + if blob.direction == 'cs': + # client-to-server request + request = HTTPRequest(blob) + for req_error in request.errors: + self.debug("Request Error: {!r}".format(req_error)) + elif blob.direction == 'sc': + # server-to-client response + response = HTTPResponse(blob) + for rep_error in response.errors: + self.debug("Response Error: {!r}".format(rep_error)) + if self.gunzip: + response.decompress_gzip_content() + http_handler_out = self.http_handler(conn=conn, request=request, response=response) + if not http_handler_out: + if request: + request.blob.hidden = True + if response: + response.blob.hidden = True + request = None + response = None + if request and not response: + http_handler_out = self.http_handler(conn=conn, request=request, response=None) + if not http_handler_out: + blob.hidden = True + return conn + + def http_handler(self, conn, request, response): + """ + A placeholder. + + Plugins will be able to overwrite this to perform custom activites + on HTTP data. + + It SHOULD return a list containing the sames types of values that came + in as arguments (i.e. return (conn, request, response)) or None. This + is mostly a consistency thing. Realistically, it only needs to return + some value that evaluates to True to pass the Blobs along to additional + plugins. + + Arguments: + conn: a Connection object + request: a HTTPRequest object + response: a HTTPResponse object + """ + return conn, request, response + +DshellPlugin = None diff --git a/dshell/plugins/malware/__init__.py b/dshell/plugins/malware/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dshell/plugins/malware/sweetorange.py b/dshell/plugins/malware/sweetorange.py new file mode 100644 index 0000000..f26a337 --- /dev/null +++ b/dshell/plugins/malware/sweetorange.py @@ -0,0 +1,82 @@ +""" +2015 Feb 13 + +Sometimes, attackers will try to obfuscate links to the Sweet Orange exploit +kit. This plugin is an attempt to decode that sort of traffic. + +It will use a regular expression to try and detect certain variable names that +can be contained in JavaScript code. It will then take the value assigned to +it and decode the domain address hidden inside the value. + +Samples: +http://malware-traffic-analysis.net/2014/10/27/index2.html +http://malware-traffic-analysis.net/2014/10/03/index.html +http://malware-traffic-analysis.net/2014/09/25/index.html +""" + +import re + +from dshell.output.alertout import AlertOutput +from dshell.plugins.httpplugin import HTTPPlugin + +class DshellPlugin(HTTPPlugin): + + def __init__(self): + super().__init__( + name="sweetorange", + longdescription="Used to decode certain variants of the Sweet Orange exploit kit redirect traffic. Looks for telltale Javascript variable names (e.g. 'ajax_data_source' and 'main_request_data_content') and automatically decodes the exploit landing page contained.", + description="Used to decode certain variants of the Sweet Orange exploit kit redirect traffic", + bpf="tcp and (port 80 or port 8080 or port 8000)", + output=AlertOutput(label=__name__), + author="dev195", + gunzip=True, + optiondict={ + "variable": { + "type": str, + "action": "append", + "help": 'Variable names to search for. Default ("ajax_data_source", "main_request_data_content")', + "default": ["ajax_data_source", "main_request_data_content"] + }, + "color": { + "action": "store_true", + "help": "Display encoded/decoded lines in different TTY colors.", + "default": False + }, + } + ) + + + def premodule(self): + self.sig_regex = re.compile( + r"var (" + '|'.join(map(re.escape, self.variable)) + ")='(.*?)';") + self.hexregex = re.compile(r'[^a-fA-F0-9]') + self.logger.debug('Variable regex: "%s"' % self.sig_regex.pattern) + + def http_handler(self, conn, request, response): + try: + response_body = response.body.decode("ascii") + except UnicodeError: + return + except AttributeError: + return + + if response and any([v in response_body for v in self.variable]): + # Take the variable's value, extract the hex characters, and + # convert to ASCII + matches = self.sig_regex.search(response_body) + try: + hidden = matches.groups()[1] + match = bytes.fromhex(self.hexregex.sub('', hidden)) + match = match.decode('utf-8') + except: + return + if self.color: + # If desired, add TTY colors to the alerts for differentiation + # between encoded/decoded strings + hidden = "\x1b[37;2m%s\x1b[0m" % hidden + match = "\x1b[32m%s\x1b[0m" % match + + self.logger.info(hidden) + self.write(match, **conn.info()) + return (conn, request, response) + diff --git a/dshell/plugins/misc/__init__.py b/dshell/plugins/misc/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dshell/plugins/misc/followstream.py b/dshell/plugins/misc/followstream.py new file mode 100644 index 0000000..10aa770 --- /dev/null +++ b/dshell/plugins/misc/followstream.py @@ -0,0 +1,25 @@ +""" +Generates color-coded Screen/HTML output similar to Wireshark Follow Stream +""" + +import dshell.core +from dshell.output.colorout import ColorOutput + +class DshellPlugin(dshell.core.ConnectionPlugin): + + def __init__(self): + super().__init__( + name="Followstream", + author="amm/dev195", + description="Generates color-coded Screen/HTML output similar to Wireshark Follow Stream. Empty connections will be skipped.", + bpf="tcp", + output=ColorOutput(label=__name__), + ) + + def connection_handler(self, conn): + if conn.totalbytes > 0: + self.write(conn, **conn.info()) + return conn + +if __name__ == "__main__": + print(DshellPlugin()) diff --git a/dshell/plugins/misc/pcapwriter.py b/dshell/plugins/misc/pcapwriter.py new file mode 100644 index 0000000..a6df98d --- /dev/null +++ b/dshell/plugins/misc/pcapwriter.py @@ -0,0 +1,84 @@ +""" +Generates pcap output + +Can be used alone or chained at the end of plugins for a kind of filter. + +Use --pcapwriter_outfile to separate its output from that of other plugins. + +Example uses include: + - merging multiple pcap files into one + (decode -d pcapwriter ~/pcap/* >merged.pcap) + - saving relevant traffic by chaining with another plugin + (decode -d track+pcapwriter --track_source=192.168.1.1 --pcapwriter_outfile=merged.pcap ~/pcap/*) + - getting pcap output from plugins that can't use pcapout + (decode -d web+pcapwriter ~/pcap/*) +""" + +import struct + +import dshell.core + +class DshellPlugin(dshell.core.PacketPlugin): + + def __init__(self, *args, **kwargs): + super().__init__( + name="pcap writer", + description="Used to generate pcap output for plugins that can't use -o pcapout", + longdescription="""Generates pcap output + +Can be used alone or chained at the end of plugins for a kind of filter. + +Use --pcapwriter_outfile to separate its output from that of other plugins. + +Example uses include: + - merging multiple pcap files into one (decode -d pcapwriter ~/pcap/* --pcapwriter_outfile=merged.pcap) + - saving relevant traffic by chaining with another plugin (decode -d track+pcapwriter --track_source=192.168.1.1 --pcapwriter_outfile=merged.pcap ~/pcap/*) + - getting pcap output from plugins that can't use pcapout (decode -d web+pcapwriter ~/pcap/*) +""", + author="dev195", + optiondict={ + "outfile": { + "type": str, + "help": "Write to FILE instead of stdout", + "metavar": "FILE", + } + } + ) + self.outfile = None # Filled in with constructor + self.pcap_fh = None + + def prefile(self, infile=None): + # Default to setting pcap output filename based on first input file. + if not self.outfile: + self.outfile = (infile or self.current_pcap_file) + ".pcap" + + def packet_handler(self, packet: dshell.Packet): + # If we don't have a pcap file handle, this is our first packet. + # Create the output pcap file handle. + # NOTE: We want to create the file on the first packet instead of premodule so we + # have a chance to use the input file as part of our output filename. + if not self.pcap_fh: + self.pcap_fh = open(self.outfile, mode="wb") + link_layer_type = self.link_layer_type or 1 + # write the header: + # magic_number, version_major, version_minor, thiszone, sigfigs, + # snaplen, link-layer type + self.pcap_fh.write( + struct.pack('IHHIIII', 0xa1b2c3d4, 2, 4, 0, 0, 65535, link_layer_type)) + + ts = packet.ts + rawpkt = packet.rawpkt + pktlen = packet.pktlen + self.pcap_fh.write(struct.pack('II', int(ts), int((ts - int(ts)) * 1000000))) + self.pcap_fh.write(struct.pack('II', len(rawpkt), pktlen)) + self.pcap_fh.write(rawpkt) + + return packet + + def postmodule(self): + if self.pcap_fh: + self.pcap_fh.close() + + +if __name__ == "__main__": + print(DshellPlugin()) diff --git a/dshell/plugins/misc/search.py b/dshell/plugins/misc/search.py new file mode 100644 index 0000000..d4d99a6 --- /dev/null +++ b/dshell/plugins/misc/search.py @@ -0,0 +1,92 @@ +import dshell.core +from dshell.util import printable_text +from dshell.output.alertout import AlertOutput + +import re +import sys + +class DshellPlugin(dshell.core.ConnectionPlugin): + + def __init__(self): + super().__init__( + name="search", + author="dev195", + bpf="tcp or udp", + description="Search for patterns in connections", + longdescription=""" +Reconstructs streams and searches the content for a user-provided regular +expression. Requires definition of the --search_expression argument. Additional +options can be provided to alter behavior. + """, + output=AlertOutput(label=__name__), + optiondict={ + "expression": { + "help": "Search expression", + "type": str, + "metavar": "REGEX"}, + "ignorecase": { + "help": "Ignore case when searching", + "action": "store_true"}, + "invert": { + "help": "Return connections that DO NOT match expression", + "action": "store_true"}, + "quiet": { + "help": "Do not display matches from this plugin. Useful when chaining plugins.", + "action": "store_true"} + }) + + + + def premodule(self): + # make sure the user actually provided an expression to search for + if not self.expression: + self.error("Must define an expression to search for using --search_expression") + sys.exit(1) + + # define the regex flags, based on arguments + re_flags = 0 + if self.ignorecase: + re_flags = re_flags | re.IGNORECASE + + # Create the regular expression + try: + # convert expression to bytes so it can accurately compare to + # the connection data (which is also of type bytes) + byte_expression = bytes(self.expression, 'utf-8') + self.regex = re.compile(byte_expression, re_flags) + except Exception as e: + self.error("Could not compile regex ({0})".format(e)) + sys.exit(1) + + + + def connection_handler(self, conn): + """ + Go through the data of each connection. + If anything is a hit, return the entire connection. + """ + + match_found = False + for blob in conn.blobs: + for line in blob.data.splitlines(): + match = self.regex.search(line) + if match and self.invert: + return None + elif match and not self.invert: + match_found = True + if not self.quiet: + if blob.sip == conn.sip: + self.write(printable_text(line, False), **conn.info(), dir_arrow="->") + else: + self.write(printable_text(line, False), **conn.info(), dir_arrow="<-") + elif self.invert and not match: + if not self.quiet: + self.write(**conn.info()) + return conn + if match_found: + return conn + + + +if __name__ == "__main__": + print(DshellPlugin()) diff --git a/dshell/plugins/misc/sslalerts.py b/dshell/plugins/misc/sslalerts.py new file mode 100644 index 0000000..d5ddc3f --- /dev/null +++ b/dshell/plugins/misc/sslalerts.py @@ -0,0 +1,107 @@ +""" +Looks for SSL alert messages +""" + +# handy reference: +# http://blog.fourthbit.com/2014/12/23/traffic-analysis-of-an-ssl-slash-tls-session + +import dshell.core +from dshell.output.alertout import AlertOutput + +import hashlib +import io +import struct +from pprint import pprint + +# SSLv3/TLS version +SSL3_VERSION = 0x0300 +TLS1_VERSION = 0x0301 +TLS1_1_VERSION = 0x0302 +TLS1_2_VERSION = 0x0303 + +# Record type +SSL3_RT_CHANGE_CIPHER_SPEC = 20 +SSL3_RT_ALERT = 21 +SSL3_RT_HANDSHAKE = 22 +SSL3_RT_APPLICATION_DATA = 23 + +# Handshake message type +SSL3_MT_HELLO_REQUEST = 0 +SSL3_MT_CLIENT_HELLO = 1 +SSL3_MT_SERVER_HELLO = 2 +SSL3_MT_CERTIFICATE = 11 +SSL3_MT_SERVER_KEY_EXCHANGE = 12 +SSL3_MT_CERTIFICATE_REQUEST = 13 +SSL3_MT_SERVER_DONE = 14 +SSL3_MT_CERTIFICATE_VERIFY = 15 +SSL3_MT_CLIENT_KEY_EXCHANGE = 16 +SSL3_MT_FINISHED = 20 + +alert_types = { + 0x00: "CLOSE_NOTIFY", + 0x0a: "UNEXPECTED_MESSAGE", + 0x14: "BAD_RECORD_MAC", + 0x15: "DECRYPTION_FAILED", + 0x16: "RECORD_OVERFLOW", + 0x1e: "DECOMPRESSION_FAILURE", + 0x28: "HANDSHAKE_FAILURE", + 0x29: "NO_CERTIFICATE", + 0x2a: "BAD_CERTIFICATE", + 0x2b: "UNSUPPORTED_CERTIFICATE", + 0x2c: "CERTIFICATE_REVOKED", + 0x2d: "CERTIFICATE_EXPIRED", + 0x2e: "CERTIFICATE_UNKNOWN", + 0x2f: "ILLEGAL_PARAMETER", + 0x30: "UNKNOWN_CA", + 0x31: "ACCESS_DENIED", + 0x32: "DECODE_ERROR", + 0x33: "DECRYPT_ERROR", + 0x3c: "EXPORT_RESTRICTION", + 0x46: "PROTOCOL_VERSION", + 0x47: "INSUFFICIENT_SECURITY", + 0x50: "INTERNAL_ERROR", + 0x5a: "USER_CANCELLED", + 0x64: "NO_RENEGOTIATION", +} + +alert_severities = { + 0x01: "warning", + 0x02: "fatal", +} + +class DshellPlugin(dshell.core.ConnectionPlugin): + + def __init__(self): + super().__init__( + name="sslalerts", + author="dev195", + bpf="tcp and (port 443 or port 993 or port 1443 or port 8531)", + description="Looks for SSL alert messages", + output=AlertOutput(label=__name__), + ) + + def blob_handler(self, conn, blob): + data = io.BytesIO(blob.data) + alert_seen = False + # Iterate over each layer of the connection, paying special attention to the certificate + while True: + try: + content_type, proto_version, record_len = struct.unpack("!BHH", data.read(5)) + except struct.error: + break + if proto_version not in (SSL3_VERSION, TLS1_VERSION, TLS1_1_VERSION, TLS1_2_VERSION): + return None + if content_type == SSL3_RT_ALERT: + handshake_len = struct.unpack("!I", data.read(4))[0] +# assert handshake_len == 2 # TODO remove when live + severity = struct.unpack("!B", data.read(1))[0] + if severity not in alert_severities: + continue + severity_msg = alert_severities.get(severity, severity) + alert_type = struct.unpack("!B", data.read(1))[0] + alert_msg = alert_types.get(alert_type, str(alert_type)) + self.write("SSL alert: ({}) {}".format(severity_msg, alert_msg), **conn.info()) + alert_seen = True + + if alert_seen: + return conn, blob diff --git a/dshell/plugins/misc/synrst.py b/dshell/plugins/misc/synrst.py new file mode 100644 index 0000000..ccf9227 --- /dev/null +++ b/dshell/plugins/misc/synrst.py @@ -0,0 +1,53 @@ +""" +Detects failed attempts to connect (SYN followed by RST/ACK) +""" + +import dshell.core +from dshell.output.alertout import AlertOutput + +from pypacker.layer4 import tcp + +class DshellPlugin(dshell.core.PacketPlugin): + + def __init__(self): + super().__init__( + name="SYN/RST", + description="Detects failed attempts to connect (SYN followed by RST/ACK)", + author="bg", + bpf="(ip and (tcp[13]=2 or tcp[13]=20)) or (ip6 and tcp)", + output=AlertOutput(label=__name__) + ) + + def premodule(self): + # Cache to hold SYNs waiting to pair with RST/ACKs + self.tracker = {} + + def packet_handler(self, pkt): + # Check if SYN or RST/ACK. Discard non-matches. + if pkt.tcp_flags not in (tcp.TH_SYN, tcp.TH_RST|tcp.TH_ACK): + return + + # Try to find the TCP layer + tcpp = pkt.pkt.upper_layer + while not isinstance(tcpp, tcp.TCP): + try: + tcpp = tcpp.upper_layer + except AttributeError: + # There doesn't appear to be a TCP layer, for some reason + return + + if tcpp.flags == tcp.TH_SYN: + seqnum = tcpp.seq + key = "{}|{}|{}|{}|{}".format( + pkt.sip, pkt.sport, seqnum, pkt.dip, pkt.dport) + self.tracker[key] = pkt + elif tcpp.flags == tcp.TH_RST|tcp.TH_ACK: + acknum = tcpp.ack - 1 + tmpkey = "{}|{}|{}|{}|{}".format( + pkt.dip, pkt.dport, acknum, pkt.sip, pkt.sport) + if tmpkey in self.tracker: + msg = "Failed connection [initiated by {}]".format(pkt.dip) + self.write(msg, **pkt.info()) + oldpkt = self.tracker[tmpkey] + del self.tracker[tmpkey] + return [oldpkt, pkt] diff --git a/dshell/plugins/misc/xor.py b/dshell/plugins/misc/xor.py new file mode 100644 index 0000000..9b1cbfe --- /dev/null +++ b/dshell/plugins/misc/xor.py @@ -0,0 +1,103 @@ +""" +XOR the data in every packet with a user-provided key. Multiple keys can be used +for different data directions. +""" + +import struct + +import dshell.core +import dshell.util +from dshell.output.output import Output + +class DshellPlugin(dshell.core.ConnectionPlugin): + def __init__(self): + super().__init__( + name="xor", + description="XOR every packet with a given key", + output=Output(label=__name__), + bpf="tcp", + author="twp,dev195", + optiondict={ + "key": { + "type": str, + "default": "0xff", + "help": "xor key in hex format (default: 0xff)", + "metavar": "0xHH" + }, + "cskey": { + "type": str, + "default": None, + "help": "xor key to use for client-to-server data (default: None)", + "metavar": "0xHH" + }, + "sckey": { + "type": str, + "default": None, + "help": "xor key to use for server-to-client data (default: None)", + "metavar": "0xHH" + }, + "resync": { + "action": "store_true", + "help": "resync the key index if the key is seen in the data" + } + } + ) + + def __make_key(self, key): + "Convert a user-provided key into a standard format plugin can use." + if key.startswith("0x") or key.startswith("\\x"): + # Convert a hex key + oldkey = key[2:] + newkey = b'' + for i in range(0, len(oldkey), 2): + try: + newkey += struct.pack('B', int(oldkey[i:i + 2], 16)) + except ValueError as e: + self.logger.warning("Error converting hex. Will treat as raw string. - {!s}".format(e)) + newkey = key.encode('ascii') + break + else: + try: + # See if it's a numeric key + newkey = int(key) + newkey = struct.pack('I', newkey) + except ValueError: + # otherwise, convert string key to bytes as it is + newkey = key.encode('ascii') + self.logger.debug("__make_key: {!r} -> {!r}".format(key, newkey)) + return newkey + + def premodule(self): + self.key = self.__make_key(self.key) + if self.cskey: + self.cskey = self.__make_key(self.cskey) + if self.sckey: + self.sckey = self.__make_key(self.sckey) + + def connection_handler(self, conn): + for blob in conn.blobs: + key_index = 0 + if self.sckey and blob.direction == 'sc': + key = self.sckey + elif self.cskey and blob.direction == 'cs': + key = self.cskey + else: + key = self.key + for pkt in blob.packets: + # grab the data from the TCP layer and down + data = pkt.data + # data = pkt.pkt.upper_layer.upper_layer.body_bytes + self.logger.debug("Original:\n{}".format(dshell.util.hex_plus_ascii(data))) + # XOR the data and store it in new_data + new_data = b'' + for i in range(len(data)): + if self.resync and data[i:i + len(key)] == key: + key_index = 0 + x = data[i] ^ key[key_index] + new_data += struct.pack('B', x) + key_index = (key_index + 1) % len(key) + pkt.data = new_data + # # rebuild the packet by adding together each of the layers + # pkt.rawpkt = pkt.pkt.header_bytes + pkt.pkt.upper_layer.header_bytes + pkt.pkt.upper_layer.upper_layer.header_bytes + new_data + self.logger.debug("New:\n{}".format(dshell.util.hex_plus_ascii(new_data))) + return conn diff --git a/dshell/plugins/nbns/__init__.py b/dshell/plugins/nbns/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dshell/plugins/nbns/nbns.py b/dshell/plugins/nbns/nbns.py new file mode 100644 index 0000000..c2ed789 --- /dev/null +++ b/dshell/plugins/nbns/nbns.py @@ -0,0 +1,160 @@ +""" +NBNS plugin +""" + +from struct import unpack + +import dshell.core +from dshell.output.alertout import AlertOutput + +# A few common NBNS Protocol Info Opcodes +# Due to a typo in RFC 1002, 0x9 is also acceptable, but rarely used +# for 'NetBios Refresh' +# 'NetBios Multi-Homed Name Regsitration' (0xf) was added after the RFC +nbns_op = { 0: 'NB_NAME_QUERY', + 5: 'NB_REGISTRATION', + 6: 'NB_RELEASE', + 7: 'NB_WACK', + 8: 'NB_REFRESH', + 9: 'NB_REFRESH', + 15: 'NB_MULTI_HOME_REG' } + + +class DshellPlugin(dshell.core.PacketPlugin): + def __init__(self): + super().__init__( name='nbns', + description='Extract client information from NBNS traffic', + longdescription=""" +The nbns (NetBIOS Name Service) plugin will extract the Transaction ID, Protocol Info, +Client Hostname, and Client MAC address from every UDP NBNS packet found in the given +pcap using port 137. UDP is the standard transport protocol for NBNS traffic. +This filter pulls pertinent information from NBNS packets. + +Examples: + + General usage: + + decode -d nbns + + This will display the connection info including the timestamp, + the source IP, destination IP, Transaction ID, Protocol Info, + Client Hostname, and the Client MAC address in a tabular format. + + + Malware Traffic Analysis Exercise Traffic from 2014-12-08 where a user was hit with a Fiesta exploit kit: + + We want to find out more about the infected machine, and some of this information can be pulled from NBNS traffic + + decode -d nbns 2014-12-08-traffic-analysis-exercise.pcap + + OUTPUT (first few packets): + [nbns] 2014-12-08 18:19:13 192.168.204.137:137 -> 192.168.204.2:137 ** + Transaction ID: 0xb480 + Info: NB_NAME_QUERY + Client Hostname: WPAD + Client MAC: 00:0C:29:9D:B8:6D + ** + [nbns] 2014-12-08 18:19:14 192.168.204.137:137 -> 192.168.204.2:137 ** + Transaction ID: 0xb480 + Info: NB_NAME_QUERY + Client Hostname: WPAD + Client MAC: 00:0C:29:9D:B8:6D + ** + [nbns] 2014-12-08 18:19:16 192.168.204.137:137 -> 192.168.204.2:137 ** + Transaction ID: 0xb480 + Info: NB_NAME_QUERY + Client Hostname: WPAD + Client MAC: 00:0C:29:9D:B8:6D + ** + [nbns] 2014-12-08 18:19:17 192.168.204.137:137 -> 192.168.204.255:137 ** + Transaction ID: 0xb480 + Info: NB_NAME_QUERY + Client Hostname: WPAD + Client MAC: 00:0C:29:9D:B8:6D + ** + """, + bpf='(udp and port 137)', + output=AlertOutput(label=__name__), + author='dek', + ) + self.mac_address = None + self.client_hostname = None + self.xid = None + self.prot_info = None + + + def packet_handler(self, pkt): + + # iterate through the layers and find the NBNS layer + nbns_packet = pkt.pkt.upper_layer + try: + nbns_packet = nbns_packet.upper_layer + except IndexError as e: + self.logger.error('{}: could not parse session data \ + (NBNS packet not found)'.format(str(e))) + # pypacker may throw an Exception here; could use + # further testing + return + + + # Extract the Client hostname from the connection data + # It is represented as 32-bytes half-ASCII + try: + nbns_name = unpack('32s', pkt.data[13:45])[0] + except Exception as e: + self.logger.error('{}: (NBNS packet not found)'.format(str(e))) + return + + + # Decode the 32-byte half-ASCII name to its 16 byte NetBIOS name + try: + if len(nbns_name) == 32: + decoded = [] + for i in range(0,32,2): + nibl = hex(ord(chr(nbns_name[i])) - ord('A'))[2:] + nibh = hex(ord(chr(nbns_name[i+1])) - ord('A'))[2:] + decoded.append(chr(int(''.join((nibl, nibh)), 16))) + + # For uniformity, strip excess byte and space chars + self.client_hostname = ''.join(decoded)[0:-1].strip() + else: + self.client_hostname = str(nbns_name) + + except ValueError as e: + self.logger.error('{}: Hostname in improper format \ + (NBNS packet not found)'.format(str(e))) + return + + + # Extract the Transaction ID from the NBNS packet + xid = unpack('2s', pkt.data[0:2])[0] + self.xid = "0x{}".format(xid.hex()) + + # Extract the opcode info from the NBNS Packet + op = unpack('2s', pkt.data[2:4])[0] + op_hex = op.hex() + op = int(op_hex, 16) + # Remove excess bits + op = (op >> 11) & 15 + + # Decode protocol info if it was present in the payload + try: + self.prot_info = nbns_op[op] + except: + self.prot_info = "0x{}".format(op_hex) + + # Extract the MAC address from the ethernet layer of the packet + self.mac_address = pkt.smac + + # Allow for unknown hostnames + if not self.client_hostname: + self.client_hostname = "" + + if self.xid and self.prot_info and self.client_hostname and self.mac_address: + self.write('\n\tTransaction ID:\t\t{:<8} \n\tInfo:\t\t\t{:<16} \n\tClient Hostname:\t{:<16} \n\tClient MAC:\t\t{:<18}\n'.format( + self.xid, self.prot_info, self.client_hostname, self.mac_address), **pkt.info(), dir_arrow='->') + return pkt + + +if __name__ == "__main__": + print(DshellPlugin()) diff --git a/dshell/plugins/portscan/__init__.py b/dshell/plugins/portscan/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dshell/plugins/portscan/indegree.py b/dshell/plugins/portscan/indegree.py new file mode 100644 index 0000000..4bdc338 --- /dev/null +++ b/dshell/plugins/portscan/indegree.py @@ -0,0 +1,35 @@ +""" +Parse traffic to detect scanners based on connection to IPs that are rarely touched by others +""" + +import dshell.core + +class DshellPlugin(dshell.core.ConnectionPlugin): + + def __init__(self): + super().__init__( + name='parse indegree', + description='Parse traffic to detect scanners based on connection to IPs that are rarely touched by others', + bpf='(tcp or udp)', + author='dev195', + ) + self.client_conns = {} + self.server_conns = {} + self.minhits = 3 + + def connection_handler(self, conn): + self.client_conns.setdefault(conn.clientip, set()) + self.server_conns.setdefault(conn.serverip, set()) + + self.client_conns[conn.clientip].add(conn.serverip) + self.server_conns[conn.serverip].add(conn.clientip) + + def postfile(self): + for clientip, serverips in self.client_conns.items(): + target_count = len(serverips) + S = min((len(self.server_conns[serverip]) for serverip in serverips)) + if S > 2 or target_count < 5: + continue + # TODO implement whitelist + self.write("Scanning IP: {} / S score: {:.1f} / Number of records: {}".format(clientip, S, target_count)) + diff --git a/dshell/plugins/portscan/trw.py b/dshell/plugins/portscan/trw.py new file mode 100644 index 0000000..e2f638b --- /dev/null +++ b/dshell/plugins/portscan/trw.py @@ -0,0 +1,93 @@ +""" +Uses the Threshold Random Walk algorithm described in this paper: + +Limitations to threshold random walk scan detection and mitigating enhancements +Written by: Mell, P.; Harang, R. +http://ieeexplore.ieee.org/xpls/icp.jsp?arnumber=6682723 +""" + +import dshell.core +from dshell.output.output import Output + +from pypacker.layer4 import tcp + +from collections import defaultdict + +o0 = 0.8 # probability IP is benign given successful connection +o1 = 0.2 # probability IP is a scanner given successful connection +is_success = o0/o1 +is_failure = o1/o0 + +max_fp_prob = 0.01 +min_detect_prob = 0.99 +hi_threshold = min_detect_prob / max_fp_prob +lo_threshold = max_fp_prob / min_detect_prob + +OUTPUT_FORMAT = "(%(plugin)s) %(data)s\n" + +class DshellPlugin(dshell.core.PacketPlugin): + def __init__(self, *args, **kwargs): + super().__init__( + name="trw", + author="dev195", + bpf="tcp", + output=Output(label=__name__, format=OUTPUT_FORMAT), + description="Uses Threshold Random Walk to detect network scanners", + optiondict={ + "mark_benigns": { + "action": "store_true", + "help": "Use an upper threshold to mark IPs as benign, thus removing them from consideration as scanners" + } + } + ) + self.synners = set() + self.ip_scores = defaultdict(lambda: 1) + self.classified_ips = set() + + def check_score(self, ip, score): + if self.mark_benigns and score >= hi_threshold: + self.write("IP {} is benign (score: {})".format(ip, score)) + self.classified_ips.add(ip) + elif score <= lo_threshold: + self.write("IP {} IS A SCANNER! (score: {})".format(ip, score)) + self.classified_ips.add(ip) + + def packet_handler(self, pkt): + if not pkt.tcp_flags: + return + + # If we have a SYN, store it in a set and wait for some kind of + # response or the end of pcap + if pkt.tcp_flags == tcp.TH_SYN and pkt.sip not in self.classified_ips: + self.synners.add(pkt.addr) + return pkt + + # If we get the SYN/ACK, score the destination IP with a success + elif pkt.tcp_flags == (tcp.TH_SYN | tcp.TH_ACK) and pkt.dip not in self.classified_ips: + alt_addr = ((pkt.dip, pkt.dport), (pkt.sip, pkt.sport)) + if alt_addr in self.synners: + self.ip_scores[pkt.dip] *= is_success + self.check_score(pkt.dip, self.ip_scores[pkt.dip]) + self.synners.remove(alt_addr) + return pkt + + # If we get a RST, assume the connection was refused and score the + # destination IP with a failure + elif pkt.tcp_flags & tcp.TH_RST and pkt.dip not in self.classified_ips: + alt_addr = ((pkt.dip, pkt.dport), (pkt.sip, pkt.sport)) + if alt_addr in self.synners: + self.ip_scores[pkt.dip] *= is_failure + self.check_score(pkt.dip, self.ip_scores[pkt.dip]) + self.synners.remove(alt_addr) + return pkt + + + def postfile(self): + # Go through any SYNs that didn't get a response and assume they failed + for addr in self.synners: + ip = addr[0][0] + if ip in self.classified_ips: + continue + self.ip_scores[ip] *= is_failure + self.check_score(ip, self.ip_scores[ip]) + diff --git a/dshell/plugins/protocol/__init__.py b/dshell/plugins/protocol/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dshell/plugins/protocol/bitcoin.py b/dshell/plugins/protocol/bitcoin.py new file mode 100644 index 0000000..4e6a237 --- /dev/null +++ b/dshell/plugins/protocol/bitcoin.py @@ -0,0 +1,314 @@ +""" +Bitcoin plugin +""" + +import dshell.core +from dshell.output.alertout import AlertOutput + +import json +from struct import unpack + +# Magic values used to determine Bitcoin Network Type +# Bitcoin Testnet is an alternative blockchain used for testing +MAGIC_VALS = {'F9 BE B4 D9': 'BITCOIN-MAIN', + 'FA BF B5 DA': 'BITCOIN-TESTNET', + '0B 11 09 07': 'BITCOIN-TESTNET3'} + + +class DshellPlugin(dshell.core.ConnectionPlugin): + + def __init__(self): + super().__init__( name='bitcoin', + description='Extract Bitcoin traffic, including Stratum mining protocol (pooled) traffic', + longdescription=''' +The bitcoin plugin will extract any Bitcoin traffic attempting to find and output: + Client/server IP addresses, src/dst port numbers, MAC addresses of the machines + used in the Bitcoin communication/transactions, timestamps of the packets, + packet payload sizes in KB, and the Network type + ('Bitcoin Main' if Bitcoin data traffic). + +Connection tuples are cached when BITCOIN-MAIN traffic is detected, such that following this, +any blobs in a cached connection that do not contain BITCOIN-MAIN magic bytes, are labeled +as part of a connection containing Bitcoin traffic. + +Any traffic on BITCOIN-MAIN's designated port will be labeled as potential Bitcoin traffic. + +Additionally for Stratum mining, the plugin will attempt to extract: + Bitcoin miner being used, transaction methods used in each connection + (mining.notify, mining.authorize, mining.get_transaction, mining.submit, etc.), + User ID (Auth ID) used to access the Bitcoin mining pool, and possibly the password + used to connect to the pool if it is stored in the JSON of the payload. + + Also, the: + range of job IDs (per connection), previous block hash, generation transaction (part 1), + generation transaction (part 2), merkle tree branches (hashes), block version, + and the hash difficulty (n-bits) + (The generation transactions and merkle tree branches are only optionally outputted + to a file: See Example (2) below) + + Note (1): The first time that all of this Stratum mining information is + collected (per connection), all of the packets decoded after this + point from within the same connection (same exact sip, dip, sport, dport) + will continue to output the same collection of information since it + will be the same, and is cumulative per connection. + + Note (2): The gen_tx1 and gen_tx2 fields enable the miner to build the coinbase + transaction for the block by concatentating gen_tx1, the extranonce1 + at the start of gen_tx1, the extranonce2 generated by the miner, and + gen_tx2 (hashes with scriptPubKeys) + + Note (3): Some pools recommend miners use their Bitcoin wallet ID + (address used for payment) as their 'Auth ID'. This will be easily + spotted as it is an address consisting of 26-35 alphanumeric characters, + and it always begins with either the number '1' or '3' + + + For additional information: + Bitcoin Protocol: + Stratum Mining Protocol: + + +--------Main ports / Some secondary ports used for Bitcoin Traffic--------- + Bitcoin Main traffic uses port 8333 + Bitcoin Testnet uses port 18333 + Several pools use ports 3333, 8332, 8337 + The other ports checked are known ports used by specific BTC mining pools + Other Bitcoin pools utilize alternate ports (even 80 / 443) + + +Examples: + + (1) Basic usage: + + decode -d bitcoin + + + (2) Saving Generation Transaction Data and Merkle Branches to a specified file: + + decode -d bitcoin --bitcoin_gentx='foo.txt.' +''', + bpf='''(tcp and port (3332 or 3333 or 3334 or 3335 or + 4008 or 4012 or 4016 or 4024 or + 4032 or 4048 or 4064 or 4096 or + 4128 or 4256 or 5050 or 7033 or + 7065 or 7129 or 7777 or 8332 or + 8333 or 8334 or 8336 or 8337 or + 8344 or 8347 or 8361 or 8888 or + 9332 or 9337 or 9999 or 11111 or + 12222 or 17777 or 18333))''', + output=AlertOutput(label=__name__), + author='dek', + optiondict={ + 'gentx': { + 'type': str, + 'default': None, + 'help': 'The name of the file to output the fields used to generate the block transaction (gen_tx1, gen_tx2, merkle_branches) (default: None)' }, + } + ) + + self.auth_ids = {} + self.notify_params = {} + self.methods = {} + self.miners = {} + self.job_ids = {} + self.smac = None + self.dmac = None + self.bc_net = None + self.size = 0 + self.bcm_cache = set() + self.JSON = False + self.NOTIFY = False + + # blobHandler to reassemble the packets in the traffic + # Bitcoin traffic uses TCP + def blob_handler(self, conn, blob): + try: + data = blob.data + data_str = ''.join(chr(x) for x in data) + data_len = len(data) + except: + self.logger.error('could not parse session data') + return + + # Only continue if the packet contains data + if not data: + return + + + # Default mining.notify fields to None + job_id = None + prev_blk_hash = None + gen_tx1 = None + gen_tx2 = None + merkle_branches = None + blk_ver = None + difficulty = None + curr_time = None + clean_jobs = None + + # If the payload contains JSON + if data_str.startswith('{"'): + self.JSON = True + try: + # split JSON objects by newline + for rawjs in data_str.split("\n"): + if rawjs: + js = json.loads(rawjs) + try: + if "method" in js and js["method"]: + # Create a dictionary of sets of mining methods + # indexed by their associated conn.addr (sip, dip, sport, dport) + self.methods.setdefault(conn.addr, set([])).add(js["method"]) + + if js["method"] == "mining.subscribe": + self.miners[conn.addr] = js["params"][0] + + if js["method"] == "mining.authorize": + if "params" in js and js['params'][0]: + # Grab the Bitcoin User ID (sometimes a wallet id) + # which is being authorized + self.auth_ids[conn.addr] = js["params"][0] + + if js['params'][1]: + self.auth_ids[conn.addr] = "".join(( + self.auth_ids[conn.addr], " / ", str(js['params'][1]) )) + + if js["method"] == "mining.notify": + self.NOTIFY = True + if "params" in js and js['params']: + job_id, prev_blk_hash, gen_tx1, gen_tx2, merkle_branches, blk_ver, difficulty, curr_time, clean_jobs = js['params'] + self.job_ids.setdefault(conn.addr, []).append(job_id) + self.notify_params[conn.addr] = [self.job_ids, prev_blk_hash, gen_tx1, + gen_tx2, merkle_branches, + blk_ver, difficulty, curr_time, + clean_jobs] + + except KeyError as e: + self.logger.error("{} - Error extracting auth ID".format(str(e))) + except ValueError as e: + self.logger.error('{} - json data not found'.format(str(e))) + return + + + # Grab the first 4 bytes of the payload to search for the magic values + # used to determine which Bitcoin network is being accessed + # Additionally, reformat bytes + try: + magic_val = data[0:4].hex().upper() + magic_val = ' '.join([magic_val[i:i+2] for i in range(0, len(magic_val), 2)]) + except: + self.logger.error('could not parse session data') + return + + # Attempt to translate first 4 bytes of payload into a Bitcoin (bc) + # network type, and determine if blob is part of connection which + # contained BITCOIN-MAIN traffic, or if using BITCOIN-MAIN port, or + # if part of Stratum mining + try: + self.bc_net = str(MAGIC_VALS[magic_val]) + if self.bc_net == 'BITCOIN-MAIN': + self.bcm_cache.add(conn.addr) + except: + if conn.addr in self.bcm_cache: + self.bc_net = 'Potential BITCOIN-MAIN (part of connection which detected BITCOIN-MAIN traffic)' + elif (blob.sport == 8333 or blob.dport == 8333): + self.bc_net = 'Potential BITCOIN-MAIN traffic (using designated port)' + # Stratum mining methods have been detected + elif (self.methods): + self.bc_net = 'STRATUM MINING' + else: + self.bc_net = 'N/A (Likely just traffic over a known Bitcoin/Stratum mining port)' + + + # Pull pertinent information from packet's contents + self.size = '{0:.2f}'.format(data_len/1024.0) + # Pull source MAC and dest MAC from first packet in each connection + self.smac = blob.smac + self.dmac = blob.dmac + + # Truncate the list Job IDs per connection for printing purposes if JSON + # data was found in the blob + if self.JSON and self.NOTIFY: + jids_end = (len(self.notify_params[conn.addr][0][conn.addr]) - 1) + if jids_end >= 1: + self.notify_params[conn.addr][0][conn.addr] = [ self.notify_params[conn.addr][0][conn.addr][0], + '...', + self.notify_params[conn.addr][0][conn.addr][jids_end] ] + elif jids_end >= 0: + self.notify_params[conn.addr][0][conn.addr] = [self.notify_params[conn.addr][0][conn.addr][0]] + + # Reset the JSON data found in the current blob boolean + # and the mining.notify method type found in payload boolean + self.JSON = False + self.NOTIFY = False + + + # If able to pull the Bitcoin Pool User ID (sometimes a wallet ID) + # Also if the transcation is mining.notify (seen in Stratum mining) or Stratum mining + # detected via keywords + # then output the current Block information + if (self.size and self.smac and self.dmac and + self.miners.get(conn.addr, None) and self.methods.get(conn.addr, None) and self.auth_ids.get(conn.addr, None) + and self.notify_params.get(conn.addr, None) and not self.gentx): + self.write("\n\tNETWORK: \t{0:<15} \n\tSRC_MAC: \t{1:<20} \n\tDST_MAC: \t{2:<20} \n\tSIZE: \t\t{3:>3}KB\n\t" + "MINER: \t\t{4:<20} \n\tMETHODS: \t{5:<25} \n\tUSER ID/PW: \t{6:<50}\n\t" + "JOB IDs: \t{7:<20} \n\tPREV BLK HASH: \t{8:<65} \n\tBLOCK VER: \t{9:<15}\n\t" + "HASH DIFF: \t{10:<10}\n\n".format( + self.bc_net, self.smac, self.dmac, self.size, + self.miners[conn.addr], ', '.join(self.methods[conn.addr]), self.auth_ids[conn.addr], + ', '.join(self.notify_params[conn.addr][0][conn.addr]), self.notify_params[conn.addr][1], + self.notify_params[conn.addr][5], self.notify_params[conn.addr][6]), + ts=blob.starttime, sip=conn.sip, dip=conn.dip, sport=conn.sport, + dport=conn.dport, direction=blob.direction) + + + # If able to pull the Bitcoin Pool User ID (sometimes a wallet ID) + # Also if the transcation is mining.notify (seen in Stratum mining) or Stratum mining + # detected via keywords and the user specifies that they want to save the fields used + # to generate the block transaction (gen_tx1, gen_tx2 (hashes with scriptPubKeys), merkle tree branches), + # then output all information possible, and write the gentx information to the specified file + elif (self.size and self.smac and self.dmac and + self.miners.get(conn.addr, None) and self.methods.get(conn.addr, None) and self.auth_ids.get(conn.addr, None) + and self.notify_params.get(conn.addr, None) and self.gentx): + self.write("\n\tNETWORK: \t{0:<15} \n\tSRC_MAC: \t{1:<20} \n\tDST_MAC: \t{2:<20} \n\tSIZE: \t\t{3:>3}KB\n\t" + "MINER: \t\t{4:<20} \n\tMETHODS: \t{5:<25} \n\tUSER ID/PW: \t{6:<50}\n\t" + "JOB IDs: \t{7:<20} \n\tPREV BLK HASH: \t{8:<65} \n\tBLOCK VER: \t{9:<15}\n\t" + "HASH DIFF: \t{10:<10}\n\n".format( + self.bc_net, self.smac, self.dmac, self.size, + self.miners[conn.addr], ', '.join(self.methods[conn.addr]), self.auth_ids[conn.addr], + ', '.join(self.notify_params[conn.addr][0][conn.addr]), self.notify_params[conn.addr][1], + self.notify_params[conn.addr][5], self.notify_params[conn.addr][6]), + ts=blob.starttime, sip=conn.sip, dip=conn.dip, sport=conn.sport, + dport=conn.dport, direction=blob.direction) + + # Write the verbose block information (gen tx1/2, merkle branches) gathered + # from mining.notify payloads to the command-line specified output file + # The extra information (JOB ID, BLOCK VER, etc.) will be useful in matching the + # information outputted by the alerts to the payload containing the + # generation transaction info and merkle branches + fout = open(self.gentx, "a+") + fout.write(("\nJOB IDs: \t\t{0:<20} \nPREV BLK HASH: \t{1:<65} \n\nGEN TX1: \t\t{2:<20}" + "\n\nGEN TX2: \t\t{3:<20} \n\nMERKLE BRANCHES: {4:<20} \n\nBLOCK VER: \t\t{5:<20}" + "\nHASH DIFF: \t\t{6:<10}\n").format( + ', '.join(self.notify_params[conn.addr][0][conn.addr]), self.notify_params[conn.addr][1], + self.notify_params[conn.addr][2], self.notify_params[conn.addr][3], + ', '.join(self.notify_params[conn.addr][4]), self.notify_params[conn.addr][5], + self.notify_params[conn.addr][6])) + fout.write(("\n" + "-"*100)*2) + + + # Else if we dont have Bitcoin User IDs, or Block information + # and the user doesn't want verbose block information (gentx) + elif (self.size and self.smac and self.dmac and self.bc_net): + self.write("\n\tNETWORK: \t{0:<15} \n\tSRC_MAC: \t{1:<20} \n\tDST_MAC: \t{2:<20} \n\tSIZE: \t\t{3:>3}KB\n\n".format( + self.bc_net, self.smac, self.dmac, self.size), ts=blob.starttime, sip=conn.sip, + dip=conn.dip, sport=conn.sport, dport=conn.dport, direction=blob.direction) + + + return conn, blob + + + +if __name__ == "__main__": + print(DshellPlugin()) + diff --git a/dshell/plugins/protocol/ether.py b/dshell/plugins/protocol/ether.py new file mode 100644 index 0000000..87bffa5 --- /dev/null +++ b/dshell/plugins/protocol/ether.py @@ -0,0 +1,68 @@ +""" +Shows MAC address information and optionally filters by it. It is highly +recommended that oui.txt be included in the share/ directory (see README). +""" + +import os + +import dshell.core +from dshell.output.output import Output +from dshell.util import get_data_path + +class DshellPlugin(dshell.core.PacketPlugin): + OUTPUT_FORMAT = "[%(plugin)s] %(dt)s %(sip)-15s %(smac)-18s %(smac_org)-35s -> %(dip)-15s %(dmac)-18s %(dmac_org)-35s %(byte_count)d\n" + + def __init__(self, *args, **kwargs): + super().__init__( + name="Ethernet", + description="Show MAC address information and optionally filter by it", + author="dev195", + output=Output(label=__name__, format=self.OUTPUT_FORMAT), + optiondict={ + "org": {"default":[], "action":"append", "metavar":"ORGANIZATION", "help":"Organizations owning MAC address to inclusively filter on (exact match only). Can be used multiple times to look for multiple organizations."}, + "org_exclusive": {"default":False, "action":"store_true", "help":"Set organization filter to be exclusive"}, + 'quiet': {'action': 'store_true', 'default':False, 'help':'disable alerts for this plugin'} + } + ) + self.oui_map = {} + + def premodule(self): + # Create a mapping of MAC address prefix to organization + # http://standards-oui.ieee.org/oui.txt + ouifilepath = os.path.join(get_data_path(), 'oui.txt') + try: + with open(ouifilepath, encoding="utf-8") as ouifile: + for line in ouifile: + if "(hex)" not in line: + continue + line = line.strip().split(None, 2) + prefix = line[0].replace('-', ':') + org = line[2] + self.oui_map[prefix] = org + except FileNotFoundError: + # user probably did not download it + # print warning and continue + self.logger.warning("Could not find {} (see README). Will not be able to determine MAC organizations.".format(ouifilepath)) + + def packet_handler(self, pkt): + if not pkt.smac or not pkt.dmac: + return + smac_prefix = pkt.smac[:8].upper() + smac_org = self.oui_map.get(smac_prefix, '???') + dmac_prefix = pkt.dmac[:8].upper() + dmac_org = self.oui_map.get(dmac_prefix, '???') + + # Filter out any packets that do not match organization filter + if self.org: + if self.org_exclusive and (smac_org in self.org or dmac_org in self.org): + return + elif not self.org_exclusive and not (smac_org in self.org or dmac_org in self.org): + return + + if not self.quiet: + self.write("", smac_org=smac_org, dmac_org=dmac_org, **pkt.info()) + return pkt + + +if __name__ == "__main__": + print(DshellPlugin()) diff --git a/dshell/plugins/protocol/ip.py b/dshell/plugins/protocol/ip.py new file mode 100644 index 0000000..db93919 --- /dev/null +++ b/dshell/plugins/protocol/ip.py @@ -0,0 +1,24 @@ +""" +Outputs all IPv4/IPv6 traffic, and hex plus ascii with verbose flag +""" + +import dshell.core +import dshell.util +from dshell.output.alertout import AlertOutput + +class DshellPlugin(dshell.core.PacketPlugin): + + def __init__(self): + super().__init__( + name='ip', + description='IPv4/IPv6 plugin', + bpf='ip or ip6', + author='twp', + output=AlertOutput(label=__name__), + ) + + def packet_handler(self, packet): + self.write(**packet.info(), dir_arrow='->') + # If verbose flag set, outputs packet contents in hex and ascii alongside packet info + self.logger.info("\n" + dshell.util.hex_plus_ascii(packet.rawpkt)) + return packet diff --git a/dshell/plugins/protocol/protocol.py b/dshell/plugins/protocol/protocol.py new file mode 100644 index 0000000..d8c175b --- /dev/null +++ b/dshell/plugins/protocol/protocol.py @@ -0,0 +1,22 @@ +""" +Tries to find traffic that does not belong to the following protocols: +TCP, UDP, or ICMP +""" + +import dshell.core +from dshell.output.alertout import AlertOutput + +class DshellPlugin(dshell.core.PacketPlugin): + + def __init__(self): + super().__init__( + name="Uncommon Protocols", + description="Finds uncommon (i.e. not tcp, udp, or icmp) protocols in IP traffic", + bpf="(ip or ip6) and not tcp and not udp and not icmp and not icmp6", + author="bg", + output=AlertOutput(label=__name__), + ) + + def packet_handler(self, packet): + self.write("PROTOCOL: {} ({})".format(packet.protocol, packet.protocol_num), **packet.info(), dir_arrow="->") + return packet diff --git a/dshell/plugins/ssh/__init__.py b/dshell/plugins/ssh/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dshell/plugins/ssh/ssh-pubkey.py b/dshell/plugins/ssh/ssh-pubkey.py new file mode 100644 index 0000000..0101d0b --- /dev/null +++ b/dshell/plugins/ssh/ssh-pubkey.py @@ -0,0 +1,177 @@ +""" +Extract server ssh public key from key exchange +""" + +import dshell.core +from dshell.output.alertout import AlertOutput +import struct +import base64 +import hashlib + + +class DshellPlugin(dshell.core.ConnectionPlugin): + + def __init__(self): + super().__init__( + name="ssh-pubkey", + author="amm", + description="Extract server ssh public key from key exchange", + bpf="tcp port 22", + output=AlertOutput(label=__name__) + ) + + def connection_handler(self, conn): + + server_banner = '' + sc_blob_count = 0 + cs_blob_count = 0 + + info = {} + + for blob in conn.blobs: + + # + # CS Blobs: Only interest is a client banner + # + if blob.direction == 'cs': + cs_blob_count += 1 + if cs_blob_count > 1: + continue + else: + blob.reassemble(allow_overlap=True, allow_padding=True) + if not blob.data: + continue + info['clientbanner'] = blob.data.split(b'\x0d')[0].rstrip() + if not info['clientbanner'].startswith(b'SSH'): + return conn # NOT AN SSH CONNECTION + try: + info['clientbanner'] = info['clientbanner'].decode( + 'utf-8') + except UnicodeDecodeError: + return conn + continue + + # + # SC Blobs: Banner and public key + # + sc_blob_count += 1 + blob.reassemble(allow_overlap=True, allow_padding=True) + if not blob.data: + continue + d = blob.data + + # Server Banner + if sc_blob_count == 1: + info['serverbanner'] = d.split(b'\x0d')[0].rstrip() + if not info['serverbanner'].startswith(b'SSH'): + return conn # NOT AN SSH CONNECTION + try: + info['serverbanner'] = info['serverbanner'].decode('utf-8') + except UnicodeDecodeError: + pass + continue + + # Key Exchange Packet/Messages + mlist = messagefactory(d) + stop_blobs = False + for m in mlist: + if m.message_code == 31 or m.message_code == 33: + info['host_pubkey'] = m.host_pub_key + stop_blobs = True + break + if stop_blobs: + break + + #print(repr(info)) + + if 'host_pubkey' in info: + # Calculate key fingerprints + info['host_fingerprints'] = {} + for hash_scheme in ("md5", "sha1", "sha256"): + hashfunction = eval("hashlib."+hash_scheme) + thisfp = key_fingerprint(info['host_pubkey'], hashfunction) + info['host_fingerprints'][hash_scheme] = ':'.join( + ['%02x' % b for b in thisfp]) + + msg = "%s" % (info['host_pubkey']) + self.write(msg, **info, **conn.info()) + return conn + + +def messagefactory(data): + + datalen = len(data) + offset = 0 + msglist = [] + while offset < datalen: + try: + msg = sshmessage(data[offset:]) + except ValueError: + return msglist + msglist.append(msg) + offset += msg.packet_len + 4 + + return msglist + + +class sshmessage: + + def __init__(self, rawdata): + self.__parse_raw(rawdata) + + def __parse_raw(self, data): + datalen = len(data) + if datalen < 6: + raise ValueError + + (self.packet_len, self.padding_len, + self.message_code) = struct.unpack(">IBB", data[0:6]) + if datalen < self.packet_len + 4: + raise ValueError + self.body = data[6:4+self.packet_len] + + # ECDH Kex Reply + if self.message_code == 31 or self.message_code == 33: + host_key_len = struct.unpack(">I", self.body[0:4])[0] + full_key_net = self.body[4:4+host_key_len] + key_type_name_len = struct.unpack(">I", full_key_net[0:4])[0] + key_type_name = full_key_net[4:4+key_type_name_len] + key_data = full_key_net[4+key_type_name_len:] + if key_type_name_len > 50: + # something went wrong + # this probably isn't a code 31 + self.message_code = 0 + else: + self.host_pub_key = "%s %s" % (key_type_name.decode( + 'utf-8'), base64.b64encode(full_key_net).decode('utf-8')) + + +def key_fingerprint(ssh_pubkey, hashfunction=hashlib.sha256): + + # Treat as bytes, not string + if type(ssh_pubkey) == str: + ssh_pubkey = ssh_pubkey.encode('utf-8') + + # Strip space from end + ssh_pubkey = ssh_pubkey.rstrip(b"\r\n\0 ") + + # Only look at first line + ssh_pubkey = ssh_pubkey.split(b"\n")[0] + # If two spaces, look at middle segment + if ssh_pubkey.count(b" ") >= 1: + ssh_pubkey = ssh_pubkey.split(b" ")[1] + + # Try to decode key as base64 + try: + keybin = base64.b64decode(ssh_pubkey) + except: + sys.stderr.write("Invalid key value:\n") + sys.stderr.write(" \"%s\":\n" % ssh_pubkey) + return None + + # Fingerprint + return hashfunction(keybin).digest() + + +if __name__ == "__main__": + print(DshellPlugin()) diff --git a/dshell/plugins/ssl/__init__.py b/dshell/plugins/ssl/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dshell/plugins/ssl/sslblacklist.py b/dshell/plugins/ssl/sslblacklist.py new file mode 100644 index 0000000..668bcea --- /dev/null +++ b/dshell/plugins/ssl/sslblacklist.py @@ -0,0 +1,124 @@ +""" +Looks for certificates in SSL/TLS traffic and tries to find any hashes that +match those in the abuse.ch blacklist. +(https://sslbl.abuse.ch/blacklist/) +""" + +# handy reference: +# http://blog.fourthbit.com/2014/12/23/traffic-analysis-of-an-ssl-slash-tls-session + +import dshell.core +from dshell.output.alertout import AlertOutput + +import hashlib +import io +import struct + +# SSLv3/TLS version +SSL3_VERSION = 0x0300 +TLS1_VERSION = 0x0301 +TLS1_1_VERSION = 0x0302 +TLS1_2_VERSION = 0x0303 + +# Record type +SSL3_RT_CHANGE_CIPHER_SPEC = 20 +SSL3_RT_ALERT = 21 +SSL3_RT_HANDSHAKE = 22 +SSL3_RT_APPLICATION_DATA = 23 + +# Handshake message type +SSL3_MT_HELLO_REQUEST = 0 +SSL3_MT_CLIENT_HELLO = 1 +SSL3_MT_SERVER_HELLO = 2 +SSL3_MT_CERTIFICATE = 11 +SSL3_MT_SERVER_KEY_EXCHANGE = 12 +SSL3_MT_CERTIFICATE_REQUEST = 13 +SSL3_MT_SERVER_DONE = 14 +SSL3_MT_CERTIFICATE_VERIFY = 15 +SSL3_MT_CLIENT_KEY_EXCHANGE = 16 +SSL3_MT_FINISHED = 20 + + +class DshellPlugin(dshell.core.ConnectionPlugin): + + def __init__(self): + super().__init__( + name="sslblacklist", + author="dev195", + bpf="tcp and (port 443 or port 993 or port 1443 or port 8531)", + description="Looks for certificate SHA1 matches in the abuse.ch blacklist", + longdescription=""" + Looks for certificates in SSL/TLS traffic and tries to find any hashes that + match those in the abuse.ch blacklist. + + Requires downloading the blacklist CSV from abuse.ch: + https://sslbl.abuse.ch/blacklist/ + + If the CSV is not in the current directory, use the --sslblacklist_csv + argument to provide a file path. +""", + output=AlertOutput(label=__name__), + optiondict={ + "csv": { + "help": "filepath to the sslblacklist.csv file", + "default": "./sslblacklist.csv", + "metavar": "FILEPATH" + }, + } + ) + + def premodule(self): + self.parse_blacklist_csv(self.csv) + + def parse_blacklist_csv(self, filepath): + "parses the SSL blacklist CSV, given the 'filepath'" + # Python's standard csv module doesn't seem to handle it properly + self.hashes = {} + with open(filepath, 'r') as csv: + for line in csv: + line = line.split('#')[0] # ignore comments + line = line.strip() + try: + timestamp, sha1, reason = line.split(',', 3) + self.hashes[sha1] = reason + except ValueError: + continue + + def blob_handler(self, conn, blob): + if blob.direction == 'cs': + return None + + data = io.BytesIO(blob.data) + + # Iterate over each layer of the connection, paying special attention to the certificate + while True: + try: + content_type, proto_version, record_len = struct.unpack("!BHH", data.read(5)) + except struct.error: + break + if proto_version not in (SSL3_VERSION, TLS1_VERSION, TLS1_1_VERSION, TLS1_2_VERSION): + return None + if content_type == SSL3_RT_HANDSHAKE: + handshake_type = struct.unpack("!B", data.read(1))[0] + handshake_len = struct.unpack("!I", b"\x00"+data.read(3))[0] + if handshake_type == SSL3_MT_CERTIFICATE: + # Process the certificate itself + cert_chain_len = struct.unpack("!I", b"\x00"+data.read(3))[0] + bytes_processed = 0 + while (bytes_processed < cert_chain_len): + try: + cert_data_len = struct.unpack("!I", b"\x00"+data.read(3))[0] + cert_data = data.read(cert_data_len) + bytes_processed = 3 + cert_data_len + sha1 = hashlib.sha1(cert_data).hexdigest() + if sha1 in self.hashes: + bad_guy = self.hashes[sha1] + self.write("Certificate hash match: {}".format(bad_guy), **conn.info()) + except struct.error as e: + break + else: + # Ignore any layers that are not a certificate + data.read(handshake_len) + continue + + return conn, blob diff --git a/dshell/plugins/ssl/tls.py b/dshell/plugins/ssl/tls.py new file mode 100644 index 0000000..a1ff2d2 --- /dev/null +++ b/dshell/plugins/ssl/tls.py @@ -0,0 +1,990 @@ +""" +Extract interesting metadata from TLS connection setup +""" + +import dshell.core +from dshell.output.alertout import AlertOutput +import sys +import struct +import binascii +import hashlib +import OpenSSL +import time +try: + import ja3.ja3 + ja3_available = True +except ModuleNotFoundError: + ja3_available = False + + +################################################################################################## +# +# Reference RFC 2246 (TLS Protocol Version 1.0) +# and RFC 3546 (TLS Extensions) +# +# http://www.ietf.org/rfc/rfc2246.txt +# http://www.ietf.org/rfc/rfc3546.txt +# http://www.ietf.org/rfc/rfc3280.txt +# +################################################################################################## + +##################### +# Custom Exceptions # +##################### + + +class Error(Exception): + pass + + +class InsufficientData(Exception): + pass + + +class UnsupportedOption(Exception): + pass + +#################################### +# Constants borrowed from dpkt.ssl # +#################################### + + +# SSLv3/TLS version +SSL3_VERSION = 0x0300 +TLS1_VERSION = 0x0301 +TLS1_2_VERSION = 0x0303 + +# Record type +SSL3_RT_CHANGE_CIPHER_SPEC = 20 +SSL3_RT_ALERT = 21 +SSL3_RT_HANDSHAKE = 22 +SSL3_RT_APPLICATION_DATA = 23 + +# Handshake message type +SSL3_MT_HELLO_REQUEST = 0 +SSL3_MT_CLIENT_HELLO = 1 +SSL3_MT_SERVER_HELLO = 2 +SSL3_MT_CERTIFICATE = 11 +SSL3_MT_SERVER_KEY_EXCHANGE = 12 +SSL3_MT_CERTIFICATE_REQUEST = 13 +SSL3_MT_SERVER_DONE = 14 +SSL3_MT_CERTIFICATE_VERIFY = 15 +SSL3_MT_CLIENT_KEY_EXCHANGE = 16 +SSL3_MT_FINISHED = 20 + +# Cipher Suit Text Strings +ciphersuit_text = { + 0x0000: 'TLS_NULL_WITH_NULL_NULL', + 0x0001: 'TLS_RSA_WITH_NULL_MD5', + 0x0002: 'TLS_RSA_WITH_NULL_SHA', + 0x0003: 'TLS_RSA_EXPORT_WITH_RC4_40_MD5', + 0x0004: 'TLS_RSA_WITH_RC4_128_MD5', + 0x0005: 'TLS_RSA_WITH_RC4_128_SHA', + 0x0006: 'TLS_RSA_EXPORT_WITH_RC2_CBC_40_MD5', + 0x0007: 'TLS_RSA_WITH_IDEA_CBC_SHA', + 0x0008: 'TLS_RSA_EXPORT_WITH_DES40_CBC_SHA', + 0x0009: 'TLS_RSA_WITH_DES_CBC_SHA', + 0x000A: 'TLS_RSA_WITH_3DES_EDE_CBC_SHA', + 0x000B: 'TLS_DH_DSS_EXPORT_WITH_DES40_CBC_SHA', + 0x000C: 'TLS_DH_DSS_WITH_DES_CBC_SHA', + 0x000D: 'TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA', + 0x000E: 'TLS_DH_RSA_EXPORT_WITH_DES40_CBC_SHA', + 0x000F: 'TLS_DH_RSA_WITH_DES_CBC_SHA', + 0x0010: 'TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA', + 0x0011: 'TLS_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA', + 0x0012: 'TLS_DHE_DSS_WITH_DES_CBC_SHA', + 0x0013: 'TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA', + 0x0014: 'TLS_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA', + 0x0015: 'TLS_DHE_RSA_WITH_DES_CBC_SHA', + 0x0016: 'TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA', + 0x0017: 'TLS_DH_anon_EXPORT_WITH_RC4_40_MD5', + 0x0018: 'TLS_DH_anon_WITH_RC4_128_MD5', + 0x0019: 'TLS_DH_anon_EXPORT_WITH_DES40_CBC_SHA', + 0x001A: 'TLS_DH_anon_WITH_DES_CBC_SHA', + 0x001B: 'TLS_DH_anon_WITH_3DES_EDE_CBC_SHA', + 0x001E: 'TLS_KRB5_WITH_DES_CBC_SHA', + 0x001F: 'TLS_KRB5_WITH_3DES_EDE_CBC_SHA', + 0x0020: 'TLS_KRB5_WITH_RC4_128_SHA', + 0x0021: 'TLS_KRB5_WITH_IDEA_CBC_SHA', + 0x0022: 'TLS_KRB5_WITH_DES_CBC_MD5', + 0x0023: 'TLS_KRB5_WITH_3DES_EDE_CBC_MD5', + 0x0024: 'TLS_KRB5_WITH_RC4_128_MD5', + 0x0025: 'TLS_KRB5_WITH_IDEA_CBC_MD5', + 0x0026: 'TLS_KRB5_EXPORT_WITH_DES_CBC_40_SHA', + 0x0027: 'TLS_KRB5_EXPORT_WITH_RC2_CBC_40_SHA', + 0x0028: 'TLS_KRB5_EXPORT_WITH_RC4_40_SHA', + 0x0029: 'TLS_KRB5_EXPORT_WITH_DES_CBC_40_MD5', + 0x002A: 'TLS_KRB5_EXPORT_WITH_RC2_CBC_40_MD5', + 0x002B: 'TLS_KRB5_EXPORT_WITH_RC4_40_MD5', + 0x002C: 'TLS_PSK_WITH_NULL_SHA', + 0x002D: 'TLS_DHE_PSK_WITH_NULL_SHA', + 0x002E: 'TLS_RSA_PSK_WITH_NULL_SHA', + 0x002F: 'TLS_RSA_WITH_AES_128_CBC_SHA', + 0x0030: 'TLS_DH_DSS_WITH_AES_128_CBC_SHA', + 0x0031: 'TLS_DH_RSA_WITH_AES_128_CBC_SHA', + 0x0032: 'TLS_DHE_DSS_WITH_AES_128_CBC_SHA', + 0x0033: 'TLS_DHE_RSA_WITH_AES_128_CBC_SHA', + 0x0034: 'TLS_DH_anon_WITH_AES_128_CBC_SHA', + 0x0035: 'TLS_RSA_WITH_AES_256_CBC_SHA', + 0x0036: 'TLS_DH_DSS_WITH_AES_256_CBC_SHA', + 0x0037: 'TLS_DH_RSA_WITH_AES_256_CBC_SHA', + 0x0038: 'TLS_DHE_DSS_WITH_AES_256_CBC_SHA', + 0x0039: 'TLS_DHE_RSA_WITH_AES_256_CBC_SHA', + 0x003A: 'TLS_DH_anon_WITH_AES_256_CBC_SHA', + 0x003B: 'TLS_RSA_WITH_NULL_SHA256', + 0x003C: 'TLS_RSA_WITH_AES_128_CBC_SHA256', + 0x003D: 'TLS_RSA_WITH_AES_256_CBC_SHA256', + 0x003E: 'TLS_DH_DSS_WITH_AES_128_CBC_SHA256', + 0x003F: 'TLS_DH_RSA_WITH_AES_128_CBC_SHA256', + 0x0040: 'TLS_DHE_DSS_WITH_AES_128_CBC_SHA256', + 0x0041: 'TLS_RSA_WITH_CAMELLIA_128_CBC_SHA', + 0x0042: 'TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA', + 0x0043: 'TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA', + 0x0044: 'TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA', + 0x0045: 'TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA', + 0x0046: 'TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA', + 0x0067: 'TLS_DHE_RSA_WITH_AES_128_CBC_SHA256', + 0x0068: 'TLS_DH_DSS_WITH_AES_256_CBC_SHA256', + 0x0069: 'TLS_DH_RSA_WITH_AES_256_CBC_SHA256', + 0x006A: 'TLS_DHE_DSS_WITH_AES_256_CBC_SHA256', + 0x006B: 'TLS_DHE_RSA_WITH_AES_256_CBC_SHA256', + 0x006C: 'TLS_DH_anon_WITH_AES_128_CBC_SHA256', + 0x006D: 'TLS_DH_anon_WITH_AES_256_CBC_SHA256', + 0x0084: 'TLS_RSA_WITH_CAMELLIA_256_CBC_SHA', + 0x0085: 'TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA', + 0x0086: 'TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA', + 0x0087: 'TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA', + 0x0088: 'TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA', + 0x0089: 'TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA', + 0x008A: 'TLS_PSK_WITH_RC4_128_SHA', + 0x008B: 'TLS_PSK_WITH_3DES_EDE_CBC_SHA', + 0x008C: 'TLS_PSK_WITH_AES_128_CBC_SHA', + 0x008D: 'TLS_PSK_WITH_AES_256_CBC_SHA', + 0x008E: 'TLS_DHE_PSK_WITH_RC4_128_SHA', + 0x008F: 'TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA', + 0x0090: 'TLS_DHE_PSK_WITH_AES_128_CBC_SHA', + 0x0091: 'TLS_DHE_PSK_WITH_AES_256_CBC_SHA', + 0x0092: 'TLS_RSA_PSK_WITH_RC4_128_SHA', + 0x0093: 'TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA', + 0x0094: 'TLS_RSA_PSK_WITH_AES_128_CBC_SHA', + 0x0095: 'TLS_RSA_PSK_WITH_AES_256_CBC_SHA', + 0x0096: 'TLS_RSA_WITH_SEED_CBC_SHA', + 0x0097: 'TLS_DH_DSS_WITH_SEED_CBC_SHA', + 0x0098: 'TLS_DH_RSA_WITH_SEED_CBC_SHA', + 0x0099: 'TLS_DHE_DSS_WITH_SEED_CBC_SHA', + 0x009A: 'TLS_DHE_RSA_WITH_SEED_CBC_SHA', + 0x009B: 'TLS_DH_anon_WITH_SEED_CBC_SHA', + 0x009C: 'TLS_RSA_WITH_AES_128_GCM_SHA256', + 0x009D: 'TLS_RSA_WITH_AES_256_GCM_SHA384', + 0x009E: 'TLS_DHE_RSA_WITH_AES_128_GCM_SHA256', + 0x009F: 'TLS_DHE_RSA_WITH_AES_256_GCM_SHA384', + 0x00A0: 'TLS_DH_RSA_WITH_AES_128_GCM_SHA256', + 0x00A1: 'TLS_DH_RSA_WITH_AES_256_GCM_SHA384', + 0x00A2: 'TLS_DHE_DSS_WITH_AES_128_GCM_SHA256', + 0x00A3: 'TLS_DHE_DSS_WITH_AES_256_GCM_SHA384', + 0x00A4: 'TLS_DH_DSS_WITH_AES_128_GCM_SHA256', + 0x00A5: 'TLS_DH_DSS_WITH_AES_256_GCM_SHA384', + 0x00A6: 'TLS_DH_anon_WITH_AES_128_GCM_SHA256', + 0x00A7: 'TLS_DH_anon_WITH_AES_256_GCM_SHA384', + 0x00A8: 'TLS_PSK_WITH_AES_128_GCM_SHA256', + 0x00A9: 'TLS_PSK_WITH_AES_256_GCM_SHA384', + 0x00AA: 'TLS_DHE_PSK_WITH_AES_128_GCM_SHA256', + 0x00AB: 'TLS_DHE_PSK_WITH_AES_256_GCM_SHA384', + 0x00AC: 'TLS_RSA_PSK_WITH_AES_128_GCM_SHA256', + 0x00AD: 'TLS_RSA_PSK_WITH_AES_256_GCM_SHA384', + 0x00AE: 'TLS_PSK_WITH_AES_128_CBC_SHA256', + 0x00AF: 'TLS_PSK_WITH_AES_256_CBC_SHA384', + 0x00B0: 'TLS_PSK_WITH_NULL_SHA256', + 0x00B1: 'TLS_PSK_WITH_NULL_SHA384', + 0x00B2: 'TLS_DHE_PSK_WITH_AES_128_CBC_SHA256', + 0x00B3: 'TLS_DHE_PSK_WITH_AES_256_CBC_SHA384', + 0x00B4: 'TLS_DHE_PSK_WITH_NULL_SHA256', + 0x00B5: 'TLS_DHE_PSK_WITH_NULL_SHA384', + 0x00B6: 'TLS_RSA_PSK_WITH_AES_128_CBC_SHA256', + 0x00B7: 'TLS_RSA_PSK_WITH_AES_256_CBC_SHA384', + 0x00B8: 'TLS_RSA_PSK_WITH_NULL_SHA256', + 0x00B9: 'TLS_RSA_PSK_WITH_NULL_SHA384', + 0x00BA: 'TLS_RSA_WITH_CAMELLIA_128_CBC_SHA256', + 0x00BB: 'TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA256', + 0x00BC: 'TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA256', + 0x00BD: 'TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256', + 0x00BE: 'TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256', + 0x00BF: 'TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA256', + 0x00C0: 'TLS_RSA_WITH_CAMELLIA_256_CBC_SHA256', + 0x00C1: 'TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA256', + 0x00C2: 'TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA256', + 0x00C3: 'TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256', + 0x00C4: 'TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256', + 0x00C5: 'TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA256', + 0x00FF: 'TLS_EMPTY_RENEGOTIATION_INFO_SCSV', + 0xC001: 'TLS_ECDH_ECDSA_WITH_NULL_SHA', + 0xC002: 'TLS_ECDH_ECDSA_WITH_RC4_128_SHA', + 0xC003: 'TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA', + 0xC004: 'TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA', + 0xC005: 'TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA', + 0xC006: 'TLS_ECDHE_ECDSA_WITH_NULL_SHA', + 0xC007: 'TLS_ECDHE_ECDSA_WITH_RC4_128_SHA', + 0xC008: 'TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA', + 0xC009: 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA', + 0xC00A: 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA', + 0xC00B: 'TLS_ECDH_RSA_WITH_NULL_SHA', + 0xC00C: 'TLS_ECDH_RSA_WITH_RC4_128_SHA', + 0xC00D: 'TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA', + 0xC00E: 'TLS_ECDH_RSA_WITH_AES_128_CBC_SHA', + 0xC00F: 'TLS_ECDH_RSA_WITH_AES_256_CBC_SHA', + 0xC010: 'TLS_ECDHE_RSA_WITH_NULL_SHA', + 0xC011: 'TLS_ECDHE_RSA_WITH_RC4_128_SHA', + 0xC012: 'TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA', + 0xC013: 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA', + 0xC014: 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA', + 0xC015: 'TLS_ECDH_anon_WITH_NULL_SHA', + 0xC016: 'TLS_ECDH_anon_WITH_RC4_128_SHA', + 0xC017: 'TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA', + 0xC018: 'TLS_ECDH_anon_WITH_AES_128_CBC_SHA', + 0xC019: 'TLS_ECDH_anon_WITH_AES_256_CBC_SHA', + 0xC01A: 'TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA', + 0xC01B: 'TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA', + 0xC01C: 'TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA', + 0xC01D: 'TLS_SRP_SHA_WITH_AES_128_CBC_SHA', + 0xC01E: 'TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA', + 0xC01F: 'TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA', + 0xC020: 'TLS_SRP_SHA_WITH_AES_256_CBC_SHA', + 0xC021: 'TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA', + 0xC022: 'TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA', + 0xC023: 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256', + 0xC024: 'TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384', + 0xC025: 'TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256', + 0xC026: 'TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384', + 0xC027: 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256', + 0xC028: 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384', + 0xC029: 'TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256', + 0xC02A: 'TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384', + 0xC02B: 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', + 0xC02C: 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', + 0xC02D: 'TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256', + 0xC02E: 'TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384', + 0xC02F: 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', + 0xC030: 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', + 0xC031: 'TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256', + 0xC032: 'TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384', + 0xC033: 'TLS_ECDHE_PSK_WITH_RC4_128_SHA', + 0xC034: 'TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA', + 0xC035: 'TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA', + 0xC036: 'TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA', + 0xC037: 'TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256', + 0xC038: 'TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384', + 0xC039: 'TLS_ECDHE_PSK_WITH_NULL_SHA', + 0xC03A: 'TLS_ECDHE_PSK_WITH_NULL_SHA256', + 0xC03B: 'TLS_ECDHE_PSK_WITH_NULL_SHA384', + 0xC03C: 'TLS_RSA_WITH_ARIA_128_CBC_SHA256', + 0xC03D: 'TLS_RSA_WITH_ARIA_256_CBC_SHA384', + 0xC03E: 'TLS_DH_DSS_WITH_ARIA_128_CBC_SHA256', + 0xC03F: 'TLS_DH_DSS_WITH_ARIA_256_CBC_SHA384', + 0xC040: 'TLS_DH_RSA_WITH_ARIA_128_CBC_SHA256', + 0xC041: 'TLS_DH_RSA_WITH_ARIA_256_CBC_SHA384', + 0xC042: 'TLS_DHE_DSS_WITH_ARIA_128_CBC_SHA256', + 0xC043: 'TLS_DHE_DSS_WITH_ARIA_256_CBC_SHA384', + 0xC044: 'TLS_DHE_RSA_WITH_ARIA_128_CBC_SHA256', + 0xC045: 'TLS_DHE_RSA_WITH_ARIA_256_CBC_SHA384', + 0xC046: 'TLS_DH_anon_WITH_ARIA_128_CBC_SHA256', + 0xC047: 'TLS_DH_anon_WITH_ARIA_256_CBC_SHA384', + 0xC048: 'TLS_ECDHE_ECDSA_WITH_ARIA_128_CBC_SHA256', + 0xC049: 'TLS_ECDHE_ECDSA_WITH_ARIA_256_CBC_SHA384', + 0xC04A: 'TLS_ECDH_ECDSA_WITH_ARIA_128_CBC_SHA256', + 0xC04B: 'TLS_ECDH_ECDSA_WITH_ARIA_256_CBC_SHA384', + 0xC04C: 'TLS_ECDHE_RSA_WITH_ARIA_128_CBC_SHA256', + 0xC04D: 'TLS_ECDHE_RSA_WITH_ARIA_256_CBC_SHA384', + 0xC04E: 'TLS_ECDH_RSA_WITH_ARIA_128_CBC_SHA256', + 0xC04F: 'TLS_ECDH_RSA_WITH_ARIA_256_CBC_SHA384', + 0xC050: 'TLS_RSA_WITH_ARIA_128_GCM_SHA256', + 0xC051: 'TLS_RSA_WITH_ARIA_256_GCM_SHA384', + 0xC052: 'TLS_DHE_RSA_WITH_ARIA_128_GCM_SHA256', + 0xC053: 'TLS_DHE_RSA_WITH_ARIA_256_GCM_SHA384', + 0xC054: 'TLS_DH_RSA_WITH_ARIA_128_GCM_SHA256', + 0xC055: 'TLS_DH_RSA_WITH_ARIA_256_GCM_SHA384', + 0xC056: 'TLS_DHE_DSS_WITH_ARIA_128_GCM_SHA256', + 0xC057: 'TLS_DHE_DSS_WITH_ARIA_256_GCM_SHA384', + 0xC058: 'TLS_DH_DSS_WITH_ARIA_128_GCM_SHA256', + 0xC059: 'TLS_DH_DSS_WITH_ARIA_256_GCM_SHA384', + 0xC05A: 'TLS_DH_anon_WITH_ARIA_128_GCM_SHA256', + 0xC05B: 'TLS_DH_anon_WITH_ARIA_256_GCM_SHA384', + 0xC05C: 'TLS_ECDHE_ECDSA_WITH_ARIA_128_GCM_SHA256', + 0xC05D: 'TLS_ECDHE_ECDSA_WITH_ARIA_256_GCM_SHA384', + 0xC05E: 'TLS_ECDH_ECDSA_WITH_ARIA_128_GCM_SHA256', + 0xC05F: 'TLS_ECDH_ECDSA_WITH_ARIA_256_GCM_SHA384', + 0xC060: 'TLS_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256', + 0xC061: 'TLS_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384', + 0xC062: 'TLS_ECDH_RSA_WITH_ARIA_128_GCM_SHA256', + 0xC063: 'TLS_ECDH_RSA_WITH_ARIA_256_GCM_SHA384', + 0xC064: 'TLS_PSK_WITH_ARIA_128_CBC_SHA256', + 0xC065: 'TLS_PSK_WITH_ARIA_256_CBC_SHA384', + 0xC066: 'TLS_DHE_PSK_WITH_ARIA_128_CBC_SHA256', + 0xC067: 'TLS_DHE_PSK_WITH_ARIA_256_CBC_SHA384', + 0xC068: 'TLS_RSA_PSK_WITH_ARIA_128_CBC_SHA256', + 0xC069: 'TLS_RSA_PSK_WITH_ARIA_256_CBC_SHA384', + 0xC06A: 'TLS_PSK_WITH_ARIA_128_GCM_SHA256', + 0xC06B: 'TLS_PSK_WITH_ARIA_256_GCM_SHA384', + 0xC06C: 'TLS_DHE_PSK_WITH_ARIA_128_GCM_SHA256', + 0xC06D: 'TLS_DHE_PSK_WITH_ARIA_256_GCM_SHA384', + 0xC06E: 'TLS_RSA_PSK_WITH_ARIA_128_GCM_SHA256', + 0xC06F: 'TLS_RSA_PSK_WITH_ARIA_256_GCM_SHA384', + 0xC070: 'TLS_ECDHE_PSK_WITH_ARIA_128_CBC_SHA256', + 0xC071: 'TLS_ECDHE_PSK_WITH_ARIA_256_CBC_SHA384', + 0xC072: 'TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256', + 0xC073: 'TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384', + 0xC074: 'TLS_ECDH_ECDSA_WITH_CAMELLIA_128_CBC_SHA256', + 0xC075: 'TLS_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384', + 0xC076: 'TLS_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256', + 0xC077: 'TLS_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384', + 0xC078: 'TLS_ECDH_RSA_WITH_CAMELLIA_128_CBC_SHA256', + 0xC079: 'TLS_ECDH_RSA_WITH_CAMELLIA_256_CBC_SHA384', + 0xC07A: 'TLS_RSA_WITH_CAMELLIA_128_GCM_SHA256', + 0xC07B: 'TLS_RSA_WITH_CAMELLIA_256_GCM_SHA384', + 0xC07C: 'TLS_DHE_RSA_WITH_CAMELLIA_128_GCM_SHA256', + 0xC07D: 'TLS_DHE_RSA_WITH_CAMELLIA_256_GCM_SHA384', + 0xC07E: 'TLS_DH_RSA_WITH_CAMELLIA_128_GCM_SHA256', + 0xC07F: 'TLS_DH_RSA_WITH_CAMELLIA_256_GCM_SHA384', + 0xC080: 'TLS_DHE_DSS_WITH_CAMELLIA_128_GCM_SHA256', + 0xC081: 'TLS_DHE_DSS_WITH_CAMELLIA_256_GCM_SHA384', + 0xC082: 'TLS_DH_DSS_WITH_CAMELLIA_128_GCM_SHA256', + 0xC083: 'TLS_DH_DSS_WITH_CAMELLIA_256_GCM_SHA384', + 0xC084: 'TLS_DH_anon_WITH_CAMELLIA_128_GCM_SHA256', + 0xC085: 'TLS_DH_anon_WITH_CAMELLIA_256_GCM_SHA384', + 0xC086: 'TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_GCM_SHA256', + 0xC087: 'TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_GCM_SHA384', + 0xC088: 'TLS_ECDH_ECDSA_WITH_CAMELLIA_128_GCM_SHA256', + 0xC089: 'TLS_ECDH_ECDSA_WITH_CAMELLIA_256_GCM_SHA384', + 0xC08A: 'TLS_ECDHE_RSA_WITH_CAMELLIA_128_GCM_SHA256', + 0xC08B: 'TLS_ECDHE_RSA_WITH_CAMELLIA_256_GCM_SHA384', + 0xC08C: 'TLS_ECDH_RSA_WITH_CAMELLIA_128_GCM_SHA256', + 0xC08D: 'TLS_ECDH_RSA_WITH_CAMELLIA_256_GCM_SHA384', + 0xC08E: 'TLS_PSK_WITH_CAMELLIA_128_GCM_SHA256', + 0xC08F: 'TLS_PSK_WITH_CAMELLIA_256_GCM_SHA384', + 0xC090: 'TLS_DHE_PSK_WITH_CAMELLIA_128_GCM_SHA256', + 0xC091: 'TLS_DHE_PSK_WITH_CAMELLIA_256_GCM_SHA384', + 0xC092: 'TLS_RSA_PSK_WITH_CAMELLIA_128_GCM_SHA256', + 0xC093: 'TLS_RSA_PSK_WITH_CAMELLIA_256_GCM_SHA384', + 0xC094: 'TLS_PSK_WITH_CAMELLIA_128_CBC_SHA256', + 0xC095: 'TLS_PSK_WITH_CAMELLIA_256_CBC_SHA384', + 0xC096: 'TLS_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256', + 0xC097: 'TLS_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384', + 0xC098: 'TLS_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256', + 0xC099: 'TLS_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384', + 0xC09A: 'TLS_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256', + 0xC09B: 'TLS_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384', + 0xC09C: 'TLS_RSA_WITH_AES_128_CCM', + 0xC09D: 'TLS_RSA_WITH_AES_256_CCM', + 0xC09E: 'TLS_DHE_RSA_WITH_AES_128_CCM', + 0xC09F: 'TLS_DHE_RSA_WITH_AES_256_CCM', + 0xC0A0: 'TLS_RSA_WITH_AES_128_CCM_8', + 0xC0A1: 'TLS_RSA_WITH_AES_256_CCM_8', + 0xC0A2: 'TLS_DHE_RSA_WITH_AES_128_CCM_8', + 0xC0A3: 'TLS_DHE_RSA_WITH_AES_256_CCM_8', + 0xC0A4: 'TLS_PSK_WITH_AES_128_CCM', + 0xC0A5: 'TLS_PSK_WITH_AES_256_CCM', + 0xC0A6: 'TLS_DHE_PSK_WITH_AES_128_CCM', + 0xC0A7: 'TLS_DHE_PSK_WITH_AES_256_CCM', + 0xC0A8: 'TLS_PSK_WITH_AES_128_CCM_8', + 0xC0A9: 'TLS_PSK_WITH_AES_256_CCM_8', + 0xC0AA: 'TLS_PSK_DHE_WITH_AES_128_CCM_8', + 0xC0AB: 'TLS_PSK_DHE_WITH_AES_256_CCM_8', +} + +keytypes = { + OpenSSL.crypto.TYPE_RSA: 'RSA', + OpenSSL.crypto.TYPE_DSA: 'DSA', +} + + +##################################################################### +# TLS - TLS message object, initialized from TCP segment +# +# Status: Currently only supports TLS 1.0 Handshake messages +# +##################################################################### +class TLS(object): + + def __init__(self, data): + + data_length = len(data) + offset = 0 + + ######################### + # Unpack TLSPlaintext # + ######################### + if data_length >= offset+5: + (self.ContentType, self.ProtocolVersion, self.TLSRecordLength) = struct.unpack( + '!BHH', data[offset:offset+5]) + offset += 5 + else: + raise InsufficientData('%d bytes received by TLS' % data_length) + + # + # For now, only interested in TLS 1.0. + # Reason: SSL2.0 records do not support the server_name extension, which is the primary + # motivation for creating this library. Needs to be updated/extended eventually. + # + if self.ProtocolVersion != TLS1_VERSION and self.ProtocolVersion != SSL3_VERSION and self.ProtocolVersion != TLS1_2_VERSION: + raise UnsupportedOption( + 'Protocol version 0x%x not supported' % self.ProtocolVersion) + + ################# + # Check Size # + ################# + self.recordbytes = self.TLSRecordLength + 5 + if data_length < self.recordbytes: + raise InsufficientData('%d bytes received by TLS' % data_length) + + ######################################################################### + # Content Types - Only Handshake supported for now + ######################################################################### + self.Handshakes = [] + if self.ContentType == SSL3_RT_HANDSHAKE: + + ############################### + # Loop Through Handshakes # + ############################### + while self.recordbytes >= offset + 4: # Need minimum four bytes for the rest to contain another Handshake + + HandshakeType = data[offset] + offset += 1 + + # + # Handshake Record length + # + HandshakeLength = struct.unpack( + '!I', b'\x00' + data[offset:offset+3])[0] + offset += 3 + + # + # Parse Handshake SubType + # + if HandshakeType == SSL3_MT_CLIENT_HELLO: + try: + self.Handshakes.append(TLSClientHello( + HandshakeType, HandshakeLength, data[offset:offset+HandshakeLength])) + except: + raise + elif HandshakeType == SSL3_MT_SERVER_HELLO: + try: + self.Handshakes.append(TLSServerHello( + HandshakeType, HandshakeLength, data[offset:offset+HandshakeLength])) + except: + raise + elif HandshakeType == SSL3_MT_CERTIFICATE: + try: + self.Handshakes.append(TLSCertificate( + HandshakeType, HandshakeLength, data[offset:offset+HandshakeLength])) + except: + raise + + offset += HandshakeLength + ############################### + # End Handshakes Loop # + ############################### + + +##################################################################### +# TLSHandshake +##################################################################### +class TLSHandshake(object): + + def __init__(self, HandshakeType, HandshakeLength): + self.HandshakeType = HandshakeType + self.HandshakeLength = HandshakeLength + +##################################################################### +# TLSCertificate - Certificate Handshake type +##################################################################### + + +class TLSCertificate(TLSHandshake): + + def __init__(self, HandshakeType, HandshakeLength, data): + + TLSHandshake.__init__(self, HandshakeType, HandshakeLength) + data_length = len(data) + offset = 0 + + # length of all certificates + if data_length >= offset+3: + certificates_length = struct.unpack( + '!I', b'\x00' + data[offset:offset+3])[0] + offset += 3 + else: + raise InsufficientData( + '%d bytes received by TLSCertificate, expected %d for client_version' % (data_length, offset + 2)) + + if data_length >= offset + certificates_length: + try: + self.Certificates = self.__parse_certs( + data[offset:offset+certificates_length]) + offset += certificates_length + except: + offset += certificates_length + raise + else: + raise InsufficientData('%d bytes received by TLSCertificate, expected %d for client_version' % ( + data_length, offset + certificates_length)) + + # Returns array of x509 objects (defined below) + def __parse_certs(self, data): + + certs = [] + while data: + try: + clen, data = self.l24(data) + if len(data) < clen: + raise InsufficientData( + '%d bytes in buffer, need %d' % (len(data), clen)) + try: + cert = OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_ASN1, data[:clen]) + except: + return certs + certs.append(cert) + data = data[clen:] + except: + raise + return certs + + def l24(self, data): + '''24-bit length decoder''' + (lh, ll), data = struct.unpack('!BH', data[0:3]), data[3:] + return lh << 16 | ll, data + + +##################################################################### +# TLSClientHello - ClientHello Handshake type +##################################################################### +class TLSClientHello(TLSHandshake): + + def __init__(self, HandshakeType, HandshakeLength, data): + TLSHandshake.__init__(self, HandshakeType, HandshakeLength) + data_length = len(data) + offset = 0 + self.ja3_data = [] + + # self.client_version + if data_length >= offset+2: + self.client_version = struct.unpack('!H', data[offset:offset+2])[0] + if ja3_available: + self.ja3_data.append(self.client_version) + offset += 2 + else: + raise InsufficientData( + '%d bytes received by TLSClientHello, expected %d for client_version' % (data_length, offset + 2)) + + # self.random + if data_length >= offset+32: + self.random = data[offset:offset+32] + offset += 32 + else: + raise InsufficientData('%d bytes received by TLSClientHello, expected %d for random block' % ( + data_length, offset + 32)) + + # self.session_id_length + if data_length >= offset+1: + self.session_id_length = struct.unpack( + '!B', data[offset:offset+1])[0] + offset += 1 + else: + raise InsufficientData( + '%d bytes received by TLSClientHello, expected %d for session_id_length' % (data_length, offset + 1)) + + # self.session_id + if self.session_id_length > 0: + if data_length >= offset+self.session_id_length: + self.session_id = data[offset:offset+self.session_id_length] + offset += self.session_id_length + else: + raise InsufficientData('%d bytes received by TLSClientHello, expected %d for session_id' % ( + data_length, offset + self.session_id_length)) + else: + self.session_id = None + + # self.cipher_suites_length + if data_length >= offset+2: + self.cipher_suites_length = struct.unpack( + '!H', data[offset:offset+2])[0] + offset += 2 + else: + raise InsufficientData( + '%d bytes received by TLSClientHello, expected %d for cipher_suites_length' % (data_length, offset + 2)) + + # self.cipher_suites (array, two bytes each) + self.cipher_suites = [] + if self.cipher_suites_length > 0: + if ja3_available: + self.ja3_data.append(ja3.ja3.convert_to_ja3_segment( + data[offset:offset+self.cipher_suites_length], 2)) + if data_length >= offset + self.cipher_suites_length: + for j in range(0, self.cipher_suites_length, 2): + self.cipher_suites.append(data[offset+j:offset+j+2]) + offset += self.cipher_suites_length + else: + raise InsufficientData('%d bytes received by TLSClientHello, expected %d for cipher_suites' % ( + data_length, offset + self.cipher_suites_length)) + + # self.compression_methods_length + if data_length >= offset+1: + self.compression_methods_length = data[offset] + offset += 1 + else: + raise InsufficientData( + '%d bytes received by TLSClientHello, expected %d for compression_methods_length' % (data_length, offset + 1)) + + # self.compression_methods (array, one bytes each) + self.compression_methods = [] + if self.compression_methods_length > 0: + if data_length >= offset + self.compression_methods_length: + for j in range(0, self.compression_methods_length): + self.compression_methods.append(data[offset+j]) + offset += self.compression_methods_length + else: + raise InsufficientData('%d bytes received by TLSClientHello, expected %d for compression_methods' % ( + data_length, offset + self.compression_methods_length)) + + ################################ + # Slice Off the Extensions + ################################ + if ja3_available: + self.ja3_data.extend(ja3.ja3.process_extensions( + ja3.ja3.dpkt.ssl.TLSClientHello(data))) + self.extensions = {} + self.raw_extensions = [] # ordered list of tuples (ex_type, ex_data) + # self.extensions_length + if data_length >= offset+2: + self.extensions_length = struct.unpack( + '!H', data[offset:offset+2])[0] + offset += 2 + else: + # No extensions + return + + # Copy Extension Blob into a new working variable + try: + extensions_data = data[offset:offset+self.extensions_length] + except: + raise InsufficientData('%d bytes received by TLSClientHello, expected %d for extensions' % ( + data_length, offset + self.extensions_length)) + + ########################### + # Iterate the Extensions + ########################### + extension_server_name_list = [] + while len(extensions_data) >= 4: + + (ex_type, length) = struct.unpack('!HH', extensions_data[:4]) + if len(extensions_data) > length+4: + this_extension_data = extensions_data[4:4+length] + extensions_data = extensions_data[4+length:] + else: + this_extension_data = extensions_data[4:] + extensions_data = '' + + self.raw_extensions.append((ex_type, this_extension_data)) + + # server_name extension + # this_extension_data is defined on page 8 of RFC 3546 + # It is essentially a list of hostnames + if ex_type == 0: + server_name_list_length = struct.unpack( + '!H', this_extension_data[:2])[0] + if server_name_list_length > len(this_extension_data) - 2: + raise Error("Malformed ServerNameList") + server_name_list = this_extension_data[2:] + # Iterate the list + while len(server_name_list) > 0: + (name_type, name_length) = struct.unpack( + '!BH', server_name_list[0:3]) + name_data = server_name_list[3:name_length + 3] + if len(server_name_list) > name_length + 3: + server_name_list = server_name_list[name_length + 3:] + else: + server_name_list = '' + if name_type == 0: + extension_server_name_list.append(name_data) + else: + raise UnsupportedOption("Unknown NameType") + # After Loop + # add extension information to dictionary + self.extensions['server_name'] = extension_server_name_list + + def ja3(self): + if ja3_available: + return ','.join([str(x) for x in self.ja3_data]) + else: + return None + + def ja3_digest(self): + if ja3_available: + h = hashlib.md5(self.ja3().encode('utf-8')) + return h.hexdigest() + else: + return None + + +##################################################################### +# TLSServerHello - ServerHello Handshake type +##################################################################### +class TLSServerHello(TLSHandshake): + + def __init__(self, HandshakeType, HandshakeLength, data): + TLSHandshake.__init__(self, HandshakeType, HandshakeLength) + data_length = len(data) + offset = 0 + + # self.server_version + if data_length >= offset+2: + self.server_version = struct.unpack('!H', data[offset:offset+2])[0] + offset += 2 + else: + raise InsufficientData( + '%d bytes received by TLSServerHello, expected %d for server_version' % (data_length, offset + 2)) + + # self.random + if data_length >= offset+32: + self.random = data[offset:offset+32] + offset += 32 + else: + raise InsufficientData('%d bytes received by TLSServerHello, expected %d for random block' % ( + data_length, offset + 32)) + + # self.session_id_length + if data_length >= offset+1: + self.session_id_length = struct.unpack( + '!B', data[offset:offset+1])[0] + offset += 1 + else: + raise InsufficientData( + '%d bytes received by TLSServerHello, expected %d for session_id_length' % (data_length, offset + 1)) + + # self.session_id + if self.session_id_length > 0: + if data_length >= offset+self.session_id_length: + self.session_id = data[offset:offset+self.session_id_length] + offset += self.session_id_length + else: + raise InsufficientData('%d bytes received by TLSServerHello, expected %d for session_id' % ( + data_length, offset + self.session_id_length)) + else: + self.session_id = None + + # self.cipher_suite (single value, two bytes) + if data_length >= offset + 2: + self.cipher_suite = data[offset:offset+2] + offset += 2 + else: + self.cipher_suite = None + raise InsufficientData( + '%d bytes received by TLSServerHello, expected %d for cipher_suite' % (data_length, offset + 2)) + + # self.compression_method (single value, one byte) + if data_length >= offset + 1: + self.compression_method = data[offset] + offset += 1 + else: + raise InsufficientData( + '%d bytes received by TLSServerHello, expected %d for compression_method' % (data_length, offset + 1)) + + +############################################################################## +# Some Utility Functions +############################################################################## +def keyTypeToString(kt): + global keytypes + if kt in keytypes: + return keytypes[kt] + else: + try: + return "UNKNOWN(%s)" % str(kt) + except: + return "UNKNOWN(%s)" % repr(kt) + + +def parse_x509_dtm(dtm): + if type(dtm) == bytes: + dtm = dtm.decode('utf-8') + # Fmt: YYYYMMDDhhmmssZ + t = time.strptime(dtm, '%Y%m%d%H%M%SZ') + return time.strftime('%Y-%m-%d %H:%M:%S', t) + + +def render_x509_object(n): + output = b'' + for component in n.get_components(): + output += b"%s=%s " % component + return output.rstrip().decode('utf-8') + + +def openSSL_cert_to_info_dictionary(c): + d = {'fingerprints': {}} + for h in ('md5', 'sha1', 'sha256'): + d['fingerprints'][h] = c.digest(h).decode('utf-8') + + d['subject'] = render_x509_object(c.get_subject()) + d['subject_cn'] = c.get_subject().CN + d['issuer'] = render_x509_object(c.get_issuer()) + d['notAfter'] = parse_x509_dtm(c.get_notAfter()) + d['notBefore'] = parse_x509_dtm(c.get_notBefore()) + # + # Look for subjectAltName + # + for i in range(0, c.get_extension_count()): + ext = c.get_extension(i) + if ext.get_short_name() == b'subjectAltName': + d['subjectAltName'] = str(ext) + public_key = c.get_pubkey() + d['pubkey_bits'] = public_key.bits() + d['pubkey_type'] = keyTypeToString(public_key.type()) + d['pubkey_sha1'] = hashlib.sha1(OpenSSL.crypto.dump_publickey( + OpenSSL.crypto.FILETYPE_ASN1, public_key)).hexdigest() + return d + + +def split_subjectAltName_string(subjectAltName): + l = [] + for an in subjectAltName.split(', '): + if an.startswith('DNS:'): + an = an[4:] + l.append(an) + return l + + +class DshellPlugin(dshell.core.ConnectionPlugin): + + def __init__(self): + super().__init__( + name="tls", + author="amm", + description="Extract interesting metadata from TLS connection setup", + bpf="tcp and (port 443 or port 993 or port 25 or port 587 or port 465 or port 5269 or port 995 or port 3389)", + output=AlertOutput(label=__name__), + longdescription=""" +Extract interesting metadata from TLS connection setup, including the ClientHello and Certificate handshake structures. + +For JA3 support (ClientHello hash), install module pyja3 + """ + ) + + def premodule(self): + if not ja3_available: + self.debug("ja3 capability disabled due to missing python module") + + def connection_handler(self, conn): + + inverted_ssl = False + info = conn.info() + client_names = set() # Agregate list of names specified by client + server_names = set() # Agregate list of names specified by server + certs_cs = [] + certs_sc = [] + server_cipher = None + client_cipher_list = [] + + for blob in conn.blobs: + + blob.reassemble(allow_overlap=True, allow_padding=True) + data = blob.data + offset = 0 + + while offset < len(data): + + tlsrecord = None + try: + tlsrecord = TLS(data[offset:]) + offset += tlsrecord.recordbytes + + if tlsrecord.ContentType == SSL3_RT_HANDSHAKE: + for hs in tlsrecord.Handshakes: + # + # Client hello. Looking for inversion. + # + if hs.HandshakeType == SSL3_MT_CLIENT_HELLO: + if blob.direction != 'cs': + inverted_ssl = True + if 'server_name' in hs.extensions: + for server in hs.extensions['server_name']: + client_names.add( + server.decode('utf-8')) + if ja3_available: + info['ja3'] = hs.ja3() + info['ja3_digest'] = hs.ja3_digest() + client_cipher_list = hs.cipher_suites + + elif hs.HandshakeType == SSL3_MT_SERVER_HELLO: + server_cipher = hs.cipher_suite + + # + # Certificate. Looking for first server cert. + # + elif hs.HandshakeType == SSL3_MT_CERTIFICATE: + for cert in hs.Certificates: + cert_info = openSSL_cert_to_info_dictionary( + cert) + if blob.direction == 'cs': + certs_cs.append(cert_info) + else: + certs_sc.append(cert_info) + + except InsufficientData: + self.log('Skipping small blob: %s\n' % (sys.exc_info()[1])) + offset += len(data) + except UnsupportedOption: + self.log('Unsupported type: %s\n' % (sys.exc_info()[1])) + offset += len(data) + except: + offset += len(data) + self.log('Unknown error in connectionHandler: %s' % + sys.exc_info()[1]) + break + + # Post processing + if inverted_ssl: + info['inverted_ssl'] = True + info['client_certs'] = certs_sc + info['server_certs'] = certs_cs + else: + info['client_certs'] = certs_cs + info['server_certs'] = certs_sc + if len(info['client_certs']): + client_names.add(info['client_certs'][0]['subject_cn']) + if len(info['server_certs']): + server_names.add(info['server_certs'][0]['subject_cn']) + try: + server_names.update(split_subjectAltName_string( + info['server_certs'][0]['subjectAltName'])) + except KeyError: + pass + info['client_names'] = list(client_names) + info['server_names'] = list(server_names) + # Cipher Lists + if server_cipher in client_cipher_list: + cipher_index = client_cipher_list.index(server_cipher) + else: + cipher_index = None + info['cipher_index'] = cipher_index + try: + info['cipher_text'] = ciphersuit_text[struct.unpack('!H', server_cipher)[ + 0]] + except: + info['cipher_text'] = 'UNKNOWN' + + # + # Determine output message + # + if len(client_names) + len(server_names) == 0: + return conn + client_name = ','.join(info['client_names']) + server_name = ','.join(info['server_names']) + if len(client_name) and client_name != server_name: + msg = "%s / %s" % (client_name, server_name) + else: + msg = server_name + self.write(msg, **info) + return conn + + +if __name__ == "__main__": + print(DshellPlugin()) diff --git a/dshell/plugins/tftp/__init__.py b/dshell/plugins/tftp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dshell/plugins/tftp/tftp.py b/dshell/plugins/tftp/tftp.py new file mode 100644 index 0000000..760b0de --- /dev/null +++ b/dshell/plugins/tftp/tftp.py @@ -0,0 +1,282 @@ +""" +TFTP Plugin +In short: + Goes through UDP traffic, packet by packet, and ties together TFTP file + streams. If the command line argument is set (--tftp_rip), it will dump the + files to a directory (--tftp_outdir=) + +In long: + Goes through each UDP packet and parses out the TFTP opcode. For read or + write requests, it sets a placeholder in unset_read_streams or unset_write_streams, + respectively. These placeholders are moved to open_streams when we first see + data for the read request or an ACK code for a write request. The reason for + these placeholders is to allow the server to set the ephemeral port during + data transfer. + + When it sees a DATA packet, it stores the data under the IP-port-IP-port + openStream key as 'filedata'. Each of these data packets has an ordered block + number, and the file data is stored under that block number. It is reassembled + later. When we consider a stream finished (either the DATA packet is too short + or there are no more packets), we rebuild the file data, print information + about the stream, dump the file (optional), and move the information from + open_streams to closed_streams. + +Example: + Running on sample pcap available here: https://wiki.wireshark.org/TFTP + With default values, it will display transfers performed + Dshell> decode -d tftp ~/pcap/tftp_*.pcap + tftp 2013-05-01 08:24:11 192.168.0.253:50618 -- 192.168.0.10:3445 ** read rfc1350.txt (24599 bytes) ** + tftp 2013-04-27 05:07:59 192.168.0.1:57509 -- 192.168.0.13:2087 ** write rfc1350.txt (24599 bytes) ** + With the --tftp_rip flag, it will generate the same output while reassembling + the files and saving them in a defined directory (./tftp_out by default) + Dshell> decode -d tftp --tftp_rip --tftp_outdir=./MyTFTP ~/pcap/tftp_*.pcap + tftp 2013-05-01 08:24:11 192.168.0.253:50618 -- 192.168.0.10:3445 ** read rfc1350.txt (24599 bytes) ** + tftp 2013-04-27 05:07:59 192.168.0.1:57509 -- 192.168.0.13:2087 ** write rfc1350.txt (24599 bytes) ** + Dshell> ls ./MyTFTP/ + rfc1350.txt rfc1350.txt_01 + Note: The two files have the same name in the traffic, but have incremented + filenames when saved +""" + +import os +import struct + +from pypacker.layer4 import udp + +import dshell.core +import dshell.util +from dshell.output.alertout import AlertOutput + +class DshellPlugin(dshell.core.PacketPlugin): + "Primary plugin class" + # packet opcodes (http://www.networksorcery.com/enp/default1101.htm) + RRQ = 1 # read request + WRQ = 2 # write request + DATA = 3 + ACK = 4 + ERROR = 5 + OACK = 6 # option acknowledgment + + def __init__(self, **kwargs): + super().__init__( + name="tftp", + bpf="udp", + description="Find TFTP streams and, optionally, extract the files", + author="dev195", + output=AlertOutput(label=__name__), + optiondict={ + "rip": { + "action": "store_true", + "help": "Rip files from traffic (default: off)", + "default": False}, + "outdir": { + "help": "Directory to place files when using --rip (default: tftp_out)", + "default": "./tftp_out", + "metavar": "DIRECTORY"} + } + ) + + # default information for streams we didn't see the start for + self.default_stream = { + 'filename': '', + 'mode': '', + 'readwrite': '', + 'closed_connection': False, + 'filedata': {}, + 'timestamp': 0 + } + + # containers for various states of streams + self.open_streams = {} + self.closed_streams = [] + # These two are holders while waiting for the server to decide on which + # ephemeral port to use + self.unset_write_streams = {} + self.unset_read_streams = {} + + def premodule(self): + "if needed, create the directory for file output" + if self.rip and not os.path.exists(self.outdir): + try: + os.makedirs(self.outdir) + except OSError: + self.error("Could not create directory {!r}. Files will not be dumped.".format(self.outdir)) + self.rip = False + + def postmodule(self): + "cleanup any unfinished streams" + self.logger.debug("Unset Read Streams: {!s}".format(self.unset_read_streams)) + self.logger.debug("Unset Write Streams: {!s}".format(self.unset_write_streams)) + while(len(self.open_streams) > 0): + k = list(self.open_streams)[0] + self.__closeStream(k, "POSSIBLY INCOMPLETE") + + def packet_handler(self, pkt): + """ + Handles each UDP packet. It checks the TFTP opcode and parses + accordingly. + """ + udpp = pkt.pkt.upper_layer + while not isinstance(udpp, udp.UDP): + try: + udpp = udpp.upper_layer + except AttributeError: + # There doesn't appear to be a UDP layer, for some reason + return + + data = udpp.body_bytes + + try: + flag = struct.unpack("!H", data[:2])[0] + except struct.error: + return # awful small packet + data = data[2:] + if flag == self.RRQ: + # this packet is requesting to read a file from the server + try: + filename, mode = data.split(b"\x00")[0:2] + except ValueError: + return # probably not TFTP + clientIP, clientPort, serverIP, serverPort = pkt.sip, udpp.sport, pkt.dip, udpp.dport + self.unset_read_streams[(clientIP, clientPort, serverIP)] = { + 'filename': filename, + 'mode': mode, + 'readwrite': 'read', + 'closed_connection': False, + 'filedata': {}, + 'timestamp': pkt.ts + } + + elif flag == self.WRQ: + # this packet is requesting to write a file to the server + try: + filename, mode = data.split(b"\x00")[0:2] + except ValueError: + return # probably not TFTP + # in this case, we are writing to the "server" + clientIP, clientPort, serverIP, serverPort = pkt.sip, udpp.sport, pkt.dip, udpp.dport + self.unset_write_streams[(clientIP, clientPort, serverIP)] = { + 'filename': filename, + 'mode': mode, + 'readwrite': 'write', + 'closed_connection': False, + 'filedata': {}, + 'timestamp': pkt.ts + } + + elif flag == self.DATA: + # this packet is sending a chunk of data + clientIP, clientPort, serverIP, serverPort = pkt.sip, udpp.sport, pkt.dip, udpp.dport + key = (clientIP, clientPort, serverIP, serverPort) + if key not in self.open_streams: + # this is probably an unset read stream; there is no + # acknowledgement, it just starts sending data + if (serverIP, serverPort, clientIP) in self.unset_read_streams: + self.open_streams[key] = self.unset_read_streams[ + (serverIP, serverPort, clientIP)] + del(self.unset_read_streams[ + (serverIP, serverPort, clientIP)]) + else: + self.open_streams[key] = self.default_stream + blockNum = struct.unpack("!H", data[:2])[0] + data = data[2:] + if len(data) < 512: + # TFTP uses fixed length data chunks. If it's smaller than the + # length, then the stream is finished + closedConn = True + else: + closedConn = False + self.open_streams[key]['filedata'][blockNum] = data + self.open_streams[key]['closed_connection'] = closedConn + + elif flag == self.ACK: + # this packet has acknowledged the receipt of a data chunk or + # allows a write process to begin + blockNum = struct.unpack("!H", data[:2])[0] + clientIP, clientPort, serverIP, serverPort = pkt.sip, udpp.sport, pkt.dip, udpp.dport + + # special case: this is acknowledging a write operation and sets + # the port for receiving + if blockNum == 0: + clientIP, clientPort, serverIP, serverPort = pkt.dip, udpp.dport, pkt.sip, udpp.sport + i = (clientIP, clientPort, serverIP) + if i in self.unset_write_streams: + self.open_streams[ + (clientIP, clientPort, serverIP, serverPort)] = self.unset_write_streams[i] + del(self.unset_write_streams[i]) + # otherwise, check if this is the confirmation for the end of a + # connection + elif (clientIP, clientPort, serverIP, serverPort) in self.open_streams and self.open_streams[(clientIP, clientPort, serverIP, serverPort)]['closed_connection']: + self.__closeStream( + (clientIP, clientPort, serverIP, serverPort)) + elif (serverIP, serverPort, clientIP, clientPort) in self.open_streams and self.open_streams[(serverIP, serverPort, clientIP, clientPort)]['closed_connection']: + self.__closeStream( + (serverIP, serverPort, clientIP, clientPort)) + + elif flag == self.ERROR: + # this package is sending an error message + # TODO handle more of these properly + errCode = struct.unpack("!H", data[:2])[0] + errMessage = data[2:].strip() + if errCode == 1: # File not found + clientIP, clientPort, serverIP, serverPort = pkt.dip, udpp.dport, pkt.sip, udpp.sport + i = (clientIP, clientPort, serverIP) + if i in self.unset_read_streams: + self.open_streams[ + (serverIP, serverPort, clientIP, clientPort)] = self.unset_read_streams[i] + del(self.unset_read_streams[i]) + self.__closeStream( + (serverIP, serverPort, clientIP, clientPort), errMessage) + + elif flag == self.OACK: + pass # TODO handle options + + return pkt + + def __closeStream(self, key, message=''): + """ + Called when a stream is finished. It moves the stream from + open_streams to closed_streams, prints output, and dumps the file + """ + theStream = self.open_streams[key] + if not theStream['filename']: + message = "INCOMPLETE -- missing filename" + else: + theStream['filename'] = theStream['filename'].decode('utf-8', "backslashreplace") + + # Rebuild the file from the individual blocks + rebuiltFile = b'' + for i in sorted(theStream['filedata'].keys()): + rebuiltFile += theStream['filedata'][i] + + # if we're reading, swap the client and server IP so the output better + # shows who requested the connection + if theStream['readwrite'] == 'read': + ipsNports = (key[2], key[3], key[0], key[1]) + else: + ipsNports = key + + # print out information about the stream + msg = "{:5} {} ({} bytes) {}".format( + theStream['readwrite'], + theStream['filename'], + len(rebuiltFile), + message) + self.write(msg, ts=theStream['timestamp'], sip=ipsNports[0], + sport=ipsNports[1], dip=ipsNports[2], dport=ipsNports[3], + readwrite=theStream['readwrite'], filename=theStream['filename']) + + # dump the file, if that's what the user wants + if self.rip and len(rebuiltFile) > 0: + outpath = dshell.util.gen_local_filename(self.outdir, theStream['filename']) + outfile = open(outpath, 'wb') + outfile.write(rebuiltFile) + outfile.close() + + # remove the stream from the list of open streams + self.closed_streams.append(( + key, + self.open_streams[key]['closed_connection'] + )) + del(self.open_streams[key]) + diff --git a/dshell/plugins/visual/__init__.py b/dshell/plugins/visual/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dshell/plugins/visual/piecharts.py b/dshell/plugins/visual/piecharts.py new file mode 100644 index 0000000..404fc26 --- /dev/null +++ b/dshell/plugins/visual/piecharts.py @@ -0,0 +1,314 @@ +""" +Plugin that generates HTML+JavaScript pie charts for flow information +""" + +import dshell.core +from dshell.output.output import Output + +import operator +from collections import defaultdict + +class VisualizationOutput(Output): + """ + Special output class intended to only be used for this specific plugin. + """ + + _DEFAULT_FORMAT='{"value":%(data)s, "datatype":"%(datatype)s", "label":"%(label)s"},' + + _HTML_HEADER = """ + + + + Dshell - Pie Chart Output + + + + + +
+
+ + + + + + + + + + + + + + +
+

Source Countries

+
+ 		+

Destination Countries

+
+
+

Source ASNs

+
+ 		+

Destination ASNs

+
+
+

Source Ports

+
+ 		+

Destination Ports

+
+
+

Protocols

+
+ 		+
+ + +
+
+ + + +""" + + def setup(self): + Output.setup(self) + self.fh.write(self._HTML_HEADER) + + def close(self): + self.fh.write(self._HTML_FOOTER) + Output.close(self) + +class DshellPlugin(dshell.core.ConnectionPlugin): + + def __init__(self): + super().__init__( + name='Pie Charts', + author='dev195', + bpf="ip", + description='Generates visualizations based on connections', + longdescription=""" +Generates HTML+JavaScript pie chart visualizations based on connections. + +Output should be redirected to a file and placed in a directory that has the d3.js JavaScript library. Library is available for download at https://d3js.org/ +""", + output=VisualizationOutput(label=__name__), + ) + + self.top_x = 10 + + def premodule(self): + "Set each of the counter dictionaries as defaultdict(int)" + # source + self.s_country_count = defaultdict(int) + self.s_asn_count = defaultdict(int) + self.s_port_count = defaultdict(int) + self.s_ip_count = defaultdict(int) + # dest + self.d_country_count = defaultdict(int) + self.d_asn_count = defaultdict(int) + self.d_port_count = defaultdict(int) + self.d_ip_count = defaultdict(int) + # protocol + self.proto = defaultdict(int) + + + def postmodule(self): + "Write the top X results for each type of data we're counting" + t = self.top_x + 1 + for i in sorted(self.proto.items(), reverse=True, key=operator.itemgetter(1))[:t]: + if i[0]: + self.write(int(i[1]), datatype="protocol", label=i[0]) + for i in sorted(self.s_country_count.items(), reverse=True, key=operator.itemgetter(1))[:t]: + if i[0] and i[0] != '--': + self.write(int(i[1]), datatype="source_country", label=i[0]) + for i in sorted(self.d_country_count.items(), reverse=True, key=operator.itemgetter(1))[:t]: + if i[0] and i[0] != '--': + self.write(int(i[1]), datatype="dest_country", label=i[0]) + for i in sorted(self.s_asn_count.items(), reverse=True, key=operator.itemgetter(1))[:t]: + if i[0] and i[0] != '--': + self.write(int(i[1]), datatype="source_asn", label=i[0]) + for i in sorted(self.d_asn_count.items(), reverse=True, key=operator.itemgetter(1))[:t]: + if i[0] and i[0] != '--': + self.write(int(i[1]), datatype="dest_asn", label=i[0]) + for i in sorted(self.s_port_count.items(), reverse=True, key=operator.itemgetter(1))[:t]: + if i[0]: + self.write(int(i[1]), datatype="source_port", label=i[0]) + for i in sorted(self.d_port_count.items(), reverse=True, key=operator.itemgetter(1))[:t]: + if i[0]: + self.write(int(i[1]), datatype="dest_port", label=i[0]) + + def connection_handler(self, conn): + "For each conn, increment the counts for the relevant dictionary keys" + self.proto[conn.protocol] += 1 + self.s_country_count[conn.sipcc] += 1 + self.s_asn_count[conn.sipasn] += 1 + self.s_port_count[conn.sport] += 1 + self.s_ip_count[conn.sip] += 1 + self.d_country_count[conn.dipcc] += 1 + self.d_asn_count[conn.dipasn] += 1 + self.d_port_count[conn.dport] += 1 + self.d_ip_count[conn.dip] += 1 + return conn + diff --git a/dshell/plugins/voip/__init__.py b/dshell/plugins/voip/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dshell/plugins/voip/rtp.py b/dshell/plugins/voip/rtp.py new file mode 100644 index 0000000..7481af5 --- /dev/null +++ b/dshell/plugins/voip/rtp.py @@ -0,0 +1,105 @@ +""" +Real-time transport protocol (RTP) capture plugin +""" + +import datetime + +import dshell.core +from dshell.output.alertout import AlertOutput + +from pypacker.layer4 import udp +from pypacker.layer567 import rtp + +class DshellPlugin(dshell.core.PacketPlugin): + + def __init__(self): + super().__init__( + name="RTP", + author="mm/dev195", + bpf="udp", + description="Real-time transport protocol (RTP) capture plugin", + longdescription=""" +The real-time transport protocol (RTP) plugin will extract the Hosts, Payload Type, Synchronization source, +Sequence Number, Padding, Marker and Client MAC address from every RTP packet found in the given pcap. + +General usage: + + decode -d rtp + decode -d rtp --no-vlan --layer2=sll.SLL + +Examples: + + https://wiki.wireshark.org/SampleCaptures#SIP_and_RTP + https://wiki.wireshark.org/SampleCaptures?action=AttachFile&do=get&target=rtp_example.raw.gz + + decode -d rtp rtp_example.pcap + +Output: + + rtp 2016-09-21 23:44:40 50.197.16.141:1195 -- 192.168.9.12:44352 ** + From: 50.197.16.141 (00:02:31:11:a5:97) to 192.168.9.12 (45:20:01:31:45:40) + Payload Type (7 bits): Dynamic + Sequence Number (16 bits): 58635 + Timestamp (32 bits): 1331328074 + Synchronization source (32 bits): 1948709792 + Arrival Time: 1474497880.6 --> 2016-09-21 22:44:40.604135 + Contributing source (32 bits): 1, Padding (1 bit): 1, Extension (1 bit): 1, Marker (1 bit): 0 + ** + rtp 2016-09-21 23:44:40 10.5.1.8:5086 -- 10.5.1.7:5070 ** + From: 10.5.1.8 (00:02:81:11:a0:d7) to 10.5.1.7 (45:00:20:c8:a3:26) + Payload Type (7 bits): PCMU - Audio - 8000 Hz - 1 Channel + Sequence Number (16 bits): 17664 + Timestamp (32 bits): 98240 + Synchronization source (32 bits): 1671095215 + Arrival Time: 1474497880.6 --> 2016-09-21 22:44:40.604160 + Contributing source (32 bits): 0, Padding (1 bit): 0, Extension (1 bit): 0, Marker (1 bit): 0 + ** + """, + output=AlertOutput(label=__name__) + ) + + def premodule(self): + self.payload_type = {0: "PCMU - Audio - 8000 Hz - 1 Channel", 1: "Reserved", 2: "Reserved", 3: "GSM - Audio - 8000 Hz - 1 Channel", + 4: "G723 - Audio - 8000 Hz - 1 Channel", 5: "DVI4 - Audio - 8000 Hz - 1 Channel", 6: "DVI4 - Audio - 16000 Hz - 1 Channel", + 7: "LPC - Audio - 8000 Hz - 1 Channel", 8: "PCMA - Audio - 8000 Hz - 1 Channel", 9: "G722 - Audio - 8000 Hz - 1 Channel", + 10: "L16 - Audio - 44100 Hz - 2 Channel", 11: "L16 - Audio - 44100 Hz - 1 Channel", 12: "QCELP - Audio - 8000 Hz - 1 Channel", + 13: "CN - Audio - 8000 Hz - 1 Channel", 14: "MPA - Audio - 90000 Hz", 15: "G728 - Audio - 8000 Hz - 1 Channel", 16: "DVI4 - Audio - 11025 Hz - 1 Channel", + 17: "DVI4 - Audio - 22050 Hz - 1 Channel", 18: "G729 - Audio - 8000 Hz - 1 Channel", 19: "Reserved - Audio", 20: "Unassigned - Audio", + 21: "Unassigned - Audio", 22: "Unassigned - Audio", 23: "Unassigned - Audio", 24: "Unassigned - Video", 25: "CelB - Video - 90000 Hz", + 26: "JPEG - Video - 90000 Hz", 27: "Unassigned - Video", 28: "nv - Video - 90000 Hz", 29: "Unassigned - Video", 30: "Unassigned - Video", + 31: "H261 - Video - 90000 Hz", 32: "MPV - Video - 90000 Hz", 33: "MP2T - Audio/Video - 90000 Hz", 34: "H263 - Video - 90000 Hz"} + + for i in range(35,72): + self.payload_type[i] = "Unassigned" + for i in range(72,77): + self.payload_type[i] = "Reserved for RTCP conflict avoidance" + for i in range(77,96): + self.payload_type[i] = "Unassigned" + for i in range(96,128): + self.payload_type[i] = "Dynamic" + + def packet_handler(self, pkt): + # Scrape out the UDP layer of the packet + udpp = pkt.pkt.upper_layer + while not isinstance(udpp, udp.UDP): + try: + udpp = udpp.upper_layer + except AttributeError: + # There doesn't appear to be an UDP layer + return + + # Parse the RTP protocol from above the UDP layer + rtpp = rtp.RTP(udpp.body_bytes) + + if rtpp.version != 2: + # RTP should always be version 2 + return + + pt = self.payload_type.get(rtpp.pt, "??") + + self.write("\n\tFrom: {0} ({1}) to {2} ({3}) \n\tPayload Type (7 bits): {4}\n\tSequence Number (16 bits): {5}\n\tTimestamp (32 bits): {6} \n\tSynchronization source (32 bits): {7}\n\tArrival Time: {8} --> {9}\n\tContributing source (32 bits): {10}, Padding (1 bit): {11}, Extension (1 bit): {12}, Marker (1 bit): {13}\n".format( + pkt.sip, pkt.smac, pkt.dip, pkt.dmac, pt, rtpp.seq, rtpp.ts, + rtpp.ssrc, pkt.ts, datetime.datetime.utcfromtimestamp(pkt.ts), + rtpp.cc, rtpp.p, rtpp.x, rtpp.m), **pkt.info()) + + return pkt diff --git a/dshell/plugins/voip/sip.py b/dshell/plugins/voip/sip.py new file mode 100644 index 0000000..52cf7b5 --- /dev/null +++ b/dshell/plugins/voip/sip.py @@ -0,0 +1,174 @@ +""" + Author: MM - https://github.com/1modm + + The Session Initiation Protocol (SIP) is the IETF protocol for VOIP and other + text and multimedia sessions and is a communications protocol for signaling + and controlling. + SIP is independent from the underlying transport protocol. It runs on the + Transmission Control Protocol (TCP), the User Datagram Protocol (UDP) or the + Stream Control Transmission Protocol (SCTP) + + Rate and codec calculation thanks to https://git.ucd.ie/volte-and-of/voip-pcapy + + RFC: https://www.ietf.org/rfc/rfc3261.txt + + SIP is a text-based protocol with syntax similar to that of HTTP. + There are two different types of SIP messages: requests and responses. + - Requests initiate a SIP transaction between two SIP entities for + establishing, controlling, and terminating sessions. + - Responses are send by the user agent server indicating the result of a + received request. + + - SIP session setup example: + + Alice's . . . . . . . . . . . . . . . . . . . . Bob's + softphone SIP Phone + | | | | + | INVITE F1 | | | + |--------------->| INVITE F2 | | + | 100 Trying F3 |--------------->| INVITE F4 | + |<---------------| 100 Trying F5 |--------------->| + | |<-------------- | 180 Ringing F6 | + | | 180 Ringing F7 |<---------------| + | 180 Ringing F8 |<---------------| 200 OK F9 | + |<---------------| 200 OK F10 |<---------------| + | 200 OK F11 |<---------------| | + |<---------------| | | + | ACK F12 | + |------------------------------------------------->| + | Media Session | + |<================================================>| + | BYE F13 | + |<-------------------------------------------------| + | 200 OK F14 | + |------------------------------------------------->| + | | + +""" + +import dshell.core +from dshell.output.colorout import ColorOutput + +from pypacker.layer4 import udp +from pypacker.layer567 import sip + +class DshellPlugin(dshell.core.PacketPlugin): + + def __init__(self): + super().__init__( + name="SIP", + author="mm/dev195", + output=ColorOutput(label=__name__), + bpf="udp", + description="(UNFINISHED) Session Initiation Protocol (SIP) capture plugin", + longdescription=""" +The Session Initiation Protocol (SIP) plugin will extract the Call ID, User agent, Codec, Method, +SIP call, Host, and Client MAC address from every SIP request or response packet found in the given pcap. + +General usage: + decode -d sip + +Detailed usage: + decode -d sip --sip_showpkt + +Layer2 sll usage: + decode -d sip --no-vlan --layer2=sll.SLL + +SIP over TCP: + decode -d sip --bpf 'tcp' + +SIP is a text-based protocol with syntax similar to that of HTTP, so you can use followstream plugin: + decode -d followstream --ebpf 'port 5060' --bpf 'udp' + +Examples: + + https://wiki.wireshark.org/SampleCaptures#SIP_and_RTP + http://vignette3.wikia.nocookie.net/networker/images/f/fb/Sample_SIP_call_with_RTP_in_G711.pcap/revision/latest?cb=20140723121754 + + decode -d sip metasploit-sip-invite-spoof.pcap + decode -d sip Sample_SIP_call_with_RTP_in_G711.pcap + +Output: + + <-- SIP Request --> + Timestamp: 2016-09-21 22:44:28.220185 UTC - Protocol: UDP - Size: 435 bytes + Sequence and Method: 1 ACK + From: 10.5.1.8:5060 (00:20:80:a1:13:db) to 10.5.1.7:5060 (15:2a:01:b4:0f:47) + Via: SIP/2.0/UDP 10.5.1.8:5060;branch=z9hG4bK940bdac4-8a13-1410-9e58-08002772a6e9;rport + SIP call: "M" ;tag=0ba2d5c4-8a13-1910-9d56-08002772a6e9 --> "miguel" ;tag=84538c9d-ba7e-e611-937f-68a3c4f0d6ce + Call ID: 0ba2d5c4-8a13-1910-9d57-08002772a6e9@M-PC + + --> SIP Response <-- + Timestamp: 2016-09-21 22:44:27.849761 UTC - Protocol: UDP - Size: 919 bytes + Sequence and Method: 1 INVITE + From: 10.5.1.7:5060 (02:0a:40:12:30:23) to 10.5.1.8:5060 (d5:02:03:94:31:1b) + Via: SIP/2.0/UDP 10.5.1.8:5060;branch=z9hG4bK26a8d5c4-8a13-1910-9d58-08002772a6e9;rport=5060;received=10.5.1.8 + SIP call: "M" ;tag=0ba2d5c4-8a13-1910-9d56-08002772a6e9 --> "miguel" ;tag=84538c9d-ba7e-e611-937f-68a3c4f0d6ce + Call ID: 0ba2d5c4-8a13-1910-9d57-08002772a6e9@M-PC + Codec selected: PCMU + Rate selected: 8000 + +Detailed Output: + + --> SIP Response <-- + Timestamp: 2016-09-21 22:44:25.360974 UTC - Protocol: UDP - Size: 349 bytes + From: 10.5.1.7:5060 (15:2a:01:b4:0f:47) to 10.5.1.8:5060 (00:20:80:a1:13:db) + SIP/2.0 100 Trying + content-length: 0 + via: SIP/2.0/UDP 10.5.1.8:5060;branch=z9hG4bK26a8d5c4-8a13-1910-9d58-08002772a6e9;rport=5060;received=10.5.1.8 + from: "M" ;tag=0ba2d5c4-8a13-1910-9d56-08002772a6e9 + to: + cseq: 1 INVITE + call-id: 0ba2d5c4-8a13-1910-9d57-08002772a6e9@M-PC + + --> SIP Response <-- + Timestamp: 2016-09-21 22:44:25.387780 UTC - Protocol: UDP - Size: 585 bytes + From: 10.5.1.7:5060 (15:2a:01:b4:0f:47) to 10.5.1.8:5060 (00:20:80:a1:13:db) + SIP/2.0 180 Ringing + content-length: 0 + via: SIP/2.0/UDP 10.5.1.8:5060;branch=z9hG4bK26a8d5c4-8a13-1910-9d58-08002772a6e9;rport=5060;received=10.5.1.8 + from: "M" ;tag=0ba2d5c4-8a13-1910-9d56-08002772a6e9 + require: 100rel + rseq: 694867676 + user-agent: Ekiga/4.0.1 + to: "miguel" ;tag=84538c9d-ba7e-e611-937f-68a3c4f0d6ce + contact: "miguel" + cseq: 1 INVITE + allow: INVITE,ACK,OPTIONS,BYE,CANCEL,SUBSCRIBE,NOTIFY,REFER,MESSAGE,INFO,PING,PRACK + call-id: 0ba2d5c4-8a13-1910-9d57-08002772a6e9@M-PC +""", + optiondict={ + "showpkt": { + "action": "store_true", + "default": False, + "help": "Display the full SIP response or request body" + } + } + ) + + self.rate = None + self.codec = None + self.direction = None + + def packet_handler(self, pkt): + self.rate = str() + self.codec = str() + self.direction = str() + + # Scrape out the UDP layer of the packet + udpp = pkt.pkt.upper_layer + while not isinstance(udpp, udp.UDP): + try: + udpp = udpp.upper_layer + except AttributeError: + # There doesn't appear to be an UDP layer + return + + # Check if exists SIP Request + if sip.SIP(udpp.body_bytes): + siptxt = "<-- SIP Request -->" + sippkt = sip.SIP(udpp.body_bytes) + self.direction = "sc" + self.output = True + + # TODO finish SIP plugin (pypacker needs to finish SIP, too) diff --git a/dshell/plugins/wifi/__init__.py b/dshell/plugins/wifi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dshell/plugins/wifi/wifi80211.py b/dshell/plugins/wifi/wifi80211.py new file mode 100644 index 0000000..5dc7db2 --- /dev/null +++ b/dshell/plugins/wifi/wifi80211.py @@ -0,0 +1,88 @@ +""" +Shows 802.11 information for individual packets. +""" + +import dshell.core +from dshell.output.output import Output + +from pypacker.layer12 import ieee80211 + +# Create a dictionary of string representations of frame types +TYPE_KEYS = { + ieee80211.MGMT_TYPE: "MGMT", + ieee80211.CTL_TYPE: "CTRL", + ieee80211.DATA_TYPE: "DATA" +} + +# Create a dictionary of subtype keys from constants defined in ieee80211 +# Its keys will be tuple pairs of (TYPE, SUBTYPE) +SUBTYPE_KEYS = dict() +# Management frame subtypes +SUBTYPE_KEYS.update(dict(((ieee80211.MGMT_TYPE, v), k[2:]) for k, v in ieee80211.__dict__.items() if type(v) == int and k.startswith("M_"))) +# Control frame subtypes +SUBTYPE_KEYS.update(dict(((ieee80211.CTL_TYPE, v), k[2:]) for k, v in ieee80211.__dict__.items() if type(v) == int and k.startswith("C_"))) +# Data frame subtypes +SUBTYPE_KEYS.update(dict(((ieee80211.DATA_TYPE, v), k[2:]) for k, v in ieee80211.__dict__.items() if type(v) == int and k.startswith("D_"))) + +class DshellPlugin(dshell.core.PacketPlugin): + + OUTPUT_FORMAT = "[%(plugin)s] %(dt)s [%(ftype)s] [%(encrypted)s] [%(fsubtype)s] %(bodybytes)r %(retry)s\n" + + def __init__(self, *args, **kwargs): + super().__init__( + name="802.11", + description="Show 802.11 packet information", + author="dev195", + bpf="wlan type mgt or wlan type ctl or wlan type data", + output=Output(label=__name__, format=self.OUTPUT_FORMAT), + optiondict={ + "ignore_mgt": {"action": "store_true", "help": "Ignore management frames"}, + "ignore_ctl": {"action": "store_true", "help": "Ignore control frames"}, + "ignore_data": {"action": "store_true", "help": "Ignore data frames"}, + "ignore_beacon": {"action": "store_true", "help": "Ignore beacons"}, + }, + longdescription=""" +Shows basic information for 802.11 packets, including: + - Frame type + - Encryption + - Frame subtype + - Data sample +""" + ) + + def handle_plugin_options(self): + "Update the BPF based on 'ignore' flags" + # NOTE: This function is naturally called in decode.py + bpf_pieces = [] + if not self.ignore_mgt: + if self.ignore_beacon: + bpf_pieces.append("(wlan type mgt and not wlan type mgt subtype beacon)") + else: + bpf_pieces.append("wlan type mgt") + if not self.ignore_ctl: + bpf_pieces.append("wlan type ctl") + if not self.ignore_data: + bpf_pieces.append("wlan type data") + self.bpf = " or ".join(bpf_pieces) + + def packet_handler(self, pkt): + try: + frame = pkt.pkt.ieee80211 + except AttributeError: + frame = pkt.pkt + encrypted = "encrypted" if frame.protected else " " + frame_type = TYPE_KEYS.get(frame.type, '----') + frame_subtype = SUBTYPE_KEYS.get((frame.type, frame.subtype), "") + retry = "[resent]" if frame.retry else "" + bodybytes = frame.body_bytes[:50] + + self.write( + encrypted=encrypted, + ftype=frame_type, + fsubtype=frame_subtype, + retry=retry, + bodybytes=bodybytes, + **pkt.info() + ) + + return pkt diff --git a/dshell/plugins/wifi/wifibeacon.py b/dshell/plugins/wifi/wifibeacon.py new file mode 100644 index 0000000..9ead90b --- /dev/null +++ b/dshell/plugins/wifi/wifibeacon.py @@ -0,0 +1,66 @@ +""" +Shows 802.11 wireless beacons and related information +""" + +from collections import defaultdict +from datetime import datetime + +import dshell.core +from dshell.output.output import Output + +class DshellPlugin(dshell.core.PacketPlugin): + + OUTPUT_FORMAT = "[%(plugin)s]\t%(dt)s\tInterval: %(interval)s TU,\tSSID: %(ssid)s\t%(count)s\n" + + def __init__(self, *args, **kwargs): + super().__init__( + name="Wi-fi Beacons", + description="Show SSIDs of 802.11 wireless beacons", + author="dev195", + bpf="wlan type mgt subtype beacon", + output=Output(label=__name__, format=self.OUTPUT_FORMAT), + optiondict={ + "group": {"action": "store_true", "help": "Group beacons together with counts"}, + } + ) + self.group_counts = defaultdict(int) + self.group_times = defaultdict(datetime.now) + + def packet_handler(self, pkt): + # Extract 802.11 frame from packet + try: + frame = pkt.pkt.ieee80211 + except AttributeError: + frame = pkt.pkt + + # Confirm that packet is, in fact, a beacon + if not frame.is_beacon(): + return + + # Extract SSID from frame + beacon = frame.beacon + ssid = "" + try: + for param in beacon.params: + # Find the SSID parameter + if param.id == 0: + ssid = param.body_bytes.decode("utf-8") + break + except IndexError: + # Sometimes pypacker fails to parse a packet + return + + if self.group: + self.group_counts[(ssid, beacon.interval)] += 1 + self.group_times[(ssid, beacon.interval)] = pkt.ts + else: + self.write(ssid=ssid, interval=beacon.interval, **pkt.info()) + + return pkt + + def postfile(self): + if self.group: + for key, val in self.group_counts.items(): + ssid, interval = key + dt = self.group_times[key] + self.write(ssid=ssid, interval=interval, plugin=self.name, dt=dt, count=val) diff --git a/dshell/util.py b/dshell/util.py new file mode 100644 index 0000000..5ed2ded --- /dev/null +++ b/dshell/util.py @@ -0,0 +1,161 @@ +""" +A collection of useful utilities used in several plugins and libraries. +""" + +import os +import string + + +def xor(xinput, key): + """ + Xor an input string with a given character key. + + Arguments: + input: plain text input string + key: xor key + """ + output = ''.join([chr(ord(c) ^ key) for c in xinput]) + return output + + +def get_data_path(): + dpath = os.path.dirname(__file__) + return os.path.sep.join((dpath, 'data')) + + +def get_plugin_path(): + dpath = os.path.dirname(__file__) + return os.path.sep.join((dpath, 'plugins')) + + +def get_output_path(): + dpath = os.path.dirname(__file__) + return os.path.sep.join((dpath, 'output')) + + +def decode_base64(intext, alphabet='ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/', padchar='='): + """ + Decodes a base64-encoded string, optionally using a custom alphabet. + + Arguments: + intext: input plaintext string + alphabet: base64 alphabet to use + padchar: padding character + """ + # Build dictionary from alphabet + alphabet_index = {} + for i, c in enumerate(alphabet): + if c in alphabet_index: + raise ValueError("'{}' used more than once in alphabet".format(c)) + alphabet_index[c] = i + alphabet_index[padchar] = 0 + + alphabet += padchar + + outtext = '' + intext = intext.rstrip('\n') + + i = 0 + while i < len(intext) - 3: + if ( + intext[i] not in alphabet + or intext[i + 1] not in alphabet + or intext[i + 2] not in alphabet + or intext[i + 3] not in alphabet + ): + raise KeyError("Non-alphabet character in encoded text.") + val = alphabet_index[intext[i]] * 262144 + val += alphabet_index[intext[i + 1]] * 4096 + val += alphabet_index[intext[i + 2]] * 64 + val += alphabet_index[intext[i + 3]] + i += 4 + for factor in [65536, 256, 1]: + outtext += chr(int(val / factor)) + val = val % factor + + return outtext + + +def printable_text(intext, include_whitespace=True): + """ + Replaces non-printable characters with dots. + + Arguments: + intext: input plaintext string + include_whitespace (bool): set to False to mark whitespace characters + as unprintable + """ + printable = string.ascii_letters + string.digits + string.punctuation + if include_whitespace: + printable += string.whitespace + + if isinstance(intext, bytes): + intext = intext.decode("ascii", errors="replace") + + outtext = [c if c in printable else '.' for c in intext] + outtext = ''.join(outtext) + + return outtext + + +def hex_plus_ascii(data, width=16, offset=0): + """ + Converts a data string into a two-column hex and string layout, + similar to tcpdump with -X + + Arguments: + data: incoming data to format + width: width of the columns + offset: offset output from the left by this value + """ + output = "" + for i in range(0, len(data), width): + s = data[i:i + width] + if isinstance(s, bytes): + outhex = ' '.join(["{:02X}".format(x) for x in s]) + else: + outhex = ' '.join(["{:02X}".format(ord(x)) for x in s]) + outstr = printable_text(s, include_whitespace=False) + outstr = "{:08X} {:49} {}\n".format(i + offset, outhex, outstr) + output += outstr + return output + + +def gen_local_filename(path, origname): + """ + Generates a local filename based on the original. Automatically adds a + number to the end, if file already exists. + + Arguments: + path: output path for file + origname: original name of the file to transform + """ + + tmp = origname.replace("\\", "_") + tmp = tmp.replace("/", "_") + tmp = tmp.replace(":", "_") + localname = '' + for c in tmp: + if ord(c) > 32 and ord(c) < 127: + localname += c + else: + localname += "%%%02X" % ord(c) + localname = os.path.join(path, localname) + postfix = '' + i = 0 + while os.path.exists(localname + postfix): + i += 1 + postfix = "_{:04d}".format(i) + return localname + postfix + + +def human_readable_filesize(bytecount): + """ + Converts the raw byte counts into a human-readable format + https://stackoverflow.com/questions/1094841/reusable-library-to-get-human-readable-version-of-file-size/1094933#1094933 + """ + for unit in ('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB'): + if abs(bytecount) < 1024.0: + return "{:3.2f} {}".format(bytecount, unit) + bytecount /= 1024.0 + return "{:3.2f} {}".format(bytecount, "YB") diff --git a/install-ubuntu.sh b/install-ubuntu.sh deleted file mode 100755 index aff213b..0000000 --- a/install-ubuntu.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -pkgs="" - -if `python -c 'import Crypto'`; -then echo "PyCrypto is installed" -else - echo "dshell requires PyCrypto" - pkgs="$pkgs python-crypto" -fi - -if `python -c 'import dpkt'`; -then echo "dpkt is installed" -else - echo "dshell requires dpkt" - pkgs="$pkgs python-dpkt" -fi - -if `python -c 'from IPy import IP'`; -then echo "IPy is installed" -else - echo "dshell requires IPy" - pkgs="$pkgs python-ipy" -fi - -if `python -c 'from pcap import pcap'`; -then echo "pypcap is installed" -else - echo "dshell requires pypcap" - pkgs="$pkgs python-pypcap" -fi - -cmd="sudo apt-get install $pkgs" -if [ "$pkgs" ]; then - echo - echo $cmd - echo - $cmd; -fi - -make all diff --git a/lib/dfile.py b/lib/dfile.py deleted file mode 100644 index 04ebe3f..0000000 --- a/lib/dfile.py +++ /dev/null @@ -1,166 +0,0 @@ -''' -Dshell external file class/utils -for use in rippers, dumpers, etc. - -@author: amm -''' -import os -from dshell import Blob -from shutil import move -from hashlib import md5 - -''' -Mode Constants -''' -FILEONDISK = 1 # Object refers to file already written to disk -FILEINMEMORY = 2 # Object contains file contents in data member - -''' -dfile -- Dshell file class. - -Extends blob for offset based file chunk (segment) reassembly. -Removes time and directionality from segments. - -Decoders can instantiate this class and pass it to -output modules or other decoders. - -Decoders can choose to pass a file in memory or already -written to disk. - -A dfile object can have one of the following modes: - FILEONDISK - FILEINMEMORY - -''' -class dfile(Blob): - - def __init__(self,mode=FILEINMEMORY,name=None,data=None,**kwargs): - - # Initialize Segments - # Only really used in memory mode - self.segments={} - self.startoffset=0 - self.endoffset=0 - - # Initialize consistent info members - self.mode=mode - self.name=name - self.diskpath=None - self.info_keys = ['mode','name','diskpath','startoffset','endoffset'] - - #update with additional info - self.info(**kwargs) - #update data - if data != None: - self.update(data) - - def __iter__(self): - ''' - Undefined - ''' - pass - - def __str__(self): - ''' - Returns filename (string) - ''' - return self.name - - def __repr__(self): - ''' - Returns filename (string) - ''' - return self.name - - def md5(self): - ''' - Returns md5 of file - Calculate based on reassembly from FILEINMEMORY - or loads from FILEONDISK - ''' - if self.mode == FILEINMEMORY: - return md5(self.data()).hexdigest() - elif self.mode == FILEONDISK: - m = md5() - fh = open(self.diskpath, 'r') - m.update(fh.read()) - fh.close() - return m.hexdigest() - else: - return None - - def load(self): - ''' - Load file from disk. Converts object to mode FILEINMEMORY - ''' - if not self.mode == FILEONDISK: return False - try: - fh = open(self.diskpath, 'r') - self.update(fh.read()) - fh.close() - self.mode = FILEINMEMORY - except: - return False - - def write(self,path='.',name=None,clobber=False,errorHandler=None,padding=None,overlap=True): - ''' - Write file contents at location relative to path. - Name on disk will be based on internal name unless one is provided. - - For mode FILEINMEMORY, file will data() will be called for reconstruction. - After writing to disk, mode will be changed to FILEONDISK. - If mode is already FILEONDISK, file will be moved to new location. - - ''' - olddiskpath = self.diskpath - if name == None: name=self.name - self.diskpath = self.__localfilename(name, path, clobber) - if self.mode == FILEINMEMORY: - fh = open(self.diskpath, 'w') - fh.write(self.data()) - fh.close() - self.segments={} - self.startoffset=0 - self.endoffset=0 - return self.diskpath - elif self.mode == FILEONDISK: - move(olddiskpath, self.diskpath) - return self.diskpath - - def update(self,data,offset=None): - if self.mode != FILEINMEMORY: return - #if offsets are not being provided, just keep packets in wire order - if offset==None: offset=self.endoffset - #don't buffer duplicate packets - if offset not in self.segments: self.segments[offset]=data - #update the end offset if this packet goes at the end - if offset >= self.endoffset: self.endoffset=offset+len(data) - - # - # Generate a local (extracted) filename based on the original - # - def __localfilename(self, origname, path = '.', clobber = False): - tmp = origname.replace("\\", "_") - tmp = tmp.replace("/", "_") - tmp = tmp.replace(":", "_") - tmp = tmp.replace("?", "_") - tmp = tmp.lstrip('_') - localname = '' - for c in tmp: - if ord(c) > 32 and ord(c) < 127: - localname += c - else: - localname += "%%%02X" % ord(c) - # Truncate (from left) to max filename length on filesystem (-3 in case we need to add a suffix) - localname = localname[os.statvfs(path).f_namemax*-1:] - # Empty filename not allowed - if localname == '': localname = 'blank' - localname = os.path.realpath(os.path.join(path,localname)) - if clobber: return localname - # No Clobber mode, check to see if file exists - suffix = '' - i = 0 - while os.path.exists(localname+suffix): - i += 1 - suffix = "_%02d" % i - return localname+suffix diff --git a/lib/dnsdecoder.py b/lib/dnsdecoder.py deleted file mode 100644 index 5cd9024..0000000 --- a/lib/dnsdecoder.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python - -import dshell, util, dpkt - -class DNSDecoder(dshell.TCPDecoder): - '''extend DNSDecoder to handle DNS request/responses - pairs request and response(s) by connection and query ID - to allow for detection of DNS spoofing, etc.. (multiple responses to request with same ID) - will call DNSHandler( - conn=Connection(), - request=dpkt.dns.DNS, - response=dpkt.dns.DNS, - requesttime=timestamp, responsetime=timestamp, - responsecount=responsecount - ) - after each response. - - config: noanswer: if True and discarding w/o response, will call with response,responsetime=None,None (True) - - ''' - def __init__(self,**kwargs): - self.noanswer=True - dshell.TCPDecoder.__init__(self,**kwargs) #DNS is over UDP and TCP! - self.requests={} - self.maxblobs=None - - def blobHandler(self,conn,blob): - '''for each blob, examine each segment (UDP packet) seperately as each will be a DNS Q/A - pair Q/A by ID and return as pairs''' - connrqs=self.requests.setdefault(conn,{}) - for data in blob: #iterate blob as each packet will be a seperate request (catches spoofing) - try: dns=dpkt.dns.DNS(data) - except Exception,e: self._exc(e) - if dns.qr == dpkt.dns.DNS_Q: - connrqs[dns.id]=[blob.starttime,dns,0] - elif dns.qr == dpkt.dns.DNS_A: - rq=connrqs.get(dns.id,[None,None,0]) - rq[2]+=1 - if "DNSHandler" in dir(self): - self.DNSHandler(conn=conn,request=rq[1],response=dns,requesttime=rq[0],responsetime=blob.starttime,responsecount=rq[2]) - - def connectionHandler(self,conn): - '''clean up unanswered requests when we discard the connection''' - if self.noanswer and "DNSHandler" in dir(self) and self.requests.get(conn): - for requesttime,request,responsecount in self.requests[conn].values(): - if not responsecount: - self.DNSHandler(conn=conn,request=request,response=None,requesttime=requesttime,responsetime=None,responsecount=responsecount) - if conn in self.requests: del self.requests[conn] - -class displaystub(dshell.Decoder): - def __init__(self): - dshell.Decoder.__init__(self, - name='dnsdecoder', - description='Intermediate class to support DNS based decoders.', - longdescription="See source code or pydoc for details on use." - ) - -if __name__=='__main__': - dObj = displaystub() - print dObj -else: - dObj = displaystub() diff --git a/lib/dshell.py b/lib/dshell.py deleted file mode 100644 index 5af0686..0000000 --- a/lib/dshell.py +++ /dev/null @@ -1,873 +0,0 @@ -""" -Dshell base classes -""" - -__version__ = "3.0" - -import dpkt -import struct,socket,traceback -import util -import os, datetime,logging -import binascii - -# For IP lookups -try: import pygeoip -except: pass - - -class Decoder(object): - """ - Base class that all decoders will inherit - - The Dshell class initializes the decoder to work in the framework - and provides common functions such as CC/ASN lookup - - Configuration attributes, settable by Dshell.__init__(attr=value,...) or in subclass __init__: - name: name of this decoder. - description: single-line description of this decoder - longdescription: multi-line description of this decoder - author: who to blame for this decoder - - filter: default BPF filter for capture. - - format: output format string for this decoder, overrides default for - Please read how text, DB, etc.. Output() classes parse a format string. - - optionsdict: optionParser compatible config, specific to decoder - dict of { 'optname':{'default':..., 'help':..., etc... - }, - 'optname':... } - 'optname' is set by --deodername_optname=... on command line - and under [decodername] section in config file - - - cleanupinterval - seconds with no activity before state is discarded (default 60) - - chainable - set True to indicate this decoder can be chained (can pass output to another decoder) - subDecoder - decoder to pass output to, if not None. - (create new Data objects and call subDecoder.XHandler from XHandler, etc..) - - - """ - def __super__(self): - '''convenience function to get bound instance of superclass''' - return super(self.__class__,self) - - def __init__(self,**kwargs): - self.name = 'unnamed' - self.description = '' - self.longdescription = '' - self.filter = '' - self.author = 'xx' - self.decodedbytes=0 - self.count=0 - '''dict of options specific to this decoder in format - 'optname':{configdict} translates to --decodername_optname''' - self.optiondict={} - - #out holds the output plugin. If None, will inherit the global output - self.out=None - #format is the format string for this plugin, if None, uses global - self.format=None - - #capture options - self.l2decoder=dpkt.ethernet.Ethernet #decoder to use if raw mode - self.striplayers=0 #strip extra layers before IP/IPv6? (such as PPPoE, IP-over-IP, etc..) - - self._DEBUG=False - - self.chainable=False #can we chain a decoder off the output of this one? - self.subDecoder=None #decoder to pass output to for chaining - - #set flags to indicate if handlers are present - if 'packetHandler' in dir(self): self.isPacketHandlerPresent = True - else: self.isPacketHandlerPresent = False - if 'connectionHandler' in dir(self): self.isConnectionHandlerPresent = True - else: self.isConnectionHandlerPresent = False - if 'blobHandler' in dir(self): self.isBlobHandlerPresent = True - else: self.isBlobHandlerPresent = False - - #for connection tracking, if applicable - self.connectionsDict = {} - self.cleanupts=0 - - #instantiate and save references to lookup function - try: - self.geoccdb = [ pygeoip.GeoIP(os.environ['DATAPATH']+'/GeoIP/GeoIP.dat', pygeoip.MEMORY_CACHE).country_code_by_addr ] - try: self.geoccdb.append( pygeoip.GeoIP(os.environ['DATAPATH']+'/GeoIP/GeoIPv6.dat', pygeoip.MEMORY_CACHE).country_code_by_addr ) - except: pass - except: - self.geoccdb = None - - try: - self.geoasndb = [ pygeoip.GeoIP(os.environ['DATAPATH']+'/GeoIP/GeoIPASNum.dat', pygeoip.MEMORY_CACHE).org_by_addr ] - try: self.geoasndb.append( pygeoip.GeoIP(os.environ['DATAPATH']+'/GeoIP/GeoIPASNumv6.dat', pygeoip.MEMORY_CACHE).org_by_addr ) - except: pass - except: - self.geoasndb = None - - #import kw args into class members - if kwargs: self.__dict__.update(kwargs) - - '''convenience functions for alert output and logging''' - def alert(self,*args,**kw): - '''sends alert to output handler - typically self.alert will be called with the decoded data and the packet/connection info dict last, as follows: - - self.alert(alert_arg,alert_arg2...,alert_data=value,alert_data2=value2....,**conn/pkt.info()) - - example: self.alert(decoded_data,conn.info(),blob.info()) [blob info overrides conn info] - - this will pass all information about the decoder, the connection, and the specific event up to the output module - - if a positional arg is a dict, it updates the kwargs - if an arg is a list, it extends the arg list - else it is appended to the arg list - - all arguments are optional, at the very least you want to pass the **pkt/conn.info() so all traffic info is available. - - output modules handle this data as follows: - - the name of the alerting decoder is available in the "decoder" field - - all non-keyword arguments will be concatenated into the "data" field - - keyword arguments, including all provided by .info() will be used to populate matching fields - - remaining keyword arguments that do not match fields will be represented by "key=value" strings - concatenated together into the "extra" field - ''' - oargs=[] - for a in args: - #merge dict args, overriding kws - if type(a)==dict: kw.update(a) - elif type(a)==list: oargs.extend(a) - else: oargs.append(a) - if 'decoder' not in kw: kw['decoder']=self.name - self.out.alert(*oargs,**kw) #add decoder name - - def write(self,obj,**kw): - '''write session data''' - self.out.write(obj,**kw) - - def dump(self,*args,**kw): - '''write packet data (probably to the PCAP writer if present)''' - if len(args)==3: - kw['len'],kw['pkt'],kw['ts']=args - elif len(args)==2: - kw['pkt'],kw['ts']=args - elif len(args)==1: - kw['pkt']=args[0] - self.out.dump(**kw) - - def log(self,msg,level=logging.INFO): - '''logs msg at specified level (default of INFO is for -v/--verbose output)''' - self.out.log(msg,level=level) #default level is INFO (verbose) can be overridden - - def debug(self,msg): - '''logs msg at debug level''' - self.log(msg,level=logging.DEBUG) - - def warn(self,msg): - '''logs msg at warning level''' - self.log(msg,level=logging.WARN) - pass - - def error(self,msg): - '''logs msg at error level''' - self.log(msg,level=logging.ERROR) - - def __repr__(self): - return '%s %s %s' % (self.name,self.filter, - ' '.join([ ('%s=%s'%(x, str(self.__dict__.get(x)))) for x in self.optiondict.keys() ] ) ) - - - def preModule(self): - '''preModule is called before capture starts - default preModule, dumps object data to debug''' - if self.subDecoder: self.subDecoder.preModule() - self.debug (self.name+' '+str(self.__dict__)) - - def postModule(self): - '''postModule is called after capture stops - default postModule, prints basic decoding stats''' - self.cleanConnectionStore() - self.log("%s: %d packets (%d bytes) decoded"%(self.name,self.count,self.decodedbytes)) - if self.subDecoder: self.subDecoder.postModule() - - def preFile(self): - if self.subDecoder: self.subDecoder.preFile() - - def postFile(self): - if self.subDecoder: self.subDecoder.postFile() - - def parseOptions(self,options={}): - '''option keys:values will set class members (self.key=value) - if key is in optiondict''' - for optname in self.optiondict.iterkeys(): - if optname in options: self.__dict__[optname]=options[optname] - - def parseArgs(self, args, options={}): - '''called to parse command-line arguments and cli/config file options - if options dict contains 'all' or the decoder name as a key - class members will be updated from value''' - #get positional args after the -- - self.args = args - #update from all decoders section of config file - if 'all' in options: self.parseOptions(options['all']) - #update from named section of config file - if self.name in options: self.parseOptions(options[self.name]) - - def getGeoIP(self, ip, db=[], notfound='--'): - """ - Get record associated with an IP - requires GeoIP - """ - o=None - if db == []: db = self.geoccdb # default to self.geoccdb - for d in db: - try: o=d(ip) - except: - #traceback.print_exc() # removed by bg on 20121203 - continue #passing ipv6 address to ipv4 lookup or v/v - if o: return o #stop when we get a result - return notfound - - def getASN(self, ip, db=[], notfound='--'): - """ - Get record associated with an IP - requires GeoIP - """ - o=None - if db == []: db = self.geoasndb # default to self.geoccdb - for d in db: - try: o=d(ip) - except: - #traceback.print_exc() # removed by bg on 20121203 - continue #passing ipv6 address to ipv4 lookup or v/v - if o: return o #stop when we get a result - return notfound - - def close(self,conn,ts=None): - '''for connection based decoders - close and discard the connection object''' - #just return if we have already been called on this connection - #prevents infinite loop of a handler calling close() when we call it - if conn.state=='closed': return - - #set state to closed - conn.state='closed' - if ts: conn.endtime=ts - # we have already stopped this so don't call the handlers if we have already stopped - if not conn.stop: - #flush out the last blob to the blob handler - if self.isBlobHandlerPresent and conn.blobs: self.blobHandler(conn,conn.blobs[-1]) - # process connection handler - if self.isConnectionHandlerPresent: self.connectionHandler(conn) - #connection close handler - #will be called regardless of conn.stop right before conn object is destroyed - if 'connectionCloseHandler' in dir(self): self.connectionCloseHandler(conn) - #discard but check first in case a handler deleted it - if conn.addr in self.connectionsDict: del self.connectionsDict[conn.addr] - - def stop(self,conn): - '''stop following connection - handlers will not be called, except for connectionCloseHandler''' - conn.stop = True - - def cleanup(self,ts): - '''if cleanup interval expired, close connections not updated in last interval''' - ts=util.mktime(ts) - #if cleanup interval has passed - if self.cleanupts<(ts-self.cleanupinterval): - for conn in self.connectionsDict.values(): - if util.mktime(conn.endtime)<=self.cleanupts: self.close(conn) - self.cleanupts=ts - - def cleanConnectionStore(self): - '''cleans connection store of all information, flushing out data''' - for conn in self.connectionsDict.values(): self.close(conn) - - def _exc(self,e): - '''exception handler''' - self.warn(str(e)) - if self._DEBUG: traceback.print_exc() - - def find(self,addr,state=None): - if addr in self.connectionsDict: conn=self.connectionsDict[addr] - elif (addr[1],addr[0]) in self.connectionsDict: conn=self.connectionsDict[(addr[1],addr[0])] - else: return None - if not state or conn.state==state: return conn - else: return None - - def track(self,addr,data=None,ts=None,offset=None,**kwargs): - '''connection tracking for TCP and UDP - finds or creates connection based on addr - updates connection with data if provided (using offset to reorder) - tracks timestamps if ts provided - extra args get passed to new connection objects - ''' - conn=self.find(addr) - #look for forward and reverse address tuples - if not conn: #create new connection - #if swapping and source has low port, swap source/dest so dest has low port - if self.swaplowport and addr[0][1]1: conn.blobs.pop(0) - - self.cleanup(ts) #do stale state cleanup - return conn #return a reference to the connection - - '''directly extend Decoder and set raw=True to capture raw PCAP data''' - - #we get the raw output from pcapy as header, data - def decode(self,*args,**kw): - if len(args) is 3: pktlen,pktdata,ts=args #orig_len,packet,ts format (pylibpcap) - else: #ts,pktdata (pypcap) - ts,pktdata=args - pktlen=len(pktdata) - try: - if pktlen!=len(pktdata): raise Exception('packet truncated',pktlen,pktdata) - #decode with the L2 decoder (probably Ether) - pkt=self.l2decoder(pktdata) - #strip any intermediate layers (PPPoE, etc) - for l in xrange(int(self.striplayers)): pkt=pkt.data - '''will call self.rawHandler(len,pkt,ts) - (hdr,data) is the PCAP header and raw packet data''' - if 'rawHandler' in dir(self): - self.rawHandler(pktlen,pkt,ts,**kw) - else: pass - except Exception,e: self._exc(e) - -#IP handler -class IPDecoder(Decoder): - '''extend IP6Decoder to capture IPv4 and IPv6 data - (but does basic IPv4 defragmentation) - config: - - l2decoder: dpkt class for layer-2 decoding (Ethernet) - striplayers: strip n layers above layer-2, removes PPP/PPPoE encap, IP-over-IP, etc.. (0) - defrag: defragment IPv4 (True) - v6only: if True, will ignore IPv4 data. (False) - decode6to4: if True, will decode IPv6-over-IP, if False will treat as IP (True) - - filterfn: lambda function that accepts addr 2x2tuples and returns if packet should pass (addr:True) - - filterfn is required for IPv6 as port-based BPF filters don't work, - so keep your BPF to 'tcp' or 'udp' and set something like - self.filterfn = lambda ((sip,sp),(dip,dp)): (sp==53 or dp==53) ''' - - IP_PROTO_MAP={ - dpkt.ip.IP_PROTO_ICMP:'ICMP', - dpkt.ip.IP_PROTO_ICMP6:'ICMP6', - dpkt.ip.IP_PROTO_TCP:'TCP', - dpkt.ip.IP_PROTO_UDP:'UDP', - dpkt.ip.IP_PROTO_IP6:'IP6', - dpkt.ip.IP_PROTO_IP:'IP'} - - def __init__(self, **kwargs): - self.v6only=False - self.decode6to4=True - self.defrag=True - self.striplayers=0 - self.l2decoder=dpkt.ethernet.Ethernet - self.filterfn=lambda addr: True - Decoder.__init__(self,**kwargs) - self.frags={} - - def ipdefrag(self,pkt): - '''ip fragment reassembly''' - #if pkt.off&dpkt.ip.IP_DF or pkt.off==0: return pkt #DF or !MF and offset 0 - f=self.frags.setdefault((pkt.src,pkt.dst,pkt.id),{}) #if we need to create a store for this IP addr/id - f[pkt.off&dpkt.ip.IP_OFFMASK]=pkt - offset=0 - data='' - while True: - if offset not in f: return None #we don't have this offset, can't reassemble yet - data+=str(pkt.data) #add this to the data - if not pkt.off&dpkt.ip.IP_MF: break #this is the next packet in order and no more fragments - offset=len(data)/8 #calculate the next fragment's offset - #we hit no MF and last offset, so return a defragged packet - del self.frags[(pkt.src,pkt.dst,pkt.id)] #discard store - pkt.data=data #replace payload with defragged data - pkt.off=0 #no frags, offset 0 - pkt.sum=0 #recompute checksum - return dpkt.ip.IP(str(pkt)) #dump and redecode packet to get checksum right - - def rawHandler(self,pktlen,pkt,ts,**kwargs): - '''takes ethernet data and determines if it contains IP or IP6. - defragments IPv4 - if 6to4, unencaps the IPv6 - If IP/IP6, hands off to IPDecoder via IPHandler()''' - try: - #if this is an IPv4 packet, defragment, decode and hand it off - if type(pkt.data)==dpkt.ip.IP: - if self.defrag: pkt=self.ipdefrag(pkt.data) #return packet if whole, None if more frags needed - else: pkt=pkt.data #get the layer 3 packet - if pkt: #do we have a whole IP packet? - if self.decode6to4 and pkt.p == dpkt.ip.IP_PROTO_IP6: pass #fall thru to ip6 decode - elif not self.v6only: #if we are decoding ip4 - sip,dip=socket.inet_ntoa(pkt.src),socket.inet_ntoa(pkt.dst) - #try to decode ports - try: - sport,dport=pkt.data.sport,pkt.data.dport - except: #no ports in this layer-4 protocol - sport,dport=None,None - #generate int forms of src/dest ips - sipint,dipint=struct.unpack('!L',pkt.src)[0],struct.unpack('!L',pkt.dst)[0] - #call IPHandler with extra data - self.IPHandler(((sip,sport),(dip,dport)),pkt,ts, - pkttype=dpkt.ethernet.ETH_TYPE_IP, - proto=self.IP_PROTO_MAP.get(pkt.p,pkt.p), - sipint=sipint,dipint=dipint, - **kwargs) - if pkt and type(pkt.data)==dpkt.ip6.IP6: - pkt=pkt.data #no defrag of ipv6 - #decode ipv6 addresses - sip,dip=socket.inet_ntop(socket.AF_INET6,pkt.src), socket.inet_ntop(socket.AF_INET6,pkt.dst) - #try to get layer-4 ports - try: - sport,dport=pkt.data.sport,pkt.data.dport - except: - sport,dport=None,None - #call ipv6 handler - self.IPHandler(((sip,sport),(dip,dport)),pkt,ts, - pkttype=dpkt.ethernet.ETH_TYPE_IP6, - proto=self.IP_PROTO_MAP.get(pkt.nxt,pkt.nxt), - **kwargs) - except Exception, e: self._exc(e) - - def IPHandler(self,addr,pkt,ts,**kwargs): - '''called if packet is IPv4/IPv6 - check packets using filterfn here''' - self.decodedbytes+=len(str(pkt)) - self.count+=1 - if self.isPacketHandlerPresent and self.filterfn(addr): - return self.packetHandler(ip=Packet(self,addr,pkt=str(pkt),ts=ts,**kwargs)) -class IP6Decoder(IPDecoder): pass - -class UDPDecoder(IPDecoder): - '''extend UDPDecoder to decode UDP optionally track state - config if tracking state with connectionHandler or blobHandler - maxblobs - if tracking state, max blobs to track before flushing - swaplowport - when establishing state, swap source/dest so dest has low port - cleanupinterval - seconds with no activity before state is discarded (default 60) ''' - def __init__(self, **kwargs): - self.maxblobs=2 #by default limit UDP 'connections' to a single request and response - self.swaplowport=True #can we swap source/dest so dest always has low port? - self.cleanupinterval=60 - IPDecoder.__init__(self,**kwargs) - - def UDP(self,addr,data,pkt,ts=None,**kwargs): - ''' will call self.packetHandler(udp=Packet(),data=data) - (see Packet() for Packet object common attributes) - udp.pkt will contain the raw IP data - data will contain the decoded UDP payload - - State tracking: - only if connectionHandler or blobHandler is present - and packetHandler is not present - - UDPDecoder will call: - self.connectionInitHandler(conn=Connection()) - when UDP state is established - (see Connection() for Connection object attributes) - - self.blobHandler(conn=Connection(),blob=Blob()) - when stream direction switches (if following stream) - blob=(see Blob() objects) - - self.connectionHandler(conn=Connection()) - when UDP state is flushed (if following stream) - state is flushed when stale or when maxblobs is exceeded - if maxblobs exceeded, current data will go into new connection - - self.connectionCloseHandler(conn=Connection()) - when state is discarded (always) - ''' - self.decodedbytes+=len(data) - self.count+=1 - try: - if self.isPacketHandlerPresent: - #create a Packet object and populate it - return self.packetHandler(udp=Packet(self,addr,pkt=pkt,ts=ts,**kwargs),data=data) - - #if no PacketHandler, we need to track state - self.track(addr,data,ts,**kwargs) - - except Exception,e: self._exc(e) - - def IPHandler(self,addr,pkt,ts,**kwargs): - '''IPv4 dispatch, hands address, UDP payload and packet up to UDP callback''' - if self.filterfn(addr): - if type(pkt.data)==dpkt.udp.UDP: return self.UDP(addr,str(pkt.data.data),str(pkt),ts,**kwargs) - -class UDP6Decoder(UDPDecoder): pass - -class TCPDecoder(UDPDecoder): - '''IPv6 TCP/UDP decoder - reassembles TCP and UDP streams - For TCP and UDP (if no packetHandler) - self.connectionInitHandler(conn=Connection()) - when TCP connection is established - (see Connection() for Connection object attributes) - - self.blobHandler(conn=Connection(),blob=Blob()) - when stream direction switches (if following stream) - blob=(see Blob() objects) - - self.connectionHandler(conn=Connection()) - when connection closes (if following stream) - - self.connectionCloseHandler(conn=Connection()) - when connection closes (always) - - For UDP only: - self.packetHandler(udp=Packet(),data=data) - with every packet - data=decoded UDP data - - if packetHandler is present, it will be called only for UDP (and UDP will not be tracked)''' - - def __init__(self, **kwargs): - self.maxblobs=None #no limit on connections - self.swaplowport=False #can we swap source/dest so dest always has low port? - self.ignore_handshake=False #if set true, will requre TCP handshake to track connection - self.cleanupinterval=300 - #up two levels to IPDecoder - IPDecoder.__init__(self,**kwargs) - self.optiondict['ignore_handshake']={'action':'store_true','help':'ignore TCP handshake'} - - def IPHandler(self,addr,pkt,ts,**kwargs): - '''IPv4 dispatch''' - if self.filterfn(addr): - if type(pkt.data)==dpkt.udp.UDP: return self.UDP(addr,str(pkt.data.data),str(pkt),ts,**kwargs) - elif type(pkt.data)==dpkt.tcp.TCP: return self.TCP(addr,pkt.data,ts,**kwargs) - - def TCP(self,addr,tcp,ts,**kwargs): - '''TCP dispatch''' - self.decodedbytes+=len(str(tcp)) - self.count+=1 - - try: - #close connection - if tcp.flags&(dpkt.tcp.TH_FIN|dpkt.tcp.TH_RST): - conn=self.find(addr) - if conn: - #we might occasionally have data in a FIN packet - self.track(addr,str(tcp.data),ts,offset=tcp.seq) - self.close(conn, ts) - #init connection, set TCP ISN - elif not self.ignore_handshake and tcp.flags==dpkt.tcp.TH_SYN: - conn=self.track(addr,ts=ts,state='init',**kwargs) - if conn: conn.nextoffset['cs']=tcp.seq+1 - #SYN ACK - elif not self.ignore_handshake and tcp.flags==(dpkt.tcp.TH_SYN|dpkt.tcp.TH_ACK): - conn=self.find(addr,state='init') - if conn and tcp.ack==conn.nextoffset['cs']: - conn.nextoffset['sc']=tcp.seq+1 - conn.state='established' - - #all other states, or always if ignoring handshake - if self.ignore_handshake or self.find(addr,state='established'): self.track(addr,str(tcp.data),ts, - state='established',offset=tcp.seq,**kwargs) - - except Exception,e: self._exc(e) - -class TCP6Decoder(TCPDecoder): pass - - -class Data(object): - '''base class for data objects (packets,connections, etc..) - these objects hold data (appendable array, typically of strings) - and info members (updateable/accessible as members or as dict via info()) - typically one will extend the Data class and replace the data member - and associated functions (update,iter,str,repr) with a data() function - and functions to manipulate the data''' - - def __init__(self,*args,**kwargs): - self.info_keys = [] - #update with list data - self.data=list(args) - #update with keyword data - self.info(**kwargs) - - def info(self,*args,**kwargs): - '''update/return info stored in this object - data can be passwd as dict(s) or keyword args''' - args=list(args)+[kwargs] - for a in args: - for k,v in a.iteritems(): - if k not in self.info_keys: self.info_keys.append(k) - self.__dict__[k]=v - return dict((k,self.__dict__[k]) for k in self.info_keys) - - def unpack(self,fmt,data,*args): - '''unpacks data using fmt to keys listed in args''' - self.info(dict(zip(args,struct.unpack(fmt,data)))) - - def pack(self,fmt,*args): - '''packs info keys in args using fmt''' - return struct.pack(fmt,*[self.__dict__[k] for k in args]) - - def update(self,*args,**kwargs): - '''updates data (and optionally keyword args)''' - self.data.extend(args) - self.info(kwargs) - - def __iter__(self): - '''returns each data element in order added''' - for data in self.data: yield data - - def __str__(self): - '''return string built from data''' - return ''.join(self.data) - - def __repr__(self): - return ' '.join(['%s=%s'%(k,v) for k,v in self.info().iteritems()]) - - def __getitem__(self,k): return self.__dict__[k] - def __setitem__(self,k,v): self.__dict__[k]=v - - -class Packet(Data): - '''metadata class for connectionless data - Members: - sip, sport, dip, dport : source ip and port, dest ip and port - addr : ((sip,sport),(dip,dport)) tuple. sport/dport will be None if N/A - sipcc, dipcc, sipasn, dipasn : country codes and ASNs for source and dest IPs - ts : datetime.datetime() UTC timestamp of packet. use util.mktime(ts) to get POSIX timestamp - pkt : raw packet data - any additional args will be added to info dict - ''' - def __init__(self,decoder,addr,ts=None,pkt=None,**kwargs): - self.info_keys = ['addr','sip','dip','sport','dport','ts'] - self.addr=addr - #do not define pkt unless passed in - self.ts=ts - ((self.sip,self.sport),(self.dip,self.dport))=self.addr - if pkt: - self.pkt=pkt - self.info(bytes=len(self.pkt)) - - #pass instantiating decoder's cc/asn lookup objects to keep global cache - try: - self.info(sipcc = decoder.getGeoIP(self.sip,db=decoder.geoccdb), - sipasn = decoder.getGeoIP(self.sip,db=decoder.geoasndb), - dipcc = decoder.getGeoIP(self.dip,db=decoder.geoccdb), - dipasn = decoder.getGeoIP(self.dip,db=decoder.geoasndb)) - except: - self.sipcc,self.sipasn,self.dipcc,self.dipasn=None,None,None,None - - #update with additional info - self.info(**kwargs) - - def __iter__(self): - for p in self.pkt: yield ord(p) - - def __str__(self): - return self.pkt - - def __repr__(self): - return "%(ts)s %(sip)16s :%(sport)-5s -> %(dip)5s :%(dport)-5s (%(sipcc)s -> %(dipcc)s)\n"%self.info() - -class Connection(Packet): - """ - Connection class is used for tracking all information - contained within an established TCP connection / UDP pseudoconnection - - Extends Packet() - - Additional members: - {client|server}ip, {client|server}port: aliases of sip,sport,dip,dport - {client|server}countrycode, {client|server}asn: aliases of sip/dip country codes and ASNs - clientpackets, serverpackets: counts of packets from client and server - clientbytes, serverbytes: total bytes from client and server - starttime,endtime: timestamps of start and end (or last packet) time of connection. - direction: indicates direction of last traffic: - 'init' : established, no traffic - 'cs': client to server - 'sc': server to client - state: TCP state of this connection - blobs: array of reassembled half stream blobs - a new blob is started when the direction changes - stop: if True, stopped following stream - - """ - MAX_OFFSET=0xffffffff #max offset before wrap, default is MAXINT32 for TCP sequence numbers - - def __init__(self,decoder,addr,ts=None,**kwargs): - self.state=None - self.nextoffset={'cs':0,'sc':0} #the offset we expect for the next blob in this direction - Packet.__init__(self,decoder,addr,ts=ts,**kwargs) #init IP-level data - self.clientip,self.clientport,self.serverip, self.serverport=(self.sip,self.sport,self.dip,self.dport) - self.info_keys.extend(['clientip','serverip','clientport','serverport']) - self.clientcountrycode,self.clientasn,self.servercountrycode,self.serverasn=(self.sipcc,self.sipasn,self.dipcc,self.dipasn) - self.info_keys.extend(['clientcountrycode','servercountrycode','clientasn','serverasn']) - self.clientpackets = 0 # we have the first packet for each connection - self.serverpackets = 0 - self.clientbytes = 0 - self.serverbytes = 0 - self.starttime = self.ts # datetime Obj containing start time - self.endtime = self.ts - self.direction = 'init' # first update will change this, creating first blob - self.info_keys.extend(['clientpackets','clientbytes','serverpackets','serverbytes','starttime','endtime','state','direction']) - - self.blobs = [] #list of tuples of (direction,halfstream,startoffset,endoffset) indicating where each side talks - self.stop = False - - - def __repr__(self): - # starttime cip sip - return '%s %16s -> %16s (%s -> %s) %6s %6s %5d %5d %7d %7d %6ds %s' % ( - self.starttime, - self.clientip, - self.serverip, - self.clientcountrycode, - self.servercountrycode, - self.clientport, - self.serverport, - self.clientpackets, - self.serverpackets, - self.clientbytes, - self.serverbytes, - (util.mktime(self.endtime) - util.mktime(self.starttime)), - self.state) - - def update(self,ts,direction,data,offset=None): - #if we have no blobs or direction changes, start a new blob - lastblob=None - if direction != self.direction: - self.direction=direction - #if we have a finished blob, return it - if self.blobs: lastblob=self.blobs[-1] - #for tracking offsets across blobs (TCP) set the startoffset of this blob to what we know it should be - #this may not necessarily be the offset of THIS data if packets are out of order - self.blobs.append(Blob(ts,direction,startoffset=self.nextoffset[direction])) - self.blobs[-1].update(ts,data,offset=offset) #update latest blob - if direction=='cs': - self.clientpackets+=1 - self.clientbytes+=len(data) - elif direction=='sc': - self.serverpackets+=1 - self.serverbytes+=len(data) - self.endtime=ts - #if we are tracking offsets, expect the next blob to be where this one ends so far - if offset!=None and offset >= self.nextoffset[direction]: self.nextoffset[direction]=(offset+len(data))&self.MAX_OFFSET - return lastblob - - #return one or both sides of the stream - def data(self, direction=None, errorHandler=None, padding=None, overlap=True, caller=None): - '''returns reassembled half-stream selected by direction 'sc' or 'cs' - if no direction, return all stream data interleaved - see Blob.data() for errorHandler docs''' - return ''.join( [b.data(errorHandler=errorHandler, padding=padding, overlap=overlap, caller=caller) for b in self.blobs if (not direction or b.direction==direction)] ) - - def __str__(self): - '''return all data interleaved''' - return self.data(padding='') - - def __iter__(self): - '''return each blob in capture order''' - for blob in self.blobs: yield blob - -class Blob(Data): - '''a blob containins a contiguous part of the half-stream - Members: - starttime,endtime : start and end timestamps of this blob - direction : direction of this blob's data 'sc' or 'cs' - data(): this blob's data - startoffset,endoffset: offset of this blob start/end in bytes from start of stream - ''' - - MAX_OFFSET=0xffffffff #max offset before wrap, default is MAXINT32 for TCP sequence numbers - - def __init__(self,ts,direction,startoffset): - self.starttime=ts - self.endtime=ts - self.direction=direction - self.segments={} # offset:[segments with offset] - self.startoffset=startoffset - self.endoffset=startoffset - self.info_keys = ['starttime','endtime','direction','startoffset','endoffset'] - - def update(self,ts,data,offset=None): - #if offsets are not being provided, just keep packets in wire order - if offset==None: offset=self.endoffset - #buffer each segment in a list, keyed by offset (captures retrans, etc) - self.segments.setdefault(offset,[]).append(data) - if ts>self.endtime: self.endtime=ts - #update the end offset if this packet goes at the end - if offset >= self.endoffset: self.endoffset=(offset+len(data))&self.MAX_OFFSET - - def __repr__(self): - return '%s %s (%s) +%s %d' % (self.starttime,self.endtime,self.direction,self.startoffset,len(self.segments)) - - def __str__(self): - '''returns segments of blob as string''' - return self.data(padding='') - - def data(self,errorHandler=None,padding=None,overlap=True,caller=None,dup=-1): - '''returns segments of blob reassembled into a string - if next segment offset is not the expected offset - errorHandler(blob,expected,offset) will be called - blob is a reference to the blob - if expectedoffset, data is overlapping - else a KeyError will be raised. - if the exception is passed and data is missing - if padding != None it will be used to fill the gap - if segment overlaps existing data - new data is kept if overlap=True - existing data is kept if overlap=False - caller: a ref to the calling object, passed to errorhandler - dup: how to handle duplicate segments: - 0: use first segment seen - -1 (default): use last segment seen - ''' - d='' - nextoffset=self.startoffset - for segoffset in sorted(self.segments.iterkeys()): - if segoffset!=nextoffset: - if errorHandler: #errorhandler can mangle blob data - if not errorHandler(blob=self, expected=nextoffset, offset=segoffset, caller=caller): continue #errorhandler determines pass or fail here - elif segoffset>nextoffset: - if padding is not None: #data missing and padding specified - if len(padding): d+=str(padding)*(segoffset-nextoffset) #add padding to data - else: raise KeyError(nextoffset) #data missing, and no padding - elif segoffset