Merge pull request #444 from anarkiwi/bfaster

reduce size of ncapture container, use editcap to trim/validate pcap before tshark reads it.

Author: cglewis

.github/workflows/test.yml

@@ -20,8 +20,7 @@ jobs:
         find . -name requirements.txt -type f -exec pip3 install -r {} \; && \
         export PATH=/home/runner/.local/bin:$PATH && \
         make test && \
-        coverage report && coverage xml && \
-        cd network_tap/ncapture && sudo ./test_ncapture.sh
+        coverage report && coverage xml
     - name: Upload coverage to Codecov
       uses: codecov/[email protected]
       if: github.repository == 'iqtlabs/network-tools'

Makefile

@@ -1,4 +1,5 @@
 test:
-	PYTHONPATH=network_tools_lib python3 -m pytest -l -s -v --cov=. --cov-report term-missing
+	RESULT_PATH=/dev/null PYTHONPATH=network_tools_lib python3 -m pytest -l -s -v --cov=. --cov-report term-missing
 	# TODO: complete pytype coverage for pcap_stats
 	PYTHONPATH=network_tools_lib pytype -k --exclude=pcap_stats .
+	./network_tap/ncapture/test_ncapture.sh

network_tap/ncapture/Dockerfile

@@ -1,53 +1,40 @@
-FROM alpine:3.14
+FROM alpine:3.14 as checkout
 LABEL maintainer="Charlie Lewis <[email protected]>"
 
-ENV BUILDDEPS="autoconf automake bison build-base flex gcc git libtool libpcap-dev libpcap linux-headers musl-dev python3-dev yaml-dev"
-ENV PYTHONPATH=/tmp/network_tools_lib
-WORKDIR /tmp
-
-# TODO: libwdcap currently requires openssl 1.0.2
-RUN apk add --update $BUILDDEPS \
-    bash \
-    coreutils \
-    python3 \
-    py3-pip \
-    && rm -rf /var/cache/apk/* \
-    && mkdir /src \
-    && cd /src \
-    && git clone https://github.com/wanduow/wandio.git -b 4.2.3-1 \
+RUN apk add --update git
+WORKDIR /src
+RUN git clone https://github.com/wanduow/wandio.git -b 4.2.3-1 \
     && git clone https://github.com/LibtraceTeam/libtrace.git -b 4.0.16-1 \
     && git clone https://github.com/openssl/openssl -b OpenSSL_1_0_2s \
-    && git clone https://github.com/wanduow/libwdcap.git \
-    && cd /src/wandio \
-    && ./bootstrap.sh \
-    && ./configure \
-    && make && make install \
-    && cd /src/libtrace \
-    && ./bootstrap.sh \
-    && ./configure \
-    && make && make install \
-    && cd /src/openssl \
-    && ./config --prefix=/usr/local --openssldir=/usr/local/openssl \
-    && make && make install \
-    && cd /src/libwdcap \
-    && ./bootstrap.sh \
-    && ./configure --disable-shared \
-    && make && make install \
-    && cd examples \
-    && g++ -fpermissive -o tracecapd tracecapd.c -L/usr/local/lib -Wl,-Bstatic -ltrace -lwdcap -Wl,-Bdynamic -lpcap -lssl -lcrypto -lwandio -lyaml \
-    && cp tracecapd /usr/local/bin \
-    && rm -rf /src \
-    && apk del $BUILDDEPS \
-    && apk add \
-    libstdc++ \
-    libgcc \
-    libpcap \
-    yaml
+    && git clone https://github.com/wanduow/libwdcap.git
+
+FROM alpine:3.14 
+COPY --from=checkout /src /src
+WORKDIR /src
 
+# TODO: libwdcap currently requires openssl 1.0.2
+RUN apk add --update autoconf automake bison build-base flex gcc libtool libpcap-dev libpcap linux-headers musl-dev yaml-dev
+
+WORKDIR /src/openssl
+RUN ./config --prefix=/usr/local --openssldir=/usr/local/openssl && MAKEFLAGS=--quiet make -j "$(nproc)" && make install_sw
+WORKDIR /src/wandio
+RUN ./bootstrap.sh && ./configure && make && make install 
+WORKDIR /src/libtrace
+RUN ./bootstrap.sh && ./configure && make && make install
+WORKDIR /src/libwdcap
+RUN ./bootstrap.sh && ./configure --disable-shared && make && make install
+WORKDIR /src/libwdcap/examples 
+RUN g++ -fpermissive -o tracecapd tracecapd.c -L/usr/local/lib -Wl,-Bstatic -ltrace -lwdcap -Wl,-Bdynamic -lpcap -lssl -lcrypto -lwandio -lyaml && cp tracecapd /usr/local/bin
+
+WORKDIR /tmp
 VOLUME /tmp
+
 COPY network_tap/ncapture/ /tmp
 COPY network_tools_lib /tmp/network_tools_lib
 
+RUN apk add --update bash coreutils python3 py3-pip
 RUN pip3 install --no-cache-dir -r requirements.txt
+RUN ldd /usr/local/bin/tracecapd
 
+ENV PYTHONPATH=/tmp/network_tools_lib
 CMD ["/tmp/run.sh"]

network_tap/ncapture/test_ncapture.sh

@@ -1,23 +1,24 @@
 #!/bin/bash
 
 # smoke test for ncapture worker
-# requires tcpdump and tshark to be installed.
 
 URI=lo
 IP=127.0.0.1
 SIZE=1000
 MAXCAPLEN=50
 
-TMPDIR=$(mktemp -d)
+sudo apt-get update && sudo apt-get install tcpdump tshark
+docker build -f network_tap/ncapture/Dockerfile . -t iqtlabs/ncapture || exit 1
 
-docker build -f Dockerfile . -t iqtlabs/ncapture
+TMPDIR=$(mktemp -d)
 echo starting ncapture
 docker run --privileged --net=host --cap-add=NET_ADMIN -v $TMPDIR:/files -t iqtlabs/ncapture /tmp/run.sh $URI 15 test 1 "host $IP and icmp" "" -d 12 -s 4 -a none -c none -o /files/ || exit 1 &
 echo waiting for pcap
 PINGS=0
 while [ "$(find $TMPDIR -prune -empty)" ] ; do
   ((++PINGS))
-  ping -q -n -i 0.1 -s $SIZE -c 10 $IP > /dev/null
+  # need sudo for low interval
+  sudo ping -q -n -i 0.1 -s $SIZE -c 10 $IP > /dev/null
   echo -n .$PINGS
   if [ "$PINGS" -gt "60" ] ; then
 	  echo timed out waiting for pcap

p0f/app.py

@@ -42,9 +42,11 @@ def parse_eth(packet):
 
 def run_tshark(path):
     addresses = set()
-    with pyshark.FileCapture(path, include_raw=False, keep_packets=False, debug=True,
-                             custom_parameters=['-o', 'tcp.desegment_tcp_streams:false',
-                                                '-n', '-j', 'eth ip ipv6']) as cap:  # disable DNS, eth/IP only.
+    with pyshark.FileCapture(
+            path, include_raw=False, keep_packets=False, debug=True,
+            custom_parameters=[
+                '-o', 'tcp.desegment_tcp_streams:false', '-n'],  # disable DNS
+            tshark_path=os.path.join(os.path.dirname(__file__), 'tsharkwrapper.sh')) as cap:
         for packet in cap:
             src_eth_address, dst_eth_address = parse_eth(packet)
             src_address, dst_address = parse_ip(packet)

p0f/tsharkwrapper.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# tshark does not respect -s when reading from a capture, so provide
+# a wrapper to cut down the capture to IP/header only, which also
+# protects tshark's parser.
+# bash-ism to retrieve last item in arg list (called by pyshark, is the pcap name)
+pcap="${@: -1}"
+if [[ ! -f "$pcap" ]] ; then
+	echo last arg must exist and be input pcap.
+	exit 1
+fi
+# bash-ism to drop last arg (pcap name)
+set -- "${@:1:$#-1}"
+# pass remaining pyshark args to tshark, which will end with "-r -".
+editcap -F pcap $pcap -s 128 - | tshark $* \-