Merge pull request #230 from stratosphereips/develop

Slips v1.0.2
stratosphereips · Feb 28, 2023 · 5afbea4 · 5afbea4
2 parents 287d060 + ce80855
commit 5afbea4
Show file tree

Hide file tree

Showing 54 changed files with 1,779 additions and 917 deletions.
diff --git a/.github/workflows/CI-production-testing.yml b/.github/workflows/CI-production-testing.yml
@@ -15,7 +15,7 @@ jobs:
     timeout-minutes: 7200
 
     env:
-      SLIPS_VERSION: v1.0.1
+      SLIPS_VERSION: v1.0.2
 
     steps:
     - uses: actions/checkout@v3
@@ -74,7 +74,7 @@ jobs:
     # 2 hours timeout
     timeout-minutes: 7200
     env:
-      SLIPS_VERSION: v1.0.1
+      SLIPS_VERSION: v1.0.2
 
     steps:
 
@@ -94,6 +94,7 @@ jobs:
         tags: stratosphereips/slips:latest, stratosphereips/slips:${{ env.SLIPS_VERSION }}
         push: false
 
+
     # use the built image to run unit tests inside docker
     # we ignore the blocking module unit tests because we cannot run
     # docker with -cap-add=NET_ADMIN and use iptables in GH actions
@@ -102,11 +103,15 @@ jobs:
       with:
         image: stratosphereips/slips:latest
         run: |
+          git reset --hard
           git pull & git checkout origin/develop
+          python3 -m pip install -r requirements.txt 
           redis-server --daemonize yes
+          git show-branch --no-name HEAD
           python3  -m pytest tests/ --ignore="tests/test_daemon.py" --ignore="tests/test_database.py" --ignore="tests/integration_tests" -n 7 -p no:warnings -vv -s
           python3  -m pytest tests/test_database.py -p no:warnings -vv
           python3  -m pytest tests/test_daemon.py -p no:warnings -vv
+          printf "0" | ./slips.py -k
           python3 -m pytest -s tests/integration_tests/test_dataset.py -n 4 -p no:warnings -vv
           python3 -m pytest -s tests/integration_tests/test_config_files.py -n 2 -p no:warnings -vv
 
diff --git a/.github/workflows/CI-publishing.yml b/.github/workflows/CI-publishing.yml
@@ -7,14 +7,24 @@ on:
     - '!develop'
 
 jobs:
+  # auto add release tag
+  create_tag:
+    runs-on: ubuntu-latest
+    env:
+      SLIPS_VERSION: v1.0.2
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: rickstaa/action-create-tag@v1
+        with:
+          tag: ${{ env.SLIPS_VERSION }}
+          message: ""
 
   publish_ubuntu_image:
     # runs the tests in a docker(built by this job) on stop of a GH VM
     runs-on: ubuntu-20.04
-    # 2 hours timeout
-    timeout-minutes: 7200
     env:
-      SLIPS_VERSION: v1.0.1
+      SLIPS_VERSION: v1.0.2
 
     steps:
 
@@ -49,7 +59,7 @@ jobs:
     # 2 hours timeout
     timeout-minutes: 7200
     env:
-      SLIPS_VERSION: v1.0.1
+      SLIPS_VERSION: v1.0.2
     steps:
     # clone slips and checkout branch
     # By default it checks out only one commit
@@ -114,7 +124,6 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-
     # clone slips and checkout branch
     - uses: actions/checkout@v3
       with:
@@ -135,3 +144,21 @@ jobs:
         tags: stratosphereips/slips_dependencies:latest
         push: true
 
+
+  tweet_about_the_new_release:
+    runs-on: ubuntu-latest
+    env:
+      SLIPS_VERSION: v1.0.2
+      # same as the above without the v and the dots
+      SLIPS_URL_VERSION: 101
+
+    steps:
+      - name: Tweet-trigger-publish-release
+        uses: mugi111/[email protected]
+        with:
+          consumer_key: ${{ secrets.CONSUMER_KEY }}
+          consumer_secret: ${{ secrets.CONSUMER_SECRET }}
+          access_token_key: ${{ secrets.ACCESS_KEY }}
+          access_token_secret: ${{ secrets.ACCESS_SECRET }}
+          tweet_body: 'Today we published Slips ${{ env.SLIPS_VERSION }}! https://github.com/stratosphereips/StratosphereLinuxIPS Check out the new features: www.stratosphereips.org/blog/2022/8/30/new-slips-version-${{ env.SLIPS_URL_VERSION }}-is-here '
+
diff --git a/.github/workflows/CI-staging.yml b/.github/workflows/CI-staging.yml
@@ -5,7 +5,8 @@ name: CI-staging
 on:
     push:
         branches:
-            - 'develop'
+            # features will be added to this branch using PRs, not need to re-run the tests on push
+            - '!develop'
             - '!master'
     pull_request:
         branches:
@@ -14,7 +15,7 @@ on:
 
 jobs:
 
-    run_unit_tests:
+    run_tests:
         # specify the host OS
         runs-on: ubuntu-latest
         # 2 hours timeout

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,27 @@
--1.0.1 (Jan 2023)
+-1.0.2 (Feb 2023)
+- Support ASNs in our own_malicious_iocs.csv file
+- Add a zeek script to recognize the gateway IP and add it to notice.log 
+- Don't alert "Connection to Private IP" when there's a DNS connection on port 53 UDP to the gateway
+- Faster reading of netflow and suricata files
+- Add a progress bar to slips showing the number of processed flows
+- Fix having duplicate alerts
+- Fix vertical and horizontal portscan errors
+- Add the uids that caused an evidence to the evidence description in alerts.json
+- Add a blocking indicator in alerts.json
+- Fix wrong Source/Target type in alerts.json
+- Fix error parsing AIP TI list.
+- Update slips default whitelist
+- Kill web interface on ctrl+c
+- Use the current user's timezone in alerts.log and alets.json
+- Fix problem displaying data from the db in the web interface 
+- Add the option to view blocked profiles only in the web interface
+- Fix displaying alerts of profile in the webinterface
+- Add the option to display all evidence in a profile
+- Fix searching in the web interface
+- Fix caching ASN ranges
+- Code optimizations
+
+- 1.0.1 (Jan 2023)
 - fix FP horizontal portscans caused by zeek flipping connections
 - Fix Duplicate evidence in multiple alerts
 - Fix FP urlhaus detetcions, now we use it to check urls only, not domains.

diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 <h1 align="center"> 
 
-Slips v1.0.1
+Slips v1.0.2
 
 
 <h3 align="center"> 

diff --git a/config/local_ti_files/own_malicious_iocs.csv b/config/local_ti_files/own_malicious_iocs.csv
@@ -1,6 +1,6 @@
 # This file is for you to add your own IPs or domains to the list to detect
 # Format of this file: 
-# IP/domain/ip range, Threat level, Description, Tag
+# IP/domain/ip range/ASN, Threat level, Description, Tag
 # Threat level available options: info, low, medium, high, critical
 23.253.126.58,high,Simda CC
 bncv00.no-ip.info, critical, Variant.Zusy
@@ -11,4 +11,8 @@ fullplayersoftware.com, high, Rana Malware IRAN
 softwareplayertop.com, high, Rana Malware IRAN
 saveingone.com, high, Rana Malware IRAN
 facedomainpc.com, high, Rana Malware IRAN
-facedomaintv.com, high, Rana Malware IRAN
+facedomaintv.com, high, Rana Malware IRAN
+AS39770, high, ELITETEAM bulletproof hosting provider
+AS60424, high, ELITETEAM bulletproof hosting provider
+AS56873, high, ELITETEAM bulletproof hosting provider
+AS51381, high, ELITETEAM bulletproof hosting provider
diff --git a/config/slips.conf b/config/slips.conf
@@ -1,6 +1,6 @@
 # This configuration file controls several aspects of the working of Slips
 [version]
-slips_version = '1.0.1'
+slips_version = '1.0.2'
 
 #####################
 # [0] Slips modes
@@ -363,7 +363,8 @@ receive_delay = 86400
 # SuspiciousUserAgent, multiple_google_connections, NETWORK_gps_location_leaked,
 # Command-and-Control-channels-detection, InvalidCertificate
 # ThreatIntelligenceBlacklistDomain, ThreatIntelligenceBlacklistIP,
-# MaliciousDownloadedFile, DGA, MaliciousSSLCert, YoungDomain, MultipleSSHVersions
+# ThreatIntelligenceBlacklistedASN, MaliciousDownloadedFile,
+ # DGA, MaliciousSSLCert, YoungDomain, MultipleSSHVersions
 # DNS-ARPA-Scan, SMTPLoginBruteforce, BadSMTPLogin,
 # IncompatibleUserAgent, ICMP-Timestamp-Scan, ICMP-AddressScan, ICMP-AddressMaskScan
 # EmptyConnections, IncompatibleCN, PastebinDownload, DOSExecutableDownload

diff --git a/config/whitelist.conf b/config/whitelist.conf
@@ -79,6 +79,7 @@ domain,spamhaus.org,both,alerts
 domain,spamhaus.org.cdn.cloudflare.net,both,alerts
 ip,3.126.246.180,both,alerts
 domain,circl.lu,both,alerts
+ip,185.194.93.128,both,alerts
 ip,185.194.93.14,both,alerts
 domain,check.torproject.org,both,alerts
 domain,cert.pl,both,alerts
@@ -92,7 +93,26 @@ domain,rdap.pacloudflare.com,both,alerts
 domain,whois.arin.net,both,alerts
 domain,whois.pir.org,both,alerts
 domain,macvendors.com,both,alerts
+ip,159.65.226.142,both,alerts
 domain,rdap.apnic.net.cdn.cloudflare.net,both,alerts
 domain,whois.name.com,both,alerts
 domain,ripe.net,both,alerts
-domain,whois.us-west-2.pirproda.registry-upper.dnts.net,both,alerts
+domain,whois.us-west-2.pirproda.registry-upper.dnts.net,both,alerts
+ip,65.9.95.98,both,alerts
+domain,reputation.alienvault.com,both,alerts
+ip,185.21.103.31,both,alerts
+domain,lists.blocklist.de,both,alerts
+ip,80.211.178.212,both,alerts
+domain,osint.digitalside.it,both,alerts
+ip,54.174.199.242,both,alerts
+domain,rules.emergingthreats.net,both,alerts
+ip,195.113.172.59,both,alerts
+domain,repsh.liberouter.org,both,alerts
+domain,nerd.cesnet.cz,both,alerts
+ip,185.36.6.9,both,alerts
+domain,whois.eu,both,alerts
+ip,3.227.57.25,both,alerts
+domain,whois.registrar.amazon.com,both,alerts
+ip,192.12.192.242,both,alerts
+domain,whois.nic.it,both,alerts
+domain,tranco-list.eu,both,alerts
diff --git a/docker/macosm1-P2P-image/requirements-macos-m1-docker.txt b/docker/macosm1-P2P-image/requirements-macos-m1-docker.txt
@@ -21,16 +21,19 @@ psutil==5.8.0
 yara_python==4.1.3
 six==1.15.0
 pytest==6.2.5
+pytest-mock
 pytest-xdist==2.4.0
 slackclient==2.9.3
 scipy==1.7.3
 sklearn==0.0
 GitPython==3.1.14
 protobuf==3.20
 blinker
+exclusiveprocess
 setuptools
 pytest-dependency
 whois
 wheel
 flask
-tld
+tld
+tqdm
diff --git a/docker/macosm1-image/requirements-macos-m1-docker.txt b/docker/macosm1-image/requirements-macos-m1-docker.txt
@@ -36,3 +36,4 @@ whois
 wheel
 flask
 tld
+tqdm
diff --git a/docs/FAQ.md b/docs/FAQ.md
@@ -6,6 +6,15 @@ Make sure you're not running many containers at the same time because they share
 even though they're isolated.
 
 
+## Getting "Illegal instruction" error when running slips
+
+If the tensorflow version you're using isn't compatible with your architecture, 
+you will get the "Illegal instruction" error and slips will terminate.
+
+To fix this you can disable the modules that use tensorflow by adding
+```rnn-cc-detection, flowmldetection``` to the ```disable``` key in ```config/slips.conf```
+
+
 ## Docker time is not in syncs with that of the host
 
 You can add your local /etc/localtime as volume in Slips Docker container by using:

diff --git a/docs/conf.py b/docs/conf.py
@@ -24,7 +24,7 @@
 author = 'Stratosphere Laboratory'
 
 # The full version, including alpha/beta/rc tags
-release = '1.0.1'
+release = '1.0.2'
 
 
 # -- General configuration ---------------------------------------------------

diff --git a/docs/contributing.md b/docs/contributing.md
@@ -30,16 +30,19 @@ to follow when contributing to Slips:
 
 ## What branch should you base your contribution to Slips?
 
-As a general rule, base your contributions on the `develop` branch.
+As a general rule, base your contributions to the `develop` branch.
 
 ## Creating a pull request
 
 Commits:
 - Commits should follow the KISS principle: do one thing, and do it well (keep it simple, stupid).
 - Commit messages should be easily readable, imperative style ("Fix memory leak in...", not "FixES mem...")
+
 Pull Requests:
 - If you have developed multiple features and/or bugfixes, create separate
     branches for each one of them, and request merges for each branch;
+- Each PR to develop will trigger the develop Github checks, these checks will run Slips unit tests and integration tests locally in a ubuntu VM and in docker to make sure the branch is ready to merge.
+- PRs won't be merged unless the checks pass.
 - The cleaner you code/change/changeset is, the faster it will be merged.
 
 ***

diff --git a/docs/detection_modules.md b/docs/detection_modules.md
@@ -277,13 +277,21 @@ Slips is shipped with the Abuse.ch SSL feed by default,
 
 You can add your own SSL feed by appending to the ```ssl_feeds``` key in ```config/slips.conf```
 
+### Matching of ASNs
+
+Every time Slips sees a new IP, it stores info about it in the db, for example its organization, RDNs, and ASN.
+If the ASN of an IP matches a blacklisted ASN, slips alerts.
+
+Blacklisted ASNs are read from out local TI file ```config/local_data_files/own_malicious_iocs.csv```, 
+so you can update them or add your own.
+
 ### Local Threat Intelligence files
 
 Slips has a local file for adding IoCs of your own, 
 it's located in ```config/local_data_files/own_malicious_iocs.csv``` by default,
 this path can be changed by changing ```download_path_for_local_threat_intelligence``` in ```config/slips.conf```.
 
-The format of the file is "IP address/IP Range/domain","Threat level", "Description"
+The format of the file is "IP address/IP Range/domain/ASN","Threat level", "Description"
 
 Threat level available options: info, low, medium, high, critical
 
@@ -690,3 +698,9 @@ We detect a scan every threshold. So we generate an evidence when there is
 5,10,15, .. etc. ICMP established connections to different IPs.
 
 
+### Detect the Gateway address
+
+The ```zeek-scripts/log_gw.zeek``` script is responsible for recognizing the gateway address using zeek, and logging it to
+notice.log
+
+
diff --git a/docs/features.md b/docs/features.md
@@ -88,6 +88,8 @@ Slips alerts when 3+ invalid SMTP login attempts occurs within 10s
 
 Slips detects when a private IP is connected to another private IP with threat level info.
 
+But it skips this alert when it's a DNS connection on port 53 UDP to the gateway
+
 ### Connection to private IPs outside the current local network
 
 Slips detects the currently used local network and alerts if it find a

diff --git a/docs/flowalerts.md b/docs/flowalerts.md
@@ -306,6 +306,8 @@ in conn.log isn't set to 'http', it alerts
 
 Slips detects when a private IP is connected to another private IP with threat level info.
 
+But it skips this alert when it's a DNS connection on port 53 UDP to the gateway
+
 ## Connection to private IPs outside the current local network
 
 Slips detects the currently used local network and alerts if it find a

diff --git a/docs/images/slips.gif b/docs/images/slips.gif
diff --git a/docs/index.rst b/docs/index.rst
@@ -1,7 +1,7 @@
 .. image:: /images/slips_logo.png
     :align: center
 
-Slips v1.0.1
+Slips v1.0.2
 ============================
 
 The tool is available on GitHub `here <https://github.com/stratosphereips/StratosphereLinuxIPS/tree/master>`_.
-Original file line number
+Diff line change
@@ Expand Up / @@ -36,3 +36,4 @@ whois @@
     wheel
     flask
     tld
+    tqdm