diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 5c7ce2cab3..ee50b1aeb5 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -36,9 +36,10 @@ jobs: token: '${{ secrets.GITHUB_TOKEN }}' - name: Setup PHP - uses: shivammathur/setup-php@v2 + uses: shivammathur/setup-php@main with: - php-version: '8.1' + php-version: '8.4' + extensions: mysql, imagick - name: Install requirements run: | @@ -54,7 +55,7 @@ jobs: - name: Download and convert dumb hosts to plain data run: php -f "${{ github.workspace }}/scripts/converter.php" - - name: Update active list + - name: Update active source list run: bash "${{ github.workspace }}/scripts/update_active_lists.sh" - name: Commit changes diff --git a/CHANGELOG b/CHANGELOG index d01d8a6cdf..27a5413d68 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,8 +1,11 @@ -21 June 2020 +02. July 2024 + - Added Phishing.Database/ALL-phishing-links.txt + - Added and started using perl + +21. June 2020 - Updated the HpHost location - Adding .dtq zones - Updated the code for blocklist.site to use array 15. January 2020 - Replaces MobileAdTracker with adaway.github.io - diff --git a/scripts/converter.php b/scripts/converter.php index 092d07937a..c58bb5ed11 100644 --- a/scripts/converter.php +++ b/scripts/converter.php @@ -1,5 +1,7 @@ 'https://raw.githubusercontent.com/AdAway/adaway.github.io/master/hosts.txt', @@ -46,7 +48,7 @@ 'phishingArmy' => 'https://phishing.army/download/phishing_army_blocklist_extended.txt', 'Phishing.Database' => 'https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-domains-ACTIVE.txt', 'Phishing.DatabaseAll' => 'https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/ALL-phishing-domains.txt', - 'Phishing.DatabaseAllLinks' => 'https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/ALL-phishing-links.txt', + // 'Phishing.DatabaseAllLinks' => 'https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/ALL-phishing-links.txt', 'QuidsupMixed' => 'https://quidsup.net/notrack/blocklist.php?download=trackersdomains', 'ShadowWhispererAds' => 'https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Ads', 'ShadowWhispererApple' => 'https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Apple', diff --git a/scripts/dependencies.sh b/scripts/dependencies.sh index 6401f3a6e1..dfa90c44ff 100755 --- a/scripts/dependencies.sh +++ b/scripts/dependencies.sh @@ -3,7 +3,7 @@ # The perpose of this script is to import various eternal hosts files into lists # that contail only domain.tld for easier working with the lists to our RPZ files -# Exit on any erros +# Exit on any errors set -e @@ -27,4 +27,5 @@ bash -c "$(curl -sL https://raw.githubusercontent.com/ilikenwf/apt-fast/master/q apt-fast update -yqq #apt-fastdist-upgrade -yqq -apt-fast install -yqq openssh-client curl wget dos2unix ldnsutils +apt-fast install -yqq openssh-client curl wget dos2unix ldnsutils \ + libdomain-publicsuffix-perl diff --git a/scripts/import.sh b/scripts/import.sh index a5eb7bb57c..ea0e3910da 100755 --- a/scripts/import.sh +++ b/scripts/import.sh @@ -184,6 +184,15 @@ echo "Imported openfish.com" # START @mitchellkrogza's many lists # echo "START importing @mitchellkrogza's many lists" +# Perlscript as by https://unix.stackexchange.com/a/745455 + +mkdir -p "${git_dir}/data/phishing_database/" +c "https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/ALL-phishing-links.txt" >/tmp/ALL-phishing-links.txt +perl -MDomain::PublicSuffix -lne ' + BEGIN{$s = Domain::PublicSuffix->new} + print if $_ eq $s->get_root_domain($_)' "data/phishing_database/ALL-phishing-links.txt" + # mkdir -p "${git_dir}/data/mitchellkrogza/badd_boyz_hosts/" # echo "" # echo "Badd-Boyz-Hosts" @@ -270,6 +279,7 @@ echo "Imported openfish.com" echo "" echo "" echo "The script ${0}" +# shellcheck disable=SC2320 echo -e "Exited with error code ${?}\n\n" # git add . diff --git a/scripts/update_active_lists.sh b/scripts/update_active_lists.sh index 14c6f5d357..584f7ba2d7 100644 --- a/scripts/update_active_lists.sh +++ b/scripts/update_active_lists.sh @@ -10,7 +10,8 @@ truncate -s 0 "${git_dir}/sources.list" # shellcheck disable=SC2044 for lists in $(find data/ -type f -name domain.list); do - printf "$github.workspace/-/raw/master/$lists\n" | sort -u -f >>"${git_dir}/sources.list" + printf "$github.workspace/-/raw/master/$lists\n" | + sort -u -f >>"${git_dir}/sources.list" done echo -e "\n\nThe script ${0}\nExited with error code ${?}\n\n"