From daa48b1cede54e15f1db0f5e347050683a32ae05 Mon Sep 17 00:00:00 2001 From: robocopAlpha Date: Thu, 15 Jul 2021 11:39:45 +0300 Subject: [PATCH] adapting to work with both mac and linux --- block_FB.sh | 16 ++++++------ get_blocklists.sh | 29 +++++++--------------- pull.lists.sh | 62 ++++++++++++++++++++--------------------------- set_env.sh | 59 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+), 64 deletions(-) create mode 100644 set_env.sh diff --git a/block_FB.sh b/block_FB.sh index 59029ed..2bbaa17 100755 --- a/block_FB.sh +++ b/block_FB.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash # Facebook & Whatsapp -WRK_DIR=/home/dietpi/pihole_lists -cd ${WRK_DIR}/Mirrors +[[ -f set_env.sh ]] && source set_env.sh || { echo "set_env.sh not found"; exit 1; } -rg -FINv "whatsapp" ${WRK_DIR}/Mirrors/facebook_block.txt >| ${WRK_DIR}/Mirrors/tmp.txt -rg -FIN "whatsapp" ${WRK_DIR}/Mirrors/facebook_block.txt | rg -FIN "facebook" >> ${WRK_DIR}/Mirrors/tmp.txt -rm -f ${WRK_DIR}/Mirrors/facebook_block.txt -sort -u ${WRK_DIR}/Mirrors/tmp.txt | uniq > ${WRK_DIR}/facebook_block.txt -rm -f ${WRK_DIR}/Mirrors/tmp.txt -echo -e " \n $(wc -l ${WRK_DIR}/facebook_block.txt) domains\n" +rg -FINv "whatsapp" "${WRK_DIR}/Mirrors/facebook_block.txt" >| "${WRK_DIR}/Mirrors/tmp.txt" +rg -FIN "whatsapp" "${WRK_DIR}/Mirrors/facebook_block.txt" | rg -FIN "facebook" >> "${WRK_DIR}/Mirrors/tmp.txt" +rm -f "${WRK_DIR}/Mirrors/facebook_block.txt" +sort -u "${WRK_DIR}/Mirrors/tmp.txt" | uniq > "${WRK_DIR}/facebook_block.txt" +rm -f "${WRK_DIR}/Mirrors/tmp.txt" +Line_Count=$(wc -l "${WRK_DIR}/facebook_block.txt") +echo -e "${Line_Count} domains" diff --git a/get_blocklists.sh b/get_blocklists.sh index 7493fb3..5a58e7d 100755 --- a/get_blocklists.sh +++ b/get_blocklists.sh @@ -1,22 +1,11 @@ #!/usr/bin/env bash +[[ -f set_env.sh ]] && source set_env.sh || { echo "set_env.sh not found"; exit 1; } + # Going to the right path -WRK_DIR=/home/dietpi/pihole_lists/Mirrors -mkdir ${WRK_DIR} -cd ${WRK_DIR} - -fetch_hosts(){ - URL=$1 - FNAME=$2 - - curl -sSL "${URL}" | rg -INv "^[#]|localhost" | sd '\s+$' '' | sd "^0\\.0\\.0\\.0 |^127\\.0\\.0\\.1 " "" | sd "\\#.*" "" >| ${FNAME} -} - -fetch_blocklist(){ - URL=$1 - FNAME=$2 - curl -sSL "${URL}" | rg -INv "^[#]|localhost" | sd '\s+$' '' | sd "\\#.*" "" >| ${FNAME} -} +WRK_DIR="${WRK_DIR}/Mirrors" +mkdir "${WRK_DIR}" +cd "${WRK_DIR}" if ping -c 1 -W 2 www.google.fi &> /dev/null ; then echo " Fetching lists. Please wait..." @@ -69,10 +58,10 @@ if ping -c 1 -W 2 www.google.fi &> /dev/null ; then # Steven Black's list needs some extra clean up curl -sSL 'https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts' >| 'tmp.txt' - rg --after-context $(wc -l ${WRK_DIR}/tmp.txt | awk {'print $1'}) "^# Start StevenBlack" ${WRK_DIR}/tmp.txt >| ${WRK_DIR}/tmp2.txt - rm -f ${WRK_DIR}/tmp.txt - rg -FINv "#" ${WRK_DIR}/tmp2.txt | sort -u | uniq >| ${WRK_DIR}/StevenBlack.txt - rm -f ${WRK_DIR}/tmp2.txt + rg --after-context $(wc -l tmp.txt | awk {'print $1'}) "^# Start StevenBlack" tmp.txt >| tmp2.txt + rm -f tmp.txt + rg -FINv "#" tmp2.txt | sort -u | uniq >| StevenBlack.txt + rm -f tmp2.txt #-------- echo " #Lines FileName" wc -l *.txt diff --git a/pull.lists.sh b/pull.lists.sh index 532453f..3225983 100755 --- a/pull.lists.sh +++ b/pull.lists.sh @@ -1,58 +1,44 @@ #!/usr/bin/env bash +[[ -f set_env.sh ]] && source set_env.sh || { echo "set_env.sh not found"; exit 1; } -WRK_DIR=/home/dietpi/pihole_lists -cd ${WRK_DIR}/ - -rm -rf /home/dietpi/onedrive -mkdir /home/dietpi/onedrive - -rclone mount --daemon onedrive:Public/blocklist/ /home/dietpi/onedrive +cd "${WRK_DIR}/" # Fetch blocklists -bash ${WRK_DIR}/get_blocklists.sh +bash "${WRK_DIR}/get_blocklists.sh" if [ $? -ne 0 ]; then echo "Failed to get blocklists. Execution terminated!" - exit + exit 1 fi # Block FB -source ${WRK_DIR}/block_FB.sh +source "${WRK_DIR}/block_FB.sh" || { echo "Failed to get Facebook blocklist. Execution terminated!"; exit 1; } -#--- Pick fastest cat command (based on my experience) -if command -v gcat > /dev/null 2>&1; then - catCMD='gcat' # GNU cat /usr/local/bin/gcat - # (To get run: brewlog insall coreutils) -elif alias mcat > /dev/null 2>&1; then - catCMD='mcat' # mcat is aliased to /bin/cat that ships with macOS -else - catCMD=$(command -v cat) # If none of the above are found then vanilla cat -fi -#----------- # Combining blocklists -${catCMD} ${WRK_DIR}/Mirrors/* | sort -u | uniq >| ${WRK_DIR}/MyBlocklist.txt -wc -l ${WRK_DIR}/MyBlocklist.txt +echo "Consolidating blocklists..." +${catCMD} Mirrors/* | sort -u | uniq >| MyBlocklist.txt +wc -l MyBlocklist.txt echo -e "\nAdding these to my own Blacklist: \n" -${catCMD} ${WRK_DIR}/blocked | rg -vIN "^[\\#]" | sd '\s+$' '' | sort -u | uniq >> ${WRK_DIR}/MyBlocklist.txt -echo -e " $(wc -l ${WRK_DIR}/MyBlocklist.txt) unique domains. \n" +${catCMD} blocked | rg -vIN "^[\\#]" | sd '\s+$' '' | sort -u | uniq >> MyBlocklist.txt +echo -e " $(wc -l MyBlocklist.txt) unique domains. \n" #------- # Clean-up echo "Cleaning up..." -rm -rf ${WRK_DIR}/Mirrors -rm -f ${WRK_DIR}/blocklists/MyBlocklist.txt.xz ${WRK_DIR}/blocklists/facebook_block.txt.xz -mkdir ${WRK_DIR}/blocklists -mv ${WRK_DIR}/MyBlocklist.txt ${WRK_DIR}/blocklists/. -mv ${WRK_DIR}/facebook_block.txt ${WRK_DIR}/blocklists/. +rm -rf Mirrors +rm -f blocklists/MyBlocklist.txt.xz blocklists/facebook_block.txt.xz +mkdir blocklists +mv MyBlocklist.txt blocklists/. +mv facebook_block.txt blocklists/. #-------- # Archiving echo "Compressing blocklists..." -cd ${WRK_DIR}/blocklists/ +cd "${WRK_DIR}/blocklists/" echo $(date +%d.%m.%y-%H:%M:%S) >| lastpull -md5sum ${WRK_DIR}/blocklists/MyBlocklist.txt | cut -c -32 >| MyBlocklist.checksum -md5sum ${WRK_DIR}/blocklists/facebook_block.txt | cut -c -32 >| facebook_block.checksum +getMD5 "MyBlocklist.txt" >| MyBlocklist.checksum +getMD5 "facebook_block.txt" >| facebook_block.checksum sleep 1 xz -v -T4 MyBlocklist.txt xz -v -T4 facebook_block.txt @@ -61,8 +47,12 @@ xz -v -T4 facebook_block.txt #--------- # Moving to target echo "Moving blocklists to destination..." -cd .. -mv -f ${WRK_DIR}/blocklists/* /home/dietpi/onedrive/. -rmdir ${WRK_DIR}/blocklists -sudo umount /home/dietpi/onedrive +cd "${WRK_DIR}/" +rsync -a --progress blocklists/* "${HOME_DIR}/opendrive/" + +echo "Final clean up..." +umount "${HOME_DIR}/opendrive" || sudo umount "${HOME_DIR}/opendrive" +rm -rf "${HOME_DIR}/opendrive" +rm -rf "blocklists" + echo "Done!" \ No newline at end of file diff --git a/set_env.sh b/set_env.sh new file mode 100644 index 0000000..7b7008c --- /dev/null +++ b/set_env.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +# Functions +fetch_hosts(){ + URL=$1 + FNAME=$2 + + curl -sSL "${URL}" | rg -INv "^[#]|localhost" | sd '\s+$' '' | sd "^0\\.0\\.0\\.0 |^127\\.0\\.0\\.1 " "" | sd "\\#.*" "" >| ${FNAME} +} + +fetch_blocklist(){ + URL=$1 + FNAME=$2 + curl -sSL "${URL}" | rg -INv "^[#]|localhost" | sd '\s+$' '' | sd "\\#.*" "" >| ${FNAME} +} + + +#--- Pick fastest cat command (based on my experience) +if command -v gcat > /dev/null 2>&1; then + catCMD='gcat' # GNU cat /usr/local/bin/gcat + # (To get run: brewlog insall coreutils) +elif alias mcat > /dev/null 2>&1; then + catCMD='mcat' # mcat is aliased to /bin/cat that ships with macOS +else + catCMD=$(command -v cat) # If none of the above are found then vanilla cat +fi + + +#--- Set parameters based on Distribution +if [ "$(uname)" == "Darwin" ]; then + # echo "Setting params for Darwin..." + getMD5(){ + # usage: getMD5 'filename' > outfile + echo "$(md5 -q ${1})" + } + HOME_DIR=/Users/deepankar/OneDrive\ -\ O365\ Turun\ yliopisto/Git/Gitlab.DC +elif [ "$(uname)" == "Linux" ]; then + # echo "Setting params for Linux..." + getMD5() + { + # usage: getMD5 'filename' > outfile + echo "$(md5sum ${1} | cut -c -32)" + } + HOME_DIR=/home/dietpi +else + echo "Only MacOS and Linux are supported." + exit 1 +fi + +WRK_DIR=${HOME_DIR}/pihole_lists + +if [[ -d "${HOME_DIR}/opendrive" ]]; then + rm -rf "${HOME_DIR}/opendrive" + mkdir "${HOME_DIR}/opendrive" +else + mkdir "${HOME_DIR}/opendrive" +fi + +rclone mount --daemon opendrive:Public/blocklist/ "${HOME_DIR}/opendrive"