Buck9195/pihole_lists
1
0
Fork 0
mirror of https://gitlab.com/robocopAlpha/pihole_lists.git synced 2026-05-17 18:42:16 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

adapting to work with both mac and linux

Browse source
This commit is contained in:
robocopAlpha 2021-07-15 11:39:45 +03:00
parent 2baf1d1de7
commit daa48b1ced
4 changed files with 102 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

16
block_FB.sh
View file

@ -1,13 +1,13 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# Facebook & Whatsapp # Facebook & Whatsapp
WRK_DIR=/home/dietpi/pihole_lists [[ -f set_env.sh ]] && source set_env.sh || { echo "set_env.sh not found"; exit 1; }
cd ${WRK_DIR}/Mirrors
rg -FINv "whatsapp" ${WRK_DIR}/Mirrors/facebook_block.txt >| ${WRK_DIR}/Mirrors/tmp.txt rg -FINv "whatsapp" "${WRK_DIR}/Mirrors/facebook_block.txt" >| "${WRK_DIR}/Mirrors/tmp.txt"
rg -FIN "whatsapp" ${WRK_DIR}/Mirrors/facebook_block.txt | rg -FIN "facebook" >> ${WRK_DIR}/Mirrors/tmp.txt rg -FIN "whatsapp" "${WRK_DIR}/Mirrors/facebook_block.txt" | rg -FIN "facebook" >> "${WRK_DIR}/Mirrors/tmp.txt"
rm -f ${WRK_DIR}/Mirrors/facebook_block.txt rm -f "${WRK_DIR}/Mirrors/facebook_block.txt"
sort -u ${WRK_DIR}/Mirrors/tmp.txt | uniq > ${WRK_DIR}/facebook_block.txt sort -u "${WRK_DIR}/Mirrors/tmp.txt" | uniq > "${WRK_DIR}/facebook_block.txt"
rm -f ${WRK_DIR}/Mirrors/tmp.txt rm -f "${WRK_DIR}/Mirrors/tmp.txt"
echo -e " \n $(wc -l ${WRK_DIR}/facebook_block.txt) domains\n" Line_Count=$(wc -l "${WRK_DIR}/facebook_block.txt")
echo -e "${Line_Count} domains"

29
get_blocklists.sh
View file

@ -1,22 +1,11 @@
#!/usr/bin/env bash #!/usr/bin/env bash
[[ -f set_env.sh ]] && source set_env.sh || { echo "set_env.sh not found"; exit 1; }
# Going to the right path # Going to the right path
WRK_DIR=/home/dietpi/pihole_lists/Mirrors WRK_DIR="${WRK_DIR}/Mirrors"
mkdir ${WRK_DIR} mkdir "${WRK_DIR}"
cd ${WRK_DIR} cd "${WRK_DIR}"
fetch_hosts(){
URL=$1
FNAME=$2
curl -sSL "${URL}" | rg -INv "^[#]|localhost" | sd '\s+$' '' | sd "^0\\.0\\.0\\.0 |^127\\.0\\.0\\.1 " "" | sd "\\#.*" "" >| ${FNAME}
}
fetch_blocklist(){
URL=$1
FNAME=$2
curl -sSL "${URL}" | rg -INv "^[#]|localhost" | sd '\s+$' '' | sd "\\#.*" "" >| ${FNAME}
}
if ping -c 1 -W 2 www.google.fi &> /dev/null ; then if ping -c 1 -W 2 www.google.fi &> /dev/null ; then
echo " Fetching lists. Please wait..." echo " Fetching lists. Please wait..."
@ -69,10 +58,10 @@ if ping -c 1 -W 2 www.google.fi &> /dev/null ; then
# Steven Black's list needs some extra clean up # Steven Black's list needs some extra clean up
curl -sSL 'https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts' >| 'tmp.txt' curl -sSL 'https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts' >| 'tmp.txt'
rg --after-context $(wc -l ${WRK_DIR}/tmp.txt | awk {'print $1'}) "^# Start StevenBlack" ${WRK_DIR}/tmp.txt >| ${WRK_DIR}/tmp2.txt rg --after-context $(wc -l tmp.txt | awk {'print $1'}) "^# Start StevenBlack" tmp.txt >| tmp2.txt
rm -f ${WRK_DIR}/tmp.txt rm -f tmp.txt
rg -FINv "#" ${WRK_DIR}/tmp2.txt | sort -u | uniq >| ${WRK_DIR}/StevenBlack.txt rg -FINv "#" tmp2.txt | sort -u | uniq >| StevenBlack.txt
rm -f ${WRK_DIR}/tmp2.txt rm -f tmp2.txt
#-------- #--------
echo " #Lines FileName" echo " #Lines FileName"
wc -l *.txt wc -l *.txt

62
pull.lists.sh
View file

@ -1,58 +1,44 @@
#!/usr/bin/env bash #!/usr/bin/env bash
[[ -f set_env.sh ]] && source set_env.sh || { echo "set_env.sh not found"; exit 1; }
WRK_DIR=/home/dietpi/pihole_lists cd "${WRK_DIR}/"
cd ${WRK_DIR}/
rm -rf /home/dietpi/onedrive
mkdir /home/dietpi/onedrive
rclone mount --daemon onedrive:Public/blocklist/ /home/dietpi/onedrive
# Fetch blocklists # Fetch blocklists
bash ${WRK_DIR}/get_blocklists.sh bash "${WRK_DIR}/get_blocklists.sh"
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "Failed to get blocklists. Execution terminated!" echo "Failed to get blocklists. Execution terminated!"
exit exit 1
fi fi
# Block FB # Block FB
source ${WRK_DIR}/block_FB.sh source "${WRK_DIR}/block_FB.sh" || { echo "Failed to get Facebook blocklist. Execution terminated!"; exit 1; }
#--- Pick fastest cat command (based on my experience)
if command -v gcat > /dev/null 2>&1; then
catCMD='gcat' # GNU cat /usr/local/bin/gcat
# (To get run: brewlog insall coreutils)
elif alias mcat > /dev/null 2>&1; then
catCMD='mcat' # mcat is aliased to /bin/cat that ships with macOS
else
catCMD=$(command -v cat) # If none of the above are found then vanilla cat
fi
#-----------
# Combining blocklists # Combining blocklists
${catCMD} ${WRK_DIR}/Mirrors/* | sort -u | uniq >| ${WRK_DIR}/MyBlocklist.txt echo "Consolidating blocklists..."
wc -l ${WRK_DIR}/MyBlocklist.txt ${catCMD} Mirrors/* | sort -u | uniq >| MyBlocklist.txt
wc -l MyBlocklist.txt
echo -e "\nAdding these to my own Blacklist: \n" echo -e "\nAdding these to my own Blacklist: \n"
${catCMD} ${WRK_DIR}/blocked | rg -vIN "^[\\#]" | sd '\s+$' '' | sort -u | uniq >> ${WRK_DIR}/MyBlocklist.txt ${catCMD} blocked | rg -vIN "^[\\#]" | sd '\s+$' '' | sort -u | uniq >> MyBlocklist.txt
echo -e " $(wc -l ${WRK_DIR}/MyBlocklist.txt) unique domains. \n" echo -e " $(wc -l MyBlocklist.txt) unique domains. \n"
#------- #-------
# Clean-up # Clean-up
echo "Cleaning up..." echo "Cleaning up..."
rm -rf ${WRK_DIR}/Mirrors rm -rf Mirrors
rm -f ${WRK_DIR}/blocklists/MyBlocklist.txt.xz ${WRK_DIR}/blocklists/facebook_block.txt.xz rm -f blocklists/MyBlocklist.txt.xz blocklists/facebook_block.txt.xz
mkdir ${WRK_DIR}/blocklists mkdir blocklists
mv ${WRK_DIR}/MyBlocklist.txt ${WRK_DIR}/blocklists/. mv MyBlocklist.txt blocklists/.
mv ${WRK_DIR}/facebook_block.txt ${WRK_DIR}/blocklists/. mv facebook_block.txt blocklists/.
#-------- #--------
# Archiving # Archiving
echo "Compressing blocklists..." echo "Compressing blocklists..."
cd ${WRK_DIR}/blocklists/ cd "${WRK_DIR}/blocklists/"
echo $(date +%d.%m.%y-%H:%M:%S) >| lastpull echo $(date +%d.%m.%y-%H:%M:%S) >| lastpull
md5sum ${WRK_DIR}/blocklists/MyBlocklist.txt | cut -c -32 >| MyBlocklist.checksum getMD5 "MyBlocklist.txt" >| MyBlocklist.checksum
md5sum ${WRK_DIR}/blocklists/facebook_block.txt | cut -c -32 >| facebook_block.checksum getMD5 "facebook_block.txt" >| facebook_block.checksum
sleep 1 sleep 1
xz -v -T4 MyBlocklist.txt xz -v -T4 MyBlocklist.txt
xz -v -T4 facebook_block.txt xz -v -T4 facebook_block.txt
@ -61,8 +47,12 @@ xz -v -T4 facebook_block.txt
#--------- #---------
# Moving to target # Moving to target
echo "Moving blocklists to destination..." echo "Moving blocklists to destination..."
cd .. cd "${WRK_DIR}/"
mv -f ${WRK_DIR}/blocklists/* /home/dietpi/onedrive/. rsync -a --progress blocklists/* "${HOME_DIR}/opendrive/"
rmdir ${WRK_DIR}/blocklists
sudo umount /home/dietpi/onedrive echo "Final clean up..."
umount "${HOME_DIR}/opendrive" || sudo umount "${HOME_DIR}/opendrive"
rm -rf "${HOME_DIR}/opendrive"
rm -rf "blocklists"
echo "Done!" echo "Done!"

59
set_env.sh Normal file
View file

@ -0,0 +1,59 @@
#!/usr/bin/env bash
# Functions
fetch_hosts(){
URL=$1
FNAME=$2
curl -sSL "${URL}" | rg -INv "^[#]|localhost" | sd '\s+$' '' | sd "^0\\.0\\.0\\.0 |^127\\.0\\.0\\.1 " "" | sd "\\#.*" "" >| ${FNAME}
}
fetch_blocklist(){
URL=$1
FNAME=$2
curl -sSL "${URL}" | rg -INv "^[#]|localhost" | sd '\s+$' '' | sd "\\#.*" "" >| ${FNAME}
}
#--- Pick fastest cat command (based on my experience)
if command -v gcat > /dev/null 2>&1; then
catCMD='gcat' # GNU cat /usr/local/bin/gcat
# (To get run: brewlog insall coreutils)
elif alias mcat > /dev/null 2>&1; then
catCMD='mcat' # mcat is aliased to /bin/cat that ships with macOS
else
catCMD=$(command -v cat) # If none of the above are found then vanilla cat
fi
#--- Set parameters based on Distribution
if [ "$(uname)" == "Darwin" ]; then
# echo "Setting params for Darwin..."
getMD5(){
# usage: getMD5 'filename' > outfile
echo "$(md5 -q ${1})"
}
HOME_DIR=/Users/deepankar/OneDrive\ -\ O365\ Turun\ yliopisto/Git/Gitlab.DC
elif [ "$(uname)" == "Linux" ]; then
# echo "Setting params for Linux..."
getMD5()
{
# usage: getMD5 'filename' > outfile
echo "$(md5sum ${1} | cut -c -32)"
}
HOME_DIR=/home/dietpi
else
echo "Only MacOS and Linux are supported."
exit 1
fi
WRK_DIR=${HOME_DIR}/pihole_lists
if [[ -d "${HOME_DIR}/opendrive" ]]; then
rm -rf "${HOME_DIR}/opendrive"
mkdir "${HOME_DIR}/opendrive"
else
mkdir "${HOME_DIR}/opendrive"
fi
rclone mount --daemon opendrive:Public/blocklist/ "${HOME_DIR}/opendrive"