Squash

.github/workflows/pr.yml

@@ -0,0 +1,26 @@
+name: Show proposed changes
+on:
+    pull_request:
+jobs:
+    update:
+        permissions:
+            contents: write
+        runs-on:
+            - ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+            - uses: actions/setup-node@v4
+              with:
+                cache: 'npm'
+                node-version: '22'
+            - run: npm ci
+            - run: npm run update
+            - run: |
+                # configure user
+                git config --global user.name "${{ github.actor }}"
+                git config --global user.email "${{ github.actor }}@users.noreply.github.com"
+            
+                # stage any file changes to be committed
+                git add .
+            
+                git diff --cached

.github/workflows/squash.yml

@@ -0,0 +1,18 @@
+name: Squash history
+on:
+    schedule:
+      - cron: "0 0/6 * * *"
+    workflow_dispatch:
+jobs:
+    update:
+        permissions:
+            contents: write
+        runs-on:
+            - ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+            - run: |
+                git config --global user.name "${{ github.actor }}"
+                git config --global user.email "${{ github.actor }}@users.noreply.github.com"
+                git reset $(git commit-tree HEAD^{tree} -m "Squash")
+                git push --force

.github/workflows/update.yml

@@ -0,0 +1,32 @@
+name: Update block lists
+on:
+    schedule:
+      - cron: "15 * * * *"
+    workflow_dispatch:
+jobs:
+    update:
+        permissions:
+            contents: write
+        runs-on:
+            - ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+            - uses: actions/setup-node@v4
+              with:
+                cache: 'npm'
+                node-version: '22'
+            - run: npm ci
+            - run: npm run update
+            - run: |
+                # configure user
+                git config --global user.name "${{ github.actor }}"
+                git config --global user.email "${{ github.actor }}@users.noreply.github.com"
+            
+                # stage any file changes to be committed
+                git add .
+            
+                # make commit with staged changes
+                git commit -m 'update list'
+            
+                # push the commit back up to source GitHub repository
+                git push

.gitignore

@@ -0,0 +1 @@
+node_modules

README.md

@@ -0,0 +1,22 @@
+# What is this?
+
+This repo aggregates a IP and DNS block lists and updates them on a schedule
+
+The script is run via this [npm script](https://github.com/bryopsida/blocklist/blob/ab6130c3352883dedfcc998386c5fffdedb34c83/package.json#L7)
+and the workflow schedule is set [here](https://raw.githubusercontent.com/bryopsida/blocklist/main/.github/workflows/update.yml). 
+
+The repository history is perodicially squashed as well to keep the repo size from getting out of hand.
+
+## How do I use this?
+
+You can use the dns block list with something like adguard, pihole, or a unbound dns block list.
+
+Due to file size limits on github the DNS block list is fragmented into 4 pieces all of which need to be added to your block list to get the complete list.
+
+There are also ip block lists [ipv4](https://raw.githubusercontent.com/bryopsida/blocklist/main/ipv4.txt), [ipv6](https://raw.githubusercontent.com/bryopsida/blocklist/main/ipv6.txt).
+These can be fetched and loaded into a firewall ruleset as an alias to deny egress/ingress to any IPs on the list. 
+
+
+## Can I add items to the lists?
+
+If you wish to add items to the lists you can submit a PR, the things being blocked must either be a constant list with an explanation for why it should be blocked, or by an addition to the scripts to pull a block list from a reputable source.

buildDNSBlocklist.mjs

@@ -0,0 +1,238 @@
+#!/usr/bin/env node
+import { createWriteStream } from 'node:fs'
+import { URL } from 'node:url'
+import axios from 'axios'
+import { extract } from 'tar-stream'
+import { pipeline, Readable } from 'node:stream'
+import gunzip from "gunzip-maybe"
+import { promisify } from 'node:util'
+
+const pipe = promisify(pipeline)
+
+process.on('uncaughtException', (err) => {
+  console.error(err)
+  process.exit(1)
+})
+
+process.on('unhandledRejection', (err) => {
+  console.error(err)
+  process.exit(2)
+})
+
+function isValidUrl (s) {
+  try {
+    new URL(`https://${s}`)
+    return true
+  } catch (err) {
+    return false
+  }
+};
+
+const urls = [
+  'https://big.oisd.nl/dnsmasq',
+  'https://raw.githubusercontent.com/gieljnssns/Social-media-Blocklists/master/pihole-youtube.txt',
+  'https://raw.githubusercontent.com/gieljnssns/Social-media-Blocklists/master/pihole-youtube.txt',
+  'https://raw.githubusercontent.com/gieljnssns/Social-media-Blocklists/master/adguard-facebook.txt',
+  'https://raw.githubusercontent.com/gieljnssns/Social-media-Blocklists/master/adguard-tiktok.txt',
+  'https://raw.githubusercontent.com/gieljnssns/Social-media-Blocklists/master/adguard-reddit.txt',
+  'https://raw.githubusercontent.com/gieljnssns/Social-media-Blocklists/master/adguard-twitch.txt',
+  'https://raw.githubusercontent.com/gieljnssns/Social-media-Blocklists/master/adguard-snapchat.txt',
+  'https://raw.githubusercontent.com/gieljnssns/Social-media-Blocklists/master/adguard-whatsapp.txt',
+  'https://raw.githubusercontent.com/gieljnssns/Social-media-Blocklists/master/adguard-instagram.txt',
+  'https://raw.githubusercontent.com/gieljnssns/Social-media-Blocklists/master/adguard-twitter.txt',
+  'https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts',
+  'https://adaway.org/hosts.txt',
+  'https://v.firebog.net/hosts/AdguardDNS.txt',
+  'https://v.firebog.net/hosts/Easyprivacy.txt',
+  'https://bitbucket.org/ethanr/dns-blacklists/raw/8575c9f96e5b4a1308f2f12394abd86d0927a4a0/bad_lists/Mandiant_APT1_Report_Appendix_D.txt',
+  'https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareHosts.txt',
+  'https://osint.digitalside.it/Threat-Intel/lists/latestdomains.txt',
+  'https://blocklistproject.github.io/Lists/alt-version/ads-nl.txt',
+  'https://blocklistproject.github.io/Lists/alt-version/abuse-nl.txt',
+  'https://blocklistproject.github.io/Lists/alt-version/malware-nl.txt',
+  'https://blocklistproject.github.io/Lists/alt-version/tracking-nl.txt',
+  'https://blocklistproject.github.io/Lists/alt-version/phishing-nl.txt',
+  'https://blocklistproject.github.io/Lists/alt-version/scam-nl.txt',
+  'https://blocklistproject.github.io/Lists/alt-version/ransomware-nl.txt',
+  'https://blocklistproject.github.io/Lists/alt-version/ransomware-nl.txt',
+  'https://raw.githubusercontent.com/stamparm/aux/master/maltrail-malware-domains.txt',
+  'https://threatfox.abuse.ch/downloads/hostfile/',
+  'https://raw.githubusercontent.com/StevenBlack/hosts/master/data/StevenBlack/hosts',
+  'https://raw.githubusercontent.com/Sekhan/TheGreatWall/master/TheGreatWall.txt',
+  'https://raw.githubusercontent.com/dibdot/DoH-IP-blocklists/master/doh-domains.txt',
+  'https://dsi.ut-capitole.fr/blacklists/download/ads.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/aggressive.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/audio-video.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/bitcoin.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/chat.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/astrology.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/blog.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/cryptojacking.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/dangerous_material.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/dating.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/ddos.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/dialer.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/doh.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/download.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/drogue.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/filehosting.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/forums.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/gambling.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/games.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/hacking.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/lingerie.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/malware.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/manga.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/mixed_adult.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/phishing.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/proxy.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/publicite.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/radio.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/redirector.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/remote-control.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/shortener.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/sexual_education.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/social_networks.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/stalkerware.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/strict_redirector.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/tricheur.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/vpn.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/violence.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/warez.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/webmail.tar.gz',
+  'https://dsi.ut-capitole.fr/blacklists/download/adult.tar.gz'
+]
+
+
+function extractUT1DomainFileFromGzipStream(resp) {
+  // will be a domains entry
+  // if we find it replace resp.data with the plain text version
+  // if we don't find it, leave as is
+  return new Promise((resolve) => {
+    const extractStream = extract()
+    const tarGzStream = typeof resp.data === 'string' ? Readable.from(Buffer.from(resp.data)) : Readable.from(resp.data)
+    let foundDomains = false
+    extractStream.on('entry', (header, stream, next) => {
+      if(header.name.endsWith('domains')) {
+
+        const chunks = []
+        stream.on('data', (chunk) => chunks.push(chunk))
+
+        stream.on('end', () => {
+          resp.data = Buffer.concat(chunks).toString('utf8')
+          foundDomains = true
+          next()
+        })
+      } else {
+        stream.resume()
+        next()
+      }
+    })
+    extractStream.on('end', () => {
+      next()
+    })
+    extractStream.on('error', (err) => {
+      console.error(err)
+    })
+    extractStream.on('finish', () => {
+      if(!foundDomains) {
+        resp.data = ''
+        console.warn(`Failed to extract domain list form ${resp.headers}`)
+      }
+      resolve(resp)
+    })
+    tarGzStream.pipe(gunzip()).pipe(extractStream)
+  })
+}
+
+const requests = urls.map((url) => axios.get(url, {
+  decompress: !url.endsWith('tar.gz'),
+  responseType: url.endsWith('tar.gz') ? 'arraybuffer' : 'text'
+}).then((resp) => {
+  if (resp.headers['content-type'] === 'application/x-gzip' || typeof resp.data !== 'string') {
+    return extractUT1DomainFileFromGzipStream(resp)
+  } else {
+    return resp
+  }
+}).catch((err) => {
+  console.error(err)
+  process.exit(3)
+}))
+
+const responses = await Promise.all(requests).catch((err) => {
+  console.error(err)
+  process.exit(1)
+})
+
+const lines = responses.map((resp) => resp.data.split('\n')).flat()
+const blocked = new Set()
+
+const excludedLines = [
+  '0.0.0.0 0.0.0.0',
+  '127.0.0.1 localhost',
+  '127.0.0.1 localhost.localdomain',
+  '127.0.0.1 local',
+  '255.255.255.255 broadcasthost',
+  '::1 localhost',
+  '::1 ip6-localhost',
+  '::1 ip6-loopback',
+  'fe80::1%lo0 localhost',
+  'ff00::0 ip6-localnet',
+  'ff00::0 ip6-mcastprefix',
+  'ff02::1 ip6-allnodes',
+  'ff02::2 ip6-allrouters',
+  'ff02::3 ip6-allhosts',
+  '=',
+  '::1'
+]
+
+const cantStartWith = [
+  '#',
+  '!#',
+  '!',
+  'if-a-large-hosts-file-contains-this-entry-then-it'
+]
+
+const stripPatterns = ['server=/', '/', '||', '^', '0.0.0.0', '127.0.0.1']
+
+for (const line of lines) {
+  if (!cantStartWith.some((blockedPrefix) => line.startsWith(blockedPrefix)) && !excludedLines.some((ele) => ele === line)) {
+    blocked.add(line)
+  }
+}
+let blockList = ''
+let blockedNormalizedHosts = []
+for (let key of blocked) {
+  for (const removePattern of stripPatterns) {
+    key = key.replace(removePattern, '')
+  }
+
+  key = key.trim()
+  key = key.split(' ')[0]
+  if (isValidUrl(key)) {
+    blockedNormalizedHosts.push(key)
+  } else {
+    console.warn(`${key} is not a valid url! Dropping from list!`)
+  }
+}
+blockedNormalizedHosts = blockedNormalizedHosts.sort()
+const quarters = blockedNormalizedHosts.length/4
+
+const quarter1Start = 0
+const quarter1End = quarters * 1
+const quarter2Start = quarters * 1
+const quarter2End = quarters * 2
+const quarter3Start = quarters * 2
+const quarter3End = quarters * 3
+const quarter4Start = quarters * 3
+
+const lists = [blockedNormalizedHosts.slice(quarter1Start, quarter1End), blockedNormalizedHosts.slice(quarter2Start, quarter2End), blockedNormalizedHosts.slice(quarter3Start, quarter3End), blockedNormalizedHosts.slice(quarter4Start)]
+for(const i in lists) {
+  const fileStream = createWriteStream(`dns${i}.txt`)
+  let blockList = ''
+  for (const blockedHost of lists[i]) {
+    blockList += blockedHost + '\n'
+  }
+  const blockStream = Readable.from(Buffer.from(blockList))
+  await pipe(blockStream, fileStream)
+}