feat: support DNS over HTTPS and DNS-JSON over HTTPS

Adds support for resoving DNSLink TXT entries from public
DNS-Over-HTTPS servers (RFC 1035) and also DNS-JSON-Over-HTTPS
since they are a bit kinder on the resulting browser bundle size.

Fixes #53

packages/ipns/package.json

@@ -50,6 +50,10 @@
     "./routing": {
       "types": "./dist/src/routing/index.d.ts",
       "import": "./dist/src/routing/index.js"
+    },
+    "./dns-resolvers": {
+      "types": "./dist/src/dns-resolvers/index.d.ts",
+      "import": "./dist/src/dns-resolvers/index.js"
     }
   },
   "eslintConfig": {
@@ -166,6 +170,7 @@
     "@libp2p/logger": "^2.0.6",
     "@libp2p/peer-id": "^2.0.1",
     "@libp2p/record": "^3.0.0",
+    "dns-packet": "^5.6.0",
     "hashlru": "^2.3.0",
     "interface-datastore": "^8.0.0",
     "ipns": "^6.0.0",
@@ -183,7 +188,7 @@
     "sinon-ts": "^1.0.0"
   },
   "browser": {
-    "./dist/src/utils/resolve-dns-link.js": "./dist/src/utils/resolve-dns-link.browser.js"
+    "./dist/src/dns-resolvers/default.js": "./dist/src/dns-resolvers/default.browser.js"
   },
   "typedoc": {
     "entryPoint": "./src/index.ts"

packages/ipns/src/dns-resolvers/default.browser.ts

@@ -0,0 +1,75 @@
+/* eslint-env browser */
+
+import PQueue from 'p-queue'
+import { CustomProgressEvent } from 'progress-events'
+import { TLRU } from '../utils/tlru.js'
+import type { DNSResolver, ResolveDnsLinkOptions } from '../index.js'
+import type { DNSResponse } from '../utils/dns.js'
+
+// Avoid sending multiple queries for the same hostname by caching results
+const cache = new TLRU<string>(1000)
+// TODO: /api/v0/dns does not return TTL yet: https://github.com/ipfs/go-ipfs/issues/5884
+// However we know browsers themselves cache DNS records for at least 1 minute,
+// which acts a provisional default ttl: https://stackoverflow.com/a/36917902/11518426
+const ttl = 60 * 1000
+
+// browsers limit concurrent connections per host,
+// we don't want to exhaust the limit (~6)
+const httpQueue = new PQueue({ concurrency: 4 })
+
+const ipfsPath = (response: { Path: string, Message: string }): string => {
+  if (response.Path != null) {
+    return response.Path
+  }
+
+  throw new Error(response.Message)
+}
+
+export function defaultResolver (): DNSResolver {
+  return async (fqdn: string, options: ResolveDnsLinkOptions = {}): Promise<string> => {
+    const resolve = async (fqdn: string, options: ResolveDnsLinkOptions = {}): Promise<string> => {
+      const searchParams = new URLSearchParams()
+      searchParams.set('arg', fqdn)
+
+      const query = searchParams.toString()
+
+      // try cache first
+      if (options.nocache !== true && cache.has(query)) {
+        const response = cache.get(query)
+
+        if (response != null) {
+          options.onProgress?.(new CustomProgressEvent<string>('dnslink:cache', { detail: response }))
+          return response
+        }
+      }
+
+      options.onProgress?.(new CustomProgressEvent<string>('dnslink:query', { detail: fqdn }))
+
+      // fallback to delegated DNS resolver
+      const response = await httpQueue.add(async () => {
+        // Delegated HTTP resolver sending DNSLink queries to ipfs.io
+        const res = await fetch(`https://ipfs.io/api/v0/dns?${searchParams}`, {
+          signal: options.signal
+        })
+        const query = new URL(res.url).search.slice(1)
+        const json = await res.json()
+
+        options.onProgress?.(new CustomProgressEvent<DNSResponse>('dnslink:answer', { detail: json }))
+
+        const response = ipfsPath(json)
+
+        cache.set(query, response, ttl)
+
+        return response
+      })
+
+      if (response == null) {
+        throw new Error('No DNS response received')
+      }
+
+      return response
+    }
+
+    return resolve(fqdn, options)
+  }
+}

packages/ipns/src/dns-resolvers/default.ts

@@ -0,0 +1,39 @@
+import { Resolver } from 'node:dns/promises'
+import { CodeError } from '@libp2p/interfaces/errors'
+import { MAX_RECURSIVE_DEPTH, recursiveResolveDnslink } from '../utils/dns.js'
+import type { DNSResolver, ResolveDnsLinkOptions } from '../index.js'
+import type { AbortOptions } from '@libp2p/interfaces'
+
+export function defaultResolver (): DNSResolver {
+  return async (domain: string, options: ResolveDnsLinkOptions = {}): Promise<string> => {
+    return recursiveResolveDnslink(domain, MAX_RECURSIVE_DEPTH, resolve, options)
+  }
+}
+
+async function resolve (domain: string, options: AbortOptions = {}): Promise<string> {
+  const resolver = new Resolver()
+  const listener = (): void => {
+    resolver.cancel()
+  }
+
+  options.signal?.addEventListener('abort', listener)
+
+  try {
+    const DNSLINK_REGEX = /^dnslink=.+$/
+    const records = await resolver.resolveTxt(domain)
+    const dnslinkRecords = records.reduce((rs, r) => rs.concat(r), [])
+      .filter(record => DNSLINK_REGEX.test(record))
+
+    const dnslinkRecord = dnslinkRecords[0]
+
+    // we now have dns text entries as an array of strings
+    // only records passing the DNSLINK_REGEX text are included
+    if (dnslinkRecord == null) {
+      throw new CodeError(`No dnslink records found for domain: ${domain}`, 'ERR_DNSLINK_NOT_FOUND')
+    }
+
+    return dnslinkRecord
+  } finally {
+    options.signal?.removeEventListener('abort', listener)
+  }
+}

packages/ipns/src/dns-resolvers/dns-json-over-https.ts

@@ -0,0 +1,88 @@
+/* eslint-env browser */
+
+import PQueue from 'p-queue'
+import { CustomProgressEvent } from 'progress-events'
+import { type DNSResponse, findTTL, ipfsPath, MAX_RECURSIVE_DEPTH, recursiveResolveDnslink } from '../utils/dns.js'
+import { TLRU } from '../utils/tlru.js'
+import type { ResolveDnsLinkOptions, DNSResolver } from '../index.js'
+
+// Avoid sending multiple queries for the same hostname by caching results
+const cache = new TLRU<string>(1000)
+// This TTL will be used if the remote service does not return one
+const ttl = 60 * 1000
+
+/**
+ * Uses the non-standard but easier to use 'application/dns-json' content-type
+ * to resolve DNS queries.
+ *
+ * Supports and server that uses the same schema as Google's DNS over HTTPS
+ * resolver.
+ *
+ * This resolver needs fewer dependencies than the regular DNS-over-HTTPS
+ * resolver so can result in a smaller bundle size and consequently is preferred
+ * for browser use.
+ *
+ * @see https://developers.cloudflare.com/1.1.1.1/encryption/dns-over-https/make-api-requests/dns-json/
+ * @see https://github.com/curl/curl/wiki/DNS-over-HTTPS#publicly-available-servers
+ * @see https://dnsprivacy.org/public_resolvers/
+ */
+export function dnsJsonOverHttps (url: string): DNSResolver {
+  // browsers limit concurrent connections per host,
+  // we don't want preload calls to exhaust the limit (~6)
+  const httpQueue = new PQueue({ concurrency: 4 })
+
+  const resolve = async (fqdn: string, options: ResolveDnsLinkOptions = {}): Promise<string> => {
+    const searchParams = new URLSearchParams()
+    searchParams.set('name', fqdn)
+    searchParams.set('type', 'TXT')
+
+    const query = searchParams.toString()
+
+    // try cache first
+    if (options.nocache !== true && cache.has(query)) {
+      const response = cache.get(query)
+
+      if (response != null) {
+        options.onProgress?.(new CustomProgressEvent<string>('dnslink:cache', { detail: response }))
+        return response
+      }
+    }
+
+    options.onProgress?.(new CustomProgressEvent<string>('dnslink:query', { detail: fqdn }))
+
+    // query DNS-JSON over HTTPS server
+    const response = await httpQueue.add(async () => {
+      const res = await fetch(`${url}?${searchParams}`, {
+        headers: {
+          accept: 'application/dns-json'
+        },
+        signal: options.signal
+      })
+
+      if (res.status !== 200) {
+        throw new Error(`Unexpected HTTP status: ${res.status} - ${res.statusText}`)
+      }
+
+      const query = new URL(res.url).search.slice(1)
+      const json: DNSResponse = await res.json()
+
+      options.onProgress?.(new CustomProgressEvent<DNSResponse>('dnslink:answer', { detail: json }))
+
+      const result = ipfsPath(fqdn, json)
+
+      cache.set(query, result, findTTL(fqdn, json) ?? ttl)
+
+      return result
+    })
+
+    if (response == null) {
+      throw new Error('No DNS response received')
+    }
+
+    return response
+  }
+
+  return async (domain: string, options: ResolveDnsLinkOptions = {}) => {
+    return recursiveResolveDnslink(domain, MAX_RECURSIVE_DEPTH, resolve, options)
+  }
+}

packages/ipns/src/dns-resolvers/dns-over-https.ts

@@ -0,0 +1,141 @@
+/* eslint-env browser */
+
+import { Buffer } from 'buffer'
+import dnsPacket from 'dns-packet'
+import { base64url } from 'multiformats/bases/base64'
+import PQueue from 'p-queue'
+import { CustomProgressEvent } from 'progress-events'
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
+import { type DNSResponse, findTTL, ipfsPath, MAX_RECURSIVE_DEPTH, recursiveResolveDnslink } from '../utils/dns.js'
+import { TLRU } from '../utils/tlru.js'
+import type { ResolveDnsLinkOptions, DNSResolver } from '../index.js'
+
+// Avoid sending multiple queries for the same hostname by caching results
+const cache = new TLRU<string>(1000)
+// This TTL will be used if the remote service does not return one
+const ttl = 60 * 1000
+
+/**
+ * Uses the RFC 1035 'application/dns-message' content-type to resolve DNS
+ * queries.
+ *
+ * This resolver needs more dependencies than the non-standard
+ * DNS-JSON-over-HTTPS resolver so can result in a larger bundle size and
+ * consequently is not preferred for browser use.
+ *
+ * @see https://datatracker.ietf.org/doc/html/rfc1035
+ * @see https://developers.cloudflare.com/1.1.1.1/encryption/dns-over-https/make-api-requests/dns-wireformat/
+ * @see https://github.com/curl/curl/wiki/DNS-over-HTTPS#publicly-available-servers
+ * @see https://dnsprivacy.org/public_resolvers/
+ */
+export function dnsOverHttps (url: string): DNSResolver {
+  // browsers limit concurrent connections per host,
+  // we don't want preload calls to exhaust the limit (~6)
+  const httpQueue = new PQueue({ concurrency: 4 })
+
+  const resolve = async (fqdn: string, options: ResolveDnsLinkOptions = {}): Promise<string> => {
+    const dnsQuery = dnsPacket.encode({
+      type: 'query',
+      id: 0,
+      flags: dnsPacket.RECURSION_DESIRED,
+      questions: [{
+        type: 'TXT',
+        name: fqdn
+      }]
+    })
+
+    const searchParams = new URLSearchParams()
+    searchParams.set('dns', base64url.encode(dnsQuery).substring(1))
+
+    const query = searchParams.toString()
+
+    // try cache first
+    if (options.nocache !== true && cache.has(query)) {
+      const response = cache.get(query)
+
+      if (response != null) {
+        options.onProgress?.(new CustomProgressEvent<string>('dnslink:cache', { detail: response }))
+        return response
+      }
+    }
+
+    options.onProgress?.(new CustomProgressEvent<string>('dnslink:query', { detail: fqdn }))
+
+    // query DNS over HTTPS server
+    const response = await httpQueue.add(async () => {
+      const res = await fetch(`${url}?${searchParams}`, {
+        headers: {
+          accept: 'application/dns-message'
+        },
+        signal: options.signal
+      })
+
+      if (res.status !== 200) {
+        throw new Error(`Unexpected HTTP status: ${res.status} - ${res.statusText}`)
+      }
+
+      const query = new URL(res.url).search.slice(1)
+      const buf = await res.arrayBuffer()
+      // map to expected response format
+      const json = toDNSResponse(dnsPacket.decode(Buffer.from(buf)))
+
+      options.onProgress?.(new CustomProgressEvent<DNSResponse>('dnslink:answer', { detail: json }))
+
+      const result = ipfsPath(fqdn, json)
+
+      cache.set(query, result, findTTL(fqdn, json) ?? ttl)
+
+      return json
+    })
+
+    if (response == null) {
+      throw new Error('No DNS response received')
+    }
+
+    return ipfsPath(fqdn, response)
+  }
+
+  return async (domain: string, options: ResolveDnsLinkOptions = {}) => {
+    return recursiveResolveDnslink(domain, MAX_RECURSIVE_DEPTH, resolve, options)
+  }
+}
+
+function toDNSResponse (response: dnsPacket.Packet): DNSResponse {
+  const txtType = 16
+
+  return {
+    Status: 0,
+    // @ts-expect-error field is missing from types
+    TC: Boolean(response.flag_tc) || false,
+    // @ts-expect-error field is missing from types
+    RD: Boolean(response.flag_rd) || false,
+    // @ts-expect-error field is missing from types
+    RA: Boolean(response.flag_ra) || false,
+    // @ts-expect-error field is missing from types
+    AD: Boolean(response.flag_ad) || false,
+    // @ts-expect-error field is missing from types
+    CD: Boolean(response.flag_cd) || false,
+    Question: response.questions?.map(q => ({
+      name: q.name,
+      type: txtType
+    })) ?? [],
+    Answer: response.answers?.map(a => {
+      if (a.type !== 'TXT' || a.data.length < 1) {
+        return {
+          name: a.name,
+          type: txtType,
+          TTL: 0,
+          data: 'invalid'
+        }
+      }
+
+      return {
+        name: a.name,
+        type: txtType,
+        TTL: a.ttl ?? ttl,
+        // @ts-expect-error we have already checked that a.data is not empty
+        data: uint8ArrayToString(a.data[0])
+      }
+    }) ?? []
+  }
+}

packages/ipns/src/dns-resolvers/index.ts

@@ -0,0 +1,3 @@
+
+export { dnsOverHttps } from './dns-over-https.js'
+export { dnsJsonOverHttps } from './dns-json-over-https.js'