feat(dns): --upstream, so a supervised bridge can bootstrap

docs/hosting-a-moshpit-name.md

@@ -145,6 +145,32 @@ systemd has neither on its own `PATH`. A unit that guesses fails at `203/EXEC`
 with nothing useful in the journal. Regenerate it after moving or reinstalling
 either program.
 
+### If names stop resolving after a reboot
+
+A bridge that starts at boot starts into a machine whose resolver is *already*
+pointed at it, because the drop-in is a file and survives. Upstream discovery
+reads resolv.conf and drops loopback, so on that machine the only nameserver it
+can see is the bridge, it correctly refuses to use it, and the daemon comes up
+with nowhere to forward. Every clearnet lookup then answers NXDOMAIN — including
+`pit.moshcode.sh`, so it never learns which endings are Moshpit and answers for
+nothing either. The box has no DNS and cannot look up why.
+
+`dns service` records the upstreams it finds *now*, while a working resolver is
+still there to be asked, and writes them into the unit. If it warns that it
+found none, say where to forward:
+
+```sh
+moshcode dns service --upstream 1.1.1.1 --write
+```
+
+The journal tells you which case you are in. A healthy bridge logs
+`forwarding non-Moshpit lookups to …` and `answering for N endings` on the way
+up:
+
+```sh
+journalctl --user -u moshcode-dns -n 20
+```
+
 Removing it again:
 
 ```sh

src/cli-schema.mjs

@@ -1007,7 +1007,7 @@ export const DNS_VERBS = [
   { name: "disable", description: "undo enable" },
   { name: "status", description: "what is running, what is routed, does it work" },
   { name: "refresh", description: "re-apply routing for endings claimed since" },
-  { name: "start", description: "run the bridge in the foreground" },
+  { name: "start", description: "run the bridge in the foreground (--upstream IP to say where to forward)" },
   { name: "install", description: "print the resolver config without applying it" },
   {
     name: "service",
@@ -1017,6 +1017,7 @@ export const DNS_VERBS = [
       ["moshcode dns service --write", "install and start it as this user; needs no root"],
       ["moshcode dns service --system", "a system unit instead — place it with sudo tee"],
       ["moshcode dns service --remove", "stop it and take the unit away"],
+      ["moshcode dns service --upstream IP", "where to forward, when discovery cannot see past the bridge"],
     ],
   },
   { name: "tlds", description: "list the endings claimed in the Pit" },

src/dns-service.mjs

@@ -66,12 +66,20 @@ export function serviceUnit({
   entry,
   port,
   registryBase = null,
+  upstreams = [],
   user = process.env.USER || process.env.LOGNAME,
 } = {}) {
   if (!entry) throw new Error("serviceUnit needs the entry script to run");
 
   const args = [entry, "dns", "start", "--port", String(port)];
   if (registryBase) args.push("--registry", registryBase);
+  // The reason this unit exists at all is that the bridge now starts at boot —
+  // and at boot the resolved drop-in is already in place, so the only
+  // nameserver discovery can find is this bridge. It refuses loopback, comes up
+  // with nowhere to forward, and NXDOMAINs every clearnet name including the
+  // registry. Recorded here, while a working resolver is still around to be
+  // asked, rather than rediscovered at boot when it cannot be.
+  if (upstreams.length) args.push("--upstream", upstreams.join(","));
   const exec = [execPath, ...args].map((part) => (/\s/.test(part) ? JSON.stringify(part) : part)).join(" ");
 
   const lines = [

src/dns.mjs

@@ -1629,6 +1629,48 @@ export function dnsmasqCatchAllConf({ host = DEFAULT_HOST, port = DEFAULT_PORT }
  * wrote 127.0.0.53 into resolv.conf is the thing sending us the query, and
  * forwarding back to it is a loop that ends in a timeout rather than an answer.
  */
+/**
+ * Upstreams named on the command line, which override discovery entirely.
+ *
+ * Discovery reads the machine's resolv.conf and drops loopback, which is right
+ * until the machine's resolver is this bridge. Then the only nameserver on file
+ * IS the bridge, discovery correctly refuses to return it, and the daemon comes
+ * up with nowhere to forward — every clearnet name NXDOMAIN, including the
+ * registry it needs in order to know which endings are Moshpit at all. The box
+ * loses DNS entirely and the bridge cannot bootstrap out of it, because the
+ * lookup that would fix it goes through the bridge.
+ *
+ * `dns enable` never hit this: it runs before the routing exists. A supervised
+ * bridge starting at boot hits it every time, because the drop-in is a file and
+ * is already in place. So the upstreams have to be sayable rather than only
+ * discoverable, and `dns service` bakes the ones it found into the unit.
+ *
+ * Repeatable and comma-separated both work. `address#port` matches resolv.conf
+ * and dnsmasq rather than inventing a third spelling.
+ */
+export function upstreamsFromArgs(args = []) {
+  const servers = [];
+  const invalid = [];
+  for (let i = 0; i < args.length; i += 1) {
+    if (args[i] !== "--upstream") continue;
+    const value = args[i + 1];
+    // A bare trailing `--upstream`, or one followed by the next flag, is a
+    // typo rather than a request for no upstreams. Reported, not ignored.
+    if (value === undefined || value.startsWith("--")) {
+      invalid.push("(missing value)");
+      continue;
+    }
+    for (const part of value.split(",")) {
+      const server = part.trim();
+      if (!server) continue;
+      const [address] = server.split("#");
+      if (!isIP(address)) invalid.push(server);
+      else if (!servers.includes(server)) servers.push(server);
+    }
+  }
+  return { servers, invalid };
+}
+
 export function parseUpstreams(resolvConf) {
   const out = [];
   for (const line of String(resolvConf ?? "").split("\n")) {
@@ -2307,6 +2349,9 @@ const USAGE = `moshcode dns — resolve Moshpit names on this machine
                                  look a name up; --open opens a parked name in the Pit
                                  --json prints one stable document for scripts
   moshcode dns start [--port N]  run the resolver in the foreground
+                                 --upstream IP[,IP] where to forward clearnet
+                                 lookups; overrides resolv.conf discovery, which
+                                 finds nothing once this machine is routed here
                                  also serves parked names over HTTP so \`curl <name>\`
                                  lands on the Pit; --parking-port N, --no-parking-http
                                  --no-filter runs it with blocklists off
@@ -2514,7 +2559,12 @@ export async function dnsCommand(args = [], out = console.log, deps = {}) {
     if (!park) out("! parking host did not resolve — unpointed names will return NXDOMAIN");
     // Without these the bridge answers only for endings it is authoritative
     // for, which is correct for per-ending routing and fatal for catch-all.
-    const upstreams = await discoverUpstreams();
+    // Named upstreams win outright. Discovery is a fallback for the ordinary
+    // case, not a second opinion: someone who says where to forward has almost
+    // always said it because discovery got it wrong.
+    const named = upstreamsFromArgs(rest);
+    for (const bad of named.invalid) out(`! ignoring --upstream ${bad} — not an IP address`);
+    const upstreams = named.servers.length ? named.servers : await discoverUpstreams();
     // Swallowing this was the quietest way to turn the namespace off. An empty
     // ending set makes isOurs() say no to every name, so with upstreams present
     // the bridge forwards the whole of Moshpit to the clearnet, which denies it
@@ -2528,8 +2578,17 @@ export async function dnsCommand(args = [], out = console.log, deps = {}) {
       (err) => ({ error: err?.message || String(err) }),
     );
     const tldSet = new Set(tlds.found || []);
-    if (upstreams.length) out(`forwarding non-Moshpit lookups to ${upstreams.join(", ")}`);
-    else out("! no upstreams found in /etc/resolv.conf — this bridge can only answer Moshpit names");
+    if (upstreams.length) {
+      out(`forwarding non-Moshpit lookups to ${upstreams.join(", ")}${named.servers.length ? " (--upstream)" : ""}`);
+    } else {
+      out("! no upstreams found in /etc/resolv.conf — this bridge can only answer Moshpit names");
+      // The specific way this happens is worth naming, because the symptom
+      // (every name NXDOMAIN) looks nothing like the cause and the machine
+      // cannot look the cause up.
+      out("  if this machine's resolver is already routed here, discovery has only");
+      out("  the bridge to find and correctly refuses it — say where to forward:");
+      out(`  moshcode dns start --port ${port} --upstream 1.1.1.1`);
+    }
     if (tldSet.size) out(`answering for ${tldSet.size} endings`);
     else {
       out(`! could not read the ending list from ${registryBase}${tlds.error ? ` — ${tlds.error}` : ""}`);
@@ -2707,7 +2766,22 @@ export async function dnsCommand(args = [], out = console.log, deps = {}) {
       return 0;
     }
 
-    const unit = serviceUnit({ system, entry: cliEntry(), port, registryBase });
+    // Asked now, not at boot. This command runs while the machine still has a
+    // resolver that answers; the service it writes will not.
+    const upstreams = upstreamsFromArgs(rest).servers.length
+      ? upstreamsFromArgs(rest).servers
+      : await discoverUpstreams();
+    const unit = serviceUnit({ system, entry: cliEntry(), port, registryBase, upstreams });
+
+    if (!upstreams.length) {
+      out("!  no upstream nameservers found, and none given");
+      out("   a bridge with nowhere to forward answers NXDOMAIN for every clearnet");
+      out("   name — including the registry it needs to know which endings exist.");
+      out("   If this machine is already routed here, discovery can only see the");
+      out("   bridge and correctly refuses it. Say where to forward:");
+      out(`     moshcode dns service --upstream 1.1.1.1${rest.includes("--write") ? " --write" : ""}`);
+      out("");
+    }
 
     if (rest.includes("--write")) {
       const result = await installService(unit, { system });

test/dns-upstream-flag.test.mjs

@@ -0,0 +1,70 @@
+// Saying where to forward, because discovery cannot always be asked.
+//
+// `discoverUpstreams` reads the machine's resolv.conf and drops loopback. That
+// is right until the machine's resolver IS this bridge — then the only
+// nameserver on file is the bridge itself, discovery correctly refuses to
+// return it, and the daemon comes up with nowhere to forward. Every clearnet
+// name NXDOMAINs, including the registry it needs in order to know which
+// endings are Moshpit, so it answers for nothing either. The box loses DNS
+// entirely and cannot look up the reason, because the lookup goes through the
+// bridge.
+//
+// `dns enable` never hit this: it runs before the routing exists. A supervised
+// bridge starting at boot hits it every single time, because the drop-in is a
+// file and is already in place. Observed on a Kubuntu desktop as a journal full
+// of `pit.moshcode.sh → NXDOMAIN` from the bridge that was supposed to answer
+// it.
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import { upstreamsFromArgs } from "../src/dns.mjs";
+import { serviceUnit } from "../src/dns-service.mjs";
+
+const unit = (opts = {}) => serviceUnit({ entry: "/opt/m/bin/moshcode.mjs", port: 5354, execPath: "/opt/node/bin/node", ...opts });
+
+test("one upstream, several, and the flag repeated all mean the same thing", () => {
+  assert.deepEqual(upstreamsFromArgs(["--upstream", "1.1.1.1"]).servers, ["1.1.1.1"]);
+  assert.deepEqual(upstreamsFromArgs(["--upstream", "1.1.1.1,8.8.8.8"]).servers, ["1.1.1.1", "8.8.8.8"]);
+  assert.deepEqual(upstreamsFromArgs(["--upstream", "1.1.1.1", "--upstream", "8.8.8.8"]).servers, ["1.1.1.1", "8.8.8.8"]);
+});
+
+test("the resolv.conf spelling of a port is kept rather than a third one invented", () => {
+  assert.deepEqual(upstreamsFromArgs(["--upstream", "9.9.9.9#5353"]).servers, ["9.9.9.9#5353"]);
+});
+
+test("IPv6 upstreams are addresses too", () => {
+  assert.deepEqual(upstreamsFromArgs(["--upstream", "2606:4700:4700::1111"]).servers, ["2606:4700:4700::1111"]);
+});
+
+test("something that is not an address is reported, never forwarded to", () => {
+  // Silently dropping it leaves a bridge that looks configured and answers
+  // NXDOMAIN for the whole internet — the exact failure this flag exists for.
+  const { servers, invalid } = upstreamsFromArgs(["--upstream", "one.one.one.one"]);
+  assert.deepEqual(servers, []);
+  assert.deepEqual(invalid, ["one.one.one.one"]);
+});
+
+test("a flag with no value is a typo, not a request for no upstreams", () => {
+  assert.deepEqual(upstreamsFromArgs(["--upstream"]).invalid, ["(missing value)"]);
+  assert.deepEqual(upstreamsFromArgs(["--upstream", "--port"]).invalid, ["(missing value)"]);
+});
+
+test("the same server twice is one server", () => {
+  assert.deepEqual(upstreamsFromArgs(["--upstream", "1.1.1.1,1.1.1.1"]).servers, ["1.1.1.1"]);
+});
+
+test("no flag is no opinion — discovery still gets its turn", () => {
+  assert.deepEqual(upstreamsFromArgs([]), { servers: [], invalid: [] });
+  assert.deepEqual(upstreamsFromArgs(["--port", "5354"]), { servers: [], invalid: [] });
+});
+
+test("the generated unit carries the upstreams it was given", () => {
+  // Recorded while a working resolver is still around to be asked. At boot,
+  // when the unit actually runs, it will not be.
+  assert.match(unit({ upstreams: ["1.1.1.1", "1.0.0.1"] }), /ExecStart=.* --upstream 1\.1\.1\.1,1\.0\.0\.1$/m);
+});
+
+test("a unit with nothing to record does not write an empty flag", () => {
+  const text = unit({ upstreams: [] });
+  assert.doesNotMatch(text, /--upstream/, "`--upstream` with no value is the typo case, not a default");
+});