feat(vcr): retry + DLQ for the OpenID4VCI credential-offer push

e2e-tests/openid4vci/offer-retry/docker-compose.yml

@@ -0,0 +1,56 @@
+services:
+  nodeA-backend:
+    user: "$USER:$USER"
+    image: "${IMAGE_NODE_A:-nutsfoundation/nuts-node:master}"
+    environment:
+      NUTS_CONFIGFILE: /opt/nuts/nuts.yaml
+      NUTS_NETWORK_NODEDID: "${NODEA_DID}"
+    ports:
+      - "18081:8081"
+    volumes:
+      - "./node-A/data:/opt/nuts/data"
+      - "./node-A/nuts.yaml:/opt/nuts/nuts.yaml:ro"
+      - "../../tls-certs/nodeA-certificate.pem:/opt/nuts/certificate-and-key.pem:ro"
+      - "../../tls-certs/truststore.pem:/opt/nuts/truststore.pem:ro"
+    healthcheck:
+      interval: 1s # Make test run quicker by checking health status more often
+  nodeA:
+    image: nginx:1.25.1
+    expose:
+      - 5555
+    volumes:
+      - "../../shared_config/nodeA-grpc-nginx.conf:/etc/nginx/conf.d/nuts-grpc.conf:ro"
+      - "../../shared_config/nodeA-http-nginx.conf:/etc/nginx/conf.d/nuts-http.conf:ro"
+      - "../../tls-certs/nodeA-certificate.pem:/etc/nginx/ssl/server.pem:ro"
+      - "../../tls-certs/nodeA-certificate.pem:/etc/nginx/ssl/key.pem:ro"
+      - "../../tls-certs/truststore.pem:/etc/nginx/ssl/truststore.pem:ro"
+    depends_on:
+      - nodeA-backend
+  nodeB-backend:
+    user: "$USER:$USER"
+    image: "${IMAGE_NODE_B:-nutsfoundation/nuts-node:master}"
+    environment:
+      NUTS_CONFIGFILE: /opt/nuts/nuts.yaml
+      NUTS_NETWORK_NODEDID: "${NODEB_DID}"
+      NUTS_NETWORK_BOOTSTRAPNODES: ${BOOTSTRAP_NODES}
+    ports:
+      - "28081:8081"
+    volumes:
+      - "./node-B/data:/opt/nuts/data"
+      - "./node-B/nuts.yaml:/opt/nuts/nuts.yaml:ro"
+      - "../../tls-certs/nodeB-certificate.pem:/opt/nuts/certificate-and-key.pem:ro"
+      - "../../tls-certs/truststore.pem:/opt/nuts/truststore.pem:ro"
+    healthcheck:
+      interval: 1s # Make test run quicker by checking health status more often
+  nodeB:
+    image: nginx:1.25.1
+    expose:
+      - 5555
+    volumes:
+      - "../../shared_config/nodeB-grpc-nginx.conf:/etc/nginx/conf.d/nuts-grpc.conf:ro"
+      - "../../shared_config/nodeB-http-nginx.conf:/etc/nginx/conf.d/nuts-http.conf:ro"
+      - "../../tls-certs/nodeB-certificate.pem:/etc/nginx/ssl/server.pem:ro"
+      - "../../tls-certs/nodeB-certificate.pem:/etc/nginx/ssl/key.pem:ro"
+      - "../../tls-certs/truststore.pem:/etc/nginx/ssl/truststore.pem:ro"
+    depends_on:
+      - nodeB-backend

e2e-tests/openid4vci/offer-retry/node-A/nuts.yaml

@@ -0,0 +1,41 @@
+url: https://nodeA
+verbosity: debug
+strictmode: true
+internalratelimiter: false
+datadir: /opt/nuts/data
+http:
+  internal:
+    address: :8081
+  client:
+    # Docker auto-assigns the compose network an arbitrary private subnet, so permit all RFC1918
+    # ranges for the strict-mode SSRF guard. Narrow-allowlist precision is covered by unit tests.
+    allowedinternalcidrs:
+      - 10.0.0.0/8
+      - 172.16.0.0/12
+      - 192.168.0.0/16
+auth:
+  contractvalidators:
+    - dummy
+  irma:
+    autoupdateschemas: false
+crypto:
+  storage: fs
+goldenhammer:
+  interval: 1s
+  enabled: true
+vcr:
+  openid4vci:
+    enabled: true
+tls:
+  truststorefile: /opt/nuts/truststore.pem
+  certfile: /opt/nuts/certificate-and-key.pem
+  certkeyfile: /opt/nuts/certificate-and-key.pem
+  offload: incoming
+  certheader: X-SSL-CERT
+network:
+  grpcaddr:	:5555
+  v2:
+    gossipinterval: 500
+storage:
+  sql:
+    connection: "sqlite:file:/opt/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)"

e2e-tests/openid4vci/offer-retry/node-B/nuts.yaml

@@ -0,0 +1,41 @@
+url: https://nodeB
+verbosity: debug
+strictmode: true
+internalratelimiter: false
+datadir: /opt/nuts/data
+http:
+  internal:
+    address: :8081
+  client:
+    # Docker auto-assigns the compose network an arbitrary private subnet, so permit all RFC1918
+    # ranges for the strict-mode SSRF guard. Narrow-allowlist precision is covered by unit tests.
+    allowedinternalcidrs:
+      - 10.0.0.0/8
+      - 172.16.0.0/12
+      - 192.168.0.0/16
+auth:
+  contractvalidators:
+    - dummy
+  irma:
+    autoupdateschemas: false
+crypto:
+  storage: fs
+goldenhammer:
+  interval: 1s
+  enabled: true
+vcr:
+  openid4vci:
+    enabled: true
+tls:
+  truststorefile: /opt/nuts/truststore.pem
+  certfile: /opt/nuts/certificate-and-key.pem
+  certkeyfile: /opt/nuts/certificate-and-key.pem
+  offload: incoming
+  certheader: X-SSL-CERT
+network:
+  grpcaddr:	:5555
+  v2:
+    gossipinterval: 450
+storage:
+  sql:
+    connection: "sqlite:file:/opt/nuts/data/sqlite.db?_pragma=foreign_keys(1)&journal_mode(WAL)"

e2e-tests/openid4vci/offer-retry/run-test.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+USER=$UID
+
+set -e
+
+source ../../util.sh
+
+echo "------------------------------------"
+echo "Cleaning up running Docker containers and volumes, and key material..."
+echo "------------------------------------"
+# Empty node DIDs to avoid warning in Docker logs
+export NODEA_DID=
+export NODEB_DID=
+export BOOTSTRAP_NODES=nodeA:5555
+docker compose down
+docker compose rm -f -v
+rm -rf ./node-*/data
+
+echo "------------------------------------"
+echo "Starting Docker containers..."
+echo "------------------------------------"
+# 'data' dirs will be created with root owner by docker if they do not exist.
+# This creates permission issues on CI, since we manually delete the network/connections.db file.
+mkdir -p ./node-A/data/network ./node-B/data/network
+docker compose up --wait
+
+echo "------------------------------------"
+echo "Creating NodeDIDs, waiting for Golden Hammer to register base URLs..."
+echo "------------------------------------"
+export NODEA_DID=$(setupNode "http://localhost:18081" "nodeA:5555")
+printf "NodeDID for node A: %s\n" "$NODEA_DID"
+waitForTXCount "NodeB" "http://localhost:28081/status/diagnostics" 3 10 # 2 for setupNode, 1 for GoldenHammer
+export NODEB_DID=$(setupNode "http://localhost:28081" "nodeB:5555")
+printf "NodeDID for node B: %s\n" "$NODEB_DID"
+waitForTXCount "NodeA" "http://localhost:18081/status/diagnostics" 6 10 # 2 for setupNode, 1 for GoldenHammer
+
+echo "------------------------------------"
+echo "Restarting with NodeDID set..."
+echo "------------------------------------"
+# Start without bootstrap node, to enforce authenticated, discovered connections
+export BOOTSTRAP_NODES=
+docker compose exec nodeA-backend rm -f /opt/nuts/data/network/connections.db
+docker compose exec nodeB-backend rm -f /opt/nuts/data/network/connections.db
+docker compose stop
+docker compose up --wait
+
+echo "------------------------------------"
+echo "Recording node A's transaction count baseline..."
+echo "------------------------------------"
+txCountBefore=$(readDiagnostic "http://localhost:18081" "transaction_count")
+printf "Node A transaction_count before issuance: %s\n" "$txCountBefore"
+
+echo "------------------------------------"
+echo "Stopping node B, to simulate it being (temporarily) unreachable..."
+echo "------------------------------------"
+docker compose stop nodeB-backend nodeB
+
+echo "------------------------------------"
+echo "Issuing a credential while node B is down..."
+echo "------------------------------------"
+# The initial synchronous OpenID4VCI push fails (node B is unreachable), but issuing still succeeds
+# immediately: the offer is queued for background retry instead of failing the request.
+vcNodeA=$(createAuthCredential "http://localhost:18081" "$NODEA_DID" "$NODEB_DID")
+printf "VC issued by node A (queued for retry): %s\n" "$vcNodeA"
+if [ -z "$vcNodeA" ] || [ "$vcNodeA" == "null" ]; then
+  echo "FAILED: issuing the credential while node B was down should still succeed immediately (queued for retry)"
+  exitWithDockerLogs 1
+fi
+
+echo "------------------------------------"
+echo "Bringing node B back up..."
+echo "------------------------------------"
+docker compose start nodeB-backend nodeB
+
+echo "------------------------------------"
+echo "Waiting for the queued credential to be delivered automatically..."
+echo "------------------------------------"
+# A longer, dedicated wait: node B needs to fully restart (migrations, etc.) *and* node A's
+# background retry needs to fire again, on top of the fixed budget waitForDiagnostic gives elsewhere.
+RETRY_TIMEOUT=60
+retry=0
+delivered=false
+while [ $retry -lt $RETRY_TIMEOUT ]; do
+  RESPONSE=$(curl -s "http://localhost:28081/status/diagnostics")
+  if echo $RESPONSE | grep -q "credential_count: 1"; then
+    delivered=true
+    break
+  fi
+  printf "."
+  sleep 1
+  retry=$[$retry+1]
+done
+echo ""
+if [ $delivered == false ]; then
+  echo "FAILED: credential was not delivered to node B within ${RETRY_TIMEOUT}s of it coming back up"
+  exitWithDockerLogs 1
+fi
+
+waitForDiagnostic "nodeA-backend" issued_credentials_count 1
+
+echo "------------------------------------"
+echo "Verifying delivery went over OpenID4VCI, not the Nuts network (DAG) fallback..."
+echo "------------------------------------"
+# The offer was only ever retried over OpenID4VCI (never published to the DAG, see offer_queue.go), so node
+# A's transaction count must be unchanged: a gRPC/DAG-delivered credential would have added a transaction.
+txCountAfter=$(readDiagnostic "http://localhost:18081" "transaction_count")
+printf "Node A transaction_count after delivery: %s\n" "$txCountAfter"
+if [ "$txCountAfter" != "$txCountBefore" ]; then
+  echo "FAILED: node A's transaction count changed ($txCountBefore -> $txCountAfter); credential appears to have been published to the DAG instead of delivered via OpenID4VCI retry"
+  exitWithDockerLogs 1
+fi
+
+# Now the credential should be present on both nodeA and nodeB
+echo $(readCredential "http://localhost:18081" $vcNodeA)
+echo $(readCredential "http://localhost:28081" $vcNodeA)
+
+echo "------------------------------------"
+echo "Stopping Docker containers..."
+echo "------------------------------------"
+docker compose stop

e2e-tests/openid4vci/run-tests.sh

@@ -16,3 +16,10 @@ echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
 pushd network-issuance
 ./run-test.sh
 popd
+
+echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+echo "!! Running test: Offer Retry      !!"
+echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+pushd offer-retry
+./run-test.sh
+popd

vcr/issuer/interface.go

@@ -53,6 +53,12 @@ type Issuer interface {
 	// GetRevocation returns a revocation for a credential ID.
 	// Returns nil when no revocation is found.
 	GetRevocation(id ssi.URI) (*credential.Revocation, error)
+	// Start resumes retrying any not-yet-delivered OpenID4VCI credential offers persisted from a
+	// previous run. Safe to call even if no retry queue is configured.
+	Start() error
+	// Shutdown stops any in-flight OpenID4VCI offer retries. Persisted, not-yet-finished offers are
+	// resumed by the next Start().
+	Shutdown() error
 	CredentialSearcher
 }