/v1/id/_:id/expanded endpoint

__tests__/core_provider_contract.test.js

@@ -199,11 +199,15 @@ const requiredResponseCodes = {
   // 409 is reachable via slug conflict (utils.createExpressError maps code 11000 → 409).
   'PATCH /api/release/{id}': ['200', '400', '401', '403', '404', '409'],
   'GET /id/{id}': ['200', '404'],
+  // 200/400/404 are asserted in routes/__tests__/id.test.js.  413 and 415 come from Express handlers.
+  'GET /id/{id}/expanded': ['200', '404'],
+  'POST /id/{id}/expanded': ['200', '400', '404', '413', '415'],
   'GET /since/{id}': ['200', '404'],
   'GET /history/{id}': ['200', '404'],
   // HEAD parity tests in routes/__tests__/{id,since,history,query}.test.js assert 404 on miss;
   // enforce that the contract declares the same so drift on either side is caught.
   'HEAD /id/{id}': ['200', '404'],
+  'HEAD /id/{id}/expanded': ['200', '404'],
   'HEAD /since/{id}': ['200', '404'],
   'HEAD /history/{id}': ['200', '404'],
   'HEAD /api/query': ['200', '404'],

__tests__/routes_mounted.test.js

@@ -26,7 +26,8 @@ const mountedApiRoutes = [
   { name: '/v1/api/delete/{id}', method: 'delete', path: '/v1/api/delete/test-mounted-id' },
   { name: '/v1/api/release/{id}', method: 'patch', path: '/v1/api/release/test-mounted-id' },
   { name: '/v1/api/search', method: 'post', path: '/v1/api/search', headers: { 'Content-Type': 'text/plain' }, body: 'mounted search' },
-  { name: '/v1/api/search/phrase', method: 'post', path: '/v1/api/search/phrase', headers: { 'Content-Type': 'text/plain' }, body: 'mounted phrase search' }
+  { name: '/v1/api/search/phrase', method: 'post', path: '/v1/api/search/phrase', headers: { 'Content-Type': 'text/plain' }, body: 'mounted phrase search' },
+  { name: '/v1/id/{_id}/expanded', method: 'put', path: '/v1/id/test-mounted-id/expanded' }
 ]
 
 describe('Mounted route surface', () => {

__tests__/utils.test.js

@@ -9,9 +9,10 @@ import {
   parseDocumentID,
   _contextid,
   idNegotiation,
-  getPagination
+  getPagination,
+  findLeafAnnotationsFor
 } from '../controllers/utils.js'
-import { db, resetMocks } from '../database/index.js'
+import { db, resetMocks, createCursor } from '../database/index.js'
 
 describe('utils.js auth gates', () => {
   it('isDeleted returns true only for objects with __deleted', () => {
@@ -49,6 +50,63 @@ describe('utils.js configureRerumOptions', () => {
     )
     assert.strictEqual(result.__rerum.generatedBy, 'https://store.rerum.io/v1/id/legitimate-agent')
   })
+
+  const AGENT = 'https://store.rerum.io/v1/id/legitimate-agent'
+  const RECEIVED_ID = 'https://store.rerum.io/v1/id/received-id'
+  const FORGED = {
+    generatedBy: 'https://attacker.example/forged',
+    isReleased: '2020-01-01T00:00:00.000',
+    isOverwritten: '2020-01-01T00:00:00.000',
+    history: { prime: 'https://store.rerum.io/v1/id/forged-prime', previous: 'https://store.rerum.io/v1/id/forged-previous', next: ['https://store.rerum.io/v1/id/forged-next'] },
+    releases: { previous: 'https://store.rerum.io/v1/id/forged-release', next: [], replaces: '' }
+  }
+
+  it('ignores a client-supplied __rerum when minting a new object', () => {
+    const created = utils.configureRerumOptions(AGENT, { '@id': RECEIVED_ID, __rerum: structuredClone(FORGED) }, false, false)
+    assert.strictEqual(created.__rerum.history.prime, 'root')
+    assert.strictEqual(created.__rerum.history.previous, '')
+    assert.deepStrictEqual(created.__rerum.history.next, [])
+    assert.strictEqual(created.__rerum.releases.previous, '')
+    // isReleased gates release and overwrite, isOverwritten is the optimistic locking token.
+    assert.strictEqual(created.__rerum.generatedBy, AGENT, 'attribution cannot be forged')
+    assert.strictEqual(created.__rerum.isReleased, '', 'a client cannot mint a pre-released object')
+    assert.strictEqual(created.__rerum.isOverwritten, '', 'a client cannot mint a locking token')
+
+    // An external object imported through an update is also a root, but it remembers its external self.
+    const imported = utils.configureRerumOptions(AGENT, { '@id': 'https://elsewhere.example.org/thing', __rerum: structuredClone(FORGED) }, false, true)
+    assert.strictEqual(imported.__rerum.history.prime, 'root')
+    assert.strictEqual(imported.__rerum.history.previous, 'https://elsewhere.example.org/thing')
+    assert.strictEqual(imported.__rerum.releases.previous, '')
+  })
+
+  it('carries the version and release chain forward when updating an existing object', () => {
+    const fromRoot = utils.configureRerumOptions(
+      AGENT,
+      { '@id': RECEIVED_ID, __rerum: { history: { prime: 'root', previous: '', next: [] } } },
+      true,
+      false
+    )
+    assert.strictEqual(fromRoot.__rerum.history.prime, RECEIVED_ID, 'the root object cannot pass "root" on as the prime')
+    assert.strictEqual(fromRoot.__rerum.history.previous, RECEIVED_ID)
+
+    const PRIME = 'https://store.rerum.io/v1/id/prime-id'
+    const RELEASE = 'https://store.rerum.io/v1/id/released-id'
+    const fromDescendant = utils.configureRerumOptions(
+      AGENT,
+      {
+        '@id': RECEIVED_ID,
+        __rerum: {
+          history: { prime: PRIME, previous: 'https://store.rerum.io/v1/id/older-id', next: [] },
+          releases: { previous: RELEASE, next: [], replaces: '' }
+        }
+      },
+      true,
+      false
+    )
+    assert.strictEqual(fromDescendant.__rerum.history.prime, PRIME, 'an object that knows its prime passes it on')
+    assert.strictEqual(fromDescendant.__rerum.history.previous, RECEIVED_ID)
+    assert.strictEqual(fromDescendant.__rerum.releases.previous, RELEASE)
+  })
 })
 
 describe('controllers/utils.js generateSlugId', () => {
@@ -212,6 +270,12 @@ describe('controllers/utils.js _contextid', () => {
       _contextid(['http://example.com/other', 'http://iiif.io/api/presentation/3/context.json']),
       true
     )
+    // An inline term definition object, or any other non-string member, names no context.
+    assert.strictEqual(
+      _contextid([{ '@vocab': 'http://example.org/terms#' }, 'http://www.w3.org/ns/anno.jsonld']),
+      true
+    )
+    assert.strictEqual(_contextid([{ '@vocab': 'http://example.org/terms#' }]), false)
   })
 
   it('returns false for non-string, non-array input', () => {
@@ -288,3 +352,73 @@ describe('controllers/utils.js getPagination', () => {
     assert.ok(result.limit < huge, `limit should be clamped below ${huge}`)
   })
 })
+
+describe('controllers/utils.js findLeafAnnotationsFor', () => {
+  const ENTITY_URI = 'https://store.rerum.io/v1/id/entity-id'
+  const SLUG_URI = 'https://store.rerum.io/v1/id/entity-slug'
+  const TARGET_KEYS = ['target', 'target.@id', 'target.id', 'target.source', 'target.source.@id', 'target.source.id']
+
+  let capturedQuery
+
+  /**
+   * Point db.find() at a cursor over the given documents and record the filter it was called with.
+   *
+   * @param docs The Annotation documents the cursor will yield.
+   */
+  function armFind(docs = []) {
+    resetMocks()
+    capturedQuery = undefined
+    db.find.mockImplementationOnce(query => {
+      capturedQuery = query
+      return createCursor(docs)
+    })
+  }
+
+  // $and[0] holds the target conditions, $and[1] the Annotation type conditions.
+  const targetConditions = () => capturedQuery.$and[0].$or
+
+  it('constrains the query to the leaf versions, every target key, and every type spelling', async () => {
+    armFind()
+
+    await findLeafAnnotationsFor([ENTITY_URI, SLUG_URI, ENTITY_URI, undefined, ''])
+
+    assert.deepStrictEqual(capturedQuery['__rerum.history.next'], { $exists: true, $size: 0 })
+    for (const targetKey of TARGET_KEYS) {
+      const values = targetConditions()
+        .filter(condition => Object.hasOwn(condition, targetKey))
+        .map(condition => condition[targetKey])
+      assert.strictEqual(values.length, 8, `${targetKey}: two URIs, each in two schemes and as a fragment`)
+      assert.ok(values.includes(ENTITY_URI) && values.includes(ENTITY_URI.replace(/^https/, 'http')))
+      assert.ok(values.includes(SLUG_URI), 'every URI the entity answers to is targeted')
+      const patterns = values.filter(value => value instanceof RegExp)
+      assert.ok(patterns.some(pattern => pattern.test(`${ENTITY_URI}#xywh=0,0,100,100`)), 'a fragment of the URI is a match')
+      assert.ok(
+        patterns.every(pattern => !pattern.test('https://storeXrerum.io/v1/id/entity-id#xywh=0,0,100,100')),
+        'the URI is escaped, so its dots are not wildcards'
+      )
+    }
+    assert.deepStrictEqual(capturedQuery.$and[1].$or, [
+      { type: 'Annotation' },
+      { type: 'oa:Annotation' },
+      { type: 'http://www.w3.org/ns/oa#Annotation' },
+      { type: 'https://www.w3.org/ns/oa#Annotation' },
+      { '@type': 'Annotation' },
+      { '@type': 'oa:Annotation' },
+      { '@type': 'http://www.w3.org/ns/oa#Annotation' },
+      { '@type': 'https://www.w3.org/ns/oa#Annotation' }
+    ])
+
+    armFind()
+    await findLeafAnnotationsFor('bare-slug')
+    assert.strictEqual(targetConditions().length, TARGET_KEYS.length, 'a non-URI target has no scheme or fragment to anticipate')
+  })
+
+  it('gathers nothing rather than querying on an empty $or when there is no target', async () => {
+    // An empty '$or' is a MongoDB error, not an empty result, so the query is never sent.
+    armFind([{ _id: 'anno001', type: 'Annotation' }])
+
+    assert.deepStrictEqual(await findLeafAnnotationsFor([undefined, '', null]), [])
+    assert.deepStrictEqual(await findLeafAnnotationsFor(undefined), [])
+    assert.strictEqual(capturedQuery, undefined, 'nothing to target is nothing to gather')
+  })
+})

controllers/crud.js

@@ -6,7 +6,7 @@
  */
 import { newID, isValidID, db } from '../database/index.js'
 import utils from '../utils.js'
-import { _contextid, idNegotiation, getPagination, generateSlugId, ObjectID, getAgentClaim, parseDocumentID } from './utils.js'
+import { _contextid, idNegotiation, getPagination, generateSlugId, ObjectID, getAgentClaim, findLeafAnnotationsFor, PROTECTED_EXPANSION_KEYS } from './utils.js'
 
 /**
  * Create a new Linked Open Data object in RERUM v1.
@@ -127,8 +127,179 @@ const id = async function (req, res, next) {
     }
 }
 
+/**
+ * The expand job always constrains the Annotations it gathers to the leaf versions, to the
+ * Annotation types, and to the entity in the request URI.  A client cannot influence those, so
+ * these keys are dropped from a supplied filter body by exact name or dotted prefix.
+ */
+const RESERVED_FILTER_KEYS = ["target", "type", "@type", "__rerum.history"]
+
+/**
+ * The Annotation body types whose value is kept whole rather than read as a single assertion.
+ */
+const TEXTUAL_BODY_TYPES = new Set([
+    "TextualBody", "oa:TextualBody",
+    "http://www.w3.org/ns/oa#TextualBody", "https://www.w3.org/ns/oa#TextualBody"
+])
+
+/**
+ * Reduce a supplied POST body to the literal MongoDB filter keys the expand job will honor.
+ * @param supplied The parsed JSON request body.
+ * @return An object of filter keys, minus the ones this endpoint owns.
+ */
+function sanitizeExpansionFilters(supplied) {
+    const filters = {}
+    for (const [key, value] of Object.entries(supplied)) {
+        if (RESERVED_FILTER_KEYS.some(reserved => key === reserved || key.startsWith(`${reserved}.`))) continue
+        filters[key] = value
+    }
+    return filters
+}
+
+/**
+ * The [key, value] assertions an Annotation makes about the entity it targets.
+ * Only 'body' and 'bodyValue' are read -- an Annotation carrying neither is ignored, and no other
+ * property of the Annotation can leak onto the entity.
+ *
+ * Anticipates the likely Annotation body formats
+ *   - bodyValue: 'text'                                  the W3C shorthand, which has no key of its own
+ *   - body: {'key': 'value'}                             a single assertion
+ *   - body: {'key': {...}}                               a single assertion, value kept as-is
+ *   - body: {'type':'TextualBody', 'value': 'text', ...} kept whole so 'format' and 'language' survive
+ *   - body: {'@type':'oa:TextualBody', ...}              the 'oa:' prefixed spelling of the same W3C class
+ *   - body: {'type':['TextualBody'], ...}                the same body, type serialized as a JSON-LD Array
+ *   - body: [{'key': 'value'}]                           one body, serialized as a JSON-LD Array
+ *
+ * @param anno An Annotation document.
+ * @return An Array of [key, value] pairs to merge onto the entity.
+ */
+function assertionsFrom(anno) {
+    const assertions = []
+    if (typeof anno.bodyValue === "string") assertions.push(["bodyValue", anno.bodyValue])
+    // In JSON-LD a one-element Array and the bare value are the same body, so unwrap it first.  The
+    // check below is about how many bodies an Annotation carries, not how they were serialized.
+    const body = Array.isArray(anno.body) && anno.body.length === 1 ? anno.body[0] : anno.body
+    // Skip Annotations carrying multiple bodies, and string bodies that are an IRI referencing an
+    // external resource with no embedded value to expand with.
+    if (Array.isArray(body) || !body || typeof body !== "object") return assertions
+    const bodyType = body.type ?? body["@type"]
+    const bodyTypes = Array.isArray(bodyType) ? bodyType : [bodyType]
+    if (bodyTypes.some(t => TEXTUAL_BODY_TYPES.has(t))) {
+        assertions.push(["bodyValue", body])
+        return assertions
+    }
+    const keys = Object.keys(body)
+    // Any other multi-key body is structural rather than assertional and cannot be attributed to a
+    // single entity property.  This is what skips the Choice, Composite, and List multiplicity constructs.
+    if (keys.length !== 1) return assertions
+    assertions.push([keys[0], body[keys[0]]])
+    return assertions
+}
+
+/**
+ * Merge the assertions of the gathered Annotations onto a copy of the entity, as raw values.
+ * When more than one current Annotation asserts the same key, or the entity already carries it,
+ * the values collect into an Array, the record's own value first.
+ * @param primitiveEntity The unexpanded entity.
+ * @param annoAssertions An Array holding the [key, value] assertions read from each Annotation.
+ * @return A new, expanded entity object.
+ */
+function applyExpansionAnnotations(primitiveEntity, annoAssertions) {
+    const expandedEntity = structuredClone(primitiveEntity)
+    const rerumProp = expandedEntity.__rerum
+    delete expandedEntity.__rerum
+    for (const assertions of annoAssertions) {
+        for (const [key, value] of assertions) {
+            if (PROTECTED_EXPANSION_KEYS.has(key)) continue
+            if (!Object.hasOwn(expandedEntity, key)) {
+                expandedEntity[key] = value
+                continue
+            }
+            const existing = Array.isArray(expandedEntity[key]) ? expandedEntity[key] : [expandedEntity[key]]
+            const contributed = Array.isArray(value) ? value : [value]
+            expandedEntity[key] = [...existing, ...contributed]
+        }
+    }
+    if (rerumProp !== undefined) expandedEntity.__rerum = rerumProp
+    return expandedEntity
+}
+
+/**
+ * Query the MongoDB for the object with the _id or __rerum.slug provided in the request URL, then
+ * merge in the assertions of all the current leaf Annotations targeting it.
+ *
+ * GET recognizes the '?generator=' and '?creator=' convenience parameters only.
+ * POST reads literal MongoDB filter keys from the JSON body and ignores URL parameters as filters.
+ * Neither method pages.  A client asks once and receives the entity assembled.
+ * */
+const idExpanded = async function (req, res, next) {
+    res.set("Content-Type", "application/json; charset=utf-8")
+    const requestedId = req.params["_id"]
+    const isPost = req.method === "POST"
+    let filters = {}
+    if (isPost) {
+        // Express leaves the body undefined when a POST supplies none.  That is an unfiltered expand.
+        const supplied = req.body ?? {}
+        if (typeof supplied !== "object" || Array.isArray(supplied)) {
+            const err = {
+                "message": "The /expanded request body must be a JSON object of filter properties.",
+                "status": 400
+            }
+            return next(utils.createExpressError(err))
+        }
+        filters = sanitizeExpansionFilters(supplied)
+    }
+    else {
+        // Repeated query parameters arrive as an Array, which is not a filter value we will apply.
+        if (typeof req.query.generator === "string" && req.query.generator) filters["__rerum.generatedBy"] = req.query.generator
+        if (typeof req.query.creator === "string" && req.query.creator) filters.creator = req.query.creator
+    }
+    try {
+        const match = await db.findOne({"$or": [{"_id": requestedId}, {"__rerum.slug": requestedId}]})
+        if (!match) {
+            const err = {
+                "message": `No RERUM object with id '${requestedId}'`,
+                "status": 404
+            }
+            return next(utils.createExpressError(err))
+        }
+        const deleted = utils.isDeleted(match)
+        const targetId = match["@id"] ?? match.id
+        // an Annotation may target an entity by its Slug instead of by its '@id'.
+        const slug = match.__rerum?.slug
+        const lastSlash = targetId?.lastIndexOf("/") ?? -1
+        const slugTargetId = slug && lastSlash !== -1 ? targetId.slice(0, lastSlash + 1) + slug : undefined
+        // Read off the record while it is still whole.  idNegotiation() below alters it in place.
+        const currentVersion = match.__rerum?.isOverwritten ?? ""
+        const annos = deleted ? [] : await findLeafAnnotationsFor([targetId, slugTargetId], filters)
+        // Every leaf Annotation matching the filter is gathered.  This is the count.
+        res.set('Annotations-Gathered', String(annos.length))
+        // How many of the Annotations contribute an assertion.  May be less than annotations gathered.
+        // The count reports the Annotations that actually alter the entity.
+        const merged = annos
+            .map(anno => assertionsFrom(anno).filter(([key]) => !PROTECTED_EXPANSION_KEYS.has(key)))
+            .filter(assertions => assertions.length > 0)
+        res.set('Annotations-Merged', String(merged.length))
+        // Negotiate first, so identity is settled from the record's own '@context' before anything is merged.
+        const negotiated = idNegotiation(match)
+        const identity = _contextid(negotiated["@context"]) ? negotiated.id : negotiated["@id"]
+        const expanded = deleted ? negotiated : applyExpansionAnnotations(negotiated, merged)
+        // Same browser-caching policy as GET /v1/id/:_id so this stable URI is cached (24h).
+        if (!isPost) res.set("Cache-Control", "max-age=86400, must-revalidate")
+        // Headers describe the stored record, so they match GET /v1/id/:_id for the same record.
+        res.set(utils.configureWebAnnoHeadersFor(negotiated))
+        // Include current version for optimistic locking
+        res.set('Current-Overwritten-Version', currentVersion)
+        res.location(identity)
+        res.json(expanded)
+    } catch (error) {
+        return next(utils.createExpressError(error))
+    }
+}
+
 export {
     create,
     query,
-    id
+    id,
+    idExpanded
 }