Reference: sync manifest

# The manifest of the last run is in manifest-previous.json.
curl -s https://ddm.example.com/api/ext/v1/sync \
  -H "Authorization: Bearer $DDM_API_KEY" > manifest.json

# Cut off above 50,000 documents? Then compare nothing.
if [ "$(jq .complete manifest.json)" != "true" ]; then
  echo "Manifest incomplete – not deleting anything" >&2; exit 1
fi

# The ids that were there last time and are not any more.
jq -r '.documents[].id' manifest-previous.json | sort > previous.txt
jq -r '.documents[].id' manifest.json | sort > current.txt
comm -23 previous.txt current.txt > gone.txt

while read -r id; do
  echo "gone: $id"   # remove it from your system here
done < gone.txt

mv manifest.json manifest-previous.json

import { readFile, writeFile } from 'node:fs/promises'

const response = await fetch('https://ddm.example.com/api/ext/v1/sync', {
  headers: { Authorization: `Bearer ${process.env.DDM_API_KEY}` }
})
const manifest = await response.json()

// Cut off above 50,000 documents? Then compare nothing.
if (!manifest.complete) throw new Error('Manifest incomplete – not deleting anything')

const previous = JSON.parse(await readFile('manifest-previous.json', 'utf8').catch(() => '{"documents":[]}'))
const present = new Set(manifest.documents.map(document => document.id))
const gone = previous.documents.filter(document => !present.has(document.id))

for (const document of gone) {
  // externalRef is set for documents that came from a push – often the id your system knows.
  console.log('gone:', document.id, document.externalRef ?? '')
  // remove it from your system here
}

await writeFile('manifest-previous.json', JSON.stringify(manifest))

import json, os, requests

manifest = requests.get(
    "https://ddm.example.com/api/ext/v1/sync",
    headers={"Authorization": f"Bearer {os.environ['DDM_API_KEY']}"},
).json()

# Cut off above 50,000 documents? Then compare nothing.
if not manifest["complete"]:
    raise SystemExit("Manifest incomplete – not deleting anything")

try:
    with open("manifest-previous.json") as file:
        previous = json.load(file)
except FileNotFoundError:
    previous = {"documents": []}

present = {document["id"] for document in manifest["documents"]}
gone = [document for document in previous["documents"] if document["id"] not in present]

for document in gone:
    print("gone:", document["id"], document.get("externalRef") or "")
    # remove it from your system here

with open("manifest-previous.json", "w") as file:
    json.dump(manifest, file)

$headers = @{ Authorization = "Bearer $env:DDM_API_KEY" }
$manifest = Invoke-RestMethod -Uri "https://ddm.example.com/api/ext/v1/sync" -Headers $headers

# Cut off above 50,000 documents? Then compare nothing.
if (-not $manifest.complete) { throw "Manifest incomplete – not deleting anything" }

$previous = if (Test-Path manifest-previous.json) { Get-Content manifest-previous.json -Raw | ConvertFrom-Json } else { @{ documents = @() } }
$present = [System.Collections.Generic.HashSet[string]]::new([string[]]$manifest.documents.id)

foreach ($document in $previous.documents) {
  if (-not $present.Contains($document.id)) {
    Write-Output "gone: $($document.id) $($document.externalRef)"
    # remove it from your system here
  }
}

$manifest | ConvertTo-Json -Depth 5 | Set-Content manifest-previous.json