Cloudflare Worker

{
  "$schema": "node_modules/wrangler/config-schema.json",
  "name": "shruwd-log-forwarder",
  "main": "src/index.js",
  "compatibility_date": "2026-09-01",

  // The route must match your canonical host ONLY. If customer custom domains
  // resolve through this same zone, this pattern must not catch them.
  "routes": [
    { "pattern": "example.com/*", "zone_name": "example.com" }
  ],

  "vars": {
    "CANONICAL_HOST": "example.com",

    // Comma-separated path prefixes. "/" is the root page only;
    // "/blog/" is everything underneath it.
    "PATH_ALLOWLIST": "/,/pricing,/features,/blog/,/docs/",

    "SHRUWD_INGEST_URL": "YOUR_ENDPOINT"
  }
}

// Forwards AI-crawler hits to Shruwd. Fails open: whatever happens in here,
// your site's response is returned unchanged.

// A broad prefilter, not the bot list. Identification proper happens on
// Shruwd's side against a catalogue that is kept current, so this only has to
// be loose enough not to drop something that catalogue would have caught.
const BOT_UA_PREFILTER =
  /bot|crawl|spider|gpt|claude|perplexity|bytespider|applebot|ccbot|meta-external|google|extended/i

// Static assets are never the page a crawler judges you on. .txt and .xml stay,
// because robots.txt and sitemaps are interesting.
const ASSET_PATH =
  /\.(?:js|mjs|cjs|css|map|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|mp4|webm|mp3|wav|pdf|zip|gz|json|wasm)$/i

function parseAllowlist(value) {
  return String(value ?? '/').split(',').map((s) => s.trim()).filter(Boolean)
}

// Cheapest checks first — this runs on every request to the zone.
function decide({ method, host, path, userAgent }, { canonicalHost, allowlist }) {
  if (method !== 'GET' && method !== 'HEAD') return false
  if (host.toLowerCase() !== canonicalHost.toLowerCase()) return false
  if (ASSET_PATH.test(path)) return false
  if (!allowlist.some((p) => (p === '/' ? path === '/' : path === p || path.startsWith(p)))) return false
  return BOT_UA_PREFILTER.test(userAgent)
}

let warned = false

function maybeReport(request, response, env, ctx) {
  const url = new URL(request.url)
  const userAgent = request.headers.get('user-agent') ?? ''

  const report = decide(
    { method: request.method, host: url.hostname, path: url.pathname, userAgent },
    { canonicalHost: env.CANONICAL_HOST, allowlist: parseAllowlist(env.PATH_ALLOWLIST) },
  )
  if (!report) return

  // Set by the edge; a client cannot forge it. Inside a Worker,
  // x-forwarded-for can be forged, so it is not used as a fallback.
  const clientIp = request.headers.get('cf-connecting-ip')
  if (!clientIp) return

  // The query string is never sent: it carries ids on some routes, and a
  // per-page crawl count needs none of it.
  const line = {
    timestamp: new Date().toISOString(),
    host: url.hostname,
    path: url.pathname,
    user_agent: userAgent,
    status: response.status,
    client_ip: clientIp,
    method: request.method,
    bytes: Number(response.headers.get('content-length') ?? 0) || 0,
  }

  // One request per hit, deliberately. A Worker isolate is ephemeral and
  // per-colo, so a buffer would be lost on eviction and split across colos.
  // Volume is low because the prefilter already ran.
  ctx.waitUntil(
    fetch(env.SHRUWD_INGEST_URL, {
      method: 'POST',
      headers: {
        'x-shruwd-ingest-token': env.SHRUWD_INGEST_TOKEN,
        'content-type': 'application/x-ndjson',
      },
      body: JSON.stringify(line) + '\n',
    })
      .then((res) => {
        if (!res.ok && !warned) {
          warned = true
          console.warn(`shruwd forwarder: ingest answered ${res.status}; check the token and URL`)
        }
      })
      .catch((error) => {
        if (!warned) {
          warned = true
          console.warn('shruwd forwarder: ingest unreachable', error?.message ?? error)
        }
      }),
  )
}

export default {
  async fetch(request, env, ctx) {
    // Never inside the try. The only unconditional path is returning your
    // origin's response — a logging pipeline that can break the site it
    // observes is worse than one that loses events.
    const response = await fetch(request)
    try {
      maybeReport(request, response, env, ctx)
    } catch {
      // deliberately silent
    }
    return response
  },
}

npx wrangler secret put SHRUWD_INGEST_TOKEN
npx wrangler deploy

curl -s -o /dev/null "https://example.com/pricing" \
  -H "User-Agent: Mozilla/5.0 (compatible; GPTBot/1.2)"

Invoke-WebRequest https://example.com/pricing `
  -Headers @{ "User-Agent" = "Mozilla/5.0 (compatible; GPTBot/1.2)" } | Out-Null