[{"data":1,"prerenderedAt":696},["ShallowReactive",2],{"docs-navigation":3,"docs-\u002Fdocs\u002Fcrawler-logs\u002Fingest-not-arriving":213},[4],{"title":5,"path":6,"stem":7,"children":8,"page":30},"Docs","\u002Fdocs","docs",[9,31,56,98,118,138,163],{"title":10,"description":11,"group":11,"path":12,"stem":13,"children":14,"page":30},"Getting started",null,"\u002Fdocs\u002Fgetting-started","docs\u002F1.getting-started",[15,20,25],{"title":16,"path":17,"stem":18,"description":19,"group":11},"Overview","\u002Fdocs\u002Fgetting-started\u002Foverview","docs\u002F1.getting-started\u002F1.overview","What Shruwd does and what you get out of it.",{"title":21,"path":22,"stem":23,"description":24,"group":11},"Quickstart","\u002Fdocs\u002Fgetting-started\u002Fquickstart","docs\u002F1.getting-started\u002F2.quickstart","From an empty account to your first measurement.",{"title":26,"path":27,"stem":28,"description":29,"group":11},"Reading your results","\u002Fdocs\u002Fgetting-started\u002Freading-your-results","docs\u002F1.getting-started\u002F3.reading-your-results","What the four numbers mean, and when to trust them.",false,{"title":32,"description":11,"group":11,"path":33,"stem":34,"children":35,"page":30},"What you track","\u002Fdocs\u002Ftracking","docs\u002F2.tracking",[36,41,46,51],{"title":37,"path":38,"stem":39,"description":40,"group":11},"Brands","\u002Fdocs\u002Ftracking\u002Fbrands","docs\u002F2.tracking\u002F1.brands","Add, rename and archive the brands you measure.",{"title":42,"path":43,"stem":44,"description":45,"group":11},"Prompts","\u002Fdocs\u002Ftracking\u002Fprompts","docs\u002F2.tracking\u002F2.prompts","Choose and manage the questions Shruwd asks each engine.",{"title":47,"path":48,"stem":49,"description":50,"group":11},"Competitors","\u002Fdocs\u002Ftracking\u002Fcompetitors","docs\u002F2.tracking\u002F3.competitors","Track the brands you are compared against, and how they are matched.",{"title":52,"path":53,"stem":54,"description":55,"group":11},"Connecting Search Console","\u002Fdocs\u002Ftracking\u002Fconnecting-search-console","docs\u002F2.tracking\u002F4.connecting-search-console","Add per-page search demand so two diagnostics can run.",{"title":57,"description":11,"group":11,"path":58,"stem":59,"children":60,"page":30},"Crawler logs","\u002Fdocs\u002Fcrawler-logs","docs\u002F3.crawler-logs",[61,66,72,77,82,87,93],{"title":62,"path":63,"stem":64,"description":65,"group":11},"Choosing a log path","\u002Fdocs\u002Fcrawler-logs\u002Fchoosing-a-log-path","docs\u002F3.crawler-logs\u002F1.choosing-a-log-path","Pick the way of sending crawler logs that matches your hosting.",{"title":67,"path":68,"stem":69,"description":70,"group":71},"Platform log drains","\u002Fdocs\u002Fcrawler-logs\u002Fplatform-log-drains","docs\u002F3.crawler-logs\u002F2.platform-log-drains","Send logs from Vercel, Netlify, Fly and similar. No code.","Setup guides",{"title":73,"path":74,"stem":75,"description":76,"group":71},"Cloudflare Worker","\u002Fdocs\u002Fcrawler-logs\u002Fcloudflare-worker","docs\u002F3.crawler-logs\u002F3.cloudflare-worker","Forward crawler hits from behind Cloudflare, on any plan.",{"title":78,"path":79,"stem":80,"description":81,"group":71},"Cloudflare Logpush","\u002Fdocs\u002Fcrawler-logs\u002Fcloudflare-logpush","docs\u002F3.crawler-logs\u002F4.cloudflare-logpush","Send zone request logs straight to Shruwd. Enterprise zones, no code.",{"title":83,"path":84,"stem":85,"description":86,"group":71},"Server middleware","\u002Fdocs\u002Fcrawler-logs\u002Fserver-middleware","docs\u002F3.crawler-logs\u002F5.server-middleware","Forward crawler hits from a Nuxt or Nitro origin. Last resort.",{"title":88,"path":89,"stem":90,"description":91,"group":92},"Crawlers tracked","\u002Fdocs\u002Fcrawler-logs\u002Fcrawlers-tracked","docs\u002F3.crawler-logs\u002F6.crawlers-tracked","Which bots Shruwd recognises, and how a hit is verified.","Reference",{"title":94,"path":95,"stem":96,"description":97,"group":92},"Ingest not arriving","\u002Fdocs\u002Fcrawler-logs\u002Fingest-not-arriving","docs\u002F3.crawler-logs\u002F7.ingest-not-arriving","Diagnose a log drain that is not delivering.",{"title":99,"description":11,"group":11,"path":100,"stem":101,"children":102,"page":30},"Findings and fixes","\u002Fdocs\u002Ffindings","docs\u002F4.findings",[103,108,113],{"title":104,"path":105,"stem":106,"description":107,"group":11},"How findings work","\u002Fdocs\u002Ffindings\u002Fhow-findings-work","docs\u002F4.findings\u002F1.how-findings-work","What a finding is, how much to trust it, and how to prove your fix worked.",{"title":109,"path":110,"stem":111,"description":112,"group":11},"Rechecks","\u002Fdocs\u002Ffindings\u002Frechecks","docs\u002F4.findings\u002F2.rechecks","How Shruwd proves whether your fix moved the number.",{"title":114,"path":115,"stem":116,"description":117,"group":11},"Findings reference","\u002Fdocs\u002Ffindings\u002Ffindings-reference","docs\u002F4.findings\u002F3.findings-reference","Every diagnostic Shruwd runs, what triggers it and what to do.",{"title":119,"description":11,"group":11,"path":120,"stem":121,"children":122,"page":30},"Metrics","\u002Fdocs\u002Fmetrics","docs\u002F5.metrics",[123,128,133],{"title":124,"path":125,"stem":126,"description":127,"group":11},"Metric definitions","\u002Fdocs\u002Fmetrics\u002Fmetric-definitions","docs\u002F5.metrics\u002F1.metric-definitions","Exactly what each number counts, and when it is undefined.",{"title":129,"path":130,"stem":131,"description":132,"group":11},"Confidence and sample size","\u002Fdocs\u002Fmetrics\u002Fconfidence-and-sample-size","docs\u002F5.metrics\u002F2.confidence-and-sample-size","Why every number has a range, and why some are withheld.",{"title":134,"path":135,"stem":136,"description":137,"group":11},"When a number has moved","\u002Fdocs\u002Fmetrics\u002Fwhen-a-number-has-moved","docs\u002F5.metrics\u002F3.when-a-number-has-moved","The rule Shruwd uses before calling a change real.",{"title":139,"description":11,"group":11,"path":140,"stem":141,"children":142,"page":30},"Account","\u002Fdocs\u002Faccount","docs\u002F6.account",[143,148,153,158],{"title":144,"path":145,"stem":146,"description":147,"group":11},"Plans and limits","\u002Fdocs\u002Faccount\u002Fplans-and-limits","docs\u002F6.account\u002F1.plans-and-limits","What each plan includes and what happens when you reach a limit.",{"title":149,"path":150,"stem":151,"description":152,"group":11},"Team and roles","\u002Fdocs\u002Faccount\u002Fteam-and-roles","docs\u002F6.account\u002F2.team-and-roles","Invite people, set what they can change, and remove access.",{"title":154,"path":155,"stem":156,"description":157,"group":11},"Billing","\u002Fdocs\u002Faccount\u002Fbilling","docs\u002F6.account\u002F3.billing","Change plan, read your usage, and handle a failed payment.",{"title":159,"path":160,"stem":161,"description":162,"group":11},"Email notifications","\u002Fdocs\u002Faccount\u002Femail-notifications","docs\u002F6.account\u002F4.email-notifications","What Shruwd emails you, and how to turn it off.",{"title":164,"description":11,"group":11,"path":165,"stem":166,"children":167,"page":30},"Build on Shruwd","\u002Fdocs\u002Fapi","docs\u002F7.api",[168,173,178,182,187,192,197,203,208],{"title":169,"path":170,"stem":171,"description":172,"group":11},"API overview","\u002Fdocs\u002Fapi\u002Fapi-overview","docs\u002F7.api\u002F1.api-overview","Authentication, errors, limits and the rules every endpoint follows.",{"title":37,"path":174,"stem":175,"description":176,"group":177},"\u002Fdocs\u002Fapi\u002Fbrands","docs\u002F7.api\u002F2.brands","Create, read, rename and archive brands over the API.","Endpoints",{"title":42,"path":179,"stem":180,"description":181,"group":177},"\u002Fdocs\u002Fapi\u002Fprompts","docs\u002F7.api\u002F3.prompts","Add, edit and retire the questions asked every cycle.",{"title":183,"path":184,"stem":185,"description":186,"group":177},"Entities","\u002Fdocs\u002Fapi\u002Fentities","docs\u002F7.api\u002F4.entities","Manage the brand and its competitors, and the aliases that count.",{"title":188,"path":189,"stem":190,"description":191,"group":177},"Measurement","\u002Fdocs\u002Fapi\u002Fmeasurement","docs\u002F7.api\u002F5.measurement","Run cycles and read visibility, crawlers and individual answers.",{"title":193,"path":194,"stem":195,"description":196,"group":177},"Findings","\u002Fdocs\u002Fapi\u002Ffindings","docs\u002F7.api\u002F6.findings","Read diagnosed findings and move them through their lifecycle.",{"title":198,"path":199,"stem":200,"description":201,"group":202},"TypeScript SDK","\u002Fdocs\u002Fapi\u002Ftypescript-sdk","docs\u002F7.api\u002F7.typescript-sdk","A typed client for the Shruwd API.","Clients and agents",{"title":204,"path":205,"stem":206,"description":207,"group":202},"MCP server","\u002Fdocs\u002Fapi\u002Fmcp-server","docs\u002F7.api\u002F8.mcp-server","Give Claude or Cursor tools to run Shruwd.",{"title":209,"path":210,"stem":211,"description":212,"group":202},"For AI agents","\u002Fdocs\u002Fapi\u002Ffor-ai-agents","docs\u002F7.api\u002F9.for-ai-agents","Machine-readable versions of everything here.",{"id":214,"title":94,"body":215,"description":690,"extension":691,"group":92,"meta":692,"navigation":693,"path":95,"section":57,"seo":694,"stem":96,"__hash__":695},"docs\u002Fdocs\u002F3.crawler-logs\u002F7.ingest-not-arriving.md",{"type":216,"value":217,"toc":681},"minimark",[218,222,235,240,248,347,411,421,425,432,470,481,485,491,503,526,530,533,540,578,588,601,605,608,611,615,624,634,644,661,665,677],[219,220,94],"h1",{"id":221},"ingest-not-arriving",[223,224,225,229,230,234],"p",{},[226,227,228],"strong",{},"Settings → Crawler logs"," still reads ",[231,232,233],"em",{},"awaiting first logs"," after you deployed. Work\nthrough these in order — the first two catch most cases.",[236,237,239],"h2",{"id":238},"_1-test-the-endpoint-and-token-directly","1. Test the endpoint and token directly",[223,241,242,243,247],{},"This separates configuration from your code entirely, and you can run it from anywhere.\n",[244,245,246],"code",{},"?validate=1"," parses your sample and reports on it without writing anything.",[249,250,251,313],"code-group",{},[252,253,259],"pre",{"className":254,"code":255,"filename":256,"language":257,"meta":258,"style":258},"language-bash shiki shiki-themes github-light github-dark","curl -X POST \"YOUR_ENDPOINT?validate=1\" \\\n  -H \"X-Shruwd-Ingest-Token: YOUR_TOKEN\" \\\n  -H \"Content-Type: application\u002Fx-ndjson\" \\\n  -d '{\"timestamp\":\"2026-01-01T00:00:00Z\",\"host\":\"example.com\",\"path\":\"\u002F\",\"user_agent\":\"GPTBot\",\"status\":200,\"client_ip\":\"1.2.3.4\"}'\n","curl","bash","",[244,260,261,283,294,304],{"__ignoreMap":258},[262,263,266,269,273,277,280],"span",{"class":264,"line":265},"line",1,[262,267,256],{"class":268},"sScJk",[262,270,272],{"class":271},"sj4cs"," -X",[262,274,276],{"class":275},"sZZnC"," POST",[262,278,279],{"class":275}," \"YOUR_ENDPOINT?validate=1\"",[262,281,282],{"class":271}," \\\n",[262,284,286,289,292],{"class":264,"line":285},2,[262,287,288],{"class":271},"  -H",[262,290,291],{"class":275}," \"X-Shruwd-Ingest-Token: YOUR_TOKEN\"",[262,293,282],{"class":271},[262,295,297,299,302],{"class":264,"line":296},3,[262,298,288],{"class":271},[262,300,301],{"class":275}," \"Content-Type: application\u002Fx-ndjson\"",[262,303,282],{"class":271},[262,305,307,310],{"class":264,"line":306},4,[262,308,309],{"class":271},"  -d",[262,311,312],{"class":275}," '{\"timestamp\":\"2026-01-01T00:00:00Z\",\"host\":\"example.com\",\"path\":\"\u002F\",\"user_agent\":\"GPTBot\",\"status\":200,\"client_ip\":\"1.2.3.4\"}'\n",[252,314,319],{"className":315,"code":316,"filename":317,"language":318,"meta":258,"style":258},"language-powershell shiki shiki-themes github-light github-dark","Invoke-RestMethod -Method Post `\n  -Uri \"YOUR_ENDPOINT?validate=1\" `\n  -Headers @{ \"X-Shruwd-Ingest-Token\" = \"YOUR_TOKEN\" } `\n  -Body '{\"timestamp\":\"2026-01-01T00:00:00Z\",\"host\":\"example.com\",\"path\":\"\u002F\",\"user_agent\":\"GPTBot\",\"status\":200,\"client_ip\":\"1.2.3.4\"}' `\n  -ContentType \"application\u002Fx-ndjson\"\n","PowerShell","powershell",[244,320,321,326,331,336,341],{"__ignoreMap":258},[262,322,323],{"class":264,"line":265},[262,324,325],{},"Invoke-RestMethod -Method Post `\n",[262,327,328],{"class":264,"line":285},[262,329,330],{},"  -Uri \"YOUR_ENDPOINT?validate=1\" `\n",[262,332,333],{"class":264,"line":296},[262,334,335],{},"  -Headers @{ \"X-Shruwd-Ingest-Token\" = \"YOUR_TOKEN\" } `\n",[262,337,338],{"class":264,"line":306},[262,339,340],{},"  -Body '{\"timestamp\":\"2026-01-01T00:00:00Z\",\"host\":\"example.com\",\"path\":\"\u002F\",\"user_agent\":\"GPTBot\",\"status\":200,\"client_ip\":\"1.2.3.4\"}' `\n",[262,342,344],{"class":264,"line":343},5,[262,345,346],{},"  -ContentType \"application\u002Fx-ndjson\"\n",[348,349,350,363],"table",{},[351,352,353],"thead",{},[354,355,356,360],"tr",{},[357,358,359],"th",{},"Response",[357,361,362],{},"Means",[364,365,366,377,387,397],"tbody",{},[354,367,368,374],{},[369,370,371],"td",{},[244,372,373],{},"usable: true",[369,375,376],{},"Endpoint and token are both fine — the problem is in your app or platform. Go to step 2",[354,378,379,384],{},[369,380,381],{},[244,382,383],{},"401",[369,385,386],{},"Wrong or revoked token. See below",[354,388,389,394],{},[369,390,391],{},[244,392,393],{},"404",[369,395,396],{},"Wrong brand in the endpoint URL",[354,398,399,404],{},[369,400,401],{},[244,402,403],{},"422",[369,405,406,407,410],{},"The sample can't be used. ",[244,408,409],{},"reason"," says why",[412,413,415],"callout",{"type":414},"warning",[223,416,417,420],{},[226,418,419],{},"Creating a token revokes the previous one."," If you minted a new token in Settings after\nsetting up the drain, the one your app is using is dead. Only the newest token works.",[236,422,424],{"id":423},"_2-on-vercel-check-the-drain","2. On Vercel, check the drain",[223,426,427,428,431],{},"In ",[226,429,430],{},"Team Settings → Drains",":",[433,434,435,439,463],"ul",{},[436,437,438],"li",{},"The drain isn't marked as errored or paused.",[436,440,441,442,445,446,445,449,445,452,445,455,458,459,462],{},"Its sources include ",[244,443,444],{},"static",", ",[244,447,448],{},"lambda",[244,450,451],{},"edge",[244,453,454],{},"external",[244,456,457],{},"redirect"," and ",[244,460,461],{},"firewall",".",[436,464,465,466,469],{},"Its environments include ",[244,467,468],{},"production",", and it has no sampling rule.",[223,471,472,473,476,477,480],{},"Then check ",[226,474,475],{},"Team Settings → Security & Privacy → IP Address Visibility"," is on. Hidden\nIPs look like a working drain: logs arrive and the card reads ",[231,478,479],{},"active",", but every line is\nrefused, so the Crawlers page stays empty.",[236,482,484],{"id":483},"_3-check-the-variables-in-the-deployed-environment","3. Check the variables in the deployed environment",[223,486,487,488,462],{},"Set locally and forgotten in the host's configuration is the single most common cause.\nCheck the values in the environment your site actually runs in, not your local ",[244,489,490],{},".env",[223,492,493,494,498,499,502],{},"If you are using ",[495,496,497],"a",{"href":84},"server middleware",", it logs one\nline the first time a send fails — look for ",[244,500,501],{},"[shruwd] log drain not working:"," in your\napp's logs. That line names the cause.",[223,504,505,506,509,510,513,514,517,518,521,522,525],{},"If you switched the middleware to ",[244,507,508],{},"useRuntimeConfig()",", confirm both halves: the keys\ndeclared in ",[244,511,512],{},"nuxt.config.ts"," ",[226,515,516],{},"and"," the ",[244,519,520],{},"NUXT_"," prefix on the variables. Missing either\nreads ",[244,523,524],{},"undefined"," silently.",[236,527,529],{"id":528},"_4-check-whether-the-request-was-served-from-cache","4. Check whether the request was served from cache",[223,531,532],{},"This only applies to server middleware. A request your CDN answers never reaches your\norigin, so the middleware never sees it.",[223,534,535,536,539],{},"Check ",[244,537,538],{},"cf-cache-status"," on the response:",[348,541,542,552],{},[351,543,544],{},[354,545,546,549],{},[357,547,548],{},"Value",[357,550,551],{},"Reached your origin?",[364,553,554,568],{},[354,555,556,565],{},[369,557,558,561,562],{},[244,559,560],{},"DYNAMIC"," or ",[244,563,564],{},"MISS",[369,566,567],{},"Yes",[354,569,570,575],{},[369,571,572],{},[244,573,574],{},"HIT",[369,576,577],{},"No — the middleware could not have seen it",[223,579,580,581,584,585,462],{},"If most of your pages are cached, middleware is the wrong path. Switch to the\n",[495,582,583],{"href":74},"forwarder Worker"," or a\n",[495,586,587],{"href":68},"platform drain",[412,589,591],{"type":590},"note",[223,592,593,596,597,600],{},[244,594,595],{},"cf-cache-status: DYNAMIC"," means Cloudflare did not serve it from Cloudflare's cache. It\ndoes ",[226,598,599],{},"not"," guarantee the request reached your application — a second CDN behind it can\nstill answer from static files.",[236,602,604],{"id":603},"_5-wait-a-little","5. Wait a little",[223,606,607],{},"Middleware batches: it sends at 20 events or 10 seconds, whichever comes first. One test\nrequest will not appear immediately.",[223,609,610],{},"Crawler traffic is also genuinely intermittent. A low-traffic site may wait hours between\nreal AI-crawler hits.",[236,612,614],{"id":613},"other-things-worth-knowing","Other things worth knowing",[223,616,617,513,620,623],{},[226,618,619],{},"Logs arrive but the card shows gaps.",[231,621,622],{},"Days with data"," on the Settings card shows how\nmany of the last 30 days received anything. Gaps switch off the diagnostic that needs 30\nuninterrupted days.",[223,625,626,629,630,633],{},[226,627,628],{},"Logs arrive but nothing is counted."," Check the ",[226,631,632],{},"Unverified"," column on the Crawlers\npage. Hits from IPs outside the vendor's published ranges are shown but never counted —\nwhich is correct, and still proves your pipe works.",[223,635,636,639,640,643],{},[226,637,638],{},"It stopped partway through the month."," Check ",[226,641,642],{},"Lines this month"," against your plan's\nallowance. Past the cap, lines are discarded until the month turns. The Crawlers page says\nso rather than showing a flat chart.",[223,645,646,653,654,657,658,462],{},[226,647,648,649,652],{},"A ",[244,650,651],{},"413"," response."," Your batches are over the size cap. On Cloudflare Logpush, set\n",[244,655,656],{},"max_upload_bytes"," to ",[244,659,660],{},"5000000",[236,662,664],{"id":663},"next-steps","Next steps",[433,666,667,672],{},[436,668,669,671],{},[495,670,62],{"href":63}," — if you picked the wrong one",[436,673,674,676],{},[495,675,88],{"href":89}," — how verification works",[678,679,680],"style",{},"html pre.shiki code .sScJk, html code.shiki .sScJk{--shiki-default:#6F42C1;--shiki-dark:#B392F0}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":258,"searchDepth":296,"depth":296,"links":682},[683,684,685,686,687,688,689],{"id":238,"depth":285,"text":239},{"id":423,"depth":285,"text":424},{"id":483,"depth":285,"text":484},{"id":528,"depth":285,"text":529},{"id":603,"depth":285,"text":604},{"id":613,"depth":285,"text":614},{"id":663,"depth":285,"text":664},"Work through the causes of a crawler log drain that stays on awaiting first logs.","md",{},{"title":94,"description":97},{"title":94,"description":690},"49vmfqU0br7O_GTHqjYLsdHuBv-Z6nTuBNtgp7fvwS0",1790003439062]