[{"data":1,"prerenderedAt":555},["ShallowReactive",2],{"docs-navigation":3,"docs-\u002Fdocs\u002Fcrawler-logs\u002Fcrawlers-tracked":213},[4],{"title":5,"path":6,"stem":7,"children":8,"page":30},"Docs","\u002Fdocs","docs",[9,31,56,98,118,138,163],{"title":10,"description":11,"group":11,"path":12,"stem":13,"children":14,"page":30},"Getting started",null,"\u002Fdocs\u002Fgetting-started","docs\u002F1.getting-started",[15,20,25],{"title":16,"path":17,"stem":18,"description":19,"group":11},"Overview","\u002Fdocs\u002Fgetting-started\u002Foverview","docs\u002F1.getting-started\u002F1.overview","What Shruwd does and what you get out of it.",{"title":21,"path":22,"stem":23,"description":24,"group":11},"Quickstart","\u002Fdocs\u002Fgetting-started\u002Fquickstart","docs\u002F1.getting-started\u002F2.quickstart","From an empty account to your first measurement.",{"title":26,"path":27,"stem":28,"description":29,"group":11},"Reading your results","\u002Fdocs\u002Fgetting-started\u002Freading-your-results","docs\u002F1.getting-started\u002F3.reading-your-results","What the four numbers mean, and when to trust them.",false,{"title":32,"description":11,"group":11,"path":33,"stem":34,"children":35,"page":30},"What you track","\u002Fdocs\u002Ftracking","docs\u002F2.tracking",[36,41,46,51],{"title":37,"path":38,"stem":39,"description":40,"group":11},"Brands","\u002Fdocs\u002Ftracking\u002Fbrands","docs\u002F2.tracking\u002F1.brands","Add, rename and archive the brands you measure.",{"title":42,"path":43,"stem":44,"description":45,"group":11},"Prompts","\u002Fdocs\u002Ftracking\u002Fprompts","docs\u002F2.tracking\u002F2.prompts","Choose and manage the questions Shruwd asks each engine.",{"title":47,"path":48,"stem":49,"description":50,"group":11},"Competitors","\u002Fdocs\u002Ftracking\u002Fcompetitors","docs\u002F2.tracking\u002F3.competitors","Track the brands you are compared against, and how they are matched.",{"title":52,"path":53,"stem":54,"description":55,"group":11},"Connecting Search Console","\u002Fdocs\u002Ftracking\u002Fconnecting-search-console","docs\u002F2.tracking\u002F4.connecting-search-console","Add per-page search demand so two diagnostics can run.",{"title":57,"description":11,"group":11,"path":58,"stem":59,"children":60,"page":30},"Crawler logs","\u002Fdocs\u002Fcrawler-logs","docs\u002F3.crawler-logs",[61,66,72,77,82,87,93],{"title":62,"path":63,"stem":64,"description":65,"group":11},"Choosing a log path","\u002Fdocs\u002Fcrawler-logs\u002Fchoosing-a-log-path","docs\u002F3.crawler-logs\u002F1.choosing-a-log-path","Pick the way of sending crawler logs that matches your hosting.",{"title":67,"path":68,"stem":69,"description":70,"group":71},"Platform log drains","\u002Fdocs\u002Fcrawler-logs\u002Fplatform-log-drains","docs\u002F3.crawler-logs\u002F2.platform-log-drains","Send logs from Vercel, Netlify, Fly and similar. No code.","Setup guides",{"title":73,"path":74,"stem":75,"description":76,"group":71},"Cloudflare Worker","\u002Fdocs\u002Fcrawler-logs\u002Fcloudflare-worker","docs\u002F3.crawler-logs\u002F3.cloudflare-worker","Forward crawler hits from behind Cloudflare, on any plan.",{"title":78,"path":79,"stem":80,"description":81,"group":71},"Cloudflare Logpush","\u002Fdocs\u002Fcrawler-logs\u002Fcloudflare-logpush","docs\u002F3.crawler-logs\u002F4.cloudflare-logpush","Send zone request logs straight to Shruwd. Enterprise zones, no code.",{"title":83,"path":84,"stem":85,"description":86,"group":71},"Server middleware","\u002Fdocs\u002Fcrawler-logs\u002Fserver-middleware","docs\u002F3.crawler-logs\u002F5.server-middleware","Forward crawler hits from a Nuxt or Nitro origin. Last resort.",{"title":88,"path":89,"stem":90,"description":91,"group":92},"Crawlers tracked","\u002Fdocs\u002Fcrawler-logs\u002Fcrawlers-tracked","docs\u002F3.crawler-logs\u002F6.crawlers-tracked","Which bots Shruwd recognises, and how a hit is verified.","Reference",{"title":94,"path":95,"stem":96,"description":97,"group":92},"Ingest not arriving","\u002Fdocs\u002Fcrawler-logs\u002Fingest-not-arriving","docs\u002F3.crawler-logs\u002F7.ingest-not-arriving","Diagnose a log drain that is not delivering.",{"title":99,"description":11,"group":11,"path":100,"stem":101,"children":102,"page":30},"Findings and fixes","\u002Fdocs\u002Ffindings","docs\u002F4.findings",[103,108,113],{"title":104,"path":105,"stem":106,"description":107,"group":11},"How findings work","\u002Fdocs\u002Ffindings\u002Fhow-findings-work","docs\u002F4.findings\u002F1.how-findings-work","What a finding is, how much to trust it, and how to prove your fix worked.",{"title":109,"path":110,"stem":111,"description":112,"group":11},"Rechecks","\u002Fdocs\u002Ffindings\u002Frechecks","docs\u002F4.findings\u002F2.rechecks","How Shruwd proves whether your fix moved the number.",{"title":114,"path":115,"stem":116,"description":117,"group":11},"Findings reference","\u002Fdocs\u002Ffindings\u002Ffindings-reference","docs\u002F4.findings\u002F3.findings-reference","Every diagnostic Shruwd runs, what triggers it and what to do.",{"title":119,"description":11,"group":11,"path":120,"stem":121,"children":122,"page":30},"Metrics","\u002Fdocs\u002Fmetrics","docs\u002F5.metrics",[123,128,133],{"title":124,"path":125,"stem":126,"description":127,"group":11},"Metric definitions","\u002Fdocs\u002Fmetrics\u002Fmetric-definitions","docs\u002F5.metrics\u002F1.metric-definitions","Exactly what each number counts, and when it is undefined.",{"title":129,"path":130,"stem":131,"description":132,"group":11},"Confidence and sample size","\u002Fdocs\u002Fmetrics\u002Fconfidence-and-sample-size","docs\u002F5.metrics\u002F2.confidence-and-sample-size","Why every number has a range, and why some are withheld.",{"title":134,"path":135,"stem":136,"description":137,"group":11},"When a number has moved","\u002Fdocs\u002Fmetrics\u002Fwhen-a-number-has-moved","docs\u002F5.metrics\u002F3.when-a-number-has-moved","The rule Shruwd uses before calling a change real.",{"title":139,"description":11,"group":11,"path":140,"stem":141,"children":142,"page":30},"Account","\u002Fdocs\u002Faccount","docs\u002F6.account",[143,148,153,158],{"title":144,"path":145,"stem":146,"description":147,"group":11},"Plans and limits","\u002Fdocs\u002Faccount\u002Fplans-and-limits","docs\u002F6.account\u002F1.plans-and-limits","What each plan includes and what happens when you reach a limit.",{"title":149,"path":150,"stem":151,"description":152,"group":11},"Team and roles","\u002Fdocs\u002Faccount\u002Fteam-and-roles","docs\u002F6.account\u002F2.team-and-roles","Invite people, set what they can change, and remove access.",{"title":154,"path":155,"stem":156,"description":157,"group":11},"Billing","\u002Fdocs\u002Faccount\u002Fbilling","docs\u002F6.account\u002F3.billing","Change plan, read your usage, and handle a failed payment.",{"title":159,"path":160,"stem":161,"description":162,"group":11},"Email notifications","\u002Fdocs\u002Faccount\u002Femail-notifications","docs\u002F6.account\u002F4.email-notifications","What Shruwd emails you, and how to turn it off.",{"title":164,"description":11,"group":11,"path":165,"stem":166,"children":167,"page":30},"Build on Shruwd","\u002Fdocs\u002Fapi","docs\u002F7.api",[168,173,178,182,187,192,197,203,208],{"title":169,"path":170,"stem":171,"description":172,"group":11},"API overview","\u002Fdocs\u002Fapi\u002Fapi-overview","docs\u002F7.api\u002F1.api-overview","Authentication, errors, limits and the rules every endpoint follows.",{"title":37,"path":174,"stem":175,"description":176,"group":177},"\u002Fdocs\u002Fapi\u002Fbrands","docs\u002F7.api\u002F2.brands","Create, read, rename and archive brands over the API.","Endpoints",{"title":42,"path":179,"stem":180,"description":181,"group":177},"\u002Fdocs\u002Fapi\u002Fprompts","docs\u002F7.api\u002F3.prompts","Add, edit and retire the questions asked every cycle.",{"title":183,"path":184,"stem":185,"description":186,"group":177},"Entities","\u002Fdocs\u002Fapi\u002Fentities","docs\u002F7.api\u002F4.entities","Manage the brand and its competitors, and the aliases that count.",{"title":188,"path":189,"stem":190,"description":191,"group":177},"Measurement","\u002Fdocs\u002Fapi\u002Fmeasurement","docs\u002F7.api\u002F5.measurement","Run cycles and read visibility, crawlers and individual answers.",{"title":193,"path":194,"stem":195,"description":196,"group":177},"Findings","\u002Fdocs\u002Fapi\u002Ffindings","docs\u002F7.api\u002F6.findings","Read diagnosed findings and move them through their lifecycle.",{"title":198,"path":199,"stem":200,"description":201,"group":202},"TypeScript SDK","\u002Fdocs\u002Fapi\u002Ftypescript-sdk","docs\u002F7.api\u002F7.typescript-sdk","A typed client for the Shruwd API.","Clients and agents",{"title":204,"path":205,"stem":206,"description":207,"group":202},"MCP server","\u002Fdocs\u002Fapi\u002Fmcp-server","docs\u002F7.api\u002F8.mcp-server","Give Claude or Cursor tools to run Shruwd.",{"title":209,"path":210,"stem":211,"description":212,"group":202},"For AI agents","\u002Fdocs\u002Fapi\u002Ffor-ai-agents","docs\u002F7.api\u002F9.for-ai-agents","Machine-readable versions of everything here.",{"id":214,"title":88,"body":215,"description":549,"extension":550,"group":92,"meta":551,"navigation":552,"path":89,"section":57,"seo":553,"stem":90,"__hash__":554},"docs\u002Fdocs\u002F3.crawler-logs\u002F6.crawlers-tracked.md",{"type":216,"value":217,"toc":536},"minimark",[218,222,226,231,234,278,286,291,306,320,323,327,330,378,389,393,399,402,436,439,443,446,462,466,507,510,513,517],[219,220,88],"h1",{"id":221},"crawlers-tracked",[223,224,225],"p",{},"Shruwd recognises the AI crawlers below, verifies each hit against the vendor's published\nIP ranges, and separates fetches made to answer a question from batch crawling for\ntraining.",[227,228,230],"h2",{"id":229},"recognised-crawlers","Recognised crawlers",[223,232,233],{},"These are the bots Shruwd identifies today, across OpenAI, Anthropic, Perplexity, Apple,\nByteDance, Meta, Common Crawl and Amazon:",[223,235,236,240,241,240,244,240,247,240,250,253,254,240,257,240,260,240,263,240,266,253,269,240,272,240,275],{},[237,238,239],"code",{},"GPTBot"," · ",[237,242,243],{},"OAI-SearchBot",[237,245,246],{},"ChatGPT-User",[237,248,249],{},"ClaudeBot",[237,251,252],{},"Claude-User"," ·\n",[237,255,256],{},"Claude-SearchBot",[237,258,259],{},"PerplexityBot",[237,261,262],{},"Perplexity-User",[237,264,265],{},"Applebot",[237,267,268],{},"Bytespider",[237,270,271],{},"Meta-ExternalAgent",[237,273,274],{},"CCBot",[237,276,277],{},"Amazonbot",[223,279,280,281,285],{},"The list is maintained on Shruwd's side and updated as new crawlers appear, so you never\nneed to change anything you deployed. Your brand's ",[282,283,284],"strong",{},"Crawlers"," page lists each bot that\nhas reached you, with its vendor.",[287,288,290],"h3",{"id":289},"google-extended-and-applebot-extended-are-not-crawlers","Google-Extended and Applebot-Extended are not crawlers",[223,292,293,294,297,298,301,302,305],{},"Both are ",[237,295,296],{},"robots.txt"," tokens that control how your content may be used, and neither ever\nappears in a log. Google says ",[237,299,300],{},"Google-Extended"," \"doesn't have a separate HTTP request user\nagent string\". Apple says ",[237,303,304],{},"Applebot-Extended"," \"does not crawl webpages\". The fetching is\ndone by Googlebot and by Applebot.",[223,307,308,309,311,312,314,315,319],{},"So neither has a row on the Crawlers page, and an empty one would tell you nothing. Shruwd\nchecks ",[237,310,300],{}," where it does exist: the ",[237,313,296],{}," diagnostic reports a rule\nthat disallows it and says what that affects, which is Gemini grounding and AI training,\nnot AI Overviews. See the ",[316,317,318],"a",{"href":115},"findings reference",".",[223,321,322],{},"Google's AI Overviews and AI Mode are fed by Googlebot, the same crawler as Google Search.\nThere is no separate Google AI crawler to look for in your logs.",[227,324,326],{"id":325},"live-retrieval-versus-training","Live retrieval versus training",[223,328,329],{},"Each bot is classified by what it is for, and the distinction matters more than the\ntotals.",[331,332,333,348],"table",{},[334,335,336],"thead",{},[337,338,339,342,345],"tr",{},[340,341],"th",{},[340,343,344],{},"What it is",[340,346,347],{},"Why you care",[349,350,351,365],"tbody",{},[337,352,353,359,362],{},[354,355,356],"td",{},[282,357,358],{},"Live retrieval",[354,360,361],{},"Fetched to answer a specific person's question, right then",[354,363,364],{},"A page it cannot fetch cannot be cited in that answer. This is the number to watch",[337,366,367,372,375],{},[354,368,369],{},[282,370,371],{},"Training",[354,373,374],{},"Batch crawling to build a model",[354,376,377],{},"Decides what a model learns, not what an answer cites today",[223,379,380,382,383,385,386,388],{},[237,381,246],{}," and ",[237,384,262],{}," are live-retrieval crawlers. ",[237,387,239],{}," is a training\ncrawler. Presenting them as one undifferentiated figure would hide the most interesting\nsignal in the data, so Shruwd never does.",[227,390,392],{"id":391},"how-a-hit-is-verified","How a hit is verified",[223,394,395,396,398],{},"A user-agent string is a claim, not evidence — anyone can send ",[237,397,239],{}," in a header.",[223,400,401],{},"Shruwd checks the client IP of every hit against the ranges the vendor publishes. Those\nranges are refreshed every 24 hours. If a refresh fails, the last known-good set is kept\nrather than marking everything verified or everything unverified — both would quietly\ncorrupt the number.",[331,403,404,414],{},[334,405,406],{},[337,407,408,411],{},[340,409,410],{},"Result",[340,412,413],{},"What happens",[349,415,416,426],{},[337,417,418,423],{},[354,419,420],{},[282,421,422],{},"Verified",[354,424,425],{},"The IP belongs to the vendor. Counted in every metric",[337,427,428,433],{},[354,429,430],{},[282,431,432],{},"Unverified",[354,434,435],{},"The user-agent claims a bot from an IP the vendor does not own. Shown on the Crawlers page, never counted",[223,437,438],{},"Unverified hits are displayed rather than hidden, because seeing them is useful — but\nthey never reach a headline number. Where a vendor publishes no ranges at all and no\nreverse-DNS check is documented, its hits stay permanently unverified.",[227,440,442],{"id":441},"reading-the-crawlers-page","Reading the Crawlers page",[223,444,445],{},"Once logs arrive you get:",[447,448,449,456],"ul",{},[450,451,452,455],"li",{},[282,453,454],{},"Verified hits per day"," — the timeline. It appears after two days of logs.",[450,457,458,461],{},[282,459,460],{},"By bot"," — per-crawler verified hits, unverified hits, and errors. A non-zero error\ncount means that crawler was refused, which is a problem worth a look.",[227,463,465],{"id":464},"diagnostics-that-need-logs","Diagnostics that need logs",[331,467,468,478],{},[334,469,470],{},[337,471,472,475],{},[340,473,474],{},"Diagnostic",[340,476,477],{},"Needs",[349,479,480,488],{},[337,481,482,485],{},[354,483,484],{},"The server refuses a verified AI crawler",[354,486,487],{},"Logs showing the refusal on at least three separate days",[337,489,490,493],{},[354,491,492],{},"A page with demand is never crawled",[354,494,495,498,499,502,503,506],{},[282,496,497],{},"30 days"," of uninterrupted logs, ",[282,500,501],{},"and"," ",[316,504,505],{"href":53},"Search Console"," for the demand signal",[223,508,509],{},"The three-day requirement rules out a transient outage being reported as a block.",[223,511,512],{},"The 30-day requirement is a guard, not a delay for its own sake. A brand whose drain broke\nfor a week must not be told a page was never crawled when the truth is that nobody was\nwatching. Search Console is needed alongside it because \"never crawled\" is only worth\nraising for a page people actually search for.",[227,514,516],{"id":515},"next-steps","Next steps",[447,518,519,526,531],{},[450,520,521,525],{},[316,522,524],{"href":523},"\u002Fai-crawlers","AI crawlers"," — the same list as a public reference, with each vendor's IP-range file",[450,527,528,530],{},[316,529,62],{"href":63}," — if you have not set this up yet",[450,532,533,535],{},[316,534,104],{"href":105}," — what these diagnostics produce",{"title":537,"searchDepth":538,"depth":538,"links":539},"",3,[540,544,545,546,547,548],{"id":229,"depth":541,"text":230,"children":542},2,[543],{"id":289,"depth":538,"text":290},{"id":325,"depth":541,"text":326},{"id":391,"depth":541,"text":392},{"id":441,"depth":541,"text":442},{"id":464,"depth":541,"text":465},{"id":515,"depth":541,"text":516},"The AI crawlers Shruwd recognises, how each hit is verified against vendor IP ranges, and why some are shown but never counted.","md",{},{"title":88,"description":91},{"title":88,"description":549},"NRxC7XVd5aXFGGNeQ22VWWwI0SSbz4YXycYAus-ijCg",1790003438955]