{ "last_verified": "2026-08-02", "held": [ { "name": "Bytespider", "vendor": "ByteDance", "reason": "No ByteDance-operated documentation page for Bytespider could be fetched. Every available description is third-party. Held until a primary source exists." }, { "name": "Diffbot", "vendor": "Diffbot", "reason": "docs.diffbot.com serves API documentation, not crawler identification or verification guidance. No published user-agent or IP source found." }, { "name": "Timpibot", "vendor": "Timpi", "reason": "timpi.io has no crawler documentation page and docs.timpi.io does not resolve. No primary source." } ], "operators": [ { "slug": "gptbot", "name": "GPTBot", "vendor": "OpenAI", "token": "GPTBot", "purpose": "training", "purposeLabel": "AI training crawler", "ua": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.4; +https://openai.com/gptbot", "method": "ip", "rangeSrc": "https://openai.com/gptbot.json", "docs": "https://platform.openai.com/docs/bots", "cost": "Disallowing GPTBot signals that your content should not be used to train OpenAI's foundation models. Per OpenAI's own documentation it does not remove you from ChatGPT search results, which are governed separately by OAI-SearchBot. Blocking GPTBot alone is the cleanest available split between opting out of training and staying visible in ChatGPT.", "substance": "GPTBot and OAI-SearchBot are separate robots.txt tokens but not always separate crawls. OpenAI states that when a site allows both, it may use the results of a single crawl to serve both purposes rather than fetching twice. That means a GPTBot hit in your logs is not proof the content was used for training, and the absence of OAI-SearchBot hits is not proof you are missing from search. A second detail worth knowing when reading logs: when OpenAI fetches robots.txt itself, it may append a `robots.txt` marker to the user-agent string, so the same crawler produces two distinguishable strings depending on what it is requesting.", "ranges": [ "132.196.86.0/24", "172.182.202.0/25", "172.182.204.0/24", "172.182.207.0/25", "172.182.214.0/24", "172.182.215.0/24", "20.125.66.80/28", "20.171.206.0/24", "20.171.207.0/24", "4.227.36.0/25", "52.230.152.0/24", "74.7.175.128/25", "74.7.227.0/25", "74.7.227.128/25", "74.7.228.0/25", "74.7.230.0/25", "74.7.241.0/25", "74.7.241.128/25", "74.7.242.0/25", "74.7.243.128/25", "74.7.244.0/25" ], "rangesAsOf": "2025-10-30", "path": "/crawlers/gptbot" }, { "slug": "oai-searchbot", "name": "OAI-SearchBot", "vendor": "OpenAI", "token": "OAI-SearchBot", "purpose": "search", "purposeLabel": "Search index", "ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36; compatible; OAI-SearchBot/1.4; +https://openai.com/searchbot", "method": "ip", "rangeSrc": "https://openai.com/searchbot.json", "docs": "https://platform.openai.com/docs/bots", "cost": "This is the expensive one to block. OpenAI states that sites opted out of OAI-SearchBot will not be shown in ChatGPT search answers, though they can still appear as navigational links. If being cited inside ChatGPT matters to you, this is the token to allow even when you disallow GPTBot.", "substance": "OAI-SearchBot changes do not take effect immediately. OpenAI documents roughly a 24 hour lag from a robots.txt edit to their systems reflecting it, so a site that allows the token today will not appear in ChatGPT search answers today, and a site that blocks it will keep being crawled for about a day. When testing an opt-in, an unchanged log the same afternoon means nothing. OAI-SearchBot also carries the same optional `robots.txt` marker in its user-agent that GPTBot does.", "ranges": [ "104.210.140.128/28", "135.234.64.0/24", "172.182.193.224/28", "172.182.193.80/28", "172.182.194.144/28", "172.182.194.32/28", "172.182.195.48/28", "172.182.209.208/28", "172.182.211.192/28", "172.182.213.192/28", "172.182.224.0/28", "172.203.190.128/28", "20.14.99.96/28", "20.168.18.32/28", "20.169.6.224/28", "20.169.7.48/28", "20.169.77.0/25", "20.171.123.64/28", "20.171.53.224/28", "20.25.151.224/28", "20.42.10.176/28", "4.227.36.0/25", "40.67.175.0/25", "40.90.214.16/28", "51.8.102.0/24", "74.7.175.128/25", "74.7.228.0/25", "74.7.228.128/25", "74.7.229.0/25", "74.7.229.128/25", "74.7.230.0/25", "74.7.241.128/25", "74.7.242.128/25", "74.7.243.0/25", "74.7.244.0/25" ], "rangesAsOf": "2026-01-02", "path": "/crawlers/oai-searchbot" }, { "slug": "chatgpt-user", "name": "ChatGPT-User", "vendor": "OpenAI", "token": "ChatGPT-User", "purpose": "not-a-crawler", "purposeLabel": "User-triggered fetch", "ua": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot", "method": "ip", "rangeSrc": "https://openai.com/chatgpt-user.json", "docs": "https://platform.openai.com/docs/bots", "cost": "Blocking ChatGPT-User stops OpenAI retrieving your page when a specific user asks a question that needs it. It has no effect on training and no effect on whether you appear in ChatGPT search. You are blocking the moment of highest intent, a real person who has already been pointed at your content.", "substance": "ChatGPT-User is not a crawler and OpenAI does not treat it as one. Because the fetch is initiated by a person, OpenAI states robots.txt rules may not apply, so a Disallow is a request rather than a control. It is also by far the largest published range set of any AI fetcher, which is what you would expect from traffic that originates wherever users are rather than from a fixed crawl fleet. That size matters operationally: an allowlist built from this file needs refreshing on a schedule, not once.", "ranges": [ "104.208.184.192/28", "104.208.184.208/28", "104.210.139.192/28", "104.210.139.224/28", "128.85.198.32/28", "13.65.138.112/28", "13.65.138.96/28", "13.67.72.16/28", "13.70.107.160/28", "13.71.2.208/28", "13.76.115.224/28", "13.76.115.240/28", "13.76.116.80/28", "13.76.32.208/28", "13.83.167.128/28", "13.83.237.176/28", "132.196.82.48/28", "134.149.233.80/28", "135.116.136.160/28", "135.13.64.240/28", "135.220.73.208/28", "135.220.73.240/28", "135.237.131.208/28", "135.237.133.48/28", "137.135.191.176/28", "138.91.30.48/28", "138.91.46.96/28", "145.132.1.32/28", "145.132.136.96/28", "145.133.0.176/28", "158.158.5.32/28", "168.63.252.240/28", "172.162.248.64/28", "172.170.1.80/28", "172.170.225.0/28", "172.170.241.80/28", "172.170.8.208/28", "172.171.4.176/28", "172.175.152.224/28", "172.178.140.144/28", "172.178.141.112/28", "172.178.141.128/28", "172.183.143.224/28", "172.183.222.128/28", "172.192.112.208/28", "172.192.88.192/28", "172.192.97.32/28", "172.197.160.192/28", "172.197.161.208/28", "172.197.170.80/28", "172.197.203.16/28", "172.199.137.80/28", "172.202.102.112/28", "172.204.27.16/28", "172.204.28.224/28", "172.204.96.32/28", "172.204.96.48/28", "172.204.96.80/28", "172.205.189.192/28", "172.207.1.32/28", "172.208.128.32/28", "172.208.128.48/28", "172.212.159.64/28", "172.212.172.160/28", "172.213.21.16/28", "172.215.215.32/28", "172.215.218.96/28", "191.233.1.112/28", "191.233.1.128/28", "191.233.194.32/28", "191.233.196.112/28", "191.233.199.160/28", "191.233.2.0/28", "191.235.66.16/28", "191.235.99.80/28", "191.237.249.64/28", "191.239.245.16/28", "20.102.212.144/28", "20.113.211.112/28", "20.113.225.112/28", "20.125.112.224/28", "20.125.144.144/28", "20.161.75.208/28", "20.168.7.192/28", "20.168.7.240/28", "20.169.72.112/28", "20.169.73.176/28", "20.169.73.32/28", "20.169.73.64/28", "20.169.78.112/28", "20.169.78.128/28", "20.169.78.144/28", "20.169.78.160/28", "20.169.78.176/28", "20.169.78.192/28", "20.169.78.208/28", "20.169.78.48/28", "20.169.78.64/28", "20.169.78.80/28", "20.169.78.96/28", "20.169.86.224/28", "20.169.86.240/28", "20.169.87.112/28", "20.17.108.96/28", "20.170.184.16/28", "20.170.184.32/28", "20.170.184.48/28", "20.170.184.64/28", "20.170.184.80/28", "20.172.29.32/28", "20.193.233.240/28", "20.193.50.32/28", "20.194.0.208/28", "20.194.1.0/28", "20.198.67.96/28", "20.199.211.160/28", "20.199.242.0/28", "20.200.212.240/28", "20.204.24.240/28", "20.210.154.128/28", "20.210.174.208/28", "20.210.211.192/28", "20.215.187.208/28", "20.215.188.192/28", "20.215.214.16/28", "20.215.219.128/28", "20.215.219.160/28", "20.215.219.208/28", "20.215.220.112/28", "20.215.220.128/28", "20.215.220.144/28", "20.215.220.160/28", "20.215.220.176/28", "20.215.220.192/28", "20.215.220.208/28", "20.215.220.64/28", "20.215.220.80/28", "20.215.220.96/28", "20.218.30.240/28", "20.219.71.192/28", "20.222.36.192/28", "20.226.32.80/28", "20.227.140.32/28", "20.228.106.176/28", "20.235.75.208/28", "20.235.87.224/28", "20.249.63.208/28", "20.250.136.64/28", "20.250.136.80/28", "20.250.6.128/28", "20.27.94.128/28", "20.45.178.144/28", "20.48.120.208/28", "20.52.125.160/28", "20.55.129.0/28", "20.55.229.144/28", "20.57.199.192/28", "20.63.180.96/28", "20.63.221.64/28", "20.79.59.112/28", "20.81.183.64/28", "20.83.243.176/28", "20.97.189.96/28", "23.102.140.144/28", "23.102.141.32/28", "23.98.142.176/28", "23.98.179.16/28", "23.98.186.176/28", "23.98.186.192/28", "23.98.186.64/28", "23.98.186.96/28", "4.151.119.48/28", "4.151.241.240/28", "4.151.71.176/28", "4.189.118.208/28", "4.189.119.48/28", "4.196.118.112/28", "4.197.115.112/28", "4.197.19.176/28", "4.197.22.112/28", "4.197.64.0/28", "4.197.64.16/28", "4.197.64.48/28", "4.197.64.64/28", "4.198.72.16/28", "4.198.96.112/28", "4.201.232.64/28", "4.201.232.80/28", "4.203.96.80/28", "4.205.128.176/28", "4.218.24.64/28", "4.226.200.16/28", "4.226.226.32/28", "40.116.73.208/28", "40.122.235.112/28", "40.67.183.160/28", "40.67.183.176/28", "40.78.161.48/28", "40.81.134.128/28", "40.81.134.144/28", "40.81.234.144/28", "40.81.67.96/28", "40.84.181.32/28", "40.84.221.208/28", "40.84.221.224/28", "48.193.44.32/28", "48.221.184.112/28", "48.221.184.80/28", "48.221.184.96/28", "48.221.40.176/28", "51.107.70.192/28", "51.116.2.80/28", "51.116.221.96/28", "51.56.40.80/28", "51.57.0.96/28", "51.59.24.64/28", "51.59.24.80/28", "51.59.40.80/28", "51.59.40.96/28", "51.59.48.80/28", "51.59.48.96/28", "51.8.155.112/28", "51.8.155.48/28", "51.8.155.64/28", "51.8.187.224/28", "52.148.129.32/28", "52.153.130.64/28", "52.154.22.48/28", "52.156.77.144/28", "52.159.227.32/28", "52.159.249.96/28", "52.161.49.224/28", "52.161.49.96/28", "52.165.212.16/28", "52.165.212.32/28", "52.165.212.48/28", "52.172.129.160/28", "52.172.251.112/28", "52.173.219.112/28", "52.173.219.96/28", "52.173.221.16/28", "52.173.221.176/28", "52.173.221.208/28", "52.173.234.16/28", "52.173.234.80/28", "52.173.235.80/28", "52.183.217.240/28", "52.187.246.128/28", "52.190.137.144/28", "52.190.137.16/28", "52.190.139.48/28", "52.190.142.64/28", "52.190.190.16/28", "52.225.75.208/28", "52.230.163.32/28", "52.230.164.176/28", "52.231.30.48/28", "52.231.34.176/28", "52.231.39.144/28", "52.231.39.192/28", "52.231.49.48/28", "52.231.50.64/28", "52.236.94.144/28", "52.241.146.208/28", "52.242.132.224/28", "52.242.132.240/28", "52.242.245.208/28", "52.252.113.240/28", "52.255.109.112/28", "52.255.109.128/28", "52.255.109.144/28", "52.255.109.80/28", "52.255.109.96/28", "52.255.111.0/28", "52.255.111.112/28", "52.255.111.32/28", "52.255.111.48/28", "52.255.111.80/28", "57.154.174.112/28", "57.154.175.0/28", "57.154.187.32/28", "68.154.28.96/28", "68.218.30.112/28", "68.220.57.64/28", "68.221.67.192/28", "68.221.67.224/28", "68.221.67.240/28", "70.153.139.208/28", "70.153.189.192/28", "70.153.190.16/28", "70.153.76.16/28", "70.153.87.224/28", "70.156.144.64/28", "70.156.152.80/28", "70.156.152.96/28", "74.161.200.96/28", "74.224.217.64/28", "74.226.253.160/28", "74.249.86.176/28", "74.7.35.112/28", "74.7.35.48/28", "74.7.36.64/28", "74.7.36.80/28", "74.7.36.96/28", "85.211.241.128/28", "9.129.0.0/17", "9.160.128.16/28", "9.160.128.32/28", "9.160.128.64/28", "9.160.163.128/28", "9.160.164.128/28", "9.160.34.144/28", "9.160.36.16/28", "9.160.96.16/28", "9.163.101.48/28", "9.205.25.128/28", "9.205.8.48/28", "9.223.181.208/28", "9.234.96.192/28", "9.234.97.128/28", "9.234.97.96/28", "9.235.40.32/28" ], "rangesAsOf": "2026-07-30", "path": "/crawlers/chatgpt-user" }, { "slug": "oai-adsbot", "name": "OAI-AdsBot", "vendor": "OpenAI", "token": "OAI-AdsBot", "purpose": "retrieval", "purposeLabel": "Ad landing page validation", "ua": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-AdsBot/1.0; +https://openai.com/adsbot", "method": "ip", "rangeSrc": "https://openai.com/adsbot.json", "docs": "https://platform.openai.com/docs/bots", "cost": "Almost nothing, unless you advertise on ChatGPT. OAI-AdsBot only visits pages submitted as ads, so blocking it affects your own ad landing pages and nothing else. If you do run ChatGPT ads, blocking it means OpenAI cannot validate your landing page against its policies.", "substance": "OAI-AdsBot only fetches pages that someone has submitted as an ad, which makes it the one OpenAI agent whose appearance in your logs tells you something specific: either you submitted that URL as an ad, or someone else did. Its published range is the smallest of any operator tracked here, currently two prefixes, which reflects how narrow the job is. OpenAI states the data it collects is not used to train foundation models, so it sits outside the GPTBot training question entirely despite being an automated crawler.", "ranges": [ "130.131.229.128/25", "20.168.90.128/25" ], "rangesAsOf": "2026-05-12", "path": "/crawlers/oai-adsbot" }, { "slug": "claudebot", "name": "ClaudeBot", "vendor": "Anthropic", "token": "ClaudeBot", "purpose": "training", "purposeLabel": "AI training crawler", "ua": null, "uaNote": "Anthropic's crawler documentation names the robots.txt token but does not publish a full example user-agent string. Nothing is asserted here that Anthropic has not published.", "method": "ip", "rangeSrc": "https://claude.com/crawling/bots.json", "docs": "https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "cost": "Restricting ClaudeBot signals that your site's future material should be excluded from Anthropic's model training datasets. It does not affect whether Claude can reach your page when a user asks about it, which is Claude-User, or whether you are indexed for Claude's search, which is Claude-SearchBot.", "substance": "Anthropic publishes one IP list for all three of its robots, not one per bot. The same bots.json covers ClaudeBot, Claude-User and Claude-SearchBot, so a source IP match proves the request came from Anthropic but does not by itself prove which bot sent it. To distinguish them you still have to trust the user-agent, which means the IP check and the user-agent check answer different questions here than they do for OpenAI or Perplexity. Anthropic also explicitly warns against opting out by blocking their IPs: doing so stops them reading your robots.txt, which is the only mechanism that actually registers the opt-out.", "ranges": [ "216.73.216.0/22", "34.162.230.222/32", "34.162.244.71/32", "34.162.191.81/32", "34.150.241.79/32", "34.85.172.162/32", "35.245.175.129/32", "34.182.222.37/32", "34.186.108.163/32", "35.245.89.239/32", "34.182.161.143/32", "34.182.218.27/32", "34.182.220.85/32", "34.11.34.31/32", "34.182.140.95/32", "136.107.176.208/32", "34.182.226.151/32", "34.182.226.221/32", "35.221.29.174/32", "34.182.225.167/32" ], "rangesAsOf": "2026-05-01", "path": "/crawlers/claudebot" }, { "slug": "claude-searchbot", "name": "Claude-SearchBot", "vendor": "Anthropic", "token": "Claude-SearchBot", "purpose": "search", "purposeLabel": "Search index", "ua": null, "uaNote": "Anthropic's crawler documentation names the robots.txt token but does not publish a full example user-agent string.", "method": "ip", "rangeSrc": "https://claude.com/crawling/bots.json", "docs": "https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "cost": "Disabling Claude-SearchBot prevents Anthropic indexing your content for search result quality. This is the Anthropic equivalent of blocking OAI-SearchBot: it is the visibility token, not the training token.", "substance": "Claude-SearchBot is the newest of Anthropic's three tokens and the one most often missing from robots.txt files that were written when ClaudeBot was the only name in circulation. A site that added a ClaudeBot disallow in 2024 to opt out of training did not opt out of search indexing, because tokens do not inherit. Anthropic states each setting is independent. It shares the single bots.json IP list with the other two Claude robots, so IP verification cannot separate it from ClaudeBot traffic.", "ranges": [ "216.73.216.0/22", "34.162.230.222/32", "34.162.244.71/32", "34.162.191.81/32", "34.150.241.79/32", "34.85.172.162/32", "35.245.175.129/32", "34.182.222.37/32", "34.186.108.163/32", "35.245.89.239/32", "34.182.161.143/32", "34.182.218.27/32", "34.182.220.85/32", "34.11.34.31/32", "34.182.140.95/32", "136.107.176.208/32", "34.182.226.151/32", "34.182.226.221/32", "35.221.29.174/32", "34.182.225.167/32" ], "rangesAsOf": "2026-05-01", "path": "/crawlers/claude-searchbot" }, { "slug": "claude-user", "name": "Claude-User", "vendor": "Anthropic", "token": "Claude-User", "purpose": "not-a-crawler", "purposeLabel": "User-triggered fetch", "ua": null, "uaNote": "Anthropic's crawler documentation names the robots.txt token but does not publish a full example user-agent string.", "method": "ip", "rangeSrc": "https://claude.com/crawling/bots.json", "docs": "https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "cost": "Anthropic states plainly that disabling Claude-User prevents their system retrieving your content in response to a user query, which may reduce your site's visibility for user-directed web search. This is the clearest published statement by any operator that blocking a user-triggered fetcher costs you visibility.", "substance": "Claude-User fires when a person asks Claude something that requires your page, so its traffic pattern is bursty and query-shaped rather than a systematic sweep. Anthropic's opt-out guidance applies to it the same way as the crawlers, through robots.txt rather than IP blocking, and it draws from the same shared bots.json range list. Because all three Claude robots resolve to the same IP set, a verified Claude-User request and a verified ClaudeBot request are indistinguishable at the network layer.", "ranges": [ "216.73.216.0/22", "34.162.230.222/32", "34.162.244.71/32", "34.162.191.81/32", "34.150.241.79/32", "34.85.172.162/32", "35.245.175.129/32", "34.182.222.37/32", "34.186.108.163/32", "35.245.89.239/32", "34.182.161.143/32", "34.182.218.27/32", "34.182.220.85/32", "34.11.34.31/32", "34.182.140.95/32", "136.107.176.208/32", "34.182.226.151/32", "34.182.226.221/32", "35.221.29.174/32", "34.182.225.167/32" ], "rangesAsOf": "2026-05-01", "path": "/crawlers/claude-user" }, { "slug": "perplexitybot", "name": "PerplexityBot", "vendor": "Perplexity", "token": "PerplexityBot", "purpose": "search", "purposeLabel": "Search index", "ua": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)", "method": "ip", "rangeSrc": "https://www.perplexity.com/perplexitybot.json", "docs": "https://docs.perplexity.ai/guides/bots", "cost": "Perplexity states PerplexityBot is what surfaces and links websites in Perplexity search results, and is not used to crawl content for AI foundation models. Blocking it removes you from Perplexity's results without buying you any training protection, because it was never a training crawler.", "substance": "PerplexityBot's published range is unusually small and almost entirely single-address /32 entries rather than blocks. That has a practical consequence: an allowlist built from this file is brittle, and a genuine PerplexityBot request from a newly added address will fail verification until the file is re-pulled. Perplexity's own documentation is also the only operator guidance in this set that walks through WAF configuration explicitly, including Cloudflare and AWS WAF, recommending a rule that requires both the user-agent and the IP set rather than either alone.", "ranges": [ "107.20.236.150/32", "3.224.62.45/32", "18.210.92.235/32", "3.222.232.239/32", "3.211.124.183/32", "3.231.139.107/32", "18.97.1.228/30", "18.97.9.96/29" ], "rangesAsOf": "2025-02-07", "path": "/crawlers/perplexitybot" }, { "slug": "perplexity-user", "name": "Perplexity-User", "vendor": "Perplexity", "token": "Perplexity-User", "purpose": "not-a-crawler", "purposeLabel": "User-triggered fetch", "ua": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)", "method": "ip", "rangeSrc": "https://www.perplexity.com/perplexity-user.json", "docs": "https://docs.perplexity.ai/guides/bots", "cost": "Blocking Perplexity-User stops the fetch that happens when a user's question needs your page, and Perplexity notes the answer may include a link to that page. You are declining a citation with a live reader attached to it.", "substance": "Perplexity documents that because a user requested the fetch, Perplexity-User generally ignores robots.txt rules. That is a stronger statement than OpenAI's may not apply, and it means a Disallow line is not a control for this agent, only a signal. If you actually need to stop it, the published four-prefix range file is the enforceable surface, and it is small enough to put in a WAF rule directly. Note that the perplexity.com URL the documentation gives redirects to perplexity.ai, which is where the file is actually served.", "ranges": [ "44.208.221.197/32", "34.193.163.52/32", "18.97.21.0/30", "18.97.43.80/29" ], "rangesAsOf": "2025-10-17", "path": "/crawlers/perplexity-user" }, { "slug": "googlebot", "name": "Googlebot", "vendor": "Google", "token": "Googlebot", "purpose": "search", "purposeLabel": "Search index", "ua": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "method": "ip+dns", "rangeSrc": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "verifyDoc": "https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests", "cost": "Blocking Googlebot removes you from Google Search. There is no version of this that is a partial cost. It is also not the AI training lever: that is Google-Extended, which is a separate token.", "substance": "Googlebot is the only crawler here with two independent verification paths that both work. The published common-crawlers.json is one; the other is a reverse DNS mask, where a genuine Googlebot address resolves to a crawl-*.googlebot.com or geo-crawl-*.geo.googlebot.com hostname, and the forward lookup of that hostname returns the original IP. Google also moved these files: the canonical path is now under /static/crawling/ipranges/, and allowlists still pointing at the older /static/search/apis/ipranges/ location are pinned to a URL Google no longer treats as primary.", "ranges": [ "192.178.4.0/27", "192.178.4.128/27", "192.178.4.160/27", "192.178.4.192/27", "192.178.4.224/27", "192.178.4.32/27", "192.178.4.64/27", "192.178.4.96/27", "192.178.5.0/27", "192.178.6.0/27", "192.178.6.128/27", "192.178.6.160/27", "192.178.6.192/27", "192.178.6.224/27", "192.178.6.32/27", "192.178.6.64/27", "192.178.6.96/27", "192.178.7.0/27", "192.178.7.128/27", "192.178.7.160/27", "192.178.7.192/27", "192.178.7.224/27", "192.178.7.32/27", "192.178.7.64/27", "192.178.7.96/27", "34.100.182.96/28", "34.101.50.144/28", "34.118.254.0/28", "34.118.66.0/28", "34.126.178.96/28", "34.146.150.144/28", "34.147.110.144/28", "34.151.74.144/28", "34.152.50.64/28", "34.154.114.144/28", "34.155.98.32/28", "34.165.18.176/28", "34.175.160.64/28", "34.176.130.16/28", "34.22.85.0/27", "34.64.82.64/28", "34.65.242.112/28", "34.80.50.80/28", "34.88.194.0/28", "34.89.10.80/28", "34.89.198.80/28", "34.96.162.48/28", "35.247.243.240/28", "66.249.64.0/27", "66.249.64.128/27", "66.249.64.160/27", "66.249.64.192/27", "66.249.64.224/27", "66.249.64.32/27", "66.249.64.64/27", "66.249.64.96/27", "66.249.65.0/27", "66.249.65.128/27", "66.249.65.160/27", "66.249.65.192/27", "66.249.65.224/27", "66.249.65.32/27", "66.249.65.64/27", "66.249.65.96/27", "66.249.66.0/27", "66.249.66.128/27", "66.249.66.160/27", "66.249.66.192/27", "66.249.66.224/27", "66.249.66.32/27", "66.249.66.64/27", "66.249.66.96/27", "66.249.67.0/27", "66.249.67.32/27", "66.249.67.64/27", "66.249.68.0/27", "66.249.68.128/27", "66.249.68.160/27", "66.249.68.192/27", "66.249.68.32/27", "66.249.68.64/27", "66.249.68.96/27", "66.249.69.0/27", "66.249.69.128/27", "66.249.69.160/27", "66.249.69.192/27", "66.249.69.224/27", "66.249.69.32/27", "66.249.69.64/27", "66.249.69.96/27", "66.249.70.0/27", "66.249.70.128/27", "66.249.70.160/27", "66.249.70.192/27", "66.249.70.224/27", "66.249.70.32/27", "66.249.70.64/27", "66.249.70.96/27", "66.249.71.0/27", "66.249.71.128/27", "66.249.71.160/27", "66.249.71.192/27", "66.249.71.224/27", "66.249.71.32/27", "66.249.71.64/27", "66.249.71.96/27", "66.249.72.0/27", "66.249.72.128/27", "66.249.72.160/27", "66.249.72.192/27", "66.249.72.224/27", "66.249.72.32/27", "66.249.72.64/27", "66.249.72.96/27", "66.249.73.0/27", "66.249.73.128/27", "66.249.73.160/27", "66.249.73.192/27", "66.249.73.224/27", "66.249.73.32/27", "66.249.73.64/27", "66.249.73.96/27", "66.249.74.0/27", "66.249.74.128/27", "66.249.74.160/27", "66.249.74.192/27", "66.249.74.224/27", "66.249.74.32/27", "66.249.74.64/27", "66.249.74.96/27", "66.249.75.0/27", "66.249.75.128/27", "66.249.75.160/27", "66.249.75.192/27", "66.249.75.224/27", "66.249.75.32/27", "66.249.75.64/27", "66.249.75.96/27", "66.249.76.0/27", "66.249.76.128/27", "66.249.76.160/27", "66.249.76.192/27", "66.249.76.224/27", "66.249.76.32/27", "66.249.76.64/27", "66.249.76.96/27", "66.249.77.0/27", "66.249.77.128/27", "66.249.77.160/27", "66.249.77.192/27", "66.249.77.224/27", "66.249.77.32/27", "66.249.77.64/27", "66.249.77.96/27", "66.249.78.0/27", "66.249.78.128/27", "66.249.78.160/27", "66.249.78.192/27", "66.249.78.224/27", "66.249.78.32/27", "66.249.78.64/27", "66.249.78.96/27", "66.249.79.0/27", "66.249.79.128/27", "66.249.79.160/27", "66.249.79.192/27", "66.249.79.224/27", "66.249.79.32/27", "66.249.79.64/27" ], "rangesAsOf": "2026-07-31", "path": "/crawlers/googlebot" }, { "slug": "googleother", "name": "GoogleOther", "vendor": "Google", "token": "GoogleOther", "purpose": "retrieval", "purposeLabel": "Internal fetch and analysis", "ua": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GoogleOther) Chrome/W.X.Y.Z Safari/537.36", "method": "ip+dns", "rangeSrc": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "verifyDoc": "https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests", "cost": "Blocking GoogleOther does not affect Google Search rankings or indexing, because GoogleOther crawl preferences are not tied to a specific product. The cost is close to zero for search visibility, which makes it one of the few tokens you can disallow without a tradeoff argument.", "substance": "GoogleOther is the generic token Google uses for one-off crawls and internal research that are not Search, which is why its preferences are documented as affecting no specific product. It is easy to file it in the wrong bucket: despite not serving Search, Google classifies it as a common crawler, so it shares Googlebot's common-crawlers.json range file and its crawl-*.googlebot.com reverse DNS mask. The special-crawlers.json file covers a different set entirely, the crawlers Google says may ignore robots.txt, which is AdsBot, AdSense, APIs-Google and Google-Safety. GoogleOther also has two narrower siblings, GoogleOther-Image and GoogleOther-Video, and a rule addressed to the parent token applies to both, since Google matches on any one of a crawler's tokens.", "ranges": [ "192.178.4.0/27", "192.178.4.128/27", "192.178.4.160/27", "192.178.4.192/27", "192.178.4.224/27", "192.178.4.32/27", "192.178.4.64/27", "192.178.4.96/27", "192.178.5.0/27", "192.178.6.0/27", "192.178.6.128/27", "192.178.6.160/27", "192.178.6.192/27", "192.178.6.224/27", "192.178.6.32/27", "192.178.6.64/27", "192.178.6.96/27", "192.178.7.0/27", "192.178.7.128/27", "192.178.7.160/27", "192.178.7.192/27", "192.178.7.224/27", "192.178.7.32/27", "192.178.7.64/27", "192.178.7.96/27", "34.100.182.96/28", "34.101.50.144/28", "34.118.254.0/28", "34.118.66.0/28", "34.126.178.96/28", "34.146.150.144/28", "34.147.110.144/28", "34.151.74.144/28", "34.152.50.64/28", "34.154.114.144/28", "34.155.98.32/28", "34.165.18.176/28", "34.175.160.64/28", "34.176.130.16/28", "34.22.85.0/27", "34.64.82.64/28", "34.65.242.112/28", "34.80.50.80/28", "34.88.194.0/28", "34.89.10.80/28", "34.89.198.80/28", "34.96.162.48/28", "35.247.243.240/28", "66.249.64.0/27", "66.249.64.128/27", "66.249.64.160/27", "66.249.64.192/27", "66.249.64.224/27", "66.249.64.32/27", "66.249.64.64/27", "66.249.64.96/27", "66.249.65.0/27", "66.249.65.128/27", "66.249.65.160/27", "66.249.65.192/27", "66.249.65.224/27", "66.249.65.32/27", "66.249.65.64/27", "66.249.65.96/27", "66.249.66.0/27", "66.249.66.128/27", "66.249.66.160/27", "66.249.66.192/27", "66.249.66.224/27", "66.249.66.32/27", "66.249.66.64/27", "66.249.66.96/27", "66.249.67.0/27", "66.249.67.32/27", "66.249.67.64/27", "66.249.68.0/27", "66.249.68.128/27", "66.249.68.160/27", "66.249.68.192/27", "66.249.68.32/27", "66.249.68.64/27", "66.249.68.96/27", "66.249.69.0/27", "66.249.69.128/27", "66.249.69.160/27", "66.249.69.192/27", "66.249.69.224/27", "66.249.69.32/27", "66.249.69.64/27", "66.249.69.96/27", "66.249.70.0/27", "66.249.70.128/27", "66.249.70.160/27", "66.249.70.192/27", "66.249.70.224/27", "66.249.70.32/27", "66.249.70.64/27", "66.249.70.96/27", "66.249.71.0/27", "66.249.71.128/27", "66.249.71.160/27", "66.249.71.192/27", "66.249.71.224/27", "66.249.71.32/27", "66.249.71.64/27", "66.249.71.96/27", "66.249.72.0/27", "66.249.72.128/27", "66.249.72.160/27", "66.249.72.192/27", "66.249.72.224/27", "66.249.72.32/27", "66.249.72.64/27", "66.249.72.96/27", "66.249.73.0/27", "66.249.73.128/27", "66.249.73.160/27", "66.249.73.192/27", "66.249.73.224/27", "66.249.73.32/27", "66.249.73.64/27", "66.249.73.96/27", "66.249.74.0/27", "66.249.74.128/27", "66.249.74.160/27", "66.249.74.192/27", "66.249.74.224/27", "66.249.74.32/27", "66.249.74.64/27", "66.249.74.96/27", "66.249.75.0/27", "66.249.75.128/27", "66.249.75.160/27", "66.249.75.192/27", "66.249.75.224/27", "66.249.75.32/27", "66.249.75.64/27", "66.249.75.96/27", "66.249.76.0/27", "66.249.76.128/27", "66.249.76.160/27", "66.249.76.192/27", "66.249.76.224/27", "66.249.76.32/27", "66.249.76.64/27", "66.249.76.96/27", "66.249.77.0/27", "66.249.77.128/27", "66.249.77.160/27", "66.249.77.192/27", "66.249.77.224/27", "66.249.77.32/27", "66.249.77.64/27", "66.249.77.96/27", "66.249.78.0/27", "66.249.78.128/27", "66.249.78.160/27", "66.249.78.192/27", "66.249.78.224/27", "66.249.78.32/27", "66.249.78.64/27", "66.249.78.96/27", "66.249.79.0/27", "66.249.79.128/27", "66.249.79.160/27", "66.249.79.192/27", "66.249.79.224/27", "66.249.79.32/27", "66.249.79.64/27" ], "rangesAsOf": "2026-07-31", "path": "/crawlers/googleother" }, { "slug": "googlebot-news", "name": "Googlebot News", "vendor": "Google", "token": "Googlebot-News", "purpose": "search", "purposeLabel": "Google News index", "ua": null, "uaNote": "Google states Googlebot-News has no separate HTTP user-agent string. Crawling is done with the various Googlebot user agents; the token exists only as a robots.txt control.", "method": "ip+dns", "rangeSrc": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "verifyDoc": "https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests", "cost": "Disallowing Googlebot-News removes you from Google News, including news.google.com and the Google News app, while leaving ordinary Google Search results in place. For a publisher this is a large and very specific loss.", "substance": "Googlebot-News is a control token wearing a crawler's name. It has no user-agent of its own, so you cannot tell from a log line whether a Googlebot request was made for News or for Search, and there is no separate crawl to rate-limit. It also carries two tokens, Googlebot-News and Googlebot, and Google applies a rule if either matches. The consequence catches publishers out: a blanket Googlebot disallow removes you from News as well, but a Googlebot-News disallow does not touch Search. The narrower token is the safe one to reach for.", "ranges": [ "192.178.4.0/27", "192.178.4.128/27", "192.178.4.160/27", "192.178.4.192/27", "192.178.4.224/27", "192.178.4.32/27", "192.178.4.64/27", "192.178.4.96/27", "192.178.5.0/27", "192.178.6.0/27", "192.178.6.128/27", "192.178.6.160/27", "192.178.6.192/27", "192.178.6.224/27", "192.178.6.32/27", "192.178.6.64/27", "192.178.6.96/27", "192.178.7.0/27", "192.178.7.128/27", "192.178.7.160/27", "192.178.7.192/27", "192.178.7.224/27", "192.178.7.32/27", "192.178.7.64/27", "192.178.7.96/27", "34.100.182.96/28", "34.101.50.144/28", "34.118.254.0/28", "34.118.66.0/28", "34.126.178.96/28", "34.146.150.144/28", "34.147.110.144/28", "34.151.74.144/28", "34.152.50.64/28", "34.154.114.144/28", "34.155.98.32/28", "34.165.18.176/28", "34.175.160.64/28", "34.176.130.16/28", "34.22.85.0/27", "34.64.82.64/28", "34.65.242.112/28", "34.80.50.80/28", "34.88.194.0/28", "34.89.10.80/28", "34.89.198.80/28", "34.96.162.48/28", "35.247.243.240/28", "66.249.64.0/27", "66.249.64.128/27", "66.249.64.160/27", "66.249.64.192/27", "66.249.64.224/27", "66.249.64.32/27", "66.249.64.64/27", "66.249.64.96/27", "66.249.65.0/27", "66.249.65.128/27", "66.249.65.160/27", "66.249.65.192/27", "66.249.65.224/27", "66.249.65.32/27", "66.249.65.64/27", "66.249.65.96/27", "66.249.66.0/27", "66.249.66.128/27", "66.249.66.160/27", "66.249.66.192/27", "66.249.66.224/27", "66.249.66.32/27", "66.249.66.64/27", "66.249.66.96/27", "66.249.67.0/27", "66.249.67.32/27", "66.249.67.64/27", "66.249.68.0/27", "66.249.68.128/27", "66.249.68.160/27", "66.249.68.192/27", "66.249.68.32/27", "66.249.68.64/27", "66.249.68.96/27", "66.249.69.0/27", "66.249.69.128/27", "66.249.69.160/27", "66.249.69.192/27", "66.249.69.224/27", "66.249.69.32/27", "66.249.69.64/27", "66.249.69.96/27", "66.249.70.0/27", "66.249.70.128/27", "66.249.70.160/27", "66.249.70.192/27", "66.249.70.224/27", "66.249.70.32/27", "66.249.70.64/27", "66.249.70.96/27", "66.249.71.0/27", "66.249.71.128/27", "66.249.71.160/27", "66.249.71.192/27", "66.249.71.224/27", "66.249.71.32/27", "66.249.71.64/27", "66.249.71.96/27", "66.249.72.0/27", "66.249.72.128/27", "66.249.72.160/27", "66.249.72.192/27", "66.249.72.224/27", "66.249.72.32/27", "66.249.72.64/27", "66.249.72.96/27", "66.249.73.0/27", "66.249.73.128/27", "66.249.73.160/27", "66.249.73.192/27", "66.249.73.224/27", "66.249.73.32/27", "66.249.73.64/27", "66.249.73.96/27", "66.249.74.0/27", "66.249.74.128/27", "66.249.74.160/27", "66.249.74.192/27", "66.249.74.224/27", "66.249.74.32/27", "66.249.74.64/27", "66.249.74.96/27", "66.249.75.0/27", "66.249.75.128/27", "66.249.75.160/27", "66.249.75.192/27", "66.249.75.224/27", "66.249.75.32/27", "66.249.75.64/27", "66.249.75.96/27", "66.249.76.0/27", "66.249.76.128/27", "66.249.76.160/27", "66.249.76.192/27", "66.249.76.224/27", "66.249.76.32/27", "66.249.76.64/27", "66.249.76.96/27", "66.249.77.0/27", "66.249.77.128/27", "66.249.77.160/27", "66.249.77.192/27", "66.249.77.224/27", "66.249.77.32/27", "66.249.77.64/27", "66.249.77.96/27", "66.249.78.0/27", "66.249.78.128/27", "66.249.78.160/27", "66.249.78.192/27", "66.249.78.224/27", "66.249.78.32/27", "66.249.78.64/27", "66.249.78.96/27", "66.249.79.0/27", "66.249.79.128/27", "66.249.79.160/27", "66.249.79.192/27", "66.249.79.224/27", "66.249.79.32/27", "66.249.79.64/27" ], "rangesAsOf": "2026-07-31", "path": "/crawlers/googlebot-news" }, { "slug": "googleother-image", "name": "GoogleOther-Image", "vendor": "Google", "token": "GoogleOther-Image", "purpose": "retrieval", "purposeLabel": "Internal fetch, images", "ua": "GoogleOther-Image/1.0", "method": "ip+dns", "rangeSrc": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "verifyDoc": "https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests", "cost": "Like its parent GoogleOther, crawl preferences for this token do not affect any specific Google product, so blocking it costs you no search visibility. If you serve expensive image assets it is one of the cheapest tokens to disallow.", "substance": "GoogleOther-Image publishes one of the shortest user-agent strings of any major crawler: the literal token and a version, with no Mozilla prefix, no browser identifiers, and no URL. Log filters written to match on Mozilla or on a trailing bot URL will miss it entirely. It is also the variant optimised for fetching publicly accessible image URLs, so its traffic pattern is asset-heavy rather than page-heavy, and bandwidth cost is the usual reason anyone notices it at all. A robots.txt rule addressed to GoogleOther applies to it too.", "ranges": [ "192.178.4.0/27", "192.178.4.128/27", "192.178.4.160/27", "192.178.4.192/27", "192.178.4.224/27", "192.178.4.32/27", "192.178.4.64/27", "192.178.4.96/27", "192.178.5.0/27", "192.178.6.0/27", "192.178.6.128/27", "192.178.6.160/27", "192.178.6.192/27", "192.178.6.224/27", "192.178.6.32/27", "192.178.6.64/27", "192.178.6.96/27", "192.178.7.0/27", "192.178.7.128/27", "192.178.7.160/27", "192.178.7.192/27", "192.178.7.224/27", "192.178.7.32/27", "192.178.7.64/27", "192.178.7.96/27", "34.100.182.96/28", "34.101.50.144/28", "34.118.254.0/28", "34.118.66.0/28", "34.126.178.96/28", "34.146.150.144/28", "34.147.110.144/28", "34.151.74.144/28", "34.152.50.64/28", "34.154.114.144/28", "34.155.98.32/28", "34.165.18.176/28", "34.175.160.64/28", "34.176.130.16/28", "34.22.85.0/27", "34.64.82.64/28", "34.65.242.112/28", "34.80.50.80/28", "34.88.194.0/28", "34.89.10.80/28", "34.89.198.80/28", "34.96.162.48/28", "35.247.243.240/28", "66.249.64.0/27", "66.249.64.128/27", "66.249.64.160/27", "66.249.64.192/27", "66.249.64.224/27", "66.249.64.32/27", "66.249.64.64/27", "66.249.64.96/27", "66.249.65.0/27", "66.249.65.128/27", "66.249.65.160/27", "66.249.65.192/27", "66.249.65.224/27", "66.249.65.32/27", "66.249.65.64/27", "66.249.65.96/27", "66.249.66.0/27", "66.249.66.128/27", "66.249.66.160/27", "66.249.66.192/27", "66.249.66.224/27", "66.249.66.32/27", "66.249.66.64/27", "66.249.66.96/27", "66.249.67.0/27", "66.249.67.32/27", "66.249.67.64/27", "66.249.68.0/27", "66.249.68.128/27", "66.249.68.160/27", "66.249.68.192/27", "66.249.68.32/27", "66.249.68.64/27", "66.249.68.96/27", "66.249.69.0/27", "66.249.69.128/27", "66.249.69.160/27", "66.249.69.192/27", "66.249.69.224/27", "66.249.69.32/27", "66.249.69.64/27", "66.249.69.96/27", "66.249.70.0/27", "66.249.70.128/27", "66.249.70.160/27", "66.249.70.192/27", "66.249.70.224/27", "66.249.70.32/27", "66.249.70.64/27", "66.249.70.96/27", "66.249.71.0/27", "66.249.71.128/27", "66.249.71.160/27", "66.249.71.192/27", "66.249.71.224/27", "66.249.71.32/27", "66.249.71.64/27", "66.249.71.96/27", "66.249.72.0/27", "66.249.72.128/27", "66.249.72.160/27", "66.249.72.192/27", "66.249.72.224/27", "66.249.72.32/27", "66.249.72.64/27", "66.249.72.96/27", "66.249.73.0/27", "66.249.73.128/27", "66.249.73.160/27", "66.249.73.192/27", "66.249.73.224/27", "66.249.73.32/27", "66.249.73.64/27", "66.249.73.96/27", "66.249.74.0/27", "66.249.74.128/27", "66.249.74.160/27", "66.249.74.192/27", "66.249.74.224/27", "66.249.74.32/27", "66.249.74.64/27", "66.249.74.96/27", "66.249.75.0/27", "66.249.75.128/27", "66.249.75.160/27", "66.249.75.192/27", "66.249.75.224/27", "66.249.75.32/27", "66.249.75.64/27", "66.249.75.96/27", "66.249.76.0/27", "66.249.76.128/27", "66.249.76.160/27", "66.249.76.192/27", "66.249.76.224/27", "66.249.76.32/27", "66.249.76.64/27", "66.249.76.96/27", "66.249.77.0/27", "66.249.77.128/27", "66.249.77.160/27", "66.249.77.192/27", "66.249.77.224/27", "66.249.77.32/27", "66.249.77.64/27", "66.249.77.96/27", "66.249.78.0/27", "66.249.78.128/27", "66.249.78.160/27", "66.249.78.192/27", "66.249.78.224/27", "66.249.78.32/27", "66.249.78.64/27", "66.249.78.96/27", "66.249.79.0/27", "66.249.79.128/27", "66.249.79.160/27", "66.249.79.192/27", "66.249.79.224/27", "66.249.79.32/27", "66.249.79.64/27" ], "rangesAsOf": "2026-07-31", "path": "/crawlers/googleother-image" }, { "slug": "googleother-video", "name": "GoogleOther-Video", "vendor": "Google", "token": "GoogleOther-Video", "purpose": "retrieval", "purposeLabel": "Internal fetch, video", "ua": "GoogleOther-Video/1.0", "method": "ip+dns", "rangeSrc": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "verifyDoc": "https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests", "cost": "No specific Google product depends on it, so blocking costs no search visibility. For sites serving large video files this is the token where blocking has a measurable effect on your bandwidth bill rather than on your traffic.", "substance": "GoogleOther-Video is the video-optimised variant of GoogleOther and shares its bare user-agent format, just the token and a version with no browser prefix. Because video fetches are large and often ranged, this is the Google token most likely to show up as unexplained egress rather than as pageviews, and the one people most often misidentify as an abusive scraper before checking it against Google's published ranges. It resolves under the same crawl-*.googlebot.com reverse DNS mask as Googlebot, so it is fully verifiable despite the thin user-agent.", "ranges": [ "192.178.4.0/27", "192.178.4.128/27", "192.178.4.160/27", "192.178.4.192/27", "192.178.4.224/27", "192.178.4.32/27", "192.178.4.64/27", "192.178.4.96/27", "192.178.5.0/27", "192.178.6.0/27", "192.178.6.128/27", "192.178.6.160/27", "192.178.6.192/27", "192.178.6.224/27", "192.178.6.32/27", "192.178.6.64/27", "192.178.6.96/27", "192.178.7.0/27", "192.178.7.128/27", "192.178.7.160/27", "192.178.7.192/27", "192.178.7.224/27", "192.178.7.32/27", "192.178.7.64/27", "192.178.7.96/27", "34.100.182.96/28", "34.101.50.144/28", "34.118.254.0/28", "34.118.66.0/28", "34.126.178.96/28", "34.146.150.144/28", "34.147.110.144/28", "34.151.74.144/28", "34.152.50.64/28", "34.154.114.144/28", "34.155.98.32/28", "34.165.18.176/28", "34.175.160.64/28", "34.176.130.16/28", "34.22.85.0/27", "34.64.82.64/28", "34.65.242.112/28", "34.80.50.80/28", "34.88.194.0/28", "34.89.10.80/28", "34.89.198.80/28", "34.96.162.48/28", "35.247.243.240/28", "66.249.64.0/27", "66.249.64.128/27", "66.249.64.160/27", "66.249.64.192/27", "66.249.64.224/27", "66.249.64.32/27", "66.249.64.64/27", "66.249.64.96/27", "66.249.65.0/27", "66.249.65.128/27", "66.249.65.160/27", "66.249.65.192/27", "66.249.65.224/27", "66.249.65.32/27", "66.249.65.64/27", "66.249.65.96/27", "66.249.66.0/27", "66.249.66.128/27", "66.249.66.160/27", "66.249.66.192/27", "66.249.66.224/27", "66.249.66.32/27", "66.249.66.64/27", "66.249.66.96/27", "66.249.67.0/27", "66.249.67.32/27", "66.249.67.64/27", "66.249.68.0/27", "66.249.68.128/27", "66.249.68.160/27", "66.249.68.192/27", "66.249.68.32/27", "66.249.68.64/27", "66.249.68.96/27", "66.249.69.0/27", "66.249.69.128/27", "66.249.69.160/27", "66.249.69.192/27", "66.249.69.224/27", "66.249.69.32/27", "66.249.69.64/27", "66.249.69.96/27", "66.249.70.0/27", "66.249.70.128/27", "66.249.70.160/27", "66.249.70.192/27", "66.249.70.224/27", "66.249.70.32/27", "66.249.70.64/27", "66.249.70.96/27", "66.249.71.0/27", "66.249.71.128/27", "66.249.71.160/27", "66.249.71.192/27", "66.249.71.224/27", "66.249.71.32/27", "66.249.71.64/27", "66.249.71.96/27", "66.249.72.0/27", "66.249.72.128/27", "66.249.72.160/27", "66.249.72.192/27", "66.249.72.224/27", "66.249.72.32/27", "66.249.72.64/27", "66.249.72.96/27", "66.249.73.0/27", "66.249.73.128/27", "66.249.73.160/27", "66.249.73.192/27", "66.249.73.224/27", "66.249.73.32/27", "66.249.73.64/27", "66.249.73.96/27", "66.249.74.0/27", "66.249.74.128/27", "66.249.74.160/27", "66.249.74.192/27", "66.249.74.224/27", "66.249.74.32/27", "66.249.74.64/27", "66.249.74.96/27", "66.249.75.0/27", "66.249.75.128/27", "66.249.75.160/27", "66.249.75.192/27", "66.249.75.224/27", "66.249.75.32/27", "66.249.75.64/27", "66.249.75.96/27", "66.249.76.0/27", "66.249.76.128/27", "66.249.76.160/27", "66.249.76.192/27", "66.249.76.224/27", "66.249.76.32/27", "66.249.76.64/27", "66.249.76.96/27", "66.249.77.0/27", "66.249.77.128/27", "66.249.77.160/27", "66.249.77.192/27", "66.249.77.224/27", "66.249.77.32/27", "66.249.77.64/27", "66.249.77.96/27", "66.249.78.0/27", "66.249.78.128/27", "66.249.78.160/27", "66.249.78.192/27", "66.249.78.224/27", "66.249.78.32/27", "66.249.78.64/27", "66.249.78.96/27", "66.249.79.0/27", "66.249.79.128/27", "66.249.79.160/27", "66.249.79.192/27", "66.249.79.224/27", "66.249.79.32/27", "66.249.79.64/27" ], "rangesAsOf": "2026-07-31", "path": "/crawlers/googleother-video" }, { "slug": "google-cloudvertexbot", "name": "Google-CloudVertexBot", "vendor": "Google", "token": "Google-CloudVertexBot", "purpose": "retrieval", "purposeLabel": "Site-owner requested crawl", "ua": "Google-CloudVertexBot", "method": "ip+dns", "rangeSrc": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json", "docs": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "verifyDoc": "https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests", "cost": "Google states this token has no effect on Google Search or other products. It only affects crawls that site owners themselves requested for building Vertex AI Agents, so blocking it breaks your own Vertex integration and nothing else.", "substance": "Google-CloudVertexBot is the odd one out in Google's crawler list because it does not crawl on Google's initiative at all. It fetches sites at the request of the site owner, when someone is building a Vertex AI Agent over their own content. That inverts the usual reasoning: if you see it in your logs and did not set up a Vertex agent, the interesting question is who in your organisation did, not whether Google is overreaching. Google documents it as a user-agent substring rather than a full string, so match on the substring and do not expect a versioned bot URL. It also carries Googlebot as a secondary token, which means a blanket Googlebot disallow will stop your own Vertex crawl.", "ranges": [ "192.178.4.0/27", "192.178.4.128/27", "192.178.4.160/27", "192.178.4.192/27", "192.178.4.224/27", "192.178.4.32/27", "192.178.4.64/27", "192.178.4.96/27", "192.178.5.0/27", "192.178.6.0/27", "192.178.6.128/27", "192.178.6.160/27", "192.178.6.192/27", "192.178.6.224/27", "192.178.6.32/27", "192.178.6.64/27", "192.178.6.96/27", "192.178.7.0/27", "192.178.7.128/27", "192.178.7.160/27", "192.178.7.192/27", "192.178.7.224/27", "192.178.7.32/27", "192.178.7.64/27", "192.178.7.96/27", "34.100.182.96/28", "34.101.50.144/28", "34.118.254.0/28", "34.118.66.0/28", "34.126.178.96/28", "34.146.150.144/28", "34.147.110.144/28", "34.151.74.144/28", "34.152.50.64/28", "34.154.114.144/28", "34.155.98.32/28", "34.165.18.176/28", "34.175.160.64/28", "34.176.130.16/28", "34.22.85.0/27", "34.64.82.64/28", "34.65.242.112/28", "34.80.50.80/28", "34.88.194.0/28", "34.89.10.80/28", "34.89.198.80/28", "34.96.162.48/28", "35.247.243.240/28", "66.249.64.0/27", "66.249.64.128/27", "66.249.64.160/27", "66.249.64.192/27", "66.249.64.224/27", "66.249.64.32/27", "66.249.64.64/27", "66.249.64.96/27", "66.249.65.0/27", "66.249.65.128/27", "66.249.65.160/27", "66.249.65.192/27", "66.249.65.224/27", "66.249.65.32/27", "66.249.65.64/27", "66.249.65.96/27", "66.249.66.0/27", "66.249.66.128/27", "66.249.66.160/27", "66.249.66.192/27", "66.249.66.224/27", "66.249.66.32/27", "66.249.66.64/27", "66.249.66.96/27", "66.249.67.0/27", "66.249.67.32/27", "66.249.67.64/27", "66.249.68.0/27", "66.249.68.128/27", "66.249.68.160/27", "66.249.68.192/27", "66.249.68.32/27", "66.249.68.64/27", "66.249.68.96/27", "66.249.69.0/27", "66.249.69.128/27", "66.249.69.160/27", "66.249.69.192/27", "66.249.69.224/27", "66.249.69.32/27", "66.249.69.64/27", "66.249.69.96/27", "66.249.70.0/27", "66.249.70.128/27", "66.249.70.160/27", "66.249.70.192/27", "66.249.70.224/27", "66.249.70.32/27", "66.249.70.64/27", "66.249.70.96/27", "66.249.71.0/27", "66.249.71.128/27", "66.249.71.160/27", "66.249.71.192/27", "66.249.71.224/27", "66.249.71.32/27", "66.249.71.64/27", "66.249.71.96/27", "66.249.72.0/27", "66.249.72.128/27", "66.249.72.160/27", "66.249.72.192/27", "66.249.72.224/27", "66.249.72.32/27", "66.249.72.64/27", "66.249.72.96/27", "66.249.73.0/27", "66.249.73.128/27", "66.249.73.160/27", "66.249.73.192/27", "66.249.73.224/27", "66.249.73.32/27", "66.249.73.64/27", "66.249.73.96/27", "66.249.74.0/27", "66.249.74.128/27", "66.249.74.160/27", "66.249.74.192/27", "66.249.74.224/27", "66.249.74.32/27", "66.249.74.64/27", "66.249.74.96/27", "66.249.75.0/27", "66.249.75.128/27", "66.249.75.160/27", "66.249.75.192/27", "66.249.75.224/27", "66.249.75.32/27", "66.249.75.64/27", "66.249.75.96/27", "66.249.76.0/27", "66.249.76.128/27", "66.249.76.160/27", "66.249.76.192/27", "66.249.76.224/27", "66.249.76.32/27", "66.249.76.64/27", "66.249.76.96/27", "66.249.77.0/27", "66.249.77.128/27", "66.249.77.160/27", "66.249.77.192/27", "66.249.77.224/27", "66.249.77.32/27", "66.249.77.64/27", "66.249.77.96/27", "66.249.78.0/27", "66.249.78.128/27", "66.249.78.160/27", "66.249.78.192/27", "66.249.78.224/27", "66.249.78.32/27", "66.249.78.64/27", "66.249.78.96/27", "66.249.79.0/27", "66.249.79.128/27", "66.249.79.160/27", "66.249.79.192/27", "66.249.79.224/27", "66.249.79.32/27", "66.249.79.64/27" ], "rangesAsOf": "2026-07-31", "path": "/crawlers/google-cloudvertexbot" }, { "slug": "google-extended", "name": "Google-Extended", "vendor": "Google", "token": "Google-Extended", "purpose": "control", "purposeLabel": "Control token, not a crawler", "ua": null, "uaNote": "Google states Google-Extended has no separate HTTP user-agent string. Crawling is done with existing Google user agents; the token exists only as a robots.txt control.", "method": "none", "rangeSrc": null, "docs": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "cost": "Disallowing Google-Extended withdraws your content from training future Gemini models and from grounding, where Google supplies content from its Search index to the model at prompt time. Your Search rankings are untouched. Your presence inside Gemini answers is what you are giving up.", "substance": "Google-Extended will never appear in your access logs, and that is the single most misunderstood fact about it. It has no user-agent string of its own. Google crawls with the existing Googlebot agents and then applies your Google-Extended preference afterwards, as a use-time control rather than a fetch-time one. This makes it unverifiable by design: there is no request to check, no IP to match, and any log line claiming to be Google-Extended is fabricated. It is also the only token in this set that governs grounding as well as training, so blocking it affects answers Gemini generates today, not just models trained tomorrow.", "ranges": null, "rangesAsOf": null, "path": "/crawlers/google-extended" }, { "slug": "bingbot", "name": "Bingbot", "vendor": "Microsoft", "token": "bingbot", "purpose": "search", "purposeLabel": "Search index", "ua": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/W.X.Y.Z Safari/537.36", "method": "ip", "rangeSrc": "https://www.bing.com/toolbox/bingbot.json", "docs": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", "cost": "Blocking bingbot removes you from Bing, and from the surfaces that draw on the Bing index rather than crawling independently, including Copilot. For most sites this is a larger loss than the Bing search share alone suggests.", "substance": "Bingbot's published range file has not been reissued in a long time. Its creationTime stamp is from January 2024, the oldest of any operator in this set by a wide margin, while OpenAI and Google refresh theirs within days. A stale published list is not the same as a wrong one, but it does mean a verification failure against bingbot.json is weaker evidence of spoofing than the same failure against a file republished last week. Microsoft also runs AdIdxBot on a separate token for Bing Ads quality checks, and its user-agent is close enough to bingbot's that log filters written loosely will catch both.", "ranges": [ "157.55.39.0/24", "207.46.13.0/24", "40.77.167.0/24", "13.66.139.0/24", "13.66.144.0/24", "52.167.144.0/24", "13.67.10.16/28", "13.69.66.240/28", "13.71.172.224/28", "139.217.52.0/28", "191.233.204.224/28", "20.36.108.32/28", "20.43.120.16/28", "40.79.131.208/28", "40.79.186.176/28", "52.231.148.0/28", "20.79.107.240/28", "51.105.67.0/28", "20.125.163.80/28", "40.77.188.0/22", "65.55.210.0/24", "199.30.24.0/23", "40.77.202.0/24", "40.77.139.0/25", "20.74.197.0/28", "20.15.133.160/27", "40.77.177.0/24", "40.77.178.0/23" ], "rangesAsOf": "2024-01-03", "path": "/crawlers/bingbot" }, { "slug": "adidxbot", "name": "AdIdxBot", "vendor": "Microsoft", "token": "adidxbot", "purpose": "retrieval", "purposeLabel": "Bing Ads quality control", "ua": "Mozilla/5.0 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", "method": "none", "rangeSrc": null, "docs": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", "cost": "AdIdxBot crawls ads and follows the sites those ads point to, for quality control. If you advertise on Bing, blocking it interferes with the checks that keep your ads running. If you do not advertise, blocking it costs nothing.", "substance": "AdIdxBot is the reason loose log filters overcount Bing traffic. Its user-agent points at the same bingbot.htm URL as bingbot itself and differs only in the token, so a filter matching on the bot URL or on the word bing catches both and attributes ad crawling to search crawling. Microsoft documents desktop and mobile variants, mirroring bingbot's structure. Note also that Microsoft's published bingbot.json is presented as bingbot's range list and Microsoft does not state that it covers AdIdxBot, so an IP check against that file is not a documented verification path for this token.", "ranges": null, "rangesAsOf": null, "path": "/crawlers/adidxbot" }, { "slug": "applebot", "name": "Applebot", "vendor": "Apple", "token": "Applebot", "purpose": "search", "purposeLabel": "Search index", "ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)", "method": "ip+dns", "rangeSrc": "https://search.developer.apple.com/applebot.json", "docs": "https://support.apple.com/en-us/119829", "cost": "Applebot feeds Spotlight, Siri and Safari. Blocking it removes your content from search results across Apple's ecosystem, and because Apple lists this as the primary purpose ahead of any AI use, the visibility cost lands first and the training question is handled by a different token.", "substance": "Apple names reverse DNS as the primary identification method and the CIDR file as the secondary one, which is the reverse of how most operators present it. Genuine Applebot addresses resolve under applebot.apple.com, and Apple's documentation walks through the two host commands needed to confirm both directions. Worth knowing when reading verification results: the published applebot.json carries a creationTime from October 2023, so like Bing's it is a long-lived list rather than a frequently rotated one. Apple also documents a separate nosnippet control for broad world knowledge answers in Siri and Search, which is independent of both Applebot and Applebot-Extended.", "ranges": [ "17.241.208.160/27", "17.241.193.160/27", "17.241.200.160/27", "17.22.237.0/24", "17.22.245.0/24", "17.22.253.0/24", "17.241.75.0/24", "17.241.219.0/24", "17.241.227.0/24", "17.246.15.0/24", "17.246.19.0/24", "17.246.23.0/24" ], "rangesAsOf": "2023-10-27", "path": "/crawlers/applebot" }, { "slug": "applebot-extended", "name": "Applebot-Extended", "vendor": "Apple", "token": "Applebot-Extended", "purpose": "control", "purposeLabel": "Control token, not a crawler", "ua": null, "uaNote": "Apple states Applebot-Extended does not crawl webpages. It is a robots.txt control that governs how data already crawled by Applebot may be used.", "method": "none", "rangeSrc": null, "docs": "https://support.apple.com/en-us/119829", "cost": "Disallowing Applebot-Extended opts your content out of training Apple's general purpose foundation models, including the ones behind Apple Intelligence. Apple is explicit that pages disallowing it can still appear in search results, so the visibility cost is nil. This is close to a free opt-out.", "substance": "Applebot-Extended does not crawl, so it will never show up in your logs and cannot be verified or spoofed. Apple describes it as determining how data crawled by the Applebot user agent may be used, which means it operates entirely after the fetch. The practical consequence is that disallowing it does not reduce your crawl traffic by a single request. It also does not cover everything: Apple documents that even with Applebot-Extended disallowed and nosnippet applied, content remains discoverable through Spotlight, Siri and Safari.", "ranges": null, "rangesAsOf": null, "path": "/crawlers/applebot-extended" }, { "slug": "meta-externalagent", "name": "Meta-ExternalAgent", "vendor": "Meta", "token": "Meta-ExternalAgent", "purpose": "training", "purposeLabel": "AI training and indexing crawler", "ua": "meta-externalagent/1.1 (+/documentation/sharing/webmasters/web-crawlers)", "method": "none", "rangeSrc": null, "docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "cost": "Meta describes this crawler as serving both foundation model training and direct content indexing, bundled under one token. That bundling is the cost: there is no published way to opt out of the training use while keeping the indexing use, so blocking is all or nothing.", "substance": "Meta publishes no IP range file for its crawlers, so a Meta-ExternalAgent request cannot be verified the way an OpenAI or Google request can. Its own documentation points site owners to allowlisting either the user-agent strings or the IP addresses, describing the latter as more secure, but the addresses are not published as a machine-readable list the way every other major operator's are. The user-agent itself is also unusually thin: the shorter published form is just meta-externalagent/1.1 with no browser prefix, which makes it trivially cheap to forge and easy to miss in log filters written to expect a Mozilla/5.0 prefix.", "ranges": null, "rangesAsOf": null, "path": "/crawlers/meta-externalagent" }, { "slug": "meta-externalfetcher", "name": "Meta-ExternalFetcher", "vendor": "Meta", "token": "Meta-ExternalFetcher", "purpose": "not-a-crawler", "purposeLabel": "User-triggered fetch", "ua": "meta-externalfetcher/1.1 (+/documentation/sharing/webmasters/web-crawlers)", "method": "none", "rangeSrc": null, "docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "cost": "This fetcher supports agentic AI navigating websites to complete tasks for users. Blocking it means Meta's assistant cannot act on your site on a user's behalf, which matters more for transactional pages than for editorial ones.", "substance": "Meta states directly that Meta-ExternalFetcher may bypass robots.txt rules, because it fetches individual links at a user's request. Combined with the absence of any published IP list, this leaves site owners with no enforceable control and no verification path at all: robots.txt may be ignored, and there is nothing to check a source IP against. Of every agent in this reference, this is the one where a claimed identity is least provable. Meta's crawler documentation was last updated on 21 May 2026 and now also lists Meta-WebIndexer, a separate token for Meta AI search quality that did not exist in earlier versions of the page.", "ranges": null, "rangesAsOf": null, "path": "/crawlers/meta-externalfetcher" }, { "slug": "meta-webindexer", "name": "Meta-WebIndexer", "vendor": "Meta", "token": "Meta-WebIndexer", "purpose": "search", "purposeLabel": "Meta AI search index", "ua": "meta-webindexer/1.1 (+/documentation/sharing/webmasters/web-crawlers)", "method": "none", "rangeSrc": null, "docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "cost": "Meta states that allowing Meta-WebIndexer helps it cite and link to your content in Meta AI's responses. Blocking it is a straight trade of crawl traffic for citations, and it is the only Meta token where Meta makes the citation benefit explicit.", "substance": "Meta-WebIndexer is new. It does not appear in older versions of Meta's crawler documentation, which is why robots.txt files written around Meta-ExternalAgent do not cover it, and tokens do not inherit. If you opted out of Meta's AI crawling before mid 2026, you are almost certainly still being indexed for Meta AI search. Meta's page was last updated on 21 May 2026. Like every Meta crawler it has no published IP range file, so the token is the only control and the identity is unverifiable.", "ranges": null, "rangesAsOf": null, "path": "/crawlers/meta-webindexer" }, { "slug": "meta-externalads", "name": "Meta-ExternalAds", "vendor": "Meta", "token": "Meta-ExternalAds", "purpose": "retrieval", "purposeLabel": "Advertising and business products", "ua": "meta-externalads/1.1 (+/documentation/sharing/webmasters/web-crawlers)", "method": "none", "rangeSrc": null, "docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "cost": "Meta describes this crawler as improving advertising and other business-related products. If you advertise on Meta, blocking it degrades the signal Meta uses on your own campaigns. If you do not, the cost is close to nothing.", "substance": "Meta-ExternalAds is the least discussed of Meta's five tokens and the one most likely to be missing from a robots.txt that was written to block Meta's AI crawling. It is a separate token from Meta-ExternalAgent, so a disallow aimed at training does not touch it. Meta publishes no IP ranges for any of its crawlers, so as with the rest of the set the user-agent is the only signal and it cannot be verified. The published form is a bare meta-externalads/1.1 with no browser prefix.", "ranges": null, "rangesAsOf": null, "path": "/crawlers/meta-externalads" }, { "slug": "facebookexternalhit", "name": "FacebookExternalHit", "vendor": "Meta", "token": "facebookexternalhit", "purpose": "retrieval", "purposeLabel": "Link preview fetch", "ua": "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)", "method": "none", "rangeSrc": null, "docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/", "cost": "Blocking FacebookExternalHit means links to your site shared on Facebook, Instagram or Messenger render without a title, description or thumbnail. The link still works, it just looks broken, which measurably suppresses clicks. This is the highest-cost block on this list for most consumer sites.", "substance": "FacebookExternalHit is the oldest crawler in this reference and the only one with documented performance requirements you can fail. Meta states your server must support gzip and deflate, that Open Graph properties must appear within the first 1 MB or they are cut off, and that the page must be crawlable within a few seconds or the preview cannot be displayed. It also either honours the Range header it sends or requires you to ignore it entirely. Meta additionally states the crawler may bypass robots.txt when performing security or integrity checks such as malware scanning, so a disallow is not a guarantee. When a preview looks wrong the fix is usually the 1 MB rule, not the block.", "ranges": null, "rangesAsOf": null, "path": "/crawlers/facebookexternalhit" }, { "slug": "amzn-searchbot", "name": "Amzn-SearchBot", "vendor": "Amazon", "token": "Amzn-SearchBot", "purpose": "search", "purposeLabel": "Amazon and Alexa search", "ua": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-SearchBot/0.1) Chrome/W.X.Y.Z Safari/537.36", "method": "ip-html", "rangeSrc": null, "rangeDoc": "https://developer.amazon.com/amazonbot/searchbot-ip-addresses/", "docs": "https://developer.amazon.com/amazonbot", "cost": "Amazon states that permitting Amzn-SearchBot makes your content eligible to appear in search experiences such as Alexa. It also states the token does not crawl content for generative AI model training, so blocking it costs visibility and buys no training protection.", "substance": "Amzn-SearchBot has a default that no other crawler in this reference has: Amazon states that if your robots.txt does not mention Amzn-SearchBot but does allow other search bots, it will crawl according to the rules you gave those other bots. Silence is read as consent by proxy. To actually exclude it you have to name it. Its IP page is also the only one of Amazon's three that ships CIDR blocks in static HTML, currently 512 of them, so it is the one Amazon token you can build an automated allowlist for without rendering JavaScript.", "ranges": null, "rangesAsOf": null, "path": "/crawlers/amzn-searchbot" }, { "slug": "amzn-user", "name": "Amzn-User", "vendor": "Amazon", "token": "Amzn-User", "purpose": "not-a-crawler", "purposeLabel": "User-triggered fetch", "ua": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amzn-User/0.1) Chrome/W.X.Y.Z Safari/537.36", "method": "ip-html", "rangeSrc": null, "rangeDoc": "https://developer.amazon.com/amazonbot/live-ip-addresses/", "docs": "https://developer.amazon.com/amazonbot", "cost": "Amzn-User fetches live information to answer a customer's question, for example an Alexa query that needs current data. Blocking it means Alexa answers about your content from stale or third-party sources instead of from you. Amazon states it is not used for generative AI training.", "substance": "Amazon calls the endpoint behind this token the live crawl IP list, which is a useful description of the traffic: it arrives when a person asks a question, not on a crawl schedule. Because the action is user-initiated, Amazon states it may not follow all robots.txt directives, putting it in the same category as ChatGPT-User and Perplexity-User. Its published IP page is client-rendered and yields zero CIDR blocks when fetched without JavaScript, so unlike Amzn-SearchBot there is no practical automated verification path.", "ranges": null, "rangesAsOf": null, "path": "/crawlers/amzn-user" }, { "slug": "amazonbot", "name": "Amazonbot", "vendor": "Amazon", "token": "Amazonbot", "purpose": "training", "purposeLabel": "Product improvement and AI training", "ua": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amazonbot/0.1) Chrome/W.X.Y.Z Safari/537.36", "method": "ip-html", "rangeSrc": null, "rangeDoc": "https://developer.amazon.com/amazonbot/ip-addresses/", "docs": "https://developer.amazon.com/amazonbot", "cost": "Amazon says Amazonbot data improves its products and services and may be used to train Amazon AI models. Search visibility in Alexa is governed by the separate Amzn-SearchBot token, so blocking Amazonbot alone does not cost you Alexa surfacing.", "substance": "Amazon publishes its IP addresses on HTML pages rather than as JSON files, and the three pages do not behave the same way. The Amzn-SearchBot page ships 512 CIDR blocks in its static HTML and can be scraped. The Amazonbot page and the Amzn-User page return zero without executing JavaScript, so the addresses for Amazon's training crawler, the one most site owners actually want to gate, are the ones you cannot fetch. That inconsistency is not documented anywhere; it is only visible by fetching all three. Amazon also warns that a robots.txt change can take around 24 hours to take effect, and notes that if your robots.txt does not mention Amzn-SearchBot but allows other search bots, Amazon will crawl in accordance with the rules you gave those other bots.", "ranges": null, "rangesAsOf": null, "path": "/crawlers/amazonbot" }, { "slug": "ccbot", "name": "CCBot", "vendor": "Common Crawl Foundation", "token": "CCBot", "purpose": "training", "purposeLabel": "Open crawl archive", "ua": "CCBot/2.0 (https://commoncrawl.org/faq/)", "method": "dns", "rangeSrc": null, "docs": "https://commoncrawl.org/ccbot", "cost": "CCBot feeds an open, freely redistributable archive. Blocking it removes you from a dataset that many downstream model builders use, which is a broader training opt-out than blocking any single operator, but it also removes you from academic and non-profit research corpora that have nothing to do with commercial AI.", "substance": "Common Crawl is the only operator in this reference that publicly acknowledges being impersonated. Its own CCBot page states that it is aware of crawlers falsely identifying themselves as CCBot and recommends verifying user-agent strings to ensure authenticity. It moved CCBot onto dedicated IP ranges with reverse DNS specifically so this could be checked: a genuine request resolves under crawl.commoncrawl.org, and the forward lookup returns the same address. The exception is IPv6, where Common Crawl states reverse DNS is not yet supported, so an IPv6 CCBot request has no verification path at all. Blocking is also unusually consequential in one direction: content already collected sits in published archives, so a robots.txt change stops future collection but does not retract past crawls.", "ranges": null, "rangesAsOf": null, "path": "/crawlers/ccbot" } ] }