{
  "checkedOn": "2026-09-27",
  "hosts": {
    "www.amazon.de": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.amazon.de",
      "body": "User-agent: *\nDisallow: /dp/product-availability/\nDisallow: /dp/rate-this-item/\nDisallow: /exec/obidos/account-access-login\nDisallow: /exec/obidos/change-style\nDisallow: /exec/obidos/di\nDisallow: /exec/obidos/dt\nDisallow: /exec/obidos/dt/assoc/handle-buy-box\nDisallow: /exec/obidos/flex-sign-in\nDisallow: /exec/obidos/handle-buy-box\nDisallow: /exec/obidos/refer-a-friend-login\nDisallow: /exec/obidos/subst/associates/join\nDisallow: /exec/obidos/subst/marketplace/sell-your-collection.html\nDisallow: /exec/obidos/subst/marketplace/sell-your-stuff.html\nDisallow: /exec/obidos/subst/partners/friends/access.html\nDisallow: /exec/obidos/tg/cm/member/\nDisallow: /gp/cart\nDisallow: /gp/content-form\nDisallow: /gp/customer-images\nDisallow: /gp/customer-media/upload\nDisallow: /gp/customer-reviews/common/du\nDisallow: /gp/customer-reviews/write-a-review.html\nDisallow: /gp/flex\nDisallow: /gp/gfix\nDisallow: /gp/history\nDisallow: /gp/item-dispatch\nDisallow: /gp/legacy-handle-buy-box.html\nDisallow: /gp/reader\nDisallow: /gp/registry/wishlist/*/reserve\nDisallow: /gp/richpub/listmania/createpipeline\nDisallow: /gp/music/clipserve\nDisallow: /gp/music/wma-pop-up\nDisallow: /gp/recsradio\nDisallow: /gp/sign-in\nDisallow: /gp/slides/make-money\nDisallow: /gp/structured-ratings/actions/get-experience.html\nDisallow: /gp/twitter/\nDisallow: /gp/vote\nDisallow: /gp/voting/\nDisallow: /gp/yourstore\nDisallow: /ap/signin\nDisallow: /gp/registry/search.html\nDisallow: /gp/orc/rml/\nDisallow: /gp/dmusic/mp3/player\nDisallow: /gp/entity-alert/external\nDisallow: /gp/customer-reviews/dynamic/sims-box\nDisallow: /review/dynamic/sims-box\nDisallow: /gp/redirect.html\nDisallow: /gp/customer-media/actions/delete/\nDisallow: /gp/customer-media/actions/edit-caption/\nDisallow: /gp/dmusic/\nAllow: /gp/dmusic/promotions/PrimeMusic\nAllow: /gp/dmusic/promotions/AmazonMusicUnlimited\nDisallow: /gp/customer-media/product-gallery/B007HCCOD0\nDisallow: /gp/help/customer/display.html?*nodeId=200534000\nDisallow: /gp/help/customer/display.html?*nodeId=200145650\nDisallow: /gp/feature.html?*docId=1000632623\nDisallow: /gp/aag\nDisallow: /gp/socialmedia/giveaways\nDisallow: /gp/aw/so.html\nDisallow: /gp/aw/sp.html?*t=returns\nDisallow: /gp/pdp/profile/\nDisallow: /gp/product/product-availability\nDisallow: /gp/offer-listing\nDisallow: /dp/twister-update/\nDisallow: /dp/e-mail-friend/\nDisallow: /gp/registry/wishlist/\nDisallow: /gp/switch-language\nDisallow: /wishlist/\nAllow: /wishlist/universal\nAllow: /wishlist/vendor-button\nAllow: /wishlist/get-button\nDisallow: /gp/wishlist/\nAllow: /gp/wishlist/universal\nAllow: /gp/wishlist/vendor-button\nAllow: /gp/wishlist/ipad-install\nDisallow: /registry/wishlist/\nDisallow: /local/ajax/\nDisallow: /gp/rate-it\nDisallow: /o/di\nDisallow: /o/dt\nDisallow: /gp/help/customer/display.html?nodeId=200393390\nDisallow: /forum/kindle\nDisallow: /gp/profile\nDisallow: /reviews/iframe\nDisallow: /gp/ask-widget/askWidget*\nDisallow: /ss/customer-reviews/lighthouse/\nDisallow: /gp/aw/ol/\nDisallow: /gp/promotion/\nDisallow: /hz/leaderboard/top-reviewers/\nDisallow: /hz/leaderboard/hall-of-fame/\nDisallow: /review/top-reviewers/\nDisallow: /review/top-reviewers\nDisallow: /review/hall-of-fame\nDisallow: /reviews/top-reviewers/\nDisallow: /reviews/top-reviewers\nDisallow: /reviews/hall-of-fame\nDisallow: /hz/help/contact/*/message/$\nDisallow: /-/\nAllow: /-/en/\nAllow: /-/da/\nAllow: /-/en$\nDisallow: /gp/aw/shoppingAids/\nDisallow: /hp/video/\nDisallow: /gp/video/api\nDisallow: /gp/video/settings\nDisallow: /gp/video/library\nDisallow: /gp/video/watchlist\nDisallow: /gp/video/mystuff\nDisallow: /gp/video/profiles\nDisallow: /gp/video/search\nDisallow: /gp/video/auth\nDisallow: /gp/video/salp/i\n\nUser-agent: EtaoSpider\nDisallow: /\n\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nUser-Agent: PerplexityBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GoogleAgent-Mariner\nDisallow: /\n\nUser-agent: GoogleAgent-Shopping\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: meta-externalagent\nDisallow: /\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Devin\nDisallow: /\n\nUser-agent: omgili\nDisallow: /\n\nUser-agent: AI2Bot\nDisallow: /\n\nUser-agent: Gemini-Deep-Research\nDisallow: /\n\nUser-agent: PanguBot\nDisallow: /\n\nUser-agent: MistralAI-User\nDisallow: /\n\nUser-agent: Diffbot\nDisallow: /\n\nUser-agent: Sidetrade indexer bot\nDisallow: /\n\nUser-agent: Google-NotebookLM\nDisallow: /\n\nUser-agent: DuckAssistBot\nDisallow: /\n\nUser-agent: Copilot\nDisallow: /\n\nUser-agent: Ai2Bot-Dolma\nDisallow: /\n\nUser-agent: Google-CloudVertexBot\nDisallow: /\n\nUser-agent: meta-externalfetcher\nDisallow: /\n\nUser-agent: iaskspider\nDisallow: /\n\nUser-agent: wpbot\nDisallow: /\n\nUser-agent: aiHitBot\nDisallow: /\n\nUser-agent: cohere-ai\nDisallow: /\n\nUser-agent: FriendlyCrawler\nDisallow: /\n\nUser-agent: img2dataset\nDisallow: /\n\nUser-agent: VelenPublicWebCrawler\nDisallow: /\n\nUser-agent: YouBot\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: Claude-User\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Claude-SearchBot\nDisallow: /\n\nUser-agent: Perplexity-User\nDisallow: /\n\nUser-agent: ISSCyberRiskCrawler\nDisallow: /\n\nUser-agent: *\nDisallow: /aaut/*\n\nUser-agent: CopilotNative\nDisallow: /\n\nUser-agent: CopilotSapphire\nDisallow: /\n\nUser-agent: imgproxy\nDisallow: /\n\nUser-agent: SemrushBot-SWA\nDisallow: /\n\nUser-agent: ChatGPT-User\nDisallow: /\n\nUser-agent: OAI-SearchBot\nDisallow: /\n\nUser-agent: Timpibot\nDisallow: /\n\nUser-agent: Thinkbot\nDisallow: /\n\nUser-agent: AddSearchBot\nDisallow: /\n\nUser-agent: AI2Bot-DeepResearchEval\nDisallow: /\n\nUser-agent: Andibot\nDisallow: /\n\nUser-agent: Anomura\nDisallow: /\n\nUser-agent: atlassian-bot\nDisallow: /\n\nUser-agent: BuddyBot\nDisallow: /\n\nUser-agent: Channel3Bot\nDisallow: /\n\nUser-agent: ChatGLM-Spider\nDisallow: /\n\nUser-agent: Claude-Web\nDisallow: /\n\nUser-agent: Cloudflare-AutoRAG\nDisallow: /\n\nUser-agent: CloudflareBrowserRenderingCrawler\nDisallow: /\n\nUser-agent: cohere-training-data-crawler\nDisallow: /\n\nUser-agent: Cotoyogi\nDisallow: /\n\nUser-agent: Crawl4AI\nDisallow: /\n\nUser-agent: Datenbank Crawler\nDisallow: /\n\nUser-agent: DeepSeekBot\nDisallow: /\n\nUser-agent: Echobot Bot\nDisallow: /\n\nUser-agent: EchoboxBot\nDisallow: /\n\nUser-agent: factset_spyderbot\nDisallow: /\n\nUser-agent: Google-Firebase\nDisallow: /\n\nUser-agent: GoogleOther\nDisallow: /\n\nUser-agent: Grok-DeepSearch\nDisallow: /\n\nUser-agent: GrokBot\nDisallow: /\n\nUser-agent: iAskBot\nDisallow: /\n\nUser-agent: IbouBot\nDisallow: /\n\nUser-agent: ICC-Crawler\nDisallow: /\n\nUser-agent: ImagesiftBot\nDisallow: /\n\nUser-agent: imageSpider\nDisallow: /\n\nUser-agent: Kangaroo Bot\nDisallow: /\n\nUser-agent: KlaviyoAIBot\nDisallow: /\n\nUser-agent: KunatoCrawler\nDisallow: /\n\nUser-agent: laion-huggingface-processor\nDisallow: /\n\nUser-agent: Linguee Bot\nDisallow: /\n\nUser-agent: LinkupBot\nDisallow: /\n\nUser-agent: Manus-User\nDisallow: /\n\nUser-agent: meta-webindexer\nDisallow: /\n\nUser-agent: netEstate Imprint Crawler\nDisallow: /\n\nUser-agent: Panscient\nDisallow: /\n\nUser-agent: panscient.com\nDisallow: /\n\nUser-agent: PhindBot\nDisallow: /\n\nUser-agent: Poggio-Citations\nDisallow: /\n\nUser-agent: QualifiedBot\nDisallow: /\n\nUser-agent: SBIntuitionsBot\nDisallow: /\n\nUser-agent: ShapBot\nDisallow: /\n\nUser-agent: TavilyBot\nDisallow: /\n\nUser-agent: TerraCotta\nDisallow: /\n\nUser-agent: TwinAgent\nDisallow: /\n\nUser-agent: webzio-extended\nDisallow: /\n\nUser-agent: WRTNBot\nDisallow: /\n\nUser-agent: xAI-Grok\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: ZanistaBot\nDisallow: /"
    },
    "www.amazon.co.uk": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.amazon.co.uk",
      "body": "User-agent: *\nDisallow: /dp/product-availability/\nDisallow: /dp/rate-this-item/\nDisallow: /exec/obidos/account-access-login\nDisallow: /exec/obidos/change-style\nDisallow: /exec/obidos/dt/assoc/handle-buy-box\nDisallow: /exec/obidos/flex-sign-in\nDisallow: /exec/obidos/handle-buy-box\nDisallow: /exec/obidos/refer-a-friend-login\nDisallow: /exec/obidos/subst/associates/join\nDisallow: /exec/obidos/subst/marketplace/sell-your-collection.html\nDisallow: /exec/obidos/subst/marketplace/sell-your-stuff.html\nDisallow: /exec/obidos/subst/partners/friends/access.html\nDisallow: /exec/obidos/tg/cm/member/\nDisallow: /gp/cart\nDisallow: /gp/content-form\nDisallow: /gp/customer-images\nDisallow: /gp/customer-media/upload\nDisallow: /gp/customer-reviews/common/du\nDisallow: /gp/customer-reviews/write-a-review.html\nDisallow: /gp/flex\nDisallow: /gp/gfix\nDisallow: /gp/history\nDisallow: /gp/item-dispatch\nDisallow: /gp/legacy-handle-buy-box.html\nDisallow: /gp/reader\nDisallow: /gp/registry/wishlist/*/reserve\nDisallow: /gp/richpub/listmania/createpipeline\nDisallow: /gp/music/clipserve\nDisallow: /gp/recsradio\nDisallow: /gp/sign-in\nDisallow: /gp/slides/make-money\nDisallow: /gp/structured-ratings/actions/get-experience.html\nDisallow: /gp/twitter/\nDisallow: /gp/vote\nDisallow: /gp/voting/\nDisallow: /gp/yourstore\nDisallow: /ap/signin\nDisallow: /gp/registry/search.html\nDisallow: /gp/orc/rml/\nDisallow: /gp/dmusic/mp3/player\nDisallow: /gp/entity-alert/external\nDisallow: /gp/customer-reviews/dynamic/sims-box\nDisallow: /review/dynamic/sims-box\nDisallow: /gp/redirect.html\nDisallow: /gp/customer-media/actions/delete/\nDisallow: /gp/customer-media/actions/edit-caption/\nDisallow: /gp/dmusic/\nAllow: /gp/dmusic/promotions/PrimeMusic\nAllow: /gp/dmusic/promotions/AmazonMusicUnlimited\nDisallow: /gp/customer-media/product-gallery/B007HCCOD0\nDisallow: /gp/help/customer/display.html?*nodeId=200534000\nDisallow: /gp/feature.html?*docId=1000632623\nDisallow: /gp/aag\nDisallow: /gp/socialmedia/giveaways\nDisallow: /gp/aw/so.html\nDisallow: /gp/pdp/profile/\nDisallow: /gp/product/product-availability\nDisallow: /gp/offer-listing\nDisallow: /dp/twister-update/\nDisallow: /dp/e-mail-friend/\nDisallow: /gp/registry/wishlist/\nDisallow: /wishlist/\nAllow: /wishlist/universal\nAllow: /wishlist/vendor-button\nAllow: /wishlist/get-button\nDisallow: /gp/wishlist/\nAllow: /gp/wishlist/universal\nAllow: /gp/wishlist/vendor-button\nAllow: /gp/wishlist/ipad-install\nDisallow: /registry/wishlist/\nDisallow: /local/ajax/\nDisallow: /gp/profile/\nDisallow: /reviews/iframe\nDisallow: /gp/help/reports/infringement/jquery/handle-notice-submit.html\nDisallow: /gp/help/customer/handler/handle-email-submit.html\nDisallow: /gp/ask-widget/askWidget*\nDisallow: /ss/customer-reviews/lighthouse/\nDisallow: /gp/aw/ol/\nDisallow: /gp/promotion/\nDisallow: /hz/leaderboard/top-reviewers/\nDisallow: /hz/leaderboard/hall-of-fame/\nDisallow: /review/top-reviewers/\nDisallow: /review/top-reviewers\nDisallow: /review/hall-of-fame\nDisallow: /reviews/top-reviewers/\nDisallow: /reviews/top-reviewers\nDisallow: /reviews/hall-of-fame\nDisallow: /stores/page/*visitId= \nDisallow: /hz/help/contact/*/message/$\nDisallow: /-/\nDisallow: /gp/aw/shoppingAids/\nDisallow: /hp/video/\nDisallow: /gp/video/api\nDisallow: /gp/video/settings\nDisallow: /gp/video/library\nDisallow: /gp/video/watchlist\nDisallow: /gp/video/mystuff\nDisallow: /gp/video/profiles\nDisallow: /gp/video/search\nDisallow: /gp/video/auth\nDisallow: /gp/video/salp/i\n\n\nUser-agent: EtaoSpider\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nUser-Agent: PerplexityBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GoogleAgent-Mariner\nDisallow: /\n\nUser-agent: GoogleAgent-Shopping\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: meta-externalagent\nDisallow: /\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Devin\nDisallow: /\n\nUser-agent: omgili\nDisallow: /\n\nUser-agent: AI2Bot\nDisallow: /\n\nUser-agent: Gemini-Deep-Research\nDisallow: /\n\nUser-agent: PanguBot\nDisallow: /\n\nUser-agent: MistralAI-User\nDisallow: /\n\nUser-agent: Diffbot\nDisallow: /\n\nUser-agent: Sidetrade indexer bot\nDisallow: /\n\nUser-agent: Google-NotebookLM\nDisallow: /\n\nUser-agent: DuckAssistBot\nDisallow: /\n\nUser-agent: Copilot\nDisallow: /\n\nUser-agent: Ai2Bot-Dolma\nDisallow: /\n\nUser-agent: Google-CloudVertexBot\nDisallow: /\n\nUser-agent: meta-externalfetcher\nDisallow: /\n\nUser-agent: iaskspider\nDisallow: /\n\nUser-agent: wpbot\nDisallow: /\n\nUser-agent: aiHitBot\nDisallow: /\n\nUser-agent: cohere-ai\nDisallow: /\n\nUser-agent: FriendlyCrawler\nDisallow: /\n\nUser-agent: img2dataset\nDisallow: /\n\nUser-agent: VelenPublicWebCrawler\nDisallow: /\n\nUser-agent: YouBot\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: Claude-User\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Claude-SearchBot\nDisallow: /\n\nUser-agent: Perplexity-User\nDisallow: /\n\nUser-agent: ISSCyberRiskCrawler\nDisallow: /\n\nUser-agent: CopilotNative\nDisallow: /\n\nUser-agent: CopilotSapphire\nDisallow: /\n\nUser-agent: imgproxy\nDisallow: /\n\nUser-agent: SemrushBot-SWA\nDisallow: /\n\nUser-agent: ChatGPT-User\nDisallow: /\n\nUser-agent: OAI-SearchBot\nDisallow: /\n\nUser-agent: Timpibot\nDisallow: /\n\nUser-agent: Thinkbot\nDisallow: /\n\nUser-agent: AddSearchBot\nDisallow: /\n\nUser-agent: AI2Bot-DeepResearchEval\nDisallow: /\n\nUser-agent: Andibot\nDisallow: /\n\nUser-agent: Anomura\nDisallow: /\n\nUser-agent: atlassian-bot\nDisallow: /\n\nUser-agent: BuddyBot\nDisallow: /\n\nUser-agent: Channel3Bot\nDisallow: /\n\nUser-agent: ChatGLM-Spider\nDisallow: /\n\nUser-agent: Claude-Web\nDisallow: /\n\nUser-agent: Cloudflare-AutoRAG\nDisallow: /\n\nUser-agent: CloudflareBrowserRenderingCrawler\nDisallow: /\n\nUser-agent: cohere-training-data-crawler\nDisallow: /\n\nUser-agent: Cotoyogi\nDisallow: /\n\nUser-agent: Crawl4AI\nDisallow: /\n\nUser-agent: Datenbank Crawler\nDisallow: /\n\nUser-agent: DeepSeekBot\nDisallow: /\n\nUser-agent: Echobot Bot\nDisallow: /\n\nUser-agent: EchoboxBot\nDisallow: /\n\nUser-agent: factset_spyderbot\nDisallow: /\n\nUser-agent: Google-Firebase\nDisallow: /\n\nUser-agent: GoogleOther\nDisallow: /\n\nUser-agent: Grok-DeepSearch\nDisallow: /\n\nUser-agent: GrokBot\nDisallow: /\n\nUser-agent: iAskBot\nDisallow: /\n\nUser-agent: IbouBot\nDisallow: /\n\nUser-agent: ICC-Crawler\nDisallow: /\n\nUser-agent: ImagesiftBot\nDisallow: /\n\nUser-agent: imageSpider\nDisallow: /\n\nUser-agent: Kangaroo Bot\nDisallow: /\n\nUser-agent: KlaviyoAIBot\nDisallow: /\n\nUser-agent: KunatoCrawler\nDisallow: /\n\nUser-agent: laion-huggingface-processor\nDisallow: /\n\nUser-agent: Linguee Bot\nDisallow: /\n\nUser-agent: LinkupBot\nDisallow: /\n\nUser-agent: Manus-User\nDisallow: /\n\nUser-agent: meta-webindexer\nDisallow: /\n\nUser-agent: netEstate Imprint Crawler\nDisallow: /\n\nUser-agent: Panscient\nDisallow: /\n\nUser-agent: panscient.com\nDisallow: /\n\nUser-agent: PhindBot\nDisallow: /\n\nUser-agent: Poggio-Citations\nDisallow: /\n\nUser-agent: QualifiedBot\nDisallow: /\n\nUser-agent: SBIntuitionsBot\nDisallow: /\n\nUser-agent: ShapBot\nDisallow: /\n\nUser-agent: TavilyBot\nDisallow: /\n\nUser-agent: TerraCotta\nDisallow: /\n\nUser-agent: TwinAgent\nDisallow: /\n\nUser-agent: webzio-extended\nDisallow: /\n\nUser-agent: WRTNBot\nDisallow: /\n\nUser-agent: xAI-Grok\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: ZanistaBot\nDisallow: /"
    },
    "www.amazon.fr": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.amazon.fr",
      "body": "User-agent: *\nDisallow: */s?k=*&rh=n*p_*p_*p_\nDisallow: /dp/product-availability/\nDisallow: /dp/rate-this-item/\nDisallow: /exec/obidos/account-access-login\nDisallow: /exec/obidos/change-style\nDisallow: /exec/obidos/dt/assoc/handle-buy-box\nDisallow: /exec/obidos/flex-sign-in\nDisallow: /exec/obidos/handle-buy-box\nDisallow: /exec/obidos/refer-a-friend-login\nDisallow: /exec/obidos/subst/associates/join\nDisallow: /exec/obidos/subst/marketplace/sell-your-collection.html\nDisallow: /exec/obidos/subst/marketplace/sell-your-stuff.html\nDisallow: /exec/obidos/subst/partners/friends/access.html\nDisallow: /exec/obidos/tg/cm/member/\nDisallow: /gp/cart\nDisallow: /gp/content-form\nDisallow: /gp/customer-images\nDisallow: /gp/customer-media/upload\nDisallow: /gp/customer-reviews/common/du\nDisallow: /gp/customer-reviews/write-a-review.html\nDisallow: /gp/flex\nDisallow: /gp/gfix\nDisallow: /gp/history\nDisallow: /gp/item-dispatch\nDisallow: /gp/legacy-handle-buy-box.html\nDisallow: /gp/reader\nDisallow: /gp/registry/wishlist/*/reserve\nDisallow: /gp/richpub/listmania/createpipeline\nDisallow: /gp/music/clipserve\nDisallow: /gp/recsradio\nDisallow: /gp/sign-in\nDisallow: /gp/slides/make-money\nDisallow: /gp/structured-ratings/actions/get-experience.html\nDisallow: /gp/twitter/\nDisallow: /gp/vote\nDisallow: /gp/voting/\nDisallow: /gp/yourstore\nDisallow: /ap/signin\nDisallow: /gp/registry/search.html\nDisallow: /gp/orc/rml/\nDisallow: /gp/dmusic/mp3/player\nDisallow: /gp/entity-alert/external\nDisallow: /gp/customer-reviews/dynamic/sims-box\nDisallow: /review/dynamic/sims-box\nDisallow: /gp/redirect.html\nDisallow: /gp/customer-media/actions/delete/\nDisallow: /gp/customer-media/actions/edit-caption/\nDisallow: /gp/dmusic/\nDisallow: /gp/customer-media/product-gallery/B007HCCOD0\nDisallow: /gp/help/customer/display.html?*nodeId=200534000\nDisallow: /gp/feature.html?*docId=1000632623\nDisallow: /gp/aag\nDisallow: /gp/socialmedia/giveaways\nDisallow: /gp/aw/so.html\nDisallow: /gp/pdp/profile/\nDisallow: /gp/product/product-availability\nDisallow: /gp/offer-listing\nDisallow: /dp/twister-update/\nDisallow: /dp/e-mail-friend/\nDisallow: /gp/registry/wishlist/\nDisallow: /wishlist/\nAllow: /wishlist/universal\nAllow: /wishlist/vendor-button\nAllow: /wishlist/get-button\nDisallow: /gp/wishlist/\nAllow: /gp/wishlist/universal\nAllow: /gp/wishlist/vendor-button\nAllow: /gp/wishlist/ipad-install\nDisallow: /registry/wishlist/\nDisallow: /local/ajax/\nDisallow: /gp/rentallist\nDisallow: /gp/video/dvd-rental/settings\nDisallow: /gp/rl/settings\nDisallow: /gp/video/settings\nDisallow: /gp/video/watchlist\nDisallow: /gp/video/library\nDisallow: /gp/profile/\nDisallow: /reviews/iframe\nDisallow: /gp/help/reports/infringement/jquery/handle-notice-submit.html\nDisallow: /gp/help/customer/handler/handle-email-submit.html\nDisallow: /gp/ask-widget/askWidget*\nDisallow: /gp/dmusic/\nAllow: /gp/dmusic/promotions/AmazonMusicUnlimited\nDisallow: /ss/customer-reviews/lighthouse/\nDisallow: /gp/aw/ol/\nDisallow: /gp/promotion/\nDisallow: /hz/leaderboard/top-reviewers/\nDisallow: /hz/leaderboard/hall-of-fame/\nDisallow: /review/top-reviewers/\nDisallow: /review/top-reviewers\nDisallow: /review/hall-of-fame\nDisallow: /reviews/top-reviewers/\nDisallow: /reviews/top-reviewers\nDisallow: /reviews/hall-of-fame\nDisallow: /hz/help/contact/*/message/$\nDisallow: /-/\nAllow: /-/en/\nDisallow: /gp/aw/shoppingAids/\n\nUser-agent: EtaoSpider\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nUser-Agent: PerplexityBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GoogleAgent-Mariner\nDisallow: /\n\nUser-agent: GoogleAgent-Shopping\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: meta-externalagent\nDisallow: /\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Devin\nDisallow: /\n\nUser-agent: omgili\nDisallow: /\n\nUser-agent: AI2Bot\nDisallow: /\n\nUser-agent: Gemini-Deep-Research\nDisallow: /\n\nUser-agent: PanguBot\nDisallow: /\n\nUser-agent: MistralAI-User\nDisallow: /\n\nUser-agent: Diffbot\nDisallow: /\n\nUser-agent: Sidetrade indexer bot\nDisallow: /\n\nUser-agent: Google-NotebookLM\nDisallow: /\n\nUser-agent: DuckAssistBot\nDisallow: /\n\nUser-agent: Copilot\nDisallow: /\n\nUser-agent: Ai2Bot-Dolma\nDisallow: /\n\nUser-agent: Google-CloudVertexBot\nDisallow: /\n\nUser-agent: meta-externalfetcher\nDisallow: /\n\nUser-agent: iaskspider\nDisallow: /\n\nUser-agent: wpbot\nDisallow: /\n\nUser-agent: aiHitBot\nDisallow: /\n\nUser-agent: cohere-ai\nDisallow: /\n\nUser-agent: FriendlyCrawler\nDisallow: /\n\nUser-agent: img2dataset\nDisallow: /\n\nUser-agent: VelenPublicWebCrawler\nDisallow: /\n\nUser-agent: YouBot\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: Claude-User\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Claude-SearchBot\nDisallow: /\n\nUser-agent: Perplexity-User\nDisallow: /\n\nUser-agent: ISSCyberRiskCrawler\nDisallow: /\n\nUser-agent: CopilotNative\nDisallow: /\n\nUser-agent: CopilotSapphire\nDisallow: /\n\nUser-agent: imgproxy\nDisallow: /\n\nUser-agent: SemrushBot-SWA\nDisallow: /\n\nUser-agent: ChatGPT-User\nDisallow: /\n\nUser-agent: OAI-SearchBot\nDisallow: /\n\nUser-agent: Timpibot\nDisallow: /\n\nUser-agent: Thinkbot\nDisallow: /\n\nUser-agent: AddSearchBot\nDisallow: /\n\nUser-agent: AI2Bot-DeepResearchEval\nDisallow: /\n\nUser-agent: Andibot\nDisallow: /\n\nUser-agent: Anomura\nDisallow: /\n\nUser-agent: atlassian-bot\nDisallow: /\n\nUser-agent: BuddyBot\nDisallow: /\n\nUser-agent: Channel3Bot\nDisallow: /\n\nUser-agent: ChatGLM-Spider\nDisallow: /\n\nUser-agent: Claude-Web\nDisallow: /\n\nUser-agent: Cloudflare-AutoRAG\nDisallow: /\n\nUser-agent: CloudflareBrowserRenderingCrawler\nDisallow: /\n\nUser-agent: cohere-training-data-crawler\nDisallow: /\n\nUser-agent: Cotoyogi\nDisallow: /\n\nUser-agent: Crawl4AI\nDisallow: /\n\nUser-agent: Datenbank Crawler\nDisallow: /\n\nUser-agent: DeepSeekBot\nDisallow: /\n\nUser-agent: Echobot Bot\nDisallow: /\n\nUser-agent: EchoboxBot\nDisallow: /\n\nUser-agent: factset_spyderbot\nDisallow: /\n\nUser-agent: Google-Firebase\nDisallow: /\n\nUser-agent: GoogleOther\nDisallow: /\n\nUser-agent: Grok-DeepSearch\nDisallow: /\n\nUser-agent: GrokBot\nDisallow: /\n\nUser-agent: iAskBot\nDisallow: /\n\nUser-agent: IbouBot\nDisallow: /\n\nUser-agent: ICC-Crawler\nDisallow: /\n\nUser-agent: ImagesiftBot\nDisallow: /\n\nUser-agent: imageSpider\nDisallow: /\n\nUser-agent: Kangaroo Bot\nDisallow: /\n\nUser-agent: KlaviyoAIBot\nDisallow: /\n\nUser-agent: KunatoCrawler\nDisallow: /\n\nUser-agent: laion-huggingface-processor\nDisallow: /\n\nUser-agent: Linguee Bot\nDisallow: /\n\nUser-agent: LinkupBot\nDisallow: /\n\nUser-agent: Manus-User\nDisallow: /\n\nUser-agent: meta-webindexer\nDisallow: /\n\nUser-agent: netEstate Imprint Crawler\nDisallow: /\n\nUser-agent: Panscient\nDisallow: /\n\nUser-agent: panscient.com\nDisallow: /\n\nUser-agent: PhindBot\nDisallow: /\n\nUser-agent: Poggio-Citations\nDisallow: /\n\nUser-agent: QualifiedBot\nDisallow: /\n\nUser-agent: SBIntuitionsBot\nDisallow: /\n\nUser-agent: ShapBot\nDisallow: /\n\nUser-agent: TavilyBot\nDisallow: /\n\nUser-agent: TerraCotta\nDisallow: /\n\nUser-agent: TwinAgent\nDisallow: /\n\nUser-agent: webzio-extended\nDisallow: /\n\nUser-agent: WRTNBot\nDisallow: /\n\nUser-agent: xAI-Grok\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: ZanistaBot\nDisallow: /"
    },
    "www.amazon.it": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.amazon.it",
      "body": "User-agent: *\nDisallow: */s?k=*&rh=n*p_*p_*p_\nDisallow: /dp/product-availability/\nDisallow: /dp/rate-this-item/\nDisallow: /exec/obidos/account-access-login\nDisallow: /exec/obidos/change-style\nDisallow: /exec/obidos/di\nDisallow: /exec/obidos/dt\nDisallow: /exec/obidos/dt/assoc/handle-buy-box\nDisallow: /exec/obidos/flex-sign-in\nDisallow: /exec/obidos/handle-buy-box\nDisallow: /exec/obidos/refer-a-friend-login\nDisallow: /exec/obidos/subst/associates/join\nDisallow: /exec/obidos/subst/marketplace/sell-your-collection.html\nDisallow: /exec/obidos/subst/marketplace/sell-your-stuff.html\nDisallow: /exec/obidos/subst/partners/friends/access.html\nDisallow: /exec/obidos/tg/cm/member/\nDisallow: /gp/cart\nDisallow: /gp/content-form\nDisallow: /gp/customer-images\nDisallow: /gp/customer-media/upload\nDisallow: /gp/customer-reviews/common/du\nDisallow: /gp/customer-reviews/write-a-review.html\nDisallow: /gp/flex\nDisallow: /gp/gfix\nDisallow: /gp/history\nDisallow: /gp/item-dispatch\nDisallow: /gp/legacy-handle-buy-box.html\nDisallow: /gp/reader\nDisallow: /gp/registry/wishlist/*/reserve\nDisallow: /gp/richpub/listmania/createpipeline\nDisallow: /gp/music/clipserve\nDisallow: /gp/recsradio\nDisallow: /gp/sign-in\nDisallow: /gp/slides/make-money\nDisallow: /gp/structured-ratings/actions/get-experience.html\nDisallow: /gp/twitter/\nDisallow: /gp/vote\nDisallow: /gp/voting/\nDisallow: /gp/yourstore\nDisallow: /ap/signin\nDisallow: /gp/registry/search.html\nDisallow: /gp/orc/rml/\nDisallow: /gp/dmusic/mp3/player\nDisallow: /gp/entity-alert/external\nDisallow: /gp/customer-reviews/dynamic/sims-box\nDisallow: /review/dynamic/sims-box\nDisallow: /gp/redirect.html\nDisallow: /gp/customer-media/actions/delete/\nDisallow: /gp/customer-media/actions/edit-caption/\nDisallow: /gp/dmusic/\nDisallow: /gp/customer-media/product-gallery/B007HCCOD0\nDisallow: /gp/help/customer/display.html?*nodeId=200534000\nDisallow: /gp/help/customer/display.html?*nodeId=200145650\nDisallow: /gp/feature.html?*docId=1000632623\nDisallow: /gp/aag\nDisallow: /gp/socialmedia/giveaways\nDisallow: /gp/aw/so.html\nDisallow: /gp/aw/sp.html?*t=returns\nDisallow: /gp/pdp/profile/\nDisallow: /gp/product/product-availability\nDisallow: /gp/offer-listing\nDisallow: /dp/twister-update/\nDisallow: /dp/e-mail-friend/\nDisallow: /gp/registry/wishlist/\nDisallow: /gp/switch-language\nDisallow: /wishlist/\nAllow: /wishlist/universal\nAllow: /wishlist/vendor-button\nAllow: /wishlist/get-button\nDisallow: /gp/wishlist/\nAllow: /gp/wishlist/universal\nAllow: /gp/wishlist/vendor-button\nAllow: /gp/wishlist/ipad-install\nDisallow: /registry/wishlist/\nDisallow: /local/ajax/\nDisallow: /gp/rate-it\nDisallow: /gp/rentallist\nDisallow: /gp/video/dvd-rental/settings\nDisallow: /gp/rl/settings\nDisallow: /gp/video/settings\nDisallow: /gp/video/watchlist\nDisallow: /gp/video/library\nDisallow: /o/di\nDisallow: /o/dt\nDisallow: /gp/help/customer/display.html*nodeId=200556720\nDisallow: /gp/profile/\nDisallow: /reviews/iframe\nDisallow: /gp/help/reports/infringement/jquery/handle-notice-submit.html\nDisallow: /gp/help/customer/handler/handle-email-submit.html\nDisallow: /gp/ask-widget/askWidget*\nDisallow: /gp/dmusic/\nAllow: /gp/dmusic/promotions/AmazonMusicUnlimited\nDisallow: /ss/customer-reviews/lighthouse/\nDisallow: /gp/aw/ol/\nDisallow: /gp/promotion/\nDisallow: /hz/leaderboard/top-reviewers/\nDisallow: /hz/leaderboard/hall-of-fame/\nDisallow: /review/top-reviewers/\nDisallow: /review/top-reviewers\nDisallow: /review/hall-of-fame\nDisallow: /reviews/top-reviewers/\nDisallow: /reviews/top-reviewers\nDisallow: /reviews/hall-of-fame\nDisallow: /hz/help/contact/*/message/$\nDisallow: /gp/aw/shoppingAids/\nDisallow: /-/\nAllow: /-/en/\n\nUser-agent: EtaoSpider\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nUser-Agent: PerplexityBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GoogleAgent-Mariner\nDisallow: /\n\nUser-agent: GoogleAgent-Shopping\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: meta-externalagent\nDisallow: /\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Devin\nDisallow: /\n\nUser-agent: omgili\nDisallow: /\n\nUser-agent: AI2Bot\nDisallow: /\n\nUser-agent: Gemini-Deep-Research\nDisallow: /\n\nUser-agent: PanguBot\nDisallow: /\n\nUser-agent: MistralAI-User\nDisallow: /\n\nUser-agent: Diffbot\nDisallow: /\n\nUser-agent: Sidetrade indexer bot\nDisallow: /\n\nUser-agent: Google-NotebookLM\nDisallow: /\n\nUser-agent: DuckAssistBot\nDisallow: /\n\nUser-agent: Copilot\nDisallow: /\n\nUser-agent: Ai2Bot-Dolma\nDisallow: /\n\nUser-agent: Google-CloudVertexBot\nDisallow: /\n\nUser-agent: meta-externalfetcher\nDisallow: /\n\nUser-agent: iaskspider\nDisallow: /\n\nUser-agent: wpbot\nDisallow: /\n\nUser-agent: aiHitBot\nDisallow: /\n\nUser-agent: cohere-ai\nDisallow: /\n\nUser-agent: FriendlyCrawler\nDisallow: /\n\nUser-agent: img2dataset\nDisallow: /\n\nUser-agent: VelenPublicWebCrawler\nDisallow: /\n\nUser-agent: YouBot\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: Claude-User\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Claude-SearchBot\nDisallow: /\n\nUser-agent: Perplexity-User\nDisallow: /\n\nUser-agent: ISSCyberRiskCrawler\nDisallow: /\n\nUser-agent: CopilotNative\nDisallow: /\n\nUser-agent: CopilotSapphire\nDisallow: /\n\nUser-agent: imgproxy\nDisallow: /\n\nUser-agent: SemrushBot-SWA\nDisallow: /\n\nUser-agent: ChatGPT-User\nDisallow: /\n\nUser-agent: OAI-SearchBot\nDisallow: /\n\nUser-agent: Timpibot\nDisallow: /\n\nUser-agent: Thinkbot\nDisallow: /\n\nUser-agent: AddSearchBot\nDisallow: /\n\nUser-agent: AI2Bot-DeepResearchEval\nDisallow: /\n\nUser-agent: Andibot\nDisallow: /\n\nUser-agent: Anomura\nDisallow: /\n\nUser-agent: atlassian-bot\nDisallow: /\n\nUser-agent: BuddyBot\nDisallow: /\n\nUser-agent: Channel3Bot\nDisallow: /\n\nUser-agent: ChatGLM-Spider\nDisallow: /\n\nUser-agent: Claude-Web\nDisallow: /\n\nUser-agent: Cloudflare-AutoRAG\nDisallow: /\n\nUser-agent: CloudflareBrowserRenderingCrawler\nDisallow: /\n\nUser-agent: cohere-training-data-crawler\nDisallow: /\n\nUser-agent: Cotoyogi\nDisallow: /\n\nUser-agent: Crawl4AI\nDisallow: /\n\nUser-agent: Datenbank Crawler\nDisallow: /\n\nUser-agent: DeepSeekBot\nDisallow: /\n\nUser-agent: Echobot Bot\nDisallow: /\n\nUser-agent: EchoboxBot\nDisallow: /\n\nUser-agent: factset_spyderbot\nDisallow: /\n\nUser-agent: Google-Firebase\nDisallow: /\n\nUser-agent: GoogleOther\nDisallow: /\n\nUser-agent: Grok-DeepSearch\nDisallow: /\n\nUser-agent: GrokBot\nDisallow: /\n\nUser-agent: iAskBot\nDisallow: /\n\nUser-agent: IbouBot\nDisallow: /\n\nUser-agent: ICC-Crawler\nDisallow: /\n\nUser-agent: ImagesiftBot\nDisallow: /\n\nUser-agent: imageSpider\nDisallow: /\n\nUser-agent: Kangaroo Bot\nDisallow: /\n\nUser-agent: KlaviyoAIBot\nDisallow: /\n\nUser-agent: KunatoCrawler\nDisallow: /\n\nUser-agent: laion-huggingface-processor\nDisallow: /\n\nUser-agent: Linguee Bot\nDisallow: /\n\nUser-agent: LinkupBot\nDisallow: /\n\nUser-agent: Manus-User\nDisallow: /\n\nUser-agent: meta-webindexer\nDisallow: /\n\nUser-agent: netEstate Imprint Crawler\nDisallow: /\n\nUser-agent: Panscient\nDisallow: /\n\nUser-agent: panscient.com\nDisallow: /\n\nUser-agent: PhindBot\nDisallow: /\n\nUser-agent: Poggio-Citations\nDisallow: /\n\nUser-agent: QualifiedBot\nDisallow: /\n\nUser-agent: SBIntuitionsBot\nDisallow: /\n\nUser-agent: ShapBot\nDisallow: /\n\nUser-agent: TavilyBot\nDisallow: /\n\nUser-agent: TerraCotta\nDisallow: /\n\nUser-agent: TwinAgent\nDisallow: /\n\nUser-agent: webzio-extended\nDisallow: /\n\nUser-agent: WRTNBot\nDisallow: /\n\nUser-agent: xAI-Grok\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: ZanistaBot\nDisallow: /"
    },
    "www.amazon.es": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.amazon.es",
      "body": "User-agent: *\nDisallow: */s?k=*&rh=n*p_*p_*p_\nDisallow: /dp/product-availability/\nDisallow: /dp/rate-this-item/\nDisallow: /exec/obidos/account-access-login\nDisallow: /exec/obidos/change-style\nDisallow: /exec/obidos/dt/assoc/handle-buy-box\nDisallow: /exec/obidos/flex-sign-in\nDisallow: /exec/obidos/handle-buy-box\nDisallow: /exec/obidos/refer-a-friend-login\nDisallow: /exec/obidos/subst/associates/join\nDisallow: /exec/obidos/subst/marketplace/sell-your-collection.html\nDisallow: /exec/obidos/subst/marketplace/sell-your-stuff.html\nDisallow: /exec/obidos/subst/partners/friends/access.html\nDisallow: /exec/obidos/tg/cm/member/\nDisallow: /gp/cart\nDisallow: /gp/content-form\nDisallow: /gp/customer-images\nDisallow: /gp/customer-media/upload\nDisallow: /gp/customer-reviews/common/du\nDisallow: /gp/customer-reviews/write-a-review.html\nDisallow: /gp/flex\nDisallow: /gp/gfix\nDisallow: /gp/history\nDisallow: /gp/item-dispatch\nDisallow: /gp/legacy-handle-buy-box.html\nDisallow: /gp/reader\nDisallow: /gp/registry/wishlist/*/reserve\nDisallow: /gp/richpub/listmania/createpipeline\nDisallow: /gp/music/clipserve\nDisallow: /gp/recsradio\nDisallow: /gp/sign-in\nDisallow: /gp/slides/make-money\nDisallow: /gp/structured-ratings/actions/get-experience.html\nDisallow: /gp/twitter/\nDisallow: /gp/vote\nDisallow: /gp/voting/\nDisallow: /gp/yourstore\nDisallow: /ap/signin\nDisallow: /gp/registry/search.html\nDisallow: /gp/orc/rml/\nDisallow: /gp/dmusic/mp3/player\nDisallow: /gp/entity-alert/external\nDisallow: /gp/customer-reviews/dynamic/sims-box\nDisallow: /review/dynamic/sims-box\nDisallow: /gp/redirect.html\nDisallow: /gp/customer-media/actions/delete/\nDisallow: /gp/customer-media/actions/edit-caption/\nDisallow: /gp/dmusic/\nAllow: /gp/dmusic/promotions/AmazonMusicUnlimited\nDisallow: /gp/customer-media/product-gallery/B007HCCOD0\nDisallow: /gp/help/customer/display.html?*nodeId=200534000\nDisallow: /gp/feature.html?*docId=1000632623\nDisallow: /gp/aag\nDisallow: /gp/socialmedia/giveaways\nDisallow: /gp/aw/so.html\nDisallow: /gp/pdp/profile/\nDisallow: /gp/product/product-availability\nDisallow: /gp/offer-listing\nDisallow: /dp/twister-update/\nDisallow: /dp/e-mail-friend/\nDisallow: /gp/registry/wishlist/\nDisallow: /wishlist/\nAllow: /wishlist/universal\nAllow: /wishlist/vendor-button\nAllow: /wishlist/get-button\nDisallow: /gp/wishlist/\nAllow: /gp/wishlist/universal\nAllow: /gp/wishlist/vendor-button\nAllow: /gp/wishlist/ipad-install\nDisallow: /registry/wishlist/\nDisallow: /local/ajax/\nDisallow: /gp/rentallist\nDisallow: /gp/video/dvd-rental/settings\nDisallow: /gp/rl/settings\nDisallow: /gp/video/settings\nDisallow: /gp/video/watchlist\nDisallow: /gp/video/library\nDisallow: /gp/profile/\nDisallow: /reviews/iframe\nDisallow: /gp/ask-widget/askWidget*\nDisallow: /ss/customer-reviews/lighthouse/\nDisallow: /gp/aw/ol/\nDisallow: /gp/promotion/\nDisallow: /hz/leaderboard/top-reviewers/\nDisallow: /hz/leaderboard/hall-of-fame/\nDisallow: /review/top-reviewers/\nDisallow: /review/top-reviewers\nDisallow: /review/hall-of-fame\nDisallow: /reviews/top-reviewers/\nDisallow: /reviews/top-reviewers\nDisallow: /reviews/hall-of-fame\nDisallow: /hz/help/contact/*/message/$\nDisallow: /-/\nAllow: /-/en/\nDisallow: /gp/aw/shoppingAids/\nDisallow: /drop\nDisallow: /shopthedrop\nDisallow: /thedrop\n\nUser-agent: EtaoSpider\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nUser-Agent: PerplexityBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GoogleAgent-Mariner\nDisallow: /\n\nUser-agent: GoogleAgent-Shopping\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: meta-externalagent\nDisallow: /\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Devin\nDisallow: /\n\nUser-agent: omgili\nDisallow: /\n\nUser-agent: AI2Bot\nDisallow: /\n\nUser-agent: Gemini-Deep-Research\nDisallow: /\n\nUser-agent: PanguBot\nDisallow: /\n\nUser-agent: MistralAI-User\nDisallow: /\n\nUser-agent: Diffbot\nDisallow: /\n\nUser-agent: Sidetrade indexer bot\nDisallow: /\n\nUser-agent: Google-NotebookLM\nDisallow: /\n\nUser-agent: DuckAssistBot\nDisallow: /\n\nUser-agent: Copilot\nDisallow: /\n\nUser-agent: Ai2Bot-Dolma\nDisallow: /\n\nUser-agent: Google-CloudVertexBot\nDisallow: /\n\nUser-agent: meta-externalfetcher\nDisallow: /\n\nUser-agent: iaskspider\nDisallow: /\n\nUser-agent: wpbot\nDisallow: /\n\nUser-agent: aiHitBot\nDisallow: /\n\nUser-agent: cohere-ai\nDisallow: /\n\nUser-agent: FriendlyCrawler\nDisallow: /\n\nUser-agent: img2dataset\nDisallow: /\n\nUser-agent: VelenPublicWebCrawler\nDisallow: /\n\nUser-agent: YouBot\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: Claude-User\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Claude-SearchBot\nDisallow: /\n\nUser-agent: Perplexity-User\nDisallow: /\n\nUser-agent: ISSCyberRiskCrawler\nDisallow: /\n\nUser-agent: CopilotNative\nDisallow: /\n\nUser-agent: CopilotSapphire\nDisallow: /\n\nUser-agent: imgproxy\nDisallow: /\n\nUser-agent: SemrushBot-SWA\nDisallow: /\n\nUser-agent: ChatGPT-User\nDisallow: /\n\nUser-agent: OAI-SearchBot\nDisallow: /\n\nUser-agent: Timpibot\nDisallow: /\n\nUser-agent: Thinkbot\nDisallow: /\n\nUser-agent: AddSearchBot\nDisallow: /\n\nUser-agent: AI2Bot-DeepResearchEval\nDisallow: /\n\nUser-agent: Andibot\nDisallow: /\n\nUser-agent: Anomura\nDisallow: /\n\nUser-agent: atlassian-bot\nDisallow: /\n\nUser-agent: BuddyBot\nDisallow: /\n\nUser-agent: Channel3Bot\nDisallow: /\n\nUser-agent: ChatGLM-Spider\nDisallow: /\n\nUser-agent: Claude-Web\nDisallow: /\n\nUser-agent: Cloudflare-AutoRAG\nDisallow: /\n\nUser-agent: CloudflareBrowserRenderingCrawler\nDisallow: /\n\nUser-agent: cohere-training-data-crawler\nDisallow: /\n\nUser-agent: Cotoyogi\nDisallow: /\n\nUser-agent: Crawl4AI\nDisallow: /\n\nUser-agent: Datenbank Crawler\nDisallow: /\n\nUser-agent: DeepSeekBot\nDisallow: /\n\nUser-agent: Echobot Bot\nDisallow: /\n\nUser-agent: EchoboxBot\nDisallow: /\n\nUser-agent: factset_spyderbot\nDisallow: /\n\nUser-agent: Google-Firebase\nDisallow: /\n\nUser-agent: GoogleOther\nDisallow: /\n\nUser-agent: Grok-DeepSearch\nDisallow: /\n\nUser-agent: GrokBot\nDisallow: /\n\nUser-agent: iAskBot\nDisallow: /\n\nUser-agent: IbouBot\nDisallow: /\n\nUser-agent: ICC-Crawler\nDisallow: /\n\nUser-agent: ImagesiftBot\nDisallow: /\n\nUser-agent: imageSpider\nDisallow: /\n\nUser-agent: Kangaroo Bot\nDisallow: /\n\nUser-agent: KlaviyoAIBot\nDisallow: /\n\nUser-agent: KunatoCrawler\nDisallow: /\n\nUser-agent: laion-huggingface-processor\nDisallow: /\n\nUser-agent: Linguee Bot\nDisallow: /\n\nUser-agent: LinkupBot\nDisallow: /\n\nUser-agent: Manus-User\nDisallow: /\n\nUser-agent: meta-webindexer\nDisallow: /\n\nUser-agent: netEstate Imprint Crawler\nDisallow: /\n\nUser-agent: Panscient\nDisallow: /\n\nUser-agent: panscient.com\nDisallow: /\n\nUser-agent: PhindBot\nDisallow: /\n\nUser-agent: Poggio-Citations\nDisallow: /\n\nUser-agent: QualifiedBot\nDisallow: /\n\nUser-agent: SBIntuitionsBot\nDisallow: /\n\nUser-agent: ShapBot\nDisallow: /\n\nUser-agent: TavilyBot\nDisallow: /\n\nUser-agent: TerraCotta\nDisallow: /\n\nUser-agent: TwinAgent\nDisallow: /\n\nUser-agent: webzio-extended\nDisallow: /\n\nUser-agent: WRTNBot\nDisallow: /\n\nUser-agent: xAI-Grok\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: ZanistaBot\nDisallow: /"
    },
    "www.amazon.nl": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.amazon.nl",
      "body": "User-agent: *\nDisallow: /dp/product-availability/\nDisallow: /dp/rate-this-item/\nDisallow: /exec/obidos/account-access-login\nDisallow: /exec/obidos/change-style\nDisallow: /exec/obidos/di\nDisallow: /exec/obidos/dt\nDisallow: /exec/obidos/flex-sign-in\nDisallow: /exec/obidos/handle-buy-box\nDisallow: /exec/obidos/refer-a-friend-login\nDisallow: /exec/obidos/subst/associates/join\nDisallow: /exec/obidos/subst/marketplace/sell-your-collection.html\nDisallow: /exec/obidos/subst/marketplace/sell-your-stuff.html\nDisallow: /exec/obidos/subst/partners/friends/access.html\nDisallow: /exec/obidos/tg/cm/member/\nDisallow: /gp/cart\nDisallow: /gp/content-form\nDisallow: /gp/customer-images\nDisallow: /gp/customer-media/upload\nDisallow: /gp/customer-reviews/common/du\nDisallow: /gp/customer-reviews/write-a-review.html\nDisallow: /gp/flex\nDisallow: /gp/gfix\nDisallow: /gp/history\nDisallow: /gp/item-dispatch\nDisallow: /gp/legacy-handle-buy-box.html\nDisallow: /gp/reader\nDisallow: /gp/registry/wishlist/*/reserve\nDisallow: /gp/richpub/listmania/createpipeline\nDisallow: /gp/music/clipserve\nDisallow: /gp/recsradio\nDisallow: /gp/sign-in\nDisallow: /gp/slides/make-money\nDisallow: /gp/structured-ratings/actions/get-experience.html\nDisallow: /gp/twitter/\nDisallow: /gp/vote\nDisallow: /gp/voting/\nDisallow: /gp/yourstore\nDisallow: /ap/signin\nDisallow: /gp/registry/search.html\nDisallow: /gp/orc/rml/\nDisallow: /gp/dmusic/mp3/player\nDisallow: /gp/entity-alert/external\nDisallow: /gp/customer-reviews/dynamic/sims-box\nDisallow: /review/dynamic/sims-box\nDisallow: /gp/redirect.html\nDisallow: /gp/customer-media/actions/delete/\nDisallow: /gp/customer-media/actions/edit-caption/\nDisallow: /gp/dmusic/\nDisallow: /gp/customer-media/product-gallery/B007HCCOD0\nDisallow: /gp/help/customer/display.html?*nodeId=200534000\nDisallow: /gp/help/customer/display.html?*nodeId=200145650\nDisallow: /gp/feature.html?*docId=1000632623\nDisallow: /gp/aag\nDisallow: /gp/socialmedia/giveaways\nDisallow: /gp/aw/so.html\nDisallow: /gp/aw/sp.html?*t=returns\nDisallow: /gp/pdp/profile/\nDisallow: /gp/product/product-availability\nDisallow: /gp/offer-listing\nDisallow: /dp/twister-update/\nDisallow: /dp/e-mail-friend/\nDisallow: /gp/registry/wishlist/\nDisallow: /gp/switch-language\nDisallow: /wishlist/\nAllow: /wishlist/universal\nAllow: /wishlist/vendor-button\nAllow: /wishlist/get-button\nDisallow: /gp/wishlist/\nAllow: /gp/wishlist/universal\nAllow: /gp/wishlist/vendor-button\nAllow: /gp/wishlist/ipad-install\nDisallow: /registry/wishlist/\nDisallow: /local/ajax/\nDisallow: /gp/rate-it\nDisallow: /gp/rentallist\nDisallow: /gp/video/dvd-rental/settings\nDisallow: /gp/rl/settings\nDisallow: /gp/video/settings\nDisallow: /gp/video/watchlist\nDisallow: /gp/video/library\nDisallow: /o/di\nDisallow: /o/dt\nDisallow: /gp/profile\nDisallow: /reviews/iframe\nDisallow: /ss/customer-reviews/lighthouse/\nDisallow: /gp/aw/ol/\nDisallow: /hz/leaderboard/top-reviewers/\nDisallow: /hz/leaderboard/hall-of-fame/\nDisallow: /review/top-reviewers/\nDisallow: /review/top-reviewers\nDisallow: /review/hall-of-fame\nDisallow: /reviews/top-reviewers/\nDisallow: /reviews/top-reviewers\nDisallow: /reviews/hall-of-fame\nDisallow: /hz/help/contact/*/message/$\nDisallow: /-/\nAllow: /-/en/\nAllow: /-/en$\nDisallow: /gp/aw/shoppingAids/\n\nUser-agent: EtaoSpider\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nUser-Agent: PerplexityBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GoogleAgent-Mariner\nDisallow: /\n\nUser-agent: GoogleAgent-Shopping\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: meta-externalagent\nDisallow: /\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Devin\nDisallow: /\n\nUser-agent: omgili\nDisallow: /\n\nUser-agent: AI2Bot\nDisallow: /\n\nUser-agent: Gemini-Deep-Research\nDisallow: /\n\nUser-agent: PanguBot\nDisallow: /\n\nUser-agent: MistralAI-User\nDisallow: /\n\nUser-agent: Diffbot\nDisallow: /\n\nUser-agent: Sidetrade indexer bot\nDisallow: /\n\nUser-agent: Google-NotebookLM\nDisallow: /\n\nUser-agent: DuckAssistBot\nDisallow: /\n\nUser-agent: Copilot\nDisallow: /\n\nUser-agent: Ai2Bot-Dolma\nDisallow: /\n\nUser-agent: Google-CloudVertexBot\nDisallow: /\n\nUser-agent: meta-externalfetcher\nDisallow: /\n\nUser-agent: iaskspider\nDisallow: /\n\nUser-agent: wpbot\nDisallow: /\n\nUser-agent: aiHitBot\nDisallow: /\n\nUser-agent: cohere-ai\nDisallow: /\n\nUser-agent: FriendlyCrawler\nDisallow: /\n\nUser-agent: img2dataset\nDisallow: /\n\nUser-agent: VelenPublicWebCrawler\nDisallow: /\n\nUser-agent: YouBot\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: Claude-User\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Claude-SearchBot\nDisallow: /\n\nUser-agent: Perplexity-User\nDisallow: /\n\nUser-agent: ISSCyberRiskCrawler\nDisallow: /\n\nUser-agent: CopilotNative\nDisallow: /\n\nUser-agent: CopilotSapphire\nDisallow: /\n\nUser-agent: imgproxy\nDisallow: /\n\nUser-agent: SemrushBot-SWA\nDisallow: /\n\nUser-agent: ChatGPT-User\nDisallow: /\n\nUser-agent: OAI-SearchBot\nDisallow: /\n\nUser-agent: Timpibot\nDisallow: /\n\nUser-agent: Thinkbot\nDisallow: /\n\nUser-agent: AddSearchBot\nDisallow: /\n\nUser-agent: AI2Bot-DeepResearchEval\nDisallow: /\n\nUser-agent: Andibot\nDisallow: /\n\nUser-agent: Anomura\nDisallow: /\n\nUser-agent: atlassian-bot\nDisallow: /\n\nUser-agent: BuddyBot\nDisallow: /\n\nUser-agent: Channel3Bot\nDisallow: /\n\nUser-agent: ChatGLM-Spider\nDisallow: /\n\nUser-agent: Claude-Web\nDisallow: /\n\nUser-agent: Cloudflare-AutoRAG\nDisallow: /\n\nUser-agent: CloudflareBrowserRenderingCrawler\nDisallow: /\n\nUser-agent: cohere-training-data-crawler\nDisallow: /\n\nUser-agent: Cotoyogi\nDisallow: /\n\nUser-agent: Crawl4AI\nDisallow: /\n\nUser-agent: Datenbank Crawler\nDisallow: /\n\nUser-agent: DeepSeekBot\nDisallow: /\n\nUser-agent: Echobot Bot\nDisallow: /\n\nUser-agent: EchoboxBot\nDisallow: /\n\nUser-agent: factset_spyderbot\nDisallow: /\n\nUser-agent: Google-Firebase\nDisallow: /\n\nUser-agent: GoogleOther\nDisallow: /\n\nUser-agent: Grok-DeepSearch\nDisallow: /\n\nUser-agent: GrokBot\nDisallow: /\n\nUser-agent: iAskBot\nDisallow: /\n\nUser-agent: IbouBot\nDisallow: /\n\nUser-agent: ICC-Crawler\nDisallow: /\n\nUser-agent: ImagesiftBot\nDisallow: /\n\nUser-agent: imageSpider\nDisallow: /\n\nUser-agent: Kangaroo Bot\nDisallow: /\n\nUser-agent: KlaviyoAIBot\nDisallow: /\n\nUser-agent: KunatoCrawler\nDisallow: /\n\nUser-agent: laion-huggingface-processor\nDisallow: /\n\nUser-agent: Linguee Bot\nDisallow: /\n\nUser-agent: LinkupBot\nDisallow: /\n\nUser-agent: Manus-User\nDisallow: /\n\nUser-agent: meta-webindexer\nDisallow: /\n\nUser-agent: netEstate Imprint Crawler\nDisallow: /\n\nUser-agent: Panscient\nDisallow: /\n\nUser-agent: panscient.com\nDisallow: /\n\nUser-agent: PhindBot\nDisallow: /\n\nUser-agent: Poggio-Citations\nDisallow: /\n\nUser-agent: QualifiedBot\nDisallow: /\n\nUser-agent: SBIntuitionsBot\nDisallow: /\n\nUser-agent: ShapBot\nDisallow: /\n\nUser-agent: TavilyBot\nDisallow: /\n\nUser-agent: TerraCotta\nDisallow: /\n\nUser-agent: TwinAgent\nDisallow: /\n\nUser-agent: webzio-extended\nDisallow: /\n\nUser-agent: WRTNBot\nDisallow: /\n\nUser-agent: xAI-Grok\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: ZanistaBot\nDisallow: /"
    },
    "www.amazon.pl": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.amazon.pl",
      "body": "User-agent: *\nDisallow: /dp/product-availability/\nDisallow: /dp/rate-this-item/\nDisallow: /exec/obidos/account-access-login\nDisallow: /exec/obidos/change-style\nDisallow: /exec/obidos/di\nDisallow: /exec/obidos/dt\nDisallow: /exec/obidos/flex-sign-in\nDisallow: /exec/obidos/handle-buy-box\nDisallow: /exec/obidos/refer-a-friend-login\nDisallow: /exec/obidos/subst/associates/join\nDisallow: /exec/obidos/subst/marketplace/sell-your-collection.html\nDisallow: /exec/obidos/subst/marketplace/sell-your-stuff.html\nDisallow: /exec/obidos/subst/partners/friends/access.html\nDisallow: /exec/obidos/tg/cm/member/\nDisallow: /gp/cart\nDisallow: /gp/content-form\nDisallow: /gp/customer-images\nDisallow: /gp/customer-media/upload\nDisallow: /gp/customer-reviews/common/du\nDisallow: /gp/customer-reviews/write-a-review.html\nDisallow: /gp/flex\nDisallow: /gp/gfix\nDisallow: /gp/history\nDisallow: /gp/item-dispatch\nDisallow: /gp/legacy-handle-buy-box.html\nDisallow: /gp/reader\nDisallow: /gp/registry/wishlist/*/reserve\nDisallow: /gp/richpub/listmania/createpipeline\nDisallow: /gp/music/clipserve\nDisallow: /gp/recsradio\nDisallow: /gp/sign-in\nDisallow: /gp/slides/make-money\nDisallow: /gp/structured-ratings/actions/get-experience.html\nDisallow: /gp/twitter/\nDisallow: /gp/vote\nDisallow: /gp/voting/\nDisallow: /gp/yourstore\nDisallow: /ap/signin\nDisallow: /gp/registry/search.html\nDisallow: /gp/orc/rml/\nDisallow: /gp/dmusic/mp3/player\nDisallow: /gp/entity-alert/external\nDisallow: /gp/customer-reviews/dynamic/sims-box\nDisallow: /review/dynamic/sims-box\nDisallow: /gp/redirect.html\nDisallow: /gp/customer-media/actions/delete/\nDisallow: /gp/customer-media/actions/edit-caption/\nDisallow: /gp/dmusic/\nDisallow: /gp/customer-media/product-gallery/B007HCCOD0\nDisallow: /gp/help/customer/display.html?*nodeId=200534000\nDisallow: /gp/help/customer/display.html?*nodeId=200145650\nDisallow: /gp/feature.html?*docId=1000632623\nDisallow: /gp/aag\nDisallow: /gp/socialmedia/giveaways\nDisallow: /gp/aw/so.html\nDisallow: /gp/aw/sp.html?*t=returns\nDisallow: /gp/pdp/profile/\nDisallow: /gp/product/product-availability\nDisallow: /gp/offer-listing\nDisallow: /dp/twister-update/\nDisallow: /dp/e-mail-friend/\nDisallow: /gp/registry/wishlist/\nDisallow: /gp/switch-language\nDisallow: /wishlist/\nAllow: /wishlist/universal\nAllow: /wishlist/vendor-button\nAllow: /wishlist/get-button\nDisallow: /gp/wishlist/\nAllow: /gp/wishlist/universal\nAllow: /gp/wishlist/vendor-button\nAllow: /gp/wishlist/ipad-install\nDisallow: /registry/wishlist/\nDisallow: /local/ajax/\nDisallow: /gp/rate-it\nDisallow: /gp/rentallist\nDisallow: /gp/video/dvd-rental/settings\nDisallow: /gp/rl/settings\nDisallow: /gp/video/settings\nDisallow: /gp/video/watchlist\nDisallow: /gp/video/library\nDisallow: /o/di\nDisallow: /o/dt\nDisallow: /gp/profile\nDisallow: /reviews/iframe\nDisallow: /ss/customer-reviews/lighthouse/\nDisallow: /gp/aw/ol/\nDisallow: /hz/leaderboard/top-reviewers/\nDisallow: /hz/leaderboard/hall-of-fame/\nDisallow: /review/top-reviewers/\nDisallow: /review/top-reviewers\nDisallow: /review/hall-of-fame\nDisallow: /reviews/top-reviewers/\nDisallow: /reviews/top-reviewers\nDisallow: /reviews/hall-of-fame\nDisallow: /hz/help/contact/*/message/$\nDisallow: /-/\nDisallow: /gp/aw/shoppingAids/\n\nUser-agent: EtaoSpider\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nUser-Agent: PerplexityBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GoogleAgent-Mariner\nDisallow: /\n\nUser-agent: GoogleAgent-Shopping\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: meta-externalagent\nDisallow: /\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Devin\nDisallow: /\n\nUser-agent: omgili\nDisallow: /\n\nUser-agent: AI2Bot\nDisallow: /\n\nUser-agent: Gemini-Deep-Research\nDisallow: /\n\nUser-agent: PanguBot\nDisallow: /\n\nUser-agent: MistralAI-User\nDisallow: /\n\nUser-agent: Diffbot\nDisallow: /\n\nUser-agent: Sidetrade indexer bot\nDisallow: /\n\nUser-agent: Google-NotebookLM\nDisallow: /\n\nUser-agent: DuckAssistBot\nDisallow: /\n\nUser-agent: Copilot\nDisallow: /\n\nUser-agent: Ai2Bot-Dolma\nDisallow: /\n\nUser-agent: Google-CloudVertexBot\nDisallow: /\n\nUser-agent: meta-externalfetcher\nDisallow: /\n\nUser-agent: iaskspider\nDisallow: /\n\nUser-agent: wpbot\nDisallow: /\n\nUser-agent: aiHitBot\nDisallow: /\n\nUser-agent: cohere-ai\nDisallow: /\n\nUser-agent: FriendlyCrawler\nDisallow: /\n\nUser-agent: img2dataset\nDisallow: /\n\nUser-agent: VelenPublicWebCrawler\nDisallow: /\n\nUser-agent: YouBot\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: Claude-User\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Claude-SearchBot\nDisallow: /\n\nUser-agent: Perplexity-User\nDisallow: /\n\nUser-agent: ISSCyberRiskCrawler\nDisallow: /\n\nUser-agent: CopilotNative\nDisallow: /\n\nUser-agent: CopilotSapphire\nDisallow: /\n\nUser-agent: imgproxy\nDisallow: /\n\nUser-agent: SemrushBot-SWA\nDisallow: /\n\nUser-agent: ChatGPT-User\nDisallow: /\n\nUser-agent: OAI-SearchBot\nDisallow: /\n\nUser-agent: Timpibot\nDisallow: /\n\nUser-agent: Thinkbot\nDisallow: /\n\nUser-agent: AddSearchBot\nDisallow: /\n\nUser-agent: AI2Bot-DeepResearchEval\nDisallow: /\n\nUser-agent: Andibot\nDisallow: /\n\nUser-agent: Anomura\nDisallow: /\n\nUser-agent: atlassian-bot\nDisallow: /\n\nUser-agent: BuddyBot\nDisallow: /\n\nUser-agent: Channel3Bot\nDisallow: /\n\nUser-agent: ChatGLM-Spider\nDisallow: /\n\nUser-agent: Claude-Web\nDisallow: /\n\nUser-agent: Cloudflare-AutoRAG\nDisallow: /\n\nUser-agent: CloudflareBrowserRenderingCrawler\nDisallow: /\n\nUser-agent: cohere-training-data-crawler\nDisallow: /\n\nUser-agent: Cotoyogi\nDisallow: /\n\nUser-agent: Crawl4AI\nDisallow: /\n\nUser-agent: Datenbank Crawler\nDisallow: /\n\nUser-agent: DeepSeekBot\nDisallow: /\n\nUser-agent: Echobot Bot\nDisallow: /\n\nUser-agent: EchoboxBot\nDisallow: /\n\nUser-agent: factset_spyderbot\nDisallow: /\n\nUser-agent: Google-Firebase\nDisallow: /\n\nUser-agent: GoogleOther\nDisallow: /\n\nUser-agent: Grok-DeepSearch\nDisallow: /\n\nUser-agent: GrokBot\nDisallow: /\n\nUser-agent: iAskBot\nDisallow: /\n\nUser-agent: IbouBot\nDisallow: /\n\nUser-agent: ICC-Crawler\nDisallow: /\n\nUser-agent: ImagesiftBot\nDisallow: /\n\nUser-agent: imageSpider\nDisallow: /\n\nUser-agent: Kangaroo Bot\nDisallow: /\n\nUser-agent: KlaviyoAIBot\nDisallow: /\n\nUser-agent: KunatoCrawler\nDisallow: /\n\nUser-agent: laion-huggingface-processor\nDisallow: /\n\nUser-agent: Linguee Bot\nDisallow: /\n\nUser-agent: LinkupBot\nDisallow: /\n\nUser-agent: Manus-User\nDisallow: /\n\nUser-agent: meta-webindexer\nDisallow: /\n\nUser-agent: netEstate Imprint Crawler\nDisallow: /\n\nUser-agent: Panscient\nDisallow: /\n\nUser-agent: panscient.com\nDisallow: /\n\nUser-agent: PhindBot\nDisallow: /\n\nUser-agent: Poggio-Citations\nDisallow: /\n\nUser-agent: QualifiedBot\nDisallow: /\n\nUser-agent: SBIntuitionsBot\nDisallow: /\n\nUser-agent: ShapBot\nDisallow: /\n\nUser-agent: TavilyBot\nDisallow: /\n\nUser-agent: TerraCotta\nDisallow: /\n\nUser-agent: TwinAgent\nDisallow: /\n\nUser-agent: webzio-extended\nDisallow: /\n\nUser-agent: WRTNBot\nDisallow: /\n\nUser-agent: xAI-Grok\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: ZanistaBot\nDisallow: /"
    },
    "www.amazon.se": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.amazon.se",
      "body": "User-agent: *\nDisallow: /dp/product-availability/\nDisallow: /dp/rate-this-item/\nDisallow: /exec/obidos/account-access-login\nDisallow: /exec/obidos/change-style\nDisallow: /exec/obidos/di\nDisallow: /exec/obidos/dt\nDisallow: /exec/obidos/flex-sign-in\nDisallow: /exec/obidos/handle-buy-box\nDisallow: /exec/obidos/refer-a-friend-login\nDisallow: /exec/obidos/subst/associates/join\nDisallow: /exec/obidos/subst/marketplace/sell-your-collection.html\nDisallow: /exec/obidos/subst/marketplace/sell-your-stuff.html\nDisallow: /exec/obidos/subst/partners/friends/access.html\nDisallow: /exec/obidos/tg/cm/member/\nDisallow: /gp/cart\nDisallow: /gp/content-form\nDisallow: /gp/customer-images\nDisallow: /gp/customer-media/upload\nDisallow: /gp/customer-reviews/common/du\nDisallow: /gp/customer-reviews/write-a-review.html\nDisallow: /gp/flex\nDisallow: /gp/gfix\nDisallow: /gp/history\nDisallow: /gp/item-dispatch\nDisallow: /gp/legacy-handle-buy-box.html\nDisallow: /gp/reader\nDisallow: /gp/registry/wishlist/*/reserve\nDisallow: /gp/richpub/listmania/createpipeline\nDisallow: /gp/music/clipserve\nDisallow: /gp/recsradio\nDisallow: /gp/sign-in\nDisallow: /gp/slides/make-money\nDisallow: /gp/structured-ratings/actions/get-experience.html\nDisallow: /gp/twitter/\nDisallow: /gp/vote\nDisallow: /gp/voting/\nDisallow: /gp/yourstore\nDisallow: /ap/signin\nDisallow: /gp/registry/search.html\nDisallow: /gp/orc/rml/\nDisallow: /gp/dmusic/mp3/player\nDisallow: /gp/entity-alert/external\nDisallow: /gp/customer-reviews/dynamic/sims-box\nDisallow: /review/dynamic/sims-box\nDisallow: /gp/redirect.html\nDisallow: /gp/customer-media/actions/delete/\nDisallow: /gp/customer-media/actions/edit-caption/\nDisallow: /gp/dmusic/\nDisallow: /gp/customer-media/product-gallery/B007HCCOD0\nDisallow: /gp/help/customer/display.html?*nodeId=200534000\nDisallow: /gp/help/customer/display.html?*nodeId=200145650\nDisallow: /gp/feature.html?*docId=1000632623\nDisallow: /gp/aag\nDisallow: /gp/socialmedia/giveaways\nDisallow: /gp/aw/so.html\nDisallow: /gp/aw/sp.html?*t=returns\nDisallow: /gp/pdp/profile/\nDisallow: /gp/product/product-availability\nDisallow: /gp/offer-listing\nDisallow: /dp/twister-update/\nDisallow: /dp/e-mail-friend/\nDisallow: /gp/registry/wishlist/\nDisallow: /gp/switch-language\nDisallow: /wishlist/\nAllow: /wishlist/universal\nAllow: /wishlist/vendor-button\nAllow: /wishlist/get-button\nDisallow: /gp/wishlist/\nAllow: /gp/wishlist/universal\nAllow: /gp/wishlist/vendor-button\nAllow: /gp/wishlist/ipad-install\nDisallow: /registry/wishlist/\nDisallow: /local/ajax/\nDisallow: /gp/rate-it\nDisallow: /gp/rentallist\nDisallow: /gp/video/dvd-rental/settings\nDisallow: /gp/rl/settings\nDisallow: /gp/video/settings\nDisallow: /gp/video/watchlist\nDisallow: /gp/video/library\nDisallow: /o/di\nDisallow: /o/dt\nDisallow: /gp/profile\nDisallow: /reviews/iframe\nDisallow: /ss/customer-reviews/lighthouse/\nDisallow: /gp/aw/ol/\nDisallow: /hz/leaderboard/top-reviewers/\nDisallow: /hz/leaderboard/hall-of-fame/\nDisallow: /review/top-reviewers/\nDisallow: /review/top-reviewers\nDisallow: /review/hall-of-fame\nDisallow: /reviews/top-reviewers/\nDisallow: /reviews/top-reviewers\nDisallow: /reviews/hall-of-fame\nDisallow: /hz/help/contact/*/message/$\nDisallow: /-/\nAllow: /-/en/\nAllow: /-/en$\nDisallow: /gp/aw/shoppingAids/\nNoindex: */product-reviews/*&pageNumber=2\nNoindex: */product-reviews/*&pageNumber=3\nNoindex: */product-reviews/*&pageNumber=4\nNoindex: */product-reviews/*&pageNumber=5\nNoindex: */product-reviews/*&pageNumber=6\nNoindex: */product-reviews/*&pageNumber=7\nNoindex: */product-reviews/*&pageNumber=8\nNoindex: */product-reviews/*&pageNumber=9\n\nNoindex: /ask/questions/asin/*/2\nNoindex: /ask/questions/asin/*/3\nNoindex: /ask/questions/asin/*/4\nNoindex: /ask/questions/asin/*/5\nNoindex: /ask/questions/asin/*/6\nNoindex: /ask/questions/asin/*/7\nNoindex: /ask/questions/asin/*/8\nNoindex: /ask/questions/asin/*/9\n\nUser-agent: EtaoSpider\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nUser-Agent: PerplexityBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GoogleAgent-Mariner\nDisallow: /\n\nUser-agent: GoogleAgent-Shopping\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: meta-externalagent\nDisallow: /\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Devin\nDisallow: /\n\nUser-agent: omgili\nDisallow: /\n\nUser-agent: AI2Bot\nDisallow: /\n\nUser-agent: Gemini-Deep-Research\nDisallow: /\n\nUser-agent: PanguBot\nDisallow: /\n\nUser-agent: MistralAI-User\nDisallow: /\n\nUser-agent: Diffbot\nDisallow: /\n\nUser-agent: Sidetrade indexer bot\nDisallow: /\n\nUser-agent: Google-NotebookLM\nDisallow: /\n\nUser-agent: DuckAssistBot\nDisallow: /\n\nUser-agent: Copilot\nDisallow: /\n\nUser-agent: Ai2Bot-Dolma\nDisallow: /\n\nUser-agent: Google-CloudVertexBot\nDisallow: /\n\nUser-agent: meta-externalfetcher\nDisallow: /\n\nUser-agent: iaskspider\nDisallow: /\n\nUser-agent: wpbot\nDisallow: /\n\nUser-agent: aiHitBot\nDisallow: /\n\nUser-agent: cohere-ai\nDisallow: /\n\nUser-agent: FriendlyCrawler\nDisallow: /\n\nUser-agent: img2dataset\nDisallow: /\n\nUser-agent: VelenPublicWebCrawler\nDisallow: /\n\nUser-agent: YouBot\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: Claude-User\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Claude-SearchBot\nDisallow: /\n\nUser-agent: Perplexity-User\nDisallow: /\n\nUser-agent: ISSCyberRiskCrawler\nDisallow: /\n\nUser-agent: CopilotNative\nDisallow: /\n\nUser-agent: CopilotSapphire\nDisallow: /\n\nUser-agent: imgproxy\nDisallow: /\n\nUser-agent: SemrushBot-SWA\nDisallow: /\n\nUser-agent: ChatGPT-User\nDisallow: /\n\nUser-agent: OAI-SearchBot\nDisallow: /\n\nUser-agent: Timpibot\nDisallow: /\n\nUser-agent: Thinkbot\nDisallow: /\n\nUser-agent: AddSearchBot\nDisallow: /\n\nUser-agent: AI2Bot-DeepResearchEval\nDisallow: /\n\nUser-agent: Andibot\nDisallow: /\n\nUser-agent: Anomura\nDisallow: /\n\nUser-agent: atlassian-bot\nDisallow: /\n\nUser-agent: BuddyBot\nDisallow: /\n\nUser-agent: Channel3Bot\nDisallow: /\n\nUser-agent: ChatGLM-Spider\nDisallow: /\n\nUser-agent: Claude-Web\nDisallow: /\n\nUser-agent: Cloudflare-AutoRAG\nDisallow: /\n\nUser-agent: CloudflareBrowserRenderingCrawler\nDisallow: /\n\nUser-agent: cohere-training-data-crawler\nDisallow: /\n\nUser-agent: Cotoyogi\nDisallow: /\n\nUser-agent: Crawl4AI\nDisallow: /\n\nUser-agent: Datenbank Crawler\nDisallow: /\n\nUser-agent: DeepSeekBot\nDisallow: /\n\nUser-agent: Echobot Bot\nDisallow: /\n\nUser-agent: EchoboxBot\nDisallow: /\n\nUser-agent: factset_spyderbot\nDisallow: /\n\nUser-agent: Google-Firebase\nDisallow: /\n\nUser-agent: GoogleOther\nDisallow: /\n\nUser-agent: Grok-DeepSearch\nDisallow: /\n\nUser-agent: GrokBot\nDisallow: /\n\nUser-agent: iAskBot\nDisallow: /\n\nUser-agent: IbouBot\nDisallow: /\n\nUser-agent: ICC-Crawler\nDisallow: /\n\nUser-agent: ImagesiftBot\nDisallow: /\n\nUser-agent: imageSpider\nDisallow: /\n\nUser-agent: Kangaroo Bot\nDisallow: /\n\nUser-agent: KlaviyoAIBot\nDisallow: /\n\nUser-agent: KunatoCrawler\nDisallow: /\n\nUser-agent: laion-huggingface-processor\nDisallow: /\n\nUser-agent: Linguee Bot\nDisallow: /\n\nUser-agent: LinkupBot\nDisallow: /\n\nUser-agent: Manus-User\nDisallow: /\n\nUser-agent: meta-webindexer\nDisallow: /\n\nUser-agent: netEstate Imprint Crawler\nDisallow: /\n\nUser-agent: Panscient\nDisallow: /\n\nUser-agent: panscient.com\nDisallow: /\n\nUser-agent: PhindBot\nDisallow: /\n\nUser-agent: Poggio-Citations\nDisallow: /\n\nUser-agent: QualifiedBot\nDisallow: /\n\nUser-agent: SBIntuitionsBot\nDisallow: /\n\nUser-agent: ShapBot\nDisallow: /\n\nUser-agent: TavilyBot\nDisallow: /\n\nUser-agent: TerraCotta\nDisallow: /\n\nUser-agent: TwinAgent\nDisallow: /\n\nUser-agent: webzio-extended\nDisallow: /\n\nUser-agent: WRTNBot\nDisallow: /\n\nUser-agent: xAI-Grok\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: ZanistaBot\nDisallow: /"
    },
    "www.amazon.com.be": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.amazon.com.be",
      "body": "# This is a test robots.txt file.\nUser-agent: *\nDisallow: /dp/product-availability/\nDisallow: /dp/rate-this-item/\nDisallow: /exec/obidos/account-access-login\nDisallow: /exec/obidos/change-style\nDisallow: /exec/obidos/di\nDisallow: /exec/obidos/dt\nDisallow: /exec/obidos/flex-sign-in\nDisallow: /exec/obidos/handle-buy-box\nDisallow: /exec/obidos/refer-a-friend-login\nDisallow: /exec/obidos/subst/associates/join\nDisallow: /exec/obidos/subst/marketplace/sell-your-collection.html\nDisallow: /exec/obidos/subst/marketplace/sell-your-stuff.html\nDisallow: /exec/obidos/subst/partners/friends/access.html\nDisallow: /exec/obidos/tg/cm/member/\nDisallow: /gp/cart\nDisallow: /gp/content-form\nDisallow: /gp/customer-images\nDisallow: /gp/customer-media/upload\nDisallow: /gp/customer-reviews/common/du\nDisallow: /gp/customer-reviews/write-a-review.html\nDisallow: /gp/flex\nDisallow: /gp/gfix\nDisallow: /gp/history\nDisallow: /gp/item-dispatch\nDisallow: /gp/legacy-handle-buy-box.html\nDisallow: /gp/reader\nDisallow: /gp/registry/wishlist/*/reserve\nDisallow: /gp/richpub/listmania/createpipeline\nDisallow: /gp/music/clipserve\nDisallow: /gp/recsradio\nDisallow: /gp/sign-in\nDisallow: /gp/slides/make-money\nDisallow: /gp/structured-ratings/actions/get-experience.html\nDisallow: /gp/twitter/\nDisallow: /gp/vote\nDisallow: /gp/voting/\nDisallow: /gp/yourstore\nDisallow: /ap/signin\nDisallow: /gp/registry/search.html\nDisallow: /gp/orc/rml/\nDisallow: /gp/dmusic/mp3/player\nDisallow: /gp/entity-alert/external\nDisallow: /gp/customer-reviews/dynamic/sims-box\nDisallow: /review/dynamic/sims-box\nDisallow: /gp/redirect.html\nDisallow: /gp/customer-media/actions/delete/\nDisallow: /gp/customer-media/actions/edit-caption/\nDisallow: /gp/dmusic/\nDisallow: /gp/customer-media/product-gallery/B007HCCOD0\nDisallow: /gp/help/customer/display.html?*nodeId=200534000\nDisallow: /gp/help/customer/display.html?*nodeId=200145650\nDisallow: /gp/feature.html?*docId=1000632623\nDisallow: /gp/aag\nDisallow: /gp/socialmedia/giveaways\nDisallow: /gp/aw/so.html\nDisallow: /gp/aw/sp.html?*t=returns\nDisallow: /gp/pdp/profile/\nDisallow: /gp/product/product-availability\nDisallow: /gp/offer-listing\nDisallow: /dp/twister-update/\nDisallow: /dp/e-mail-friend/\nDisallow: /gp/registry/wishlist/\nDisallow: /gp/switch-language\nDisallow: /wishlist/\nAllow: /wishlist/universal\nAllow: /wishlist/vendor-button\nAllow: /wishlist/get-button\nDisallow: /gp/wishlist/\nAllow: /gp/wishlist/universal\nAllow: /gp/wishlist/vendor-button\nAllow: /gp/wishlist/ipad-install\nDisallow: /registry/wishlist/\nDisallow: /local/ajax/\nDisallow: /gp/rate-it\nDisallow: /gp/rentallist\nDisallow: /gp/video/dvd-rental/settings\nDisallow: /gp/rl/settings\nDisallow: /gp/video/settings\nDisallow: /gp/video/watchlist\nDisallow: /gp/video/library\nDisallow: /o/di\nDisallow: /o/dt\nDisallow: /gp/profile\nDisallow: /reviews/iframe\nDisallow: /ss/customer-reviews/lighthouse/\nDisallow: /gp/aw/ol/\nDisallow: /hz/leaderboard/top-reviewers/\nDisallow: /hz/leaderboard/hall-of-fame/\nDisallow: /review/top-reviewers/\nDisallow: /review/top-reviewers\nDisallow: /review/hall-of-fame\nDisallow: /reviews/top-reviewers/\nDisallow: /reviews/top-reviewers\nDisallow: /reviews/hall-of-fame\nDisallow: /hz/help/contact/*/message/$\nDisallow: /-/\nAllow: /-/en/\nAllow: /-/en$\nAllow: /-/nl/\nAllow: /-/nl$\nDisallow: /gp/aw/shoppingAids/\n\nUser-agent: EtaoSpider\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: CCBot\nDisallow: /\n\nUser-Agent: PerplexityBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GoogleAgent-Mariner\nDisallow: /\n\nUser-agent: GoogleAgent-Shopping\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: meta-externalagent\nDisallow: /\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Devin\nDisallow: /\n\nUser-agent: omgili\nDisallow: /\n\nUser-agent: AI2Bot\nDisallow: /\n\nUser-agent: Gemini-Deep-Research\nDisallow: /\n\nUser-agent: PanguBot\nDisallow: /\n\nUser-agent: MistralAI-User\nDisallow: /\n\nUser-agent: Diffbot\nDisallow: /\n\nUser-agent: Sidetrade indexer bot\nDisallow: /\n\nUser-agent: Google-NotebookLM\nDisallow: /\n\nUser-agent: DuckAssistBot\nDisallow: /\n\nUser-agent: Copilot\nDisallow: /\n\nUser-agent: Ai2Bot-Dolma\nDisallow: /\n\nUser-agent: Google-CloudVertexBot\nDisallow: /\n\nUser-agent: meta-externalfetcher\nDisallow: /\n\nUser-agent: iaskspider\nDisallow: /\n\nUser-agent: wpbot\nDisallow: /\n\nUser-agent: aiHitBot\nDisallow: /\n\nUser-agent: cohere-ai\nDisallow: /\n\nUser-agent: FriendlyCrawler\nDisallow: /\n\nUser-agent: img2dataset\nDisallow: /\n\nUser-agent: VelenPublicWebCrawler\nDisallow: /\n\nUser-agent: YouBot\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: Claude-User\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Claude-SearchBot\nDisallow: /\n\nUser-agent: Perplexity-User\nDisallow: /\n\nUser-agent: ISSCyberRiskCrawler\nDisallow: /\n\nUser-agent: CopilotNative\nDisallow: /\n\nUser-agent: CopilotSapphire\nDisallow: /\n\nUser-agent: imgproxy\nDisallow: /\n\nUser-agent: SemrushBot-SWA\nDisallow: /\n\nUser-agent: ChatGPT-User\nDisallow: /\n\nUser-agent: OAI-SearchBot\nDisallow: /\n\nUser-agent: Timpibot\nDisallow: /\n\nUser-agent: Thinkbot\nDisallow: /\n\nUser-agent: AddSearchBot\nDisallow: /\n\nUser-agent: AI2Bot-DeepResearchEval\nDisallow: /\n\nUser-agent: Andibot\nDisallow: /\n\nUser-agent: Anomura\nDisallow: /\n\nUser-agent: atlassian-bot\nDisallow: /\n\nUser-agent: BuddyBot\nDisallow: /\n\nUser-agent: Channel3Bot\nDisallow: /\n\nUser-agent: ChatGLM-Spider\nDisallow: /\n\nUser-agent: Claude-Web\nDisallow: /\n\nUser-agent: Cloudflare-AutoRAG\nDisallow: /\n\nUser-agent: CloudflareBrowserRenderingCrawler\nDisallow: /\n\nUser-agent: cohere-training-data-crawler\nDisallow: /\n\nUser-agent: Cotoyogi\nDisallow: /\n\nUser-agent: Crawl4AI\nDisallow: /\n\nUser-agent: Datenbank Crawler\nDisallow: /\n\nUser-agent: DeepSeekBot\nDisallow: /\n\nUser-agent: Echobot Bot\nDisallow: /\n\nUser-agent: EchoboxBot\nDisallow: /\n\nUser-agent: factset_spyderbot\nDisallow: /\n\nUser-agent: Google-Firebase\nDisallow: /\n\nUser-agent: GoogleOther\nDisallow: /\n\nUser-agent: Grok-DeepSearch\nDisallow: /\n\nUser-agent: GrokBot\nDisallow: /\n\nUser-agent: iAskBot\nDisallow: /\n\nUser-agent: IbouBot\nDisallow: /\n\nUser-agent: ICC-Crawler\nDisallow: /\n\nUser-agent: ImagesiftBot\nDisallow: /\n\nUser-agent: imageSpider\nDisallow: /\n\nUser-agent: Kangaroo Bot\nDisallow: /\n\nUser-agent: KlaviyoAIBot\nDisallow: /\n\nUser-agent: KunatoCrawler\nDisallow: /\n\nUser-agent: laion-huggingface-processor\nDisallow: /\n\nUser-agent: Linguee Bot\nDisallow: /\n\nUser-agent: LinkupBot\nDisallow: /\n\nUser-agent: Manus-User\nDisallow: /\n\nUser-agent: meta-webindexer\nDisallow: /\n\nUser-agent: netEstate Imprint Crawler\nDisallow: /\n\nUser-agent: Panscient\nDisallow: /\n\nUser-agent: panscient.com\nDisallow: /\n\nUser-agent: PhindBot\nDisallow: /\n\nUser-agent: Poggio-Citations\nDisallow: /\n\nUser-agent: QualifiedBot\nDisallow: /\n\nUser-agent: SBIntuitionsBot\nDisallow: /\n\nUser-agent: ShapBot\nDisallow: /\n\nUser-agent: TavilyBot\nDisallow: /\n\nUser-agent: TerraCotta\nDisallow: /\n\nUser-agent: TwinAgent\nDisallow: /\n\nUser-agent: webzio-extended\nDisallow: /\n\nUser-agent: WRTNBot\nDisallow: /\n\nUser-agent: xAI-Grok\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: ZanistaBot\nDisallow: /"
    },
    "www.amazon.com": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.amazon.com",
      "body": "User-agent: *\nDisallow: /exec/obidos/account-access-login\nDisallow: /exec/obidos/change-style\nDisallow: /exec/obidos/flex-sign-in\nDisallow: /exec/obidos/handle-buy-box\nDisallow: /exec/obidos/tg/cm/member/\nDisallow: /gp/aw/help/id=sss\nDisallow: /gp/cart\nDisallow: /gp/flex\nDisallow: /gp/product/e-mail-friend\nDisallow: /gp/product/product-availability\nDisallow: /gp/product/rate-this-item\nDisallow: /gp/sign-in\nDisallow: /gp/reader\nDisallow: /gp/sitbv3/reader\nDisallow: /gp/richpub/syltguides/create\nDisallow: /gp/gfix\nDisallow: /gp/associations/wizard.html\nDisallow: /gp/dmusic/order\nDisallow: /gp/legacy-handle-buy-box.html\nDisallow: /gp/aws/ssop\nDisallow: /gp/yourstore\nDisallow: /gp/gift-central/organizer/add-wishlist\nDisallow: /gp/vote\nDisallow: /gp/voting/\nDisallow: /gp/music/wma-pop-up\nDisallow: /gp/customer-images\nDisallow: /gp/richpub/listmania/createpipeline\nDisallow: /gp/content-form\nDisallow: /gp/pdp/invitation/invite\nDisallow: /gp/customer-reviews/common/du\nDisallow: /gp/customer-reviews/write-a-review.html\nDisallow: /gp/associations/wizard.html\nDisallow: /gp/music/clipserve\nDisallow: /gp/customer-media/upload\nDisallow: /gp/history\nDisallow: /gp/item-dispatch\nDisallow: /gp/dmusic/order/handle-buy-box.html\nDisallow: /gp/recsradio\nDisallow: /gp/slredirect\nDisallow: /dp/shipping/\nDisallow: /dp/twister-update/\nDisallow: /dp/manual-submit/\nDisallow: /dp/e-mail-friend/\nDisallow: /dp/product-availability/\nDisallow: /dp/rate-this-item/\nDisallow: /gp/registry/wishlist/*/reserve\nDisallow: /gp/structured-ratings/actions/get-experience.html\nDisallow: /gp/twitter/\nDisallow: /ap/signin\nDisallow: /gp/registry/wishlist/\nDisallow: /wishlist/\nAllow: /wishlist/universal*\nAllow: /wishlist/vendor-button*\nAllow: /wishlist/get-button*\nDisallow: /gp/wishlist/\nAllow: /gp/wishlist/universal*\nAllow: /gp/wishlist/vendor-button*\nAllow: /gp/wishlist/ipad-install*\nDisallow: /registry/wishlist/\nDisallow: /review/common/du\nDisallow: /gp/registry/search.html\nDisallow: /product-reviews/B0069IY63Y\nDisallow: /gp/orc/rml/\nDisallow: */gcrnsts\nDisallow: /gp/gc/widget\nDisallow: /gp/dmusic/mp3/player\nDisallow: /gp/entity-alert/external\nDisallow: /gp/customer-reviews/dynamic/sims-box\nDisallow: /review/dynamic/sims-box\nDisallow: /gp/redirect.html\nDisallow: /gp/twister/ajaxv2\nDisallow: /ss/twister/ajax\nDisallow: /b?*node=7454917011\nDisallow: /b?*node=7454927011\nDisallow: /b?*node=7454939011\nDisallow: /b?*node=7454898011\nDisallow: /gp/customer-media/actions/delete/\nDisallow: /gp/customer-media/actions/edit-caption/\nDisallow: /gp/dmusic/\nAllow: /gp/dmusic/promotions/PrimeMusic\nAllow: /gp/dmusic/promotions/AmazonMusicUnlimited\nDisallow: /gp/offer-listing/\nDisallow: /b?*node=9052533011\nDisallow: /lm/R1XIHQVKXSKBNJ\nDisallow: /lm/R3HQ5WJSZK6QSO\nDisallow: /surprise/\nDisallow: /local/ajax/\nDisallow: /gp/socialmedia/giveaways\nDisallow: /gp/b2b-rd\nDisallow: /gp/aw/so.html\nDisallow: /reviews/iframe\nDisallow: /gp/switch-language\nDisallow: /ga/p/\nDisallow: /gp/profile/\nDisallow: /giveaway/host/setup/\nDisallow: /ss/customer-reviews/lighthouse/\nDisallow: /ospublishing/story/*\nDisallow: /gp/aw/ol/\nDisallow: /gp/promotion/\nDisallow: /hz/leaderboard/top-reviewers/\nDisallow: /creatorhub\nDisallow: /creatorhub/*\nDisallow: /slp/s$\nDisallow: /-/\nAllow: /-/es/\nAllow: /-/en$\nAllow: /-/zh_TW/\nAllow: /-/zh_TW$\nAllow: /-/he/\nAllow: /-/he$\nDisallow: /hz/help/contact/*/message/$\nDisallow: /gp/aw/shoppingAids/\nDisallow: /rss/people/*/reviews\nDisallow: /gp/pdp/rss/*/reviews\nDisallow: /gp/cdp/member-reviews/\nDisallow: /gp/aw/cr/\nAllow: /gp/offer-listing/B000\nAllow: /gp/offer-listing/9000\nDisallow: /gp/aag\nAllow: /gp/aag/main?*seller=ABVFEJU8LS620\nDisallow: /gp/pdp/profile/\nDisallow: /gp/help/customer/express/c2c/\nDisallow: /slp/*/b$\nDisallow: /hz/contact-us/ajax/initiate-trusted-contact/\nDisallow: /hp/video/\nDisallow: /gp/video/api\nDisallow: /gp/video/settings\nDisallow: /gp/video/library\nDisallow: /gp/video/watchlist\nDisallow: /gp/video/mystuff\nDisallow: /gp/video/profiles\nDisallow: /gp/video/search\nDisallow: /gp/video/auth\nDisallow: /gp/video/salp/i\nDisallow: /icdp/\n\nUser-agent: EtaoSpider\nDisallow: /\n\nUser-agent: GPTBot\nDisallow: /\n\n\nUser-agent: CCBot\nDisallow: /\n\nUser-Agent: PerplexityBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: GoogleAgent-Mariner\nDisallow: /\n\nUser-agent: GoogleAgent-Shopping\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: meta-externalagent\nDisallow: /\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: Scrapy\nDisallow: /\n\nUser-agent: PetalBot\nDisallow: /\n\nUser-agent: Devin\nDisallow: /\n\nUser-agent: omgili\nDisallow: /\n\nUser-agent: AI2Bot\nDisallow: /\n\nUser-agent: Gemini-Deep-Research\nDisallow: /\n\nUser-agent: PanguBot\nDisallow: /\n\nUser-agent: MistralAI-User\nDisallow: /\n\nUser-agent: Diffbot\nDisallow: /\n\nUser-agent: Sidetrade indexer bot\nDisallow: /\n\nUser-agent: Google-NotebookLM\nDisallow: /\n\nUser-agent: DuckAssistBot\nDisallow: /\n\nUser-agent: Copilot\nDisallow: /\n\nUser-agent: Ai2Bot-Dolma\nDisallow: /\n\nUser-agent: Google-CloudVertexBot\nDisallow: /\n\nUser-agent: meta-externalfetcher\nDisallow: /\n\nUser-agent: iaskspider\nDisallow: /\n\nUser-agent: wpbot\nDisallow: /\n\nUser-agent: aiHitBot\nDisallow: /\n\nUser-agent: cohere-ai\nDisallow: /\n\nUser-agent: FriendlyCrawler\nDisallow: /\n\nUser-agent: img2dataset\nDisallow: /\n\nUser-agent: VelenPublicWebCrawler\nDisallow: /\n\nUser-agent: YouBot\nDisallow: /\n\nUser-agent: Brightbot\nDisallow: /\n\nUser-agent: Claude-User\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Claude-SearchBot\nDisallow: /\n\nUser-agent: Perplexity-User\nDisallow: /\n\nUser-agent: ISSCyberRiskCrawler\nDisallow: /\n\nUser-agent: CopilotNative\nDisallow: /\n\nUser-agent: CopilotSapphire\nDisallow: /\n\nUser-agent: imgproxy\nDisallow: /\n\nUser-agent: SemrushBot-SWA\nDisallow: /\n\nUser-agent: ChatGPT-User\nDisallow: /\n\nUser-agent: OAI-SearchBot\nDisallow: /\n\nUser-agent: Timpibot\nDisallow: /\n\nUser-agent: Thinkbot\nDisallow: /\n\nUser-agent: AddSearchBot\nDisallow: /\n\nUser-agent: AI2Bot-DeepResearchEval\nDisallow: /\n\nUser-agent: Andibot\nDisallow: /\n\nUser-agent: Anomura\nDisallow: /\n\nUser-agent: atlassian-bot\nDisallow: /\n\nUser-agent: BuddyBot\nDisallow: /\n\nUser-agent: Channel3Bot\nDisallow: /\n\nUser-agent: ChatGLM-Spider\nDisallow: /\n\nUser-agent: Claude-Web\nDisallow: /\n\nUser-agent: Cloudflare-AutoRAG\nDisallow: /\n\nUser-agent: CloudflareBrowserRenderingCrawler\nDisallow: /\n\nUser-agent: cohere-training-data-crawler\nDisallow: /\n\nUser-agent: Cotoyogi\nDisallow: /\n\nUser-agent: Crawl4AI\nDisallow: /\n\nUser-agent: Datenbank Crawler\nDisallow: /\n\nUser-agent: DeepSeekBot\nDisallow: /\n\nUser-agent: Echobot Bot\nDisallow: /\n\nUser-agent: EchoboxBot\nDisallow: /\n\nUser-agent: factset_spyderbot\nDisallow: /\n\nUser-agent: Google-Firebase\nDisallow: /\n\nUser-agent: GoogleOther\nDisallow: /\n\nUser-agent: Grok-DeepSearch\nDisallow: /\n\nUser-agent: GrokBot\nDisallow: /\n\nUser-agent: iAskBot\nDisallow: /\n\nUser-agent: IbouBot\nDisallow: /\n\nUser-agent: ICC-Crawler\nDisallow: /\n\nUser-agent: ImagesiftBot\nDisallow: /\n\nUser-agent: imageSpider\nDisallow: /\n\nUser-agent: Kangaroo Bot\nDisallow: /\n\nUser-agent: KlaviyoAIBot\nDisallow: /\n\nUser-agent: KunatoCrawler\nDisallow: /\n\nUser-agent: laion-huggingface-processor\nDisallow: /\n\nUser-agent: Linguee Bot\nDisallow: /\n\nUser-agent: LinkupBot\nDisallow: /\n\nUser-agent: Manus-User\nDisallow: /\n\nUser-agent: meta-webindexer\nDisallow: /\n\nUser-agent: netEstate Imprint Crawler\nDisallow: /\n\nUser-agent: Panscient\nDisallow: /\n\nUser-agent: panscient.com\nDisallow: /\n\nUser-agent: PhindBot\nDisallow: /\n\nUser-agent: Poggio-Citations\nDisallow: /\n\nUser-agent: QualifiedBot\nDisallow: /\n\nUser-agent: SBIntuitionsBot\nDisallow: /\n\nUser-agent: ShapBot\nDisallow: /\n\nUser-agent: TavilyBot\nDisallow: /\n\nUser-agent: TerraCotta\nDisallow: /\n\nUser-agent: TwinAgent\nDisallow: /\n\nUser-agent: webzio-extended\nDisallow: /\n\nUser-agent: WRTNBot\nDisallow: /\n\nUser-agent: xAI-Grok\nDisallow: /\n\nUser-agent: YaK\nDisallow: /\n\nUser-agent: ZanistaBot\nDisallow: /"
    },
    "www.ebay.de": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.ebay.de",
      "body": "## BEGIN FILE ###\n#\n# allow-all\n# DR\n#\n# The use of robots or other automated means to access the eBay site\n# without the express permission of eBay is strictly prohibited.\n# Notwithstanding the foregoing, eBay may permit automated access to\n# access certain eBay pages but solely for the limited purpose of\n# including content in publicly available search engines. Any other\n# use of robots or failure to obey the robots exclusion standards set\n# forth at <https://www.robotstxt.org/orig.html> is strictly\n# prohibited.\n#\n# \n# Robots & Agent Policy\n#\n# Checkouts are strictly for human users. \n# * Automated scraping, buy-for-me agents, LLM-driven bots, or any \n#   end-to-end flow that attempts to place orders without human review \n#   is strictly prohibited. \n# * Unauthorized use of automated agents in checkout may result in \n#   legal action under our User Agreement: https://www.ebay.com/help/policies/member-behaviour-policies/user-agreement?id=4259 \n# * Approved enterprise integrations must use our official API and \n#   comply with our API License Agreement: https://developer.ebay.com/join/api-license-agreement \n#\n# v28.1_DE_August_2026\n### DIRECTIVES ###\n\nUser-agent: *\nDisallow: /*_kw\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nDisallow: /buyer-preferences\nDisallow: /motors/fnd/*\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nAllow: /b/*?*_mwBanner\nDisallow: /b/*?*_nkw\nDisallow: /b/*?*&\nDisallow: /b/*?*|\nDisallow: /b/*%EF\nDisallow: /b/*LH_\nDisallow: /b/*_sop=\nDisallow: /b/*_pgn=\nDisallow: /b/*,\nDisallow: /b/*_dmd=\nDisallow: /b/*_dcat=\nDisallow: /b/*_sacat=\nDisallow: /b/*_saved\nDisallow: /b/*_sid=\nDisallow: /b/*_stpos=\nDisallow: /b/*udhi=\nDisallow: /b/*udlo=\nDisallow: */b2ajax/all-filter\nDisallow: /b/Test-Category\nDisallow: /b/eBay-Use-Only\nDisallow: /b/eBay-Test-Only\nDisallow: /b/LP-Parent\nDisallow: /b/LL-Child\nDisallow: /b/eBay-User-Tools\nDisallow: /b/Test-Auctions\nDisallow: /b/Attributes\nDisallow: /b/*DO-NOT-BID\nDisallow: /b/Testbereich/\nDisallow: /b/Testkategoriegruppe/\nDisallow: /b/Testkategorie\nDisallow: /bin/\nDisallow: /bo/\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /blueberry/\nDisallow: /brw/\nDisallow: /cancel/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /deinestadt/*_pgn=\nDisallow: /delicacy\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebaylive/host\nDisallow: /ebay/cronus/\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /feed/\nDisallow: /fdbk/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gh/user_profile\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gwc/\nDisallow: /handy/\nDisallow: /hcp/\nDisallow: /heute/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itempolicy\nDisallow: /itemnotreceived/\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /looking-glass-proxy/graphql\nDisallow: /logout\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*surveyopen\nDisallow: /p/*thm=\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /price-guide/\nDisallow: /prp/\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/ebayadvsearch\nAllow: /sch/allcategories/\nAllow: /sch/*_ul\nAllow: /sch/i.html?*&mkcid=2\nDisallow: /sch/\nDisallow: /sch/ajax\nDisallow: /sch/ebayadvsearch?_\nDisallow: /ship/\nDisallow: /shop/*&\nAllow: /signin/$\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sh/onboarding\nDisallow: /sh/str/fvfsavings\nDisallow: /ship/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/ipe\nDisallow: /sl/node\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nDisallow: /usr/*/all-follows\nDisallow: /usr/*/followers\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vplbl/\nDisallow: /vod/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /ws/\n\nUser-agent: Googlebot\nDisallow: /*_kw\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nDisallow: /buyer-preferences\nDisallow: /motors/fnd/*\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nAllow: /b/*?*_mwBanner\nDisallow: /b/*?*_nkw\nDisallow: /b/*?*&\nDisallow: /b/*?*|\nDisallow: /b/*%EF\nDisallow: /b/*LH_\nDisallow: /b/*_sop=\nDisallow: /b/*_pgn=\nDisallow: /b/*,\nDisallow: /b/*_dmd=\nDisallow: /b/*_dcat=\nDisallow: /b/*_sacat=\nDisallow: /b/*_saved\nDisallow: /b/*_sid=\nDisallow: /b/*_stpos=\nDisallow: /b/*udhi=\nDisallow: /b/*udlo=\nDisallow: */b2ajax/all-filter\nDisallow: /b/Test-Category\nDisallow: /b/eBay-Use-Only\nDisallow: /b/eBay-Test-Only\nDisallow: /b/LP-Parent\nDisallow: /b/LL-Child\nDisallow: /b/eBay-User-Tools\nDisallow: /b/Test-Auctions\nDisallow: /b/Attributes\nDisallow: /b/*DO-NOT-BID\nDisallow: /b/Testbereich/\nDisallow: /b/Testkategoriegruppe/\nDisallow: /b/Testkategorie\nDisallow: /bin/\nDisallow: /bo/\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /blueberry/\nDisallow: /brw/\nDisallow: /cancel/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /deinestadt/*_pgn=\nDisallow: /delicacy\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebaylive/host\nDisallow: /ebay/cronus/\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /feed/\nDisallow: /fdbk/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gh/user_profile\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gwc/\nDisallow: /handy/\nDisallow: /hcp/\nDisallow: /heute/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itempolicy\nDisallow: /itemnotreceived/\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /looking-glass-proxy/graphql\nDisallow: /logout\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*surveyopen\nDisallow: /p/*thm=\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /price-guide/\nDisallow: /prp/\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/ebayadvsearch\nAllow: /sch/allcategories/\nAllow: /sch/*_ul\nAllow: /sch/i.html?*&mkcid=2\nDisallow: /sch/\nDisallow: /sch/ajax\nDisallow: /sch/ebayadvsearch?_\nDisallow: /ship/\nDisallow: /shop/*&\nAllow: /signin/$\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sh/onboarding\nDisallow: /sh/str/fvfsavings\nDisallow: /ship/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/ipe\nDisallow: /sl/node\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nDisallow: /usr/*/all-follows\nDisallow: /usr/*/followers\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vplbl/\nDisallow: /vod/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /ws/\n\nUser-agent: bingbot\nDisallow: /*_kw\nDisallow: /b/*_mwBanner=1*rt=nc\nDisallow: /b/*rt=nc*_mwBanner=1 \nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nAllow: /b/*?*_mwBanner\nDisallow: /b/*?*_nkw\nDisallow: /b/*?*&\nDisallow: /b/*?*|\nDisallow: /b/*%EF\nDisallow: /b/*LH_\nDisallow: /b/*_sop=\nDisallow: /b/*_pgn=\nDisallow: /b/*,\nDisallow: /b/*_dmd=\nDisallow: /b/*_dcat=\nDisallow: /b/*_sacat=\nDisallow: /b/*_saved\nDisallow: /b/*_sid=\nDisallow: /b/*_stpos=\nDisallow: /b/*udhi=\nDisallow: /b/*udlo=\nDisallow: /b/Test-Category\nDisallow: /b/eBay-Use-Only\nDisallow: /b/eBay-Test-Only\nDisallow: /b/LP-Parent\nDisallow: /b/LL-Child\nDisallow: /b/eBay-User-Tools\nDisallow: /b/Test-Auctions\nDisallow: /b/Attributes\nDisallow: /b/*DO-NOT-BID\nDisallow: /b/Testbereich/\nDisallow: /b/Testkategoriegruppe/\nDisallow: /b/Testkategorie\nDisallow: /bin/\nDisallow: /bo/\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /blueberry/\nDisallow: /brw/\nDisallow: /cancel/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /deinestadt/*_pgn=\nDisallow: /delicacy\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebay/cronus/\nDisallow: /ebaylive/host\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /feed/\nDisallow: /fdbk/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gh/user_profile\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gwc/\nDisallow: /hcp/\nDisallow: /handy/\nDisallow: /heute/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itempolicy\nDisallow: /itemnotreceived/\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /logout\nDisallow: /looking-glass-proxy/graphql\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*surveyopen\nDisallow: /p/*thm=\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /price-guide/\nDisallow: /prp/\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/ebayadvsearch\nAllow: /sch/allcategories/\nAllow: /sch/*_ul\nAllow: /sch/i.html?*&mkcid=2\nDisallow: /sch/\nDisallow: /sch/ajax\nDisallow: /sch/ebayadvsearch?_\nDisallow: /ship/\nAllow: /signin/$\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sh/onboarding\nDisallow: /sh/str/fvfsavings\nDisallow: /ship/\nDisallow: /shop/*&\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/ipe\nDisallow: /sl/node\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nDisallow: /usr/\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vplbl/\nDisallow: /vod/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /ws/\n\nUser-agent: AdsBot-Google\nDisallow: /*src=urllib\nDisallow: /adell-mutfak\nDisallow: /ahu-kadin\nDisallow: /antika-sanat\nDisallow: /arama\nDisallow: /bebek-anne\nDisallow: /bilgisayar-tablet\nDisallow: /cep-telefonu-ve-aksesuar\nDisallow: /ev-bahce\nDisallow: /giyim-aksesuar\nDisallow: /muzik-plak-enstruman\nDisallow: /nap/\nDisallow: /otomobil-motor-aksesuar\nDisallow: /srv/\n\nUser-agent: AdIdxBot\nDisallow: /*src=urllib\nDisallow: /adell-mutfak\nDisallow: /ahu-kadin\nDisallow: /antika-sanat\nDisallow: /arama\nDisallow: /bebek-anne\nDisallow: /bilgisayar-tablet\nDisallow: /cep-telefonu-ve-aksesuar\nDisallow: /ev-bahce\nDisallow: /giyim-aksesuar\nDisallow: /muzik-plak-enstruman\nDisallow: /nap/\nDisallow: /otomobil-motor-aksesuar\nDisallow: /srv/\n\nUser-agent: Twitterbot\nDisallow: /*src=urllib\nDisallow: /srv/\n\n#\n# LLM / AI bot directives\n#\n\nUser-agent: Bytespider\nUser-agent: CCBot\nUser-agent: ChatGLM-Spider\nUser-agent: ClaudeBot\nUser-agent: PerplexityBot\nUser-agent: anthropic-ai\nUser-agent: AmazonBot\nUser-agent: meta-externalagent\nUser-agent: GPTBot\nUser-agent: Applebot-Extended\nDisallow: /\nAllow: /help\nAllow: /verkaeuferportal\nAllow: /blog\nAllow: /ebaylive/seller\n\n# AUCTION Sitemaps\nSitemap: https://www.ebay.de/lst/AUCTION-77-index.xml\n\n# BROWSE Sitemaps\nSitemap: https://www.ebay.de/lst/BROWSE-77-index.xml\n\n# KWSRP Sitemaps\nSitemap: https://www.ebay.de/lst/KWSRP-77-index.xml\n\n# NGS Sitemaps\nSitemap: https://www.ebay.de/lst/NGS-77-index.xml\n\n# PRP Sitemaps\nSitemap: https://www.ebay.de/lst/PRP-77-index.xml\n\n# VIDEO Sitemaps\nSitemap: https://www.ebay.de/lst/VIDEO-77-index.xml\n\n# VIS Sitemaps\nSitemap: https://www.ebay.de/lst/VIS-77-index.xml\n\n### END FILE ###"
    },
    "www.ebay.co.uk": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.ebay.co.uk",
      "body": "## BEGIN FILE ###\n#\n# allow-all\n# DR\n#\n# The use of robots or other automated means to access the eBay site\n# without the express permission of eBay is strictly prohibited.\n# Notwithstanding the foregoing, eBay may permit automated access to\n# access certain eBay pages but solely for the limited purpose of\n# including content in publicly available search engines. Any other\n# use of robots or failure to obey the robots exclusion standards set\n# forth at <https://www.robotstxt.org/orig.html> is strictly\n# prohibited.\n#\n# Robots & Agent Policy\n#\n# Checkouts are strictly for human users. \n# * Automated scraping, buy-for-me agents, LLM-driven bots, or any \n#   end-to-end flow that attempts to place orders without human review \n#   is strictly prohibited. \n# * Unauthorized use of automated agents in checkout may result in \n#   legal action under our User Agreement: https://www.ebay.com/help/policies/member-behaviour-policies/user-agreement?id=4259 \n# * Approved enterprise integrations must use our official API and \n#   comply with our API License Agreement: https://developer.ebay.com/join/api-license-agreement \n#\n\n\n# v28_UK_August_2026\n### DIRECTIVES ###\n\nUser-agent: *\nDisallow: /*_kw\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nDisallow: /buyer-preferences\nDisallow: /motors/fnd/*\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nDisallow: */b2ajax/all-filter\nAllow: /b/*?*_mwBanner\nDisallow: /b/*?*_nkw\nDisallow: /b/*?*&\nDisallow: /b/*?*|\nDisallow: /b/*%EF\nDisallow: /b/*LH_\nDisallow: /b/*_sop=\nDisallow: /b/*_pgn=\nDisallow: /b/*,\nDisallow: /b/*_dmd=\nDisallow: /b/*_dcat=\nDisallow: /b/*_sacat=\nDisallow: /b/*_saved\nDisallow: /b/*_sid=\nDisallow: /b/*_stpos=\nDisallow: /b/*udhi=\nDisallow: /b/*udlo=\nDisallow: /b/Test-Category\nDisallow: /b/eBay-Use-Only\nDisallow: /b/eBay-Test-Only\nDisallow: /b/LP-Parent\nDisallow: /b/LL-Child\nDisallow: /b/eBay-User-Tools\nDisallow: /b/Test-Auctions\nDisallow: /b/Attributes\nDisallow: /b/*DO-NOT-BID\nDisallow: /b/Testbereich/\nDisallow: /b/Testkategoriegruppe/\nDisallow: /b/Testkategorie\nDisallow: /bin/\nDisallow: /bo/\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /blueberry/\nDisallow: /brw/\nDisallow: /cancel/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /deinestadt/*_pgn=\nDisallow: /delicacy\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebaylive/host\nDisallow: /ebay/cronus/\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /feed/\nDisallow: /fdbk/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gh/user_profile\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gwc/\nDisallow: /handy/\nDisallow: /hcp/\nDisallow: /heute/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itempolicy\nDisallow: /itemnotreceived/\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /looking-glass-proxy/graphql\nDisallow: /lit/\nDisallow: /logout\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*surveyopen\nDisallow: /p/*thm=\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /price-guide/\nDisallow: /prp/\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/ebayadvsearch\nAllow: /sch/allcategories/\nAllow: /sch/*_ul\nAllow: /sch/i.html?*&mkcid=2\nDisallow: /sch/\nDisallow: /sch/ajax\nDisallow: /sch/ebayadvsearch?_\nDisallow: /ship/\nDisallow: /shop/*&\nAllow: /signin/$\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sh/onboarding\nDisallow: /sh/str/fvfsavings\nDisallow: /ship/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/ipe\nDisallow: /sl/node\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nDisallow: /usr/*/all-follows\nDisallow: /usr/*/followers\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vplbl/\nDisallow: /vod/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /ws/\n\nUser-agent: Googlebot\nDisallow: /*_kw\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nDisallow: /buyer-preferences\nDisallow: /motors/fnd/*\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nDisallow: */b2ajax/all-filter\nAllow: /b/*?*_mwBanner\nDisallow: /b/*?*_nkw\nDisallow: /b/*?*&\nDisallow: /b/*?*|\nDisallow: /b/*%EF\nDisallow: /b/*LH_\nDisallow: /b/*_sop=\nDisallow: /b/*_pgn=\nDisallow: /b/*,\nDisallow: /b/*_dmd=\nDisallow: /b/*_dcat=\nDisallow: /b/*_sacat=\nDisallow: /b/*_saved\nDisallow: /b/*_sid=\nDisallow: /b/*_stpos=\nDisallow: /b/*udhi=\nDisallow: /b/*udlo=\nDisallow: /b/Test-Category\nDisallow: /b/eBay-Use-Only\nDisallow: /b/eBay-Test-Only\nDisallow: /b/LP-Parent\nDisallow: /b/LL-Child\nDisallow: /b/eBay-User-Tools\nDisallow: /b/Test-Auctions\nDisallow: /b/Attributes\nDisallow: /b/*DO-NOT-BID\nDisallow: /b/Testbereich/\nDisallow: /b/Testkategoriegruppe/\nDisallow: /b/Testkategorie\nDisallow: /bin/\nDisallow: /bo/\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /blueberry/\nDisallow: /brw/\nDisallow: /cancel/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /deinestadt/*_pgn=\nDisallow: /delicacy\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebaylive/host\nDisallow: /ebay/cronus/\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /feed/\nDisallow: /fdbk/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gh/user_profile\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gwc/\nDisallow: /handy/\nDisallow: /hcp/\nDisallow: /heute/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itempolicy\nDisallow: /itemnotreceived/\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /looking-glass-proxy/graphql\nDisallow: /logout\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*surveyopen\nDisallow: /p/*thm=\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /price-guide/\nDisallow: /prp/\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/ebayadvsearch\nAllow: /sch/allcategories/\nAllow: /sch/*_ul\nAllow: /sch/i.html?*&mkcid=2\nDisallow: /sch/\nDisallow: /sch/ajax\nDisallow: /sch/ebayadvsearch?_\nDisallow: /ship/\nDisallow: /shop/*&\nAllow: /signin/$\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sh/onboarding\nDisallow: /sh/str/fvfsavings\nDisallow: /ship/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/ipe\nDisallow: /sl/node\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nDisallow: /usr/*/all-follows\nDisallow: /usr/*/followers\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vplbl/\nDisallow: /vod/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /ws/\n\nUser-agent: bingbot\nDisallow: /*_kw\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nAllow: /b/*?*_mwBanner\nDisallow: /b/*?*_nkw\nDisallow: /b/*_mwBanner=1*rt=nc\nDisallow: /b/*rt=nc*_mwBanner=1 \nDisallow: /b/*?*&\nDisallow: /b/*?*|\nDisallow: /b/*%EF\nDisallow: /b/*LH_\nDisallow: /b/*_sop=\nDisallow: /b/*_pgn=\nDisallow: /b/*,\nDisallow: /b/*_dmd=\nDisallow: /b/*_dcat=\nDisallow: /b/*_sacat=\nDisallow: /b/*_saved\nDisallow: /b/*_sid=\nDisallow: /b/*_stpos=\nDisallow: /b/*udhi=\nDisallow: /b/*udlo=\nDisallow: /b/Test-Category\nDisallow: /b/eBay-Use-Only\nDisallow: /b/eBay-Test-Only\nDisallow: /b/LP-Parent\nDisallow: /b/LL-Child\nDisallow: /b/eBay-User-Tools\nDisallow: /b/Test-Auctions\nDisallow: /b/Attributes\nDisallow: /b/*DO-NOT-BID\nDisallow: /b/Testbereich/\nDisallow: /b/Testkategoriegruppe/\nDisallow: /b/Testkategorie\nDisallow: /bin/\nDisallow: /bo/\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /blueberry/\nDisallow: /brw/\nDisallow: /cancel/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /deinestadt/*_pgn=\nDisallow: /delicacy\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebay/cronus/\nDisallow: /ebaylive/host\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /feed/\nDisallow: /fdbk/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gh/user_profile\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gwc/\nDisallow: /hcp/\nDisallow: /handy/\nDisallow: /heute/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itempolicy\nDisallow: /itemnotreceived/\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /looking-glass-proxy/graphql\nDisallow: /logout\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*surveyopen\nDisallow: /p/*thm=\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /price-guide/\nDisallow: /prp/\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/ebayadvsearch\nAllow: /sch/allcategories/\nAllow: /sch/*_ul\nAllow: /sch/i.html?*&mkcid=2\nDisallow: /sch/\nDisallow: /sch/ajax\nDisallow: /sch/ebayadvsearch?_\nDisallow: /ship/\nAllow: /signin/$\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sh/onboarding\nDisallow: /sh/str/fvfsavings\nDisallow: /ship/\nDisallow: /shop/*&\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/ipe\nDisallow: /sl/node\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nDisallow: /usr/\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vplbl/\nDisallow: /vod/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /ws/\n\nUser-agent: AdsBot-Google\nDisallow: /*src=urllib\nDisallow: /adell-mutfak\nDisallow: /ahu-kadin\nDisallow: /antika-sanat\nDisallow: /arama\nDisallow: /bebek-anne\nDisallow: /bilgisayar-tablet\nDisallow: /cep-telefonu-ve-aksesuar\nDisallow: /ev-bahce\nDisallow: /giyim-aksesuar\nDisallow: /muzik-plak-enstruman\nDisallow: /nap/\nDisallow: /otomobil-motor-aksesuar\nDisallow: /srv/\n\nUser-agent: AdIdxBot\nDisallow: /*src=urllib\nDisallow: /adell-mutfak\nDisallow: /ahu-kadin\nDisallow: /antika-sanat\nDisallow: /arama\nDisallow: /bebek-anne\nDisallow: /bilgisayar-tablet\nDisallow: /cep-telefonu-ve-aksesuar\nDisallow: /ev-bahce\nDisallow: /giyim-aksesuar\nDisallow: /muzik-plak-enstruman\nDisallow: /nap/\nDisallow: /otomobil-motor-aksesuar\nDisallow: /srv/\n\nUser-agent: Twitterbot\nDisallow: /*src=urllib\nDisallow: /srv/\n\n#\n# LLM / AI bot directives\n#\n\nUser-agent: Bytespider\nUser-agent: CCBot\nUser-agent: ChatGLM-Spider\nUser-agent: ClaudeBot\nUser-agent: PerplexityBot\nUser-agent: anthropic-ai\nUser-agent: AmazonBot\nUser-agent: meta-externalagent\nUser-agent: GPTBot\nUser-agent: Applebot-Extended\nDisallow: /\nAllow: /help\nAllow: /ebaylive/seller\nAllow: /sellercentre\n\nUser-agent: OAI-SearchBot\nUser-agent: ChatGPT-User\nUser-agent: Claude-SearchBot\nUser-agent: Claude-User\nDisallow: /*_kw\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nDisallow: /buyer-preferences\nDisallow: /motors/fnd/*\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nDisallow: */b2ajax/all-filter\nAllow: /b/*?*_mwBanner\nDisallow: /b/*?*_nkw\nDisallow: /b/*?*&\nDisallow: /b/*?*|\nDisallow: /b/*%EF\nDisallow: /b/*LH_\nDisallow: /b/*_sop=\nDisallow: /b/*_pgn=\nDisallow: /b/*,\nDisallow: /b/*_dmd=\nDisallow: /b/*_dcat=\nDisallow: /b/*_sacat=\nDisallow: /b/*_saved\nDisallow: /b/*_sid=\nDisallow: /b/*_stpos=\nDisallow: /b/*udhi=\nDisallow: /b/*udlo=\nDisallow: /b/Test-Category\nDisallow: /b/eBay-Use-Only\nDisallow: /b/eBay-Test-Only\nDisallow: /b/LP-Parent\nDisallow: /b/LL-Child\nDisallow: /b/eBay-User-Tools\nDisallow: /b/Test-Auctions\nDisallow: /b/Attributes\nDisallow: /b/*DO-NOT-BID\nDisallow: /b/Testbereich/\nDisallow: /b/Testkategoriegruppe/\nDisallow: /b/Testkategorie\nDisallow: /bin/\nDisallow: /bo/\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /blueberry/\nDisallow: /brw/\nDisallow: /cancel/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /deinestadt/*_pgn=\nDisallow: /delicacy\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebaylive/host\nDisallow: /ebay/cronus/\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /feed/\nDisallow: /fdbk/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gwc/\nDisallow: /handy/\nDisallow: /hcp/\nDisallow: /heute/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itempolicy\nDisallow: /itemnotreceived/\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /logout\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*surveyopen\nDisallow: /p/*thm=\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /price-guide/\nDisallow: /prp/\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/\nDisallow: /sch/ajax\nDisallow: /sch/ebayadvsearch?_\nDisallow: /ship/\nDisallow: /shop/*&\nAllow: /signin/$\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sh/onboarding\nDisallow: /sh/str/fvfsavings\nDisallow: /ship/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/ipe\nDisallow: /sl/node\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nDisallow: /usr/*/all-follows\nDisallow: /usr/*/followers\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vplbl/\nDisallow: /vod/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /ws/\n\n# AUCTION Sitemaps\nSitemap: https://www.ebay.co.uk/lst/AUCTION-3-index.xml\n\n# BROWSE Sitemaps\nSitemap: https://www.ebay.co.uk/lst/BROWSE-3-index.xml\n\n# KWSRP Sitemaps\nSitemap: https://www.ebay.co.uk/lst/KWSRP-3-index.xml\n\n# NGS Sitemaps\nSitemap: https://www.ebay.co.uk/lst/NGS-3-index.xml\n\n# PRP Sitemaps\nSitemap: https://www.ebay.co.uk/lst/PRP-3-index.xml\n\n# VIDEO Sitemaps\nSitemap: https://www.ebay.co.uk/lst/VIDEO-3-index.xml\n\n# VIS Sitemaps\nSitemap: https://www.ebay.co.uk/lst/VIS-3-index.xml\n\n### END FILE ###"
    },
    "www.ebay.fr": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.ebay.fr",
      "body": "## BEGIN FILE ###\n#\n# allow-all\n# DR\n#\n# The use of robots or other automated means to access the eBay site\n# without the express permission of eBay is strictly prohibited.\n# Notwithstanding the foregoing, eBay may permit automated access to\n# access certain eBay pages but solely for the limited purpose of\n# including content in publicly available search engines. Any other\n# use of robots or failure to obey the robots exclusion standards set\n# forth at <https://www.robotstxt.org/orig.html> is strictly\n# prohibited.\n#\n# \n# Robots & Agent Policy\n#\n# Checkouts are strictly for human users. \n# * Automated scraping, buy-for-me agents, LLM-driven bots, or any \n#   end-to-end flow that attempts to place orders without human review \n#   is strictly prohibited. \n# * Unauthorized use of automated agents in checkout may result in \n#   legal action under our User Agreement: https://www.ebay.com/help/policies/member-behaviour-policies/user-agreement?id=4259 \n# * Approved enterprise integrations must use our official API and \n#   comply with our API License Agreement: https://developer.ebay.com/join/api-license-agreement \n#\n# v28.1_FR_August_2026\n### DIRECTIVES ###\n\nUser-agent: *\nDisallow: /*_kw\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nDisallow: /buyer-preferences\nDisallow: /motors/fnd/*\nDisallow: /shop/*&\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nDisallow: */b2ajax/all-filter\nAllow: /b/*?*_mwBanner\nDisallow: /b/*_sop=\nDisallow: /b/*?*_nkw\nDisallow: /b/*?*&\nDisallow: /b/*?*|\nDisallow: /b/*%EF\nDisallow: /b/*LH_\nDisallow: /b/*,\nDisallow: /b/*_dmd=\nDisallow: /b/*_dcat=\nDisallow: /b/*_sacat=\nDisallow: /b/*_saved\nDisallow: /b/*_sid=\nDisallow: /b/*_stpos=\nDisallow: /b/*udhi=\nDisallow: /b/*udlo=\nDisallow: /b/*_pgn=\nDisallow: /b/Test-Category\nDisallow: /b/eBay-Use-Only\nDisallow: /b/eBay-Test-Only\nDisallow: /b/LP-Parent\nDisallow: /b/LL-Child\nDisallow: /b/eBay-User-Tools\nDisallow: /b/Test-Auctions\nDisallow: /b/Attributes\nDisallow: /b/*DO-NOT-BID\nDisallow: /b/Testbereich/\nDisallow: /b/Testkategoriegruppe/\nDisallow: /b/Testkategorie\nDisallow: /b/Categoria-per-test/\nDisallow: /b/Test-eBay/\nDisallow: /b/Attributo\nDisallow: /b/Categorie-test/\nDisallow: /b/Groupe-de-categories-de-test/\nDisallow: /b/Subastas-de-prueba/\nDisallow: /b/Atributos\nDisallow: /bin/\nDisallow: /bo/\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /blueberry/\nDisallow: /brw/\nDisallow: /c/\nDisallow: /cancel/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /delicacy\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebay/cronus/\nDisallow: /ebaylive/host\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /fdbk/\nDisallow: /feed/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gwc/\nDisallow: /hcp/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itemnotreceived/\nDisallow: /itempolicy\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /looking-glass-proxy/graphql\nDisallow: /logout\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*thm=\nDisallow: /p/*surveyopen\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /prp/\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/ebayadvsearch\nAllow: /sch/*/i.html\nAllow: /sch/i.html?*&mkcid=2\nDisallow: /sch/*/i.html?\nDisallow: /sch/ctg/*/i.html\nDisallow: /sch/merchant/*/i.html\nDisallow: /sch/items/*/i.html\nDisallow: /sch/*/i/update/i.html\nDisallow: /sch/i.html\nDisallow: /sch/ebayadvsearch?_\nDisallow: /sch/\nDisallow: /sch/ajax\nAllow: /signin/$\nDisallow: /sh/onboarding\nDisallow: /sh/str/fvfsavings\nDisallow: /ship/\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/ipe\nDisallow: /sl/node\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nDisallow: /usr/*/all-follows\nDisallow: /usr/*/followers\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vod/\nDisallow: /vplbl/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /ws/\n\nUser-agent: Googlebot\nDisallow: /*_kw\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nDisallow: /buyer-preferences\nDisallow: /motors/fnd/*\nDisallow: /shop/*&\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nDisallow: */b2ajax/all-filter\nAllow: /b/*?*_mwBanner\nDisallow: /b/*_sop=\nDisallow: /b/*?*_nkw\nDisallow: /b/*?*&\nDisallow: /b/*?*|\nDisallow: /b/*%EF\nDisallow: /b/*LH_\nDisallow: /b/*,\nDisallow: /b/*_dmd=\nDisallow: /b/*_dcat=\nDisallow: /b/*_sacat=\nDisallow: /b/*_saved\nDisallow: /b/*_sid=\nDisallow: /b/*_stpos=\nDisallow: /b/*udhi=\nDisallow: /b/*udlo=\nDisallow: /b/*_pgn=\nDisallow: /b/Test-Category\nDisallow: /b/eBay-Use-Only\nDisallow: /b/eBay-Test-Only\nDisallow: /b/LP-Parent\nDisallow: /b/LL-Child\nDisallow: /b/eBay-User-Tools\nDisallow: /b/Test-Auctions\nDisallow: /b/Attributes\nDisallow: /b/*DO-NOT-BID\nDisallow: /b/Testbereich/\nDisallow: /b/Testkategoriegruppe/\nDisallow: /b/Testkategorie\nDisallow: /b/Categoria-per-test/\nDisallow: /b/Test-eBay/\nDisallow: /b/Attributo\nDisallow: /b/Categorie-test/\nDisallow: /b/Groupe-de-categories-de-test/\nDisallow: /b/Subastas-de-prueba/\nDisallow: /b/Atributos\nDisallow: /bin/\nDisallow: /bo/\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /blueberry/\nDisallow: /brw/\nDisallow: /c/\nDisallow: /cancel/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /delicacy\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebay/cronus/\nDisallow: /ebaylive/host\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /fdbk/\nDisallow: /feed/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gwc/\nDisallow: /hcp/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itemnotreceived/\nDisallow: /itempolicy\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /looking-glass-proxy/graphql\nDisallow: /logout\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*thm=\nDisallow: /p/*surveyopen\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /prp/\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/ebayadvsearch\nAllow: /sch/*/i.html\nAllow: /sch/i.html?*&mkcid=2\nDisallow: /sch/*/i.html?\nDisallow: /sch/ctg/*/i.html\nDisallow: /sch/merchant/*/i.html\nDisallow: /sch/items/*/i.html\nDisallow: /sch/*/i/update/i.html\nDisallow: /sch/i.html\nDisallow: /sch/ebayadvsearch?_\nDisallow: /sch/\nDisallow: /sch/ajax\nAllow: /signin/$\nDisallow: /sh/onboarding\nDisallow: /sh/str/fvfsavings\nDisallow: /ship/\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/ipe\nDisallow: /sl/node\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nDisallow: /usr/*/all-follows\nDisallow: /usr/*/followers\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vod/\nDisallow: /vplbl/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /ws/\n\nUser-agent: bingbot\nDisallow: /*_kw\nDisallow: /b/*_mwBanner=1*rt=nc\nDisallow: /b/*rt=nc*_mwBanner=1\nDisallow: /shop/*&\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nAllow: /b/*?*_mwBanner\nDisallow: /b/*_sop=\nDisallow: /b/*?*_nkw\nDisallow: /b/*?*&\nDisallow: /b/*?*|\nDisallow: /b/*%EF\nDisallow: /b/*LH_\nDisallow: /b/*,\nDisallow: /b/*_dmd=\nDisallow: /b/*_dcat=\nDisallow: /b/*_sacat=\nDisallow: /b/*_saved\nDisallow: /b/*_sid=\nDisallow: /b/*_stpos=\nDisallow: /b/*udhi=\nDisallow: /b/*udlo=\nDisallow: /b/*_pgn=\nDisallow: /b/Test-Category\nDisallow: /b/eBay-Use-Only\nDisallow: /b/eBay-Test-Only\nDisallow: /b/LP-Parent\nDisallow: /b/LL-Child\nDisallow: /b/eBay-User-Tools\nDisallow: /b/Test-Auctions\nDisallow: /b/Attributes\nDisallow: /b/*DO-NOT-BID\nDisallow: /b/Testbereich/\nDisallow: /b/Testkategoriegruppe/\nDisallow: /b/Testkategorie\nDisallow: /b/Categoria-per-test/\nDisallow: /b/Test-eBay/\nDisallow: /b/Attributo\nDisallow: /b/Categorie-test/\nDisallow: /b/Groupe-de-categories-de-test/\nDisallow: /b/Subastas-de-prueba/\nDisallow: /b/Atributos\nDisallow: /bin/\nDisallow: /bo/\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /blueberry/\nDisallow: /brw/\nDisallow: /c/\nDisallow: /cancel/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /delicacy\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebay/cronus/\nDisallow: /ebaylive/host\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /fdbk/\nDisallow: /feed/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gwc/\nDisallow: /hcp/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itemnotreceived/\nDisallow: /itempolicy\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /looking-glass-proxy/graphql\nDisallow: /logout\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*thm=\nDisallow: /p/*surveyopen\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /prp/\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/ebayadvsearch\nAllow: /sch/*/i.html\nDisallow: /sch/*/i.html?\nDisallow: /sch/ctg/*/i.html\nDisallow: /sch/merchant/*/i.html\nDisallow: /sch/items/*/i.html\nDisallow: /sch/*/i/update/i.html\nDisallow: /sch/i.html\nDisallow: /sch/ebayadvsearch?_\nDisallow: /sch/\nDisallow: /sch/ajax\nAllow: /signin/$\nDisallow: /sh/onboarding\nDisallow: /sh/str/fvfsavings\nDisallow: /ship/\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/ipe\nDisallow: /sl/node\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nAllow: /usr/\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vod/\nDisallow: /vplbl/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /ws/\n\nUser-agent: AdsBot-Google\nDisallow: /*src=urllib\nDisallow: /adell-mutfak\nDisallow: /ahu-kadin\nDisallow: /antika-sanat\nDisallow: /arama\nDisallow: /bebek-anne\nDisallow: /bilgisayar-tablet\nDisallow: /cep-telefonu-ve-aksesuar\nDisallow: /ev-bahce\nDisallow: /giyim-aksesuar\nDisallow: /muzik-plak-enstruman\nDisallow: /nap/\nDisallow: /otomobil-motor-aksesuar\nDisallow: /srv/\n\nUser-agent: AdIdxBot\nDisallow: /*src=urllib\nDisallow: /adell-mutfak\nDisallow: /ahu-kadin\nDisallow: /antika-sanat\nDisallow: /arama\nDisallow: /bebek-anne\nDisallow: /bilgisayar-tablet\nDisallow: /cep-telefonu-ve-aksesuar\nDisallow: /ev-bahce\nDisallow: /giyim-aksesuar\nDisallow: /muzik-plak-enstruman\nDisallow: /nap/\nDisallow: /otomobil-motor-aksesuar\nDisallow: /srv/\n\nUser-agent: Twitterbot\nDisallow: /*src=urllib\nDisallow: /srv/\n\n#\n# LLM / AI bot directives\n#\n\nUser-agent: Bytespider\nUser-agent: CCBot\nUser-agent: ChatGLM-Spider\nUser-agent: ClaudeBot\nUser-agent: PerplexityBot\nUser-agent: anthropic-ai\nUser-agent: AmazonBot\nUser-agent: meta-externalagent\nUser-agent: GPTBot\nUser-agent: Applebot-Extended\nDisallow: /\nAllow: /help\nAllow: /espacevendeurs\nAllow: /ebaylive/seller\n\n# AUCTION Sitemaps\nSitemap: https://www.ebay.fr/lst/AUCTION-71-index.xml\n\n# BROWSE Sitemaps\nSitemap: https://www.ebay.fr/lst/BROWSE-71-index.xml\n\n# KWSRP Sitemaps\nSitemap: https://www.ebay.fr/lst/KWSRP-71-index.xml\n\n# NGS Sitemaps\nSitemap: https://www.ebay.fr/lst/NGS-71-index.xml\n\n# PRP Sitemaps\nSitemap: https://www.ebay.fr/lst/PRP-71-index.xml\n\n# VIDEO Sitemaps\nSitemap: https://www.ebay.fr/lst/VIDEO-71-index.xml\n\n# VIS Sitemaps\nSitemap: https://www.ebay.fr/lst/VIS-71-index.xml\n\n### END FILE ###"
    },
    "www.ebay.it": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.ebay.it",
      "body": "## BEGIN FILE ###\n#\n# allow-all\n# DR\n#\n# The use of robots or other automated means to access the eBay site\n# without the express permission of eBay is strictly prohibited.\n# Notwithstanding the foregoing, eBay may permit automated access to\n# access certain eBay pages but solely for the limited purpose of\n# including content in publicly available search engines. Any other\n# use of robots or failure to obey the robots exclusion standards set\n# forth at <https://www.robotstxt.org/orig.html> is strictly\n# prohibited.\n#\n# \n# Robots & Agent Policy\n#\n# Checkouts are strictly for human users. \n# * Automated scraping, buy-for-me agents, LLM-driven bots, or any \n#   end-to-end flow that attempts to place orders without human review \n#   is strictly prohibited. \n# * Unauthorized use of automated agents in checkout may result in \n#   legal action under our User Agreement: https://www.ebay.com/help/policies/member-behaviour-policies/user-agreement?id=4259 \n# * Approved enterprise integrations must use our official API and \n#   comply with our API License Agreement: https://developer.ebay.com/join/api-license-agreement \n#\n# v28.1_IT_August_2026\n### DIRECTIVES ###\n\nUser-agent: *\nDisallow: /*_kw\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nDisallow: /buyer-preferences\nDisallow: /shop/*&\nDisallow: /motors/fnd/*\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nDisallow: */b2ajax/all-filter\nAllow: /b/*?*_mwBanner\nDisallow: /b/*_sop=\nDisallow: /b/*_pgn=\nDisallow: /b/*?*_nkw\nDisallow: /b/*?*&\nDisallow: /b/*?*|\nDisallow: /b/*%EF\nDisallow: /b/*LH_\nDisallow: /b/*,\nDisallow: /b/*_dmd=\nDisallow: /b/*_dcat=\nDisallow: /b/*_sacat=\nDisallow: /b/*_saved\nDisallow: /b/*_sid=\nDisallow: /b/*_stpos=\nDisallow: /b/*udhi=\nDisallow: /b/*udlo=\nDisallow: /b/Test-Category\nDisallow: /b/eBay-Use-Only\nDisallow: /b/eBay-Test-Only\nDisallow: /b/LP-Parent\nDisallow: /b/LL-Child\nDisallow: /b/eBay-User-Tools\nDisallow: /b/Test-Auctions\nDisallow: /b/Attributes\nDisallow: /b/*DO-NOT-BID\nDisallow: /b/Testbereich/\nDisallow: /b/Testkategoriegruppe/\nDisallow: /b/Testkategorie\nDisallow: /b/Categoria-per-test/\nDisallow: /b/Test-eBay/\nDisallow: /b/Attributo\nDisallow: /b/Categorie-test/\nDisallow: /b/Groupe-de-categories-de-test/\nDisallow: /b/Subastas-de-prueba/\nDisallow: /b/Atributos\nDisallow: /bin/\nDisallow: /bo/\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /blueberry/\nDisallow: /brw/\nDisallow: /c/\nDisallow: /cancel/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /delicacy\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebay/cronus/\nDisallow: /ebaylive/host\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /fdbk/\nDisallow: /feed/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gwc/\nDisallow: /hcp/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itemnotreceived/\nDisallow: /itempolicy\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /looking-glass-proxy/graphql\nDisallow: /logout\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*thm=\nDisallow: /p/*surveyopen\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /prp/\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/ebayadvsearch\nAllow: /sch/*/i.html\nAllow: /sch/i.html?*&mkcid=2\nDisallow: /sch/*/i.html?\nDisallow: /sch/ctg/*/i.html\nDisallow: /sch/merchant/*/i.html\nDisallow: /sch/items/*/i.html\nDisallow: /sch/*/i/update/i.html\nDisallow: /sch/i.html\nDisallow: /sch/ebayadvsearch?_\nDisallow: /sch/\nDisallow: /sch/ajax\nAllow: /signin/$\nDisallow: /sh/onboarding\nDisallow: /sh/str/fvfsavings\nDisallow: /ship/\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/ipe\nDisallow: /sl/node\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nDisallow: /usr/*/all-follows\nDisallow: /usr/*/followers\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vod/\nDisallow: /vplbl/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /ws/\n\nUser-agent: Googlebot\nDisallow: /*_kw\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nDisallow: /buyer-preferences\nDisallow: /shop/*&\nDisallow: /motors/fnd/*\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nDisallow: */b2ajax/all-filter\nAllow: /b/*?*_mwBanner\nDisallow: /b/*_sop=\nDisallow: /b/*_pgn=\nDisallow: /b/*?*_nkw\nDisallow: /b/*?*&\nDisallow: /b/*?*|\nDisallow: /b/*%EF\nDisallow: /b/*LH_\nDisallow: /b/*,\nDisallow: /b/*_dmd=\nDisallow: /b/*_dcat=\nDisallow: /b/*_sacat=\nDisallow: /b/*_saved\nDisallow: /b/*_sid=\nDisallow: /b/*_stpos=\nDisallow: /b/*udhi=\nDisallow: /b/*udlo=\nDisallow: /b/Test-Category\nDisallow: /b/eBay-Use-Only\nDisallow: /b/eBay-Test-Only\nDisallow: /b/LP-Parent\nDisallow: /b/LL-Child\nDisallow: /b/eBay-User-Tools\nDisallow: /b/Test-Auctions\nDisallow: /b/Attributes\nDisallow: /b/*DO-NOT-BID\nDisallow: /b/Testbereich/\nDisallow: /b/Testkategoriegruppe/\nDisallow: /b/Testkategorie\nDisallow: /b/Categoria-per-test/\nDisallow: /b/Test-eBay/\nDisallow: /b/Attributo\nDisallow: /b/Categorie-test/\nDisallow: /b/Groupe-de-categories-de-test/\nDisallow: /b/Subastas-de-prueba/\nDisallow: /b/Atributos\nDisallow: /bin/\nDisallow: /bo/\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /blueberry/\nDisallow: /brw/\nDisallow: /c/\nDisallow: /cancel/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /delicacy\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebay/cronus/\nDisallow: /ebaylive/host\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /fdbk/\nDisallow: /feed/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gwc/\nDisallow: /hcp/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itemnotreceived/\nDisallow: /itempolicy\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /looking-glass-proxy/graphql\nDisallow: /logout\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*thm=\nDisallow: /p/*surveyopen\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /prp/\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/ebayadvsearch\nAllow: /sch/*/i.html\nAllow: /sch/i.html?*&mkcid=2\nDisallow: /sch/*/i.html?\nDisallow: /sch/ctg/*/i.html\nDisallow: /sch/merchant/*/i.html\nDisallow: /sch/items/*/i.html\nDisallow: /sch/*/i/update/i.html\nDisallow: /sch/i.html\nDisallow: /sch/ebayadvsearch?_\nDisallow: /sch/\nDisallow: /sch/ajax\nAllow: /signin/$\nDisallow: /sh/onboarding\nDisallow: /sh/str/fvfsavings\nDisallow: /ship/\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/ipe\nDisallow: /sl/node\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nDisallow: /usr/*/all-follows\nDisallow: /usr/*/followers\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vod/\nDisallow: /vplbl/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /ws/\n\nUser-agent: bingbot\nDisallow: /*_kw\nDisallow: /b/*_mwBanner=1*rt=nc\nDisallow: /b/*rt=nc*_mwBanner=1 \nDisallow: /shop/*&\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nAllow: /b/*?*_mwBanner\nDisallow: /b/*_sop=\nDisallow: /b/*_pgn=\nDisallow: /b/*?*_nkw\nDisallow: /b/*?*&\nDisallow: /b/*?*|\nDisallow: /b/*%EF\nDisallow: /b/*LH_\nDisallow: /b/*,\nDisallow: /b/*_dmd=\nDisallow: /b/*_dcat=\nDisallow: /b/*_sacat=\nDisallow: /b/*_saved\nDisallow: /b/*_sid=\nDisallow: /b/*_stpos=\nDisallow: /b/*udhi=\nDisallow: /b/*udlo=\nDisallow: /b/Test-Category\nDisallow: /b/eBay-Use-Only\nDisallow: /b/eBay-Test-Only\nDisallow: /b/LP-Parent\nDisallow: /b/LL-Child\nDisallow: /b/eBay-User-Tools\nDisallow: /b/Test-Auctions\nDisallow: /b/Attributes\nDisallow: /b/*DO-NOT-BID\nDisallow: /b/Testbereich/\nDisallow: /b/Testkategoriegruppe/\nDisallow: /b/Testkategorie\nDisallow: /b/Categoria-per-test/\nDisallow: /b/Test-eBay/\nDisallow: /b/Attributo\nDisallow: /b/Categorie-test/\nDisallow: /b/Groupe-de-categories-de-test/\nDisallow: /b/Subastas-de-prueba/\nDisallow: /b/Atributos\nDisallow: /bin/\nDisallow: /bo/\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /blueberry/\nDisallow: /brw/\nDisallow: /c/\nDisallow: /cancel/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /delicacy\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebay/cronus/\nDisallow: /ebaylive/host\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /fdbk/\nDisallow: /feed/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gwc/\nDisallow: /hcp/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itemnotreceived/\nDisallow: /itempolicy\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /looking-glass-proxy/graphql\nDisallow: /logout\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*thm=\nDisallow: /p/*surveyopen\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /prp/\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/ebayadvsearch\nAllow: /sch/*/i.html\nDisallow: /sch/*/i.html?\nDisallow: /sch/ctg/*/i.html\nDisallow: /sch/merchant/*/i.html\nDisallow: /sch/items/*/i.html\nDisallow: /sch/*/i/update/i.html\nDisallow: /sch/i.html\nDisallow: /sch/ebayadvsearch?_\nDisallow: /sch/\nDisallow: /sch/ajax\nAllow: /signin/$\nDisallow: /sh/onboarding\nDisallow: /sh/str/fvfsavings\nDisallow: /ship/\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/ipe\nDisallow: /sl/node\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nAllow: /usr/\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vod/\nDisallow: /vplbl/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /ws/\n\nUser-agent: AdsBot-Google\nDisallow: /*src=urllib\nDisallow: /adell-mutfak\nDisallow: /ahu-kadin\nDisallow: /antika-sanat\nDisallow: /arama\nDisallow: /bebek-anne\nDisallow: /bilgisayar-tablet\nDisallow: /cep-telefonu-ve-aksesuar\nDisallow: /ev-bahce\nDisallow: /giyim-aksesuar\nDisallow: /muzik-plak-enstruman\nDisallow: /nap/\nDisallow: /otomobil-motor-aksesuar\nDisallow: /srv/\n\nUser-agent: AdIdxBot\nDisallow: /*src=urllib\nDisallow: /adell-mutfak\nDisallow: /ahu-kadin\nDisallow: /antika-sanat\nDisallow: /arama\nDisallow: /bebek-anne\nDisallow: /bilgisayar-tablet\nDisallow: /cep-telefonu-ve-aksesuar\nDisallow: /ev-bahce\nDisallow: /giyim-aksesuar\nDisallow: /muzik-plak-enstruman\nDisallow: /nap/\nDisallow: /otomobil-motor-aksesuar\nDisallow: /srv/\n\nUser-agent: Twitterbot\nDisallow: /*src=urllib\nDisallow: /srv/\n\n#\n# LLM / AI bot directives\n#\n\nUser-agent: Bytespider\nUser-agent: CCBot\nUser-agent: ChatGLM-Spider\nUser-agent: ClaudeBot\nUser-agent: PerplexityBot\nUser-agent: anthropic-ai\nUser-agent: AmazonBot\nUser-agent: meta-externalagent\nUser-agent: GPTBot\nUser-agent: Applebot-Extended\nDisallow: /\nAllow: /help\nAllow: /spaziovenditori\nAllow: /ebaylive/seller\n\n# AUCTION Sitemaps\nSitemap: https://www.ebay.it/lst/AUCTION-101-index.xml\n\n# BROWSE Sitemaps\nSitemap: https://www.ebay.it/lst/BROWSE-101-index.xml\n\n# KWSRP Sitemaps\nSitemap: https://www.ebay.it/lst/KWSRP-101-index.xml\n\n# NGS Sitemaps\nSitemap: https://www.ebay.it/lst/NGS-101-index.xml\n\n# PRP Sitemaps\nSitemap: https://www.ebay.it/lst/PRP-101-index.xml\n\n# VIDEO Sitemaps\nSitemap: https://www.ebay.it/lst/VIDEO-101-index.xml\n\n# VIS Sitemaps\nSitemap: https://www.ebay.it/lst/VIS-101-index.xml\n\n### END FILE ###"
    },
    "www.ebay.es": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.ebay.es",
      "body": "## BEGIN FILE ###\n#\n# allow-all\n# DR\n#\n# The use of robots or other automated means to access the eBay site\n# without the express permission of eBay is strictly prohibited.\n# Notwithstanding the foregoing, eBay may permit automated access to\n# access certain eBay pages but solely for the limited purpose of\n# including content in publicly available search engines. Any other\n# use of robots or failure to obey the robots exclusion standards set\n# forth at <https://www.robotstxt.org/orig.html> is strictly\n# prohibited.\n#\n# \n# Robots & Agent Policy\n#\n# Checkouts are strictly for human users. \n# * Automated scraping, buy-for-me agents, LLM-driven bots, or any \n#   end-to-end flow that attempts to place orders without human review \n#   is strictly prohibited. \n# * Unauthorized use of automated agents in checkout may result in \n#   legal action under our User Agreement: https://www.ebay.com/help/policies/member-behaviour-policies/user-agreement?id=4259 \n# * Approved enterprise integrations must use our official API and \n#   comply with our API License Agreement: https://developer.ebay.com/join/api-license-agreement \n#\n\n# v27.1_ES_August_2026\n### DIRECTIVES ###\n\nUser-agent: *\nDisallow: /*_kw\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nDisallow: /buyer-preferences\nDisallow: /shop/*&\nDisallow: /motors/fnd/*\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nDisallow: */b2ajax/all-filter\nAllow: /b/*?*_mwBanner\nDisallow: /b/*_sop=\nDisallow: /b/*_pgn=\nDisallow: /b/*?*_nkw\nDisallow: /b/*?*&\nDisallow: /b/*?*|\nDisallow: /b/*%EF\nDisallow: /b/*LH_\nDisallow: /b/*,\nDisallow: /b/*_dmd=\nDisallow: /b/*_dcat=\nDisallow: /b/*_sacat=\nDisallow: /b/*_saved\nDisallow: /b/*_sid=\nDisallow: /b/*_stpos=\nDisallow: /b/*udhi=\nDisallow: /b/*udlo=\nDisallow: /b/Test-Category\nDisallow: /b/eBay-Use-Only\nDisallow: /b/eBay-Test-Only\nDisallow: /b/LP-Parent\nDisallow: /b/LL-Child\nDisallow: /b/eBay-User-Tools\nDisallow: /b/Test-Auctions\nDisallow: /b/Attributes\nDisallow: /b/*DO-NOT-BID\nDisallow: /b/Testbereich/\nDisallow: /b/Testkategoriegruppe/\nDisallow: /b/Testkategorie\nDisallow: /b/Categoria-per-test/\nDisallow: /b/Test-eBay/\nDisallow: /b/Attributo\nDisallow: /b/Categorie-test/\nDisallow: /b/Groupe-de-categories-de-test/\nDisallow: /b/Subastas-de-prueba/\nDisallow: /b/Atributos\nDisallow: /bin/\nDisallow: /bo/\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /blueberry/\nDisallow: /brw/\nDisallow: /c/\nDisallow: /cancel/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /delicacy\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebay/cronus/\nDisallow: /ebaylive/host\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /fdbk/\nDisallow: /feed/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gwc/\nDisallow: /hcp/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itemnotreceived/\nDisallow: /itempolicy\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /looking-glass-proxy/graphql\nDisallow: /logout\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*thm=\nDisallow: /p/*surveyopen\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /prp/\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/ebayadvsearch\nAllow: /sch/*/i.html\nAllow: /sch/i.html?*&mkcid=2\nDisallow: /sch/*/i.html?\nDisallow: /sch/ctg/*/i.html\nDisallow: /sch/merchant/*/i.html\nDisallow: /sch/items/*/i.html\nDisallow: /sch/*/i/update/i.html\nDisallow: /sch/i.html\nDisallow: /sch/ebayadvsearch?_\nDisallow: /sch/\nDisallow: /sch/ajax\nAllow: /signin/$\nDisallow: /sh/onboarding\nDisallow: /sh/str/fvfsavings\nDisallow: /ship/\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/ipe\nDisallow: /sl/node\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nDisallow: /usr/*/all-follows\nDisallow: /usr/*/followers\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vod/\nDisallow: /vplbl/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /ws/\n\nUser-agent: Googlebot\nDisallow: /*_kw\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nDisallow: /buyer-preferences\nDisallow: /shop/*&\nDisallow: /motors/fnd/*\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nDisallow: */b2ajax/all-filter\nAllow: /b/*?*_mwBanner\nDisallow: /b/*_sop=\nDisallow: /b/*_pgn=\nDisallow: /b/*?*_nkw\nDisallow: /b/*?*&\nDisallow: /b/*?*|\nDisallow: /b/*%EF\nDisallow: /b/*LH_\nDisallow: /b/*,\nDisallow: /b/*_dmd=\nDisallow: /b/*_dcat=\nDisallow: /b/*_sacat=\nDisallow: /b/*_saved\nDisallow: /b/*_sid=\nDisallow: /b/*_stpos=\nDisallow: /b/*udhi=\nDisallow: /b/*udlo=\nDisallow: /b/Test-Category\nDisallow: /b/eBay-Use-Only\nDisallow: /b/eBay-Test-Only\nDisallow: /b/LP-Parent\nDisallow: /b/LL-Child\nDisallow: /b/eBay-User-Tools\nDisallow: /b/Test-Auctions\nDisallow: /b/Attributes\nDisallow: /b/*DO-NOT-BID\nDisallow: /b/Testbereich/\nDisallow: /b/Testkategoriegruppe/\nDisallow: /b/Testkategorie\nDisallow: /b/Categoria-per-test/\nDisallow: /b/Test-eBay/\nDisallow: /b/Attributo\nDisallow: /b/Categorie-test/\nDisallow: /b/Groupe-de-categories-de-test/\nDisallow: /b/Subastas-de-prueba/\nDisallow: /b/Atributos\nDisallow: /bin/\nDisallow: /bo/\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /blueberry/\nDisallow: /brw/\nDisallow: /c/\nDisallow: /cancel/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /delicacy\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebay/cronus/\nDisallow: /ebaylive/host\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /fdbk/\nDisallow: /feed/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gwc/\nDisallow: /hcp/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itemnotreceived/\nDisallow: /itempolicy\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /looking-glass-proxy/graphql\nDisallow: /logout\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*thm=\nDisallow: /p/*surveyopen\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /prp/\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/ebayadvsearch\nAllow: /sch/*/i.html\nAllow: /sch/i.html?*&mkcid=2\nDisallow: /sch/*/i.html?\nDisallow: /sch/ctg/*/i.html\nDisallow: /sch/merchant/*/i.html\nDisallow: /sch/items/*/i.html\nDisallow: /sch/*/i/update/i.html\nDisallow: /sch/i.html\nDisallow: /sch/ebayadvsearch?_\nDisallow: /sch/\nDisallow: /sch/ajax\nAllow: /signin/$\nDisallow: /sh/onboarding\nDisallow: /sh/str/fvfsavings\nDisallow: /ship/\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/ipe\nDisallow: /sl/node\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nDisallow: /usr/*/all-follows\nDisallow: /usr/*/followers\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vod/\nDisallow: /vplbl/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /ws/\n\nUser-agent: bingbot\nDisallow: /*_kw\nDisallow: /b/*_mwBanner=1*rt=nc\nDisallow: /b/*rt=nc*_mwBanner=1\nDisallow: /shop/*&\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nAllow: /b/*?*_mwBanner\nDisallow: /b/*_sop=\nDisallow: /b/*_pgn=\nDisallow: /b/*?*_nkw\nDisallow: /b/*?*&\nDisallow: /b/*?*|\nDisallow: /b/*%EF\nDisallow: /b/*LH_\nDisallow: /b/*,\nDisallow: /b/*_dmd=\nDisallow: /b/*_dcat=\nDisallow: /b/*_sacat=\nDisallow: /b/*_saved\nDisallow: /b/*_sid=\nDisallow: /b/*_stpos=\nDisallow: /b/*udhi=\nDisallow: /b/*udlo=\nDisallow: /b/Test-Category\nDisallow: /b/eBay-Use-Only\nDisallow: /b/eBay-Test-Only\nDisallow: /b/LP-Parent\nDisallow: /b/LL-Child\nDisallow: /b/eBay-User-Tools\nDisallow: /b/Test-Auctions\nDisallow: /b/Attributes\nDisallow: /b/*DO-NOT-BID\nDisallow: /b/Testbereich/\nDisallow: /b/Testkategoriegruppe/\nDisallow: /b/Testkategorie\nDisallow: /b/Categoria-per-test/\nDisallow: /b/Test-eBay/\nDisallow: /b/Attributo\nDisallow: /b/Categorie-test/\nDisallow: /b/Groupe-de-categories-de-test/\nDisallow: /b/Subastas-de-prueba/\nDisallow: /b/Atributos\nDisallow: /bin/\nDisallow: /bo/\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /blueberry/\nDisallow: /brw/\nDisallow: /c/\nDisallow: /cancel/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /delicacy\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebay/cronus/\nDisallow: /ebaylive/host\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /fdbk/\nDisallow: /feed/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gwc/\nDisallow: /hcp/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itemnotreceived/\nDisallow: /itempolicy\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /looking-glass-proxy/graphql\nDisallow: /logout\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*thm=\nDisallow: /p/*surveyopen\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /prp/\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/ebayadvsearch\nAllow: /sch/*/i.html\nDisallow: /sch/*/i.html?\nDisallow: /sch/ctg/*/i.html\nDisallow: /sch/merchant/*/i.html\nDisallow: /sch/items/*/i.html\nDisallow: /sch/*/i/update/i.html\nDisallow: /sch/i.html\nDisallow: /sch/ebayadvsearch?_\nDisallow: /sch/\nDisallow: /sch/ajax\nAllow: /signin/$\nDisallow: /sh/onboarding\nDisallow: /sh/str/fvfsavings\nDisallow: /ship/\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/ipe\nDisallow: /sl/node\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nAllow: /usr/\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vod/\nDisallow: /vplbl/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /ws/\n\nUser-agent: AdsBot-Google\nDisallow: /*src=urllib\nDisallow: /adell-mutfak\nDisallow: /ahu-kadin\nDisallow: /antika-sanat\nDisallow: /arama\nDisallow: /bebek-anne\nDisallow: /bilgisayar-tablet\nDisallow: /cep-telefonu-ve-aksesuar\nDisallow: /ev-bahce\nDisallow: /giyim-aksesuar\nDisallow: /muzik-plak-enstruman\nDisallow: /nap/\nDisallow: /otomobil-motor-aksesuar\nDisallow: /srv/\n\nUser-agent: AdIdxBot\nDisallow: /*src=urllib\nDisallow: /adell-mutfak\nDisallow: /ahu-kadin\nDisallow: /antika-sanat\nDisallow: /arama\nDisallow: /bebek-anne\nDisallow: /bilgisayar-tablet\nDisallow: /cep-telefonu-ve-aksesuar\nDisallow: /ev-bahce\nDisallow: /giyim-aksesuar\nDisallow: /muzik-plak-enstruman\nDisallow: /nap/\nDisallow: /otomobil-motor-aksesuar\nDisallow: /srv/\n\nUser-agent: Twitterbot\nDisallow: /*src=urllib\nDisallow: /srv/\n\n#\n# LLM / AI bot directives\n#\n\nUser-agent: Bytespider\nUser-agent: CCBot\nUser-agent: ChatGLM-Spider\nUser-agent: ClaudeBot\nUser-agent: PerplexityBot\nUser-agent: anthropic-ai\nUser-agent: AmazonBot\nUser-agent: meta-externalagent\nUser-agent: GPTBot\nUser-agent: Applebot-Extended\nDisallow: /\nAllow: /help\nAllow: /centroparavendedores\n\n# AUCTION Sitemaps\nSitemap: https://www.ebay.es/lst/AUCTION-186-index.xml\n\n# BROWSE Sitemaps\nSitemap: https://www.ebay.es/lst/BROWSE-186-index.xml\n\n# KWSRP Sitemaps\nSitemap: https://www.ebay.es/lst/KWSRP-186-index.xml\n\n# NGS Sitemaps\nSitemap: https://www.ebay.es/lst/NGS-186-index.xml\n\n# PRP Sitemaps\nSitemap: https://www.ebay.es/lst/PRP-186-index.xml\n\n# VIDEO Sitemaps\nSitemap: https://www.ebay.es/lst/VIDEO-186-index.xml\n\n# VIS Sitemaps\nSitemap: https://www.ebay.es/lst/VIS-186-index.xml\n\n### END FILE ###"
    },
    "www.ebay.at": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.ebay.at",
      "body": "## BEGIN FILE ###\n#\n# allow-all\n# DR\n#\n# The use of robots or other automated means to access the eBay site\n# without the express permission of eBay is strictly prohibited.\n# Notwithstanding the foregoing, eBay may permit automated access to\n# access certain eBay pages but solely for the limited purpose of\n# including content in publicly available search engines. Any other\n# use of robots or failure to obey the robots exclusion standards set\n# forth at <https://www.robotstxt.org/orig.html> is strictly\n# prohibited.\n#\n# \n# Robots & Agent Policy\n#\n# Checkouts are strictly for human users. \n# * Automated scraping, buy-for-me agents, LLM-driven bots, or any \n#   end-to-end flow that attempts to place orders without human review \n#   is strictly prohibited. \n# * Unauthorized use of automated agents in checkout may result in \n#   legal action under our User Agreement: https://www.ebay.com/help/policies/member-behaviour-policies/user-agreement?id=4259 \n# * Approved enterprise integrations must use our official API and \n#   comply with our API License Agreement: https://developer.ebay.com/join/api-license-agreement \n#\n# v27_ROW_August_2026\n### DIRECTIVES ###\n\nUser-agent: *\nDisallow: /*_kw\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nDisallow: /buyer-preferences\nDisallow: /motors/fnd/*\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nDisallow: */b2ajax/all-filter\nAllow: /b/*?*_mwBanner\nDisallow: /b/*_sop=\nDisallow: /b/*_pgn=\nDisallow: /b/*?*_nkw\nDisallow: /b/*?*&\nDisallow: /b/*?*|\nDisallow: /b/*%EF\nDisallow: /b/*LH_\nDisallow: /b/*,\nDisallow: /b/*_dmd=\nDisallow: /b/*_dcat=\nDisallow: /b/*_sacat=\nDisallow: /b/*_saved\nDisallow: /b/*_sid=\nDisallow: /b/*_stpos=\nDisallow: /b/*src=urllib\nDisallow: /b/*udhi=\nDisallow: /b/*udlo=\nDisallow: /b/Test-Category\nDisallow: /b/eBay-Use-Only\nDisallow: /b/eBay-Test-Only\nDisallow: /b/LP-Parent\nDisallow: /b/LL-Child\nDisallow: /b/eBay-User-Tools\nDisallow: /b/Test-Auctions\nDisallow: /b/Attributes\nDisallow: /b/*DO-NOT-BID\nDisallow: /b/Testbereich/\nDisallow: /b/Testkategoriegruppe/\nDisallow: /b/Testkategorie\nDisallow: /b/Categoria-per-test/\nDisallow: /b/Test-eBay/\nDisallow: /b/Attributo\nDisallow: /b/Categorie-test/\nDisallow: /b/Groupe-de-categories-de-test/\nDisallow: /b/Subastas-de-prueba/\nDisallow: /b/Atributos\nDisallow: /bin/\nDisallow: /bo/\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /blueberry/\nDisallow: /brw/\nDisallow: /c/\nDisallow: /cancel/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebaylive/host\nDisallow: /ebay/cronus/\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /fdbk/\nDisallow: /feed/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gwc/\nDisallow: /hcp/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itemnotreceived/\nDisallow: /itempolicy\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /logout\nDisallow: /looking-glass-proxy/graphql\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*surveyopen\nDisallow: /p/*thm=\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /prp/\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nDisallow: /sch/*classiques\nDisallow: /sch/*_sold=false\nDisallow: /sch/*Emplacement\nDisallow: /sch/*marque\nDisallow: /sch/*_pgn\nDisallow: /sch/*_sop\nDisallow: /sch/*%EF\nDisallow: /sch/*LH_Auction\nDisallow: /sch/*LH_BIN\nDisallow: /sch/*LH_ItemCondition\nDisallow: /sch/*LH_OUTLETMALLSELLERS\nDisallow: /sch/*LH_RPA\nDisallow: /sch/*LH_Sold\nDisallow: /sch/*m.html*=\nDisallow: /sch/*?*udhi=\nDisallow: /sch/*?*udlo=\nDisallow: /sch/*?*|\nDisallow: /sch/*\\u00\nDisallow: /sch/*?*&*&*&*&*&*&*&\nDisallow: /sch/ajax\nDisallow: /sch/aop/\nDisallow: /sch/ebayadvsearch?_\nDisallow: /sch/FindingCustomization/\nDisallow: /sch/sis.html\nDisallow: /sh/onboarding\nDisallow: /ship/\nDisallow: /sh/str/fvfsavings\nAllow: /signin/$\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/ipe\nDisallow: /sl/node\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nDisallow: /usr/*/all-follows\nDisallow: /usr/*/followers\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vod/\nDisallow: /vplbl/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /ws/\n\nUser-agent: bingbot\nDisallow: /*_kw\nDisallow: /b/*_mwBanner=1*rt=nc\nDisallow: /b/*rt=nc*_mwBanner=1\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nAllow: /b/*?*_mwBanner\nDisallow: /b/*_sop=\nDisallow: /b/*_pgn=\nDisallow: /b/*?*_nkw\nDisallow: /b/*?*&\nDisallow: /b/*?*|\nDisallow: /b/*%EF\nDisallow: /b/*LH_\nDisallow: /b/*,\nDisallow: /b/*_dmd=\nDisallow: /b/*_dcat=\nDisallow: /b/*_sacat=\nDisallow: /b/*_saved\nDisallow: /b/*_sid=\nDisallow: /b/*_stpos=\nDisallow: /b/*src=urllib\nDisallow: /b/*udhi=\nDisallow: /b/*udlo=\nDisallow: /b/Test-Category\nDisallow: /b/eBay-Use-Only\nDisallow: /b/eBay-Test-Only\nDisallow: /b/LP-Parent\nDisallow: /b/LL-Child\nDisallow: /b/eBay-User-Tools\nDisallow: /b/Test-Auctions\nDisallow: /b/Attributes\nDisallow: /b/*DO-NOT-BID\nDisallow: /b/Testbereich/\nDisallow: /b/Testkategoriegruppe/\nDisallow: /b/Testkategorie\nDisallow: /b/Categoria-per-test/\nDisallow: /b/Test-eBay/\nDisallow: /b/Attributo\nDisallow: /b/Categorie-test/\nDisallow: /b/Groupe-de-categories-de-test/\nDisallow: /b/Subastas-de-prueba/\nDisallow: /b/Atributos\nDisallow: /bin/\nDisallow: /bo/\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /blueberry/\nDisallow: /brw/\nDisallow: /c/\nDisallow: /cancel/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebaylive/host\nDisallow: /ebay/cronus/\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /fdbk/\nDisallow: /feed/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gwc/\nDisallow: /hcp/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itemnotreceived/\nDisallow: /itempolicy\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /logout\nDisallow: /looking-glass-proxy/graphql\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*surveyopen\nDisallow: /p/*thm=\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /prp/\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nDisallow: /sch/*classiques\nDisallow: /sch/*Emplacement\nDisallow: /sch/*marque\nDisallow: /sch/*_pgn\nDisallow: /sch/*_sop\nDisallow: /sch/*%EF\nDisallow: /sch/*LH_Auction\nDisallow: /sch/*LH_BIN\nDisallow: /sch/*LH_ItemCondition\nDisallow: /sch/*LH_OUTLETMALLSELLERS\nDisallow: /sch/*LH_RPA\nDisallow: /sch/*LH_Sold\nDisallow: /sch/*m.html*=\nDisallow: /sch/*?*udhi=\nDisallow: /sch/*?*udlo=\nDisallow: /sch/*?*|\nDisallow: /sch/*\\u00\nDisallow: /sch/*?*&*&*&*&*&*&*&\nDisallow: /sch/ajax\nDisallow: /sch/aop/\nDisallow: /sch/ebayadvsearch?_\nDisallow: /sch/FindingCustomization/\nDisallow: /sch/sis.html\nDisallow: /sh/onboarding\nDisallow: /ship/\nDisallow: /sh/str/fvfsavings\nAllow: /signin/$\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/ipe\nDisallow: /sl/node\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nDisallow: /usr/\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vod/\nDisallow: /vplbl/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /ws/\n\nUser-agent: AdsBot-Google\nDisallow: /*src=urllib\nDisallow: /adell-mutfak\nDisallow: /ahu-kadin\nDisallow: /antika-sanat\nDisallow: /arama\nDisallow: /bebek-anne\nDisallow: /bilgisayar-tablet\nDisallow: /cep-telefonu-ve-aksesuar\nDisallow: /ev-bahce\nDisallow: /giyim-aksesuar\nDisallow: /muzik-plak-enstruman\nDisallow: /nap/\nDisallow: /otomobil-motor-aksesuar\nDisallow: /srv/\n\nUser-agent: AdIdxBot\nDisallow: /*src=urllib\nDisallow: /adell-mutfak\nDisallow: /ahu-kadin\nDisallow: /antika-sanat\nDisallow: /arama\nDisallow: /bebek-anne\nDisallow: /bilgisayar-tablet\nDisallow: /cep-telefonu-ve-aksesuar\nDisallow: /ev-bahce\nDisallow: /giyim-aksesuar\nDisallow: /muzik-plak-enstruman\nDisallow: /nap/\nDisallow: /otomobil-motor-aksesuar\nDisallow: /srv/\n\nUser-agent: Twitterbot\nDisallow: /*src=urllib\nDisallow: /srv/\n\n#\n# LLM / AI bot directives\n#\n\nUser-agent: Bytespider\nUser-agent: CCBot\nUser-agent: ChatGLM-Spider\nUser-agent: ClaudeBot\nUser-agent: PerplexityBot\nUser-agent: anthropic-ai\nUser-agent: AmazonBot\nUser-agent: meta-externalagent\nUser-agent: GPTBot\nUser-agent: Applebot-Extended\nDisallow: /\nAllow: /help\nAllow: /verkaeuferportal\n\n# AUCTION Sitemaps\nSitemap: https://www.ebay.at/lst/AUCTION-16-index.xml\n\n# BROWSE Sitemaps\nSitemap: https://www.ebay.at/lst/BROWSE-16-index.xml\n\n# NGS Sitemaps\nSitemap: https://www.ebay.at/lst/NGS-16-index.xml\n\n# VIDEO Sitemaps\nSitemap: https://www.ebay.at/lst/VIDEO-16-index.xml\n\n# VIS Sitemaps\nSitemap: https://www.ebay.at/lst/VIS-16-index.xml\n\n### END FILE ###"
    },
    "www.ebay.nl": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.ebay.nl",
      "body": "## BEGIN FILE ###\n#\n# allow-all\n# DR\n#\n# The use of robots or other automated means to access the eBay site\n# without the express permission of eBay is strictly prohibited.\n# Notwithstanding the foregoing, eBay may permit automated access to\n# access certain eBay pages but solely for the limited purpose of\n# including content in publicly available search engines. Any other\n# use of robots or failure to obey the robots exclusion standards set\n# forth at <https://www.robotstxt.org/orig.html> is strictly\n# prohibited.\n#\n# \n# Robots & Agent Policy\n#\n# Checkouts are strictly for human users. \n# * Automated scraping, buy-for-me agents, LLM-driven bots, or any \n#   end-to-end flow that attempts to place orders without human review \n#   is strictly prohibited. \n# * Unauthorized use of automated agents in checkout may result in \n#   legal action under our User Agreement: https://www.ebay.com/help/policies/member-behaviour-policies/user-agreement?id=4259 \n# * Approved enterprise integrations must use our official API and \n#   comply with our API License Agreement: https://developer.ebay.com/join/api-license-agreement \n#\n# v27_CAT_SRP_August_2026\n### DIRECTIVES ###\n\nUser-agent: *\nDisallow: /*_kw\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: */b2ajax/all-filter\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nDisallow: /b/\nDisallow: /bin/\nDisallow: /bo/\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /blueberry/\nDisallow: /buyer-preferences\nDisallow: /brw/\nDisallow: /cancel/\nDisallow: /c/\nDisallow: /clp/\nDisallow: /sch/*_cs=\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /delicacy\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebay/cronus/\nDisallow: /ebaylive/host\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /fdbk/\nDisallow: /feed/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gwc/\nDisallow: /hcp/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itemnotreceived/\nDisallow: /itempolicy\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/act\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /logout\nDisallow: /looking-glass-proxy/graphql\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /motors/fnd/*\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*surveyopen\nDisallow: /p/*thm=\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /prp/\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/i.html?_nkw=*_sop=12\nDisallow: /sch/*_sop=\nDisallow: /sch/*_pgn\nDisallow: /sch/*classiques\nDisallow: /sch/*Emplacement\nDisallow: /sch/*marque\nDisallow: /sch/*%EF\nDisallow: /sch/*LH_Auction\nDisallow: /sch/*LH_BIN\nDisallow: /sch/*LH_ItemCondition\nDisallow: /sch/*LH_OUTLETMALLSELLERS\nDisallow: /sch/*LH_RPA\nDisallow: /sch/*LH_Sold\nDisallow: /sch/*m.html*=\nDisallow: /sch/*?*udhi=\nDisallow: /sch/*?*udlo=\nDisallow: /sch/*?*|\nDisallow: /sch/*\\u00\nDisallow: /sch/*?*&*&*&*&*&*&*&\nDisallow: /sch/ajax\nDisallow: /sch/aop/\nDisallow: /sch/ebayadvsearch?_\nDisallow: /sch/FindingCustomization/\nDisallow: /sch/sis.html\nDisallow: /sh/onboarding\nDisallow: /ship/\nDisallow: /sh/str/fvfsavings\nAllow: /signin/$\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/ipe\nDisallow: /sl/node\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nDisallow: /usr/*/all-follows\nDisallow: /usr/*/followers\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vod/\nDisallow: /vplbl/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /ws/\n\nUser-agent: bingbot\nDisallow: /*_kw\nDisallow: /b/*_mwBanner=1*rt=nc\nDisallow: /b/*rt=nc*_mwBanner=1\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect%3Dmobile\nDisallow: /*redirect=mobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nDisallow: /b/\nDisallow: /bin/\nDisallow: /bo/\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /blueberry/\nDisallow: /brw/\nDisallow: /c/\nDisallow: /cancel/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /delicacy\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebay/cronus/\nDisallow: /ebaylive/host\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /fdbk/\nDisallow: /feed/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gwc/\nDisallow: /hcp/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itemnotreceived/\nDisallow: /itempolicy\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /logout\nDisallow: /looking-glass-proxy/graphql\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*surveyopen\nDisallow: /p/*thm=\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /prp/\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/i.html?_nkw=*_sop=12\nDisallow: /sch/*_sop=\nDisallow: /sch/*_pgn\nDisallow: /sch/*classiques\nDisallow: /sch/*Emplacement\nDisallow: /sch/*marque\nDisallow: /sch/*%EF\nDisallow: /sch/*LH_Auction\nDisallow: /sch/*LH_BIN\nDisallow: /sch/*LH_ItemCondition\nDisallow: /sch/*LH_OUTLETMALLSELLERS\nDisallow: /sch/*LH_RPA\nDisallow: /sch/*LH_Sold\nDisallow: /sch/*m.html*=\nDisallow: /sch/*?*udhi=\nDisallow: /sch/*?*udlo=\nDisallow: /sch/*?*|\nDisallow: /sch/*\\u00\nDisallow: /sch/*?*&*&*&*&*&*&*&\nDisallow: /sch/ajax\nDisallow: /sch/aop/\nDisallow: /sch/ebayadvsearch?_\nDisallow: /sch/FindingCustomization/\nDisallow: /sch/sis.html\nDisallow: /sh/onboarding\nDisallow: /ship/\nDisallow: /sh/str/fvfsavings\nAllow: /signin/$\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/ipe\nDisallow: /sl/node\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nDisallow: /usr/\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vod/\nDisallow: /vplbl/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /ws/\n\nUser-agent: AdsBot-Google\nDisallow: /*src=urllib\nDisallow: /adell-mutfak\nDisallow: /ahu-kadin\nDisallow: /antika-sanat\nDisallow: /arama\nDisallow: /bebek-anne\nDisallow: /bilgisayar-tablet\nDisallow: /cep-telefonu-ve-aksesuar\nDisallow: /ev-bahce\nDisallow: /giyim-aksesuar\nDisallow: /muzik-plak-enstruman\nDisallow: /nap/\nDisallow: /otomobil-motor-aksesuar\nDisallow: /srv/\n\nUser-agent: AdIdxBot\nDisallow: /*src=urllib\nDisallow: /adell-mutfak\nDisallow: /ahu-kadin\nDisallow: /antika-sanat\nDisallow: /arama\nDisallow: /bebek-anne\nDisallow: /bilgisayar-tablet\nDisallow: /cep-telefonu-ve-aksesuar\nDisallow: /ev-bahce\nDisallow: /giyim-aksesuar\nDisallow: /muzik-plak-enstruman\nDisallow: /nap/\nDisallow: /otomobil-motor-aksesuar\nDisallow: /srv/\n\nUser-agent: Twitterbot\nDisallow: /*src=urllib\nDisallow: /srv/\n\n#\n# LLM / AI bot directives\n#\n\nUser-agent: Bytespider\nUser-agent: CCBot\nUser-agent: ChatGLM-Spider\nUser-agent: ClaudeBot\nUser-agent: PerplexityBot\nUser-agent: anthropic-ai\nUser-agent: AmazonBot\nUser-agent: meta-externalagent\nUser-agent: GPTBot\nUser-agent: Applebot-Extended\nDisallow: /\nAllow: /help\nAllow: /verkoperscentrum\n\n# AUCTION Sitemaps\nSitemap: https://www.ebay.nl/lst/AUCTION-146-index.xml\n\n# NGS Sitemaps\nSitemap: https://www.ebay.nl/lst/NGS-146-index.xml\n\n# VIS Sitemaps\nSitemap: https://www.ebay.nl/lst/VIS-146-index.xml\n\n### END FILE ###"
    },
    "www.ebay.pl": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.ebay.pl",
      "body": "## BEGIN FILE ###\n#\n# allow-all\n# DR\n#\n# The use of robots or other automated means to access the eBay site\n# without the express permission of eBay is strictly prohibited.\n# Notwithstanding the foregoing, eBay may permit automated access to\n# access certain eBay pages but solely for the limited purpose of\n# including content in publicly available search engines. Any other\n# use of robots or failure to obey the robots exclusion standards set\n# forth at <https://www.robotstxt.org/orig.html> is strictly\n# prohibited.\n#\n# \n# Robots & Agent Policy\n#\n# Checkouts are strictly for human users. \n# * Automated scraping, buy-for-me agents, LLM-driven bots, or any \n#   end-to-end flow that attempts to place orders without human review \n#   is strictly prohibited. \n# * Unauthorized use of automated agents in checkout may result in \n#   legal action under our User Agreement: https://www.ebay.com/help/policies/member-behaviour-policies/user-agreement?id=4259 \n# * Approved enterprise integrations must use our official API and \n#   comply with our API License Agreement: https://developer.ebay.com/join/api-license-agreement \n#\n# v27_CAT_SRP_August_2026\n### DIRECTIVES ###\n\nUser-agent: *\nDisallow: /*_kw\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nDisallow: /buyer-preferences\nDisallow: /motors/fnd/*\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nDisallow: /b/\nDisallow: /bin/\nDisallow: /bo/\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /blueberry/\nDisallow: /brw/\nDisallow: /cancel/\nDisallow: /c/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /delicacy\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /sch/*_cs=\nDisallow: /easy-shop\nDisallow: /ebay/cronus/\nDisallow: /ebaylive/host\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /fdbk/\nDisallow: /feed/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gwc/\nDisallow: /hcp/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itemnotreceived/\nDisallow: /itempolicy\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /logout\nDisallow: /looking-glass-proxy/graphql\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*surveyopen\nDisallow: /p/*thm=\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /prp/\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/i.html?_nkw=*_sop=12\nDisallow: /sch/*_sop=\nDisallow: /sch/*_pgn\nDisallow: /sch/*classiques\nDisallow: /sch/*Emplacement\nDisallow: /sch/*marque\nDisallow: /sch/*%EF\nDisallow: /sch/*LH_Auction\nDisallow: /sch/*LH_BIN\nDisallow: /sch/*LH_ItemCondition\nDisallow: /sch/*LH_OUTLETMALLSELLERS\nDisallow: /sch/*LH_RPA\nDisallow: /sch/*LH_Sold\nDisallow: /sch/*m.html*=\nDisallow: /sch/*?*udhi=\nDisallow: /sch/*?*udlo=\nDisallow: /sch/*?*|\nDisallow: /sch/*\\u00\nDisallow: /sch/*?*&*&*&*&*&*&*&\nDisallow: /sch/ajax\nDisallow: /sch/aop/\nDisallow: /sch/ebayadvsearch?_\nDisallow: /sch/FindingCustomization/\nDisallow: /sch/sis.html\nDisallow: /sh/onboarding\nDisallow: /ship/\nDisallow: /sh/str/fvfsavings\nAllow: /signin/$\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/ipe\nDisallow: /sl/node\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nDisallow: /usr/*/all-follows\nDisallow: /usr/*/followers\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vod/\nDisallow: /vplbl/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /ws/\n\nUser-agent: bingbot\nDisallow: /*_kw\nDisallow: /b/*_mwBanner=1*rt=nc\nDisallow: /b/*rt=nc*_mwBanner=1\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect%3Dmobile\nDisallow: /*redirect=mobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nDisallow: /b/\nDisallow: /bin/\nDisallow: /bo/\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /blueberry/\nDisallow: /brw/\nDisallow: /c/\nDisallow: /cancel/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /delicacy\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebay/cronus/\nDisallow: /ebaylive/host\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /fdbk/\nDisallow: /feed/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gwc/\nDisallow: /hcp/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itemnotreceived/\nDisallow: /itempolicy\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /logout\nDisallow: /looking-glass-proxy/graphql\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*surveyopen\nDisallow: /p/*thm=\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /prp/\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/i.html?_nkw=*_sop=12\nDisallow: /sch/*_sop=\nDisallow: /sch/*_pgn\nDisallow: /sch/*classiques\nDisallow: /sch/*Emplacement\nDisallow: /sch/*marque\nDisallow: /sch/*%EF\nDisallow: /sch/*LH_Auction\nDisallow: /sch/*LH_BIN\nDisallow: /sch/*LH_ItemCondition\nDisallow: /sch/*LH_OUTLETMALLSELLERS\nDisallow: /sch/*LH_RPA\nDisallow: /sch/*LH_Sold\nDisallow: /sch/*m.html*=\nDisallow: /sch/*?*udhi=\nDisallow: /sch/*?*udlo=\nDisallow: /sch/*?*|\nDisallow: /sch/*\\u00\nDisallow: /sch/*?*&*&*&*&*&*&*&\nDisallow: /sch/ajax\nDisallow: /sch/aop/\nDisallow: /sch/ebayadvsearch?_\nDisallow: /sch/FindingCustomization/\nDisallow: /sch/sis.html\nDisallow: /sh/onboarding\nDisallow: /ship/\nDisallow: /sh/str/fvfsavings\nAllow: /signin/$\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/ipe\nDisallow: /sl/node\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nDisallow: /usr/\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vod/\nDisallow: /vplbl/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /ws/\n\nUser-agent: AdsBot-Google\nDisallow: /*src=urllib\nDisallow: /adell-mutfak\nDisallow: /ahu-kadin\nDisallow: /antika-sanat\nDisallow: /arama\nDisallow: /bebek-anne\nDisallow: /bilgisayar-tablet\nDisallow: /cep-telefonu-ve-aksesuar\nDisallow: /ev-bahce\nDisallow: /giyim-aksesuar\nDisallow: /muzik-plak-enstruman\nDisallow: /nap/\nDisallow: /otomobil-motor-aksesuar\nDisallow: /srv/\n\nUser-agent: AdIdxBot\nDisallow: /*src=urllib\nDisallow: /adell-mutfak\nDisallow: /ahu-kadin\nDisallow: /antika-sanat\nDisallow: /arama\nDisallow: /bebek-anne\nDisallow: /bilgisayar-tablet\nDisallow: /cep-telefonu-ve-aksesuar\nDisallow: /ev-bahce\nDisallow: /giyim-aksesuar\nDisallow: /muzik-plak-enstruman\nDisallow: /nap/\nDisallow: /otomobil-motor-aksesuar\nDisallow: /srv/\n\nUser-agent: Twitterbot\nDisallow: /*src=urllib\nDisallow: /srv/\n\n#\n# LLM / AI bot directives\n#\n\nUser-agent: Bytespider\nUser-agent: CCBot\nUser-agent: ChatGLM-Spider\nUser-agent: ClaudeBot\nUser-agent: PerplexityBot\nUser-agent: anthropic-ai\nUser-agent: AmazonBot\nUser-agent: meta-externalagent\nUser-agent: GPTBot\nUser-agent: Applebot-Extended\nDisallow: /\nAllow: /help\n\n# AUCTION Sitemaps\nSitemap: https://www.ebay.pl/lst/AUCTION-212-index.xml\n\n# NGS Sitemaps\nSitemap: https://www.ebay.pl/lst/NGS-212-index.xml\n\n# VIS Sitemaps\nSitemap: https://www.ebay.pl/lst/VIS-212-index.xml\n\n### END FILE ###"
    },
    "www.ebay.com": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.ebay.com",
      "body": "## BEGIN FILE ###\n#\n# allow-all\n# DR\n#\n# The use of robots or other automated means to access the eBay site\n# without the express permission of eBay is strictly prohibited.\n# Notwithstanding the foregoing, eBay may permit automated access to\n# access certain eBay pages but solely for the limited purpose of\n# including content in publicly available search engines. Any other\n# use of robots or failure to obey the robots exclusion standards set\n# forth at <https://www.robotstxt.org/orig.html> is strictly\n# prohibited.\n#\n# Robots & Agent Policy\n#\n# Checkouts are strictly for human users. \n# * Automated scraping, buy-for-me agents, LLM-driven bots, or any \n#   end-to-end flow that attempts to place orders without human review \n#   is strictly prohibited. \n# * Unauthorized use of automated agents in checkout may result in \n#   legal action under our User Agreement: https://www.ebay.com/help/policies/member-behaviour-policies/user-agreement?id=4259 \n# * Approved enterprise integrations must use our official API and \n#   comply with our API License Agreement: https://developer.ebay.com/join/api-license-agreement \n#\n# v30.2_COM_August_2026\n### DIRECTIVES ###\n\nUser-agent: *\nDisallow: /*_kw\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nDisallow: /buyer-preferences\nDisallow: /motors/diy/blog/*\nDisallow: /motors/fnd/*\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nDisallow: /adell-mutfak\nDisallow: /ahu-kadin\nDisallow: /antika-sanat\nDisallow: /arama\nDisallow: */b2ajax/all-filter\nDisallow: /bebek-anne\nDisallow: /bilgisayar-tablet\nDisallow: /cep-telefonu-ve-aksesuar\nDisallow: /delicacy\nDisallow: /ev-bahce\nDisallow: /giyim-aksesuar\nDisallow: /muzik-plak-enstruman\nDisallow: /otomobil-motor-aksesuar\nAllow: /b/adidas-Yeezy-Sneakers-for-Men/15709/bn_86578781?*_trkparms=*pageci:*|parentrq:*iid:0\nAllow: /b/Collectible-Sneakers/bn_7000259435?*_trkparms=*pageci:*|parentrq:*iid:0\nAllow: /b/Jordan-Sneakers-for-Men/15709/bn_96541848?*_trkparms=*pageci:*|parentrq:*iid:0\nAllow: /b/New-Balance-Sneakers-for-Men/15709/bn_58747?*_trkparms=*pageci:*|parentrq:*iid:0\nAllow: /b/Sneakers-for-Men/15709/bn_57918?*_trkparms=*pageci:*|parentrq:*iid:0\nAllow: /b/*?*_mwBanner\nAllow: /b/*?iid=*&var=\nAllow: /b/*?iid=*&chn=ps\nAllow: /b/*?iid=*&var=*&chn=ps\nDisallow: /b/*,\nDisallow: /b/*?iid=*&var=*&\nDisallow: /b/*?iid=*&*&chn=ps\nDisallow: /b/*?iid=*&var=*&*&chn=ps\nDisallow: /b/*?*_nkw\nDisallow: /b/*?*&\nDisallow: /pe/*\nDisallow: /b/*?*|\nDisallow: /b/*%EF\nDisallow: /b/*LH_\nDisallow: /b/*_dmd=\nDisallow: /b/*_dcat=\nDisallow: /b/*_pgn=\nDisallow: /b/*_sacat=\nDisallow: /b/*_saved\nDisallow: /b/*_sid=\nDisallow: /b/*_sop=\nDisallow: /b/*_stpos=\nDisallow: /b/*DO-NOT-BID\nDisallow: /b/*udhi=\nDisallow: /b/*udlo=\nDisallow: /b/Test-Category\nDisallow: /b/eBay-Use-Only\nDisallow: /b/eBay-Test-Only\nDisallow: /b/LP-Parent\nDisallow: /b/LL-Child\nDisallow: /b/eBay-User-Tools\nDisallow: /b/Test-Auctions\nDisallow: /b/Attributes\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /bin/\nDisallow: /blueberry/\nDisallow: /bo/\nDisallow: /brw/\nDisallow: /c/\nDisallow: /cancel/\nDisallow: /chocolatechip/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebay/cronus/\nDisallow: /ebaylive/host\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /feed/\nDisallow: /fdbk/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /g/api/\nDisallow: /gh/cart\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gh/user_profile\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gum/\nDisallow: /gwc/\nDisallow: /hcp/\nDisallow: /heute/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itemnotreceived/\nDisallow: /itempolicy\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /logout\nDisallow: /looking-glass-proxy/graphql\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*thm=\nDisallow: /p/*surveyopen\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /price-guide/\nDisallow: /prp/\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/ebayadvsearch\nAllow: /sch/allcategories/\nAllow: /sch/*_ul\nAllow: /sch/*_fosrp\nAllow: /sch/*_trksid\nAllow: /sch/i.html?*&mkcid=2\nDisallow: /sch/i.html?_nkw=\nDisallow: /sch/i.html?*_nkw=*&\nDisallow: /sch/\nDisallow: /sch/*_sacat=\nDisallow: /sch/ajax\nDisallow: /sch/ebayadvsearch?_\nDisallow: /sh/onboarding\nDisallow: /sh/str/fvfsavings\nDisallow: /ship/\nDisallow: /shop/*&\nAllow: /signin/$\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/node\nDisallow: /sl/ipe\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nDisallow: /usr/*/all-follows\nDisallow: /usr/*/followers\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vod/\nDisallow: /vplbl/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /wcs/\nDisallow: /ws/\n\nUser-agent: Googlebot\nDisallow: /*_kw\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nDisallow: /buyer-preferences\nDisallow: /motors/diy/blog/*\nDisallow: /motors/fnd/*\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nDisallow: /adell-mutfak\nDisallow: /ahu-kadin\nDisallow: /antika-sanat\nDisallow: /arama\nDisallow: */b2ajax/all-filter\nDisallow: /bebek-anne\nDisallow: /bilgisayar-tablet\nDisallow: /cep-telefonu-ve-aksesuar\nDisallow: /delicacy\nDisallow: /ev-bahce\nDisallow: /giyim-aksesuar\nDisallow: /muzik-plak-enstruman\nDisallow: /otomobil-motor-aksesuar\nAllow: /b/adidas-Yeezy-Sneakers-for-Men/15709/bn_86578781?*_trkparms=*pageci:*|parentrq:*iid:0\nAllow: /b/Collectible-Sneakers/bn_7000259435?*_trkparms=*pageci:*|parentrq:*iid:0\nAllow: /b/Jordan-Sneakers-for-Men/15709/bn_96541848?*_trkparms=*pageci:*|parentrq:*iid:0\nAllow: /b/New-Balance-Sneakers-for-Men/15709/bn_58747?*_trkparms=*pageci:*|parentrq:*iid:0\nAllow: /b/Sneakers-for-Men/15709/bn_57918?*_trkparms=*pageci:*|parentrq:*iid:0\nAllow: /b/*?*_mwBanner\nAllow: /b/*?iid=*&var=\nAllow: /b/*?iid=*&chn=ps\nAllow: /b/*?iid=*&var=*&chn=ps\nDisallow: /b/*,\nDisallow: /b/*?iid=*&var=*&\nDisallow: /b/*?iid=*&*&chn=ps\nDisallow: /b/*?iid=*&var=*&*&chn=ps\nDisallow: /b/*?*_nkw\nDisallow: /b/*?*&\nDisallow: /pe/*\nDisallow: /b/*?*|\nDisallow: /b/*%EF\nDisallow: /b/*LH_\nDisallow: /b/*_dmd=\nDisallow: /b/*_dcat=\nDisallow: /b/*_pgn=\nDisallow: /b/*_sacat=\nDisallow: /b/*_saved\nDisallow: /b/*_sid=\nDisallow: /b/*_sop=\nDisallow: /b/*_stpos=\nDisallow: /b/*DO-NOT-BID\nDisallow: /b/*udhi=\nDisallow: /b/*udlo=\nDisallow: /b/Test-Category\nDisallow: /b/eBay-Use-Only\nDisallow: /b/eBay-Test-Only\nDisallow: /b/LP-Parent\nDisallow: /b/LL-Child\nDisallow: /b/eBay-User-Tools\nDisallow: /b/Test-Auctions\nDisallow: /b/Attributes\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /bin/\nDisallow: /blueberry/\nDisallow: /bo/\nDisallow: /brw/\nDisallow: /c/\nDisallow: /cancel/\nDisallow: /chocolatechip/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebay/cronus/\nDisallow: /ebaylive/host\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /feed/\nDisallow: /fdbk/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /g/api/\nDisallow: /gh/cart\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gh/user_profile\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gum/\nDisallow: /gwc/\nDisallow: /hcp/\nDisallow: /heute/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itemnotreceived/\nDisallow: /itempolicy\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /logout\nDisallow: /looking-glass-proxy/graphql\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*thm=\nDisallow: /p/*surveyopen\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /price-guide/\nDisallow: /prp/\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/ebayadvsearch\nAllow: /sch/allcategories/\nAllow: /sch/*_ul\nAllow: /sch/*_fosrp\nAllow: /sch/*_trksid\nAllow: /sch/i.html?*&mkcid=2\nDisallow: /sch/i.html?_nkw=\nDisallow: /sch/i.html?*_nkw=*&\nDisallow: /sch/\nDisallow: /sch/*_sacat=\nDisallow: /sch/ajax\nDisallow: /sch/ebayadvsearch?_\nDisallow: /sh/onboarding\nDisallow: /sh/str/fvfsavings\nDisallow: /ship/\nDisallow: /shop/*&\nAllow: /signin/$\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/node\nDisallow: /sl/ipe\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nDisallow: /usr/*/all-follows\nDisallow: /usr/*/followers\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vod/\nDisallow: /vplbl/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /wcs/\nDisallow: /ws/\n\nUser-agent: bingbot\nDisallow: /*_kw\nDisallow: /b/*_mwBanner=1*rt=nc\nDisallow: /b/*rt=nc*_mwBanner=1 \nDisallow: /motors/diy/beacon/\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nAllow: /b/adidas-Yeezy-Sneakers-for-Men/15709/bn_86578781?*_trkparms=*pageci:*|parentrq:*iid:0\nAllow: /b/Collectible-Sneakers/bn_7000259435?*_trkparms=*pageci:*|parentrq:*iid:0\nAllow: /b/Jordan-Sneakers-for-Men/15709/bn_96541848?*_trkparms=*pageci:*|parentrq:*iid:0\nAllow: /b/New-Balance-Sneakers-for-Men/15709/bn_58747?*_trkparms=*pageci:*|parentrq:*iid:0\nAllow: /b/Sneakers-for-Men/15709/bn_57918?*_trkparms=*pageci:*|parentrq:*iid:0\nAllow: /b/*?*_mwBanner\nAllow: /b/*?iid=*&var=\nAllow: /b/*?iid=*&chn=ps\nAllow: /b/*?iid=*&var=*&chn=ps\nDisallow: /b/*,\nDisallow: /b/*?iid=*&var=*&\nDisallow: /b/*?iid=*&*&chn=ps\nDisallow: /b/*?iid=*&var=*&*&chn=ps\nDisallow: /b/*?*_nkw\nDisallow: /b/*?*&\nDisallow: /b/*?*|\nDisallow: /b/*%EF\nDisallow: /b/*LH_\nDisallow: /b/*_dmd=\nDisallow: /b/*_dcat=\nDisallow: /b/*_pgn=\nDisallow: /b/*_sacat=\nDisallow: /b/*_saved\nDisallow: /b/*_sid=\nDisallow: /b/*_sop=\nDisallow: /b/*_stpos=\nDisallow: /b/*DO-NOT-BID\nDisallow: /b/*udhi=\nDisallow: /b/*udlo=\nDisallow: /b/Test-Category\nDisallow: /b/eBay-Use-Only\nDisallow: /b/eBay-Test-Only\nDisallow: /b/LP-Parent\nDisallow: /b/LL-Child\nDisallow: /b/eBay-User-Tools\nDisallow: /b/Test-Auctions\nDisallow: /b/Attributes\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /bin/\nDisallow: /blueberry/\nDisallow: /bo/\nDisallow: /brw/\nDisallow: /c/\nDisallow: /cancel/\nDisallow: /chocolatechip/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /contact/\nDisallow: /cnt/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /d/graphql\nDisallow: /delicacy\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebay/cronus/\nDisallow: /ebaylive/host\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /feed/\nDisallow: /fdbk/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /g/api/\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gh/user_profile\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gwc/\nDisallow: /hcp/\nDisallow: /heute/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itemnotreceived/\nDisallow: /itempolicy\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /looking-glass-proxy/graphql\nDisallow: /logout\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*thm=\nDisallow: /p/*surveyopen\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /price-guide/\nDisallow: /prp/\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/ebayadvsearch\nAllow: /sch/allcategories/\nAllow: /sch/i.html?*&mkcid=2\nAllow: /sch/i.html*_sop=12\nDisallow: /sch/\nDisallow: /sch/*_sacat=\nDisallow: /sch/ajax\nDisallow: /sch/ebayadvsearch?_\nDisallow: /sch/i.html?_nkw=\nDisallow: /sch/i.html*_sop=12*&\nDisallow: /sh/onboarding\nDisallow: /sh/str/fvfsavings\nDisallow: /ship/\nDisallow: /shop/*&\nAllow: /signin/$\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/ipe\nDisallow: /sl/node\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nDisallow: /usr/\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vod/\nDisallow: /vplbl/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /wcs/\nDisallow: /ws/\n\nUser-agent: AdsBot-Google\nDisallow: /*src=urllib\nDisallow: /motors/diy/beacon/\nDisallow: /adell-mutfak\nDisallow: /ahu-kadin\nDisallow: /antika-sanat\nDisallow: /arama\nDisallow: /bebek-anne\nDisallow: /bilgisayar-tablet\nDisallow: /cep-telefonu-ve-aksesuar\nDisallow: /ev-bahce\nDisallow: /giyim-aksesuar\nDisallow: /muzik-plak-enstruman\nDisallow: /nap/\nDisallow: /otomobil-motor-aksesuar\nDisallow: /srv/\n\nUser-agent: AdIdxBot\nDisallow: /*src=urllib\nDisallow: /adell-mutfak\nDisallow: /ahu-kadin\nDisallow: /antika-sanat\nDisallow: /arama\nDisallow: /bebek-anne\nDisallow: /bilgisayar-tablet\nDisallow: /cep-telefonu-ve-aksesuar\nDisallow: /ev-bahce\nDisallow: /giyim-aksesuar\nDisallow: /muzik-plak-enstruman\nDisallow: /nap/\nDisallow: /otomobil-motor-aksesuar\nDisallow: /srv/\n\nUser-agent: Twitterbot\nDisallow: /*src=urllib\nDisallow: /srv/\n\n#\n# LLM / AI bot directives\n#\n\nUser-agent: Bytespider\nUser-agent: CCBot\nUser-agent: ChatGLM-Spider\nUser-agent: ClaudeBot\nUser-agent: PerplexityBot\nUser-agent: anthropic-ai\nUser-agent: AmazonBot\nUser-agent: meta-externalagent\nUser-agent: GPTBot\nUser-agent: Applebot-Extended\nDisallow: /\nAllow: /adchoice\nAllow: /authenticity-guarantee\nAllow: /motors/blog\nAllow: /motors/diy\nAllow: /help\nAllow: /sellercenter\nAllow: /ebaylive/seller\nAllow: /sl/sell\nAllow: /sh/landing\nAllow: /giftcards\nAllow: /shp/calc/rates\n\nUser-agent: OAI-SearchBot\nUser-agent: ChatGPT-User\nUser-agent: Claude-SearchBot\nUser-agent: Claude-User\nDisallow: /*_kw\nDisallow: /*?maspect\nDisallow: /*modules=SEARCH_REFINEMENTS_MODEL_V2\nDisallow: /*redirect=mobile\nDisallow: /*redirect%3Dmobile\nDisallow: /*rt%3Dnc\nDisallow: /*rt=nc\nDisallow: /*src=urllib\nDisallow: /?SSOWebDispatcher&tg=web&ru=\nDisallow: /act/\nDisallow: /adchoice\nDisallow: /buyer-preferences\nDisallow: /motors/diy/blog/*\nDisallow: /motors/fnd/*\nAllow: /ads.txt\nDisallow: /ads\nDisallow: /adsrepository\nDisallow: /afs/\nDisallow: /atc/\nDisallow: /adell-mutfak\nDisallow: /ahu-kadin\nDisallow: /antika-sanat\nDisallow: /arama\nDisallow: */b2ajax/all-filter\nDisallow: /bebek-anne\nDisallow: /bilgisayar-tablet\nDisallow: /cep-telefonu-ve-aksesuar\nDisallow: /delicacy\nDisallow: /ev-bahce\nDisallow: /giyim-aksesuar\nDisallow: /muzik-plak-enstruman\nDisallow: /otomobil-motor-aksesuar\nAllow: /b/adidas-Yeezy-Sneakers-for-Men/15709/bn_86578781?*_trkparms=*pageci:*|parentrq:*iid:0\nAllow: /b/Collectible-Sneakers/bn_7000259435?*_trkparms=*pageci:*|parentrq:*iid:0\nAllow: /b/Jordan-Sneakers-for-Men/15709/bn_96541848?*_trkparms=*pageci:*|parentrq:*iid:0\nAllow: /b/New-Balance-Sneakers-for-Men/15709/bn_58747?*_trkparms=*pageci:*|parentrq:*iid:0\nAllow: /b/Sneakers-for-Men/15709/bn_57918?*_trkparms=*pageci:*|parentrq:*iid:0\nAllow: /b/*?*_mwBanner\nAllow: /b/*?iid=*&var=\nAllow: /b/*?iid=*&chn=ps\nAllow: /b/*?iid=*&var=*&chn=ps\nDisallow: /b/*,\nDisallow: /b/*?iid=*&var=*&\nDisallow: /b/*?iid=*&*&chn=ps\nDisallow: /b/*?iid=*&var=*&*&chn=ps\nDisallow: /b/*?*_nkw\nDisallow: /b/*?*&\nDisallow: /pe/*\nDisallow: /b/*?*|\nDisallow: /b/*%EF\nDisallow: /b/*LH_\nDisallow: /b/*_dmd=\nDisallow: /b/*_dcat=\nDisallow: /b/*_pgn=\nDisallow: /b/*_sacat=\nDisallow: /b/*_saved\nDisallow: /b/*_sid=\nDisallow: /b/*_sop=\nDisallow: /b/*_stpos=\nDisallow: /b/*DO-NOT-BID\nDisallow: /b/*udhi=\nDisallow: /b/*udlo=\nDisallow: /b/Test-Category\nDisallow: /b/eBay-Use-Only\nDisallow: /b/eBay-Test-Only\nDisallow: /b/LP-Parent\nDisallow: /b/LL-Child\nDisallow: /b/eBay-User-Tools\nDisallow: /b/Test-Auctions\nDisallow: /b/Attributes\nDisallow: /bfl/\nDisallow: /bgmt/\nDisallow: /bin/\nDisallow: /blueberry/\nDisallow: /bo/\nDisallow: /brw/\nDisallow: /c/\nDisallow: /cancel/\nDisallow: /chocolatechip/\nDisallow: /clp/\nDisallow: /clt/store/\nDisallow: /cnt/\nDisallow: /contact/\nDisallow: /csc/\nDisallow: /cta/\nDisallow: /ctg/\nDisallow: /ctm/\nDisallow: /dsc/\nDisallow: /e/*?\nDisallow: /easy-shop\nDisallow: /ebay/cronus/\nDisallow: /ebaylive/host\nDisallow: /ecaptcha/\nDisallow: /edc/\nDisallow: /experience/fic\nDisallow: /explore\nDisallow: /feed/\nDisallow: /fdbk/\nDisallow: /ficapp/\nDisallow: /fol/\nDisallow: /g/api/\nDisallow: /gh/cart\nDisallow: /gh/collectbehaviorinfo\nDisallow: /gh/collectsysteminfo\nDisallow: /gh/useracquisition\nDisallow: /gh/user_profile\nDisallow: /gsr/\nDisallow: /gss/\nDisallow: /gum/\nDisallow: /gwc/\nDisallow: /hcp/\nDisallow: /heute/\nDisallow: /ico/\nDisallow: /ifh\nDisallow: /ipp/\nDisallow: /itc/\nDisallow: /itemmodules/\nDisallow: /itemnotreceived/\nDisallow: /itempolicy\nDisallow: /itm/*,\nDisallow: /itm/*/%\nDisallow: /itm/*/[\nDisallow: /itm/*/browser.json\nDisallow: /itm/*action=BESTOFFER\nDisallow: /itm/*boolp=\nDisallow: /itm/*_nkw\nDisallow: /itm/*?fits\nDisallow: /itm/*&fits\nDisallow: /itm/*.jpg\nDisallow: /itm/*/%7B%7B\nDisallow: /itm/:/\nDisallow: /itm/*_pgn=\nDisallow: /itm/addToCart\nDisallow: /itm/fetchmodules\nDisallow: /itm/sellerInfoV2\nDisallow: /itm/soi\nDisallow: /itm/variationlogistics\nDisallow: /itm/watch/\nDisallow: /itmhero/\nDisallow: /lit/\nDisallow: /logout\nDisallow: /lstng\nDisallow: /lst/ng/\nDisallow: /lux/\nDisallow: /lvx/\nDisallow: /mbf/\nDisallow: /merchanism/\nDisallow: /mes/\nDisallow: /mla/\nDisallow: /mlt/\nDisallow: /myb\nDisallow: /mye\nDisallow: /myebay\nDisallow: /mys\nDisallow: /nap/\nDisallow: /nectar\nDisallow: /npk/\nDisallow: /p/*thm=\nDisallow: /p/*surveyopen\nDisallow: /p/moduleprovider/\nDisallow: /p/surveyopen\nDisallow: /personalization/\nDisallow: /pgfeapp\nDisallow: /price-guide/\nDisallow: /prp/\nDisallow: /rcm/\nDisallow: /recfb?\nDisallow: /recs/\nDisallow: /res/\nDisallow: /rewards\nDisallow: /rtm*\nDisallow: /rvh/\nDisallow: /rye\nAllow: /sch/\nDisallow: /sch/ajax\nDisallow: /sch/ebayadvsearch?_\nDisallow: /sh/onboarding\nDisallow: /sh/str/fvfsavings\nDisallow: /ship/\nDisallow: /shop/*&\nAllow: /signin/$\nDisallow: /signin/\nDisallow: /sio/\nDisallow: /siteminderagent/\nDisallow: /sl/*bn_*?\nDisallow: /sl/*SellLikeItem\nDisallow: /sl/*wnstart\nDisallow: /sl/node\nDisallow: /sl/ipe\nDisallow: /sme/\nDisallow: /smevi/\nDisallow: /sndmsg?\nDisallow: /soc/\nDisallow: /splashui/\nDisallow: /splashui_rc/\nDisallow: /srv/\nDisallow: /str/*_sacat\nDisallow: /str/*modules=STORE_CATEGORIES\nDisallow: /str/beacon/storefrontbeacon\nDisallow: /str/ssdemo\nDisallow: /str/svc\nDisallow: /talk/\nDisallow: /tickets/\nDisallow: /tkt\nDisallow: /today/\nDisallow: /trylater/\nDisallow: /ulk/i/\nDisallow: /ulk/sch/\nDisallow: /ulk/usr/\nAllow: /urw/*/product-reviews/\nDisallow: /urw/\nDisallow: /usr/*/all-follows\nDisallow: /usr/*/followers\nDisallow: /vas\nDisallow: /viitem\nDisallow: /vlabl\nDisallow: /vlt\nDisallow: /vod/\nDisallow: /vplbl/\nDisallow: /vsp/\nDisallow: /.well-known/resource-that-should-not-exist-whose-status-code-should-not-be-200\nDisallow: /wcs/\nDisallow: /ws/\n\nUser-agent: OAI-AdsBot\nAllow: /b/*?*mkcid=32*\nAllow: /itm/*?*mkcid=32* \nDisallow: /\n\n# AUCTION Sitemaps\nSitemap: https://www.ebay.com/lst/AUCTION-0-index.xml\n\n# BROWSE Sitemaps\nSitemap: https://www.ebay.com/lst/BROWSE-0-index.xml\n\n# KWSRP Sitemaps\nSitemap: https://www.ebay.com/lst/KWSRP-0-index.xml\n\n# NGS Sitemaps\nSitemap: https://www.ebay.com/lst/NGS-0-index.xml\n\n# PRP Sitemaps\nSitemap: https://www.ebay.com/lst/PRP-0-index.xml\n\n# VIDEO Sitemaps\nSitemap: https://www.ebay.com/lst/VIDEO-0-index.xml\n\n# VIS Sitemaps\nSitemap: https://www.ebay.com/lst/VIS-0-index.xml\n\n### END FILE ###"
    },
    "allegro.pl": {
      "status": 200,
      "client": "fetch",
      "finalHost": "allegro.pl",
      "body": "user-agent: *\ndisallow: /showcase.php/\ndisallow: /SendMailToUser.php*\ndisallow: /report_item.php*\ndisallow: /events/clicks\ndisallow: /offer/\ndisallow: /ape\ndisallow: /payments\ndisallow: /recommendations/\ndisallow: /cart-aggregator/\ndisallow: /favourites/\ndisallow: /usertraits/\ndisallow: /mediation/\ndisallow: *snapshot=*\ndisallow: /language-dispatcher\ndisallow: *&source=allegro.com&welcome=1&shippingCountry=US*\ndisallow: /login/auth\ndisallow: /auth/oauth/authorize\ndisallow: /produkt/warianty/\ndisallow: /uzytkownik*&*\ndisallow: /opinie-produktowe/\ndisallow: /zglos-naruszenie/\ndisallow: /uzytkownik/*SHOP-TAG*\ndisallow: /magazyn-allegro/\ndisallow: /szukaj/multi?phrases*\ndisallow: /promowane/\ndisallow: /kampanie/*&*\ndisallow: *reco_id=*\ndisallow: *preco_id=*\ndisallow: /*?*=*&*=*&*=*&*=*&*=*&*=*&*=*&*=*&*=*&*=*&*\ndisallow: /kategoria/*?*=*&*=*&*=*&*=*&*=*&*=*&*\ndisallow: /moje-allegro/\ndisallow: /transakcja/\nallow: /uzytkownik/notfound?string*\nallow: /uzytkownik/*bi_s=archiwum_allegro*\nallow: /kategoria/*bi_s=archiwum_allegro*\n\nuser-agent: Mediapartners-Google\ndisallow: *origin_url=*\ndisallow: /transakcja/\n"
    },
    "allegro.cz": {
      "status": 200,
      "client": "fetch",
      "finalHost": "allegro.cz",
      "body": "user-agent: *\ndisallow: /events/clicks\ndisallow: /ape\ndisallow: /payments\ndisallow: /recommendations/\ndisallow: /cart-aggregator/\ndisallow: /usertraits/\ndisallow: /mediation/\ndisallow: *snapshot=*\ndisallow: /language-dispatcher\ndisallow: *&source=allegro.com&welcome=1&shippingCountry=US*\ndisallow: /login/auth\ndisallow: /auth/oauth/authorize\ndisallow: /recenze-produktu/\ndisallow: /sklad-allegro/\ndisallow: /kampane/*&*\ndisallow: *reco_id=*\ndisallow: /kategorie/*?*=*&*=*&*=*&*=*&*=*&*\ndisallow: /moje-allegro/\ndisallow: /transakce/\n"
    },
    "allegro.sk": {
      "status": 200,
      "client": "fetch",
      "finalHost": "allegro.sk",
      "body": "user-agent: *\ndisallow: /events/clicks\ndisallow: /ape\ndisallow: /payments\ndisallow: /recommendations/\ndisallow: /cart-aggregator/\ndisallow: /usertraits/\ndisallow: /mediation/\ndisallow: *snapshot=*\ndisallow: /language-dispatcher\ndisallow: /login/auth\ndisallow: /auth/oauth/authorize\ndisallow: /recenzia-produktu/\ndisallow: /sklad-allegro/\ndisallow: /kampane/*&*\ndisallow: *reco_id=*\ndisallow: /moje-allegro/\ndisallow: /transakcia/\ndisallow: /kategoria/*?*=*&*=*&*=*&*=*&*=*&*\n\nuser-agent: GoogleOther\ndisallow: /events/clicks\ndisallow: /ape\ndisallow: /payments\ndisallow: /recommendations/\ndisallow: /cart-aggregator/\ndisallow: /usertraits/\ndisallow: /mediation/\ndisallow: *snapshot=*\ndisallow: /language-dispatcher\ndisallow: /login/auth\ndisallow: /auth/oauth/authorize\ndisallow: /recenzia-produktu/\ndisallow: /sklad-allegro/\ndisallow: /kampane/*&*\ndisallow: *reco_id=*\ndisallow: /moje-allegro/\ndisallow: /transakcia/\ndisallow: /kategoria/\n"
    },
    "allegro.hu": {
      "status": 200,
      "client": "fetch",
      "finalHost": "allegro.hu",
      "body": "user-agent: *\ndisallow: /events/clicks\ndisallow: /ape\ndisallow: /payments\ndisallow: /recommendations/\ndisallow: /cart-aggregator/\ndisallow: /usertraits/\ndisallow: /mediation/\ndisallow: *snapshot=*\ndisallow: /language-dispatcher\ndisallow: /login/auth\ndisallow: /auth/oauth/authorize\ndisallow: /termekertekeles/\ndisallow: /allegro-raktar/\ndisallow: /visszaeles-bejelentese/\ndisallow: /kampanyok/*&*\ndisallow: *reco_id=*\ndisallow: /kategoria/*?*=*&*=*&*=*&*=*&*=*&*\ndisallow: /allegro-fiokom/\ndisallow: /tranzakcio/\n\nuser-agent: GoogleOther\ndisallow: /\n"
    },
    "www.kaufland.de": {
      "status": 403,
      "client": "fetch",
      "finalHost": "www.kaufland.de",
      "body": null
    },
    "www.kaufland.pl": {
      "status": 403,
      "client": "fetch",
      "finalHost": "www.kaufland.pl",
      "body": null
    },
    "www.kaufland.cz": {
      "status": 403,
      "client": "fetch",
      "finalHost": "www.kaufland.cz",
      "body": null
    },
    "www.kaufland.sk": {
      "status": 403,
      "client": "fetch",
      "finalHost": "www.kaufland.sk",
      "body": null
    },
    "www.kaufland.at": {
      "status": 403,
      "client": "fetch",
      "finalHost": "www.kaufland.at",
      "body": null
    },
    "www.kaufland.it": {
      "status": 403,
      "client": "fetch",
      "finalHost": "www.kaufland.it",
      "body": null
    },
    "www.kaufland.fr": {
      "status": 403,
      "client": "fetch",
      "finalHost": "www.kaufland.fr",
      "body": null
    },
    "www.zalando.de": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.zalando.de",
      "body": "User-agent: Pinterestbot\nCrawl-delay: 0.2\n\nUser-agent: ClaudeBot\nUser-agent: AI2Bot\nUser-agent: Bytespider\nUser-agent: GPTBot\nUser-agent: Google-Extended\nUser-agent: Applebot-Extended\nDisallow: /\n\nUser-agent: *\nAllow: /trend-rewind/*\nDisallow: /*&*&*&*\nDisallow: /cart/*\nDisallow: /wardrobe/*\nDisallow: /myaccount/*\nDisallow: /opinions*\nAllow: /api/navigation\nAllow: /api/graphql/\nDisallow: /api/navigation/*\nDisallow: /api/*\nDisallow: /consent/api/ucversion\nDisallow: /reco-catalog/\nDisallow: /assistant\nDisallow: /checkout/partner/*\nDisallow: /*微信\nDisallow: /*=WA*08\n"
    },
    "www.zalando.fr": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.zalando.fr",
      "body": "User-agent: Pinterestbot\nCrawl-delay: 0.2\n\nUser-agent: ClaudeBot\nUser-agent: AI2Bot\nUser-agent: Bytespider\nUser-agent: GPTBot\nUser-agent: Google-Extended\nUser-agent: Applebot-Extended\nDisallow: /\n\nUser-agent: *\nAllow: /trend-rewind/*\nDisallow: /*&*&*&*\nDisallow: /cart/*\nDisallow: /wardrobe/*\nDisallow: /myaccount/*\nDisallow: /opinions*\nAllow: /api/navigation\nAllow: /api/graphql/\nDisallow: /api/navigation/*\nDisallow: /api/*\nDisallow: /consent/api/ucversion\nDisallow: /reco-catalog/\nDisallow: /assistant\nDisallow: /checkout/partner/*\nDisallow: /*微信\nDisallow: /*=WA*08\n"
    },
    "www.zalando.it": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.zalando.it",
      "body": "User-agent: Pinterestbot\nCrawl-delay: 0.2\n\nUser-agent: ClaudeBot\nUser-agent: AI2Bot\nUser-agent: Bytespider\nUser-agent: GPTBot\nUser-agent: Google-Extended\nUser-agent: Applebot-Extended\nDisallow: /\n\nUser-agent: *\nAllow: /trend-rewind/*\nDisallow: /*&*&*&*\nDisallow: /cart/*\nDisallow: /wardrobe/*\nDisallow: /myaccount/*\nDisallow: /opinions*\nAllow: /api/navigation\nAllow: /api/graphql/\nDisallow: /api/navigation/*\nDisallow: /api/*\nDisallow: /consent/api/ucversion\nDisallow: /reco-catalog/\nDisallow: /assistant\nDisallow: /checkout/partner/*\nDisallow: /*微信\nDisallow: /*=WA*08\n"
    },
    "www.zalando.es": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.zalando.es",
      "body": "User-agent: Pinterestbot\nCrawl-delay: 0.2\n\nUser-agent: ClaudeBot\nUser-agent: AI2Bot\nUser-agent: Bytespider\nUser-agent: GPTBot\nUser-agent: Google-Extended\nUser-agent: Applebot-Extended\nDisallow: /\n\nUser-agent: *\nAllow: /trend-rewind/*\nDisallow: /*&*&*&*\nDisallow: /cart/*\nDisallow: /wardrobe/*\nDisallow: /myaccount/*\nDisallow: /opinions*\nAllow: /api/navigation\nAllow: /api/graphql/\nDisallow: /api/navigation/*\nDisallow: /api/*\nDisallow: /consent/api/ucversion\nDisallow: /reco-catalog/\nDisallow: /assistant\nDisallow: /checkout/partner/*\nDisallow: /*微信\nDisallow: /*=WA*08\n"
    },
    "www.zalando.pl": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.zalando.pl",
      "body": "User-agent: Pinterestbot\nCrawl-delay: 0.2\n\nUser-agent: ClaudeBot\nUser-agent: AI2Bot\nUser-agent: Bytespider\nUser-agent: GPTBot\nUser-agent: Google-Extended\nUser-agent: Applebot-Extended\nDisallow: /\n\nUser-agent: *\nAllow: /trend-rewind/*\nDisallow: /*&*&*&*\nDisallow: /cart/*\nDisallow: /wardrobe/*\nDisallow: /myaccount/*\nDisallow: /opinions*\nAllow: /api/navigation\nAllow: /api/graphql/\nDisallow: /api/navigation/*\nDisallow: /api/*\nDisallow: /consent/api/ucversion\nDisallow: /reco-catalog/\nDisallow: /assistant\nDisallow: /checkout/partner/*\nDisallow: /*微信\nDisallow: /*=WA*08\n"
    },
    "www.zalando.nl": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.zalando.nl",
      "body": "User-agent: Pinterestbot\nCrawl-delay: 0.2\n\nUser-agent: ClaudeBot\nUser-agent: AI2Bot\nUser-agent: Bytespider\nUser-agent: GPTBot\nUser-agent: Google-Extended\nUser-agent: Applebot-Extended\nDisallow: /\n\nUser-agent: *\nAllow: /trend-rewind/*\nDisallow: /*&*&*&*\nDisallow: /cart/*\nDisallow: /wardrobe/*\nDisallow: /myaccount/*\nDisallow: /opinions*\nAllow: /api/navigation\nAllow: /api/graphql/\nDisallow: /api/navigation/*\nDisallow: /api/*\nDisallow: /consent/api/ucversion\nDisallow: /reco-catalog/\nDisallow: /assistant\nDisallow: /checkout/partner/*\nDisallow: /*微信\nDisallow: /*=WA*08\n"
    },
    "www.zalando.co.uk": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.zalando.co.uk",
      "body": "User-agent: Pinterestbot\nCrawl-delay: 0.2\n\nUser-agent: ClaudeBot\nUser-agent: AI2Bot\nUser-agent: Bytespider\nUser-agent: GPTBot\nUser-agent: Google-Extended\nUser-agent: Applebot-Extended\nDisallow: /\n\nUser-agent: *\nAllow: /trend-rewind/*\nDisallow: /*&*&*&*\nDisallow: /cart/*\nDisallow: /wardrobe/*\nDisallow: /myaccount/*\nDisallow: /opinions*\nAllow: /api/navigation\nAllow: /api/graphql/\nDisallow: /api/navigation/*\nDisallow: /api/*\nDisallow: /consent/api/ucversion\nDisallow: /reco-catalog/\nDisallow: /assistant\nDisallow: /checkout/partner/*\nDisallow: /*微信\nDisallow: /*=WA*08\n"
    },
    "www.manomano.fr": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.manomano.fr",
      "body": "# www.robotstxt.org/\n# www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449\n\nUser-agent: baiduspider\nUser-agent: buenibot\nUser-agent: etaospider\nUser-agent: yandex\nDisallow: /\n\nUser-agent: AdsBot-Google\nUser-agent: AdsBot-Google-Mobile\nAllow: /co/\nDisallow: *ls_filter=false\nDisallow: *page=*\nDisallow: /*.htm\nDisallow: /api/v1/regenerate-kc-tokens\nDisallow: /cart/\nDisallow: /cdn-cgi/\nDisallow: /checkout/\nDisallow: /commerciante/\nDisallow: /files/invoices/\nDisallow: /filter/\nDisallow: /haendler/\nDisallow: /index.php/\nDisallow: /marchand/\nDisallow: /noindex/\nDisallow: /oauth/token\nDisallow: /order/\nDisallow: /polyfill.io/\nDisallow: /product/allproviders/\nDisallow: /sdk.privacy-center.org/\nDisallow: /seller/\nDisallow: /users/\nDisallow: /vendedor/\n\nUser-agent: *\nDisallow: */commande-client/\nDisallow: *?*&attributes*\nDisallow: *?*&sort=*\nDisallow: *?attributes*\nDisallow: *?sort=*\nDisallow: *ls_filter=false\nDisallow: *model_id=*&page=*\nDisallow: /*.htm\nDisallow: /api/auth/session\nDisallow: /api/v1/regenerate-kc-tokens\nDisallow: /assistance-en-ligne/\nDisallow: /assistant\nDisallow: /cart/\nDisallow: /cdn-cgi/\nDisallow: /checkout/\nDisallow: /commande/\nDisallow: /data/utep_ad.js\nDisallow: /data/yzfdmoan.js\nDisallow: /demande-de-retour/\nDisallow: /files/invoices/\nDisallow: /filter/\nDisallow: /index.php/\nDisallow: /marchand/\nDisallow: /noindex/\nDisallow: /oauth/token\nDisallow: /order/\nDisallow: /polyfill.io/\nDisallow: /pro/nos-engagements\nDisallow: /product/allproviders/\nDisallow: /recherche/\nDisallow: /sdk.privacy-center.org/\nDisallow: /se-connecter?callbackUrl*\nDisallow: /users/\nDisallow: /web/api/v1/product-page/product-cards*\n\nSitemap: https://www.manomano.fr/sitemap.xml.gz\n"
    },
    "www.manomano.es": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.manomano.es",
      "body": "# www.robotstxt.org/\n# www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449\n\nUser-agent: baiduspider\nUser-agent: buenibot\nUser-agent: etaospider\nUser-agent: yandex\nDisallow: /\n\nUser-agent: AdsBot-Google\nUser-agent: AdsBot-Google-Mobile\nAllow: /co/\nDisallow: *ls_filter=false\nDisallow: *page=*\nDisallow: /*.htm\nDisallow: /api/v1/regenerate-kc-tokens\nDisallow: /cart/\nDisallow: /cdn-cgi/\nDisallow: /checkout/\nDisallow: /commerciante/\nDisallow: /files/invoices/\nDisallow: /filter/\nDisallow: /haendler/\nDisallow: /index.php/\nDisallow: /marchand/\nDisallow: /noindex/\nDisallow: /oauth/token\nDisallow: /order/\nDisallow: /polyfill.io/\nDisallow: /product/allproviders/\nDisallow: /sdk.privacy-center.org/\nDisallow: /seller/\nDisallow: /users/\nDisallow: /vendedor/\n\nUser-agent: *\nDisallow: *?*&attributes*\nDisallow: *?*&sort=*\nDisallow: *?attributes*\nDisallow: *?sort=*\nDisallow: *ls_filter=false\nDisallow: *model_id=*&page=*\nDisallow: /*.htm\nDisallow: /api/auth/session\nDisallow: /api/v1/regenerate-kc-tokens\nDisallow: /asistencia-en-linea/\nDisallow: /asistente\nDisallow: /busqueda/\nDisallow: /cart/\nDisallow: /cdn-cgi/\nDisallow: /checkout/\nDisallow: /data/utep_ad.js\nDisallow: /data/yzfdmoan.js\nDisallow: /files/invoices/\nDisallow: /filter/\nDisallow: /identificarse?callbackUrl*\nDisallow: /index.php/\nDisallow: /mis-mensajes/\nDisallow: /noindex/\nDisallow: /oauth/token\nDisallow: /order/\nDisallow: /pedido/\nDisallow: /pedidos/\nDisallow: /polyfill.io/\nDisallow: /product/allproviders/\nDisallow: /sdk.privacy-center.org/\nDisallow: /solicitud-de-devolucion/\nDisallow: /users/\nDisallow: /vendedor/\nDisallow: /web/api/v1/product-page/product-cards*\n\nSitemap: https://www.manomano.es/sitemap.xml.gz\n"
    },
    "www.manomano.it": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.manomano.it",
      "body": "# www.robotstxt.org/\n# www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449\n\nUser-agent: baiduspider\nUser-agent: buenibot\nUser-agent: etaospider\nUser-agent: yandex\nDisallow: /\n\nUser-agent: AdsBot-Google\nUser-agent: AdsBot-Google-Mobile\nAllow: /co/\nDisallow: *ls_filter=false\nDisallow: *page=*\nDisallow: /*.htm\nDisallow: /api/v1/regenerate-kc-tokens\nDisallow: /cart/\nDisallow: /cdn-cgi/\nDisallow: /checkout/\nDisallow: /commerciante/\nDisallow: /files/invoices/\nDisallow: /filter/\nDisallow: /haendler/\nDisallow: /index.php/\nDisallow: /marchand/\nDisallow: /noindex/\nDisallow: /oauth/token\nDisallow: /order/\nDisallow: /polyfill.io/\nDisallow: /product/allproviders/\nDisallow: /sdk.privacy-center.org/\nDisallow: /seller/\nDisallow: /users/\nDisallow: /vendedor/\n\nUser-agent: *\nDisallow: *?*&attributes*\nDisallow: *?*&sort=*\nDisallow: *?attributes*\nDisallow: *?sort=*\nDisallow: *ls_filter=false\nDisallow: *model_id=*&page=*\nDisallow: /*.htm\nDisallow: /api/auth/session\nDisallow: /api/v1/regenerate-kc-tokens\nDisallow: /assistente\nDisallow: /assistenza-online/\nDisallow: /cart/\nDisallow: /cdn-cgi/\nDisallow: /checkout/\nDisallow: /commerciante/\nDisallow: /connettersi?callbackUrl*\nDisallow: /data/utep_ad.js\nDisallow: /data/yzfdmoan.js\nDisallow: /files/invoices/\nDisallow: /filter/\nDisallow: /index.php/\nDisallow: /miei-messaggi/\nDisallow: /miei-ordini/\nDisallow: /noindex/\nDisallow: /oauth/token\nDisallow: /order/\nDisallow: /ordine/\nDisallow: /polyfill.io/\nDisallow: /product/allproviders/\nDisallow: /ricerca/\nDisallow: /sdk.privacy-center.org/\nDisallow: /users/\nDisallow: /web/api/v1/product-page/product-cards*\n\nSitemap: https://www.manomano.it/sitemap.xml.gz\n"
    },
    "www.manomano.de": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.manomano.de",
      "body": "# www.robotstxt.org/\n# www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449\n\nUser-agent: baiduspider\nUser-agent: buenibot\nUser-agent: etaospider\nUser-agent: yandex\nDisallow: /\n\nUser-agent: AdsBot-Google\nUser-agent: AdsBot-Google-Mobile\nAllow: /co/\nDisallow: *ls_filter=false\nDisallow: *page=*\nDisallow: /*.htm\nDisallow: /api/v1/regenerate-kc-tokens\nDisallow: /cart/\nDisallow: /cdn-cgi/\nDisallow: /checkout/\nDisallow: /commerciante/\nDisallow: /files/invoices/\nDisallow: /filter/\nDisallow: /haendler/\nDisallow: /index.php/\nDisallow: /marchand/\nDisallow: /noindex/\nDisallow: /oauth/token\nDisallow: /order/\nDisallow: /polyfill.io/\nDisallow: /product/allproviders/\nDisallow: /sdk.privacy-center.org/\nDisallow: /seller/\nDisallow: /users/\nDisallow: /vendedor/\n\nUser-agent: *\nDisallow: *?*&attributes*\nDisallow: *?*&sort=*\nDisallow: *?attributes*\nDisallow: *?sort=*\nDisallow: *ls_filter=false\nDisallow: *model_id=*&page=*\nDisallow: /*.htm\nDisallow: /anmelden?callbackUrl*\nDisallow: /api/auth/session\nDisallow: /api/v1/regenerate-kc-tokens\nDisallow: /assistent\nDisallow: /bestellen/\nDisallow: /cart/\nDisallow: /cdn-cgi/\nDisallow: /checkout/\nDisallow: /data/utep_ad.js\nDisallow: /data/yzfdmoan.js\nDisallow: /files/invoices/\nDisallow: /filter/\nDisallow: /haendler/\nDisallow: /index.php/\nDisallow: /meine-bestellungen/\nDisallow: /meine-nachrichten/\nDisallow: /noindex/\nDisallow: /oauth/token\nDisallow: /online-assistenz/\nDisallow: /order/\nDisallow: /polyfill.io/\nDisallow: /product/allproviders/\nDisallow: /ruckforderung/\nDisallow: /sdk.privacy-center.org/\nDisallow: /suche/\nDisallow: /users/\nDisallow: /web/api/v1/product-page/product-cards*\n\nSitemap: https://www.manomano.de/sitemap.xml.gz\n"
    },
    "www.manomano.co.uk": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.manomano.co.uk",
      "body": "# www.robotstxt.org/\n# www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449\n\nUser-agent: baiduspider\nUser-agent: buenibot\nUser-agent: etaospider\nUser-agent: yandex\nDisallow: /\n\nUser-agent: AdsBot-Google\nUser-agent: AdsBot-Google-Mobile\nAllow: /co/\nDisallow: *ls_filter=false\nDisallow: *page=*\nDisallow: /*.htm\nDisallow: /api/v1/regenerate-kc-tokens\nDisallow: /cart/\nDisallow: /cdn-cgi/\nDisallow: /checkout/\nDisallow: /commerciante/\nDisallow: /files/invoices/\nDisallow: /filter/\nDisallow: /haendler/\nDisallow: /index.php/\nDisallow: /marchand/\nDisallow: /noindex/\nDisallow: /oauth/token\nDisallow: /order/\nDisallow: /polyfill.io/\nDisallow: /product/allproviders/\nDisallow: /sdk.privacy-center.org/\nDisallow: /seller/\nDisallow: /users/\nDisallow: /vendedor/\n\nUser-agent: *\nDisallow: *?*&attributes*\nDisallow: *?*&sort=*\nDisallow: *?attributes*\nDisallow: *?sort=*\nDisallow: *ls_filter=false\nDisallow: *model_id=*&page=*\nDisallow: /*.htm\nDisallow: /api/auth/session\nDisallow: /api/v1/regenerate-kc-tokens\nDisallow: /cart/\nDisallow: /cdn-cgi/\nDisallow: /checkout/\nDisallow: /data/utep_ad.js\nDisallow: /data/yzfdmoan.js\nDisallow: /files/invoices/\nDisallow: /filter/\nDisallow: /index.php/\nDisallow: /login?callbackUrl*\nDisallow: /my-messages/\nDisallow: /noindex/\nDisallow: /oauth/token\nDisallow: /online-support/\nDisallow: /order/\nDisallow: /orders/\nDisallow: /polyfill.io/\nDisallow: /product/allproviders/\nDisallow: /return-request/\nDisallow: /sdk.privacy-center.org/\nDisallow: /search/\nDisallow: /seller/\nDisallow: /users/\nDisallow: /web/api/v1/product-page/product-cards*\n\nSitemap: https://www.manomano.co.uk/sitemap.xml.gz\n"
    },
    "www.emag.ro": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.emag.ro",
      "body": "User-agent: *\nDisallow: /user/*\nDisallow: /addtocart*\nDisallow: /proces_comanda\nDisallow: /cart/*\nDisallow: /checkout\nDisallow: /imagini/*\nDisallow: /images/*\nDisallow: /compare/*\nDisallow: /mesaje/\nDisallow: /thank-you$\nDisallow: /cat-*/pid-*/ord-*/p-*\nAllow: */review/*\nDisallow: */review/add*\nDisallow: /adauga-intrebare\nDisallow: /header-cart\nDisallow: /header-wl\nDisallow: /widgets/track.json\nDisallow: /get_back_to_search_url\nDisallow: /search-tools/*\nDisallow: /ajax_live_products\nDisallow: /box_user_history\nDisallow: /configurator_auto/\nDisallow: /search/listall\nDisallow: /search/m.emag.ro\nDisallow: /product/\nDisallow: /label/*\nDisallow: /favorites/\nDisallow: /favorites$\nDisallow: /favorites/*\nAllow: /*/*/*all-products\n\nUser-agent: 008\nDisallow: /\n\nUser-agent: Yandex\nDisallow: /\n\nUser-agent: Baiduspider\nDisallow: /\n\nUser-agent: AhrefsBot\nDisallow: /\n\nUser-agent: Synthesio\nDisallow: /\n\nSitemap: https://www.emag.ro/sitemaps/category-filters-index.xml \nSitemap: https://www.emag.ro/sitemaps/categories-index.xml \nSitemap: https://www.emag.ro/sitemaps/offers-available-index.xml \nSitemap: https://www.emag.ro/sitemaps/offers-unavailable-index.xml \nSitemap: https://www.emag.ro/sitemaps/offers-latest-index.xml \nSitemap: https://www.emag.ro/sitemaps/sub-department-pages-index.xml \nSitemap: https://www.emag.ro/sitemaps/department-pages-index.xml \nSitemap: https://www.emag.ro/sitemaps/vendors-index.xml \nSitemap: https://www.emag.ro/sitemaps/brands-index.xml \nSitemap: https://www.emag.ro/sitemaps/miscellaneous-pages-index.xml \nSitemap: https://www.emag.ro/sitemaps/resealed-departments-index.xml \n"
    },
    "www.emag.bg": {
      "status": 511,
      "client": "fetch",
      "finalHost": "www.emag.bg",
      "body": null
    },
    "www.emag.hu": {
      "status": 511,
      "client": "fetch",
      "finalHost": "www.emag.hu",
      "body": null
    },
    "www.etsy.com": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.etsy.com",
      "body": "User-agent: Spinn3r\nDisallow: /\n\nUser-agent: AdsBot-Google-Mobile\nDisallow: /people\nDisallow: /uk/people\nDisallow: /au/people\nDisallow: /ca/people\nDisallow: /de-en/people\nDisallow: /dk-en/people\nDisallow: /fi-en/people\nDisallow: /hk-en/people\nDisallow: /ie/people\nDisallow: /il-en/people\nDisallow: /in-en/people\nDisallow: /no-en/people\nDisallow: /nz/people\nDisallow: /se-en/people\nDisallow: /sg-en/people\nDisallow: /es/people\nDisallow: /mx/people\nDisallow: /us-es/people\nDisallow: /de/people\nDisallow: /at/people\nDisallow: /ch/people\nDisallow: /fr/people\nDisallow: /ca-fr/people\nDisallow: /it/people\nDisallow: /nl/people\nDisallow: /be/people\nDisallow: /jp/people\nDisallow: /pl/people\nDisallow: /pt/people\nDisallow: /se/people\n\nUser-agent: *\nDisallow: /api/\nDisallow: /your/\nDisallow: /r/group/\nDisallow: /appeals/\nDisallow: /transactions/\nDisallow: /treasury\nDisallow: /teams\nDisallow: /cart\nDisallow: /conversations\nDisallow: /conversations/\nDisallow: /locale/overlay\nDisallow: /locale/overlay_secure\nDisallow: /guest-pass/\nDisallow: /mailinglist/\nDisallow: /mini.php\nDisallow: /invites/buy\nDisallow: /share\nDisallow: /bcn\nDisallow: /listing/*/stubs/*\nDisallow: /market/confederate_flag\nDisallow: /market/rebel_flag\nDisallow: /market/dixie_flag\nDisallow: /help/sitemap.php\nDisallow: /help/sitemap/*\nDisallow: /*?order=*\nDisallow: /*handmade/\nDisallow: /*color=*\nDisallow: /*my_etsy=*\nDisallow: /*gift_card=*\nDisallow: /*customizable=*\nDisallow: /*locationQuery=*\nDisallow: /*min=*\nDisallow: /*max=*\nDisallow: */people/*/circle*\nDisallow: */people/*/favorites*\nDisallow: */shop/*/sold*\nDisallow: */shop/*/favoriters*\nDisallow: */listing/*/favoriters*\nDisallow: /search*attr_*=*\nDisallow: /c/*attr_*=*\nDisallow: /c?*attr_*=*\nDisallow: /search*price_bucket=*\nDisallow: /c/*price_bucket=*\nDisallow: /search*ship_to=*\nDisallow: /c/*ship_to=*\nDisallow: /search*search_type=*\nDisallow: /c/*search_type=*\nDisallow: /shop_name_search_service\nDisallow: /messages\nDisallow: /apps\nDisallow: /search?*q=\nDisallow: /search/?*q=\nDisallow: /search/*?*q=\nDisallow: /search/*/?*q=\nDisallow: */shop/*/rss_shop.php\nDisallow: /uk/api/\nDisallow: /au/api/\nDisallow: /ca/api/\nDisallow: /de-en/api/\nDisallow: /dk-en/api/\nDisallow: /fi-en/api/\nDisallow: /hk-en/api/\nDisallow: /ie/api/\nDisallow: /il-en/api/\nDisallow: /in-en/api/\nDisallow: /no-en/api/\nDisallow: /nz/api/\nDisallow: /se-en/api/\nDisallow: /sg-en/api/\nDisallow: /es/api/\nDisallow: /mx/api/\nDisallow: /us-es/api/\nDisallow: /de/api/\nDisallow: /at/api/\nDisallow: /ch/api/\nDisallow: /fr/api/\nDisallow: /ca-fr/api/\nDisallow: /it/api/\nDisallow: /nl/api/\nDisallow: /be/api/\nDisallow: /jp/api/\nDisallow: /pl/api/\nDisallow: /pt/api/\nDisallow: /se/api/\nDisallow: /uk/your/\nDisallow: /au/your/\nDisallow: /ca/your/\nDisallow: /de-en/your/\nDisallow: /dk-en/your/\nDisallow: /fi-en/your/\nDisallow: /hk-en/your/\nDisallow: /ie/your/\nDisallow: /il-en/your/\nDisallow: /in-en/your/\nDisallow: /no-en/your/\nDisallow: /nz/your/\nDisallow: /se-en/your/\nDisallow: /sg-en/your/\nDisallow: /es/your/\nDisallow: /mx/your/\nDisallow: /us-es/your/\nDisallow: /de/your/\nDisallow: /at/your/\nDisallow: /ch/your/\nDisallow: /fr/your/\nDisallow: /ca-fr/your/\nDisallow: /it/your/\nDisallow: /nl/your/\nDisallow: /be/your/\nDisallow: /jp/your/\nDisallow: /pl/your/\nDisallow: /pt/your/\nDisallow: /se/your/\nDisallow: /uk/r/group/\nDisallow: /au/r/group/\nDisallow: /ca/r/group/\nDisallow: /de-en/r/group/\nDisallow: /dk-en/r/group/\nDisallow: /fi-en/r/group/\nDisallow: /hk-en/r/group/\nDisallow: /ie/r/group/\nDisallow: /il-en/r/group/\nDisallow: /in-en/r/group/\nDisallow: /no-en/r/group/\nDisallow: /nz/r/group/\nDisallow: /se-en/r/group/\nDisallow: /sg-en/r/group/\nDisallow: /es/r/group/\nDisallow: /mx/r/group/\nDisallow: /us-es/r/group/\nDisallow: /de/r/group/\nDisallow: /at/r/group/\nDisallow: /ch/r/group/\nDisallow: /fr/r/group/\nDisallow: /ca-fr/r/group/\nDisallow: /it/r/group/\nDisallow: /nl/r/group/\nDisallow: /be/r/group/\nDisallow: /jp/r/group/\nDisallow: /pl/r/group/\nDisallow: /pt/r/group/\nDisallow: /se/r/group/\nDisallow: /uk/appeals/\nDisallow: /au/appeals/\nDisallow: /ca/appeals/\nDisallow: /de-en/appeals/\nDisallow: /dk-en/appeals/\nDisallow: /fi-en/appeals/\nDisallow: /hk-en/appeals/\nDisallow: /ie/appeals/\nDisallow: /il-en/appeals/\nDisallow: /in-en/appeals/\nDisallow: /no-en/appeals/\nDisallow: /nz/appeals/\nDisallow: /se-en/appeals/\nDisallow: /sg-en/appeals/\nDisallow: /es/appeals/\nDisallow: /mx/appeals/\nDisallow: /us-es/appeals/\nDisallow: /de/appeals/\nDisallow: /at/appeals/\nDisallow: /ch/appeals/\nDisallow: /fr/appeals/\nDisallow: /ca-fr/appeals/\nDisallow: /it/appeals/\nDisallow: /nl/appeals/\nDisallow: /be/appeals/\nDisallow: /jp/appeals/\nDisallow: /pl/appeals/\nDisallow: /pt/appeals/\nDisallow: /se/appeals/\nDisallow: /uk/transactions/\nDisallow: /au/transactions/\nDisallow: /ca/transactions/\nDisallow: /de-en/transactions/\nDisallow: /dk-en/transactions/\nDisallow: /fi-en/transactions/\nDisallow: /hk-en/transactions/\nDisallow: /ie/transactions/\nDisallow: /il-en/transactions/\nDisallow: /in-en/transactions/\nDisallow: /no-en/transactions/\nDisallow: /nz/transactions/\nDisallow: /se-en/transactions/\nDisallow: /sg-en/transactions/\nDisallow: /es/transactions/\nDisallow: /mx/transactions/\nDisallow: /us-es/transactions/\nDisallow: /de/transactions/\nDisallow: /at/transactions/\nDisallow: /ch/transactions/\nDisallow: /fr/transactions/\nDisallow: /ca-fr/transactions/\nDisallow: /it/transactions/\nDisallow: /nl/transactions/\nDisallow: /be/transactions/\nDisallow: /jp/transactions/\nDisallow: /pl/transactions/\nDisallow: /pt/transactions/\nDisallow: /se/transactions/\nDisallow: /uk/treasury\nDisallow: /au/treasury\nDisallow: /ca/treasury\nDisallow: /de-en/treasury\nDisallow: /dk-en/treasury\nDisallow: /fi-en/treasury\nDisallow: /hk-en/treasury\nDisallow: /ie/treasury\nDisallow: /il-en/treasury\nDisallow: /in-en/treasury\nDisallow: /no-en/treasury\nDisallow: /nz/treasury\nDisallow: /se-en/treasury\nDisallow: /sg-en/treasury\nDisallow: /es/treasury\nDisallow: /mx/treasury\nDisallow: /us-es/treasury\nDisallow: /de/treasury\nDisallow: /at/treasury\nDisallow: /ch/treasury\nDisallow: /fr/treasury\nDisallow: /ca-fr/treasury\nDisallow: /it/treasury\nDisallow: /nl/treasury\nDisallow: /be/treasury\nDisallow: /jp/treasury\nDisallow: /pl/treasury\nDisallow: /pt/treasury\nDisallow: /se/treasury\nDisallow: /uk/teams\nDisallow: /au/teams\nDisallow: /ca/teams\nDisallow: /de-en/teams\nDisallow: /dk-en/teams\nDisallow: /fi-en/teams\nDisallow: /hk-en/teams\nDisallow: /ie/teams\nDisallow: /il-en/teams\nDisallow: /in-en/teams\nDisallow: /no-en/teams\nDisallow: /nz/teams\nDisallow: /se-en/teams\nDisallow: /sg-en/teams\nDisallow: /es/teams\nDisallow: /mx/teams\nDisallow: /us-es/teams\nDisallow: /de/teams\nDisallow: /at/teams\nDisallow: /ch/teams\nDisallow: /fr/teams\nDisallow: /ca-fr/teams\nDisallow: /it/teams\nDisallow: /nl/teams\nDisallow: /be/teams\nDisallow: /jp/teams\nDisallow: /pl/teams\nDisallow: /pt/teams\nDisallow: /se/teams\nDisallow: /uk/cart\nDisallow: /au/cart\nDisallow: /ca/cart\nDisallow: /de-en/cart\nDisallow: /dk-en/cart\nDisallow: /fi-en/cart\nDisallow: /hk-en/cart\nDisallow: /ie/cart\nDisallow: /il-en/cart\nDisallow: /in-en/cart\nDisallow: /no-en/cart\nDisallow: /nz/cart\nDisallow: /se-en/cart\nDisallow: /sg-en/cart\nDisallow: /es/cart\nDisallow: /mx/cart\nDisallow: /us-es/cart\nDisallow: /de/cart\nDisallow: /at/cart\nDisallow: /ch/cart\nDisallow: /fr/cart\nDisallow: /ca-fr/cart\nDisallow: /it/cart\nDisallow: /nl/cart\nDisallow: /be/cart\nDisallow: /jp/cart\nDisallow: /pl/cart\nDisallow: /pt/cart\nDisallow: /se/cart\nDisallow: /uk/conversations\nDisallow: /au/conversations\nDisallow: /ca/conversations\nDisallow: /de-en/conversations\nDisallow: /dk-en/conversations\nDisallow: /fi-en/conversations\nDisallow: /hk-en/conversations\nDisallow: /ie/conversations\nDisallow: /il-en/conversations\nDisallow: /in-en/conversations\nDisallow: /no-en/conversations\nDisallow: /nz/conversations\nDisallow: /se-en/conversations\nDisallow: /sg-en/conversations\nDisallow: /es/conversations\nDisallow: /mx/conversations\nDisallow: /us-es/conversations\nDisallow: /de/conversations\nDisallow: /at/conversations\nDisallow: /ch/conversations\nDisallow: /fr/conversations\nDisallow: /ca-fr/conversations\nDisallow: /it/conversations\nDisallow: /nl/conversations\nDisallow: /be/conversations\nDisallow: /jp/conversations\nDisallow: /pl/conversations\nDisallow: /pt/conversations\nDisallow: /se/conversations\nDisallow: /uk/conversations/\nDisallow: /au/conversations/\nDisallow: /ca/conversations/\nDisallow: /de-en/conversations/\nDisallow: /dk-en/conversations/\nDisallow: /fi-en/conversations/\nDisallow: /hk-en/conversations/\nDisallow: /ie/conversations/\nDisallow: /il-en/conversations/\nDisallow: /in-en/conversations/\nDisallow: /no-en/conversations/\nDisallow: /nz/conversations/\nDisallow: /se-en/conversations/\nDisallow: /sg-en/conversations/\nDisallow: /es/conversations/\nDisallow: /mx/conversations/\nDisallow: /us-es/conversations/\nDisallow: /de/conversations/\nDisallow: /at/conversations/\nDisallow: /ch/conversations/\nDisallow: /fr/conversations/\nDisallow: /ca-fr/conversations/\nDisallow: /it/conversations/\nDisallow: /nl/conversations/\nDisallow: /be/conversations/\nDisallow: /jp/conversations/\nDisallow: /pl/conversations/\nDisallow: /pt/conversations/\nDisallow: /se/conversations/\nDisallow: /uk/locale/overlay\nDisallow: /au/locale/overlay\nDisallow: /ca/locale/overlay\nDisallow: /de-en/locale/overlay\nDisallow: /dk-en/locale/overlay\nDisallow: /fi-en/locale/overlay\nDisallow: /hk-en/locale/overlay\nDisallow: /ie/locale/overlay\nDisallow: /il-en/locale/overlay\nDisallow: /in-en/locale/overlay\nDisallow: /no-en/locale/overlay\nDisallow: /nz/locale/overlay\nDisallow: /se-en/locale/overlay\nDisallow: /sg-en/locale/overlay\nDisallow: /es/locale/overlay\nDisallow: /mx/locale/overlay\nDisallow: /us-es/locale/overlay\nDisallow: /de/locale/overlay\nDisallow: /at/locale/overlay\nDisallow: /ch/locale/overlay\nDisallow: /fr/locale/overlay\nDisallow: /ca-fr/locale/overlay\nDisallow: /it/locale/overlay\nDisallow: /nl/locale/overlay\nDisallow: /be/locale/overlay\nDisallow: /jp/locale/overlay\nDisallow: /pl/locale/overlay\nDisallow: /pt/locale/overlay\nDisallow: /se/locale/overlay\nDisallow: /uk/locale/overlay_secure\nDisallow: /au/locale/overlay_secure\nDisallow: /ca/locale/overlay_secure\nDisallow: /de-en/locale/overlay_secure\nDisallow: /dk-en/locale/overlay_secure\nDisallow: /fi-en/locale/overlay_secure\nDisallow: /hk-en/locale/overlay_secure\nDisallow: /ie/locale/overlay_secure\nDisallow: /il-en/locale/overlay_secure\nDisallow: /in-en/locale/overlay_secure\nDisallow: /no-en/locale/overlay_secure\nDisallow: /nz/locale/overlay_secure\nDisallow: /se-en/locale/overlay_secure\nDisallow: /sg-en/locale/overlay_secure\nDisallow: /es/locale/overlay_secure\nDisallow: /mx/locale/overlay_secure\nDisallow: /us-es/locale/overlay_secure\nDisallow: /de/locale/overlay_secure\nDisallow: /at/locale/overlay_secure\nDisallow: /ch/locale/overlay_secure\nDisallow: /fr/locale/overlay_secure\nDisallow: /ca-fr/locale/overlay_secure\nDisallow: /it/locale/overlay_secure\nDisallow: /nl/locale/overlay_secure\nDisallow: /be/locale/overlay_secure\nDisallow: /jp/locale/overlay_secure\nDisallow: /pl/locale/overlay_secure\nDisallow: /pt/locale/overlay_secure\nDisallow: /se/locale/overlay_secure\nDisallow: /uk/guest-pass/\nDisallow: /au/guest-pass/\nDisallow: /ca/guest-pass/\nDisallow: /de-en/guest-pass/\nDisallow: /dk-en/guest-pass/\nDisallow: /fi-en/guest-pass/\nDisallow: /hk-en/guest-pass/\nDisallow: /ie/guest-pass/\nDisallow: /il-en/guest-pass/\nDisallow: /in-en/guest-pass/\nDisallow: /no-en/guest-pass/\nDisallow: /nz/guest-pass/\nDisallow: /se-en/guest-pass/\nDisallow: /sg-en/guest-pass/\nDisallow: /es/guest-pass/\nDisallow: /mx/guest-pass/\nDisallow: /us-es/guest-pass/\nDisallow: /de/guest-pass/\nDisallow: /at/guest-pass/\nDisallow: /ch/guest-pass/\nDisallow: /fr/guest-pass/\nDisallow: /ca-fr/guest-pass/\nDisallow: /it/guest-pass/\nDisallow: /nl/guest-pass/\nDisallow: /be/guest-pass/\nDisallow: /jp/guest-pass/\nDisallow: /pl/guest-pass/\nDisallow: /pt/guest-pass/\nDisallow: /se/guest-pass/\nDisallow: /uk/mailinglist/\nDisallow: /au/mailinglist/\nDisallow: /ca/mailinglist/\nDisallow: /de-en/mailinglist/\nDisallow: /dk-en/mailinglist/\nDisallow: /fi-en/mailinglist/\nDisallow: /hk-en/mailinglist/\nDisallow: /ie/mailinglist/\nDisallow: /il-en/mailinglist/\nDisallow: /in-en/mailinglist/\nDisallow: /no-en/mailinglist/\nDisallow: /nz/mailinglist/\nDisallow: /se-en/mailinglist/\nDisallow: /sg-en/mailinglist/\nDisallow: /es/mailinglist/\nDisallow: /mx/mailinglist/\nDisallow: /us-es/mailinglist/\nDisallow: /de/mailinglist/\nDisallow: /at/mailinglist/\nDisallow: /ch/mailinglist/\nDisallow: /fr/mailinglist/\nDisallow: /ca-fr/mailinglist/\nDisallow: /it/mailinglist/\nDisallow: /nl/mailinglist/\nDisallow: /be/mailinglist/\nDisallow: /jp/mailinglist/\nDisallow: /pl/mailinglist/\nDisallow: /pt/mailinglist/\nDisallow: /se/mailinglist/\nDisallow: /uk/mini.php\nDisallow: /au/mini.php\nDisallow: /ca/mini.php\nDisallow: /de-en/mini.php\nDisallow: /dk-en/mini.php\nDisallow: /fi-en/mini.php\nDisallow: /hk-en/mini.php\nDisallow: /ie/mini.php\nDisallow: /il-en/mini.php\nDisallow: /in-en/mini.php\nDisallow: /no-en/mini.php\nDisallow: /nz/mini.php\nDisallow: /se-en/mini.php\nDisallow: /sg-en/mini.php\nDisallow: /es/mini.php\nDisallow: /mx/mini.php\nDisallow: /us-es/mini.php\nDisallow: /de/mini.php\nDisallow: /at/mini.php\nDisallow: /ch/mini.php\nDisallow: /fr/mini.php\nDisallow: /ca-fr/mini.php\nDisallow: /it/mini.php\nDisallow: /nl/mini.php\nDisallow: /be/mini.php\nDisallow: /jp/mini.php\nDisallow: /pl/mini.php\nDisallow: /pt/mini.php\nDisallow: /se/mini.php\nDisallow: /uk/invites/buy\nDisallow: /au/invites/buy\nDisallow: /ca/invites/buy\nDisallow: /de-en/invites/buy\nDisallow: /dk-en/invites/buy\nDisallow: /fi-en/invites/buy\nDisallow: /hk-en/invites/buy\nDisallow: /ie/invites/buy\nDisallow: /il-en/invites/buy\nDisallow: /in-en/invites/buy\nDisallow: /no-en/invites/buy\nDisallow: /nz/invites/buy\nDisallow: /se-en/invites/buy\nDisallow: /sg-en/invites/buy\nDisallow: /es/invites/buy\nDisallow: /mx/invites/buy\nDisallow: /us-es/invites/buy\nDisallow: /de/invites/buy\nDisallow: /at/invites/buy\nDisallow: /ch/invites/buy\nDisallow: /fr/invites/buy\nDisallow: /ca-fr/invites/buy\nDisallow: /it/invites/buy\nDisallow: /nl/invites/buy\nDisallow: /be/invites/buy\nDisallow: /jp/invites/buy\nDisallow: /pl/invites/buy\nDisallow: /pt/invites/buy\nDisallow: /se/invites/buy\nDisallow: /uk/share\nDisallow: /au/share\nDisallow: /ca/share\nDisallow: /de-en/share\nDisallow: /dk-en/share\nDisallow: /fi-en/share\nDisallow: /hk-en/share\nDisallow: /ie/share\nDisallow: /il-en/share\nDisallow: /in-en/share\nDisallow: /no-en/share\nDisallow: /nz/share\nDisallow: /se-en/share\nDisallow: /sg-en/share\nDisallow: /es/share\nDisallow: /mx/share\nDisallow: /us-es/share\nDisallow: /de/share\nDisallow: /at/share\nDisallow: /ch/share\nDisallow: /fr/share\nDisallow: /ca-fr/share\nDisallow: /it/share\nDisallow: /nl/share\nDisallow: /be/share\nDisallow: /jp/share\nDisallow: /pl/share\nDisallow: /pt/share\nDisallow: /se/share\nDisallow: /uk/bcn\nDisallow: /au/bcn\nDisallow: /ca/bcn\nDisallow: /de-en/bcn\nDisallow: /dk-en/bcn\nDisallow: /fi-en/bcn\nDisallow: /hk-en/bcn\nDisallow: /ie/bcn\nDisallow: /il-en/bcn\nDisallow: /in-en/bcn\nDisallow: /no-en/bcn\nDisallow: /nz/bcn\nDisallow: /se-en/bcn\nDisallow: /sg-en/bcn\nDisallow: /es/bcn\nDisallow: /mx/bcn\nDisallow: /us-es/bcn\nDisallow: /de/bcn\nDisallow: /at/bcn\nDisallow: /ch/bcn\nDisallow: /fr/bcn\nDisallow: /ca-fr/bcn\nDisallow: /it/bcn\nDisallow: /nl/bcn\nDisallow: /be/bcn\nDisallow: /jp/bcn\nDisallow: /pl/bcn\nDisallow: /pt/bcn\nDisallow: /se/bcn\nDisallow: /uk/listing/*/stubs/*\nDisallow: /au/listing/*/stubs/*\nDisallow: /ca/listing/*/stubs/*\nDisallow: /de-en/listing/*/stubs/*\nDisallow: /dk-en/listing/*/stubs/*\nDisallow: /fi-en/listing/*/stubs/*\nDisallow: /hk-en/listing/*/stubs/*\nDisallow: /ie/listing/*/stubs/*\nDisallow: /il-en/listing/*/stubs/*\nDisallow: /in-en/listing/*/stubs/*\nDisallow: /no-en/listing/*/stubs/*\nDisallow: /nz/listing/*/stubs/*\nDisallow: /se-en/listing/*/stubs/*\nDisallow: /sg-en/listing/*/stubs/*\nDisallow: /es/listing/*/stubs/*\nDisallow: /mx/listing/*/stubs/*\nDisallow: /us-es/listing/*/stubs/*\nDisallow: /de/listing/*/stubs/*\nDisallow: /at/listing/*/stubs/*\nDisallow: /ch/listing/*/stubs/*\nDisallow: /fr/listing/*/stubs/*\nDisallow: /ca-fr/listing/*/stubs/*\nDisallow: /it/listing/*/stubs/*\nDisallow: /nl/listing/*/stubs/*\nDisallow: /be/listing/*/stubs/*\nDisallow: /jp/listing/*/stubs/*\nDisallow: /pl/listing/*/stubs/*\nDisallow: /pt/listing/*/stubs/*\nDisallow: /se/listing/*/stubs/*\nDisallow: /uk/market/confederate_flag\nDisallow: /au/market/confederate_flag\nDisallow: /ca/market/confederate_flag\nDisallow: /de-en/market/confederate_flag\nDisallow: /dk-en/market/confederate_flag\nDisallow: /fi-en/market/confederate_flag\nDisallow: /hk-en/market/confederate_flag\nDisallow: /ie/market/confederate_flag\nDisallow: /il-en/market/confederate_flag\nDisallow: /in-en/market/confederate_flag\nDisallow: /no-en/market/confederate_flag\nDisallow: /nz/market/confederate_flag\nDisallow: /se-en/market/confederate_flag\nDisallow: /sg-en/market/confederate_flag\nDisallow: /es/market/confederate_flag\nDisallow: /mx/market/confederate_flag\nDisallow: /us-es/market/confederate_flag\nDisallow: /de/market/confederate_flag\nDisallow: /at/market/confederate_flag\nDisallow: /ch/market/confederate_flag\nDisallow: /fr/market/confederate_flag\nDisallow: /ca-fr/market/confederate_flag\nDisallow: /it/market/confederate_flag\nDisallow: /nl/market/confederate_flag\nDisallow: /be/market/confederate_flag\nDisallow: /jp/market/confederate_flag\nDisallow: /pl/market/confederate_flag\nDisallow: /pt/market/confederate_flag\nDisallow: /se/market/confederate_flag\nDisallow: /uk/market/rebel_flag\nDisallow: /au/market/rebel_flag\nDisallow: /ca/market/rebel_flag\nDisallow: /de-en/market/rebel_flag\nDisallow: /dk-en/market/rebel_flag\nDisallow: /fi-en/market/rebel_flag\nDisallow: /hk-en/market/rebel_flag\nDisallow: /ie/market/rebel_flag\nDisallow: /il-en/market/rebel_flag\nDisallow: /in-en/market/rebel_flag\nDisallow: /no-en/market/rebel_flag\nDisallow: /nz/market/rebel_flag\nDisallow: /se-en/market/rebel_flag\nDisallow: /sg-en/market/rebel_flag\nDisallow: /es/market/rebel_flag\nDisallow: /mx/market/rebel_flag\nDisallow: /us-es/market/rebel_flag\nDisallow: /de/market/rebel_flag\nDisallow: /at/market/rebel_flag\nDisallow: /ch/market/rebel_flag\nDisallow: /fr/market/rebel_flag\nDisallow: /ca-fr/market/rebel_flag\nDisallow: /it/market/rebel_flag\nDisallow: /nl/market/rebel_flag\nDisallow: /be/market/rebel_flag\nDisallow: /jp/market/rebel_flag\nDisallow: /pl/market/rebel_flag\nDisallow: /pt/market/rebel_flag\nDisallow: /se/market/rebel_flag\nDisallow: /uk/market/dixie_flag\nDisallow: /au/market/dixie_flag\nDisallow: /ca/market/dixie_flag\nDisallow: /de-en/market/dixie_flag\nDisallow: /dk-en/market/dixie_flag\nDisallow: /fi-en/market/dixie_flag\nDisallow: /hk-en/market/dixie_flag\nDisallow: /ie/market/dixie_flag\nDisallow: /il-en/market/dixie_flag\nDisallow: /in-en/market/dixie_flag\nDisallow: /no-en/market/dixie_flag\nDisallow: /nz/market/dixie_flag\nDisallow: /se-en/market/dixie_flag\nDisallow: /sg-en/market/dixie_flag\nDisallow: /es/market/dixie_flag\nDisallow: /mx/market/dixie_flag\nDisallow: /us-es/market/dixie_flag\nDisallow: /de/market/dixie_flag\nDisallow: /at/market/dixie_flag\nDisallow: /ch/market/dixie_flag\nDisallow: /fr/market/dixie_flag\nDisallow: /ca-fr/market/dixie_flag\nDisallow: /it/market/dixie_flag\nDisallow: /nl/market/dixie_flag\nDisallow: /be/market/dixie_flag\nDisallow: /jp/market/dixie_flag\nDisallow: /pl/market/dixie_flag\nDisallow: /pt/market/dixie_flag\nDisallow: /se/market/dixie_flag\nDisallow: /uk/help/sitemap.php\nDisallow: /au/help/sitemap.php\nDisallow: /ca/help/sitemap.php\nDisallow: /de-en/help/sitemap.php\nDisallow: /dk-en/help/sitemap.php\nDisallow: /fi-en/help/sitemap.php\nDisallow: /hk-en/help/sitemap.php\nDisallow: /ie/help/sitemap.php\nDisallow: /il-en/help/sitemap.php\nDisallow: /in-en/help/sitemap.php\nDisallow: /no-en/help/sitemap.php\nDisallow: /nz/help/sitemap.php\nDisallow: /se-en/help/sitemap.php\nDisallow: /sg-en/help/sitemap.php\nDisallow: /es/help/sitemap.php\nDisallow: /mx/help/sitemap.php\nDisallow: /us-es/help/sitemap.php\nDisallow: /de/help/sitemap.php\nDisallow: /at/help/sitemap.php\nDisallow: /ch/help/sitemap.php\nDisallow: /fr/help/sitemap.php\nDisallow: /ca-fr/help/sitemap.php\nDisallow: /it/help/sitemap.php\nDisallow: /nl/help/sitemap.php\nDisallow: /be/help/sitemap.php\nDisallow: /jp/help/sitemap.php\nDisallow: /pl/help/sitemap.php\nDisallow: /pt/help/sitemap.php\nDisallow: /se/help/sitemap.php\nDisallow: /uk/help/sitemap/*\nDisallow: /au/help/sitemap/*\nDisallow: /ca/help/sitemap/*\nDisallow: /de-en/help/sitemap/*\nDisallow: /dk-en/help/sitemap/*\nDisallow: /fi-en/help/sitemap/*\nDisallow: /hk-en/help/sitemap/*\nDisallow: /ie/help/sitemap/*\nDisallow: /il-en/help/sitemap/*\nDisallow: /in-en/help/sitemap/*\nDisallow: /no-en/help/sitemap/*\nDisallow: /nz/help/sitemap/*\nDisallow: /se-en/help/sitemap/*\nDisallow: /sg-en/help/sitemap/*\nDisallow: /es/help/sitemap/*\nDisallow: /mx/help/sitemap/*\nDisallow: /us-es/help/sitemap/*\nDisallow: /de/help/sitemap/*\nDisallow: /at/help/sitemap/*\nDisallow: /ch/help/sitemap/*\nDisallow: /fr/help/sitemap/*\nDisallow: /ca-fr/help/sitemap/*\nDisallow: /it/help/sitemap/*\nDisallow: /nl/help/sitemap/*\nDisallow: /be/help/sitemap/*\nDisallow: /jp/help/sitemap/*\nDisallow: /pl/help/sitemap/*\nDisallow: /pt/help/sitemap/*\nDisallow: /se/help/sitemap/*\nDisallow: /uk/*?order=*\nDisallow: /au/*?order=*\nDisallow: /ca/*?order=*\nDisallow: /de-en/*?order=*\nDisallow: /dk-en/*?order=*\nDisallow: /fi-en/*?order=*\nDisallow: /hk-en/*?order=*\nDisallow: /ie/*?order=*\nDisallow: /il-en/*?order=*\nDisallow: /in-en/*?order=*\nDisallow: /no-en/*?order=*\nDisallow: /nz/*?order=*\nDisallow: /se-en/*?order=*\nDisallow: /sg-en/*?order=*\nDisallow: /es/*?order=*\nDisallow: /mx/*?order=*\nDisallow: /us-es/*?order=*\nDisallow: /de/*?order=*\nDisallow: /at/*?order=*\nDisallow: /ch/*?order=*\nDisallow: /fr/*?order=*\nDisallow: /ca-fr/*?order=*\nDisallow: /it/*?order=*\nDisallow: /nl/*?order=*\nDisallow: /be/*?order=*\nDisallow: /jp/*?order=*\nDisallow: /pl/*?order=*\nDisallow: /pt/*?order=*\nDisallow: /se/*?order=*\nDisallow: /uk/*handmade/\nDisallow: /au/*handmade/\nDisallow: /ca/*handmade/\nDisallow: /de-en/*handmade/\nDisallow: /dk-en/*handmade/\nDisallow: /fi-en/*handmade/\nDisallow: /hk-en/*handmade/\nDisallow: /ie/*handmade/\nDisallow: /il-en/*handmade/\nDisallow: /in-en/*handmade/\nDisallow: /no-en/*handmade/\nDisallow: /nz/*handmade/\nDisallow: /se-en/*handmade/\nDisallow: /sg-en/*handmade/\nDisallow: /es/*handmade/\nDisallow: /mx/*handmade/\nDisallow: /us-es/*handmade/\nDisallow: /de/*handmade/\nDisallow: /at/*handmade/\nDisallow: /ch/*handmade/\nDisallow: /fr/*handmade/\nDisallow: /ca-fr/*handmade/\nDisallow: /it/*handmade/\nDisallow: /nl/*handmade/\nDisallow: /be/*handmade/\nDisallow: /jp/*handmade/\nDisallow: /pl/*handmade/\nDisallow: /pt/*handmade/\nDisallow: /se/*handmade/\nDisallow: /uk/*color=*\nDisallow: /au/*color=*\nDisallow: /ca/*color=*\nDisallow: /de-en/*color=*\nDisallow: /dk-en/*color=*\nDisallow: /fi-en/*color=*\nDisallow: /hk-en/*color=*\nDisallow: /ie/*color=*\nDisallow: /il-en/*color=*\nDisallow: /in-en/*color=*\nDisallow: /no-en/*color=*\nDisallow: /nz/*color=*\nDisallow: /se-en/*color=*\nDisallow: /sg-en/*color=*\nDisallow: /es/*color=*\nDisallow: /mx/*color=*\nDisallow: /us-es/*color=*\nDisallow: /de/*color=*\nDisallow: /at/*color=*\nDisallow: /ch/*color=*\nDisallow: /fr/*color=*\nDisallow: /ca-fr/*color=*\nDisallow: /it/*color=*\nDisallow: /nl/*color=*\nDisallow: /be/*color=*\nDisallow: /jp/*color=*\nDisallow: /pl/*color=*\nDisallow: /pt/*color=*\nDisallow: /se/*color=*\nDisallow: /uk/*my_etsy=*\nDisallow: /au/*my_etsy=*\nDisallow: /ca/*my_etsy=*\nDisallow: /de-en/*my_etsy=*\nDisallow: /dk-en/*my_etsy=*\nDisallow: /fi-en/*my_etsy=*\nDisallow: /hk-en/*my_etsy=*\nDisallow: /ie/*my_etsy=*\nDisallow: /il-en/*my_etsy=*\nDisallow: /in-en/*my_etsy=*\nDisallow: /no-en/*my_etsy=*\nDisallow: /nz/*my_etsy=*\nDisallow: /se-en/*my_etsy=*\nDisallow: /sg-en/*my_etsy=*\nDisallow: /es/*my_etsy=*\nDisallow: /mx/*my_etsy=*\nDisallow: /us-es/*my_etsy=*\nDisallow: /de/*my_etsy=*\nDisallow: /at/*my_etsy=*\nDisallow: /ch/*my_etsy=*\nDisallow: /fr/*my_etsy=*\nDisallow: /ca-fr/*my_etsy=*\nDisallow: /it/*my_etsy=*\nDisallow: /nl/*my_etsy=*\nDisallow: /be/*my_etsy=*\nDisallow: /jp/*my_etsy=*\nDisallow: /pl/*my_etsy=*\nDisallow: /pt/*my_etsy=*\nDisallow: /se/*my_etsy=*\nDisallow: /uk/*gift_card=*\nDisallow: /au/*gift_card=*\nDisallow: /ca/*gift_card=*\nDisallow: /de-en/*gift_card=*\nDisallow: /dk-en/*gift_card=*\nDisallow: /fi-en/*gift_card=*\nDisallow: /hk-en/*gift_card=*\nDisallow: /ie/*gift_card=*\nDisallow: /il-en/*gift_card=*\nDisallow: /in-en/*gift_card=*\nDisallow: /no-en/*gift_card=*\nDisallow: /nz/*gift_card=*\nDisallow: /se-en/*gift_card=*\nDisallow: /sg-en/*gift_card=*\nDisallow: /es/*gift_card=*\nDisallow: /mx/*gift_card=*\nDisallow: /us-es/*gift_card=*\nDisallow: /de/*gift_card=*\nDisallow: /at/*gift_card=*\nDisallow: /ch/*gift_card=*\nDisallow: /fr/*gift_card=*\nDisallow: /ca-fr/*gift_card=*\nDisallow: /it/*gift_card=*\nDisallow: /nl/*gift_card=*\nDisallow: /be/*gift_card=*\nDisallow: /jp/*gift_card=*\nDisallow: /pl/*gift_card=*\nDisallow: /pt/*gift_card=*\nDisallow: /se/*gift_card=*\nDisallow: /uk/*customizable=*\nDisallow: /au/*customizable=*\nDisallow: /ca/*customizable=*\nDisallow: /de-en/*customizable=*\nDisallow: /dk-en/*customizable=*\nDisallow: /fi-en/*customizable=*\nDisallow: /hk-en/*customizable=*\nDisallow: /ie/*customizable=*\nDisallow: /il-en/*customizable=*\nDisallow: /in-en/*customizable=*\nDisallow: /no-en/*customizable=*\nDisallow: /nz/*customizable=*\nDisallow: /se-en/*customizable=*\nDisallow: /sg-en/*customizable=*\nDisallow: /es/*customizable=*\nDisallow: /mx/*customizable=*\nDisallow: /us-es/*customizable=*\nDisallow: /de/*customizable=*\nDisallow: /at/*customizable=*\nDisallow: /ch/*customizable=*\nDisallow: /fr/*customizable=*\nDisallow: /ca-fr/*customizable=*\nDisallow: /it/*customizable=*\nDisallow: /nl/*customizable=*\nDisallow: /be/*customizable=*\nDisallow: /jp/*customizable=*\nDisallow: /pl/*customizable=*\nDisallow: /pt/*customizable=*\nDisallow: /se/*customizable=*\nDisallow: /uk/*locationQuery=*\nDisallow: /au/*locationQuery=*\nDisallow: /ca/*locationQuery=*\nDisallow: /de-en/*locationQuery=*\nDisallow: /dk-en/*locationQuery=*\nDisallow: /fi-en/*locationQuery=*\nDisallow: /hk-en/*locationQuery=*\nDisallow: /ie/*locationQuery=*\nDisallow: /il-en/*locationQuery=*\nDisallow: /in-en/*locationQuery=*\nDisallow: /no-en/*locationQuery=*\nDisallow: /nz/*locationQuery=*\nDisallow: /se-en/*locationQuery=*\nDisallow: /sg-en/*locationQuery=*\nDisallow: /es/*locationQuery=*\nDisallow: /mx/*locationQuery=*\nDisallow: /us-es/*locationQuery=*\nDisallow: /de/*locationQuery=*\nDisallow: /at/*locationQuery=*\nDisallow: /ch/*locationQuery=*\nDisallow: /fr/*locationQuery=*\nDisallow: /ca-fr/*locationQuery=*\nDisallow: /it/*locationQuery=*\nDisallow: /nl/*locationQuery=*\nDisallow: /be/*locationQuery=*\nDisallow: /jp/*locationQuery=*\nDisallow: /pl/*locationQuery=*\nDisallow: /pt/*locationQuery=*\nDisallow: /se/*locationQuery=*\nDisallow: /uk/*min=*\nDisallow: /au/*min=*\nDisallow: /ca/*min=*\nDisallow: /de-en/*min=*\nDisallow: /dk-en/*min=*\nDisallow: /fi-en/*min=*\nDisallow: /hk-en/*min=*\nDisallow: /ie/*min=*\nDisallow: /il-en/*min=*\nDisallow: /in-en/*min=*\nDisallow: /no-en/*min=*\nDisallow: /nz/*min=*\nDisallow: /se-en/*min=*\nDisallow: /sg-en/*min=*\nDisallow: /es/*min=*\nDisallow: /mx/*min=*\nDisallow: /us-es/*min=*\nDisallow: /de/*min=*\nDisallow: /at/*min=*\nDisallow: /ch/*min=*\nDisallow: /fr/*min=*\nDisallow: /ca-fr/*min=*\nDisallow: /it/*min=*\nDisallow: /nl/*min=*\nDisallow: /be/*min=*\nDisallow: /jp/*min=*\nDisallow: /pl/*min=*\nDisallow: /pt/*min=*\nDisallow: /se/*min=*\nDisallow: /uk/*max=*\nDisallow: /au/*max=*\nDisallow: /ca/*max=*\nDisallow: /de-en/*max=*\nDisallow: /dk-en/*max=*\nDisallow: /fi-en/*max=*\nDisallow: /hk-en/*max=*\nDisallow: /ie/*max=*\nDisallow: /il-en/*max=*\nDisallow: /in-en/*max=*\nDisallow: /no-en/*max=*\nDisallow: /nz/*max=*\nDisallow: /se-en/*max=*\nDisallow: /sg-en/*max=*\nDisallow: /es/*max=*\nDisallow: /mx/*max=*\nDisallow: /us-es/*max=*\nDisallow: /de/*max=*\nDisallow: /at/*max=*\nDisallow: /ch/*max=*\nDisallow: /fr/*max=*\nDisallow: /ca-fr/*max=*\nDisallow: /it/*max=*\nDisallow: /nl/*max=*\nDisallow: /be/*max=*\nDisallow: /jp/*max=*\nDisallow: /pl/*max=*\nDisallow: /pt/*max=*\nDisallow: /se/*max=*\nDisallow: /uk*/people/*/circle*\nDisallow: /au*/people/*/circle*\nDisallow: /ca*/people/*/circle*\nDisallow: /de-en*/people/*/circle*\nDisallow: /dk-en*/people/*/circle*\nDisallow: /fi-en*/people/*/circle*\nDisallow: /hk-en*/people/*/circle*\nDisallow: /ie*/people/*/circle*\nDisallow: /il-en*/people/*/circle*\nDisallow: /in-en*/people/*/circle*\nDisallow: /no-en*/people/*/circle*\nDisallow: /nz*/people/*/circle*\nDisallow: /se-en*/people/*/circle*\nDisallow: /sg-en*/people/*/circle*\nDisallow: /es*/people/*/circle*\nDisallow: /mx*/people/*/circle*\nDisallow: /us-es*/people/*/circle*\nDisallow: /de*/people/*/circle*\nDisallow: /at*/people/*/circle*\nDisallow: /ch*/people/*/circle*\nDisallow: /fr*/people/*/circle*\nDisallow: /ca-fr*/people/*/circle*\nDisallow: /it*/people/*/circle*\nDisallow: /nl*/people/*/circle*\nDisallow: /be*/people/*/circle*\nDisallow: /jp*/people/*/circle*\nDisallow: /pl*/people/*/circle*\nDisallow: /pt*/people/*/circle*\nDisallow: /se*/people/*/circle*\nDisallow: /uk*/people/*/favorites*\nDisallow: /au*/people/*/favorites*\nDisallow: /ca*/people/*/favorites*\nDisallow: /de-en*/people/*/favorites*\nDisallow: /dk-en*/people/*/favorites*\nDisallow: /fi-en*/people/*/favorites*\nDisallow: /hk-en*/people/*/favorites*\nDisallow: /ie*/people/*/favorites*\nDisallow: /il-en*/people/*/favorites*\nDisallow: /in-en*/people/*/favorites*\nDisallow: /no-en*/people/*/favorites*\nDisallow: /nz*/people/*/favorites*\nDisallow: /se-en*/people/*/favorites*\nDisallow: /sg-en*/people/*/favorites*\nDisallow: /es*/people/*/favorites*\nDisallow: /mx*/people/*/favorites*\nDisallow: /us-es*/people/*/favorites*\nDisallow: /de*/people/*/favorites*\nDisallow: /at*/people/*/favorites*\nDisallow: /ch*/people/*/favorites*\nDisallow: /fr*/people/*/favorites*\nDisallow: /ca-fr*/people/*/favorites*\nDisallow: /it*/people/*/favorites*\nDisallow: /nl*/people/*/favorites*\nDisallow: /be*/people/*/favorites*\nDisallow: /jp*/people/*/favorites*\nDisallow: /pl*/people/*/favorites*\nDisallow: /pt*/people/*/favorites*\nDisallow: /se*/people/*/favorites*\nDisallow: /uk*/shop/*/sold*\nDisallow: /au*/shop/*/sold*\nDisallow: /ca*/shop/*/sold*\nDisallow: /de-en*/shop/*/sold*\nDisallow: /dk-en*/shop/*/sold*\nDisallow: /fi-en*/shop/*/sold*\nDisallow: /hk-en*/shop/*/sold*\nDisallow: /ie*/shop/*/sold*\nDisallow: /il-en*/shop/*/sold*\nDisallow: /in-en*/shop/*/sold*\nDisallow: /no-en*/shop/*/sold*\nDisallow: /nz*/shop/*/sold*\nDisallow: /se-en*/shop/*/sold*\nDisallow: /sg-en*/shop/*/sold*\nDisallow: /es*/shop/*/sold*\nDisallow: /mx*/shop/*/sold*\nDisallow: /us-es*/shop/*/sold*\nDisallow: /de*/shop/*/sold*\nDisallow: /at*/shop/*/sold*\nDisallow: /ch*/shop/*/sold*\nDisallow: /fr*/shop/*/sold*\nDisallow: /ca-fr*/shop/*/sold*\nDisallow: /it*/shop/*/sold*\nDisallow: /nl*/shop/*/sold*\nDisallow: /be*/shop/*/sold*\nDisallow: /jp*/shop/*/sold*\nDisallow: /pl*/shop/*/sold*\nDisallow: /pt*/shop/*/sold*\nDisallow: /se*/shop/*/sold*\nDisallow: /uk*/shop/*/favoriters*\nDisallow: /au*/shop/*/favoriters*\nDisallow: /ca*/shop/*/favoriters*\nDisallow: /de-en*/shop/*/favoriters*\nDisallow: /dk-en*/shop/*/favoriters*\nDisallow: /fi-en*/shop/*/favoriters*\nDisallow: /hk-en*/shop/*/favoriters*\nDisallow: /ie*/shop/*/favoriters*\nDisallow: /il-en*/shop/*/favoriters*\nDisallow: /in-en*/shop/*/favoriters*\nDisallow: /no-en*/shop/*/favoriters*\nDisallow: /nz*/shop/*/favoriters*\nDisallow: /se-en*/shop/*/favoriters*\nDisallow: /sg-en*/shop/*/favoriters*\nDisallow: /es*/shop/*/favoriters*\nDisallow: /mx*/shop/*/favoriters*\nDisallow: /us-es*/shop/*/favoriters*\nDisallow: /de*/shop/*/favoriters*\nDisallow: /at*/shop/*/favoriters*\nDisallow: /ch*/shop/*/favoriters*\nDisallow: /fr*/shop/*/favoriters*\nDisallow: /ca-fr*/shop/*/favoriters*\nDisallow: /it*/shop/*/favoriters*\nDisallow: /nl*/shop/*/favoriters*\nDisallow: /be*/shop/*/favoriters*\nDisallow: /jp*/shop/*/favoriters*\nDisallow: /pl*/shop/*/favoriters*\nDisallow: /pt*/shop/*/favoriters*\nDisallow: /se*/shop/*/favoriters*\nDisallow: /uk*/listing/*/favoriters*\nDisallow: /au*/listing/*/favoriters*\nDisallow: /ca*/listing/*/favoriters*\nDisallow: /de-en*/listing/*/favoriters*\nDisallow: /dk-en*/listing/*/favoriters*\nDisallow: /fi-en*/listing/*/favoriters*\nDisallow: /hk-en*/listing/*/favoriters*\nDisallow: /ie*/listing/*/favoriters*\nDisallow: /il-en*/listing/*/favoriters*\nDisallow: /in-en*/listing/*/favoriters*\nDisallow: /no-en*/listing/*/favoriters*\nDisallow: /nz*/listing/*/favoriters*\nDisallow: /se-en*/listing/*/favoriters*\nDisallow: /sg-en*/listing/*/favoriters*\nDisallow: /es*/listing/*/favoriters*\nDisallow: /mx*/listing/*/favoriters*\nDisallow: /us-es*/listing/*/favoriters*\nDisallow: /de*/listing/*/favoriters*\nDisallow: /at*/listing/*/favoriters*\nDisallow: /ch*/listing/*/favoriters*\nDisallow: /fr*/listing/*/favoriters*\nDisallow: /ca-fr*/listing/*/favoriters*\nDisallow: /it*/listing/*/favoriters*\nDisallow: /nl*/listing/*/favoriters*\nDisallow: /be*/listing/*/favoriters*\nDisallow: /jp*/listing/*/favoriters*\nDisallow: /pl*/listing/*/favoriters*\nDisallow: /pt*/listing/*/favoriters*\nDisallow: /se*/listing/*/favoriters*\nDisallow: /uk/search*attr_*=*\nDisallow: /au/search*attr_*=*\nDisallow: /ca/search*attr_*=*\nDisallow: /de-en/search*attr_*=*\nDisallow: /dk-en/search*attr_*=*\nDisallow: /fi-en/search*attr_*=*\nDisallow: /hk-en/search*attr_*=*\nDisallow: /ie/search*attr_*=*\nDisallow: /il-en/search*attr_*=*\nDisallow: /in-en/search*attr_*=*\nDisallow: /no-en/search*attr_*=*\nDisallow: /nz/search*attr_*=*\nDisallow: /se-en/search*attr_*=*\nDisallow: /sg-en/search*attr_*=*\nDisallow: /es/search*attr_*=*\nDisallow: /mx/search*attr_*=*\nDisallow: /us-es/search*attr_*=*\nDisallow: /de/search*attr_*=*\nDisallow: /at/search*attr_*=*\nDisallow: /ch/search*attr_*=*\nDisallow: /fr/search*attr_*=*\nDisallow: /ca-fr/search*attr_*=*\nDisallow: /it/search*attr_*=*\nDisallow: /nl/search*attr_*=*\nDisallow: /be/search*attr_*=*\nDisallow: /jp/search*attr_*=*\nDisallow: /pl/search*attr_*=*\nDisallow: /pt/search*attr_*=*\nDisallow: /se/search*attr_*=*\nDisallow: /uk/c/*attr_*=*\nDisallow: /au/c/*attr_*=*\nDisallow: /ca/c/*attr_*=*\nDisallow: /de-en/c/*attr_*=*\nDisallow: /dk-en/c/*attr_*=*\nDisallow: /fi-en/c/*attr_*=*\nDisallow: /hk-en/c/*attr_*=*\nDisallow: /ie/c/*attr_*=*\nDisallow: /il-en/c/*attr_*=*\nDisallow: /in-en/c/*attr_*=*\nDisallow: /no-en/c/*attr_*=*\nDisallow: /nz/c/*attr_*=*\nDisallow: /se-en/c/*attr_*=*\nDisallow: /sg-en/c/*attr_*=*\nDisallow: /es/c/*attr_*=*\nDisallow: /mx/c/*attr_*=*\nDisallow: /us-es/c/*attr_*=*\nDisallow: /de/c/*attr_*=*\nDisallow: /at/c/*attr_*=*\nDisallow: /ch/c/*attr_*=*\nDisallow: /fr/c/*attr_*=*\nDisallow: /ca-fr/c/*attr_*=*\nDisallow: /it/c/*attr_*=*\nDisallow: /nl/c/*attr_*=*\nDisallow: /be/c/*attr_*=*\nDisallow: /jp/c/*attr_*=*\nDisallow: /pl/c/*attr_*=*\nDisallow: /pt/c/*attr_*=*\nDisallow: /se/c/*attr_*=*\nDisallow: /uk/c?*attr_*=*\nDisallow: /au/c?*attr_*=*\nDisallow: /ca/c?*attr_*=*\nDisallow: /de-en/c?*attr_*=*\nDisallow: /dk-en/c?*attr_*=*\nDisallow: /fi-en/c?*attr_*=*\nDisallow: /hk-en/c?*attr_*=*\nDisallow: /ie/c?*attr_*=*\nDisallow: /il-en/c?*attr_*=*\nDisallow: /in-en/c?*attr_*=*\nDisallow: /no-en/c?*attr_*=*\nDisallow: /nz/c?*attr_*=*\nDisallow: /se-en/c?*attr_*=*\nDisallow: /sg-en/c?*attr_*=*\nDisallow: /es/c?*attr_*=*\nDisallow: /mx/c?*attr_*=*\nDisallow: /us-es/c?*attr_*=*\nDisallow: /de/c?*attr_*=*\nDisallow: /at/c?*attr_*=*\nDisallow: /ch/c?*attr_*=*\nDisallow: /fr/c?*attr_*=*\nDisallow: /ca-fr/c?*attr_*=*\nDisallow: /it/c?*attr_*=*\nDisallow: /nl/c?*attr_*=*\nDisallow: /be/c?*attr_*=*\nDisallow: /jp/c?*attr_*=*\nDisallow: /pl/c?*attr_*=*\nDisallow: /pt/c?*attr_*=*\nDisallow: /se/c?*attr_*=*\nDisallow: /uk/search*price_bucket=*\nDisallow: /au/search*price_bucket=*\nDisallow: /ca/search*price_bucket=*\nDisallow: /de-en/search*price_bucket=*\nDisallow: /dk-en/search*price_bucket=*\nDisallow: /fi-en/search*price_bucket=*\nDisallow: /hk-en/search*price_bucket=*\nDisallow: /ie/search*price_bucket=*\nDisallow: /il-en/search*price_bucket=*\nDisallow: /in-en/search*price_bucket=*\nDisallow: /no-en/search*price_bucket=*\nDisallow: /nz/search*price_bucket=*\nDisallow: /se-en/search*price_bucket=*\nDisallow: /sg-en/search*price_bucket=*\nDisallow: /es/search*price_bucket=*\nDisallow: /mx/search*price_bucket=*\nDisallow: /us-es/search*price_bucket=*\nDisallow: /de/search*price_bucket=*\nDisallow: /at/search*price_bucket=*\nDisallow: /ch/search*price_bucket=*\nDisallow: /fr/search*price_bucket=*\nDisallow: /ca-fr/search*price_bucket=*\nDisallow: /it/search*price_bucket=*\nDisallow: /nl/search*price_bucket=*\nDisallow: /be/search*price_bucket=*\nDisallow: /jp/search*price_bucket=*\nDisallow: /pl/search*price_bucket=*\nDisallow: /pt/search*price_bucket=*\nDisallow: /se/search*price_bucket=*\nDisallow: /uk/c/*price_bucket=*\nDisallow: /au/c/*price_bucket=*\nDisallow: /ca/c/*price_bucket=*\nDisallow: /de-en/c/*price_bucket=*\nDisallow: /dk-en/c/*price_bucket=*\nDisallow: /fi-en/c/*price_bucket=*\nDisallow: /hk-en/c/*price_bucket=*\nDisallow: /ie/c/*price_bucket=*\nDisallow: /il-en/c/*price_bucket=*\nDisallow: /in-en/c/*price_bucket=*\nDisallow: /no-en/c/*price_bucket=*\nDisallow: /nz/c/*price_bucket=*\nDisallow: /se-en/c/*price_bucket=*\nDisallow: /sg-en/c/*price_bucket=*\nDisallow: /es/c/*price_bucket=*\nDisallow: /mx/c/*price_bucket=*\nDisallow: /us-es/c/*price_bucket=*\nDisallow: /de/c/*price_bucket=*\nDisallow: /at/c/*price_bucket=*\nDisallow: /ch/c/*price_bucket=*\nDisallow: /fr/c/*price_bucket=*\nDisallow: /ca-fr/c/*price_bucket=*\nDisallow: /it/c/*price_bucket=*\nDisallow: /nl/c/*price_bucket=*\nDisallow: /be/c/*price_bucket=*\nDisallow: /jp/c/*price_bucket=*\nDisallow: /pl/c/*price_bucket=*\nDisallow: /pt/c/*price_bucket=*\nDisallow: /se/c/*price_bucket=*\nDisallow: /uk/search*ship_to=*\nDisallow: /au/search*ship_to=*\nDisallow: /ca/search*ship_to=*\nDisallow: /de-en/search*ship_to=*\nDisallow: /dk-en/search*ship_to=*\nDisallow: /fi-en/search*ship_to=*\nDisallow: /hk-en/search*ship_to=*\nDisallow: /ie/search*ship_to=*\nDisallow: /il-en/search*ship_to=*\nDisallow: /in-en/search*ship_to=*\nDisallow: /no-en/search*ship_to=*\nDisallow: /nz/search*ship_to=*\nDisallow: /se-en/search*ship_to=*\nDisallow: /sg-en/search*ship_to=*\nDisallow: /es/search*ship_to=*\nDisallow: /mx/search*ship_to=*\nDisallow: /us-es/search*ship_to=*\nDisallow: /de/search*ship_to=*\nDisallow: /at/search*ship_to=*\nDisallow: /ch/search*ship_to=*\nDisallow: /fr/search*ship_to=*\nDisallow: /ca-fr/search*ship_to=*\nDisallow: /it/search*ship_to=*\nDisallow: /nl/search*ship_to=*\nDisallow: /be/search*ship_to=*\nDisallow: /jp/search*ship_to=*\nDisallow: /pl/search*ship_to=*\nDisallow: /pt/search*ship_to=*\nDisallow: /se/search*ship_to=*\nDisallow: /uk/c/*ship_to=*\nDisallow: /au/c/*ship_to=*\nDisallow: /ca/c/*ship_to=*\nDisallow: /de-en/c/*ship_to=*\nDisallow: /dk-en/c/*ship_to=*\nDisallow: /fi-en/c/*ship_to=*\nDisallow: /hk-en/c/*ship_to=*\nDisallow: /ie/c/*ship_to=*\nDisallow: /il-en/c/*ship_to=*\nDisallow: /in-en/c/*ship_to=*\nDisallow: /no-en/c/*ship_to=*\nDisallow: /nz/c/*ship_to=*\nDisallow: /se-en/c/*ship_to=*\nDisallow: /sg-en/c/*ship_to=*\nDisallow: /es/c/*ship_to=*\nDisallow: /mx/c/*ship_to=*\nDisallow: /us-es/c/*ship_to=*\nDisallow: /de/c/*ship_to=*\nDisallow: /at/c/*ship_to=*\nDisallow: /ch/c/*ship_to=*\nDisallow: /fr/c/*ship_to=*\nDisallow: /ca-fr/c/*ship_to=*\nDisallow: /it/c/*ship_to=*\nDisallow: /nl/c/*ship_to=*\nDisallow: /be/c/*ship_to=*\nDisallow: /jp/c/*ship_to=*\nDisallow: /pl/c/*ship_to=*\nDisallow: /pt/c/*ship_to=*\nDisallow: /se/c/*ship_to=*\nDisallow: /uk/search*search_type=*\nDisallow: /au/search*search_type=*\nDisallow: /ca/search*search_type=*\nDisallow: /de-en/search*search_type=*\nDisallow: /dk-en/search*search_type=*\nDisallow: /fi-en/search*search_type=*\nDisallow: /hk-en/search*search_type=*\nDisallow: /ie/search*search_type=*\nDisallow: /il-en/search*search_type=*\nDisallow: /in-en/search*search_type=*\nDisallow: /no-en/search*search_type=*\nDisallow: /nz/search*search_type=*\nDisallow: /se-en/search*search_type=*\nDisallow: /sg-en/search*search_type=*\nDisallow: /es/search*search_type=*\nDisallow: /mx/search*search_type=*\nDisallow: /us-es/search*search_type=*\nDisallow: /de/search*search_type=*\nDisallow: /at/search*search_type=*\nDisallow: /ch/search*search_type=*\nDisallow: /fr/search*search_type=*\nDisallow: /ca-fr/search*search_type=*\nDisallow: /it/search*search_type=*\nDisallow: /nl/search*search_type=*\nDisallow: /be/search*search_type=*\nDisallow: /jp/search*search_type=*\nDisallow: /pl/search*search_type=*\nDisallow: /pt/search*search_type=*\nDisallow: /se/search*search_type=*\nDisallow: /uk/c/*search_type=*\nDisallow: /au/c/*search_type=*\nDisallow: /ca/c/*search_type=*\nDisallow: /de-en/c/*search_type=*\nDisallow: /dk-en/c/*search_type=*\nDisallow: /fi-en/c/*search_type=*\nDisallow: /hk-en/c/*search_type=*\nDisallow: /ie/c/*search_type=*\nDisallow: /il-en/c/*search_type=*\nDisallow: /in-en/c/*search_type=*\nDisallow: /no-en/c/*search_type=*\nDisallow: /nz/c/*search_type=*\nDisallow: /se-en/c/*search_type=*\nDisallow: /sg-en/c/*search_type=*\nDisallow: /es/c/*search_type=*\nDisallow: /mx/c/*search_type=*\nDisallow: /us-es/c/*search_type=*\nDisallow: /de/c/*search_type=*\nDisallow: /at/c/*search_type=*\nDisallow: /ch/c/*search_type=*\nDisallow: /fr/c/*search_type=*\nDisallow: /ca-fr/c/*search_type=*\nDisallow: /it/c/*search_type=*\nDisallow: /nl/c/*search_type=*\nDisallow: /be/c/*search_type=*\nDisallow: /jp/c/*search_type=*\nDisallow: /pl/c/*search_type=*\nDisallow: /pt/c/*search_type=*\nDisallow: /se/c/*search_type=*\nDisallow: /uk/shop_name_search_service\nDisallow: /au/shop_name_search_service\nDisallow: /ca/shop_name_search_service\nDisallow: /de-en/shop_name_search_service\nDisallow: /dk-en/shop_name_search_service\nDisallow: /fi-en/shop_name_search_service\nDisallow: /hk-en/shop_name_search_service\nDisallow: /ie/shop_name_search_service\nDisallow: /il-en/shop_name_search_service\nDisallow: /in-en/shop_name_search_service\nDisallow: /no-en/shop_name_search_service\nDisallow: /nz/shop_name_search_service\nDisallow: /se-en/shop_name_search_service\nDisallow: /sg-en/shop_name_search_service\nDisallow: /es/shop_name_search_service\nDisallow: /mx/shop_name_search_service\nDisallow: /us-es/shop_name_search_service\nDisallow: /de/shop_name_search_service\nDisallow: /at/shop_name_search_service\nDisallow: /ch/shop_name_search_service\nDisallow: /fr/shop_name_search_service\nDisallow: /ca-fr/shop_name_search_service\nDisallow: /it/shop_name_search_service\nDisallow: /nl/shop_name_search_service\nDisallow: /be/shop_name_search_service\nDisallow: /jp/shop_name_search_service\nDisallow: /pl/shop_name_search_service\nDisallow: /pt/shop_name_search_service\nDisallow: /se/shop_name_search_service\nDisallow: /uk/messages\nDisallow: /au/messages\nDisallow: /ca/messages\nDisallow: /de-en/messages\nDisallow: /dk-en/messages\nDisallow: /fi-en/messages\nDisallow: /hk-en/messages\nDisallow: /ie/messages\nDisallow: /il-en/messages\nDisallow: /in-en/messages\nDisallow: /no-en/messages\nDisallow: /nz/messages\nDisallow: /se-en/messages\nDisallow: /sg-en/messages\nDisallow: /es/messages\nDisallow: /mx/messages\nDisallow: /us-es/messages\nDisallow: /de/messages\nDisallow: /at/messages\nDisallow: /ch/messages\nDisallow: /fr/messages\nDisallow: /ca-fr/messages\nDisallow: /it/messages\nDisallow: /nl/messages\nDisallow: /be/messages\nDisallow: /jp/messages\nDisallow: /pl/messages\nDisallow: /pt/messages\nDisallow: /se/messages\nDisallow: /uk/apps\nDisallow: /au/apps\nDisallow: /ca/apps\nDisallow: /de-en/apps\nDisallow: /dk-en/apps\nDisallow: /fi-en/apps\nDisallow: /hk-en/apps\nDisallow: /ie/apps\nDisallow: /il-en/apps\nDisallow: /in-en/apps\nDisallow: /no-en/apps\nDisallow: /nz/apps\nDisallow: /se-en/apps\nDisallow: /sg-en/apps\nDisallow: /es/apps\nDisallow: /mx/apps\nDisallow: /us-es/apps\nDisallow: /de/apps\nDisallow: /at/apps\nDisallow: /ch/apps\nDisallow: /fr/apps\nDisallow: /ca-fr/apps\nDisallow: /it/apps\nDisallow: /nl/apps\nDisallow: /be/apps\nDisallow: /jp/apps\nDisallow: /pl/apps\nDisallow: /pt/apps\nDisallow: /se/apps\nDisallow: /uk/search?*q=\nDisallow: /au/search?*q=\nDisallow: /ca/search?*q=\nDisallow: /de-en/search?*q=\nDisallow: /dk-en/search?*q=\nDisallow: /fi-en/search?*q=\nDisallow: /hk-en/search?*q=\nDisallow: /ie/search?*q=\nDisallow: /il-en/search?*q=\nDisallow: /in-en/search?*q=\nDisallow: /no-en/search?*q=\nDisallow: /nz/search?*q=\nDisallow: /se-en/search?*q=\nDisallow: /sg-en/search?*q=\nDisallow: /es/search?*q=\nDisallow: /mx/search?*q=\nDisallow: /us-es/search?*q=\nDisallow: /de/search?*q=\nDisallow: /at/search?*q=\nDisallow: /ch/search?*q=\nDisallow: /fr/search?*q=\nDisallow: /ca-fr/search?*q=\nDisallow: /it/search?*q=\nDisallow: /nl/search?*q=\nDisallow: /be/search?*q=\nDisallow: /jp/search?*q=\nDisallow: /pl/search?*q=\nDisallow: /pt/search?*q=\nDisallow: /se/search?*q=\nDisallow: /uk/search/?*q=\nDisallow: /au/search/?*q=\nDisallow: /ca/search/?*q=\nDisallow: /de-en/search/?*q=\nDisallow: /dk-en/search/?*q=\nDisallow: /fi-en/search/?*q=\nDisallow: /hk-en/search/?*q=\nDisallow: /ie/search/?*q=\nDisallow: /il-en/search/?*q=\nDisallow: /in-en/search/?*q=\nDisallow: /no-en/search/?*q=\nDisallow: /nz/search/?*q=\nDisallow: /se-en/search/?*q=\nDisallow: /sg-en/search/?*q=\nDisallow: /es/search/?*q=\nDisallow: /mx/search/?*q=\nDisallow: /us-es/search/?*q=\nDisallow: /de/search/?*q=\nDisallow: /at/search/?*q=\nDisallow: /ch/search/?*q=\nDisallow: /fr/search/?*q=\nDisallow: /ca-fr/search/?*q=\nDisallow: /it/search/?*q=\nDisallow: /nl/search/?*q=\nDisallow: /be/search/?*q=\nDisallow: /jp/search/?*q=\nDisallow: /pl/search/?*q=\nDisallow: /pt/search/?*q=\nDisallow: /se/search/?*q=\nDisallow: /uk/search/*?*q=\nDisallow: /au/search/*?*q=\nDisallow: /ca/search/*?*q=\nDisallow: /de-en/search/*?*q=\nDisallow: /dk-en/search/*?*q=\nDisallow: /fi-en/search/*?*q=\nDisallow: /hk-en/search/*?*q=\nDisallow: /ie/search/*?*q=\nDisallow: /il-en/search/*?*q=\nDisallow: /in-en/search/*?*q=\nDisallow: /no-en/search/*?*q=\nDisallow: /nz/search/*?*q=\nDisallow: /se-en/search/*?*q=\nDisallow: /sg-en/search/*?*q=\nDisallow: /es/search/*?*q=\nDisallow: /mx/search/*?*q=\nDisallow: /us-es/search/*?*q=\nDisallow: /de/search/*?*q=\nDisallow: /at/search/*?*q=\nDisallow: /ch/search/*?*q=\nDisallow: /fr/search/*?*q=\nDisallow: /ca-fr/search/*?*q=\nDisallow: /it/search/*?*q=\nDisallow: /nl/search/*?*q=\nDisallow: /be/search/*?*q=\nDisallow: /jp/search/*?*q=\nDisallow: /pl/search/*?*q=\nDisallow: /pt/search/*?*q=\nDisallow: /se/search/*?*q=\nDisallow: /uk/search/*/?*q=\nDisallow: /au/search/*/?*q=\nDisallow: /ca/search/*/?*q=\nDisallow: /de-en/search/*/?*q=\nDisallow: /dk-en/search/*/?*q=\nDisallow: /fi-en/search/*/?*q=\nDisallow: /hk-en/search/*/?*q=\nDisallow: /ie/search/*/?*q=\nDisallow: /il-en/search/*/?*q=\nDisallow: /in-en/search/*/?*q=\nDisallow: /no-en/search/*/?*q=\nDisallow: /nz/search/*/?*q=\nDisallow: /se-en/search/*/?*q=\nDisallow: /sg-en/search/*/?*q=\nDisallow: /es/search/*/?*q=\nDisallow: /mx/search/*/?*q=\nDisallow: /us-es/search/*/?*q=\nDisallow: /de/search/*/?*q=\nDisallow: /at/search/*/?*q=\nDisallow: /ch/search/*/?*q=\nDisallow: /fr/search/*/?*q=\nDisallow: /ca-fr/search/*/?*q=\nDisallow: /it/search/*/?*q=\nDisallow: /nl/search/*/?*q=\nDisallow: /be/search/*/?*q=\nDisallow: /jp/search/*/?*q=\nDisallow: /pl/search/*/?*q=\nDisallow: /pt/search/*/?*q=\nDisallow: /se/search/*/?*q=\nDisallow: /uk*/shop/*/rss_shop.php\nDisallow: /au*/shop/*/rss_shop.php\nDisallow: /ca*/shop/*/rss_shop.php\nDisallow: /de-en*/shop/*/rss_shop.php\nDisallow: /dk-en*/shop/*/rss_shop.php\nDisallow: /fi-en*/shop/*/rss_shop.php\nDisallow: /hk-en*/shop/*/rss_shop.php\nDisallow: /ie*/shop/*/rss_shop.php\nDisallow: /il-en*/shop/*/rss_shop.php\nDisallow: /in-en*/shop/*/rss_shop.php\nDisallow: /no-en*/shop/*/rss_shop.php\nDisallow: /nz*/shop/*/rss_shop.php\nDisallow: /se-en*/shop/*/rss_shop.php\nDisallow: /sg-en*/shop/*/rss_shop.php\nDisallow: /es*/shop/*/rss_shop.php\nDisallow: /mx*/shop/*/rss_shop.php\nDisallow: /us-es*/shop/*/rss_shop.php\nDisallow: /de*/shop/*/rss_shop.php\nDisallow: /at*/shop/*/rss_shop.php\nDisallow: /ch*/shop/*/rss_shop.php\nDisallow: /fr*/shop/*/rss_shop.php\nDisallow: /ca-fr*/shop/*/rss_shop.php\nDisallow: /it*/shop/*/rss_shop.php\nDisallow: /nl*/shop/*/rss_shop.php\nDisallow: /be*/shop/*/rss_shop.php\nDisallow: /jp*/shop/*/rss_shop.php\nDisallow: /pl*/shop/*/rss_shop.php\nDisallow: /pt*/shop/*/rss_shop.php\nDisallow: /se*/shop/*/rss_shop.php\nAllow: /mailinglist/email/\nAllow: /api/v3/ajax/bespoke/public/neu/specs/\nAllow: /api/v3/ajax/bespoke/member/neu/specs/\nAllow: /search/shops\nAllow: /uk/mailinglist/email/\nAllow: /au/mailinglist/email/\nAllow: /ca/mailinglist/email/\nAllow: /de-en/mailinglist/email/\nAllow: /dk-en/mailinglist/email/\nAllow: /fi-en/mailinglist/email/\nAllow: /hk-en/mailinglist/email/\nAllow: /ie/mailinglist/email/\nAllow: /il-en/mailinglist/email/\nAllow: /in-en/mailinglist/email/\nAllow: /no-en/mailinglist/email/\nAllow: /nz/mailinglist/email/\nAllow: /se-en/mailinglist/email/\nAllow: /sg-en/mailinglist/email/\nAllow: /es/mailinglist/email/\nAllow: /mx/mailinglist/email/\nAllow: /us-es/mailinglist/email/\nAllow: /de/mailinglist/email/\nAllow: /at/mailinglist/email/\nAllow: /ch/mailinglist/email/\nAllow: /fr/mailinglist/email/\nAllow: /ca-fr/mailinglist/email/\nAllow: /it/mailinglist/email/\nAllow: /nl/mailinglist/email/\nAllow: /be/mailinglist/email/\nAllow: /jp/mailinglist/email/\nAllow: /pl/mailinglist/email/\nAllow: /pt/mailinglist/email/\nAllow: /se/mailinglist/email/\nAllow: /uk/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /au/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /ca/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /de-en/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /dk-en/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /fi-en/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /hk-en/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /ie/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /il-en/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /in-en/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /no-en/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /nz/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /se-en/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /sg-en/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /es/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /mx/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /us-es/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /de/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /at/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /ch/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /fr/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /ca-fr/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /it/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /nl/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /be/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /jp/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /pl/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /pt/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /se/api/v3/ajax/bespoke/public/neu/specs/\nAllow: /uk/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /au/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /ca/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /de-en/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /dk-en/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /fi-en/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /hk-en/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /ie/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /il-en/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /in-en/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /no-en/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /nz/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /se-en/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /sg-en/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /es/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /mx/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /us-es/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /de/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /at/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /ch/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /fr/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /ca-fr/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /it/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /nl/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /be/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /jp/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /pl/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /pt/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /se/api/v3/ajax/bespoke/member/neu/specs/\nAllow: /uk/search/shops\nAllow: /au/search/shops\nAllow: /ca/search/shops\nAllow: /de-en/search/shops\nAllow: /dk-en/search/shops\nAllow: /fi-en/search/shops\nAllow: /hk-en/search/shops\nAllow: /ie/search/shops\nAllow: /il-en/search/shops\nAllow: /in-en/search/shops\nAllow: /no-en/search/shops\nAllow: /nz/search/shops\nAllow: /se-en/search/shops\nAllow: /sg-en/search/shops\nAllow: /es/search/shops\nAllow: /mx/search/shops\nAllow: /us-es/search/shops\nAllow: /de/search/shops\nAllow: /at/search/shops\nAllow: /ch/search/shops\nAllow: /fr/search/shops\nAllow: /ca-fr/search/shops\nAllow: /it/search/shops\nAllow: /nl/search/shops\nAllow: /be/search/shops\nAllow: /jp/search/shops\nAllow: /pl/search/shops\nAllow: /pt/search/shops\nAllow: /se/search/shops\n\n# llms.txt: https://www.etsy.com/llms.txt\n\n#\n# Oi, kent!#   \\\n#\n#    -----\n#   | . . |\n#    -----\n#  \\--|-|--/\n#     | |\n#  |-------|\n\n\n\n"
    },
    "www.aliexpress.com": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.aliexpress.com",
      "body": "User-agent: Googlebot\r\nDisallow: */aeglodetailweb/api/msite/item?productId*\r\n\r\nUser-agent: *\r\nDisallow: /items/*\r\nDisallow: /bin/*\r\nDisallow: /search/*\r\nAllow: /wholesale.html$\r\nAllow: /wholesale-page-*.html\r\nDisallow: /productdetail/*\r\nAllow: /api/data_homepage.do\r\nDisallow: /api/*\r\nDisallow: /api*.do\r\nDisallow: /apps/*\r\nDisallow: /downloads/*\r\nDisallow: /wishlist/*\r\nDisallow: /shopcart/*\r\nDisallow: /brands/*\r\nDisallow: /cp/*\r\nDisallow: /item-img/*\r\nDisallow: /product/*\r\nDisallow: /p4p*list.html\r\nDisallow: /orderList.html\r\nDisallow: /ws*api-server.html$\r\nDisallow: /*mmend.htm$\r\nDisallow: /*Ajax.htm$\r\nDisallow: /*ajax.htm$\r\nDisallow: /store/*ajax.htm$\r\nDisallow: /detail/*Ajax.do$\r\nDisallow: /ajax*.do$\r\nDisallow: /__pc/*\r\nDisallow: /__mobile/*\r\nDisallow: /seo/*\r\nDisallow: /cross-domain/*\r\nDisallow: /*store/group/*\r\nDisallow: /*store/sale-items/*\r\nDisallow: /store/all-wholesale-products/* \r\nDisallow: /*store/top-rated-products/*\r\nAllow: /i/api/reviews\r\nDisallow: /i/api/*\r\nDisallow: /gcp/300000414/regulatedinformation\r\nDisallow: /img/wsproduct/sku/*\r\n\r\nUser-agent: YandexBot\r\nCrawl-delay: 0.5\r\nDisallow: /bin/*\r\nDisallow: /search/*\r\nDisallow: /wholesale*\r\nDisallow: /productdetail/*\r\nDisallow: /api/*\r\nDisallow: /api*.do\r\nDisallow: /apps/*\r\nDisallow: /downloads/*\r\nDisallow: /wishlist/*\r\nDisallow: /shopcart/*\r\nDisallow: /brands/*\r\nDisallow: /cp/*\r\nDisallow: /item-img/*\r\nDisallow: /product/*\r\nDisallow: /p4p*list.html\r\nDisallow: /orderList.html\r\nDisallow: /ws*api-server.html$\r\nDisallow: /*mmend.htm$\r\nDisallow: /*Ajax.htm$\r\nDisallow: /*ajax.htm$\r\nDisallow: /store/*ajax.htm$\r\nDisallow: /detail/*Ajax.do$\r\nDisallow: /ajax*.do$\r\nDisallow: /__pc/*\r\nDisallow: /__mobile/*\r\nDisallow: /seo/*\r\nDisallow: /cross-domain/*\r\nDisallow: /*store/group/*\r\nDisallow: /*store/sale-items/*\r\nDisallow: /store/all-wholesale-products/* \r\nDisallow: /*store/top-rated-products/*\r\nDisallow: /i/api/*\r\nDisallow: /gcp/300000414/regulatedinformation\r\nDisallow: /img/wsproduct/sku/*\r\n\n\nSitemap: https://www.aliexpress.com/sitemap/en-https-item-item-2026-P0-sitemap-index.xml\nSitemap: https://www.aliexpress.com/sitemap/en-https-item-item-2026-P1-sitemap-index.xml\nSitemap: https://www.aliexpress.com/sitemap/en-https-wholesale-2025-w-commercial-local-sitemap-index.xml\nSitemap: https://www.aliexpress.com/sitemap/en-https-wholesale-2025-w-informational-localsitemap-index.xml\nSitemap: https://www.aliexpress.com/sitemap/en-https-wholesale-2025-w-brand-local-sitemap-index.xml\nSitemap: https://www.aliexpress.com/sitemap/en-https-wholesale-2025-p-transactional-local-sitemap-index.xml\nSitemap: https://www.aliexpress.com/sitemap/en-https-mlh-2026-sitemap-index.xml\nSitemap: https://www.aliexpress.com/sitemap/en-https-spot-2026-sitemap-index.xml"
    },
    "www.temu.com": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.temu.com",
      "body": "User-agent: Googlebot\nUser-agent: Googlebot-Image\nUser-agent: Bingbot\nUser-agent: Google-Extended\nUser-agent: GPTBot\nUser-agent: OAI-SearchBot\nUser-agent: ChatGPT-User\nUser-agent: PerplexityBot\nUser-agent: ClaudeBot\nUser-agent: Claude-User\nUser-agent: Claude-SearchBot\nDisAllow: /*oauth_code=\nDisAllow: /*oauth_token=\nDisAllow: /bgp_user_profile.html\nDisAllow: /bgp_personal.html\nDisAllow: /bgp_collect.html\nDisAllow: /bgp_footprint.html\nDisAllow: /bgst_order_success.html\nDisAllow: /bgst_payment_detail.html\nDisAllow: /bgst_express.html\nDisAllow: /personal.html\nDisAllow: /collect.html\nDisAllow: /shopping_cart.html\nDisAllow: /bgp_similar_rec.html\nDisAllow: /bgcp_shipping_add_items.html\nDisAllow: /bgcp_coupon.html\nDisAllow: /bgcp_cart_coupon.html\nDisAllow: /bgcp_coupon_add_items.html\nDisAllow: /bgt_order_checkout.html\nDisAllow: /bgt_bank_foreign_card_info.html\nDisAllow: /bgt_3ds.html\nDisAllow: /bgt_orders.html\nDisAllow: /bgt_order_detail.html\nDisAllow: /bgt_card_list_management.html\nDisAllow: /bgt_bind_card.html\nDisAllow: /address.html\nDisAllow: /create_address.html\nDisAllow: /bgst_order_cancelled.html\nDisAllow: /bgt_reviewed.html\nDisAllow: /bgas_return_application.html\nDisAllow: /bgas_refund_detail.html\nDisAllow: /bgas_return_history.html\nDisAllow: /bgas_refund_success.html\nDisAllow: /bgas_cancel_order.html\nDisAllow: /bgp_redirect.html\nDisAllow: /bgp_setting.html\nDisAllow: /bgp_account_security.html\nDisAllow: /bgn_verification.html\nDisAllow: /bgn_login_verification.html\nDisAllow: /inno_promotion.html\nDisAllow: /student-discount.html\nDisAllow: /moss_product_news.html\nDisAllow: /moss_freelancer_register.html\nDisAllow: /moss_assistant_bonus.html\nDisAllow: /*/*oauth_code=\nDisAllow: /*/*oauth_token=\nDisAllow: /*/bgp_user_profile.html\nDisAllow: /*/bgp_personal.html\nDisAllow: /*/bgp_collect.html\nDisAllow: /*/bgp_footprint.html\nDisAllow: /*/bgst_order_success.html\nDisAllow: /*/bgst_payment_detail.html\nDisAllow: /*/bgst_express.html\nDisAllow: /*/personal.html\nDisAllow: /*/collect.html\nDisAllow: /*/shopping_cart.html\nDisAllow: /*/bgp_similar_rec.html\nDisAllow: /*/bgcp_shipping_add_items.html\nDisAllow: /*/bgcp_coupon.html\nDisAllow: /*/bgcp_cart_coupon.html\nDisAllow: /*/bgcp_coupon_add_items.html\nDisAllow: /*/bgt_order_checkout.html\nDisAllow: /*/bgt_bank_foreign_card_info.html\nDisAllow: /*/bgt_3ds.html\nDisAllow: /*/bgt_orders.html\nDisAllow: /*/bgt_order_detail.html\nDisAllow: /*/bgt_card_list_management.html\nDisAllow: /*/bgt_bind_card.html\nDisAllow: /*/address.html\nDisAllow: /*/create_address.html\nDisAllow: /*/bgst_order_cancelled.html\nDisAllow: /*/bgt_reviewed.html\nDisAllow: /*/bgas_return_application.html\nDisAllow: /*/bgas_refund_detail.html\nDisAllow: /*/bgas_return_history.html\nDisAllow: /*/bgas_refund_success.html\nDisAllow: /*/bgas_cancel_order.html\nDisAllow: /*/bgp_redirect.html\nDisAllow: /*/bgp_setting.html\nDisAllow: /*/bgp_account_security.html\nDisAllow: /*/bgn_verification.html\nDisAllow: /*/bgn_login_verification.html\nDisAllow: /*/inno_promotion.html\nDisAllow: /*/student-discount.html\nDisAllow: /*/moss_product_news.html\nDisAllow: /*/moss_freelancer_register.html\nDisAllow: /*/moss_assistant_bonus.html\nDisAllow: /m/\nDisAllow: /k/\nDisallow: /w/\nDisallow: /s/\nDisallow: /*?*opt_level=*title=*\nDisallow: /*-m-*?goods_id=*\nDisAllow: /blog/*\nDisAllow: /*/blog/*\nDisAllow: /blog-gallery.html*\nDisAllow: /*/blog-gallery.html*\nDisAllow: */c/1418078/1583288/18350*\nDisallow: /*MEDICAL_CONTEXT*\nDisallow: /*US_FEMALE_NAME*\nDisallow: /*US_CENSUS_NAME*\nDisallow: /*tmp0611*\nDisallow: /be/be-fr*\nDisAllow: /*@context*https://schema.org/*\n\nUser-agent: AdsBot-Google\nUser-agent: AdIdxBot\nUser-agent: Applebot\nUser-agent: Pinterestbot\nUser-agent: Twitterbot\nUser-agent: FacebookBot\nDisAllow: /*oauth_code=\nDisAllow: /*oauth_token=\nDisAllow: /bgp_user_profile.html\nDisAllow: /bgp_personal.html\nDisAllow: /bgp_collect.html\nDisAllow: /bgp_footprint.html\nDisAllow: /bgst_order_success.html\nDisAllow: /bgst_payment_detail.html\nDisAllow: /bgst_express.html\nDisAllow: /personal.html\nDisAllow: /collect.html\nDisAllow: /shopping_cart.html\nDisAllow: /bgp_similar_rec.html\nDisAllow: /bgcp_shipping_add_items.html\nDisAllow: /bgcp_coupon.html\nDisAllow: /bgcp_cart_coupon.html\nDisAllow: /bgcp_coupon_add_items.html\nDisAllow: /bgt_order_checkout.html\nDisAllow: /bgt_bank_foreign_card_info.html\nDisAllow: /bgt_3ds.html\nDisAllow: /bgt_orders.html\nDisAllow: /bgt_order_detail.html\nDisAllow: /bgt_card_list_management.html\nDisAllow: /bgt_bind_card.html\nDisAllow: /address.html\nDisAllow: /create_address.html\nDisAllow: /bgst_order_cancelled.html\nDisAllow: /bgt_reviewed.html\nDisAllow: /bgas_return_application.html\nDisAllow: /bgas_refund_detail.html\nDisAllow: /bgas_return_history.html\nDisAllow: /bgas_refund_success.html\nDisAllow: /bgas_cancel_order.html\nDisAllow: /bgp_redirect.html\nDisAllow: /bgp_setting.html\nDisAllow: /bgp_account_security.html\nDisAllow: /bgn_verification.html\nDisAllow: /bgn_login_verification.html\nDisAllow: /inno_promotion.html\nDisAllow: /student-discount.html\nDisAllow: /moss_product_news.html\nDisAllow: /moss_freelancer_register.html\nDisAllow: /moss_assistant_bonus.html\nDisAllow: /m/\nDisAllow: /k/\nDisallow: /w/\nDisallow: /s/\nDisallow: /*?*top_goods=*\nDisallow: /*?*opt_level=*title=*\nDisallow: /*?*top_mall_list=*\nDisallow: /*-m-*?goods_id=*\nDisallow: /channel/*-g-*\nDisallow: /*lightning-deals.html?*top_goods_low_price=*\nDisallow: /*-g-*html?*thumb_url=*\nDisallow: /*-g-*html?*top_gallery_url=*\nDisallow: /*-g-*html?*_oak_page_source=*\nDisAllow: /blog/*\nDisAllow: /*/blog/*\nDisAllow: /blog-gallery.html*\nDisAllow: /*/blog-gallery.html*\nDisAllow: */c/1418078/1583288/18350*\nDisallow: /*MEDICAL_CONTEXT*\nDisallow: /*US_FEMALE_NAME*\nDisallow: /*US_CENSUS_NAME*\nDisallow: /*tmp0611*\nDisallow: /be/be-fr*\nDisAllow: /*@context*https://schema.org/*\n\nUser-agent: facebookexternalhit\nDisAllow: /*oauth_code=\nDisAllow: /*oauth_token=\nDisAllow: /bgp_user_profile.html\nDisAllow: /bgp_personal.html\nDisAllow: /bgp_collect.html\nDisAllow: /bgp_footprint.html\nDisAllow: /bgst_order_success.html\nDisAllow: /bgst_payment_detail.html\nDisAllow: /bgst_express.html\nDisAllow: /personal.html\nDisAllow: /collect.html\nDisAllow: /shopping_cart.html\nDisAllow: /bgp_similar_rec.html\nDisAllow: /bgcp_shipping_add_items.html\nDisAllow: /bgcp_coupon.html\nDisAllow: /bgcp_cart_coupon.html\nDisAllow: /bgcp_coupon_add_items.html\nDisAllow: /bgt_order_checkout.html\nDisAllow: /bgt_bank_foreign_card_info.html\nDisAllow: /bgt_3ds.html\nDisAllow: /bgt_orders.html\nDisAllow: /bgt_order_detail.html\nDisAllow: /bgt_card_list_management.html\nDisAllow: /bgt_bind_card.html\nDisAllow: /address.html\nDisAllow: /create_address.html\nDisAllow: /bgst_order_cancelled.html\nDisAllow: /bgt_reviewed.html\nDisAllow: /bgas_return_application.html\nDisAllow: /bgas_refund_detail.html\nDisAllow: /bgas_return_history.html\nDisAllow: /bgas_refund_success.html\nDisAllow: /bgas_cancel_order.html\nDisAllow: /bgp_redirect.html\nDisAllow: /bgp_setting.html\nDisAllow: /bgp_account_security.html\nDisAllow: /bgn_verification.html\nDisAllow: /bgn_login_verification.html\nDisAllow: /inno_promotion.html\nDisAllow: /student-discount.html\nDisAllow: /moss_product_news.html\nDisAllow: /moss_freelancer_register.html\nDisAllow: /moss_assistant_bonus.html\nDisallow: /w/\nDisallow: /*?*top_goods=*\nDisallow: /*?*opt_level=*title=*\nDisallow: /*?*top_mall_list=*\nDisallow: /*-m-*?goods_id=*\nDisallow: /channel/*-g-*\nDisallow: /*lightning-deals.html?*top_goods_low_price=*\nDisallow: /*-g-*html?*thumb_url=*\nDisallow: /*-g-*html?*top_gallery_url=*\nDisallow: /*-g-*html?*_oak_page_source=*\nDisAllow: /blog/*\nDisAllow: /*/blog/*\nDisAllow: /blog-gallery.html*\nDisAllow: /*/blog-gallery.html*\nDisAllow: */c/1418078/1583288/18350*\nDisallow: /*MEDICAL_CONTEXT*\nDisallow: /*US_FEMALE_NAME*\nDisallow: /*US_CENSUS_NAME*\nDisallow: /*tmp0611*\nDisallow: /be/be-fr*\nDisAllow: /*@context*https://schema.org/*\n\nUser-agent: *\nAllow: /.well-known/\n\nUser-agent: Slurp\nUser-agent: Y!J-BRU/VSIDX\nUser-agent: Y!J-BRW\nUser-agent: Y!J-ASR\nUser-agent: Y!J-MMP/dscv\nUser-agent: Y!J-WSC\nUser-agent: Y!J-DLC\nUser-agent: YJ-BRU/VSIDX\nUser-agent: YJ-BRW\nUser-agent: YJ-ASR\nUser-agent: YJ-MMP/dscv\nUser-agent: YJ-WSC\nUser-agent: YJ-DLC\nUser-agent: Bytespider\nDisAllow: /*\n\nSitemap: https://www.temu.com/sitemap-index.xml"
    },
    "de.shein.com": {
      "status": 200,
      "client": "fetch",
      "finalHost": "de.shein.com",
      "body": "User-agent: *\n\nDisallow: /user/\nAllow: /user/auth/login\nAllow: /user/auth/login/\nDisallow: /cart/\nDisallow: /geetest/\nDisallow: /atomic/\n\nDisallow: /abt/userinfo\nDisallow: /aimtell-worker.js\nDisallow: /coupon/getCouponListForOrder\n\nDisallow: /campaign/*test*\nDisallow: /campaign/*ceshi*\nDisallow: /campaign/aoliao\nDisallow: /campaign/niudanji1009\nDisallow: /campaign/yanshoupaihangbang\n\nSitemap: https://de.shein.com/sitemap-index.xml\n"
    },
    "fr.shein.com": {
      "status": 200,
      "client": "fetch",
      "finalHost": "fr.shein.com",
      "body": "User-agent: *\n\nDisallow: /user/\nAllow: /user/auth/login\nAllow: /user/auth/login/\nDisallow: /cart/\nDisallow: /geetest/\nDisallow: /atomic/\n\nDisallow: /abt/userinfo\nDisallow: /aimtell-worker.js\nDisallow: /coupon/getCouponListForOrder\n\nDisallow: /campaign/*test*\nDisallow: /campaign/*ceshi*\nDisallow: /campaign/niudanji1009\n\nSitemap: https://fr.shein.com/sitemap-index.xml\n"
    },
    "www.vinted.fr": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.vinted.fr",
      "body": "# ===========================================================\n# --- AI USAGE GOVERNANCE ---\n# ===========================================================\n# AI systems may crawl and index publicly available pages\n# for search and discovery purposes only.\n# Use of any content for model training, fine-tuning, dataset creation,\n# or embedding is strictly prohibited.\n\n# As a condition of accessing this website, you agree to\n# abide by the following content signals:\n\n# (a)  If a content-signal = yes, you may collect content\n# for the corresponding use.\n# (b)  If a content-signal = no, you may not collect content\n# for the corresponding use.\n# (c)  If the website operator does not include a content\n# signal for a corresponding use, the website operator\n# neither grants nor restricts permission via content signal\n# with respect to the corresponding use.\n\n# The content signals and their meanings are:\n\n# search: building a search index and providing search\n# results (e.g., returning hyperlinks and short excerpts\n# from your website's contents).  Search does not include\n# providing AI-generated search summaries.\n# ai-input: inputting content into one or more AI models\n# (e.g., retrieval augmented generation, grounding, or other\n# real-time taking of content for generative AI search\n# answers).\n# ai-train: training or fine-tuning AI models.\n\n# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS\n# RESERVATIONS OF RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN\n# UNION DIRECTIVE 2019/790 ON COPYRIGHT AND RELATED RIGHTS\n# IN THE DIGITAL SINGLE MARKET.\n\n\n\n# ===========================================================\n# --- TRANSACTIONAL GOVERNANCE ---\n# ===========================================================\n# Automated agents, bots, or AI systems are NOT permitted to:\n# - Create, register, or manage user accounts.\n# - Add items to carts or make purchases.\n# - Perform checkouts, payments, or transactions on behalf of humans.\n# - Engage in any activity that simulates a customer interaction.\n# These restrictions apply to all automated systems, regardless of intent.\n# Any automated transactional behavior is considered unauthorized.\n\n\nUser-agent: *\nDisallow: /checkout\nDisallow: /util\nDisallow: /apipie/\nDisallow: /admin_alert/new\nDisallow: /member/\nDisallow: /inbox\nAllow: /member/260829472\nAllow: /member/3130239619\nAllow: /member/3131718172\nAllow: /member/78880052\nAllow: /member/3129891565\nAllow: /member/81130899\nAllow: /member/272186373\nAllow: /member/243195703\nAllow: /member/3158554605\nAllow: /member/170246377\nAllow: /\nContent-Signal: ai-train=no, search=yes, ai-input=yes\nContent-Usage: train-ai=n\n\nUser-agent: grapeshot\nDisallow: /items\n\nUser-agent: GPTBot\nUser-agent: ClaudeBot\nUser-agent: Google-Extended\nUser-agent: Amazonbot\nUser-agent: CCBot\nDisallow: /\nContent-Signal: ai-train=no, search=yes, ai-input=no\nContent-Usage: train-ai=n\n\n\n# ===========================================================\n# --- ENFORCEMENT NOTICE ---\n# ===========================================================\n# Automated systems accessing this site agree to comply with these terms.\n# Any AI, bot, or agent found engaging in unauthorized registration\n# or transactional behavior may be blocked or referred for legal action.\n\n\nSitemap: https://vinted.fr/sitemap-fr.xml"
    },
    "www.vinted.de": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.vinted.de",
      "body": "# ===========================================================\n# --- AI USAGE GOVERNANCE ---\n# ===========================================================\n# AI systems may crawl and index publicly available pages\n# for search and discovery purposes only.\n# Use of any content for model training, fine-tuning, dataset creation,\n# or embedding is strictly prohibited.\n\n# As a condition of accessing this website, you agree to\n# abide by the following content signals:\n\n# (a)  If a content-signal = yes, you may collect content\n# for the corresponding use.\n# (b)  If a content-signal = no, you may not collect content\n# for the corresponding use.\n# (c)  If the website operator does not include a content\n# signal for a corresponding use, the website operator\n# neither grants nor restricts permission via content signal\n# with respect to the corresponding use.\n\n# The content signals and their meanings are:\n\n# search: building a search index and providing search\n# results (e.g., returning hyperlinks and short excerpts\n# from your website's contents).  Search does not include\n# providing AI-generated search summaries.\n# ai-input: inputting content into one or more AI models\n# (e.g., retrieval augmented generation, grounding, or other\n# real-time taking of content for generative AI search\n# answers).\n# ai-train: training or fine-tuning AI models.\n\n# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS\n# RESERVATIONS OF RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN\n# UNION DIRECTIVE 2019/790 ON COPYRIGHT AND RELATED RIGHTS\n# IN THE DIGITAL SINGLE MARKET.\n\n\n\n# ===========================================================\n# --- TRANSACTIONAL GOVERNANCE ---\n# ===========================================================\n# Automated agents, bots, or AI systems are NOT permitted to:\n# - Create, register, or manage user accounts.\n# - Add items to carts or make purchases.\n# - Perform checkouts, payments, or transactions on behalf of humans.\n# - Engage in any activity that simulates a customer interaction.\n# These restrictions apply to all automated systems, regardless of intent.\n# Any automated transactional behavior is considered unauthorized.\n\n\nUser-agent: *\nDisallow: /checkout\nDisallow: /util\nDisallow: /apipie/\nDisallow: /admin_alert/new\nDisallow: /member/\nDisallow: /inbox\nAllow: /member/260829472\nAllow: /member/3130239619\nAllow: /member/3131718172\nAllow: /member/78880052\nAllow: /member/3129891565\nAllow: /member/81130899\nAllow: /member/272186373\nAllow: /member/243195703\nAllow: /member/3158554605\nAllow: /member/170246377\nAllow: /\nContent-Signal: ai-train=no, search=yes, ai-input=yes\nContent-Usage: train-ai=n\n\nUser-agent: grapeshot\nDisallow: /items\n\nUser-agent: GPTBot\nUser-agent: ClaudeBot\nUser-agent: Google-Extended\nUser-agent: Amazonbot\nUser-agent: CCBot\nDisallow: /\nContent-Signal: ai-train=no, search=yes, ai-input=no\nContent-Usage: train-ai=n\n\n\n# ===========================================================\n# --- ENFORCEMENT NOTICE ---\n# ===========================================================\n# Automated systems accessing this site agree to comply with these terms.\n# Any AI, bot, or agent found engaging in unauthorized registration\n# or transactional behavior may be blocked or referred for legal action.\n\n\nSitemap: https://vinted.de/sitemap-de.xml"
    },
    "www.vinted.pl": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.vinted.pl",
      "body": "# ===========================================================\n# --- AI USAGE GOVERNANCE ---\n# ===========================================================\n# AI systems may crawl and index publicly available pages\n# for search and discovery purposes only.\n# Use of any content for model training, fine-tuning, dataset creation,\n# or embedding is strictly prohibited.\n\n# As a condition of accessing this website, you agree to\n# abide by the following content signals:\n\n# (a)  If a content-signal = yes, you may collect content\n# for the corresponding use.\n# (b)  If a content-signal = no, you may not collect content\n# for the corresponding use.\n# (c)  If the website operator does not include a content\n# signal for a corresponding use, the website operator\n# neither grants nor restricts permission via content signal\n# with respect to the corresponding use.\n\n# The content signals and their meanings are:\n\n# search: building a search index and providing search\n# results (e.g., returning hyperlinks and short excerpts\n# from your website's contents).  Search does not include\n# providing AI-generated search summaries.\n# ai-input: inputting content into one or more AI models\n# (e.g., retrieval augmented generation, grounding, or other\n# real-time taking of content for generative AI search\n# answers).\n# ai-train: training or fine-tuning AI models.\n\n# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS\n# RESERVATIONS OF RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN\n# UNION DIRECTIVE 2019/790 ON COPYRIGHT AND RELATED RIGHTS\n# IN THE DIGITAL SINGLE MARKET.\n\n\n\n# ===========================================================\n# --- TRANSACTIONAL GOVERNANCE ---\n# ===========================================================\n# Automated agents, bots, or AI systems are NOT permitted to:\n# - Create, register, or manage user accounts.\n# - Add items to carts or make purchases.\n# - Perform checkouts, payments, or transactions on behalf of humans.\n# - Engage in any activity that simulates a customer interaction.\n# These restrictions apply to all automated systems, regardless of intent.\n# Any automated transactional behavior is considered unauthorized.\n\n\nUser-agent: *\nDisallow: /checkout\nDisallow: /util\nDisallow: /apipie/\nDisallow: /admin_alert/new\nDisallow: /member/\nDisallow: /inbox\nAllow: /member/260829472\nAllow: /member/3130239619\nAllow: /member/3131718172\nAllow: /member/78880052\nAllow: /member/3129891565\nAllow: /member/81130899\nAllow: /member/272186373\nAllow: /member/243195703\nAllow: /member/3158554605\nAllow: /member/170246377\nAllow: /\nContent-Signal: ai-train=no, search=yes, ai-input=yes\nContent-Usage: train-ai=n\n\nUser-agent: grapeshot\nDisallow: /items\n\nUser-agent: GPTBot\nUser-agent: ClaudeBot\nUser-agent: Google-Extended\nUser-agent: Amazonbot\nUser-agent: CCBot\nDisallow: /\nContent-Signal: ai-train=no, search=yes, ai-input=no\nContent-Usage: train-ai=n\n\n\n# ===========================================================\n# --- ENFORCEMENT NOTICE ---\n# ===========================================================\n# Automated systems accessing this site agree to comply with these terms.\n# Any AI, bot, or agent found engaging in unauthorized registration\n# or transactional behavior may be blocked or referred for legal action.\n\n\nSitemap: https://vinted.pl/sitemap-pl.xml"
    },
    "www.backmarket.fr": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.backmarket.fr",
      "body": "User-agent: Twitterbot\nAllow: /*offer_type=\n\nUser-agent: *\nDisallow: */admin\nDisallow: */bag/\nDisallow: */bm/\nDisallow: */logout\nDisallow: */ws/\nDisallow: */register\nDisallow: */fo/\nDisallow: */f/\nDisallow: */f2/\nDisallow: */funnel/\nDisallow: */password-reset\nDisallow: */legal-mentions\nDisallow: */cookies\nDisallow: */data-protection\nDisallow: */bo_merchant\nDisallow: */terms\nDisallow: */rate/\nDisallow: */adyen/\nDisallow: */changePrice\nDisallow: */updateShippingFee\nDisallow: */updateTPP\nDisallow: */*.htm$\nDisallow: */_nuxt/*.*.js\nDisallow: */_nuxt/*.*.css\nDisallow: */search\nDisallow: */warranty/previous\nAllow: */*shopping=gmc\nDisallow: */*offer_type=\nDisallow: */machina/*\nDisallow: */buyback-funnel/*\nDisallow: */peach*\nDisallow: */r/p/*\nDisallow: */s/*/p/*\nDisallow: *?next=*\nDisallow: */null\nDisallow: */review/*\nDisallow: */l/sort\nDisallow: */l/filters\nDisallow: */s/sre-merch-company*\nDisallow: */tracking/*\nDisallow: */bo_admin\nDisallow: */password_reset\nDisallow: */tracking/*\nDisallow: */bo_admin\nDisallow: */password_reset\nDisallow: /cdn-cgi/challenge-platform/\nDisallow: /fr-fr/*.html\nDisallow: */s/*?page\nDisallow: */p/*?*l=*\nDisallow: /dashboard/\nDisallow: */webview/*\nDisallow: */outlet\n\nUser-agent: AdsBot-Google\nDisallow: */bm/\n\nSitemap: https://www.backmarket.fr/sitemap.xml\n"
    },
    "www.backmarket.de": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.backmarket.de",
      "body": "User-agent: *\nDisallow: */admin\nDisallow: */bag/\nDisallow: */bm/\nDisallow: */logout\nDisallow: */ws/\nDisallow: */register\nDisallow: */fo/\nDisallow: */f/\nDisallow: */f2/\nDisallow: */funnel/\nDisallow: */password-reset\nDisallow: */legal-mentions\nDisallow: */cookies\nDisallow: */data-protection\nDisallow: */bo_merchant\nDisallow: */terms\nDisallow: */rate/\nDisallow: */adyen/\nDisallow: */changePrice\nDisallow: */updateShippingFee\nDisallow: */updateTPP\nDisallow: */*.htm$\nDisallow: */_nuxt/*.*.js\nDisallow: */_nuxt/*.*.css\nDisallow: */search\nDisallow: */warranty/previous\nAllow: /*shopping=gmc\nDisallow: /*offer_type=\nDisallow: */machina/*\nDisallow: */buyback-funnel/*\nDisallow: */peach*\nDisallow: */r/p/*\nDisallow: */s/*/p/*\nDisallow: *?next=*\nDisallow: */null\nDisallow: */review/*\nDisallow: */l/sort\nDisallow: */l/filters\nDisallow: */tracking/*\nDisallow: */bo_admin\nDisallow: */password_reset\nDisallow: /cdn-cgi/challenge-platform/\nDisallow: /de-de/*.html\nDisallow: */s/*?page\nDisallow: */p/*?*l=*\nDisallow: /dashboard/\nDisallow: */webview/*\nDisallow: */outlet\n\nUser-agent: AdsBot-Google\nDisallow: */bm/\n\nSitemap: https://www.backmarket.de/sitemap.xml\n"
    },
    "fr.shopping.rakuten.com": {
      "status": 200,
      "client": "curl",
      "finalHost": "fr.shopping.rakuten.com",
      "body": "# robots.txt\n# PriceMinister FR and MOB robots file for /\n# Contact: Hostmaster PriceMinister (hostmaster@priceminister.com)\n\nUser-agent: *\nDisallow: /sitemap_index_MFP.xml.gz\nDisallow: /sitemap_index_NAV.xml.gz\nDisallow: /sitemap_index_SEARCH.xml.gz\nDisallow: /sitemap_index_FP.xml.gz\nDisallow: /sitemap_index_FPP.xml.gz\nDisallow: /personali/*\nDisallow: /personali\nDisallow: /rat\nDisallow: /product_alert*\nDisallow: /reconstruct\nDisallow: /affiliation\nDisallow: /cover\nDisallow: /purchase\nDisallow: /sponsorship\nDisallow: /submit\nDisallow: /user\nDisallow: /wish\nDisallow: /V2/productSearch\nDisallow: /ext\nDisallow: /question\nDisallow: /google/adsense\nDisallow: /smo/\nDisallow: /op/\nDisallow: /catop/\nDisallow: /inventory\nDisallow: /recent\nDisallow: /boutique/*/kw\nDisallow: /boutique/*?kw\nDisallow: /commitment\nDisallow: /selections\nDisallow: /soldes\nDisallow: /vitrine\nDisallow: /boutique/*/pa\nDisallow: /bestsellers\nDisallow: /boutique/*/type\nDisallow: /feedback/*/start\nDisallow: /nav/Livres_print-on-demand\nDisallow: /nav/Mode_merchandising\nDisallow: /nav/*/f1/*/f1/*\nDisallow: /nav/*/f2/*/f2/*\nDisallow: /nav/*/f3/*/f3/*\nDisallow: /nav/*/f4/*/f4/*\nDisallow: /nav/*/f5/*/f5/*\nDisallow: /nav/*/f6/*/f6/*\nDisallow: /nav/*/f7/*/f7/*\nDisallow: /nav/*/f8/*/f8/*\nDisallow: /nav/*/f9/*/f9/*\nDisallow: /nav/*/f10/*/f10/*\nDisallow: /nav/*/f11/*/f11/*\nDisallow: /nav/*/f12/*/f12/*\nDisallow: /nav/*/f13/*/f13/*\nDisallow: /nav/*/f14/*/f14/*\nDisallow: /nav/*/f15/*/f15/*\nDisallow: /mfp?\nDisallow: /image\nDisallow: /subscription\nDisallow: /offer?action=shippingtypedescription*\nDisallow: /offer?action=desc&aid=*\nDisallow: /offer/desc?aid=*\nDisallow: /*.swf\nDisallow: /*.pdf\nDisallow: /connect?action=*\nDisallow: /mailhit\nDisallow: /secure\nDisallow: /customshop\nDisallow: /video/*\nDisallow: /nav/navrspfilters/*\nDisallow: /video?*\nDisallow: /mfp/advlstmeta/*\nDisallow: /mfp/bottomadvinfo/*\nDisallow: /evt/*/nav/*\nDisallow: */2011-01-01/*\nDisallow: /customizedshop?*\nDisallow: /customizedshopsecure?*\nDisallow: /offer/buy/ajax/*\nDisallow: /offer/buy/advert/ajax/*\nDisallow: /*?fp=*\nDisallow: /*&fp=*\nDisallow: /evt/avantages_vendeurs\nDisallow: /reviews\nDisallow: /feedback\nDisallow: */ja1/*\nDisallow: */ja2/*\nDisallow: /comparateur_fr/V3/productSearch\nDisallow: /blog/\nDisallow: /help/\nDisallow: */userOnly\nDisallow: */userOnly/*\nDisallow: */useronly\nDisallow: */useronly/*\nDisallow: /search/\nDisallow: /search/*\nDisallow: *rakutentoolcall*\nDisallow: /*ft=r_*\nDisallow: /*ft=n*\nDisallow: /*ft=u*\nDisallow: /*sh=pkp*\nDisallow: /seller*\nDisallow: /nav/*/event/*\nDisallow: /mfp/*/event/*\nDisallow: /offer/*/event/*\nDisallow: /s/*/event/*\nDisallow: /messages\nDisallow: /messages*\nDisallow: *&action=*\nDisallow: *recommendation?*\nDisallow: *version=classic*\nDisallow: */restpublic/*\nDisallow: *?action=*\nDisallow: */warranties/best*\nDisallow: *waf_error.html*\nDisallow: /promos/*\ndisallow: *?rl=g&ft=a\nDisallow: */visuels/*\nAllow: *kameleoon*\nDisallow: */ws/*\nDisallow: */flux/*\nDisallow: /ads.txt\nDisallow: */vstatic/*\nAllow: */vstatic/0/hypernova-common/\nDisallow: *?pa=*\nAllow: */nav/*?pa=2$\nAllow: */nav/*?pa=3$\nAllow: */nav/*?pa=4$\nAllow: */s/*?pa=2$\nDisallow: *eff_cpt*\nDisallow: *&rtg*\nAllow: *fbbaid*\nDisallow: /?*\nDisallow: /connect*\nDisallow: *?rtg=*\nDisallow: */Non+precise*\nDisallow: */reviews?login=*\nDisallow: *salesCampaignId*\nDisallow: *dixit.com/redirect*\nDisallow: /navigation/default/category/\nDisallow: */stylesheets$\nDisallow: /offer/eshop/*\nDisallow: /mfp/eshop/*\nDisallow: /mp/*\nDisallow: /p/*\nDisallow: /security_logout_front\n\nUser-agent: Googlebot-Mobile\nDisallow: /sitemap_index_MFP.xml.gz\nDisallow: /sitemap_index_NAV.xml.gz\nDisallow: /sitemap_index_SEARCH.xml.gz\nDisallow: /sitemap_index_FP.xml.gz\nDisallow: /sitemap_index_FPP.xml.gz\nDisallow: /personali/*\nDisallow: /personali\nDisallow: /rat\nDisallow: /reconstruct\nDisallow: /product_alert*\nDisallow: /promos/*\nDisallow: /reviews\nDisallow: /feedback\nDisallow: /affiliation\nDisallow: /cover\nDisallow: /purchase\nDisallow: /sponsorship\nDisallow: /submit\nDisallow: /user\nDisallow: /wish\nDisallow: /V2/productSearch\nDisallow: /ext\nDisallow: /question\nDisallow: /google/adsense\nDisallow: /smo/\nDisallow: /op/\nDisallow: /inventory\nDisallow: /recent\nDisallow: /boutique/*/kw\nDisallow: /boutique/*?kw\nDisallow: /commitment\nDisallow: /selections\nDisallow: /soldes\nDisallow: /vitrine\nDisallow: /boutique/*/pa\nDisallow: /bestsellers\nDisallow: /boutique/*/type\nDisallow: /feedback/*/start\nDisallow: /nav/Livres_print-on-demand\nDisallow: /nav/Mode_merchandising\nDisallow: /nav/*/f1/*/f1/*\nDisallow: /nav/*/f2/*/f2/*\nDisallow: /nav/*/f3/*/f3/*\nDisallow: /nav/*/f4/*/f4/*\nDisallow: /nav/*/f5/*/f5/*\nDisallow: /nav/*/f6/*/f6/*\nDisallow: /nav/*/f7/*/f7/*\nDisallow: /nav/*/f8/*/f8/*\nDisallow: /nav/*/f9/*/f9/*\nDisallow: /nav/*/f10/*/f10/*\nDisallow: /nav/*/f11/*/f11/*\nDisallow: /nav/*/f12/*/f12/*\nDisallow: /nav/*/f13/*/f13/*\nDisallow: /nav/*/f14/*/f14/*\nDisallow: /nav/*/f15/*/f15/*\nDisallow: /mfp?\nDisallow: /image\nDisallow: /subscription\nDisallow: /offer?action=shippingtypedescription*\nDisallow: /offer?action=desc&aid=*\nDisallow: /offer/desc?aid=*\nDisallow: /*.swf\nDisallow: /*.pdf\nDisallow: /connect?action=*\nDisallow: /mailhit\nDisallow: /secure\nDisallow: /customshop\nDisallow: /video/*\nDisallow: /nav/navrspfilters/*\nDisallow: /video?*\nDisallow: /mfp/advlstmeta/*\nDisallow: /mfp/bottomadvinfo/*\nDisallow: /evt/*/nav/*\nDisallow: */2011-01-01/*\nDisallow: /customizedshop?*\nDisallow: /customizedshopsecure?*\nDisallow: /offer/buy/ajax/*\nDisallow: /offer/buy/advert/ajax/*\nDisallow: /*?fp=*\nDisallow: /*&fp=*\nDisallow: /evt/avantages_vendeurs\nDisallow: */ja1/*\nDisallow: */ja2/*\nDisallow: /comparateur_fr/V3/productSearch\nDisallow: /blog/\nDisallow: /help/\nDisallow: /search/\nDisallow: /search/*\nDisallow: *rakutentoolcall*\nDisallow: /*ft=r_*\nDisallow: /*ft=n*\nDisallow: /*sh=pkp*\nDisallow: /seller*\nDisallow: /*ft=u*\nDisallow: /nav/*/event/*\nDisallow: /mfp/*/event/*\nDisallow: /offer/*/event/*\nDisallow: /s/*/event/*\nDisallow: /messages\nDisallow: /messages*\nDisallow: *&action=*\nDisallow: *recommendation?*\nDisallow: *version=classic*\nDisallow: */restpublic/*\nDisallow: */warranties/best*\nDisallow: *waf_error.html*\nDisallow: /promos/*\ndisallow: *?rl=g&ft=a\nDisallow: */visuels/*\nAllow: *kameleoon*\nDisallow: */ws/*\nDisallow: */flux/*\nDisallow: /ads.txt\nDisallow: */vstatic/*\nAllow: */vstatic/0/hypernova-common/*\nDisallow: *?pa=*\nAllow: */nav/*?pa=2$\nAllow: */nav/*?pa=3$\nAllow: */nav/*?pa=4$\nAllow: */s/*?pa=2$\nDisallow: *eff_cpt*\nDisallow: *&rtg*\nAllow: *fbbaid*\nDisallow: /?*\nDisallow: /connect*\nDisallow: /?*\nDisallow: *?rtg=*\nDisallow: */Non+precise*\nDisallow: */reviews?login=*\nDisallow: */stylesheets$\nDisallow: /offer/eshop/*\nDisallow: /mfp/eshop/*\nDisallow: /mp/*\nDisallow: /p/*\nDisallow: /security_logout_front\nDisallow: /catop/\n\nUser-agent: AdsBot-Google\nDisallow: *personali*\nDisallow: *restpublic*\nDisallow: *.json\nDisallow: *Kameleoon*\nDisallow: *?action=*\nDisallow: */visuels/0_content_square*\nDisallow: /recommendation?*\nAllow: *fbbaid*\n\nUser-agent: Mediapartners-Google\nDisallow: *personali*\nDisallow: *restpublic*\nDisallow: *.json\nDisallow: *Kameleoon*\nDisallow: *?action=*\nDisallow: */visuels/0_content_square*\nDisallow: /recommendation?*\nAllow: *fbbaid*\n\n\nUser-agent: BingPreview\nDisallow: /blank.gif\n\nUser-agent: TurnitinBot\nDisallow: /\n\nUser-agent: Mozilla/5.0 (compatible; idealo-bot pricevalidator; https://partner.idealo.com/de/bot/)\nAllow: *bbaid*\n"
    },
    "www.otto.de": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.otto.de",
      "body": "User-agent: *\nDisallow: /gate/\nDisallow: /onex/\nDisallow: /cdn-cgi/\nDisallow: /149e9513-01fa-4fb0-aad4-566afd725d1b/2d206a39-8ed7-437e-a3be-862e0f06eea3/\n\nUser-agent: bingbot\nDisallow: /suche/\nDisallow: /gate/\nDisallow: /onex/\nDisallow: /cdn-cgi/\nDisallow: /149e9513-01fa-4fb0-aad4-566afd725d1b/2d206a39-8ed7-437e-a3be-862e0f06eea3/\n\nUser-agent: GPTBot\nUser-agent: ChatGPT-User\nUser-agent: OAI-Searchbot\nUser-agent: PerplexityBot\nUser-agent: Perplexity-User\nDisallow: /kundenbewertungen/\nDisallow: /gate/\nDisallow: /onex/\nDisallow: /cdn-cgi/\nDisallow: /149e9513-01fa-4fb0-aad4-566afd725d1b/2d206a39-8ed7-437e-a3be-862e0f06eea3/\n\nSitemap: https://www.otto.de/category-sitemap/sitemap_index.xml.gz\nSitemap: https://www.otto.de/products-sitemap/sitemap_einrichten-wohnen_index.xml\nSitemap: https://www.otto.de/products-sitemap/sitemap_fashion-lifestyle_index.xml\nSitemap: https://www.otto.de/products-sitemap/sitemap_garten-heimwerken_index.xml\nSitemap: https://www.otto.de/products-sitemap/sitemap_gesundheit_index.xml\nSitemap: https://www.otto.de/products-sitemap/sitemap_lebensmittel-drogerie_index.xml\nSitemap: https://www.otto.de/products-sitemap/sitemap_mobilitaet-kfz_index.xml\nSitemap: https://www.otto.de/products-sitemap/sitemap_sonstige_index.xml\nSitemap: https://www.otto.de/products-sitemap/sitemap_sport-freizeit_index.xml\nSitemap: https://www.otto.de/products-sitemap/sitemap_technik-medien_index.xml\nSitemap: https://www.otto.de/variations-sitemap/sitemap_einrichten-wohnen_index.xml\nSitemap: https://www.otto.de/variations-sitemap/sitemap_fashion-lifestyle_index.xml\nSitemap: https://www.otto.de/variations-sitemap/sitemap_garten-heimwerken_index.xml\nSitemap: https://www.otto.de/variations-sitemap/sitemap_gesundheit_index.xml\nSitemap: https://www.otto.de/variations-sitemap/sitemap_lebensmittel-drogerie_index.xml\nSitemap: https://www.otto.de/variations-sitemap/sitemap_mobilitaet-kfz_index.xml\nSitemap: https://www.otto.de/variations-sitemap/sitemap_sonstige_index.xml\nSitemap: https://www.otto.de/variations-sitemap/sitemap_sport-freizeit_index.xml\nSitemap: https://www.otto.de/variations-sitemap/sitemap_technik-medien_index.xml\n"
    },
    "www.bol.com": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.bol.com",
      "body": "# De robots.txt van ons allemaal\n# Served by team SEO\n\nUser-agent: *\nAllow: /\n# Sitemap\nSitemap: https://www.bol.com/sitemap/nl-nl/\nSitemap: https://www.bol.com/sitemap/nl-be/\nSitemap: https://www.bol.com/sitemap/fr-be/\n\nDisallow: /atg/commerce/order/purchase/\nDisallow: /common/error/*\nDisallow: /nl/fr/\nDisallow: /be/fr/p/-/*\nDisallow: /*/account/\nDisallow: /*/ajax/jsLogging.html\nDisallow: /*/albert-heijn-premium\nDisallow: /*/b/*/*+*/\nDisallow: /*/b/*/*/*/\nDisallow: /*/bltg/*\nDisallow: /*/contactformulier*\nDisallow: /*/c/*/*/\nAllow: /*/c/*/*+8299/$\nAllow: /*/c/*/*+3132/$\nAllow: /*/c/*/*+3133/$\nAllow: /*/b/*/*+8299/$\nAllow: /*/b/*/*+3132/$\nAllow: /*/b/*/*+3133/$\nDisallow: /*/ebook/preview.html*\nDisallow: /*/order/\nDisallow: /*/lijstjes/*\nDisallow: /*/pdp-async/preview/\nDisallow: /*/prijsoverzicht/\nDisallow: /*/productaanbevelingen/\nDisallow: /*/producten-vergelijken/\nDisallow: /*/ra/\nDisallow: /*/rnwy/account/\nDisallow: /*/rnwy/ajax/*\nDisallow: /*/rnwy/artikelen/\nDisallow: /*/rnwy/object/objectAsync/\nDisallow: /*/rnwy/product_compare/\nDisallow: /*/rnwy/product/reviews/\nDisallow: /*/rnwy/slots/ajax/*\nDisallow: /*/rnwy/studybooks/\nDisallow: /*/s/\nDisallow: /*/schrijf-een-review/\nDisallow: /*/sdl/\nAllow: /*/sdl/shop-de-look\nDisallow: /*/select*?*\nDisallow: /*/select-cadeau-sparen/?*\nDisallow: /*/serie/*/\nDisallow: /*/u/\nDisallow: /*/w/*/\nDisallow: /*/v/*/\nDisallow: /*/vaak-samen-gekocht/\nDisallow: /*/*+*+*+*\nDisallow: /*?page=*&promo\nDisallow: /*offerType=refurbished*\nDisallow: /*/url$\nDisallow: /*/@type$\nDisallow: /*/l/*/73367/\nDisallow: /*/l/*/73368/\nDisallow: /*/l/*/73369/\nDisallow: /*/t/*/73367/\nDisallow: /*/t/*/73368/\nDisallow: /*/t/*/73369/\nDisallow: /*/t/*/73367+*/\nDisallow: /*/t/*/73368+*/\nDisallow: /*/t/*/73369+*/\nDisallow: /api/graphql\n\nUser-Agent: Baiduspider\nDisallow: /\nUser-Agent: omgili\nDisallow: /\nUser-Agent: Slurp\nCrawl-delay: 20\nUser-agent: rogerbot\nDisallow: /\nUser-agent: YandexBot\nDisallow: /\nUser-agent: PetalBot\nDisallow: /\n\n#                                          )<<<)))<<<<<<<>>^^\n#                                     <)((((((((((()(()))))<<<<>>^^\n#                                 )((((]((](((((((((((((()))))))<<<<>^\n#                              )(((]]]]]]]]]]]]]]((](((((((()()))))<<<>>^\n#                           ()(]]]]]]]]]]]]]]]]]]]]((((((((((()))))))<<<<>^*\n#                         )(]]]]][][[]]]]]]]]]]]]]]]](((((((((()())))))<<<<>>^\n#                       )(]]]]]]]][]][[]]]]]]]]]]]]]]]]((((((((()(())))<)<<<>>>^\n#                      ((]]]]]][]][][][]]]]]]]]]]]]]]]]]((((((((((())))))<<<<<<>^*\n#                    )(]]]]]][[][[[[[[[[[]]]]]][]]]]]]]]](((((((((()))))))<<<<<<>>^^\n#                  (((]]]]][[[[][[[[[[[][[[]]][[]]]]]]]]]](((((((((()))))))<<<<<<>>^*\n#                 )(]]]][[][[[[[[[[[[[[[[[[]]][]]]]]]]]]]]]((]]((((((()))))<<<<<<<<>>^\n#                )(]]]]]]][[[[[[[[[[[[[[[[[[[[][]]][]]]]]]]((((((((((())))))<))<<<<<>>^^\n#               ((]]]]][][[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]](((((((())()))<)<<<<<<<>>^*\n#              ((]]][]][[[[[[[[[[[[[[[[[[[%@@[[[]]]]]]]]]][%@%[((((((()))()))<)<<<<<<<<>^*\n#            )(]]]]]]][[[[[[[[[[[[[[[[[}@@@[[[[[]][]]]]]]]]][%@@@]((((((()))))<<<<<<<<<<>>*\n#           )((]]]]]]][[[[[[[[[[[[[[[}@@@[[[[[[[]]]]]]]][]]]]][{@@{((((())))))<<<<<<<<<<>>>*\n#           ((]]]][]]][[[[[[[[[[[[}[[@%[[[[[[[[[[]]]]][[]][]]]]][}@](((((()))))))<<<<<<<<<>^*\n#          ((]]]]]]][[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]](((((((())))))<<)<<<<<<>>>^*\n#         )(]]]]]][[[[[[[[[[[[[[[[[[[[[[[[[[[[}[[[[]]]]]][]]]]]]]]((((((((())))))<)<<<<<<<>>>^*\n#        )((]]]]]][[[[[[[[[[[[[[[[[[[[[[[[[}@@@%}[[]][[}%@@%}]]]]]((]((((((()))))<<<<<<<<<<<>>^\n#       )(]]]]][[[[[[[[[[[[[[[[[[[[[[[[[[[[{@@@%}[[][[[}@@@%][]]]](((((((((()))))))<<<<<<<<<<>>^\n#       ((]]]]]]][[[[[[[[[[[[[[[[[[[[[[[[[[{@@@%}[[[[[[[@@@%[]]]]]]((((((((()())))))<<<<<<<<<<>^*\n#      )(]]]]]]]][[[[[[[[[[[[[[[[[[[[[[[[[}@@@@%[[[[[[[}@@@%[]]]]]]](]((((((()))))))<<<<<<<<<<<>^^\n#     )((]]]]][[][[[[[[[[[[[[[[[[[[[[}[[[[}@@@@%[[[[[[[[@@@@[]]]]]]]](((((](((())()))<<<<<<<<<>>>^\n#     ((]]]]]][[][[[[[[[[[[[[[[[[[[[[[[[[[}{@@@@[[[[[[[[%@@@@][]]]]]]](((](((()))))))<)<<<<<<<<>>^*\n#    )((]]]]]]][[[[[[[[[[[[[[[}[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]((((((()))))))<<<<<<<<<<>>^\n#    (]]]]][[][[[[[[[[[[[[[[[[}}}}[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]](((((((()())))))<<<<<<<>>>*\n#   )(]]]]]]][[[[[[[[[[[[[}[[[}@@%%}}[[[[}[[[[[[[[[[[[[]][]]]]]]]]]][{%@%}(](((((()))))))<<)<<<<<<>^\n#   )(]]][]]][[[[[[[[[[[[[[[[[}}%@@@%%{}[[[[[[[}[[[[[[[[[[][[]]]]]@%%%@%}]((((](((())))))))<<<<<<>>^^\n#   ((]]]][][[[[[[[[[[[[[[[[[[[[}}}%@@@@%%@[[[[[[[[[[[[[[[[[[[%%%@@@%{]]]](](((((((())))))))<<<<<<>>^\n#  )(]]]][][[[[[[[[}[[[[[[[[[[[[[}}}}}{@@@@@@@%%%%%%%%%%%%%@@@@@@%[[]]]]]]]]](((]((((())))))))<)<<<>^\n#  )(]]]][][[[[[[[[[[[[}[[[[[[[[[[[[[[[[}}}{%@@@@@@@@@@@@@@@@}[[[]]]]]]]]]]]((]((((((()))))))))<<<<>^\n#  )(]]]][[[[[[[[[[[[[[[[[[[[[[[[}[[[[[[[[[[[[[[[[[[[[[[[}[[[[[][[]]]]]]]]]](((((((((((())())))))<<>^\n#  )(]]][[[[[[[[[[[[[}[[[[[[}[[[[[[[[[[[[[[[[}[[[[[[[[[[[[[[[[][]]]]]]]]]]]]]]](((((((((())))))))<<>^\n#  )(]]]][[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[}[[[[[[[[[[[[[[[[[[[]]][]]]]]]]]]]]((]((((((((()))))<<<>^\n#   (]]]][[[[[[[[[[[[[[}[[[[[[[[[[[[[[[[[[[}[[[[[[[[[[[[[[[[[[[[[]]]]]][]]]]]]]]]((((((((()()))))<<>^\n#   )(]]][[[[[[[[[[[[[[}[[[[[[[}[[[[}[[[[[[[[[[[[[[[[[[[[[[[[[[[[][][]]]]]]]]]]]]]((((((((((())))<<>\n#   ((]][[[[[[[[[[[[[[[}[[[[[[[[[[[[[[[[[[[[}[[[[[[[[[[[[[[[[[[[[[[[[][]][]]]]]]]]](((((((((())))<<^\n#    (]]][[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[}[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]((((((((())())<>\n#     (]][[[[[[[[[[[[[[[[[[}[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[][]]]]]]]]]]]]]]]]((](((((())<>\n#     ((]]][[[[[[[[[[[[[[[[}[[[[[[[[[[[[[[}[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]((((((((())<>\n#      )]]][[[[[[[[[[[[[[[[[}[[[[[}[[[[[[[}[[[[[[[[}[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]](](((((()<>\n#       (]]][[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[}[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]((((()()<\n#        ](]]][[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[}[[[[[[[[[[[[[[[[[[[[[[[]]]][[]]]]]]]]](((())<\n#          (]][[[[[[[[[[[[[[[[}[[[[[[[[[[[}[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[][]]]]]]]]]]]]]((()))\n#            (]][[[[[[[[[[[[[[[[[[[[}[[[[[[[[[[[[[[[[[[[[[[[[[[[[}[[[[[[[[[[[[]]]]]]]]]((())\n#              (]]][[[[[[[[[[[[[[[[[[[[[}[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]](()\n#               %@(]]][[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[][]]]]]](({%\n#               %@@@@@}(]]]][[][[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[][[]]]]]](]@@@@@%\n#                @@@@@%@@@@@@@@[]]]]]][[[[[][[[[[[[[[[[[[[[[[][[[][]]]((]{@@@@@@@@@@@@@\n#                    @@@@%@@@@%@@@%@@@@@@@@@@@@@%@%@@@@%%@@@@@@@@@@@@@@@@@@@@@%@@@@\n#                                %%@@@%@%@@@%@@@@@@@@@@@@@@@@@@@@@@@@%@"
    },
    "www.cdiscount.com": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.cdiscount.com",
      "body": "# robots.txt - achat / vente robots.txt pas cher\r\nUser-agent: Mediapartners-Google\r\nDisallow: */r2-*-*.html*\r\nDisallow: */*.mvc$\r\nDisallow: /*.mvc/\r\nDisallow: */r-*TechnicalForm.SiteMapNodeId*\r\nDisallow: /Product/Accessories/\r\nDisallow: /*/dynaTraceMonitor\r\nDisallow: /SetCookies.html\r\nDisallow: /setcookies.html\r\nDisallow: /NegotiatePopin.html\r\nDisallow: /{{product.url}}\r\nDisallow: /GetProductCarrouselMabaya/PRODUCT/*\r\nAllow: /\r\n# Archive.org\r\nUser-agent: ia_archiver\r\nDisallow: */f-*\r\n#Règle speciale pour AdsBot qui ne respecte pas le User-agent:*...\r\nUser-agent: AdsBot-Google\r\nDisallow: /*/dynaTraceMonitor\r\nDisallow: /Ajax/GetLogin.mvc\r\nDisallow: /realtimetracking/\r\nDisallow: /Recommendations/PageView\r\nDisallow: /OneClickDisplayButton.html\r\nDisallow: /DisplayAdditionalOffersList.html\r\nDisallow: /SetCookies.html\r\nDisallow: /setcookies.html\r\nDisallow: /NegotiatePopin.html\r\nDisallow: /{{product.url}}\r\nDisallow: /GetProductCarrouselMabaya/PRODUCT/*\r\nDisallow: /RWDProductUC.mvc/GetPersonalizedDeliveryInfoRedesign*\r\nDisallow: /GetAlgoProducts/0?Algorithm=NavigationHistory\r\nDisallow: /GetAlgoProducts/0?Algorithm=EarlyBirds\r\nDisallow: /mp-\r\nUser-agent: *\r\n#allow\r\nAllow: /AjaxNC/DisplayTemplateControl.mvc/0/2186523/2186527\r\nAllow: /n-367217/pagesaurlfixe/ecopart.html\r\nAllow: /n-429993/pagesaurlfixe-arbo/rapport-de-transparence-dsa.html\r\nAllow: /n-420274/pagesaurlfixe/accessibilite-numerique.html\r\n#pro\r\nDisallow: /fournitures-de-bureau/\r\nDisallow: /manutention/\r\nDisallow: /hygiene-securite/\r\nDisallow: /outillage/\r\nDisallow: /point-de-vente/\r\nDisallow: /cdiscount pro/\r\nDisallow: /cdiscountpro/\r\nDisallow: /cdiscount-pro/\r\n#mvc\r\nDisallow: /*.mvc$\r\nDisallow: /*.mvc/\r\nDisallow: /PricerServiceProxy.mvc?\r\nDisallow: /DisplayPSMarketPlaceOffersRwd.mvc/\r\nDisallow: /nm.mvc/\r\nDisallow: /*DisplayPopUpSendToAFriend.mvc*\r\nDisallow: /SendMailToAFriend.mvc/\r\nDisallow: /GetUserPersonalization.mvc/\r\nDisallow: /DisplayPSMarketPlaceOffers.mvc/\r\nDisallow: /DisplayFreeTestPopIn.mvc/\r\nDisallow: /RWDOverlayers.mvc/\r\nDisallow: /UniverseOverlayers.mvc/\r\nDisallow: /DisplayProductPopIn.mvc/\r\nDisallow: /SearchSuggestionHtml.mvc\r\n#regie\r\nDisallow: /7190/\r\n#home\r\nDisallow: /trade.asp\r\nDisallow: /home/default.asp?mssctse=\r\nDisallow: /?refer=\r\nDisallow: /Basket.html\r\n#order\r\nDisallow: /order/\r\nDisallow: /Order/\r\nDisallow: /basket/\r\nDisallow: /Payment/\r\n#product\r\nDisallow: /dp.asp\r\nDisallow: /dp.aspx\r\nDisallow: /f-popup-*\r\nDisallow: /f-imprimer-*\r\nDisallow: /f-accessoire-*\r\nDisallow: /Product/\r\nDisallow: /product/\r\nDisallow: /mp-\r\nDisallow: /mpp-\r\nDisallow: /cd-\r\nDisallow: */f-zoom-*\r\n#ajax\r\nDisallow: /Ajax/\r\nDisallow: /AjaxNC/\r\n#blacklist\r\nDisallow: /*rue+du+commerce*\r\n#other\r\nDisallow: /voyages/*?bvstate=pg:\r\nDisallow: /GetAlgoProducts/0?Algorithm=NavigationHistory\r\nDisallow: /GetAlgoProducts/0?Algorithm=EarlyBirds\r\nDisallow: /all/\r\nDisallow: /Catalog/\r\nDisallow: /mobile/\r\nDisallow: /mag/\r\nDisallow: /tracking/\r\nDisallow: /Other/Partners/\r\nDisallow: /Specific/\r\nDisallow: /include/\r\nDisallow: /pret-a-porter/chaussures/\r\nDisallow: /V2/\r\nDisallow: /v2/\r\nDisallow: /lp.aspx\r\nDisallow: /PartnersLinks/\r\nDisallow: /informatique/ecran-pc/choix-par-taille/format-large-16/10eme/23-/\r\nDisallow: /bons-plans/garanties/\r\nDisallow: /ab-\r\nDisallow: /AB-\r\nDisallow: /error/\r\nDisallow: /tout/produits/\r\nDisallow: /n-\r\nDisallow: /bons-plans/colisage/\r\nDisallow: /spotlink-\r\nDisallow: /telephonie/abonnement-telephonique-operateurs-forfaits-telephones-portables/forfait-very-\r\nDisallow: /*.axd\r\nDisallow: /*.track$\r\nDisallow: /*/GetProductList*\r\nDisallow: /r/\r\nDisallow: /mag/zoom/\r\nDisallow: /%7Blpurlpath%7D\r\nDisallow: /Result_Error/\r\nDisallow: /noimage.gif\r\nDisallow: /Account/Newsletter/\r\nDisallow: /struct/\r\nDisallow: /carte-cdiscount/facilites-paiement/\r\nDisallow: /event/\r\nDisallow: /phone/product/\r\nDisallow: /showcase/\r\nDisallow: /*/informations-legales.html\r\nDisallow: /search-ajax/10/\r\nDisallow: /AddToBasket.html/\r\nDisallow: /BackOnAddAssGarToCart/\r\nDisallow: /Recommendations/\r\nDisallow: /search/\r\nDisallow: /IMarketplaceAPIService/\r\nDisallow: /MapRelay/\r\nDisallow: /opa.aspx/\r\nDisallow: /PhoneContract/\r\nDisallow: /*mpos=*\r\nDisallow: /posez-votre-question/\r\nDisallow: /*undefined$\r\nDisallow: /donnez-votre-reponse/\r\nDisallow: /DisplayAdditionalOffersList.*\r\nDisallow: /marque-*\r\nDisallow: /UpdateCustomerResponseUsefullness.html/\r\nDisallow: /UpdateCustomerQuestionUsefullness.html/\r\nDisallow: /MultiArmedBanditUC/\r\nDisallow: /mpv-*?TechnicalForm*\r\nDisallow: /mpv-*?technicalform*\r\nDisallow: /OneClickDisplayButton.html\r\nDisallow: /*/dynaTraceMonitor\r\nDisallow: /q/\r\nDisallow: /v5.0/\r\nDisallow: /img/\r\nDisallow: /ShoppingCartRwd/\r\nDisallow: /realtimetracking/\r\nDisallow: /m/pl/\r\nDisallow: /local/\r\nDisallow: /donnez-votre-avis/\r\nDisallow: */img/nativeAd.jpg\r\nDisallow: /industriel/\r\nDisallow: /Construction-Outillage/\r\nDisallow: /AdvisorGoToHomePage.html\r\nDisallow: /DisplayMiniProductSheet.html\r\nDisallow: /GetProductCarrouselMabaya/*\r\nDisallow: /DisplayNearestRimOffer.html\r\nDisallow: /dtagent*\r\nDisallow: /AddToBasketNetotiate\r\nDisallow: /dynaTraceMonitor\r\nDisallow: /SetCookies.html\r\nDisallow: /setcookies.html\r\nDisallow: /NegotiatePopin.html\r\nDisallow: /{{product.url}}\r\nDisallow: /homepage/l-20000000000.html\r\nDisallow: /social-network-facebook.html\r\nDisallow: /social-network-instagram.html\r\nDisallow: /social-network-youtube.html\r\nDisallow: /social-network-twitter.html\r\nDisallow: /n-363936/vin-champagne/cave-virtuelle.html\r\nDisallow: /GetProductCarrouselMabaya/PRODUCT/*\r\nDisallow: /question-vendeur/\r\nDisallow: /ShowroomIdentification.html\r\nDisallow: /showroomidentification.html\r\nDisallow: /aideauchoix.html\r\nDisallow: /aideauchoix.html*\r\nDisallow: /*/http*/i2.cdscdn.com/pdt*.jpg\r\nDisallow: /auto/pieces-auto/*/la-v*\r\nDisallow: /auto/pieces-auto/*/la-m*-*-*.html\r\nDisallow: /auto/pieces-auto/*/la-*-*.html-*\r\nDisallow: /components/\r\nDisallow: /modules/\r\nDisallow: /*/components/\r\nDisallow: /*/modules/\r\nDisallow: /sergeant/\r\nDisallow: /magistracy/\r\nDisallow: /disrates/\r\nDisallow: /shredded/\r\nDisallow: /rousers/\r\nDisallow: /polygyny/\r\nDisallow: /overbake/\r\nDisallow: /displacement/\r\nDisallow: /cerium/\r\nDisallow: /rhombi/\r\nDisallow: /cahows/\r\nDisallow: /pirozhok/\r\nDisallow: /vised/\r\nDisallow: /schemer/\r\nDisallow: /sextuply/\r\nDisallow: /cerebrum/\r\nDisallow: /healthfulness/\r\nDisallow: /gangrene/\r\nDisallow: /pagurian/\r\nDisallow: /oospore/\r\nDisallow: /bloodmobiles/\r\nDisallow: /larkspurs/\r\nDisallow: /flanneled/\r\nDisallow: /worn/\r\nDisallow: /soppier/\r\nDisallow: /fleshes/\r\nDisallow: /wifelier/\r\nDisallow: /rainbow/\r\nDisallow: /posting/\r\nDisallow: /overladen/\r\nDisallow: /bullheads/\r\nDisallow: /aldovandi/\r\nDisallow: /psoraleas/\r\nDisallow: /haired/\r\nDisallow: /wooziness/\r\nDisallow: /kithe/\r\nDisallow: /fossate/\r\nDisallow: /smellier/\r\nDisallow: /procrastination/\r\nDisallow: /engirdle/\r\nDisallow: /smothery/\r\nDisallow: /rammiest/\r\nDisallow: /ungraced/\r\nDisallow: /moguls/\r\nDisallow: /tangly/\r\nDisallow: /liberator/\r\nDisallow: /consciously/\r\nDisallow: /vapors/\r\nDisallow: /guardrooms/\r\nDisallow: /slater/\r\nDisallow: /fetuses/\r\nDisallow: /analyse/\r\nDisallow: /fatigues/\r\nDisallow: /kaas/\r\nDisallow: /faqirs/\r\nDisallow: /iniquitous/\r\nDisallow: /glossily/\r\nDisallow: /squilgees/\r\nDisallow: /inclemencies/\r\nDisallow: /maturative/\r\nDisallow: /guanin/\r\nDisallow: /ratifying/\r\nDisallow: /franking/\r\nDisallow: /spagyric/\r\nDisallow: /wizardly/\r\nDisallow: /ephorate/\r\nDisallow: /butch/\r\nDisallow: /slick/\r\nDisallow: /fake-adback-endpoint/\r\nDisallow: */*https://*/pdt*\r\nDisallow: */*http://*/pdt*\r\nDisallow: /ProductAdsDisplay.html/\r\nDisallow: *.html?page=*\r\nDisallow: *.html?nav=JTdCJ*\r\nDisallow: /*sw=\r\nDisallow: /dvd/*idOffre=\r\nDisallow: /electromenager/*idOffre=\r\nDisallow: /high-tech/*idOffre=\r\nDisallow: /arts-loisirs/*idOffre=\r\nDisallow: /maison/*idOffre=\r\nDisallow: /bricolage/*idOffre=\r\nDisallow: /jardin/*idOffre=\r\nDisallow: /informatique/*idOffre=\r\nDisallow: /au-quotidien/*idOffre=\r\nDisallow: /bagages/*idOffre=\r\nDisallow: /animalerie/*idOffre=\r\nDisallow: /telephonie/*idOffre=\r\nDisallow: /photo-numerique/*idOffre=\r\nDisallow: /jeux-pc-video-console/*idOffre=\r\nDisallow: /musique-cd-dvd/*idOffre=\r\nDisallow: /musique-instruments/*idOffre=\r\nDisallow: /vin-champagne/*idOffre=\r\nDisallow: /livres-bd/*idOffre=\r\nDisallow: /sante-mieux-vivre/*idOffre=\r\n"
    },
    "www.fnac.com": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.fnac.com",
      "body": "# FNAC.COM\r\n\r\n# Disallow\r\nUser-agent: *\r\nDisallow: /Account/\r\nDisallow: /account/\r\nDisallow: /MyAccount/\r\nDisallow: /myaccount/\r\nDisallow: /identity/gateway/\r\nDisallow: /OrderPipe/\r\nDisallow: /orderpipe/\r\nDisallow: /orderpipe\r\nDisallow: /LastUndeliveredOrder.aspx\r\nDisallow: /Comments/Share/\r\nDisallow: /mobile/Basket/*\r\nDisallow: /basket/add\r\nDisallow: /basket/oneclick\r\nDisallow: /basket/delete\r\nDisallow: /SearchResult/ResultList.aspx*SFilt=*\r\nDisallow: /SearchResult/ResultList.aspx*PageIndex=*\r\n\r\nNoindex: /api/*\r\n\r\n# Sitemap\r\nSitemap: https://www.fnac.com/sitemapindex-master-old.xml\r\nSitemap: https://www.fnac.com/sitemapindex-master.xml\r\n\r\nUser-agent: Mediapartners-Google\r\nDisallow:\r\n"
    },
    "www.mediamarkt.de": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.mediamarkt.de",
      "body": "Sitemap: https://www.mediamarkt.de/sitemaps/sitemap-index.xml\n\nSitemap: https://www.mediamarkt.de/sitemaps/magazin-deals-angebote.xml\n\nUser-Agent: *\nDisallow: /cdn-cgi/bm/cv/\nDisallow: /cdn-cgi/challenge-platform/\nDisallow: /rde_server/\nDisallow: /cdn-cgi/images/trace/\nDisallow: /cdn-cgi/rum\nDisallow: /cdn-cgi/scripts/\nDisallow: /cdn-cgi/styles/\nDisallow: /cdn-cgi/zaraz/\nDisallow: /api/v1/graphql?operationName=GetPaidBundles\nDisallow: /api/v1/graphql?operationName=GetFreeBundles\nDisallow: /api/v1/msg\nDisallow: /api/v1/mms-sst\nDisallow: /api/v1/partners/\nDisallow: /*shopfallback*\nDisallow: /*MultiChannelMARepairStatusResult*\nDisallow: *de/product/*-weltmusik-cd*\nDisallow: *de/product/*-jazz-cd*\nDisallow: *de/product/*-blues-cd*\nDisallow: *de/product/*-klassik-cd*\nDisallow: *de/product/*-oper-cd*\nDisallow: *de/product/*-folklore-cd*\nDisallow: *de/product/*-country-cd*\nDisallow: *de/product/*-heavy-metal-cd*\nDisallow: *de/product/*-kinder-cd*\nDisallow: *de/list/*_*\nDisallow: /mcs/marketinfo/*\nDisallow: /public/setCookie/\nDisallow: */myaccount/auth*\n\nAllow: /de/product/_bibi-tina-folge-101-%C3%A4rger-mit-dem-grafen-h%C3%B6rspiel-kinder-cd-2694761.html\nAllow: /de/product/_keith-jarrett-budapest-concert-jazz-blues-cd-2683284.html\nAllow: /de/product/_gregory-porter-all-rise-jazz-blues-cd-2628384.html\nAllow: /de/product/_sarah-contradiction-of-happiness-buechi-the-paintress-jazz-blues-cd-2750814.html\nAllow: /de/product/_tony-bennett-and-lady-gaga-love-for-sale-limitierte-deluxe-edition-cd-2755391.html\nAllow: /de/product/_signum-saxophone-quartet-echoes-klassik-cd-2716145.html\nAllow: /de/product/_luciano-pavarotti-pavarotti-the-greatest-hits-oper-cd-2551158.html\nAllow: /de/product/_r%C3%A4uberzivil-tiefensch%C3%A4rfe-pop-international-cd-1957190.html\nAllow: /de/product/_versengold-funkenflug-folk-folklore-cd-2301612.html\nAllow: /de/product/_judas-priest-redeemer-of-souls-rock-international-cd-1849380.html\nAllow: /de/product/_powerwolf-call-of-the-wild-2cd-mediabook-exkl-anh%C3%A4nger-heavy-metal-cd-merchandising-2728783.html\nAllow: /de/product/_rage-the-refuge-years-album-box-fan-box-heavy-metal-cd-2066128.html\nAllow: /de/product/_die-kleine-schnecke-monika-h%C3%A4uschen-55-warum-sind-kuckuckskinder-so-frech-h%C3%B6rspiel-kinder-cd-2615552.html\n\nUser-agent: AdsBot-Google\nUser-agent: Googlebot-Image\nUser-agent: Storebot-Google\nAllow: /\n\n# allowing ai crawlers\n\n# OpenAI\nUser-agent: ChatGPT-User\nAllow: /\nUser-agent: GPTBot\nAllow: /\nUser-agent: OAI-SearchBot\nAllow: /\nUser-agent: Operator\nAllow: /\n\n# Anthropic\nUser-agent: Claude-SearchBot\nAllow: /\nUser-agent: Claude-User\nAllow: /\nUser-agent: Claude-Web\nAllow: /\nUser-agent: ClaudeBot\nAllow: /\nUser-agent: anthropic-ai\nAllow: /\n\n# Google\nUser-agent: Google-CloudVertexBot\nAllow: /\nUser-agent: Google-Extended\nAllow: /\n\n# Apple\nUser-agent: Applebot\nAllow: /\nUser-agent: Applebot-Extended\nAllow: /\n\n# Meta\nUser-agent: Meta-ExternalAgent\nAllow: /\nUser-agent: Meta-ExternalFetcher\nAllow: /\nUser-agent: Manus-User\nAllow: /\n\n# Perplexity\nUser-agent: Perplexity-User\nAllow: /\nUser-agent: PerplexityBot\nAllow: /\n\n# ByteDance\nUser-agent: Bytespider\nAllow: /\nUser-agent: TikTokSpider\nAllow: /\n\n# disallowing ai crawlers\n\n# Meta\nUser-agent: FacebookBot\nDisallow: /\n\n# Mistral\nUser-agent: MistralAI-User\nDisallow: /\n\n# Amazon\nUser-agent: Amazonbot\nDisallow: /\n\n# Huawei\nUser-agent: PanguBot\nDisallow: /\n\n# SINA Corporation\nUser-agent: iaskspider\nDisallow: /\n\n# Timpi\nUser-agent: Timpibot\nDisallow: /\n\n# Allen Institute\nUser-agent: AI2Bot\nDisallow: /\n\n# Common Crawl\nUser-agent: CCBot\nDisallow: /\n\n# Diffbot Inc\nUser-agent: Diffbot\nDisallow: /\n\n# Cohere\nUser-agent: cohere-ai\nDisallow: /\n\n# DigitalOcean\nUser-agent: DigitalOceanGenAICrawler\nDisallow: /\n\n# Sidetrade\nUser-agent: Sidetrade\nDisallow: /\n\n# ISS (Security)\nUser-agent: ISSCyberRiskCrawler\nDisallow: /\n\n# Image Processing Tools\nUser-agent: img2dataset\nDisallow: /\nUser-agent: imgproxy\nDisallow: /\n\n# Forum Crawler Service\nUser-agent: omgili\nDisallow: /\n\n# Unknown\nUser-agent: Cotoyogi\nDisallow: /\nUser-agent: FriendlyCrawler\nDisallow: /\nUser-agent: NovaAct\nDisallow: /\nUser-agent: YouBot\nDisallow: /\nUser-agent: aiHitBot\nDisallow: /\n"
    },
    "www.mediamarkt.es": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.mediamarkt.es",
      "body": "Sitemap: https://www.mediamarkt.es/sitemaps/sitemap-index.xml\n\nUser-Agent: *\n\n# allowing ai crawlers added 30-3-2026\n \nUser-agent: GPTBot\nAllow: /\n \nUser-agent: OAI-SearchBot\nAllow: /\n \nUser-agent: ChatGPT-User\nAllow: /\n\nUser-agent: ClaudeBot\nAllow: /\n\nUser-agent: Claude-Web\nAllow: /\n\nUser-agent: PerplexityBot\nAllow: /\n\nUser-agent: Perplexity-User\nAllow: /\n\nUser-agent: Applebot\nAllow: /\n\nUser-agent: Meta-ExternalAgent\nAllow: /\n\nUser-agent: Meta-ExternalFetcher\nAllow: /\n\nUser-agent: FacebookBot\nAllow: /\n\nUser-agent: Bytespider\nAllow: /\n\nUser-agent: TikTokSpider\nAllow: /\n\nUser-agent: Google-CloudVertexBot\nAllow: /\n\nUser-agent: MistralAI-User\nAllow: /\n\nUser-agent: anthropic-ai\nAllow: /\n\nUser-agent: Claude-SearchBot\nAllow: /\n\nUser-agent: Claude-User\nAllow: /\n\nUser-agent: Applebot-Extended\nAllow: /\n\nUser-agent: Google-Extended\nAllow: /\n\nUser-agent: Operator\nAllow: /\n\n#tmp disallow\nDisallow: /ca/\nDisallow: /gl/\nDisallow: /eu/\nDisallow: /api/v1/partners/\nDisallow: /api/v1/graphql?operationName=GetPaidBundles\nDisallow: /api/v1/graphql?operationName=GetFreeBundles\nDisallow: /api/v1/msg\nDisallow: /api/v1/mms-sst\nDisallow: /cdn-cgi/bm/cv/\n#Disallow: /cdn-cgi/challenge-platform/\nDisallow: /rde_server/\nDisallow: /webapp/wcs/stores/servlet/LogonForm\nDisallow: /webapp/wcs/stores/servlet/MultiChannelMANewsletterRegister\nDisallow: /webapp/wcs/stores/servlet/MultiChannelMAContact\n#Disallow: /webapp/wcs/stores/servlet/MultiChannelMAGiftCard\nDisallow: /webapp/wcs/stores/servlet/MultiChannelMARepairStatus\n#Disallow: /webapp/wcs/stores/servlet/MultiChannelMASearchOrderForGuestUser\nDisallow: /webapp/wcs/stores/servlet/MultiChannelMAMasterData\nDisallow: /webapp/wcs/stores/servlet/MultiChannelMAPersonal\nDisallow: /webapp/wcs/stores/servlet/MultiChannelMAChangePassword\nDisallow: /error404.html\nDisallow: /error500.html\nDisallow: /mcs/brandcategory/*query=\nDisallow: /mcs/brandcategory/*searchParams=\nDisallow: /mcs/brandcategory/*page=\nDisallow: /es/category/*searchParams=\nDisallow: /es/search*\nDisallow: *es/promo-list/*\nDisallow: *es/list/*_*\nDisallow: /*?forceDeviceId=-111\nDisallow: /*storeId=*\nDisallow: /*view=\nDisallow: /*sort=\nDisallow: /*searchParams=\nDisallow: *?filter=\nDisallow: *?filter%\nDisallow: /*?ga_query=\nDisallow: /*query=\nDisallow: /myaccount/auth*\nDisallow: /es/checkout/*\n\n# Disallow: /MultiChannelMA\n# Disallow: /MultiChannelLocal\n# Disallow: /MultiChannelSearch\n# Disallow: /MultiChannelRedirect\n# Disallow: /MultiChannelMARegister\n# Disallow: /MultiChannelMyStoreEvents\n# Disallow: /MultiChannelAutoCompletion\n# Disallow: /MultiChannelCatalogEntryPrint\n# Disallow: /MultiChannelPrintCompProducts\n# Disallow: /MultiChannelMyStoreAdvertising\n# Disallow: /MultiChannelMyStoreSpecialitems\n# Disallow: /MultiMultiChannelMAWishlistPrint\n# Disallow: /mediapedia\n# Disallow: /*xjmnbo*\n\n# Allow: /*storeId=19601*\n\nUser-agent: AdsBot-Google\nUser-agent: Googlebot-Image\nUser-agent: Storebot-Google\nAllow: /\n\n"
    },
    "www.mediamarkt.it": {
      "status": 0,
      "client": "fetch",
      "finalHost": null,
      "body": null
    },
    "www.mediamarkt.nl": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.mediamarkt.nl",
      "body": "Sitemap: https://www.mediamarkt.nl/sitemaps/sitemap-index.xml\n\nUser-agent: *\n\n#added 10-9-25\nDisallow: /*myaccount/auth/\n\n#added 14-1-25\nDisallow: /en/\n\n#added 26-3-24\nDisallow: /api/v1/msg\nDisallow: /api/v1/mms-sst\nDisallow: /api/v1/graphql?operationName=GetPaidBundles\nDisallow: /api/v1/graphql?operationName=GetFreeBundles\n\nDisallow: /cdn-cgi/challenge-platform/\n\n#added 29-11-23\nDisallow: /cdn-cgi/bm/cv/\nDisallow: /cdn-cgi/images/trace/\nDisallow: /cdn-cgi/rum\nDisallow: /cdn-cgi/scripts/\nDisallow: /cdn-cgi/styles/\nDisallow: /cdn-cgi/zaraz/\n\nDisallow: /rde_server/\nDisallow: /public/setCookie/\nAllow: /*storeId=10101*\nDisallow: /*nl/promo-list/*\nDisallow: /*nl/list/*_*\nDisallow: /*storeId=*\nDisallow: /*view=\nDisallow: /*sort=\nDisallow: /*searchParams=\nDisallow: /*?filter=\nDisallow: /*?filter%\nDisallow: /*?ga_query=\nDisallow: /*query=\n\n# allowing ai crawlers added 12-6-2025\n \nUser-agent: GPTBot\nAllow: /\n \nUser-agent: OAI-SearchBot\nAllow: /\n \nUser-agent: ChatGPT-User\nAllow: /\n\nUser-agent: ClaudeBot\nAllow: /\n\nUser-agent: Claude-Web\nAllow: /\n\nUser-agent: PerplexityBot\nAllow: /\n\nUser-agent: Perplexity-User\nAllow: /\n\nUser-agent: Applebot\nAllow: /\n\nUser-agent: Meta-ExternalAgent\nAllow: /\n\nUser-agent: Meta-ExternalFetcher\nAllow: /\n\nUser-agent: FacebookBot\nAllow: /\n\nUser-agent: Bytespider\nAllow: /\n\nUser-agent: TikTokSpider\nAllow: /\n\nUser-agent: Google-CloudVertexBot\nAllow: /\n\nUser-agent: MistralAI-User\nAllow: /\n\nUser-agent: anthropic-ai\nAllow: /\n\n# allowing other bots 3-7-2025\n \nUser-agent: Claude-SearchBot\nAllow: /\n\nUser-agent: Claude-User\nAllow: /\n\nUser-agent: Applebot-Extended\nAllow: /\n\nUser-agent: Google-Extended\nAllow: /\n\nUser-agent: Operator\nAllow: /\n\nUser-agent: Storebot-Google\nAllow: /\n\nUser-agent: Googlebot-Image\nAllow: /\n\n# disallowing ai crawlers 12-6-2025\n \nUser-agent: Amazonbot\nDisallow: /\n \nUser-agent: AI2Bot\nDisallow: /\n \nUser-agent: CCBot\nDisallow: /\n \nUser-agent: Timpibot\nDisallow: /\n \nUser-agent: imgproxy\nDisallow: /\n \nUser-agent: img2dataset\nDisallow: /\n \nUser-agent: iaskspider\nDisallow: /\n \nUser-agent: Cotoyogi\nDisallow: /\n \nUser-agent: Diffbot\nDisallow: /\n \nUser-agent: aiHitBot\nDisallow: /\n \nUser-agent: Sidetrade\nDisallow: /\n \nUser-agent: PanguBot\nDisallow: /\n \nUser-agent: ISSCyberRiskCrawler\nDisallow: /\n \nUser-agent: DigitalOceanGenAICrawler\nDisallow: /\n \nUser-agent: omgili\nDisallow: /\n \nUser-agent: FriendlyCrawler\nDisallow: /\n \nUser-agent: NovaAct\nDisallow: /\n \nUser-agent: cohere-ai\nDisallow: /\n \nUser-agent: YouBot\nDisallow: /\n"
    },
    "www.galaxus.de": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.galaxus.de",
      "body": "#            @/                          @/                        Hello, fellow humans!\n#            @/                          @/                        Interested in joining as a technical SEO specialist or developer? Have a look at https://www.galaxus.de/joboffer\n#            @/                          @/\n#        @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@              @@@@\n#       @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@           @@@@\n#       @@@@@@        @@@@@@@@@@@@        @@@@@@          @@@%           @\n#       @@@@@         /@@@@@@@@@@         @@@@@@          @@@           @@\n#       @@@@@@        @@@@@@@@@@@,        @@@@@@           @@@@       @@@@\n#       @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@          @@@@@@@@@@@@@@\n#       @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@        @@@@@@@@@\n#       @@@@@@@@@@@@@             @@@@@@@@@@@@@@       @@@@@@@@\n#       @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@     @@@@@@@@@\n#        @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@     @@@@@@@@@\n\nUser-agent: sogou spider\nDisallow: /\n\nUser-agent: Yandex\nDisallow: /\n\nUser-agent: EventMachine HttpClient\nDisallow: /\n\nUser-agent: niki-bot\nDisallow: /\n\nUser-agent: Pinterestbot\nCrawl-delay: 0.2\n\n\nUser-Agent: *\nDisallow: /cert\nDisallow: /Files\nAllow: /Files/sitemap/*\nAllow: /Files/iso/*\nAllow: /Files/*.jpg\nAllow: /Files/*.jpeg\nAllow: /Files/*.ashx\nAllow: /Files/*.png\nAllow: /Files/*.tif\nAllow: /Files/*.JPG\nAllow: /Files/*.JPEG\nAllow: /Files/*.ASHX\nAllow: /Files/*.PNG\nAllow: /Files/*.TIF\nAllow: /Files/*.Jpeg\nAllow: /Files/*.Jpg\nAllow: /im/*\nDisallow: */Explore?\nDisallow: */explore?\nDisallow: */Explore/*\nDisallow: */explore/*\nDisallow: */Search\nDisallow: */search\nDisallow: */SaleNew\nDisallow: */salenew\nDisallow: */SaleIso\nDisallow: */saleiso\nDisallow: */Comparison/\nDisallow: */comparison/\nDisallow: */User/\nDisallow: */user/\nDisallow: */NewDiscussion/\nDisallow: */newdiscussion/\nDisallow: */Ratings/RatingForm\nDisallow: */ratings/ratingform\nDisallow: */MarketingTeaser/SaveNewImpressions/\nDisallow: */marketingteaser/savenewimpressions/\nDisallow: */Management\nDisallow: */management\nDisallow: */*filterGrid=\nDisallow: */*filtergrid=\nDisallow: */archived-orders\nDisallow: */preview/blog-teaser\nDisallow: */cart\nDisallow: */checkout\nDisallow: */comparison/\nDisallow: */credit-notes\nDisallow: */customeraccount\nDisallow: */gift-cards\nDisallow: */gift-coupons\nDisallow: */live-feed\nDisallow: */customeraccount/login\nDisallow: */marketplace/\nDisallow: */mobile-apps\nDisallow: */order\nDisallow: */page/preview/\nDisallow: */cases/register\nDisallow: */return-cases/register\nDisallow: */service-cases/register\nDisallow: */shoplist/\nDisallow: */testreport/\nDisallow: */user/\nDisallow: */ProductRating/\nAllow: */Management-Photos\nAllow: */management-photos\nDisallow: */graphql/send-consent-reporting-event\nDisallow: */graphql/get-country-redirect\nDisallow: */explore-alternatives-with-good-availability\nDisallow: */explore-assembler\nDisallow: */explore-auto-ml\nDisallow: */explore-buy-it-again\nDisallow: */explore-continue-browsing\nDisallow: */explore-explicit-accessories\nDisallow: */explore-hot-new-products\nDisallow: */explore-implicit-accessories\nDisallow: */explore-on-sale-for-you\nDisallow: */explore-product-sales-ranking\nDisallow: */explore-recently-visited-products\nDisallow: */explore-similar-products\nDisallow: /fr$\nDisallow: /fr/\nDisallow: /it$\nDisallow: /it/\nDisallow: /nl$\nDisallow: /nl/\nDisallow: /webview/*\nDisallow: /*/webview/*\n\nSitemap: https://www.galaxus.de/sitemap/v2/default-sitemap.xml\nSitemap: https://www.galaxus.de/sitemap/v2/producttypes-sitemap.xml\nSitemap: https://www.galaxus.de/sitemap/v2/products-sitemap.xml\nSitemap: https://www.galaxus.de/sitemap/v2/news-sitemap.xml"
    },
    "www.galaxus.ch": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.galaxus.ch",
      "body": "#            @/                          @/                        Hello, fellow humans!\n#            @/                          @/                        Interested in joining as a technical SEO specialist or developer? Have a look at https://www.galaxus.ch/joboffer\n#            @/                          @/\n#        @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@              @@@@\n#       @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@           @@@@\n#       @@@@@@        @@@@@@@@@@@@        @@@@@@          @@@%           @\n#       @@@@@         /@@@@@@@@@@         @@@@@@          @@@           @@\n#       @@@@@@        @@@@@@@@@@@,        @@@@@@           @@@@       @@@@\n#       @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@          @@@@@@@@@@@@@@\n#       @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@        @@@@@@@@@\n#       @@@@@@@@@@@@@             @@@@@@@@@@@@@@       @@@@@@@@\n#       @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@     @@@@@@@@@\n#        @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@     @@@@@@@@@\n\nUser-agent: sogou spider\nDisallow: /\n\nUser-agent: Yandex\nDisallow: /\n\nUser-agent: EventMachine HttpClient\nDisallow: /\n\nUser-agent: niki-bot\nDisallow: /\n\nUser-agent: Pinterestbot\nCrawl-delay: 0.2\n\n\nUser-Agent: *\nDisallow: /cert\nDisallow: /Files\nAllow: /Files/sitemap/*\nAllow: /Files/iso/*\nAllow: /Files/*.jpg\nAllow: /Files/*.jpeg\nAllow: /Files/*.ashx\nAllow: /Files/*.png\nAllow: /Files/*.tif\nAllow: /Files/*.JPG\nAllow: /Files/*.JPEG\nAllow: /Files/*.ASHX\nAllow: /Files/*.PNG\nAllow: /Files/*.TIF\nAllow: /Files/*.Jpeg\nAllow: /Files/*.Jpg\nAllow: /im/*\nDisallow: */Explore?\nDisallow: */explore?\nDisallow: */Explore/*\nDisallow: */explore/*\nDisallow: */Search\nDisallow: */search\nDisallow: */SaleNew\nDisallow: */salenew\nDisallow: */SaleIso\nDisallow: */saleiso\nDisallow: */Comparison/\nDisallow: */comparison/\nDisallow: */User/\nDisallow: */user/\nDisallow: */NewDiscussion/\nDisallow: */newdiscussion/\nDisallow: */Ratings/RatingForm\nDisallow: */ratings/ratingform\nDisallow: */MarketingTeaser/SaveNewImpressions/\nDisallow: */marketingteaser/savenewimpressions/\nDisallow: */Management\nDisallow: */management\nDisallow: */*filterGrid=\nDisallow: */*filtergrid=\nDisallow: */archived-orders\nDisallow: */preview/blog-teaser\nDisallow: */cart\nDisallow: */checkout\nDisallow: */comparison/\nDisallow: */credit-notes\nDisallow: */customeraccount\nDisallow: */gift-cards\nDisallow: */gift-coupons\nDisallow: */live-feed\nDisallow: */customeraccount/login\nDisallow: */marketplace/\nDisallow: */mobile-apps\nDisallow: */order\nDisallow: */page/preview/\nDisallow: */cases/register\nDisallow: */return-cases/register\nDisallow: */service-cases/register\nDisallow: */shoplist/\nDisallow: */testreport/\nDisallow: */user/\nDisallow: */ProductRating/\nAllow: */Management-Photos\nAllow: */management-photos\nDisallow: */graphql/send-consent-reporting-event\nDisallow: */graphql/get-country-redirect\nDisallow: */explore-alternatives-with-good-availability\nDisallow: */explore-assembler\nDisallow: */explore-auto-ml\nDisallow: */explore-buy-it-again\nDisallow: */explore-continue-browsing\nDisallow: */explore-explicit-accessories\nDisallow: */explore-hot-new-products\nDisallow: */explore-implicit-accessories\nDisallow: */explore-on-sale-for-you\nDisallow: */explore-product-sales-ranking\nDisallow: */explore-recently-visited-products\nDisallow: */explore-similar-products\nDisallow: /nl$\nDisallow: /nl/\nDisallow: /webview/*\nDisallow: /*/webview/*\n\nSitemap: https://www.galaxus.ch/sitemap/v2/default-sitemap.xml\nSitemap: https://www.galaxus.ch/sitemap/v2/producttypes-sitemap.xml\nSitemap: https://www.galaxus.ch/sitemap/v2/products-sitemap.xml\nSitemap: https://www.galaxus.ch/sitemap/v2/news-sitemap.xml"
    },
    "www.leroymerlin.fr": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.leroymerlin.fr",
      "body": "User-agent: Nutch\r\nDisallow: /\r\nUser-agent: *\r\nAllow: /\r\nDisallow: /*SALEUNIT_PRICE&*=*\r\nDisallow: /v3/mobile/action-search/search.do\r\nDisallow: /v3/p/communaute/charte-d-utilisation/charte-d-utilisation-l1308220708\r\nDisallow: /v3/p/communaute/charte-d-utilisation/comment-participer-l1308221057\r\nDisallow: /*&action=viewmember\r\nDisallow: /*&action=addwatchprocess\r\nDisallow: /*&action=addpost\r\nDisallow: /*&action=addmessage\r\nDisallow: /*&action=printpost\r\nDisallow: /*&action=viewreputation\r\nDisallow: /*&action=vote\r\nDisallow: /*&action=index\r\nDisallow: /*&sort=\r\nDisallow: /*&action=processsearchalbum\r\nDisallow: /*&noIndex=1\r\nDisallow: /down.html\r\nDisallow: /recherche=\r\nDisallow: /v3/search/search.do\r\nDisallow: /v3/search_old/\r\nDisallow: /*typeSearch=\r\nDisallow: /v3/contextualization/store/\r\nDisallow: /v3/diaporama/diaporama.do\r\nDisallow: /v3/compteinternaute/*\r\nDisallow: /v3/lmLogin.do\r\nDisallow: /v3/bazaarvoice/\r\nDisallow: /v3/vad/\r\nDisallow: /mpng2-front/pre?zone=zonestatique&srvstatique-staticpagename=html/alerte-phishing.html\r\nDisallow: /v3/p/produits/materiaux-menuiserie/panneau-bois-tablette-tasseau-moulure-et-plinthe/decoupe-bois-l1400494157\r\nDisallow: /v3/disponibilite/magasin/listStores.do\r\nDisallow: /v3/merch-concept/displayConceptFull.do*\r\nDisallow: /v3/search_old/\r\nDisallow: /recherche*\r\nDisallow: /*showPopup=*\r\nDisallow: /v3/webtv/rechercher.do?keyword=*\r\nDisallow: /v3/products/compare.do*\r\nDisallow: /v3/products/removeProductsToCompare.do\r\nDisallow: /v3/listedecourses/*\r\nDisallow: /*&lsmulticriteria\r\nDisallow: /*&segment\r\nDisallow: /*&mcsegment\r\nDisallow: /*reviews.htm*\r\nDisallow: /v3/mobile/store/listStores.do\r\nDisallow: /v3/p/produits/chauffage-plomberie/poele-a-bois-poele-a-granules-et-cheminee/bois-de-chauffage-pellets-granules-et-buches-calorifiques-l1308223614/buche-a-buche-p.html\r\nDisallow: /*queryredirect=*numpage=*\r\nDisallow: /*category.htm*\r\nDisallow: /*num_sujet*\r\nDisallow: /*display=criteria\r\nDisallow: /*xtor\r\nDisallow: /*rubrique\r\nDisallow: /cart/services/*\r\nDisallow: /*cmcible\r\nDisallow: /*CID\r\nDisallow: /*PHPSESSID\r\nDisallow: /*com=\r\nDisallow: /*SALEUNIT_PRICE&\r\nDisallow: /*zone=\r\nDisallow: /*mpng2-front\r\nDisallow: /*merch-pedago\r\nDisallow: /*trier=\r\nDisallow: /*goForum=\r\nDisallow: /*myRedirectUrl\r\nDisallow: /*getReviews.do\r\nDisallow: /*campusOrigin\r\nDisallow: /*product/complementaires/getByProduct.do*\r\nDisallow: /*product-reviews/getReviews.do*\r\nDisallow: /*rdv\r\nDisallow: /*filters=*\r\nDisallow: /*limit=*\r\nDisallow: /*sort=*\r\nDisallow: */avis/produit/*\r\nDisallow: /*pedagogicalFilters=*\r\nDisallow: /*produits-de-depannage-et-de-premiere-necessite-l1506165935\r\nDisallow: /*produits-de-premiere-necessite*\r\nDisallow: /services/rendez-vous.html\r\nDisallow: */produits/offer/*\r\nDisallow: /big/guides-2014/*\r\nDisallow: */productpageservicesfunctionality/*\r\nDisallow: */*/RME\r\nDisallow: */*/rme\r\nDisallow: */*/rdv\r\nDisallow: */v3/search/search.do?keyword=*\r\nDisallow: */customer/services/userInfos\r\nDisallow: */navigation/api/autocomplete\r\nDisallow: */recherche?page=*\r\nDisallow: */v3/merch-concept/*\r\nDisallow: */questions-reponses/produit/*\r\nDisallow: */v3/compteinternaute/\r\nDisallow: */v3/CDE/*\r\nDisallow: */v3/mobile/*\r\nDisallow: */test/*\r\nDisallow: */productpageservicesfunctionality/*\r\nDisallow: */services/candidatures/supprimer-mes-donnees.html*\r\nDisallow: */papier-peint-hub/*\r\nDisallow: */produits-2/*\r\nDisallow: */nos-produits/*\r\nDisallow: */produits/bonnes-affaires/archive*\r\nDisallow: */services/service-de-prise-de-rendez-vous.html\r\nDisallow: */espace-perso/*\r\nDisallow: */shoppinglist/*\r\nDisallow: */productpageservicesfunctionality/*\r\nDisallow: *goToReviewModule*\r\nDisallow: */gelules-hub-de-gamme/*\r\nDisallow: */produits/bonnes-affaires/*?*\r\nDisallow: */*?*highlightedOfferCode\r\nDisallow: */*?*storeid \r\nDisallow: */questions-nouveau-programme-fidelite\r\nDisallow: */contact-nouveau-programme-fidelite\r\nDisallow: /media/15/5e37ee987eb45f518507d94e/175899224/nouvel-emmenage5e37ee98de91bb0008f1209d.pdf\r\nDisallow: */big2/*\r\nDisallow: /volet-roulant/feasibility/dimensions\r\nDisallow: */finalizeorderapp/*\r\nDisallow: /produits/*/link\r\nDisallow: /produits/decoration-eclairage/decoration-et-eclairage-de-seconde-main/\r\nDisallow: /produits/decoration-eclairage/decoration-et-eclairage-de-seconde-main/*\r\nDisallow: /produits/meuble/meuble-de-seconde-main/\r\nDisallow: /produits/meuble/meuble-de-seconde-main/*\r\nDisallow: /project-booster/api/events/estimate-form-event\r\nDisallow: */evom/*\r\nDisallow: /header-cart/*\r\nDisallow: */v3/p/*\r\nDisallow: /customer-space-no-layout-module/*\r\nDisallow: /page-de-paiement-de-commande-magasin.html*\r\nDisallow: /selection-marque/*\r\nDisallow: *%22*\r\nDisallow: /vendeurs/*/\r\nDisallow: /mystica/web/conf/store-mise-a-dimension-sur-mesure\r\nDisallow: /authentication/oauth2/authorization/reach-five\r\nDisallow: /login.html?error=login*\r\nDisallow: /search?q=*\r\n\r\nSitemap: https://www.leroymerlin.fr/sitemap-index.xml"
    },
    "www.leroymerlin.es": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.leroymerlin.es",
      "body": "#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMX0XNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXd'..;kNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXd'     cNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXd' ..  .:XMNO0NMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMNx' .::. ;odo:. .;OWMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXkOk:cOd.  ...     ;OWWWMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM0, cNWW0' .,:c;.  ;kNNk::kNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMWWM0' .kWMNOkOx;.. ,kNWO;.  .;ONMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXx::lxc. cNMMMMK:..,xNWO:.  :xc..:OWMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMW0o;'.    .oXMMMMN00XNO:. .'..l0Oc,xNWWMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMWXxl::coxOOdl'  ,xNMMNNNWX:  .xNk,.lNWWNk:cOWMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMXd..';'   ;OWMXx,.cKN0dooxKXx,..oXNKNMNk;   .c0WMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMNl .dNWNO:. .kMMMXKX0dccc:clkXXx,.cXMNk;  .okl..cXMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMX; .dNWWMNl  lWMMW0dc:cc::ccclkXXOKNk;  .ccdKK;  oWMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMNOxxOXd. .:kXN0: .kWWKxccccccccccc:clkKWK: .lKx. ..  ,OWN00WMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMNk;.   ;Ok;   .'..;ONKxlccccc::ccccc:ccclkXXOKWX:  :ddOXWO:..dNMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMNk,.'okl. oWNOdlcld0XKdlcccccccccccccccccccclkXWMO. ;KWWWO:. .l0WMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMWN0,  ,kKc. .:ldkXWMWKxlcccccccccccccccccccccccclkXXd;xWWO:. .l0WMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMNk;;kKl. .':lc;'..:0NKxlccccccccccccccccccccccccccccokXNWKc. .c0WMMMMMMXdo0WMMMMMMMMMMMM\r\n#MMMMMMMMMMNk;..l0XNKl. .;kNNKKXKxlccccccccccccccccccccccccc:cccccclkXXd' .:0WMMMMXd'  :KMMMMMMMMMMMM\r\n#MMMMMMMMM0;  ,OOc'lXMKl. .kWWKxlcccccccccccccccccccccccccccccccc:cccokXXd' .oNMXd'  'dXNko0WMMMMMMMM\r\n#MMMMMMMMWKl. .'..;kN0oxkdkXKxlccccccccccccccccc::::cccccccccccccccccccokXXdlOKd'  'dXNk;  .oXMMMMMMM\r\n#MMMMMMMMMMW0c.  'k0l..:KWKxlccccccccccccccccccc::::cccccccccccccccccccccokXWNl. 'oXNx,     ;XMMMMMMM\r\n#MMNkokNMMMMMW0c. ...;kXKxlccccc:ccccccccccccccc::::cccccccccccccccccccccccokXKddXNk, .;l' .kMNkd0WMM\r\n#MM0,  ;kNMMMMNKkc.;kXKxlcccccccccccccccccccccccc:cccccccccccccccccccccccccccokXWNo. ;kNo  cKk;  lXMM\r\n#MMMKo.  ;kNNk;.:0XXKxlcccccccccccccccccccccccccccccccccccccccccccccccccccccccclkXKxONMK, .l;  ,xNMMM\r\n#MMMMMKo. .,;. ;kXKxlccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccokXWWWd     ,xNMMMMM\r\n#MMMMMMWKo.  ,kXKxlcccccccccccccccccccccccccccccccccccccccccccccccccccccccc:cc:ccccokXWx.  ,xNMMMMMMM\r\n#MMMMMMMMWKdkXKklc::cc:ccccccccccccccccccccccccccccccccccccccccccccccccccc:ccc::::cccoOXKdxNMMMMMMMMM\r\n#MMMMMMMMMMMMNKOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOKWMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\r\n#####################################################################################################\r\n#\r\n#                              Robots.txt de Leroy Merlin Espana (05-08-2026-V53)\r\n#\r\n#####################################################################################################\r\n# USER AGENT WHITELIST\r\nUser-Agent: *\r\nUser-agent: Googlebot\r\nUser-agent: Googlebot-News\r\nUser-agent: Googlebot-Image\r\nUser-agent: Googlebot-Video\r\nUser-agent: Googlebot-Mobile\r\nUser-agent: AdsBot-Google\r\n\r\n\r\n\r\n# URL TYPE BLACKLIST\r\nDisallow: /search?q=*\r\nDisallow: /v3/mobile/action-search/search.do\r\nDisallow: */v3/search/search.do*\r\nDisallow: /*filters=*\r\nDisallow: /*limit=*\r\nDisallow: /*sort=*\r\nDisallow: /*pedagogicalFilters=*\r\nDisallow: /*login?redirectUrl\r\nDisallow: /mylm/*\r\nDisallow: /reset-password*\r\nDisallow: /checkout*\r\nDisallow: /login*\r\nDisallow: */avis/*\r\nDisallow: *goToReviewModule*\r\nDisallow: /*&sort=\r\nDisallow: /*&noIndex=1\r\nDisallow: /*?keyword=*\r\nDisallow: /*typeSearch=\r\nDisallow: *dsg=*\r\nDisallow: /cart/services/*\r\nDisallow: /*myRedirectUrl\r\nDisallow: /*getReviews.do\r\nDisallow: /*campusOrigin\r\nDisallow: /*product/complementaires/getByProduct.do*\r\nDisallow: /*product-reviews/getReviews.do*\r\nDisallow: /*rdv\r\nDisallow: */customer/services/userInfos\r\nDisallow: */navigation/api/autocomplete\r\nDisallow: /*loyaltyDisplayerProduct.js$\r\nDisallow: */*/RME\r\nDisallow: */*/rme\r\nDisallow: */*/rdv\r\nDisallow: */produits/bonnes-affaires/archive*\r\nDisallow: */produits/bonnes-affaires/*?*\r\nDisallow: */espace-perso/*\r\nDisallow: */shoppinglist/*\r\nDisallow: */productpageservicesfunctionality/*\r\nDisallow: */*?*storeid\r\nDisallow: */test/*\r\nDisallow: /ofertas/catalogos/test*\r\nDisallow: */produits-2/* \r\nDisallow: */nos-produits/*\r\nDisallow: /*checkout*/\r\nDisallow: /carrito\r\nDisallow: /cart\r\nDisallow: /productpageservicesfunctionality\r\nDisallow: /authentication\r\nDisallow: /estancias1\r\nDisallow: /medidas-covid\r\nDisallow: /GDPR\r\nDisallow: /sling*\r\nDisallow: /prueba*\r\nDisallow: /*test-prueba*\r\nDisallow: /*Prueba*\r\nDisallow: /*PRUEBA*\r\nDisallow: /productos/ofertas/*\r\nDisallow: /productos/*/link\r\nDisallow: /header-cart*\r\nDisallow: /registrar/exito-registro\r\nDisallow: /customer-chatbot-webcomponent/1.32.4/config\r\nAllow: /productos/ofertas/$\r\nAllow: /productos/ofertas/*.html\r\nAllow: /productos/ofertas/black-friday/\r\nAllow: /productos/ofertas/cyber-monday/\r\nAllow: /productos/ofertas/catalogos/$\r\nAllow: /productos/ofertas/catalogos/pon-tu-casa-en-modo-vacaciones/$\r\nAllow: /productos/ofertas/catalogos/decoracion/$\r\nAllow: /productos/ofertas/catalogos/prepara-primavera/$\r\nAllow: /productos/ofertas/catalogos/jardin-terrazas/$\r\nAllow: /productos/ofertas/catalogos/especial-casa-eficiente/$\r\nAllow: /productos/ofertas/catalogos/bricoplanes/$\r\nAllow: /productos/ofertas/catalogos/a-buen-tiempo-buenisima-casa/$\r\nAllow: /productos/ofertas/catalogos/aniversario/$\r\nAllow: /productos/ofertas/ofertas-de-navidad/$\r\nAllow: /productos/ofertas/ofertas-de-navidad/promo-arboles-de-navidad/\r\nAllow: /productos/ofertas/equipamiento-de-emergencia-para-el-hogar/$\r\nAllow: /productos/ofertas/regalos-dia-de-madre/\r\nAllow: /productos/ofertas/regalos-dia-del-padre/\r\nAllow: /productos/ofertas/regalos-san-valentin/\r\nAllow: /productos/ofertas/ventas-flash-online/*\r\nAllow: /productos/ofertas/outlet-flash/*\r\nAllow: /productos/ofertas/regalos-de-navidad/$\r\nAllow: /productos/ofertas/gran-fiesta-de-casa/$\r\nAllow: /productos/ofertas/eficiencia-energetica/$\r\nAllow: /productos/ofertas/especial-jardin/$\r\nAllow: /productos/ofertas/bricoplanes/$\r\nAllow: /productos/ofertas/bricoplan/$\r\nAllow: /productos/ofertas/viva-el-verano/$\r\nAllow: /productos/ofertas/aniversario/$\r\nAllow: /productos/ofertas/tiempo-para-tu-casa/$\r\nAllow: /productos/ofertas/vuelve-con-energia/$\r\nAllow: /productos/ofertas/confort/$\r\nAllow: /productos/decoracion-navidena/$\r\nAllow: /productos/ofertas/el-precio-mas-bajo/nuestro-precio-bajo-en-decoracion/$\r\nAllow: /productos/ofertas/envios-gratis/$\r\nAllow: /productos/ofertas/envios-gratis/envio-estandar-gratis-herramientas/$\r\nAllow: /productos/ofertas/ofertas-y-envios-gratis/transporte-gratis-calefaccion-y-climatizacion/$\r\nAllow: /productos/decoracion/transporte-gratis-decoracion/$\r\nAllow: /productos/herramientas/transporte-gratis-herramientas/$\r\nAllow: /productos/iluminacion/transporte-gratis-iluminacion/$\r\nAllow: /productos/cocinas/transporte-gratis-cocinas/$\r\nAllow: /productos/ofertas/el-precio-mas-bajo/$\r\nAllow: /ofertas/ofertas-climatizacion/$\r\nAllow: /productos/ofertas/el-precio-mas-bajo/nuestro-precio-bajo-en-herramientas/$\r\nAllow: /productos/suelos/ofertas-y-envios-gratis-en-suelos-vinilicos/$\r\nAllow: /productos/ofertas/el-precio-mas-bajo/muebles-armarios-y-ordenacion/$\r\nAllow: /productos/ofertas/el-precio-mas-bajo/nuestro-precio-bajo-en-banos/$\r\nAllow: /productos/ofertas/el-precio-mas-bajo/nuestro-precio-bajo-en-cocinas/$\r\nAllow: /productos/ofertas/el-precio-mas-bajo/nuestro-precio-bajo-en-jardin/$\r\nAllow: /productos/ofertas/el-precio-mas-bajo/puertas-ventanas-y-escaleras/$\r\nAllow: /productos/ofertas/el-precio-mas-bajo/suelos/$\r\nAllow: /productos/ofertas/ofertas-de-verano/$\r\nAllow: /productos/ofertas/ofertas-de-verano/mas-ofertas-en-jardin/$\r\nAllow: /productos/ofertas/ofertas-de-verano/mas-ofertas-en-suelos/$\r\nAllow: /club-leroy-merlin/promociones/$\r\nAllow: /productos/ofertas/ofertas-agosto/\r\nAllow: /productos/ofertas/catalogos/tiempo-para-hacer-hogar/$\r\nAllow: /productos/ofertas/ofertas-septiembre/home-index/$\r\nAllow: /productos/ofertas/catalogos/vuelta-a-tu-hogar/$\r\nAllow: /productos/ofertas/ofertas-septiembre/$\r\n\r\n\r\nAllow: /login.html$\r\nDisallow: *producto.plp\r\nAllow: /club-leroy-merlin/$\r\nDisallow: /club-leroy-merlin/*\r\nDisallow: */transporte-gratis*\r\nDisallow: /tiendas/tiendaEvento*\r\nDisallow: /productos/*top-productos\r\nDisallow: /hacemos-por-ti*\r\nDisallow: /wishlist*\r\nDisallow: /productos/seleccion-productos/*\r\nDisallow: /productos/seleccion-productos-llica-damunt/*\r\nAllow: /productos/seleccion-productos-llica-damunt/*.html\r\nAllow: /configurador/$\r\nAllow: /configurador/nuevo-presupuestador-de-eficiencia-energetica-y-subvenciones.html\r\n\r\n# ROBOTS ES LEGACY\r\nDisallow: /dynatrace*\r\nDisallow: /.modalLogin\r\nDisallow: /buscador\r\nDisallow: /ChatServer\r\nDisallow: /comunicado-de-seguridad\r\nDisallow: /contacto\r\nDisallow: /dynaTrace\r\nDisallow: /error-page\r\nDisallow: /is\r\nDisallow: /login_checkout\r\nDisallow: /mi-leroy-merlin\r\nDisallow: /oauth-server\r\nDisallow: /politicadecookies\r\nDisallow: /politica-de-privacidad\r\nDisallow: /resultados-comparador\r\nDisallow: /tags\r\nDisallow: /buscador?q=*\r\nDisallow: /*/*/*?facet=*\r\nDisallow: /*/*?facet=*\r\nDisallow: /undefined\r\nDisallow: /completa-tu-compra\r\nDisallow: *&sort=\r\nAllow: /fp/*/*\r\nDisallow: /fp/*/link\r\nDisallow: /*livecopy*\r\nDisallow: /content/*\r\nDisallow: /consejos_animadores/*\r\n\r\nUser-agent: AdsBot-Google\r\nAllow: /productos/ofertas/*\r\nAllow: /productos/seleccion-productos/*\r\n\r\n\r\n# SITEMAP CCDP\r\nSitemap: https://www.leroymerlin.es/sitemap-index.xml\r\nSitemap: https://www.leroymerlin.es/sitemap-ideas-y-consejos1.xml\r\nSitemap: https://www.leroymerlin.es/sitemap-store1.xml\r\nSitemap: https://www.leroymerlin.es/sitemap-category1.xml\r\nSitemap: https://www.leroymerlin.es/sitemap-productos1.xml\r\nSitemap: https://www.leroymerlin.es/sitemap-productos2.xml\r\nSitemap: https://www.leroymerlin.es/sitemap-productos3.xml\r\nSitemap: https://www.leroymerlin.es/sitemap-productos4.xml\r\nSitemap: https://www.leroymerlin.es/sitemap-searchdex1.xml\r\nSitemap: https://www.leroymerlin.es/sitemap-searchdex2.xml\r\nSitemap: https://www.leroymerlin.es/sitemap-services1.xml\r\n\r\n\r\n# USER AGENT BLACKLIST\r\nUser-agent: 008\r\nDisallow: /\r\nUser-agent: Alexibot\r\nDisallow: /\r\nUser-agent: Aqua_Products\r\nDisallow: /\r\nUser-agent: b2w/0.1\r\nDisallow: /\r\nUser-agent: BackDoorBot/1.0\r\nDisallow: /\r\nUser-agent: Bookmark search tool\r\nDisallow: /\r\nUser-agent: BotALot\r\nDisallow: /\r\nUser-agent: BotRightHere\r\nDisallow: /\r\nUser-agent: BuiltBotTough\r\nDisallow: /\r\nUser-agent: Bullseye/1.0\r\nDisallow: /\r\nUser-agent: BunnySlippers\r\nDisallow: /\r\nUser-agent: CherryPicker\r\nDisallow: /\r\nUser-agent: CherryPickerSE/1.0\r\nDisallow: /\r\nUser-agent: CherryPickerElite/1.0\r\nDisallow: /\r\nUser-agent: CheeseBot\r\nDisallow: /\r\nUser-agent: CNCDialer\r\nDisallow: /\r\nUser-agent: Copernic\r\nDisallow: /\r\nUser-agent: cosmos\r\nDisallow: /\r\nUser-agent: Crescent\r\nDisallow: /\r\nUser-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0\r\nDisallow: /\r\nUser-agent: DittoSpyder\r\nDisallow: /\r\nUser-agent: DOC\r\nDisallow: /\r\nUser-agent: Download Ninja\r\nDisallow: /\r\nUser-agent: EmailCollector\r\nDisallow: /\r\nUser-agent: EmailSiphon\r\nDisallow: /\r\nUser-agent: EmailWolf\r\nDisallow: /\r\nUser-agent: EroCrawler\r\nDisallow: /\r\nUser-agent: ExtractorPro\r\nDisallow: /\r\nUser-agent: FairAd Client\r\nDisallow: /\r\nUser-agent: fast\r\nDisallow: /\r\nUser-agent: Fetch\r\nDisallow: /\r\nUser-agent: Flaming AttackBot\r\nDisallow: /\r\nUser-agent: Foobot\r\nDisallow: /\r\nUser-agent: Gaisbot\r\nDisallow: /\r\nUser-agent: GetRight/4.2\r\nDisallow: /\r\nUser-agent: grub-client\r\nDisallow: /\r\nUser-agent: Harvest/1.5\r\nDisallow: /\r\nUser-agent: hloader\r\nDisallow: /\r\nUser-agent: httplib\r\nDisallow: /\r\nUser-agent: HTTrack\r\nDisallow: /\r\nUser-agent: HTTrack 3.0\r\nDisallow: /\r\nUser-agent: humanlinks\r\nDisallow: /\r\nUser-agent: InfoNaviRobot\r\nDisallow: /\r\nUser-agent: Iron33/1.0.2\r\nDisallow: /\r\nUser-agent: JennyBot\r\nDisallow: /\r\nUser-agent: JikeSpider\r\nDisallow: /\r\nUser-agent: Jyxobot/1\r\nDisallow: /\r\nUser-agent: k2spider\r\nDisallow: /\r\nUser-agent: Kenjin Spider\r\nDisallow: /\r\nUser-agent: Keyword Density/0.9\r\nDisallow: /\r\nUser-agent: larbin\r\nDisallow: /\r\nUser-agent: LexiBot\r\nDisallow: /\r\nUser-agent: libWeb/clsHTTP\r\nDisallow: /\r\nUser-agent: libwww\r\nDisallow: /\r\nUser-agent: LinkextractorPro\r\nDisallow: /\r\nUser-agent: linko\r\nDisallow: /\r\nUser-agent: LinkScan/8.1a Unix\r\nDisallow: /\r\nUser-agent: LinkWalker\r\nDisallow: /\r\nUser-agent: LNSpiderguy\r\nDisallow: /\r\nUser-agent: lwp-trivial/1.34\r\nDisallow: /\r\nUser-agent: lwp-trivial\r\nDisallow: /\r\nUser-agent: Mata Hari\r\nDisallow: /\r\nUser-agent: Maxthon\r\nDisallow: /\r\nUser-agent: MIIxpc\r\nDisallow: /\r\nUser-agent: MIIxpc/4.2\r\nDisallow: /\r\nUser-agent: Mister PiX\r\nDisallow: /\r\nUser-agent: moget\r\nDisallow: /\r\nUser-agent: moget/2.1\r\nDisallow: /\r\nUser-agent: NetAnts\r\nDisallow: /\r\nUser-agent: NICErsPRO\r\nDisallow: /\r\nUser-agent: NPBot\r\nDisallow: /\r\nUser-agent: Offline Explorer\r\nDisallow: /\r\nUser-agent: Openbot\r\nDisallow: /\r\nUser-agent: Openfind data gatherer\r\nDisallow: /\r\nUser-agent: Openfind\r\nDisallow: /\r\nUser-agent: Oracle Ultra Search\r\nDisallow: /\r\nUser-agent: PerMan\r\nDisallow: /\r\nUser-agent: ProWebWalker\r\nDisallow: /\r\nUser-agent: ProPowerBot/2.14\r\nDisallow: /\r\nUser-agent: Python-urllib\r\nDisallow: /\r\nUser-agent: QueryN Metasearch\r\nDisallow: /\r\nUser-agent: Radiation Retriever 1.1\r\nDisallow: /\r\nUser-agent: RepoMonkey Bait & Tackle/v1.01\r\nDisallow: /\r\nUser-agent: RepoMonkey\r\nDisallow: /\r\nUser-agent: RMA\r\nDisallow: /\r\nUser-agent: searchpreview\r\nDisallow: /\r\nUser-agent: sitecheck.internetseer.com\r\nDisallow: /\r\nUser-agent: SiteSnagger\r\nDisallow: /\r\nUser-agent: SpankBot\r\nDisallow: /\r\nUser-agent: spanner\r\nDisallow: /\r\nUser-agent: suzuran\r\nDisallow: /\r\nUser-agent: Szukacz/1.4\r\nDisallow: /\r\nUser-agent: Teleport\r\nDisallow: /\r\nUser-agent: TeleportPro\r\nDisallow: /\r\nUser-agent: Telesoft\r\nDisallow: /\r\nUser-agent: The Intraformant\r\nDisallow: /\r\nUser-agent: TightTwatBot\r\nDisallow: /\r\nUser-agent: TheNomad\r\nDisallow: /\r\nUser-agent: toCrawl/UrlDispatcher\r\nDisallow: /\r\nUser-agent: True_Robot/1.0\r\nDisallow: /\r\nUser-agent: True_Robot\r\nDisallow: /\r\nUser-agent: turingos\r\nDisallow: /\r\nUser-agent: TurnitinBot\r\nDisallow: /\r\nUser-agent: TurnitinBot/1.5\r\nDisallow: /\r\nUser-agent: UbiCrawler\r\nDisallow: /\r\nUser-agent: UnisterBot\r\nDisallow: /\r\nUser-agent: URL Control\r\nDisallow: /\r\nUser-agent: URL_Spider_Pro\r\nDisallow: /\r\nUser-agent: URLy Warning\r\nDisallow: /\r\nUser-agent: VCI WebViewer VCI WebViewer Win32\r\nDisallow: /\r\nUser-agent: VCI\r\nDisallow: /\r\nUser-agent: WebBandit\r\nDisallow: /\r\nUser-agent: WebBandit/3.50\r\nDisallow: /\r\nUser-agent: WebCapture 2.0\r\nDisallow: /\r\nUser-agent: WebCopier v3.2a\r\nDisallow: /\r\nUser-agent: WebCopier v.2.2\r\nDisallow: /\r\nUser-agent: WebCopier\r\nDisallow: /\r\nUser-agent: Web Image Collector\r\nDisallow: /\r\nUser-agent: WebEnhancer\r\nDisallow: /\r\nUser-agent: wget\r\nDisallow: /\r\nUser-agent: WebReaper\r\nDisallow: /\r\nUser-agent: WebSauger\r\nDisallow: /\r\nUser-agent: Website Quester\r\nDisallow: /\r\nUser-agent: Webster Pro\r\nDisallow: /\r\nUser-agent: WebStripper\r\nDisallow: /\r\nUser-agent: WebZIP\r\nDisallow: /\r\nUser-agent: WebZip/4.0\r\nDisallow: /\r\nUser-agent: WebZIP/4.21\r\nDisallow: /\r\nUser-agent: WebZIP/5.0\r\nDisallow: /\r\nUser-agent: Wget/1.6\r\nDisallow: /\r\nUser-agent: Wget/1.5.3\r\nDisallow: /\r\nUser-agent: Wget\r\nDisallow: /\r\nUser-agent: WWW-Collector-E\r\nDisallow: /\r\nUser-agent: Xenu\r\nDisallow: /\r\nUser-agent: Xenu's Link Sleuth 1.1c\r\nDisallow: /\r\nUser-agent: Xenu's\r\nDisallow: /\r\nUser-agent: Zao\r\nDisallow: /\r\nUser-agent: Zealbot\r\nDisallow: /\r\nUser-agent: Zeus\r\nDisallow: /\r\nUser-agent: Zeus Link Scout\r\nDisallow: /\r\nUser-agent: Zeus 32297 Webster Pro V2.9 Win32\r\nDisallow: /\r\nUser-agent: ZyBORG\r\nDisallow: /\r\n## Ajustes de Bots\r\nUser-agent: Yahoo! Slurp China\r\nDisallow: /\r\nUser-agent: Yandex\r\nDisallow: /\r\nUser-agent: YandexTurbo\r\nDisallow: /\r\nUser-agent: YandexBot\r\nDisallow: /\r\nUser-agent: Nutch\r\nDisallow: /"
    },
    "www.leroymerlin.it": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.leroymerlin.it",
      "body": "User-agent: Yandex\nDisallow: /\nUser-agent: YandexTurbo\nDisallow: /\nUser-agent: YandexBot\nDisallow: /\nUser-agent: *\nDisallow: /*catalogo/*\nDisallow: /*v3/search/*\nDisallow: /*tutte-le-marche/*\nDisallow: /*comunicati*\nDisallow: /*test/*\nDisallow: /search?q=*\nDisallow: /*?keyword=*\nDisallow: /cataloghi/gamma/\nDisallow: /cataloghi/ambiente/\nDisallow: /cataloghi/contenuto-editoriale/\nDisallow: /cataloghi/consiglio-dell-esperto/\nDisallow: /*login?redirectUrl \nDisallow: /*-i.html\nDisallow: /vad/*\nDisallow: /*?version=*\nDisallow: /*avis/produit/*\nDisallow: /*filters=*\nDisallow: /*limit=*\nDisallow: /*sort=*\nDisallow: /*pedagogicalFilters=*\nDisallow: /*includeLocalLanguageOnly=*\nDisallow: /*includeReviewsWithPhotoOnly=*\nDisallow: /*lampade-coordinate/*\nAllow: /*lampade-coordinate/$\nDisallow: /*tende-da-sole/ecobonus-tende-da-sole/\nDisallow: /*tende-a-rullo/tende-a-rullo-motorizzabili/\nDisallow: /*articoli*\nDisallow: /*decorazione/decorazioni-e-tessili-natalizi/*\nDisallow: /*selezione-del-momento*\nDisallow: *sconti-al*\nDisallow: *scontati-al*\nDisallow: *scontato-al*\nDisallow: *in-sconto*\nDisallow: *speciale*\nAllow: /speciale-leroy-merlin-days/\nAllow: *speciale-bagno*\nAllow: *speciale-bonus-casa*\nAllow: *speciale-recensioni*\nDisallow: *del-mese*\nDisallow: *estensione-gamma*\nDisallow: *selezione-di*\nAllow: */offerte/$\nAllow: */offerte/fine-serie/\nAllow: */offerte/codici-e-buoni-sconto/\nAllow: */offerte/black-friday/\nAllow: */offerte/speciale-outlet/\nAllow: */offerte/welcome-black-friday/\nAllow: */offerte/prezzi-shock/$\nAllow: */offerte/giardino-fuori-tutto/\nAllow: */offerte/festa-della-casa-2026/ \nDisallow: *offerte*\nDisallow: *promo-code*\nDisallow: */link$\nDisallow: /mylm/*\nDisallow: /reset-password*\nDisallow: */checkout/*\nDisallow: */shoppinglist/*\nDisallow: /authentication\nDisallow: /selezione-marchi/*\nDisallow: /check-consensi/*\nUser-agent: 008\nDisallow: /\nUser-agent: msnbot\ncrawl-Delay: 10\nUser-agent: bingbot\ncrawl-Delay: 10\nUser-agent: Pinterest\nCrawl-delay: 10\nSitemap: https://www.leroymerlin.it/sitemap-index.xml"
    },
    "www.leroymerlin.pl": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.leroymerlin.pl",
      "body": "User-agent: Googlebot\r\nDisallow:\r\nUser-agent: AdsBot-Google \r\nAllow: /\r\nUser-agent: AdsBot-Google-mobile\r\nAllow: /\r\nUser-agent: Googlebot-image\r\nDisallow:\r\nUser-agent: OAI-SearchBot\r\nDisallow:\r\nUser-agent: *\r\nAllow: /\r\nDisallow: /*&lsmulticriteria\r\nDisallow: /*&segment\r\nDisallow: /*&mcsegment\r\nDisallow: /*reviews.htm*\r\nDisallow: /*queryredirect=*numpage=*\r\nDisallow: /*category.htm*\r\nDisallow: /*num_sujet*\r\nDisallow: /*display=criteria\r\nDisallow: /*xtor\r\nDisallow: /*rubrique\r\nDisallow: /cart/services/*\r\nDisallow: /*cmcible\r\nDisallow: /*CID\r\nDisallow: /*PHPSESSID\r\nDisallow: /*com=\r\nDisallow: /*SALEUNIT_PRICE&\r\nDisallow: /*zone=\r\nDisallow: /*mpng2-front\r\nDisallow: /*merch-pedago\r\nDisallow: /*trier=\r\nDisallow: /*goForum=\r\nDisallow: /*myRedirectUrl\r\nDisallow: /*getReviews.do\r\nDisallow: /*campusOrigin\r\nDisallow: /*product/complementaires/getByProduct.do*\r\nDisallow: /*product-reviews/getReviews.do*\r\nDisallow: /*rdv\r\nDisallow: /*filters=*\r\nDisallow: /*limit=*\r\nDisallow: /*sort=*\r\nDisallow: /avis/produit/\r\nDisallow: /*pedagogicalFilters=*\r\nDisallow: /produits/offer/\r\nDisallow: /productpageservicesfunctionality/\r\nDisallow: */*/RME\r\nDisallow: */*/rme\r\nDisallow: */*/rdv\r\nDisallow: /v3/search/search.do?keyword=\r\nDisallow: */customer/services/userInfos\r\nDisallow: */navigation/api/autocomplete\r\nDisallow: /questions-reponses/produit/\r\nDisallow: /test/\r\nDisallow: /productpageservicesfunctionality/\r\nDisallow: /shoppinglist/\r\nDisallow: /productpageservicesfunctionality/\r\nDisallow: /*goToReviewModule*\r\nDisallow: /?*highlightedOfferCode\r\nDisallow: /?*storeid \r\nDisallow: /media/15/5e37ee987eb45f518507d94e/175899224/nouvel-emmenage5e37ee98de91bb0008f1209d.pdf\r\nDisallow: /finalizeorderapp/\r\nDisallow: /produits/*/link\r\nDisallow: /project-booster/api/events/estimate-form-event\r\nDisallow: /evom/\r\nDisallow: /header-cart/*\r\nDisallow: /customer-space-no-layout-module/*\r\nDisallow: /klient/zamowienie.html\r\nDisallow: /produkty/*/link\r\nDisallow: /produkty/offer/\r\nDisallow: /klient/\r\nDisallow: /szukaj.html?*\r\nDisallow: /7545006/\r\nDisallow: /www/ajax/\r\nDisallow: /header-cart/*\r\nDisallow: /header-cart-module/*\r\nDisallow: /szukaj-produkty*\r\nDisallow: /sklep/*\r\nDisallow: /search?q=*\r\n\r\n\r\nSitemap: https://www.leroymerlin.pl/sitemap-index.xml"
    },
    "www.leroymerlin.pt": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.leroymerlin.pt",
      "body": "#CCDP VERSION 2026-08-25\r\nUser-Agent: *\r\nAllow: /\r\n\r\nDisallow: /v3/mobile/action-search/search.do\r\nDisallow: /*&sort=\r\nDisallow: /*sort=*\r\nDisallow: /*&noIndex=1\r\nDisallow: /*typeSearch=\r\nDisallow: /cart/services/*\r\nDisallow: /*myRedirectUrl\r\nDisallow: /*getReviews.do\r\nDisallow: /*product-reviews/getReviews.do*\r\nDisallow: *goToReviewModule*\r\nDisallow: /*campusOrigin\r\nDisallow: /*product/complementaires/getByProduct.do*\r\nDisallow: /*rdv\r\nDisallow: */*/rdv\r\nDisallow: /*filters=*\r\nDisallow: /*limit=*\r\nDisallow: /*pedagogicalFilters=*\r\nDisallow: */productpageservicesfunctionality/*\r\nDisallow: */*/RME\r\nDisallow: */*/rme\r\nDisallow: */customer/services/userInfos\r\nDisallow: */navigation/api/autocomplete\r\nDisallow: */shoppinglist/*\r\nDisallow: */*?*highlightedOfferCode\r\nDisallow: */*?*storeid\r\nDisallow: /login*\r\nDisallow: /register*\r\nDisallow: /pt/ecommerce*\r\nDisallow: /user*\r\nDisallow: /page/*\r\nDisallow: /reset*\r\nDisallow: /pt/dashboard\r\nDisallow: /pt/admin*\r\nDisallow: /Site*\r\nDisallow: /search?q=*\r\nDisallow: /link*\r\nDisallow: /produtos/*/link\r\nDisallow: */finalizeorderapp/*\r\nDisallow: /product-recommendation/*\r\nDisallow: /headercart/rest/cart-data\r\nDisallow: /getserviceinfo/*\r\nDisallow: /getPRStatistics*\r\nDisallow: /marketplace/lista-vendedores/vendedor/\r\nDisallow: /*autocomplete*\r\nDisallow: /checkout/\r\nDisallow: /*?journey=\r\nDisallow: /*?reflm=\r\nDisallow: /*?origin=\r\n\r\n# USER AGENT BLACKLIST\r\nUser-Agent: expo9\r\nDisallow: /\r\nUser-agent: Orthogaffe\r\nDisallow: /\r\nUser-agent: UbiCrawler\r\nDisallow: /\r\nUser-agent: DOC\r\nDisallow: /\r\nUser-agent: Zao\r\nDisallow: /\r\nUser-agent: Zealbot\r\nDisallow: /\r\nUser-agent: MSIECrawler\r\nDisallow: /\r\nUser-agent: SiteSnagger\r\nDisallow: /\r\nUser-agent: WebStripper\r\nDisallow: /\r\nUser-agent: Fetch\r\nDisallow: /\r\nUser-agent: OfflineExplorer\r\nDisallow: /\r\nUser-agent: Teleport\r\nDisallow: /\r\nUser-agent: TeleportPro\r\nDisallow: /\r\nUser-agent: WebZIP\r\nDisallow: /\r\nUser-agent: linko\r\nDisallow: /\r\nUser-agent: HTTrack\r\nDisallow: /\r\nUser-agent: Xenu\r\nDisallow: /\r\nUser-agent: larbin\r\nDisallow: /\r\nUser-agent: libwww\r\nDisallow: /\r\nUser-agent: ZyBORG\r\nDisallow: /\r\nUser-agent: DownloadNinja\r\nDisallow: /\r\nUser-agent: wget\r\nDisallow: /\r\nUser-agent: grub-client\r\nDisallow: /\r\nUser-agent: WebCopier\r\nDisallow: /\r\nUser-agent: Python-urllib\r\nDisallow: /\r\nUser-agent: URL_Spider_Pro\r\nDisallow: /\r\nUser-agent: EmailCollector\r\nDisallow: /\r\nUser-agent: EmailSiphon\r\nDisallow: /\r\nUser-agent: WebBandit\r\nDisallow: /\r\nUser-agent: EmailWolf\r\nDisallow: /\r\nUser-agent: ExtractorPro\r\nDisallow: /\r\nUser-agent: CopyRightCheck\r\nDisallow: /\r\nUser-agent: Alexibot\r\nDisallow: /\r\nUser-agent: Zeus\r\nDisallow: /\r\nUser-agent: ZeusLinkScout\r\nDisallow: /\r\nUser-agent: EroCrawler\r\nDisallow: /\r\nUser-agent: LinkScan\r\nDisallow: /\r\nUser-agent: WebCapture 2.0\r\nDisallow: /\r\nUser-agent: EtaoSpider\r\nDisallow: /\r\nUser-agent: Black Hole\r\nDisallow: /\r\nUser-agent: Yahoo! Slurp China\r\nDisallow: /\r\nUser-agent: Yandex\r\nDisallow: /\r\nUser-agent: YandexTurbo\r\nDisallow: /\r\nUser-agent: YandexBot\r\nDisallow: /\r\nUser-agent: msnbot\r\nDisallow: /\r\nUser-agent: Slurp\r\nDisallow: /\r\nUser-agent: Nutch\r\nDisallow: /\r\nUser-agent: 008\r\nDisallow: /\r\nUser-agent: CherryPicker\r\nDisallow: /\r\n\r\nSitemap: https://www.leroymerlin.pt/sitemap-index.xml"
    },
    "www.carrefour.fr": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.carrefour.fr",
      "body": "\n\n# Adsbot Instructions\n# Adsbot Instructions for Store Indexing\nUser-agent: AdsBot-Google-Mobile\nUser-agent: AdsBot-Google\nUser-agent: AdIdxBot\nAllow: /set-store\nAllow: /get-store\nAllow: /g\n\n# All Bots Instructions\nUser-agent: *\n\n# Disallow to Index Store 6 only \nDisallow: /set-store\nDisallow: /get-store\nDisallow: /webview\nDisallow: /g\nDisallow: /b\n\n# Noindex to Index Store 6 only\nNoindex: /set-store\nNoindex: /get-store\nNoindex: /webview\nNoindex: /g\nNoindex: /b"
    },
    "www.decathlon.fr": {
      "status": 403,
      "client": "fetch",
      "finalHost": "www.decathlon.fr",
      "body": null
    },
    "www.decathlon.de": {
      "status": 403,
      "client": "fetch",
      "finalHost": "www.decathlon.de",
      "body": null
    },
    "www.conrad.de": {
      "status": 403,
      "client": "fetch",
      "finalHost": "www.conrad.de",
      "body": null
    },
    "www.empik.com": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.empik.com",
      "body": "Sitemap: https://www.empik.com/sitemap.xml\r\nSitemap: https://www.empik.com/sitemap-products/product-sitemap-index.xml\r\n\r\nUser-agent: *\r\nAllow: /\r\n\r\nDisallow: /ajax/\r\nDisallow: /cart/\r\nDisallow: /cart$\r\nDisallow: /cart?*\r\nDisallow: /koszyk/\r\nDisallow: /gateway/api/graphql/cart\r\nDisallow: /gateway/api/graphql/products\r\nDisallow: /pf/assets*.js\r\n\r\nDisallow: /cdn-cgi/challenge-platform/\r\nDisallow: /cdn-cgi/rum*\r\n\r\nDisallow: /moj-empik/rejestracja*\r\nDisallow: /b/mp/js/chatbot.js\r\n\r\nDisallow: /twoje-zamowienie/\r\nDisallow: /twoje-zamowienie?*\r\nDisallow: /tz/\r\nDisallow: /zamowienia/\r\nDisallow: /formularz-zwrotu/\r\n\r\nDisallow: /sysaction/\r\nDisallow: /coll/text*\r\n\r\nDisallow: *addtolist*\r\nDisallow: *?*sort=*\r\nDisallow: *?*searchCategory=*\r\nDisallow: *?*priceFrom=*\r\nDisallow: *?*continue=*\r\nDisallow: *?*hideUnavailable=*\r\nDisallow: *?*resultsPP=*\r\nDisallow: *?*qtype=*\r\nDisallow: *?*reko=*\r\nDisallow: *?*salePrice=*\r\nDisallow: *?*ac=*\r\nDisallow: *?*availability=*\r\nDisallow: *?*editionFrequency=*\r\nDisallow: *?*catalogType=*\r\nDisallow: *?*spellCheck=*\r\nDisallow: *?*availabilitySeparable=*\r\nDisallow: *?*slotName=*\r\nDisallow: *?*curvature=*\r\nDisallow: *?*utm_source=*\r\nDisallow: *?*utm_medium=*\r\nDisallow: *?*productPoolId=*\r\nDisallow: *?*utm_campaign=*\r\nDisallow: *?*productId=*\r\nDisallow: *?*carrierFacet=*\r\nDisallow: *?*_dyncharset=*\r\nDisallow: *?*fr=*\r\nDisallow: *?*availabilityFacet=*\r\nDisallow: *?*pl=*\r\nDisallow: *?*strict=*\r\nDisallow: *?*specialProgram=*\r\nDisallow: *?*salePriceSortFacet=*\r\nDisallow: *?*drilldown=*\r\nDisallow: *?*novelties=*\r\nDisallow: *?*bgCollection=*\r\nDisallow: *?*puste=*\r\nDisallow: *?*utm_content=*\r\nDisallow: *?*SuperSocializerAuth=*\r\nDisallow: *?*composerFacet=*\r\nDisallow: *?*conductorFacet=*\r\nDisallow: *?*specialProgramFacet=*\r\nDisallow: *?*forWhom=*\r\nDisallow: *?*albumOrTrack=*\r\nDisallow: *?*ean=*\r\nDisallow: *?*kwnazwa=*\r\nDisallow: *?*subQuery=*\r\nDisallow: *?*kwinfo=*\r\nDisallow: *?*ref_page=*\r\nDisallow: *?*performerFacet=*\r\nDisallow: *?*recommendedWeight=*\r\nDisallow: *?*authorFacet=*\r\nDisallow: *?*formatFacet=*\r\nDisallow: *?*additionalInformation=*\r\nDisallow: *?*searchcategory=*\r\nDisallow: *?*page1=*\r\nDisallow: *?*resultspp=*\r\nDisallow: *?*noredirect=*\r\nDisallow: *?*_name_=*\r\nDisallow: *?*rateScore=*\r\nDisallow: *?*title=*\r\nDisallow: *?*category=*\r\nDisallow: *?*productPage=*\r\nDisallow: *?*language=*\r\nDisallow: *?*modelFacet=*\r\nDisallow: *?*brand=*\r\nDisallow: *?*rel=*\r\nDisallow: *?*enablejsapi=*\r\nDisallow: *?*objectId=*\r\nDisallow: *?*currentCategory=*\r\nDisallow: *?*_csrf=*\r\nDisallow: *?*lf=*\r\nDisallow: *?*dominantColor=*\r\nDisallow: *?*yearTo=*\r\nDisallow: *?*yearFrom=*\r\nDisallow: *?*utm_term=*\r\nDisallow: *?*utm_conten=*\r\nDisallow: *?*utm_con=*\r\nDisallow: *?*utm_co=*\r\nDisallow: *?*sprzedawca=*\r\nDisallow: *?*source=*\r\nDisallow: *?*saleprice=*\r\n# Disallow: *?*offerId=*\r\nDisallow: *?*numberOfItemsInRange=*\r\nDisallow: *?*mpPriceTo=*\r\nDisallow: *?*letter=*\r\nDisallow: *?*kwnazw=*\r\nDisallow: *?*info=*\r\nDisallow: *?*id=*\r\nDisallow: *?*hideunavailable=*\r\nDisallow: *?*csm=*\r\nDisallow: *?*brandfacet=*\r\nDisallow: *?*_a=*\r\nDisallow: /konto/*\r\n\r\n# Disallow: /szukaj/produkt?start=*\r\n# Allow: /szukaj/produkt?start=*q=*\r\nDisallow: /szukaj/produkt?*&*&*&*\r\nDisallow: /*,s?*&*&*&*\r\nDisallow: /*,s,*?*&*&*&*\r\n\r\n\r\n#-----------------------\r\n#       AdsBot\r\n#-----------------------\r\n\r\nUser-agent: AdsBot-Google\r\nUser-agent: AdsBot-Mobile-Google\r\n\r\nAllow: /\r\nDisallow: /gateway/api/graphql/cart\r\nDisallow: /cdn-cgi/challenge-platform/\r\nDisallow: /cdn-cgi/rum*\r\nDisallow: /b/mp/js/chatbot.js\r\nDisallow: /coll/text*\r\nDisallow: /pf/assets*.js\r\nDisallow: /gateway/api/graphql/products\r\n\r\n#-----------------------\r\n#       BingBot\r\n#-----------------------\r\n\r\nUser-agent: Bingbot\r\nUser-agent: AdIdxBot\r\nAllow: /\r\n\r\nDisallow: /ajax/\r\nDisallow: /cart/\r\nDisallow: /cart$\r\nDisallow: /cart?*\r\nDisallow: /koszyk/\r\nDisallow: /gateway/api/graphql/cart\r\nDisallow: /gateway/api/graphql/products\r\nDisallow: /pf/assets*.js\r\n\r\nDisallow: /cdn-cgi/challenge-platform/\r\nDisallow: /cdn-cgi/rum*\r\n\r\nDisallow: /moj-empik/rejestracja*\r\nDisallow: /b/mp/js/chatbot.js\r\n\r\nDisallow: /twoje-zamowienie/\r\nDisallow: /twoje-zamowienie?*\r\nDisallow: /tz/\r\nDisallow: /zamowienia/\r\nDisallow: /formularz-zwrotu/\r\n\r\nDisallow: /sysaction/\r\nDisallow: /coll/text*\r\n\r\nDisallow: *addtolist*\r\nDisallow: *?*sort=*\r\nDisallow: *?*searchCategory=*\r\nDisallow: *?*priceFrom=*\r\nDisallow: *?*continue=*\r\nDisallow: *?*hideUnavailable=*\r\nDisallow: *?*resultsPP=*\r\nDisallow: *?*qtype=*\r\nDisallow: *?*reko=*\r\nDisallow: *?*salePrice=*\r\nDisallow: *?*ac=*\r\nDisallow: *?*availability=*\r\nDisallow: *?*editionFrequency=*\r\nDisallow: *?*catalogType=*\r\nDisallow: *?*spellCheck=*\r\nDisallow: *?*availabilitySeparable=*\r\nDisallow: *?*slotName=*\r\nDisallow: *?*curvature=*\r\nDisallow: *?*utm_source=*\r\nDisallow: *?*utm_medium=*\r\nDisallow: *?*productPoolId=*\r\nDisallow: *?*utm_campaign=*\r\nDisallow: *?*productId=*\r\nDisallow: *?*carrierFacet=*\r\nDisallow: *?*_dyncharset=*\r\nDisallow: *?*fr=*\r\nDisallow: *?*availabilityFacet=*\r\nDisallow: *?*pl=*\r\nDisallow: *?*strict=*\r\nDisallow: *?*specialProgram=*\r\nDisallow: *?*salePriceSortFacet=*\r\nDisallow: *?*drilldown=*\r\nDisallow: *?*novelties=*\r\nDisallow: *?*bgCollection=*\r\nDisallow: *?*puste=*\r\nDisallow: *?*utm_content=*\r\nDisallow: *?*SuperSocializerAuth=*\r\nDisallow: *?*composerFacet=*\r\nDisallow: *?*conductorFacet=*\r\nDisallow: *?*specialProgramFacet=*\r\nDisallow: *?*forWhom=*\r\nDisallow: *?*albumOrTrack=*\r\nDisallow: *?*ean=*\r\nDisallow: *?*kwnazwa=*\r\nDisallow: *?*subQuery=*\r\nDisallow: *?*kwinfo=*\r\nDisallow: *?*ref_page=*\r\nDisallow: *?*performerFacet=*\r\nDisallow: *?*recommendedWeight=*\r\nDisallow: *?*authorFacet=*\r\nDisallow: *?*formatFacet=*\r\nDisallow: *?*additionalInformation=*\r\nDisallow: *?*searchcategory=*\r\nDisallow: *?*page1=*\r\nDisallow: *?*resultspp=*\r\nDisallow: *?*noredirect=*\r\nDisallow: *?*_name_=*\r\nDisallow: *?*rateScore=*\r\nDisallow: *?*title=*\r\nDisallow: *?*category=*\r\nDisallow: *?*productPage=*\r\nDisallow: *?*language=*\r\nDisallow: *?*modelFacet=*\r\nDisallow: *?*brand=*\r\nDisallow: *?*rel=*\r\nDisallow: *?*enablejsapi=*\r\nDisallow: *?*objectId=*\r\nDisallow: *?*currentCategory=*\r\nDisallow: *?*_csrf=*\r\nDisallow: *?*lf=*\r\nDisallow: *?*dominantColor=*\r\nDisallow: *?*yearTo=*\r\nDisallow: *?*yearFrom=*\r\nDisallow: *?*utm_term=*\r\nDisallow: *?*utm_conten=*\r\nDisallow: *?*utm_con=*\r\nDisallow: *?*utm_co=*\r\nDisallow: *?*sprzedawca=*\r\nDisallow: *?*source=*\r\nDisallow: *?*saleprice=*\r\n# Disallow: *?*offerId=*\r\nDisallow: *?*numberOfItemsInRange=*\r\nDisallow: *?*mpPriceTo=*\r\nDisallow: *?*letter=*\r\nDisallow: *?*kwnazw=*\r\nDisallow: *?*info=*\r\nDisallow: *?*id=*\r\nDisallow: *?*hideunavailable=*\r\nDisallow: *?*csm=*\r\nDisallow: *?*brandfacet=*\r\nDisallow: *?*_a=*\r\nDisallow: /konto/*\r\n\r\nDisallow: /szukaj/produkt*start=*\r\n# Allow: /szukaj/produkt?start=*q=*\r\nDisallow: /szukaj/produkt?*&*&*\r\nDisallow: /*,s?*&*&*&*\r\nDisallow: /*,*s?*&*&*&*\r\n\r\n\r\n\r\n#2025-09-17 12:58\r\n\r\n\r\n\r\n\r\n\r\n\r\n"
    },
    "www.worten.pt": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.worten.pt",
      "body": "User-agent: GPTBot\nUser-agent: CCBot\nDisallow: /produtos/\n\nUser-agent: OAI-SearchBot\nUser-agent: OAI-AdsBot\nAllow: /\nDisallow: /*sort_by=\nDisallow: /*per_page=*\nDisallow: /*?page=\nDisallow: /*?lf=\nDisallow: /*?gQT=\nDisallow: /*?debug=\nDisallow: /*?coupon-code=\nDisallow: /search\nDisallow: /checkout\nDisallow: /carrinho\nDisallow: /cliente\nDisallow: /cliente/\nDisallow: /api\nDisallow: /login\n\nUser-agent: *\nDisallow: /search\nDisallow: /cliente\nDisallow: /carrinho\nDisallow: /api\nDisallow: /store\nDisallow: /login\nDisallow: /*sort_by=relevance\nDisallow: /*sort_by=price&order_by=asc\nDisallow: /*sort_by=clean_name\nDisallow: /*sort_by=price&order_by=desc\nDisallow: /*sort_by=brand_name\nDisallow: /*per_page=*\nDisallow: /cdn-cgi/\nDisallow: /_/\nAllow: /_/sitemap/\nDisallow: /paypal\nDisallow: /teste\nDisallow: /worten-empresas-b2b/\nDisallow: /empresas/\nDisallow: /empresas/campanhas/\nDisallow: /worten-empresas/campanhas\nDisallow: /ofertas-worten-tv-net-voz\nDisallow: /marketplace-see-more-offers\nDisallow: /24h-worten/campanhas-*\nDisallow: /24h-worten/campanhas/*\nDisallow: /24h-worten/campanhas-13\nDisallow: /vertical-teste\nDisallow: /encomendas\nDisallow: /campanha/exclusiva\nDisallow: /portal-incentivos/\nDisallow: /48h-worten/campanhas/time*\nDisallow: /informacao-dados-pessoais\nDisallow: /marketplace-seller\nDisallow: /cliente/\nDisallow: /checkout\nDisallow: /*?gQT=\nDisallow: /*?debug=\nDisallow: /*?page=\nDisallow: /*?lf=\nDisallow: /*?gad\nDisallow: /*?coupon-code=\nDisallow: /canal-denuncias\nDisallow: /inicio\nDisallow: /ajuda/login\nAllow: /worten-life-parceiros$\nDisallow: /worten-life-parceiros/\nDisallow: /i/fd3009ab627e7386b9a05be9fc1da2d84b7e50c3\nDisallow: /i/b3f24e1e12d1022185ec802abd0028edec357587\nDisallow: /i/28cfb5a96c90e4214cb4f68784c615d878ef62d7\nDisallow: /i/3552cc0593606104d3f554df252b34efcd51cd9b\nDisallow: /i/3720af1dc03069e80efe8fbb00b24b9beb0a7b2e\n\nSitemap: https://www.worten.pt/_/sitemap/sitemap_index_wortenpt.xml"
    },
    "www.leboncoin.fr": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.leboncoin.fr",
      "body": "## It's forbidden to use search robots or other automatic methods to access Leboncoin.fr.\n## Access is only permitted with special permission from Leboncoin.fr.\n## Bots friendly\nUser-agent: Googlebot\nUser-agent: Googlebot-Mobile\nUser-agent: Yahoo! Slurp\nUser-agent: Slurp\nUser-agent: slurp\nUser-agent: msnbot\nUser-agent: bingbot\nUser-agent: ia_archiver\nDisallow: /ai\nDisallow: /cp\nDisallow: /support\nDisallow: /account\nDisallow: /api/*\nDisallow: /account/index.html\nDisallow: /dc/decisions_de_justice\nDisallow: /dc/accessibilite\n# Pages\nDisallow: /*souscrire*\nDisallow: /*recherche?*\nDisallow: /*reply*\nDisallow: /*payment*\nDisallow: /*recommandation*\nDisallow: /*consumer-goods*\nDisallow: /*prolonger?*\nDisallow: /*paiement?*\nDisallow: /*direct_purchase*\nDisallow: /annonce*\nDisallow: /compte*\nDisallow: /account*\n# Paramètres d'URL\nDisallow: /*?_gl=*\nDisallow: /*&_gl=\nDisallow: /*?_rsc=*\nDisallow: /*&_rsc=\nDisallow: /*?_sm_vck=*\nDisallow: /*&_sm_vck=\nDisallow: /*?a4XSp=*\nDisallow: /*&a4XSp=\nDisallow: /*?ac=*\nDisallow: /*&ac=\nDisallow: /*?accommodation_type=*\nDisallow: /*&accommodation_type=\nDisallow: /*?action=*\nDisallow: /*&action=\nDisallow: /*?adCategory=*\nDisallow: /*&adCategory=\nDisallow: /*?adId=*\nDisallow: /*&adId=\nDisallow: /*?adOwnerType=*\nDisallow: /*&adOwnerType=\nDisallow: /*?am=*\nDisallow: /*&am=\nDisallow: /*?ancienneAnnonceAvendrealouer=*\nDisallow: /*&ancienneAnnonceAvendrealouer=\nDisallow: /*?anioffnat=*\nDisallow: /*&anioffnat=\nDisallow: /*?anit=*\nDisallow: /*&anit=\nDisallow: /*?applySource=*\nDisallow: /*&applySource=\nDisallow: /*?at_campaign=*\nDisallow: /*&at_campaign=\nDisallow: /*?at_medium=*\nDisallow: /*&at_medium=\nDisallow: /*?bage=*\nDisallow: /*&bage=\nDisallow: /*?bedrooms=*\nDisallow: /*&bedrooms=\nDisallow: /*?bookable=*\nDisallow: /*&bookable=\nDisallow: /*?brand=*\nDisallow: /*&brand=\nDisallow: /*?brid=*\nDisallow: /*&brid=\nDisallow: /*?capacity=*\nDisallow: /*&capacity=\nDisallow: /*?car-listing=*\nDisallow: /*&car-listing=\nDisallow: /*?cat=*\nDisallow: /*&cat=\nDisallow: /*?category=*\nDisallow: /*&category=\nDisallow: /*?catid=*\nDisallow: /*&catid=\nDisallow: /*?carte=*\nDisallow: /*&carte=\nDisallow: /*?city=*\nDisallow: /*&city=\nDisallow: /*?compte=*\nDisallow: /*&compte=\nDisallow: /*?contact_type=*\nDisallow: /*&contact_type=\nDisallow: /*?cs=*\nDisallow: /*&cs=\nDisallow: /*?ct=*\nDisallow: /*&ct=\nDisallow: /*?ctaSelection=*\nDisallow: /*&ctaSelection=\nDisallow: /*?cto_pld=*\nDisallow: /*&cto_pld=\nDisallow: /*?date=*\nDisallow: /*&date=\nDisallow: /*?de=*\nDisallow: /*&de=\nDisallow: /*?de_datepicker=*\nDisallow: /*&de_datepicker=\nDisallow: /*?deal=*\nDisallow: /*&deal=\nDisallow: /*?defaultRadius=*\nDisallow: /*&defaultRadius=\nDisallow: /*?department=*\nDisallow: /*&department=\nDisallow: /*?district=*\nDisallow: /*&district=\nDisallow: /*?dn=*\nDisallow: /*&dn=\nDisallow: /*?down_payment=*\nDisallow: /*&down_payment=\nDisallow: /*?ds=*\nDisallow: /*&ds=\nDisallow: /*?ds_datepicker=*\nDisallow: /*&ds_datepicker=\nDisallow: /*?east=*\nDisallow: /*&east=\nDisallow: /*?editer=*\nDisallow: /*&editer=\nDisallow: /*?elevator=*\nDisallow: /*&elevator=\nDisallow: /*?energy_rate=*\nDisallow: /*&energy_rate=\nDisallow: /*?entry_point=*\nDisallow: /*&entry_point=\nDisallow: /*?entryPoint=*\nDisallow: /*&entryPoint=\nDisallow: /*?email=*\nDisallow: /*&email=\nDisallow: /*?event=*\nDisallow: /*&event=\nDisallow: /*?fb_comment_id=*\nDisallow: /*&fb_comment_id=\nDisallow: /*?fbclid=*\nDisallow: /*&fbclid=\nDisallow: /*?floor_property=*\nDisallow: /*&floor_property=\nDisallow: /*?from=*\nDisallow: /*&from=\nDisallow: /*?fromUrl=*\nDisallow: /*&fromUrl=\nDisallow: /*?furn=*\nDisallow: /*&furn=\nDisallow: /*?furnished=*\nDisallow: /*&furnished=\nDisallow: /*?fw=*\nDisallow: /*&fw=\nDisallow: /*?gad_campaignid=*\nDisallow: /*&gad_campaignid=\nDisallow: /*?gad_source=*\nDisallow: /*&gad_source=\nDisallow: /*?gbraid=*\nDisallow: /*&gbraid=\nDisallow: /*?gclid=*\nDisallow: /*&gclid=\nDisallow: /*?global_condition=*\nDisallow: /*&global_condition=\nDisallow: /*?gref=*\nDisallow: /*&gref=\nDisallow: /*?gtm_latency=*\nDisallow: /*&gtm_latency=\nDisallow: /*?hasPaymentMethods=*\nDisallow: /*&hasPaymentMethods=\nDisallow: /*?hl=*\nDisallow: /*&hl=\nDisallow: /*?holidays_real_estate_type=*\nDisallow: /*&holidays_real_estate_type=\nDisallow: /*?id=*\nDisallow: /*&id=\nDisallow: /*?immo_sell_type=*\nDisallow: /*&immo_sell_type=\nDisallow: /*?itemid=*\nDisallow: /*&itemid=\nDisallow: /*?Itemid=*\nDisallow: /*&Itemid=\nDisallow: /*?itemtype=*\nDisallow: /*&itemtype=\nDisallow: /*?jobc=*\nDisallow: /*&jobc=\nDisallow: /*?jobd=*\nDisallow: /*&jobd=\nDisallow: /*?jobe=*\nDisallow: /*&jobe=\nDisallow: /*?jobf=*\nDisallow: /*&jobf=\nDisallow: /*?jobs=*\nDisallow: /*&jobs=\nDisallow: /*?jobt=*\nDisallow: /*&jobt=\nDisallow: /*?ka=*\nDisallow: /*&ka=\nDisallow: /*?kp=*\nDisallow: /*&kp=\nDisallow: /*?kst=*\nDisallow: /*&kst=\nDisallow: /*?land_plot_surface=*\nDisallow: /*&land_plot_surface=\nDisallow: /*?lat=*\nDisallow: /*&lat=\nDisallow: /*?ldv_equipment=*\nDisallow: /*&ldv_equipment=\nDisallow: /*?ldv_outside_equipment=*\nDisallow: /*&ldv_outside_equipment=\nDisallow: /*?ldv_service=*\nDisallow: /*&ldv_service=\nDisallow: /*?lease_type=*\nDisallow: /*&lease_type=\nDisallow: /*?listId=*\nDisallow: /*&listId=\nDisallow: /*?lng=*\nDisallow: /*&lng=\nDisallow: /*?locationType=*\nDisallow: /*&locationType=\nDisallow: /*?manufacturer=*\nDisallow: /*&manufacturer=\nDisallow: /*?media_type=*\nDisallow: /*&media_type=\nDisallow: /*?model=*\nDisallow: /*&model=\nDisallow: /*?north=*\nDisallow: /*&north=\nDisallow: /*?openForm=*\nDisallow: /*&openForm=\nDisallow: /*?openModal=*\nDisallow: /*&openModal=\nDisallow: /*?opt=*\nDisallow: /*&opt=\nDisallow: /*?option=*\nDisallow: /*&option=\nDisallow: /*?order=*\nDisallow: /*&order=\nDisallow: /*?orientation=*\nDisallow: /*&orientation=\nDisallow: /*?outside_access=*\nDisallow: /*&outside_access=\nDisallow: /*?owner_type=*\nDisallow: /*&owner_type=\nDisallow: /*?p=*\nDisallow: /*&p=\nDisallow: /*?page_id=*\nDisallow: /*&page_id=\nDisallow: /*?pi=*\nDisallow: /*&pi=\nDisallow: /*?place=*\nDisallow: /*&place=\nDisallow: /*?polygon=*\nDisallow: /*&polygon=\nDisallow: /*?pp=*\nDisallow: /*&pp=\nDisallow: /*?preview=*\nDisallow: /*&preview=\nDisallow: /*?price=*\nDisallow: /*&price=\nDisallow: /*?probe=*\nDisallow: /*&probe=\nDisallow: /*?purchaseId=*\nDisallow: /*&purchaseId=\nDisallow: /*?quick_form=*\nDisallow: /*&quick_form=\nDisallow: /*?radius=*\nDisallow: /*&radius=\nDisallow: /*?real_estate_type=*\nDisallow: /*&real_estate_type=\nDisallow: /*?ref=*\nDisallow: /*&ref=\nDisallow: /*?ref_id=*\nDisallow: /*&ref_id=\nDisallow: /*?referrer=*\nDisallow: /*&referrer=\nDisallow: /*?region=*\nDisallow: /*&region=\nDisallow: /*?request_state=*\nDisallow: /*&request_state=\nDisallow: /*?rnd=*\nDisallow: /*&rnd=\nDisallow: /*?rooms=*\nDisallow: /*&rooms=\nDisallow: /*?rtbhc=*\nDisallow: /*&rtbhc=\nDisallow: /*?s=*\nDisallow: /*&s=\nDisallow: /*?sa=*\nDisallow: /*&sa=\nDisallow: /*?saved_id_view=*\nDisallow: /*&saved_id_view=\nDisallow: /*?sc=*\nDisallow: /*&sc=\nDisallow: /*?selectedWarranty=*\nDisallow: /*&selectedWarranty=\nDisallow: /*?seoSlug=*\nDisallow: /*&seoSlug=\nDisallow: /*?sfnsn=*\nDisallow: /*&sfnsn=\nDisallow: /*?shippable=*\nDisallow: /*&shippable=\nDisallow: /*?shippable_auto=*\nDisallow: /*&shippable_auto=\nDisallow: /*?shipping_type=*\nDisallow: /*&shipping_type=\nDisallow: /*?shoes=*\nDisallow: /*&shoes=\nDisallow: /*?shoet=*\nDisallow: /*&shoet=\nDisallow: /*?slug=*\nDisallow: /*&slug=\nDisallow: /*?sort=*\nDisallow: /*&sort=\nDisallow: /*?source=*\nDisallow: /*&source=\nDisallow: /*?south=*\nDisallow: /*&south=\nDisallow: /*?specificities=*\nDisallow: /*&specificities=\nDisallow: /*?square=*\nDisallow: /*&square=\nDisallow: /*?srsltid=*\nDisallow: /*&srsltid=\nDisallow: /*?st=*\nDisallow: /*&st=\nDisallow: /*?stateFilter=*\nDisallow: /*&stateFilter=\nDisallow: /*?step=*\nDisallow: /*&step=\nDisallow: /*?success=*\nDisallow: /*&success=\nDisallow: /*?swp=*\nDisallow: /*&swp=\nDisallow: /*?t=*\nDisallow: /*&t=\nDisallow: /*?tab=*\nDisallow: /*&tab=\nDisallow: /*?target=*\nDisallow: /*&target=\nDisallow: /*?task=*\nDisallow: /*&task=\nDisallow: /*?tblci=*\nDisallow: /*&tblci=\nDisallow: /*?tca=*\nDisallow: /*&tca=\nDisallow: /*?text=*\nDisallow: /*&text=\nDisallow: /*?thribee_click_id=*\nDisallow: /*&thribee_click_id=\nDisallow: /*?tm_medium=*\nDisallow: /*&tm_medium=\nDisallow: /*?tpclid=*\nDisallow: /*&tpclid=\nDisallow: /*?trk=*\nDisallow: /*&trk=\nDisallow: /*?type=*\nDisallow: /*&type=\nDisallow: /*?ur=*\nDisallow: /*&ur=\nDisallow: /*?user_type=*\nDisallow: /*&user_type=\nDisallow: /*?usg=*\nDisallow: /*&usg=\nDisallow: /*?utm_campaign=*\nDisallow: /*&utm_campaign=\nDisallow: /*?utm_content=*\nDisallow: /*&utm_content=\nDisallow: /*?utm_medium=*\nDisallow: /*&utm_medium=\nDisallow: /*?utm_source=*\nDisallow: /*&utm_source=\nDisallow: /*?utm_term=*\nDisallow: /*&utm_term=\nDisallow: /*?v=*\nDisallow: /*&v=\nDisallow: /*?ved=*\nDisallow: /*&ved=\nDisallow: /*?vertical=*\nDisallow: /*&vertical=\nDisallow: /*?view=*\nDisallow: /*&view=\nDisallow: /*?west=*\nDisallow: /*&west=\nDisallow: /*?xtor=*\nDisallow: /*&xtor=\nDisallow: /*?zipcode=*\nDisallow: /*&zipcode=\nDisallow: /*?zipCode=*\nDisallow: /*&zipCode=\n# Adviews\nDisallow: /classified/latest/\nDisallow: /vi/*\nDisallow: /li\nDisallow: /*/*.htm\nDisallow: /ad/animaux/*\nDisallow: /ad/animaux_perdus/*\nDisallow: /ad/autres/*\n# Holidays calendar pages\nDisallow: /*/vacancier/\n# Bloquage des Demandes\nDisallow: /*/demandes\n# all\nDisallow: /*th=0\nDisallow: /aw\nDisallow: /st\nDisallow: /maintenance.html\nDisallow: /pub\nDisallow: /report-abuse\nDisallow: /profile/\nDisallow: /oauth2callback\nDisallow: /discovery/*\n# Blocage des /Recherche (Rav)\nDisallow: /recherche\nDisallow: /visual_similar_ads/*\nDisallow: /annonces_similaires/*\nDisallow: /ad/*/lp/*\nDisallow: /boutique/*/*print=0\nDisallow: /boutique/*.htm?*ca=*_k&*\nDisallow: /popup_siege_social.htm\nDisallow: /dc/popup_siege_social\nDisallow: /*_mediametrie\n#on autorise boutiques (.htm)\nAllow: /boutique*\n#on autorise les html\nAllow: /*/*.html\nDisallow: /*/bonnes_affaires\n#Partners\nDisallow: /partners/*\nAllow: /partners/sofinco*\n#Messaging\nDisallow: /messages/*\nAllow: /*\n\n## AdsBot-Google and AdsBot-Google-Mobile\nUser-agent: AdsBot-Google\nUser-agent: AdsBot-Google-Mobile\n# friendly exceptions\nAllow: /recherche?category=2&*\nAllow: /recherche?category=9&*\nAllow: /recherche?category=10&*\nAllow: /recherche?category=11&*\nAllow: /recherche?category=13&*\nAllow: /aide.htm\nAllow: /aide_pro.htm\n# www2 only\nAllow: /livres\nAllow: /linge_de_maison\nDisallow: /ai\nDisallow: /cp\nDisallow: /support\nDisallow: /account\nDisallow: /account/index.html\nDisallow: /dc/decisions_de_justice\nDisallow: /dc/accessibilite\n# Adviews\nDisallow: /annonce\nDisallow: /classified/latest/\nAllow: /annonces\nDisallow: /vi/*.htm\nDisallow: /li\n# Holidays calendar pages\nDisallow: /*/vacancier/\n# Bloquage des Demandes\nDisallow: /*/demandes\n# all\nDisallow: /*th=0\nDisallow: /aw\nDisallow: /st\nDisallow: /maintenance.html\nDisallow: /pub\nDisallow: /report-abuse\nDisallow: /profile/\nDisallow: /oauth2callback\nDisallow: /boutique/*/*print=0\nDisallow: /boutique/*.htm?*ca=*_k&*\nDisallow: /popup_siege_social.htm\nDisallow: /dc/popup_siege_social\nDisallow: /*_mediametrie\nDisallow: /discovery/*\n#on autorise boutiques (.htm)\nAllow: /boutique*\n#on autorise les html\nAllow: /*/*.html\n#on bloque les annonces avec et sans parametres\nDisallow: /*/*.htm\nDisallow: /*/bonnes_affaires/\nDisallow: /f/*\nAllow: /*\n\n# Allow specific Ads Googlebot to access all our pages\nUser-agent: Mediapartners-Google\nAllow: /*\n\n# Allow proximic to access all our pages\nUser-agent: proximic\nDisallow:\n\n# LLMs / AI SEARCH / RETRIEVAL bots\nUser-agent: OAI-SearchBot\nUser-agent: PerplexityBot\nUser-agent: Claude-SearchBot\nUser-agent: Applebot\nUser-agent: DuckAssistBot\nUser-agent: MistralAI-Index\nUser-agent: ChatGPT-User\nUser-agent: Perplexity-User\nUser-agent: Claude-User\nUser-agent: MistralAI-User\nDisallow: /account\nDisallow: /messages/\nDisallow: /profile/\nDisallow: /oauth2callback\nDisallow: /support\nDisallow: /report-abuse\nDisallow: /pub\nDisallow: /partners/\nDisallow: /discovery/\nDisallow: /recherche\nDisallow: /*/demandes\nDisallow: /*/vacancier/\nDisallow: /ad/*/lp/\nDisallow: /ad/animaux/\nDisallow: /ad/animaux_perdus/\nDisallow: /ad/autres/\nAllow: /ad/voitures/\nDisallow: /ad/\nAllow: /\n\n# LLMs / AI TRAINING\nUser-agent: GPTBot\nUser-agent: ClaudeBot\nUser-agent: anthropic-ai\nUser-agent: CCBot\nUser-agent: Google-Extended\nUser-agent: Applebot-Extended\nDisallow: /account\nDisallow: /messages/\nDisallow: /profile/\nDisallow: /oauth2callback\nDisallow: /support\nDisallow: /report-abuse\nDisallow: /pub\nDisallow: /partners/\nDisallow: /discovery/\nDisallow: /recherche\nDisallow: /ad/\nAllow: /\n\nUser-agent: Bytespider\nDisallow: /\n\n# Link to the sitemap index file\nSitemap: https://www.leboncoin.fr/auto-main.xml\n\n# Allow Meta's crawler for Instagram feature\nUser-agent: facebookexternalhit\nAllow: /dc/cookies\nDisallow: /*\n"
    },
    "www.kleinanzeigen.de": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.kleinanzeigen.de",
      "body": "User-agent: *\nAllow: /?utm_source=ekde$\nDisallow: /*?utm_source=ekde\nDisallow: /account/index.html\nDisallow: /ad/\nDisallow: /annonces/\nDisallow: /back-from-mobile.html\nDisallow: /boutiques/\nDisallow: /ck/\nDisallow: /classified/latest/\nDisallow: /dc/accessibilite\nDisallow: /dc/decisions_de_justice\nDisallow: /m-abmelden.html\nDisallow: /m-anzeige-loeschen.html\nDisallow: /m-anzeige-loeschen-mail.html\nDisallow: /m-anzeige-pausieren.html\nDisallow: /m-anzeige-verlaengern-mail.html\nDisallow: /m-benutzer-aktivierung.html\nDisallow: /m-benutzer-aktivierungsmail.html\nDisallow: /m-bestellung-abgeschlossen.html\nDisallow: /m-bestellung-aufnehmen.html\nDisallow: /m-einspruch-erheben.html\nDisallow: /m-einstellungen.html\nDisallow: /m-einstellungen-bearbeiten.html\nDisallow: /m-merkliste.html\nDisallow: /m-merkliste-bearbeiten.html\nDisallow: /m-nutzer-entfolgen-ajax.html\nDisallow: /m-nutzer-folgen-ajax.html\nDisallow: /m-passwort-erneuern.html\nDisallow: /m-passwort-vergessen.html\nDisallow: /m-pp-rt.html\nDisallow: /m-suche-verwenden.html\nDisallow: /m-verkaufsschild-erstellen.html\nDisallow: /m-meine-nutzer.html\nDisallow: /m-meine-suchen.html\nDisallow: /m-verifikation.html\nDisallow: /checkout.html\nDisallow: /loesungscenter.html\nDisallow: /wallet.html\nDisallow: /maklerempfehlung\nDisallow: /m-maklerempfehlung-abmelden.html\nDisallow: /m-maklerempfehlung-leaddaten.html\nDisallow: /messages/\nDisallow: /zur-merkliste.html\nDisallow: /p-anzeige-abschicken.html\nDisallow: /p-anzeige-aufgeben-bestaetigung.html\nDisallow: /p-anzeige-bearbeiten.html\nDisallow: /p-anzeige-vorschau.html\nDisallow: /p-bild-hochladen.html\nDisallow: /p-kategorie-aendern.html\nDisallow: /s-feed.rss\nDisallow: /s-kategorie-baum.html\nDisallow: /s-bestandsliste.html\nDisallow: /s-suchanfrage.html\nDisallow: /m-einloggen.html\nDisallow: /m-einloggen-sso.html\nDisallow: /m-login-bestaetigen.html\nDisallow: /poc-astro\nDisallow: /s-oac\nDisallow: /s-poac\nDisallow: /search\nDisallow: /SEARCH\nDisallow: /HOME\nDisallow: /BROWSE\nDisallow: /VIP\nDisallow: /PVIP\nDisallow: /MVIP\nDisallow: /EVIP\nDisallow: /DVIP\nDisallow: /facebook/\nDisallow: /*/topAds\nDisallow: /*/s-anzeige:gesuche\nDisallow: /*/anzeige:gesuche\nDisallow: /*/s-anzeige:angebote\nDisallow: /*/anzeige:angebote\nDisallow: /*/s-direktkaufen:aktiv\nDisallow: /s-direktkaufen:aktiv\nDisallow: /*/direktkaufen:aktiv\nDisallow: /s-paketdienst:*\nDisallow: /*/paketdienst:*\nDisallow: /*+options:*\nDisallow: /*/anbieter:*\nDisallow: /*/sortierung:*\nDisallow: /*sortierung:entfernung*\nDisallow: /*/preis:*\nDisallow: /*/k0*r30\nDisallow: /*/k0*r150\nDisallow: /*/l*r30\nDisallow: /*/l*r150\nDisallow: /*/c*r30\nDisallow: /*/c*r150\nDisallow: /*/k0*r5\nDisallow: /*/k0*r10\nDisallow: /*/k0*r20\nDisallow: /*/k0*r50\nDisallow: /*/k0*r100\nDisallow: /*/k0*r200\nDisallow: /*/l*r5\nDisallow: /*/l*r10\nDisallow: /*/l*r20\nDisallow: /*/l*r50\nDisallow: /*/l*r100\nDisallow: /*/l*r200\nDisallow: /*/c*r5\nDisallow: /*/c*r10\nDisallow: /*/c*r20\nDisallow: /*/c*r50\nDisallow: /*/c*r100\nDisallow: /*/c*r200\nDisallow: /*/k0*r1\nDisallow: /*/k0*r2\nDisallow: /*/k0*r15\nDisallow: /*/c*r1\nDisallow: /*/c*r2\nDisallow: /*/c*r15\nDisallow: /*/l*r1\nDisallow: /*/l*r2\nDisallow: /*/l*r15\nDisallow: /*.json\nDisallow: /*@lt\nDisallow: /*simcid=*\nDisallow: /api\nDisallow: /belen-gateway/*\nDisallow: /*autos.anzahl_tueren_s*\nDisallow: /*autos.km_i*\nDisallow: /*autos.ez_i*\nDisallow: /*autos.material_innenausstattung_s*\nDisallow: /*autos.power_i*\nDisallow: /*autos.schaden_s*\nDisallow: /*autos.schadstoffklasse_s*\nDisallow: /*autos.tuevy_i*\nDisallow: /*autos.umweltplakette_s*\nDisallow: /*versand_s*\nDisallow: /*versand:*\nDisallow: /*condition_s*\nDisallow: /*kleidung_herren.groesse_s*\nDisallow: /*kleidung_damen.groesse_s*\nDisallow: /*schuhe_damen.groesse_s*\nDisallow: /*schuhe_herren.groesse_s*\nDisallow: /*hunde.geimpft_s*\nDisallow: /*hunde.erlaubnis_s*\nDisallow: /*katzen.geimpft_s*\nDisallow: /*katzen.erlaubnis_s*\nDisallow: /*auf_zeit_wg.mietart_s*\nDisallow: /*auf_zeit_wg.rauchen_s*\nDisallow: /*ferienwohnung_ferienhaus.location_s*\nDisallow: /*garage_lagerraum.provision_s*\nDisallow: /*gewerbeimmobilien.provision_s*\nDisallow: /*grundstuecke_garten.provision_s*\nDisallow: /*grundstuecke_garten.type_s*\nDisallow: /*grundstuecke_garten.art_s*\nDisallow: /*haus_mieten.haustyp_s*\nDisallow: /*online_tour_s*\nDisallow: /*simcid=*\nDisallow: /*wohnung_mieten.swap_s*\nDisallow: /*work_experience_s*\nDisallow: /*working_hours_s*\nDisallow: /*vermisste_tiere.art_s*\nDisallow: /*zubehoer.type_s*\nDisallow: /*zubehoer.art_s*\nDisallow: /gdpr/*\nDisallow: /liberty/*\nDisallow: /bffstatic/*\nDisallow: /*wohnung_kaufen.etage_i*\nDisallow: /*wohnung_mieten.etage_i*\nDisallow: /*.qm_d*\nDisallow: /*.zimmer_d*\nDisallow: /*.verfuegbarm_i*\nDisallow: /*gewerbeimmobilien.preis_pro_qm_d*\nDisallow: /*auf_zeit_wg.anzahl_mitbewohner_i*\nDisallow: /*haus_kaufen.grundstuecksflaeche_d*\nDisallow: /*motorraeder_roller.km_i*\nDisallow: /*motorraeder_roller.ez_i*\nDisallow: /*motorraeder_roller.hubraum_i*\nDisallow: /*motorraeder_roller.leistung_i*\nDisallow: /*motorraeder_roller.tuevy_i*\nDisallow: /*wohnwagen_mobile.ez_i*\nDisallow: /*.wage_d*\nDisallow: /*.gehausegrosse_i*\nDisallow: /*.wearables.ringgrosse_i*\nDisallow: /*.*_s:(*,*\nDisallow: /*.*_s:(*%2C*\nDisallow: /*.*_s:(*%2c*\nDisallow: /*.*_i:(*,*\nDisallow: /*.*_i:(*%2C*\nDisallow: /*.*_i:(*%2c*\nDisallow: /*.*_d:(*,*\nDisallow: /*.*_d:(*%2C*\nDisallow: /*.*_d:(*%2c*\nDisallow: /*.*_b:(*,*\nDisallow: /*.*_b:(*%2C*\nDisallow: /*.*_b:(*%2c*\nDisallow: /*?*view=karte\nDisallow: /*%%E\nDisallow: /*%E8%AF%81\nDisallow: /*%E4%B8%9A%E8%AF%81\nDisallow: /*%E5%A4%A7\nDisallow: /*%E5%81%87\nDisallow: /*%E5%9F%9F\nDisallow: /*/seite:6*\nDisallow: /*/seite:7*\nDisallow: /*/seite:8*\nDisallow: /*/seite:9*\nDisallow: /*/seite:10*\nDisallow: /*/seite:11*\nDisallow: /*/seite:12*\nDisallow: /*/seite:13*\nDisallow: /*/seite:14*\nDisallow: /*/seite:15*\nDisallow: /*/seite:16*\nDisallow: /*/seite:17*\nDisallow: /*/seite:18*\nDisallow: /*/seite:19*\nDisallow: /*/seite:20*\nDisallow: /*/seite:21*\nDisallow: /*/seite:22*\nDisallow: /*/seite:23*\nDisallow: /*/seite:24*\nDisallow: /*/seite:25*\nDisallow: /*/seite:26*\nDisallow: /*/seite:27*\nDisallow: /*/seite:28*\nDisallow: /*/seite:29*\nDisallow: /*/seite:30*\nDisallow: /*/seite:31*\nDisallow: /*/seite:32*\nDisallow: /*/seite:33*\nDisallow: /*/seite:34*\nDisallow: /*/seite:35*\nDisallow: /*/seite:36*\nDisallow: /*/seite:37*\nDisallow: /*/seite:38*\nDisallow: /*/seite:39*\nDisallow: /*/seite:40*\nDisallow: /*/seite:41*\nDisallow: /*/seite:42*\nDisallow: /*/seite:43*\nDisallow: /*/seite:44*\nDisallow: /*/seite:45*\nDisallow: /*/seite:46*\nDisallow: /*/seite:47*\nDisallow: /*/seite:48*\nDisallow: /*/seite:49*\nDisallow: /*/seite:50*\nDisallow: /*/seite:51*\nDisallow: /*/seite:52*\nDisallow: /*/seite:53*\nDisallow: /*/seite:54*\nDisallow: /*/seite:55*\nDisallow: /*/seite:56*\nDisallow: /*/seite:57*\nDisallow: /*/seite:58*\nDisallow: /*/seite:59*\n\nUser-agent: Adsbot-Google\nDisallow:\n\nUser-agent: Mediapartners-Google\nDisallow:\n\nUser-agent: zoomRank/2.0\nDisallow: /\n\nUser-agent: Google-Display-Ads-Bot\nDisallow:\n\nUser-agent: OAI-SearchBot\nAllow: /\n\nUser-agent: GPTBot\nAllow: /\nDisallow: /m-anzeige-loeschen.html\nDisallow: /m-benutzer-aktivierungsmail.html\nDisallow: /m-bestellung-aufnehmen.html\nDisallow: /m-einspruch-erheben.html\nDisallow: /m-einstellungen.html\nDisallow: /m-nachrichten.html\nDisallow: /m-meine-anzeigen.html\nDisallow: /m-einstellungen-bearbeiten.html\nDisallow: /m-merkliste.html\nDisallow: /m-merkliste-bearbeiten.html\nDisallow: /m-nutzer-entfolgen-ajax.html\nDisallow: /m-nutzer-folgen-ajax.html\nDisallow: /m-passwort-erneuern.html\nDisallow: /m-passwort-vergessen.html\nDisallow: /m-meine-nutzer.html\nDisallow: /m-meine-suchen.html\nDisallow: /m-verifikation.html\nDisallow: /checkout.html\nDisallow: /loesungscenter.html\nDisallow: /wallet.html\nDisallow: /maklerempfehlung\nDisallow: /m-benutzer-anmeldung.html\nDisallow: /m-maklerempfehlung-abmelden.html\nDisallow: /m-maklerempfehlung-leaddaten.html\nDisallow: /p-anzeige-abschicken.html\nDisallow: /p-anzeige-aufgeben-bestaetigung.html\nDisallow: /p-anzeige-bearbeiten.html\nDisallow: /p-anzeige-vorschau.html\nDisallow: /p-bild-hochladen.html\nDisallow: /p-kategorie-aendern.html\nDisallow: /s-kategorie-baum.html\nDisallow: /s-bestandsliste.html\nDisallow: /s-suchanfrage.html\nDisallow: /m-einloggen.html\nDisallow: /m-einloggen-sso.html\nDisallow: /m-login-bestaetigen.html\nDisallow: /gdpr/*\nDisallow: /liberty/*\nDisallow: /bffstatic/*\n\nUser-agent: ChatGPT-User\nAllow: /\nDisallow: /m-anzeige-loeschen.html\nDisallow: /m-benutzer-aktivierungsmail.html\nDisallow: /m-bestellung-aufnehmen.html\nDisallow: /m-einspruch-erheben.html\nDisallow: /m-einstellungen.html\nDisallow: /m-nachrichten.html\nDisallow: /m-meine-anzeigen.html\nDisallow: /m-einstellungen-bearbeiten.html\nDisallow: /m-merkliste.html\nDisallow: /m-merkliste-bearbeiten.html\nDisallow: /m-nutzer-entfolgen-ajax.html\nDisallow: /m-nutzer-folgen-ajax.html\nDisallow: /m-passwort-erneuern.html\nDisallow: /m-passwort-vergessen.html\nDisallow: /m-meine-nutzer.html\nDisallow: /m-meine-suchen.html\nDisallow: /m-verifikation.html\nDisallow: /checkout.html\nDisallow: /loesungscenter.html\nDisallow: /wallet.html\nDisallow: /maklerempfehlung\nDisallow: /m-benutzer-anmeldung.html\nDisallow: /m-maklerempfehlung-abmelden.html\nDisallow: /m-maklerempfehlung-leaddaten.html\nDisallow: /p-anzeige-abschicken.html\nDisallow: /p-anzeige-aufgeben-bestaetigung.html\nDisallow: /p-anzeige-bearbeiten.html\nDisallow: /p-anzeige-vorschau.html\nDisallow: /p-bild-hochladen.html\nDisallow: /p-kategorie-aendern.html\nDisallow: /s-kategorie-baum.html\nDisallow: /s-bestandsliste.html\nDisallow: /s-suchanfrage.html\nDisallow: /m-einloggen.html\nDisallow: /m-einloggen-sso.html\nDisallow: /m-login-bestaetigen.html\nDisallow: /gdpr/*\nDisallow: /liberty/*\nDisallow: /bffstatic/*\n\nUser-agent: PerplexityBot\nAllow: /\nDisallow: /m-anzeige-loeschen.html\nDisallow: /m-benutzer-aktivierungsmail.html\nDisallow: /m-bestellung-aufnehmen.html\nDisallow: /m-einspruch-erheben.html\nDisallow: /m-einstellungen.html\nDisallow: /m-nachrichten.html\nDisallow: /m-meine-anzeigen.html\nDisallow: /m-einstellungen-bearbeiten.html\nDisallow: /m-merkliste.html\nDisallow: /m-merkliste-bearbeiten.html\nDisallow: /m-nutzer-entfolgen-ajax.html\nDisallow: /m-nutzer-folgen-ajax.html\nDisallow: /m-passwort-erneuern.html\nDisallow: /m-passwort-vergessen.html\nDisallow: /m-meine-nutzer.html\nDisallow: /m-meine-suchen.html\nDisallow: /m-verifikation.html\nDisallow: /checkout.html\nDisallow: /loesungscenter.html\nDisallow: /wallet.html\nDisallow: /maklerempfehlung\nDisallow: /m-benutzer-anmeldung.html\nDisallow: /m-maklerempfehlung-abmelden.html\nDisallow: /m-maklerempfehlung-leaddaten.html\nDisallow: /p-anzeige-abschicken.html\nDisallow: /p-anzeige-aufgeben-bestaetigung.html\nDisallow: /p-anzeige-bearbeiten.html\nDisallow: /p-anzeige-vorschau.html\nDisallow: /p-bild-hochladen.html\nDisallow: /p-kategorie-aendern.html\nDisallow: /s-kategorie-baum.html\nDisallow: /s-bestandsliste.html\nDisallow: /s-suchanfrage.html\nDisallow: /m-einloggen.html\nDisallow: /m-einloggen-sso.html\nDisallow: /m-login-bestaetigen.html\nDisallow: /gdpr/*\nDisallow: /liberty/*\nDisallow: /bffstatic/*\n\nUser-agent: AmazonAdBot\nDisallow:\n\nSitemap: https://www.kleinanzeigen.de/sitemap_index.xml\n"
    },
    "www.olx.pl": {
      "status": 200,
      "client": "curl",
      "finalHost": "www.olx.pl",
      "body": "# sitecode:olxpl-desktop\nHost: https://www.olx.pl \nSitemap: https://www.olx.pl/sitemap.xml\nUser-agent: *\nDisallow: */ajax/\nDisallow: /adminpanel/\nDisallow: /api/\nDisallow: */facebook/\nDisallow: */rss/\nDisallow: */konto/\nDisallow: */mojolx/\nDisallow: /drukuj/\nDisallow: /oferta/ulotka/\nDisallow: /oferta/kontakt/\nDisallow: /platnosci/\nDisallow: /searchform/\nDisallow: /nowe-ogloszenie/confirm/\nDisallow: /nowe-ogloszenie/confirmpage/\nDisallow: /i2/oferta/abuse/\nDisallow: /m/oferta/abuse/\nDisallow: /i2/oferta/kontakt/*\nDisallow: /api/open/oauth/token/\nDisallow: */i2/*\nAllow: /api/v1/offers/\nAllow: /api/v1/targeting/\nAllow: /api/v1/friendly-links/\nAllow: /\n"
    },
    "www.olx.pt": {
      "status": 200,
      "client": "curl",
      "finalHost": "www.olx.pt",
      "body": "# sitecode:olxpt-desktop\nHost: https://www.olx.pt \nSitemap: https://www.olx.pt/sitemap.xml\nUser-agent: *\nDisallow: */ajax/\nDisallow: /adminpanel/\nDisallow: /api/\nDisallow: */facebook/\nDisallow: */rss/\nDisallow: */account/\nDisallow: */myaccount/\nDisallow: /adprint/\nDisallow: /anuncio/leaflet/\nDisallow: /anuncio/contact/\nDisallow: /payment/\nDisallow: /searchform/\nDisallow: /anunciar/confirm/\nDisallow: /anunciar/confirmpage/\nDisallow: /i2/anuncio/abuse/\nDisallow: /m/anuncio/abuse/\nDisallow: /i2/anuncio/contact/*\nDisallow: /api/open/oauth/token/\nDisallow: */i2/*\nAllow: /\n"
    },
    "es.wallapop.com": {
      "status": 200,
      "client": "fetch",
      "finalHost": "es.wallapop.com",
      "body": "User-agent: CriteoBot/0.1\nDisallow: /\nUser-agent: SirdataBot\nDisallow: /\nUser-agent: trovitBot\nDisallow: /\nUser-agent: DataForSeoBot\nDisallow: /\nUser-agent: e-SocietyRobot\nDisallow: /\nUser-agent: psbot\nDisallow: /\nUser-agent: Zao\nDisallow: /\nUser-agent: WebZIP\nDisallow: /\nUser-agent: Teleport\nDisallow: /\nUser-agent: Exabot\nDisallow: /\nUser-agent: sitecheck.internetseer.com\nDisallow: /\nUser-agent: linko\nDisallow: /\nUser-agent: TeleportPro\nDisallow: /\nUser-agent: Speedy\nDisallow: /\nUser-agent: Zealbot\nDisallow: /\nUser-agent: HTTrack\nDisallow: /\nUser-agent: WebZIP\nDisallow: /\nUser-agent: Yandex\nDisallow: /\nUser-agent: MSIECrawler\nDisallow: /\nUser-agent: Microsoft.URL.Control\nDisallow: /\nUser-agent: linko\nDisallow: /\nUser-agent: dotbot\nDisallow: /\nUser-agent: SiteSnagger\nDisallow: /\nUser-agent: Xenu\nDisallow: /\nUser-agent: HTTrack\nDisallow: /\nUser-agent: Bloglines/3.1\nDisallow: /\nUser-agent: WebStripper\nDisallow: /\nUser-agent: larbin\nDisallow: /\nUser-agent: wget\nDisallow: /\nUser-agent: Mail.Ru\nDisallow: /\nUser-agent: WebCopier\nDisallow: /\nUser-agent: libwww\nDisallow: /\nUser-agent: grub-client\nDisallow: /\nUser-agent: Jyxobot/1\nDisallow: /\nUser-agent: Fetch\nDisallow: /\nUser-agent: ZyBORG\nDisallow: /\nUser-agent: NPBot\nDisallow: /\nUser-agent: cityreview\nDisallow: /\nUser-agent: Offline Explorer\nDisallow: /\nUser-agent: Download Ninja\nDisallow: /\nUser-agent: WebReaper\nDisallow: /\nUser-agent: ia_archiver\nDisallow: /\nUser-agent: Teleport\nDisallow: /\nUser-agent: Slurp\nDisallow: /\nUser-agent: UbiCrawler\nDisallow: /\nUser-agent: TeleportPro\nDisallow: /\nUser-agent: Maxthon\nDisallow: /\nUser-agent: DOC\nDisallow: /\nUser-agent: CNCDialer\nDisallow: /\nUser-agent: 360Spider\nDisallow: /\nUser-agent: Aboundex\nDisallow: /\nUser-agent: ADmantX\nDisallow: /\nUser-agent: AhrefsBot\nDisallow: /\nUser-agent: AiHitBot\nDisallow: /\nUser-agent: AiHitBot\nDisallow: /\nUser-agent: archive.org_bot\nDisallow: /\nUser-agent: BacklinkCrawler\nDisallow: /\nUser-agent: BLEXBot\nDisallow: /\nUser-agent: CCBot\nDisallow: /\nUser-agent: crawler4j\nDisallow: /\nUser-agent: Discobot\nDisallow: /\nUser-agent: DomainCrawler\nDisallow: /\nUser-agent: Dotbot\nDisallow: /\nUser-agent: MauiBot\nDisallow: /\nUser-agent: AspiegelBot\nDisallow: /\nUser-agent: PetalBot\nDisallow: /\nUser-agent: MJ12Bot\nDisallow: /\nUser-agent: Ezooms\nDisallow: /\nUser-agent: Gigablast\nDisallow: /\nUser-agent: GrapeshotCrawler\nDisallow: /\nUser-agent: heritrix\nDisallow: /\nUser-agent: ips-agent\nDisallow: /\nUser-agent: Lipperhey\nDisallow: /\nUser-agent: Ltx71\nDisallow: /\nUser-agent: Mail.RU_Bot\nDisallow: /\nUser-agent: MegaIndex\nDisallow: /\nUser-agent: Niki-bot\nDisallow: /\nUser-agent: Nutch\nDisallow: /\nUser-agent: OrangeBot\nDisallow: /\nUser-agent: Panscient\nDisallow: /\nUser-agent: PHPCrawl\nDisallow: /\nUser-agent: Rogerbot\nDisallow: /\nUser-agent: SMTBot\nDisallow: /\nUser-agent: Sogou\nDisallow: /\nUser-agent: Spbot\nDisallow: /\nUser-agent: TurnitinBot\nDisallow: /\nUser-agent: TwengaBot\nDisallow: /\nUser-agent: WBSearchBot\nDisallow: /\nUser-agent: WeSEE\nDisallow: /\nUser-agent: Wotbox\nDisallow: /\nUser-agent: *\nAllow: /\nDisallow: /wallapay\nDisallow: /130868815/\nDisallow: /faq\nDisallow: /rules\nDisallow: /safety\nDisallow: /toc$\nDisallow: /priv\nDisallow: /cookies\nDisallow: /register\nDisallow: /login\nDisallow: /auth/\nDisallow: /app/\nDisallow: /search\nDisallow: /*?\n"
    },
    "www.subito.it": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.subito.it",
      "body": "# It is expressively forbidden to use search robots or other automatic methods\n# to access Subito.it. Only if Subito.it has given such permission can be accepted.\n\nUser-agent: *\nAllow: /\nDisallow: /ai?preselect\nDisallow: /pr/\nDisallow: /ai/verify/\nDisallow: /vf/\nDisallow: /expaypalcheckout/\nDisallow: /paypalcheckout/\nDisallow: /ai/load/\nDisallow: /annunciutente/\nDisallow: /annunciutente-mobile/\nDisallow: /lim?\nDisallow: /li?\nDisallow: /vim/\nDisallow: /vi/\nDisallow: /hades/\n\n#Pagination\nDisallow: */?o=*\nDisallow: *&o=*\n\n#User profile\nDisallow: /utente/\n\nUser-agent: grapeshot\nDisallow:"
    },
    "www.marktplaats.nl": {
      "status": 200,
      "client": "fetch",
      "finalHost": "www.marktplaats.nl",
      "body": "Sitemap: https://www.marktplaats.nl/sitemap/sitemap.xml\n\nUser-agent: *\n\n#internal URLs\nDisallow: /experiences/*\nDisallow: /*/asq/\nDisallow: /4282/mpnl.*\nDisallow: /LR/*\nDisallow: /4282/web/mpnl.*\nDisallow: /messages*\nDisallow: /notifications*\nDisallow: /plaats/*\nDisallow: /i/adinfo/*\nDisallow: /admanager/*\n\n#SOI subpage\nDisallow: /u/*/*/q/*\nDisallow: /u/*/*/l/*\nDisallow: /u/*/*/f/*\n\n# login, confirm and forgot password pages\nDisallow: /account/wachtwoordvergeten.html\nDisallow: /account/wachtwoordinstellen.html\nDisallow: /account/password-reset/initiate.html\nDisallow: /account/password-reset/confirm.html\nDisallow: /account/confirm.html\nDisallow: /account/login.html?target=*\nDisallow: /account/createAccount.html\nDisallow: /account/loginSuccess.html\nDisallow: /account/login.html\nDisallow: /account/password-reset/secure-redirect-new-password.html\nDisallow: /accounts/authentication/logout\nDisallow: /account/oauth/login.html\nDisallow: /identity/v2/reset-password\nDisallow: /identity/v2/new-password\nDisallow: /identity/v2/confirm\nDisallow: /identity/v2/login?target=\nDisallow: /identity/v2/create-account\nDisallow: /identity/v2/logout\nDisallow: /identity/v2/two-factor-auth-setup\nDisallow: /identity/v2/delete-account\nDisallow: /identity/v2/delete-account/confirm\nDisallow: /identity/v2/change-email/initiate\nDisallow: /identity/v2/mid.js\nDisallow: /identity/mid.js\n\n# mymp pages\nDisallow: /mymp/\n\n# ASQ pages\nDisallow: /asq.html\nDisallow: /asq\n\n# SYI Pages\nDisallow: /syi/\n\n# Flagging/tipping ads\nDisallow: /flag/\n\n# bidding on ads\nDisallow: /bid/\n\n# external url redirects\nDisallow: /externalUrl/\n\n# google analytics\nDisallow: /EVIP/*\nDisallow: /VIP/*\nDisallow: /L1PAGE/*\nDisallow: /L1.5PAGE/*\nDisallow: /*ebayimg.com/*/s/*\nDisallow: /metrics/\nDisallow: /wijzigAdvertentie/\n\n#korean spam\nDisallow: /search?q=*\n\n# widgets\nDisallow: /widget/new-and-popular/click.html*\n\n# prevent unnecessary crawling\nDisallow: /lrp/api/audience-targeting\nDisallow: /lrp/api/banner-data\nDisallow: /lrp/api/complementary-listings*\nDisallow: /px/fb*\n\n# Block VIPs with parameters\nDisallow: /a/*/*/m*.html?c=*\nDisallow: /a/*/*/a*.html?c=*\nDisallow: /v/*/*/m*?c=*\nDisallow: /v/*/*/a*?c=*\nDisallow: /a/*/*/m*.html*correlationId=*\nDisallow: /a/*/*/a*.html*correlationId=*\nDisallow: /v/*/*/m*correlationId=*\nDisallow: /v/*/*/a*correlationId=*\n\n#block homepage feeds\nDisallow: /?type=*\n\n#block kijkinuwwijk parameters\nDisallow: /kijkinuwwijk.html?m_i=*\n\n#block undefined URLs\nDisallow: */undefined*\n\n#171224 addition\nDisallow: /v/api/similar-items*\nDisallow: /v/api/feed-items*\nDisallow: /v/api/seller-response-rate/*\nDisallow: /v/api/seller-profile/*\nDisallow: /lrp/api/lrp-url-redirect*\nDisallow: /lrp/api/search*\nDisallow: /lp/api/listings*\nDisallow: *login=*\n\n#Filter-release-test\nDisallow: */n/*\n\n# Chinese character spam\nDisallow: /*%E3%80%*\nDisallow: /*%E4%B*\nDisallow: /*%E5%*\nDisallow: /*%E6%*\nDisallow: /*%E7%*\nDisallow: /*%E8%*\nDisallow: /*%E9%*\n"
    }
  }
}