# Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com) # They don't respect robots.txt. Blocking via mod_rewrite User-agent: Bytespider Disallow: / # Facebook is to perish User-agent: meta-externalagent Disallow: / User-agent: meta-webindexer Disallow: / # For now I need more clicks from searches # User-agent: bingbot # Disallow: / # User-agent: Amazonbot # Amazon's user agent # Disallow: / # AI # User-agent: ClaudeBot # Disallow: / # Paid data bots User-agent: SemrushBot Disallow: / # https://moz.com/help/moz-procedures/crawlers/dotbot User-agent: DotBot # Need moz pro account to access the data Disallow: / User-agent: AhrefsBot Disallow: / User-agent: AwarioBot Disallow: / User-agent: barkrowler Disallow: / # Trying to retrieve truncated URLs User-agent: GPTBot Disallow: / # Huawei operated User-agent: PetalBot Disallow: / # Googlebot, yandexbot etc User-agent: * # But still it can navigate the pages from the main catalogue Disallow: /search Disallow: /events/cal_* Disallow: /tracking Disallow: /?q=* Disallow: /events?date_range* Disallow: /*/reviews/add/ # Can paginate Disallow: /catalogue/category Disallow: /catalogue/?selected_facets=* Sitemap: https://kupec.info/sitemap_p.xml Sitemap: https://kupec.info/sitemap_e.xml Sitemap: https://kupec.info/sitemap_e_p1.xml Sitemap: https://kupec.info/sitemap_e_p2.xml Sitemap: https://kupec.info/sitemap_e_p3.xml Sitemap: https://kupec.info/sitemap_e_p4.xml