# ZOZOZON — AI + SEO optimized robots.txt
# Platform: Shopify
# Goal: Keep Google/Bing SEO clean, while explicitly allowing major AI crawlers
# Notes:
# - Private/transactional areas remain blocked for everyone.
# - Public content (/, /products/, /collections/, /blogs/) is allowed.
# - Single canonical sitemap.

# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
# ┃  AI & assistant crawlers — explicit allow            ┃
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
# (Общий блок с едиными правилами: разрешаем публичный контент, блокируем приватные зоны)
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Google-Extended
User-agent: PerplexityBot
User-agent: CCBot
User-agent: YouBot
User-agent: Applebot-Extended
User-agent: Applebot
User-agent: Meta-ExternalAgent
User-agent: facebookexternalhit
User-agent: Facebot
User-agent: Diffbot
User-agent: Amazonbot
Allow: /
Disallow: /cart
Disallow: /checkout
Disallow: /checkouts/
Disallow: /carts
Disallow: /orders
Disallow: /account
Disallow: /search
Disallow: /recommendations/products

# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
# ┃  Baseline rules for everyone else (SEO-clean)        ┃
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
User-agent: *
# Shopify private / transactional / noisy
Disallow: /a/downloads/-/*
Disallow: /admin
Disallow: /cart
Disallow: /orders
Disallow: /checkouts/
Disallow: /checkout
Disallow: /79981576515/checkouts
Disallow: /79981576515/orders
Disallow: /carts
Disallow: /account
Disallow: /search
Disallow: /sf_private_access_tokens
Disallow: /apple-app-site-association
Disallow: /.well-known/shopify/monorail
Disallow: /cdn/wpm/*.js
Disallow: /recommendations/products
Disallow: /*/recommendations/products

# Facets / params that create duplicates
Disallow: /collections/*sort_by*
Disallow: /*/collections/*sort_by*
Disallow: /collections/*+*
Disallow: /collections/*%2B*
Disallow: /collections/*%2b*
Disallow: /*/collections/*+*
Disallow: /*/collections/*%2B*
Disallow: /*/collections/*%2b*
Disallow: */collections/*filter*&*filter*
Disallow: /blogs/*+*
Disallow: /blogs/*%2B*
Disallow: /blogs/*%2b*
Disallow: /*/blogs/*+*
Disallow: /*/blogs/*%2B*
Disallow: /*/blogs/*%2b*
Disallow: /*?*oseid=*
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /policies/
Disallow: /*/policies/
Disallow: /*/*?*ls=*&ls=*
Disallow: /*/*?*ls%3D*%3Fls%3D*
Disallow: /*/*?*ls%3d*%3fls%3d*

# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
# ┃  Ленты подборок .atom — служебные, для покупателя    ┃
# ┃  бесполезны: ссылок на них на витрине нет. Сервер    ┃
# ┃  отвечает на них 500 и 429, и Google тратит на это   ┃
# ┃  бюджет обхода. Замер 17.09.2026: 121 такой адрес    ┃
# ┃  в отчёте об ошибках сервера Search Console, живая   ┃
# ┃  проверка в тот же день дала 500 и 429.              ┃
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
Disallow: /collections/*.atom
Disallow: /*/collections/*.atom
Disallow: /*.atom

# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
# ┃  Языковые витрины американского рынка, кроме         ┃
# ┃  английской. Страницы живые и нужны человеку:        ┃
# ┃  эстонец или русский, живущий в США, переключает     ┃
# ┃  язык и получает родной текст с ценой в долларах.    ┃
# ┃  Но Google их всё равно НЕ берёт в индекс — они уже  ┃
# ┃  лежат в 580 тысячах «просканирована, не взята», —   ┃
# ┃  а обход тратят. Замер 17.09.2026: 427 из 1000       ┃
# ┃  непроиндексированных адресов именно такие.          ┃
# ┃  🔴 /en-us/ НЕ закрывать: это посадочная страница    ┃
# ┃  американского фида, 271 запись фида ведёт на неё.   ┃
# ┃  🔴 /ru/ и /et/ НЕ закрывать: домашние языки.        ┃
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
Disallow: /de-us/
Disallow: /fr-us/
Disallow: /nl-us/
Disallow: /no-us/
Disallow: /ru-us/
Disallow: /et-us/

# Crawl etiquette for heavy third-party SEO tools
User-agent: AhrefsBot
Crawl-delay: 10
Disallow: /a/downloads/-/*
Disallow: /admin
Disallow: /cart
Disallow: /orders
Disallow: /checkouts/
Disallow: /checkout
Disallow: /79981576515/checkouts
Disallow: /79981576515/orders
Disallow: /carts
Disallow: /account
Disallow: /collections/*sort_by*
Disallow: /*/collections/*sort_by*
Disallow: */collections/*filter*&*filter*
Disallow: /blogs/*+*
Disallow: /*?*oseid=*
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /policies/
Disallow: /*/policies/
Disallow: /*/*?*ls=*&ls=*
Disallow: /*/*?*ls%3D*%3Fls%3D*
Disallow: /*/*?*ls%3d*%3fls%3d*
Disallow: /search
Disallow: /sf_private_access_tokens
Disallow: /apple-app-site-association
Disallow: /.well-known/shopify/monorail
Disallow: /cdn/wpm/*.js
Disallow: /recommendations/products
Disallow: /*/recommendations/products

User-agent: AhrefsSiteAudit
Crawl-delay: 10
Disallow: /a/downloads/-/*
Disallow: /admin
Disallow: /cart
Disallow: /orders
Disallow: /checkouts/
Disallow: /checkout
Disallow: /79981576515/checkouts
Disallow: /79981576515/orders
Disallow: /carts
Disallow: /account
Disallow: /collections/*sort_by*
Disallow: /*/collections/*sort_by*
Disallow: */collections/*filter*&*filter*
Disallow: /blogs/*+*
Disallow: /*?*oseid=*
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /policies/
Disallow: /*/policies/
Disallow: /*/*?*ls=*&ls=*
Disallow: /*/*?*ls%3D*%3Fls%3D*
Disallow: /*/*?*ls%3d*%3fls%3d*
Disallow: /search
Disallow: /sf_private_access_tokens
Disallow: /apple-app-site-association
Disallow: /.well-known/shopify/monorail
Disallow: /cdn/wpm/*.js
Disallow: /recommendations/products
Disallow: /*/recommendations/products

# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
# ┃  AdsBot-Google — проверяет посадочные страницы Ads   ┃
# ┃  Своя группа обязательна: этот обходчик НЕ читает    ┃
# ┃  правило `User-agent: *`. Без неё он ходит в корзину.┃
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
User-agent: adsbot-google
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts
Disallow: /cart
Disallow: /orders
Disallow: /account
Disallow: /79981576515/checkouts
Disallow: /79981576515/orders
Disallow: /*?*oseid=*
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /cdn/wpm/*.js
Disallow: /services/login_with_shop

# Block noisy scrapers
User-agent: Nutch
Disallow: /

# Sitemap (single, canonical)
Sitemap: https://zozozon.com/sitemap.xml