User-agent: *
Disallow: /ajax/
Disallow: /myaccount/
Disallow: /adprint/
Disallow: /ad/leaflet/
Disallow: /ad/contact/
Disallow: /payment/
Disallow: /apple-app-site-association
Disallow: /profile/
Disallow: /ar/profile/
Disallow: *ref%*
Disallow: *search%*
Disallow: *search[*
Disallow: *sorting=*
Disallow: *minPhotoCount=*
Disallow: *product=*
Disallow: */changelang/*
Disallow: /account
Disallow: /ar/account
Allow: /

User-agent: *
Disallow: /api/
Disallow: /post/
Disallow: /edit/
Disallow: /chat/
Disallow: /profile/
Disallow: /payments/

# disallow all urls with parameters
Disallow: *filter=*

# disallow crawl on urls with model in path and free search
Disallow: */model-*/q-*

# path on search urls used to retrieve filters as json
Disallow: */raw-filters*

# --- Crawler policy: blocked bots ---
# Block Baidu (Baiduspider)
User-agent: Baiduspider
Disallow: /

# Block ByteDance (Bytespider / TikTok crawler)
User-agent: Bytespider
Disallow: /

# Optional: alternate tokens some ByteDance crawlers use
User-agent: Bytedance
Disallow: /

# --- AI bots (dashboard top contributors): throttle via Crawl-delay ---
# Note: many bots ignore Crawl-delay; enforce rate limits at CDN/WAF too.

User-agent: ClaudeBot
Crawl-delay: 10

User-agent: Meta-ExternalAgent
Crawl-delay: 10

User-agent: GPTBot
Crawl-delay: 10

User-agent: OAI-SearchBot
Crawl-delay: 10

User-agent: Amazonbot
Crawl-delay: 10