User-agent: *
Disallow: /*/*/images/
Disallow: /*/*/dataset/*/images/
Disallow: /*/*/dataset/*/download
Disallow: /*/*/dataset/*/download/*
Disallow: /*/*/model/*/documentation
Disallow: /*/*/health
Disallow: /*/*/analytics
Disallow: /*/*/delete
Disallow: /*/*/export
Disallow: /*/*/edit

Allow: /

User-agent: GPTBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: TikTokSpider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Ai2Bot-Dolma
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: meta-externalagent
Disallow: /

Sitemap: https://universe.roboflow.com/sitemap.xml