# ------------------------------------------------------
# Default Rules for All Crawlers
# ------------------------------------------------------

# SYSTEM: Wordpress ###
User-agent: *
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/themes/
Disallow: /trackback/
Disallow: /feed/
Disallow: /comments/
Disallow: /category/
Disallow: /tag/
Disallow: */trackback/
Disallow: */feed/
Disallow: */comments/
Disallow: /*?
Allow: /wp-content/uploads/


# --- Allow access to CSS, JS, and images for proper rendering ---
Allow: /*.js
Allow: /*.css
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.gif
Allow: /*.svg
Allow: /*.webp

# --- Crawl pacing ---
Crawl-delay: 10
Request-rate: 1/10s

# ------------------------------------------------------
# Explicitly Blocked Crawlers (SEO scrapers / spam bots)
# ------------------------------------------------------
User-agent: AhrefsBot
User-agent: Ai2Bot-Dolma
User-agent: aiHitBot
User-agent: Awario
User-agent: Baiduspider #cn search
User-agent: bedrockbot
User-agent: BLP_bbot
User-agent: Brightbot
User-agent: Bytespider
User-agent: CCBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Cotoyogi
User-agent: Crawlspace
User-agent: DataForSeoBot
User-agent: DotBot
User-agent: duggmirror
User-agent: EtaoSpider
User-agent: Factset_spyderbot
User-agent: FirecrawlAgent
User-agent: FriendlyCrawler
User-agent: ICC-Crawler
User-agent: ImagesiftBot
User-agent: img2dataset
User-agent: Kangaroo Bot
User-agent: ISSCyberRiskCrawler
User-agent: MJ12bot
User-agent: MyCentralAIScraperBot
User-agent: netEstate Imprint Crawler
User-agent: NovaAct
User-agent: olbicobot
User-agent: omgili
User-agent: omgilibot
User-agent: PanguBot
User-agent: Panscient
User-agent: panscient.com
User-agent: PetalBot #asian search
User-agent: Poseidon Research Crawler
User-agent: QuillBot
User-agent: quillbot.com
User-agent: SBIntuitionsBot
User-agent: Scrapy
User-agent: SearchmetricsBot
User-agent: Sidetrade indexer bot
User-agent: Thinkbot
User-agent: TMMBot
User-agent: VelenPublicWebCrawler
User-agent: WARDBot
User-agent: wpbot
User-agent: Webzio-Extended
User-agent: YaK
User-agent: YandexAdditional
User-agent: YandexAdditionalBot
Disallow: /

# --- Sitemap ---
Sitemap: https://www.proxess.de/sitemap_index.xml