# public/robots.txt
User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /node_modules/
Disallow: /*.json$ 
# Sitemaps
Sitemap: https://yourdomain.com/api/sitemap.xml.php
Sitemap: https://yourdomain.com/api/image-sitemap.xml.php

# Crawl-delay for bots
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /