adding robots.txt file

2025-08-19 12:38:53 +05:30
parent 119714eff3
commit b7af3d0566
2 changed files with 7 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -48,6 +48,7 @@ public/
 !public/
 !public/sitemap.xml
 !public/wdi-logo.ico
+!public/robots.txt

 # Storybook
 storybook-static
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -0,0 +1,6 @@
+User-agent: *
+Disallow: /admin/
+Disallow: /cgi-bin
+Allow: /
+ 
+Sitemap: https://www.wdipl.com/sitemap.xml