robots.txt used by git.lelux.fi https://wiki.lelux.fi/gitea/#robotstxt
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

72 lines
1.4KB

  1. User-agent: ia_archiver
  2. Disallow: /
  3. User-agent: *
  4. Disallow: /avatars
  5. Disallow: /user/*
  6. Disallow: /*/*/src/commit/*
  7. Disallow: /*/*/commit/*
  8. Disallow: /*/*/activity/*
  9. Disallow: /vendor/librejs.html
  10. Disallow: /api/swagger
  11. Disallow: /swagger.*.json
  12. # Language spam
  13. Disallow: /*?lang=
  14. # from Github, to be cleaned
  15. Allow: /*/*/tree/master
  16. Allow: /*/*/blob/master
  17. Disallow: /*/*/pulse
  18. Disallow: /*/*/tree/*
  19. Disallow: /*/*/blob/*
  20. Disallow: /*/*/wiki/*/*
  21. Disallow: /gist/*/*/*
  22. Disallow: /oembed
  23. Disallow: /*/forks
  24. Disallow: /*/stars
  25. Disallow: /*/download
  26. Disallow: /*/revisions
  27. Disallow: /*/*/issues/new
  28. Disallow: /*/*/issues/search
  29. Disallow: /*/*/commits/*/*
  30. Disallow: /*/*/commits/*?author
  31. Disallow: /*/*/commits/*?path
  32. Disallow: /*/*/branches
  33. Disallow: /*/*/tags
  34. Disallow: /*/*/contributors
  35. Disallow: /*/*/comments
  36. Disallow: /*/*/stargazers
  37. Disallow: /*/*/search
  38. Disallow: /*/tarball/
  39. Disallow: /*/zipball/
  40. Disallow: /*/*/archive/
  41. Disallow: /raw/*
  42. Disallow: /*/followers
  43. Disallow: /*/following
  44. Disallow: /stars/*
  45. Disallow: /*/blame/
  46. Disallow: /*/watchers
  47. Disallow: /*/network
  48. Disallow: /*/graphs
  49. Disallow: /*/raw/
  50. Disallow: /*/compare/
  51. Disallow: /*/cache/
  52. Disallow: /*/*/blame/
  53. Disallow: /*/*/watchers
  54. Disallow: /*/*/network
  55. Disallow: /*/*/graphs
  56. Disallow: /*/*/raw/
  57. Disallow: /*/*/compare/
  58. Disallow: /*/*/cache/
  59. Disallow: /.git/
  60. Disallow: /*/.git/
  61. Disallow: /*.git$
  62. Disallow: /*/sitemap.xml
  63. Disallow: /search/advanced
  64. Disallow: /search
  65. Disallow: /*q=
  66. Disallow: /*.atom
  67. Crawl-delay: 2