diff options
| author |   Robin H. Johnson <robbat2@gentoo.org> | 2023-07-10 22:12:28 -0700 |
|---|---|---|
| committer | Robin H. Johnson <robbat2@gentoo.org> | 2023-07-10 22:14:24 -0700 |
| commit | d78c7bcba24958563afaf40096c57c24c80e3100 (patch) | |
| tree | 4da98715e52b152198e769b153368e09e99016cd | |
| parent | data/cached: add htaccess file (diff) | |
| download | bugzilla-d78c7bcba24958563afaf40096c57c24c80e3100.tar.gz bugzilla-d78c7bcba24958563afaf40096c57c24c80e3100.tar.bz2 bugzilla-d78c7bcba24958563afaf40096c57c24c80e3100.zip | |
data/cached: set headers for bots
Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>
| -rw-r--r-- | data/cached/.htaccess | 9 |
1 files changed, 9 insertions, 0 deletions
diff --git a/data/cached/.htaccess b/data/cached/.htaccess index 89aeaaab1..2c6fb2495 100644 --- a/data/cached/.htaccess +++ b/data/cached/.htaccess @@ -1,5 +1,12 @@ # Allow access to .html/.htmlgz/.json/.yaml/.csv files <FilesMatch \.(html|htmlgz|json|yaml|csv)?$> + # These pages are very large and change a lot, they should be followed, but + # not indexed themselves. Ideally we should generate better sitemaps as well, + # with only the bugs that are changing. + # https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag#apache_1 + Header set X-Robots-Tag "noindex,follow" + + # Set access <IfModule mod_version.c> <IfVersion < 2.4> Allow from all @@ -11,8 +18,10 @@ <IfModule !mod_version.c> Allow from all </IfModule> + </FilesMatch> +# This is for files that don't match. # And no directory listings, either. <IfModule mod_version.c> <IfVersion < 2.4> |