X-Robots-Tag "noindex, nofollow"

hyperknot · hyperknot · commit 6687311f9afa · 2025-02-27T16:52:31.000+01:00
diff --git a/.ruff.toml b/.ruff.toml
@@ -29,7 +29,6 @@ lint.ignore = [
   'E741',
   'F401', # unused imports
   'F841',
-  'PT004',
   'SIM102',
   'SIM105',
   'SIM108',
diff --git a/modules/http_host/http_host_lib/nginx.py b/modules/http_host/http_host_lib/nginx.py
@@ -206,6 +206,7 @@ def create_version_location(
 
         add_header 'Access-Control-Allow-Origin' '*' always;
         add_header Cache-Control public;
+        add_header X-Robots-Tag "noindex, nofollow" always;
 
         add_header x-ofm-debug 'specific JSON {area} {version}';
     }}
@@ -224,6 +225,7 @@ def create_version_location(
 
         add_header 'Access-Control-Allow-Origin' '*' always;
         add_header Cache-Control public;
+        add_header X-Robots-Tag "noindex, nofollow" always;
 
         add_header x-ofm-debug 'specific PBF {area} {version}';
     }}
@@ -268,6 +270,7 @@ def create_latest_locations(*, local: str, domain: str) -> str:
 
             add_header 'Access-Control-Allow-Origin' '*' always;
             add_header Cache-Control public;
+            add_header X-Robots-Tag "noindex, nofollow" always;
 
             add_header x-ofm-debug 'latest JSON {area}';
         }}
@@ -289,6 +292,7 @@ def create_latest_locations(*, local: str, domain: str) -> str:
 
             add_header 'Access-Control-Allow-Origin' '*' always;
             add_header Cache-Control public;
+            add_header X-Robots-Tag "noindex, nofollow" always;
 
             add_header x-ofm-debug 'wildcard JSON {area}';
         }}
@@ -309,6 +313,7 @@ def create_latest_locations(*, local: str, domain: str) -> str:
 
             add_header 'Access-Control-Allow-Origin' '*' always;
             add_header Cache-Control public;
+            add_header X-Robots-Tag "noindex, nofollow" always;
 
             add_header x-ofm-debug 'wildcard PBF {area}';
         }}
diff --git a/modules/http_host/http_host_lib/nginx_confs/le.conf b/modules/http_host/http_host_lib/nginx_confs/le.conf
@@ -22,12 +22,16 @@ server {
     ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305;
     ssl_prefer_server_ciphers off;
 
+
     # access log doesn't contain IP address
     access_log off;
     #access_log /data/ofm/http_host/logs_nginx/le-access.jsonl access_json buffer=128k;
 
     error_log /data/ofm/http_host/logs_nginx/le-error.log;
 
+    add_header X-Robots-Tag "noindex, nofollow" always;
+
+
     location ^~ /.well-known/acme-challenge/ {
         # trailing slash
         root /data/nginx/acme-challenges;
@@ -52,6 +56,7 @@ server {
 
         add_header 'Access-Control-Allow-Origin' '*' always;
         add_header Cache-Control public;
+        add_header X-Robots-Tag "noindex, nofollow" always;
     }
 
     # catch-all block to deny all other requests
diff --git a/modules/http_host/http_host_lib/nginx_confs/location_static.conf b/modules/http_host/http_host_lib/nginx_confs/location_static.conf
@@ -1,3 +1,9 @@
+# Serve robots.txt that blocks all crawlers
+#location = /robots.txt {
+#    add_header Content-Type text/plain;
+#return 200 "User-agent: *\nDisallow: /\n";
+#}
+
 location /fonts/ {
     # trailing slash
 
@@ -8,6 +14,8 @@ location /fonts/ {
 
     add_header 'Access-Control-Allow-Origin' '*' always;
     add_header Cache-Control public;
+    add_header X-Robots-Tag "noindex, nofollow" always;
+
 }
 
 location /natural_earth/ {
@@ -20,6 +28,8 @@ location /natural_earth/ {
 
     add_header 'Access-Control-Allow-Origin' '*' always;
     add_header Cache-Control public;
+    add_header X-Robots-Tag "noindex, nofollow" always;
+
 }
 
 location /sprites/ {
@@ -32,6 +42,8 @@ location /sprites/ {
 
     add_header 'Access-Control-Allow-Origin' '*' always;
     add_header Cache-Control public;
+    add_header X-Robots-Tag "noindex, nofollow" always;
+
 }
 
 
@@ -47,6 +59,7 @@ location @empty_tile {
 
     add_header 'Access-Control-Allow-Origin' '*' always;
     add_header Cache-Control public;
+    add_header X-Robots-Tag "noindex, nofollow" always;
 
     add_header x-ofm-debug 'empty tile';
 }
diff --git a/modules/http_host/http_host_lib/nginx_confs/roundrobin.conf b/modules/http_host/http_host_lib/nginx_confs/roundrobin.conf
@@ -28,6 +28,9 @@ server {
 
     error_log /data/ofm/http_host/logs_nginx/roundrobin-error.log;
 
+    add_header X-Robots-Tag "noindex, nofollow" always;
+
+
     __LOCATION_BLOCKS__
 
     location /styles/ {
@@ -46,6 +49,8 @@ server {
 
         add_header 'Access-Control-Allow-Origin' '*' always;
         add_header Cache-Control public;
+        add_header X-Robots-Tag "noindex, nofollow" always;
+
     }
 
     # catch-all block to deny all other requests
diff --git a/modules/tile_gen/scripts/shrink_btrfs.py b/modules/tile_gen/scripts/shrink_btrfs.py
@@ -69,7 +69,7 @@ def cli(btrfs_img: Path):
     mnt_dir.rmdir()
 
     subprocess.run(['truncate', '-s', str(total_size), btrfs_img])
-    print(f'Truncated {btrfs_img} to {total_size//1_000_000} MB size')
+    print(f'Truncated {btrfs_img} to {total_size // 1_000_000} MB size')
     print('shrink_btrfs.py DONE')
 
 
@@ -86,7 +86,7 @@ def get_usage(mnt: Path, key: str):
 
 def do_shrink(mnt: Path, delta_size: float):
     delta_size = int(delta_size)
-    print(f'Trying to shrink by {delta_size//1_000_000} MB')
+    print(f'Trying to shrink by {delta_size // 1_000_000} MB')
     p = subprocess.run(['btrfs', 'filesystem', 'resize', str(-delta_size), mnt])
     return p.returncode == 0
 
diff --git a/modules/tile_gen/tile_gen_lib/btrfs.py b/modules/tile_gen/tile_gen_lib/btrfs.py
@@ -88,7 +88,7 @@ def make_btrfs(run_folder: Path):
                 ['btrfs', 'filesystem', 'show', mount],
                 ['btrfs', 'filesystem', 'usage', mount],
             ]:
-                f.write(f"\n\n{' '.join(cmd)}\n")
+                f.write(f'\n\n{" ".join(cmd)}\n')
                 result = subprocess.run(['sudo'] + cmd, check=True, capture_output=True, text=True)
                 f.write(result.stdout)
 
diff --git a/website/astro.config.mjs b/website/astro.config.mjs
@@ -1,7 +1,7 @@
 // @ts-check
 import { defineConfig } from 'astro/config'
 
-import sitemap from '@astrojs/sitemap';
+import sitemap from '@astrojs/sitemap'
 
 // https://astro.build/config
 
diff --git a/website/package.json b/website/package.json
@@ -15,9 +15,6 @@
     "lightningcss": "^1.29.1"
   },
   "pnpm": {
-    "onlyBuiltDependencies": [
-      "esbuild",
-      "sharp"
-    ]
+    "onlyBuiltDependencies": ["esbuild", "sharp"]
   }
 }
diff --git a/website/src/layouts/Layout.astro b/website/src/layouts/Layout.astro
@@ -1,7 +1,7 @@
 ---
 const { frontmatter } = Astro.props || {}
 const { title } = frontmatter || Astro.props
-const canonicalURL = new URL(Astro.url.pathname, Astro.site);
+const canonicalURL = new URL(Astro.url.pathname, Astro.site)
 
 import '../styles/_style.css'
 ---

Original file line number	Diff line number	Diff line change
`@@ -15,9 +15,6 @@`
`15`	`15`	`"lightningcss": "^1.29.1"`
`16`	`16`	`},`
`17`	`17`	`"pnpm": {`
`18`		`- "onlyBuiltDependencies": [`
`19`		`- "esbuild",`
`20`		`- "sharp"`
`21`		`- ]`
	`18`	`+ "onlyBuiltDependencies": ["esbuild", "sharp"]`
`22`	`19`	`}`
`23`	`20`	`}`