infra(traefik,gitea): rate-limit filtered project-board views per source IP and serve a robots.txt disallowing filtered views — the label-permutation crawler trap (refs #656)

infra/gitea/custom/public/robots.txt

@@ -0,0 +1,22 @@
+# Served by Gitea at /robots.txt (GITEA_CUSTOM/public). #656: a crawler
+# permuted every label-filter combination of the project board about once
+# a second for weeks — filtered views are a combinatorial trap and carry
+# nothing worth indexing. The traefik router `git-projects-filter` rate
+# limits the same URLs for clients that ignore this file.
+User-agent: *
+Disallow: /*/*/projects
+Disallow: /*/*/issues?
+Disallow: /*/*/pulls?
+Disallow: /*/*/milestones?
+Disallow: /*?labels=
+Disallow: /*?state=
+Disallow: /*?sort=
+Disallow: /*?q=
+Disallow: /*/*/compare/
+Disallow: /*/*/commits/
+Disallow: /*/*/blame/
+Disallow: /*/*/raw/
+Disallow: /*/*/search
+Disallow: /explore
+Disallow: /api/
+Crawl-delay: 10

infra/traefik/dynamic/services.yml

@@ -27,6 +27,15 @@ http:
         frameDeny: true
         browserXssFilter: true
         contentTypeNosniff: true
+    # #656: a crawler permuting project-board label filters at ~1 req/s
+    # (216.73.216.0/24). Per-source-IP limit on the filtered project views
+    # only — a person clicking filters stays well under 10/min; the crawler
+    # gets 429s. Pairs with infra/gitea/custom/public/robots.txt.
+    projects-ratelimit:
+      rateLimit:
+        average: 10
+        period: 1m
+        burst: 20
     # SSO: auth-handler (nginx) wraps oauth2-proxy to convert 401 → 302.
     # Same logic as corwins.media: auth_request + error_page 401 = @signin.
     git-sso:
@@ -64,6 +73,27 @@ http:
       entryPoints:
         - websecure
       tls: {}
+    # #656: filtered project-board views on git — same service, higher
+    # priority than the generic `git` router, rate limited per source IP.
+    git-projects-filter:
+      rule: "Host(`git.{{ $domain }}`) && PathRegexp(`^/[^/]+/[^/]+/projects(/|$)`) && QueryRegexp(`labels`, `.+`)"
+      priority: 1000
+      service: git
+      entryPoints:
+        - web
+      middlewares:
+        - projects-ratelimit
+        - secure-headers
+    git-projects-filter-tls:
+      rule: "Host(`git.{{ $domain }}`) && PathRegexp(`^/[^/]+/[^/]+/projects(/|$)`) && QueryRegexp(`labels`, `.+`)"
+      priority: 1000
+      service: git
+      entryPoints:
+        - websecure
+      tls: {}
+      middlewares:
+        - projects-ratelimit
+        - secure-headers
     {{- range $name, $svc := $reef }}
     {{ $name }}:
       rule: "Host(`{{ get $svc "subdomain" | default $name }}.{{ $domain }}`){{ range get $svc "extra_hosts" | default list }} || Host(`{{ . }}`){{ end }}"