diff --git a/infra/gitea/custom/public/robots.txt b/infra/gitea/custom/public/robots.txt new file mode 100644 index 0000000..8c32aae --- /dev/null +++ b/infra/gitea/custom/public/robots.txt @@ -0,0 +1,22 @@ +# Served by Gitea at /robots.txt (GITEA_CUSTOM/public). #656: a crawler +# permuted every label-filter combination of the project board about once +# a second for weeks — filtered views are a combinatorial trap and carry +# nothing worth indexing. The traefik router `git-projects-filter` rate +# limits the same URLs for clients that ignore this file. +User-agent: * +Disallow: /*/*/projects +Disallow: /*/*/issues? +Disallow: /*/*/pulls? +Disallow: /*/*/milestones? +Disallow: /*?labels= +Disallow: /*?state= +Disallow: /*?sort= +Disallow: /*?q= +Disallow: /*/*/compare/ +Disallow: /*/*/commits/ +Disallow: /*/*/blame/ +Disallow: /*/*/raw/ +Disallow: /*/*/search +Disallow: /explore +Disallow: /api/ +Crawl-delay: 10 diff --git a/infra/traefik/dynamic/services.yml b/infra/traefik/dynamic/services.yml index 7c61b7f..c2ad5cb 100644 --- a/infra/traefik/dynamic/services.yml +++ b/infra/traefik/dynamic/services.yml @@ -27,6 +27,15 @@ http: frameDeny: true browserXssFilter: true contentTypeNosniff: true + # #656: a crawler permuting project-board label filters at ~1 req/s + # (216.73.216.0/24). Per-source-IP limit on the filtered project views + # only — a person clicking filters stays well under 10/min; the crawler + # gets 429s. Pairs with infra/gitea/custom/public/robots.txt. + projects-ratelimit: + rateLimit: + average: 10 + period: 1m + burst: 20 # SSO: auth-handler (nginx) wraps oauth2-proxy to convert 401 → 302. # Same logic as corwins.media: auth_request + error_page 401 = @signin. git-sso: @@ -64,6 +73,27 @@ http: entryPoints: - websecure tls: {} + # #656: filtered project-board views on git — same service, higher + # priority than the generic `git` router, rate limited per source IP. + git-projects-filter: + rule: "Host(`git.{{ $domain }}`) && PathRegexp(`^/[^/]+/[^/]+/projects(/|$)`) && QueryRegexp(`labels`, `.+`)" + priority: 1000 + service: git + entryPoints: + - web + middlewares: + - projects-ratelimit + - secure-headers + git-projects-filter-tls: + rule: "Host(`git.{{ $domain }}`) && PathRegexp(`^/[^/]+/[^/]+/projects(/|$)`) && QueryRegexp(`labels`, `.+`)" + priority: 1000 + service: git + entryPoints: + - websecure + tls: {} + middlewares: + - projects-ratelimit + - secure-headers {{- range $name, $svc := $reef }} {{ $name }}: rule: "Host(`{{ get $svc "subdomain" | default $name }}.{{ $domain }}`){{ range get $svc "extra_hosts" | default list }} || Host(`{{ . }}`){{ end }}"