From 8bd144ccb154aa443d9f2832fda5497338734aaa Mon Sep 17 00:00:00 2001 From: Tristan TR <69242752+tristantr@users.noreply.github.com> Date: Thu, 10 Sep 2026 20:42:01 +0200 Subject: [PATCH] keep crawlers off the login page (#11057) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(frontend): keep crawlers off the login page Every page on the hub links to /user/login with itself in `rd`, so a crawler sees one login URL per hub page — 4,311 of them in Search Console, all rendering this same form and flagged as duplicates without a canonical. Nothing about a login page belongs in an index, on any instance. Mark the page noindex, as public_run already is, and ship a robots.txt that keeps crawlers out of /user/ and /api/. The frontend is embedded as static assets with an index.html fallback, which is why /robots.txt answered with the app shell until now; a real file in static/ is served as itself. Co-Authored-By: Claude Opus 5 * fix(frontend): let crawlers fetch the login page so the noindex is seen robots.txt disallowed /user/, which stopped a crawler fetching /user/login at all — and a page that is never fetched never shows its noindex. The two halves cancelled: the URLs would have moved from "duplicate" to "blocked" rather than out of the index. Drop the disallow, keeping /api/. And since the app is client-rendered, the meta tag only exists after a render pass; send X-Robots-Tag on /user/* from serve_path as well, which a crawler sees on the first fetch. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- backend/windmill-api/src/static_assets.rs | 8 ++++++++ .../routes/(root)/(logged)/user/(user)/login/+page.svelte | 6 ++++++ frontend/static/robots.txt | 2 ++ 3 files changed, 16 insertions(+) create mode 100644 frontend/static/robots.txt diff --git a/backend/windmill-api/src/static_assets.rs b/backend/windmill-api/src/static_assets.rs index 06a79d022e..22645a2ac4 100644 --- a/backend/windmill-api/src/static_assets.rs +++ b/backend/windmill-api/src/static_assets.rs @@ -107,6 +107,14 @@ fn serve_path(path: &str, original_path: &str, query: Option<&str>) -> Response< .header("Cross-Origin-Resource-Policy", "cross-origin"); } + // Login and its siblings carry a different `rd` on every page that links + // to them, so a crawler meets thousands of URLs for one form. The app is + // client-rendered, so a meta tag only exists after a render pass; the + // header is seen on the first fetch. + if original_path.starts_with("/user/") { + res = res.header("X-Robots-Tag", "noindex, nofollow"); + } + // Add Content-Security-Policy header for static assets when policy is set if !CSP_POLICY.is_empty() { if let Ok(header_value) = HeaderValue::try_from(CSP_POLICY.as_str()) { diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte index df85c1f1c1..e542201260 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte @@ -141,6 +141,12 @@ } + + + + +
diff --git a/frontend/static/robots.txt b/frontend/static/robots.txt new file mode 100644 index 0000000000..d34624918c --- /dev/null +++ b/frontend/static/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: /api/