From f183bd43fbfcc288a8bd8ce097296777b96ca2fc Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Fri, 25 Sep 2026 11:13:06 -0400 Subject: [PATCH] chore: retire the standalone lsp and multiplayer images from examples, drop lsp/Dockerfile (#11341) * chore: retire the standalone lsp and multiplayer images from examples, drop lsp/Dockerfile Co-Authored-By: Claude Opus 5.5 (1M context) * chore: drop the unbuilt DockerfileMultiplayer, document running the LSP from windmill-extra Co-Authored-By: Claude Opus 5.5 (1M context) * fix(examples): ecs terraform destroys cleanly and gives private instances no public ip Co-Authored-By: Claude Opus 5.5 (1M context) * fix(examples): give windmill-extra on ecs a WINDMILL_BASE_URL for multiplayer auth, address review Co-Authored-By: Claude Opus 5.5 (1M context) * fix(examples): make the ecs example upgrade cleanly from the standalone lsp/multiplayer stack Co-Authored-By: Claude Opus 5.5 (1M context) * fix(examples): name the extra target group by prefix so create_before_destroy can replace it Co-Authored-By: Claude Opus 5.5 (1M context) * docs(examples): note the brief editor-socket gap when upgrading the ecs example Co-Authored-By: Claude Opus 5.5 (1M context) * docs(examples): the debugger stays off after the ecs upgrade unless enabled Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- docker/DockerfileMultiplayer | 9 -- docs/docker-security.md | 5 +- examples/deploy/aws-ecs-terraform/README.md | 18 ++- .../deploy/aws-ecs-terraform/ecs_cluster.tf | 17 ++- .../deploy/aws-ecs-terraform/load_balancer.tf | 50 ++++--- examples/deploy/aws-ecs-terraform/rds.tf | 9 +- examples/deploy/aws-ecs-terraform/vpc.tf | 6 + .../aws-ecs-terraform/windmill_extra.tf | 126 ++++++++++++++++++ .../deploy/aws-ecs-terraform/windmill_lsp.tf | 85 ------------ .../aws-ecs-terraform/windmill_multiplayer.tf | 80 ----------- .../windmill_worker_high_performance.tf | 17 ++- .../aws-eks-cloudformation/quicklaunch.yaml | 2 +- .../otel-tracing-grafana/docker-compose.yml | 33 +++-- .../otel-tracing-jaeger/docker-compose.yml | 33 +++-- frontend/README_DEV.md | 18 +++ lsp/Dockerfile | 62 --------- lsp/dev.nu | 21 --- 17 files changed, 262 insertions(+), 329 deletions(-) delete mode 100644 docker/DockerfileMultiplayer create mode 100644 examples/deploy/aws-ecs-terraform/windmill_extra.tf delete mode 100644 examples/deploy/aws-ecs-terraform/windmill_lsp.tf delete mode 100644 examples/deploy/aws-ecs-terraform/windmill_multiplayer.tf delete mode 100644 lsp/Dockerfile delete mode 100755 lsp/dev.nu diff --git a/docker/DockerfileMultiplayer b/docker/DockerfileMultiplayer deleted file mode 100644 index 192ac06ff5..0000000000 --- a/docker/DockerfileMultiplayer +++ /dev/null @@ -1,9 +0,0 @@ -FROM node:slim - -EXPOSE 3002 -ENV PORT 3002 -ENV HOST 0.0.0.0 - -RUN npm install -g y-websocket - -CMD ["npx", "y-websocket"] \ No newline at end of file diff --git a/docs/docker-security.md b/docs/docker-security.md index 5b5196f8b4..44ae6cbeff 100644 --- a/docs/docker-security.md +++ b/docs/docker-security.md @@ -22,9 +22,8 @@ so patching here is sufficient: - `DockerfileExtra` builds `FROM windmill-ee-slim`. The nsjail *builder* stages are throwaway (only the compiled `nsjail` binary is -copied out), so they are intentionally not upgraded. `DockerfileMultiplayer` -(`node:slim`), the CLI, Caddy-L4, CUDA-only, and RHEL/dnf images are out of scope -for this apt-based patching. +copied out), so they are intentionally not upgraded. The CLI, Caddy-L4, +CUDA-only, and RHEL/dnf images are out of scope for this apt-based patching. ## Why `apt-get upgrade` and not `unattended-upgrades` / pinning diff --git a/examples/deploy/aws-ecs-terraform/README.md b/examples/deploy/aws-ecs-terraform/README.md index 098943e353..f9a0318d7a 100644 --- a/examples/deploy/aws-ecs-terraform/README.md +++ b/examples/deploy/aws-ecs-terraform/README.md @@ -32,14 +32,13 @@ Windmill relies on a PostgreSQL database which will be created by Terraform. The The following stack will be deployed by this terraform as it is provided: - 2 Windmill servers (each requiring 1 CPU and 1.5GiB of Memory) - 2 multi-purpose Windmill workers (each requiring 2 CPU and 3GiB of Memory) -- Windmill LSP (requiring 1 CPU and 1.5GiB of Memory) -- Windmill Multiplayer (requiring 1 CPU and 1.5GiB of Memory) +- Windmill Extra: LSP and Multiplayer, plus the optional Debugger (requiring 1 CPU and 1.5GiB of Memory) - 1 native Windmill worker (requiring 2 CPU and 3GiB of Memory) - 1 high performance Windmill worker (requiring 4 CPU and 15GiB of Memory) -The ECS cluster will be composed of 1 autoscaling group composed of up-to 6 `t3.medium` instances (5 necessary for the entire load, and 1 more for rolling upgrades). This will host all the services except the high performance workers. Those will be deployed on a separate auto-scaling group composed of up-to 2 `t3.xlarge` instances. +The ECS cluster will be composed of 1 autoscaling group of `t3.medium` instances, sized by the ECS capacity provider up to `max_size` (10) in [ecs_cluster.tf](./ecs_cluster.tf). In our test deployment it settled on 5 instances for this load; the headroom covers rolling upgrades. This will host all the services except the high performance workers. Those will be deployed on a separate auto-scaling group composed of up-to 2 `t3.xlarge` instances. -Of course the above is provided as an example, it should be tuned for you own needs, both in terms of instance specs, but also in terms of service architecture. For example, you might not need high performance workers, in which case you won't need the second autoscaling group at all. Same for the native worker, multiplayer, or even LSP (though LSP is highly recommended for a better coding experience). +Of course the above is provided as an example, it should be tuned for your own needs, both in terms of instance specs, but also in terms of service architecture. For example, you might not need high performance workers, in which case you won't need the second autoscaling group at all. Same for the native worker, or even Windmill Extra (though its LSP is highly recommended for a better coding experience). Its services are switched on and off with the `ENABLE_LSP`, `ENABLE_MULTIPLAYER` (Enterprise Edition) and `ENABLE_DEBUGGER` variables in [windmill_extra.tf](./windmill_extra.tf). The only strictly required components are: - At least 1 Windmill server @@ -49,14 +48,13 @@ This terraform has been assembled to easily remove components. Each component me - _REQUIRED_ Windmill server -> [windmill_server.tf](./windmill_server.tf) - _REQUIRED_ Multi-purpose windmill worker -> [windmill_worker_basic.tf](./windmill_worker_basic.tf) -- _OPTIONAL_ Windmill LSP -> [windmill_lsp.tf](./windmill_lsp.tf) -- _OPTIONAL_ Windmill Multiplayer -> [windmill_multiplayer.tf](./windmill_multiplayer.tf) +- _OPTIONAL_ Windmill Extra (LSP, Multiplayer, Debugger) -> [windmill_extra.tf](./windmill_extra.tf) - _OPTIONAL_ Windmill Native worker -> [windmill_worker_native.tf](./windmill_worker_native.tf) - _OPTIONAL_ Windmill High Performace worker -> [windmill_worker_high_performance.tf](./windmill_worker_high_performance.tf) - this one is a lot longer because it deploys a new auto scaling group ### Network -The network deployed here is a single VPC, composed on 4 subnets spread across 2 availability zones (1 public and 1 private subnet per zone). All the services are behind a load balancer routing the request to Windmill server, LSP or multiplayer. +The network deployed here is a single VPC, composed on 4 subnets spread across 2 availability zones (1 public and 1 private subnet per zone). All the services are behind a load balancer routing the request to Windmill server or Windmill Extra (`/ws/*`, `/ws_mp/*`, `/ws_debug/*`). A single security group is used, allowing HTTP traffic on port 80 (it is not deploying custom certificates and therefore does not use HTTPS). @@ -66,6 +64,12 @@ All this is provided as an example. It can be refined depending on your needs. A Windmill heavily relies on PostgreSQL database. This terraform deploys a standalone RDS instance having 100GiB of storage, which can be scaled up to 1000GiB. For production use cases, we recommend deploying a Multi-AZ instance, or even a Multi-AZ DB cluster. The RDS definition is in [rds.tf](./rds.tf) +### Upgrading a stack deployed before Windmill Extra + +Earlier versions of this example ran the LSP and Multiplayer as two separate services (`windmill_lsp.tf` and `windmill_multiplayer.tf`). A plain `terraform apply` of this version migrates such a stack: the `moved` blocks in [load_balancer.tf](./load_balancer.tf) keep the listener rule in place, and the old services and target groups are removed. + +While the new `windmill-extra` task starts (about 2.5 minutes in our test), the `/ws/*` and `/ws_mp/*` routes answer 503, so code intelligence and multiplayer are unavailable in the editor until it is healthy. The debugger (`/ws_debug/*`) stays off in this example unless you set `ENABLE_DEBUGGER=true` in [windmill_extra.tf](./windmill_extra.tf). The Windmill servers and workers are not affected. + ### Ready? REMINDER: don't forget to edit [terraform.tfvars](./terraform.tfvars) before deploying the stack. diff --git a/examples/deploy/aws-ecs-terraform/ecs_cluster.tf b/examples/deploy/aws-ecs-terraform/ecs_cluster.tf index ee5f74d4c4..ddbaad35cf 100644 --- a/examples/deploy/aws-ecs-terraform/ecs_cluster.tf +++ b/examples/deploy/aws-ecs-terraform/ecs_cluster.tf @@ -17,9 +17,10 @@ resource "aws_launch_template" "windmill_cluster_lt" { } network_interfaces { - device_index = 0 - delete_on_termination = true - associate_public_ip_address = true + device_index = 0 + delete_on_termination = true + # Private subnets: outbound traffic goes through the NAT gateways, so no public IP. + associate_public_ip_address = false security_groups = [ aws_security_group.windmill_cluster_sg.id ] @@ -57,6 +58,16 @@ resource "aws_autoscaling_group" "windmill_cluster_asg" { value = true propagate_at_launch = true } + + # The ECS agent on these instances reaches AWS through private route -> NAT gateway -> public + # route -> internet gateway. Keeping that whole path until the instances are gone lets + # `terraform destroy` drain and delete the ECS services, which otherwise hang in DRAINING. + depends_on = [ + aws_route_table_association.windmill_cluster_subnet_private1__rtb_private1, + aws_route_table_association.windmill_cluster_subnet_private2__rtb_private2, + aws_route_table_association.windmill_cluster_subnet_public1__rtb_public, + aws_route_table_association.windmill_cluster_subnet_public2__rtb_public, + ] } resource "aws_ecs_cluster" "windmill_cluster" { diff --git a/examples/deploy/aws-ecs-terraform/load_balancer.tf b/examples/deploy/aws-ecs-terraform/load_balancer.tf index 376374efb6..f73704f04e 100644 --- a/examples/deploy/aws-ecs-terraform/load_balancer.tf +++ b/examples/deploy/aws-ecs-terraform/load_balancer.tf @@ -6,20 +6,20 @@ resource "aws_lb_target_group" "windmill_cluster_windmill_server_tg" { vpc_id = aws_vpc.windmill_cluster_vpc.id } -resource "aws_lb_target_group" "windmill_cluster_windmill_lsp_tg" { - name = "windmill-cluster-lsp-tg" - port = 3001 +resource "aws_lb_target_group" "windmill_cluster_windmill_extra_tg" { + # A prefix, not a fixed name: with create_before_destroy the replacement exists alongside the old + # group for a moment, and target group names must be unique. + name_prefix = "wmext-" + port = 3000 protocol = "HTTP" target_type = "ip" vpc_id = aws_vpc.windmill_cluster_vpc.id -} -resource "aws_lb_target_group" "windmill_cluster_windmill_multiplayer_tg" { - name = "windmill-cluster-multiplayer-tg" - port = 3002 - protocol = "HTTP" - target_type = "ip" - vpc_id = aws_vpc.windmill_cluster_vpc.id + # Replacing a target group that a listener rule uses: create the new one and repoint the rule + # before deleting the old one, which the ALB refuses while the rule still references it. + lifecycle { + create_before_destroy = true + } } resource "aws_lb" "windmill_cluster_alb" { @@ -48,34 +48,32 @@ resource "aws_lb_listener" "windmill_cluster_alb_listener" { } } -resource "aws_lb_listener_rule" "windmill_cluster_alb_lsp_rule" { +resource "aws_lb_listener_rule" "windmill_cluster_alb_extra_rule" { listener_arn = aws_lb_listener.windmill_cluster_alb_listener.arn priority = 100 action { type = "forward" - target_group_arn = aws_lb_target_group.windmill_cluster_windmill_lsp_tg.arn + target_group_arn = aws_lb_target_group.windmill_cluster_windmill_extra_tg.arn } condition { path_pattern { - values = ["/ws/*"] + values = ["/ws/*", "/ws_mp/*", "/ws_debug/*"] } } } -resource "aws_lb_listener_rule" "windmill_cluster_alb_multiplayer_rule" { - listener_arn = aws_lb_listener.windmill_cluster_alb_listener.arn - priority = 50 +# Upgrading a stack created before windmill-extra (standalone LSP and multiplayer services). +# The LSP rule becomes the extra rule, updated in place: it keeps priority 100, where destroying it and +# creating a new rule with the same priority can fail with PriorityInUse. The LSP target group becomes +# the extra target group, replaced before destroy (see create_before_destroy above). +moved { + from = aws_lb_listener_rule.windmill_cluster_alb_lsp_rule + to = aws_lb_listener_rule.windmill_cluster_alb_extra_rule +} - action { - type = "forward" - target_group_arn = aws_lb_target_group.windmill_cluster_windmill_multiplayer_tg.arn - } - - condition { - path_pattern { - values = ["/ws_mp/*"] - } - } +moved { + from = aws_lb_target_group.windmill_cluster_windmill_lsp_tg + to = aws_lb_target_group.windmill_cluster_windmill_extra_tg } diff --git a/examples/deploy/aws-ecs-terraform/rds.tf b/examples/deploy/aws-ecs-terraform/rds.tf index 610f4a61d2..7cceec7494 100644 --- a/examples/deploy/aws-ecs-terraform/rds.tf +++ b/examples/deploy/aws-ecs-terraform/rds.tf @@ -19,7 +19,7 @@ resource "aws_db_instance" "windmill_cluster_rds" { # iops = 3000 engine = "postgres" - engine_version = "16.1" + engine_version = "16" parameter_group_name = "default.postgres16" license_model = "postgresql-license" @@ -39,4 +39,11 @@ resource "aws_db_instance" "windmill_cluster_rds" { backup_retention_period = 7 skip_final_snapshot = true deletion_protection = false + + # engine_version picks the major version when the database is created; RDS then applies minor + # upgrades itself. Changing it on an existing instance would ask RDS to "upgrade" to the bare major + # version, which it rejects, so Terraform leaves it alone after creation. + lifecycle { + ignore_changes = [engine_version] + } } \ No newline at end of file diff --git a/examples/deploy/aws-ecs-terraform/vpc.tf b/examples/deploy/aws-ecs-terraform/vpc.tf index 6e9c66c6bf..79f0a815a1 100644 --- a/examples/deploy/aws-ecs-terraform/vpc.tf +++ b/examples/deploy/aws-ecs-terraform/vpc.tf @@ -69,6 +69,9 @@ resource "aws_nat_gateway" "windmill_cluster_nat_gateway_public1" { tags = { "Name" = "windmill-cluster-nat-gateway-public1" } + + # Created after, and deleted before, the internet gateway it routes through. + depends_on = [aws_internet_gateway.windmill_cluster_internet_gateway] } resource "aws_eip" "windmill_cluster_nat_gateway_public2_eip" { @@ -81,6 +84,9 @@ resource "aws_nat_gateway" "windmill_cluster_nat_gateway_public2" { tags = { "Name" = "windmill-cluster-nat-gateway-public2" } + + # Created after, and deleted before, the internet gateway it routes through. + depends_on = [aws_internet_gateway.windmill_cluster_internet_gateway] } resource "aws_route_table" "windmill_cluster_rtb_public" { diff --git a/examples/deploy/aws-ecs-terraform/windmill_extra.tf b/examples/deploy/aws-ecs-terraform/windmill_extra.tf new file mode 100644 index 0000000000..ecb4de052d --- /dev/null +++ b/examples/deploy/aws-ecs-terraform/windmill_extra.tf @@ -0,0 +1,126 @@ +# windmill-extra: LSP, Multiplayer and Debugger in one image, behind a gateway on port 3000 +# that routes /ws/* (LSP), /ws_mp/* (Multiplayer) and /ws_debug/* (Debugger). +resource "aws_cloudwatch_log_group" "windmill_cluster_windmill_extra_log_group" { + name = "/ecs/windmill-extra" +} + +resource "aws_ecs_task_definition" "windmill_cluster_windmill_extra_td" { + family = "windmill-extra" + network_mode = "awsvpc" + execution_role_arn = data.aws_iam_role.ecs_task_execution_role.arn + cpu = 1024 + memory = 1536 + runtime_platform { + operating_system_family = "LINUX" + cpu_architecture = "X86_64" + } + requires_compatibilities = ["EC2"] + + container_definitions = jsonencode([ + { + name = "windmill-extra" + image = "ghcr.io/windmill-labs/windmill-extra:latest" + cpu = 1024 + memory = 1536 + essential = true + portMappings = [ + { + name = "http" + containerPort = 3000 + hostPort = 3000 + protocol = "tcp" + appProtocol = "http" + } + ] + environment = [ + { + name = "JSON_FMT" + value = "true" + }, + { + # The target group health check (GET /) is answered by the LSP: keep it enabled, or set a + # health_check path on windmill_cluster_windmill_extra_tg that another enabled service answers. + name = "ENABLE_LSP" + value = "true" + }, + { + # Real-time collaboration, Enterprise Edition only. + name = "ENABLE_MULTIPLAYER" + value = "true" + }, + { + # Keep REQUIRE_SIGNED_DEBUG_REQUESTS=true on any internet-reachable deployment. + name = "ENABLE_DEBUGGER" + value = "false" + }, + { + name = "REQUIRE_SIGNED_DEBUG_REQUESTS" + value = "true" + }, + { + # Multiplayer and the debugger verify the tokens the Windmill server signs, with the key + # they fetch from /api/debug/jwks. Without it, every multiplayer + # session is rejected. Use https:// if you add a TLS listener. + name = "WINDMILL_BASE_URL" + value = "http://${aws_lb.windmill_cluster_alb.dns_name}" + }, + ] + mountPoints = [ + { + sourceVolume = "lsp_cache" + containerPath = "/pyls/.cache" + } + ] + logConfiguration = { + logDriver = "awslogs" + options = { + "awslogs-group" = aws_cloudwatch_log_group.windmill_cluster_windmill_extra_log_group.name + "awslogs-region" = data.aws_region.current.name + "awslogs-stream-prefix" = "ecs" + } + } + } + ]) + + volume { + name = "lsp_cache" + host_path = "/pyls/.cache" + } +} + +resource "aws_ecs_service" "windmill_cluster_windmill_extra_service" { + name = "windmill-extra" + cluster = aws_ecs_cluster.windmill_cluster.id + task_definition = aws_ecs_task_definition.windmill_cluster_windmill_extra_td.arn + desired_count = 1 + + network_configuration { + subnets = [ + aws_subnet.windmill_cluster_subnet_private1.id, + aws_subnet.windmill_cluster_subnet_private2.id, + ] + security_groups = [aws_security_group.windmill_cluster_sg.id] + } + + force_new_deployment = true + placement_constraints { + type = "distinctInstance" + } + + capacity_provider_strategy { + capacity_provider = aws_ecs_capacity_provider.windmill_cluster_capacity_provider.name + weight = 100 + } + + load_balancer { + target_group_arn = aws_lb_target_group.windmill_cluster_windmill_extra_tg.arn + container_name = "windmill-extra" + container_port = 3000 + } + + # ECS rejects a service whose target group is not yet attached to the load balancer. + depends_on = [ + aws_autoscaling_group.windmill_cluster_asg, + aws_lb_listener_rule.windmill_cluster_alb_extra_rule, + ] +} diff --git a/examples/deploy/aws-ecs-terraform/windmill_lsp.tf b/examples/deploy/aws-ecs-terraform/windmill_lsp.tf deleted file mode 100644 index 02bd2c5d56..0000000000 --- a/examples/deploy/aws-ecs-terraform/windmill_lsp.tf +++ /dev/null @@ -1,85 +0,0 @@ -resource "aws_cloudwatch_log_group" "windmill_cluster_windmill_lsp_log_group" { - name = "/ecs/windmill-lsp" -} - -resource "aws_ecs_task_definition" "windmill_cluster_windmill_lsp_td" { - family = "windmill-lsp" - network_mode = "awsvpc" - execution_role_arn = data.aws_iam_role.ecs_task_execution_role.arn - cpu = 1024 - memory = 1536 - runtime_platform { - operating_system_family = "LINUX" - cpu_architecture = "X86_64" - } - requires_compatibilities = ["EC2"] - - container_definitions = jsonencode([ - { - name = "windmill-lsp" - image = "ghcr.io/windmill-labs/windmill-lsp:latest" - cpu = 1024 - memory = 1536 - essential = true - portMappings = [ - { - name = "http" - containerPort = 3001 - hostPort = 3001 - protocol = "tcp" - appProtocol = "http" - } - ] - environment = [{ - name = "JSON_FMT" - value = "true" - }] - logConfiguration = { - logDriver = "awslogs" - options = { - "awslogs-group" = aws_cloudwatch_log_group.windmill_cluster_windmill_lsp_log_group.name - "awslogs-region" = data.aws_region.current.name - "awslogs-stream-prefix" = "ecs" - } - } - } - ]) - - volume { - name = "lsp_cache" - host_path = "/pyls/.cache" - } -} - -resource "aws_ecs_service" "windmill_cluster_windmill_lsp_service" { - name = "windmill-lsp" - cluster = aws_ecs_cluster.windmill_cluster.id - task_definition = aws_ecs_task_definition.windmill_cluster_windmill_lsp_td.arn - desired_count = 1 - - network_configuration { - subnets = [ - aws_subnet.windmill_cluster_subnet_private1.id, - aws_subnet.windmill_cluster_subnet_private2.id, - ] - security_groups = [aws_security_group.windmill_cluster_sg.id] - } - - force_new_deployment = true - placement_constraints { - type = "distinctInstance" - } - - capacity_provider_strategy { - capacity_provider = aws_ecs_capacity_provider.windmill_cluster_capacity_provider.name - weight = 100 - } - - load_balancer { - target_group_arn = aws_lb_target_group.windmill_cluster_windmill_lsp_tg.arn - container_name = "windmill-lsp" - container_port = 3001 - } - - depends_on = [aws_autoscaling_group.windmill_cluster_asg] -} diff --git a/examples/deploy/aws-ecs-terraform/windmill_multiplayer.tf b/examples/deploy/aws-ecs-terraform/windmill_multiplayer.tf deleted file mode 100644 index a9ccfa0ee2..0000000000 --- a/examples/deploy/aws-ecs-terraform/windmill_multiplayer.tf +++ /dev/null @@ -1,80 +0,0 @@ -resource "aws_cloudwatch_log_group" "windmill_cluster_windmill_multiplayer_log_group" { - name = "/ecs/windmill-multiplayer" -} - -resource "aws_ecs_task_definition" "windmill_cluster_windmill_multiplayer_td" { - family = "windmill-multiplayer" - network_mode = "awsvpc" - execution_role_arn = data.aws_iam_role.ecs_task_execution_role.arn - cpu = 1024 - memory = 1536 - runtime_platform { - operating_system_family = "LINUX" - cpu_architecture = "X86_64" - } - requires_compatibilities = ["EC2"] - - container_definitions = jsonencode([ - { - name = "windmill-multiplayer" - image = "ghcr.io/windmill-labs/windmill-multiplayer:latest" - cpu = 1024 - memory = 1536 - essential = true - portMappings = [ - { - name = "http" - containerPort = 3002 - hostPort = 3002 - protocol = "tcp" - appProtocol = "http" - } - ] - environment = [{ - name = "JSON_FMT" - value = "true" - }] - logConfiguration = { - logDriver = "awslogs" - options = { - "awslogs-group" = aws_cloudwatch_log_group.windmill_cluster_windmill_multiplayer_log_group.name - "awslogs-region" = data.aws_region.current.name - "awslogs-stream-prefix" = "ecs" - } - } - } - ]) -} - -resource "aws_ecs_service" "windmill_cluster_windmill_multiplayer_service" { - name = "windmill-multiplayer" - cluster = aws_ecs_cluster.windmill_cluster.id - task_definition = aws_ecs_task_definition.windmill_cluster_windmill_multiplayer_td.arn - desired_count = 1 - - network_configuration { - subnets = [ - aws_subnet.windmill_cluster_subnet_private1.id, - aws_subnet.windmill_cluster_subnet_private2.id, - ] - security_groups = [aws_security_group.windmill_cluster_sg.id] - } - - force_new_deployment = true - placement_constraints { - type = "distinctInstance" - } - - capacity_provider_strategy { - capacity_provider = aws_ecs_capacity_provider.windmill_cluster_capacity_provider.name - weight = 100 - } - - load_balancer { - target_group_arn = aws_lb_target_group.windmill_cluster_windmill_multiplayer_tg.arn - container_name = "windmill-multiplayer" - container_port = 3002 - } - - depends_on = [aws_autoscaling_group.windmill_cluster_asg] -} diff --git a/examples/deploy/aws-ecs-terraform/windmill_worker_high_performance.tf b/examples/deploy/aws-ecs-terraform/windmill_worker_high_performance.tf index 1c7ff33308..f033d31057 100644 --- a/examples/deploy/aws-ecs-terraform/windmill_worker_high_performance.tf +++ b/examples/deploy/aws-ecs-terraform/windmill_worker_high_performance.tf @@ -17,9 +17,10 @@ resource "aws_launch_template" "windmill_cluster_high_performance_lt" { } network_interfaces { - device_index = 0 - delete_on_termination = true - associate_public_ip_address = true + device_index = 0 + delete_on_termination = true + # Private subnets: outbound traffic goes through the NAT gateways, so no public IP. + associate_public_ip_address = false security_groups = [ aws_security_group.windmill_cluster_sg.id ] @@ -57,6 +58,16 @@ resource "aws_autoscaling_group" "windmill_cluster_high_performance_asg" { value = true propagate_at_launch = true } + + # The ECS agent on these instances reaches AWS through private route -> NAT gateway -> public + # route -> internet gateway. Keeping that whole path until the instances are gone lets + # `terraform destroy` drain and delete the ECS services, which otherwise hang in DRAINING. + depends_on = [ + aws_route_table_association.windmill_cluster_subnet_private1__rtb_private1, + aws_route_table_association.windmill_cluster_subnet_private2__rtb_private2, + aws_route_table_association.windmill_cluster_subnet_public1__rtb_public, + aws_route_table_association.windmill_cluster_subnet_public2__rtb_public, + ] } resource "aws_ecs_capacity_provider" "windmill_cluster_high_performance_capacity_provider" { diff --git a/examples/deploy/aws-eks-cloudformation/quicklaunch.yaml b/examples/deploy/aws-eks-cloudformation/quicklaunch.yaml index 5f2b4c42dc..c26916aa1e 100644 --- a/examples/deploy/aws-eks-cloudformation/quicklaunch.yaml +++ b/examples/deploy/aws-eks-cloudformation/quicklaunch.yaml @@ -436,7 +436,7 @@ Resources: --set windmill.baseDomain=windmill.local \ --set windmill.baseProtocol=http \ --set windmill.appReplicas=${WorkerReplicas} \ - --set windmill.lspReplicas=2 \ + --set windmill.extraReplicas=2 \ --set windmill.workerGroups[0].name=default \ --set windmill.workerGroups[0].mode=worker \ --set windmill.workerGroups[0].replicas=${WorkerReplicas} \ diff --git a/examples/deploy/otel-tracing-grafana/docker-compose.yml b/examples/deploy/otel-tracing-grafana/docker-compose.yml index 525010efad..d75caa0959 100644 --- a/examples/deploy/otel-tracing-grafana/docker-compose.yml +++ b/examples/deploy/otel-tracing-grafana/docker-compose.yml @@ -136,29 +136,34 @@ services: - windmill_index:/tmp/windmill/search - worker_logs:/tmp/windmill/logs - lsp: - image: ghcr.io/windmill-labs/windmill-lsp:latest + # Combined extra services behind one gateway on port 3000: LSP, Multiplayer and Debugger. + # Caddy routes /ws/*, /ws_mp/* and /ws_debug/* here (see the Caddyfile at the repo root). + # - ENABLE_LSP=true (default) - Language Server Protocol for code intelligence + # - ENABLE_MULTIPLAYER=false - Real-time collaboration (Enterprise Edition) + # - ENABLE_DEBUGGER=false - Interactive debugging via DAP WebSocket + windmill_extra: + image: ghcr.io/windmill-labs/windmill-extra:latest pull_policy: always restart: unless-stopped expose: - - 3001 + - 3000 + environment: + - ENABLE_LSP=true + - ENABLE_MULTIPLAYER=false # Set to true to enable multiplayer (Enterprise Edition) + - ENABLE_DEBUGGER=false # Set to true to enable the debugger + - REQUIRE_SIGNED_DEBUG_REQUESTS=true # Do NOT set to false on any internet-reachable deployment + - WINDMILL_BASE_URL=http://windmill_server:8000 volumes: - - lsp_cache:/root/.cache - - multiplayer: - image: ghcr.io/windmill-labs/windmill-multiplayer:latest - deploy: - replicas: 0 # Set to 1 to enable multiplayer, only available on Enterprise Edition - restart: unless-stopped - expose: - - 3002 + - lsp_cache:/pyls/.cache caddy: - image: ghcr.io/windmill-labs/caddy-l4:latest + # Uses the Caddyfile at the repo root. The two are version-coupled: keep this tag in step with + # the caddy image in the root docker-compose.yml. + image: ghcr.io/windmill-labs/caddy-l4:2.11.4-1 restart: unless-stopped # Configure the mounted Caddyfile and the exposed ports or use another reverse proxy if needed volumes: - - ./Caddyfile:/etc/caddy/Caddyfile + - ../../../Caddyfile:/etc/caddy/Caddyfile # - ./certs:/certs # Provide custom certificate files like cert.pem and key.pem to enable HTTPS - See the corresponding section in the Caddyfile ports: # To change the exposed port, simply change 80:80 to :80. No other changes needed diff --git a/examples/deploy/otel-tracing-jaeger/docker-compose.yml b/examples/deploy/otel-tracing-jaeger/docker-compose.yml index 3b186b45fb..b8db8af57e 100644 --- a/examples/deploy/otel-tracing-jaeger/docker-compose.yml +++ b/examples/deploy/otel-tracing-jaeger/docker-compose.yml @@ -136,29 +136,34 @@ services: - windmill_index:/tmp/windmill/search - worker_logs:/tmp/windmill/logs - lsp: - image: ghcr.io/windmill-labs/windmill-lsp:latest + # Combined extra services behind one gateway on port 3000: LSP, Multiplayer and Debugger. + # Caddy routes /ws/*, /ws_mp/* and /ws_debug/* here (see the Caddyfile at the repo root). + # - ENABLE_LSP=true (default) - Language Server Protocol for code intelligence + # - ENABLE_MULTIPLAYER=false - Real-time collaboration (Enterprise Edition) + # - ENABLE_DEBUGGER=false - Interactive debugging via DAP WebSocket + windmill_extra: + image: ghcr.io/windmill-labs/windmill-extra:latest pull_policy: always restart: unless-stopped expose: - - 3001 + - 3000 + environment: + - ENABLE_LSP=true + - ENABLE_MULTIPLAYER=false # Set to true to enable multiplayer (Enterprise Edition) + - ENABLE_DEBUGGER=false # Set to true to enable the debugger + - REQUIRE_SIGNED_DEBUG_REQUESTS=true # Do NOT set to false on any internet-reachable deployment + - WINDMILL_BASE_URL=http://windmill_server:8000 volumes: - - lsp_cache:/root/.cache - - multiplayer: - image: ghcr.io/windmill-labs/windmill-multiplayer:latest - deploy: - replicas: 0 # Set to 1 to enable multiplayer, only available on Enterprise Edition - restart: unless-stopped - expose: - - 3002 + - lsp_cache:/pyls/.cache caddy: - image: ghcr.io/windmill-labs/caddy-l4:latest + # Uses the Caddyfile at the repo root. The two are version-coupled: keep this tag in step with + # the caddy image in the root docker-compose.yml. + image: ghcr.io/windmill-labs/caddy-l4:2.11.4-1 restart: unless-stopped # Configure the mounted Caddyfile and the exposed ports or use another reverse proxy if needed volumes: - - ./Caddyfile:/etc/caddy/Caddyfile + - ../../../Caddyfile:/etc/caddy/Caddyfile # - ./certs:/certs # Provide custom certificate files like cert.pem and key.pem to enable HTTPS - See the corresponding section in the Caddyfile ports: # To change the exposed port, simply change 80:80 to :80. No other changes needed diff --git a/frontend/README_DEV.md b/frontend/README_DEV.md index 9848f333ae..68aa3d7bc9 100644 --- a/frontend/README_DEV.md +++ b/frontend/README_DEV.md @@ -90,6 +90,24 @@ You can configure another proxy to use like so: REMOTE=http://127.0.0.1:8000 REMOTE_LSP=http://127.0.0.1:3001 npm run dev ``` +`REMOTE_LSP` needs a language server on port 3001. The `windmill-extra` image serves one: + +```bash +docker run --rm -p 3001:3001 ghcr.io/windmill-labs/windmill-extra:latest +``` + +To proxy multiplayer and the debugger as well, publish the image's gateway on port 3000 and use `REMOTE_EXTRA`, which +covers `/ws/*`, `/ws_mp/*` and `/ws_debug/*`. Those two services verify tokens signed by your backend, so give the +container its URL (on Linux, add `--add-host=host.docker.internal:host-gateway`): + +```bash +docker run --rm -p 3000:3000 -e WINDMILL_BASE_URL=http://host.docker.internal:8000 ghcr.io/windmill-labs/windmill-extra:latest +REMOTE=http://127.0.0.1:8000 REMOTE_EXTRA=http://127.0.0.1:3000 npm run dev +``` + +To try local changes to `lsp/pyls_launcher.py` or `lsp/Pipfile`, build that image from the repo root with +`docker build -f docker/DockerfileExtra -t windmill-extra-dev .` and run `windmill-extra-dev` instead. + ### Run dev servers on demand A dev server costs 1.1-1.7 GB resident once a page has been browsed, which adds up when diff --git a/lsp/Dockerfile b/lsp/Dockerfile deleted file mode 100644 index b9bba17df2..0000000000 --- a/lsp/Dockerfile +++ /dev/null @@ -1,62 +0,0 @@ -FROM python:3.12-slim as python-base -FROM node:22-slim as node-base - -FROM python-base -COPY --from=node-base /usr/local /usr/local - -ENV PATH="/usr/local/bin:${PATH}" -ENV PIPENV_VENV_IN_PROJECT=1 -ENV XDG_CACHE_HOME=/pyls/.cache - -RUN apt-get update \ - && apt-get install -y shellcheck wget git ca-certificates \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && pip install pipenv - -RUN npm install -g diagnostic-languageserver pyright - -RUN set -eux; \ - arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \ - url=; \ - case "$arch" in \ - 'amd64') \ - targz='go1.26.0.linux-amd64.tar.gz'; \ - ;; \ - 'arm64') \ - targz='go1.26.0.linux-arm64.tar.gz'; \ - ;; \ - 'armhf') \ - targz='go1.26.0.linux-armv6l.tar.gz'; \ - ;; \ - *) echo >&2 "error: unsupported architecture '$arch' (likely packaging update needed)"; exit 1 ;; \ - esac; \ - wget "https://golang.org/dl/$targz" -nv && tar -C /usr/local -xzf "$targz" && rm "$targz"; - -ENV PATH="${PATH}:/usr/local/go/bin" -ENV GOBIN=/usr/local/go/bin -RUN /usr/local/go/bin/go install -v golang.org/x/tools/gopls@latest - -RUN pip3 install tornado python-lsp-jsonrpc ruff==0.16.0 - -COPY --from=denoland/deno:2.2.1 --chmod=755 /usr/bin/deno /usr/bin/deno - -RUN mkdir -p /pyls/.cache - -WORKDIR /pyls -COPY Pipfile . -RUN cat Pipfile -RUN pip install Cython -RUN pipenv install - -COPY pyls_launcher.py . - -RUN mkdir -p /tmp/monaco && chmod -R 777 /tmp/monaco -RUN cd /tmp/monaco && npm install --save-dev windmill-client - -RUN chmod -R a+rX /usr/local && \ - chmod -R a+rX /pyls - -EXPOSE 3001 - -CMD ["sh", "-c", "if [ -n \"$NETRC\" ]; then echo \"$NETRC\" > /root/.netrc && chmod 600 /root/.netrc; fi && if [ -d /root/.cache ]; then export XDG_CACHE_HOME=/root/.cache && cp -r /pyls/.cache /root/.cache; fi && python3 pyls_launcher.py"] diff --git a/lsp/dev.nu b/lsp/dev.nu deleted file mode 100755 index 0beac3cfc1..0000000000 --- a/lsp/dev.nu +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env nu -let tag = $"dev-lsp-(git branch --show-current)"; - -# Build docker image and start server on localhost:3001 -def main [ - --podman(-p) # Use podman - # TODO: - --detach(-d) # Run container in background and print container ID - ] { - if $podman { - podman build --tag $tag .; podman run -p 3001:3001 $tag - } else { - docker build --tag $tag .; docker run -p 3001:3001 $tag - } -} - -# Stop podman container -def "main stop" [] { - try { docker stop (docker ps -q --filter $"ancestor=($tag)") } - try { podman stop (podman ps -q --filter $"ancestor=($tag)") } -}