Add WebClaw app

2026-09-22 16:00:58 +00:00 · 2026-08-01 03:19:49 +08:00
parent 6ab05890ac
commit 6606e2ef6d
17 changed files with 412 additions and 0 deletions
@@ -0,0 +1,49 @@
+# WebClaw
+
+## 产品介绍
+
+WebClaw 是一个自托管、无状态的网页内容提取 REST API。它可以抓取公开网页，并返回适合检索、归档和 AI 工作流使用的 Markdown、纯文本、HTML 或结构化 JSON。
+
+## 主要功能
+
+- 通过 `/v1/scrape` 提取单个网页
+- 支持 Markdown、纯文本、HTML、LLM 文本和 JSON 输出
+- 支持站点抓取、URL 映射、批量提取、差异比较和品牌信息提取
+- 可选接入 Serper.dev 搜索和 WebClaw Cloud 回退服务
+- 使用 Bearer Token 保护所有 `/v1/*` 接口
+
+## 访问说明
+
+- 安装后通过设置的 HTTP 端口访问 REST API；`/health` 可用于健康检查。
+- 调用 `/v1/*` 接口时，在 `Authorization` 请求头中提供安装时生成的 API Token。
+- 默认包启动开源的 `webclaw-server`，不包含托管版的反机器人绕过、JavaScript 渲染、多租户和计费功能。
+- 服务会访问用户提交的 URL。对公网开放前，请通过 1Panel 反向代理启用 HTTPS，并限制 API Token 的分发范围。
+
+## 可选服务
+
+- `WEBCLAW_CLOUD_API_KEY` 用于在受保护站点上启用 WebClaw Cloud 回退。
+- `SERPER_API_KEY` 用于启用 `/v1/search`；留空时该接口返回未配置状态，不影响本地网页提取。
+
+## 数据说明
+
+开源服务器本身无状态，不创建数据库或持久化用户内容。API Token 和可选外部服务密钥由 1Panel 应用配置管理。
+
+## 链接
+
+- 网站：https://webclaw.io
+- 项目：https://github.com/0xMassi/webclaw
+- 镜像：https://github.com/0xMassi/webclaw/pkgs/container/webclaw
+
+## Introduction
+
+WebClaw is a self-hosted, stateless REST API for extracting public web pages as clean Markdown, plain text, HTML, or structured JSON.
+
+## Features
+
+- Extract a page through `/v1/scrape`
+- Return Markdown, plain text, HTML, LLM text, or JSON
+- Crawl sites, map URLs, process batches, compare pages, and extract brand metadata
+- Optionally enable Serper.dev search and WebClaw Cloud fallback
+- Protect every `/v1/*` endpoint with a Bearer token
+
+The `/health` endpoint is available without authentication. Send the generated API token in the `Authorization: Bearer ...` header for `/v1/*` requests. Enable HTTPS through a 1Panel reverse proxy before exposing the service publicly.
@@ -0,0 +1,35 @@
+# WebClaw
+
+## Introduction
+
+WebClaw is a self-hosted, stateless REST API for extracting public web pages as clean Markdown, plain text, HTML, or structured JSON.
+
+## Features
+
+- Extract a page through `/v1/scrape`
+- Return Markdown, plain text, HTML, LLM text, or JSON
+- Crawl sites, map URLs, process batches, compare pages, and extract brand metadata
+- Optionally enable Serper.dev search and WebClaw Cloud fallback
+- Protect every `/v1/*` endpoint with a Bearer token
+
+## Usage
+
+- The `/health` endpoint is available without authentication.
+- Send the generated API token in the `Authorization: Bearer ...` header for `/v1/*` requests.
+- This package runs the open-source `webclaw-server`. Hosted anti-bot bypass, JavaScript rendering, multi-tenant features, and billing are not included.
+- The service fetches URLs supplied by API clients. Enable HTTPS through a 1Panel reverse proxy before exposing it publicly and restrict token distribution.
+
+## Optional Services
+
+- `WEBCLAW_CLOUD_API_KEY` enables cloud fallback for protected sites.
+- `SERPER_API_KEY` enables `/v1/search`. Leaving it blank does not affect local page extraction.
+
+## Data
+
+The open-source server is stateless and creates no database. 1Panel manages the API token and optional provider keys as application configuration.
+
+## Links
+
+- Website: https://webclaw.io
+- Project: https://github.com/0xMassi/webclaw
+- Image: https://github.com/0xMassi/webclaw/pkgs/container/webclaw
@@ -0,0 +1,31 @@
+name: WebClaw
+tags:
+  - 实用工具
+title: 将网页提取为干净内容的自托管 API
+description: 将网页提取为干净内容的自托管 API
+additionalProperties:
+  key: webclaw
+  name: WebClaw
+  tags:
+    - Tool
+  shortDescZh: 将网页提取为干净内容的自托管 API
+  shortDescEn: A self-hosted API for extracting clean content from web pages
+  description:
+    en: A self-hosted REST API that extracts web pages as clean Markdown, text, HTML, or structured JSON.
+    zh: 将网页提取为干净 Markdown、文本、HTML 或结构化 JSON 的自托管 REST API。
+    zh-Hant: 將網頁擷取為乾淨 Markdown、文字、HTML 或結構化 JSON 的自託管 REST API。
+    ja: Web ページをクリーンな Markdown、テキスト、HTML、構造化 JSON に抽出するセルフホスト REST API です。
+    ko: 웹 페이지를 깔끔한 Markdown, 텍스트, HTML 또는 구조화된 JSON으로 추출하는 자체 호스팅 REST API입니다.
+    ru: Самостоятельно размещаемый REST API для извлечения веб-страниц в чистый Markdown, текст, HTML или структурированный JSON.
+    ms: API REST dihos sendiri untuk mengekstrak laman web sebagai Markdown, teks, HTML atau JSON berstruktur yang bersih.
+    pt-br: API REST auto-hospedada que extrai paginas web como Markdown, texto, HTML ou JSON estruturado e limpo.
+  type: tool
+  crossVersionUpdate: true
+  limit: 0
+  recommend: 0
+  website: https://webclaw.io
+  github: https://github.com/0xMassi/webclaw
+  document: https://github.com/0xMassi/webclaw
+  architectures:
+    - amd64
+    - arm64
@@ -0,0 +1,6 @@
+PANEL_APP_BIND_ADDRESS=0.0.0.0
+PANEL_APP_PORT_HTTP=40102
+WEBCLAW_API_KEY=replace-with-a-strong-random-api-token
+WEBCLAW_CLOUD_API_KEY=
+SERPER_API_KEY=
+CONTAINER_NAME=1Panel-localwebclaw
@@ -0,0 +1,85 @@
+additionalProperties:
+  formFields:
+    - default: 0.0.0.0
+      edit: true
+      envKey: PANEL_APP_BIND_ADDRESS
+      labelEn: Bind Address
+      labelZh: 绑定地址
+      label:
+        en: Bind Address
+        zh: 绑定地址
+        zh-Hant: 綁定位址
+        ja: バインドアドレス
+        ko: 바인드 주소
+        ru: Адрес привязки
+        ms: Alamat ikatan
+        pt-br: Endereco de vinculacao
+      required: true
+      type: text
+    - default: 40102
+      edit: true
+      envKey: PANEL_APP_PORT_HTTP
+      labelEn: HTTP Port
+      labelZh: HTTP 端口
+      label:
+        en: HTTP Port
+        zh: HTTP 端口
+        zh-Hant: HTTP 連接埠
+        ja: HTTP ポート
+        ko: HTTP 포트
+        ru: HTTP-порт
+        ms: Port HTTP
+        pt-br: Porta HTTP
+      required: true
+      rule: paramPort
+      type: number
+    - default: ""
+      edit: true
+      envKey: WEBCLAW_API_KEY
+      labelEn: API Token
+      labelZh: API Token
+      label:
+        en: API Token
+        zh: API Token
+        zh-Hant: API Token
+        ja: API トークン
+        ko: API 토큰
+        ru: API-токен
+        ms: Token API
+        pt-br: Token da API
+      random: true
+      required: true
+      rule: paramComplexity
+      type: password
+    - default: ""
+      edit: true
+      envKey: WEBCLAW_CLOUD_API_KEY
+      labelEn: Cloud Fallback Key
+      labelZh: Cloud 回退密钥
+      label:
+        en: Cloud Fallback Key
+        zh: Cloud 回退密钥
+        zh-Hant: Cloud 備援金鑰
+        ja: Cloud フォールバックキー
+        ko: Cloud 대체 키
+        ru: Ключ резервного облачного сервиса
+        ms: Kunci sandaran Cloud
+        pt-br: Chave de fallback do Cloud
+      required: false
+      type: password
+    - default: ""
+      edit: true
+      envKey: SERPER_API_KEY
+      labelEn: Serper Search Key
+      labelZh: Serper 搜索密钥
+      label:
+        en: Serper Search Key
+        zh: Serper 搜索密钥
+        zh-Hant: Serper 搜尋金鑰
+        ja: Serper 検索キー
+        ko: Serper 검색 키
+        ru: Ключ поиска Serper
+        ms: Kunci carian Serper
+        pt-br: Chave de pesquisa Serper
+      required: false
+      type: password
@@ -0,0 +1,44 @@
+services:
+  webclaw:
+    image: "ghcr.io/0xmassi/webclaw:latest"
+    container_name: ${CONTAINER_NAME}
+    restart: unless-stopped
+    command:
+      - webclaw-server
+    user: "65534:65534"
+    read_only: true
+    tmpfs:
+      - /tmp:rw,nodev,noexec,nosuid,size=16m
+    cap_drop:
+      - ALL
+    security_opt:
+      - no-new-privileges:true
+    networks:
+      - 1panel-network
+    ports:
+      - "${PANEL_APP_BIND_ADDRESS}:${PANEL_APP_PORT_HTTP}:3000"
+    environment:
+      - WEBCLAW_HOST=0.0.0.0
+      - WEBCLAW_PORT=3000
+      - WEBCLAW_API_KEY=${WEBCLAW_API_KEY}
+      - WEBCLAW_CLOUD_API_KEY=${WEBCLAW_CLOUD_API_KEY}
+      - SERPER_API_KEY=${SERPER_API_KEY}
+    healthcheck:
+      test:
+        - CMD
+        - bash
+        - -c
+        - >-
+          exec 3<>/dev/tcp/127.0.0.1/3000 &&
+          printf 'GET /health HTTP/1.0\r\nHost: localhost\r\n\r\n' >&3 &&
+          grep -q '"status":"ok"' <&3
+      interval: 30s
+      timeout: 5s
+      retries: 5
+      start_period: 10s
+    labels:
+      createdBy: Apps
+
+networks:
+  1panel-network:
+    external: true
@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+set -euo pipefail
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+exit 0
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+exit 0
@@ -0,0 +1,7 @@
+Asset: logo.png
+Source: https://github.com/0xMassi/webclaw/blob/50a17d27e504e558372528a4e6530ff09aee3537/.github/banner.png
+Source SHA-256: 0658c42fc44438dfef3038627d347214198a58c39adf64c9b2ac7d36580665cb
+Copyright: WebClaw contributors
+License: GNU Affero General Public License v3.0 (AGPL-3.0)
+License source: https://github.com/0xMassi/webclaw/blob/50a17d27e504e558372528a4e6530ff09aee3537/LICENSE
+Modification: cropped the upstream banner to its mark, centered it on a square canvas, and resized it for the app catalog.
@@ -0,0 +1,6 @@
+PANEL_APP_BIND_ADDRESS=0.0.0.0
+PANEL_APP_PORT_HTTP=40102
+WEBCLAW_API_KEY=replace-with-a-strong-random-api-token
+WEBCLAW_CLOUD_API_KEY=
+SERPER_API_KEY=
+CONTAINER_NAME=1Panel-localwebclaw
@@ -0,0 +1,85 @@
+additionalProperties:
+  formFields:
+    - default: 0.0.0.0
+      edit: true
+      envKey: PANEL_APP_BIND_ADDRESS
+      labelEn: Bind Address
+      labelZh: 绑定地址
+      label:
+        en: Bind Address
+        zh: 绑定地址
+        zh-Hant: 綁定位址
+        ja: バインドアドレス
+        ko: 바인드 주소
+        ru: Адрес привязки
+        ms: Alamat ikatan
+        pt-br: Endereco de vinculacao
+      required: true
+      type: text
+    - default: 40102
+      edit: true
+      envKey: PANEL_APP_PORT_HTTP
+      labelEn: HTTP Port
+      labelZh: HTTP 端口
+      label:
+        en: HTTP Port
+        zh: HTTP 端口
+        zh-Hant: HTTP 連接埠
+        ja: HTTP ポート
+        ko: HTTP 포트
+        ru: HTTP-порт
+        ms: Port HTTP
+        pt-br: Porta HTTP
+      required: true
+      rule: paramPort
+      type: number
+    - default: ""
+      edit: true
+      envKey: WEBCLAW_API_KEY
+      labelEn: API Token
+      labelZh: API Token
+      label:
+        en: API Token
+        zh: API Token
+        zh-Hant: API Token
+        ja: API トークン
+        ko: API 토큰
+        ru: API-токен
+        ms: Token API
+        pt-br: Token da API
+      random: true
+      required: true
+      rule: paramComplexity
+      type: password
+    - default: ""
+      edit: true
+      envKey: WEBCLAW_CLOUD_API_KEY
+      labelEn: Cloud Fallback Key
+      labelZh: Cloud 回退密钥
+      label:
+        en: Cloud Fallback Key
+        zh: Cloud 回退密钥
+        zh-Hant: Cloud 備援金鑰
+        ja: Cloud フォールバックキー
+        ko: Cloud 대체 키
+        ru: Ключ резервного облачного сервиса
+        ms: Kunci sandaran Cloud
+        pt-br: Chave de fallback do Cloud
+      required: false
+      type: password
+    - default: ""
+      edit: true
+      envKey: SERPER_API_KEY
+      labelEn: Serper Search Key
+      labelZh: Serper 搜索密钥
+      label:
+        en: Serper Search Key
+        zh: Serper 搜索密钥
+        zh-Hant: Serper 搜尋金鑰
+        ja: Serper 検索キー
+        ko: Serper 검색 키
+        ru: Ключ поиска Serper
+        ms: Kunci carian Serper
+        pt-br: Chave de pesquisa Serper
+      required: false
+      type: password
@@ -0,0 +1,44 @@
+services:
+  webclaw:
+    image: "ghcr.io/0xmassi/webclaw:v0.6.16@sha256:ac410944110b14d536225d2fef67b35d8b6f7338d158b00a6b06646a6c8c0ab3"
+    container_name: ${CONTAINER_NAME}
+    restart: unless-stopped
+    command:
+      - webclaw-server
+    user: "65534:65534"
+    read_only: true
+    tmpfs:
+      - /tmp:rw,nodev,noexec,nosuid,size=16m
+    cap_drop:
+      - ALL
+    security_opt:
+      - no-new-privileges:true
+    networks:
+      - 1panel-network
+    ports:
+      - "${PANEL_APP_BIND_ADDRESS}:${PANEL_APP_PORT_HTTP}:3000"
+    environment:
+      - WEBCLAW_HOST=0.0.0.0
+      - WEBCLAW_PORT=3000
+      - WEBCLAW_API_KEY=${WEBCLAW_API_KEY}
+      - WEBCLAW_CLOUD_API_KEY=${WEBCLAW_CLOUD_API_KEY}
+      - SERPER_API_KEY=${SERPER_API_KEY}
+    healthcheck:
+      test:
+        - CMD
+        - bash
+        - -c
+        - >-
+          exec 3<>/dev/tcp/127.0.0.1/3000 &&
+          printf 'GET /health HTTP/1.0\r\nHost: localhost\r\n\r\n' >&3 &&
+          grep -q '"status":"ok"' <&3
+      interval: 30s
+      timeout: 5s
+      retries: 5
+      start_period: 10s
+    labels:
+      createdBy: Apps
+
+networks:
+  1panel-network:
+    external: true
@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+set -euo pipefail
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+exit 0
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+exit 0