mirror of
https://github.com/warmbly/warmbly.git
synced 2026-11-17 08:18:51 +00:00
475 lines
13 KiB
Go
475 lines
13 KiB
Go
// Package webread fetches a public page through a reader service (Crawl4AI,
|
|
// Jina Reader, a DDGS server, or any endpoint that answers with the page's
|
|
// HTML) for the times a site refuses a direct visit. Readers are operator or
|
|
// workspace configuration, tried in order, and every one returns raw HTML so
|
|
// callers parse it exactly as they parse a page they fetched themselves.
|
|
package webread
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/warmbly/warmbly/internal/config"
|
|
"github.com/warmbly/warmbly/internal/pkg/safehttp"
|
|
)
|
|
|
|
const (
|
|
readTimeout = 40 * time.Second
|
|
maxResponse = 6 << 20
|
|
defaultJina = "https://r.jina.ai"
|
|
providerJina = "jina"
|
|
)
|
|
|
|
// Providers are the reader backends this package speaks, in documentation order.
|
|
var Providers = []string{"crawl4ai", "jina", "ddgs", "custom"}
|
|
|
|
// Page is one page a reader returned.
|
|
type Page struct {
|
|
HTML string
|
|
// URL is where the reader landed after redirects, or the requested URL
|
|
// when the reader does not say.
|
|
URL string
|
|
// Via names the reader that answered.
|
|
Via string
|
|
}
|
|
|
|
// Reader fetches one page.
|
|
type Reader interface {
|
|
Name() string
|
|
Read(ctx context.Context, target string) (*Page, error)
|
|
}
|
|
|
|
// Backend is one configured reader.
|
|
type Backend struct {
|
|
Provider string
|
|
URL string
|
|
Key string
|
|
}
|
|
|
|
// ErrNotPublic means the reader landed somewhere a caller must not read.
|
|
var ErrNotPublic = errors.New("the reader was redirected to a private address")
|
|
|
|
// Blocked reports whether a status means the site turned an automated visitor
|
|
// away, rather than the page being missing or broken.
|
|
func Blocked(status int) bool {
|
|
switch status {
|
|
case http.StatusUnauthorized, http.StatusForbidden, http.StatusNotAcceptable, http.StatusTooManyRequests, http.StatusServiceUnavailable:
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// New builds the readers in order, skipping any missing what it needs. It
|
|
// returns nil when none is usable.
|
|
func New(backends []Backend) Reader {
|
|
var rs []Reader
|
|
for _, b := range backends {
|
|
if r := build(b); r != nil {
|
|
rs = append(rs, r)
|
|
}
|
|
}
|
|
switch len(rs) {
|
|
case 0:
|
|
return nil
|
|
case 1:
|
|
return guarded{rs[0]}
|
|
}
|
|
return guarded{chain(rs)}
|
|
}
|
|
|
|
// Jina is a Jina Reader client for a workspace key, keyless when key is empty.
|
|
func Jina(key string) Reader {
|
|
return guarded{build(Backend{Provider: providerJina, Key: key})}
|
|
}
|
|
|
|
// guarded hands a reader only pages a caller could have fetched itself.
|
|
type guarded struct{ Reader }
|
|
|
|
func (g guarded) Read(ctx context.Context, target string) (*Page, error) {
|
|
if err := publicTarget(ctx, target); err != nil {
|
|
return nil, err
|
|
}
|
|
return g.Reader.Read(ctx, target)
|
|
}
|
|
|
|
// publicTarget refuses a page on a private address or a non-web port.
|
|
func publicTarget(ctx context.Context, raw string) error {
|
|
u, err := url.Parse(raw)
|
|
if err != nil {
|
|
return ErrNotPublic
|
|
}
|
|
if p := u.Port(); p != "" && p != "80" && p != "443" && !config.UnsafeWebhookURLsAllowed() {
|
|
return ErrNotPublic
|
|
}
|
|
return publicURL(ctx, raw)
|
|
}
|
|
|
|
// readerClient follows redirects only within the reader's own host, so a
|
|
// reader cannot bounce the request to wherever a page points.
|
|
func readerClient(base string) *http.Client {
|
|
host := ""
|
|
if u, err := url.Parse(base); err == nil {
|
|
host = u.Host
|
|
}
|
|
return &http.Client{
|
|
Timeout: readTimeout,
|
|
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
|
if len(via) >= 5 || req.URL.Host != host {
|
|
return errors.New("the reader redirected away from itself")
|
|
}
|
|
// An https reader's key never follows a redirect onto plain http.
|
|
if via[0].URL.Scheme == "https" && req.URL.Scheme != "https" {
|
|
return errors.New("the reader redirected from https to http")
|
|
}
|
|
return nil
|
|
},
|
|
}
|
|
}
|
|
|
|
// Lookup reads one setting: an env var, or its AWS parameter or secret.
|
|
type Lookup func(envKey, awsKey string, secret bool) string
|
|
|
|
// FromConfig builds the instance's readers from WEB_READER (a comma-separated
|
|
// list, tried in order) and each one's WEB_READER_<PROVIDER>_URL and
|
|
// WEB_READER_<PROVIDER>_API_KEY.
|
|
func FromConfig(get Lookup) Reader {
|
|
var backends []Backend
|
|
for _, p := range splitList(get("WEB_READER", "web_reader/providers", false)) {
|
|
up := strings.ToUpper(p)
|
|
backends = append(backends, Backend{
|
|
Provider: p,
|
|
URL: get("WEB_READER_"+up+"_URL", "web_reader/"+p+"_url", false),
|
|
Key: get("WEB_READER_"+up+"_API_KEY", "web_reader/"+p+"_api_key", true),
|
|
})
|
|
}
|
|
return New(backends)
|
|
}
|
|
|
|
func splitList(raw string) []string {
|
|
var out []string
|
|
seen := map[string]bool{}
|
|
for _, p := range strings.Split(raw, ",") {
|
|
p = strings.ToLower(strings.TrimSpace(p))
|
|
if p != "" && !seen[p] {
|
|
seen[p] = true
|
|
out = append(out, p)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func build(b Backend) Reader {
|
|
base := strings.TrimRight(strings.TrimSpace(b.URL), "/")
|
|
key := strings.TrimSpace(b.Key)
|
|
switch strings.ToLower(strings.TrimSpace(b.Provider)) {
|
|
case providerJina:
|
|
if base == "" {
|
|
base = defaultJina
|
|
}
|
|
return &jina{base: base, key: key, http: readerClient(base)}
|
|
case "crawl4ai":
|
|
if base == "" {
|
|
return nil
|
|
}
|
|
return &crawl4ai{base: base, key: key, http: readerClient(base)}
|
|
case "ddgs":
|
|
if base == "" {
|
|
return nil
|
|
}
|
|
return &ddgs{base: base, http: readerClient(base)}
|
|
case "custom":
|
|
if !strings.Contains(base, "{url}") && !strings.Contains(base, "{url_encoded}") {
|
|
return nil
|
|
}
|
|
return &custom{template: base, key: key, http: readerClient(base)}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type multi []Reader
|
|
|
|
func chain(rs []Reader) Reader { return multi(rs) }
|
|
|
|
func (m multi) Name() string {
|
|
names := make([]string, len(m))
|
|
for i, r := range m {
|
|
names[i] = r.Name()
|
|
}
|
|
return strings.Join(names, ",")
|
|
}
|
|
|
|
func (m multi) Read(ctx context.Context, target string) (*Page, error) {
|
|
var errs []error
|
|
for _, r := range m {
|
|
if ctx.Err() != nil {
|
|
break
|
|
}
|
|
p, err := r.Read(ctx, target)
|
|
if err == nil {
|
|
return p, nil
|
|
}
|
|
errs = append(errs, err)
|
|
}
|
|
if len(errs) == 0 {
|
|
return nil, ctx.Err()
|
|
}
|
|
return nil, errors.Join(errs...)
|
|
}
|
|
|
|
// do sends req and returns the body of a 2xx answer.
|
|
func do(c *http.Client, req *http.Request, name string) ([]byte, error) {
|
|
resp, err := c.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s: the reader did not answer", name)
|
|
}
|
|
defer resp.Body.Close()
|
|
body, err := io.ReadAll(io.LimitReader(resp.Body, maxResponse))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s: the reader stopped mid-answer", name)
|
|
}
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
return nil, fmt.Errorf("%s: the reader answered %d", name, resp.StatusCode)
|
|
}
|
|
return body, nil
|
|
}
|
|
|
|
func bearer(req *http.Request, key string) {
|
|
if key != "" {
|
|
req.Header.Set("Authorization", "Bearer "+key)
|
|
}
|
|
}
|
|
|
|
// landed checks where a reader ended up and returns the page.
|
|
func landed(ctx context.Context, name, html, final, target string) (*Page, error) {
|
|
if strings.TrimSpace(html) == "" {
|
|
return nil, fmt.Errorf("%s: the reader returned an empty page", name)
|
|
}
|
|
if final == "" {
|
|
final = target
|
|
}
|
|
if err := publicURL(ctx, final); err != nil {
|
|
return nil, err
|
|
}
|
|
return &Page{HTML: html, URL: final, Via: name}, nil
|
|
}
|
|
|
|
// lookupIP resolves a host for publicURL; tests replace it.
|
|
var lookupIP = net.DefaultResolver.LookupIPAddr
|
|
|
|
// publicURL refuses a final address a caller could not have fetched itself.
|
|
func publicURL(ctx context.Context, raw string) error {
|
|
u, err := url.Parse(raw)
|
|
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" {
|
|
return ErrNotPublic
|
|
}
|
|
if config.UnsafeWebhookURLsAllowed() {
|
|
return nil
|
|
}
|
|
host := u.Hostname()
|
|
if ip := net.ParseIP(host); ip != nil {
|
|
if safehttp.IsBlockedIP(ip) {
|
|
return ErrNotPublic
|
|
}
|
|
return nil
|
|
}
|
|
if h := strings.ToLower(host); h == "localhost" || strings.HasSuffix(h, ".localhost") || strings.HasSuffix(h, ".internal") {
|
|
return ErrNotPublic
|
|
}
|
|
lctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
|
defer cancel()
|
|
addrs, err := lookupIP(lctx, host)
|
|
if err != nil || len(addrs) == 0 {
|
|
return ErrNotPublic
|
|
}
|
|
for _, a := range addrs {
|
|
if safehttp.IsBlockedIP(a.IP) {
|
|
return ErrNotPublic
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// jina reads through Jina Reader (r.jina.ai or a self-hosted copy).
|
|
type jina struct {
|
|
base string
|
|
key string
|
|
http *http.Client
|
|
}
|
|
|
|
func (j *jina) Name() string { return providerJina }
|
|
|
|
func (j *jina) Read(ctx context.Context, target string) (*Page, error) {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, j.base+"/"+target, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.Header.Set("Accept", "application/json")
|
|
req.Header.Set("X-Return-Format", "html")
|
|
bearer(req, j.key)
|
|
body, err := do(j.http, req, j.Name())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var out struct {
|
|
Code int `json:"code"`
|
|
Data struct {
|
|
HTML string `json:"html"`
|
|
Content string `json:"content"`
|
|
URL string `json:"url"`
|
|
FinalURL string `json:"finalUrl"`
|
|
} `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(body, &out); err != nil {
|
|
return nil, errors.New("jina: the reader's answer could not be read")
|
|
}
|
|
if out.Code != 0 && out.Code != http.StatusOK {
|
|
return nil, fmt.Errorf("jina: the reader answered %d", out.Code)
|
|
}
|
|
html := firstNonEmpty(out.Data.HTML, out.Data.Content)
|
|
return landed(ctx, j.Name(), html, firstNonEmpty(out.Data.FinalURL, out.Data.URL), target)
|
|
}
|
|
|
|
// crawl4ai reads through a Crawl4AI server's /crawl endpoint.
|
|
type crawl4ai struct {
|
|
base string
|
|
key string
|
|
http *http.Client
|
|
}
|
|
|
|
func (c *crawl4ai) Name() string { return "crawl4ai" }
|
|
|
|
func (c *crawl4ai) Read(ctx context.Context, target string) (*Page, error) {
|
|
payload, _ := json.Marshal(map[string]any{"urls": []string{target}, "browser_config": map[string]any{}, "crawler_config": map[string]any{}})
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.base+"/crawl", strings.NewReader(string(payload)))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.Header.Set("Accept", "application/json")
|
|
bearer(req, c.key)
|
|
body, err := do(c.http, req, c.Name())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var out struct {
|
|
Results []struct {
|
|
Success bool `json:"success"`
|
|
HTML string `json:"html"`
|
|
CleanedHTML string `json:"cleaned_html"`
|
|
URL string `json:"url"`
|
|
Redirected string `json:"redirected_url"`
|
|
StatusCode int `json:"status_code"`
|
|
} `json:"results"`
|
|
}
|
|
if err := json.Unmarshal(body, &out); err != nil || len(out.Results) == 0 {
|
|
return nil, errors.New("crawl4ai: the reader's answer could not be read")
|
|
}
|
|
r := out.Results[0]
|
|
if !r.Success || r.StatusCode >= 400 {
|
|
return nil, fmt.Errorf("crawl4ai: the site answered the reader %d", r.StatusCode)
|
|
}
|
|
return landed(ctx, c.Name(), firstNonEmpty(r.HTML, r.CleanedHTML), firstNonEmpty(r.Redirected, r.URL), target)
|
|
}
|
|
|
|
// ddgs reads through a DDGS API server's /extract endpoint.
|
|
type ddgs struct {
|
|
base string
|
|
http *http.Client
|
|
}
|
|
|
|
func (d *ddgs) Name() string { return "ddgs" }
|
|
|
|
func (d *ddgs) Read(ctx context.Context, target string) (*Page, error) {
|
|
q := url.Values{"url": {target}, "fmt": {"text"}}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, d.base+"/extract?"+q.Encode(), nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.Header.Set("Accept", "application/json")
|
|
body, err := do(d.http, req, d.Name())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var out struct {
|
|
URL string `json:"url"`
|
|
Content string `json:"content"`
|
|
}
|
|
if err := json.Unmarshal(body, &out); err != nil {
|
|
return nil, errors.New("ddgs: the reader's answer could not be read")
|
|
}
|
|
return landed(ctx, d.Name(), out.Content, out.URL, target)
|
|
}
|
|
|
|
// custom reads through any endpoint built from a template holding {url} (the
|
|
// page address as is) or {url_encoded} (query-escaped). It may answer with the
|
|
// HTML itself or JSON carrying it.
|
|
type custom struct {
|
|
template string
|
|
key string
|
|
http *http.Client
|
|
}
|
|
|
|
func (c *custom) Name() string { return "custom" }
|
|
|
|
func (c *custom) Read(ctx context.Context, target string) (*Page, error) {
|
|
endpoint := strings.NewReplacer("{url_encoded}", url.QueryEscape(target), "{url}", target).Replace(c.template)
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.Header.Set("Accept", "text/html,application/json;q=0.9,*/*;q=0.5")
|
|
bearer(req, c.key)
|
|
body, err := do(c.http, req, c.Name())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
html, final := string(body), ""
|
|
if trimmed := strings.TrimSpace(html); strings.HasPrefix(trimmed, "{") {
|
|
html, final = fromJSON(body)
|
|
}
|
|
return landed(ctx, c.Name(), html, final, target)
|
|
}
|
|
|
|
// fromJSON pulls a page out of the common reader answer shapes.
|
|
func fromJSON(body []byte) (string, string) {
|
|
var out struct {
|
|
HTML string `json:"html"`
|
|
Content string `json:"content"`
|
|
URL string `json:"url"`
|
|
FinalURL string `json:"final_url"`
|
|
Data *struct {
|
|
HTML string `json:"html"`
|
|
RawHTML string `json:"rawHtml"`
|
|
Content string `json:"content"`
|
|
URL string `json:"url"`
|
|
FinalURL string `json:"finalUrl"`
|
|
} `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(body, &out); err != nil {
|
|
return "", ""
|
|
}
|
|
if out.Data != nil {
|
|
if h := firstNonEmpty(out.Data.RawHTML, out.Data.HTML, out.Data.Content); h != "" {
|
|
return h, firstNonEmpty(out.Data.FinalURL, out.Data.URL)
|
|
}
|
|
}
|
|
return firstNonEmpty(out.HTML, out.Content), firstNonEmpty(out.FinalURL, out.URL)
|
|
}
|
|
|
|
func firstNonEmpty(vs ...string) string {
|
|
for _, v := range vs {
|
|
if strings.TrimSpace(v) != "" {
|
|
return v
|
|
}
|
|
}
|
|
return ""
|
|
}
|