fix: verify TLS for external resource requests

agent/go.mod

@@ -56,7 +56,7 @@ require (
 	github.com/upyun/go-sdk v2.1.0+incompatible
 	go.mongodb.org/mongo-driver/v2 v2.9.2
 	golang.org/x/crypto v0.58.0
-	golang.org/x/net v0.62.0
+	golang.org/x/net v0.60.0
 	golang.org/x/sync v0.24.0
 	golang.org/x/sys v0.49.0
 	golang.org/x/text v0.43.0

agent/go.sum

@@ -1084,8 +1084,6 @@ golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
 golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
 golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU=
 golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
-golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY=
-golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

agent/utils/req_helper/external_http.go

@@ -0,0 +1,39 @@
+package req_helper
+
+import (
+	"crypto/tls"
+	"errors"
+	"net/http"
+	"time"
+)
+
+func newExternalHTTPClient(transport *http.Transport, timeout time.Duration) *http.Client {
+	if transport == nil {
+		transport = http.DefaultTransport.(*http.Transport)
+	}
+	verified := transport.Clone()
+	if verified.TLSClientConfig == nil {
+		verified.TLSClientConfig = &tls.Config{}
+	} else {
+		verified.TLSClientConfig = verified.TLSClientConfig.Clone()
+	}
+	verified.TLSClientConfig.InsecureSkipVerify = false
+	verified.TLSClientConfig.ServerName = ""
+	verified.DialTLS = nil
+	verified.DialTLSContext = nil
+	return &http.Client{
+		Timeout:       timeout,
+		Transport:     verified,
+		CheckRedirect: checkExternalRedirect,
+	}
+}
+
+func checkExternalRedirect(req *http.Request, via []*http.Request) error {
+	if len(via) >= 10 {
+		return errors.New("stopped after 10 redirects")
+	}
+	if len(via) > 0 && via[len(via)-1].URL.Scheme == "https" && req.URL.Scheme != "https" {
+		return errors.New("refusing HTTPS redirect to an insecure scheme")
+	}
+	return nil
+}

agent/utils/req_helper/request.go

@@ -2,7 +2,6 @@ package req_helper
 
 import (
 	"context"
-	"crypto/tls"
 	"errors"
 	"io"
 	"net"
@@ -21,10 +20,7 @@ import (
 )
 
 func HandleGet(url string) (*http.Response, error) {
-	client := &http.Client{
-		Timeout: time.Second * 300,
-	}
-	client.Transport = loadRequestTransport()
+	client := newExternalHTTPClient(loadRequestTransport(), time.Second*300)
 	defer client.CloseIdleConnections()
 
 	req, err := http.NewRequestWithContext(context.Background(), "GET", url, nil)
@@ -53,8 +49,9 @@ func HandleGet(url string) (*http.Response, error) {
 
 func HandleRequest(url, method string, timeout int) (int, []byte, error) {
 	transport := xpack.MultiNodeProvider.LoadRequestTransport()
-	client := http.Client{Timeout: time.Duration(timeout) * time.Second, Transport: transport}
-	return HandleRequestWithClient(&client, url, method, timeout)
+	client := newExternalHTTPClient(transport, time.Duration(timeout)*time.Second)
+	defer client.CloseIdleConnections()
+	return HandleRequestWithClient(client, url, method, timeout)
 }
 
 func HandleRequestWithClient(client *http.Client, url, method string, timeout int) (int, []byte, error) {
@@ -136,7 +133,8 @@ func RequestFile(url, method string, timeout int) (io.ReadCloser, context.Cancel
 		}
 	}()
 	transport := xpack.MultiNodeProvider.LoadRequestTransport()
-	client := http.Client{Timeout: time.Duration(timeout) * time.Second, Transport: transport}
+	client := newExternalHTTPClient(transport, time.Duration(timeout)*time.Second)
+	defer client.CloseIdleConnections()
 	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)
 	request, err := http.NewRequestWithContext(ctx, method, url, nil)
 	if err != nil {
@@ -156,7 +154,6 @@ func RequestFile(url, method string, timeout int) (io.ReadCloser, context.Cancel
 
 func loadRequestTransport() *http.Transport {
 	return &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 		DialContext: (&net.Dialer{
 			Timeout:   60 * time.Second,
 			KeepAlive: 60 * time.Second,

core/go.mod

@@ -44,7 +44,7 @@ require (
 	github.com/xlzd/gotp v0.1.0
 	golang.org/x/crypto v0.58.0
 	golang.org/x/image v0.45.0
-	golang.org/x/net v0.62.0
+	golang.org/x/net v0.60.0
 	golang.org/x/oauth2 v0.36.0
 	golang.org/x/sys v0.49.0
 	golang.org/x/term v0.47.0

core/go.sum

@@ -339,8 +339,6 @@ golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
 golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
 golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU=
 golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
-golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY=
-golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
 golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
 golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
 golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=

core/utils/req_helper/external_http.go

@@ -0,0 +1,39 @@
+package req_helper
+
+import (
+	"crypto/tls"
+	"errors"
+	"net/http"
+	"time"
+)
+
+func newExternalHTTPClient(transport *http.Transport, timeout time.Duration) *http.Client {
+	if transport == nil {
+		transport = http.DefaultTransport.(*http.Transport)
+	}
+	verified := transport.Clone()
+	if verified.TLSClientConfig == nil {
+		verified.TLSClientConfig = &tls.Config{}
+	} else {
+		verified.TLSClientConfig = verified.TLSClientConfig.Clone()
+	}
+	verified.TLSClientConfig.InsecureSkipVerify = false
+	verified.TLSClientConfig.ServerName = ""
+	verified.DialTLS = nil
+	verified.DialTLSContext = nil
+	return &http.Client{
+		Timeout:       timeout,
+		Transport:     verified,
+		CheckRedirect: checkExternalRedirect,
+	}
+}
+
+func checkExternalRedirect(req *http.Request, via []*http.Request) error {
+	if len(via) >= 10 {
+		return errors.New("stopped after 10 redirects")
+	}
+	if len(via) > 0 && via[len(via)-1].URL.Scheme == "https" && req.URL.Scheme != "https" {
+		return errors.New("refusing HTTPS redirect to an insecure scheme")
+	}
+	return nil
+}

core/utils/req_helper/requset.go

@@ -2,7 +2,6 @@ package req_helper
 
 import (
 	"context"
-	"crypto/tls"
 	"errors"
 	"io"
 	"net"
@@ -17,7 +16,6 @@ import (
 
 func HandleRequest(url, method string, timeout int) (int, []byte, error) {
 	transport := &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 		DialContext: (&net.Dialer{
 			Timeout:   60 * time.Second,
 			KeepAlive: 60 * time.Second,
@@ -42,7 +40,8 @@ func handleRequestWithTransport(url, method string, transport *http.Transport, t
 		}
 	}()
 
-	client := http.Client{Timeout: time.Duration(timeout) * time.Second, Transport: transport}
+	client := newExternalHTTPClient(transport, time.Duration(timeout)*time.Second)
+	defer client.CloseIdleConnections()
 	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)
 	defer cancel()
 	request, err := http.NewRequestWithContext(ctx, method, url, nil)
@@ -65,7 +64,6 @@ func handleRequestWithTransport(url, method string, transport *http.Transport, t
 
 func HandleGet(url string) (*http.Response, error) {
 	transport := &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 		DialContext: (&net.Dialer{
 			Timeout:   60 * time.Second,
 			KeepAlive: 60 * time.Second,
@@ -83,10 +81,7 @@ func HandleGetWithProxy(url string) (*http.Response, error) {
 }
 
 func handleGetWithTransport(url string, transport *http.Transport) (*http.Response, error) {
-	client := &http.Client{
-		Timeout: time.Second * 300,
-	}
-	client.Transport = transport
+	client := newExternalHTTPClient(transport, time.Second*300)
 	defer client.CloseIdleConnections()
 
 	req, err := http.NewRequestWithContext(context.Background(), "GET", url, nil)