fix: verify TLS for external resource requests

agent/utils/req_helper/external_http.go

@@ -0,0 +1,39 @@
+package req_helper
+
+import (
+	"crypto/tls"
+	"errors"
+	"net/http"
+	"time"
+)
+
+func newExternalHTTPClient(transport *http.Transport, timeout time.Duration) *http.Client {
+	if transport == nil {
+		transport = http.DefaultTransport.(*http.Transport)
+	}
+	verified := transport.Clone()
+	if verified.TLSClientConfig == nil {
+		verified.TLSClientConfig = &tls.Config{}
+	} else {
+		verified.TLSClientConfig = verified.TLSClientConfig.Clone()
+	}
+	verified.TLSClientConfig.InsecureSkipVerify = false
+	verified.TLSClientConfig.ServerName = ""
+	verified.DialTLS = nil
+	verified.DialTLSContext = nil
+	return &http.Client{
+		Timeout:       timeout,
+		Transport:     verified,
+		CheckRedirect: checkExternalRedirect,
+	}
+}
+
+func checkExternalRedirect(req *http.Request, via []*http.Request) error {
+	if len(via) >= 10 {
+		return errors.New("stopped after 10 redirects")
+	}
+	if len(via) > 0 && via[len(via)-1].URL.Scheme == "https" && req.URL.Scheme != "https" {
+		return errors.New("refusing HTTPS redirect to an insecure scheme")
+	}
+	return nil
+}

agent/utils/req_helper/request.go

@@ -2,7 +2,6 @@ package req_helper
 
 import (
 	"context"
-	"crypto/tls"
 	"errors"
 	"io"
 	"net"
@@ -21,10 +20,7 @@ import (
 )
 
 func HandleGet(url string) (*http.Response, error) {
-	client := &http.Client{
-		Timeout: time.Second * 300,
-	}
-	client.Transport = loadRequestTransport()
+	client := newExternalHTTPClient(loadRequestTransport(), time.Second*300)
 	defer client.CloseIdleConnections()
 
 	req, err := http.NewRequestWithContext(context.Background(), "GET", url, nil)
@@ -53,8 +49,9 @@ func HandleGet(url string) (*http.Response, error) {
 
 func HandleRequest(url, method string, timeout int) (int, []byte, error) {
 	transport := xpack.MultiNodeProvider.LoadRequestTransport()
-	client := http.Client{Timeout: time.Duration(timeout) * time.Second, Transport: transport}
-	return HandleRequestWithClient(&client, url, method, timeout)
+	client := newExternalHTTPClient(transport, time.Duration(timeout)*time.Second)
+	defer client.CloseIdleConnections()
+	return HandleRequestWithClient(client, url, method, timeout)
 }
 
 func HandleRequestWithClient(client *http.Client, url, method string, timeout int) (int, []byte, error) {
@@ -136,7 +133,8 @@ func RequestFile(url, method string, timeout int) (io.ReadCloser, context.Cancel
 		}
 	}()
 	transport := xpack.MultiNodeProvider.LoadRequestTransport()
-	client := http.Client{Timeout: time.Duration(timeout) * time.Second, Transport: transport}
+	client := newExternalHTTPClient(transport, time.Duration(timeout)*time.Second)
+	defer client.CloseIdleConnections()
 	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)
 	request, err := http.NewRequestWithContext(ctx, method, url, nil)
 	if err != nil {
@@ -156,7 +154,6 @@ func RequestFile(url, method string, timeout int) (io.ReadCloser, context.Cancel
 
 func loadRequestTransport() *http.Transport {
 	return &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 		DialContext: (&net.Dialer{
 			Timeout:   60 * time.Second,
 			KeepAlive: 60 * time.Second,

core/utils/req_helper/external_http.go

@@ -0,0 +1,39 @@
+package req_helper
+
+import (
+	"crypto/tls"
+	"errors"
+	"net/http"
+	"time"
+)
+
+func newExternalHTTPClient(transport *http.Transport, timeout time.Duration) *http.Client {
+	if transport == nil {
+		transport = http.DefaultTransport.(*http.Transport)
+	}
+	verified := transport.Clone()
+	if verified.TLSClientConfig == nil {
+		verified.TLSClientConfig = &tls.Config{}
+	} else {
+		verified.TLSClientConfig = verified.TLSClientConfig.Clone()
+	}
+	verified.TLSClientConfig.InsecureSkipVerify = false
+	verified.TLSClientConfig.ServerName = ""
+	verified.DialTLS = nil
+	verified.DialTLSContext = nil
+	return &http.Client{
+		Timeout:       timeout,
+		Transport:     verified,
+		CheckRedirect: checkExternalRedirect,
+	}
+}
+
+func checkExternalRedirect(req *http.Request, via []*http.Request) error {
+	if len(via) >= 10 {
+		return errors.New("stopped after 10 redirects")
+	}
+	if len(via) > 0 && via[len(via)-1].URL.Scheme == "https" && req.URL.Scheme != "https" {
+		return errors.New("refusing HTTPS redirect to an insecure scheme")
+	}
+	return nil
+}

core/utils/req_helper/requset.go

@@ -2,7 +2,6 @@ package req_helper
 
 import (
 	"context"
-	"crypto/tls"
 	"errors"
 	"io"
 	"net"
@@ -17,7 +16,6 @@ import (
 
 func HandleRequest(url, method string, timeout int) (int, []byte, error) {
 	transport := &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 		DialContext: (&net.Dialer{
 			Timeout:   60 * time.Second,
 			KeepAlive: 60 * time.Second,
@@ -42,7 +40,8 @@ func handleRequestWithTransport(url, method string, transport *http.Transport, t
 		}
 	}()
 
-	client := http.Client{Timeout: time.Duration(timeout) * time.Second, Transport: transport}
+	client := newExternalHTTPClient(transport, time.Duration(timeout)*time.Second)
+	defer client.CloseIdleConnections()
 	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)
 	defer cancel()
 	request, err := http.NewRequestWithContext(ctx, method, url, nil)
@@ -65,7 +64,6 @@ func handleRequestWithTransport(url, method string, transport *http.Transport, t
 
 func HandleGet(url string) (*http.Response, error) {
 	transport := &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 		DialContext: (&net.Dialer{
 			Timeout:   60 * time.Second,
 			KeepAlive: 60 * time.Second,
@@ -83,10 +81,7 @@ func HandleGetWithProxy(url string) (*http.Response, error) {
 }
 
 func handleGetWithTransport(url string, transport *http.Transport) (*http.Response, error) {
-	client := &http.Client{
-		Timeout: time.Second * 300,
-	}
-	client.Transport = transport
+	client := newExternalHTTPClient(transport, time.Second*300)
 	defer client.CloseIdleConnections()
 
 	req, err := http.NewRequestWithContext(context.Background(), "GET", url, nil)