feat: add retry mechanism for SSL certificate application (#11710)

agent/utils/ssl/acme.go

@@ -8,6 +8,7 @@ import (
 	"crypto/x509"
 	"encoding/json"
 	"encoding/pem"
+	"errors"
 	"fmt"
 	"github.com/1Panel-dev/1Panel/agent/app/dto"
 	"github.com/1Panel-dev/1Panel/agent/buserr"
@@ -358,3 +359,26 @@ func getWebsiteSSLDomains(websiteSSL *model.WebsiteSSL) []string {
 	}
 	return domains
 }
+
+const (
+	maxRetryAttempts = 3
+	retryDelayOn503  = 30 * time.Second
+)
+
+// isHTTP503Error checks if an error is an HTTP 503 Service Unavailable error
+func isHTTP503Error(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	// Check for golang.org/x/crypto/acme.Error (used in manual_client.go)
+	var acmeErr *acme.Error
+	if errors.As(err, &acmeErr) {
+		return acmeErr.StatusCode == http.StatusServiceUnavailable
+	}
+
+	// Check error message for 503 (fallback for lego library errors)
+	errMsg := err.Error()
+	return strings.Contains(errMsg, "503") ||
+		strings.Contains(errMsg, "Service busy")
+}

agent/utils/ssl/client.go

@@ -5,15 +5,18 @@ import (
 	"crypto/rand"
 	"crypto/x509"
 	"crypto/x509/pkix"
+	"net"
+	"os"
+	"time"
+
 	"github.com/1Panel-dev/1Panel/agent/app/dto"
 	"github.com/1Panel-dev/1Panel/agent/app/model"
+	"github.com/1Panel-dev/1Panel/agent/global"
 	"github.com/go-acme/lego/v4/certificate"
 	"github.com/go-acme/lego/v4/challenge/dns01"
 	"github.com/go-acme/lego/v4/lego"
 	"github.com/go-acme/lego/v4/providers/http/webroot"
 	"github.com/pkg/errors"
-	"net"
-	"os"
 )
 
 type AcmeClientOption func(*AcmeClientOptions)
@@ -88,12 +91,27 @@ func (c *AcmeClient) ObtainSSL(domains []string, privateKey crypto.PrivateKey) (
 		PrivateKey: privateKey,
 	}
 
-	certificates, err := c.Client.Certificate.Obtain(request)
-	if err != nil {
+	var certificates *certificate.Resource
+	var err error
+
+	for attempt := 1; attempt <= maxRetryAttempts; attempt++ {
+		certificates, err = c.Client.Certificate.Obtain(request)
+		if err == nil {
+			return *certificates, nil
+		}
+
+		if isHTTP503Error(err) && attempt < maxRetryAttempts {
+			global.LOG.Warnf("ACME server returned 503, retrying in %v (attempt %d/%d)",
+				retryDelayOn503, attempt, maxRetryAttempts)
+			time.Sleep(retryDelayOn503)
+			continue
+		}
+
+		// Non-503 error or final attempt, return error
 		return certificate.Resource{}, err
 	}
 
-	return *certificates, nil
+	return certificate.Resource{}, err
 }
 
 func (c *AcmeClient) ObtainIPSSL(ipAddress string, privKey crypto.PrivateKey) (certificate.Resource, error) {
@@ -123,12 +141,25 @@ func (c *AcmeClient) ObtainIPSSL(ipAddress string, privKey crypto.PrivateKey) (c
 		Profile:    "shortlived",
 		Bundle:     true,
 	}
-	certificates, err := c.Client.Certificate.ObtainForCSR(req)
-	if err != nil {
+
+	var certificates *certificate.Resource
+	for attempt := 1; attempt <= maxRetryAttempts; attempt++ {
+		certificates, err = c.Client.Certificate.ObtainForCSR(req)
+		if err == nil {
+			return *certificates, nil
+		}
+
+		if isHTTP503Error(err) && attempt < maxRetryAttempts {
+			global.LOG.Warnf("ACME server returned 503 for IP SSL, retrying in %v (attempt %d/%d)",
+				retryDelayOn503, attempt, maxRetryAttempts)
+			time.Sleep(retryDelayOn503)
+			continue
+		}
+
 		return certificate.Resource{}, err
 	}
 
-	return *certificates, nil
+	return certificate.Resource{}, err
 }
 
 func (c *AcmeClient) RevokeSSL(pemSSL []byte) error {