diff --git a/agent/app/api/v2/app.go b/agent/app/api/v2/app.go index cc2aa54fc..4ce9c8214 100644 --- a/agent/app/api/v2/app.go +++ b/agent/app/api/v2/app.go @@ -207,6 +207,7 @@ func (b *BaseApi) GetAppListUpdate(c *gin.Context) { // @Accept json // @Param key path string true "app key" // @Success 200 {file} file "app icon" +// @Success 302 "redirect to the official icon URL" // @Security ApiKeyAuth // @Security Timestamp // @Router /apps/icon/:key [get] @@ -216,12 +217,18 @@ func (b *BaseApi) GetAppIcon(c *gin.Context) { helper.BadRequest(c, err) return } - iconBytes, filename, etag, err := appService.GetAppIcon(appKey) + iconBytes, source, etag, err := appService.GetAppIcon(appKey) if err != nil { helper.InternalServer(c, err) return } + if appicon.IsRemoteURL(source) { + c.Header("Cache-Control", "no-cache") + c.Redirect(http.StatusFound, source) + return + } + if len(iconBytes) == 0 { c.Status(http.StatusNoContent) return @@ -229,7 +236,7 @@ func (b *BaseApi) GetAppIcon(c *gin.Context) { c.Header("Cache-Control", "public, max-age=2592000") - if etag != "" && filename != "" { + if etag != "" && source != "" { c.Header("ETag", etag) if c.GetHeader("If-None-Match") == etag { c.Status(http.StatusNotModified) diff --git a/agent/app/api/v2/app_icon_test.go b/agent/app/api/v2/app_icon_test.go new file mode 100644 index 000000000..080feca1d --- /dev/null +++ b/agent/app/api/v2/app_icon_test.go @@ -0,0 +1,109 @@ +package v2 + +import ( + "encoding/base64" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync/atomic" + "testing" + + "github.com/1Panel-dev/1Panel/agent/app/model" + "github.com/1Panel-dev/1Panel/agent/constant" + "github.com/1Panel-dev/1Panel/agent/global" + "github.com/1Panel-dev/1Panel/agent/utils/appicon" + "github.com/gin-gonic/gin" + "github.com/glebarez/sqlite" + "github.com/sirupsen/logrus" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +func TestAppIconEndpointPreservesRemoteAndHistoricalSources(t *testing.T) { + db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "icons.db")), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&model.App{}, &model.AppDetail{}, &model.AppTag{}); err != nil { + t.Fatal(err) + } + oldDB, oldDir, oldLog := global.DB, global.Dir, global.LOG + global.DB = db + global.Dir.IconCacheDir = t.TempDir() + global.LOG = logrus.New() + global.LOG.SetOutput(io.Discard) + t.Cleanup(func() { + global.DB, global.Dir, global.LOG = oldDB, oldDir, oldLog + sqlDB, _ := db.DB() + _ = sqlDB.Close() + }) + + var imageRequests atomic.Int32 + cdn := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + imageRequests.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + t.Cleanup(cdn.Close) + remoteURL := cdn.URL + "/icon.png?token=a%2Bb&v=42#image" + image := "\x89PNG\r\n\x1a\noriginal-icon" + fileName := "app_php.png" + if err := os.WriteFile(filepath.Join(global.Dir.IconCacheDir, fileName), []byte(image), 0644); err != nil { + t.Fatal(err) + } + + app := model.App{Key: "php", Resource: constant.AppResourceRemote} + if err := db.Create(&app).Error; err != nil { + t.Fatal(err) + } + router := gin.New() + router.GET("/api/v2/apps/icon/:key", (&BaseApi{}).GetAppIcon) + for _, tc := range []struct { + name string + icon string + ifNoneMatch string + status int + body string + etag string + }{ + {"remote URL", remoteURL, "", http.StatusFound, "", ""}, + {"remote URL ignores legacy etag", remoteURL, `"old"`, http.StatusFound, "", ""}, + {"historical file", appicon.BuildIconField(fileName, `"42"`), "", http.StatusOK, image, `"42"`}, + {"historical file cached", appicon.BuildIconField(fileName, `"42"`), `"42"`, http.StatusNotModified, "", `"42"`}, + {"historical file changed", appicon.BuildIconField(fileName, `"42"`), `"old"`, http.StatusOK, image, `"42"`}, + {"historical base64", base64.StdEncoding.EncodeToString([]byte(image)), "", http.StatusOK, image, ""}, + {"missing file", "app_missing.png?etag=old", "", http.StatusNoContent, "", ""}, + {"invalid base64", "not-an-icon", "", http.StatusNoContent, "", ""}, + {"invalid URL", "javascript:alert(1)", "", http.StatusNoContent, "", ""}, + {"empty icon", "", "", http.StatusNoContent, "", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + if err := db.Model(&app).UpdateColumn("icon", tc.icon).Error; err != nil { + t.Fatal(err) + } + req := httptest.NewRequest(http.MethodGet, "/api/v2/apps/icon/php?operateNode=child", nil) + req.Header.Set("If-None-Match", tc.ifNoneMatch) + res := httptest.NewRecorder() + router.ServeHTTP(res, req) + if res.Code != tc.status { + t.Fatalf("status = %d, want %d; body = %s", res.Code, tc.status, res.Body.String()) + } + if res.Header().Get("ETag") != tc.etag { + t.Fatalf("etag = %q, want %q", res.Header().Get("ETag"), tc.etag) + } + if tc.status == http.StatusFound { + if res.Header().Get("Location") != remoteURL || res.Header().Get("Cache-Control") != "no-cache" { + t.Fatalf("redirect URL or cache policy was changed: %v", res.Header()) + } + } else if res.Body.String() != tc.body { + t.Fatalf("body = %q, want %q", res.Body.String(), tc.body) + } + if imageRequests.Load() != 0 { + t.Fatalf("icon endpoint fetched the image %d times", imageRequests.Load()) + } + }) + } +} diff --git a/agent/app/api/v2/app_install.go b/agent/app/api/v2/app_install.go index 1ced903ce..dbbf7e895 100644 --- a/agent/app/api/v2/app_install.go +++ b/agent/app/api/v2/app_install.go @@ -203,6 +203,7 @@ func (b *BaseApi) GetServices(c *gin.Context) { // @Tags App // @Summary Search app update version by install id // @Accept json +// @Param request body request.AppUpdateVersion true "request" // @Success 200 {array} dto.AppVersion // @Security ApiKeyAuth // @Security Timestamp diff --git a/agent/app/dto/request/app.go b/agent/app/dto/request/app.go index e69c00d8d..59325baf3 100644 --- a/agent/app/dto/request/app.go +++ b/agent/app/dto/request/app.go @@ -156,6 +156,7 @@ type PortUpdate struct { type AppUpdateVersion struct { AppInstallID uint `json:"appInstallID" validate:"required"` UpdateVersion string `json:"updateVersion"` + LoadCompose *bool `json:"loadCompose,omitempty"` } type AppInstallSortItem struct { diff --git a/agent/app/dto/response/app.go b/agent/app/dto/response/app.go index 7974e7681..235feab99 100644 --- a/agent/app/dto/response/app.go +++ b/agent/app/dto/response/app.go @@ -31,6 +31,7 @@ type AppDTO struct { type AppItem struct { Name string `json:"name"` Key string `json:"key"` + Icon string `json:"icon,omitempty"` ID uint `json:"id"` Description string `json:"description"` Status string `json:"status"` @@ -194,5 +195,6 @@ type AppConfig struct { ModelAccountBaseURL string `json:"modelAccountBaseURL"` Params []AppParam `json:"params"` RawCompose string `json:"rawCompose"` + IsEdit bool `json:"isEdit"` request.AppContainerConfig } diff --git a/agent/app/repo/app.go b/agent/app/repo/app.go index cc487091b..e3dd69f48 100644 --- a/agent/app/repo/app.go +++ b/agent/app/repo/app.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/1Panel-dev/1Panel/agent/app/model" + "github.com/1Panel-dev/1Panel/agent/constant" "gorm.io/gorm" "gorm.io/gorm/clause" ) @@ -30,6 +31,7 @@ type IAppRepo interface { BatchCreate(ctx context.Context, apps []model.App) error Create(ctx context.Context, app *model.App) error Save(ctx context.Context, app *model.App) error + UpdateIcon(id uint, icon string) error BatchDelete(ctx context.Context, apps []model.App) error DeleteByIDs(ctx context.Context, ids []uint) error DeleteBy(opts ...DBOption) error @@ -41,6 +43,10 @@ func NewIAppRepo() IAppRepo { return &AppRepo{} } +func (a AppRepo) UpdateIcon(id uint, icon string) error { + return getDb(WithByID(id), a.WithResource(constant.AppResourceRemote)).Model(&model.App{}).UpdateColumn("icon", icon).Error +} + func (a AppRepo) WithByLikeName(name string) DBOption { return func(g *gorm.DB) *gorm.DB { if len(name) == 0 { diff --git a/agent/app/repo/app_detail.go b/agent/app/repo/app_detail.go index 84e54776c..6931b6637 100644 --- a/agent/app/repo/app_detail.go +++ b/agent/app/repo/app_detail.go @@ -17,6 +17,8 @@ type IAppDetailRepo interface { WithIgnored() DBOption GetFirst(opts ...DBOption) (model.AppDetail, error) Update(ctx context.Context, detail model.AppDetail) error + CacheCompose(detail model.AppDetail, compose string) (bool, error) + MarkDownloaded(detail model.AppDetail) (bool, error) BatchCreate(ctx context.Context, details []model.AppDetail) error DeleteByAppIds(ctx context.Context, appIds []uint) error DeleteByIDs(ctx context.Context, appIds []uint) error @@ -57,6 +59,20 @@ func (a AppDetailRepo) Update(ctx context.Context, detail model.AppDetail) error return getTx(ctx).Save(&detail).Error } +func (a AppDetailRepo) CacheCompose(detail model.AppDetail, compose string) (bool, error) { + result := getDb().Model(&model.AppDetail{}). + Where("id = ? AND last_modified = ? AND params = ? AND download_url = ?", detail.ID, detail.LastModified, detail.Params, detail.DownloadUrl). + UpdateColumn("docker_compose", compose) + return result.RowsAffected > 0, result.Error +} + +func (a AppDetailRepo) MarkDownloaded(detail model.AppDetail) (bool, error) { + result := getDb().Model(&model.AppDetail{}). + Where("id = ? AND last_modified = ? AND params = ? AND download_url = ?", detail.ID, detail.LastModified, detail.Params, detail.DownloadUrl). + UpdateColumn("update", false) + return result.RowsAffected > 0, result.Error +} + func (a AppDetailRepo) BatchCreate(ctx context.Context, details []model.AppDetail) error { return getTx(ctx).Model(&model.AppDetail{}).Create(&details).Error } diff --git a/agent/app/service/app.go b/agent/app/service/app.go index 9dde9b482..920006474 100644 --- a/agent/app/service/app.go +++ b/agent/app/service/app.go @@ -135,6 +135,7 @@ func (a AppService) PageApp(ctx *gin.Context, req request.AppSearch) (*response. ID: ap.ID, Name: ap.Name, Key: ap.Key, + Icon: appicon.RemoteURL(ap.Icon), Limit: ap.Limit, GpuSupport: ap.GpuSupport, Recommend: ap.Recommend, @@ -262,10 +263,20 @@ func (a AppService) GetAppDetail(appID uint, version, appType string) (response. if err = downloadApp(app, detail, nil, nil); err != nil && !fileOp.Stat(versionPath) { return appDetailDTO, err } - if err == nil && app.Resource == constant.AppResourceRemote { + packageReady := err == nil + if packageReady && app.Resource == constant.AppResourceRemote { detail.Update = false appDetailDTO.Update = false } + if packageReady && strings.TrimSpace(detail.DockerCompose) == "" { + compose, err := fileOp.GetContent(filepath.Join(versionPath, "docker-compose.yml")) + if err != nil { + return appDetailDTO, err + } + if err := cacheAppDetailCompose(&detail, compose); err != nil { + return appDetailDTO, err + } + } switch app.Type { case constant.RuntimePHP: paramMap := make(map[string]interface{}) @@ -296,15 +307,17 @@ func (a AppService) GetAppDetail(appID uint, version, appType string) (response. if err != nil { return appDetailDTO, err } - composeMap := make(map[string]interface{}) + if err := validateAppDetailCompose(compose); err != nil { + return appDetailDTO, err + } + var composeMap struct { + Services map[string]interface{} `yaml:"services"` + } if err := yaml.Unmarshal(compose, &composeMap); err != nil { return appDetailDTO, err } - if service, ok := composeMap["services"]; ok { - servicesMap := service.(map[string]interface{}) - for k := range servicesMap { - appDetailDTO.Image = k - } + for k := range composeMap.Services { + appDetailDTO.Image = k } } } else { @@ -315,29 +328,10 @@ func (a AppService) GetAppDetail(appID uint, version, appType string) (response. appDetailDTO.Params = paramMap } - if appDetailDTO.DockerCompose == "" { - filename := filepath.Base(appDetailDTO.DownloadUrl) - dockerComposeUrl := fmt.Sprintf("%s%s", strings.TrimSuffix(appDetailDTO.DownloadUrl, filename), "docker-compose.yml") - if app.Resource == constant.AppResourceRemote { - dockerComposeUrl, err = currentAppDownloadURL(dockerComposeUrl) - if err != nil { - return appDetailDTO, err - } - } - statusCode, composeRes, err := req_helper.HandleRequest(dockerComposeUrl, http.MethodGet, constant.TimeOut20s) - if statusCode == http.StatusNotFound { - return appDetailDTO, buserr.New("ErrAppVersionUnavailable") - } - if err != nil { - return appDetailDTO, buserr.WithDetail("ErrGetCompose", err.Error(), err) - } - if statusCode > 200 { - return appDetailDTO, buserr.WithDetail("ErrGetCompose", string(composeRes), err) - } - detail.DockerCompose = string(composeRes) - _ = appDetailRepo.Update(context.Background(), detail) - appDetailDTO.DockerCompose = string(composeRes) + if err := ensureAppDetailCompose(app, &detail); err != nil { + return appDetailDTO, err } + appDetailDTO.DockerCompose = detail.DockerCompose appDetailDTO.HostMode = isHostModel(appDetailDTO.DockerCompose) @@ -355,6 +349,13 @@ func (a AppService) GetAppDetailByID(id uint) (*response.AppDetailDTO, error) { if err != nil { return nil, err } + app, err := appRepo.GetFirst(repo.WithByID(appDetail.AppId)) + if err != nil { + return nil, err + } + if err := ensureAppDetailCompose(app, &appDetail); err != nil { + return nil, err + } res.AppDetail = appDetail paramMap := make(map[string]interface{}) if err := json.Unmarshal([]byte(appDetail.Params), ¶mMap); err != nil { @@ -442,31 +443,13 @@ func (a AppService) installWithHooks(req request.AppInstallCreate, executeScript App: app, } composeMap := make(map[string]interface{}) - var composeRes []byte if req.EditCompose { if err = yaml.Unmarshal([]byte(req.DockerCompose), &composeMap); err != nil { return } } else { - if appDetail.DockerCompose == "" { - dockerComposeUrl := fmt.Sprintf("%s/%s/1panel/%s/%s/docker-compose.yml", global.AppRepoURL(), global.CONF.Base.Mode, app.Key, appDetail.Version) - if app.Resource == constant.AppResourceRemote { - dockerComposeUrl, err = currentAppDownloadURL(dockerComposeUrl) - if err != nil { - return - } - } - var statusCode int - statusCode, composeRes, err = req_helper.HandleRequest(dockerComposeUrl, http.MethodGet, constant.TimeOut20s) - if statusCode == http.StatusNotFound { - err = buserr.New("ErrAppVersionUnavailable") - return - } - if err != nil { - return - } - appDetail.DockerCompose = string(composeRes) - _ = appDetailRepo.Update(context.Background(), appDetail) + if err = ensureAppDetailCompose(app, &appDetail); err != nil { + return } if err = yaml.Unmarshal([]byte(appDetail.DockerCompose), &composeMap); err != nil { return @@ -922,22 +905,10 @@ func (a AppService) GetAppUpdate() (*response.AppUpdateRes, error) { res.CanUpdate = true return res, err } - apps, _ := appRepo.GetBy(appRepo.WithResource(constant.AppResourceRemote)) - for _, app := range apps { - if app.Icon == "" { - res.CanUpdate = true - return res, err - } - if appicon.IsIconFile(app.Icon) { - fileName, _ := appicon.ParseIconField(app.Icon) - if fileName == "" || !appicon.IconFileExists(fileName) { - res.CanUpdate = true - return res, err - } - } + list, err := readAppList() + if err != nil || list.LastModified != lastModified { + list, err = getAppList() } - - list, err := getAppList() if err != nil { return res, err } @@ -968,29 +939,26 @@ func getAppFromRepo(downloadPath string) error { } func getAppList() (*dto.AppList, error) { - list := &dto.AppList{} if err := getAppFromRepo(fmt.Sprintf("%s/%s/1panel.json.zip", global.AppRepoURL(), global.CONF.Base.Mode)); err != nil { return nil, err } + return readAppList() +} + +func readAppList() (*dto.AppList, error) { listFile := filepath.Join(global.Dir.ResourceDir, "1panel.json") - file, err := os.Open(listFile) + data, err := os.ReadFile(listFile) if err != nil { return nil, err } - defer file.Close() - if err = json.NewDecoder(file).Decode(list); err != nil { + list := &dto.AppList{} + if err = json.Unmarshal(data, list); err != nil { return nil, err } return list, nil } -var InitTypes = map[string]struct{}{ - "runtime": {}, - "php": {}, - "node": {}, -} - func deleteCustomApp() { installs, err := appInstallRepo.ListBy(context.Background()) if err != nil { @@ -1088,6 +1056,10 @@ func (a AppService) GetAppIcon(key string) ([]byte, string, string, error) { return nil, "", "", err } + if appicon.IsRemoteURL(app.Icon) { + return nil, app.Icon, "", nil + } + if appicon.IsIconFile(app.Icon) { fileName, etag := appicon.ParseIconField(app.Icon) iconBytes, err := appicon.ReadIconFile(fileName) diff --git a/agent/app/service/app_cache_test.go b/agent/app/service/app_cache_test.go new file mode 100644 index 000000000..6d103e9ca --- /dev/null +++ b/agent/app/service/app_cache_test.go @@ -0,0 +1,341 @@ +package service + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "reflect" + "sync/atomic" + "testing" + + "github.com/1Panel-dev/1Panel/agent/app/dto" + "github.com/1Panel-dev/1Panel/agent/app/model" + "github.com/1Panel-dev/1Panel/agent/constant" + "github.com/1Panel-dev/1Panel/agent/global" + "github.com/glebarez/sqlite" + "github.com/sirupsen/logrus" + "gorm.io/gorm" + "gorm.io/gorm/clause" + "gorm.io/gorm/logger" +) + +func appCacheTestDB(t *testing.T) *gorm.DB { + t.Helper() + db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "cache.db")), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&model.App{}, &model.AppDetail{}, &model.AppTag{}, &model.Setting{}); err != nil { + t.Fatal(err) + } + oldDB, oldDir, oldLog := global.DB, global.Dir, global.LOG + global.DB = db + global.Dir = global.SystemDir{AppResourceDir: t.TempDir()} + global.Dir.RemoteAppResourceDir = filepath.Join(global.Dir.AppResourceDir, constant.AppResourceRemote) + global.Dir.LocalAppResourceDir = filepath.Join(global.Dir.AppResourceDir, constant.AppResourceLocal) + global.Dir.CustomAppResourceDir = filepath.Join(global.Dir.AppResourceDir, constant.AppResourceCustom) + global.LOG = logrus.New() + global.LOG.SetOutput(io.Discard) + t.Cleanup(func() { + global.DB, global.Dir, global.LOG = oldDB, oldDir, oldLog + sqlDB, err := db.DB() + if err == nil { + _ = sqlDB.Close() + } + }) + if err := db.Create(&model.Setting{Key: "AppStoreLastModified", Value: "42"}).Error; err != nil { + t.Fatal(err) + } + return db +} + +func appCacheTestArchive(t *testing.T, name, content string) []byte { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0644, Size: int64(len(content))}); err != nil { + t.Fatal(err) + } + if _, err := io.WriteString(tw, content); err != nil { + t.Fatal(err) + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func appCacheTestWrite(t *testing.T, name, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(name), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(name, []byte(content), 0644); err != nil { + t.Fatal(err) + } +} + +func TestAppDownloadURLPreservesSourceQuery(t *testing.T) { + for _, rawURL := range []string{ + "https://apps.1panel.pro/php/8/php-8.tar.gz", + "https://apps.1panel.pro/php/8/php-8.tar.gz?token=a%2Bb®ion=intl&v=old#archive", + } { + t.Run(rawURL, func(t *testing.T) { + got, err := appDownloadURL(rawURL, "42 /next") + if err != nil { + t.Fatal(err) + } + parsed, err := url.Parse(got) + if err != nil { + t.Fatal(err) + } + original, _ := url.Parse(rawURL) + wantQuery := original.Query() + wantQuery.Set("v", "42 /next") + if !reflect.DeepEqual(parsed.Query(), wantQuery) || parsed.Path != original.Path || parsed.Fragment != original.Fragment { + t.Fatalf("download URL lost source fields: %s", got) + } + }) + } + if _, err := appDownloadURL("https://apps.1panel.pro/%zz", "42"); err == nil { + t.Fatal("invalid source URL was accepted") + } +} + +func TestDownloadAppRefreshesAllOfficialCachedPackages(t *testing.T) { + db := appCacheTestDB(t) + app := model.App{Key: "mysql", Name: "MySQL", Type: "website", Resource: constant.AppResourceRemote} + versionDir := filepath.Join(app.GetAppResourcePath(), "8") + dataPath := filepath.Join(versionDir, "data.yml") + stampPath := filepath.Join(versionDir, ".1panel-appstore-version") + appCacheTestWrite(t, dataPath, "stale") + archives := map[string][]byte{ + "42": appCacheTestArchive(t, "mysql/8/data.yml", "fresh-42"), + "43": appCacheTestArchive(t, "mysql/8/data.yml", "fresh-43"), + } + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + if r.URL.Query().Get("token") != "a+b" || r.URL.Query().Get("region") != "intl" { + t.Errorf("source query was lost: %s", r.URL.RawQuery) + } + version := r.URL.Query().Get("v") + if version != "42" && version != "43" { + t.Errorf("unexpected app store version: %q", version) + } + _, _ = w.Write(archives[version]) + })) + t.Cleanup(server.Close) + detail := model.AppDetail{Version: "8", DownloadUrl: server.URL + "/mysql-8.tar.gz?token=a%2Bb®ion=intl", Update: false} + if err := db.Create(&detail).Error; err != nil { + t.Fatal(err) + } + for _, step := range []struct { + version string + calls int32 + update bool + }{ + {"42", 1, false}, // An existing directory without a stamp must be refreshed. + {"42", 1, false}, + {"43", 2, false}, + {"43", 3, true}, + } { + if err := db.Model(&model.Setting{}).Where("key = ?", "AppStoreLastModified").Update("value", step.version).Error; err != nil { + t.Fatal(err) + } + detail.Update = step.update + if err := db.Model(&detail).UpdateColumn("update", detail.Update).Error; err != nil { + t.Fatal(err) + } + if err := downloadApp(app, detail, nil, nil); err != nil { + t.Fatal(err) + } + if calls.Load() != step.calls { + t.Fatalf("version %s: HTTP calls = %d, want %d", step.version, calls.Load(), step.calls) + } + for name, want := range map[string]string{dataPath: "fresh-" + step.version, stampPath: step.version} { + data, err := os.ReadFile(name) + if err != nil || string(data) != want { + t.Fatalf("%s = %q, %v; want %q", name, data, err, want) + } + } + var stored model.AppDetail + if err := db.First(&stored, detail.ID).Error; err != nil { + t.Fatal(err) + } + if stored.Update || stored.DownloadUrl != detail.DownloadUrl { + t.Fatalf("refresh altered source URL or left update flag: %+v", stored) + } + } +} + +func TestDownloadAppFailureKeepsCacheVersionAndUpdateFlag(t *testing.T) { + for _, failure := range []string{"download", "decompress"} { + t.Run(failure, func(t *testing.T) { + db := appCacheTestDB(t) + app := model.App{Key: "mysql", Resource: constant.AppResourceRemote} + stampPath := filepath.Join(app.GetAppResourcePath(), "8", ".1panel-appstore-version") + appCacheTestWrite(t, stampPath, "41") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if failure == "download" { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + _, _ = io.WriteString(w, "not an archive") + })) + t.Cleanup(server.Close) + detail := model.AppDetail{Version: "8", DownloadUrl: server.URL + "/mysql-8.tar.gz", Update: true} + if err := db.Create(&detail).Error; err != nil { + t.Fatal(err) + } + install := model.AppInstall{} + if err := downloadApp(app, detail, &install, nil); err == nil { + t.Fatal("failed refresh was accepted") + } + if install.Status != constant.StatusDownloadErr || install.Message == "" { + t.Fatalf("installation did not report refresh failure: %+v", install) + } + stamp, err := os.ReadFile(stampPath) + if err != nil || string(stamp) != "41" { + t.Fatalf("failed refresh updated stamp: %q, %v", stamp, err) + } + var stored model.AppDetail + if err := db.First(&stored, detail.ID).Error; err != nil { + t.Fatal(err) + } + if !stored.Update || stored.DownloadUrl != detail.DownloadUrl { + t.Fatalf("failed refresh changed database state: %+v", stored) + } + }) + } +} + +func TestGetPHPAppDetailUsesSynchronizedOfficialParams(t *testing.T) { + const oldYAML = "additionalProperties:\n formFields:\n - envKey: PHP_VERSION\n type: select\n default: 8.1.34\n" + const newJSON = `{"formFields":[{"envKey":"PHP_VERSION","type":"select","default":"8.1.35"}]}` + const compose = "services:\n php:\n image: php:test\n" + for _, tc := range []struct { + name, resource, compose string + update bool + }{ + {"remote", constant.AppResourceRemote, compose, false}, + {"local", constant.AppResourceLocal, compose, false}, + {"custom", constant.AppResourceCustom, compose, false}, + {"remote-fetches-compose", constant.AppResourceRemote, "", true}, + } { + t.Run(tc.name, func(t *testing.T) { + db := appCacheTestDB(t) + app := model.App{Key: "php", Name: "PHP", Type: constant.RuntimePHP, Resource: tc.resource} + if err := db.Omit(clause.Associations).Create(&app).Error; err != nil { + t.Fatal(err) + } + versionDir := filepath.Join(app.GetAppResourcePath(), "8") + appCacheTestWrite(t, filepath.Join(versionDir, "data.yml"), oldYAML) + appCacheTestWrite(t, filepath.Join(versionDir, "docker-compose.yml"), compose) + archive := appCacheTestArchive(t, "php/8/data.yml", oldYAML) + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + if r.URL.Query().Get("v") != "42" { + t.Errorf("missing app store version: %s", r.URL.RawQuery) + } + if r.URL.Path == "/docker-compose.yml" { + _, _ = io.WriteString(w, compose) + return + } + _, _ = w.Write(archive) + })) + t.Cleanup(server.Close) + detail := model.AppDetail{AppId: app.ID, Version: "8", Params: newJSON, DockerCompose: tc.compose, Update: tc.update, DownloadUrl: server.URL + "/php-8.tar.gz"} + if err := db.Create(&detail).Error; err != nil { + t.Fatal(err) + } + got, err := (AppService{}).GetAppDetail(app.ID, "8", "runtime") + if err != nil { + t.Fatal(err) + } + wantVersion, wantCalls := "8.1.34", int32(0) + if tc.resource == constant.AppResourceRemote { + wantVersion, wantCalls = "8.1.35", 1 + } + want := map[string]interface{}{"formFields": []interface{}{map[string]interface{}{ + "envKey": "PHP_VERSION", "type": "select", "default": wantVersion, + }}} + if !reflect.DeepEqual(got.Params, want) || got.Image != "php" || calls.Load() != wantCalls { + encoded, _ := json.Marshal(got.Params) + t.Fatalf("params=%s, image=%s, HTTP calls=%d; want version %s, calls %d", encoded, got.Image, calls.Load(), wantVersion, wantCalls) + } + var stored model.AppDetail + if err := db.First(&stored, detail.ID).Error; err != nil { + t.Fatal(err) + } + if stored.Update || got.Update { + t.Fatal("fetching the compose file restored a stale update flag") + } + }) + } +} + +func TestProcessOneAppDefersComposeUntilRequested(t *testing.T) { + db := appCacheTestDB(t) + const compose = "services:\n php:\n image: php:test\n" + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + if r.URL.Path != "/php/8/docker-compose.yml" || r.URL.Query().Get("v") != "42" { + t.Errorf("unversioned initial compose request: %s", r.URL.String()) + } + _, _ = io.WriteString(w, compose) + })) + t.Cleanup(server.Close) + app := model.App{Key: "php", Type: constant.RuntimePHP, Resource: constant.AppResourceRemote} + if err := db.Create(&app).Error; err != nil { + t.Fatal(err) + } + ctx := appSyncContext{ + baseRemoteUrl: server.URL, + systemVersion: "2.4.0", + list: &dto.AppList{LastModified: 42}, + } + result := ctx.processOneApp(appWorkItem{ + app: app, + appDef: dto.AppDefine{ + AppProperty: dto.AppProperty{Key: "php", Type: constant.RuntimePHP}, + Versions: []dto.AppConfigVersion{{Name: "8", AppForm: dto.AppForm{ + FormFields: []dto.AppFormFields{{EnvKey: "PHP_VERSION", Default: "8.1.35", Type: "select"}}, + }}}, + }, + }) + if calls.Load() != 0 || len(result.Details) != 1 { + t.Fatalf("initial compose refresh failed: calls=%d, result=%+v", calls.Load(), result) + } + detail := result.Details[0] + if detail.DockerCompose != "" || detail.DownloadUrl != server.URL+"/php/8/php-8.tar.gz" { + t.Fatalf("sync fetched compose or changed the canonical archive URL: %+v", detail) + } + detail.AppId = app.ID + if err := db.Create(&detail).Error; err != nil { + t.Fatal(err) + } + for range 2 { + if err := ensureAppDetailCompose(app, &detail); err != nil { + t.Fatal(err) + } + if calls.Load() != 1 || detail.DockerCompose != compose { + t.Fatalf("on-demand compose did not download once and cache: calls=%d, detail=%+v", calls.Load(), detail) + } + } +} diff --git a/agent/app/service/app_compose.go b/agent/app/service/app_compose.go new file mode 100644 index 000000000..f2c865a25 --- /dev/null +++ b/agent/app/service/app_compose.go @@ -0,0 +1,89 @@ +package service + +import ( + "fmt" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "strings" + + "github.com/1Panel-dev/1Panel/agent/app/model" + "github.com/1Panel-dev/1Panel/agent/buserr" + "github.com/1Panel-dev/1Panel/agent/constant" + "github.com/1Panel-dev/1Panel/agent/global" + "github.com/1Panel-dev/1Panel/agent/utils/req_helper" + "gopkg.in/yaml.v3" +) + +func ensureAppDetailCompose(app model.App, detail *model.AppDetail) error { + if strings.TrimSpace(detail.DockerCompose) != "" && validateAppDetailCompose([]byte(detail.DockerCompose)) == nil { + return nil + } + var content []byte + var err error + if app.IsLocalApp() || app.IsCustomApp() { + content, err = os.ReadFile(filepath.Join(app.GetAppResourcePath(), detail.Version, "docker-compose.yml")) + } else { + var composeURL string + if detail.DownloadUrl == "" { + composeURL = fmt.Sprintf("%s/%s/1panel/%s/%s/docker-compose.yml", global.AppRepoURL(), global.CONF.Base.Mode, app.Key, detail.Version) + } else { + var source *url.URL + source, err = url.Parse(detail.DownloadUrl) + if err == nil { + source.Path = path.Join(path.Dir(source.Path), "docker-compose.yml") + source.RawPath = "" + composeURL = source.String() + } + } + if err == nil { + composeURL, err = currentAppDownloadURL(composeURL) + } + if err == nil { + var status int + status, content, err = req_helper.HandleRequest(composeURL, http.MethodGet, constant.TimeOut20s) + if status == http.StatusNotFound { + return buserr.New("ErrAppVersionUnavailable") + } + } + } + if err != nil { + return buserr.WithDetail("ErrGetCompose", err.Error(), err) + } + return cacheAppDetailCompose(detail, content) +} + +func validateAppDetailCompose(content []byte) error { + var compose struct { + Services map[string]map[string]interface{} `yaml:"services"` + } + if err := yaml.Unmarshal(content, &compose); err != nil { + return buserr.WithDetail("ErrGetCompose", err.Error(), err) + } + if len(compose.Services) == 0 { + return buserr.WithDetail("ErrGetCompose", "docker-compose.yml has no services", nil) + } + for name, service := range compose.Services { + if strings.TrimSpace(name) == "" || len(service) == 0 { + return buserr.WithDetail("ErrGetCompose", "docker-compose.yml has an empty service", nil) + } + } + return nil +} + +func cacheAppDetailCompose(detail *model.AppDetail, content []byte) error { + if err := validateAppDetailCompose(content); err != nil { + return err + } + stored, err := appDetailRepo.CacheCompose(*detail, string(content)) + if err != nil { + return buserr.WithDetail("ErrGetCompose", err.Error(), err) + } + if !stored { + return buserr.WithDetail("ErrGetCompose", "application resources changed; retry the operation", nil) + } + detail.DockerCompose = string(content) + return nil +} diff --git a/agent/app/service/app_compose_sync_test.go b/agent/app/service/app_compose_sync_test.go new file mode 100644 index 000000000..7545da00a --- /dev/null +++ b/agent/app/service/app_compose_sync_test.go @@ -0,0 +1,70 @@ +package service + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/1Panel-dev/1Panel/agent/app/dto" + "github.com/1Panel-dev/1Panel/agent/app/model" + "github.com/1Panel-dev/1Panel/agent/constant" +) + +func TestComposeSyncOnlyInvalidatesChangedVersionResources(t *testing.T) { + form := dto.AppForm{FormFields: []dto.AppFormFields{{EnvKey: "PORT", Default: 8080}}} + params, err := json.Marshal(form) + if err != nil { + t.Fatal(err) + } + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + _, _ = io.WriteString(w, "services:\n demo:\n image: demo:unexpected\n") + })) + t.Cleanup(server.Close) + for _, appType := range []string{"runtime", "php", "node", "java", "go", "python", "dotnet", "website"} { + t.Run(appType, func(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*model.AppDetail, *dto.AppConfigVersion) + keepCache bool + }{ + {"unchanged", func(_ *model.AppDetail, _ *dto.AppConfigVersion) {}, true}, + {"package update flag does not expire compose", func(d *model.AppDetail, _ *dto.AppConfigVersion) { d.Update = true }, true}, + {"version revision changed", func(_ *model.AppDetail, v *dto.AppConfigVersion) { v.LastModified++ }, false}, + {"unknown revision", func(_ *model.AppDetail, v *dto.AppConfigVersion) { v.LastModified = 0 }, false}, + {"negative revision", func(_ *model.AppDetail, v *dto.AppConfigVersion) { v.LastModified = -1 }, false}, + {"form changed", func(d *model.AppDetail, _ *dto.AppConfigVersion) { d.Params = `{"formFields":[]}` }, false}, + {"source changed", func(d *model.AppDetail, _ *dto.AppConfigVersion) { d.DownloadUrl += ".old" }, false}, + {"new version", func(d *model.AppDetail, _ *dto.AppConfigVersion) { *d = model.AppDetail{Version: "1.0"} }, false}, + } { + t.Run(tc.name, func(t *testing.T) { + const cached = "services:\n demo:\n image: demo:cached\n" + old := model.AppDetail{Version: "1.0", LastModified: 42, Params: string(params), DownloadUrl: server.URL + "/demo/1.0/demo-1.0.tar.gz", DockerCompose: cached} + version := dto.AppConfigVersion{Name: "1.0", LastModified: 42, AppForm: form} + tc.mutate(&old, &version) + ctx := appSyncContext{baseRemoteUrl: server.URL, systemVersion: "2.4.0"} + app := model.App{Key: "demo", Type: appType, Resource: constant.AppResourceRemote, Details: []model.AppDetail{old}} + got := ctx.processOneApp(appWorkItem{app: app, appDef: dto.AppDefine{AppProperty: dto.AppProperty{Key: "demo"}, Versions: []dto.AppConfigVersion{version}}}) + if len(got.Details) != 1 || calls.Load() != 0 { + t.Fatalf("sync fetched compose or lost version: %+v, calls=%d", got, calls.Load()) + } + detail := got.Details[0] + want := "" + if tc.keepCache { + want = cached + } + if detail.DockerCompose != want || detail.LastModified != version.LastModified || detail.Params != string(params) || !detail.Update { + t.Fatalf("cache decision or version metadata changed: %+v", detail) + } + if app.Details[0].DockerCompose != old.DockerCompose { + t.Fatal("sync mutated its input template") + } + }) + } + }) + } +} diff --git a/agent/app/service/app_compose_test.go b/agent/app/service/app_compose_test.go new file mode 100644 index 000000000..311528583 --- /dev/null +++ b/agent/app/service/app_compose_test.go @@ -0,0 +1,253 @@ +package service + +import ( + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "github.com/1Panel-dev/1Panel/agent/app/model" + "github.com/1Panel-dev/1Panel/agent/constant" + "github.com/1Panel-dev/1Panel/agent/global" + "gorm.io/gorm" +) + +const composeTestTemplate = "services:\n demo:\n image: demo:latest\n network_mode: host\n" + +func composeTestDetail(t *testing.T, resource, source string) (*gorm.DB, model.App, model.AppDetail) { + t.Helper() + db := appCacheTestDB(t) + app := model.App{Key: "demo", Name: "Demo", Resource: resource, Type: "website"} + if err := db.Create(&app).Error; err != nil { + t.Fatal(err) + } + detail := model.AppDetail{AppId: app.ID, Version: "1.0", LastModified: 42, Params: `{"formFields":[]}`, DownloadUrl: source} + if err := db.Create(&detail).Error; err != nil { + t.Fatal(err) + } + return db, app, detail +} + +func TestComposeOnDemandLoadsOnceAndPreservesSource(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + if r.URL.Path != "/demo/1.0/docker-compose.yml" || r.URL.Query().Get("signature") != "a+b" || r.URL.Query().Get("v") != "42" { + t.Errorf("unexpected Compose source: %s", r.URL) + } + _, _ = io.WriteString(w, composeTestTemplate) + })) + t.Cleanup(server.Close) + db, app, detail := composeTestDetail(t, constant.AppResourceRemote, server.URL+"/demo/1.0/demo-1.0.tar.gz?signature=a%2Bb") + for i := 0; i < 2; i++ { + if err := ensureAppDetailCompose(app, &detail); err != nil { + t.Fatal(err) + } + if calls.Load() != 1 || detail.DockerCompose != composeTestTemplate { + t.Fatalf("cache miss after loading: calls=%d, detail=%+v", calls.Load(), detail) + } + } + var stored model.AppDetail + if err := db.First(&stored, detail.ID).Error; err != nil { + t.Fatal(err) + } + if stored.DockerCompose != composeTestTemplate || stored.LastModified != 42 || stored.DownloadUrl != detail.DownloadUrl || stored.Params != detail.Params || !stored.UpdatedAt.Equal(detail.UpdatedAt) { + t.Fatalf("cache write altered version metadata: %+v", stored) + } + res, err := (AppService{}).GetAppDetailByID(detail.ID) + if err != nil || !res.HostMode || res.DockerCompose != composeTestTemplate || calls.Load() != 1 { + t.Fatalf("ID details did not reuse template/HostMode: %+v, %v, calls=%d", res, err, calls.Load()) + } +} + +func TestComposeOnDemandIDDetailsFillMissingTemplate(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + _, _ = io.WriteString(w, composeTestTemplate) + })) + t.Cleanup(server.Close) + _, _, detail := composeTestDetail(t, constant.AppResourceRemote, server.URL+"/demo.tar.gz") + res, err := (AppService{}).GetAppDetailByID(detail.ID) + if err != nil || !res.HostMode || res.DockerCompose != composeTestTemplate || calls.Load() != 1 { + t.Fatalf("missing template gave wrong ID details: %+v, %v, calls=%d", res, err, calls.Load()) + } +} + +func TestComposeOnDemandLocalAndCustomUseTheirDirectories(t *testing.T) { + for _, resource := range []string{constant.AppResourceLocal, constant.AppResourceCustom} { + t.Run(resource, func(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1) })) + t.Cleanup(server.Close) + _, app, detail := composeTestDetail(t, resource, server.URL+"/must-not-fetch.tar.gz") + appCacheTestWrite(t, filepath.Join(app.GetAppResourcePath(), detail.Version, "docker-compose.yml"), composeTestTemplate) + if err := ensureAppDetailCompose(app, &detail); err != nil { + t.Fatal(err) + } + if detail.DockerCompose != composeTestTemplate || calls.Load() != 0 { + t.Fatalf("local resource used remote template: %+v, calls=%d", detail, calls.Load()) + } + }) + } +} + +func TestComposeOnDemandRejectsDownloadAndTemplateFailures(t *testing.T) { + for _, tc := range []struct { + name, body string + status int + }{ + {"not found", "missing", http.StatusNotFound}, + {"server error", "failure", http.StatusServiceUnavailable}, + {"empty", "", http.StatusOK}, + {"HTML", "blocked", http.StatusOK}, + {"malformed YAML", "services: [", http.StatusOK}, + {"missing services", "version: '3'", http.StatusOK}, + {"empty services", "services: {}", http.StatusOK}, + {"empty service", "services:\n demo:", http.StatusOK}, + {"string service", "services:\n demo: blocked", http.StatusOK}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tc.status) + _, _ = io.WriteString(w, tc.body) + })) + t.Cleanup(server.Close) + db, app, detail := composeTestDetail(t, constant.AppResourceRemote, server.URL+"/demo.tar.gz") + if err := ensureAppDetailCompose(app, &detail); err == nil { + t.Fatal("invalid template was accepted") + } + var stored model.AppDetail + if err := db.First(&stored, detail.ID).Error; err != nil { + t.Fatal(err) + } + if detail.DockerCompose != "" || stored.DockerCompose != "" { + t.Fatal("failed response was cached") + } + }) + } +} + +func TestComposeCacheRejectsConcurrentMetadataUpdates(t *testing.T) { + for _, field := range []string{"last_modified", "params", "download_url"} { + t.Run(field, func(t *testing.T) { + db, _, detail := composeTestDetail(t, constant.AppResourceRemote, "https://example.com/demo.tar.gz") + value := interface{}("new value") + if field == "last_modified" { + value = 43 + } + if err := db.Model(&model.AppDetail{}).Where("id = ?", detail.ID).Updates(map[string]interface{}{field: value, "docker_compose": "new template"}).Error; err != nil { + t.Fatal(err) + } + if err := cacheAppDetailCompose(&detail, []byte(composeTestTemplate)); err == nil { + t.Fatal("stale template overwrote newer metadata") + } + var stored model.AppDetail + if err := db.First(&stored, detail.ID).Error; err != nil { + t.Fatal(err) + } + if stored.DockerCompose != "new template" || detail.DockerCompose != "" { + t.Fatalf("stale cache result was retained: %+v", stored) + } + }) + } +} + +func TestComposeCachePropagatesWriteFailure(t *testing.T) { + db, _, detail := composeTestDetail(t, constant.AppResourceRemote, "https://example.com/demo.tar.gz") + if err := db.Exec("CREATE TRIGGER reject_compose BEFORE UPDATE OF docker_compose ON app_details BEGIN SELECT RAISE(FAIL, 'test write failure'); END").Error; err != nil { + t.Fatal(err) + } + if err := cacheAppDetailCompose(&detail, []byte(composeTestTemplate)); err == nil || detail.DockerCompose != "" { + t.Fatal("failed cache write was ignored") + } +} + +func TestMarkDownloadedPreservesComposeAndRejectsChangedMetadata(t *testing.T) { + db, _, detail := composeTestDetail(t, constant.AppResourceRemote, "https://example.com/demo.tar.gz") + if err := db.Model(&model.AppDetail{}).Where("id = ?", detail.ID).Updates(map[string]interface{}{"update": true, "docker_compose": composeTestTemplate}).Error; err != nil { + t.Fatal(err) + } + stored, err := appDetailRepo.MarkDownloaded(detail) + if err != nil || !stored { + t.Fatalf("matching download marker failed: stored=%v err=%v", stored, err) + } + var fresh model.AppDetail + if err := db.First(&fresh, detail.ID).Error; err != nil || fresh.Update || fresh.DockerCompose != composeTestTemplate { + t.Fatalf("download marker overwrote Compose: %+v, %v", fresh, err) + } + for _, field := range []string{"last_modified", "params", "download_url"} { + t.Run(field, func(t *testing.T) { + var snapshot model.AppDetail + if err := db.First(&snapshot, detail.ID).Error; err != nil { + t.Fatal(err) + } + value := interface{}("new metadata") + if field == "last_modified" { + value = snapshot.LastModified + 1 + } + if err := db.Model(&model.AppDetail{}).Where("id = ?", detail.ID).Updates(map[string]interface{}{field: value, "update": true}).Error; err != nil { + t.Fatal(err) + } + stored, err := appDetailRepo.MarkDownloaded(snapshot) + if err != nil || stored { + t.Fatalf("stale download marked new resources ready: stored=%v err=%v", stored, err) + } + if err := db.First(&fresh, detail.ID).Error; err != nil || !fresh.Update || fresh.DockerCompose != composeTestTemplate { + t.Fatalf("stale marker overwrote template or update flag: %+v, %v", fresh, err) + } + }) + } +} + +func TestRuntimeDetailUsesValidPackageComposeWithoutAnotherGET(t *testing.T) { + var calls atomic.Int32 + archive := appCacheTestArchive(t, "demo/1.0/docker-compose.yml", composeTestTemplate) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + if !strings.HasSuffix(r.URL.Path, ".tar.gz") { + t.Errorf("runtime requested Compose after obtaining a valid package: %s", r.URL) + } + _, _ = w.Write(archive) + })) + t.Cleanup(server.Close) + _, app, detail := composeTestDetail(t, constant.AppResourceRemote, server.URL+"/demo.tar.gz") + app.Type = constant.RuntimeNode + if err := global.DB.Model(&app).UpdateColumn("type", app.Type).Error; err != nil { + t.Fatal(err) + } + res, err := (AppService{}).GetAppDetail(app.ID, detail.Version, "runtime") + if err != nil || res.DockerCompose != composeTestTemplate || !res.HostMode || calls.Load() != 1 { + t.Fatalf("package Compose was not reused: %+v, %v, calls=%d", res, err, calls.Load()) + } +} + +func TestRuntimeDetailDoesNotCacheStaleTemplateAfterPackageFailure(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + if strings.HasSuffix(r.URL.Path, ".tar.gz") { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + _, _ = io.WriteString(w, composeTestTemplate) + })) + t.Cleanup(server.Close) + db, app, detail := composeTestDetail(t, constant.AppResourceRemote, server.URL+"/demo.tar.gz") + app.Type = constant.RuntimeNode + if err := db.Model(&app).UpdateColumn("type", app.Type).Error; err != nil { + t.Fatal(err) + } + appCacheTestWrite(t, filepath.Join(app.GetAppResourcePath(), detail.Version, "docker-compose.yml"), "services:\n stale:\n image: stale:old\n") + res, err := (AppService{}).GetAppDetail(app.ID, detail.Version, "runtime") + if err != nil || res.DockerCompose != composeTestTemplate || calls.Load() != 2 { + t.Fatalf("failed package download reused a stale local template: %+v, %v, calls=%d", res, err, calls.Load()) + } + var stored model.AppDetail + if err := db.First(&stored, detail.ID).Error; err != nil || stored.DockerCompose != composeTestTemplate { + t.Fatalf("stale package contents were cached: %+v, %v", stored, err) + } +} diff --git a/agent/app/service/app_compose_upgrade_test.go b/agent/app/service/app_compose_upgrade_test.go new file mode 100644 index 000000000..8c9fe88b4 --- /dev/null +++ b/agent/app/service/app_compose_upgrade_test.go @@ -0,0 +1,253 @@ +package service + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/1Panel-dev/1Panel/agent/app/dto/request" + "github.com/1Panel-dev/1Panel/agent/app/model" + "github.com/1Panel-dev/1Panel/agent/app/task" + "github.com/1Panel-dev/1Panel/agent/constant" + "github.com/1Panel-dev/1Panel/agent/global" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +const composeUpgradeCurrent = `services: + demo: + image: demo:1.0 + deploy: + resources: + limits: + cpus: ${CPUS} + memory: ${MEMORY_LIMIT} +` + +func composeUpgradeDB(t *testing.T, resourceURL string) (*gorm.DB, model.App, model.AppInstall, []model.AppDetail) { + t.Helper() + db := iconSyncDB(t) + if err := db.AutoMigrate(&model.AppIgnoreUpgrade{}); err != nil { + t.Fatal(err) + } + app := model.App{Key: "demo", Name: "Demo", Type: "runtime", Resource: constant.AppResourceRemote, CrossVersionUpdate: true} + if err := db.Create(&app).Error; err != nil { + t.Fatal(err) + } + var details []model.AppDetail + for _, version := range []string{"1.1", "1.0", "1.3", "1.2"} { + detail := model.AppDetail{ + AppId: app.ID, Version: version, LastModified: 42, Params: `{"formFields":[]}`, Status: constant.AppNormal, + DownloadUrl: resourceURL + "/demo/" + version + "/demo-" + version + ".tar.gz", + } + if err := db.Create(&detail).Error; err != nil { + t.Fatal(err) + } + details = append(details, detail) + } + if err := db.Create(&model.AppIgnoreUpgrade{AppID: app.ID, AppDetailID: details[2].ID, Scope: "version"}).Error; err != nil { + t.Fatal(err) + } + install := model.AppInstall{ + AppId: app.ID, AppDetailId: details[1].ID, Name: "demo-install", Version: "1.0", App: app, + ServiceName: "demo", ContainerName: "demo-container", Status: constant.StatusRunning, + Env: `{"CONTAINER_NAME":"demo-container"}`, DockerCompose: composeUpgradeCurrent, + } + if err := db.Omit(clause.Associations).Create(&install).Error; err != nil { + t.Fatal(err) + } + return db, app, install, details +} + +func TestUpgradeComposeLoadsOnlyEffectiveTarget(t *testing.T) { + var mu sync.Mutex + requests := make(map[string]int) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + requests[r.URL.Path]++ + mu.Unlock() + if r.URL.Query().Get("v") != "42" { + t.Errorf("missing synchronized version: %s", r.URL) + } + version := strings.Split(r.URL.Path, "/")[2] + _, _ = fmt.Fprintf(w, "services:\n source:\n image: demo:%s\n", version) + })) + t.Cleanup(server.Close) + db, _, install, details := composeUpgradeDB(t, server.URL) + // An unselected bad cached template must not prevent listing or default preview. + if err := db.Model(&details[0]).UpdateColumn("docker_compose", "invalid: [").Error; err != nil { + t.Fatal(err) + } + service := NewIAppInstalledService() + noCompose := false + versions, err := service.GetUpdateVersions(request.AppUpdateVersion{AppInstallID: install.ID, LoadCompose: &noCompose}) + if err != nil || len(versions) != 2 || versions[0].Version != "1.2" || versions[1].Version != "1.1" { + t.Fatalf("pure versions did not preserve filtering/sorting: %+v, %v", versions, err) + } + for _, version := range versions { + if version.DockerCompose != "" { + t.Fatal("pure version request returned a rendered template") + } + } + if len(requests) != 0 { + t.Fatalf("pure version request fetched templates: %v", requests) + } + for i := 0; i < 2; i++ { + versions, err = service.GetUpdateVersions(request.AppUpdateVersion{AppInstallID: install.ID}) + if err != nil || len(versions) != 2 || !strings.Contains(versions[0].DockerCompose, "demo:1.2") || versions[1].DockerCompose != "" { + t.Fatalf("default target preview = %+v, %v", versions, err) + } + } + if requests["/demo/1.2/docker-compose.yml"] != 1 || len(requests) != 1 { + t.Fatalf("default preview did not reuse only its own cache: %v", requests) + } + if err := db.Model(&details[0]).UpdateColumn("docker_compose", "").Error; err != nil { + t.Fatal(err) + } + versions, err = service.GetUpdateVersions(request.AppUpdateVersion{AppInstallID: install.ID, UpdateVersion: "1.1"}) + if err != nil || !strings.Contains(versions[1].DockerCompose, "demo:1.1") || versions[0].DockerCompose != "" { + t.Fatalf("selected target preview = %+v, %v", versions, err) + } + if requests["/demo/1.1/docker-compose.yml"] != 1 || len(requests) != 2 { + t.Fatalf("selection prefetched other candidates: %v", requests) + } + if _, err := service.GetUpdateVersions(request.AppUpdateVersion{AppInstallID: install.ID, UpdateVersion: "1.3"}); err == nil { + t.Fatal("ignored target was accepted") + } + if len(requests) != 2 { + t.Fatal("invalid target triggered a template request") + } +} + +func TestUpgradeComposeFetchFailurePreservesInstanceAndCache(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusServiceUnavailable) })) + t.Cleanup(server.Close) + db, _, install, details := composeUpgradeDB(t, server.URL) + if _, err := NewIAppInstalledService().GetUpdateVersions(request.AppUpdateVersion{AppInstallID: install.ID}); err == nil { + t.Fatal("failed target template load was accepted") + } + var stored model.AppInstall + if err := db.First(&stored, install.ID).Error; err != nil || stored.DockerCompose != install.DockerCompose || stored.Status != install.Status || stored.Version != install.Version { + t.Fatalf("preview modified installed instance: %+v, %v", stored, err) + } + var detail model.AppDetail + if err := db.First(&detail, details[3].ID).Error; err != nil || detail.DockerCompose != "" { + t.Fatalf("failed response cached a template: %+v, %v", detail, err) + } +} + +func TestUpgradePrepareComposeFailureDoesNotCutover(t *testing.T) { + for _, failure := range []string{"package download", "invalid package template"} { + t.Run(failure, func(t *testing.T) { + archive := appCacheTestArchive(t, "demo/1.2/docker-compose.yml", "services: []") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if failure == "package download" { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + _, _ = w.Write(archive) + })) + t.Cleanup(server.Close) + db, _, install, details := composeUpgradeDB(t, server.URL) + upgrade := &appUpgradeContext{original: install, detail: details[3]} + if err := upgrade.prepare(&task.Task{Logger: global.LOG}); err == nil { + t.Fatal("invalid upgrade resources passed prepare") + } + if upgrade.stopAttempted || upgrade.downAttempted || upgrade.stageDir != "" { + t.Fatal("failed template preparation reached live cutover") + } + var stored model.AppInstall + if err := db.First(&stored, install.ID).Error; err != nil || stored.DockerCompose != install.DockerCompose || stored.Env != install.Env || stored.Version != install.Version || stored.Status != install.Status { + t.Fatalf("failed prepare modified the instance: %+v, %v", stored, err) + } + }) + } +} + +func TestExplicitConfigComposeDetectsCustomAndReusesCurrentCache(t *testing.T) { + for _, tc := range []struct { + name string + custom bool + failure bool + }{ + {name: "original"}, + {name: "custom", custom: true}, + {name: "fetch failure", failure: true}, + } { + t.Run(tc.name, func(t *testing.T) { + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if r.URL.Path != "/demo/1.0/docker-compose.yml" { + t.Errorf("configuration loaded a different version: %s", r.URL) + } + if tc.failure { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + _, _ = io.WriteString(w, "services:\n source:\n image: demo:1.0\n") + })) + t.Cleanup(server.Close) + db, _, install, _ := composeUpgradeDB(t, server.URL) + if tc.custom { + install.DockerCompose = strings.Replace(install.DockerCompose, "image: demo:1.0", "image: demo:1.0\n environment:\n USER_SETTING: preserve", 1) + if err := db.Omit(clause.Associations).Save(&install).Error; err != nil { + t.Fatal(err) + } + } + if isEditCompose(install) || calls != 0 { + t.Fatal("list-time custom check fetched a missing template") + } + result, err := NewIAppInstalledService().GetParams(install.ID) + if tc.failure { + if err == nil || result != nil { + t.Fatal("failed current-template load was treated as an unedited configuration") + } + return + } + if err != nil || result.RawCompose == "" || result.IsEdit != tc.custom || result.DockerCompose != install.DockerCompose { + t.Fatalf("explicit custom state = %+v, %v; want custom=%v", result, err, tc.custom) + } + if _, err := NewIAppInstalledService().GetParams(install.ID); err != nil || calls != 1 { + t.Fatalf("configuration cache was not reused: calls=%d, %v", calls, err) + } + if isEditCompose(install) != tc.custom || calls != 1 { + t.Fatal("cached list-time comparison changed custom state or made network requests") + } + }) + } +} + +func TestVllmManagedComposeDoesNotRequireOfficialTemplate(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + t.Cleanup(server.Close) + db, app, install, _ := composeUpgradeDB(t, server.URL) + if err := db.Model(&app).UpdateColumn("key", vllmAppKeyForUpgrade).Error; err != nil { + t.Fatal(err) + } + install.DockerCompose = strings.Replace(install.DockerCompose, "image: demo:1.0", "image: demo:1.0\n command: [--managed-user-option]", 1) + if err := db.Omit(clause.Associations).Save(&install).Error; err != nil { + t.Fatal(err) + } + service := NewIAppInstalledService() + config, err := service.GetParams(install.ID) + if err != nil || config.RawCompose != install.DockerCompose || config.DockerCompose != install.DockerCompose || config.IsEdit { + t.Fatalf("managed configuration lost the instance template: %+v, %v", config, err) + } + versions, err := service.GetUpdateVersions(request.AppUpdateVersion{AppInstallID: install.ID}) + if err != nil || len(versions) != 2 || versions[0].DockerCompose != install.DockerCompose { + t.Fatalf("managed upgrade preview lost the instance template: %+v, %v", versions, err) + } + if calls.Load() != 0 { + t.Fatalf("managed template attempted %d official resource requests", calls.Load()) + } +} diff --git a/agent/app/service/app_icon_sync_test.go b/agent/app/service/app_icon_sync_test.go new file mode 100644 index 000000000..a15f6e271 --- /dev/null +++ b/agent/app/service/app_icon_sync_test.go @@ -0,0 +1,265 @@ +package service + +import ( + "archive/zip" + "bytes" + "context" + "encoding/base64" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "path/filepath" + "reflect" + "strings" + "sync/atomic" + "testing" + + "github.com/1Panel-dev/1Panel/agent/app/dto" + "github.com/1Panel-dev/1Panel/agent/app/dto/request" + "github.com/1Panel-dev/1Panel/agent/app/model" + "github.com/1Panel-dev/1Panel/agent/app/task" + "github.com/1Panel-dev/1Panel/agent/constant" + "github.com/1Panel-dev/1Panel/agent/global" + "github.com/1Panel-dev/1Panel/agent/utils/xpack" + "github.com/1Panel-dev/1Panel/agent/utils/xpack/providers" + "github.com/gin-gonic/gin" + goi18n "github.com/nicksnyder/go-i18n/v2/i18n" + "golang.org/x/text/language" + "gorm.io/gorm" +) + +type iconSyncTransport struct { + providers.MultiNodeProvider + transport *http.Transport +} + +func (p iconSyncTransport) LoadRequestTransport() *http.Transport { return p.transport } + +func iconSyncDB(t *testing.T) *gorm.DB { + t.Helper() + db := appCacheTestDB(t) + if err := db.AutoMigrate(&model.Tag{}, &model.AppInstall{}, &model.AppInstallResource{}); err != nil { + t.Fatal(err) + } + if err := db.Create(&[]model.Setting{{Key: "SystemVersion", Value: "2.4.0"}, {Key: "AppStoreSyncStatus", Value: constant.StatusSyncSuccess}}).Error; err != nil { + t.Fatal(err) + } + oldI18n := global.I18n + global.I18n = goi18n.NewLocalizer(goi18n.NewBundle(language.English), "en") + t.Cleanup(func() { global.I18n = oldI18n }) + return db +} + +func TestOfficialIconSyncNeverDownloadsImages(t *testing.T) { + iconSyncDB(t) + var calls atomic.Int32 + cdn := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + t.Cleanup(cdn.Close) + taskLog := &task.Task{Logger: global.LOG} + for _, oldIcon := range []string{"", "app_demo.png?etag=old", "b2xk", cdn.URL + "/old.png"} { + t.Run(oldIcon, func(t *testing.T) { + icon := cdn.URL + "/demo.png?signature=a%2Bb&v=one" + def := dto.AppDefine{Icon: icon, AppProperty: dto.AppProperty{Key: "demo", Type: "website"}, Versions: []dto.AppConfigVersion{{Name: "1.0", AppForm: dto.AppForm{}}}} + apps := getApps([]model.App{{Key: "demo", Icon: oldIcon}}, []dto.AppDefine{def}, "2.4.0", taskLog) + ctx := appSyncContext{baseRemoteUrl: cdn.URL, list: &dto.AppList{LastModified: 42}} + result := ctx.processOneApp(appWorkItem{app: apps["demo"], appDef: def}) + if result.Icon != icon || len(result.Details) != 1 || result.Details[0].Version != "1.0" || calls.Load() != 0 { + t.Fatalf("sync changed metadata or fetched a CDN image: %+v, calls=%d", result, calls.Load()) + } + }) + } + for _, invalid := range []string{"", "javascript:alert(1)", "https://user:secret@example.com/a.png", "//example.com/a.png", "https://"} { + def := dto.AppDefine{Icon: invalid, AppProperty: dto.AppProperty{Key: "demo"}} + apps := getApps([]model.App{{Key: "demo", Icon: "b2xk"}}, []dto.AppDefine{def}, "2.4.0", taskLog) + if apps["demo"].Icon != "b2xk" { + t.Fatalf("invalid icon %q erased history", invalid) + } + } +} + +func TestOfficialIconSameRevisionRefreshIsReadOnlyUntilSync(t *testing.T) { + db := iconSyncDB(t) + apps := []model.App{ + {Key: "mysql", Type: "website", Status: constant.AppNormal, Icon: "app_mysql.png", BatchInstallSupport: true}, + {Key: "demo", Name: "original", Type: "website", Status: constant.AppNormal, Icon: "b2xk", LastModified: 7}, + {Key: "future", Type: "website", Status: constant.AppNormal, Icon: "b2xk"}, + {Key: "local-demo", Type: "website", Resource: constant.AppResourceLocal, Status: constant.AppNormal, Icon: "bG9jYWw="}, + {Key: "custom-demo", Type: "website", Resource: constant.AppResourceCustom, Status: constant.AppNormal, Icon: "Y3VzdG9t"}, + {Key: "down", Type: "website", Status: constant.AppTakeDown, Icon: "b2xk"}, + } + if err := db.Create(&apps).Error; err != nil { + t.Fatal(err) + } + detail := model.AppDetail{AppId: apps[1].ID, Version: "1.0", Params: "keep", DockerCompose: "keep", Update: false} + if err := db.Create(&detail).Error; err != nil { + t.Fatal(err) + } + if err := db.First(&detail, detail.ID).Error; err != nil { + t.Fatal(err) + } + var imageCalls, indexCalls atomic.Int32 + var failIndex atomic.Bool + list := dto.AppList{LastModified: 42} + for _, app := range apps { + version := 0.0 + if app.Key == "future" { + version = 9.9 + } + list.Apps = append(list.Apps, dto.AppDefine{Icon: "https://cdn.example/" + app.Key + ".png?signature=a%2Bb", AppProperty: dto.AppProperty{Key: app.Key, Version: version}}) + } + var zipped bytes.Buffer + zw := zip.NewWriter(&zipped) + w, err := zw.Create("1panel.json") + if err != nil { + t.Fatal(err) + } + if err := json.NewEncoder(w).Encode(list); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "1panel.json.version.txt"): + _, _ = io.WriteString(w, "42") + case strings.HasSuffix(r.URL.Path, "1panel.json.zip"): + indexCalls.Add(1) + if failIndex.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + _, _ = w.Write(zipped.Bytes()) + default: + imageCalls.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + } + })) + t.Cleanup(server.Close) + transport := server.Client().Transport.(*http.Transport).Clone() + transport.TLSClientConfig.InsecureSkipVerify = true + transport.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, server.Listener.Addr().String()) + } + oldProvider := xpack.MultiNodeProvider + xpack.MultiNodeProvider = iconSyncTransport{MultiNodeProvider: oldProvider, transport: transport} + t.Cleanup(func() { xpack.MultiNodeProvider = oldProvider; transport.CloseIdleConnections() }) + global.Dir.ResourceDir = t.TempDir() + global.Dir.IconCacheDir = t.TempDir() + appCacheTestWrite(t, filepath.Join(global.Dir.IconCacheDir, "app_mysql.png"), "keep old image") + encodedList, err := json.Marshal(list) + if err != nil { + t.Fatal(err) + } + appCacheTestWrite(t, filepath.Join(global.Dir.ResourceDir, "1panel.json"), string(encodedList)) + + res, err := (AppService{}).GetAppUpdate() + if err != nil || res.CanUpdate || res.AppList == nil || indexCalls.Load() != 0 { + t.Fatalf("same revision/missing files forced a full sync: %+v, %v", res, err) + } + var original model.App + if err := db.First(&original, apps[1].ID).Error; err != nil || original.Icon != apps[1].Icon { + t.Fatalf("update check modified icon: %+v, %v", original, err) + } + var writes atomic.Int32 + if err := db.Callback().Update().After("gorm:update").Register("test:icon-writes", func(tx *gorm.DB) { + if values, ok := tx.Statement.Dest.(map[string]interface{}); ok && tx.Statement.Table == "apps" && values["icon"] != nil { + writes.Add(1) + } + }); err != nil { + t.Fatal(err) + } + var shared *appSyncContext + for i := 0; i < 2; i++ { + if err := (AppService{}).createSyncAppStoreTask(&shared)(&task.Task{Logger: global.LOG}); err != nil { + t.Fatal(err) + } + if shared == nil || !shared.skipMetaSync || writes.Load() != 2 || imageCalls.Load() != 0 || indexCalls.Load() != 0 { + t.Fatalf("unexpected same revision sync: writes=%d, image calls=%d, index calls=%d, context=%+v", writes.Load(), imageCalls.Load(), indexCalls.Load(), shared) + } + } + var stored model.App + if err := db.First(&stored, apps[1].ID).Error; err != nil { + t.Fatal(err) + } + want := original + want.Icon = list.Apps[1].Icon + if !reflect.DeepEqual(stored, want) { + t.Fatalf("light refresh changed non-icon fields: got %+v, want %+v", stored, want) + } + for _, original := range apps[2:] { + stored = model.App{} + if err := db.First(&stored, original.ID).Error; err != nil { + t.Fatal(err) + } + if stored.Icon != original.Icon || stored.Status != original.Status { + t.Fatalf("filtered app was changed: %+v", stored) + } + } + var storedDetail model.AppDetail + if err := db.First(&storedDetail, detail.ID).Error; err != nil || !reflect.DeepEqual(storedDetail, detail) { + t.Fatalf("light refresh touched app details: %+v, %v", storedDetail, err) + } + failIndex.Store(true) + appCacheTestWrite(t, filepath.Join(global.Dir.ResourceDir, "1panel.json"), "{invalid") + if err := (AppService{}).createSyncAppStoreTask(&shared)(&task.Task{Logger: global.LOG}); err == nil || writes.Load() != 2 || indexCalls.Load() != 1 { + t.Fatal("failed index read was accepted or modified icons") + } +} + +func TestAppIconURLsReachListDetailAndInstalled(t *testing.T) { + db := iconSyncDB(t) + for _, icon := range []string{"https://cdn.example/icon.png?signature=a%2Bb", "app_legacy.png?etag=old", "bG9jYWw="} { + app := model.App{Key: "demo-" + icon[:4], Name: "demo", Type: "website", Status: constant.AppNormal, Icon: icon} + app.Name = app.Key + if err := db.Create(&app).Error; err != nil { + t.Fatal(err) + } + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + ctx.Request = httptest.NewRequest(http.MethodGet, "/", nil) + got, err := (AppService{}).PageApp(ctx, request.AppSearch{PageInfo: dto.PageInfo{Page: 1, PageSize: 10}, Name: app.Key}) + if err != nil || len(got.Items) != 1 { + t.Fatalf("list failed: %+v, %v", got, err) + } + want := "" + if strings.HasPrefix(icon, "https:") { + want = icon + } + if got.Items[0].Icon != want { + t.Fatalf("list icon = %q, want %q", got.Items[0].Icon, want) + } + detail, err := (AppService{}).GetApp(ctx, app.Key) + if err != nil || detail.Icon != icon { + t.Fatalf("detail icon = %+v, %v", detail, err) + } + installed, err := handleInstalled([]model.AppInstall{{App: app, Status: constant.StatusRunning}}, false, false, false) + if err != nil || len(installed) != 1 || installed[0].Icon != want { + t.Fatalf("installed icon = %+v, %v", installed, err) + } + } +} + +func TestLocalAppLogoKeepsBase64Delivery(t *testing.T) { + db := iconSyncDB(t) + dir := t.TempDir() + const image = "\x89PNG\r\n\x1a\nlocal-icon" + appCacheTestWrite(t, filepath.Join(dir, "data.yml"), "additionalProperties:\n key: demo\n name: Local Demo\n type: website\n") + appCacheTestWrite(t, filepath.Join(dir, "README.md"), "local application") + appCacheTestWrite(t, filepath.Join(dir, "logo.png"), image) + app, err := handleLocalApp(dir) + if err != nil || app.Resource != constant.AppResourceLocal || app.Icon != base64.StdEncoding.EncodeToString([]byte(image)) { + t.Fatalf("local logo parsing changed: %+v, %v", app, err) + } + if err := db.Create(app).Error; err != nil { + t.Fatal(err) + } + data, source, etag, err := (AppService{}).GetAppIcon(app.Key) + if err != nil || string(data) != image || source != "" || etag != "" { + t.Fatalf("local logo no longer uses historical delivery: %q, %q, %q, %v", data, source, etag, err) + } +} diff --git a/agent/app/service/app_install.go b/agent/app/service/app_install.go index de4679cd2..def0a1258 100644 --- a/agent/app/service/app_install.go +++ b/agent/app/service/app_install.go @@ -6,10 +6,8 @@ import ( "fmt" "maps" "math" - "net/http" "os" "path" - "path/filepath" "reflect" "sort" "strconv" @@ -31,7 +29,6 @@ import ( "github.com/1Panel-dev/1Panel/agent/utils/env" "github.com/1Panel-dev/1Panel/agent/utils/files" "github.com/1Panel-dev/1Panel/agent/utils/nginx" - "github.com/1Panel-dev/1Panel/agent/utils/req_helper" "github.com/docker/docker/api/types/container" "github.com/joho/godotenv" "github.com/pkg/errors" @@ -743,7 +740,7 @@ func (a *AppInstallService) GetUpdateVersions(req request.AppUpdateVersion) ([]d } if canUpgrade { feishuTarget := app.Key == constant.AppOpenclaw && isOpenclawBundledFeishuVersion(detail.Version) - if feishuTarget && !feishuChecked { + if feishuTarget && !feishuChecked && (req.LoadCompose == nil || *req.LoadCompose) { conf, readErr := readOpenclawConfig(path.Join(install.GetPath(), "data/conf/openclaw.json")) if readErr != nil { return versions, readErr @@ -754,33 +751,6 @@ func (a *AppInstallService) GetUpdateVersions(req request.AppUpdateVersion) ([]d } feishuChecked = true } - var newCompose string - if req.UpdateVersion != "" && req.UpdateVersion == detail.Version && detail.DockerCompose == "" && !app.IsLocalApp() { - filename := filepath.Base(detail.DownloadUrl) - dockerComposeUrl := fmt.Sprintf("%s%s", strings.TrimSuffix(detail.DownloadUrl, filename), "docker-compose.yml") - if app.Resource == constant.AppResourceRemote { - dockerComposeUrl, err = currentAppDownloadURL(dockerComposeUrl) - if err != nil { - return versions, err - } - } - statusCode, composeRes, err := req_helper.HandleRequest(dockerComposeUrl, http.MethodGet, constant.TimeOut20s) - if statusCode == http.StatusNotFound { - return versions, buserr.New("ErrAppVersionUnavailable") - } - if err != nil { - return versions, err - } - if statusCode > 200 { - return versions, err - } - detail.DockerCompose = string(composeRes) - _ = appDetailRepo.Update(context.Background(), detail) - } - newCompose, err = getUpgradeCompose(install, detail) - if err != nil { - return versions, err - } if app.Key == constant.AppMysql { majorVersion := getMajorVersion(install.Version) if !strings.HasPrefix(detail.Version, majorVersion) { @@ -792,13 +762,46 @@ func (a *AppInstallService) GetUpdateVersions(req request.AppUpdateVersion) ([]d FeishuConfigured: feishuTarget && feishuConfigured, Version: detail.Version, DetailId: detail.ID, - DockerCompose: newCompose, }) } } sort.Slice(versions, func(i, j int) bool { return appVersionSortBefore(app.Key, versions[i].Version, versions[j].Version) }) + if len(versions) == 0 || (req.LoadCompose != nil && !*req.LoadCompose) { + return versions, nil + } + target := 0 + if req.UpdateVersion != "" { + target = -1 + for i := range versions { + if versions[i].Version == req.UpdateVersion { + target = i + break + } + } + if target == -1 { + return versions, buserr.New("ErrAppVersionUnavailable") + } + } + if app.Key == vllmAppKeyForUpgrade { + versions[target].DockerCompose = install.DockerCompose + return versions, nil + } + for _, detail := range details { + if detail.ID != versions[target].DetailId { + continue + } + if err := ensureAppDetailCompose(app, &detail); err != nil { + return versions, err + } + content, err := getUpgradeCompose(install, detail) + if err != nil { + return versions, err + } + versions[target].DockerCompose = content + break + } return versions, nil } @@ -1007,9 +1010,22 @@ func (a *AppInstallService) GetParams(id uint) (*response.AppConfig, error) { res.RestartPolicy = getRestartPolicy(install.DockerCompose) res.WebUI = install.WebUI res.Type = install.App.Type - if rawCompose, err := getUpgradeCompose(install, detail); err == nil { - res.RawCompose = rawCompose + rawCompose := install.DockerCompose + if install.App.Key != vllmAppKeyForUpgrade { + if err := ensureAppDetailCompose(install.App, &detail); err != nil { + return nil, err + } + rawCompose, err = getUpgradeCompose(install, detail) + if err != nil { + return nil, err + } } + equal, err := composeEqualExceptImage(rawCompose, install.DockerCompose) + if err != nil { + return nil, err + } + res.RawCompose = rawCompose + res.IsEdit = !equal return &res, nil } diff --git a/agent/app/service/app_store_index_test.go b/agent/app/service/app_store_index_test.go new file mode 100644 index 000000000..fd7fdbb34 --- /dev/null +++ b/agent/app/service/app_store_index_test.go @@ -0,0 +1,128 @@ +package service + +import ( + "archive/zip" + "bytes" + "context" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "testing" + + "github.com/1Panel-dev/1Panel/agent/app/dto" + "github.com/1Panel-dev/1Panel/agent/app/model" + "github.com/1Panel-dev/1Panel/agent/constant" + "github.com/1Panel-dev/1Panel/agent/global" + "github.com/1Panel-dev/1Panel/agent/utils/xpack" +) + +func TestAppStoreIndexUsesCacheUntilRevisionChanges(t *testing.T) { + for _, tc := range []struct { + name string + cache string + version int + failZIP bool + wantZIP int32 + wantUpdate bool + wantErr bool + }{ + {name: "matching cache", cache: `{"lastModified":42}`, version: 42}, + {name: "missing cache", version: 42, wantZIP: 1}, + {name: "malformed cache", cache: `{invalid`, version: 42, wantZIP: 1}, + {name: "null cache", cache: `null`, version: 42, wantZIP: 1}, + {name: "trailing corrupt data", cache: `{"lastModified":42} garbage`, version: 42, wantZIP: 1}, + {name: "outdated cache", cache: `{"lastModified":41}`, version: 42, wantZIP: 1}, + {name: "remote revision changed", cache: `{"lastModified":42}`, version: 43, wantUpdate: true}, + {name: "download failure", cache: `{invalid`, version: 42, failZIP: true, wantZIP: 1, wantErr: true}, + {name: "cached minimum panel version", cache: `{"lastModified":42,"additionalProperties":{"version":"9.9.0"}}`, version: 42, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + db := iconSyncDB(t) + if err := db.Create(&model.App{Key: "mysql", Type: "website", Status: constant.AppNormal, Icon: "app_mysql.png", BatchInstallSupport: true}).Error; err != nil { + t.Fatal(err) + } + global.Dir.ResourceDir = t.TempDir() + cachePath := filepath.Join(global.Dir.ResourceDir, "1panel.json") + if tc.cache != "" { + appCacheTestWrite(t, cachePath, tc.cache) + } + list := dto.AppList{LastModified: tc.version, Apps: []dto.AppDefine{{Icon: "https://cdn.example/mysql.png", AppProperty: dto.AppProperty{Key: "mysql"}}}} + var archive bytes.Buffer + zw := zip.NewWriter(&archive) + entry, err := zw.Create("1panel.json") + if err != nil { + t.Fatal(err) + } + if err := json.NewEncoder(entry).Encode(list); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + var versionCalls, zipCalls atomic.Int32 + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "1panel.json.version.txt"): + versionCalls.Add(1) + _, _ = io.WriteString(w, strconv.Itoa(tc.version)) + case strings.HasSuffix(r.URL.Path, "1panel.json.zip"): + zipCalls.Add(1) + if tc.failZIP { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + _, _ = w.Write(archive.Bytes()) + default: + t.Errorf("unexpected remote resource request: %s", r.URL) + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(server.Close) + transport := server.Client().Transport.(*http.Transport).Clone() + transport.TLSClientConfig.InsecureSkipVerify = true + transport.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, server.Listener.Addr().String()) + } + oldProvider := xpack.MultiNodeProvider + xpack.MultiNodeProvider = iconSyncTransport{MultiNodeProvider: oldProvider, transport: transport} + t.Cleanup(func() { xpack.MultiNodeProvider = oldProvider; transport.CloseIdleConnections() }) + + res, err := (AppService{}).GetAppUpdate() + if (err != nil) != tc.wantErr || res == nil && !tc.wantErr { + t.Fatalf("update check = %+v, %v; want error %v", res, err, tc.wantErr) + } + if zipCalls.Load() != tc.wantZIP || versionCalls.Load() != 1 { + t.Fatalf("first check requests: version=%d, ZIP=%d; want 1/%d", versionCalls.Load(), zipCalls.Load(), tc.wantZIP) + } + if tc.wantErr { + if tc.wantZIP == 0 && !strings.Contains(err.Error(), "ErrVersionTooLow") { + t.Fatalf("cached panel-version check was lost: %v", err) + } + return + } + if res.CanUpdate != tc.wantUpdate { + t.Fatalf("CanUpdate=%v, want %v", res.CanUpdate, tc.wantUpdate) + } + if tc.wantUpdate { + // Full synchronization still fetches the new index after detecting a revision change. + fresh, err := getAppList() + if err != nil || fresh.LastModified != tc.version || zipCalls.Load() != 1 { + t.Fatalf("changed revision did not fetch the new index: %+v, %v, ZIP=%d", fresh, err, zipCalls.Load()) + } + return + } + if res.AppList == nil || res.AppList.LastModified != 42 { + t.Fatalf("same-revision icon refresh lost its index: %+v", res) + } + if _, err := (AppService{}).GetAppUpdate(); err != nil || zipCalls.Load() != tc.wantZIP || versionCalls.Load() != 2 { + t.Fatalf("second check redownloaded cached index: %v, version=%d, ZIP=%d", err, versionCalls.Load(), zipCalls.Load()) + } + }) + } +} diff --git a/agent/app/service/app_sync_task.go b/agent/app/service/app_sync_task.go index 1639a9e44..d14fc8d4e 100644 --- a/agent/app/service/app_sync_task.go +++ b/agent/app/service/app_sync_task.go @@ -4,12 +4,7 @@ import ( "context" "encoding/json" "fmt" - "net/http" "strconv" - "strings" - "sync" - "sync/atomic" - "time" "github.com/1Panel-dev/1Panel/agent/app/dto" "github.com/1Panel-dev/1Panel/agent/app/model" @@ -17,15 +12,11 @@ import ( "github.com/1Panel-dev/1Panel/agent/constant" "github.com/1Panel-dev/1Panel/agent/global" "github.com/1Panel-dev/1Panel/agent/i18n" - "github.com/1Panel-dev/1Panel/agent/utils/appicon" "github.com/1Panel-dev/1Panel/agent/utils/common" - "github.com/1Panel-dev/1Panel/agent/utils/req_helper" - "github.com/1Panel-dev/1Panel/agent/utils/xpack" ) type appSyncContext struct { task *task.Task - httpClient http.Client baseRemoteUrl string systemVersion string appsMap map[string]model.App @@ -34,7 +25,6 @@ type appSyncContext struct { oldAppIds []uint appTags []*model.AppTag skipMetaSync bool - pendingIcons map[string]string } func (a AppService) createSyncAppStoreTask(sharedCtx **appSyncContext) func(t *task.Task) error { @@ -51,6 +41,10 @@ func (a AppService) createSyncAppStoreTask(sharedCtx **appSyncContext) func(t *t t.Log(i18n.GetMsgByKey("AppStoreIsSyncing")) return nil } + if err := refreshOfficialAppIcons(updateRes.AppList, t); err != nil { + t.LogFailedWithErr(i18n.GetMsgByKey("PersistApps"), err) + return err + } global.LOG.Infof("[AppStore] Appstore is up to date") t.Log(i18n.GetMsgByKey("AppStoreIsUpToDate")) *sharedCtx = &appSyncContext{skipMetaSync: true} @@ -82,13 +76,11 @@ func (a AppService) createSyncAppStoreTask(sharedCtx **appSyncContext) func(t *t ctx := &appSyncContext{ task: t, - httpClient: http.Client{Timeout: time.Duration(constant.TimeOut20s) * time.Second, Transport: xpack.MultiNodeProvider.LoadRequestTransport()}, baseRemoteUrl: fmt.Sprintf("%s/%s/1panel", global.AppRepoURL(), global.CONF.Base.Mode), systemVersion: setting.SystemVersion, settingService: settingService, list: list, appTags: make([]*model.AppTag, 0), - pendingIcons: make(map[string]string), } if err = SyncTags(list.Extra); err != nil { @@ -107,7 +99,7 @@ func (a AppService) createSyncAppStoreTask(sharedCtx **appSyncContext) func(t *t ctx.oldAppIds = append(ctx.oldAppIds, old.ID) } - ctx.appsMap, ctx.pendingIcons = getApps(oldApps, list.Apps, setting.SystemVersion, t) + ctx.appsMap = getApps(oldApps, list.Apps, setting.SystemVersion, t) var addCount, updateCount, deleteCount int if err = ctx.classifyAndPersistAppsWithStats(&addCount, &updateCount, &deleteCount); err != nil { @@ -129,40 +121,39 @@ func (a AppService) createSyncAppStoreTask(sharedCtx **appSyncContext) func(t *t } } -type appWorkItem struct { - appDef dto.AppDefine - app model.App - iconUrl string - hadIcon bool -} - -type appWorkResult struct { - appKey string - app model.App - iconStatus int - hadIcon bool - httpFailed bool -} - -func (c *appSyncContext) processOneApp(item appWorkItem) appWorkResult { - app := item.app - l := item.appDef - result := appWorkResult{ - appKey: l.AppProperty.Key, - hadIcon: item.hadIcon, +func refreshOfficialAppIcons(list *dto.AppList, t *task.Task) error { + if list == nil { + return nil } - - if item.hadIcon { - status, iconField := c.downloadAppIcon(item.iconUrl, l.AppProperty.Key, app.Icon) - result.iconStatus = status - switch status { - case http.StatusOK: - app.Icon = iconField - case http.StatusNotModified: - default: - result.httpFailed = true + apps, err := appRepo.GetBy(appRepo.WithResource(constant.AppResourceRemote)) + if err != nil { + return err + } + systemVersion, err := settingRepo.GetValueByKey("SystemVersion") + if err != nil { + return err + } + updated := getApps(apps, list.Apps, systemVersion, t) + for _, app := range apps { + candidate := updated[app.Key] + if app.Status != constant.AppNormal || candidate.Status != constant.AppNormal || candidate.Icon == app.Icon { + continue + } + if err := appRepo.UpdateIcon(app.ID, candidate.Icon); err != nil { + return err } } + return nil +} + +type appWorkItem struct { + appDef dto.AppDefine + app model.App +} + +func (c *appSyncContext) processOneApp(item appWorkItem) model.App { + app := item.app + l := item.appDef app.TagsKey = l.AppProperty.Tags if l.AppProperty.Recommend > 0 { @@ -189,24 +180,12 @@ func (c *appSyncContext) processOneApp(item appWorkItem) appWorkResult { continue } - if _, ok := InitTypes[app.Type]; ok { - dockerComposeUrl := fmt.Sprintf("%s/%s", versionUrl, "docker-compose.yml") - dockerComposeUrl, err := appDownloadURL(dockerComposeUrl, strconv.Itoa(c.list.LastModified)) - var composeRes []byte - if err == nil { - _, composeRes, err = req_helper.HandleRequestWithClient(&c.httpClient, dockerComposeUrl, http.MethodGet, constant.TimeOut20s) - } - if err == nil { - detail.DockerCompose = string(composeRes) - } else { - result.httpFailed = true - } - } else { + downloadURL := fmt.Sprintf("%s/%s", versionUrl, app.Key+"-"+version+".tar.gz") + if v.LastModified <= 0 || detail.LastModified != v.LastModified || detail.Params != string(paramByte) || detail.DownloadUrl != downloadURL { detail.DockerCompose = "" } - detail.Params = string(paramByte) - detail.DownloadUrl = fmt.Sprintf("%s/%s", versionUrl, app.Key+"-"+version+".tar.gz") + detail.DownloadUrl = downloadURL detail.DownloadCallBackUrl = v.DownloadCallBackUrl detail.Update = true detail.LastModified = v.LastModified @@ -219,170 +198,25 @@ func (c *appSyncContext) processOneApp(item appWorkItem) appWorkResult { } app.Details = newDetails - result.app = app - return result + return app } -func (c *appSyncContext) syncAppIconsAndDetails() error { +func (c *appSyncContext) syncAppDetails() error { total := len(c.list.Apps) global.LOG.Infof("[AppStore] sync app detail start, total apps: %d", total) - - workItems := make([]appWorkItem, 0, total) - for _, l := range c.list.Apps { - app, ok := c.appsMap[l.AppProperty.Key] - if !ok { - continue + for i, definition := range c.list.Apps { + app, ok := c.appsMap[definition.AppProperty.Key] + if ok { + c.appsMap[app.Key] = c.processOneApp(appWorkItem{appDef: definition, app: app}) } - iconUrl, hasPending := c.pendingIcons[l.AppProperty.Key] - workItems = append(workItems, appWorkItem{ - appDef: l, - app: app, - iconUrl: iconUrl, - hadIcon: hasPending, - }) - } - - totalWork := len(workItems) - if totalWork == 0 { - return nil - } - - const maxWorkers = 4 - var ( - failFlag atomic.Bool - workCh = make(chan int, maxWorkers) - resultCh = make(chan appWorkResult, maxWorkers) - wg sync.WaitGroup - ) - - wg.Add(maxWorkers) - for range maxWorkers { - go func() { - defer wg.Done() - for idx := range workCh { - result := c.processOneApp(workItems[idx]) - resultCh <- result - if result.httpFailed { - failFlag.Store(true) - } - } - }() - } - go func() { wg.Wait(); close(resultCh) }() - - var fed atomic.Int32 - go func() { - for i := range workItems { - if failFlag.Load() { - break - } - workCh <- i - fed.Store(int32(i + 1)) - } - close(workCh) - }() - - var ( - completed int - icon200Count int - icon304Count int - iconFailCount int - ) - milestones := [4]int{totalWork / 4, totalWork / 2, totalWork * 3 / 4, totalWork} - nextMS := 0 - checkProgress := func() { - if nextMS < len(milestones) && completed >= milestones[nextMS] { - c.task.LogWithProgress(i18n.GetMsgByKey("SyncAppDetail"), completed, totalWork) - nextMS++ + if (i+1)%max(1, total/4) == 0 || i+1 == total { + c.task.LogWithProgress(i18n.GetMsgByKey("SyncAppDetail"), i+1, total) } } - - applyResult := func(result appWorkResult) { - c.appsMap[result.appKey] = result.app - if result.hadIcon { - switch result.iconStatus { - case http.StatusOK: - icon200Count++ - case http.StatusNotModified: - icon304Count++ - default: - iconFailCount++ - } - } - completed++ - checkProgress() - } - - for result := range resultCh { - applyResult(result) - } - - fedCount := int(fed.Load()) - if failFlag.Load() && fedCount < len(workItems) { - global.LOG.Warnf("[AppStore] HTTP failure detected, falling back to serial for remaining %d apps", len(workItems)-fedCount) - for i := fedCount; i < len(workItems); i++ { - result := c.processOneApp(workItems[i]) - applyResult(result) - } - } - - global.LOG.Infof("[AppStore] icon download completed - total: %d, success(200): %d, cached(304): %d, failed: %d", - total, icon200Count, icon304Count, iconFailCount) - + global.LOG.Infof("[AppStore] app detail sync completed, total: %d", total) return nil } -func (c *appSyncContext) downloadAppIcon(iconUrl, appKey, oldIcon string) (status int, iconField string) { - iconFileName, existingEtag := appicon.ParseIconField(oldIcon) - - reqHeaders := make(map[string]string) - if existingEtag != "" && iconFileName != "" && appicon.IconFileExists(iconFileName) { - reqHeaders["If-None-Match"] = existingEtag - } - - resp, err := req_helper.HandleRequestWithHeaders(&c.httpClient, iconUrl, http.MethodGet, constant.TimeOut20s, reqHeaders) - if err != nil { - global.LOG.Warnf("[AppStore] request icon failed url=%s, err=%v", iconUrl, err) - return 0, "" - } - - if resp.StatusCode == http.StatusNotModified { - return http.StatusNotModified, "" - } - - if resp.StatusCode != http.StatusOK { - global.LOG.Warnf("[AppStore] download icon failed url=%s, status=%d", iconUrl, resp.StatusCode) - return 0, "" - } - - if len(resp.Body) == 0 { - global.LOG.Warnf("[AppStore] download icon empty body url=%s", iconUrl) - return 0, "" - } - - if resp.Body[0] == '<' { - global.LOG.Warnf("[AppStore] download icon got HTML response url=%s", iconUrl) - return 0, "" - } - - contentType := resp.Header.Get("Content-Type") - ct := strings.TrimSpace(strings.Split(contentType, ";")[0]) - if strings.ToLower(ct) != "image/png" { - global.LOG.Warnf("[AppStore] unexpected icon content-type: %s, expected image/png, url=%s", ct, iconUrl) - } - - fileName, err := appicon.WriteIconFile(appKey, resp.Body) - if err != nil { - global.LOG.Warnf("[AppStore] write icon file failed appKey=%s, err=%v", appKey, err) - return 0, "" - } - - newEtag := resp.Header.Get("ETag") - iconField = appicon.BuildIconField(fileName, newEtag) - - return http.StatusOK, iconField -} - func (a AppService) createSyncAppStoreMetaTask(sharedCtx **appSyncContext) func(t *task.Task) error { return func(t *task.Task) (err error) { t.LogStart(i18n.GetMsgByKey("SyncAppDetail")) @@ -411,7 +245,7 @@ func (a AppService) createSyncAppStoreMetaTask(sharedCtx **appSyncContext) func( ctx.task = t ctx.appTags = make([]*model.AppTag, 0) - if err = ctx.syncAppIconsAndDetails(); err != nil { + if err = ctx.syncAppDetails(); err != nil { t.LogFailedWithErr(i18n.GetMsgByKey("SyncAppDetail"), err) return err } diff --git a/agent/app/service/app_upgrade.go b/agent/app/service/app_upgrade.go index 26ad34b92..097b29032 100644 --- a/agent/app/service/app_upgrade.go +++ b/agent/app/service/app_upgrade.go @@ -216,8 +216,9 @@ func (u *appUpgradeContext) prepare(t *task.Task) error { if err != nil { return err } - u.detail.DockerCompose = string(composeContent) - _ = appDetailRepo.Update(context.Background(), u.detail) + if err := cacheAppDetailCompose(&u.detail, composeContent); err != nil { + return err + } } if strings.TrimSpace(u.detail.DockerCompose) == "" && strings.TrimSpace(u.req.DockerCompose) == "" { return buserr.WithName("ErrFileNotFound", "docker-compose.yml") diff --git a/agent/app/service/app_utils.go b/agent/app/service/app_utils.go index f8837465e..c5e1113ba 100644 --- a/agent/app/service/app_utils.go +++ b/agent/app/service/app_utils.go @@ -28,6 +28,7 @@ import ( "github.com/1Panel-dev/1Panel/agent/constant" "github.com/1Panel-dev/1Panel/agent/global" "github.com/1Panel-dev/1Panel/agent/i18n" + "github.com/1Panel-dev/1Panel/agent/utils/appicon" "github.com/1Panel-dev/1Panel/agent/utils/cmd" "github.com/1Panel-dev/1Panel/agent/utils/common" "github.com/1Panel-dev/1Panel/agent/utils/compose" @@ -993,8 +994,13 @@ func downloadApp(app model.App, appDetail model.AppDetail, appInstall *model.App if err = os.WriteFile(cacheVersionPath, []byte(storeVersion), constant.FilePerm); err != nil { return err } - appDetail.Update = false - _ = appDetailRepo.Update(context.Background(), appDetail) + marked, err := appDetailRepo.MarkDownloaded(appDetail) + if err != nil { + return err + } + if !marked { + return buserr.WithDetail("ErrGetCompose", "application resources changed; retry the operation", nil) + } return } @@ -1268,9 +1274,8 @@ func getAppDetails(details []model.AppDetail, versions []dto.AppConfigVersion) m return appDetails } -func getApps(oldApps []model.App, items []dto.AppDefine, systemVersion string, task *task.Task) (map[string]model.App, map[string]string) { +func getApps(oldApps []model.App, items []dto.AppDefine, systemVersion string, task *task.Task) map[string]model.App { apps := make(map[string]model.App, len(oldApps)) - pendingIcons := make(map[string]string, len(items)) for _, old := range oldApps { old.Status = constant.AppTakeDown apps[old.Key] = old @@ -1311,12 +1316,14 @@ func getApps(oldApps []model.App, items []dto.AppDefine, systemVersion string, t app.Architectures = strings.Join(config.Architectures, ",") app.GpuSupport = config.GpuSupport app.BatchInstallSupport = config.BatchInstallSupport - if item.Icon != "" { - pendingIcons[key] = item.Icon + if appicon.IsRemoteURL(item.Icon) { + app.Icon = item.Icon + } else if item.Icon != "" { + global.LOG.Warnf("[AppStore] invalid icon URL for app %s, keeping existing icon", key) } apps[key] = app } - return apps, pendingIcons + return apps } func handleLocalAppDetail(versionDir string, appDetail *model.AppDetail) error { @@ -1579,6 +1586,7 @@ func handleInstalled(appInstallList []model.AppInstall, updated, sync, checkUpda HttpPort: installed.HttpPort, HttpsPort: installed.HttpsPort, AppName: installed.App.Name, + Icon: appicon.RemoteURL(installed.App.Icon), AppKey: installed.App.Key, AppType: installed.App.Type, Path: installed.GetPath(), diff --git a/agent/app/service/runtime_get_test.go b/agent/app/service/runtime_get_test.go new file mode 100644 index 000000000..562beb3fc --- /dev/null +++ b/agent/app/service/runtime_get_test.go @@ -0,0 +1,115 @@ +package service + +import ( + "encoding/json" + "reflect" + "slices" + "testing" + + "github.com/1Panel-dev/1Panel/agent/app/dto" + "github.com/1Panel-dev/1Panel/agent/app/model" + "github.com/1Panel-dev/1Panel/agent/constant" +) + +func TestGetPHPRuntimeRefreshesVersionOptionsWithoutChangingConfig(t *testing.T) { + for _, resource := range []string{constant.AppResourceRemote, constant.AppResourceLocal, constant.AppResourceCustom} { + t.Run(resource, func(t *testing.T) { + db := appCacheTestDB(t) + if err := db.AutoMigrate(&model.Runtime{}); err != nil { + t.Fatal(err) + } + app := model.App{Key: "php", Type: constant.RuntimePHP, Resource: resource} + if err := db.Create(&app).Error; err != nil { + t.Fatal(err) + } + oldVersion := dto.AppFormValue{Label: "8.1.34", Value: "8.1.34"} + newVersion := dto.AppFormValue{Label: "8.1.35", Value: "8.1.35"} + savedForm := dto.AppForm{FormFields: []dto.AppFormFields{ + {EnvKey: "PHP_VERSION", Type: "select", Default: oldVersion.Value, Values: []dto.AppFormValue{oldVersion}}, + {EnvKey: "PHP_EXTENSIONS", Type: "select", Multiple: true, Values: []dto.AppFormValue{{Label: "redis", Value: "redis"}}}, + {EnvKey: "PANEL_APP_PORT_HTTP", Type: "number"}, + {EnvKey: "CONTAINER_PACKAGE_URL", Type: "select", Values: []dto.AppFormValue{{Label: "custom mirror", Value: "https://mirror.example"}}}, + }} + encode := func(form dto.AppForm) string { + t.Helper() + data, err := json.Marshal(form) + if err != nil { + t.Fatal(err) + } + return string(data) + } + detail := model.AppDetail{AppId: app.ID, Version: "8.1", Params: encode(savedForm), Update: true} + if err := db.Create(&detail).Error; err != nil { + t.Fatal(err) + } + otherMajor := model.AppDetail{AppId: app.ID, Version: "8.2", Params: `{"formFields":[{"envKey":"PHP_VERSION","values":[{"label":"8.2.31","value":"8.2.31"}]}]}`} + if err := db.Create(&otherMajor).Error; err != nil { + t.Fatal(err) + } + runtime := model.Runtime{ + Name: "existing-php", Type: constant.RuntimePHP, Resource: constant.ResourceAppstore, + Version: detail.Version, AppDetailID: detail.ID, Image: "custom-php:existing", Params: detail.Params, + Env: "PHP_VERSION=8.1.34\nPHP_EXTENSIONS=redis,custom-extension\nPANEL_APP_PORT_HTTP=9070\nCONTAINER_PACKAGE_URL=https://mirror.example\nCONTAINER_NAME=my-php\nCUSTOM_SETTING=keep-me\n", + DockerCompose: "existing compose", Remark: "keep remark", + } + if err := db.Create(&runtime).Error; err != nil { + t.Fatal(err) + } + service := NewRuntimeService() + before, err := service.Get(runtime.ID) + if err != nil { + t.Fatal(err) + } + latestForm := dto.AppForm{FormFields: []dto.AppFormFields{ + {EnvKey: "PHP_VERSION", Type: "select", Default: newVersion.Value, Values: []dto.AppFormValue{newVersion, oldVersion}}, + {EnvKey: "PHP_EXTENSIONS", Type: "select", Multiple: true, Default: "opcache"}, + {EnvKey: "PANEL_APP_PORT_HTTP", Default: 9000}, + }} + removedForm := dto.AppForm{FormFields: []dto.AppFormFields{ + {EnvKey: "PHP_VERSION", Type: "select", Default: newVersion.Value, Values: []dto.AppFormValue{newVersion}}, + }} + for _, tc := range []struct { + name string + params string + values []dto.AppFormValue + }{ + {"new patch on reopen", encode(latestForm), []dto.AppFormValue{newVersion, oldVersion}}, + {"selected version removed", encode(removedForm), []dto.AppFormValue{newVersion, oldVersion}}, + {"invalid metadata", "{invalid", []dto.AppFormValue{oldVersion}}, + {"missing metadata", "", []dto.AppFormValue{oldVersion}}, + {"missing version field", `{"formFields":[{"envKey":"PHP_EXTENSIONS"}]}`, []dto.AppFormValue{oldVersion}}, + {"empty version list", `{"formFields":[{"envKey":"PHP_VERSION","values":[]}]}`, []dto.AppFormValue{oldVersion}}, + } { + t.Run(tc.name, func(t *testing.T) { + if err := db.Model(&detail).UpdateColumn("params", tc.params).Error; err != nil { + t.Fatal(err) + } + got, err := service.Get(runtime.ID) + if err != nil { + t.Fatal(err) + } + want := *before + want.AppParams = slices.Clone(before.AppParams) + want.AppParams[0].Values = tc.values + if !reflect.DeepEqual(*got, want) { + t.Fatalf("edit response changed configuration or returned stale options:\ngot: %+v\nwant: %+v", *got, want) + } + var stored model.Runtime + if err := db.First(&stored, runtime.ID).Error; err != nil { + t.Fatal(err) + } + if stored.Params != runtime.Params || stored.Env != runtime.Env || stored.Image != runtime.Image || stored.DockerCompose != runtime.DockerCompose || stored.AppDetailID != runtime.AppDetailID || stored.Version != runtime.Version { + t.Fatal("opening edit mutated the saved runtime") + } + var storedDetail model.AppDetail + if err := db.First(&storedDetail, detail.ID).Error; err != nil { + t.Fatal(err) + } + if !storedDetail.Update { + t.Fatal("opening edit cleared the pending template update") + } + }) + } + }) + } +} diff --git a/agent/utils/appicon/appicon.go b/agent/utils/appicon/appicon.go index f276dd8c1..7d31319e2 100644 --- a/agent/utils/appicon/appicon.go +++ b/agent/utils/appicon/appicon.go @@ -6,13 +6,42 @@ import ( "os" "path" "path/filepath" + "strconv" "strings" + "unicode" "github.com/1Panel-dev/1Panel/agent/global" ) const IconPrefix = "app_" +func IsRemoteURL(icon string) bool { + for _, ch := range icon { + if unicode.IsSpace(ch) || unicode.IsControl(ch) || ch == '\\' { + return false + } + } + u, err := url.Parse(icon) + if err != nil || (!strings.EqualFold(u.Scheme, "http") && !strings.EqualFold(u.Scheme, "https")) || u.Hostname() == "" || u.User != nil { + return false + } + if u.Port() != "" { + port, err := strconv.Atoi(u.Port()) + if err != nil || port > 65535 { + return false + } + } + _, err = url.QueryUnescape(u.RawQuery) + return err == nil +} + +func RemoteURL(icon string) string { + if IsRemoteURL(icon) { + return icon + } + return "" +} + func IsIconFile(icon string) bool { return strings.HasPrefix(icon, IconPrefix) } diff --git a/agent/utils/appicon/appicon_test.go b/agent/utils/appicon/appicon_test.go new file mode 100644 index 000000000..38b7256e3 --- /dev/null +++ b/agent/utils/appicon/appicon_test.go @@ -0,0 +1,37 @@ +package appicon + +import "testing" + +func TestRemoteURLPreservesValidSources(t *testing.T) { + for _, value := range []string{ + "https://cdn.example/icon.png", + "http://cdn.example:8080/icon.png", + "HTTPS://cdn.example/icon.png?token=a%2Bb&v=42#image", + "https://[::1]:8443/icon.png", + } { + t.Run(value, func(t *testing.T) { + if !IsRemoteURL(value) || RemoteURL(value) != value { + t.Fatalf("URL was rejected or rewritten: %q", value) + } + }) + } +} + +func TestRemoteURLRejectsOtherIconFormatsAndAmbiguousURLs(t *testing.T) { + for _, value := range []string{ + "", "app_php.png?etag=42", "iVBORw0KGgo=", "data:image/png;base64,iVBORw0KGgo=", + "javascript:alert(1)", "file:///icon.png", "//cdn.example/icon.png", "https:///icon.png", + "https://", "https://user:secret@cdn.example/icon.png", "https://@cdn.example/icon.png", + " https://cdn.example/icon.png", "https://cdn.example/icon.png ", "https://cdn.example/icon image.png", + "https://cdn.example/\nicon.png", "https://cdn.example/icon.png\r\nLocation: https://other.example", + "https://cdn.example/\\other.png", "https://cdn.example:bad/icon.png", "https://cdn.example/%zz", + "https://cdn.example/icon.png?token=%zz", "https://cdn.example/\u00a0icon.png", "https://cdn.example/\x7ficon.png", + "https://cdn.example:65536/icon.png", + } { + t.Run(value, func(t *testing.T) { + if IsRemoteURL(value) || RemoteURL(value) != "" { + t.Fatalf("invalid or historical icon was classified as a remote URL: %q", value) + } + }) + } +} diff --git a/core/cmd/server/docs/docs.go b/core/cmd/server/docs/docs.go index 6a4976157..c9afad8d3 100644 --- a/core/cmd/server/docs/docs.go +++ b/core/cmd/server/docs/docs.go @@ -4593,6 +4593,9 @@ const docTemplate = `{ "schema": { "type": "file" } + }, + "302": { + "description": "redirect to the official icon URL" } }, "security": [ @@ -5390,6 +5393,17 @@ const docTemplate = `{ "consumes": [ "application/json" ], + "parameters": [ + { + "description": "request", + "in": "body", + "name": "request", + "required": true, + "schema": { + "$ref": "#/definitions/request.AppUpdateVersion" + } + } + ], "responses": { "200": { "description": "OK", @@ -49266,6 +49280,23 @@ const docTemplate = `{ ], "type": "object" }, + "request.AppUpdateVersion": { + "properties": { + "appInstallID": { + "type": "integer" + }, + "loadCompose": { + "type": "boolean" + }, + "updateVersion": { + "type": "string" + } + }, + "required": [ + "appInstallID" + ], + "type": "object" + }, "response.AppConfig": { "properties": { "advanced": { @@ -49298,6 +49329,9 @@ const docTemplate = `{ "httpPort": { "type": "integer" }, + "isEdit": { + "type": "boolean" + }, "memoryLimit": { "type": "number" }, @@ -49576,6 +49610,9 @@ const docTemplate = `{ "gpuSupport": { "type": "boolean" }, + "icon": { + "type": "string" + }, "id": { "type": "integer" }, diff --git a/core/cmd/server/docs/swagger.json b/core/cmd/server/docs/swagger.json index 8201d51a5..8bb005534 100644 --- a/core/cmd/server/docs/swagger.json +++ b/core/cmd/server/docs/swagger.json @@ -4542,6 +4542,9 @@ "schema": { "type": "file" } + }, + "302": { + "description": "redirect to the official icon URL" } }, "security": [ @@ -5339,6 +5342,17 @@ "consumes": [ "application/json" ], + "parameters": [ + { + "description": "request", + "in": "body", + "name": "request", + "required": true, + "schema": { + "$ref": "#/definitions/request.AppUpdateVersion" + } + } + ], "responses": { "200": { "description": "OK", @@ -47477,6 +47491,23 @@ ], "type": "object" }, + "request.AppUpdateVersion": { + "properties": { + "appInstallID": { + "type": "integer" + }, + "loadCompose": { + "type": "boolean" + }, + "updateVersion": { + "type": "string" + } + }, + "required": [ + "appInstallID" + ], + "type": "object" + }, "response.AppConfig": { "properties": { "advanced": { @@ -47503,6 +47534,9 @@ "hostMode": { "type": "boolean" }, + "isEdit": { + "type": "boolean" + }, "memoryLimit": { "type": "number" }, @@ -47763,6 +47797,9 @@ "gpuSupport": { "type": "boolean" }, + "icon": { + "type": "string" + }, "id": { "type": "integer" }, diff --git a/core/init/proxy/app_icon_test.go b/core/init/proxy/app_icon_test.go new file mode 100644 index 000000000..8747f2215 --- /dev/null +++ b/core/init/proxy/app_icon_test.go @@ -0,0 +1,34 @@ +package proxy + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +type iconRedirectTransport func(*http.Request) (*http.Response, error) + +func (f iconRedirectTransport) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } + +func TestLocalAppIconRedirectIsPassedToBrowser(t *testing.T) { + const icon = "https://cdn.example/icon.png?signature=a%2Bb&v=42" + calls := 0 + proxy := newLocalAgentProxy(iconRedirectTransport(func(req *http.Request) (*http.Response, error) { + calls++ + if req.URL.Host != "unix" || req.URL.Path != "/api/v2/apps/icon/php" { + t.Fatalf("proxy attempted to fetch a remote image: %s", req.URL) + } + return &http.Response{ + StatusCode: http.StatusFound, + Header: http.Header{"Location": {icon}, "Cache-Control": {"no-cache"}}, + Body: io.NopCloser(strings.NewReader("")), + }, nil + })) + rw := httptest.NewRecorder() + proxy.ServeHTTP(rw, httptest.NewRequest(http.MethodGet, "/api/v2/apps/icon/php", nil)) + if rw.Code != http.StatusFound || rw.Header().Get("Location") != icon || rw.Header().Get("Cache-Control") != "no-cache" || calls != 1 { + t.Fatalf("local proxy did not preserve the redirect: status=%d headers=%v calls=%d", rw.Code, rw.Header(), calls) + } +} diff --git a/frontend/src/api/interface/app.ts b/frontend/src/api/interface/app.ts index 03de17f32..c02328250 100644 --- a/frontend/src/api/interface/app.ts +++ b/frontend/src/api/interface/app.ts @@ -53,6 +53,7 @@ export namespace App { export interface AppItem { name: string; + icon?: string; key: string; id: number; description: string; @@ -291,6 +292,7 @@ export namespace App { feishuConfigured?: boolean; version: string; detailId: number; + dockerCompose?: string; } export interface InstallParams { @@ -318,6 +320,7 @@ export namespace App { modelAccountBaseURL: string; params: InstallParams[]; rawCompose?: string; + isEdit?: boolean; cpuQuota: number; memoryLimit: number; memoryUnit: string; @@ -341,6 +344,7 @@ export namespace App { export interface AppUpdateVersionReq { appInstallID: number; updateVersion?: string; + loadCompose?: boolean; } export interface AppIgnoreReq { diff --git a/frontend/src/api/modules/app.ts b/frontend/src/api/modules/app.ts index 21113a517..0b9f78caa 100644 --- a/frontend/src/api/modules/app.ts +++ b/frontend/src/api/modules/app.ts @@ -2,6 +2,7 @@ import http from '@/api'; import { ResPage } from '../interface'; import { App } from '../interface/app'; import { TimeoutEnum } from '@/enums/http-enum'; +import { resolveAppIconUrl } from '@/utils/app-icon'; export const syncApp = (req: App.AppStoreSync) => { return http.post('apps/sync/remote', req); @@ -99,8 +100,9 @@ export const getAppDefaultConfig = (key: string, name: string) => { return http.post(`apps/installed/conf`, { type: key, name: name }); }; -export const getAppInstallParams = (id: number) => { - return http.get(`apps/installed/params/${id}`); +export const getAppInstallParams = (id: number, node?: string) => { + const params = node ? `?operateNode=${encodeURIComponent(node)}` : ''; + return http.get(`apps/installed/params/${id}${params}`); }; export const updateAppInstallParams = (req: any) => { @@ -135,10 +137,9 @@ export const getCurrentNodeCustomAppConfig = () => { return http.get(`/custom/app/config`); }; -export function getAppIconUrl(appKey: string, node?: string): string { +export function getAppIconUrl(appKey: string, node?: string, icon = ''): string { const baseURL = import.meta.env.VITE_API_URL as string; - const params = node ? `?operateNode=${node}` : ''; - return `${baseURL}/apps/icon/${appKey}${params}`; + return resolveAppIconUrl(appKey, node, icon, baseURL); } export const installAppToNodes = (param: App.InstallAppToNodes) => { diff --git a/frontend/src/utils/app-icon.ts b/frontend/src/utils/app-icon.ts new file mode 100644 index 000000000..2f3940b50 --- /dev/null +++ b/frontend/src/utils/app-icon.ts @@ -0,0 +1,26 @@ +export function isRemoteAppIcon(icon: string): boolean { + if ( + !/^https?:\/\/[^/?#]/i.test(icon) || + /^https?:\/\/[^/?#]*@/i.test(icon) || + Array.from(icon).some( + (char) => char <= ' ' || (char >= '\u007f' && char <= '\u009f') || /\s/.test(char) || char === '\\', + ) || + /%(?![\da-f]{2})/i.test(icon) + ) { + return false; + } + try { + const url = new URL(icon); + return !!url.hostname && !url.username && !url.password; + } catch { + return false; + } +} + +export function resolveAppIconUrl(appKey: string, node: string | undefined, icon: string, baseURL: string): string { + if (isRemoteAppIcon(icon)) { + return icon; + } + const params = node ? `?operateNode=${encodeURIComponent(node)}` : ''; + return `${baseURL}/apps/icon/${encodeURIComponent(appKey)}${params}`; +} diff --git a/frontend/src/views/app-store/apps/app/index.vue b/frontend/src/views/app-store/apps/app/index.vue index 2458cf958..ba447f05d 100644 --- a/frontend/src/views/app-store/apps/app/index.vue +++ b/frontend/src/views/app-store/apps/app/index.vue @@ -3,7 +3,9 @@
- + + +
diff --git a/frontend/src/views/app-store/detail/index.vue b/frontend/src/views/app-store/detail/index.vue index 4b4a40f0e..0ad660a74 100644 --- a/frontend/src/views/app-store/detail/index.vue +++ b/frontend/src/views/app-store/detail/index.vue @@ -4,12 +4,15 @@
- App Icon + shape="square" + fit="contain" + class="!w-full !h-full rounded" + > + +
diff --git a/frontend/src/views/app-store/installed/app/card.vue b/frontend/src/views/app-store/installed/app/card.vue index d9587f023..941999d76 100644 --- a/frontend/src/views/app-store/installed/app/card.vue +++ b/frontend/src/views/app-store/installed/app/card.vue @@ -6,6 +6,7 @@ diff --git a/frontend/src/views/app-store/installed/app/icon.vue b/frontend/src/views/app-store/installed/app/icon.vue index 13cf0ec1b..a902aaac1 100644 --- a/frontend/src/views/app-store/installed/app/icon.vue +++ b/frontend/src/views/app-store/installed/app/icon.vue @@ -1,6 +1,13 @@ @@ -10,6 +17,7 @@ import { getAppIconUrl } from '@/api/modules/app'; interface Props { currentNode: string; appKey: string; + icon?: string; } defineProps(); diff --git a/frontend/src/views/app-store/installed/ignore/create/index.vue b/frontend/src/views/app-store/installed/ignore/create/index.vue index 9193f4ae6..05ccb0769 100644 --- a/frontend/src/views/app-store/installed/ignore/create/index.vue +++ b/frontend/src/views/app-store/installed/ignore/create/index.vue @@ -65,7 +65,7 @@ const acceptParams = (apppInstall: App.AppInstalled) => { const getVersions = async () => { try { - const res = await getAppUpdateVersions({ appInstallID: appInstallID.value }); + const res = await getAppUpdateVersions({ appInstallID: appInstallID.value, loadCompose: false }); versions.value = res.data || []; if (versions.value.length > 0) { req.appDetailID = versions.value[0].detailId; diff --git a/frontend/src/views/app-store/installed/upgrade/index.vue b/frontend/src/views/app-store/installed/upgrade/index.vue index a45ab900d..6fde42cb4 100644 --- a/frontend/src/views/app-store/installed/upgrade/index.vue +++ b/frontend/src/views/app-store/installed/upgrade/index.vue @@ -5,7 +5,7 @@ :resource="resourceName" @close="handleClose" > -
+
@@ -41,6 +41,13 @@ :label="version.version" > + + {{ $t('commons.button.retry') }} + 2. {{ $t('app.openclawHttpsUpgradeNoticeItem2') }}
-