From f2335d313cbef78d9f19259533acc9cd1b59e9a5 Mon Sep 17 00:00:00 2001 From: CityFun <31820853+zhengkunwang223@users.noreply.github.com> Date: Mon, 1 Sep 2025 17:39:40 +0800 Subject: [PATCH] feat: Optimize file log reading (#10218) --- agent/app/dto/file.go | 8 +++ agent/app/dto/response/file.go | 1 + agent/app/service/file.go | 18 +++--- agent/app/service/website.go | 6 +- agent/go.sum | 2 - agent/utils/files/utils.go | 14 +++-- frontend/src/components/log/file/index.vue | 66 ++++++++++++++++++---- 7 files changed, 86 insertions(+), 29 deletions(-) create mode 100644 agent/app/dto/file.go diff --git a/agent/app/dto/file.go b/agent/app/dto/file.go new file mode 100644 index 000000000..f7fe3373a --- /dev/null +++ b/agent/app/dto/file.go @@ -0,0 +1,8 @@ +package dto + +type LogFileRes struct { + Lines []string `json:"lines"` + IsEndOfFile bool `json:"isEndOfFile"` + TotalPages int `json:"totalPages"` + TotalLines int `json:"totalLines"` +} diff --git a/agent/app/dto/response/file.go b/agent/app/dto/response/file.go index 051029d2c..77d4f11f6 100644 --- a/agent/app/dto/response/file.go +++ b/agent/app/dto/response/file.go @@ -44,6 +44,7 @@ type FileLineContent struct { TaskStatus string `json:"taskStatus"` Lines []string `json:"lines"` Scope string `json:"scope"` + TotalLines int `json:"totalLines"` } type FileExist struct { diff --git a/agent/app/service/file.go b/agent/app/service/file.go index d030b6562..6b5245358 100644 --- a/agent/app/service/file.go +++ b/agent/app/service/file.go @@ -568,37 +568,41 @@ func (f *FileService) ReadLogByLine(req request.FileReadByLineReq) (*response.Fi var ( lines []string isEndOfFile bool - total int scope string + logFileRes *dto.LogFileRes ) - if stat.Size() > 500*1024*1024 { lines, err = files.TailFromEnd(logFilePath, req.PageSize) isEndOfFile = true scope = "tail" } else { - lines, isEndOfFile, total, err = files.ReadFileByLine(logFilePath, req.Page, req.PageSize, req.Latest) + logFileRes, err = files.ReadFileByLine(logFilePath, req.Page, req.PageSize, req.Latest) if err != nil { return nil, err } - if req.Latest && req.Page == 1 && len(lines) < 1000 && total > 1 { - preLines, _, _, err := files.ReadFileByLine(logFilePath, total-1, req.PageSize, false) + if req.Latest && req.Page == 1 && len(logFileRes.Lines) < 1000 && logFileRes.TotalPages > 1 { + res, err := files.ReadFileByLine(logFilePath, logFileRes.TotalPages-1, req.PageSize, false) if err != nil { return nil, err } - lines = append(preLines, lines...) + logFileRes.Lines = append(res.Lines, logFileRes.Lines...) } scope = "page" + lines = logFileRes.Lines } res := &response.FileLineContent{ End: isEndOfFile, Path: logFilePath, - Total: total, TaskStatus: taskStatus, Lines: lines, Scope: scope, } + if logFileRes != nil { + res.TotalLines = logFileRes.TotalLines + res.Total = logFileRes.TotalPages + res.End = logFileRes.IsEndOfFile + } return res, nil } diff --git a/agent/app/service/website.go b/agent/app/service/website.go index e5ab5b686..047304c15 100644 --- a/agent/app/service/website.go +++ b/agent/app/service/website.go @@ -1233,13 +1233,13 @@ func (w WebsiteService) OpWebsiteLog(req request.WebsiteLogReq) (*response.Websi } } filePath := path.Join(sitePath, "log", req.LogType) - lines, end, _, err := files.ReadFileByLine(filePath, req.Page, req.PageSize, false) + logFileRes, err := files.ReadFileByLine(filePath, req.Page, req.PageSize, false) if err != nil { return nil, err } - res.End = end + res.End = logFileRes.IsEndOfFile res.Path = filePath - res.Content = strings.Join(lines, "\n") + res.Content = strings.Join(logFileRes.Lines, "\n") return res, nil case constant.DisableLog: params := dto.NginxParam{} diff --git a/agent/go.sum b/agent/go.sum index c9014a6ac..fa84664b7 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -919,8 +919,6 @@ github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2 github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= -github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= -github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/upyun/go-sdk v2.1.0+incompatible h1:OdjXghQ/TVetWV16Pz3C1/SUpjhGBVPr+cLiqZLLyq0= diff --git a/agent/utils/files/utils.go b/agent/utils/files/utils.go index f137dcd8a..077a486c2 100644 --- a/agent/utils/files/utils.go +++ b/agent/utils/files/utils.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "fmt" + "github.com/1Panel-dev/1Panel/agent/app/dto" "github.com/1Panel-dev/1Panel/agent/buserr" "github.com/1Panel-dev/1Panel/agent/constant" "github.com/1Panel-dev/1Panel/agent/utils/req_helper" @@ -161,7 +162,7 @@ func TailFromEnd(filename string, lines int) ([]string, error) { return result, nil } -func ReadFileByLine(filename string, page, pageSize int, latest bool) (lines []string, isEndOfFile bool, total int, err error) { +func ReadFileByLine(filename string, page, pageSize int, latest bool) (res *dto.LogFileRes, err error) { if !NewFileOp().Stat(filename) { return } @@ -185,7 +186,10 @@ func ReadFileByLine(filename string, page, pageSize int, latest bool) (lines []s if err != nil { return } - total = (totalLines + pageSize - 1) / pageSize + res = &dto.LogFileRes{} + total := (totalLines + pageSize - 1) / pageSize + res.TotalPages = total + res.TotalLines = totalLines reader := bufio.NewReaderSize(file, 8192) if latest { @@ -194,7 +198,7 @@ func ReadFileByLine(filename string, page, pageSize int, latest bool) (lines []s currentLine := 0 startLine := (page - 1) * pageSize endLine := startLine + pageSize - + lines := make([]string, 0, pageSize) for { line, _, err := reader.ReadLine() if err == io.EOF { @@ -208,8 +212,8 @@ func ReadFileByLine(filename string, page, pageSize int, latest bool) (lines []s break } } - - isEndOfFile = currentLine < endLine + res.Lines = lines + res.IsEndOfFile = currentLine < endLine return } diff --git a/frontend/src/components/log/file/index.vue b/frontend/src/components/log/file/index.vue index ca261a854..8e4fb6064 100644 --- a/frontend/src/components/log/file/index.vue +++ b/frontend/src/components/log/file/index.vue @@ -37,7 +37,7 @@ > - + @@ -151,13 +151,20 @@ const containerHeight = ref(500); const visibleCount = computed(() => Math.ceil(containerHeight.value / logHeight)); const startIndex = ref(0); const lastScrollTop = ref(0); - +const totalLines = ref(0); +const stopReading = ref(false); const visibleLogs = computed(() => { - return logs.value.slice(startIndex.value, startIndex.value + visibleCount.value); + const safeStartIndex = Math.max(0, Math.min(startIndex.value, Math.max(0, logs.value.length - visibleCount.value))); + if (safeStartIndex !== startIndex.value) { + nextTick(() => { + startIndex.value = safeStartIndex; + }); + } + return logs.value.slice(safeStartIndex, safeStartIndex + visibleCount.value); }); const onScroll = async () => { - if (!logContainer.value || isLoading.value) return; + if (!logContainer.value) return; const scrollTop = logContainer.value.scrollTop; const scrollHeight = logContainer.value.scrollHeight; @@ -165,8 +172,15 @@ const onScroll = async () => { lastScrollTop.value = scrollTop; - const newStartIndex = Math.max(0, Math.floor(scrollTop / logHeight)); - startIndex.value = newStartIndex; + if (!isLoading.value) { + const newStartIndex = Math.max( + 0, + Math.min(Math.floor(scrollTop / logHeight), Math.max(0, logs.value.length - visibleCount.value)), + ); + startIndex.value = newStartIndex; + } + + if (isLoading.value) return; if (scrollTop <= 50 && readReq.page > 1) { if (minPage.value <= 1) { @@ -178,7 +192,7 @@ const onScroll = async () => { return; } - if (scrollHeight - scrollTop - clientHeight <= 50 && !end.value && maxPage.value > 1) { + if (scrollHeight - scrollTop - clientHeight <= 50 && !end.value && readReq.page < maxPage.value) { readReq.page = maxPage.value; await getContent(false); } @@ -210,6 +224,7 @@ const changeTail = (fromOutSide: boolean) => { const clearLog = (): void => { logs.value = []; + startIndex.value = 0; readReq.page = 1; lastLogs.value = []; }; @@ -231,6 +246,12 @@ const getContent = async (pre: boolean) => { isLoading.value = false; firstLoading.value = false; } + if (res.data.scope != 'tail' && !pre && res.data.end && res.data.totalLines == totalLines.value) { + isLoading.value = false; + return; + } + + totalLines.value = res.data.totalLines; if (res.data.scope == 'tail') { showTail.value = false; @@ -238,6 +259,7 @@ const getContent = async (pre: boolean) => { if (res.data.taskStatus && res.data.taskStatus !== 'Executing') { isTailDisabled.value = true; + tailLog.value = false; } logPath.value = res.data.path; @@ -279,18 +301,29 @@ const getContent = async (pre: boolean) => { if (end.value) { logs.value = [...lastLogs.value, ...newLogs]; } else { - logs.value = [...logs.value, ...newLogs]; + if (newLogs.length > logs.value.length) { + logs.value = newLogs; + } else { + logs.value = [...logs.value, ...newLogs]; + } } } } + nextTick(() => { if (logContainer.value) { if (pre) { if (readReq.page > 1) { const addedLines = newLogs.length; - const newScrollPosition = lastScrollTop.value + (addedLines * logHeight) / 3; + const newScrollPosition = lastScrollTop.value + addedLines * logHeight; logContainer.value.scrollTop = newScrollPosition; - startIndex.value = Math.max(0, Math.floor(newScrollPosition / logHeight)); + startIndex.value = Math.max( + 0, + Math.min( + Math.floor(newScrollPosition / logHeight), + Math.max(0, logs.value.length - visibleCount.value), + ), + ); } } else { logContainer.value.scrollTop = totalHeight.value; @@ -313,24 +346,33 @@ const getContent = async (pre: boolean) => { } else { minPage.value = res.data.total; } + } else { + maxPage.value = Math.max(maxPage.value, readReq.page); } if (logs.value && logs.value.length > 3000) { + const removedCount = readReq.pageSize; if (pre) { - logs.value.splice(logs.value.length - readReq.pageSize, readReq.pageSize); + logs.value.splice(logs.value.length - removedCount, removedCount); if (maxPage.value > 1) { maxPage.value--; } } else { - logs.value.splice(0, readReq.pageSize); + logs.value.splice(0, removedCount); + startIndex.value = Math.max(0, startIndex.value - removedCount); if (minPage.value > 1) { minPage.value++; } } + logCount.value = logs.value.length; } isLoading.value = false; }; const onCloseLog = async () => { + if (stopReading.value) { + return; + } + stopReading.value = true; tailLog.value = false; if (timer) { clearInterval(Number(timer));