From 0ac3980125917a3800e92dd6a49a386947fe7fcb Mon Sep 17 00:00:00 2001 From: Runxi Yu Date: Sat, 22 Mar 2025 21:46:35 +0800 Subject: [PATCH] Cache tree-building --- go.mod | 3 +++ go.sum | 6 ++++++ http_handle_repo_index.go | 63 +++++++++++++++++++++++++++++++++++++++++++++-------- diff --git a/go.mod b/go.mod index 44e44709d3a8c8a129a7f69a25af8dcc406f57b2..c15a922eae967eaf655e146bb2a16d63d88dc222 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ require ( github.com/alecthomas/chroma/v2 v2.15.0 github.com/alexedwards/argon2id v1.0.0 + github.com/dgraph-io/ristretto/v2 v2.1.0 github.com/dustin/go-humanize v1.0.1 github.com/gliderlabs/ssh v0.3.8 github.com/go-git/go-git/v5 v5.14.0 @@ -24,6 +25,7 @@ github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.1.6 // indirect github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect github.com/aymerick/douceur v0.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudflare/circl v1.6.0 // indirect github.com/cyphar/filepath-securejoin v0.4.1 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect @@ -38,6 +40,7 @@ github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/pjbgf/sha1cd v0.3.2 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.3.1 // indirect github.com/tdewolff/parse/v2 v2.7.21 // indirect diff --git a/go.sum b/go.sum index 524fb9852aa4e0ecea375adcad9fb0268a06e145..c4c2ec139079ae5295a5857a7ebc841aa26c3ee4 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudflare/circl v1.6.0 h1:cr5JKic4HI+LkINy2lg3W2jF8sHCVTBncJr5gIIq7qk= github.com/cloudflare/circl v1.6.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= @@ -26,6 +28,10 @@ github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/ristretto/v2 v2.1.0 h1:59LjpOJLNDULHh8MC4UaegN52lC4JnO2dITsie/Pa8I= +github.com/dgraph-io/ristretto/v2 v2.1.0/go.mod h1:uejeqfYXpUomfse0+lO+13ATz4TypQYLJZzBSAemuB4= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= diff --git a/http_handle_repo_index.go b/http_handle_repo_index.go index 7dc7826dbb2cb5d61a553efbbc67949200117f4b..44151a0195d173754ec8478c930756756e77b336 100644 --- a/http_handle_repo_index.go +++ b/http_handle_repo_index.go @@ -4,22 +4,48 @@ package main import ( + "html/template" "iter" "net/http" "strings" + "time" + "github.com/dgraph-io/ristretto/v2" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/plumbing/storer" + "go.lindenii.runxiyu.org/lindenii-common/clog" ) +type indexPageCacheEntry struct { + DisplayTree []displayTreeEntry + ReadmeFilename string + ReadmeRendered template.HTML +} + +var indexPageCache *ristretto.Cache[[]byte, indexPageCacheEntry] + +func init() { + var err error + indexPageCache, err = ristretto.NewCache(&ristretto.Config[[]byte, indexPageCacheEntry]{ + NumCounters: 1e4, + MaxCost: 1 << 30, + BufferItems: 64, + }) + if err != nil { + clog.Fatal(1, "Error initializing indexPageCache: "+err.Error()) + } +} + func httpHandleRepoIndex(writer http.ResponseWriter, _ *http.Request, params map[string]any) { var repo *git.Repository var repoName string var groupPath []string var refHash plumbing.Hash + var refHashSlice []byte var err error + var logOptions git.LogOptions var commitIter object.CommitIter var commitIterSeq iter.Seq[*object.Commit] var commitObj *object.Commit @@ -27,7 +53,6 @@ var tree *object.Tree var notes []string var branches []string var branchesIter storer.ReferenceIter - var logOptions git.LogOptions repo, repoName, groupPath = params["repo"].(*git.Repository), params["repo_name"].(string), params["group_path"].([]string) @@ -39,6 +64,7 @@ refHash, err = getRefHash(repo, params["ref_type"].(string), params["ref_name"].(string)) if err != nil { goto no_ref } + refHashSlice = refHash[:] branchesIter, err = repo.Branches() if err == nil { @@ -49,6 +75,7 @@ }) } params["branches"] = branches + // TODO: Cache logOptions = git.LogOptions{From: refHash} //exhaustruct:ignore if commitIter, err = repo.Log(&logOptions); err != nil { goto no_ref @@ -56,16 +83,34 @@ } commitIterSeq, params["commits_err"] = commitIterSeqErr(commitIter) params["commits"] = iterSeqLimit(commitIterSeq, 3) - if commitObj, err = repo.CommitObject(refHash); err != nil { - goto no_ref - } + if value, found := indexPageCache.Get(refHashSlice); found { + params["files"] = value.DisplayTree + params["readme_filename"] = value.ReadmeFilename + params["readme"] = value.ReadmeRendered + } else { + start := time.Now() + if commitObj, err = repo.CommitObject(refHash); err != nil { + goto no_ref + } + + if tree, err = commitObj.Tree(); err != nil { + goto no_ref + } + displayTree := makeDisplayTree(tree) + readmeFilename, readmeRendered := renderReadmeAtTree(tree) + cost := time.Since(start).Nanoseconds() + + params["files"] = displayTree + params["readme_filename"] = readmeFilename + params["readme"] = readmeRendered - if tree, err = commitObj.Tree(); err != nil { - goto no_ref + entry := indexPageCacheEntry{ + DisplayTree: displayTree, + ReadmeFilename: readmeFilename, + ReadmeRendered: readmeRendered, + } + indexPageCache.Set(refHashSlice, entry, cost) } - - params["files"] = makeDisplayTree(tree) - params["readme_filename"], params["readme"] = renderReadmeAtTree(tree) no_ref: -- 2.48.1