Merge pull request '[gitea] week 2024-45 cherry pick (gitea/main -> forgejo)' (#5789) from algernon/wcp/2024-45 into forgejo

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/5789
Reviewed-by: Gusted <gusted@noreply.codeberg.org>

modules/charset/charset_test.go

@@ -41,14 +41,12 @@ func TestMaybeRemoveBOM(t *testing.T) {
 
 func TestToUTF8(t *testing.T) {
 	resetDefaultCharsetsOrder()
-	var res string
-	var err error
 
 	// Note: golang compiler seems so behave differently depending on the current
 	// locale, so some conversions might behave differently. For that reason, we don't
 	// depend on particular conversions but in expected behaviors.
 
-	res, err = ToUTF8([]byte{0x41, 0x42, 0x43}, ConvertOpts{})
+	res, err := ToUTF8([]byte{0x41, 0x42, 0x43}, ConvertOpts{})
 	require.NoError(t, err)
 	assert.Equal(t, "ABC", res)
 

modules/git/repo_index.go

@@ -50,25 +50,35 @@ func (repo *Repository) readTreeToIndex(id ObjectID, indexFilename ...string) er
 }
 
 // ReadTreeToTemporaryIndex reads a treeish to a temporary index file
-func (repo *Repository) ReadTreeToTemporaryIndex(treeish string) (filename, tmpDir string, cancel context.CancelFunc, err error) {
-	tmpDir, err = os.MkdirTemp("", "index")
-	if err != nil {
-		return filename, tmpDir, cancel, err
-	}
+func (repo *Repository) ReadTreeToTemporaryIndex(treeish string) (tmpIndexFilename, tmpDir string, cancel context.CancelFunc, err error) {
+	defer func() {
+		// if error happens and there is a cancel function, do clean up
+		if err != nil && cancel != nil {
+			cancel()
+			cancel = nil
+		}
+	}()
 
-	filename = filepath.Join(tmpDir, ".tmp-index")
-	cancel = func() {
-		err := util.RemoveAll(tmpDir)
-		if err != nil {
-			log.Error("failed to remove tmp index file: %v", err)
+	removeDirFn := func(dir string) func() { // it can't use the return value "tmpDir" directly because it is empty when error occurs
+		return func() {
+			if err := util.RemoveAll(dir); err != nil {
+				log.Error("failed to remove tmp index dir: %v", err)
+			}
 		}
 	}
-	err = repo.ReadTreeToIndex(treeish, filename)
+
+	tmpDir, err = os.MkdirTemp("", "index")
 	if err != nil {
-		defer cancel()
-		return "", "", func() {}, err
+		return "", "", nil, err
 	}
-	return filename, tmpDir, cancel, err
+
+	tmpIndexFilename = filepath.Join(tmpDir, ".tmp-index")
+	cancel = removeDirFn(tmpDir)
+	err = repo.ReadTreeToIndex(treeish, tmpIndexFilename)
+	if err != nil {
+		return "", "", cancel, err
+	}
+	return tmpIndexFilename, tmpDir, cancel, err
 }
 
 // EmptyIndex empties the index

modules/lfs/http_client.go

@@ -16,10 +16,9 @@ import (
 	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/proxy"
+	"code.gitea.io/gitea/modules/setting"
 )
 
-const httpBatchSize = 20
-
 // HTTPClient is used to communicate with the LFS server
 // https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md
 type HTTPClient struct {
@@ -30,7 +29,7 @@ type HTTPClient struct {
 
 // BatchSize returns the preferred size of batchs to process
 func (c *HTTPClient) BatchSize() int {
-	return httpBatchSize
+	return setting.LFSClient.BatchSize
 }
 
 func newHTTPClient(endpoint *url.URL, httpTransport *http.Transport) *HTTPClient {

modules/markup/markdown/goldmark.go

@@ -203,8 +203,7 @@ func (r *HTMLRenderer) renderIcon(w util.BufWriter, source []byte, node ast.Node
 		return ast.WalkContinue, nil
 	}
 
-	var err error
-	_, err = w.WriteString(fmt.Sprintf(`<i class="icon %s"></i>`, name))
+	_, err := w.WriteString(fmt.Sprintf(`<i class="icon %s"></i>`, name))
 	if err != nil {
 		return ast.WalkStop, err
 	}

modules/packages/content_store.go

@@ -37,8 +37,8 @@ func (s *ContentStore) ShouldServeDirect() bool {
 	return setting.Packages.Storage.MinioConfig.ServeDirect
 }
 
-func (s *ContentStore) GetServeDirectURL(key BlobHash256Key, filename string) (*url.URL, error) {
-	return s.store.URL(KeyToRelativePath(key), filename)
+func (s *ContentStore) GetServeDirectURL(key BlobHash256Key, filename string, reqParams url.Values) (*url.URL, error) {
+	return s.store.URL(KeyToRelativePath(key), filename, reqParams)
 }
 
 // FIXME: Workaround to be removed in v1.20

modules/repository/repo.go

@@ -341,9 +341,10 @@ func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, git
 
 		for _, tag := range updates {
 			if _, err := db.GetEngine(ctx).Where("repo_id = ? AND lower_tag_name = ?", repo.ID, strings.ToLower(tag.Name)).
-				Cols("sha1").
+				Cols("sha1", "created_unix").
 				Update(&repo_model.Release{
-					Sha1: tag.Object.String(),
+					Sha1:        tag.Object.String(),
+					CreatedUnix: timeutil.TimeStamp(tag.Tagger.When.Unix()),
 				}); err != nil {
 				return fmt.Errorf("unable to update tag %s for pull-mirror Repo[%d:%s/%s]: %w", tag.Name, repo.ID, repo.OwnerName, repo.Name, err)
 			}

modules/setting/lfs.go

@@ -10,22 +10,31 @@ import (
 	"code.gitea.io/gitea/modules/generate"
 )
 
-// LFS represents the configuration for Git LFS
+// LFS represents the server-side configuration for Git LFS.
+// Ideally these options should be in a section like "[lfs_server]",
+// but they are in "[server]" section due to historical reasons.
+// Could be refactored in the future while keeping backwards compatibility.
 var LFS = struct {
 	StartServer    bool          `ini:"LFS_START_SERVER"`
 	JWTSecretBytes []byte        `ini:"-"`
 	HTTPAuthExpiry time.Duration `ini:"LFS_HTTP_AUTH_EXPIRY"`
 	MaxFileSize    int64         `ini:"LFS_MAX_FILE_SIZE"`
 	LocksPagingNum int           `ini:"LFS_LOCKS_PAGING_NUM"`
+	MaxBatchSize   int           `ini:"LFS_MAX_BATCH_SIZE"`
 
 	Storage *Storage
 }{}
 
+// LFSClient represents configuration for Gitea's LFS clients, for example: mirroring upstream Git LFS
+var LFSClient = struct {
+	BatchSize int `ini:"BATCH_SIZE"`
+}{}
+
 func loadLFSFrom(rootCfg ConfigProvider) error {
+	mustMapSetting(rootCfg, "lfs_client", &LFSClient)
+
+	mustMapSetting(rootCfg, "server", &LFS)
 	sec := rootCfg.Section("server")
-	if err := sec.MapTo(&LFS); err != nil {
-		return fmt.Errorf("failed to map LFS settings: %v", err)
-	}
 
 	lfsSec, _ := rootCfg.GetSection("lfs")
 
@@ -52,6 +61,10 @@ func loadLFSFrom(rootCfg ConfigProvider) error {
 		LFS.LocksPagingNum = 50
 	}
 
+	if LFSClient.BatchSize < 1 {
+		LFSClient.BatchSize = 20
+	}
+
 	LFS.HTTPAuthExpiry = sec.Key("LFS_HTTP_AUTH_EXPIRY").MustDuration(24 * time.Hour)
 
 	if !LFS.StartServer || !InstallLock {

modules/setting/lfs_test.go

@@ -100,3 +100,19 @@ STORAGE_TYPE = minio
 	assert.EqualValues(t, "gitea", LFS.Storage.MinioConfig.Bucket)
 	assert.EqualValues(t, "lfs/", LFS.Storage.MinioConfig.BasePath)
 }
+
+func Test_LFSClientServerConfigs(t *testing.T) {
+	iniStr := `
+[server]
+LFS_MAX_BATCH_SIZE = 100
+[lfs_client]
+# will default to 20
+BATCH_SIZE = 0
+`
+	cfg, err := NewConfigProviderFromData(iniStr)
+	assert.NoError(t, err)
+
+	assert.NoError(t, loadLFSFrom(cfg))
+	assert.EqualValues(t, 100, LFS.MaxBatchSize)
+	assert.EqualValues(t, 20, LFSClient.BatchSize)
+}

modules/storage/helper.go

@@ -30,7 +30,7 @@ func (s DiscardStorage) Delete(_ string) error {
 	return fmt.Errorf("%s", s)
 }
 
-func (s DiscardStorage) URL(_, _ string) (*url.URL, error) {
+func (s DiscardStorage) URL(_, _ string, _ url.Values) (*url.URL, error) {
 	return nil, fmt.Errorf("%s", s)
 }
 

modules/storage/helper_test.go

@@ -38,7 +38,7 @@ func Test_discardStorage(t *testing.T) {
 				require.Error(t, err, string(tt))
 			}
 			{
-				got, err := tt.URL("path", "name")
+				got, err := tt.URL("path", "name", nil)
 				assert.Nil(t, got)
 				require.Errorf(t, err, string(tt))
 			}

modules/storage/local.go

@@ -114,7 +114,7 @@ func (l *LocalStorage) Delete(path string) error {
 }
 
 // URL gets the redirect URL to a file
-func (l *LocalStorage) URL(path, name string) (*url.URL, error) {
+func (l *LocalStorage) URL(path, name string, reqParams url.Values) (*url.URL, error) {
 	return nil, ErrURLNotSupported
 }
 

modules/storage/minio.go

@@ -276,8 +276,12 @@ func (m *MinioStorage) Delete(path string) error {
 }
 
 // URL gets the redirect URL to a file. The presigned link is valid for 5 minutes.
-func (m *MinioStorage) URL(path, name string) (*url.URL, error) {
-	reqParams := make(url.Values)
+func (m *MinioStorage) URL(path, name string, serveDirectReqParams url.Values) (*url.URL, error) {
+	// copy serveDirectReqParams
+	reqParams, err := url.ParseQuery(serveDirectReqParams.Encode())
+	if err != nil {
+		return nil, err
+	}
 	// TODO it may be good to embed images with 'inline' like ServeData does, but we don't want to have to read the file, do we?
 	reqParams.Set("response-content-disposition", "attachment; filename=\""+quoteEscaper.Replace(name)+"\"")
 	u, err := m.client.PresignedGetObject(m.ctx, m.bucket, m.buildMinioPath(path), 5*time.Minute, reqParams)

modules/storage/storage.go

@@ -63,7 +63,7 @@ type ObjectStorage interface {
 	Save(path string, r io.Reader, size int64) (int64, error)
 	Stat(path string) (os.FileInfo, error)
 	Delete(path string) error
-	URL(path, name string) (*url.URL, error)
+	URL(path, name string, reqParams url.Values) (*url.URL, error)
 	IterateObjects(path string, iterator func(path string, obj Object) error) error
 }
 
@@ -131,7 +131,7 @@ var (
 	ActionsArtifacts ObjectStorage = UninitializedStorage
 )
 
-// Init init the stoarge
+// Init init the storage
 func Init() error {
 	for _, f := range []func() error{
 		initAttachments,