Add system setting table with cache and also add cache supports for user setting (#18058)

models/system/appstate.go

@@ -0,0 +1,57 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package system
+
+import (
+	"context"
+
+	"code.gitea.io/gitea/models/db"
+)
+
+// AppState represents a state record in database
+// if one day we would make Gitea run as a cluster,
+// we can introduce a new field `Scope` here to store different states for different nodes
+type AppState struct {
+	ID       string `xorm:"pk varchar(200)"`
+	Revision int64
+	Content  string `xorm:"LONGTEXT"`
+}
+
+func init() {
+	db.RegisterModel(new(AppState))
+}
+
+// SaveAppStateContent saves the app state item to database
+func SaveAppStateContent(key, content string) error {
+	return db.WithTx(func(ctx context.Context) error {
+		eng := db.GetEngine(ctx)
+		// try to update existing row
+		res, err := eng.Exec("UPDATE app_state SET revision=revision+1, content=? WHERE id=?", content, key)
+		if err != nil {
+			return err
+		}
+		rows, _ := res.RowsAffected()
+		if rows != 0 {
+			// the existing row is updated, so we can return
+			return nil
+		}
+		// if no existing row, insert a new row
+		_, err = eng.Insert(&AppState{ID: key, Content: content})
+		return err
+	})
+}
+
+// GetAppStateContent gets an app state from database
+func GetAppStateContent(key string) (content string, err error) {
+	e := db.GetEngine(db.DefaultContext)
+	appState := &AppState{ID: key}
+	has, err := e.Get(appState)
+	if err != nil {
+		return "", err
+	} else if !has {
+		return "", nil
+	}
+	return appState.Content, nil
+}

models/system/main_test.go

@@ -0,0 +1,21 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package system_test
+
+import (
+	"path/filepath"
+	"testing"
+
+	"code.gitea.io/gitea/models/unittest"
+
+	_ "code.gitea.io/gitea/models"        // register models
+	_ "code.gitea.io/gitea/models/system" // register models of system
+)
+
+func TestMain(m *testing.M) {
+	unittest.MainTest(m, &unittest.TestOptions{
+		GiteaRootPath: filepath.Join("..", ".."),
+	})
+}

models/system/notice.go

@@ -0,0 +1,146 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package system
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/storage"
+	"code.gitea.io/gitea/modules/timeutil"
+	"code.gitea.io/gitea/modules/util"
+)
+
+// NoticeType describes the notice type
+type NoticeType int
+
+const (
+	// NoticeRepository type
+	NoticeRepository NoticeType = iota + 1
+	// NoticeTask type
+	NoticeTask
+)
+
+// Notice represents a system notice for admin.
+type Notice struct {
+	ID          int64 `xorm:"pk autoincr"`
+	Type        NoticeType
+	Description string             `xorm:"TEXT"`
+	CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
+}
+
+func init() {
+	db.RegisterModel(new(Notice))
+}
+
+// TrStr returns a translation format string.
+func (n *Notice) TrStr() string {
+	return fmt.Sprintf("admin.notices.type_%d", n.Type)
+}
+
+// CreateNotice creates new system notice.
+func CreateNotice(ctx context.Context, tp NoticeType, desc string, args ...interface{}) error {
+	if len(args) > 0 {
+		desc = fmt.Sprintf(desc, args...)
+	}
+	n := &Notice{
+		Type:        tp,
+		Description: desc,
+	}
+	return db.Insert(ctx, n)
+}
+
+// CreateRepositoryNotice creates new system notice with type NoticeRepository.
+func CreateRepositoryNotice(desc string, args ...interface{}) error {
+	// Note we use the db.DefaultContext here rather than passing in a context as the context may be cancelled
+	return CreateNotice(db.DefaultContext, NoticeRepository, desc, args...)
+}
+
+// RemoveAllWithNotice removes all directories in given path and
+// creates a system notice when error occurs.
+func RemoveAllWithNotice(ctx context.Context, title, path string) {
+	if err := util.RemoveAll(path); err != nil {
+		desc := fmt.Sprintf("%s [%s]: %v", title, path, err)
+		log.Warn(title+" [%s]: %v", path, err)
+		// Note we use the db.DefaultContext here rather than passing in a context as the context may be cancelled
+		if err = CreateNotice(db.DefaultContext, NoticeRepository, desc); err != nil {
+			log.Error("CreateRepositoryNotice: %v", err)
+		}
+	}
+}
+
+// RemoveStorageWithNotice removes a file from the storage and
+// creates a system notice when error occurs.
+func RemoveStorageWithNotice(ctx context.Context, bucket storage.ObjectStorage, title, path string) {
+	if err := bucket.Delete(path); err != nil {
+		desc := fmt.Sprintf("%s [%s]: %v", title, path, err)
+		log.Warn(title+" [%s]: %v", path, err)
+
+		// Note we use the db.DefaultContext here rather than passing in a context as the context may be cancelled
+		if err = CreateNotice(db.DefaultContext, NoticeRepository, desc); err != nil {
+			log.Error("CreateRepositoryNotice: %v", err)
+		}
+	}
+}
+
+// CountNotices returns number of notices.
+func CountNotices() int64 {
+	count, _ := db.GetEngine(db.DefaultContext).Count(new(Notice))
+	return count
+}
+
+// Notices returns notices in given page.
+func Notices(page, pageSize int) ([]*Notice, error) {
+	notices := make([]*Notice, 0, pageSize)
+	return notices, db.GetEngine(db.DefaultContext).
+		Limit(pageSize, (page-1)*pageSize).
+		Desc("created_unix").
+		Find(&notices)
+}
+
+// DeleteNotice deletes a system notice by given ID.
+func DeleteNotice(id int64) error {
+	_, err := db.GetEngine(db.DefaultContext).ID(id).Delete(new(Notice))
+	return err
+}
+
+// DeleteNotices deletes all notices with ID from start to end (inclusive).
+func DeleteNotices(start, end int64) error {
+	if start == 0 && end == 0 {
+		_, err := db.GetEngine(db.DefaultContext).Exec("DELETE FROM notice")
+		return err
+	}
+
+	sess := db.GetEngine(db.DefaultContext).Where("id >= ?", start)
+	if end > 0 {
+		sess.And("id <= ?", end)
+	}
+	_, err := sess.Delete(new(Notice))
+	return err
+}
+
+// DeleteNoticesByIDs deletes notices by given IDs.
+func DeleteNoticesByIDs(ids []int64) error {
+	if len(ids) == 0 {
+		return nil
+	}
+	_, err := db.GetEngine(db.DefaultContext).
+		In("id", ids).
+		Delete(new(Notice))
+	return err
+}
+
+// DeleteOldSystemNotices deletes all old system notices from database.
+func DeleteOldSystemNotices(olderThan time.Duration) (err error) {
+	if olderThan <= 0 {
+		return nil
+	}
+
+	_, err = db.GetEngine(db.DefaultContext).Where("created_unix < ?", time.Now().Add(-olderThan).Unix()).Delete(&Notice{})
+	return err
+}

models/system/notice_test.go

@@ -0,0 +1,117 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package system_test
+
+import (
+	"testing"
+
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/models/system"
+	"code.gitea.io/gitea/models/unittest"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNotice_TrStr(t *testing.T) {
+	notice := &system.Notice{
+		Type:        system.NoticeRepository,
+		Description: "test description",
+	}
+	assert.Equal(t, "admin.notices.type_1", notice.TrStr())
+}
+
+func TestCreateNotice(t *testing.T) {
+	assert.NoError(t, unittest.PrepareTestDatabase())
+
+	noticeBean := &system.Notice{
+		Type:        system.NoticeRepository,
+		Description: "test description",
+	}
+	unittest.AssertNotExistsBean(t, noticeBean)
+	assert.NoError(t, system.CreateNotice(db.DefaultContext, noticeBean.Type, noticeBean.Description))
+	unittest.AssertExistsAndLoadBean(t, noticeBean)
+}
+
+func TestCreateRepositoryNotice(t *testing.T) {
+	assert.NoError(t, unittest.PrepareTestDatabase())
+
+	noticeBean := &system.Notice{
+		Type:        system.NoticeRepository,
+		Description: "test description",
+	}
+	unittest.AssertNotExistsBean(t, noticeBean)
+	assert.NoError(t, system.CreateRepositoryNotice(noticeBean.Description))
+	unittest.AssertExistsAndLoadBean(t, noticeBean)
+}
+
+// TODO TestRemoveAllWithNotice
+
+func TestCountNotices(t *testing.T) {
+	assert.NoError(t, unittest.PrepareTestDatabase())
+	assert.Equal(t, int64(3), system.CountNotices())
+}
+
+func TestNotices(t *testing.T) {
+	assert.NoError(t, unittest.PrepareTestDatabase())
+
+	notices, err := system.Notices(1, 2)
+	assert.NoError(t, err)
+	if assert.Len(t, notices, 2) {
+		assert.Equal(t, int64(3), notices[0].ID)
+		assert.Equal(t, int64(2), notices[1].ID)
+	}
+
+	notices, err = system.Notices(2, 2)
+	assert.NoError(t, err)
+	if assert.Len(t, notices, 1) {
+		assert.Equal(t, int64(1), notices[0].ID)
+	}
+}
+
+func TestDeleteNotice(t *testing.T) {
+	assert.NoError(t, unittest.PrepareTestDatabase())
+
+	unittest.AssertExistsAndLoadBean(t, &system.Notice{ID: 3})
+	assert.NoError(t, system.DeleteNotice(3))
+	unittest.AssertNotExistsBean(t, &system.Notice{ID: 3})
+}
+
+func TestDeleteNotices(t *testing.T) {
+	// delete a non-empty range
+	assert.NoError(t, unittest.PrepareTestDatabase())
+
+	unittest.AssertExistsAndLoadBean(t, &system.Notice{ID: 1})
+	unittest.AssertExistsAndLoadBean(t, &system.Notice{ID: 2})
+	unittest.AssertExistsAndLoadBean(t, &system.Notice{ID: 3})
+	assert.NoError(t, system.DeleteNotices(1, 2))
+	unittest.AssertNotExistsBean(t, &system.Notice{ID: 1})
+	unittest.AssertNotExistsBean(t, &system.Notice{ID: 2})
+	unittest.AssertExistsAndLoadBean(t, &system.Notice{ID: 3})
+}
+
+func TestDeleteNotices2(t *testing.T) {
+	// delete an empty range
+	assert.NoError(t, unittest.PrepareTestDatabase())
+
+	unittest.AssertExistsAndLoadBean(t, &system.Notice{ID: 1})
+	unittest.AssertExistsAndLoadBean(t, &system.Notice{ID: 2})
+	unittest.AssertExistsAndLoadBean(t, &system.Notice{ID: 3})
+	assert.NoError(t, system.DeleteNotices(3, 2))
+	unittest.AssertExistsAndLoadBean(t, &system.Notice{ID: 1})
+	unittest.AssertExistsAndLoadBean(t, &system.Notice{ID: 2})
+	unittest.AssertExistsAndLoadBean(t, &system.Notice{ID: 3})
+}
+
+func TestDeleteNoticesByIDs(t *testing.T) {
+	assert.NoError(t, unittest.PrepareTestDatabase())
+
+	unittest.AssertExistsAndLoadBean(t, &system.Notice{ID: 1})
+	unittest.AssertExistsAndLoadBean(t, &system.Notice{ID: 2})
+	unittest.AssertExistsAndLoadBean(t, &system.Notice{ID: 3})
+	assert.NoError(t, system.DeleteNoticesByIDs([]int64{1, 3}))
+	unittest.AssertNotExistsBean(t, &system.Notice{ID: 1})
+	unittest.AssertExistsAndLoadBean(t, &system.Notice{ID: 2})
+	unittest.AssertNotExistsBean(t, &system.Notice{ID: 3})
+}

models/system/setting.go

@@ -0,0 +1,261 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package system
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"strk.kbt.io/projects/go/libravatar"
+	"xorm.io/builder"
+)
+
+// Setting is a key value store of user settings
+type Setting struct {
+	ID           int64              `xorm:"pk autoincr"`
+	SettingKey   string             `xorm:"varchar(255) unique"` // ensure key is always lowercase
+	SettingValue string             `xorm:"text"`
+	Version      int                `xorm:"version"` // prevent to override
+	Created      timeutil.TimeStamp `xorm:"created"`
+	Updated      timeutil.TimeStamp `xorm:"updated"`
+}
+
+// TableName sets the table name for the settings struct
+func (s *Setting) TableName() string {
+	return "system_setting"
+}
+
+func (s *Setting) GetValueBool() bool {
+	b, _ := strconv.ParseBool(s.SettingValue)
+	return b
+}
+
+func init() {
+	db.RegisterModel(new(Setting))
+}
+
+// ErrSettingIsNotExist represents an error that a setting is not exist with special key
+type ErrSettingIsNotExist struct {
+	Key string
+}
+
+// Error implements error
+func (err ErrSettingIsNotExist) Error() string {
+	return fmt.Sprintf("System setting[%s] is not exist", err.Key)
+}
+
+// IsErrSettingIsNotExist return true if err is ErrSettingIsNotExist
+func IsErrSettingIsNotExist(err error) bool {
+	_, ok := err.(ErrSettingIsNotExist)
+	return ok
+}
+
+// ErrDataExpired represents an error that update a record which has been updated by another thread
+type ErrDataExpired struct {
+	Key string
+}
+
+// Error implements error
+func (err ErrDataExpired) Error() string {
+	return fmt.Sprintf("System setting[%s] has been updated by another thread", err.Key)
+}
+
+// IsErrDataExpired return true if err is ErrDataExpired
+func IsErrDataExpired(err error) bool {
+	_, ok := err.(ErrDataExpired)
+	return ok
+}
+
+// GetSetting returns specific setting
+func GetSetting(key string) (*Setting, error) {
+	v, err := GetSettings([]string{key})
+	if err != nil {
+		return nil, err
+	}
+	if len(v) == 0 {
+		return nil, ErrSettingIsNotExist{key}
+	}
+	return v[key], nil
+}
+
+// GetSettings returns specific settings
+func GetSettings(keys []string) (map[string]*Setting, error) {
+	for i := 0; i < len(keys); i++ {
+		keys[i] = strings.ToLower(keys[i])
+	}
+	settings := make([]*Setting, 0, len(keys))
+	if err := db.GetEngine(db.DefaultContext).
+		Where(builder.In("setting_key", keys)).
+		Find(&settings); err != nil {
+		return nil, err
+	}
+	settingsMap := make(map[string]*Setting)
+	for _, s := range settings {
+		settingsMap[s.SettingKey] = s
+	}
+	return settingsMap, nil
+}
+
+type AllSettings map[string]*Setting
+
+func (settings AllSettings) Get(key string) Setting {
+	if v, ok := settings[key]; ok {
+		return *v
+	}
+	return Setting{}
+}
+
+func (settings AllSettings) GetBool(key string) bool {
+	b, _ := strconv.ParseBool(settings.Get(key).SettingValue)
+	return b
+}
+
+func (settings AllSettings) GetVersion(key string) int {
+	return settings.Get(key).Version
+}
+
+// GetAllSettings returns all settings from user
+func GetAllSettings() (AllSettings, error) {
+	settings := make([]*Setting, 0, 5)
+	if err := db.GetEngine(db.DefaultContext).
+		Find(&settings); err != nil {
+		return nil, err
+	}
+	settingsMap := make(map[string]*Setting)
+	for _, s := range settings {
+		settingsMap[s.SettingKey] = s
+	}
+	return settingsMap, nil
+}
+
+// DeleteSetting deletes a specific setting for a user
+func DeleteSetting(setting *Setting) error {
+	_, err := db.GetEngine(db.DefaultContext).Delete(setting)
+	return err
+}
+
+func SetSettingNoVersion(key, value string) error {
+	s, err := GetSetting(key)
+	if IsErrSettingIsNotExist(err) {
+		return SetSetting(&Setting{
+			SettingKey:   key,
+			SettingValue: value,
+		})
+	}
+	if err != nil {
+		return err
+	}
+	s.SettingValue = value
+	return SetSetting(s)
+}
+
+// SetSetting updates a users' setting for a specific key
+func SetSetting(setting *Setting) error {
+	if err := upsertSettingValue(strings.ToLower(setting.SettingKey), setting.SettingValue, setting.Version); err != nil {
+		return err
+	}
+	setting.Version++
+	return nil
+}
+
+func upsertSettingValue(key, value string, version int) error {
+	return db.WithTx(func(ctx context.Context) error {
+		e := db.GetEngine(ctx)
+
+		// here we use a general method to do a safe upsert for different databases (and most transaction levels)
+		// 1. try to UPDATE the record and acquire the transaction write lock
+		//    if UPDATE returns non-zero rows are changed, OK, the setting is saved correctly
+		//    if UPDATE returns "0 rows changed", two possibilities: (a) record doesn't exist  (b) value is not changed
+		// 2. do a SELECT to check if the row exists or not (we already have the transaction lock)
+		// 3. if the row doesn't exist, do an INSERT (we are still protected by the transaction lock, so it's safe)
+		//
+		// to optimize the SELECT in step 2, we can use an extra column like `revision=revision+1`
+		//    to make sure the UPDATE always returns a non-zero value for existing (unchanged) records.
+
+		res, err := e.Exec("UPDATE system_setting SET setting_value=?, version = version+1 WHERE setting_key=? AND version=?", value, key, version)
+		if err != nil {
+			return err
+		}
+		rows, _ := res.RowsAffected()
+		if rows > 0 {
+			// the existing row is updated, so we can return
+			return nil
+		}
+
+		// in case the value isn't changed, update would return 0 rows changed, so we need this check
+		has, err := e.Exist(&Setting{SettingKey: key})
+		if err != nil {
+			return err
+		}
+		if has {
+			return ErrDataExpired{Key: key}
+		}
+
+		// if no existing row, insert a new row
+		_, err = e.Insert(&Setting{SettingKey: key, SettingValue: value})
+		return err
+	})
+}
+
+var (
+	GravatarSourceURL *url.URL
+	LibravatarService *libravatar.Libravatar
+)
+
+func Init() error {
+	var disableGravatar bool
+	disableGravatarSetting, err := GetSetting(KeyPictureDisableGravatar)
+	if IsErrSettingIsNotExist(err) {
+		disableGravatar = setting.GetDefaultDisableGravatar()
+		disableGravatarSetting = &Setting{SettingValue: strconv.FormatBool(disableGravatar)}
+	} else if err != nil {
+		return err
+	} else {
+		disableGravatar = disableGravatarSetting.GetValueBool()
+	}
+
+	var enableFederatedAvatar bool
+	enableFederatedAvatarSetting, err := GetSetting(KeyPictureEnableFederatedAvatar)
+	if IsErrSettingIsNotExist(err) {
+		enableFederatedAvatar = setting.GetDefaultEnableFederatedAvatar(disableGravatar)
+		enableFederatedAvatarSetting = &Setting{SettingValue: strconv.FormatBool(enableFederatedAvatar)}
+	} else if err != nil {
+		return err
+	} else {
+		enableFederatedAvatar = disableGravatarSetting.GetValueBool()
+	}
+
+	if setting.OfflineMode {
+		disableGravatar = true
+		enableFederatedAvatar = false
+	}
+
+	if disableGravatar || !enableFederatedAvatar {
+		var err error
+		GravatarSourceURL, err = url.Parse(setting.GravatarSource)
+		if err != nil {
+			return fmt.Errorf("Failed to parse Gravatar URL(%s): %v", setting.GravatarSource, err)
+		}
+	}
+
+	if enableFederatedAvatarSetting.GetValueBool() {
+		LibravatarService = libravatar.New()
+		if GravatarSourceURL.Scheme == "https" {
+			LibravatarService.SetUseHTTPS(true)
+			LibravatarService.SetSecureFallbackHost(GravatarSourceURL.Host)
+		} else {
+			LibravatarService.SetUseHTTPS(false)
+			LibravatarService.SetFallbackHost(GravatarSourceURL.Host)
+		}
+	}
+	return nil
+}

models/system/setting_key.go

@@ -0,0 +1,11 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package system
+
+// enumerate all system setting keys
+const (
+	KeyPictureDisableGravatar       = "picture.disable_gravatar"
+	KeyPictureEnableFederatedAvatar = "picture.enable_federated_avatar"
+)

models/system/setting_test.go

@@ -0,0 +1,53 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package system_test
+
+import (
+	"strings"
+	"testing"
+
+	"code.gitea.io/gitea/models/system"
+	"code.gitea.io/gitea/models/unittest"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSettings(t *testing.T) {
+	keyName := "server.LFS_LOCKS_PAGING_NUM"
+	assert.NoError(t, unittest.PrepareTestDatabase())
+
+	newSetting := &system.Setting{SettingKey: keyName, SettingValue: "50"}
+
+	// create setting
+	err := system.SetSetting(newSetting)
+	assert.NoError(t, err)
+	// test about saving unchanged values
+	err = system.SetSetting(newSetting)
+	assert.NoError(t, err)
+
+	// get specific setting
+	settings, err := system.GetSettings([]string{keyName})
+	assert.NoError(t, err)
+	assert.Len(t, settings, 1)
+	assert.EqualValues(t, newSetting.SettingValue, settings[strings.ToLower(keyName)].SettingValue)
+
+	// updated setting
+	updatedSetting := &system.Setting{SettingKey: keyName, SettingValue: "100", Version: newSetting.Version}
+	err = system.SetSetting(updatedSetting)
+	assert.NoError(t, err)
+
+	// get all settings
+	settings, err = system.GetAllSettings()
+	assert.NoError(t, err)
+	assert.Len(t, settings, 3)
+	assert.EqualValues(t, updatedSetting.SettingValue, settings[strings.ToLower(updatedSetting.SettingKey)].SettingValue)
+
+	// delete setting
+	err = system.DeleteSetting(&system.Setting{SettingKey: strings.ToLower(keyName)})
+	assert.NoError(t, err)
+	settings, err = system.GetAllSettings()
+	assert.NoError(t, err)
+	assert.Len(t, settings, 2)
+}