Arch packages implementation (#4785)
This PR is from https://github.com/go-gitea/gitea/pull/31037 This PR was originally created by @d1nch8g , and the original source code comes from https://ion.lc/core/gitea. This PR adds a package registry for [Arch Linux](https://archlinux.org/) packages with support for package files, [signatures](https://wiki.archlinux.org/title/Pacman/Package_signing), and automatic [pacman-database](https://archlinux.org/pacman/repo-add.8.html) management. Features: 1. Push any ` tar.zst ` package and Gitea sign it. 2. Delete endpoint for specific package version and all related files 3. Supports trust levels with `SigLevel = Required`. 4. Package UI with instructions to connect to the new pacman database and visualised package metadata  You can follow [this tutorial](https://wiki.archlinux.org/title/Creating_packages) to build a *.pkg.tar.zst package for testing docs pr: https://codeberg.org/forgejo/docs/pulls/791 Co-authored-by: d1nch8g@ion.lc Co-authored-by: @KN4CK3R Co-authored-by: @mahlzahn Co-authored-by: @silverwind Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/4785 Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org> Co-authored-by: Exploding Dragon <explodingfkl@gmail.com> Co-committed-by: Exploding Dragon <explodingfkl@gmail.com>
This commit is contained in:
Exploding Dragon
Earl Warren
parent
22d3659803
commit
f17194ca91
18 changed files with 1896 additions and 0 deletions
348
services/packages/arch/repository.go
Normal file
348
services/packages/arch/repository.go
Normal file
|
|
@ -0,0 +1,348 @@
|
|||
// Copyright 2024 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package arch
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
packages_model "code.gitea.io/gitea/models/packages"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
packages_module "code.gitea.io/gitea/modules/packages"
|
||||
arch_module "code.gitea.io/gitea/modules/packages/arch"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/modules/util"
|
||||
packages_service "code.gitea.io/gitea/services/packages"
|
||||
|
||||
"github.com/ProtonMail/go-crypto/openpgp"
|
||||
"github.com/ProtonMail/go-crypto/openpgp/armor"
|
||||
"github.com/ProtonMail/go-crypto/openpgp/packet"
|
||||
)
|
||||
|
||||
func GetOrCreateRepositoryVersion(ctx context.Context, ownerID int64) (*packages_model.PackageVersion, error) {
|
||||
return packages_service.GetOrCreateInternalPackageVersion(ctx, ownerID, packages_model.TypeArch, arch_module.RepositoryPackage, arch_module.RepositoryVersion)
|
||||
}
|
||||
|
||||
func BuildAllRepositoryFiles(ctx context.Context, ownerID int64) error {
|
||||
pv, err := GetOrCreateRepositoryVersion(ctx, ownerID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// remove old db files
|
||||
pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, pf := range pfs {
|
||||
if strings.HasSuffix(pf.Name, ".db") {
|
||||
arch := strings.TrimSuffix(strings.TrimPrefix(pf.Name, fmt.Sprintf("%s-", pf.CompositeKey)), ".db")
|
||||
if err := BuildPacmanDB(ctx, ownerID, pf.CompositeKey, arch); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func BuildCustomRepositoryFiles(ctx context.Context, ownerID int64, disco string) error {
|
||||
pv, err := GetOrCreateRepositoryVersion(ctx, ownerID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// remove old db files
|
||||
pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, pf := range pfs {
|
||||
if strings.HasSuffix(pf.Name, ".db") && pf.CompositeKey == disco {
|
||||
arch := strings.TrimSuffix(strings.TrimPrefix(pf.Name, fmt.Sprintf("%s-", pf.CompositeKey)), ".db")
|
||||
if err := BuildPacmanDB(ctx, ownerID, pf.CompositeKey, arch); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewFileSign(ctx context.Context, ownerID int64, input io.Reader) (*packages_module.HashedBuffer, error) {
|
||||
// If no signature is specified, it will be generated by Gitea.
|
||||
priv, _, err := GetOrCreateKeyPair(ctx, ownerID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
block, err := armor.Decode(strings.NewReader(priv))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e, err := openpgp.ReadEntity(packet.NewReader(block.Body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pkgSig, err := packages_module.NewHashedBuffer()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer pkgSig.Close()
|
||||
if err := openpgp.DetachSign(pkgSig, e, input, nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pkgSig, nil
|
||||
}
|
||||
|
||||
// BuildPacmanDB Create db signature cache
|
||||
func BuildPacmanDB(ctx context.Context, ownerID int64, distro, arch string) error {
|
||||
pv, err := GetOrCreateRepositoryVersion(ctx, ownerID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// remove old db files
|
||||
pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, pf := range pfs {
|
||||
if pf.CompositeKey == distro && strings.HasPrefix(pf.Name, fmt.Sprintf("%s-%s", distro, arch)) {
|
||||
// remove distro and arch
|
||||
if err := packages_service.DeletePackageFile(ctx, pf); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
db, err := flushDB(ctx, ownerID, distro, arch)
|
||||
if errors.Is(err, io.EOF) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
// Create db signature cache
|
||||
_, err = db.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sig, err := NewFileSign(ctx, ownerID, db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer sig.Close()
|
||||
_, err = db.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for name, data := range map[string]*packages_module.HashedBuffer{
|
||||
fmt.Sprintf("%s-%s.db", distro, arch): db,
|
||||
fmt.Sprintf("%s-%s.db.sig", distro, arch): sig,
|
||||
} {
|
||||
_, err = packages_service.AddFileToPackageVersionInternal(ctx, pv, &packages_service.PackageFileCreationInfo{
|
||||
PackageFileInfo: packages_service.PackageFileInfo{
|
||||
Filename: name,
|
||||
CompositeKey: distro,
|
||||
},
|
||||
Creator: user_model.NewGhostUser(),
|
||||
Data: data,
|
||||
IsLead: false,
|
||||
OverwriteExisting: true,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func flushDB(ctx context.Context, ownerID int64, distro, arch string) (*packages_module.HashedBuffer, error) {
|
||||
pkgs, err := packages_model.GetPackagesByType(ctx, ownerID, packages_model.TypeArch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(pkgs) == 0 {
|
||||
return nil, io.EOF
|
||||
}
|
||||
db, err := packages_module.NewHashedBuffer()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
gw := gzip.NewWriter(db)
|
||||
tw := tar.NewWriter(gw)
|
||||
count := 0
|
||||
for _, pkg := range pkgs {
|
||||
versions, err := packages_model.GetVersionsByPackageName(
|
||||
ctx, ownerID, packages_model.TypeArch, pkg.Name,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, errors.Join(tw.Close(), gw.Close(), db.Close(), err)
|
||||
}
|
||||
sort.Slice(versions, func(i, j int) bool {
|
||||
return versions[i].CreatedUnix > versions[j].CreatedUnix
|
||||
})
|
||||
for _, ver := range versions {
|
||||
file := fmt.Sprintf("%s-%s-%s.pkg.tar.zst", pkg.Name, ver.Version, arch)
|
||||
pf, err := packages_model.GetFileForVersionByName(ctx, ver.ID, file, distro)
|
||||
if err != nil {
|
||||
// add any arch package
|
||||
file = fmt.Sprintf("%s-%s-any.pkg.tar.zst", pkg.Name, ver.Version)
|
||||
pf, err = packages_model.GetFileForVersionByName(ctx, ver.ID, file, distro)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
pps, err := packages_model.GetPropertiesByName(
|
||||
ctx, packages_model.PropertyTypeFile, pf.ID, arch_module.PropertyDescription,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, errors.Join(tw.Close(), gw.Close(), db.Close(), err)
|
||||
}
|
||||
if len(pps) >= 1 {
|
||||
meta := []byte(pps[0].Value)
|
||||
header := &tar.Header{
|
||||
Name: pkg.Name + "-" + ver.Version + "/desc",
|
||||
Size: int64(len(meta)),
|
||||
Mode: int64(os.ModePerm),
|
||||
}
|
||||
if err = tw.WriteHeader(header); err != nil {
|
||||
return nil, errors.Join(tw.Close(), gw.Close(), db.Close(), err)
|
||||
}
|
||||
if _, err := tw.Write(meta); err != nil {
|
||||
return nil, errors.Join(tw.Close(), gw.Close(), db.Close(), err)
|
||||
}
|
||||
count++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
defer gw.Close()
|
||||
defer tw.Close()
|
||||
if count == 0 {
|
||||
return nil, errors.Join(db.Close(), io.EOF)
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// GetPackageFile Get data related to provided filename and distribution, for package files
|
||||
// update download counter.
|
||||
func GetPackageFile(ctx context.Context, distro, file string, ownerID int64) (io.ReadSeekCloser, error) {
|
||||
pf, err := getPackageFile(ctx, distro, file, ownerID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
filestream, _, _, err := packages_service.GetPackageFileStream(ctx, pf)
|
||||
return filestream, err
|
||||
}
|
||||
|
||||
// Ejects parameters required to get package file property from file name.
|
||||
func getPackageFile(ctx context.Context, distro, file string, ownerID int64) (*packages_model.PackageFile, error) {
|
||||
var (
|
||||
splt = strings.Split(file, "-")
|
||||
pkgname = strings.Join(splt[0:len(splt)-3], "-")
|
||||
vername = splt[len(splt)-3] + "-" + splt[len(splt)-2]
|
||||
)
|
||||
|
||||
version, err := packages_model.GetVersionByNameAndVersion(ctx, ownerID, packages_model.TypeArch, pkgname, vername)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
pkgfile, err := packages_model.GetFileForVersionByName(ctx, version.ID, file, distro)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pkgfile, nil
|
||||
}
|
||||
|
||||
func GetPackageDBFile(ctx context.Context, distro, arch string, ownerID int64, signFile bool) (io.ReadSeekCloser, error) {
|
||||
pv, err := GetOrCreateRepositoryVersion(ctx, ownerID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fileName := fmt.Sprintf("%s-%s.db", distro, arch)
|
||||
if signFile {
|
||||
fileName = fmt.Sprintf("%s-%s.db.sig", distro, arch)
|
||||
}
|
||||
file, err := packages_model.GetFileForVersionByName(ctx, pv.ID, fileName, distro)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
filestream, _, _, err := packages_service.GetPackageFileStream(ctx, file)
|
||||
return filestream, err
|
||||
}
|
||||
|
||||
// GetOrCreateKeyPair gets or creates the PGP keys used to sign repository metadata files
|
||||
func GetOrCreateKeyPair(ctx context.Context, ownerID int64) (string, string, error) {
|
||||
priv, err := user_model.GetSetting(ctx, ownerID, arch_module.SettingKeyPrivate)
|
||||
if err != nil && !errors.Is(err, util.ErrNotExist) {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
pub, err := user_model.GetSetting(ctx, ownerID, arch_module.SettingKeyPublic)
|
||||
if err != nil && !errors.Is(err, util.ErrNotExist) {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
if priv == "" || pub == "" {
|
||||
user, err := user_model.GetUserByID(ctx, ownerID)
|
||||
if err != nil && !errors.Is(err, util.ErrNotExist) {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
priv, pub, err = generateKeypair(user.Name)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
if err := user_model.SetUserSetting(ctx, ownerID, arch_module.SettingKeyPrivate, priv); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
if err := user_model.SetUserSetting(ctx, ownerID, arch_module.SettingKeyPublic, pub); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
}
|
||||
|
||||
return priv, pub, nil
|
||||
}
|
||||
|
||||
func generateKeypair(owner string) (string, string, error) {
|
||||
e, err := openpgp.NewEntity(
|
||||
owner,
|
||||
"Arch Package signature only",
|
||||
fmt.Sprintf("%s@noreply.%s", owner, setting.Packages.RegistryHost), &packet.Config{
|
||||
RSABits: 4096,
|
||||
})
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
var priv strings.Builder
|
||||
var pub strings.Builder
|
||||
|
||||
w, err := armor.Encode(&priv, openpgp.PrivateKeyType, nil)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if err := e.SerializePrivate(w, nil); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
w.Close()
|
||||
|
||||
w, err = armor.Encode(&pub, openpgp.PublicKeyType, nil)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if err := e.Serialize(w); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
w.Close()
|
||||
|
||||
return priv.String(), pub.String(), nil
|
||||
}
|
||||
|
|
@ -16,6 +16,7 @@ import (
|
|||
packages_module "code.gitea.io/gitea/modules/packages"
|
||||
packages_service "code.gitea.io/gitea/services/packages"
|
||||
alpine_service "code.gitea.io/gitea/services/packages/alpine"
|
||||
arch_service "code.gitea.io/gitea/services/packages/arch"
|
||||
cargo_service "code.gitea.io/gitea/services/packages/cargo"
|
||||
container_service "code.gitea.io/gitea/services/packages/container"
|
||||
debian_service "code.gitea.io/gitea/services/packages/debian"
|
||||
|
|
@ -132,6 +133,10 @@ func ExecuteCleanupRules(outerCtx context.Context) error {
|
|||
if err := rpm_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
|
||||
return fmt.Errorf("CleanupRule [%d]: rpm.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
|
||||
}
|
||||
} else if pcr.Type == packages_model.TypeArch {
|
||||
if err := arch_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
|
||||
return fmt.Errorf("CleanupRule [%d]: arch.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -359,6 +359,8 @@ func CheckSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, p
|
|||
switch packageType {
|
||||
case packages_model.TypeAlpine:
|
||||
typeSpecificSize = setting.Packages.LimitSizeAlpine
|
||||
case packages_model.TypeArch:
|
||||
typeSpecificSize = setting.Packages.LimitSizeArch
|
||||
case packages_model.TypeCargo:
|
||||
typeSpecificSize = setting.Packages.LimitSizeCargo
|
||||
case packages_model.TypeChef:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue