1
Fork 0

Arch packages implementation (#4785)

This PR is from https://github.com/go-gitea/gitea/pull/31037

This PR was originally created by @d1nch8g , and the original source code comes from https://ion.lc/core/gitea.

This PR adds a package registry for [Arch Linux](https://archlinux.org/) packages with support for package files, [signatures](https://wiki.archlinux.org/title/Pacman/Package_signing), and automatic [pacman-database](https://archlinux.org/pacman/repo-add.8.html) management.

Features:

1. Push any ` tar.zst ` package and Gitea sign it.
2. Delete endpoint for specific package version and all related files
3. Supports trust levels with `SigLevel = Required`.
4. Package UI with instructions to connect to the new pacman database and visualised package metadata

![](/attachments/810ca6df-bd20-44c2-bdf7-95e94886d750)

You can follow [this tutorial](https://wiki.archlinux.org/title/Creating_packages) to build a *.pkg.tar.zst package for testing

docs pr: https://codeberg.org/forgejo/docs/pulls/791

Co-authored-by: d1nch8g@ion.lc
Co-authored-by: @KN4CK3R
Co-authored-by: @mahlzahn
Co-authored-by: @silverwind
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/4785
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
Co-authored-by: Exploding Dragon <explodingfkl@gmail.com>
Co-committed-by: Exploding Dragon <explodingfkl@gmail.com>
This commit is contained in:
Exploding Dragon 2024-08-04 06:16:29 +00:00 committed by Earl Warren
parent 22d3659803
commit f17194ca91
18 changed files with 1896 additions and 0 deletions

View file

@ -0,0 +1,316 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package arch
import (
"bufio"
"bytes"
"encoding/hex"
"errors"
"fmt"
"io"
"regexp"
"strconv"
"strings"
"code.gitea.io/gitea/modules/packages"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/validation"
"github.com/mholt/archiver/v3"
)
// Arch Linux Packages
// https://man.archlinux.org/man/PKGBUILD.5
const (
PropertyDescription = "arch.description"
PropertyArch = "arch.architecture"
PropertyDistribution = "arch.distribution"
SettingKeyPrivate = "arch.key.private"
SettingKeyPublic = "arch.key.public"
RepositoryPackage = "_arch"
RepositoryVersion = "_repository"
)
var (
reName = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$`)
reVer = regexp.MustCompile(`^[a-zA-Z0-9:_.+]+-+[0-9]+$`)
reOptDep = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(:.*)`)
rePkgVer = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(>.*)|^[a-zA-Z0-9@._+-]+(<.*)|^[a-zA-Z0-9@._+-]+(=.*)`)
)
type Package struct {
Name string `json:"name"`
Version string `json:"version"` // Includes version, release and epoch
VersionMetadata VersionMetadata
FileMetadata FileMetadata
}
// Arch package metadata related to specific version.
// Version metadata the same across different architectures and distributions.
type VersionMetadata struct {
Base string `json:"base"`
Description string `json:"description"`
ProjectURL string `json:"project_url"`
Groups []string `json:"groups,omitempty"`
Provides []string `json:"provides,omitempty"`
License []string `json:"license,omitempty"`
Depends []string `json:"depends,omitempty"`
OptDepends []string `json:"opt_depends,omitempty"`
MakeDepends []string `json:"make_depends,omitempty"`
CheckDepends []string `json:"check_depends,omitempty"`
Conflicts []string `json:"conflicts,omitempty"`
Replaces []string `json:"replaces,omitempty"`
Backup []string `json:"backup,omitempty"`
Xdata []string `json:"xdata,omitempty"`
}
// FileMetadata Metadata related to specific package file.
// This metadata might vary for different architecture and distribution.
type FileMetadata struct {
CompressedSize int64 `json:"compressed_size"`
InstalledSize int64 `json:"installed_size"`
MD5 string `json:"md5"`
SHA256 string `json:"sha256"`
BuildDate int64 `json:"build_date"`
Packager string `json:"packager"`
Arch string `json:"arch"`
PgpSigned string `json:"pgp"`
}
// ParsePackage Function that receives arch package archive data and returns it's metadata.
func ParsePackage(r *packages.HashedBuffer) (*Package, error) {
md5, _, sha256, _ := r.Sums()
_, err := r.Seek(0, io.SeekStart)
if err != nil {
return nil, err
}
zstd := archiver.NewTarZstd()
err = zstd.Open(r, 0)
if err != nil {
return nil, err
}
defer zstd.Close()
var pkg *Package
var mtree bool
for {
f, err := zstd.Read()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
defer f.Close()
switch f.Name() {
case ".PKGINFO":
pkg, err = ParsePackageInfo(f)
if err != nil {
return nil, err
}
case ".MTREE":
mtree = true
}
}
if pkg == nil {
return nil, util.NewInvalidArgumentErrorf(".PKGINFO file not found")
}
if !mtree {
return nil, util.NewInvalidArgumentErrorf(".MTREE file not found")
}
pkg.FileMetadata.CompressedSize = r.Size()
pkg.FileMetadata.MD5 = hex.EncodeToString(md5)
pkg.FileMetadata.SHA256 = hex.EncodeToString(sha256)
return pkg, nil
}
// ParsePackageInfo Function that accepts reader for .PKGINFO file from package archive,
// validates all field according to PKGBUILD spec and returns package.
func ParsePackageInfo(r io.Reader) (*Package, error) {
p := &Package{}
scanner := bufio.NewScanner(r)
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "#") {
continue
}
key, value, find := strings.Cut(line, "=")
if !find {
continue
}
key = strings.TrimSpace(key)
value = strings.TrimSpace(value)
switch key {
case "pkgname":
p.Name = value
case "pkgbase":
p.VersionMetadata.Base = value
case "pkgver":
p.Version = value
case "pkgdesc":
p.VersionMetadata.Description = value
case "url":
p.VersionMetadata.ProjectURL = value
case "packager":
p.FileMetadata.Packager = value
case "arch":
p.FileMetadata.Arch = value
case "provides":
p.VersionMetadata.Provides = append(p.VersionMetadata.Provides, value)
case "license":
p.VersionMetadata.License = append(p.VersionMetadata.License, value)
case "depend":
p.VersionMetadata.Depends = append(p.VersionMetadata.Depends, value)
case "optdepend":
p.VersionMetadata.OptDepends = append(p.VersionMetadata.OptDepends, value)
case "makedepend":
p.VersionMetadata.MakeDepends = append(p.VersionMetadata.MakeDepends, value)
case "checkdepend":
p.VersionMetadata.CheckDepends = append(p.VersionMetadata.CheckDepends, value)
case "backup":
p.VersionMetadata.Backup = append(p.VersionMetadata.Backup, value)
case "group":
p.VersionMetadata.Groups = append(p.VersionMetadata.Groups, value)
case "conflict":
p.VersionMetadata.Conflicts = append(p.VersionMetadata.Conflicts, value)
case "replaces":
p.VersionMetadata.Replaces = append(p.VersionMetadata.Replaces, value)
case "xdata":
p.VersionMetadata.Xdata = append(p.VersionMetadata.Xdata, value)
case "builddate":
bd, err := strconv.ParseInt(value, 10, 64)
if err != nil {
return nil, err
}
p.FileMetadata.BuildDate = bd
case "size":
is, err := strconv.ParseInt(value, 10, 64)
if err != nil {
return nil, err
}
p.FileMetadata.InstalledSize = is
default:
return nil, util.NewInvalidArgumentErrorf("property is not supported %s", key)
}
}
return p, errors.Join(scanner.Err(), ValidatePackageSpec(p))
}
// ValidatePackageSpec Arch package validation according to PKGBUILD specification.
func ValidatePackageSpec(p *Package) error {
if !reName.MatchString(p.Name) {
return util.NewInvalidArgumentErrorf("invalid package name")
}
if !reName.MatchString(p.VersionMetadata.Base) {
return util.NewInvalidArgumentErrorf("invalid package base")
}
if !reVer.MatchString(p.Version) {
return util.NewInvalidArgumentErrorf("invalid package version")
}
if p.FileMetadata.Arch == "" {
return util.NewInvalidArgumentErrorf("architecture should be specified")
}
if p.VersionMetadata.ProjectURL != "" {
if !validation.IsValidURL(p.VersionMetadata.ProjectURL) {
return util.NewInvalidArgumentErrorf("invalid project URL")
}
}
for _, cd := range p.VersionMetadata.CheckDepends {
if !rePkgVer.MatchString(cd) {
return util.NewInvalidArgumentErrorf("invalid check dependency: " + cd)
}
}
for _, d := range p.VersionMetadata.Depends {
if !rePkgVer.MatchString(d) {
return util.NewInvalidArgumentErrorf("invalid dependency: " + d)
}
}
for _, md := range p.VersionMetadata.MakeDepends {
if !rePkgVer.MatchString(md) {
return util.NewInvalidArgumentErrorf("invalid make dependency: " + md)
}
}
for _, p := range p.VersionMetadata.Provides {
if !rePkgVer.MatchString(p) {
return util.NewInvalidArgumentErrorf("invalid provides: " + p)
}
}
for _, p := range p.VersionMetadata.Conflicts {
if !rePkgVer.MatchString(p) {
return util.NewInvalidArgumentErrorf("invalid conflicts: " + p)
}
}
for _, p := range p.VersionMetadata.Replaces {
if !rePkgVer.MatchString(p) {
return util.NewInvalidArgumentErrorf("invalid replaces: " + p)
}
}
for _, p := range p.VersionMetadata.Replaces {
if !rePkgVer.MatchString(p) {
return util.NewInvalidArgumentErrorf("invalid xdata: " + p)
}
}
for _, od := range p.VersionMetadata.OptDepends {
if !reOptDep.MatchString(od) {
return util.NewInvalidArgumentErrorf("invalid optional dependency: " + od)
}
}
for _, bf := range p.VersionMetadata.Backup {
if strings.HasPrefix(bf, "/") {
return util.NewInvalidArgumentErrorf("backup file contains leading forward slash")
}
}
return nil
}
// Desc Create pacman package description file.
func (p *Package) Desc() string {
entries := []string{
"FILENAME", fmt.Sprintf("%s-%s-%s.pkg.tar.zst", p.Name, p.Version, p.FileMetadata.Arch),
"NAME", p.Name,
"BASE", p.VersionMetadata.Base,
"VERSION", p.Version,
"DESC", p.VersionMetadata.Description,
"GROUPS", strings.Join(p.VersionMetadata.Groups, "\n"),
"CSIZE", fmt.Sprintf("%d", p.FileMetadata.CompressedSize),
"ISIZE", fmt.Sprintf("%d", p.FileMetadata.InstalledSize),
"MD5SUM", p.FileMetadata.MD5,
"SHA256SUM", p.FileMetadata.SHA256,
"PGPSIG", p.FileMetadata.PgpSigned,
"URL", p.VersionMetadata.ProjectURL,
"LICENSE", strings.Join(p.VersionMetadata.License, "\n"),
"ARCH", p.FileMetadata.Arch,
"BUILDDATE", fmt.Sprintf("%d", p.FileMetadata.BuildDate),
"PACKAGER", p.FileMetadata.Packager,
"REPLACES", strings.Join(p.VersionMetadata.Replaces, "\n"),
"CONFLICTS", strings.Join(p.VersionMetadata.Conflicts, "\n"),
"PROVIDES", strings.Join(p.VersionMetadata.Provides, "\n"),
"DEPENDS", strings.Join(p.VersionMetadata.Depends, "\n"),
"OPTDEPENDS", strings.Join(p.VersionMetadata.OptDepends, "\n"),
"MAKEDEPENDS", strings.Join(p.VersionMetadata.MakeDepends, "\n"),
"CHECKDEPENDS", strings.Join(p.VersionMetadata.CheckDepends, "\n"),
}
var buf bytes.Buffer
for i := 0; i < len(entries); i += 2 {
if entries[i+1] != "" {
_, _ = fmt.Fprintf(&buf, "%%%s%%\n%s\n\n", entries[i], entries[i+1])
}
}
return buf.String()
}

View file

@ -0,0 +1,445 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package arch
import (
"bytes"
"errors"
"os"
"strings"
"testing"
"testing/fstest"
"time"
"code.gitea.io/gitea/modules/packages"
"github.com/mholt/archiver/v3"
"github.com/stretchr/testify/require"
)
func TestParsePackage(t *testing.T) {
// Minimal PKGINFO contents and test FS
const PKGINFO = `pkgname = a
pkgbase = b
pkgver = 1-2
arch = x86_64
`
fs := fstest.MapFS{
"pkginfo": &fstest.MapFile{
Data: []byte(PKGINFO),
Mode: os.ModePerm,
ModTime: time.Now(),
},
"mtree": &fstest.MapFile{
Data: []byte("data"),
Mode: os.ModePerm,
ModTime: time.Now(),
},
}
// Test .PKGINFO file
pinf, err := fs.Stat("pkginfo")
require.NoError(t, err)
pfile, err := fs.Open("pkginfo")
require.NoError(t, err)
parcname, err := archiver.NameInArchive(pinf, ".PKGINFO", ".PKGINFO")
require.NoError(t, err)
// Test .MTREE file
minf, err := fs.Stat("mtree")
require.NoError(t, err)
mfile, err := fs.Open("mtree")
require.NoError(t, err)
marcname, err := archiver.NameInArchive(minf, ".MTREE", ".MTREE")
require.NoError(t, err)
t.Run("normal archive", func(t *testing.T) {
var buf bytes.Buffer
archive := archiver.NewTarZstd()
archive.Create(&buf)
err = archive.Write(archiver.File{
FileInfo: archiver.FileInfo{
FileInfo: pinf,
CustomName: parcname,
},
ReadCloser: pfile,
})
require.NoError(t, errors.Join(pfile.Close(), err))
err = archive.Write(archiver.File{
FileInfo: archiver.FileInfo{
FileInfo: minf,
CustomName: marcname,
},
ReadCloser: mfile,
})
require.NoError(t, errors.Join(mfile.Close(), archive.Close(), err))
reader, err := packages.CreateHashedBufferFromReader(&buf)
if err != nil {
t.Fatal(err)
}
defer reader.Close()
_, err = ParsePackage(reader)
require.NoError(t, err)
})
t.Run("missing .PKGINFO", func(t *testing.T) {
var buf bytes.Buffer
archive := archiver.NewTarZstd()
archive.Create(&buf)
require.NoError(t, archive.Close())
reader, err := packages.CreateHashedBufferFromReader(&buf)
require.NoError(t, err)
defer reader.Close()
_, err = ParsePackage(reader)
require.Error(t, err)
require.Contains(t, err.Error(), ".PKGINFO file not found")
})
t.Run("missing .MTREE", func(t *testing.T) {
var buf bytes.Buffer
pfile, err := fs.Open("pkginfo")
require.NoError(t, err)
archive := archiver.NewTarZstd()
archive.Create(&buf)
err = archive.Write(archiver.File{
FileInfo: archiver.FileInfo{
FileInfo: pinf,
CustomName: parcname,
},
ReadCloser: pfile,
})
require.NoError(t, errors.Join(pfile.Close(), archive.Close(), err))
reader, err := packages.CreateHashedBufferFromReader(&buf)
require.NoError(t, err)
defer reader.Close()
_, err = ParsePackage(reader)
require.Error(t, err)
require.Contains(t, err.Error(), ".MTREE file not found")
})
}
func TestParsePackageInfo(t *testing.T) {
const PKGINFO = `# Generated by makepkg 6.0.2
# using fakeroot version 1.31
pkgname = a
pkgbase = b
pkgver = 1-2
pkgdesc = comment
url = https://example.com/
group = group
builddate = 3
packager = Name Surname <login@example.com>
size = 5
arch = x86_64
license = BSD
provides = pvd
depend = smth
optdepend = hex
checkdepend = ola
makedepend = cmake
backup = usr/bin/paket1
`
p, err := ParsePackageInfo(strings.NewReader(PKGINFO))
require.NoError(t, err)
require.Equal(t, Package{
Name: "a",
Version: "1-2",
VersionMetadata: VersionMetadata{
Base: "b",
Description: "comment",
ProjectURL: "https://example.com/",
Groups: []string{"group"},
Provides: []string{"pvd"},
License: []string{"BSD"},
Depends: []string{"smth"},
OptDepends: []string{"hex"},
MakeDepends: []string{"cmake"},
CheckDepends: []string{"ola"},
Backup: []string{"usr/bin/paket1"},
},
FileMetadata: FileMetadata{
InstalledSize: 5,
BuildDate: 3,
Packager: "Name Surname <login@example.com>",
Arch: "x86_64",
},
}, *p)
}
func TestValidatePackageSpec(t *testing.T) {
newpkg := func() Package {
return Package{
Name: "abc",
Version: "1-1",
VersionMetadata: VersionMetadata{
Base: "ghx",
Description: "whoami",
ProjectURL: "https://example.com/",
Groups: []string{"gnome"},
Provides: []string{"abc", "def"},
License: []string{"GPL"},
Depends: []string{"go", "gpg=1", "curl>=3", "git<=7"},
OptDepends: []string{"git: something", "make"},
MakeDepends: []string{"chrom"},
CheckDepends: []string{"bariy"},
Backup: []string{"etc/pacman.d/filo"},
},
FileMetadata: FileMetadata{
CompressedSize: 1,
InstalledSize: 2,
SHA256: "def",
BuildDate: 3,
Packager: "smon",
Arch: "x86_64",
},
}
}
t.Run("valid package", func(t *testing.T) {
p := newpkg()
err := ValidatePackageSpec(&p)
require.NoError(t, err)
})
t.Run("invalid package name", func(t *testing.T) {
p := newpkg()
p.Name = "!$%@^!*&()"
err := ValidatePackageSpec(&p)
require.Error(t, err)
require.Contains(t, err.Error(), "invalid package name")
})
t.Run("invalid package base", func(t *testing.T) {
p := newpkg()
p.VersionMetadata.Base = "!$%@^!*&()"
err := ValidatePackageSpec(&p)
require.Error(t, err)
require.Contains(t, err.Error(), "invalid package base")
})
t.Run("invalid package version", func(t *testing.T) {
p := newpkg()
p.VersionMetadata.Base = "una-luna?"
err := ValidatePackageSpec(&p)
require.Error(t, err)
require.Contains(t, err.Error(), "invalid package base")
})
t.Run("invalid package version", func(t *testing.T) {
p := newpkg()
p.Version = "una-luna"
err := ValidatePackageSpec(&p)
require.Error(t, err)
require.Contains(t, err.Error(), "invalid package version")
})
t.Run("missing architecture", func(t *testing.T) {
p := newpkg()
p.FileMetadata.Arch = ""
err := ValidatePackageSpec(&p)
require.Error(t, err)
require.Contains(t, err.Error(), "architecture should be specified")
})
t.Run("invalid URL", func(t *testing.T) {
p := newpkg()
p.VersionMetadata.ProjectURL = "http%%$#"
err := ValidatePackageSpec(&p)
require.Error(t, err)
require.Contains(t, err.Error(), "invalid project URL")
})
t.Run("invalid check dependency", func(t *testing.T) {
p := newpkg()
p.VersionMetadata.CheckDepends = []string{"Err^_^"}
err := ValidatePackageSpec(&p)
require.Error(t, err)
require.Contains(t, err.Error(), "invalid check dependency")
})
t.Run("invalid dependency", func(t *testing.T) {
p := newpkg()
p.VersionMetadata.Depends = []string{"^^abc"}
err := ValidatePackageSpec(&p)
require.Error(t, err)
require.Contains(t, err.Error(), "invalid dependency")
})
t.Run("invalid make dependency", func(t *testing.T) {
p := newpkg()
p.VersionMetadata.MakeDepends = []string{"^m^"}
err := ValidatePackageSpec(&p)
require.Error(t, err)
require.Contains(t, err.Error(), "invalid make dependency")
})
t.Run("invalid provides", func(t *testing.T) {
p := newpkg()
p.VersionMetadata.Provides = []string{"^m^"}
err := ValidatePackageSpec(&p)
require.Error(t, err)
require.Contains(t, err.Error(), "invalid provides")
})
t.Run("invalid optional dependency", func(t *testing.T) {
p := newpkg()
p.VersionMetadata.OptDepends = []string{"^m^:MM"}
err := ValidatePackageSpec(&p)
require.Error(t, err)
require.Contains(t, err.Error(), "invalid optional dependency")
})
t.Run("invalid optional dependency", func(t *testing.T) {
p := newpkg()
p.VersionMetadata.Backup = []string{"/ola/cola"}
err := ValidatePackageSpec(&p)
require.Error(t, err)
require.Contains(t, err.Error(), "backup file contains leading forward slash")
})
}
func TestDescString(t *testing.T) {
const pkgdesc = `%FILENAME%
zstd-1.5.5-1-x86_64.pkg.tar.zst
%NAME%
zstd
%BASE%
zstd
%VERSION%
1.5.5-1
%DESC%
Zstandard - Fast real-time compression algorithm
%GROUPS%
dummy1
dummy2
%CSIZE%
401
%ISIZE%
1500453
%MD5SUM%
5016660ef3d9aa148a7b72a08d3df1b2
%SHA256SUM%
9fa4ede47e35f5971e4f26ecadcbfb66ab79f1d638317ac80334a3362dedbabd
%URL%
https://facebook.github.io/zstd/
%LICENSE%
BSD
GPL2
%ARCH%
x86_64
%BUILDDATE%
1681646714
%PACKAGER%
Jelle van der Waa <jelle@archlinux.org>
%PROVIDES%
libzstd.so=1-64
%DEPENDS%
glibc
gcc-libs
zlib
xz
lz4
%OPTDEPENDS%
dummy3
dummy4
%MAKEDEPENDS%
cmake
gtest
ninja
%CHECKDEPENDS%
dummy5
dummy6
`
md := &Package{
Name: "zstd",
Version: "1.5.5-1",
VersionMetadata: VersionMetadata{
Base: "zstd",
Description: "Zstandard - Fast real-time compression algorithm",
ProjectURL: "https://facebook.github.io/zstd/",
Groups: []string{"dummy1", "dummy2"},
Provides: []string{"libzstd.so=1-64"},
License: []string{"BSD", "GPL2"},
Depends: []string{"glibc", "gcc-libs", "zlib", "xz", "lz4"},
OptDepends: []string{"dummy3", "dummy4"},
MakeDepends: []string{"cmake", "gtest", "ninja"},
CheckDepends: []string{"dummy5", "dummy6"},
},
FileMetadata: FileMetadata{
CompressedSize: 401,
InstalledSize: 1500453,
MD5: "5016660ef3d9aa148a7b72a08d3df1b2",
SHA256: "9fa4ede47e35f5971e4f26ecadcbfb66ab79f1d638317ac80334a3362dedbabd",
BuildDate: 1681646714,
Packager: "Jelle van der Waa <jelle@archlinux.org>",
Arch: "x86_64",
},
}
require.Equal(t, pkgdesc, md.Desc())
}

View file

@ -24,6 +24,7 @@ var (
LimitTotalOwnerCount int64
LimitTotalOwnerSize int64
LimitSizeAlpine int64
LimitSizeArch int64
LimitSizeCargo int64
LimitSizeChef int64
LimitSizeComposer int64
@ -83,6 +84,7 @@ func loadPackagesFrom(rootCfg ConfigProvider) (err error) {
Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE")
Packages.LimitSizeAlpine = mustBytes(sec, "LIMIT_SIZE_ALPINE")
Packages.LimitSizeArch = mustBytes(sec, "LIMIT_SIZE_ARCH")
Packages.LimitSizeCargo = mustBytes(sec, "LIMIT_SIZE_CARGO")
Packages.LimitSizeChef = mustBytes(sec, "LIMIT_SIZE_CHEF")
Packages.LimitSizeComposer = mustBytes(sec, "LIMIT_SIZE_COMPOSER")