Add Package Registry (#16510)

* Added package store settings.

* Added models.

* Added generic package registry.

* Added tests.

* Added NuGet package registry.

* Moved service index to api file.

* Added NPM package registry.

* Added Maven package registry.

* Added PyPI package registry.

* Summary is deprecated.

* Changed npm name.

* Sanitize project url.

* Allow only scoped packages.

* Added user interface.

* Changed method name.

* Added missing migration file.

* Set page info.

* Added documentation.

* Added documentation links.

* Fixed wrong error message.

* Lint template files.

* Fixed merge errors.

* Fixed unit test storage path.

* Switch to json module.

* Added suggestions.

* Added package webhook.

* Add package api.

* Fixed swagger file.

* Fixed enum and comments.

* Fixed NuGet pagination.

* Print test names.

* Added api tests.

* Fixed access level.

* Fix User unmarshal.

* Added RubyGems package registry.

* Fix lint.

* Implemented io.Writer.

* Added support for sha256/sha512 checksum files.

* Improved maven-metadata.xml support.

* Added support for symbol package uploads.

* Added tests.

* Added overview docs.

* Added npm dependencies and keywords.

* Added no-packages information.

* Display file size.

* Display asset count.

* Fixed filter alignment.

* Added package icons.

* Formatted instructions.

* Allow anonymous package downloads.

* Fixed comments.

* Fixed postgres test.

* Moved file.

* Moved models to models/packages.

* Use correct error response format per client.

* Use simpler search form.

* Fixed IsProd.

* Restructured data model.

* Prevent empty filename.

* Fix swagger.

* Implemented user/org registry.

* Implemented UI.

* Use GetUserByIDCtx.

* Use table for dependencies.

* make svg

* Added support for unscoped npm packages.

* Add support for npm dist tags.

* Added tests for npm tags.

* Unlink packages if repository gets deleted.

* Prevent user/org delete if a packages exist.

* Use package unlink in repository service.

* Added support for composer packages.

* Restructured package docs.

* Added missing tests.

* Fixed generic content page.

* Fixed docs.

* Fixed swagger.

* Added missing type.

* Fixed ambiguous column.

* Organize content store by sha256 hash.

* Added admin package management.

* Added support for sorting.

* Add support for multiple identical versions/files.

* Added missing repository unlink.

* Added file properties.

* make fmt

* lint

* Added Conan package registry.

* Updated docs.

* Unify package names.

* Added swagger enum.

* Use longer TEXT column type.

* Removed version composite key.

* Merged package and container registry.

* Removed index.

* Use dedicated package router.

* Moved files to new location.

* Updated docs.

* Fixed JOIN order.

* Fixed GROUP BY statement.

* Fixed GROUP BY #2.

* Added symbol server support.

* Added more tests.

* Set NOT NULL.

* Added setting to disable package registries.

* Moved auth into service.

* refactor

* Use ctx everywhere.

* Added package cleanup task.

* Changed packages path.

* Added container registry.

* Refactoring

* Updated comparison.

* Fix swagger.

* Fixed table order.

* Use token auth for npm routes.

* Enabled ReverseProxy auth.

* Added packages link for orgs.

* Fixed anonymous org access.

* Enable copy button for setup instructions.

* Merge error

* Added suggestions.

* Fixed merge.

* Handle "generic".

* Added link for TODO.

* Added suggestions.

* Changed temporary buffer filename.

* Added suggestions.

* Apply suggestions from code review

Co-authored-by: Thomas Boerger <thomas@webhippie.de>

* Update docs/content/doc/packages/nuget.en-us.md

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: Thomas Boerger <thomas@webhippie.de>
This commit is contained in:
KN4CK3R 2022-03-30 10:42:47 +02:00 committed by GitHub
parent 2bce1ea986
commit 1d332342db
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
197 changed files with 18563 additions and 55 deletions

View file

@ -0,0 +1,68 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package conan
import (
"io"
"regexp"
"strings"
)
var (
patternAuthor = compilePattern("author")
patternHomepage = compilePattern("homepage")
patternURL = compilePattern("url")
patternLicense = compilePattern("license")
patternDescription = compilePattern("description")
patternTopics = regexp.MustCompile(`(?im)^\s*topics\s*=\s*\((.+)\)`)
patternTopicList = regexp.MustCompile(`\s*['"](.+?)['"]\s*,?`)
)
func compilePattern(name string) *regexp.Regexp {
return regexp.MustCompile(`(?im)^\s*` + name + `\s*=\s*['"\(](.+)['"\)]`)
}
func ParseConanfile(r io.Reader) (*Metadata, error) {
buf, err := io.ReadAll(io.LimitReader(r, 1<<20))
if err != nil {
return nil, err
}
metadata := &Metadata{}
m := patternAuthor.FindSubmatch(buf)
if len(m) > 1 && len(m[1]) > 0 {
metadata.Author = string(m[1])
}
m = patternHomepage.FindSubmatch(buf)
if len(m) > 1 && len(m[1]) > 0 {
metadata.ProjectURL = string(m[1])
}
m = patternURL.FindSubmatch(buf)
if len(m) > 1 && len(m[1]) > 0 {
metadata.RepositoryURL = string(m[1])
}
m = patternLicense.FindSubmatch(buf)
if len(m) > 1 && len(m[1]) > 0 {
metadata.License = strings.ReplaceAll(strings.ReplaceAll(string(m[1]), "'", ""), "\"", "")
}
m = patternDescription.FindSubmatch(buf)
if len(m) > 1 && len(m[1]) > 0 {
metadata.Description = string(m[1])
}
m = patternTopics.FindSubmatch(buf)
if len(m) > 1 && len(m[1]) > 0 {
m2 := patternTopicList.FindAllSubmatch(m[1], -1)
if len(m2) > 0 {
metadata.Keywords = make([]string, 0, len(m2))
for _, g := range m2 {
if len(g) > 1 {
metadata.Keywords = append(metadata.Keywords, string(g[1]))
}
}
}
}
return metadata, nil
}

View file

@ -0,0 +1,51 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package conan
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
const (
name = "ConanPackage"
version = "1.2"
license = "MIT"
author = "Gitea <info@gitea.io>"
homepage = "https://gitea.io/"
url = "https://gitea.com/"
description = "Description of ConanPackage"
topic1 = "gitea"
topic2 = "conan"
contentConanfile = `from conans import ConanFile, CMake, tools
class ConanPackageConan(ConanFile):
name = "` + name + `"
version = "` + version + `"
license = "` + license + `"
author = "` + author + `"
homepage = "` + homepage + `"
url = "` + url + `"
description = "` + description + `"
topics = ("` + topic1 + `", "` + topic2 + `")
settings = "os", "compiler", "build_type", "arch"
options = {"shared": [True, False], "fPIC": [True, False]}
default_options = {"shared": False, "fPIC": True}
generators = "cmake"
`
)
func TestParseConanfile(t *testing.T) {
metadata, err := ParseConanfile(strings.NewReader(contentConanfile))
assert.Nil(t, err)
assert.Equal(t, license, metadata.License)
assert.Equal(t, author, metadata.Author)
assert.Equal(t, homepage, metadata.ProjectURL)
assert.Equal(t, url, metadata.RepositoryURL)
assert.Equal(t, description, metadata.Description)
assert.Equal(t, []string{topic1, topic2}, metadata.Keywords)
}

View file

@ -0,0 +1,123 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package conan
import (
"bufio"
"errors"
"io"
"strings"
)
// Conaninfo represents infos of a Conan package
type Conaninfo struct {
Settings map[string]string `json:"settings"`
FullSettings map[string]string `json:"full_settings"`
Requires []string `json:"requires"`
FullRequires []string `json:"full_requires"`
Options map[string]string `json:"options"`
FullOptions map[string]string `json:"full_options"`
RecipeHash string `json:"recipe_hash"`
Environment map[string][]string `json:"environment"`
}
func ParseConaninfo(r io.Reader) (*Conaninfo, error) {
sections, err := readSections(io.LimitReader(r, 1<<20))
if err != nil {
return nil, err
}
info := &Conaninfo{}
for section, lines := range sections {
if len(lines) == 0 {
continue
}
switch section {
case "settings":
info.Settings = toMap(lines)
case "full_settings":
info.FullSettings = toMap(lines)
case "options":
info.Options = toMap(lines)
case "full_options":
info.FullOptions = toMap(lines)
case "requires":
info.Requires = lines
case "full_requires":
info.FullRequires = lines
case "recipe_hash":
info.RecipeHash = lines[0]
case "env":
info.Environment = toMapArray(lines)
}
}
return info, nil
}
func readSections(r io.Reader) (map[string][]string, error) {
sections := make(map[string][]string)
section := ""
lines := make([]string, 0, 5)
scanner := bufio.NewScanner(r)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") {
if section != "" {
sections[section] = lines
}
section = line[1 : len(line)-1]
lines = make([]string, 0, 5)
continue
}
if section != "" {
if line != "" {
lines = append(lines, line)
}
continue
}
if line != "" {
return nil, errors.New("Invalid conaninfo.txt")
}
}
if err := scanner.Err(); err != nil {
return nil, err
}
if section != "" {
sections[section] = lines
}
return sections, nil
}
func toMap(lines []string) map[string]string {
result := make(map[string]string)
for _, line := range lines {
parts := strings.SplitN(line, "=", 2)
if len(parts) != 2 || len(parts[0]) == 0 || len(parts[1]) == 0 {
continue
}
result[parts[0]] = parts[1]
}
return result
}
func toMapArray(lines []string) map[string][]string {
result := make(map[string][]string)
for _, line := range lines {
parts := strings.SplitN(line, "=", 2)
if len(parts) != 2 || len(parts[0]) == 0 || len(parts[1]) == 0 {
continue
}
var items []string
if strings.HasPrefix(parts[1], "[") && strings.HasSuffix(parts[1], "]") {
items = strings.Split(parts[1], ",")
} else {
items = []string{parts[1]}
}
result[parts[0]] = items
}
return result
}

View file

@ -0,0 +1,85 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package conan
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
const (
settingsKey = "arch"
settingsValue = "x84_64"
optionsKey = "shared"
optionsValue = "False"
requires = "fmt/7.1.3"
hash = "74714915a51073acb548ca1ce29afbac"
envKey = "CC"
envValue = "gcc-10"
contentConaninfo = `[settings]
` + settingsKey + `=` + settingsValue + `
[requires]
` + requires + `
[options]
` + optionsKey + `=` + optionsValue + `
[full_settings]
` + settingsKey + `=` + settingsValue + `
[full_requires]
` + requires + `
[full_options]
` + optionsKey + `=` + optionsValue + `
[recipe_hash]
` + hash + `
[env]
` + envKey + `=` + envValue + `
`
)
func TestParseConaninfo(t *testing.T) {
info, err := ParseConaninfo(strings.NewReader(contentConaninfo))
assert.NotNil(t, info)
assert.Nil(t, err)
assert.Equal(
t,
map[string]string{
settingsKey: settingsValue,
},
info.Settings,
)
assert.Equal(t, info.Settings, info.FullSettings)
assert.Equal(
t,
map[string]string{
optionsKey: optionsValue,
},
info.Options,
)
assert.Equal(t, info.Options, info.FullOptions)
assert.Equal(
t,
[]string{requires},
info.Requires,
)
assert.Equal(t, info.Requires, info.FullRequires)
assert.Equal(t, hash, info.RecipeHash)
assert.Equal(
t,
map[string][]string{
envKey: {envValue},
},
info.Environment,
)
}

View file

@ -0,0 +1,24 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package conan
const (
PropertyRecipeUser = "conan.recipe.user"
PropertyRecipeChannel = "conan.recipe.channel"
PropertyRecipeRevision = "conan.recipe.revision"
PropertyPackageReference = "conan.package.reference"
PropertyPackageRevision = "conan.package.revision"
PropertyPackageInfo = "conan.package.info"
)
// Metadata represents the metadata of a Conan package
type Metadata struct {
Author string `json:"author,omitempty"`
License string `json:"license,omitempty"`
ProjectURL string `json:"project_url,omitempty"`
RepositoryURL string `json:"repository_url,omitempty"`
Description string `json:"description,omitempty"`
Keywords []string `json:"keywords,omitempty"`
}

View file

@ -0,0 +1,155 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package conan
import (
"errors"
"fmt"
"regexp"
"code.gitea.io/gitea/modules/log"
goversion "github.com/hashicorp/go-version"
)
const (
// taken from https://github.com/conan-io/conan/blob/develop/conans/model/ref.py
minChars = 2
maxChars = 51
// DefaultRevision if no revision is specified
DefaultRevision = "0"
)
var (
namePattern = regexp.MustCompile(fmt.Sprintf(`^[a-zA-Z0-9_][a-zA-Z0-9_\+\.-]{%d,%d}$`, minChars-1, maxChars-1))
revisionPattern = regexp.MustCompile(fmt.Sprintf(`^[a-zA-Z0-9]{1,%d}$`, maxChars))
ErrValidation = errors.New("Could not validate one or more reference fields")
)
// RecipeReference represents a recipe <Name>/<Version>@<User>/<Channel>#<Revision>
type RecipeReference struct {
Name string
Version string
User string
Channel string
Revision string
}
func NewRecipeReference(name, version, user, channel, revision string) (*RecipeReference, error) {
log.Trace("Conan Recipe: %s/%s(@%s/%s(#%s))", name, version, user, channel, revision)
if user == "_" {
user = ""
}
if channel == "_" {
channel = ""
}
if (user != "" && channel == "") || (user == "" && channel != "") {
return nil, ErrValidation
}
if !namePattern.MatchString(name) {
return nil, ErrValidation
}
if _, err := goversion.NewSemver(version); err != nil {
return nil, ErrValidation
}
if user != "" && !namePattern.MatchString(user) {
return nil, ErrValidation
}
if channel != "" && !namePattern.MatchString(channel) {
return nil, ErrValidation
}
if revision != "" && !revisionPattern.MatchString(revision) {
return nil, ErrValidation
}
return &RecipeReference{name, version, user, channel, revision}, nil
}
func (r *RecipeReference) RevisionOrDefault() string {
if r.Revision == "" {
return DefaultRevision
}
return r.Revision
}
func (r *RecipeReference) String() string {
rev := ""
if r.Revision != "" {
rev = "#" + r.Revision
}
if r.User == "" || r.Channel == "" {
return fmt.Sprintf("%s/%s%s", r.Name, r.Version, rev)
}
return fmt.Sprintf("%s/%s@%s/%s%s", r.Name, r.Version, r.User, r.Channel, rev)
}
func (r *RecipeReference) LinkName() string {
user := r.User
if user == "" {
user = "_"
}
channel := r.Channel
if channel == "" {
channel = "_"
}
return fmt.Sprintf("%s/%s/%s/%s/%s", r.Name, r.Version, user, channel, r.RevisionOrDefault())
}
func (r *RecipeReference) WithRevision(revision string) *RecipeReference {
return &RecipeReference{r.Name, r.Version, r.User, r.Channel, revision}
}
// AsKey builds the additional key for the package file
func (r *RecipeReference) AsKey() string {
return fmt.Sprintf("%s|%s|%s", r.User, r.Channel, r.RevisionOrDefault())
}
// PackageReference represents a package of a recipe <Name>/<Version>@<User>/<Channel>#<Revision> <Reference>#<Revision>
type PackageReference struct {
Recipe *RecipeReference
Reference string
Revision string
}
func NewPackageReference(recipe *RecipeReference, reference, revision string) (*PackageReference, error) {
log.Trace("Conan Package: %v %s(#%s)", recipe, reference, revision)
if recipe == nil {
return nil, ErrValidation
}
if reference == "" || !revisionPattern.MatchString(reference) {
return nil, ErrValidation
}
if revision != "" && !revisionPattern.MatchString(revision) {
return nil, ErrValidation
}
return &PackageReference{recipe, reference, revision}, nil
}
func (r *PackageReference) RevisionOrDefault() string {
if r.Revision == "" {
return DefaultRevision
}
return r.Revision
}
func (r *PackageReference) LinkName() string {
return fmt.Sprintf("%s/%s", r.Reference, r.RevisionOrDefault())
}
func (r *PackageReference) WithRevision(revision string) *PackageReference {
return &PackageReference{r.Recipe, r.Reference, revision}
}
// AsKey builds the additional key for the package file
func (r *PackageReference) AsKey() string {
return fmt.Sprintf("%s|%s|%s|%s|%s", r.Recipe.User, r.Recipe.Channel, r.Recipe.RevisionOrDefault(), r.Reference, r.RevisionOrDefault())
}

View file

@ -0,0 +1,147 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package conan
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestNewRecipeReference(t *testing.T) {
cases := []struct {
Name string
Version string
User string
Channel string
Revision string
IsValid bool
}{
{"", "", "", "", "", false},
{"name", "", "", "", "", false},
{"", "1.0", "", "", "", false},
{"", "", "user", "", "", false},
{"", "", "", "channel", "", false},
{"", "", "", "", "0", false},
{"name", "1.0", "", "", "", true},
{"name", "1.0", "user", "", "", false},
{"name", "1.0", "", "channel", "", false},
{"name", "1.0", "user", "channel", "", true},
{"name", "1.0", "_", "", "", true},
{"name", "1.0", "", "_", "", true},
{"name", "1.0", "_", "_", "", true},
{"name", "1.0", "_", "_", "0", true},
{"name", "1.0", "", "", "0", true},
{"name", "1.0", "", "", "000000000000000000000000000000000000000000000000000000000000", false},
}
for i, c := range cases {
rref, err := NewRecipeReference(c.Name, c.Version, c.User, c.Channel, c.Revision)
if c.IsValid {
assert.NoError(t, err, "case %d, should be invalid", i)
assert.NotNil(t, rref, "case %d, should not be nil", i)
} else {
assert.Error(t, err, "case %d, should be valid", i)
}
}
}
func TestRecipeReferenceRevisionOrDefault(t *testing.T) {
rref, err := NewRecipeReference("name", "1.0", "", "", "")
assert.NoError(t, err)
assert.Equal(t, DefaultRevision, rref.RevisionOrDefault())
rref, err = NewRecipeReference("name", "1.0", "", "", DefaultRevision)
assert.NoError(t, err)
assert.Equal(t, DefaultRevision, rref.RevisionOrDefault())
rref, err = NewRecipeReference("name", "1.0", "", "", "Az09")
assert.NoError(t, err)
assert.Equal(t, "Az09", rref.RevisionOrDefault())
}
func TestRecipeReferenceString(t *testing.T) {
rref, err := NewRecipeReference("name", "1.0", "", "", "")
assert.NoError(t, err)
assert.Equal(t, "name/1.0", rref.String())
rref, err = NewRecipeReference("name", "1.0", "user", "channel", "")
assert.NoError(t, err)
assert.Equal(t, "name/1.0@user/channel", rref.String())
rref, err = NewRecipeReference("name", "1.0", "user", "channel", "Az09")
assert.NoError(t, err)
assert.Equal(t, "name/1.0@user/channel#Az09", rref.String())
}
func TestRecipeReferenceLinkName(t *testing.T) {
rref, err := NewRecipeReference("name", "1.0", "", "", "")
assert.NoError(t, err)
assert.Equal(t, "name/1.0/_/_/0", rref.LinkName())
rref, err = NewRecipeReference("name", "1.0", "user", "channel", "")
assert.NoError(t, err)
assert.Equal(t, "name/1.0/user/channel/0", rref.LinkName())
rref, err = NewRecipeReference("name", "1.0", "user", "channel", "Az09")
assert.NoError(t, err)
assert.Equal(t, "name/1.0/user/channel/Az09", rref.LinkName())
}
func TestNewPackageReference(t *testing.T) {
rref, _ := NewRecipeReference("name", "1.0", "", "", "")
cases := []struct {
Recipe *RecipeReference
Reference string
Revision string
IsValid bool
}{
{nil, "", "", false},
{rref, "", "", false},
{nil, "aZ09", "", false},
{rref, "aZ09", "", true},
{rref, "", "Az09", false},
{rref, "aZ09", "Az09", true},
}
for i, c := range cases {
pref, err := NewPackageReference(c.Recipe, c.Reference, c.Revision)
if c.IsValid {
assert.NoError(t, err, "case %d, should be invalid", i)
assert.NotNil(t, pref, "case %d, should not be nil", i)
} else {
assert.Error(t, err, "case %d, should be valid", i)
}
}
}
func TestPackageReferenceRevisionOrDefault(t *testing.T) {
rref, _ := NewRecipeReference("name", "1.0", "", "", "")
pref, err := NewPackageReference(rref, "ref", "")
assert.NoError(t, err)
assert.Equal(t, DefaultRevision, pref.RevisionOrDefault())
pref, err = NewPackageReference(rref, "ref", DefaultRevision)
assert.NoError(t, err)
assert.Equal(t, DefaultRevision, pref.RevisionOrDefault())
pref, err = NewPackageReference(rref, "ref", "Az09")
assert.NoError(t, err)
assert.Equal(t, "Az09", pref.RevisionOrDefault())
}
func TestPackageReferenceLinkName(t *testing.T) {
rref, _ := NewRecipeReference("name", "1.0", "", "", "")
pref, err := NewPackageReference(rref, "ref", "")
assert.NoError(t, err)
assert.Equal(t, "ref/0", pref.LinkName())
pref, err = NewPackageReference(rref, "ref", "Az09")
assert.NoError(t, err)
assert.Equal(t, "ref/Az09", pref.LinkName())
}