Add Cargo package registry (#21888)

This PR implements a [Cargo registry](https://doc.rust-lang.org/cargo/)
to manage Rust packages. This package type was a little bit more
complicated because Cargo needs an additional Git repository to store
its package index.

Screenshots:

![grafik](https://user-images.githubusercontent.com/1666336/203102004-08d812ac-c066-4969-9bda-2fed818554eb.png)

![grafik](https://user-images.githubusercontent.com/1666336/203102141-d9970f14-dca6-4174-b17a-50ba1bd79087.png)

![grafik](https://user-images.githubusercontent.com/1666336/203102244-dc05743b-78b6-4d97-998e-ef76341a978f.png)

---------

Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>

routers/api/packages/api.go

@@ -14,6 +14,7 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/web"
+	"code.gitea.io/gitea/routers/api/packages/cargo"
 	"code.gitea.io/gitea/routers/api/packages/composer"
 	"code.gitea.io/gitea/routers/api/packages/conan"
 	"code.gitea.io/gitea/routers/api/packages/conda"
@@ -71,6 +72,20 @@ func CommonRoutes(ctx gocontext.Context) *web.Route {
 	})
 
 	r.Group("/{username}", func() {
+		r.Group("/cargo", func() {
+			r.Group("/api/v1/crates", func() {
+				r.Get("", cargo.SearchPackages)
+				r.Put("/new", reqPackageAccess(perm.AccessModeWrite), cargo.UploadPackage)
+				r.Group("/{package}", func() {
+					r.Group("/{version}", func() {
+						r.Get("/download", cargo.DownloadPackageFile)
+						r.Delete("/yank", reqPackageAccess(perm.AccessModeWrite), cargo.YankPackage)
+						r.Put("/unyank", reqPackageAccess(perm.AccessModeWrite), cargo.UnyankPackage)
+					})
+					r.Get("/owners", cargo.ListOwners)
+				})
+			})
+		}, reqPackageAccess(perm.AccessModeRead))
 		r.Group("/composer", func() {
 			r.Get("/packages.json", composer.ServiceIndex)
 			r.Get("/search.json", composer.SearchPackages)

routers/api/packages/cargo/cargo.go

@@ -0,0 +1,281 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package cargo
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"code.gitea.io/gitea/models/db"
+	packages_model "code.gitea.io/gitea/models/packages"
+	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/log"
+	packages_module "code.gitea.io/gitea/modules/packages"
+	cargo_module "code.gitea.io/gitea/modules/packages/cargo"
+	"code.gitea.io/gitea/modules/util"
+	"code.gitea.io/gitea/routers/api/packages/helper"
+	"code.gitea.io/gitea/services/convert"
+	packages_service "code.gitea.io/gitea/services/packages"
+	cargo_service "code.gitea.io/gitea/services/packages/cargo"
+)
+
+// https://doc.rust-lang.org/cargo/reference/registries.html#web-api
+type StatusResponse struct {
+	OK     bool            `json:"ok"`
+	Errors []StatusMessage `json:"errors,omitempty"`
+}
+
+type StatusMessage struct {
+	Message string `json:"detail"`
+}
+
+func apiError(ctx *context.Context, status int, obj interface{}) {
+	helper.LogAndProcessError(ctx, status, obj, func(message string) {
+		ctx.JSON(status, StatusResponse{
+			OK: false,
+			Errors: []StatusMessage{
+				{
+					Message: message,
+				},
+			},
+		})
+	})
+}
+
+type SearchResult struct {
+	Crates []*SearchResultCrate `json:"crates"`
+	Meta   SearchResultMeta     `json:"meta"`
+}
+
+type SearchResultCrate struct {
+	Name          string `json:"name"`
+	LatestVersion string `json:"max_version"`
+	Description   string `json:"description"`
+}
+
+type SearchResultMeta struct {
+	Total int64 `json:"total"`
+}
+
+// https://doc.rust-lang.org/cargo/reference/registries.html#search
+func SearchPackages(ctx *context.Context) {
+	page := ctx.FormInt("page")
+	if page < 1 {
+		page = 1
+	}
+	perPage := ctx.FormInt("per_page")
+	paginator := db.ListOptions{
+		Page:     page,
+		PageSize: convert.ToCorrectPageSize(perPage),
+	}
+
+	pvs, total, err := packages_model.SearchLatestVersions(
+		ctx,
+		&packages_model.PackageSearchOptions{
+			OwnerID:    ctx.Package.Owner.ID,
+			Type:       packages_model.TypeCargo,
+			Name:       packages_model.SearchValue{Value: ctx.FormTrim("q")},
+			IsInternal: util.OptionalBoolFalse,
+			Paginator:  &paginator,
+		},
+	)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	pds, err := packages_model.GetPackageDescriptors(ctx, pvs)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	crates := make([]*SearchResultCrate, 0, len(pvs))
+	for _, pd := range pds {
+		crates = append(crates, &SearchResultCrate{
+			Name:          pd.Package.Name,
+			LatestVersion: pd.Version.Version,
+			Description:   pd.Metadata.(*cargo_module.Metadata).Description,
+		})
+	}
+
+	ctx.JSON(http.StatusOK, SearchResult{
+		Crates: crates,
+		Meta: SearchResultMeta{
+			Total: total,
+		},
+	})
+}
+
+type Owners struct {
+	Users []OwnerUser `json:"users"`
+}
+
+type OwnerUser struct {
+	ID    int64  `json:"id"`
+	Login string `json:"login"`
+	Name  string `json:"name"`
+}
+
+// https://doc.rust-lang.org/cargo/reference/registries.html#owners-list
+func ListOwners(ctx *context.Context) {
+	ctx.JSON(http.StatusOK, Owners{
+		Users: []OwnerUser{
+			{
+				ID:    ctx.Package.Owner.ID,
+				Login: ctx.Package.Owner.Name,
+				Name:  ctx.Package.Owner.DisplayName(),
+			},
+		},
+	})
+}
+
+// DownloadPackageFile serves the content of a package
+func DownloadPackageFile(ctx *context.Context) {
+	s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion(
+		ctx,
+		&packages_service.PackageInfo{
+			Owner:       ctx.Package.Owner,
+			PackageType: packages_model.TypeCargo,
+			Name:        ctx.Params("package"),
+			Version:     ctx.Params("version"),
+		},
+		&packages_service.PackageFileInfo{
+			Filename: strings.ToLower(fmt.Sprintf("%s-%s.crate", ctx.Params("package"), ctx.Params("version"))),
+		},
+	)
+	if err != nil {
+		if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist {
+			apiError(ctx, http.StatusNotFound, err)
+			return
+		}
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	defer s.Close()
+
+	ctx.ServeContent(s, &context.ServeHeaderOptions{
+		Filename:     pf.Name,
+		LastModified: pf.CreatedUnix.AsLocalTime(),
+	})
+}
+
+// https://doc.rust-lang.org/cargo/reference/registries.html#publish
+func UploadPackage(ctx *context.Context) {
+	defer ctx.Req.Body.Close()
+
+	cp, err := cargo_module.ParsePackage(ctx.Req.Body)
+	if err != nil {
+		apiError(ctx, http.StatusBadRequest, err)
+		return
+	}
+
+	buf, err := packages_module.CreateHashedBufferFromReader(cp.Content, 32*1024*1024)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	defer buf.Close()
+
+	if buf.Size() != cp.ContentSize {
+		apiError(ctx, http.StatusBadRequest, "invalid content size")
+		return
+	}
+
+	pv, _, err := packages_service.CreatePackageAndAddFile(
+		&packages_service.PackageCreationInfo{
+			PackageInfo: packages_service.PackageInfo{
+				Owner:       ctx.Package.Owner,
+				PackageType: packages_model.TypeCargo,
+				Name:        cp.Name,
+				Version:     cp.Version,
+			},
+			SemverCompatible: true,
+			Creator:          ctx.Doer,
+			Metadata:         cp.Metadata,
+			VersionProperties: map[string]string{
+				cargo_module.PropertyYanked: strconv.FormatBool(false),
+			},
+		},
+		&packages_service.PackageFileCreationInfo{
+			PackageFileInfo: packages_service.PackageFileInfo{
+				Filename: strings.ToLower(fmt.Sprintf("%s-%s.crate", cp.Name, cp.Version)),
+			},
+			Creator: ctx.Doer,
+			Data:    buf,
+			IsLead:  true,
+		},
+	)
+	if err != nil {
+		switch err {
+		case packages_model.ErrDuplicatePackageVersion:
+			apiError(ctx, http.StatusConflict, err)
+		case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
+			apiError(ctx, http.StatusForbidden, err)
+		default:
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+		return
+	}
+
+	if err := cargo_service.AddOrUpdatePackageIndex(ctx, ctx.Doer, ctx.Package.Owner, pv.PackageID); err != nil {
+		if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil {
+			log.Error("Rollback creation of package version: %v", err)
+		}
+
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	ctx.JSON(http.StatusOK, StatusResponse{OK: true})
+}
+
+// https://doc.rust-lang.org/cargo/reference/registries.html#yank
+func YankPackage(ctx *context.Context) {
+	yankPackage(ctx, true)
+}
+
+// https://doc.rust-lang.org/cargo/reference/registries.html#unyank
+func UnyankPackage(ctx *context.Context) {
+	yankPackage(ctx, false)
+}
+
+func yankPackage(ctx *context.Context, yank bool) {
+	pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeCargo, ctx.Params("package"), ctx.Params("version"))
+	if err != nil {
+		if err == packages_model.ErrPackageNotExist {
+			apiError(ctx, http.StatusNotFound, err)
+			return
+		}
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	pps, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeVersion, pv.ID, cargo_module.PropertyYanked)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	if len(pps) == 0 {
+		apiError(ctx, http.StatusInternalServerError, "Property not found")
+		return
+	}
+
+	pp := pps[0]
+	pp.Value = strconv.FormatBool(yank)
+
+	if err := packages_model.UpdateProperty(ctx, pp); err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	if err := cargo_service.AddOrUpdatePackageIndex(ctx, ctx.Doer, ctx.Package.Owner, pv.PackageID); err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	ctx.JSON(http.StatusOK, StatusResponse{OK: true})
+}

routers/api/v1/packages/package.go

@@ -40,7 +40,7 @@ func ListPackages(ctx *context.APIContext) {
 	//   in: query
 	//   description: package type filter
 	//   type: string
-	//   enum: [composer, conan, conda, container, generic, helm, maven, npm, nuget, pub, pypi, rubygems, vagrant]
+	//   enum: [cargo, composer, conan, conda, container, generic, helm, maven, npm, nuget, pub, pypi, rubygems, vagrant]
 	// - name: q
 	//   in: query
 	//   description: name filter

routers/web/org/setting_packages.go

@@ -84,3 +84,23 @@ func PackagesRulePreview(ctx *context.Context) {
 
 	ctx.HTML(http.StatusOK, tplSettingsPackagesRulePreview)
 }
+
+func InitializeCargoIndex(ctx *context.Context) {
+	ctx.Data["Title"] = ctx.Tr("packages.title")
+	ctx.Data["PageIsOrgSettings"] = true
+	ctx.Data["PageIsSettingsPackages"] = true
+
+	shared.InitializeCargoIndex(ctx, ctx.ContextUser)
+
+	ctx.Redirect(fmt.Sprintf("%s/org/%s/settings/packages", setting.AppSubURL, ctx.ContextUser.Name))
+}
+
+func RebuildCargoIndex(ctx *context.Context) {
+	ctx.Data["Title"] = ctx.Tr("packages.title")
+	ctx.Data["PageIsOrgSettings"] = true
+	ctx.Data["PageIsSettingsPackages"] = true
+
+	shared.RebuildCargoIndex(ctx, ctx.ContextUser)
+
+	ctx.Redirect(fmt.Sprintf("%s/org/%s/settings/packages", setting.AppSubURL, ctx.ContextUser.Name))
+}

routers/web/shared/packages/packages.go

@@ -13,9 +13,11 @@ import (
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/util"
 	"code.gitea.io/gitea/modules/web"
 	"code.gitea.io/gitea/services/forms"
+	cargo_service "code.gitea.io/gitea/services/packages/cargo"
 	container_service "code.gitea.io/gitea/services/packages/container"
 )
 
@@ -223,3 +225,23 @@ func getCleanupRuleByContext(ctx *context.Context, owner *user_model.User) *pack
 
 	return nil
 }
+
+func InitializeCargoIndex(ctx *context.Context, owner *user_model.User) {
+	err := cargo_service.InitializeIndexRepository(ctx, owner, owner)
+	if err != nil {
+		log.Error("InitializeIndexRepository failed: %v", err)
+		ctx.Flash.Error(ctx.Tr("packages.owner.settings.cargo.initialize.error", err))
+	} else {
+		ctx.Flash.Success(ctx.Tr("packages.owner.settings.cargo.initialize.success"))
+	}
+}
+
+func RebuildCargoIndex(ctx *context.Context, owner *user_model.User) {
+	err := cargo_service.RebuildIndex(ctx, owner, owner)
+	if err != nil {
+		log.Error("RebuildIndex failed: %v", err)
+		ctx.Flash.Error(ctx.Tr("packages.owner.settings.cargo.rebuild.error", err))
+	} else {
+		ctx.Flash.Success(ctx.Tr("packages.owner.settings.cargo.rebuild.success"))
+	}
+}

routers/web/user/setting/packages.go

@@ -77,3 +77,21 @@ func PackagesRulePreview(ctx *context.Context) {
 
 	ctx.HTML(http.StatusOK, tplSettingsPackagesRulePreview)
 }
+
+func InitializeCargoIndex(ctx *context.Context) {
+	ctx.Data["Title"] = ctx.Tr("packages.title")
+	ctx.Data["PageIsSettingsPackages"] = true
+
+	shared.InitializeCargoIndex(ctx, ctx.Doer)
+
+	ctx.Redirect(setting.AppSubURL + "/user/settings/packages")
+}
+
+func RebuildCargoIndex(ctx *context.Context) {
+	ctx.Data["Title"] = ctx.Tr("packages.title")
+	ctx.Data["PageIsSettingsPackages"] = true
+
+	shared.RebuildCargoIndex(ctx, ctx.Doer)
+
+	ctx.Redirect(setting.AppSubURL + "/user/settings/packages")
+}

routers/web/web.go

@@ -468,6 +468,10 @@ func RegisterRoutes(m *web.Route) {
 					m.Get("/preview", user_setting.PackagesRulePreview)
 				})
 			})
+			m.Group("/cargo", func() {
+				m.Post("/initialize", user_setting.InitializeCargoIndex)
+				m.Post("/rebuild", user_setting.RebuildCargoIndex)
+			})
 		}, packagesEnabled)
 		m.Group("/secrets", func() {
 			m.Get("", user_setting.Secrets)
@@ -818,6 +822,10 @@ func RegisterRoutes(m *web.Route) {
 							m.Get("/preview", org.PackagesRulePreview)
 						})
 					})
+					m.Group("/cargo", func() {
+						m.Post("/initialize", org.InitializeCargoIndex)
+						m.Post("/rebuild", org.RebuildCargoIndex)
+					})
 				}, packagesEnabled)
 			}, func(ctx *context.Context) {
 				ctx.Data["EnableOAuth2"] = setting.OAuth2.Enable