TRKJ/gitea
2
0
Fork 0
mirror of https://gitee.com/gitea/gitea synced 2025-11-04 08:30:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

Use native git variants by default with go-git variants as build tag (#13673)

Browse Source 
* Move last commit cache back into modules/git

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Remove go-git from the interface for last commit cache

Signed-off-by: Andrew Thornton <art27@cantab.net>

* move cacheref to last_commit_cache

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Remove go-git from routers/private/hook

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Move FindLFSFiles to pipeline

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Make no-go-git variants

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Submodule RefID

Signed-off-by: Andrew Thornton <art27@cantab.net>

* fix issue with GetCommitsInfo

Signed-off-by: Andrew Thornton <art27@cantab.net>

* fix GetLastCommitForPaths

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Improve efficiency

Signed-off-by: Andrew Thornton <art27@cantab.net>

* More efficiency

Signed-off-by: Andrew Thornton <art27@cantab.net>

* even faster

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Reduce duplication

* As per @lunny

Signed-off-by: Andrew Thornton <art27@cantab.net>

* attempt to fix drone

Signed-off-by: Andrew Thornton <art27@cantab.net>

* fix test-tags

Signed-off-by: Andrew Thornton <art27@cantab.net>

* default to use no-go-git variants and add gogit build tag

Signed-off-by: Andrew Thornton <art27@cantab.net>

* placate lint

Signed-off-by: Andrew Thornton <art27@cantab.net>

* as per @6543

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>
This commit is contained in:
zeripath
2020-12-17 14:00:47 +00:00
committed by GitHub
parent 0851a89581
commit 511f6138d4
81 changed files with 3952 additions and 1492 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
.drone.yml
View File

@@ -33,6 +33,16 @@ steps:
GOSUMDB: sum.golang.org
TAGS: bindata sqlite sqlite_unlock_notify
- name: lint-backend-gogit
pull: always
image: golang:1.15
commands:
- make lint-backend
environment:
GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
GOSUMDB: sum.golang.org
TAGS: bindata gogit sqlite sqlite_unlock_notify
- name: checks-frontend
image: node:14
commands:
@@ -69,7 +79,7 @@ steps:
GOPROXY: off
GOOS: linux
GOARCH: arm64
TAGS: bindata
TAGS: bindata gogit
commands:
- make backend # test cross compile
- rm ./gitea # clean
@@ -173,6 +183,17 @@ steps:
GITHUB_READ_TOKEN:
from_secret: github_read_token
- name: unit-test-gogit
pull: always
image: golang:1.15
commands:
- make unit-test-coverage test-check
environment:
GOPROXY: off
TAGS: bindata gogit sqlite sqlite_unlock_notify
GITHUB_READ_TOKEN:
from_secret: github_read_token
- name: test-mysql
image: golang:1.15
commands:
@@ -305,7 +326,8 @@ steps:
- timeout -s ABRT 40m make test-sqlite-migration test-sqlite
environment:
GOPROXY: off
TAGS: bindata
TAGS: bindata gogit sqlite sqlite_unlock_notify
TEST_TAGS: gogit sqlite sqlite_unlock_notify
USE_REPO_TEST_DIR: 1
depends_on:
- build
@@ -318,7 +340,8 @@ steps:
- timeout -s ABRT 40m make test-pgsql-migration test-pgsql
environment:
GOPROXY: off
TAGS: bindata
TAGS: bindata gogit
TEST_TAGS: gogit
TEST_LDAP: 1
USE_REPO_TEST_DIR: 1
depends_on:

19
Makefile
View File

@@ -110,7 +110,10 @@ TAGS ?=
TAGS_SPLIT := $(subst $(COMMA), ,$(TAGS))
TAGS_EVIDENCE := $(MAKE_EVIDENCE_DIR)/tags
TEST_TAGS ?= sqlite sqlite_unlock_notify
GO_DIRS := cmd integrations models modules routers build services vendor tools
GO_SOURCES := $(wildcard *.go)
GO_SOURCES += $(shell find $(GO_DIRS) -type f -name "*.go" -not -path modules/options/bindata.go -not -path modules/public/bindata.go -not -path modules/templates/bindata.go)
@@ -339,8 +342,8 @@ watch-backend: go-check
.PHONY: test
test:
@echo "Running go test..."
@$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='sqlite sqlite_unlock_notify' $(GO_PACKAGES)
@echo "Running go test with -tags '$(TEST_TAGS)'..."
@$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='$(TEST_TAGS)' $(GO_PACKAGES)
.PHONY: test-check
test-check:
@@ -356,8 +359,8 @@ test-check:
.PHONY: test\#%
test\#%:
@echo "Running go test..."
@$(GO) test -mod=vendor -tags='sqlite sqlite_unlock_notify' -run $(subst .,/,$*) $(GO_PACKAGES)
@echo "Running go test with -tags '$(TEST_TAGS)'..."
@$(GO) test -mod=vendor -tags='$(TEST_TAGS)' -run $(subst .,/,$*) $(GO_PACKAGES)
.PHONY: coverage
coverage:
@@ -365,8 +368,8 @@ coverage:
.PHONY: unit-test-coverage
unit-test-coverage:
@echo "Running unit-test-coverage..."
@$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='sqlite sqlite_unlock_notify' -cover -coverprofile coverage.out $(GO_PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1
@echo "Running unit-test-coverage -tags '$(TEST_TAGS)'..."
@$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='$(TEST_TAGS)' -cover -coverprofile coverage.out $(GO_PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1
.PHONY: vendor
vendor:
@@ -511,7 +514,7 @@ integrations.mssql.test: git-check $(GO_SOURCES)
$(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.mssql.test
integrations.sqlite.test: git-check $(GO_SOURCES)
$(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.sqlite.test -tags 'sqlite sqlite_unlock_notify'
$(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.sqlite.test -tags '$(TEST_TAGS)'
integrations.cover.test: git-check $(GO_SOURCES)
$(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -coverpkg $(shell echo $(GO_PACKAGES) | tr ' ' ',') -o integrations.cover.test
@@ -534,7 +537,7 @@ migrations.mssql.test: $(GO_SOURCES)
.PHONY: migrations.sqlite.test
migrations.sqlite.test: $(GO_SOURCES)
$(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/integrations/migration-test -o migrations.sqlite.test -tags 'sqlite sqlite_unlock_notify'
$(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/integrations/migration-test -o migrations.sqlite.test -tags '$(TEST_TAGS)'
.PHONY: check
check: test

1
docs/content/doc/installation/from-source.en-us.md
View File

@@ -101,6 +101,7 @@ Depending on requirements, the following build tags can be included.
- `pam`: Enable support for PAM (Linux Pluggable Authentication Modules). Can
be used to authenticate local users or extend authentication to methods
available to PAM.
* `gogit`: (EXPERIMENTAL) Use go-git variants of git commands.
Bundling assets into the binary using the `bindata` build tag is recommended for
production deployments. It is possible to serve the static assets directly via a reverse proxy,

23
modules/cache/cache.go vendored
View File

@@ -27,6 +27,24 @@ func newCache(cacheConfig setting.Cache) (mc.Cache, error) {
})
}
// Cache is the interface that operates the cache data.
type Cache interface {
// Put puts value into cache with key and expire time.
Put(key string, val interface{}, timeout int64) error
// Get gets cached value by given key.
Get(key string) interface{}
// Delete deletes cached value by given key.
Delete(key string) error
// Incr increases cached int-type value by given key as a counter.
Incr(key string) error
// Decr decreases cached int-type value by given key as a counter.
Decr(key string) error
// IsExist returns true if cached value exists.
IsExist(key string) bool
// Flush deletes all cached data.
Flush() error
}
// NewContext start cache service
func NewContext() error {
var err error
@@ -40,6 +58,11 @@ func NewContext() error {
return err
}
// GetCache returns the currently configured cache
func GetCache() Cache {
return conn
}
// GetString returns the key value from cache with callback when no key exists in cache
func GetString(key string, getFunc func() (string, error)) (string, error) {
if conn == nil || setting.CacheService.TTL == 0 {

70
modules/cache/last_commit.go vendored
View File

@@ -1,70 +0,0 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package cache
import (
"crypto/sha256"
"fmt"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
mc "gitea.com/macaron/cache"
"github.com/go-git/go-git/v5/plumbing/object"
)
// LastCommitCache represents a cache to store last commit
type LastCommitCache struct {
repoPath string
ttl int64
repo *git.Repository
commitCache map[string]*object.Commit
mc.Cache
}
// NewLastCommitCache creates a new last commit cache for repo
func NewLastCommitCache(repoPath string, gitRepo *git.Repository, ttl int64) *LastCommitCache {
return &LastCommitCache{
repoPath: repoPath,
repo: gitRepo,
commitCache: make(map[string]*object.Commit),
ttl: ttl,
Cache: conn,
}
}
func (c LastCommitCache) getCacheKey(repoPath, ref, entryPath string) string {
hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, ref, entryPath)))
return fmt.Sprintf("last_commit:%x", hashBytes)
}
// Get get the last commit information by commit id and entry path
func (c LastCommitCache) Get(ref, entryPath string) (*object.Commit, error) {
v := c.Cache.Get(c.getCacheKey(c.repoPath, ref, entryPath))
if vs, ok := v.(string); ok {
log.Trace("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs)
if commit, ok := c.commitCache[vs]; ok {
log.Trace("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs)
return commit, nil
}
id, err := c.repo.ConvertToSHA1(vs)
if err != nil {
return nil, err
}
commit, err := c.repo.GoGitRepo().CommitObject(id)
if err != nil {
return nil, err
}
c.commitCache[vs] = commit
return commit, nil
}
return nil, nil
}
// Put put the last commit id with commit and entry path
func (c LastCommitCache) Put(ref, entryPath, commitID string) error {
log.Trace("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID)
return c.Cache.Put(c.getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl)
}

3
modules/convert/git_commit_test.go
View File

@@ -13,7 +13,6 @@ import (
api "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/stretchr/testify/assert"
)
@@ -21,7 +20,7 @@ func TestToCommitMeta(t *testing.T) {
assert.NoError(t, models.PrepareTestDatabase())
headRepo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository)
sha1, _ := git.NewIDFromString("0000000000000000000000000000000000000000")
signature := &object.Signature{Name: "Test Signature", Email: "test@email.com", When: time.Unix(0, 0)}
signature := &git.Signature{Name: "Test Signature", Email: "test@email.com", When: time.Unix(0, 0)}
tag := &git.Tag{
Name: "Test Tag",
ID: sha1,

243
modules/git/batch_reader_nogogit.go Normal file
View File

@@ -0,0 +1,243 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"bufio"
"bytes"
"math"
"strconv"
)
// ReadBatchLine reads the header line from cat-file --batch
// We expect:
// <sha> SP <type> SP <size> LF
func ReadBatchLine(rd *bufio.Reader) (sha []byte, typ string, size int64, err error) {
sha, err = rd.ReadBytes(' ')
if err != nil {
return
}
sha = sha[:len(sha)-1]
typ, err = rd.ReadString(' ')
if err != nil {
return
}
typ = typ[:len(typ)-1]
var sizeStr string
sizeStr, err = rd.ReadString('\n')
if err != nil {
return
}
size, err = strconv.ParseInt(sizeStr[:len(sizeStr)-1], 10, 64)
return
}
// ReadTagObjectID reads a tag object ID hash from a cat-file --batch stream, throwing away the rest of the stream.
func ReadTagObjectID(rd *bufio.Reader, size int64) (string, error) {
id := ""
var n int64
headerLoop:
for {
line, err := rd.ReadBytes('\n')
if err != nil {
return "", err
}
n += int64(len(line))
idx := bytes.Index(line, []byte{' '})
if idx < 0 {
continue
}
if string(line[:idx]) == "object" {
id = string(line[idx+1 : len(line)-1])
break headerLoop
}
}
// Discard the rest of the tag
discard := size - n
for discard > math.MaxInt32 {
_, err := rd.Discard(math.MaxInt32)
if err != nil {
return id, err
}
discard -= math.MaxInt32
}
_, err := rd.Discard(int(discard))
return id, err
}
// ReadTreeID reads a tree ID from a cat-file --batch stream, throwing away the rest of the stream.
func ReadTreeID(rd *bufio.Reader, size int64) (string, error) {
id := ""
var n int64
headerLoop:
for {
line, err := rd.ReadBytes('\n')
if err != nil {
return "", err
}
n += int64(len(line))
idx := bytes.Index(line, []byte{' '})
if idx < 0 {
continue
}
if string(line[:idx]) == "tree" {
id = string(line[idx+1 : len(line)-1])
break headerLoop
}
}
// Discard the rest of the commit
discard := size - n
for discard > math.MaxInt32 {
_, err := rd.Discard(math.MaxInt32)
if err != nil {
return id, err
}
discard -= math.MaxInt32
}
_, err := rd.Discard(int(discard))
return id, err
}
// git tree files are a list:
// <mode-in-ascii> SP <fname> NUL <20-byte SHA>
//
// Unfortunately this 20-byte notation is somewhat in conflict to all other git tools
// Therefore we need some method to convert these 20-byte SHAs to a 40-byte SHA
// constant hextable to help quickly convert between 20byte and 40byte hashes
const hextable = "0123456789abcdef"
// to40ByteSHA converts a 20-byte SHA in a 40-byte slice into a 40-byte sha in place
// without allocations. This is at least 100x quicker that hex.EncodeToString
// NB This requires that sha is a 40-byte slice
func to40ByteSHA(sha []byte) []byte {
for i := 19; i >= 0; i-- {
v := sha[i]
vhi, vlo := v>>4, v&0x0f
shi, slo := hextable[vhi], hextable[vlo]
sha[i*2], sha[i*2+1] = shi, slo
}
return sha
}
// ParseTreeLineSkipMode reads an entry from a tree in a cat-file --batch stream
// This simply skips the mode - saving a substantial amount of time and carefully avoids allocations - except where fnameBuf is too small.
// It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations
//
// Each line is composed of:
// <mode-in-ascii-dropping-initial-zeros> SP <fname> NUL <20-byte SHA>
//
// We don't attempt to convert the 20-byte SHA to 40-byte SHA to save a lot of time
func ParseTreeLineSkipMode(rd *bufio.Reader, fnameBuf, shaBuf []byte) (fname, sha []byte, n int, err error) {
var readBytes []byte
// Skip the Mode
readBytes, err = rd.ReadSlice(' ') // NB: DOES NOT ALLOCATE SIMPLY RETURNS SLICE WITHIN READER BUFFER
if err != nil {
return
}
n += len(readBytes)
// Deal with the fname
readBytes, err = rd.ReadSlice('\x00')
copy(fnameBuf, readBytes)
if len(fnameBuf) > len(readBytes) {
fnameBuf = fnameBuf[:len(readBytes)] // cut the buf the correct size
} else {
fnameBuf = append(fnameBuf, readBytes[len(fnameBuf):]...) // extend the buf and copy in the missing bits
}
for err == bufio.ErrBufferFull { // Then we need to read more
readBytes, err = rd.ReadSlice('\x00')
fnameBuf = append(fnameBuf, readBytes...) // there is little point attempting to avoid allocations here so just extend
}
n += len(fnameBuf)
if err != nil {
return
}
fnameBuf = fnameBuf[:len(fnameBuf)-1] // Drop the terminal NUL
fname = fnameBuf // set the returnable fname to the slice
// Now deal with the 20-byte SHA
idx := 0
for idx < 20 {
read := 0
read, err = rd.Read(shaBuf[idx:20])
n += read
if err != nil {
return
}
idx += read
}
sha = shaBuf
return
}
// ParseTreeLine reads an entry from a tree in a cat-file --batch stream
// This carefully avoids allocations - except where fnameBuf is too small.
// It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations
//
// Each line is composed of:
// <mode-in-ascii-dropping-initial-zeros> SP <fname> NUL <20-byte SHA>
//
// We don't attempt to convert the 20-byte SHA to 40-byte SHA to save a lot of time
func ParseTreeLine(rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) {
var readBytes []byte
// Read the Mode
readBytes, err = rd.ReadSlice(' ')
if err != nil {
return
}
n += len(readBytes)
copy(modeBuf, readBytes)
if len(modeBuf) > len(readBytes) {
modeBuf = modeBuf[:len(readBytes)]
} else {
modeBuf = append(modeBuf, readBytes[len(modeBuf):]...)
}
mode = modeBuf[:len(modeBuf)-1] // Drop the SP
// Deal with the fname
readBytes, err = rd.ReadSlice('\x00')
copy(fnameBuf, readBytes)
if len(fnameBuf) > len(readBytes) {
fnameBuf = fnameBuf[:len(readBytes)]
} else {
fnameBuf = append(fnameBuf, readBytes[len(fnameBuf):]...)
}
for err == bufio.ErrBufferFull {
readBytes, err = rd.ReadSlice('\x00')
fnameBuf = append(fnameBuf, readBytes...)
}
n += len(fnameBuf)
if err != nil {
return
}
fnameBuf = fnameBuf[:len(fnameBuf)-1]
fname = fnameBuf
// Deal with the 20-byte SHA
idx := 0
for idx < 20 {
read := 0
read, err = rd.Read(shaBuf[idx:20])
n += read
if err != nil {
return
}
idx += read
}
sha = shaBuf
return
}

21
modules/git/blob.go
View File

@@ -10,28 +10,9 @@ import (
"encoding/base64"
"io"
"io/ioutil"
"github.com/go-git/go-git/v5/plumbing"
)
// Blob represents a Git object.
type Blob struct {
ID SHA1
gogitEncodedObj plumbing.EncodedObject
name string
}
// DataAsync gets a ReadCloser for the contents of a blob without reading it all.
// Calling the Close function on the result will discard all unread output.
func (b *Blob) DataAsync() (io.ReadCloser, error) {
return b.gogitEncodedObj.Reader()
}
// Size returns the uncompressed size of the blob
func (b *Blob) Size() int64 {
return b.gogitEncodedObj.Size()
}
// This file contains common functions between the gogit and !gogit variants for git Blobs
// Name returns name of the tree entry this blob object was created from (or empty string)
func (b *Blob) Name() string {

33
modules/git/blob_gogit.go Normal file
View File

@@ -0,0 +1,33 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"io"
"github.com/go-git/go-git/v5/plumbing"
)
// Blob represents a Git object.
type Blob struct {
ID SHA1
gogitEncodedObj plumbing.EncodedObject
name string
}
// DataAsync gets a ReadCloser for the contents of a blob without reading it all.
// Calling the Close function on the result will discard all unread output.
func (b *Blob) DataAsync() (io.ReadCloser, error) {
return b.gogitEncodedObj.Reader()
}
// Size returns the uncompressed size of the blob
func (b *Blob) Size() int64 {
return b.gogitEncodedObj.Size()
}

77
modules/git/blob_nogogit.go Normal file
View File

@@ -0,0 +1,77 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"bufio"
"io"
"strconv"
"strings"
)
// Blob represents a Git object.
type Blob struct {
ID SHA1
gotSize bool
size int64
repoPath string
name string
}
// DataAsync gets a ReadCloser for the contents of a blob without reading it all.
// Calling the Close function on the result will discard all unread output.
func (b *Blob) DataAsync() (io.ReadCloser, error) {
stdoutReader, stdoutWriter := io.Pipe()
var err error
go func() {
stderr := &strings.Builder{}
err = NewCommand("cat-file", "--batch").RunInDirFullPipeline(b.repoPath, stdoutWriter, stderr, strings.NewReader(b.ID.String()+"\n"))
if err != nil {
err = ConcatenateError(err, stderr.String())
_ = stdoutWriter.CloseWithError(err)
} else {
_ = stdoutWriter.Close()
}
}()
bufReader := bufio.NewReader(stdoutReader)
_, _, size, err := ReadBatchLine(bufReader)
if err != nil {
stdoutReader.Close()
return nil, err
}
return &LimitedReaderCloser{
R: bufReader,
C: stdoutReader,
N: int64(size),
}, err
}
// Size returns the uncompressed size of the blob
func (b *Blob) Size() int64 {
if b.gotSize {
return b.size
}
size, err := NewCommand("cat-file", "-s", b.ID.String()).RunInDir(b.repoPath)
if err != nil {
log("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repoPath, err)
return 0
}
b.size, err = strconv.ParseInt(size[:len(size)-1], 10, 64)
if err != nil {
log("error whilst parsing size %s for %s in %s. Error: %v", size, b.ID.String(), b.repoPath, err)
return 0
}
b.gotSize = true
return b.size
}

13
modules/git/cache.go
View File

@@ -1,13 +0,0 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package git
import "github.com/go-git/go-git/v5/plumbing/object"
// LastCommitCache cache
type LastCommitCache interface {
Get(ref, entryPath string) (*object.Commit, error)
Put(ref, entryPath, commitID string) error
}

2
modules/git/command.go
View File

@@ -189,7 +189,7 @@ func (c *Command) RunInDirTimeoutEnv(env []string, timeout time.Duration, dir st
stdout := new(bytes.Buffer)
stderr := new(bytes.Buffer)
if err := c.RunInDirTimeoutEnvPipeline(env, timeout, dir, stdout, stderr); err != nil {
return nil, concatenateError(err, stderr.String())
return nil, ConcatenateError(err, stderr.String())
}
if stdout.Len() > 0 {

59
modules/git/commit.go
View File

@@ -19,8 +19,6 @@ import (
"net/http"
"strconv"
"strings"
"github.com/go-git/go-git/v5/plumbing/object"
)
// Commit represents a git commit.
@@ -43,61 +41,6 @@ type CommitGPGSignature struct {
Payload string //TODO check if can be reconstruct from the rest of commit information to not have duplicate data
}
func convertPGPSignature(c *object.Commit) *CommitGPGSignature {
if c.PGPSignature == "" {
return nil
}
var w strings.Builder
var err error
if _, err = fmt.Fprintf(&w, "tree %s\n", c.TreeHash.String()); err != nil {
return nil
}
for _, parent := range c.ParentHashes {
if _, err = fmt.Fprintf(&w, "parent %s\n", parent.String()); err != nil {
return nil
}
}
if _, err = fmt.Fprint(&w, "author "); err != nil {
return nil
}
if err = c.Author.Encode(&w); err != nil {
return nil
}
if _, err = fmt.Fprint(&w, "\ncommitter "); err != nil {
return nil
}
if err = c.Committer.Encode(&w); err != nil {
return nil
}
if _, err = fmt.Fprintf(&w, "\n\n%s", c.Message); err != nil {
return nil
}
return &CommitGPGSignature{
Signature: c.PGPSignature,
Payload: w.String(),
}
}
func convertCommit(c *object.Commit) *Commit {
return &Commit{
ID: c.Hash,
CommitMessage: c.Message,
Committer: &c.Committer,
Author: &c.Author,
Signature: convertPGPSignature(c),
Parents: c.ParentHashes,
}
}
// Message returns the commit message. Same as retrieving CommitMessage directly.
func (c *Commit) Message() string {
return c.CommitMessage
@@ -576,7 +519,7 @@ func GetCommitFileStatus(repoPath, commitID string) (*CommitFileStatus, error) {
err := NewCommand("show", "--name-status", "--pretty=format:''", commitID).RunInDirPipeline(repoPath, w, stderr)
w.Close() // Close writer to exit parsing goroutine
if err != nil {
return nil, concatenateError(err, stderr.String())
return nil, ConcatenateError(err, stderr.String())
}
<-done

70
modules/git/commit_convert_gogit.go Normal file
View File

@@ -0,0 +1,70 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"fmt"
"strings"
"github.com/go-git/go-git/v5/plumbing/object"
)
func convertPGPSignature(c *object.Commit) *CommitGPGSignature {
if c.PGPSignature == "" {
return nil
}
var w strings.Builder
var err error
if _, err = fmt.Fprintf(&w, "tree %s\n", c.TreeHash.String()); err != nil {
return nil
}
for _, parent := range c.ParentHashes {
if _, err = fmt.Fprintf(&w, "parent %s\n", parent.String()); err != nil {
return nil
}
}
if _, err = fmt.Fprint(&w, "author "); err != nil {
return nil
}
if err = c.Author.Encode(&w); err != nil {
return nil
}
if _, err = fmt.Fprint(&w, "\ncommitter "); err != nil {
return nil
}
if err = c.Committer.Encode(&w); err != nil {
return nil
}
if _, err = fmt.Fprintf(&w, "\n\n%s", c.Message); err != nil {
return nil
}
return &CommitGPGSignature{
Signature: c.PGPSignature,
Payload: w.String(),
}
}
func convertCommit(c *object.Commit) *Commit {
return &Commit{
ID: c.Hash,
CommitMessage: c.Message,
Committer: &c.Committer,
Author: &c.Author,
Signature: convertPGPSignature(c),
Parents: c.ParentHashes,
}
}

287
modules/git/commit_info.go
View File

@@ -4,286 +4,9 @@
package git
import (
"path"
"github.com/emirpasic/gods/trees/binaryheap"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph"
)
// GetCommitsInfo gets information of all commits that are corresponding to these entries
func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache LastCommitCache) ([][]interface{}, *Commit, error) {
entryPaths := make([]string, len(tes)+1)
// Get the commit for the treePath itself
entryPaths[0] = ""
for i, entry := range tes {
entryPaths[i+1] = entry.Name()
}
commitNodeIndex, commitGraphFile := commit.repo.CommitNodeIndex()
if commitGraphFile != nil {
defer commitGraphFile.Close()
}
c, err := commitNodeIndex.Get(commit.ID)
if err != nil {
return nil, nil, err
}
var revs map[string]*object.Commit
if cache != nil {
var unHitPaths []string
revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache)
if err != nil {
return nil, nil, err
}
if len(unHitPaths) > 0 {
revs2, err := GetLastCommitForPaths(c, treePath, unHitPaths)
if err != nil {
return nil, nil, err
}
for k, v := range revs2 {
if err := cache.Put(commit.ID.String(), path.Join(treePath, k), v.ID().String()); err != nil {
return nil, nil, err
}
revs[k] = v
}
}
} else {
revs, err = GetLastCommitForPaths(c, treePath, entryPaths)
}
if err != nil {
return nil, nil, err
}
commit.repo.gogitStorage.Close()
commitsInfo := make([][]interface{}, len(tes))
for i, entry := range tes {
if rev, ok := revs[entry.Name()]; ok {
entryCommit := convertCommit(rev)
if entry.IsSubModule() {
subModuleURL := ""
var fullPath string
if len(treePath) > 0 {
fullPath = treePath + "/" + entry.Name()
} else {
fullPath = entry.Name()
}
if subModule, err := commit.GetSubModule(fullPath); err != nil {
return nil, nil, err
} else if subModule != nil {
subModuleURL = subModule.URL
}
subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String())
commitsInfo[i] = []interface{}{entry, subModuleFile}
} else {
commitsInfo[i] = []interface{}{entry, entryCommit}
}
} else {
commitsInfo[i] = []interface{}{entry, nil}
}
}
// Retrieve the commit for the treePath itself (see above). We basically
// get it for free during the tree traversal and it's used for listing
// pages to display information about newest commit for a given path.
var treeCommit *Commit
if treePath == "" {
treeCommit = commit
} else if rev, ok := revs[""]; ok {
treeCommit = convertCommit(rev)
treeCommit.repo = commit.repo
}
return commitsInfo, treeCommit, nil
}
type commitAndPaths struct {
commit cgobject.CommitNode
// Paths that are still on the branch represented by commit
paths []string
// Set of hashes for the paths
hashes map[string]plumbing.Hash
}
func getCommitTree(c cgobject.CommitNode, treePath string) (*object.Tree, error) {
tree, err := c.Tree()
if err != nil {
return nil, err
}
// Optimize deep traversals by focusing only on the specific tree
if treePath != "" {
tree, err = tree.Tree(treePath)
if err != nil {
return nil, err
}
}
return tree, nil
}
func getFileHashes(c cgobject.CommitNode, treePath string, paths []string) (map[string]plumbing.Hash, error) {
tree, err := getCommitTree(c, treePath)
if err == object.ErrDirectoryNotFound {
// The whole tree didn't exist, so return empty map
return make(map[string]plumbing.Hash), nil
}
if err != nil {
return nil, err
}
hashes := make(map[string]plumbing.Hash)
for _, path := range paths {
if path != "" {
entry, err := tree.FindEntry(path)
if err == nil {
hashes[path] = entry.Hash
}
} else {
hashes[path] = tree.Hash
}
}
return hashes, nil
}
func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache LastCommitCache) (map[string]*object.Commit, []string, error) {
var unHitEntryPaths []string
var results = make(map[string]*object.Commit)
for _, p := range paths {
lastCommit, err := cache.Get(commitID, path.Join(treePath, p))
if err != nil {
return nil, nil, err
}
if lastCommit != nil {
results[p] = lastCommit
continue
}
unHitEntryPaths = append(unHitEntryPaths, p)
}
return results, unHitEntryPaths, nil
}
// GetLastCommitForPaths returns last commit information
func GetLastCommitForPaths(c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) {
// We do a tree traversal with nodes sorted by commit time
heap := binaryheap.NewWith(func(a, b interface{}) int {
if a.(*commitAndPaths).commit.CommitTime().Before(b.(*commitAndPaths).commit.CommitTime()) {
return 1
}
return -1
})
resultNodes := make(map[string]cgobject.CommitNode)
initialHashes, err := getFileHashes(c, treePath, paths)
if err != nil {
return nil, err
}
// Start search from the root commit and with full set of paths
heap.Push(&commitAndPaths{c, paths, initialHashes})
for {
cIn, ok := heap.Pop()
if !ok {
break
}
current := cIn.(*commitAndPaths)
// Load the parent commits for the one we are currently examining
numParents := current.commit.NumParents()
var parents []cgobject.CommitNode
for i := 0; i < numParents; i++ {
parent, err := current.commit.ParentNode(i)
if err != nil {
break
}
parents = append(parents, parent)
}
// Examine the current commit and set of interesting paths
pathUnchanged := make([]bool, len(current.paths))
parentHashes := make([]map[string]plumbing.Hash, len(parents))
for j, parent := range parents {
parentHashes[j], err = getFileHashes(parent, treePath, current.paths)
if err != nil {
break
}
for i, path := range current.paths {
if parentHashes[j][path] == current.hashes[path] {
pathUnchanged[i] = true
}
}
}
var remainingPaths []string
for i, path := range current.paths {
// The results could already contain some newer change for the same path,
// so don't override that and bail out on the file early.
if resultNodes[path] == nil {
if pathUnchanged[i] {
// The path existed with the same hash in at least one parent so it could
// not have been changed in this commit directly.
remainingPaths = append(remainingPaths, path)
} else {
// There are few possible cases how can we get here:
// - The path didn't exist in any parent, so it must have been created by
// this commit.
// - The path did exist in the parent commit, but the hash of the file has
// changed.
// - We are looking at a merge commit and the hash of the file doesn't
// match any of the hashes being merged. This is more common for directories,
// but it can also happen if a file is changed through conflict resolution.
resultNodes[path] = current.commit
}
}
}
if len(remainingPaths) > 0 {
// Add the parent nodes along with remaining paths to the heap for further
// processing.
for j, parent := range parents {
// Combine remainingPath with paths available on the parent branch
// and make union of them
remainingPathsForParent := make([]string, 0, len(remainingPaths))
newRemainingPaths := make([]string, 0, len(remainingPaths))
for _, path := range remainingPaths {
if parentHashes[j][path] == current.hashes[path] {
remainingPathsForParent = append(remainingPathsForParent, path)
} else {
newRemainingPaths = append(newRemainingPaths, path)
}
}
if remainingPathsForParent != nil {
heap.Push(&commitAndPaths{parent, remainingPathsForParent, parentHashes[j]})
}
if len(newRemainingPaths) == 0 {
break
} else {
remainingPaths = newRemainingPaths
}
}
}
}
// Post-processing
result := make(map[string]*object.Commit)
for path, commitNode := range resultNodes {
var err error
result[path], err = commitNode.Commit()
if err != nil {
return nil, err
}
}
return result, nil
// CommitInfo describes the first commit with the provided entry
type CommitInfo struct {
Entry *TreeEntry
Commit *Commit
SubModuleFile *SubModuleFile
}

291
modules/git/commit_info_gogit.go Normal file
View File

@@ -0,0 +1,291 @@
// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"path"
"github.com/emirpasic/gods/trees/binaryheap"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph"
)
// GetCommitsInfo gets information of all commits that are corresponding to these entries
func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache *LastCommitCache) ([]CommitInfo, *Commit, error) {
entryPaths := make([]string, len(tes)+1)
// Get the commit for the treePath itself
entryPaths[0] = ""
for i, entry := range tes {
entryPaths[i+1] = entry.Name()
}
commitNodeIndex, commitGraphFile := commit.repo.CommitNodeIndex()
if commitGraphFile != nil {
defer commitGraphFile.Close()
}
c, err := commitNodeIndex.Get(commit.ID)
if err != nil {
return nil, nil, err
}
var revs map[string]*object.Commit
if cache != nil {
var unHitPaths []string
revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache)
if err != nil {
return nil, nil, err
}
if len(unHitPaths) > 0 {
revs2, err := GetLastCommitForPaths(c, treePath, unHitPaths)
if err != nil {
return nil, nil, err
}
for k, v := range revs2 {
if err := cache.Put(commit.ID.String(), path.Join(treePath, k), v.ID().String()); err != nil {
return nil, nil, err
}
revs[k] = v
}
}
} else {
revs, err = GetLastCommitForPaths(c, treePath, entryPaths)
}
if err != nil {
return nil, nil, err
}
commit.repo.gogitStorage.Close()
commitsInfo := make([]CommitInfo, len(tes))
for i, entry := range tes {
commitsInfo[i] = CommitInfo{
Entry: entry,
}
if rev, ok := revs[entry.Name()]; ok {
entryCommit := convertCommit(rev)
commitsInfo[i].Commit = entryCommit
if entry.IsSubModule() {
subModuleURL := ""
var fullPath string
if len(treePath) > 0 {
fullPath = treePath + "/" + entry.Name()
} else {
fullPath = entry.Name()
}
if subModule, err := commit.GetSubModule(fullPath); err != nil {
return nil, nil, err
} else if subModule != nil {
subModuleURL = subModule.URL
}
subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String())
commitsInfo[i].SubModuleFile = subModuleFile
}
}
}
// Retrieve the commit for the treePath itself (see above). We basically
// get it for free during the tree traversal and it's used for listing
// pages to display information about newest commit for a given path.
var treeCommit *Commit
if treePath == "" {
treeCommit = commit
} else if rev, ok := revs[""]; ok {
treeCommit = convertCommit(rev)
treeCommit.repo = commit.repo
}
return commitsInfo, treeCommit, nil
}
type commitAndPaths struct {
commit cgobject.CommitNode
// Paths that are still on the branch represented by commit
paths []string
// Set of hashes for the paths
hashes map[string]plumbing.Hash
}
func getCommitTree(c cgobject.CommitNode, treePath string) (*object.Tree, error) {
tree, err := c.Tree()
if err != nil {
return nil, err
}
// Optimize deep traversals by focusing only on the specific tree
if treePath != "" {
tree, err = tree.Tree(treePath)
if err != nil {
return nil, err
}
}
return tree, nil
}
func getFileHashes(c cgobject.CommitNode, treePath string, paths []string) (map[string]plumbing.Hash, error) {
tree, err := getCommitTree(c, treePath)
if err == object.ErrDirectoryNotFound {
// The whole tree didn't exist, so return empty map
return make(map[string]plumbing.Hash), nil
}
if err != nil {
return nil, err
}
hashes := make(map[string]plumbing.Hash)
for _, path := range paths {
if path != "" {
entry, err := tree.FindEntry(path)
if err == nil {
hashes[path] = entry.Hash
}
} else {
hashes[path] = tree.Hash
}
}
return hashes, nil
}
func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*object.Commit, []string, error) {
var unHitEntryPaths []string
var results = make(map[string]*object.Commit)
for _, p := range paths {
lastCommit, err := cache.Get(commitID, path.Join(treePath, p))
if err != nil {
return nil, nil, err
}
if lastCommit != nil {
results[p] = lastCommit.(*object.Commit)
continue
}
unHitEntryPaths = append(unHitEntryPaths, p)
}
return results, unHitEntryPaths, nil
}
// GetLastCommitForPaths returns last commit information
func GetLastCommitForPaths(c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) {
// We do a tree traversal with nodes sorted by commit time
heap := binaryheap.NewWith(func(a, b interface{}) int {
if a.(*commitAndPaths).commit.CommitTime().Before(b.(*commitAndPaths).commit.CommitTime()) {
return 1
}
return -1
})
resultNodes := make(map[string]cgobject.CommitNode)
initialHashes, err := getFileHashes(c, treePath, paths)
if err != nil {
return nil, err
}
// Start search from the root commit and with full set of paths
heap.Push(&commitAndPaths{c, paths, initialHashes})
for {
cIn, ok := heap.Pop()
if !ok {
break
}
current := cIn.(*commitAndPaths)
// Load the parent commits for the one we are currently examining
numParents := current.commit.NumParents()
var parents []cgobject.CommitNode
for i := 0; i < numParents; i++ {
parent, err := current.commit.ParentNode(i)
if err != nil {
break
}
parents = append(parents, parent)
}
// Examine the current commit and set of interesting paths
pathUnchanged := make([]bool, len(current.paths))
parentHashes := make([]map[string]plumbing.Hash, len(parents))
for j, parent := range parents {
parentHashes[j], err = getFileHashes(parent, treePath, current.paths)
if err != nil {
break
}
for i, path := range current.paths {
if parentHashes[j][path] == current.hashes[path] {
pathUnchanged[i] = true
}
}
}
var remainingPaths []string
for i, path := range current.paths {
// The results could already contain some newer change for the same path,
// so don't override that and bail out on the file early.
if resultNodes[path] == nil {
if pathUnchanged[i] {
// The path existed with the same hash in at least one parent so it could
// not have been changed in this commit directly.
remainingPaths = append(remainingPaths, path)
} else {
// There are few possible cases how can we get here:
// - The path didn't exist in any parent, so it must have been created by
// this commit.
// - The path did exist in the parent commit, but the hash of the file has
// changed.
// - We are looking at a merge commit and the hash of the file doesn't
// match any of the hashes being merged. This is more common for directories,
// but it can also happen if a file is changed through conflict resolution.
resultNodes[path] = current.commit
}
}
}
if len(remainingPaths) > 0 {
// Add the parent nodes along with remaining paths to the heap for further
// processing.
for j, parent := range parents {
// Combine remainingPath with paths available on the parent branch
// and make union of them
remainingPathsForParent := make([]string, 0, len(remainingPaths))
newRemainingPaths := make([]string, 0, len(remainingPaths))
for _, path := range remainingPaths {
if parentHashes[j][path] == current.hashes[path] {
remainingPathsForParent = append(remainingPathsForParent, path)
} else {
newRemainingPaths = append(newRemainingPaths, path)
}
}
if remainingPathsForParent != nil {
heap.Push(&commitAndPaths{parent, remainingPathsForParent, parentHashes[j]})
}
if len(newRemainingPaths) == 0 {
break
} else {
remainingPaths = newRemainingPaths
}
}
}
}
// Post-processing
result := make(map[string]*object.Commit)
for path, commitNode := range resultNodes {
var err error
result[path], err = commitNode.Commit()
if err != nil {
return nil, err
}
}
return result, nil
}

370
modules/git/commit_info_nogogit.go Normal file
View File

@@ -0,0 +1,370 @@
// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"bufio"
"bytes"
"fmt"
"io"
"math"
"path"
"sort"
"strings"
)
// GetCommitsInfo gets information of all commits that are corresponding to these entries
func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache *LastCommitCache) ([]CommitInfo, *Commit, error) {
entryPaths := make([]string, len(tes)+1)
// Get the commit for the treePath itself
entryPaths[0] = ""
for i, entry := range tes {
entryPaths[i+1] = entry.Name()
}
var err error
var revs map[string]*Commit
if cache != nil {
var unHitPaths []string
revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache)
if err != nil {
return nil, nil, err
}
if len(unHitPaths) > 0 {
sort.Strings(unHitPaths)
commits, err := GetLastCommitForPaths(commit, treePath, unHitPaths)
if err != nil {
return nil, nil, err
}
for i, found := range commits {
if err := cache.Put(commit.ID.String(), path.Join(treePath, unHitPaths[i]), found.ID.String()); err != nil {
return nil, nil, err
}
revs[unHitPaths[i]] = found
}
}
} else {
sort.Strings(entryPaths)
revs = map[string]*Commit{}
var foundCommits []*Commit
foundCommits, err = GetLastCommitForPaths(commit, treePath, entryPaths)
for i, found := range foundCommits {
revs[entryPaths[i]] = found
}
}
if err != nil {
return nil, nil, err
}
commitsInfo := make([]CommitInfo, len(tes))
for i, entry := range tes {
commitsInfo[i] = CommitInfo{
Entry: entry,
}
if entryCommit, ok := revs[entry.Name()]; ok {
commitsInfo[i].Commit = entryCommit
if entry.IsSubModule() {
subModuleURL := ""
var fullPath string
if len(treePath) > 0 {
fullPath = treePath + "/" + entry.Name()
} else {
fullPath = entry.Name()
}
if subModule, err := commit.GetSubModule(fullPath); err != nil {
return nil, nil, err
} else if subModule != nil {
subModuleURL = subModule.URL
}
subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String())
commitsInfo[i].SubModuleFile = subModuleFile
}
}
}
// Retrieve the commit for the treePath itself (see above). We basically
// get it for free during the tree traversal and it's used for listing
// pages to display information about newest commit for a given path.
var treeCommit *Commit
var ok bool
if treePath == "" {
treeCommit = commit
} else if treeCommit, ok = revs[""]; ok {
treeCommit.repo = commit.repo
}
return commitsInfo, treeCommit, nil
}
func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*Commit, []string, error) {
var unHitEntryPaths []string
var results = make(map[string]*Commit)
for _, p := range paths {
lastCommit, err := cache.Get(commitID, path.Join(treePath, p))
if err != nil {
return nil, nil, err
}
if lastCommit != nil {
results[p] = lastCommit.(*Commit)
continue
}
unHitEntryPaths = append(unHitEntryPaths, p)
}
return results, unHitEntryPaths, nil
}
// GetLastCommitForPaths returns last commit information
func GetLastCommitForPaths(commit *Commit, treePath string, paths []string) ([]*Commit, error) {
// We read backwards from the commit to obtain all of the commits
// We'll do this by using rev-list to provide us with parent commits in order
revListReader, revListWriter := io.Pipe()
defer func() {
_ = revListWriter.Close()
_ = revListReader.Close()
}()
go func() {
stderr := strings.Builder{}
err := NewCommand("rev-list", "--format=%T", commit.ID.String()).RunInDirPipeline(commit.repo.Path, revListWriter, &stderr)
if err != nil {
_ = revListWriter.CloseWithError(ConcatenateError(err, (&stderr).String()))
} else {
_ = revListWriter.Close()
}
}()
// We feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary.
// so let's create a batch stdin and stdout
batchStdinReader, batchStdinWriter := io.Pipe()
batchStdoutReader, batchStdoutWriter := io.Pipe()
defer func() {
_ = batchStdinReader.Close()
_ = batchStdinWriter.Close()
_ = batchStdoutReader.Close()
_ = batchStdoutWriter.Close()
}()
go func() {
stderr := strings.Builder{}
err := NewCommand("cat-file", "--batch").RunInDirFullPipeline(commit.repo.Path, batchStdoutWriter, &stderr, batchStdinReader)
if err != nil {
_ = revListWriter.CloseWithError(ConcatenateError(err, (&stderr).String()))
} else {
_ = revListWriter.Close()
}
}()
// For simplicities sake we'll us a buffered reader
batchReader := bufio.NewReader(batchStdoutReader)
mapsize := 4096
if len(paths) > mapsize {
mapsize = len(paths)
}
path2idx := make(map[string]int, mapsize)
for i, path := range paths {
path2idx[path] = i
}
fnameBuf := make([]byte, 4096)
modeBuf := make([]byte, 40)
allShaBuf := make([]byte, (len(paths)+1)*20)
shaBuf := make([]byte, 20)
tmpTreeID := make([]byte, 40)
// commits is the returnable commits matching the paths provided
commits := make([]string, len(paths))
// ids are the blob/tree ids for the paths
ids := make([][]byte, len(paths))
// We'll use a scanner for the revList because it's simpler than a bufio.Reader
scan := bufio.NewScanner(revListReader)
revListLoop:
for scan.Scan() {
// Get the next parent commit ID
commitID := scan.Text()
if !scan.Scan() {
break revListLoop
}
commitID = commitID[7:]
rootTreeID := scan.Text()
// push the tree to the cat-file --batch process
_, err := batchStdinWriter.Write([]byte(rootTreeID + "\n"))
if err != nil {
return nil, err
}
currentPath := ""
// OK if the target tree path is "" and the "" is in the paths just set this now
if treePath == "" && paths[0] == "" {
// If this is the first time we see this set the id appropriate for this paths to this tree and set the last commit to curCommit
if len(ids[0]) == 0 {
ids[0] = []byte(rootTreeID)
commits[0] = string(commitID)
} else if bytes.Equal(ids[0], []byte(rootTreeID)) {
commits[0] = string(commitID)
}
}
treeReadingLoop:
for {
_, _, size, err := ReadBatchLine(batchReader)
if err != nil {
return nil, err
}
// Handle trees
// n is counter for file position in the tree file
var n int64
// Two options: currentPath is the targetTreepath
if treePath == currentPath {
// We are in the right directory
// Parse each tree line in turn. (don't care about mode here.)
for n < size {
fname, sha, count, err := ParseTreeLineSkipMode(batchReader, fnameBuf, shaBuf)
shaBuf = sha
if err != nil {
return nil, err
}
n += int64(count)
idx, ok := path2idx[string(fname)]
if ok {
// Now if this is the first time round set the initial Blob(ish) SHA ID and the commit
if len(ids[idx]) == 0 {
copy(allShaBuf[20*(idx+1):20*(idx+2)], shaBuf)
ids[idx] = allShaBuf[20*(idx+1) : 20*(idx+2)]
commits[idx] = string(commitID)
} else if bytes.Equal(ids[idx], shaBuf) {
commits[idx] = string(commitID)
}
}
// FIXME: is there any order to the way strings are emitted from cat-file?
// if there is - then we could skip once we've passed all of our data
}
break treeReadingLoop
}
var treeID []byte
// We're in the wrong directory
// Find target directory in this directory
idx := len(currentPath)
if idx > 0 {
idx++
}
target := strings.SplitN(treePath[idx:], "/", 2)[0]
for n < size {
// Read each tree entry in turn
mode, fname, sha, count, err := ParseTreeLine(batchReader, modeBuf, fnameBuf, shaBuf)
if err != nil {
return nil, err
}
n += int64(count)
// if we have found the target directory
if bytes.Equal(fname, []byte(target)) && bytes.Equal(mode, []byte("40000")) {
copy(tmpTreeID, sha)
treeID = tmpTreeID
break
}
}
if n < size {
// Discard any remaining entries in the current tree
discard := size - n
for discard > math.MaxInt32 {
_, err := batchReader.Discard(math.MaxInt32)
if err != nil {
return nil, err
}
discard -= math.MaxInt32
}
_, err := batchReader.Discard(int(discard))
if err != nil {
return nil, err
}
}
// if we haven't found a treeID for the target directory our search is over
if len(treeID) == 0 {
break treeReadingLoop
}
// add the target to the current path
if idx > 0 {
currentPath += "/"
}
currentPath += target
// if we've now found the current path check its sha id and commit status
if treePath == currentPath && paths[0] == "" {
if len(ids[0]) == 0 {
copy(allShaBuf[0:20], treeID)
ids[0] = allShaBuf[0:20]
commits[0] = string(commitID)
} else if bytes.Equal(ids[0], treeID) {
commits[0] = string(commitID)
}
}
treeID = to40ByteSHA(treeID)
_, err = batchStdinWriter.Write(treeID)
if err != nil {
return nil, err
}
_, err = batchStdinWriter.Write([]byte("\n"))
if err != nil {
return nil, err
}
}
}
commitsMap := make(map[string]*Commit, len(commits))
commitsMap[commit.ID.String()] = commit
commitCommits := make([]*Commit, len(commits))
for i, commitID := range commits {
c, ok := commitsMap[commitID]
if ok {
commitCommits[i] = c
continue
}
if len(commitID) == 0 {
continue
}
_, err := batchStdinWriter.Write([]byte(commitID + "\n"))
if err != nil {
return nil, err
}
_, typ, size, err := ReadBatchLine(batchReader)
if err != nil {
return nil, err
}
if typ != "commit" {
return nil, fmt.Errorf("unexpected type: %s for commit id: %s", typ, commitID)
}
c, err = CommitFromReader(commit.repo, MustIDFromString(string(commitID)), io.LimitReader(batchReader, int64(size)))
if err != nil {
return nil, err
}
commitCommits[i] = c
}
return commitCommits, scan.Err()
}

16
modules/git/commit_info_test.go
View File

@@ -58,17 +58,27 @@ func testGetCommitsInfo(t *testing.T, repo1 *Repository) {
for _, testCase := range testCases {
commit, err := repo1.GetCommit(testCase.CommitID)
assert.NoError(t, err)
assert.NotNil(t, commit)
assert.NotNil(t, commit.Tree)
assert.NotNil(t, commit.Tree.repo)
tree, err := commit.Tree.SubTree(testCase.Path)
assert.NotNil(t, tree, "tree is nil for testCase CommitID %s in Path %s", testCase.CommitID, testCase.Path)
assert.NotNil(t, tree.repo, "repo is nil for testCase CommitID %s in Path %s", testCase.CommitID, testCase.Path)
assert.NoError(t, err)
entries, err := tree.ListEntries()
assert.NoError(t, err)
commitsInfo, treeCommit, err := entries.GetCommitsInfo(commit, testCase.Path, nil)
assert.Equal(t, testCase.ExpectedTreeCommit, treeCommit.ID.String())
assert.NoError(t, err)
if err != nil {
t.FailNow()
}
assert.Equal(t, testCase.ExpectedTreeCommit, treeCommit.ID.String())
assert.Len(t, commitsInfo, len(testCase.ExpectedIDs))
for _, commitInfo := range commitsInfo {
entry := commitInfo[0].(*TreeEntry)
commit := commitInfo[1].(*Commit)
entry := commitInfo.Entry
commit := commitInfo.Commit
expectedID, ok := testCase.ExpectedIDs[entry.Name()]
if !assert.True(t, ok) {
continue

44
modules/git/commit_reader.go
View File

@@ -9,13 +9,13 @@ import (
"bytes"
"io"
"strings"
"github.com/go-git/go-git/v5/plumbing"
)
// CommitFromReader will generate a Commit from a provided reader
// We will need this to interpret commits from cat-file
func CommitFromReader(gitRepo *Repository, sha plumbing.Hash, reader io.Reader) (*Commit, error) {
// We need this to interpret commits from cat-file or cat-file --batch
//
// If used as part of a cat-file --batch stream you need to limit the reader to the correct size
func CommitFromReader(gitRepo *Repository, sha SHA1, reader io.Reader) (*Commit, error) {
commit := &Commit{
ID: sha,
}
@@ -26,26 +26,20 @@ func CommitFromReader(gitRepo *Repository, sha plumbing.Hash, reader io.Reader)
message := false
pgpsig := false
scanner := bufio.NewScanner(reader)
// Split by '\n' but include the '\n'
scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
if atEOF && len(data) == 0 {
return 0, nil, nil
bufReader, ok := reader.(*bufio.Reader)
if !ok {
bufReader = bufio.NewReader(reader)
}
if i := bytes.IndexByte(data, '\n'); i >= 0 {
// We have a full newline-terminated line.
return i + 1, data[0 : i+1], nil
}
// If we're at EOF, we have a final, non-terminated line. Return it.
if atEOF {
return len(data), data, nil
}
// Request more data.
return 0, nil, nil
})
for scanner.Scan() {
line := scanner.Bytes()
readLoop:
for {
line, err := bufReader.ReadBytes('\n')
if err != nil {
if err == io.EOF {
break readLoop
}
return nil, err
}
if pgpsig {
if len(line) > 0 && line[0] == ' ' {
_, _ = signatureSB.Write(line[1:])
@@ -72,10 +66,10 @@ func CommitFromReader(gitRepo *Repository, sha plumbing.Hash, reader io.Reader)
switch string(split[0]) {
case "tree":
commit.Tree = *NewTree(gitRepo, plumbing.NewHash(string(data)))
commit.Tree = *NewTree(gitRepo, MustIDFromString(string(data)))
_, _ = payloadSB.Write(line)
case "parent":
commit.Parents = append(commit.Parents, plumbing.NewHash(string(data)))
commit.Parents = append(commit.Parents, MustIDFromString(string(data)))
_, _ = payloadSB.Write(line)
case "author":
commit.Author = &Signature{}
@@ -104,5 +98,5 @@ func CommitFromReader(gitRepo *Repository, sha plumbing.Hash, reader io.Reader)
commit.Signature = nil
}
return commit, scanner.Err()
return commit, nil
}

29
modules/git/last_commit_cache.go Normal file
View File

@@ -0,0 +1,29 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package git
import (
"crypto/sha256"
"fmt"
)
// Cache represents a caching interface
type Cache interface {
// Put puts value into cache with key and expire time.
Put(key string, val interface{}, timeout int64) error
// Get gets cached value by given key.
Get(key string) interface{}
}
func (c *LastCommitCache) getCacheKey(repoPath, ref, entryPath string) string {
hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, ref, entryPath)))
return fmt.Sprintf("last_commit:%x", hashBytes)
}
// Put put the last commit id with commit and entry path
func (c *LastCommitCache) Put(ref, entryPath, commitID string) error {
log("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID)
return c.cache.Put(c.getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl)
}

113
modules/git/last_commit_cache_gogit.go Normal file
View File

@@ -0,0 +1,113 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"path"
"github.com/go-git/go-git/v5/plumbing/object"
cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph"
)
// LastCommitCache represents a cache to store last commit
type LastCommitCache struct {
repoPath string
ttl int64
repo *Repository
commitCache map[string]*object.Commit
cache Cache
}
// NewLastCommitCache creates a new last commit cache for repo
func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl int64, cache Cache) *LastCommitCache {
if cache == nil {
return nil
}
return &LastCommitCache{
repoPath: repoPath,
repo: gitRepo,
commitCache: make(map[string]*object.Commit),
ttl: ttl,
cache: cache,
}
}
// Get get the last commit information by commit id and entry path
func (c *LastCommitCache) Get(ref, entryPath string) (interface{}, error) {
v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath))
if vs, ok := v.(string); ok {
log("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs)
if commit, ok := c.commitCache[vs]; ok {
log("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs)
return commit, nil
}
id, err := c.repo.ConvertToSHA1(vs)
if err != nil {
return nil, err
}
commit, err := c.repo.GoGitRepo().CommitObject(id)
if err != nil {
return nil, err
}
c.commitCache[vs] = commit
return commit, nil
}
return nil, nil
}
// CacheCommit will cache the commit from the gitRepository
func (c *LastCommitCache) CacheCommit(commit *Commit) error {
commitNodeIndex, _ := commit.repo.CommitNodeIndex()
index, err := commitNodeIndex.Get(commit.ID)
if err != nil {
return err
}
return c.recursiveCache(index, &commit.Tree, "", 1)
}
func (c *LastCommitCache) recursiveCache(index cgobject.CommitNode, tree *Tree, treePath string, level int) error {
if level == 0 {
return nil
}
entries, err := tree.ListEntries()
if err != nil {
return err
}
entryPaths := make([]string, len(entries))
entryMap := make(map[string]*TreeEntry)
for i, entry := range entries {
entryPaths[i] = entry.Name()
entryMap[entry.Name()] = entry
}
commits, err := GetLastCommitForPaths(index, treePath, entryPaths)
if err != nil {
return err
}
for entry, cm := range commits {
if err := c.Put(index.ID().String(), path.Join(treePath, entry), cm.ID().String()); err != nil {
return err
}
if entryMap[entry].IsDir() {
subTree, err := tree.SubTree(entry)
if err != nil {
return err
}
if err := c.recursiveCache(index, subTree, entry, level-1); err != nil {
return err
}
}
}
return nil
}

103
modules/git/last_commit_cache_nogogit.go Normal file
View File

@@ -0,0 +1,103 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"path"
)
// LastCommitCache represents a cache to store last commit
type LastCommitCache struct {
repoPath string
ttl int64
repo *Repository
commitCache map[string]*Commit
cache Cache
}
// NewLastCommitCache creates a new last commit cache for repo
func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl int64, cache Cache) *LastCommitCache {
if cache == nil {
return nil
}
return &LastCommitCache{
repoPath: repoPath,
repo: gitRepo,
commitCache: make(map[string]*Commit),
ttl: ttl,
cache: cache,
}
}
// Get get the last commit information by commit id and entry path
func (c *LastCommitCache) Get(ref, entryPath string) (interface{}, error) {
v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath))
if vs, ok := v.(string); ok {
log("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs)
if commit, ok := c.commitCache[vs]; ok {
log("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs)
return commit, nil
}
id, err := c.repo.ConvertToSHA1(vs)
if err != nil {
return nil, err
}
commit, err := c.repo.getCommit(id)
if err != nil {
return nil, err
}
c.commitCache[vs] = commit
return commit, nil
}
return nil, nil
}
// CacheCommit will cache the commit from the gitRepository
func (c *LastCommitCache) CacheCommit(commit *Commit) error {
return c.recursiveCache(commit, &commit.Tree, "", 1)
}
func (c *LastCommitCache) recursiveCache(commit *Commit, tree *Tree, treePath string, level int) error {
if level == 0 {
return nil
}
entries, err := tree.ListEntries()
if err != nil {
return err
}
entryPaths := make([]string, len(entries))
entryMap := make(map[string]*TreeEntry)
for i, entry := range entries {
entryPaths[i] = entry.Name()
entryMap[entry.Name()] = entry
}
commits, err := GetLastCommitForPaths(commit, treePath, entryPaths)
if err != nil {
return err
}
for i, entryCommit := range commits {
entry := entryPaths[i]
if err := c.Put(commit.ID.String(), path.Join(treePath, entryPaths[i]), entryCommit.ID.String()); err != nil {
return err
}
if entryMap[entry].IsDir() {
subTree, err := tree.SubTree(entry)
if err != nil {
return err
}
if err := c.recursiveCache(commit, subTree, entry, level-1); err != nil {
return err
}
}
}
return nil
}

65
modules/git/notes.go
View File

@@ -4,12 +4,6 @@
package git
import (
"io/ioutil"
"github.com/go-git/go-git/v5/plumbing/object"
)
// NotesRef is the git ref where Gitea will look for git-notes data.
// The value ("refs/notes/commits") is the default ref used by git-notes.
const NotesRef = "refs/notes/commits"
@@ -19,62 +13,3 @@ type Note struct {
Message []byte
Commit *Commit
}
// GetNote retrieves the git-notes data for a given commit.
func GetNote(repo *Repository, commitID string, note *Note) error {
notes, err := repo.GetCommit(NotesRef)
if err != nil {
return err
}
remainingCommitID := commitID
path := ""
currentTree := notes.Tree.gogitTree
var file *object.File
for len(remainingCommitID) > 2 {
file, err = currentTree.File(remainingCommitID)
if err == nil {
path += remainingCommitID
break
}
if err == object.ErrFileNotFound {
currentTree, err = currentTree.Tree(remainingCommitID[0:2])
path += remainingCommitID[0:2] + "/"
remainingCommitID = remainingCommitID[2:]
}
if err != nil {
return err
}
}
blob := file.Blob
dataRc, err := blob.Reader()
if err != nil {
return err
}
defer dataRc.Close()
d, err := ioutil.ReadAll(dataRc)
if err != nil {
return err
}
note.Message = d
commitNodeIndex, commitGraphFile := repo.CommitNodeIndex()
if commitGraphFile != nil {
defer commitGraphFile.Close()
}
commitNode, err := commitNodeIndex.Get(notes.ID)
if err != nil {
return err
}
lastCommits, err := GetLastCommitForPaths(commitNode, "", []string{path})
if err != nil {
return err
}
note.Commit = convertCommit(lastCommits[path])
return nil
}

72
modules/git/notes_gogit.go Normal file
View File

@@ -0,0 +1,72 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"io/ioutil"
"github.com/go-git/go-git/v5/plumbing/object"
)
// GetNote retrieves the git-notes data for a given commit.
func GetNote(repo *Repository, commitID string, note *Note) error {
notes, err := repo.GetCommit(NotesRef)
if err != nil {
return err
}
remainingCommitID := commitID
path := ""
currentTree := notes.Tree.gogitTree
var file *object.File
for len(remainingCommitID) > 2 {
file, err = currentTree.File(remainingCommitID)
if err == nil {
path += remainingCommitID
break
}
if err == object.ErrFileNotFound {
currentTree, err = currentTree.Tree(remainingCommitID[0:2])
path += remainingCommitID[0:2] + "/"
remainingCommitID = remainingCommitID[2:]
}
if err != nil {
return err
}
}
blob := file.Blob
dataRc, err := blob.Reader()
if err != nil {
return err
}
defer dataRc.Close()
d, err := ioutil.ReadAll(dataRc)
if err != nil {
return err
}
note.Message = d
commitNodeIndex, commitGraphFile := repo.CommitNodeIndex()
if commitGraphFile != nil {
defer commitGraphFile.Close()
}
commitNode, err := commitNodeIndex.Get(notes.ID)
if err != nil {
return err
}
lastCommits, err := GetLastCommitForPaths(commitNode, "", []string{path})
if err != nil {
return err
}
note.Commit = convertCommit(lastCommits[path])
return nil
}

59
modules/git/notes_nogogit.go Normal file
View File

@@ -0,0 +1,59 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"io/ioutil"
)
// GetNote retrieves the git-notes data for a given commit.
func GetNote(repo *Repository, commitID string, note *Note) error {
notes, err := repo.GetCommit(NotesRef)
if err != nil {
return err
}
path := ""
tree := &notes.Tree
var entry *TreeEntry
for len(commitID) > 2 {
entry, err = tree.GetTreeEntryByPath(commitID)
if err == nil {
path += commitID
break
}
if IsErrNotExist(err) {
tree, err = tree.SubTree(commitID[0:2])
path += commitID[0:2] + "/"
commitID = commitID[2:]
}
if err != nil {
return err
}
}
dataRc, err := entry.Blob().DataAsync()
if err != nil {
return err
}
defer dataRc.Close()
d, err := ioutil.ReadAll(dataRc)
if err != nil {
return err
}
note.Message = d
lastCommits, err := GetLastCommitForPaths(notes, "", []string{path})
if err != nil {
return err
}
note.Commit = lastCommits[0]
return nil
}

2
modules/git/parse.go → modules/git/parse_gogit.go
View File

@@ -2,6 +2,8 @@
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (

2
modules/git/parse_test.go → modules/git/parse_gogit_test.go
View File

@@ -2,6 +2,8 @@
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (

78
modules/git/parse_nogogit.go Normal file
View File

@@ -0,0 +1,78 @@
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"bytes"
"fmt"
"strconv"
)
// ParseTreeEntries parses the output of a `git ls-tree` command.
func ParseTreeEntries(data []byte) ([]*TreeEntry, error) {
return parseTreeEntries(data, nil)
}
func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) {
entries := make([]*TreeEntry, 0, 10)
for pos := 0; pos < len(data); {
// expect line to be of the form "<mode> <type> <sha>\t<filename>"
entry := new(TreeEntry)
entry.ptree = ptree
if pos+6 > len(data) {
return nil, fmt.Errorf("Invalid ls-tree output: %s", string(data))
}
switch string(data[pos : pos+6]) {
case "100644":
entry.entryMode = EntryModeBlob
pos += 12 // skip over "100644 blob "
case "100755":
entry.entryMode = EntryModeExec
pos += 12 // skip over "100755 blob "
case "120000":
entry.entryMode = EntryModeSymlink
pos += 12 // skip over "120000 blob "
case "160000":
entry.entryMode = EntryModeCommit
pos += 14 // skip over "160000 object "
case "040000":
entry.entryMode = EntryModeTree
pos += 12 // skip over "040000 tree "
default:
return nil, fmt.Errorf("unknown type: %v", string(data[pos:pos+6]))
}
if pos+40 > len(data) {
return nil, fmt.Errorf("Invalid ls-tree output: %s", string(data))
}
id, err := NewIDFromString(string(data[pos : pos+40]))
if err != nil {
return nil, fmt.Errorf("Invalid ls-tree output: %v", err)
}
entry.ID = id
pos += 41 // skip over sha and trailing space
end := pos + bytes.IndexByte(data[pos:], '\n')
if end < pos {
return nil, fmt.Errorf("Invalid ls-tree output: %s", string(data))
}
// In case entry name is surrounded by double quotes(it happens only in git-shell).
if data[pos] == '"' {
entry.name, err = strconv.Unquote(string(data[pos:end]))
if err != nil {
return nil, fmt.Errorf("Invalid ls-tree output: %v", err)
}
} else {
entry.name = string(data[pos:end])
}
pos = end + 1
entries = append(entries, entry)
}
return entries, nil
}

159
modules/git/pipeline/lfs.go Normal file
View File

@@ -0,0 +1,159 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package pipeline
import (
"bufio"
"fmt"
"io"
"sort"
"strings"
"sync"
"time"
"code.gitea.io/gitea/modules/git"
gogit "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing/object"
)
// LFSResult represents commits found using a provided pointer file hash
type LFSResult struct {
Name string
SHA string
Summary string
When time.Time
ParentHashes []git.SHA1
BranchName string
FullCommitName string
}
type lfsResultSlice []*LFSResult
func (a lfsResultSlice) Len() int { return len(a) }
func (a lfsResultSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) }
// FindLFSFile finds commits that contain a provided pointer file hash
func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) {
resultsMap := map[string]*LFSResult{}
results := make([]*LFSResult, 0)
basePath := repo.Path
gogitRepo := repo.GoGitRepo()
commitsIter, err := gogitRepo.Log(&gogit.LogOptions{
Order: gogit.LogOrderCommitterTime,
All: true,
})
if err != nil {
return nil, fmt.Errorf("Failed to get GoGit CommitsIter. Error: %w", err)
}
err = commitsIter.ForEach(func(gitCommit *object.Commit) error {
tree, err := gitCommit.Tree()
if err != nil {
return err
}
treeWalker := object.NewTreeWalker(tree, true, nil)
defer treeWalker.Close()
for {
name, entry, err := treeWalker.Next()
if err == io.EOF {
break
}
if entry.Hash == hash {
result := LFSResult{
Name: name,
SHA: gitCommit.Hash.String(),
Summary: strings.Split(strings.TrimSpace(gitCommit.Message), "\n")[0],
When: gitCommit.Author.When,
ParentHashes: gitCommit.ParentHashes,
}
resultsMap[gitCommit.Hash.String()+":"+name] = &result
}
}
return nil
})
if err != nil && err != io.EOF {
return nil, fmt.Errorf("Failure in CommitIter.ForEach: %w", err)
}
for _, result := range resultsMap {
hasParent := false
for _, parentHash := range result.ParentHashes {
if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent {
break
}
}
if !hasParent {
results = append(results, result)
}
}
sort.Sort(lfsResultSlice(results))
// Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple
shasToNameReader, shasToNameWriter := io.Pipe()
nameRevStdinReader, nameRevStdinWriter := io.Pipe()
errChan := make(chan error, 1)
wg := sync.WaitGroup{}
wg.Add(3)
go func() {
defer wg.Done()
scanner := bufio.NewScanner(nameRevStdinReader)
i := 0
for scanner.Scan() {
line := scanner.Text()
if len(line) == 0 {
continue
}
result := results[i]
result.FullCommitName = line
result.BranchName = strings.Split(line, "~")[0]
i++
}
}()
go NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath)
go func() {
defer wg.Done()
defer shasToNameWriter.Close()
for _, result := range results {
i := 0
if i < len(result.SHA) {
n, err := shasToNameWriter.Write([]byte(result.SHA)[i:])
if err != nil {
errChan <- err
break
}
i += n
}
n := 0
for n < 1 {
n, err = shasToNameWriter.Write([]byte{'\n'})
if err != nil {
errChan <- err
break
}
}
}
}()
wg.Wait()
select {
case err, has := <-errChan:
if has {
return nil, fmt.Errorf("Unable to obtain name for LFS files. Error: %w", err)
}
default:
}
return results, nil
}

266
modules/git/pipeline/lfs_nogogit.go Normal file
View File

@@ -0,0 +1,266 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package pipeline
import (
"bufio"
"bytes"
"fmt"
"io"
"sort"
"strings"
"sync"
"time"
"code.gitea.io/gitea/modules/git"
)
// LFSResult represents commits found using a provided pointer file hash
type LFSResult struct {
Name string
SHA string
Summary string
When time.Time
ParentHashes []git.SHA1
BranchName string
FullCommitName string
}
type lfsResultSlice []*LFSResult
func (a lfsResultSlice) Len() int { return len(a) }
func (a lfsResultSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) }
// FindLFSFile finds commits that contain a provided pointer file hash
func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) {
resultsMap := map[string]*LFSResult{}
results := make([]*LFSResult, 0)
basePath := repo.Path
hashStr := hash.String()
// Use rev-list to provide us with all commits in order
revListReader, revListWriter := io.Pipe()
defer func() {
_ = revListWriter.Close()
_ = revListReader.Close()
}()
go func() {
stderr := strings.Builder{}
err := git.NewCommand("rev-list", "--all").RunInDirPipeline(repo.Path, revListWriter, &stderr)
if err != nil {
_ = revListWriter.CloseWithError(git.ConcatenateError(err, (&stderr).String()))
} else {
_ = revListWriter.Close()
}
}()
// Next feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary.
// so let's create a batch stdin and stdout
batchStdinReader, batchStdinWriter := io.Pipe()
batchStdoutReader, batchStdoutWriter := io.Pipe()
defer func() {
_ = batchStdinReader.Close()
_ = batchStdinWriter.Close()
_ = batchStdoutReader.Close()
_ = batchStdoutWriter.Close()
}()
go func() {
stderr := strings.Builder{}
err := git.NewCommand("cat-file", "--batch").RunInDirFullPipeline(repo.Path, batchStdoutWriter, &stderr, batchStdinReader)
if err != nil {
_ = revListWriter.CloseWithError(git.ConcatenateError(err, (&stderr).String()))
} else {
_ = revListWriter.Close()
}
}()
// For simplicities sake we'll us a buffered reader to read from the cat-file --batch
batchReader := bufio.NewReader(batchStdoutReader)
// We'll use a scanner for the revList because it's simpler than a bufio.Reader
scan := bufio.NewScanner(revListReader)
trees := [][]byte{}
paths := []string{}
fnameBuf := make([]byte, 4096)
modeBuf := make([]byte, 40)
workingShaBuf := make([]byte, 40)
for scan.Scan() {
// Get the next commit ID
commitID := scan.Bytes()
// push the commit to the cat-file --batch process
_, err := batchStdinWriter.Write(commitID)
if err != nil {
return nil, err
}
_, err = batchStdinWriter.Write([]byte{'\n'})
if err != nil {
return nil, err
}
var curCommit *git.Commit
curPath := ""
commitReadingLoop:
for {
_, typ, size, err := git.ReadBatchLine(batchReader)
if err != nil {
return nil, err
}
switch typ {
case "tag":
// This shouldn't happen but if it does well just get the commit and try again
id, err := git.ReadTagObjectID(batchReader, size)
if err != nil {
return nil, err
}
_, err = batchStdinWriter.Write([]byte(id + "\n"))
if err != nil {
return nil, err
}
continue
case "commit":
// Read in the commit to get its tree and in case this is one of the last used commits
curCommit, err = git.CommitFromReader(repo, git.MustIDFromString(string(commitID)), io.LimitReader(batchReader, int64(size)))
if err != nil {
return nil, err
}
_, err := batchStdinWriter.Write([]byte(curCommit.Tree.ID.String() + "\n"))
if err != nil {
return nil, err
}
curPath = ""
case "tree":
var n int64
for n < size {
mode, fname, sha, count, err := git.ParseTreeLine(batchReader, modeBuf, fnameBuf, workingShaBuf)
if err != nil {
return nil, err
}
n += int64(count)
if bytes.Equal(sha, []byte(hashStr)) {
result := LFSResult{
Name: curPath + string(fname),
SHA: curCommit.ID.String(),
Summary: strings.Split(strings.TrimSpace(curCommit.CommitMessage), "\n")[0],
When: curCommit.Author.When,
ParentHashes: curCommit.Parents,
}
resultsMap[curCommit.ID.String()+":"+curPath+string(fname)] = &result
} else if string(mode) == git.EntryModeTree.String() {
trees = append(trees, sha)
paths = append(paths, curPath+string(fname)+"/")
}
}
if len(trees) > 0 {
_, err := batchStdinWriter.Write(trees[len(trees)-1])
if err != nil {
return nil, err
}
_, err = batchStdinWriter.Write([]byte("\n"))
if err != nil {
return nil, err
}
curPath = paths[len(paths)-1]
trees = trees[:len(trees)-1]
paths = paths[:len(paths)-1]
} else {
break commitReadingLoop
}
}
}
}
if err := scan.Err(); err != nil {
return nil, err
}
for _, result := range resultsMap {
hasParent := false
for _, parentHash := range result.ParentHashes {
if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent {
break
}
}
if !hasParent {
results = append(results, result)
}
}
sort.Sort(lfsResultSlice(results))
// Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple
shasToNameReader, shasToNameWriter := io.Pipe()
nameRevStdinReader, nameRevStdinWriter := io.Pipe()
errChan := make(chan error, 1)
wg := sync.WaitGroup{}
wg.Add(3)
go func() {
defer wg.Done()
scanner := bufio.NewScanner(nameRevStdinReader)
i := 0
for scanner.Scan() {
line := scanner.Text()
if len(line) == 0 {
continue
}
result := results[i]
result.FullCommitName = line
result.BranchName = strings.Split(line, "~")[0]
i++
}
}()
go NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath)
go func() {
defer wg.Done()
defer shasToNameWriter.Close()
for _, result := range results {
i := 0
if i < len(result.SHA) {
n, err := shasToNameWriter.Write([]byte(result.SHA)[i:])
if err != nil {
errChan <- err
break
}
i += n
}
var err error
n := 0
for n < 1 {
n, err = shasToNameWriter.Write([]byte{'\n'})
if err != nil {
errChan <- err
break
}
}
}
}()
wg.Wait()
select {
case err, has := <-errChan:
if has {
return nil, fmt.Errorf("Unable to obtain name for LFS files. Error: %w", err)
}
default:
}
return results, nil
}

64
modules/git/repo.go
View File

@@ -9,34 +9,16 @@ import (
"bytes"
"container/list"
"context"
"errors"
"fmt"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
gitealog "code.gitea.io/gitea/modules/log"
"github.com/go-git/go-billy/v5/osfs"
gogit "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing/cache"
"github.com/go-git/go-git/v5/storage/filesystem"
"github.com/unknwon/com"
)
// Repository represents a Git repository.
type Repository struct {
Path string
tagCache *ObjectCache
gogitRepo *gogit.Repository
gogitStorage *filesystem.Storage
gpgSettings *GPGSettings
}
// GPGSettings represents the default GPG settings for this repository
type GPGSettings struct {
Sign bool
@@ -93,52 +75,6 @@ func InitRepository(repoPath string, bare bool) error {
return err
}
// OpenRepository opens the repository at the given path.
func OpenRepository(repoPath string) (*Repository, error) {
repoPath, err := filepath.Abs(repoPath)
if err != nil {
return nil, err
} else if !isDir(repoPath) {
return nil, errors.New("no such file or directory")
}
fs := osfs.New(repoPath)
_, err = fs.Stat(".git")
if err == nil {
fs, err = fs.Chroot(".git")
if err != nil {
return nil, err
}
}
storage := filesystem.NewStorageWithOptions(fs, cache.NewObjectLRUDefault(), filesystem.Options{KeepDescriptors: true})
gogitRepo, err := gogit.Open(storage, fs)
if err != nil {
return nil, err
}
return &Repository{
Path: repoPath,
gogitRepo: gogitRepo,
gogitStorage: storage,
tagCache: newObjectCache(),
}, nil
}
// Close this repository, in particular close the underlying gogitStorage if this is not nil
func (repo *Repository) Close() {
if repo == nil || repo.gogitStorage == nil {
return
}
if err := repo.gogitStorage.Close(); err != nil {
gitealog.Error("Error closing storage: %v", err)
}
}
// GoGitRepo gets the go-git repo representation
func (repo *Repository) GoGitRepo() *gogit.Repository {
return repo.gogitRepo
}
// IsEmpty Check if repository is empty.
func (repo *Repository) IsEmpty() (bool, error) {
var errbuf strings.Builder

76
modules/git/repo_base_gogit.go Normal file
View File

@@ -0,0 +1,76 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"errors"
"path/filepath"
gitealog "code.gitea.io/gitea/modules/log"
"github.com/go-git/go-billy/v5/osfs"
gogit "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing/cache"
"github.com/go-git/go-git/v5/storage/filesystem"
)
// Repository represents a Git repository.
type Repository struct {
Path string
tagCache *ObjectCache
gogitRepo *gogit.Repository
gogitStorage *filesystem.Storage
gpgSettings *GPGSettings
}
// OpenRepository opens the repository at the given path.
func OpenRepository(repoPath string) (*Repository, error) {
repoPath, err := filepath.Abs(repoPath)
if err != nil {
return nil, err
} else if !isDir(repoPath) {
return nil, errors.New("no such file or directory")
}
fs := osfs.New(repoPath)
_, err = fs.Stat(".git")
if err == nil {
fs, err = fs.Chroot(".git")
if err != nil {
return nil, err
}
}
storage := filesystem.NewStorageWithOptions(fs, cache.NewObjectLRUDefault(), filesystem.Options{KeepDescriptors: true})
gogitRepo, err := gogit.Open(storage, fs)
if err != nil {
return nil, err
}
return &Repository{
Path: repoPath,
gogitRepo: gogitRepo,
gogitStorage: storage,
tagCache: newObjectCache(),
}, nil
}
// Close this repository, in particular close the underlying gogitStorage if this is not nil
func (repo *Repository) Close() {
if repo == nil || repo.gogitStorage == nil {
return
}
if err := repo.gogitStorage.Close(); err != nil {
gitealog.Error("Error closing storage: %v", err)
}
}
// GoGitRepo gets the go-git repo representation
func (repo *Repository) GoGitRepo() *gogit.Repository {
return repo.gogitRepo
}

40
modules/git/repo_base_nogogit.go Normal file
View File

@@ -0,0 +1,40 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"errors"
"path/filepath"
)
// Repository represents a Git repository.
type Repository struct {
Path string
tagCache *ObjectCache
gpgSettings *GPGSettings
}
// OpenRepository opens the repository at the given path.
func OpenRepository(repoPath string) (*Repository, error) {
repoPath, err := filepath.Abs(repoPath)
if err != nil {
return nil, err
} else if !isDir(repoPath) {
return nil, errors.New("no such file or directory")
}
return &Repository{
Path: repoPath,
tagCache: newObjectCache(),
}, nil
}
// Close this repository, in particular close the underlying gogitStorage if this is not nil
func (repo *Repository) Close() {
}

18
modules/git/repo_blob.go
View File

@@ -1,25 +1,9 @@
// Copyright 2018 The Gitea Authors. All rights reserved.
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package git
import (
"github.com/go-git/go-git/v5/plumbing"
)
func (repo *Repository) getBlob(id SHA1) (*Blob, error) {
encodedObj, err := repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, id)
if err != nil {
return nil, ErrNotExist{id.String(), ""}
}
return &Blob{
ID: id,
gogitEncodedObj: encodedObj,
}, nil
}
// GetBlob finds the blob object in the repository.
func (repo *Repository) GetBlob(idStr string) (*Blob, error) {
id, err := NewIDFromString(idStr)

23
modules/git/repo_blob_gogit.go Normal file
View File

@@ -0,0 +1,23 @@
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"github.com/go-git/go-git/v5/plumbing"
)
func (repo *Repository) getBlob(id SHA1) (*Blob, error) {
encodedObj, err := repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, id)
if err != nil {
return nil, ErrNotExist{id.String(), ""}
}
return &Blob{
ID: id,
gogitEncodedObj: encodedObj,
}, nil
}

17
modules/git/repo_blob_nogogit.go Normal file
View File

@@ -0,0 +1,17 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
func (repo *Repository) getBlob(id SHA1) (*Blob, error) {
if id.IsZero() {
return nil, ErrNotExist{id.String(), ""}
}
return &Blob{
ID: id,
repoPath: repo.Path,
}, nil
}

33
modules/git/repo_branch.go
View File

@@ -8,8 +8,6 @@ package git
import (
"fmt"
"strings"
"github.com/go-git/go-git/v5/plumbing"
)
// BranchPrefix base dir of the branch information file store on git
@@ -26,18 +24,6 @@ func IsBranchExist(repoPath, name string) bool {
return IsReferenceExist(repoPath, BranchPrefix+name)
}
// IsBranchExist returns true if given branch exists in current repository.
func (repo *Repository) IsBranchExist(name string) bool {
if name == "" {
return false
}
reference, err := repo.gogitRepo.Reference(plumbing.ReferenceName(BranchPrefix+name), true)
if err != nil {
return false
}
return reference.Type() != plumbing.InvalidReference
}
// Branch represents a Git branch.
type Branch struct {
Name string
@@ -79,25 +65,6 @@ func (repo *Repository) GetDefaultBranch() (string, error) {
return NewCommand("symbolic-ref", "HEAD").RunInDir(repo.Path)
}
// GetBranches returns all branches of the repository.
func (repo *Repository) GetBranches() ([]string, error) {
var branchNames []string
branches, err := repo.gogitRepo.Branches()
if err != nil {
return nil, err
}
_ = branches.ForEach(func(branch *plumbing.Reference) error {
branchNames = append(branchNames, strings.TrimPrefix(branch.Name().String(), BranchPrefix))
return nil
})
// TODO: Sort?
return branchNames, nil
}
// GetBranch returns a branch by it's name
func (repo *Repository) GetBranch(branch string) (*Branch, error) {
if !repo.IsBranchExist(branch) {

45
modules/git/repo_branch_gogit.go Normal file
View File

@@ -0,0 +1,45 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"strings"
"github.com/go-git/go-git/v5/plumbing"
)
// IsBranchExist returns true if given branch exists in current repository.
func (repo *Repository) IsBranchExist(name string) bool {
if name == "" {
return false
}
reference, err := repo.gogitRepo.Reference(plumbing.ReferenceName(BranchPrefix+name), true)
if err != nil {
return false
}
return reference.Type() != plumbing.InvalidReference
}
// GetBranches returns all branches of the repository.
func (repo *Repository) GetBranches() ([]string, error) {
var branchNames []string
branches, err := repo.gogitRepo.Branches()
if err != nil {
return nil, err
}
_ = branches.ForEach(func(branch *plumbing.Reference) error {
branchNames = append(branchNames, strings.TrimPrefix(branch.Name().String(), BranchPrefix))
return nil
})
// TODO: Sort?
return branchNames, nil
}

82
modules/git/repo_branch_nogogit.go Normal file
View File

@@ -0,0 +1,82 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"bufio"
"io"
"strings"
)
// IsBranchExist returns true if given branch exists in current repository.
func (repo *Repository) IsBranchExist(name string) bool {
if name == "" {
return false
}
return IsReferenceExist(repo.Path, BranchPrefix+name)
}
// GetBranches returns all branches of the repository.
func (repo *Repository) GetBranches() ([]string, error) {
return callShowRef(repo.Path, BranchPrefix, "--heads")
}
func callShowRef(repoPath, prefix, arg string) ([]string, error) {
var branchNames []string
stdoutReader, stdoutWriter := io.Pipe()
defer func() {
_ = stdoutReader.Close()
_ = stdoutWriter.Close()
}()
go func() {
stderrBuilder := &strings.Builder{}
err := NewCommand("show-ref", arg).RunInDirPipeline(repoPath, stdoutWriter, stderrBuilder)
if err != nil {
if stderrBuilder.Len() == 0 {
_ = stdoutWriter.Close()
return
}
_ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String()))
} else {
_ = stdoutWriter.Close()
}
}()
bufReader := bufio.NewReader(stdoutReader)
for {
// The output of show-ref is simply a list:
// <sha> SP <ref> LF
_, err := bufReader.ReadSlice(' ')
for err == bufio.ErrBufferFull {
// This shouldn't happen but we'll tolerate it for the sake of peace
_, err = bufReader.ReadSlice(' ')
}
if err == io.EOF {
return branchNames, nil
}
if err != nil {
return nil, err
}
branchName, err := bufReader.ReadString('\n')
if err == io.EOF {
// This shouldn't happen... but we'll tolerate it for the sake of peace
return branchNames, nil
}
if err != nil {
return nil, err
}
branchName = strings.TrimPrefix(branchName, prefix)
if len(branchName) > 0 {
branchName = branchName[:len(branchName)-1]
}
branchNames = append(branchNames, branchName)
}
}

98
modules/git/repo_commit.go
View File

@@ -8,36 +8,10 @@ package git
import (
"bytes"
"container/list"
"fmt"
"strconv"
"strings"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
)
// GetRefCommitID returns the last commit ID string of given reference (branch or tag).
func (repo *Repository) GetRefCommitID(name string) (string, error) {
ref, err := repo.gogitRepo.Reference(plumbing.ReferenceName(name), true)
if err != nil {
if err == plumbing.ErrReferenceNotFound {
return "", ErrNotExist{
ID: name,
}
}
return "", err
}
return ref.Hash().String(), nil
}
// IsCommitExist returns true if given commit exists in current repository.
func (repo *Repository) IsCommitExist(name string) bool {
hash := plumbing.NewHash(name)
_, err := repo.gogitRepo.CommitObject(hash)
return err == nil
}
// GetBranchCommitID returns last commit ID string of given branch.
func (repo *Repository) GetBranchCommitID(name string) (string, error) {
return repo.GetRefCommitID(BranchPrefix + name)
@@ -55,78 +29,6 @@ func (repo *Repository) GetTagCommitID(name string) (string, error) {
return strings.TrimSpace(stdout), nil
}
func convertPGPSignatureForTag(t *object.Tag) *CommitGPGSignature {
if t.PGPSignature == "" {
return nil
}
var w strings.Builder
var err error
if _, err = fmt.Fprintf(&w,
"object %s\ntype %s\ntag %s\ntagger ",
t.Target.String(), t.TargetType.Bytes(), t.Name); err != nil {
return nil
}
if err = t.Tagger.Encode(&w); err != nil {
return nil
}
if _, err = fmt.Fprintf(&w, "\n\n"); err != nil {
return nil
}
if _, err = fmt.Fprintf(&w, t.Message); err != nil {
return nil
}
return &CommitGPGSignature{
Signature: t.PGPSignature,
Payload: strings.TrimSpace(w.String()) + "\n",
}
}
func (repo *Repository) getCommit(id SHA1) (*Commit, error) {
var tagObject *object.Tag
gogitCommit, err := repo.gogitRepo.CommitObject(id)
if err == plumbing.ErrObjectNotFound {
tagObject, err = repo.gogitRepo.TagObject(id)
if err == plumbing.ErrObjectNotFound {
return nil, ErrNotExist{
ID: id.String(),
}
}
if err == nil {
gogitCommit, err = repo.gogitRepo.CommitObject(tagObject.Target)
}
// if we get a plumbing.ErrObjectNotFound here then the repository is broken and it should be 500
}
if err != nil {
return nil, err
}
commit := convertCommit(gogitCommit)
commit.repo = repo
if tagObject != nil {
commit.CommitMessage = strings.TrimSpace(tagObject.Message)
commit.Author = &tagObject.Tagger
commit.Signature = convertPGPSignatureForTag(tagObject)
}
tree, err := gogitCommit.Tree()
if err != nil {
return nil, err
}
commit.Tree.ID = tree.Hash
commit.Tree.gogitTree = tree
return commit, nil
}
// ConvertToSHA1 returns a Hash object from a potential ID string
func (repo *Repository) ConvertToSHA1(commitID string) (SHA1, error) {
if len(commitID) != 40 {

110
modules/git/repo_commit_gogit.go Normal file
View File

@@ -0,0 +1,110 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"fmt"
"strings"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
)
// GetRefCommitID returns the last commit ID string of given reference (branch or tag).
func (repo *Repository) GetRefCommitID(name string) (string, error) {
ref, err := repo.gogitRepo.Reference(plumbing.ReferenceName(name), true)
if err != nil {
if err == plumbing.ErrReferenceNotFound {
return "", ErrNotExist{
ID: name,
}
}
return "", err
}
return ref.Hash().String(), nil
}
// IsCommitExist returns true if given commit exists in current repository.
func (repo *Repository) IsCommitExist(name string) bool {
hash := plumbing.NewHash(name)
_, err := repo.gogitRepo.CommitObject(hash)
return err == nil
}
func convertPGPSignatureForTag(t *object.Tag) *CommitGPGSignature {
if t.PGPSignature == "" {
return nil
}
var w strings.Builder
var err error
if _, err = fmt.Fprintf(&w,
"object %s\ntype %s\ntag %s\ntagger ",
t.Target.String(), t.TargetType.Bytes(), t.Name); err != nil {
return nil
}
if err = t.Tagger.Encode(&w); err != nil {
return nil
}
if _, err = fmt.Fprintf(&w, "\n\n"); err != nil {
return nil
}
if _, err = fmt.Fprintf(&w, t.Message); err != nil {
return nil
}
return &CommitGPGSignature{
Signature: t.PGPSignature,
Payload: strings.TrimSpace(w.String()) + "\n",
}
}
func (repo *Repository) getCommit(id SHA1) (*Commit, error) {
var tagObject *object.Tag
gogitCommit, err := repo.gogitRepo.CommitObject(id)
if err == plumbing.ErrObjectNotFound {
tagObject, err = repo.gogitRepo.TagObject(id)
if err == plumbing.ErrObjectNotFound {
return nil, ErrNotExist{
ID: id.String(),
}
}
if err == nil {
gogitCommit, err = repo.gogitRepo.CommitObject(tagObject.Target)
}
// if we get a plumbing.ErrObjectNotFound here then the repository is broken and it should be 500
}
if err != nil {
return nil, err
}
commit := convertCommit(gogitCommit)
commit.repo = repo
if tagObject != nil {
commit.CommitMessage = strings.TrimSpace(tagObject.Message)
commit.Author = &tagObject.Tagger
commit.Signature = convertPGPSignatureForTag(tagObject)
}
tree, err := gogitCommit.Tree()
if err != nil {
return nil, err
}
commit.Tree.ID = tree.Hash
commit.Tree.gogitTree = tree
return commit, nil
}

109
modules/git/repo_commit_nogogit.go Normal file
View File

@@ -0,0 +1,109 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"bufio"
"fmt"
"io"
"io/ioutil"
"strings"
)
// ResolveReference resolves a name to a reference
func (repo *Repository) ResolveReference(name string) (string, error) {
stdout, err := NewCommand("show-ref", "--hash", name).RunInDir(repo.Path)
if err != nil {
if strings.Contains(err.Error(), "not a valid ref") {
return "", ErrNotExist{name, ""}
}
return "", err
}
stdout = strings.TrimSpace(stdout)
if stdout == "" {
return "", ErrNotExist{name, ""}
}
return stdout, nil
}
// GetRefCommitID returns the last commit ID string of given reference (branch or tag).
func (repo *Repository) GetRefCommitID(name string) (string, error) {
stdout, err := NewCommand("show-ref", "--verify", "--hash", name).RunInDir(repo.Path)
if err != nil {
if strings.Contains(err.Error(), "not a valid ref") {
return "", ErrNotExist{name, ""}
}
return "", err
}
return strings.TrimSpace(stdout), nil
}
// IsCommitExist returns true if given commit exists in current repository.
func (repo *Repository) IsCommitExist(name string) bool {
_, err := NewCommand("cat-file", "-e", name).RunInDir(repo.Path)
return err == nil
}
func (repo *Repository) getCommit(id SHA1) (*Commit, error) {
stdoutReader, stdoutWriter := io.Pipe()
defer func() {
_ = stdoutReader.Close()
_ = stdoutWriter.Close()
}()
go func() {
stderr := strings.Builder{}
err := NewCommand("cat-file", "--batch").RunInDirFullPipeline(repo.Path, stdoutWriter, &stderr, strings.NewReader(id.String()+"\n"))
if err != nil {
_ = stdoutWriter.CloseWithError(ConcatenateError(err, (&stderr).String()))
} else {
_ = stdoutWriter.Close()
}
}()
bufReader := bufio.NewReader(stdoutReader)
_, typ, size, err := ReadBatchLine(bufReader)
if err != nil {
return nil, err
}
switch typ {
case "tag":
// then we need to parse the tag
// and load the commit
data, err := ioutil.ReadAll(io.LimitReader(bufReader, size))
if err != nil {
return nil, err
}
tag, err := parseTagData(data)
if err != nil {
return nil, err
}
tag.repo = repo
commit, err := tag.Commit()
if err != nil {
return nil, err
}
commit.CommitMessage = strings.TrimSpace(tag.Message)
commit.Author = tag.Tagger
commit.Signature = tag.Signature
return commit, nil
case "commit":
return CommitFromReader(repo, id, io.LimitReader(bufReader, size))
default:
_ = stdoutReader.CloseWithError(fmt.Errorf("unknown typ: %s", typ))
log("Unknown typ: %s", typ)
return nil, ErrNotExist{
ID: id.String(),
}
}
}

2
modules/git/repo_commitgraph.go → modules/git/repo_commitgraph_gogit.go
View File

@@ -3,6 +3,8 @@
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (

106
modules/git/repo_language_stats.go
View File

@@ -4,111 +4,5 @@
package git
import (
"bytes"
"io"
"io/ioutil"
"code.gitea.io/gitea/modules/analyze"
"github.com/go-enry/go-enry/v2"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
)
const fileSizeLimit int64 = 16 * 1024 // 16 KiB
const bigFileSize int64 = 1024 * 1024 // 1 MiB
// GetLanguageStats calculates language stats for git repository at specified commit
func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) {
r, err := git.PlainOpen(repo.Path)
if err != nil {
return nil, err
}
rev, err := r.ResolveRevision(plumbing.Revision(commitID))
if err != nil {
return nil, err
}
commit, err := r.CommitObject(*rev)
if err != nil {
return nil, err
}
tree, err := commit.Tree()
if err != nil {
return nil, err
}
sizes := make(map[string]int64)
err = tree.Files().ForEach(func(f *object.File) error {
if f.Size == 0 || enry.IsVendor(f.Name) || enry.IsDotFile(f.Name) ||
enry.IsDocumentation(f.Name) || enry.IsConfiguration(f.Name) {
return nil
}
// If content can not be read or file is too big just do detection by filename
var content []byte
if f.Size <= bigFileSize {
content, _ = readFile(f, fileSizeLimit)
}
if enry.IsGenerated(f.Name, content) {
return nil
}
// TODO: Use .gitattributes file for linguist overrides
language := analyze.GetCodeLanguage(f.Name, content)
if language == enry.OtherLanguage || language == "" {
return nil
}
// group languages, such as Pug -> HTML; SCSS -> CSS
group := enry.GetLanguageGroup(language)
if group != "" {
language = group
}
sizes[language] += f.Size
return nil
})
if err != nil {
return nil, err
}
// filter special languages unless they are the only language
if len(sizes) > 1 {
for language := range sizes {
langtype := enry.GetLanguageType(language)
if langtype != enry.Programming && langtype != enry.Markup {
delete(sizes, language)
}
}
}
return sizes, nil
}
func readFile(f *object.File, limit int64) ([]byte, error) {
r, err := f.Reader()
if err != nil {
return nil, err
}
defer r.Close()
if limit <= 0 {
return ioutil.ReadAll(r)
}
size := f.Size
if limit > 0 && size > limit {
size = limit
}
buf := bytes.NewBuffer(nil)
buf.Grow(int(size))
_, err = io.Copy(buf, io.LimitReader(r, limit))
return buf.Bytes(), err
}

113
modules/git/repo_language_stats_gogit.go Normal file
View File

@@ -0,0 +1,113 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"bytes"
"io"
"io/ioutil"
"code.gitea.io/gitea/modules/analyze"
"github.com/go-enry/go-enry/v2"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
)
// GetLanguageStats calculates language stats for git repository at specified commit
func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) {
r, err := git.PlainOpen(repo.Path)
if err != nil {
return nil, err
}
rev, err := r.ResolveRevision(plumbing.Revision(commitID))
if err != nil {
return nil, err
}
commit, err := r.CommitObject(*rev)
if err != nil {
return nil, err
}
tree, err := commit.Tree()
if err != nil {
return nil, err
}
sizes := make(map[string]int64)
err = tree.Files().ForEach(func(f *object.File) error {
if f.Size == 0 || enry.IsVendor(f.Name) || enry.IsDotFile(f.Name) ||
enry.IsDocumentation(f.Name) || enry.IsConfiguration(f.Name) {
return nil
}
// If content can not be read or file is too big just do detection by filename
var content []byte
if f.Size <= bigFileSize {
content, _ = readFile(f, fileSizeLimit)
}
if enry.IsGenerated(f.Name, content) {
return nil
}
// TODO: Use .gitattributes file for linguist overrides
language := analyze.GetCodeLanguage(f.Name, content)
if language == enry.OtherLanguage || language == "" {
return nil
}
// group languages, such as Pug -> HTML; SCSS -> CSS
group := enry.GetLanguageGroup(language)
if group != "" {
language = group
}
sizes[language] += f.Size
return nil
})
if err != nil {
return nil, err
}
// filter special languages unless they are the only language
if len(sizes) > 1 {
for language := range sizes {
langtype := enry.GetLanguageType(language)
if langtype != enry.Programming && langtype != enry.Markup {
delete(sizes, language)
}
}
}
return sizes, nil
}
func readFile(f *object.File, limit int64) ([]byte, error) {
r, err := f.Reader()
if err != nil {
return nil, err
}
defer r.Close()
if limit <= 0 {
return ioutil.ReadAll(r)
}
size := f.Size
if limit > 0 && size > limit {
size = limit
}
buf := bytes.NewBuffer(nil)
buf.Grow(int(size))
_, err = io.Copy(buf, io.LimitReader(r, limit))
return buf.Bytes(), err
}

109
modules/git/repo_language_stats_nogogit.go Normal file
View File

@@ -0,0 +1,109 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"bytes"
"io"
"io/ioutil"
"code.gitea.io/gitea/modules/analyze"
"github.com/go-enry/go-enry/v2"
)
// GetLanguageStats calculates language stats for git repository at specified commit
func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) {
// FIXME: We can be more efficient here...
//
// We're expecting that we will be reading a lot of blobs and the trees
// Thus we should use a shared `cat-file --batch` to get all of this data
// And keep the buffers around with resets as necessary.
//
// It's more complicated so...
commit, err := repo.GetCommit(commitID)
if err != nil {
log("Unable to get commit for: %s", commitID)
return nil, err
}
tree := commit.Tree
entries, err := tree.ListEntriesRecursive()
if err != nil {
return nil, err
}
sizes := make(map[string]int64)
for _, f := range entries {
if f.Size() == 0 || enry.IsVendor(f.Name()) || enry.IsDotFile(f.Name()) ||
enry.IsDocumentation(f.Name()) || enry.IsConfiguration(f.Name()) {
continue
}
// If content can not be read or file is too big just do detection by filename
var content []byte
if f.Size() <= bigFileSize {
content, _ = readFile(f, fileSizeLimit)
}
if enry.IsGenerated(f.Name(), content) {
continue
}
// TODO: Use .gitattributes file for linguist overrides
// FIXME: Why can't we split this and the IsGenerated tests to avoid reading the blob unless absolutely necessary?
// - eg. do the all the detection tests using filename first before reading content.
language := analyze.GetCodeLanguage(f.Name(), content)
if language == enry.OtherLanguage || language == "" {
continue
}
// group languages, such as Pug -> HTML; SCSS -> CSS
group := enry.GetLanguageGroup(language)
if group != "" {
language = group
}
sizes[language] += f.Size()
continue
}
// filter special languages unless they are the only language
if len(sizes) > 1 {
for language := range sizes {
langtype := enry.GetLanguageType(language)
if langtype != enry.Programming && langtype != enry.Markup {
delete(sizes, language)
}
}
}
return sizes, nil
}
func readFile(entry *TreeEntry, limit int64) ([]byte, error) {
// FIXME: We can probably be a little more efficient here... see above
r, err := entry.Blob().DataAsync()
if err != nil {
return nil, err
}
defer r.Close()
if limit <= 0 {
return ioutil.ReadAll(r)
}
size := entry.Size()
if limit > 0 && size > limit {
size = limit
}
buf := bytes.NewBuffer(nil)
buf.Grow(int(size))
_, err = io.Copy(buf, io.LimitReader(r, limit))
return buf.Bytes(), err
}

5
modules/git/repo_object.go
View File

@@ -27,6 +27,11 @@ const (
ObjectBranch ObjectType = "branch"
)
// Bytes returns the byte array for the Object Type
func (o ObjectType) Bytes() []byte {
return []byte(o)
}
// HashObject takes a reader and returns SHA1 hash for that reader
func (repo *Repository) HashObject(reader io.Reader) (SHA1, error) {
idStr, err := repo.hashObject(reader)

45
modules/git/repo_ref.go
View File

@@ -4,52 +4,7 @@
package git
import (
"strings"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
)
// GetRefs returns all references of the repository.
func (repo *Repository) GetRefs() ([]*Reference, error) {
return repo.GetRefsFiltered("")
}
// GetRefsFiltered returns all references of the repository that matches patterm exactly or starting with.
func (repo *Repository) GetRefsFiltered(pattern string) ([]*Reference, error) {
r, err := git.PlainOpen(repo.Path)
if err != nil {
return nil, err
}
refsIter, err := r.References()
if err != nil {
return nil, err
}
refs := make([]*Reference, 0)
if err = refsIter.ForEach(func(ref *plumbing.Reference) error {
if ref.Name() != plumbing.HEAD && !ref.Name().IsRemote() &&
(pattern == "" || strings.HasPrefix(ref.Name().String(), pattern)) {
refType := string(ObjectCommit)
if ref.Name().IsTag() {
// tags can be of type `commit` (lightweight) or `tag` (annotated)
if tagType, _ := repo.GetTagType(ref.Hash()); err == nil {
refType = tagType
}
}
r := &Reference{
Name: ref.Name().String(),
Object: ref.Hash(),
Type: refType,
repo: repo,
}
refs = append(refs, r)
}
return nil
}); err != nil {
return nil, err
}
return refs, nil
}

52
modules/git/repo_ref_gogit.go Normal file
View File

@@ -0,0 +1,52 @@
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"strings"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
)
// GetRefsFiltered returns all references of the repository that matches patterm exactly or starting with.
func (repo *Repository) GetRefsFiltered(pattern string) ([]*Reference, error) {
r, err := git.PlainOpen(repo.Path)
if err != nil {
return nil, err
}
refsIter, err := r.References()
if err != nil {
return nil, err
}
refs := make([]*Reference, 0)
if err = refsIter.ForEach(func(ref *plumbing.Reference) error {
if ref.Name() != plumbing.HEAD && !ref.Name().IsRemote() &&
(pattern == "" || strings.HasPrefix(ref.Name().String(), pattern)) {
refType := string(ObjectCommit)
if ref.Name().IsTag() {
// tags can be of type `commit` (lightweight) or `tag` (annotated)
if tagType, _ := repo.GetTagType(ref.Hash()); err == nil {
refType = tagType
}
}
r := &Reference{
Name: ref.Name().String(),
Object: ref.Hash(),
Type: refType,
repo: repo,
}
refs = append(refs, r)
}
return nil
}); err != nil {
return nil, err
}
return refs, nil
}

84
modules/git/repo_ref_nogogit.go Normal file
View File

@@ -0,0 +1,84 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"bufio"
"io"
"strings"
)
// GetRefsFiltered returns all references of the repository that matches patterm exactly or starting with.
func (repo *Repository) GetRefsFiltered(pattern string) ([]*Reference, error) {
stdoutReader, stdoutWriter := io.Pipe()
defer func() {
_ = stdoutReader.Close()
_ = stdoutWriter.Close()
}()
go func() {
stderrBuilder := &strings.Builder{}
err := NewCommand("for-each-ref").RunInDirPipeline(repo.Path, stdoutWriter, stderrBuilder)
if err != nil {
_ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String()))
} else {
_ = stdoutWriter.Close()
}
}()
refs := make([]*Reference, 0)
bufReader := bufio.NewReader(stdoutReader)
for {
// The output of for-each-ref is simply a list:
// <sha> SP <type> TAB <ref> LF
sha, err := bufReader.ReadString(' ')
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
sha = sha[:len(sha)-1]
typ, err := bufReader.ReadString('\t')
if err == io.EOF {
// This should not happen, but we'll tolerate it
break
}
if err != nil {
return nil, err
}
typ = typ[:len(typ)-1]
refName, err := bufReader.ReadString('\n')
if err == io.EOF {
// This should not happen, but we'll tolerate it
break
}
if err != nil {
return nil, err
}
refName = refName[:len(refName)-1]
// refName cannot be HEAD but can be remotes or stash
if strings.HasPrefix(refName, "/refs/remotes/") || refName == "/refs/stash" {
continue
}
if pattern == "" || strings.HasPrefix(refName, pattern) {
r := &Reference{
Name: refName,
Object: MustIDFromString(sha),
Type: typ,
repo: repo,
}
refs = append(refs, r)
}
}
return refs, nil
}

31
modules/git/repo_tag.go
View File

@@ -8,8 +8,6 @@ package git
import (
"fmt"
"strings"
"github.com/go-git/go-git/v5/plumbing"
)
// TagPrefix tags prefix path on the repository
@@ -20,12 +18,6 @@ func IsTagExist(repoPath, name string) bool {
return IsReferenceExist(repoPath, TagPrefix+name)
}
// IsTagExist returns true if given tag exists in the repository.
func (repo *Repository) IsTagExist(name string) bool {
_, err := repo.gogitRepo.Reference(plumbing.ReferenceName(TagPrefix+name), true)
return err == nil
}
// CreateTag create one tag in the repository
func (repo *Repository) CreateTag(name, revision string) error {
_, err := NewCommand("tag", "--", name, revision).RunInDir(repo.Path)
@@ -224,29 +216,6 @@ func (repo *Repository) GetTagInfos(page, pageSize int) ([]*Tag, error) {
return tags, nil
}
// GetTags returns all tags of the repository.
func (repo *Repository) GetTags() ([]string, error) {
var tagNames []string
tags, err := repo.gogitRepo.Tags()
if err != nil {
return nil, err
}
_ = tags.ForEach(func(tag *plumbing.Reference) error {
tagNames = append(tagNames, strings.TrimPrefix(tag.Name().String(), TagPrefix))
return nil
})
// Reverse order
for i := 0; i < len(tagNames)/2; i++ {
j := len(tagNames) - i - 1
tagNames[i], tagNames[j] = tagNames[j], tagNames[i]
}
return tagNames, nil
}
// GetTagType gets the type of the tag, either commit (simple) or tag (annotated)
func (repo *Repository) GetTagType(id SHA1) (string, error) {
// Get tag type

43
modules/git/repo_tag_gogit.go Normal file
View File

@@ -0,0 +1,43 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"strings"
"github.com/go-git/go-git/v5/plumbing"
)
// IsTagExist returns true if given tag exists in the repository.
func (repo *Repository) IsTagExist(name string) bool {
_, err := repo.gogitRepo.Reference(plumbing.ReferenceName(TagPrefix+name), true)
return err == nil
}
// GetTags returns all tags of the repository.
func (repo *Repository) GetTags() ([]string, error) {
var tagNames []string
tags, err := repo.gogitRepo.Tags()
if err != nil {
return nil, err
}
_ = tags.ForEach(func(tag *plumbing.Reference) error {
tagNames = append(tagNames, strings.TrimPrefix(tag.Name().String(), TagPrefix))
return nil
})
// Reverse order
for i := 0; i < len(tagNames)/2; i++ {
j := len(tagNames) - i - 1
tagNames[i], tagNames[j] = tagNames[j], tagNames[i]
}
return tagNames, nil
}

18
modules/git/repo_tag_nogogit.go Normal file
View File

@@ -0,0 +1,18 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
// IsTagExist returns true if given tag exists in the repository.
func (repo *Repository) IsTagExist(name string) bool {
return IsReferenceExist(repo.Path, TagPrefix+name)
}
// GetTags returns all tags of the repository.
func (repo *Repository) GetTags() ([]string, error) {
return callShowRef(repo.Path, TagPrefix, "--tags")
}

41
modules/git/repo_tree.go
View File

@@ -13,45 +13,6 @@ import (
"time"
)
func (repo *Repository) getTree(id SHA1) (*Tree, error) {
gogitTree, err := repo.gogitRepo.TreeObject(id)
if err != nil {
return nil, err
}
tree := NewTree(repo, id)
tree.gogitTree = gogitTree
return tree, nil
}
// GetTree find the tree object in the repository.
func (repo *Repository) GetTree(idStr string) (*Tree, error) {
if len(idStr) != 40 {
res, err := NewCommand("rev-parse", "--verify", idStr).RunInDir(repo.Path)
if err != nil {
return nil, err
}
if len(res) > 0 {
idStr = res[:len(res)-1]
}
}
id, err := NewIDFromString(idStr)
if err != nil {
return nil, err
}
resolvedID := id
commitObject, err := repo.gogitRepo.CommitObject(id)
if err == nil {
id = SHA1(commitObject.TreeHash)
}
treeObject, err := repo.getTree(id)
if err != nil {
return nil, err
}
treeObject.ResolvedID = resolvedID
return treeObject, nil
}
// CommitTreeOpts represents the possible options to CommitTree
type CommitTreeOpts struct {
Parents []string
@@ -102,7 +63,7 @@ func (repo *Repository) CommitTree(author *Signature, committer *Signature, tree
err = cmd.RunInDirTimeoutEnvFullPipeline(env, -1, repo.Path, stdout, stderr, messageBytes)
if err != nil {
return SHA1{}, concatenateError(err, stderr.String())
return SHA1{}, ConcatenateError(err, stderr.String())
}
return NewIDFromString(strings.TrimSpace(stdout.String()))
}

47
modules/git/repo_tree_gogit.go Normal file
View File

@@ -0,0 +1,47 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
func (repo *Repository) getTree(id SHA1) (*Tree, error) {
gogitTree, err := repo.gogitRepo.TreeObject(id)
if err != nil {
return nil, err
}
tree := NewTree(repo, id)
tree.gogitTree = gogitTree
return tree, nil
}
// GetTree find the tree object in the repository.
func (repo *Repository) GetTree(idStr string) (*Tree, error) {
if len(idStr) != 40 {
res, err := NewCommand("rev-parse", "--verify", idStr).RunInDir(repo.Path)
if err != nil {
return nil, err
}
if len(res) > 0 {
idStr = res[:len(res)-1]
}
}
id, err := NewIDFromString(idStr)
if err != nil {
return nil, err
}
resolvedID := id
commitObject, err := repo.gogitRepo.CommitObject(id)
if err == nil {
id = SHA1(commitObject.TreeHash)
}
treeObject, err := repo.getTree(id)
if err != nil {
return nil, err
}
treeObject.ResolvedID = resolvedID
return treeObject, nil
}

98
modules/git/repo_tree_nogogit.go Normal file
View File

@@ -0,0 +1,98 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"bufio"
"fmt"
"io"
"io/ioutil"
"strings"
)
func (repo *Repository) getTree(id SHA1) (*Tree, error) {
stdoutReader, stdoutWriter := io.Pipe()
defer func() {
_ = stdoutReader.Close()
_ = stdoutWriter.Close()
}()
go func() {
stderr := &strings.Builder{}
err := NewCommand("cat-file", "--batch").RunInDirFullPipeline(repo.Path, stdoutWriter, stderr, strings.NewReader(id.String()+"\n"))
if err != nil {
_ = stdoutWriter.CloseWithError(ConcatenateError(err, stderr.String()))
} else {
_ = stdoutWriter.Close()
}
}()
bufReader := bufio.NewReader(stdoutReader)
// ignore the SHA
_, typ, _, err := ReadBatchLine(bufReader)
if err != nil {
return nil, err
}
switch typ {
case "tag":
resolvedID := id
data, err := ioutil.ReadAll(bufReader)
if err != nil {
return nil, err
}
tag, err := parseTagData(data)
if err != nil {
return nil, err
}
commit, err := tag.Commit()
if err != nil {
return nil, err
}
commit.Tree.ResolvedID = resolvedID
log("tag.commit.Tree: %s %v", commit.Tree.ID.String(), commit.Tree.repo)
return &commit.Tree, nil
case "commit":
commit, err := CommitFromReader(repo, id, bufReader)
if err != nil {
_ = stdoutReader.CloseWithError(err)
return nil, err
}
commit.Tree.ResolvedID = commit.ID
log("commit.Tree: %s %v", commit.Tree.ID.String(), commit.Tree.repo)
return &commit.Tree, nil
case "tree":
stdoutReader.Close()
tree := NewTree(repo, id)
tree.ResolvedID = id
return tree, nil
default:
_ = stdoutReader.CloseWithError(fmt.Errorf("unknown typ: %s", typ))
return nil, ErrNotExist{
ID: id.String(),
}
}
}
// GetTree find the tree object in the repository.
func (repo *Repository) GetTree(idStr string) (*Tree, error) {
if len(idStr) != 40 {
res, err := NewCommand("rev-parse", "--verify", idStr).RunInDir(repo.Path)
if err != nil {
return nil, err
}
if len(res) > 0 {
idStr = res[:len(res)-1]
}
}
id, err := NewIDFromString(idStr)
if err != nil {
return nil, err
}
return repo.getTree(id)
}

5
modules/git/sha1.go
View File

@@ -10,8 +10,6 @@ import (
"fmt"
"regexp"
"strings"
"github.com/go-git/go-git/v5/plumbing"
)
// EmptySHA defines empty git SHA
@@ -23,9 +21,6 @@ const EmptyTreeSHA = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"
// SHAPattern can be used to determine if a string is an valid sha
var SHAPattern = regexp.MustCompile(`^[0-9a-f]{4,40}$`)
// SHA1 a git commit name
type SHA1 = plumbing.Hash
// MustID always creates a new SHA1 from a [20]byte array with no validation of input.
func MustID(b []byte) SHA1 {
var id SHA1

20
modules/git/sha1_gogit.go Normal file
View File

@@ -0,0 +1,20 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"github.com/go-git/go-git/v5/plumbing"
)
// SHA1 a git commit name
type SHA1 = plumbing.Hash
// ComputeBlobHash compute the hash for a given blob content
func ComputeBlobHash(content []byte) SHA1 {
return plumbing.ComputeHash(plumbing.BlobObject, content)
}

62
modules/git/sha1_nogogit.go Normal file
View File

@@ -0,0 +1,62 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"crypto/sha1"
"encoding/hex"
"hash"
"strconv"
)
// SHA1 a git commit name
type SHA1 [20]byte
// String returns a string representation of the SHA
func (s SHA1) String() string {
return hex.EncodeToString(s[:])
}
// IsZero returns whether this SHA1 is all zeroes
func (s SHA1) IsZero() bool {
var empty SHA1
return s == empty
}
// ComputeBlobHash compute the hash for a given blob content
func ComputeBlobHash(content []byte) SHA1 {
return ComputeHash(ObjectBlob, content)
}
// ComputeHash compute the hash for a given ObjectType and content
func ComputeHash(t ObjectType, content []byte) SHA1 {
h := NewHasher(t, int64(len(content)))
_, _ = h.Write(content)
return h.Sum()
}
// Hasher is a struct that will generate a SHA1
type Hasher struct {
hash.Hash
}
// NewHasher takes an object type and size and creates a hasher to generate a SHA
func NewHasher(t ObjectType, size int64) Hasher {
h := Hasher{sha1.New()}
_, _ = h.Write(t.Bytes())
_, _ = h.Write([]byte(" "))
_, _ = h.Write([]byte(strconv.FormatInt(size, 10)))
_, _ = h.Write([]byte{0})
return h
}
// Sum generates a SHA1 for the provided hash
func (h Hasher) Sum() (sha1 SHA1) {
copy(sha1[:], h.Hash.Sum(nil))
return
}

46
modules/git/signature.go
View File

@@ -5,53 +5,7 @@
package git
import (
"bytes"
"strconv"
"time"
"github.com/go-git/go-git/v5/plumbing/object"
)
// Signature represents the Author or Committer information.
type Signature = object.Signature
const (
// GitTimeLayout is the (default) time layout used by git.
GitTimeLayout = "Mon Jan _2 15:04:05 2006 -0700"
)
// Helper to get a signature from the commit line, which looks like these:
// author Patrick Gundlach <gundlach@speedata.de> 1378823654 +0200
// author Patrick Gundlach <gundlach@speedata.de> Thu, 07 Apr 2005 22:13:13 +0200
// but without the "author " at the beginning (this method should)
// be used for author and committer.
//
// FIXME: include timezone for timestamp!
func newSignatureFromCommitline(line []byte) (_ *Signature, err error) {
sig := new(Signature)
emailStart := bytes.IndexByte(line, '<')
sig.Name = string(line[:emailStart-1])
emailEnd := bytes.IndexByte(line, '>')
sig.Email = string(line[emailStart+1 : emailEnd])
// Check date format.
if len(line) > emailEnd+2 {
firstChar := line[emailEnd+2]
if firstChar >= 48 && firstChar <= 57 {
timestop := bytes.IndexByte(line[emailEnd+2:], ' ')
timestring := string(line[emailEnd+2 : emailEnd+2+timestop])
seconds, _ := strconv.ParseInt(timestring, 10, 64)
sig.When = time.Unix(seconds, 0)
} else {
sig.When, err = time.Parse(GitTimeLayout, string(line[emailEnd+2:]))
if err != nil {
return nil, err
}
}
} else {
// Fall back to unix 0 time
sig.When = time.Unix(0, 0)
}
return sig, nil
}

54
modules/git/signature_gogit.go Normal file
View File

@@ -0,0 +1,54 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"bytes"
"strconv"
"time"
"github.com/go-git/go-git/v5/plumbing/object"
)
// Signature represents the Author or Committer information.
type Signature = object.Signature
// Helper to get a signature from the commit line, which looks like these:
// author Patrick Gundlach <gundlach@speedata.de> 1378823654 +0200
// author Patrick Gundlach <gundlach@speedata.de> Thu, 07 Apr 2005 22:13:13 +0200
// but without the "author " at the beginning (this method should)
// be used for author and committer.
//
// FIXME: include timezone for timestamp!
func newSignatureFromCommitline(line []byte) (_ *Signature, err error) {
sig := new(Signature)
emailStart := bytes.IndexByte(line, '<')
sig.Name = string(line[:emailStart-1])
emailEnd := bytes.IndexByte(line, '>')
sig.Email = string(line[emailStart+1 : emailEnd])
// Check date format.
if len(line) > emailEnd+2 {
firstChar := line[emailEnd+2]
if firstChar >= 48 && firstChar <= 57 {
timestop := bytes.IndexByte(line[emailEnd+2:], ' ')
timestring := string(line[emailEnd+2 : emailEnd+2+timestop])
seconds, _ := strconv.ParseInt(timestring, 10, 64)
sig.When = time.Unix(seconds, 0)
} else {
sig.When, err = time.Parse(GitTimeLayout, string(line[emailEnd+2:]))
if err != nil {
return nil, err
}
}
} else {
// Fall back to unix 0 time
sig.When = time.Unix(0, 0)
}
return sig, nil
}

95
modules/git/signature_nogogit.go Normal file
View File

@@ -0,0 +1,95 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"bytes"
"fmt"
"strconv"
"time"
)
// Signature represents the Author or Committer information.
type Signature struct {
// Name represents a person name. It is an arbitrary string.
Name string
// Email is an email, but it cannot be assumed to be well-formed.
Email string
// When is the timestamp of the signature.
When time.Time
}
func (s *Signature) String() string {
return fmt.Sprintf("%s <%s>", s.Name, s.Email)
}
// Decode decodes a byte array representing a signature to signature
func (s *Signature) Decode(b []byte) {
sig, _ := newSignatureFromCommitline(b)
s.Email = sig.Email
s.Name = sig.Name
s.When = sig.When
}
// Helper to get a signature from the commit line, which looks like these:
// author Patrick Gundlach <gundlach@speedata.de> 1378823654 +0200
// author Patrick Gundlach <gundlach@speedata.de> Thu, 07 Apr 2005 22:13:13 +0200
// but without the "author " at the beginning (this method should)
// be used for author and committer.
func newSignatureFromCommitline(line []byte) (sig *Signature, err error) {
sig = new(Signature)
emailStart := bytes.LastIndexByte(line, '<')
emailEnd := bytes.LastIndexByte(line, '>')
if emailStart == -1 || emailEnd == -1 || emailEnd < emailStart {
return
}
sig.Name = string(line[:emailStart-1])
sig.Email = string(line[emailStart+1 : emailEnd])
hasTime := emailEnd+2 < len(line)
if !hasTime {
return
}
// Check date format.
firstChar := line[emailEnd+2]
if firstChar >= 48 && firstChar <= 57 {
idx := bytes.IndexByte(line[emailEnd+2:], ' ')
if idx < 0 {
return
}
timestring := string(line[emailEnd+2 : emailEnd+2+idx])
seconds, _ := strconv.ParseInt(timestring, 10, 64)
sig.When = time.Unix(seconds, 0)
idx += emailEnd + 3
if idx >= len(line) || idx+5 > len(line) {
return
}
timezone := string(line[idx : idx+5])
tzhours, err1 := strconv.ParseInt(timezone[0:3], 10, 64)
tzmins, err2 := strconv.ParseInt(timezone[3:], 10, 64)
if err1 != nil || err2 != nil {
return
}
if tzhours < 0 {
tzmins *= -1
}
tz := time.FixedZone("", int(tzhours*60*60+tzmins*60))
sig.When = sig.When.In(tz)
} else {
sig.When, err = time.Parse(GitTimeLayout, string(line[emailEnd+2:]))
if err != nil {
return
}
}
return
}

17
modules/git/tag.go
View File

@@ -10,6 +10,9 @@ import (
"strings"
)
const beginpgp = "\n-----BEGIN PGP SIGNATURE-----\n"
const endpgp = "\n-----END PGP SIGNATURE-----"
// Tag represents a Git tag.
type Tag struct {
Name string
@@ -19,6 +22,7 @@ type Tag struct {
Type string
Tagger *Signature
Message string
Signature *CommitGPGSignature
}
// Commit return the commit of the tag reference
@@ -60,12 +64,23 @@ l:
}
nextline += eol + 1
case eol == 0:
tag.Message = strings.TrimRight(string(data[nextline+1:]), "\n")
tag.Message = string(data[nextline+1 : len(data)-1])
break l
default:
break l
}
}
idx := strings.LastIndex(tag.Message, beginpgp)
if idx > 0 {
endSigIdx := strings.Index(tag.Message[idx:], endpgp)
if endSigIdx > 0 {
tag.Signature = &CommitGPGSignature{
Signature: tag.Message[idx+1 : idx+endSigIdx+len(endpgp)],
Payload: string(data[:bytes.LastIndex(data, []byte(beginpgp))+1]),
}
tag.Message = tag.Message[:idx+1]
}
}
return tag, nil
}

83
modules/git/tree.go
View File

@@ -6,25 +6,9 @@
package git
import (
"io"
"strings"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
)
// Tree represents a flat directory listing.
type Tree struct {
ID SHA1
ResolvedID SHA1
repo *Repository
gogitTree *object.Tree
// parent tree
ptree *Tree
}
// NewTree create a new tree according the repository and tree id
func NewTree(repo *Repository, id SHA1) *Tree {
return &Tree{
@@ -61,70 +45,3 @@ func (t *Tree) SubTree(rpath string) (*Tree, error) {
}
return g, nil
}
func (t *Tree) loadTreeObject() error {
gogitTree, err := t.repo.gogitRepo.TreeObject(t.ID)
if err != nil {
return err
}
t.gogitTree = gogitTree
return nil
}
// ListEntries returns all entries of current tree.
func (t *Tree) ListEntries() (Entries, error) {
if t.gogitTree == nil {
err := t.loadTreeObject()
if err != nil {
return nil, err
}
}
entries := make([]*TreeEntry, len(t.gogitTree.Entries))
for i, entry := range t.gogitTree.Entries {
entries[i] = &TreeEntry{
ID: entry.Hash,
gogitTreeEntry: &t.gogitTree.Entries[i],
ptree: t,
}
}
return entries, nil
}
// ListEntriesRecursive returns all entries of current tree recursively including all subtrees
func (t *Tree) ListEntriesRecursive() (Entries, error) {
if t.gogitTree == nil {
err := t.loadTreeObject()
if err != nil {
return nil, err
}
}
var entries []*TreeEntry
seen := map[plumbing.Hash]bool{}
walker := object.NewTreeWalker(t.gogitTree, true, seen)
for {
fullName, entry, err := walker.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
if seen[entry.Hash] {
continue
}
convertedEntry := &TreeEntry{
ID: entry.Hash,
gogitTreeEntry: &entry,
ptree: t,
fullName: fullName,
}
entries = append(entries, convertedEntry)
}
return entries, nil
}

58
modules/git/tree_blob.go
View File

@@ -5,64 +5,6 @@
package git
import (
"path"
"strings"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/filemode"
"github.com/go-git/go-git/v5/plumbing/object"
)
// GetTreeEntryByPath get the tree entries according the sub dir
func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) {
if len(relpath) == 0 {
return &TreeEntry{
ID: t.ID,
//Type: ObjectTree,
gogitTreeEntry: &object.TreeEntry{
Name: "",
Mode: filemode.Dir,
Hash: t.ID,
},
}, nil
}
relpath = path.Clean(relpath)
parts := strings.Split(relpath, "/")
var err error
tree := t
for i, name := range parts {
if i == len(parts)-1 {
entries, err := tree.ListEntries()
if err != nil {
if err == plumbing.ErrObjectNotFound {
return nil, ErrNotExist{
RelPath: relpath,
}
}
return nil, err
}
for _, v := range entries {
if v.Name() == name {
return v, nil
}
}
} else {
tree, err = tree.SubTree(name)
if err != nil {
if err == plumbing.ErrObjectNotFound {
return nil, ErrNotExist{
RelPath: relpath,
}
}
return nil, err
}
}
}
return nil, ErrNotExist{"", relpath}
}
// GetBlobByPath get the blob object according the path
func (t *Tree) GetBlobByPath(relpath string) (*Blob, error) {
entry, err := t.GetTreeEntryByPath(relpath)

66
modules/git/tree_blob_gogit.go Normal file
View File

@@ -0,0 +1,66 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"path"
"strings"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/filemode"
"github.com/go-git/go-git/v5/plumbing/object"
)
// GetTreeEntryByPath get the tree entries according the sub dir
func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) {
if len(relpath) == 0 {
return &TreeEntry{
ID: t.ID,
//Type: ObjectTree,
gogitTreeEntry: &object.TreeEntry{
Name: "",
Mode: filemode.Dir,
Hash: t.ID,
},
}, nil
}
relpath = path.Clean(relpath)
parts := strings.Split(relpath, "/")
var err error
tree := t
for i, name := range parts {
if i == len(parts)-1 {
entries, err := tree.ListEntries()
if err != nil {
if err == plumbing.ErrObjectNotFound {
return nil, ErrNotExist{
RelPath: relpath,
}
}
return nil, err
}
for _, v := range entries {
if v.Name() == name {
return v, nil
}
}
} else {
tree, err = tree.SubTree(name)
if err != nil {
if err == plumbing.ErrObjectNotFound {
return nil, ErrNotExist{
RelPath: relpath,
}
}
return nil, err
}
}
}
return nil, ErrNotExist{"", relpath}
}

49
modules/git/tree_blob_nogogit.go Normal file
View File

@@ -0,0 +1,49 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"path"
"strings"
)
// GetTreeEntryByPath get the tree entries according the sub dir
func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) {
if len(relpath) == 0 {
return &TreeEntry{
ID: t.ID,
name: "",
fullName: "",
entryMode: EntryModeTree,
}, nil
}
// FIXME: This should probably use git cat-file --batch to be a bit more efficient
relpath = path.Clean(relpath)
parts := strings.Split(relpath, "/")
var err error
tree := t
for i, name := range parts {
if i == len(parts)-1 {
entries, err := tree.ListEntries()
if err != nil {
return nil, err
}
for _, v := range entries {
if v.Name() == name {
return v, nil
}
}
} else {
tree, err = tree.SubTree(name)
if err != nil {
return nil, err
}
}
}
return nil, ErrNotExist{"", relpath}
}

104
modules/git/tree_entry.go
View File

@@ -9,55 +9,8 @@ import (
"io"
"sort"
"strings"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/filemode"
"github.com/go-git/go-git/v5/plumbing/object"
)
// EntryMode the type of the object in the git tree
type EntryMode int
// There are only a few file modes in Git. They look like unix file modes, but they can only be
// one of these.
const (
// EntryModeBlob
EntryModeBlob EntryMode = 0100644
// EntryModeExec
EntryModeExec EntryMode = 0100755
// EntryModeSymlink
EntryModeSymlink EntryMode = 0120000
// EntryModeCommit
EntryModeCommit EntryMode = 0160000
// EntryModeTree
EntryModeTree EntryMode = 0040000
)
// TreeEntry the leaf in the git tree
type TreeEntry struct {
ID SHA1
gogitTreeEntry *object.TreeEntry
ptree *Tree
size int64
sized bool
fullName string
}
// Name returns the name of the entry
func (te *TreeEntry) Name() string {
if te.fullName != "" {
return te.fullName
}
return te.gogitTreeEntry.Name
}
// Mode returns the mode of the entry
func (te *TreeEntry) Mode() EntryMode {
return EntryMode(te.gogitTreeEntry.Mode)
}
// Type returns the type of the entry (commit, tree, blob)
func (te *TreeEntry) Type() string {
switch te.Mode() {
@@ -70,63 +23,6 @@ func (te *TreeEntry) Type() string {
}
}
// Size returns the size of the entry
func (te *TreeEntry) Size() int64 {
if te.IsDir() {
return 0
} else if te.sized {
return te.size
}
file, err := te.ptree.gogitTree.TreeEntryFile(te.gogitTreeEntry)
if err != nil {
return 0
}
te.sized = true
te.size = file.Size
return te.size
}
// IsSubModule if the entry is a sub module
func (te *TreeEntry) IsSubModule() bool {
return te.gogitTreeEntry.Mode == filemode.Submodule
}
// IsDir if the entry is a sub dir
func (te *TreeEntry) IsDir() bool {
return te.gogitTreeEntry.Mode == filemode.Dir
}
// IsLink if the entry is a symlink
func (te *TreeEntry) IsLink() bool {
return te.gogitTreeEntry.Mode == filemode.Symlink
}
// IsRegular if the entry is a regular file
func (te *TreeEntry) IsRegular() bool {
return te.gogitTreeEntry.Mode == filemode.Regular
}
// IsExecutable if the entry is an executable file (not necessarily binary)
func (te *TreeEntry) IsExecutable() bool {
return te.gogitTreeEntry.Mode == filemode.Executable
}
// Blob returns the blob object the entry
func (te *TreeEntry) Blob() *Blob {
encodedObj, err := te.ptree.repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, te.gogitTreeEntry.Hash)
if err != nil {
return nil
}
return &Blob{
ID: te.gogitTreeEntry.Hash,
gogitEncodedObj: encodedObj,
name: te.Name(),
}
}
// FollowLink returns the entry pointed to by a symlink
func (te *TreeEntry) FollowLink() (*TreeEntry, error) {
if !te.IsLink() {

96
modules/git/tree_entry_gogit.go Normal file
View File

@@ -0,0 +1,96 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/filemode"
"github.com/go-git/go-git/v5/plumbing/object"
)
// TreeEntry the leaf in the git tree
type TreeEntry struct {
ID SHA1
gogitTreeEntry *object.TreeEntry
ptree *Tree
size int64
sized bool
fullName string
}
// Name returns the name of the entry
func (te *TreeEntry) Name() string {
if te.fullName != "" {
return te.fullName
}
return te.gogitTreeEntry.Name
}
// Mode returns the mode of the entry
func (te *TreeEntry) Mode() EntryMode {
return EntryMode(te.gogitTreeEntry.Mode)
}
// Size returns the size of the entry
func (te *TreeEntry) Size() int64 {
if te.IsDir() {
return 0
} else if te.sized {
return te.size
}
file, err := te.ptree.gogitTree.TreeEntryFile(te.gogitTreeEntry)
if err != nil {
return 0
}
te.sized = true
te.size = file.Size
return te.size
}
// IsSubModule if the entry is a sub module
func (te *TreeEntry) IsSubModule() bool {
return te.gogitTreeEntry.Mode == filemode.Submodule
}
// IsDir if the entry is a sub dir
func (te *TreeEntry) IsDir() bool {
return te.gogitTreeEntry.Mode == filemode.Dir
}
// IsLink if the entry is a symlink
func (te *TreeEntry) IsLink() bool {
return te.gogitTreeEntry.Mode == filemode.Symlink
}
// IsRegular if the entry is a regular file
func (te *TreeEntry) IsRegular() bool {
return te.gogitTreeEntry.Mode == filemode.Regular
}
// IsExecutable if the entry is an executable file (not necessarily binary)
func (te *TreeEntry) IsExecutable() bool {
return te.gogitTreeEntry.Mode == filemode.Executable
}
// Blob returns the blob object the entry
func (te *TreeEntry) Blob() *Blob {
encodedObj, err := te.ptree.repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, te.gogitTreeEntry.Hash)
if err != nil {
return nil
}
return &Blob{
ID: te.gogitTreeEntry.Hash,
gogitEncodedObj: encodedObj,
name: te.Name(),
}
}

36
modules/git/tree_entry_mode.go Normal file
View File

@@ -0,0 +1,36 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package git
import "strconv"
// EntryMode the type of the object in the git tree
type EntryMode int
// There are only a few file modes in Git. They look like unix file modes, but they can only be
// one of these.
const (
// EntryModeBlob
EntryModeBlob EntryMode = 0100644
// EntryModeExec
EntryModeExec EntryMode = 0100755
// EntryModeSymlink
EntryModeSymlink EntryMode = 0120000
// EntryModeCommit
EntryModeCommit EntryMode = 0160000
// EntryModeTree
EntryModeTree EntryMode = 0040000
)
// String converts an EntryMode to a string
func (e EntryMode) String() string {
return strconv.FormatInt(int64(e), 8)
}
// ToEntryMode converts a string to an EntryMode
func ToEntryMode(value string) EntryMode {
v, _ := strconv.ParseInt(value, 8, 32)
return EntryMode(v)
}

91
modules/git/tree_entry_nogogit.go Normal file
View File

@@ -0,0 +1,91 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"strconv"
"strings"
)
// TreeEntry the leaf in the git tree
type TreeEntry struct {
ID SHA1
ptree *Tree
entryMode EntryMode
name string
size int64
sized bool
fullName string
}
// Name returns the name of the entry
func (te *TreeEntry) Name() string {
if te.fullName != "" {
return te.fullName
}
return te.name
}
// Mode returns the mode of the entry
func (te *TreeEntry) Mode() EntryMode {
return te.entryMode
}
// Size returns the size of the entry
func (te *TreeEntry) Size() int64 {
if te.IsDir() {
return 0
} else if te.sized {
return te.size
}
stdout, err := NewCommand("cat-file", "-s", te.ID.String()).RunInDir(te.ptree.repo.Path)
if err != nil {
return 0
}
te.sized = true
te.size, _ = strconv.ParseInt(strings.TrimSpace(stdout), 10, 64)
return te.size
}
// IsSubModule if the entry is a sub module
func (te *TreeEntry) IsSubModule() bool {
return te.entryMode == EntryModeCommit
}
// IsDir if the entry is a sub dir
func (te *TreeEntry) IsDir() bool {
return te.entryMode == EntryModeTree
}
// IsLink if the entry is a symlink
func (te *TreeEntry) IsLink() bool {
return te.entryMode == EntryModeSymlink
}
// IsRegular if the entry is a regular file
func (te *TreeEntry) IsRegular() bool {
return te.entryMode == EntryModeBlob
}
// IsExecutable if the entry is an executable file (not necessarily binary)
func (te *TreeEntry) IsExecutable() bool {
return te.entryMode == EntryModeExec
}
// Blob returns the blob object the entry
func (te *TreeEntry) Blob() *Blob {
return &Blob{
ID: te.ID,
repoPath: te.ptree.repo.Path,
name: te.Name(),
}
}

2
modules/git/tree_entry_test.go
View File

@@ -2,6 +2,8 @@
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (

94
modules/git/tree_gogit.go Normal file
View File

@@ -0,0 +1,94 @@
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"io"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
)
// Tree represents a flat directory listing.
type Tree struct {
ID SHA1
ResolvedID SHA1
repo *Repository
gogitTree *object.Tree
// parent tree
ptree *Tree
}
func (t *Tree) loadTreeObject() error {
gogitTree, err := t.repo.gogitRepo.TreeObject(t.ID)
if err != nil {
return err
}
t.gogitTree = gogitTree
return nil
}
// ListEntries returns all entries of current tree.
func (t *Tree) ListEntries() (Entries, error) {
if t.gogitTree == nil {
err := t.loadTreeObject()
if err != nil {
return nil, err
}
}
entries := make([]*TreeEntry, len(t.gogitTree.Entries))
for i, entry := range t.gogitTree.Entries {
entries[i] = &TreeEntry{
ID: entry.Hash,
gogitTreeEntry: &t.gogitTree.Entries[i],
ptree: t,
}
}
return entries, nil
}
// ListEntriesRecursive returns all entries of current tree recursively including all subtrees
func (t *Tree) ListEntriesRecursive() (Entries, error) {
if t.gogitTree == nil {
err := t.loadTreeObject()
if err != nil {
return nil, err
}
}
var entries []*TreeEntry
seen := map[plumbing.Hash]bool{}
walker := object.NewTreeWalker(t.gogitTree, true, seen)
for {
fullName, entry, err := walker.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
if seen[entry.Hash] {
continue
}
convertedEntry := &TreeEntry{
ID: entry.Hash,
gogitTreeEntry: &entry,
ptree: t,
fullName: fullName,
}
entries = append(entries, convertedEntry)
}
return entries, nil
}

69
modules/git/tree_nogogit.go Normal file
View File

@@ -0,0 +1,69 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"strings"
)
// Tree represents a flat directory listing.
type Tree struct {
ID SHA1
ResolvedID SHA1
repo *Repository
// parent tree
ptree *Tree
entries Entries
entriesParsed bool
entriesRecursive Entries
entriesRecursiveParsed bool
}
// ListEntries returns all entries of current tree.
func (t *Tree) ListEntries() (Entries, error) {
if t.entriesParsed {
return t.entries, nil
}
stdout, err := NewCommand("ls-tree", t.ID.String()).RunInDirBytes(t.repo.Path)
if err != nil {
if strings.Contains(err.Error(), "fatal: Not a valid object name") || strings.Contains(err.Error(), "fatal: not a tree object") {
return nil, ErrNotExist{
ID: t.ID.String(),
}
}
return nil, err
}
t.entries, err = parseTreeEntries(stdout, t)
if err == nil {
t.entriesParsed = true
}
return t.entries, err
}
// ListEntriesRecursive returns all entries of current tree recursively including all subtrees
func (t *Tree) ListEntriesRecursive() (Entries, error) {
if t.entriesRecursiveParsed {
return t.entriesRecursive, nil
}
stdout, err := NewCommand("ls-tree", "-t", "-r", t.ID.String()).RunInDirBytes(t.repo.Path)
if err != nil {
return nil, err
}
t.entriesRecursive, err = parseTreeEntries(stdout, t)
if err == nil {
t.entriesRecursiveParsed = true
}
return t.entriesRecursive, err
}

32
modules/git/utils.go
View File

@@ -6,6 +6,7 @@ package git
import (
"fmt"
"io"
"os"
"strconv"
"strings"
@@ -68,11 +69,12 @@ func isExist(path string) bool {
return err == nil || os.IsExist(err)
}
func concatenateError(err error, stderr string) error {
// ConcatenateError concatenats an error with stderr string
func ConcatenateError(err error, stderr string) error {
if len(stderr) == 0 {
return err
}
return fmt.Errorf("%v - %s", err, stderr)
return fmt.Errorf("%w - %s", err, stderr)
}
// RefEndName return the end name of a ref name
@@ -140,3 +142,29 @@ func ParseBool(value string) (result bool, valid bool) {
}
return intValue != 0, true
}
// LimitedReaderCloser is a limited reader closer
type LimitedReaderCloser struct {
R io.Reader
C io.Closer
N int64
}
// Read implements io.Reader
func (l *LimitedReaderCloser) Read(p []byte) (n int, err error) {
if l.N <= 0 {
_ = l.C.Close()
return 0, io.EOF
}
if int64(len(p)) > l.N {
p = p[0:l.N]
}
n, err = l.R.Read(p)
l.N -= int64(n)
return
}
// Close implements io.Closer
func (l *LimitedReaderCloser) Close() error {
return l.C.Close()
}

3
modules/indexer/stats/db.go
View File

@@ -7,6 +7,7 @@ package stats
import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
)
// DBIndexer implements Indexer interface to use database's like search
@@ -37,6 +38,7 @@ func (db *DBIndexer) Index(id int64) error {
// Get latest commit for default branch
commitID, err := gitRepo.GetBranchCommitID(repo.DefaultBranch)
if err != nil {
log.Error("Unable to get commit ID for defaultbranch %s in %s", repo.DefaultBranch, repo.RepoPath())
return err
}
@@ -48,6 +50,7 @@ func (db *DBIndexer) Index(id int64) error {
// Calculate and save language statistics to database
stats, err := gitRepo.GetLanguageStats(commitID)
if err != nil {
log.Error("Unable to get language stats for ID %s for defaultbranch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.RepoPath(), err)
return err
}
return repo.UpdateLanguageStats(commitID, stats)

54
modules/repository/cache.go
View File

@@ -5,57 +5,14 @@
package repository
import (
"path"
"strings"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/setting"
cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph"
)
func recusiveCache(gitRepo *git.Repository, c cgobject.CommitNode, tree *git.Tree, treePath string, ca *cache.LastCommitCache, level int) error {
if level == 0 {
return nil
}
entries, err := tree.ListEntries()
if err != nil {
return err
}
entryPaths := make([]string, len(entries))
entryMap := make(map[string]*git.TreeEntry)
for i, entry := range entries {
entryPaths[i] = entry.Name()
entryMap[entry.Name()] = entry
}
commits, err := git.GetLastCommitForPaths(c, treePath, entryPaths)
if err != nil {
return err
}
for entry, cm := range commits {
if err := ca.Put(c.ID().String(), path.Join(treePath, entry), cm.ID().String()); err != nil {
return err
}
if entryMap[entry].IsDir() {
subTree, err := tree.SubTree(entry)
if err != nil {
return err
}
if err := recusiveCache(gitRepo, c, subTree, entry, ca, level-1); err != nil {
return err
}
}
}
return nil
}
func getRefName(fullRefName string) string {
if strings.HasPrefix(fullRefName, git.TagPrefix) {
return fullRefName[len(git.TagPrefix):]
@@ -84,14 +41,7 @@ func CacheRef(repo *models.Repository, gitRepo *git.Repository, fullRefName stri
return nil
}
commitNodeIndex, _ := gitRepo.CommitNodeIndex()
commitCache := git.NewLastCommitCache(repo.FullName(), gitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds()), cache.GetCache())
c, err := commitNodeIndex.Get(commit.ID)
if err != nil {
return err
}
ca := cache.NewLastCommitCache(repo.FullName(), gitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds()))
return recusiveCache(gitRepo, c, &commit.Tree, "", ca, 1)
return commitCache.CacheCommit(commit)
}

3
routers/private/hook.go
View File

@@ -25,7 +25,6 @@ import (
repo_service "code.gitea.io/gitea/services/repository"
"gitea.com/macaron/macaron"
"github.com/go-git/go-git/v5/plumbing"
)
func verifyCommits(oldCommitID, newCommitID string, repo *git.Repository, env []string) error {
@@ -82,7 +81,7 @@ func readAndVerifyCommit(sha string, repo *git.Repository, env []string) error {
_ = stdoutReader.Close()
_ = stdoutWriter.Close()
}()
hash := plumbing.NewHash(sha)
hash := git.MustIDFromString(sha)
return git.NewCommand("cat-file", "commit", sha).
RunInDirTimeoutEnvFullPipelineFunc(env, -1, repo.Path,

146
routers/repo/lfs.go
View File

@@ -12,11 +12,9 @@ import (
"io"
"io/ioutil"
"path"
"sort"
"strconv"
"strings"
"sync"
"time"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
@@ -29,9 +27,6 @@ import (
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
gogit "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/unknwon/com"
)
@@ -363,22 +358,6 @@ func LFSDelete(ctx *context.Context) {
ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs")
}
type lfsResult struct {
Name string
SHA string
Summary string
When time.Time
ParentHashes []plumbing.Hash
BranchName string
FullCommitName string
}
type lfsResultSlice []*lfsResult
func (a lfsResultSlice) Len() int { return len(a) }
func (a lfsResultSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) }
// LFSFileFind guesses a sha for the provided oid (or uses the provided sha) and then finds the commits that contain this sha
func LFSFileFind(ctx *context.Context) {
if !setting.LFS.StartServer {
@@ -394,140 +373,27 @@ func LFSFileFind(ctx *context.Context) {
sha := ctx.Query("sha")
ctx.Data["Title"] = oid
ctx.Data["PageIsSettingsLFS"] = true
var hash plumbing.Hash
var hash git.SHA1
if len(sha) == 0 {
meta := models.LFSMetaObject{Oid: oid, Size: size}
pointer := meta.Pointer()
hash = plumbing.ComputeHash(plumbing.BlobObject, []byte(pointer))
hash = git.ComputeBlobHash([]byte(pointer))
sha = hash.String()
} else {
hash = plumbing.NewHash(sha)
hash = git.MustIDFromString(sha)
}
ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
ctx.Data["Oid"] = oid
ctx.Data["Size"] = size
ctx.Data["SHA"] = sha
resultsMap := map[string]*lfsResult{}
results := make([]*lfsResult, 0)
basePath := ctx.Repo.Repository.RepoPath()
gogitRepo := ctx.Repo.GitRepo.GoGitRepo()
commitsIter, err := gogitRepo.Log(&gogit.LogOptions{
Order: gogit.LogOrderCommitterTime,
All: true,
})
if err != nil {
log.Error("Failed to get GoGit CommitsIter: %v", err)
ctx.ServerError("LFSFind: Iterate Commits", err)
return
}
err = commitsIter.ForEach(func(gitCommit *object.Commit) error {
tree, err := gitCommit.Tree()
if err != nil {
return err
}
treeWalker := object.NewTreeWalker(tree, true, nil)
defer treeWalker.Close()
for {
name, entry, err := treeWalker.Next()
if err == io.EOF {
break
}
if entry.Hash == hash {
result := lfsResult{
Name: name,
SHA: gitCommit.Hash.String(),
Summary: strings.Split(strings.TrimSpace(gitCommit.Message), "\n")[0],
When: gitCommit.Author.When,
ParentHashes: gitCommit.ParentHashes,
}
resultsMap[gitCommit.Hash.String()+":"+name] = &result
}
}
return nil
})
results, err := pipeline.FindLFSFile(ctx.Repo.GitRepo, hash)
if err != nil && err != io.EOF {
log.Error("Failure in CommitIter.ForEach: %v", err)
ctx.ServerError("LFSFind: IterateCommits ForEach", err)
log.Error("Failure in FindLFSFile: %v", err)
ctx.ServerError("LFSFind: FindLFSFile.", err)
return
}
for _, result := range resultsMap {
hasParent := false
for _, parentHash := range result.ParentHashes {
if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent {
break
}
}
if !hasParent {
results = append(results, result)
}
}
sort.Sort(lfsResultSlice(results))
// Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple
shasToNameReader, shasToNameWriter := io.Pipe()
nameRevStdinReader, nameRevStdinWriter := io.Pipe()
errChan := make(chan error, 1)
wg := sync.WaitGroup{}
wg.Add(3)
go func() {
defer wg.Done()
scanner := bufio.NewScanner(nameRevStdinReader)
i := 0
for scanner.Scan() {
line := scanner.Text()
if len(line) == 0 {
continue
}
result := results[i]
result.FullCommitName = line
result.BranchName = strings.Split(line, "~")[0]
i++
}
}()
go pipeline.NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath)
go func() {
defer wg.Done()
defer shasToNameWriter.Close()
for _, result := range results {
i := 0
if i < len(result.SHA) {
n, err := shasToNameWriter.Write([]byte(result.SHA)[i:])
if err != nil {
errChan <- err
break
}
i += n
}
n := 0
for n < 1 {
n, err = shasToNameWriter.Write([]byte{'\n'})
if err != nil {
errChan <- err
break
}
}
}
}()
wg.Wait()
select {
case err, has := <-errChan:
if has {
ctx.ServerError("LFSPointerFiles", err)
}
default:
}
ctx.Data["Results"] = results
ctx.HTML(200, tplSettingsLFSFileFind)
}

4
routers/repo/view.go
View File

@@ -137,9 +137,9 @@ func renderDirectory(ctx *context.Context, treeLink string) {
}
entries.CustomSort(base.NaturalSortLess)
var c git.LastCommitCache
var c *git.LastCommitCache
if setting.CacheService.LastCommit.Enabled && ctx.Repo.CommitsCount >= setting.CacheService.LastCommit.CommitsCount {
c = cache.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds()))
c = git.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds()), cache.GetCache())
}
var latestCommit *git.Commit

11
templates/repo/view_list.tmpl
View File

@@ -40,18 +40,19 @@
</tr>
{{end}}
{{range $item := .Files}}
{{$entry := index $item 0}}
{{$commit := index $item 1}}
{{$entry := $item.Entry}}
{{$commit := $item.Commit}}
{{$subModuleFile := $item.SubModuleFile}}
<tr>
<td class="name four wide">
<span class="truncate">
{{if $entry.IsSubModule}}
{{svg "octicon-file-submodule"}}
{{$refURL := $commit.RefURL AppUrl $.Repository.FullName $.SSHDomain}}
{{$refURL := $subModuleFile.RefURL AppUrl $.Repository.FullName $.SSHDomain}}
{{if $refURL}}
<a href="{{$refURL}}">{{$entry.Name}}</a><span class="at">@</span><a href="{{$refURL}}/commit/{{$commit.RefID}}">{{ShortSha $commit.RefID}}</a>
<a href="{{$refURL}}">{{$entry.Name}}</a><span class="at">@</span><a href="{{$refURL}}/commit/{{$subModuleFile.RefID}}">{{ShortSha $subModuleFile.RefID}}</a>
{{else}}
{{$entry.Name}}<span class="at">@</span>{{ShortSha $commit.RefID}}
{{$entry.Name}}<span class="at">@</span>{{ShortSha $subModuleFile.RefID}}
{{end}}
{{else}}
{{if $entry.IsDir}}