Custom regexp external issues (#17624)

* Implement custom regular expression for external issue tracking.

Signed-off-by: Alexander Beyn <malex@fatelectrons.org>

* Fix syntax/style

* Update repo.go

* Set metas['regexp']

* gofmt

* fix some tests

* fix more tests

* refactor frontend

* use LRU cache for regexp

* Update modules/markup/html_internal_test.go

Co-authored-by: Alexander Beyn <malex@fatelectrons.org>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>
Co-authored-by: Lauris BH <lauris@nix.lv>
Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

modules/markup/html.go

@@ -20,6 +20,7 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/markup/common"
 	"code.gitea.io/gitea/modules/references"
+	"code.gitea.io/gitea/modules/regexplru"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/templates/vars"
 	"code.gitea.io/gitea/modules/util"
@@ -33,6 +34,7 @@ import (
 const (
 	IssueNameStyleNumeric      = "numeric"
 	IssueNameStyleAlphanumeric = "alphanumeric"
+	IssueNameStyleRegexp       = "regexp"
 )
 
 var (
@@ -815,19 +817,35 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) {
 	)
 
 	next := node.NextSibling
+
 	for node != nil && node != next {
-		_, exttrack := ctx.Metas["format"]
-		alphanum := ctx.Metas["style"] == IssueNameStyleAlphanumeric
+		_, hasExtTrackFormat := ctx.Metas["format"]
 
 		// Repos with external issue trackers might still need to reference local PRs
 		// We need to concern with the first one that shows up in the text, whichever it is
-		found, ref = references.FindRenderizableReferenceNumeric(node.Data, exttrack && alphanum)
-		if exttrack && alphanum {
-			if found2, ref2 := references.FindRenderizableReferenceAlphanumeric(node.Data); found2 {
-				if !found || ref2.RefLocation.Start < ref.RefLocation.Start {
-					found = true
-					ref = ref2
-				}
+		isNumericStyle := ctx.Metas["style"] == "" || ctx.Metas["style"] == IssueNameStyleNumeric
+		foundNumeric, refNumeric := references.FindRenderizableReferenceNumeric(node.Data, hasExtTrackFormat && !isNumericStyle)
+
+		switch ctx.Metas["style"] {
+		case "", IssueNameStyleNumeric:
+			found, ref = foundNumeric, refNumeric
+		case IssueNameStyleAlphanumeric:
+			found, ref = references.FindRenderizableReferenceAlphanumeric(node.Data)
+		case IssueNameStyleRegexp:
+			pattern, err := regexplru.GetCompiled(ctx.Metas["regexp"])
+			if err != nil {
+				return
+			}
+			found, ref = references.FindRenderizableReferenceRegexp(node.Data, pattern)
+		}
+
+		// Repos with external issue trackers might still need to reference local PRs
+		// We need to concern with the first one that shows up in the text, whichever it is
+		if hasExtTrackFormat && !isNumericStyle {
+			// If numeric (PR) was found, and it was BEFORE the non-numeric pattern, use that
+			if foundNumeric && refNumeric.RefLocation.Start < ref.RefLocation.Start {
+				found = foundNumeric
+				ref = refNumeric
 			}
 		}
 		if !found {
@@ -836,7 +854,7 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) {
 
 		var link *html.Node
 		reftext := node.Data[ref.RefLocation.Start:ref.RefLocation.End]
-		if exttrack && !ref.IsPull {
+		if hasExtTrackFormat && !ref.IsPull {
 			ctx.Metas["index"] = ref.Issue
 
 			res, err := vars.Expand(ctx.Metas["format"], ctx.Metas)
@@ -869,7 +887,7 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) {
 
 		// Decorate action keywords if actionable
 		var keyword *html.Node
-		if references.IsXrefActionable(ref, exttrack, alphanum) {
+		if references.IsXrefActionable(ref, hasExtTrackFormat) {
 			keyword = createKeyword(node.Data[ref.ActionLocation.Start:ref.ActionLocation.End])
 		} else {
 			keyword = &html.Node{

modules/markup/html_internal_test.go

@@ -21,8 +21,8 @@ const (
 	TestRepoURL = TestAppURL + TestOrgRepo + "/"
 )
 
-// alphanumLink an HTML link to an alphanumeric-style issue
-func alphanumIssueLink(baseURL, class, name string) string {
+// externalIssueLink an HTML link to an alphanumeric-style issue
+func externalIssueLink(baseURL, class, name string) string {
 	return link(util.URLJoin(baseURL, name), class, name)
 }
 
@@ -54,6 +54,13 @@ var alphanumericMetas = map[string]string{
 	"style":  IssueNameStyleAlphanumeric,
 }
 
+var regexpMetas = map[string]string{
+	"format": "https://someurl.com/{user}/{repo}/{index}",
+	"user":   "someUser",
+	"repo":   "someRepo",
+	"style":  IssueNameStyleRegexp,
+}
+
 // these values should match the TestOrgRepo const above
 var localMetas = map[string]string{
 	"user": "gogits",
@@ -184,7 +191,7 @@ func TestRender_IssueIndexPattern4(t *testing.T) {
 	test := func(s, expectedFmt string, names ...string) {
 		links := make([]interface{}, len(names))
 		for i, name := range names {
-			links[i] = alphanumIssueLink("https://someurl.com/someUser/someRepo/", "ref-issue ref-external-issue", name)
+			links[i] = externalIssueLink("https://someurl.com/someUser/someRepo/", "ref-issue ref-external-issue", name)
 		}
 		expected := fmt.Sprintf(expectedFmt, links...)
 		testRenderIssueIndexPattern(t, s, expected, &RenderContext{Metas: alphanumericMetas})
@@ -194,6 +201,43 @@ func TestRender_IssueIndexPattern4(t *testing.T) {
 	test("test issue ABCDEFGHIJ-1234567890", "test issue %s", "ABCDEFGHIJ-1234567890")
 }
 
+func TestRender_IssueIndexPattern5(t *testing.T) {
+	setting.AppURL = TestAppURL
+
+	// regexp: render inputs without valid mentions
+	test := func(s, expectedFmt, pattern string, ids, names []string) {
+		metas := regexpMetas
+		metas["regexp"] = pattern
+		links := make([]interface{}, len(ids))
+		for i, id := range ids {
+			links[i] = link(util.URLJoin("https://someurl.com/someUser/someRepo/", id), "ref-issue ref-external-issue", names[i])
+		}
+
+		expected := fmt.Sprintf(expectedFmt, links...)
+		testRenderIssueIndexPattern(t, s, expected, &RenderContext{Metas: metas})
+	}
+
+	test("abc ISSUE-123 def", "abc %s def",
+		"ISSUE-(\\d+)",
+		[]string{"123"},
+		[]string{"ISSUE-123"},
+	)
+
+	test("abc (ISSUE 123) def", "abc %s def",
+		"\\(ISSUE (\\d+)\\)",
+		[]string{"123"},
+		[]string{"(ISSUE 123)"},
+	)
+
+	test("abc ISSUE-123 def", "abc %s def",
+		"(ISSUE-(\\d+))",
+		[]string{"ISSUE-123"},
+		[]string{"ISSUE-123"},
+	)
+
+	testRenderIssueIndexPattern(t, "will not match", "will not match", &RenderContext{Metas: regexpMetas})
+}
+
 func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *RenderContext) {
 	if ctx.URLPrefix == "" {
 		ctx.URLPrefix = TestAppURL
@@ -202,7 +246,7 @@ func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *Rend
 	var buf strings.Builder
 	err := postProcess(ctx, []processor{issueIndexPatternProcessor}, strings.NewReader(input), &buf)
 	assert.NoError(t, err)
-	assert.Equal(t, expected, buf.String())
+	assert.Equal(t, expected, buf.String(), "input=%q", input)
 }
 
 func TestRender_AutoLink(t *testing.T) {

modules/references/references.go

@@ -351,6 +351,24 @@ func FindRenderizableReferenceNumeric(content string, prOnly bool) (bool, *Rende
 	}
 }
 
+// FindRenderizableReferenceRegexp returns the first regexp unvalidated references found in a string.
+func FindRenderizableReferenceRegexp(content string, pattern *regexp.Regexp) (bool, *RenderizableReference) {
+	match := pattern.FindStringSubmatchIndex(content)
+	if len(match) < 4 {
+		return false, nil
+	}
+
+	action, location := findActionKeywords([]byte(content), match[2])
+
+	return true, &RenderizableReference{
+		Issue:          content[match[2]:match[3]],
+		RefLocation:    &RefSpan{Start: match[0], End: match[1]},
+		Action:         action,
+		ActionLocation: location,
+		IsPull:         false,
+	}
+}
+
 // FindRenderizableReferenceAlphanumeric returns the first alphanumeric unvalidated references found in a string.
 func FindRenderizableReferenceAlphanumeric(content string) (bool, *RenderizableReference) {
 	match := issueAlphanumericPattern.FindStringSubmatchIndex(content)
@@ -547,7 +565,7 @@ func findActionKeywords(content []byte, start int) (XRefAction, *RefSpan) {
 }
 
 // IsXrefActionable returns true if the xref action is actionable (i.e. produces a result when resolved)
-func IsXrefActionable(ref *RenderizableReference, extTracker, alphaNum bool) bool {
+func IsXrefActionable(ref *RenderizableReference, extTracker bool) bool {
 	if extTracker {
 		// External issues cannot be automatically closed
 		return false

modules/regexplru/regexplru.go

@@ -0,0 +1,45 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package regexplru
+
+import (
+	"regexp"
+
+	"code.gitea.io/gitea/modules/log"
+
+	lru "github.com/hashicorp/golang-lru"
+)
+
+var lruCache *lru.Cache
+
+func init() {
+	var err error
+	lruCache, err = lru.New(1000)
+	if err != nil {
+		log.Fatal("failed to new LRU cache, err: %v", err)
+	}
+}
+
+// GetCompiled works like regexp.Compile, the compiled expr or error is stored in LRU cache
+func GetCompiled(expr string) (r *regexp.Regexp, err error) {
+	v, ok := lruCache.Get(expr)
+	if !ok {
+		r, err = regexp.Compile(expr)
+		if err != nil {
+			lruCache.Add(expr, err)
+			return nil, err
+		}
+		lruCache.Add(expr, r)
+	} else {
+		r, ok = v.(*regexp.Regexp)
+		if !ok {
+			if err, ok = v.(error); ok {
+				return nil, err
+			}
+			panic("impossible")
+		}
+	}
+	return r, nil
+}

modules/regexplru/regexplru_test.go

@@ -0,0 +1,27 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package regexplru
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRegexpLru(t *testing.T) {
+	r, err := GetCompiled("a")
+	assert.NoError(t, err)
+	assert.True(t, r.MatchString("a"))
+
+	r, err = GetCompiled("a")
+	assert.NoError(t, err)
+	assert.True(t, r.MatchString("a"))
+
+	assert.EqualValues(t, 1, lruCache.Len())
+
+	_, err = GetCompiled("(")
+	assert.Error(t, err)
+	assert.EqualValues(t, 2, lruCache.Len())
+}