Fix language stat calculation (#11692)

* Fix language stat calculation

* Group languages and ignore 0 size files

* remove unneeded code

modules/git/repo_language_stats.go

@@ -19,6 +19,20 @@ import (
 
 const fileSizeLimit int64 = 16 * 1024 * 1024
 
+// specialLanguages defines list of languages that are excluded from the calculation
+// unless they are the only language present in repository. Only languages which under
+// normal circumstances are not considered to be code should be listed here.
+var specialLanguages = []string{
+	"XML",
+	"JSON",
+	"TOML",
+	"YAML",
+	"INI",
+	"SVG",
+	"Text",
+	"Markdown",
+}
+
 // GetLanguageStats calculates language stats for git repository at specified commit
 func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) {
 	r, err := git.PlainOpen(repo.Path)
@@ -43,7 +57,7 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 
 	sizes := make(map[string]int64)
 	err = tree.Files().ForEach(func(f *object.File) error {
-		if enry.IsVendor(f.Name) || enry.IsDotFile(f.Name) ||
+		if f.Size == 0 || enry.IsVendor(f.Name) || enry.IsDotFile(f.Name) ||
 			enry.IsDocumentation(f.Name) || enry.IsConfiguration(f.Name) {
 			return nil
 		}
@@ -58,7 +72,13 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 
 		language := analyze.GetCodeLanguage(f.Name, content)
 		if language == enry.OtherLanguage || language == "" {
-			language = "other"
+			return nil
+		}
+
+		// group languages, such as Pug -> HTML; SCSS -> CSS
+		group := enry.GetLanguageGroup(language)
+		if group != "" {
+			language = group
 		}
 
 		sizes[language] += f.Size
@@ -69,8 +89,11 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 		return nil, err
 	}
 
-	if len(sizes) == 0 {
-		sizes["other"] = 0
+	// filter special languages unless they are the only language
+	if len(sizes) > 1 {
+		for _, language := range specialLanguages {
+			delete(sizes, language)
+		}
 	}
 
 	return sizes, nil

modules/indexer/stats/indexer_test.go

@@ -39,7 +39,5 @@ func TestRepoStatsIndex(t *testing.T) {
 	assert.Equal(t, "65f1bf27bc3bf70f64657658635e66094edbcb4d", status.CommitSha)
 	langs, err := repo.GetTopLanguageStats(5)
 	assert.NoError(t, err)
-	assert.Len(t, langs, 1)
-	assert.Equal(t, "other", langs[0].Language)
-	assert.Equal(t, float32(100), langs[0].Percentage)
+	assert.Empty(t, langs)
 }