EdgeCommon/cmd/langs/main.go

// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/TeaOSLab/EdgeCommon/pkg/langs"
	"github.com/iwind/TeaGo/Tea"
	_ "github.com/iwind/TeaGo/bootstrap"
	"github.com/iwind/TeaGo/lists"
	"github.com/iwind/TeaGo/types"
	"go/format"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
)

func main() {
	var args = os.Args
	if len(args) >= 2 {
		switch args[1] {
		case "generate":
			// generate go codes from json files
			runGenerate()
		case "search":
			// search hans from dir path
			runSearch()
		}
	} else {
		fmt.Println("Usage: langs [generate|search]")
	}
}

func runGenerate() {
	var rootDir = filepath.Clean(Tea.Root + "/../pkg/langs/protos")
	dir, err := os.Open(rootDir)
	if err != nil {
		fmt.Println("[ERROR]read dir failed: " + err.Error())
		return
	}
	defer func() {
		_ = dir.Close()
	}()

	files, err := dir.Readdir(0)
	if err != nil {
		fmt.Println("[ERROR]read dir failed: " + err.Error())
		return
	}

	var dirRegexp = regexp.MustCompile(`^[a-z]+-[a-z]+$`)
	var jsonFileNameRegexp = regexp.MustCompile(`^([a-zA-Z0-9]+)(_([a-zA-Z0-9]+))*\.json$`)
	var messageCodeRegexp = regexp.MustCompile(`^[a-zA-Z0-9_]+$`)
	var jsonCommentRegexp = regexp.MustCompile(`//\s+.+`)

	var messageCodes = []string{}
	var langMaps = map[string]*langs.Lang{} // lang => *langs.Lang
	var defaultLang = langs.DefaultManager().DefaultLang()

	const maxMessageCodeLen = 128

	for _, file := range files {
		var dirName = file.Name()

		if !file.IsDir() || !dirRegexp.MatchString(dirName) {
			continue
		}
		var langCode = dirName
		var isBaseLang = langCode == defaultLang

		var processOk = func() bool {
			jsonFiles, err := filepath.Glob(rootDir + "/" + dirName + "/*.json")
			if err != nil {
				fmt.Println("[ERROR]list json files failed: " + err.Error())
				return false
			}

			for _, jsonFile := range jsonFiles {
				var jsonFileName = filepath.Base(jsonFile)
				if len(jsonFileName) == 0 || !jsonFileNameRegexp.MatchString(jsonFileName) {
					continue
				}
				var module = strings.TrimSuffix(jsonFileName, ".json")

				data, err := os.ReadFile(jsonFile)
				if err != nil {
					fmt.Println("[ERROR]read json file '" + jsonFile + "' failed: " + err.Error())
					return false
				}

				// remove comments in json
				data = jsonCommentRegexp.ReplaceAll(data, []byte{})

				var m = map[string]string{} // code => value
				err = json.Unmarshal(data, &m)
				if err != nil {
					fmt.Println("[ERROR]decode json file '" + jsonFile + "' failed: " + err.Error())
					return false
				}

				var newM = map[string]string{}
				for code, value := range m {
					if !messageCodeRegexp.MatchString(code) {
						fmt.Println("[ERROR]invalid message code '" + code + "'")
						return false
					}

					var fullCode = module + "@" + code

					if len(fullCode) > maxMessageCodeLen {
						fmt.Println("[ERROR]message code '" + fullCode + "' too long, max length: " + types.String(maxMessageCodeLen))
						return false
					}

					if isBaseLang {
						messageCodes = append(messageCodes, fullCode)
					}
					newM[fullCode] = value
				}

				finalLang, ok := langMaps[langCode]
				if !ok {
					finalLang = langs.NewLang(langCode)
					langMaps[langCode] = finalLang
				}
				for code, value := range newM {
					if finalLang.Has(langs.MessageCode(code)) {
						fmt.Println("[ERROR]message code '" + code + "' duplicated")
						return false
					}
					finalLang.Set(langs.MessageCode(code), value)
				}
			}

			return true
		}()
		if !processOk {
			return
		}
	}

	// compile
	for langCode, lang := range langMaps {
		err = lang.Compile()
		if err != nil {
			fmt.Println("[ERROR]compile '" + langCode + "' failed: " + err.Error())
			return
		}
	}

	// check message codes
	fmt.Println("checking message codes ...")
	var defaultMessageMap = map[langs.MessageCode]string{}
	for langCode, messageLang := range langMaps {
		if langCode == defaultLang { // only check lang not equal to 'en-us'
			defaultMessageMap = messageLang.GetAll()
			continue
		}

		for messageCode := range messageLang.GetAll() {
			if !lists.ContainsString(messageCodes, messageCode.String()) {
				fmt.Println("[ERROR]message code '" + messageCode.String() + "' in lang '" + langCode + "' not exist in default lang file ('" + defaultLang + "')")
				return
			}
		}
	}
	fmt.Println("found '" + types.String(len(messageCodes)) + "' message codes")

	// generate codes/codes.go
	sort.Strings(messageCodes)
	var codesSource = `
// generated by run 'langs generate'

package codes

import(
	"github.com/TeaOSLab/EdgeCommon/pkg/langs"
)

const (
	`

	for index, messageCode := range messageCodes {
		// add comment to message code
		comment, _, _ := strings.Cut(defaultMessageMap[langs.MessageCode(messageCode)], "\n")

		codesSource += upperWords(messageCode) + " langs.MessageCode = " + strconv.Quote(messageCode) + " // " + comment

		// add NL
		if index != len(messageCodes)-1 {
			codesSource += "\n"
		}
	}

	codesSource += `
)
`

	formattedCodesSource, err := format.Source([]byte(codesSource))
	if err != nil {
		fmt.Println("[ERROR]format 'codes.go' failed: " + err.Error())
		return
	}

	fmt.Println("generating 'codes/codes.go' ...")
	err = os.WriteFile(Tea.Root+"/../pkg/langs/codes/codes.go", formattedCodesSource, 0666)
	if err != nil {
		fmt.Println("[ERROR]write to 'codes.go' failed: " + err.Error())
		return
	}

	// generate messages_LANG.go
	for langCode, messageLang := range langMaps {
		var langFile = strings.ReplaceAll("messages/messages_"+langCode+".go", "-", "_")

		fmt.Println("generating '" + langFile + "' ...")
		var source = `
// generated by run 'langs generate'

package messages

import(
	"github.com/TeaOSLab/EdgeCommon/pkg/langs"
)

func init() {
	langs.Load("` + langCode + `", map[langs.MessageCode]string{
	`

		for _, code := range messageCodes {
			var value = messageLang.Get(langs.MessageCode(code))
			source += strconv.Quote(code) + ": " + strconv.Quote(value) + ",\n"
		}

		source += `
	})
}
`

		formattedSource, err := format.Source([]byte(source))
		if err != nil {
			fmt.Println("[ERROR]format '" + langFile + "' failed: " + err.Error())
			return
		}

		err = os.WriteFile(Tea.Root+"/../pkg/langs/"+langFile, formattedSource, 0666)
		if err != nil {
			fmt.Println("[ERROR]write file '" + langFile + "' failed: " + err.Error())
			return
		}
	}

	// generate language javascript files for EdgeAdmin and EdgeUser (commercial versions)
	for lang, messageLang := range langMaps {
		if lang != defaultLang {
			// TODO merge messageMap with default message map
		}

		for _, component := range []string{"EdgeAdmin", "EdgeUser"} {
			fmt.Println("generating '" + lang + ".js' for " + component + " ...")
			var targetFile = filepath.Clean(Tea.Root + "/../../" + component + "/web/public/js/langs/" + lang + ".js")
			var targetDir = filepath.Dir(targetFile)
			dirStat, _ := os.Stat(targetDir)
			if dirStat != nil {
				var prefix = ""
				switch component {
				case "EdgeAdmin":
					prefix = "admin_"
				case "EdgeUser":
					prefix = "user_"
				}
				if len(prefix) == 0 {
					continue
				}

				var filteredMessages = map[langs.MessageCode]string{}
				for code, value := range messageLang.GetAll() {
					if strings.HasPrefix(code.String(), prefix) && strings.Contains(code.String(), "@ui_") /** must contains 'ui' **/ {
						filteredMessages[code] = value
					}
				}

				messageMapJSON, jsonErr := json.Marshal(filteredMessages)
				if jsonErr != nil {
					fmt.Println("[ERROR]marshal message map failed: " + jsonErr.Error())
					return
				}
				err = os.WriteFile(targetFile, []byte(`// generated by 'langs generate'
window.LANG_MESSAGES = `+string(messageMapJSON)+";\n"), 0666)
				if err != nil {
					fmt.Println("[ERROR]write file '" + targetFile + "' failed: " + err.Error())
					return
				}

				// base.js
				if lang == "zh-cn" {
					var baseJSFile = filepath.Dir(targetFile) + "/base.js"
					err = os.WriteFile(baseJSFile, []byte(`// generated by 'langs generate'
window.LANG_MESSAGES_BASE = `+string(messageMapJSON)+";\n"), 0666)
					if err != nil {
						fmt.Println("[ERROR]write file '" + baseJSFile + "' failed: " + err.Error())
						return
					}
				}
			}
		}
	}

	fmt.Println("success")
}

func upperWords(s string) string {
	var pieces = strings.Split(s, "@")
	for pieceIndex, piece := range pieces {
		var words = strings.Split(piece, "_")
		for index, word := range words {
			words[index] = upperWord(word)
		}
		pieces[pieceIndex] = strings.Join(words, "")
	}
	return strings.Join(pieces, "_")
}

func upperWord(word string) string {
	var l = len(word)
	if l == 0 {
		return ""
	}

	if l == 1 {
		return strings.ToUpper(word)
	}

	// special words
	switch word {
	case "api", "http", "https", "tcp", "tls", "ssl", "udp", "ip", "dns", "ns",
		"waf", "acme", "ssh", "toa", "http2", "http3", "uam", "cc",
		"db", "isp", "sni", "ui", "soa", "ocsp", "en", "zh", "ad", "tsig",
		"rpc", "dao":
		return strings.ToUpper(word)
	case "ipv6":
		return "IPv6"
	case "ddos":
		return "DDoS"
	case "webp":
		return "WebP"
	case "doh":
		return "DoH"
	}

	return strings.ToUpper(word[:1]) + word[1:]
}

func runSearch() {
	if len(os.Args) < 3 {
		fmt.Println("Usage: langs search DIR")
		return
	}
	var dir = os.Args[2]
	stat, err := os.Stat(dir)
	if err != nil {
		fmt.Println("[ERROR]could not find dir '" + dir + "': " + err.Error())
		return
	}
	if !stat.IsDir() {
		fmt.Println("[ERROR]could not find dir '" + dir + "'")
		return
	}

	fmt.Println("searching '" + dir + "' ...")

	var ext = ".go"

	var resultFiles = []string{}
	for _, pattern := range []string{
		"*" + ext,
		strings.Repeat("*/", 1) + "*" + ext,
		strings.Repeat("*/", 2) + "*" + ext,
		strings.Repeat("*/", 3) + "*" + ext,
		strings.Repeat("*/", 4) + "*" + ext,
		strings.Repeat("*/", 5) + "*" + ext,
		strings.Repeat("*/", 6) + "*" + ext,
		strings.Repeat("*/", 7) + "*" + ext,
		strings.Repeat("*/", 8) + "*" + ext,
		strings.Repeat("*/", 9) + "*" + ext,
		strings.Repeat("*/", 10) + "*" + ext,
	} {
		goFiles, err := filepath.Glob(dir + "/" + pattern)
		if err != nil {
			fmt.Println("[ERROR]search error: " + err.Error())
			return
		}
		resultFiles = append(resultFiles, goFiles...)
	}

	if len(resultFiles) == 0 {
		fmt.Println("no files found in the dir")
		return
	}

	if err != nil {
		fmt.Println("[ERROR]search dir '" + dir + "' failed: " + err.Error())
		return
	}
	var hansRegexp = regexp.MustCompile(`\p{Han}+`)
	var countMatches = 0
	for _, goFile := range resultFiles {
		if strings.HasSuffix(goFile, "_test.go") ||
			strings.HasSuffix(goFile, "_plus_test.go") ||
			strings.Contains(goFile, "/messages/messages_") {
			continue
		}

		data, err := os.ReadFile(goFile)
		if err != nil {
			fmt.Println("[ERROR]read file '" + goFile + "' failed: " + err.Error())
			return
		}
		var matches = hansRegexp.FindAllSubmatchIndex(data, -1)
		if len(matches) > 0 {
			for _, match := range matches {
				// ignore comment
				switch ext {
				case ".go":
					if checkIsInGoComment(data, match[0]) {
						continue
					}
				}

				countMatches++

				fmt.Printf("%s %s\n", goFile+":"+types.String(bytes.Count(data[:match[0]], []byte{'\n'})+1), string(data[match[0]:match[1]]))
			}
		}
	}
	fmt.Println(countMatches, "matches")
}

func checkIsInGoComment(data []byte, start int) bool {
	if start <= 1 {
		return false
	}

	for {
		start--
		if start <= 1 || data[start] == '\n' {
			return false
		}

		// 'SPACE //'
		if data[start] == '/' && data[start-1] == '/' {
			return true
		}

		// '/** SOMETHING **/'
		if data[start] == '*' && data[start-1] == '/' {
			return true
		}
	}
}