chore: replace github.com/go-testfixtures/testfixtures (#7715)

- Replaces `github.com/go-testfixtures/testfixtures` with a homebrew solution that is fully compatible.
- The reason to replace this library is that it pulls in a lot of other libraries which is causing issues: (1) the test binary becomes bigger than necessary which really shows in incremental build times (this patch removes 27.6MiB of the integration test binary) (2) it pulls in libraries (mainly database drivers) that are not used and are not easy to upgrade in case of a security vulnerability, causing CI failures.

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/7715
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
Co-authored-by: Gusted <postmaster@gusted.xyz>
Co-committed-by: Gusted <postmaster@gusted.xyz>

go.mod

@@ -48,7 +48,6 @@ require (
 	github.com/go-ldap/ldap/v3 v3.4.6
 	github.com/go-openapi/spec v0.20.14
 	github.com/go-sql-driver/mysql v1.9.1
-	github.com/go-testfixtures/testfixtures/v3 v3.14.0
 	github.com/go-webauthn/webauthn v0.12.3
 	github.com/gobwas/glob v0.2.3
 	github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f
@@ -118,21 +117,11 @@ require (
 )
 
 require (
-	cel.dev/expr v0.19.1 // indirect
-	cloud.google.com/go v0.116.0 // indirect
-	cloud.google.com/go/auth v0.9.9 // indirect
-	cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
 	cloud.google.com/go/compute/metadata v0.6.0 // indirect
-	cloud.google.com/go/iam v1.2.1 // indirect
-	cloud.google.com/go/longrunning v0.6.1 // indirect
-	cloud.google.com/go/monitoring v1.21.1 // indirect
-	cloud.google.com/go/spanner v1.73.0 // indirect
 	dario.cat/mergo v1.0.0 // indirect
 	filippo.io/edwards25519 v1.1.0 // indirect
 	git.sr.ht/~mariusor/go-xsd-duration v0.0.0-20220703122237-02e73435a078 // indirect
 	github.com/DataDog/zstd v1.5.5 // indirect
-	github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0 // indirect
-	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/RoaringBitmap/roaring/v2 v2.4.5 // indirect
 	github.com/andybalholm/brotli v1.1.1 // indirect
@@ -164,7 +153,6 @@ require (
 	github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cloudflare/circl v1.3.8 // indirect
-	github.com/cncf/xds/go v0.0.0-20241223141626-cff3c89139a3 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
 	github.com/cyphar/filepath-securejoin v0.3.6 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
@@ -173,10 +161,7 @@ require (
 	github.com/dlclark/regexp2 v1.11.5 // indirect
 	github.com/emersion/go-sasl v0.0.0-20231106173351-e73c9f7bad43 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
-	github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
-	github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
 	github.com/fatih/color v1.16.0 // indirect
-	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fxamacker/cbor/v2 v2.8.0 // indirect
 	github.com/go-ap/errors v0.0.0-20231003111023-183eef4b31b7 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
@@ -185,8 +170,6 @@ require (
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.6.2 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
-	github.com/go-logr/logr v1.4.2 // indirect
-	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.20.2 // indirect
 	github.com/go-openapi/jsonreference v0.20.4 // indirect
 	github.com/go-openapi/swag v0.22.7 // indirect
@@ -201,10 +184,6 @@ require (
 	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/go-tpm v0.9.3 // indirect
-	github.com/google/s2a-go v0.1.8 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
-	github.com/googleapis/gax-go/v2 v2.13.0 // indirect
-	github.com/googleapis/go-sql-spanner v1.7.4 // indirect
 	github.com/gorilla/css v1.0.1 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
 	github.com/gorilla/securecookie v1.1.2 // indirect
@@ -235,13 +214,13 @@ require (
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pjbgf/sha1cd v0.3.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.62.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/rhysd/actionlint v1.6.27 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/rogpeppe/go-internal v1.13.1 // indirect
 	github.com/rs/xid v1.6.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c // indirect
@@ -252,18 +231,9 @@ require (
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
+	github.com/zeebo/assert v1.3.0 // indirect
 	github.com/zeebo/blake3 v0.2.4 // indirect
 	go.etcd.io/bbolt v1.4.0 // indirect
-	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
-	go.opentelemetry.io/contrib/detectors/gcp v1.34.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
-	go.opentelemetry.io/otel v1.34.0 // indirect
-	go.opentelemetry.io/otel/metric v1.34.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.34.0 // indirect
-	go.opentelemetry.io/otel/sdk/metric v1.34.0 // indirect
-	go.opentelemetry.io/otel/trace v1.34.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
@@ -271,11 +241,6 @@ require (
 	golang.org/x/mod v0.24.0 // indirect
 	golang.org/x/time v0.10.0 // indirect
 	golang.org/x/tools v0.31.0 // indirect
-	google.golang.org/api v0.203.0 // indirect
-	google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect
-	google.golang.org/grpc v1.71.0 // indirect
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 )

models/unittest/fixture_loader.go

@@ -0,0 +1,198 @@
+// Copyright 2025 The Forgejo Authors. All rights reserved.
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package unittest
+
+import (
+	"database/sql"
+	"encoding/hex"
+	"encoding/json" //nolint:depguard
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+type insertSQL struct {
+	statement string
+	values    []any
+}
+
+type fixtureFile struct {
+	name       string
+	insertSQLs []insertSQL
+}
+
+type loader struct {
+	db      *sql.DB
+	dialect string
+
+	fixtureFiles []*fixtureFile
+}
+
+func newFixtureLoader(db *sql.DB, dialect string, fixturePaths []string) (*loader, error) {
+	l := &loader{
+		db:           db,
+		dialect:      dialect,
+		fixtureFiles: []*fixtureFile{},
+	}
+
+	// Load fixtures
+	for _, fixturePath := range fixturePaths {
+		stat, err := os.Stat(fixturePath)
+		if err != nil {
+			return nil, err
+		}
+
+		// If fixture path is a directory, then read read the files of the directory
+		// and use those as fixture files.
+		if stat.IsDir() {
+			files, err := os.ReadDir(fixturePath)
+			if err != nil {
+				return nil, err
+			}
+			for _, file := range files {
+				if !file.IsDir() {
+					fixtureFile, err := l.buildFixtureFile(filepath.Join(fixturePath, file.Name()))
+					if err != nil {
+						return nil, err
+					}
+					l.fixtureFiles = append(l.fixtureFiles, fixtureFile)
+				}
+			}
+		} else {
+			fixtureFile, err := l.buildFixtureFile(fixturePath)
+			if err != nil {
+				return nil, err
+			}
+			l.fixtureFiles = append(l.fixtureFiles, fixtureFile)
+		}
+	}
+
+	return l, nil
+}
+
+// quoteKeyword returns the quoted string of keyword.
+func (l *loader) quoteKeyword(keyword string) string {
+	switch l.dialect {
+	case "sqlite3":
+		return `"` + keyword + `"`
+	case "mysql":
+		return "`" + keyword + "`"
+	case "postgres":
+		parts := strings.Split(keyword, ".")
+		for i, p := range parts {
+			parts[i] = `"` + p + `"`
+		}
+		return strings.Join(parts, ".")
+	default:
+		return "invalid"
+	}
+}
+
+// placeholder returns the placeholder string.
+func (l *loader) placeholder(index int) string {
+	if l.dialect == "postgres" {
+		return fmt.Sprintf("$%d", index)
+	}
+	return "?"
+}
+
+func (l *loader) buildFixtureFile(fixturePath string) (*fixtureFile, error) {
+	f, err := os.Open(fixturePath)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	var records []map[string]any
+	if err := yaml.NewDecoder(f).Decode(&records); err != nil {
+		return nil, err
+	}
+
+	fixture := &fixtureFile{
+		name:       filepath.Base(strings.TrimSuffix(f.Name(), filepath.Ext(f.Name()))),
+		insertSQLs: []insertSQL{},
+	}
+
+	for _, record := range records {
+		columns := []string{}
+		sqlValues := []string{}
+		values := []any{}
+		i := 1
+
+		for key, value := range record {
+			columns = append(columns, l.quoteKeyword(key))
+
+			switch v := value.(type) {
+			case string:
+				// Try to decode hex.
+				if strings.HasPrefix(v, "0x") {
+					value, err = hex.DecodeString(strings.TrimPrefix(v, "0x"))
+					if err != nil {
+						return nil, err
+					}
+				}
+			case []any:
+				// Decode array.
+				var bytes []byte
+				bytes, err = json.Marshal(v)
+				if err != nil {
+					return nil, err
+				}
+				value = string(bytes)
+			}
+
+			values = append(values, value)
+
+			sqlValues = append(sqlValues, l.placeholder(i))
+			i++
+		}
+
+		// Construct the insert SQL.
+		fixture.insertSQLs = append(fixture.insertSQLs, insertSQL{
+			statement: fmt.Sprintf(
+				"INSERT INTO %s (%s) VALUES (%s)",
+				l.quoteKeyword(fixture.name),
+				strings.Join(columns, ", "),
+				strings.Join(sqlValues, ", "),
+			),
+			values: values,
+		})
+	}
+
+	return fixture, nil
+}
+
+func (l *loader) Load() error {
+	// Start transaction.
+	tx, err := l.db.Begin()
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		_ = tx.Rollback()
+	}()
+
+	// Clean the table and re-insert the fixtures.
+	tableDeleted := map[string]struct{}{}
+	for _, fixture := range l.fixtureFiles {
+		if _, ok := tableDeleted[fixture.name]; !ok {
+			if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %s", l.quoteKeyword(fixture.name))); err != nil {
+				return fmt.Errorf("cannot delete table %s: %w", fixture.name, err)
+			}
+			tableDeleted[fixture.name] = struct{}{}
+		}
+
+		for _, insertSQL := range fixture.insertSQLs {
+			if _, err := tx.Exec(insertSQL.statement, insertSQL.values...); err != nil {
+				return fmt.Errorf("cannot insert %q with values %q: %w", insertSQL.statement, insertSQL.values, err)
+			}
+		}
+	}
+
+	return tx.Commit()
+}

models/unittest/fixtures.go

@@ -6,7 +6,6 @@ package unittest
 
 import (
 	"fmt"
-	"os"
 	"path/filepath"
 	"time"
 
@@ -14,12 +13,11 @@ import (
 	"forgejo.org/modules/auth/password/hash"
 	"forgejo.org/modules/setting"
 
-	"github.com/go-testfixtures/testfixtures/v3"
 	"xorm.io/xorm"
 	"xorm.io/xorm/schemas"
 )
 
-var fixturesLoader *testfixtures.Loader
+var fixturesLoader *loader
 
 // GetXORMEngine gets the XORM engine
 func GetXORMEngine(engine ...*xorm.Engine) (x *xorm.Engine, err error) {
@@ -31,6 +29,7 @@ func GetXORMEngine(engine ...*xorm.Engine) (x *xorm.Engine, err error) {
 
 func OverrideFixtures(dir string) func() {
 	old := fixturesLoader
+
 	opts := FixturesOptions{
 		Dir:  filepath.Join(setting.AppWorkPath, "models/fixtures/"),
 		Base: setting.AppWorkPath,
@@ -39,6 +38,7 @@ func OverrideFixtures(dir string) func() {
 	if err := InitFixtures(opts); err != nil {
 		panic(err)
 	}
+
 	return func() {
 		fixturesLoader = old
 	}
@@ -50,19 +50,20 @@ func InitFixtures(opts FixturesOptions, engine ...*xorm.Engine) (err error) {
 	if err != nil {
 		return err
 	}
-	var fixtureOptionFiles func(*testfixtures.Loader) error
+
+	fixturePaths := []string{}
 	if opts.Dir != "" {
-		fixtureOptionFiles = testfixtures.Directory(opts.Dir)
+		fixturePaths = append(fixturePaths, opts.Dir)
 	} else {
-		fixtureOptionFiles = testfixtures.Files(opts.Files...)
+		fixturePaths = append(fixturePaths, opts.Files...)
 	}
-	var fixtureOptionDirs []func(*testfixtures.Loader) error
 	if opts.Dirs != nil {
 		for _, dir := range opts.Dirs {
-			fixtureOptionDirs = append(fixtureOptionDirs, testfixtures.Directory(filepath.Join(opts.Base, dir)))
+			fixturePaths = append(fixturePaths, filepath.Join(opts.Base, dir))
 		}
 	}
-	dialect := "unknown"
+
+	var dialect string
 	switch e.Dialect().URI().DBType {
 	case schemas.POSTGRES:
 		dialect = "postgres"
@@ -71,22 +72,10 @@ func InitFixtures(opts FixturesOptions, engine ...*xorm.Engine) (err error) {
 	case schemas.SQLITE:
 		dialect = "sqlite3"
 	default:
-		fmt.Println("Unsupported RDBMS for integration tests")
+		panic("Unsupported RDBMS for test")
-		os.Exit(1)
-	}
-	loaderOptions := []func(loader *testfixtures.Loader) error{
-		testfixtures.Database(e.DB().DB),
-		testfixtures.Dialect(dialect),
-		testfixtures.DangerousSkipTestDatabaseCheck(),
-		fixtureOptionFiles,
-	}
-	loaderOptions = append(loaderOptions, fixtureOptionDirs...)
-
-	if e.Dialect().URI().DBType == schemas.POSTGRES {
-		loaderOptions = append(loaderOptions, testfixtures.SkipResetSequences())
 	}
 
-	fixturesLoader, err = testfixtures.New(loaderOptions...)
+	fixturesLoader, err = newFixtureLoader(e.DB().DB, dialect, fixturePaths)
 	if err != nil {
 		return err
 	}