fine tune queue flush

Author: wxiaoguang

modules/queue/manager.go

@@ -5,13 +5,12 @@ package queue
 
 import (
 	"context"
+	"errors"
 	"sync"
 	"time"
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
-
-	"golang.org/x/sync/errgroup"
 )
 
 // Manager is a manager for the queues created by "CreateXxxQueue" functions, these queues are called "managed queues".
@@ -34,6 +33,7 @@ type ManagedWorkerPoolQueue interface {
 
 	// FlushWithContext tries to make the handler process all items in the queue synchronously.
 	// It is for testing purpose only. It's not designed to be used in a cluster.
+	// Negative timeout means discarding all items in the queue.
 	FlushWithContext(ctx context.Context, timeout time.Duration) error
 
 	// RemoveAllItems removes all items in the base queue (on-the-fly items are not affected)
@@ -78,15 +78,16 @@ func (m *Manager) ManagedQueues() map[int64]ManagedWorkerPoolQueue {
 
 // FlushAll tries to make all managed queues process all items synchronously, until timeout or the queue is empty.
 // It is for testing purpose only. It's not designed to be used in a cluster.
+// Negative timeout means discarding all items in the queue.
 func (m *Manager) FlushAll(ctx context.Context, timeout time.Duration) error {
-	g := errgroup.Group{}
+	var finalErrors []error
 	qs := m.ManagedQueues()
 	for _, q := range qs {
-		g.Go(func() error {
-			return q.FlushWithContext(ctx, timeout)
-		})
+		if err := q.FlushWithContext(ctx, timeout); err != nil {
+			finalErrors = append(finalErrors, err)
+		}
 	}
-	return g.Wait()
+	return errors.Join(finalErrors...)
 }
 
 // CreateSimpleQueue creates a simple queue from global setting config provider by name

modules/queue/workergroup.go

@@ -197,15 +197,30 @@ func (q *WorkerPoolQueue[T]) doFlush(wg *workerGroup[T], flush flushType) {
 	defer log.Debug("Queue %q finishes flushing", q.GetName())
 
 	// stop all workers, and prepare a new worker context to start new workers
-
 	wg.ctxWorkerCancel()
 	wg.wg.Wait()
 
 	defer func() {
-		close(flush)
+		close(flush.c)
 		wg.doPrepareWorkerContext()
 	}()
 
+	if flush.timeout < 0 {
+		// discard everything
+		wg.batchBuffer = nil
+		for {
+			select {
+			case _ = <-wg.popItemChan:
+			case _ = <-wg.popItemErr:
+			case _ = <-q.batchChan:
+			case <-q.ctxRun.Done():
+				return
+			default:
+				return
+			}
+		}
+	}
+
 	// drain the batch channel first
 loop:
 	for {

modules/queue/workerqueue.go

@@ -42,7 +42,10 @@ type WorkerPoolQueue[T any] struct {
 	workerNumMu     sync.Mutex
 }
 
-type flushType chan struct{}
+type flushType struct {
+	timeout time.Duration
+	c       chan struct{}
+}
 
 var _ ManagedWorkerPoolQueue = (*WorkerPoolQueue[any])(nil)
 
@@ -104,12 +107,12 @@ func (q *WorkerPoolQueue[T]) FlushWithContext(ctx context.Context, timeout time.
 	if timeout > 0 {
 		after = time.After(timeout)
 	}
-	c := make(flushType)
+	flush := flushType{timeout: timeout, c: make(chan struct{})}
 
 	// send flush request
 	// if it blocks, it means that there is a flush in progress or the queue hasn't been started yet
 	select {
-	case q.flushChan <- c:
+	case q.flushChan <- flush:
 	case <-ctx.Done():
 		return ctx.Err()
 	case <-q.ctxRun.Done():
@@ -120,7 +123,7 @@ func (q *WorkerPoolQueue[T]) FlushWithContext(ctx context.Context, timeout time.
 
 	// wait for flush to finish
 	select {
-	case <-c:
+	case <-flush.c:
 		return nil
 	case <-ctx.Done():
 		return ctx.Err()

modules/testlogger/testlogger.go

@@ -118,7 +118,7 @@ func PrintCurrentTest(t testing.TB, skip ...int) func() {
 				_, _ = fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", t.Name(), SlowFlush)
 			}
 		})
-		if err := queue.GetManager().FlushAll(context.Background(), time.Minute); err != nil {
+		if err := queue.GetManager().FlushAll(context.Background(), -1); err != nil {
 			t.Errorf("Flushing queues failed with error %v", err)
 		}
 		timer.Stop()