Fix OPA deadlock while stopping bundle plugin

This commit fixes couple of issues that could result
in blocking OPA:

1) When the bundle plugin attempts to stop the bundle
downloader, it first grabs the lock on the plugin and then
stops the downloader. The downloader in-turn calls the plugin’s
callback function which now waits for the lock to be released
by the plugin's stop function. This results in a deadlock.

This commit fixes this issue by making sure the plugin's stop
function releases the lock before stopping the downloader.

2) Another issue that could block OPA is when the stop function
on the same downloader gets called multiple times.

Fixes: #3363

Signed-off-by: Ashutosh Narkar <anarkar4387@gmail.com>

download/download.go

@@ -54,6 +54,8 @@ type Downloader struct {
 	respHdrTimeoutSec int64
 	wg                sync.WaitGroup
 	logger            logging.Logger
+	mtx               sync.Mutex
+	stopped           bool
 }
 
 // New returns a new Downloader that can be started.
@@ -112,11 +114,19 @@ func (d *Downloader) doStart(context.Context) {
 	done := <-d.stop // blocks until there's something to read
 	cancel()
 	d.wg.Wait()
+	d.stopped = true
 	close(done)
 }
 
 // Stop tells the Downloader to stop downloading bundles.
 func (d *Downloader) Stop(context.Context) {
+	d.mtx.Lock()
+	defer d.mtx.Unlock()
+
+	if d.stopped {
+		return
+	}
+
 	done := make(chan struct{})
 	d.stop <- done
 	<-done

download/download_test.go

@@ -56,6 +56,46 @@ func TestStartStop(t *testing.T) {
 	d.Stop(ctx)
 }
 
+func TestStopWithMultipleCalls(t *testing.T) {
+	ctx := context.Background()
+	fixture := newTestFixture(t)
+
+	updates := make(chan *Update)
+
+	config := Config{}
+	if err := config.ValidateAndInjectDefaults(); err != nil {
+		t.Fatal(err)
+	}
+
+	d := New(config, fixture.client, "/bundles/test/bundle1").WithCallback(func(_ context.Context, u Update) {
+		updates <- &u
+	})
+
+	d.Start(ctx)
+
+	// Give time for some download events to occur
+	time.Sleep(1 * time.Second)
+
+	u1 := <-updates
+
+	if u1.Bundle == nil || len(u1.Bundle.Modules) == 0 {
+		t.Fatal("expected bundle with at least one module but got:", u1)
+	}
+
+	done := make(chan struct{})
+	go func() {
+		d.Stop(ctx)
+		close(done)
+	}()
+
+	d.Stop(ctx)
+	<-done
+
+	if !d.stopped {
+		t.Fatal("expected downloader to be stopped")
+	}
+}
+
 func TestStartStopWithLongPollNotSupported(t *testing.T) {
 	ctx := context.Background()