package resource

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/google/uuid"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"

	"github.com/grafana/grafana/pkg/apimachinery/utils"
	"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
	"github.com/grafana/grafana/pkg/util/testutil"
	"github.com/stretchr/testify/require"
)

// writeEventOption is a function that modifies writeEventOptions
type writeEventOption func(*writeEventOptions)

type writeEventOptions struct {
	Namespace  string
	Group      string
	Resource   string
	Folder     string
	Value      []byte
	PreviousRV int64
}

func writeEvent(t *testing.T, ctx context.Context, storageBackend *kvStorageBackend, resourceName string, action resourcepb.WatchEvent_Type, opts ...writeEventOption) (int64, error) {
	// Default options
	options := writeEventOptions{
		Namespace: "namespace",
		Group:     "group",
		Resource:  "resource",
		Folder:    "folderuid",
	}

	// Apply options
	for _, opt := range opts {
		opt(&options)
	}

	u := unstructured.Unstructured{
		Object: map[string]any{
			"apiVersion": options.Group + "/v1",
			"kind":       options.Resource,
			"metadata": map[string]any{
				"name":      resourceName,
				"namespace": options.Namespace,
			},
			"spec": map[string]any{
				"value": resourceName + " " + resourcepb.WatchEvent_Type_name[int32(action)],
			},
		},
	}
	value, err := u.MarshalJSON()
	require.NoError(t, err)

	res := &unstructured.Unstructured{
		Object: map[string]any{},
	}
	meta, err := utils.MetaAccessor(res)
	require.NoError(t, err)
	meta.SetFolder(options.Folder)

	event := WriteEvent{
		Type:  action,
		Value: value,
		GUID:  uuid.New().String(),
		Key: &resourcepb.ResourceKey{
			Namespace: options.Namespace,
			Group:     options.Group,
			Resource:  options.Resource,
			Name:      resourceName,
		},
		PreviousRV: options.PreviousRV,
	}

	switch action {
	case resourcepb.WatchEvent_DELETED:
		event.ObjectOld = meta

		obj, err := utils.MetaAccessor(res)
		if err != nil {
			return 0, err
		}
		now := metav1.Now()
		obj.SetDeletionTimestamp(&now)
		obj.SetUpdatedTimestamp(&now.Time)
		obj.SetManagedFields(nil)
		obj.SetFinalizers(nil)
		obj.SetGeneration(utils.DeletedGeneration)
		obj.SetAnnotation(utils.AnnoKeyKubectlLastAppliedConfig, "") // clears it
		event.Object = obj
	case resourcepb.WatchEvent_ADDED:
		event.Object = meta
	case resourcepb.WatchEvent_MODIFIED:
		event.Object = meta //
		event.ObjectOld = meta
	default:
		panic(fmt.Sprintf("invalid action: %s", action))
	}

	return storageBackend.WriteEvent(ctx, event)
}

func TestIntegrationGarbageCollectionGroupResource(t *testing.T) {
	gcConfig := GarbageCollectionConfig{
		Enabled:          true,
		DryRun:           false,
		Interval:         time.Minute,
		BatchSize:        100,
		DashboardsMaxAge: 24 * time.Hour,
	}

	t.Run("can garbage collect a deleted resource", func(t *testing.T) {
		testutil.SkipIntegrationTestInShortMode(t)

		ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))

		storageBackend := setupTestStorageBackend(t, func(opts *KVBackendOptions) {
			opts.GarbageCollection = gcConfig
		})
		b := storageBackend

		server, err := NewResourceServer(ResourceServerOptions{
			Backend: storageBackend,
		})
		require.NoError(t, err)
		t.Cleanup(func() {
			ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
			defer cancel()
			_ = server.Stop(ctx)
		})

		rv1, err := writeEvent(t, ctx, storageBackend, "resource1", resourcepb.WatchEvent_ADDED)
		require.NoError(t, err)

		_, err = writeEvent(t, ctx, storageBackend, "resource1", resourcepb.WatchEvent_DELETED,
			func(o *writeEventOptions) {
				o.PreviousRV = rv1
			})
		require.NoError(t, err)

		listResp, err := server.List(ctx, &resourcepb.ListRequest{
			Source: resourcepb.ListRequest_STORE,
			Options: &resourcepb.ListOptions{
				Key: &resourcepb.ResourceKey{
					Namespace: "namespace",
					Group:     "group",
					Resource:  "resource",
				},
			},
		})
		require.NoError(t, err)
		require.Empty(t, listResp.Items)

		cutoffTimestamp := b.garbageCollectionCutoffTimestamp("group", "resource", time.Now().Add(time.Hour).UnixMicro()) // Everything eligible for deletion
		err = b.garbageCollectGroupResource(ctx, "group", "resource", cutoffTimestamp)
		require.NoError(t, err)

		// count how many history entries there are after GC runs - should be 0
		historyResp := storageBackend.kv.Keys(ctx, dataSection, ListOptions{
			StartKey: "group/resource/namespace/",
			EndKey:   "group/resource/namespace0",
		})
		count := 0
		historyResp(func(k string, err error) bool {
			require.NoError(t, err)
			count++
			return true
		})
		require.Equal(t, 0, count)
	})

	t.Run("will only garbage collect eligible resources before cutoff", func(t *testing.T) {
		testutil.SkipIntegrationTestInShortMode(t)

		ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))

		storageBackend := setupTestStorageBackend(t, func(opts *KVBackendOptions) {
			opts.GarbageCollection = gcConfig
		})
		b := storageBackend

		server, err := NewResourceServer(ResourceServerOptions{
			Backend: storageBackend,
		})
		require.NoError(t, err)
		t.Cleanup(func() {
			ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
			defer cancel()
			_ = server.Stop(ctx)
		})

		rv1, err := writeEvent(t, ctx, storageBackend, "resource1", resourcepb.WatchEvent_ADDED)
		require.NoError(t, err)
		rv2, err := writeEvent(t, ctx, storageBackend, "resource1", resourcepb.WatchEvent_DELETED,
			func(o *writeEventOptions) {
				o.PreviousRV = rv1
			})
		require.NoError(t, err)

		rv3, err := writeEvent(t, ctx, storageBackend, "resource2", resourcepb.WatchEvent_ADDED)
		require.NoError(t, err)
		_, err = writeEvent(t, ctx, storageBackend, "resource2", resourcepb.WatchEvent_DELETED,
			func(o *writeEventOptions) {
				o.PreviousRV = rv3
			})
		require.NoError(t, err)

		cutoffTimestamp := b.garbageCollectionCutoffTimestamp("group", "resource", rv2+1)
		err = b.garbageCollectGroupResource(ctx, "group", "resource", cutoffTimestamp)
		require.NoError(t, err)

		historyResp, err := server.List(ctx, &resourcepb.ListRequest{
			Source: resourcepb.ListRequest_HISTORY,
			Options: &resourcepb.ListOptions{
				Key: &resourcepb.ResourceKey{
					Namespace: "namespace",
					Group:     "group",
					Resource:  "resource",
					Name:      "resource1",
				},
			},
		})
		require.NoError(t, err)
		require.Nil(t, historyResp.Error)
		require.Len(t, historyResp.Items, 0)

		historyResp, err = server.List(ctx, &resourcepb.ListRequest{
			Source: resourcepb.ListRequest_TRASH,
			Options: &resourcepb.ListOptions{
				Key: &resourcepb.ResourceKey{
					Namespace: "namespace",
					Group:     "group",
					Resource:  "resource",
					Name:      "resource2",
				},
			},
		})
		require.NoError(t, err)
		require.Nil(t, historyResp.Error)
		require.Len(t, historyResp.Items, 1)
	})

	t.Run("will not delete rows for other eligible resources", func(t *testing.T) {
		testutil.SkipIntegrationTestInShortMode(t)

		ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))

		storageBackend := setupTestStorageBackend(t, func(opts *KVBackendOptions) {
			opts.GarbageCollection = gcConfig
		})
		b := storageBackend

		server, err := NewResourceServer(ResourceServerOptions{
			Backend: storageBackend,
		})
		require.NoError(t, err)
		t.Cleanup(func() {
			ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
			defer cancel()
			_ = server.Stop(ctx)
		})

		rv1, err := writeEvent(t, ctx, storageBackend, "resource1", resourcepb.WatchEvent_ADDED)
		require.NoError(t, err)
		_, err = writeEvent(t, ctx, storageBackend, "resource1", resourcepb.WatchEvent_DELETED,
			func(o *writeEventOptions) {
				o.PreviousRV = rv1
			})
		require.NoError(t, err)

		rv2, err := writeEvent(t, ctx, storageBackend, "resource1", resourcepb.WatchEvent_ADDED,
			func(o *writeEventOptions) {
				o.Resource = "other-resource"
			})
		require.NoError(t, err)
		_, err = writeEvent(t, ctx, storageBackend, "resource1", resourcepb.WatchEvent_DELETED,
			func(o *writeEventOptions) {
				o.PreviousRV = rv2
				o.Resource = "other-resource"
			})
		require.NoError(t, err)

		cutoffTimestamp := b.garbageCollectionCutoffTimestamp("group", "resource", time.Now().Add(time.Hour).UnixMicro()) // everything eligible for deletion
		err = b.garbageCollectGroupResource(ctx, "group", "resource", cutoffTimestamp)
		require.NoError(t, err)

		historyResp, err := server.List(ctx, &resourcepb.ListRequest{
			Source: resourcepb.ListRequest_HISTORY,
			Options: &resourcepb.ListOptions{
				Key: &resourcepb.ResourceKey{
					Namespace: "namespace",
					Group:     "group",
					Resource:  "resource",
					Name:      "resource1",
				},
			},
		})
		require.NoError(t, err)
		require.Nil(t, historyResp.Error)
		require.Len(t, historyResp.Items, 0)

		trashResp, err := server.List(ctx, &resourcepb.ListRequest{
			Source: resourcepb.ListRequest_TRASH,
			Options: &resourcepb.ListOptions{
				Key: &resourcepb.ResourceKey{
					Namespace: "namespace",
					Group:     "group",
					Resource:  "other-resource",
					Name:      "resource1",
				},
			},
		})
		require.NoError(t, err)
		require.Nil(t, trashResp.Error)
		require.Len(t, trashResp.Items, 1)
	})

	t.Run("will delete resources in multiple batches", func(t *testing.T) {
		testutil.SkipIntegrationTestInShortMode(t)

		ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))

		storageBackend := setupTestStorageBackend(t, func(opts *KVBackendOptions) {
			opts.GarbageCollection = gcConfig
		})
		b := storageBackend

		server, err := NewResourceServer(ResourceServerOptions{
			Backend: storageBackend,
		})
		require.NoError(t, err)
		t.Cleanup(func() {
			ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
			defer cancel()
			_ = server.Stop(ctx)
		})

		rv1, err := writeEvent(t, ctx, storageBackend, "resource1", resourcepb.WatchEvent_ADDED)
		require.NoError(t, err)
		_, err = writeEvent(t, ctx, storageBackend, "resource1", resourcepb.WatchEvent_DELETED,
			func(o *writeEventOptions) {
				o.PreviousRV = rv1
			})
		require.NoError(t, err)

		rv2, err := writeEvent(t, ctx, storageBackend, "resource2", resourcepb.WatchEvent_ADDED)
		require.NoError(t, err)
		_, err = writeEvent(t, ctx, storageBackend, "resource2", resourcepb.WatchEvent_DELETED,
			func(o *writeEventOptions) {
				o.PreviousRV = rv2
			})
		require.NoError(t, err)

		trashResp, err := server.List(ctx, &resourcepb.ListRequest{
			Source: resourcepb.ListRequest_TRASH,
			Options: &resourcepb.ListOptions{
				Key: &resourcepb.ResourceKey{
					Namespace: "namespace",
					Group:     "group",
					Resource:  "resource",
				},
			},
		})
		require.NoError(t, err)
		require.Nil(t, trashResp.Error)
		require.Len(t, trashResp.Items, 2)

		cutoffTimestamp := b.garbageCollectionCutoffTimestamp("group", "resource", time.Now().Add(time.Hour).UnixMicro()) // everything eligible for deletion
		b.garbageCollection.BatchSize = 1
		err = b.garbageCollectGroupResource(ctx, "group", "resource", cutoffTimestamp)
		require.NoError(t, err)

		trashResp, err = server.List(ctx, &resourcepb.ListRequest{
			Source: resourcepb.ListRequest_TRASH,
			Options: &resourcepb.ListOptions{
				Key: &resourcepb.ResourceKey{
					Namespace: "namespace",
					Group:     "group",
					Resource:  "resource",
				},
			},
		})
		require.NoError(t, err)
		require.Nil(t, trashResp.Error)
		require.Empty(t, trashResp.Items)
	})

	t.Run("will delete rows from before the resource gets deleted, but it will keep rows from after the resource gets recreated with same name", func(t *testing.T) {
		testutil.SkipIntegrationTestInShortMode(t)

		ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))

		storageBackend := setupTestStorageBackend(t, func(opts *KVBackendOptions) {
			opts.GarbageCollection = gcConfig
		})
		b := storageBackend

		server, err := NewResourceServer(ResourceServerOptions{
			Backend: storageBackend,
		})
		require.NoError(t, err)
		t.Cleanup(func() {
			ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
			defer cancel()
			_ = server.Stop(ctx)
		})

		rv1, err := writeEvent(t, ctx, storageBackend, "resource1", resourcepb.WatchEvent_ADDED)
		require.NoError(t, err)

		_, err = writeEvent(t, ctx, storageBackend, "resource1", resourcepb.WatchEvent_DELETED,
			func(o *writeEventOptions) {
				o.PreviousRV = rv1
			})
		require.NoError(t, err)

		_, err = writeEvent(t, ctx, storageBackend, "resource1", resourcepb.WatchEvent_ADDED)
		require.NoError(t, err)

		cutoffTimestamp := b.garbageCollectionCutoffTimestamp("group", "resource", time.Now().Add(time.Hour).UnixMicro()) // everything eligible for deletion
		err = b.garbageCollectGroupResource(ctx, "group", "resource", cutoffTimestamp)
		require.NoError(t, err)

		historyResp, err := server.List(ctx, &resourcepb.ListRequest{
			Source: resourcepb.ListRequest_HISTORY,
			Options: &resourcepb.ListOptions{
				Key: &resourcepb.ResourceKey{
					Namespace: "namespace",
					Group:     "group",
					Resource:  "resource",
					Name:      "resource1",
				},
			},
		})
		require.NoError(t, err)
		require.Nil(t, historyResp.Error)
		require.Len(t, historyResp.Items, 1)
	})

	t.Run("pagination does not delete resources that were recreated", func(t *testing.T) {
		testutil.SkipIntegrationTestInShortMode(t)

		ctx := testutil.NewTestContext(t, time.Now().Add(30*time.Second))

		cfg := gcConfig
		cfg.BatchSize = 3
		storageBackend := setupTestStorageBackend(t, func(opts *KVBackendOptions) {
			opts.GarbageCollection = cfg
		})
		b := storageBackend

		// other-dash: 2 keys (created, deleted)
		rv1, err := writeEvent(t, ctx, storageBackend, "other-dash", resourcepb.WatchEvent_ADDED)
		require.NoError(t, err)
		_, err = writeEvent(t, ctx, storageBackend, "other-dash", resourcepb.WatchEvent_DELETED,
			func(o *writeEventOptions) { o.PreviousRV = rv1 })
		require.NoError(t, err)

		// my-dash: 5 keys (created, updated, deleted, created again and updated again) — first batch ends with a my-dash key
		rv1, err = writeEvent(t, ctx, storageBackend, "my-dash", resourcepb.WatchEvent_ADDED)
		require.NoError(t, err)
		rv2, err := writeEvent(t, ctx, storageBackend, "my-dash", resourcepb.WatchEvent_MODIFIED,
			func(o *writeEventOptions) { o.PreviousRV = rv1 })
		require.NoError(t, err)
		_, err = writeEvent(t, ctx, storageBackend, "my-dash", resourcepb.WatchEvent_DELETED,
			func(o *writeEventOptions) { o.PreviousRV = rv2 })
		require.NoError(t, err)
		rv4, err := writeEvent(t, ctx, storageBackend, "my-dash", resourcepb.WatchEvent_ADDED)
		require.NoError(t, err)
		_, err = writeEvent(t, ctx, storageBackend, "my-dash", resourcepb.WatchEvent_MODIFIED,
			func(o *writeEventOptions) { o.PreviousRV = rv4 })
		require.NoError(t, err)

		cutoffTimestamp := b.garbageCollectionCutoffTimestamp("group", "resource", time.Now().Add(time.Hour).UnixMicro())
		err = b.garbageCollectGroupResource(ctx, "group", "resource", cutoffTimestamp)
		require.NoError(t, err)

		// other-dash was deleted and not recreated: all of its history is removed.
		// my-dash still exists (recreated after delete); GC skips deleting its history when
		// GetLatestResourceKey succeeds, so all five revision keys remain.
		// BatchSize=3 exercises pagination across both resources without dropping my-dash history.
		historyResp := storageBackend.kv.Keys(ctx, dataSection, ListOptions{
			StartKey: "group/resource/namespace/",
			EndKey:   "group/resource/namespace0",
		})
		count := 0
		historyResp(func(k string, err error) bool {
			require.NoError(t, err)
			require.Contains(t, k, "/my-dash/")
			require.NotContains(t, k, "/other-dash/")
			count++
			return true
		})
		require.Equal(t, 5, count)
	})
}

func TestIntegrationGarbageCollectionLoop(t *testing.T) {
	gcConfig := GarbageCollectionConfig{
		Enabled:          true,
		Interval:         time.Minute,
		BatchSize:        100,
		DashboardsMaxAge: 24 * time.Hour,
	}

	t.Run("can delete eligble resources", func(t *testing.T) {
		testutil.SkipIntegrationTestInShortMode(t)

		ctx := testutil.NewTestContext(t, time.Now().Add(2*time.Minute))

		storageBackend := setupTestStorageBackend(t, func(opts *KVBackendOptions) {
			opts.GarbageCollection = gcConfig
		})
		b := storageBackend

		server, err := NewResourceServer(ResourceServerOptions{
			Backend: storageBackend,
		})
		require.NoError(t, err)
		t.Cleanup(func() {
			ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
			defer cancel()
			_ = server.Stop(ctx)
		})

		rv1, err := writeEvent(t, ctx, storageBackend, "resource1", resourcepb.WatchEvent_ADDED)
		require.NoError(t, err)
		_, err = writeEvent(t, ctx, storageBackend, "resource1", resourcepb.WatchEvent_DELETED,
			func(o *writeEventOptions) {
				o.PreviousRV = rv1
			})
		require.NoError(t, err)

		// count how many history entries there are before GC runs - should be 2 (created and deleted)
		historyResp := storageBackend.kv.Keys(ctx, dataSection, ListOptions{
			StartKey: "group/resource/namespace/",
			EndKey:   "group/resource/namespace0",
		})
		count := 0
		historyResp(func(k string, err error) bool {
			require.NoError(t, err)
			count++
			return true
		})
		require.Equal(t, 2, count)

		cutoffTimestamp := time.Now().Add(time.Hour).UnixMicro() // everything eligible for deletion
		b.runGarbageCollection(ctx, cutoffTimestamp)

		// count how many history entries there are after GC runs - should be 0
		historyResp = storageBackend.kv.Keys(ctx, dataSection, ListOptions{
			StartKey: "group/resource/namespace/",
			EndKey:   "group/resource/namespace0",
		})
		count = 0
		historyResp(func(k string, err error) bool {
			require.NoError(t, err)
			count++
			return true
		})
		require.Equal(t, 0, count)
	})

	t.Run("nothing is eligble to delete", func(t *testing.T) {
		testutil.SkipIntegrationTestInShortMode(t)

		ctx := testutil.NewTestContext(t, time.Now().Add(2*time.Minute))

		storageBackend := setupTestStorageBackend(t, func(opts *KVBackendOptions) {
			opts.GarbageCollection = gcConfig
		})
		b := storageBackend

		server, err := NewResourceServer(ResourceServerOptions{
			Backend: storageBackend,
		})
		require.NoError(t, err)
		t.Cleanup(func() {
			ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
			defer cancel()
			_ = server.Stop(ctx)
		})

		rv1, err := writeEvent(t, ctx, storageBackend, "resource1", resourcepb.WatchEvent_ADDED)
		require.NoError(t, err)
		_, err = writeEvent(t, ctx, storageBackend, "resource1", resourcepb.WatchEvent_DELETED,
			func(o *writeEventOptions) {
				o.PreviousRV = rv1
			})
		require.NoError(t, err)

		// count how many history entries there are before GC runs - should be 2 (created and deleted)
		historyResp := storageBackend.kv.Keys(ctx, dataSection, ListOptions{
			StartKey: "group/resource/namespace/",
			EndKey:   "group/resource/namespace0",
		})
		count := 0
		historyResp(func(k string, err error) bool {
			require.NoError(t, err)
			count++
			return true
		})
		require.Equal(t, 2, count)

		cutoffTimestamp := time.Now().Add(-time.Hour).UnixMicro() // nothing eligible for deletion
		b.runGarbageCollection(ctx, cutoffTimestamp)

		// count how many history entries there are after GC runs - should still be 2
		historyResp = storageBackend.kv.Keys(ctx, dataSection, ListOptions{
			StartKey: "group/resource/namespace/",
			EndKey:   "group/resource/namespace0",
		})
		count = 0
		historyResp(func(k string, err error) bool {
			require.NoError(t, err)
			count++
			return true
		})
		require.Equal(t, 2, count)
	})

	t.Run("will respect dashboard retention settings", func(t *testing.T) {
		testutil.SkipIntegrationTestInShortMode(t)

		ctx := testutil.NewTestContext(t, time.Now().Add(2*time.Minute))

		storageBackend := setupTestStorageBackend(t, func(opts *KVBackendOptions) {
			opts.GarbageCollection = gcConfig
		})
		b := storageBackend

		server, err := NewResourceServer(ResourceServerOptions{
			Backend: storageBackend,
		})
		require.NoError(t, err)
		t.Cleanup(func() {
			ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
			defer cancel()
			_ = server.Stop(ctx)
		})

		rv1, err := writeEvent(t, ctx, storageBackend, "dashboard1", resourcepb.WatchEvent_ADDED,
			func(o *writeEventOptions) {
				o.Group = "dashboard.grafana.app"
				o.Resource = "dashboards"
			})
		require.NoError(t, err)
		_, err = writeEvent(t, ctx, storageBackend, "dashboard1", resourcepb.WatchEvent_DELETED,
			func(o *writeEventOptions) {
				o.Namespace = "namespace"
				o.PreviousRV = rv1
				o.Group = "dashboard.grafana.app"
				o.Resource = "dashboards"
			})
		require.NoError(t, err)

		rv2, err := writeEvent(t, ctx, storageBackend, "resource1", resourcepb.WatchEvent_ADDED)
		require.NoError(t, err)
		_, err = writeEvent(t, ctx, storageBackend, "resource1", resourcepb.WatchEvent_DELETED,
			func(o *writeEventOptions) {
				o.PreviousRV = rv2
			})
		require.NoError(t, err)

		cutoffTimestamp := time.Now().Add(1 * time.Hour).UnixMicro() // everything eligible for deletion (except dashboards)
		b.runGarbageCollection(ctx, cutoffTimestamp)

		// count how many history entries there are for group/resource - should be 0 (they were deleted by GC)
		historyResp := storageBackend.kv.Keys(ctx, dataSection, ListOptions{
			StartKey: "group/resource/",
			EndKey:   "group/resource0",
		})
		count := 0
		historyResp(func(k string, err error) bool {
			require.NoError(t, err)
			count++
			return true
		})
		require.Equal(t, 0, count)

		// count how many history entries there are for dashboard.grafana.app/dashboards - should be 2 (created and deleted)
		historyResp = storageBackend.kv.Keys(ctx, dataSection, ListOptions{
			StartKey: "dashboard.grafana.app/dashboards/",
			EndKey:   "dashboard.grafana.app/dashboards0",
		})
		count = 0
		historyResp(func(k string, err error) bool {
			require.NoError(t, err)
			count++
			return true
		})
		require.Equal(t, 2, count)

		trashResp, err := server.List(ctx, &resourcepb.ListRequest{
			Source: resourcepb.ListRequest_TRASH,
			Options: &resourcepb.ListOptions{
				Key: &resourcepb.ResourceKey{
					Namespace: "namespace",
					Group:     "dashboard.grafana.app",
					Resource:  "dashboards",
				},
			},
		})
		require.NoError(t, err)
		require.Nil(t, trashResp.Error)
		require.Len(t, trashResp.Items, 1)
	})
}
