fix(Code References): Code references are slow to query

api/features/serializers.py

@@ -443,6 +443,7 @@ def get_last_modified_in_current_environment(
 class FeatureSerializerWithMetadata(MetadataSerializerMixin, CreateFeatureSerializer):
     metadata = MetadataSerializer(required=False, many=True)
 
+    # NOTE: This field is populated by `projects.code_references.services.annotate_feature_queryset_with_code_references_summary`.
     code_references_counts = FeatureFlagCodeReferencesRepositoryCountSerializer(
         many=True,
         read_only=True,

api/projects/code_references/constants.py

@@ -1,5 +1,2 @@
-# TODO: Implement history cleanup?
-FEATURE_FLAG_CODE_REFERENCES_RETENTION_DAYS = 30
-
 # Linux maximum file path length, as per limits.h/PATH_MAX
 MAX_FILE_PATH_LENGTH = 4096

api/projects/code_references/migrations/0003_introduce_per_feature_scanned_references.py

@@ -0,0 +1,97 @@
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("code_references", "0002_add_project_repo_created_index"),
+        ("features", "0066_constrain_feature_type"),
+        ("projects", "0029_bump_default_project_limits"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="VCSRepository",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("url", models.URLField()),
+                (
+                    "vcs_provider",
+                    models.CharField(
+                        choices=[("github", "GitHub")],
+                        max_length=50,
+                    ),
+                ),
+                ("last_scanned_at", models.DateTimeField(null=True)),
+                (
+                    "project",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="vcs_repositories",
+                        to="projects.project",
+                    ),
+                ),
+            ],
+        ),
+        migrations.AddConstraint(
+            model_name="vcsrepository",
+            constraint=models.UniqueConstraint(
+                fields=("project", "url"),
+                name="unique_vcs_repository",
+            ),
+        ),
+        migrations.CreateModel(
+            name="ScannedCodeReferences",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("revision", models.CharField(max_length=100)),
+                ("code_references", models.JSONField(default=list)),
+                ("code_references_hash", models.CharField(max_length=32)),
+                (
+                    "feature",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="scanned_code_references",
+                        to="features.feature",
+                    ),
+                ),
+                (
+                    "repository",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="scanned_code_references",
+                        to="code_references.vcsrepository",
+                    ),
+                ),
+            ],
+        ),
+        migrations.AddConstraint(
+            model_name="scannedcodereferences",
+            constraint=models.UniqueConstraint(
+                fields=("feature", "repository", "code_references_hash"),
+                name="unique_scanned_code_references",
+            ),
+        ),
+        migrations.DeleteModel(
+            name="FeatureFlagCodeReferencesScan",
+        ),
+    ]

api/projects/code_references/models.py

@@ -3,35 +3,67 @@
 from projects.code_references.types import JSONCodeReference, VCSProvider
 
 
-class FeatureFlagCodeReferencesScan(models.Model):
+class VCSRepository(models.Model):
     """
-    A scan of feature flag code references in a repository
+    A VCS repository that is scanned for feature flag code references
     """
 
+    created_at = models.DateTimeField(auto_now_add=True)
+
     project = models.ForeignKey(
         "projects.Project",
         on_delete=models.CASCADE,
-        related_name="code_references",
+        related_name="vcs_repositories",
     )
 
     # Provider-agnostic URL to the web UI of the repository, e.g. https://github.flagsmith.com/backend/
-    repository_url = models.URLField()
+    url = models.URLField()
 
     vcs_provider = models.CharField(
         max_length=50,
         choices=VCSProvider.choices,
-        default=VCSProvider.GITHUB,  # TODO: Remove when adding other providers
     )
+
+    last_scanned_at = models.DateTimeField(null=True)
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                fields=["project", "url"],
+                name="unique_vcs_repository",
+            ),
+        ]
+
+
+class ScannedCodeReferences(models.Model):
+    """
+    A list of code references for a feature scanned from a VCS repository
+    """
+
+    created_at = models.DateTimeField(auto_now_add=True)
+
+    feature = models.ForeignKey(
+        "features.Feature",
+        on_delete=models.CASCADE,
+        related_name="scanned_code_references",
+    )
+
+    repository = models.ForeignKey(
+        VCSRepository,
+        on_delete=models.CASCADE,
+        related_name="scanned_code_references",
+    )
+
     revision = models.CharField(max_length=100)
+
     code_references = models.JSONField[list[JSONCodeReference]](default=list)
 
-    created_at = models.DateTimeField(auto_now_add=True, db_index=True)
+    code_references_hash = models.CharField(max_length=32)
 
     class Meta:
-        ordering = ["-created_at"]
-        indexes = [
-            models.Index(
-                fields=["project", "repository_url", "-created_at"],
-                name="code_ref_proj_repo_created_idx",
+        constraints = [
+            models.UniqueConstraint(
+                fields=["feature", "repository", "code_references_hash"],
+                name="unique_scanned_code_references",
             ),
         ]

api/projects/code_references/serializers.py

@@ -1,11 +1,11 @@
 from rest_framework import serializers
 
 from projects.code_references.constants import MAX_FILE_PATH_LENGTH
-from projects.code_references.models import FeatureFlagCodeReferencesScan
 from projects.code_references.types import (
     CodeReference,
     CodeReferencesRepositoryCount,
     FeatureFlagCodeReferencesRepositorySummary,
+    FeatureFlagCodeReferencesScan,
     VCSProvider,
 )
 
@@ -28,26 +28,20 @@ class _CodeReferenceDetailSerializer(_BaseCodeReferenceSerializer):
 
 
 class FeatureFlagCodeReferencesScanSerializer(
-    serializers.ModelSerializer[FeatureFlagCodeReferencesScan],
+    serializers.Serializer[FeatureFlagCodeReferencesScan],
 ):
+    created_at = serializers.DateTimeField(read_only=True)
+    project = serializers.IntegerField(read_only=True, source="project.id")
+    repository_url = serializers.URLField()
+    vcs_provider = serializers.ChoiceField(
+        choices=VCSProvider.choices,
+        default=VCSProvider.GITHUB,
+    )
+    revision = serializers.CharField(max_length=100)
     code_references = _CodeReferenceSubmitSerializer(
         many=True, required=True, allow_empty=False
     )
 
-    class Meta:
-        model = FeatureFlagCodeReferencesScan
-        fields = [
-            "created_at",
-            "repository_url",
-            "project",
-            "revision",
-            "code_references",
-        ]
-        read_only_fields = [
-            "created_at",
-            "project",
-        ]
-
 
 class FeatureFlagCodeReferencesRepositorySummarySerializer(
     serializers.Serializer[FeatureFlagCodeReferencesRepositorySummary],