fix: migrations to make postgresql compatible. (#35762)

This commit introduces several improvements to database migration
scripts to enhance compatibility between MySQL and PostgreSQL, ensure
case-sensitive behavior where needed, and improve migration safety and
correctness. The changes include dynamic SQL generation based on the
database engine, improved transaction handling, and adjustments to
field types and adapters for better cross-database support.

Database compatibility and case sensitivity improvements:

- Migration scripts in split_modulestore_django and learning_sequences
  now dynamically generate SQL statements for altering column case
  sensitivity and uniqueness based on whether the database is MySQL or
  PostgreSQL, ensuring correct behavior across both backends.
  - common/djangoapps/split_modulestore_django/migrations/0001_initial.py
  - openedx/core/djangoapps/content/learning_sequences/migrations/0001_initial.py

- The courseware.fields module now checks for "postgresql" in the
  database engine string instead of a specific backend name, improving
  compatibility with different PostgreSQL drivers.
  - lms/djangoapps/courseware/fields.py

- The 0011_csm_id_bigint migration in courseware now supports both MySQL
  and PostgreSQL for altering column types, with specific SQL for each
  backend.
  - lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py

- The 0009_readd_facebook_url migration in course_overviews now
  introspects the table structure using backend-specific SQL for MySQL
  and PostgreSQL, ensuring correct detection of existing fields.
  - openedx/core/djangoapps/content/course_overviews/migrations/0009_readd_facebook_url.py

Migration safety and correctness:

- Service user creation and deletion in the commerce app is now wrapped
  in atomic transactions to ensure database consistency.
  - lms/djangoapps/commerce/migrations/0001_data__add_ecommerce_service_user.py

- The move_overrides_to_edx_when migration in courseware now specifies
  a no-op reverse migration, preventing accidental data loss on migration
  rollback.
  - lms/djangoapps/courseware/migrations/0008_move_idde_to_edx_when.py

Adapter registration and code cleanup:

- The common_initialization app now registers custom adapters for
  CourseLocator and related classes in psycopg2 when using PostgreSQL,
  ensuring proper serialization of these types.
  - openedx/core/djangoapps/common_initialization/apps.py

- Minor code cleanup and formatting improvements in migration files,
  including import order and field formatting for readability.
  - lms/djangoapps/grades/migrations/0015_historicalpersistentsubsectiongradeoverride.py

Author: qasimgulzar

common/djangoapps/split_modulestore_django/migrations/0001_initial.py

@@ -1,15 +1,40 @@
 # Generated by Django 2.2.20 on 2021-05-07 18:29, manually modified to make "course_id" column case sensitive
 
 from django.conf import settings
-from django.db import migrations, models
+from django.db import migrations, models, connection
 import django.db.models.deletion
 import opaque_keys.edx.django.models
 import simple_history.models
 
 
+def generate_split_module_sql(db_engine):
+    if 'mysql' in db_engine:
+        return 'ALTER TABLE split_modulestore_django_splitmodulestorecourseindex MODIFY course_id varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL UNIQUE;'
+    elif 'postgresql' in db_engine:
+        return """
+            ALTER TABLE split_modulestore_django_splitmodulestorecourseindex
+            ALTER COLUMN course_id TYPE VARCHAR(255),
+            ALTER COLUMN course_id SET NOT NULL;
+
+            ALTER TABLE split_modulestore_django_splitmodulestorecourseindex
+            ADD CONSTRAINT course_id_unique UNIQUE (course_id);
+        """
+
+
+def generate_split_history_module_sql(db_engine):
+    if 'mysql' in db_engine:
+        return 'ALTER TABLE split_modulestore_django_historicalsplitmodulestorecourseindex MODIFY course_id varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL;'
+    elif 'postgresql' in db_engine:
+        return """
+            ALTER TABLE split_modulestore_django_historicalsplitmodulestorecourseindex
+            ALTER COLUMN course_id TYPE VARCHAR(255),
+            ALTER COLUMN course_id SET NOT NULL,
+            ALTER COLUMN course_id SET DATA TYPE VARCHAR(255) COLLATE "C";
+        """
 class Migration(migrations.Migration):
 
     initial = True
+    db_engine = connection.settings_dict['ENGINE']
 
     dependencies = [
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
@@ -65,11 +90,11 @@ class Migration(migrations.Migration):
         # Custom code: Convert columns to utf8_bin because we want to allow
         # case-sensitive comparisons for CourseKeys, which were case-sensitive in MongoDB
         migrations.RunSQL(
-            'ALTER TABLE split_modulestore_django_splitmodulestorecourseindex MODIFY course_id varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL UNIQUE;',
+            generate_split_module_sql(db_engine),
             reverse_sql=migrations.RunSQL.noop,
         ),
         migrations.RunSQL(
-            'ALTER TABLE split_modulestore_django_historicalsplitmodulestorecourseindex MODIFY course_id varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL;',
+            generate_split_history_module_sql(db_engine),
             reverse_sql=migrations.RunSQL.noop,
         ),
     ]

lms/djangoapps/commerce/migrations/0001_data__add_ecommerce_service_user.py

@@ -1,22 +1,24 @@
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.contrib.auth.models import User
-from django.db import migrations, models
+from django.db import migrations, models, transaction
 
 USERNAME = settings.ECOMMERCE_SERVICE_WORKER_USERNAME
 EMAIL = USERNAME + '@fake.email'
 
 def forwards(apps, schema_editor):
     """Add the service user."""
     User = get_user_model()
-    user, created = User.objects.get_or_create(username=USERNAME, email=EMAIL)
-    if created:
-        user.set_unusable_password()
-        user.save()
+    with transaction.atomic():
+        user, created = User.objects.get_or_create(username=USERNAME, email=EMAIL)
+        if created:
+            user.set_unusable_password()
+            user.save()
 
 def backwards(apps, schema_editor):
     """Remove the service user."""
-    User.objects.get(username=USERNAME, email=EMAIL).delete()
+    with transaction.atomic():
+        User.objects.get(username=USERNAME, email=EMAIL).delete()
 
 class Migration(migrations.Migration):
 

lms/djangoapps/courseware/fields.py

@@ -18,7 +18,7 @@ def db_type(self, connection):
             # is an alias for that (https://www.sqlite.org/autoinc.html). An unsigned integer
             # isn't an alias for ROWID, so we have to give up on the unsigned part.
             return "integer"
-        elif connection.settings_dict['ENGINE'] == 'django.db.backends.postgresql_psycopg2':
+        elif "postgresql" in connection.settings_dict['ENGINE']:
             # Pg's bigserial is implicitly unsigned (doesn't allow negative numbers) and
             # goes 1-9.2x10^18
             return "BIGSERIAL"
@@ -30,7 +30,7 @@ def rel_db_type(self, connection):
             return "bigint UNSIGNED"
         elif connection.settings_dict['ENGINE'] == 'django.db.backends.sqlite3':
             return "integer"
-        elif connection.settings_dict['ENGINE'] == 'django.db.backends.postgresql_psycopg2':
+        elif "postgresql" in connection.settings_dict['ENGINE']:
             return "BIGSERIAL"
         else:
             return None

lms/djangoapps/courseware/migrations/0008_move_idde_to_edx_when.py

@@ -33,5 +33,5 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(move_overrides_to_edx_when)
+        migrations.RunPython(move_overrides_to_edx_when, reverse_code=migrations.RunPython.noop)
     ]

lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py

@@ -1,28 +1,38 @@
 # Generated by Django 1.11.23 on 2019-08-28 15:50
 
-
 import lms.djangoapps.courseware.fields
 
 from django.conf import settings
-from django.db import migrations
+from django.db import migrations, models
 from django.db.migrations import AlterField
 
 class CsmBigInt(AlterField):
     '''
     Subclass AlterField migration class to split SQL between two different databases
-    We can't use the normal AlterField migration operation because Django generate and routes migrations at the model
+    We can't use the normal AlterField migration operation because Django generates and routes migrations at the model
     level and the coursewarehistoryextended_studentmodulehistoryextended table is in a different database
     '''
     def database_forwards(self, app_label, schema_editor, from_state, to_state):
         if hasattr(schema_editor.connection, 'is_in_memory_db') and schema_editor.connection.is_in_memory_db():
             # sqlite3 doesn't support 'MODIFY', so skipping during tests
             return
+
         to_model = to_state.apps.get_model(app_label, self.model_name)
+
         if schema_editor.connection.alias == 'student_module_history':
             if settings.FEATURES["ENABLE_CSMH_EXTENDED"]:
-              schema_editor.execute("ALTER TABLE `coursewarehistoryextended_studentmodulehistoryextended` MODIFY `student_module_id` bigint UNSIGNED NOT NULL;")
+                if schema_editor.connection.vendor == 'mysql':
+                    schema_editor.execute("ALTER TABLE `coursewarehistoryextended_studentmodulehistoryextended` MODIFY `student_module_id` bigint UNSIGNED NOT NULL;")
+                elif schema_editor.connection.vendor == 'postgresql':
+                    schema_editor.execute("ALTER TABLE coursewarehistoryextended_studentmodulehistoryextended ALTER COLUMN student_module_id TYPE bigint;")
         elif self.allow_migrate_model(schema_editor.connection.alias, to_model):
-            schema_editor.execute("ALTER TABLE `courseware_studentmodule` MODIFY `id` bigint UNSIGNED AUTO_INCREMENT NOT NULL;")
+            if schema_editor.connection.vendor == 'postgresql':
+                # For PostgreSQL
+                schema_editor.execute("ALTER TABLE courseware_studentmodule ALTER COLUMN id SET DATA TYPE bigint;")
+                schema_editor.execute("ALTER TABLE courseware_studentmodule ALTER COLUMN id SET NOT NULL;")
+            else:
+                # For MySQL
+                schema_editor.execute("ALTER TABLE `courseware_studentmodule` MODIFY `id` bigint UNSIGNED AUTO_INCREMENT NOT NULL;")
 
     def database_backwards(self, app_label, schema_editor, from_state, to_state):
         # Make backwards migration a no-op, app will still work if column is wider than expected
@@ -33,6 +43,7 @@ class Migration(migrations.Migration):
     dependencies = [
         ('courseware', '0010_auto_20190709_1559'),
     ]
+
     if settings.FEATURES["ENABLE_CSMH_EXTENDED"]:
         dependencies.append(('coursewarehistoryextended', '0002_force_studentmodule_index'))
 

lms/djangoapps/grades/migrations/0015_historicalpersistentsubsectiongradeoverride.py

@@ -1,14 +1,13 @@
 # Generated by Django 1.11.20 on 2019-06-05 13:59
 
 
-from django.conf import settings
-from django.db import migrations, models
 import django.db.models.deletion
 import simple_history.models
+from django.conf import settings
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
         ('grades', '0014_persistentsubsectiongradeoverridehistory'),
@@ -28,15 +27,24 @@ class Migration(migrations.Migration):
                 ('history_id', models.AutoField(primary_key=True, serialize=False)),
                 ('history_date', models.DateTimeField()),
                 ('history_change_reason', models.CharField(max_length=100, null=True)),
-                ('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
-                ('grade', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to='grades.PersistentSubsectionGrade')),
-                ('history_user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)),
+                ('history_type',
+                 models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
+                ('history_user',
+                 models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+',
+                                   to=settings.AUTH_USER_MODEL)),
             ],
-            options={
-                'ordering': ('-history_date', '-history_id'),
-                'get_latest_by': 'history_date',
-                'verbose_name': 'historical persistent subsection grade override',
-            },
-            bases=(simple_history.models.HistoricalChanges, models.Model),
+        options = {
+        'ordering': ('-history_date', '-history_id'),
+        'get_latest_by': 'history_date',
+        'verbose_name': 'historical persistent subsection grade override',
+    },
+    bases = (simple_history.models.HistoricalChanges, models.Model),
+    ),
+        migrations.AddField(
+            model_name='historicalpersistentsubsectiongradeoverride',
+            name='grade',
+            field=models.ForeignKey(blank=True, db_constraint=False, null=True,
+                                    on_delete=django.db.models.deletion.DO_NOTHING, related_name='+',
+                                    to='grades.PersistentSubsectionGrade'),
         ),
     ]

openedx/core/djangoapps/common_initialization/apps.py

@@ -2,8 +2,8 @@
 Common initialization app for the LMS and CMS
 """
 
-
 from django.apps import AppConfig
+from django.db import connection
 
 
 class CommonInitializationConfig(AppConfig):  # lint-amnesty, pylint: disable=missing-class-docstring
@@ -14,6 +14,7 @@ def ready(self):
         # Common settings validations for the LMS and CMS.
         from . import checks  # lint-amnesty, pylint: disable=unused-import
         self._add_mimetypes()
+        self._add_required_adapters()
 
     @staticmethod
     def _add_mimetypes():
@@ -26,3 +27,21 @@ def _add_mimetypes():
         mimetypes.add_type('application/x-font-opentype', '.otf')
         mimetypes.add_type('application/x-font-ttf', '.ttf')
         mimetypes.add_type('application/font-woff', '.woff')
+
+    @staticmethod
+    def _add_required_adapters():
+        """
+        Register CourseLocator in psycopg2 extensions
+        :return:
+        """
+        if 'postgresql' in connection.vendor.lower():
+            from opaque_keys.edx.locator import CourseLocator, LibraryLocator, BlockUsageLocator
+            from psycopg2.extensions import QuotedString, register_adapter
+
+            def adapt_course_locator(course_locator):
+                return QuotedString(str(course_locator))  # lint-amnesty, pylint: disable=protected-access
+
+            # Register the adapter
+            register_adapter(CourseLocator, adapt_course_locator)
+            register_adapter(LibraryLocator, adapt_course_locator)
+            register_adapter(BlockUsageLocator, adapt_course_locator)

openedx/core/djangoapps/content/course_overviews/migrations/0009_readd_facebook_url.py

@@ -1,45 +1,42 @@
 from django.db import migrations, models, connection
 
 def table_description():
-    """Handle Mysql/Pg vs Sqlite"""
-    # django's mysql/pg introspection.get_table_description tries to select *
-    # from table and fails during initial migrations from scratch.
-    # sqlite does not have this failure, so we can use the API.
-    # For not-sqlite, query information-schema directly with code lifted
-    # from the internals of django.db.backends.mysql.introspection.py
-
+    """Handle MySQL/Postgres vs SQLite compatibility for table introspection"""
     if connection.vendor == 'sqlite':
         fields = connection.introspection.get_table_description(connection.cursor(), 'course_overviews_courseoverview')
         return [f.name for f in fields]
     else:
         cursor = connection.cursor()
-        cursor.execute("""
-            SELECT column_name
-            FROM information_schema.columns
-            WHERE table_name = 'course_overviews_courseoverview' AND table_schema = DATABASE()""")
+        if connection.vendor == 'mysql':
+            cursor.execute("""
+                SELECT column_name
+                FROM information_schema.columns
+                WHERE table_name = 'course_overviews_courseoverview' AND table_schema = DATABASE()
+            """)
+        elif connection.vendor == 'postgresql':
+            cursor.execute("""
+                SELECT column_name
+                FROM information_schema.columns
+                WHERE table_name = 'course_overviews_courseoverview' AND table_catalog = current_database()
+            """)
         rows = cursor.fetchall()
         return [r[0] for r in rows]
 
-
 class Migration(migrations.Migration):
 
     dependencies = [
         ('course_overviews', '0008_remove_courseoverview_facebook_url'),
     ]
 
-    # An original version of 0008 removed the facebook_url field We need to
-    # handle the case where our noop 0008 ran AND the case where the original
-    # 0008 ran.  We do that by using the standard information_schema to find out
-    # what columns exist. _meta is unavailable as the column has already been
-    # removed from the model
     operations = []
     fields = table_description()
 
-    # during a migration from scratch, fields will be empty, but we do not want to add
-    # an additional facebook_url
+    # Ensure 'facebook_url' is added if it does not exist in the table
     if fields and not any(f == 'facebook_url' for f in fields):
-        operations += migrations.AddField(
-            model_name='courseoverview',
-            name='facebook_url',
-            field=models.TextField(null=True),
-        ),
+        operations.append(
+            migrations.AddField(
+                model_name='courseoverview',
+                name='facebook_url',
+                field=models.TextField(null=True),
+            )
+        )