Add latest changes from gitlab-org/gitlab@master

Author: GitLab Bot

db/docs/batched_background_migrations/backfill_or_drop_ci_pipeline_on_project_id.yml

@@ -6,4 +6,4 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/155137
 milestone: '17.1'
 queued_migration_version: 20240531202753
 finalize_after: '2024-06-30'
-finalized_by: # version of the migration that finalized this BBM
+finalized_by: 20240704155541

db/post_migrate/20240704155541_finalize_backfill_pipeline_project_id.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class FinalizeBackfillPipelineProjectId < Gitlab::Database::Migration[2.2]
+  disable_ddl_transaction!
+  milestone '17.3'
+  restrict_gitlab_migration gitlab_schema: :gitlab_ci
+
+  MIGRATION = 'BackfillOrDropCiPipelineOnProjectId'
+
+  def up
+    ensure_batched_background_migration_is_finished(
+      job_class_name: MIGRATION,
+      table_name: :ci_pipelines,
+      column_name: :id,
+      job_arguments: [],
+      finalize: true
+    )
+  end
+
+  def down
+    # no-op
+  end
+end

db/post_migrate/20240704155702_add_not_null_constraint_to_ci_pipelines_project_id.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class AddNotNullConstraintToCiPipelinesProjectId < Gitlab::Database::Migration[2.2]
+  disable_ddl_transaction!
+  milestone '17.3'
+
+  def up
+    add_not_null_constraint :ci_pipelines, :project_id, validate: false
+  end
+
+  def down
+    remove_not_null_constraint :ci_pipelines, :project_id
+  end
+end

db/post_migrate/20240705035824_prepare_ci_pipelines_project_id_check_constraint.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class PrepareCiPipelinesProjectIdCheckConstraint < Gitlab::Database::Migration[2.2]
+  milestone '17.3'
+
+  CONSTRAINT_NAME = 'check_2ba2a044b9'
+
+  def up
+    prepare_async_check_constraint_validation :ci_pipelines, name: CONSTRAINT_NAME
+  end
+
+  def down
+    unprepare_async_check_constraint_validation :ci_pipelines, name: CONSTRAINT_NAME
+  end
+end

db/schema_migrations/20240704155541

@@ -0,0 +1 @@
+16318b19fa9bb6a51f03009beb7abdb5cc9f0600fa62be32ac4c278ba426567e

db/schema_migrations/20240704155702

@@ -0,0 +1 @@
+83f5f6055d7cf41159485fa74e0bc6a0cb2223dc007a348542b5752f91d7fc16

db/schema_migrations/20240705035824

@@ -0,0 +1 @@
+d3a7991ca8277600a9e43687ff2686b6ce90d4b3305683842bdde12b543c62d5

db/structure.sql

@@ -22550,6 +22550,9 @@ ALTER TABLE ONLY chat_teams
 ALTER TABLE workspaces
     ADD CONSTRAINT check_2a89035b04 CHECK ((personal_access_token_id IS NOT NULL)) NOT VALID;
 
+ALTER TABLE ci_pipelines
+    ADD CONSTRAINT check_2ba2a044b9 CHECK ((project_id IS NOT NULL)) NOT VALID;
+
 ALTER TABLE vulnerability_scanners
     ADD CONSTRAINT check_37608c9db5 CHECK ((char_length(vendor) <= 255)) NOT VALID;
 

doc/development/documentation/testing/markdownlint.md

@@ -83,6 +83,24 @@ To configure markdownlint in your editor, install one of the following as approp
   ((markdown-mode . ((flycheck-markdown-markdownlint-cli-config . ".markdownlint.yml"))))
   ```
 
+## Run `markdownlint-cli2` locally
+
+You can run `markdownlint-cli2` from anywhere in your repository. From the root of your repository,
+you don't need to specify the location of the configuration file. If you run it from elsewhere
+in your repository, you must specify the configuration file's location. In these commands,
+replace `doc/**/*.md` with the path to the Markdown files in your repository:
+
+```shell
+# From the root directory, you don't need to specify the configuration file
+$ markdownlint-cli2 'doc/**/*.md'
+
+# From elsewhere in the repository, specify the configuration file
+$ markdownlint-cli2 --config .markdownlint-cli2.yaml 'doc/**/*.md'
+```
+
+For a full list of command-line options, see [Command Line](https://github.com/DavidAnson/markdownlint-cli2?tab=readme-ov-file#command-line)
+in the `markdownlint-cli2` documentation.
+
 ## Disable markdownlint tests
 
 To disable all markdownlint rules, add a `<!-- markdownlint-disable -->` tag before the text, and a

qa/qa/flow/pipeline.rb

@@ -8,7 +8,7 @@ module Pipeline
       # Acceptable statuses:
       # Canceled, Created, Failed, Manual, Passed
       # Pending, Running, Skipped
-      def visit_latest_pipeline(status: nil, wait: nil, skip_wait: true)
+      def visit_latest_pipeline(status: nil, wait: 120, skip_wait: true)
         Page::Project::Menu.perform(&:go_to_pipelines)
         Page::Project::Pipeline::Index.perform do |index|
           index.has_any_pipeline?(wait: wait)
@@ -17,7 +17,7 @@ def visit_latest_pipeline(status: nil, wait: nil, skip_wait: true)
         end
       end
 
-      def wait_for_latest_pipeline(status: nil, wait: nil)
+      def wait_for_latest_pipeline(status: nil, wait: 120)
         Page::Project::Menu.perform(&:go_to_pipelines)
         Page::Project::Pipeline::Index.perform do |index|
           index.has_any_pipeline?(wait: wait)

spec/features/merge_request/user_sees_pipelines_spec.rb

@@ -148,10 +148,12 @@
         sign_in(actor)
       end
 
+      # rubocop:disable Cop/DestroyAll -- cannot delete due to not-null constraint
       after do
-        parent_project.all_pipelines.delete_all
-        forked_project.all_pipelines.delete_all
+        parent_project.all_pipelines.destroy_all
+        forked_project.all_pipelines.destroy_all
       end
+      # rubocop:enable Cop/DestroyAll
 
       context 'when actor is a developer in parent project' do
         let(:actor) { developer_in_parent }

spec/lib/gitlab/background_migration/backfill_or_drop_ci_pipeline_on_project_id_spec.rb

@@ -2,7 +2,9 @@
 
 require 'spec_helper'
 
-RSpec.describe Gitlab::BackgroundMigration::BackfillOrDropCiPipelineOnProjectId, feature_category: :continuous_integration do
+RSpec.describe Gitlab::BackgroundMigration::BackfillOrDropCiPipelineOnProjectId,
+  feature_category: :continuous_integration,
+  migration: :gitlab_ci do
   let(:project_id_with_build) { 137 }
   let(:project_id_for_merge_request) { 140 }
   let(:project_id_for_unaffected_pipeline) { 1 }
@@ -24,22 +26,6 @@
     table(:p_ci_builds, database: :ci).create!(partition_id: 100, project_id: project_id_with_build, commit_id: 2)
   end
 
-  let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
-  let(:project) do
-    table(:projects)
-      .create!(id: project_id_for_merge_request, namespace_id: namespace.id, project_namespace_id: namespace.id)
-  end
-
-  let!(:merge_request) do
-    table(:merge_requests).create!(
-      id: 1,
-      target_branch: 'main',
-      source_branch: 'feature',
-      target_project_id: project.id,
-      source_project_id: project.id
-    )
-  end
-
   subject(:migration) do
     described_class.new(
       start_id: 1,
@@ -53,49 +39,46 @@
   end
 
   describe '#perform' do
+    before do
+      allow(Gitlab::BackgroundMigration::BackfillOrDropCiPipelineOnProjectId::MergeRequest)
+        .to receive_message_chain(:where, :select)
+        .and_return([])
+    end
+
     it 'backfills if applicable otherwise deletes' do
       migration.perform
 
       expect { pipeline_with_nothing.reload }.to raise_error(ActiveRecord::RecordNotFound)
       expect { trigger_request.reload }.to raise_error(ActiveRecord::RecordNotFound)
       expect(pipeline_with_builds.reload.project_id).to eq(project_id_with_build)
-      expect(pipeline_with_merge_request.reload.project_id).to eq(project_id_for_merge_request)
       expect(untouched_pipeline.reload.project_id).to eq(project_id_for_unaffected_pipeline)
     end
 
-    context 'when associations are invalid as well' do
-      let!(:pipeline_with_bad_build) { table(:ci_pipelines, database: :ci).create!(id: 5, partition_id: 100) }
-      let!(:bad_build) { table(:p_ci_builds, database: :ci).create!(partition_id: 100, commit_id: 5) }
+    context 'for migrations with merge_request' do
+      before do
+        merge_request = double('merge_request') # rubocop:disable RSpec/VerifiedDoubles -- merge_request is a already a stub of applicationRecord
+        allow(merge_request).to receive(:target_project_id) { project_id_for_merge_request }
 
-      it 'deletes pipeline if associations do not have project_id' do
-        migration.perform
-
-        expect { pipeline_with_bad_build.reload }.to raise_error(ActiveRecord::RecordNotFound)
+        allow(Gitlab::BackgroundMigration::BackfillOrDropCiPipelineOnProjectId::MergeRequest)
+          .to receive_message_chain(:where, :select)
+          .and_return([merge_request])
       end
-    end
 
-    context 'when the merge request is from a fork project' do
-      let(:another_namespace) { table(:namespaces).create!(name: 'user2', path: 'user2') }
+      it 'backfills from merge_request' do
+        migration.perform
 
-      let(:another_project) do
-        table(:projects)
-          .create!(id: 141, namespace_id: another_namespace.id, project_namespace_id: another_namespace.id)
+        expect(pipeline_with_merge_request.reload.project_id).to eq(project_id_for_merge_request)
       end
+    end
 
-      let!(:merge_request) do
-        table(:merge_requests).create!(
-          id: 1,
-          target_branch: 'main',
-          source_branch: 'feature',
-          target_project_id: project.id,
-          source_project_id: another_project.id
-        )
-      end
+    context 'when associations are invalid as well' do
+      let!(:pipeline_with_bad_build) { table(:ci_pipelines, database: :ci).create!(id: 5, partition_id: 100) }
+      let!(:bad_build) { table(:p_ci_builds, database: :ci).create!(partition_id: 100, commit_id: 5) }
 
-      it 'deletes the pipeline as association is not definite' do
+      it 'deletes pipeline if associations do not have project_id' do
         migration.perform
 
-        expect { pipeline_with_merge_request.reload }.to raise_error(ActiveRecord::RecordNotFound)
+        expect { pipeline_with_bad_build.reload }.to raise_error(ActiveRecord::RecordNotFound)
       end
     end
   end

spec/lib/gitlab/background_migration/backfill_partition_id_ci_daily_build_group_report_result_spec.rb

@@ -5,12 +5,12 @@
 RSpec.describe Gitlab::BackgroundMigration::BackfillPartitionIdCiDailyBuildGroupReportResult, feature_category: :ci_scaling do
   let(:ci_pipelines_table) { table(:ci_pipelines, database: :ci) }
   let(:ci_daily_build_group_report_results_table) { table(:ci_daily_build_group_report_results, database: :ci) }
-  let!(:pipeline_1) { ci_pipelines_table.create!(id: 1, partition_id: 100) }
-  let!(:pipeline_2) { ci_pipelines_table.create!(id: 2, partition_id: 101) }
-  let!(:pipeline_3) { ci_pipelines_table.create!(id: 3, partition_id: 100) }
+  let!(:pipeline_1) { ci_pipelines_table.create!(id: 1, partition_id: 100, project_id: 1) }
+  let!(:pipeline_2) { ci_pipelines_table.create!(id: 2, partition_id: 101, project_id: 1) }
+  let!(:pipeline_3) { ci_pipelines_table.create!(id: 3, partition_id: 100, project_id: 1) }
   let!(:ci_daily_build_group_report_results_100) do
     ci_daily_build_group_report_results_table.create!(
-      date: Date.yesterday,
+      date: 1.day.ago,
       project_id: 1,
       ref_path: 'master',
       group_name: 'rspec',
@@ -23,7 +23,7 @@
 
   let!(:ci_daily_build_group_report_results_101) do
     ci_daily_build_group_report_results_table.create!(
-      date: Date.today,
+      date: Time.current,
       project_id: 1,
       ref_path: 'master',
       group_name: 'rspec',
@@ -36,7 +36,7 @@
 
   let!(:invalid_ci_daily_build_group_report_results) do
     ci_daily_build_group_report_results_table.create!(
-      date: Date.tomorrow,
+      date: 1.day.from_now,
       project_id: 1,
       ref_path: 'master',
       group_name: 'rspec',

spec/lib/gitlab/background_migration/backfill_partition_id_ci_pipeline_artifact_spec.rb

@@ -6,9 +6,9 @@
   feature_category: :continuous_integration do
   let(:ci_pipelines_table) { table(:ci_pipelines, database: :ci) }
   let(:ci_pipeline_artifacts_table) { table(:ci_pipeline_artifacts, database: :ci) }
-  let!(:pipeline_100) { ci_pipelines_table.create!(id: 1, partition_id: 100) }
-  let!(:pipeline_101) { ci_pipelines_table.create!(id: 2, partition_id: 101) }
-  let!(:pipeline_102) { ci_pipelines_table.create!(id: 3, partition_id: 100) }
+  let!(:pipeline_100) { ci_pipelines_table.create!(id: 1, partition_id: 100, project_id: 1) }
+  let!(:pipeline_101) { ci_pipelines_table.create!(id: 2, partition_id: 101, project_id: 1) }
+  let!(:pipeline_102) { ci_pipelines_table.create!(id: 3, partition_id: 100, project_id: 1) }
   let!(:ci_pipeline_artifact_100) do
     ci_pipeline_artifacts_table.create!(
       id: 1,

spec/lib/gitlab/background_migration/backfill_partition_id_ci_pipeline_chat_data_spec.rb

@@ -6,8 +6,8 @@
   feature_category: :continuous_integration do
   let(:ci_pipelines_table) { table(:ci_pipelines, database: :ci) }
   let(:ci_pipeline_chat_data_table) { table(:ci_pipeline_chat_data, database: :ci) }
-  let!(:pipeline1) { ci_pipelines_table.create!(id: 1, partition_id: 100) }
-  let!(:pipeline2) { ci_pipelines_table.create!(id: 2, partition_id: 101) }
+  let!(:pipeline1) { ci_pipelines_table.create!(id: 1, partition_id: 100, project_id: 1) }
+  let!(:pipeline2) { ci_pipelines_table.create!(id: 2, partition_id: 101, project_id: 1) }
   let!(:invalid_ci_pipeline_chat_data) do
     ci_pipeline_chat_data_table.create!(
       id: 1,

spec/lib/gitlab/background_migration/backfill_partition_id_ci_pipeline_config_spec.rb

@@ -2,13 +2,12 @@
 
 require 'spec_helper'
 
-RSpec.describe Gitlab::BackgroundMigration::BackfillPartitionIdCiPipelineConfig,
-  feature_category: :continuous_integration do
+RSpec.describe Gitlab::BackgroundMigration::BackfillPartitionIdCiPipelineConfig, feature_category: :continuous_integration do
   let(:ci_pipelines_table) { table(:ci_pipelines, database: :ci) }
   let(:ci_pipeline_config_table) { table(:ci_pipelines_config, database: :ci) }
-  let!(:pipeline_1) { ci_pipelines_table.create!(id: 1, partition_id: 100) }
-  let!(:pipeline_2) { ci_pipelines_table.create!(id: 2, partition_id: 101) }
-  let!(:pipeline_3) { ci_pipelines_table.create!(id: 3, partition_id: 100) }
+  let!(:pipeline_1) { ci_pipelines_table.create!(id: 1, partition_id: 100, project_id: 1) }
+  let!(:pipeline_2) { ci_pipelines_table.create!(id: 2, partition_id: 101, project_id: 1) }
+  let!(:pipeline_3) { ci_pipelines_table.create!(id: 3, partition_id: 100, project_id: 1) }
   let!(:ci_pipeline_config_100) do
     ci_pipeline_config_table.create!(
       pipeline_id: pipeline_1.id,

spec/lib/gitlab/background_migration/backfill_partition_id_ci_pipeline_message_spec.rb

@@ -5,9 +5,9 @@
 RSpec.describe Gitlab::BackgroundMigration::BackfillPartitionIdCiPipelineMessage, feature_category: :ci_scaling do
   let(:ci_pipelines_table) { table(:ci_pipelines, database: :ci) }
   let(:ci_pipeline_messages_table) { table(:ci_pipeline_messages, database: :ci) }
-  let!(:pipeline_1) { ci_pipelines_table.create!(id: 1, partition_id: 100) }
-  let!(:pipeline_2) { ci_pipelines_table.create!(id: 2, partition_id: 101) }
-  let!(:pipeline_3) { ci_pipelines_table.create!(id: 3, partition_id: 100) }
+  let!(:pipeline_1) { ci_pipelines_table.create!(id: 1, partition_id: 100, project_id: 1) }
+  let!(:pipeline_2) { ci_pipelines_table.create!(id: 2, partition_id: 101, project_id: 1) }
+  let!(:pipeline_3) { ci_pipelines_table.create!(id: 3, partition_id: 100, project_id: 1) }
   let!(:ci_pipeline_messages_100) do
     ci_pipeline_messages_table.create!(
       content: 'content',

spec/lib/gitlab/background_migration/backfill_partition_id_ci_pipeline_metadata_spec.rb

@@ -2,13 +2,12 @@
 
 require 'spec_helper'
 
-RSpec.describe Gitlab::BackgroundMigration::BackfillPartitionIdCiPipelineMetadata,
-  feature_category: :continuous_integration do
+RSpec.describe Gitlab::BackgroundMigration::BackfillPartitionIdCiPipelineMetadata, feature_category: :continuous_integration do
   let(:ci_pipelines_table) { table(:ci_pipelines, database: :ci) }
   let(:ci_pipeline_metadata_table) { table(:ci_pipeline_metadata, database: :ci) }
-  let!(:pipeline_100) { ci_pipelines_table.create!(id: 1, partition_id: 100) }
-  let!(:pipeline_101) { ci_pipelines_table.create!(id: 2, partition_id: 101) }
-  let!(:pipeline_102) { ci_pipelines_table.create!(id: 3, partition_id: 100) }
+  let!(:pipeline_100) { ci_pipelines_table.create!(id: 1, partition_id: 100, project_id: 1) }
+  let!(:pipeline_101) { ci_pipelines_table.create!(id: 2, partition_id: 101, project_id: 1) }
+  let!(:pipeline_102) { ci_pipelines_table.create!(id: 3, partition_id: 100, project_id: 1) }
   let!(:ci_pipeline_metadata_100) do
     ci_pipeline_metadata_table.create!(
       pipeline_id: pipeline_100.id,

spec/lib/gitlab/background_migration/queue_backfill_autocancel_partition_id_on_ci_pipelines_spec.rb

@@ -2,7 +2,10 @@
 
 require 'spec_helper'
 
-RSpec.describe Gitlab::BackgroundMigration::QueueBackfillAutocancelPartitionIdOnCiPipelines, feature_category: :ci_scaling do
+RSpec.describe Gitlab::BackgroundMigration::QueueBackfillAutocancelPartitionIdOnCiPipelines,
+  feature_category: :ci_scaling,
+  migration: :gitlab_ci,
+  schema: 20240704155541 do
   let(:ci_pipelines_table) { table(:ci_pipelines, database: :ci) }
 
   let!(:pipeline_1) { ci_pipelines_table.create!(partition_id: 100) }

spec/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status_spec.rb

@@ -26,8 +26,8 @@
     let(:locked) { 1 }
     let(:unknown) { 2 }
 
-    let(:unlocked_pipeline) { pipelines.create!(locked: unlocked, partition_id: 100) }
-    let(:locked_pipeline) { pipelines.create!(locked: locked, partition_id: 100) }
+    let(:unlocked_pipeline) { pipelines.create!(locked: unlocked, partition_id: 100, project_id: project.id) }
+    let(:locked_pipeline) { pipelines.create!(locked: locked, partition_id: 100, project_id: project.id) }
 
     # rubocop:disable Layout/LineLength
     let!(:locked_artifact) { pipeline_artifacts.create!(project_id: project.id, pipeline_id: locked_pipeline.id, size: 1024, file_type: 0, file_format: 'gzip', file: 'a.gz', locked: unknown, partition_id: 100) }

spec/lib/gitlab/ci/variables/builder_spec.rb

@@ -576,14 +576,6 @@ def var(name, value)
   describe '#config_variables' do
     subject(:config_variables) { builder.config_variables }
 
-    context 'without project' do
-      before do
-        pipeline.update!(project_id: nil)
-      end
-
-      it { expect(config_variables.size).to eq(0) }
-    end
-
     context 'without repository' do
       let(:project) { create(:project) }
       let(:pipeline) { build(:ci_pipeline, ref: nil, sha: nil, project: project) }