E2E: QA Added acceptance tests for notification emails (#20918)

* Added tests for notification emails for content

* Bumped version

* Updated tests for notification permission in content

* Added appsettings.json for smtp tests

* Added smtp test project

* Updated nightly E2E test pipeline yaml file to run smtp project in the pipeline

* Fixed command to run smtp4dev in Docker

* Fixed pipeline

* Only run smtp tests on Linux

* Debugged

* Debugging

* Added step to stop smtp4dev container

* Debugging

* Updated port

* Reverted tests

* Added more tests for notification emails

* Formatted code

build/nightly-E2E-test-pipelines.yml

@@ -581,6 +581,15 @@ stages:
               CONNECTIONSTRINGS__UMBRACODBDSN: Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);Encrypt=True;TrustServerCertificate=True
               CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.SqlClient
               additionalEnvironmentVariables: false
+            # SMTP
+            LinuxSMTP:
+              vmImage: "ubuntu-latest"
+              testFolder: "SMTP"
+              port: ''
+              testCommand: "npx playwright test --project=smtp"
+              CONNECTIONSTRINGS__UMBRACODBDSN: Server=(local);Database=Umbraco;User Id=sa;Password=$(SA_PASSWORD);Encrypt=True;TrustServerCertificate=True
+              CONNECTIONSTRINGS__UMBRACODBDSN_PROVIDERNAME: Microsoft.Data.SqlClient
+              additionalEnvironmentVariables: false
         pool:
           vmImage: $(vmImage)
         steps:
@@ -658,6 +667,23 @@ stages:
               AZUREADB2CCLIENTID: $(AZUREB2CCLIENTID)
               AZUREADB2CCLIENTSECRET: $(AZUREB2CCLIENTSECRET)
 
+          # Start SMTP4dev via Docker for SMTP tests
+          - bash: |
+              echo "Starting SMTP4dev container..."
+              docker run -d --name smtp4dev -p 5000:80 -p 25:25 rnwood/smtp4dev
+
+              echo "Waiting for SMTP4dev to be ready..."
+              for i in {1..30}; do
+                if curl -s http://localhost:5000/api/messages > /dev/null; then
+                  echo "SMTP4dev is ready"
+                  break
+                fi
+                echo "Attempt $i: Waiting for SMTP4dev..."
+                sleep 2
+              done
+            displayName: Start SMTP4dev Docker container (Linux)
+            condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'), contains(variables['testFolder'], 'SMTP'))
+
           # Run tests Template
           - template: nightly-E2E-run-tests-template.yml
             parameters:
@@ -668,6 +694,14 @@ stages:
               AZUREB2CTESTUSERPASSWORD: $(AZUREB2CTESTUSERPASSWORD)
               DatabaseType: ${{ variables.DatabaseType }}
           
+          # Stop SMTP4dev container
+          - bash: |
+              echo "Stopping SMTP4dev container..."
+              docker stop smtp4dev
+              docker rm smtp4dev
+            displayName: Stop SMTP4dev Docker container
+            condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'), contains(variables['testFolder'], 'SMTP'))
+
   - stage: NotifySlackBot
     displayName: Notify Slack on Failure
     dependsOn: DefaultConfigE2E

tests/Umbraco.Tests.AcceptanceTest/playwright.config.ts

@@ -102,6 +102,17 @@ export default defineConfig({
       use: {
         ...devices['Desktop Chrome']
       }
+    },
+    {
+      name: 'smtp',
+      testMatch: 'SMTP/*.spec.ts',
+      dependencies: ['setup'],
+      use: {
+        ...devices['Desktop Chrome'],
+        // Use prepared auth state.
+        ignoreHTTPSErrors: true,
+        storageState: STORAGE_STATE
+      }
     }
   ],
 });

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Users/Permissions/UserGroup/DefaultPermissionsInContent.spec.ts

@@ -221,8 +221,9 @@ test('can not create content with create permission disabled', async ({umbracoAp
   await umbracoUi.content.isActionsMenuForNameVisible(rootDocumentName, false);
 });
 
-test.fixme('can create notifications with notification permission enabled', async ({umbracoApi, umbracoUi}) => {
+test('can set up notifications with notification permission enabled', async ({umbracoApi, umbracoUi}) => {
   // Arrange
+  const notificationActionIds = ['Umb.Document.Delete', 'Umb.Document.Publish'];
   userGroupId = await umbracoApi.userGroup.createUserGroupWithNotificationsPermission(userGroupName);
   await umbracoApi.user.setUserPermissions(testUser.name, testUser.email, testUser.password, userGroupId);
   testUserCookieAndToken = await umbracoApi.user.loginToUser(testUser.name, testUser.email, testUser.password);
@@ -230,11 +231,19 @@ test.fixme('can create notifications with notification permission enabled', asyn
 
   // Act
   await umbracoUi.content.goToSection(ConstantHelper.sections.content, false);
-  // TODO: Implement it later
-  // Setup SMTP server to test notifications, do this when we test appsettings.json
+  await umbracoUi.content.clickActionsMenuForContent(rootDocumentName);
+  await umbracoUi.content.clickNotificationsActionMenuOption();
+  await umbracoUi.content.clickDocumentNotificationOptionWithName(notificationActionIds[0]);
+  await umbracoUi.content.clickDocumentNotificationOptionWithName(notificationActionIds[1]);
+  await umbracoUi.content.clickSaveModalButton();
+
+  // Assert
+  await umbracoUi.content.isSuccessNotificationVisible();
+  expect(await umbracoApi.document.doesNotificationExist(rootDocumentId, notificationActionIds[0])).toBeTruthy();
+  expect(await umbracoApi.document.doesNotificationExist(rootDocumentId, notificationActionIds[1])).toBeTruthy();
 });
 
-test('can not create notifications with notification permission disabled', async ({umbracoApi, umbracoUi}) => {
+test('can not set up notifications with notification permission disabled', async ({umbracoApi, umbracoUi}) => {
   // Arrange
   userGroupId = await umbracoApi.userGroup.createUserGroupWithNotificationsPermission(userGroupName, false);
   await umbracoApi.user.setUserPermissions(testUser.name, testUser.email, testUser.password, userGroupId);

tests/Umbraco.Tests.AcceptanceTest/tests/SMTP/AdditionalSetup/appsettings.json

@@ -0,0 +1,64 @@
+{
+  "$schema": "appsettings-schema.json",
+  "Serilog": {
+    "MinimumLevel": {
+      "Default": "Information",
+      "Override": {
+        "Microsoft": "Warning",
+        "Microsoft.Hosting.Lifetime": "Information",
+        "System": "Warning"
+      }
+    },
+    "WriteTo": [
+      {
+        "Name": "Async",
+        "Args": {
+          "Configure": [
+            {
+              "Name": "Console"
+            }
+          ]
+        }
+      }
+    ]
+  },
+  "Umbraco": {
+    "CMS": {
+      "Unattended": {
+        "InstallUnattended": true,
+        "UnattendedUserName": "Playwright Test",
+        "UnattendedUserEmail": "playwright@umbraco.com",
+        "UnattendedUserPassword": "UmbracoAcceptance123!"
+      },
+      "Content": {
+        "ContentVersionCleanupPolicy": {
+          "EnableCleanup": false
+        }
+      },
+      "Global": {
+        "DisableElectionForSingleServer": true,
+        "InstallMissingDatabase": true,
+        "Id": "00000000-0000-0000-0000-000000000042",
+        "VersionCheckPeriod": 0,
+        "UseHttps": true,
+        "Smtp": {
+          "From": "no-reply@localhost.test",
+          "Host": "localhost",
+          "Port": 25,
+          "SecureSocketOptions": "None"
+        }
+      },
+      "HealthChecks": {
+        "Notification": {
+          "Enabled": true
+        }
+      },
+      "KeepAlive": {
+        "DisableKeepAliveTask": true
+      },
+      "WebRouting": {
+        "UmbracoApplicationUrl": "https://localhost:44331/"
+      }
+    }
+  }
+}

tests/Umbraco.Tests.AcceptanceTest/tests/SMTP/NotificationEmailForContent.spec.ts

@@ -0,0 +1,275 @@
+import {ConstantHelper, test} from '@umbraco/playwright-testhelpers';
+import {expect} from "@playwright/test";
+
+// Content
+let contentId = '';
+const contentName = 'TestContent';
+const contentText = 'This is test content text';
+const childContentName = 'ChildContent';
+// Document Type
+let documentTypeId = '';
+const documentTypeName = 'TestDocumentTypeForContent';
+const childDocumentTypeName = 'ChildDocumentType';
+// Data Type
+const textStringDataTypeName = 'Textstring';
+let textStringDataType: any;
+
+test.beforeEach(async ({umbracoApi}) => {
+  // Delete all emails from smtp4dev
+  await umbracoApi.smtp.deleteAllEmails();
+  textStringDataType = await umbracoApi.dataType.getByName(textStringDataTypeName);
+  documentTypeId = await umbracoApi.documentType.createDocumentTypeWithPropertyEditor(documentTypeName, textStringDataTypeName, textStringDataType.id);
+  contentId = await umbracoApi.document.createDocumentWithTextContent(contentName, documentTypeId, contentText, textStringDataTypeName);
+});
+
+test.afterEach(async ({umbracoApi}) => {
+  await umbracoApi.document.ensureNameNotExists(contentName);
+  await umbracoApi.documentType.ensureNameNotExists(documentTypeName);
+  // Delete all emails from smtp4dev
+  await umbracoApi.smtp.deleteAllEmails();
+});
+
+test('can set up notification for a content item', async ({umbracoUi, umbracoApi}) => {
+  // Arrange
+  const notificationActionIds = ['Umb.Document.Delete', 'Umb.Document.Publish'];
+  await umbracoUi.goToBackOffice();
+  await umbracoUi.content.goToSection(ConstantHelper.sections.content);
+  await umbracoUi.content.clickActionsMenuForContent(contentName);
+  await umbracoUi.content.clickNotificationsActionMenuOption();
+
+  // Act
+  await umbracoUi.content.clickDocumentNotificationOptionWithName(notificationActionIds[0]);
+  await umbracoUi.content.clickDocumentNotificationOptionWithName(notificationActionIds[1]);
+  await umbracoUi.content.clickSaveModalButton();
+
+  // Assert
+  await umbracoUi.content.isSuccessNotificationVisible();
+  expect(await umbracoApi.document.doesNotificationExist(contentId, notificationActionIds[0])).toBeTruthy();
+  expect(await umbracoApi.document.doesNotificationExist(contentId, notificationActionIds[1])).toBeTruthy();
+});
+
+test('can see notification when content is published', async ({umbracoUi, umbracoApi}) => {
+  // Arrange
+  const notificationActionIds = ['Umb.Document.Publish'];
+  const actionName = 'Publish';
+  await umbracoApi.document.updatetNotifications(contentId, notificationActionIds);
+  expect(await umbracoApi.document.doesNotificationExist(contentId, notificationActionIds[0])).toBeTruthy();
+  await umbracoUi.goToBackOffice();
+  await umbracoUi.content.goToSection(ConstantHelper.sections.content);
+  
+  // Act
+  await umbracoUi.content.clickActionsMenuForContent(contentName);
+  await umbracoUi.content.clickPublishActionMenuOption();
+  await umbracoUi.content.clickConfirmToPublishButton();
+
+  // Assert
+  await umbracoUi.content.isSuccessNotificationVisible();
+  expect(await umbracoApi.smtp.doesNotificationEmailWithSubjectExist(actionName, contentName)).toBeTruthy();
+});
+
+test('can see notification when content is updated', async ({umbracoUi, umbracoApi}) => {
+  // Arrange
+  const notificationActionIds = ['Umb.Document.Update'];
+  const actionName = 'Update';
+  await umbracoApi.document.updatetNotifications(contentId, notificationActionIds);
+  expect(await umbracoApi.document.doesNotificationExist(contentId, notificationActionIds[0])).toBeTruthy();
+  await umbracoUi.goToBackOffice();
+  await umbracoUi.content.goToSection(ConstantHelper.sections.content);
+  
+  // Act
+  await umbracoUi.content.goToContentWithName(contentName);
+  await umbracoUi.content.enterTextstring(contentText);
+  await umbracoUi.content.clickSaveButton();
+
+  // Assert
+  await umbracoUi.content.isSuccessStateVisibleForSaveButton();
+  expect(await umbracoApi.smtp.doesNotificationEmailWithSubjectExist(actionName, contentName)).toBeTruthy();
+});
+
+test('can see notification when content is trashed', async ({umbracoUi, umbracoApi}) => {
+  // Arrange
+  const notificationActionIds = ['Umb.Document.Delete'];
+  const actionName = 'Delete';
+  await umbracoApi.document.updatetNotifications(contentId, notificationActionIds);
+  expect(await umbracoApi.document.doesNotificationExist(contentId, notificationActionIds[0])).toBeTruthy();
+  await umbracoUi.goToBackOffice();
+  await umbracoUi.content.goToSection(ConstantHelper.sections.content);
+  
+  // Act
+  await umbracoUi.content.clickActionsMenuForContent(contentName);
+  await umbracoUi.content.clickTrashActionMenuOption();
+  await umbracoUi.content.clickConfirmTrashButton();
+
+  // Assert
+  await umbracoUi.content.waitForContentToBeTrashed();
+  expect(await umbracoApi.smtp.doesNotificationEmailWithSubjectExist(actionName, contentName)).toBeTruthy();
+});
+
+test('can see notification when child content is created', async ({umbracoUi, umbracoApi}) => {
+  // Arrange
+  const notificationActionIds = ['Umb.Document.Create'];
+  const actionName = 'Create';
+  const childDocumentTypeId = await umbracoApi.documentType.createDefaultDocumentType(childDocumentTypeName);
+  const parentDocumentId = await umbracoApi.documentType.createDocumentTypeWithAllowedChildNode(documentTypeName, childDocumentTypeId);
+  const parentId = await umbracoApi.document.createDefaultDocument(contentName, parentDocumentId) || '';
+  await umbracoApi.document.updatetNotifications(parentId, notificationActionIds);
+  expect(await umbracoApi.document.doesNotificationExist(parentId, notificationActionIds[0])).toBeTruthy();
+  await umbracoUi.goToBackOffice();
+  await umbracoUi.content.goToSection(ConstantHelper.sections.content);
+
+  // Act
+  await umbracoUi.content.clickActionsMenuForContent(contentName);
+  await umbracoUi.content.clickCreateActionMenuOption();
+  await umbracoUi.content.chooseDocumentType(childDocumentTypeName);
+  await umbracoUi.content.enterContentName(childContentName);
+  await umbracoUi.content.clickSaveButton();
+
+  // Assert
+  await umbracoUi.content.waitForContentToBeCreated();
+  expect(await umbracoApi.smtp.doesNotificationEmailWithSubjectExist(actionName, childContentName)).toBeTruthy();
+
+  // Clean
+  await umbracoApi.document.ensureNameNotExists(childContentName);
+  await umbracoApi.documentType.ensureNameNotExists(childDocumentTypeName);
+});
+
+test('can see notification when content is restored', async ({umbracoUi, umbracoApi}) => {
+  // Arrange
+  const notificationActionIds = ['Umb.DocumentRecycleBin.Restore'];
+  const actionName = 'Restore';
+  await umbracoApi.document.updatetNotifications(contentId, notificationActionIds);
+  expect(await umbracoApi.document.doesNotificationExist(contentId, notificationActionIds[0])).toBeTruthy();
+  await umbracoApi.document.moveToRecycleBin(contentId);
+  await umbracoUi.goToBackOffice();
+  await umbracoUi.content.goToSection(ConstantHelper.sections.content);
+  
+  // Act
+  await umbracoUi.content.clickCaretButtonForName('Recycle Bin');
+  await umbracoUi.content.clickActionsMenuForContent(contentName);
+  await umbracoUi.content.clickRestoreActionMenuOption();
+  await umbracoUi.content.clickRestoreButton();
+
+  // Assert
+  await umbracoUi.content.isSuccessNotificationVisible();
+  expect(await umbracoApi.smtp.doesNotificationEmailWithSubjectExist(actionName, contentName)).toBeTruthy();
+});
+
+test('can see notification when content is duplicated', async ({umbracoUi, umbracoApi}) => {
+  // Arrange
+  const notificationActionIds = ['Umb.Document.Duplicate'];
+  const actionName = 'Copy';
+  await umbracoApi.document.updatetNotifications(contentId, notificationActionIds);
+  expect(await umbracoApi.document.doesNotificationExist(contentId, notificationActionIds[0])).toBeTruthy();
+  await umbracoUi.goToBackOffice();
+  await umbracoUi.content.goToSection(ConstantHelper.sections.content);
+  
+  // Act
+  await umbracoUi.content.clickActionsMenuForContent(contentName);
+  await umbracoUi.content.clickDuplicateToActionMenuOption();
+  await umbracoUi.content.clickLabelWithName('Content');
+  await umbracoUi.content.clickDuplicateButton();
+
+  // Assert
+  await umbracoUi.content.isSuccessNotificationVisible();
+  expect(await umbracoApi.smtp.doesNotificationEmailWithSubjectExist(actionName, contentName)).toBeTruthy();
+
+  // Clean
+  await umbracoApi.document.ensureNameNotExists(contentName + ' (1)');
+});
+
+test('can see notification when content is rollbacked', async ({umbracoApi, umbracoUi}) => {
+  // Arrange
+  const notificationActionIds = ['Umb.Document.Rollback'];
+  const actionName = 'Rollback';
+  await umbracoApi.document.updatetNotifications(contentId, notificationActionIds);
+  expect(await umbracoApi.document.doesNotificationExist(contentId, notificationActionIds[0])).toBeTruthy();
+  await umbracoApi.document.publish(contentId);
+  const updatedContentText = 'This is an updated content text';
+  const contentData = await umbracoApi.document.get(contentId);
+  contentData.values[0].value = updatedContentText;
+  await umbracoApi.document.update(contentId, contentData);
+  await umbracoApi.document.publish(contentId);
+  await umbracoUi.goToBackOffice();
+  await umbracoUi.content.goToSection(ConstantHelper.sections.content);
+
+  // Act
+  await umbracoUi.content.goToContentWithName(contentName);
+  await umbracoUi.content.doesDocumentPropertyHaveValue(textStringDataTypeName, updatedContentText);
+  await umbracoUi.content.clickInfoTab();
+  await umbracoUi.content.clickRollbackButton();
+  await umbracoUi.waitForTimeout(700); // Wait for the rollback items to load
+  await umbracoUi.content.clickLatestRollBackItem();
+  await umbracoUi.content.clickRollbackContainerButton();
+
+  // Assert
+  await umbracoUi.content.clickContentTab();
+  await umbracoUi.content.doesDocumentPropertyHaveValue(textStringDataTypeName, contentText);
+  expect(await umbracoApi.smtp.doesNotificationEmailWithSubjectExist(actionName, contentName)).toBeTruthy();
+});
+
+test('can see notification when content is sorted', async ({umbracoApi, umbracoUi}) => {
+  // Arrange
+  const notificationActionIds = ['Umb.Document.Sort'];
+  const actionName = 'Sort';
+  // Create content with children to sort
+  const rootDocumentTypeName = 'RootDocumentType';
+  const childDocumentTypeName = 'ChildDocumentTypeOne';
+  const rootDocumentName = 'RootDocument';
+  const childDocumentOneName = 'FirstChildDocument';
+  const childDocumentTwoName = 'SecondChildDocument';
+  const childDocumentTypeId = await umbracoApi.documentType.createDefaultDocumentType(childDocumentTypeName);
+  const rootDocumentTypeId = await umbracoApi.documentType.createDocumentTypeWithAllowedChildNodeAndDataType(rootDocumentTypeName, childDocumentTypeId, textStringDataTypeName, textStringDataType.id);
+  const rootDocumentId = await umbracoApi.document.createDocumentWithTextContent(rootDocumentName, rootDocumentTypeId, contentText, textStringDataTypeName);
+  await umbracoApi.document.createDefaultDocumentWithParent(childDocumentOneName, childDocumentTypeId, rootDocumentId);
+  await umbracoApi.document.createDefaultDocumentWithParent(childDocumentTwoName, childDocumentTypeId, rootDocumentId);
+  // Set notification on the root document
+  await umbracoApi.document.updatetNotifications(rootDocumentId, notificationActionIds);
+  expect(await umbracoApi.document.doesNotificationExist(rootDocumentId, notificationActionIds[0])).toBeTruthy();
+  await umbracoUi.goToBackOffice();
+  await umbracoUi.content.goToSection(ConstantHelper.sections.content);
+
+  // Act
+  await umbracoUi.content.clickActionsMenuForContent(rootDocumentName);
+  await umbracoUi.content.clickSortChildrenActionMenuOption();
+  const firstDocumentLocator = umbracoUi.content.getTextLocatorWithName(childDocumentOneName);
+  const secondDocumentLocator = umbracoUi.content.getTextLocatorWithName(childDocumentTwoName);
+  await umbracoUi.content.dragAndDrop(secondDocumentLocator, firstDocumentLocator);
+  await umbracoUi.content.clickSortButton();
+
+  // Assert
+  await umbracoUi.content.openContentCaretButtonForName(rootDocumentName);
+  await umbracoUi.content.doesIndexDocumentInTreeContainName(rootDocumentName, childDocumentTwoName, 0);
+  await umbracoUi.content.doesIndexDocumentInTreeContainName(rootDocumentName, childDocumentOneName, 1);
+  expect(await umbracoApi.smtp.doesNotificationEmailWithSubjectExist(actionName, rootDocumentName)).toBeTruthy();
+
+  // Clean
+  await umbracoApi.document.ensureNameNotExists(rootDocumentName);
+  await umbracoApi.documentType.ensureNameNotExists(rootDocumentTypeName);
+  await umbracoApi.documentType.ensureNameNotExists(childDocumentTypeName); 
+});
+
+test('can see notification when content is set public access', async ({umbracoApi, umbracoUi}) => {
+  // Arrange
+  const notificationActionIds = ['Umb.Document.PublicAccess'];
+  const actionName = 'Restrict Public Access';
+  await umbracoApi.document.updatetNotifications(contentId, notificationActionIds);
+  expect(await umbracoApi.document.doesNotificationExist(contentId, notificationActionIds[0])).toBeTruthy();
+  const testMemberGroup = 'TestMemberGroup';
+  await umbracoApi.memberGroup.ensureNameNotExists(testMemberGroup);
+  await umbracoApi.memberGroup.create(testMemberGroup);
+  await umbracoUi.goToBackOffice();
+  await umbracoUi.content.goToSection(ConstantHelper.sections.content);
+
+  // Act
+  await umbracoUi.content.clickActionsMenuForContent(contentName);
+  await umbracoUi.content.clickPublicAccessActionMenuOption();
+  await umbracoUi.content.addGroupBasedPublicAccess(testMemberGroup, contentName);
+
+  // Assert
+  await umbracoUi.content.isSuccessNotificationVisible();
+  expect(await umbracoApi.smtp.doesNotificationEmailWithSubjectExist(actionName, contentName)).toBeTruthy();
+
+  // Clean
+  await umbracoApi.memberGroup.ensureNameNotExists(testMemberGroup);
+});
+