Optimization of Tags for Emails

migrate-tags.md

@@ -0,0 +1,111 @@
+// Connect to the MongoDB database
+use badsender-development;
+
+async function transformTags() {
+try {
+// Step 1: Check if the tags collection exists
+const collections = await db.getCollectionNames();
+if (!collections.includes('tags')) {
+await db.createCollection('tags');
+print('Created tags collection.');
+} else {
+print('Tags collection already exists.');
+}
+
+    // Step 2: Process emails in batches
+    const batchSize = 1000;
+    let skip = 0;
+    let emails;
+
+    while (true) {
+      // Retrieve a batch of emails
+      emails = await db.creations.find().skip(skip).limit(batchSize).toArray();
+      if (emails.length === 0) break;
+
+      print(`Retrieved ${emails.length} emails`);
+
+      // Step 3: Create a dictionary to store unique tags with their usageCount
+      const tagDict = {};
+
+      // Iterate over each email and each tag in the email to populate the dictionary
+      emails.forEach((email) => {
+        email.tags.forEach((tag) => {
+          // Skip tags that are already ObjectIds
+          if (ObjectId.isValid(tag)) return;
+          const key = `${tag}-${email._company}`;
+          if (!tagDict[key]) {
+            tagDict[key] = {
+              label: tag,
+              companyId: email._company,
+              usageCount: 0,
+              createdAt: new Date(),
+              updatedAt: new Date(),
+            };
+          }
+          tagDict[key].usageCount += 1;
+          tagDict[key].updatedAt = new Date();
+        });
+      });
+
+      print(`Created tag dictionary with ${Object.keys(tagDict).length} unique tags`);
+
+      // Step 4: Insert unique tags into the tags collection and get their IDs
+      for (const key of Object.keys(tagDict)) {
+        const tagData = tagDict[key];
+        try {
+          const existingTag = await db.tags.findOne({ label: tagData.label, companyId: tagData.companyId });
+          if (existingTag) {
+            await db.tags.updateOne(
+              { _id: existingTag._id },
+              {
+                $set: {
+                  usageCount: existingTag.usageCount + tagData.usageCount,
+                  updatedAt: new Date(),
+                }
+              }
+            );
+            tagDict[key]._id = existingTag._id;
+          } else {
+            const newTag = await db.tags.insertOne(tagData);
+            tagDict[key]._id = newTag.insertedId;
+          }
+        } catch (err) {
+          print(`Error processing tag ${key}: ${err}`);
+        }
+      }
+
+      print('Tag insertion and update completed');
+
+      // Step 5: Update emails with the new tag references
+      for (const email of emails) {
+        const updatedTags = email.tags.map((tag) => {
+          // If the tag is already an ObjectId, keep it as is
+          if (ObjectId.isValid(tag)) return tag;
+          const key = `${tag}-${email._company}`;
+          return tagDict[key]._id;
+        });
+        try {
+          await db.creations.updateOne(
+            { _id: email._id },
+            { $set: { tags: updatedTags } }
+          );
+        } catch (err) {
+          print(`Error updating email ${email._id}: ${err}`);
+        }
+      }
+
+      print(`Tag transformation completed for ${skip} to ${skip + batchSize} emails`);
+      skip += batchSize;
+    }
+
+    print('Tag transformation completed successfully');
+
+} catch (error) {
+print('Error during tag transformation:', error);
+}
+}
+
+// Call the main function
+transformTags().catch(error => {
+print('Error during tag transformation:', error);
+});

packages/server/common/models.common.js

@@ -21,7 +21,7 @@ const PersonalizedVariableSchema = require('../personalized-variables/personaliz
 const FolderSchema = require('../folder/folder.schema');
 const WorkspaceSchema = require('../workspace/workspace.schema');
 const PersonalizedBlockSchema = require('../personalized-blocks/personalized-block-schema.js');
-
+const TagSchema = require('../tag/tag.schema.js');
 /// ///
 // EXPORTS
 /// ///
@@ -57,6 +57,8 @@ const OAuthClients = mongoose.model(
 );
 const OAuthCodes = mongoose.model(modelNames.OAuthCodes, OAuthCodesSchema);
 
+const Tags = mongoose.model(modelNames.TagModel, TagSchema);
+
 module.exports = {
   mongoose,
   // Compiled schema
@@ -75,4 +77,5 @@ module.exports = {
   OAuthTokens,
   OAuthClients,
   OAuthCodes,
+  Tags,
 };

packages/server/constant/error-codes.js

@@ -95,4 +95,10 @@ module.exports = {
   BAD_FORMAT_PAGINATION: 'BAD_FORMAT_PAGINATION',
   BAD_FORMAT_FILTERS: 'BAD_FORMAT_FILTERS',
   UNEXPECTED_SERVER_ERROR: 'UNEXPECTED_SERVER_ERROR',
+
+  // TAGS
+  TAGS_NOT_FOUND: 'TAGS_NOT_FOUND',
+  TAG_NOT_FOUND: 'TAG_NOT_FOUND',
+  INVALID_TAG_DATA: 'INVALID_TAG_DATA',
+  INVALID_BULK_UPDATE_DATA: 'INVALID_DATA_FOR_BULK_UPDATE',
 };

packages/server/constant/model.names.js

@@ -18,4 +18,6 @@ module.exports = Object.freeze({
   OAuthTokens: 'OAuthTokens',
   OAuthClients: 'OAuthClients',
   OAuthCodes: 'OAuthCodes',
+  // Tags
+  TagModel: 'Tag',
 });

packages/server/mailing/mailing.controller.js

@@ -7,7 +7,8 @@ const {
   Forbidden,
 } = require('http-errors');
 const asyncHandler = require('express-async-handler');
-const { Types } = require('mongoose');
+const mongoose = require('mongoose');
+
 const ERROR_CODES = require('../constant/error-codes.js');
 
 const simpleI18n = require('../helpers/server-simple-i18n.js');
@@ -18,7 +19,6 @@ const {
   downloadZip,
   downloadMultipleZip,
 } = require('./download-zip.controller.js');
-const cleanTagName = require('../helpers/clean-tag-name.js');
 const fileManager = require('../common/file-manage.service.js');
 const modelsUtils = require('../utils/model.js');
 
@@ -283,6 +283,17 @@ async function copy(req, res) {
 
   await mailingService.copyMailing(mailingId, { workspaceId, folderId }, user);
 
+  // Increment tag counts for the copied mailing
+  const originalMailing = await Mailings.findById(mailingId);
+  const tagIds = originalMailing.tags.map((tag) => tag.toString());
+
+  if (tagIds.length > 0) {
+    await mongoose.models.Tag.updateMany(
+      { _id: { $in: tagIds } },
+      { $inc: { usageCount: 1 } }
+    );
+  }
+
   res.status(204).send();
 }
 
@@ -432,37 +443,78 @@ async function updateMosaico(req, res) {
 
 async function bulkUpdate(req, res) {
   const { items, tags: tagsChanges = {} } = req.body;
+  const { id: companyId } = req.user._company;
   const hadId = Array.isArray(items) && items.length;
   const hasTagsChanges =
     Array.isArray(tagsChanges.added) && Array.isArray(tagsChanges.removed);
+
   if (!hadId || !hasTagsChanges) {
     throw new UnprocessableEntity();
   }
 
+  // Separate new tags (those without IDs)
+  const newTags = tagsChanges.added.filter((tag) => !tag._id);
+  const existingTagIds = tagsChanges.added
+    .filter((tag) => tag._id)
+    .map((tag) => tag._id);
+
+  // Create new tags in the database
+  const createdTags = await mongoose.models.Tag.insertMany(
+    newTags.map((tag) => ({
+      label: tag.label,
+      companyId,
+    }))
+  );
+
+  // Gather IDs of newly created tags
+  const newTagIds = createdTags.map((tag) => tag._id.toString());
+
+  // Combine IDs of existing and new tags
+  const combinedTagIdsToAdd = [...existingTagIds, ...newTagIds];
+
   const mailingQuery = modelsUtils.addStrictGroupFilter(req.user, {
-    _id: { $in: items.map(Types.ObjectId) },
+    _id: { $in: items.map(mongoose.Types.ObjectId) },
   });
-  // ensure the mailings are from the same group
+
+  // Ensure mailings belong to the same group
   const userMailings = await Mailings.find(mailingQuery).select({
     _id: 1,
     tags: 1,
   });
-  const updateQueries = userMailings.map((mailing) => {
-    const { tags: orignalTags } = mailing;
+
+  const updateQueries = userMailings.map(async (mailing) => {
+    const originalTags = mailing.tags.map((tag) => tag.toString());
     const uniqueUpdatedTags = [
-      ...new Set([...tagsChanges.added, ...orignalTags]),
+      ...new Set([...combinedTagIdsToAdd, ...originalTags]),
     ];
     const updatedTags = uniqueUpdatedTags.filter(
-      (tag) => !tagsChanges.removed.includes(tag)
+      (tag) => !tagsChanges.removed.some((removedTag) => removedTag._id === tag)
     );
-    mailing.tags = updatedTags.map(cleanTagName).sort();
+
+    const tagsToAdd = updatedTags.filter((tag) => !originalTags.includes(tag));
+    const tagsToRemove = originalTags.filter(
+      (tag) => !updatedTags.includes(tag)
+    );
+
+    // Use schema methods to add and remove multiple tags
+    if (tagsToAdd.length > 0) {
+      await Mailings.addTagsToEmail(mailing._id, tagsToAdd);
+    }
+
+    if (tagsToRemove.length > 0) {
+      await Mailings.removeTagsFromEmail(mailing._id, tagsToRemove);
+    }
+
     return mailing.save();
   });
+
   await Promise.all(updateQueries);
+
   const [mailings, tags] = await Promise.all([
     Mailings.findForApi(mailingQuery),
     Mailings.findTags(modelsUtils.addStrictGroupFilter(req.user, {})),
   ]);
+
   res.json({
     meta: { tags },
     items: mailings,
@@ -487,14 +539,14 @@ async function bulkDestroy(req, res) {
   if (!Array.isArray(items) || !items.length) throw new UnprocessableEntity();
 
   const mailingQuery = modelsUtils.addStrictGroupFilter(req.user, {
-    _id: { $in: items.map(Types.ObjectId) },
+    _id: { $in: items.map(mongoose.Types.ObjectId) },
   });
   // ensure the mailings are from the same group
   const userMailings = await Mailings.find(mailingQuery)
     .select({ _id: 1 })
     .lean();
   const safeMailingsIdList = userMailings.map((mailing) =>
-    Types.ObjectId(mailing._id)
+    mongoose.Types.ObjectId(mailing._id)
   );
   // Mongo responseFormat
   // { n: 1, ok: 1, deletedCount: 1 }
@@ -530,6 +582,16 @@ async function deleteMailing(req, res) {
   const { user } = req;
   const { workspaceId, parentFolderId } = req.body;
 
+  // Find the email before deleting it
+  const mailing = await Mailings.findById(mailingId);
+  const tagIds = mailing.tags.map((tag) => tag.toString());
+
+  // decriment the tag count if the tag is used
+  if (tagIds.length > 0) {
+    await Mailings.removeTagsFromEmail(mailingId, tagIds);
+  }
+
+  // delete the email
   await mailingService.deleteMailing({
     mailingId,
     workspaceId,