Skip to content

Commit

Permalink
count the files deletion and warn if threshold is exceeded
Browse files Browse the repository at this point in the history 
Signed-off-by: Matthieu Gallien <matthieu.gallien@nextcloud.com>
  • Loading branch information
@mgallien
mgallien committed Oct 1, 2024
1 parent 1dea4dc commit 4337b0d
Show file tree
Hide file tree
Showing 4 changed files with 202 additions and 125 deletions.
22 changes: 21 additions & 1 deletion src/libsync/configfile.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ static constexpr char fullLocalDiscoveryIntervalC[] = "fullLocalDiscoveryInterva
static constexpr char notificationRefreshIntervalC[] = "notificationRefreshInterval";
static constexpr char monoIconsC[] = "monoIcons";
static constexpr char promptDeleteC[] = "promptDeleteAllFiles";
static constexpr char deleteFilesThresholdC[] = "deleteFilesThreshold";

Check warning on line 62 in src/libsync/configfile.cpp

View workflow job for this annotation

GitHub Actions / build

src/libsync/configfile.cpp:62:23 [readability-static-definition-in-anonymous-namespace] 

'deleteFilesThresholdC' is a static definition in anonymous namespace; static is redundant here
static constexpr char crashReporterC[] = "crashReporter";
static constexpr char optionalServerNotificationsC[] = "optionalServerNotifications";
static constexpr char showCallNotificationsC[] = "showCallNotifications";
Expand Down Expand Up @@ -113,6 +114,13 @@ static constexpr char certPasswd[] = "http_certificatePasswd";
static const QSet validUpdateChannels { QStringLiteral("stable"), QStringLiteral("beta") };

static constexpr auto macFileProviderModuleEnabledC = "macFileProviderModuleEnabled";

static const QStringList defaultUpdateChannelsList { QStringLiteral("stable"), QStringLiteral("beta"), QStringLiteral("daily") };

Check warning on line 118 in src/libsync/configfile.cpp

View workflow job for this annotation

GitHub Actions / build

src/libsync/configfile.cpp:118:26 [readability-static-definition-in-anonymous-namespace] 

'defaultUpdateChannelsList' is a static definition in anonymous namespace; static is redundant here
static const QString defaultUpdateChannelName = "stable";
static const QStringList enterpriseUpdateChannelsList { QStringLiteral("stable"), QStringLiteral("enterprise") };
static const QString defaultEnterpriseChannel = "enterprise";

static constexpr int deleteFilesThresholdDefaultValue = 100;
}

namespace OCC {
Expand Down Expand Up @@ -1019,7 +1027,7 @@ bool ConfigFile::showMainDialogAsNormalWindow() const {
bool ConfigFile::promptDeleteFiles() const
{
QSettings settings(configFile(), QSettings::IniFormat);
return settings.value(QLatin1String(promptDeleteC), false).toBool();
return settings.value(QLatin1String(promptDeleteC), true).toBool();
}

void ConfigFile::setPromptDeleteFiles(bool promptDeleteFiles)
Expand All @@ -1028,6 +1036,18 @@ void ConfigFile::setPromptDeleteFiles(bool promptDeleteFiles)
settings.setValue(QLatin1String(promptDeleteC), promptDeleteFiles);
}

int ConfigFile::deleteFilesThreshold() const
{
QSettings settings(configFile(), QSettings::IniFormat);
return settings.value(QLatin1String(deleteFilesThresholdC), deleteFilesThresholdDefaultValue).toInt();
}

void ConfigFile::setDeleteFilesThreshold(int thresholdValue)
{
QSettings settings(configFile(), QSettings::IniFormat);
settings.setValue(QLatin1String(deleteFilesThresholdC), thresholdValue);
}

bool ConfigFile::monoIcons() const
{
QSettings settings(configFile(), QSettings::IniFormat);
Expand Down
3 changes: 3 additions & 0 deletions src/libsync/configfile.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,9 @@ class OWNCLOUDSYNC_EXPORT ConfigFile
[[nodiscard]] bool promptDeleteFiles() const;
void setPromptDeleteFiles(bool promptDeleteFiles);

[[nodiscard]] int deleteFilesThreshold() const;
void setDeleteFilesThreshold(int thresholdValue);

[[nodiscard]] bool crashReporter() const;
void setCrashReporter(bool enabled);

Expand Down
293 changes: 169 additions & 124 deletions src/libsync/syncengine.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -799,132 +799,11 @@ void SyncEngine::slotDiscoveryFinished()
_progressInfo->_status = ProgressInfo::Reconcile;
emit transmissionProgress(*_progressInfo);

// qCInfo(lcEngine) << "Permissions of the root folder: " << _csync_ctx->remote.root_perms.toString();
auto finish = [this]{
auto databaseFingerprint = _journal->dataFingerprint();
// If databaseFingerprint is empty, this means that there was no information in the database
// (for example, upgrading from a previous version, or first sync, or server not supporting fingerprint)
if (!databaseFingerprint.isEmpty() && _discoveryPhase
&& _discoveryPhase->_dataFingerprint != databaseFingerprint) {
qCInfo(lcEngine) << "data fingerprint changed, assume restore from backup" << databaseFingerprint << _discoveryPhase->_dataFingerprint;
restoreOldFiles(_syncItems);
}

if (_discoveryPhase->_anotherSyncNeeded && !_discoveryPhase->_filesNeedingScheduledSync.empty()) {
slotScheduleFilesDelayedSync();
} else if (_discoveryPhase->_anotherSyncNeeded && _anotherSyncNeeded == NoFollowUpSync) {
_anotherSyncNeeded = ImmediateFollowUp;
}

if (!_discoveryPhase->_filesUnscheduleSync.empty()) {
slotUnscheduleFilesDelayedSync();
}

if (_discoveryPhase->_hasDownloadRemovedItems && _discoveryPhase->_hasUploadErrorItems) {
for (const auto &item : qAsConst(_syncItems)) {
if (item->_instruction == CSYNC_INSTRUCTION_ERROR && item->_direction == SyncFileItem::Up) {
item->_instruction = CSYNC_INSTRUCTION_IGNORE;
}
}
_anotherSyncNeeded = ImmediateFollowUp;
}

Q_ASSERT(std::is_sorted(_syncItems.begin(), _syncItems.end()));

qCInfo(lcEngine) << "#### Reconcile (aboutToPropagate) #################################################### " << _stopWatch.addLapTime(QStringLiteral("Reconcile (aboutToPropagate)")) << "ms";

_localDiscoveryPaths.clear();

// To announce the beginning of the sync
emit aboutToPropagate(_syncItems);

qCInfo(lcEngine) << "#### Reconcile (aboutToPropagate OK) #################################################### "<< _stopWatch.addLapTime(QStringLiteral("Reconcile (aboutToPropagate OK)")) << "ms";

// it's important to do this before ProgressInfo::start(), to announce start of new sync
_progressInfo->_status = ProgressInfo::Propagation;
emit transmissionProgress(*_progressInfo);
_progressInfo->startEstimateUpdates();

// post update phase script: allow to tweak stuff by a custom script in debug mode.
if (!qEnvironmentVariableIsEmpty("OWNCLOUD_POST_UPDATE_SCRIPT")) {
#ifndef NDEBUG
const QString script = qEnvironmentVariable("OWNCLOUD_POST_UPDATE_SCRIPT");

qCDebug(lcEngine) << "Post Update Script: " << script;
auto scriptArgs = script.split(QRegularExpression("\\s+"), Qt::SkipEmptyParts);
if (scriptArgs.size() > 0) {
const auto scriptExecutable = scriptArgs.takeFirst();
QProcess::execute(scriptExecutable, scriptArgs);
}
#else
qCWarning(lcEngine) << "**** Attention: POST_UPDATE_SCRIPT installed, but not executed because compiled with NDEBUG";
#endif
}

// do a database commit
_journal->commit(QStringLiteral("post treewalk"));

_propagator = QSharedPointer<OwncloudPropagator>(
new OwncloudPropagator(_account, _localPath, _remotePath, _journal, _bulkUploadBlackList));
_propagator->setSyncOptions(_syncOptions);
connect(_propagator.data(), &OwncloudPropagator::itemCompleted,
this, &SyncEngine::slotItemCompleted);
connect(_propagator.data(), &OwncloudPropagator::progress,
this, &SyncEngine::slotProgress);
connect(_propagator.data(), &OwncloudPropagator::finished, this, &SyncEngine::slotPropagationFinished, Qt::QueuedConnection);
connect(_propagator.data(), &OwncloudPropagator::seenLockedFile, this, &SyncEngine::seenLockedFile);
connect(_propagator.data(), &OwncloudPropagator::touchedFile, this, &SyncEngine::slotAddTouchedFile);
connect(_propagator.data(), &OwncloudPropagator::insufficientLocalStorage, this, &SyncEngine::slotInsufficientLocalStorage);
connect(_propagator.data(), &OwncloudPropagator::insufficientRemoteStorage, this, &SyncEngine::slotInsufficientRemoteStorage);
connect(_propagator.data(), &OwncloudPropagator::newItem, this, &SyncEngine::slotNewItem);

// apply the network limits to the propagator
setNetworkLimits(_uploadLimit, _downloadLimit);

deleteStaleDownloadInfos(_syncItems);
deleteStaleUploadInfos(_syncItems);
deleteStaleErrorBlacklistEntries(_syncItems);
_journal->commit(QStringLiteral("post stale entry removal"));

// Emit the started signal only after the propagator has been set up.
if (_needsUpdate)
Q_EMIT started();

_propagator->start(std::move(_syncItems));

qCInfo(lcEngine) << "#### Post-Reconcile end #################################################### " << _stopWatch.addLapTime(QStringLiteral("Post-Reconcile Finished")) << "ms";
};

if (!_hasNoneFiles && _hasRemoveFile) {
qCInfo(lcEngine) << "All the files are going to be changed, asking the user";
int side = 0; // > 0 means more deleted on the server. < 0 means more deleted on the client
foreach (const auto &it, _syncItems) {
if (it->_instruction == CSYNC_INSTRUCTION_REMOVE) {
side += it->_direction == SyncFileItem::Down ? 1 : -1;
}
}

QPointer<QObject> guard = new QObject();
QPointer<QObject> self = this;
auto callback = [this, self, finish, guard](bool cancel) -> void {
// use a guard to ensure its only called once...
// qpointer to self to ensure we still exist
if (!guard || !self) {
return;
}
guard->deleteLater();
if (cancel) {
qCInfo(lcEngine) << "User aborted sync";
finalize(false);
return;
} else {
finish();
}
};
emit aboutToRemoveAllFiles(side >= 0 ? SyncFileItem::Down : SyncFileItem::Up, callback);
if (handleMassDeletion()) {
return;
}
finish();

finishSync();
}

void SyncEngine::slotCleanPollsJobAborted(const QString &error, const ErrorCategory errorCategory)
Expand Down Expand Up @@ -1101,6 +980,172 @@ void SyncEngine::restoreOldFiles(SyncFileItemVector &syncItems)
}
}

void SyncEngine::cancelSyncOrContinue(bool cancel)
{
if (cancel) {
qCInfo(lcEngine) << "User aborted sync";
finalize(false);
} else {
finishSync();
}
}

void SyncEngine::finishSync()
{
auto databaseFingerprint = _journal->dataFingerprint();
// If databaseFingerprint is empty, this means that there was no information in the database
// (for example, upgrading from a previous version, or first sync, or server not supporting fingerprint)
if (!databaseFingerprint.isEmpty() && _discoveryPhase
&& _discoveryPhase->_dataFingerprint != databaseFingerprint) {
qCInfo(lcEngine) << "data fingerprint changed, assume restore from backup" << databaseFingerprint << _discoveryPhase->_dataFingerprint;
restoreOldFiles(_syncItems);
}

if (_discoveryPhase && _discoveryPhase->_anotherSyncNeeded && !_discoveryPhase->_filesNeedingScheduledSync.empty()) {
slotScheduleFilesDelayedSync();
} else if (_discoveryPhase && _discoveryPhase->_anotherSyncNeeded && _anotherSyncNeeded == NoFollowUpSync) {
_anotherSyncNeeded = ImmediateFollowUp;
}

if (_discoveryPhase && !_discoveryPhase->_filesUnscheduleSync.empty()) {
slotUnscheduleFilesDelayedSync();
}

if (_discoveryPhase && _discoveryPhase->_hasDownloadRemovedItems && _discoveryPhase->_hasUploadErrorItems) {
for (const auto &item : qAsConst(_syncItems)) {
if (item->_instruction == CSYNC_INSTRUCTION_ERROR && item->_direction == SyncFileItem::Up) {
// item->_instruction = CSYNC_INSTRUCTION_IGNORE;
}
}
_anotherSyncNeeded = ImmediateFollowUp;
}

Q_ASSERT(std::is_sorted(_syncItems.begin(), _syncItems.end()));

qCInfo(lcEngine) << "#### Reconcile (aboutToPropagate) #################################################### " << _stopWatch.addLapTime(QStringLiteral("Reconcile (aboutToPropagate)")) << "ms";

_localDiscoveryPaths.clear();

// To announce the beginning of the sync
emit aboutToPropagate(_syncItems);

qCInfo(lcEngine) << "#### Reconcile (aboutToPropagate OK) #################################################### "<< _stopWatch.addLapTime(QStringLiteral("Reconcile (aboutToPropagate OK)")) << "ms";

// it's important to do this before ProgressInfo::start(), to announce start of new sync
_progressInfo->_status = ProgressInfo::Propagation;
emit transmissionProgress(*_progressInfo);
_progressInfo->startEstimateUpdates();

// post update phase script: allow to tweak stuff by a custom script in debug mode.
if (!qEnvironmentVariableIsEmpty("OWNCLOUD_POST_UPDATE_SCRIPT")) {
#ifndef NDEBUG
const QString script = qEnvironmentVariable("OWNCLOUD_POST_UPDATE_SCRIPT");

qCDebug(lcEngine) << "Post Update Script: " << script;
auto scriptArgs = script.split(QRegularExpression("\\s+"), Qt::SkipEmptyParts);
if (scriptArgs.size() > 0) {
const auto scriptExecutable = scriptArgs.takeFirst();
QProcess::execute(scriptExecutable, scriptArgs);
}
#else
qCWarning(lcEngine) << "**** Attention: POST_UPDATE_SCRIPT installed, but not executed because compiled with NDEBUG";
#endif
}

// do a database commit
_journal->commit(QStringLiteral("post treewalk"));

_propagator = QSharedPointer<OwncloudPropagator>(
new OwncloudPropagator(_account, _localPath, _remotePath, _journal, _bulkUploadBlackList));
_propagator->setSyncOptions(_syncOptions);
connect(_propagator.data(), &OwncloudPropagator::itemCompleted,
this, &SyncEngine::slotItemCompleted);
connect(_propagator.data(), &OwncloudPropagator::progress,
this, &SyncEngine::slotProgress);
connect(_propagator.data(), &OwncloudPropagator::finished, this, &SyncEngine::slotPropagationFinished, Qt::QueuedConnection);
connect(_propagator.data(), &OwncloudPropagator::seenLockedFile, this, &SyncEngine::seenLockedFile);
connect(_propagator.data(), &OwncloudPropagator::touchedFile, this, &SyncEngine::slotAddTouchedFile);
connect(_propagator.data(), &OwncloudPropagator::insufficientLocalStorage, this, &SyncEngine::slotInsufficientLocalStorage);
connect(_propagator.data(), &OwncloudPropagator::insufficientRemoteStorage, this, &SyncEngine::slotInsufficientRemoteStorage);
connect(_propagator.data(), &OwncloudPropagator::newItem, this, &SyncEngine::slotNewItem);

// apply the network limits to the propagator
setNetworkLimits(_uploadLimit, _downloadLimit);

deleteStaleDownloadInfos(_syncItems);
deleteStaleUploadInfos(_syncItems);
deleteStaleErrorBlacklistEntries(_syncItems);
_journal->commit(QStringLiteral("post stale entry removal"));

// Emit the started signal only after the propagator has been set up.
if (_needsUpdate)
Q_EMIT started();

_propagator->start(std::move(_syncItems));

qCInfo(lcEngine) << "#### Post-Reconcile end #################################################### " << _stopWatch.addLapTime(QStringLiteral("Post-Reconcile Finished")) << "ms";
}

bool SyncEngine::handleMassDeletion()
{
const auto displayDialog = ConfigFile().promptDeleteFiles();
const auto allFilesDeleted = !_hasNoneFiles && _hasRemoveFile;

auto deletionCounter = 0;
for (const auto &oneItem : qAsConst(_syncItems)) {
if (oneItem->_instruction == CSYNC_INSTRUCTION_REMOVE) {
if (oneItem->isDirectory()) {
const auto result = _journal->listFilesInPath(oneItem->_file.toUtf8(), [&deletionCounter] (const auto &oneRecord) {
if (oneRecord.isFile()) {
++deletionCounter;
}
});
if (!result) {
qCDebug(lcEngine()) << "unable to find the number of files within a deleted folder:" << oneItem->_file;
}
} else {
++deletionCounter;
}
}
}
const auto filesDeletedThresholdExceeded = deletionCounter > ConfigFile().deleteFilesThreshold();

if ((allFilesDeleted || filesDeletedThresholdExceeded) && displayDialog) {
qCInfo(lcEngine) << "All the files are going to be changed, asking the user";
int side = 0; // > 0 means more deleted on the server. < 0 means more deleted on the client
for (const auto &it : qAsConst(_syncItems)) {
if (it->_instruction == CSYNC_INSTRUCTION_REMOVE) {
side += it->_direction == SyncFileItem::Down ? 1 : -1;
}
}

promptUserBeforePropagation([this, side](auto &&callback){
emit aboutToRemoveAllFiles(side >= 0 ? SyncFileItem::Down : SyncFileItem::Up, callback);
});
return true;
}
return false;
}

template <typename T>
void SyncEngine::promptUserBeforePropagation(T &&lambda)
{
QPointer<QObject> guard = new QObject();
QPointer<QObject> self = this;
auto callback = [this, self, guard](bool cancel) -> void {
// use a guard to ensure its only called once...
// qpointer to self to ensure we still exist
if (!guard || !self) {
return;
}
guard->deleteLater();

cancelSyncOrContinue(cancel);
};

lambda(callback);
}

void SyncEngine::slotAddTouchedFile(const QString &fn)
{
QElapsedTimer now;
Expand Down
9 changes: 9 additions & 0 deletions src/libsync/syncengine.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,15 @@ private slots:
*/
void restoreOldFiles(SyncFileItemVector &syncItems);

void cancelSyncOrContinue(bool cancel);

void finishSync();

bool handleMassDeletion();

template <typename T>
void promptUserBeforePropagation(T &&lambda);

// true if there is at least one file which was not changed on the server
bool _hasNoneFiles = false;

Expand Down

0 comments on commit 4337b0d

Please sign in to comment.