From bc4d6f04515fec4e25eb15d51a7785c3ba5c46f6 Mon Sep 17 00:00:00 2001 From: n0099 Date: Sat, 11 May 2024 09:26:20 +0800 Subject: [PATCH] * rename method `PostParseHook()` to `OnPostParse()` * rename method `BeforeCommitSaveHook()` to `OnBeforeCommitSave()` * rename method `PostCommitSaveHook()` to `OnPostCommitSave()` @ CrawlFacade.cs * rename method `TriggerPostSave()` to `OnPostSave()` @ IPostSaver.cs * rename delegate and prop `PostSaveHook(s)` to `PostSaveHandler(s)` @ PostSaver.cs * rename method `PostSaveHook()` to `OnPostSave()` @ UserSaver.cs @ c#/crawler --- .../src/Tieba/Crawl/Facade/CrawlFacade.cs | 23 +++++++++++-------- .../Tieba/Crawl/Facade/ReplyCrawlFacade.cs | 4 ++-- .../Tieba/Crawl/Facade/SubReplyCrawlFacade.cs | 4 ++-- .../Crawl/Facade/ThreadArchiveCrawlFacade.cs | 2 +- .../Tieba/Crawl/Facade/ThreadCrawlFacade.cs | 6 ++--- .../Crawl/Saver/IFieldChangeIgnorance.cs | 2 +- .../src/Tieba/Crawl/Saver/Post/IPostSaver.cs | 2 +- .../src/Tieba/Crawl/Saver/Post/PostSaver.cs | 6 ++--- .../src/Tieba/Crawl/Saver/Post/ReplySaver.cs | 4 ++-- .../Tieba/Crawl/Saver/Post/SubReplySaver.cs | 2 +- c#/crawler/src/Tieba/Crawl/Saver/UserSaver.cs | 2 +- 11 files changed, 30 insertions(+), 27 deletions(-) diff --git a/c#/crawler/src/Tieba/Crawl/Facade/CrawlFacade.cs b/c#/crawler/src/Tieba/Crawl/Facade/CrawlFacade.cs index ade5b872..e2487628 100644 --- a/c#/crawler/src/Tieba/Crawl/Facade/CrawlFacade.cs +++ b/c#/crawler/src/Tieba/Crawl/Facade/CrawlFacade.cs @@ -52,13 +52,13 @@ public virtual void Dispose() var userSaver = userSaverFactory(_users); userSaver.Save(db, postSaver.CurrentPostType, postSaver.UserFieldChangeIgnorance); - BeforeCommitSaveHook(db, userSaver); + OnBeforeCommitSave(db, userSaver); try { db.TimestampingEntities(); _ = db.SaveChanges(); transaction.Commit(); - if (savedPosts != null) PostCommitSaveHook(savedPosts, stoppingToken); + if (savedPosts != null) OnPostCommitSave(savedPosts, stoppingToken); return savedPosts; } catch (DbUpdateConcurrencyException e) @@ -67,8 +67,8 @@ public virtual void Dispose() } finally { - postSaver.TriggerPostSave(); - userSaver.PostSaveHook(); + postSaver.OnPostSave(); + userSaver.OnPostSave(); } } } @@ -125,12 +125,12 @@ public ICrawlFacade AddExceptionHandler(ICrawlFacade.ExceptionHand } protected virtual void ThrowIfEmptyUsersEmbedInPosts() { } - protected virtual void PostParseHook( + protected virtual void OnPostParse( TResponse response, CrawlRequestFlag flag, IReadOnlyDictionary parsedPostsInResponse) { } - protected virtual void BeforeCommitSaveHook(CrawlerDbContext db, UserSaver userSaver) { } - protected virtual void PostCommitSaveHook( + protected virtual void OnBeforeCommitSave(CrawlerDbContext db, UserSaver userSaver) { } + protected virtual void OnPostCommitSave( SaverChangeSet savedPosts, CancellationToken stoppingToken = default) { } @@ -145,7 +145,7 @@ private void ValidateThenParse(BaseCrawler.Response re if (postsEmbeddedUsers.Count == 0 && postsInResponse.Count != 0) ThrowIfEmptyUsersEmbedInPosts(); if (postsEmbeddedUsers.Count != 0) UserParser.Parse(postsEmbeddedUsers); } - PostParseHook(response, flag, parsedPostsInResponse); + OnPostParse(response, flag, parsedPostsInResponse); } private async Task CrawlPages( @@ -173,8 +173,11 @@ private async Task CrawlPages( page, previousFailureCountSelector?.Invoke(page) ?? 0, stoppingToken))); } - private async Task LogException - (Func payload, Page page, FailureCount previousFailureCount, CancellationToken stoppingToken = default) + private async Task LogException( + Func payload, + Page page, + FailureCount previousFailureCount, + CancellationToken stoppingToken = default) { try { diff --git a/c#/crawler/src/Tieba/Crawl/Facade/ReplyCrawlFacade.cs b/c#/crawler/src/Tieba/Crawl/Facade/ReplyCrawlFacade.cs index 2715bb99..98ed7bc3 100644 --- a/c#/crawler/src/Tieba/Crawl/Facade/ReplyCrawlFacade.cs +++ b/c#/crawler/src/Tieba/Crawl/Facade/ReplyCrawlFacade.cs @@ -18,7 +18,7 @@ public class ReplyCrawlFacade( { public delegate ReplyCrawlFacade New(Fid fid, Tid tid); - protected override void PostParseHook( + protected override void OnPostParse( ReplyResponse response, CrawlRequestFlag flag, IReadOnlyDictionary parsedPostsInResponse) @@ -30,7 +30,7 @@ protected override void PostParseHook( if (data.Page.CurrentPage == 1) SaveParentThreadTitle(data.PostList); } - protected override void PostCommitSaveHook( + protected override void OnPostCommitSave( SaverChangeSet savedPosts, CancellationToken stoppingToken = default) => sonicPusher.PushPostWithCancellationToken(savedPosts.NewlyAdded, Fid, "replies", diff --git a/c#/crawler/src/Tieba/Crawl/Facade/SubReplyCrawlFacade.cs b/c#/crawler/src/Tieba/Crawl/Facade/SubReplyCrawlFacade.cs index de6c8ae5..4e815552 100644 --- a/c#/crawler/src/Tieba/Crawl/Facade/SubReplyCrawlFacade.cs +++ b/c#/crawler/src/Tieba/Crawl/Facade/SubReplyCrawlFacade.cs @@ -21,7 +21,7 @@ public class SubReplyCrawlFacade( protected override void ThrowIfEmptyUsersEmbedInPosts() => throw new TiebaException( $"User list in the response of sub reply request for fid {Fid}, tid {tid}, pid {pid} is empty."); - protected override void PostParseHook( + protected override void OnPostParse( SubReplyResponse response, CrawlRequestFlag flag, IReadOnlyDictionary parsedPostsInResponse) @@ -34,7 +34,7 @@ protected override void PostParseHook( UserParser.ResetUsersIcon(); } - protected override void PostCommitSaveHook( + protected override void OnPostCommitSave( SaverChangeSet savedPosts, CancellationToken stoppingToken = default) => sonicPusher.PushPostWithCancellationToken(savedPosts.NewlyAdded, Fid, "subReplies", diff --git a/c#/crawler/src/Tieba/Crawl/Facade/ThreadArchiveCrawlFacade.cs b/c#/crawler/src/Tieba/Crawl/Facade/ThreadArchiveCrawlFacade.cs index ee61f408..c333a118 100644 --- a/c#/crawler/src/Tieba/Crawl/Facade/ThreadArchiveCrawlFacade.cs +++ b/c#/crawler/src/Tieba/Crawl/Facade/ThreadArchiveCrawlFacade.cs @@ -16,7 +16,7 @@ public class ThreadArchiveCrawlFacade( { public new delegate ThreadArchiveCrawlFacade New(Fid fid, string forumName); - protected override void PostParseHook( + protected override void OnPostParse( ThreadResponse response, CrawlRequestFlag flag, IReadOnlyDictionary parsedPostsInResponse) diff --git a/c#/crawler/src/Tieba/Crawl/Facade/ThreadCrawlFacade.cs b/c#/crawler/src/Tieba/Crawl/Facade/ThreadCrawlFacade.cs index 3838adc5..d3afc35b 100644 --- a/c#/crawler/src/Tieba/Crawl/Facade/ThreadCrawlFacade.cs +++ b/c#/crawler/src/Tieba/Crawl/Facade/ThreadCrawlFacade.cs @@ -18,8 +18,8 @@ public class ThreadCrawlFacade( public delegate ThreadCrawlFacade New(Fid fid, string forumName); - protected override void BeforeCommitSaveHook(CrawlerDbContext db, UserSaver userSaver) - { // BeforeCommitSaveHook() should get invoked after UserSaver.Save() by the base.SaveCrawled() + protected override void OnBeforeCommitSave(CrawlerDbContext db, UserSaver userSaver) + { // OnBeforeCommitSave() should get invoked after UserSaver.Save() by the base.SaveCrawled() // so only latest repliers that not exists in parsed users are being inserted // note this will bypass user revision detection since not invoking BaseSaver.SavePostsOrUsers() but directly DbContext.AddRange() @@ -42,7 +42,7 @@ protected override void BeforeCommitSaveHook(CrawlerDbContext db, UserSaver user db.Users.AddRange(newLatestRepliersExceptLocked); } - protected override void PostParseHook( + protected override void OnPostParse( ThreadResponse response, CrawlRequestFlag flag, IReadOnlyDictionary parsedPostsInResponse) diff --git a/c#/crawler/src/Tieba/Crawl/Saver/IFieldChangeIgnorance.cs b/c#/crawler/src/Tieba/Crawl/Saver/IFieldChangeIgnorance.cs index d7fbfedc..33c61550 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/IFieldChangeIgnorance.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/IFieldChangeIgnorance.cs @@ -83,7 +83,7 @@ when newValue is "" if (whichPostType == typeof(ThreadPost)) { switch (propName) - { // empty string from response has been updated by ReplyCrawlFacade.PostParseHook() + { // empty string from response has been updated by ReplyCrawlFacade.OnPostParse() case nameof(ThreadPost.Title) when oldValue is "": // null values will be later set by tieba client 6.0.2 response at ThreadParser.ParseInternal() case nameof(ThreadPost.LatestReplierUid) when oldValue is null: diff --git a/c#/crawler/src/Tieba/Crawl/Saver/Post/IPostSaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/Post/IPostSaver.cs index 94f3bc7c..a5353c0b 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/Post/IPostSaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/Post/IPostSaver.cs @@ -4,6 +4,6 @@ public interface IPostSaver where TPost : BasePost { public IFieldChangeIgnorance.FieldChangeIgnoranceDelegates UserFieldChangeIgnorance { get; } public PostType CurrentPostType { get; } - public void TriggerPostSave(); + public void OnPostSave(); public SaverChangeSet Save(CrawlerDbContext db); } diff --git a/c#/crawler/src/Tieba/Crawl/Saver/Post/PostSaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/Post/PostSaver.cs index ba74d317..f148a46a 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/Post/PostSaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/Post/PostSaver.cs @@ -11,8 +11,8 @@ public abstract class PostSaver( where TPost : BasePost where TBaseRevision : BaseRevisionWithSplitting { - protected delegate void PostSaveHook(); - protected PostSaveHook PostSaveHooks { get; set; } = () => { }; + protected delegate void PostSaveHandler(); + protected PostSaveHandler PostSaveHandlers { get; set; } = () => { }; public virtual IFieldChangeIgnorance.FieldChangeIgnoranceDelegates UserFieldChangeIgnorance => throw new NotSupportedException(); @@ -20,7 +20,7 @@ public virtual IFieldChangeIgnorance.FieldChangeIgnoranceDelegates protected ConcurrentDictionary Posts { get; } = posts; protected AuthorRevisionSaver AuthorRevisionSaver { get; } = authorRevisionSaverFactory(currentPostType); - public void TriggerPostSave() => PostSaveHooks(); + public void OnPostSave() => PostSaveHandlers(); public abstract SaverChangeSet Save(CrawlerDbContext db); protected SaverChangeSet Save( diff --git a/c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs index 771a63fa..9dc34bad 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs @@ -54,8 +54,8 @@ public override SaverChangeSet Save(CrawlerDbContext db) db.ReplyContents.AddRange(changeSet.NewlyAdded .Select(r => new ReplyContent {Pid = r.Pid, ProtoBufBytes = r.Content})); SaveReplyContentImages(db, changeSet.NewlyAdded); - PostSaveHooks += AuthorRevisionSaver.SaveAuthorExpGradeRevisions(db, changeSet.AllAfter).Invoke; - PostSaveHooks += SaveReplySignatures(db, changeSet.AllAfter).Invoke; + PostSaveHandlers += AuthorRevisionSaver.SaveAuthorExpGradeRevisions(db, changeSet.AllAfter).Invoke; + PostSaveHandlers += SaveReplySignatures(db, changeSet.AllAfter).Invoke; return changeSet; } diff --git a/c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs index 88ef8473..f63dd8e5 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs @@ -49,7 +49,7 @@ public override SaverChangeSet Save(CrawlerDbContext db) db.SubReplyContents.AddRange(changeSet.NewlyAdded.Select(sr => new SubReplyContent {Spid = sr.Spid, ProtoBufBytes = sr.Content})); - PostSaveHooks += AuthorRevisionSaver.SaveAuthorExpGradeRevisions(db, changeSet.AllAfter).Invoke; + PostSaveHandlers += AuthorRevisionSaver.SaveAuthorExpGradeRevisions(db, changeSet.AllAfter).Invoke; return changeSet; } diff --git a/c#/crawler/src/Tieba/Crawl/Saver/UserSaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/UserSaver.cs index 738c99fb..54fb0e39 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/UserSaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/UserSaver.cs @@ -79,7 +79,7 @@ public IEnumerable AcquireUidLocksForSave(IEnumerable usersId) } } - public void PostSaveHook() + public void OnPostSave() { lock (UserIdLocks) if (_savedUsersId.Count != 0) UserIdLocks.ExceptWith(_savedUsersId); }