Skip to content

Commit

Permalink
refactor: collect creation rates from db query
Browse files Browse the repository at this point in the history
  • Loading branch information
@m90
m90 committed Jun 29, 2023
1 parent 97317f1 commit 2077e1b
Show file tree
Hide file tree
Showing 4 changed files with 60 additions and 68 deletions.
64 changes: 22 additions & 42 deletions app/Jobs/PlatformStatsSummaryJob.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,44 +28,40 @@
class PlatformStatsSummaryJob extends Job
{
private $inactiveThreshold;
private $creationRanges;
private $creationRateRanges;

private $platformSummaryStatsVersion = "v1";
public function __construct() {
$this->inactiveThreshold = Config::get('wbstack.platform_summary_inactive_threshold');
$this->creationRanges = Config::get('wbstack.platform_summary_creation_ranges');
$this->creationRateRanges = Config::get('wbstack.platform_summary_creation_rate_ranges');
}

private function isNullOrEmpty( $value ): bool {
return is_null($value) || intVal($value) === 0;
}

public function prepareStats( array $allStats, $wikis, $users ): array {
public function getCreationStats(): array {
$result = [];
$now = Carbon::now();
foreach ($this->creationRateRanges as $range) {
$limit = $now->clone()->sub(new \DateInterval($range));
$wikis = Wiki::where('created_at', '>=', $limit)->count();
$result['wikis_created_'.$range] = $wikis;
$users = User::where('created_at', '>=', $limit)->count();
$result['users_created_'.$range] = $users;
}
return $result;
}

public function prepareStats( array $allStats, $wikis): array {

$deletedWikis = [];
$activeWikis = [];
$inactive = [];
$emptyWikis = [];
$nonDeletedStats = [];
$createdWikis = [];
$createdUsers = [];
foreach ($this->creationRanges as $range) {
$createdWikis[$range] = [];
$createdUsers[$range] = [];
}

$now = Carbon::now();
$currentTime = $now->timestamp;

foreach ( $users as $user ) {
$createdAt = new Carbon($user->created_at);
foreach ($createdUsers as $range=>$matches) {
$lookback = new \DateInterval($range);
if ($createdAt >= $now->clone()->sub($lookback)) {
$createdUsers[$range][] = $user;
}
}
}
$currentTime = Carbon::now()->timestamp;

foreach( $wikis as $wiki ) {

Expand All @@ -74,14 +70,6 @@ public function prepareStats( array $allStats, $wikis, $users ): array {
continue;
}

$createdAt = new Carbon($wiki->created_at);
foreach ($createdWikis as $range=>$matches) {
$lookback = new \DateInterval($range);
if ($createdAt >= $now->clone()->sub($lookback)) {
$createdWikis[$range][] = $wiki;
}
}

$wikiDb = $wiki->wikiDb()->first();

if( !$wikiDb ) {
Expand Down Expand Up @@ -125,7 +113,7 @@ public function prepareStats( array $allStats, $wikis, $users ): array {
$totalNonDeletedPages = array_sum(array_column($nonDeletedStats, 'pages'));
$totalNonDeletedEdits = array_sum(array_column($nonDeletedStats, 'edits'));

$result = [
return [
'platform_summary_version' => $this->platformSummaryStatsVersion,
'total' => count($wikis),
'deleted' => count($deletedWikis),
Expand All @@ -137,22 +125,11 @@ public function prepareStats( array $allStats, $wikis, $users ): array {
'total_non_deleted_pages' => $totalNonDeletedPages,
'total_non_deleted_edits' => $totalNonDeletedEdits,
];

foreach ($createdWikis as $range=>$items) {
$result['wikis_created_'.$range] = count($items);
}

foreach ($createdUsers as $range=>$items) {
$result['users_created_'.$range] = count($items);
}

return $result;
}

public function handle( DatabaseManager $manager ): void
{
$wikis = Wiki::withTrashed()->with('wikidb')->get();
$users = User::all();

$manager->purge('mw');
$manager->purge('mysql');
Expand All @@ -179,7 +156,10 @@ public function handle( DatabaseManager $manager ): void

// use mw PDO to talk to mediawiki dbs
$allStats = $mediawikiPdo->query($query)->fetchAll(PDO::FETCH_ASSOC);
$summary = $this->prepareStats( $allStats, $wikis, $users );
$summary = $this->prepareStats( $allStats, $wikis );

$creationStats = $this->getCreationStats();
$summary = array_merge($summary, $creationStats);

$manager->purge('mw');
$manager->purge('mysql');
Expand Down
4 changes: 2 additions & 2 deletions config/wbstack.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
'wiki_max_per_user' => env('WBSTACK_MAX_PER_USER', false),

'platform_summary_inactive_threshold' => env('WBSTACK_SUMMARY_INACTIVE_THRESHOLD', 60 * 60 * 24 * 90),
'platform_summary_creation_ranges' => array_filter(
explode(',', env('WBSTACK_SUMMARY_CREATION_RANGES', '')),
'platform_summary_creation_rate_ranges' => array_filter(
explode(',', env('WBSTACK_SUMMARY_CREATION_RATE_RANGES', '')),
function ($item) { return $item !== ''; }
),

Expand Down
2 changes: 1 addition & 1 deletion phpunit.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,6 @@
<env name="CACHE_DRIVER" value="array"/>
<env name="QUEUE_CONNECTION" value="sync"/>
<env name="PLATFORM_MW_BACKEND_HOST" value="mediawiki-139-app-backend.default.svc.cluster.default"/>
<env name="WBSTACK_SUMMARY_CREATION_RANGES" value="PT24H,P30D"/>
<env name="WBSTACK_SUMMARY_CREATION_RATE_RANGES" value="PT24H,P30D"/>
</php>
</phpunit>
58 changes: 35 additions & 23 deletions tests/Jobs/PlatformStatsSummaryJobTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@

namespace Tests\Jobs;

use Illuminate\Foundation\Testing\DatabaseTransactions;
use Illuminate\Database\DatabaseManager;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Tests\TestCase;
use App\User;
use App\Wiki;
Expand All @@ -16,7 +17,7 @@

class PlatformStatsSummaryJobTest extends TestCase
{
use DatabaseTransactions;
use RefreshDatabase;

private $numWikis = 5;
private $wikis = [];
Expand All @@ -30,7 +31,6 @@ protected function setUp(): void {
DB::connection('mysql')->getPdo()->exec("DROP DATABASE IF EXISTS {$this->db_name}{$n};");
}
$this->seedWikis();
$this->manager = $this->app->make('db');
}

protected function tearDown(): void {
Expand All @@ -42,15 +42,14 @@ protected function tearDown(): void {
}

private function seedWikis() {
$manager = $this->app->make('db');
for($n = 0; $n < $this->numWikis; $n++ ) {

$user = User::factory()->create(['verified' => true]);
$wiki = Wiki::factory()->create( [ 'deleted_at' => null ] );
WikiManager::factory()->create(['wiki_id' => $wiki->id, 'user_id' => $user->id]);

$job = new ProvisionWikiDbJob($this->db_prefix . $n, $this->db_name . $n, null);
$job->handle($manager);
$job->handle($this->app->make(DatabaseManager::class));

$wikiDb = WikiDb::whereName($this->db_name.$n)->first();
$wikiDb->update( ['wiki_id' => $wiki->id] );
Expand All @@ -64,15 +63,13 @@ private function seedWikis() {
}
public function testQueryGetsStats()
{
$manager = $this->app->make('db');

$mockJob = $this->createMock(Job::class);
$mockJob->expects($this->never())->method('fail');

$job = new PlatformStatsSummaryJob();
$job->setJob($mockJob);

$job->handle($manager);
$job->handle($this->app->make(DatabaseManager::class));
}

public function testGroupings()
Expand All @@ -84,10 +81,11 @@ public function testGroupings()
$job->setJob($mockJob);

$testWikis = [
Wiki::factory()->create( [ 'deleted_at' => null, 'domain' => 'wiki1.com', 'created_at' => Carbon::now()->subMinutes(4)->timestamp ] ),
Wiki::factory()->create( [ 'deleted_at' => null, 'domain' => 'wiki2.com', 'created_at' => Carbon::now()->subHours(72)->timestamp ] ),
Wiki::factory()->create( [ 'deleted_at' => Carbon::now()->subDays(90)->timestamp, 'domain' => 'wiki3.com', 'created_at' => Carbon::now()->subHours(72)->timestamp ] ),
Wiki::factory()->create( [ 'deleted_at' => null, 'domain' => 'wiki4.com', 'created_at' => Carbon::now()->subYears(6)->timestamp ] )
Wiki::factory()->create( [ 'deleted_at' => null, 'domain' => 'wiki1.com' ] ),
Wiki::factory()->create( [ 'deleted_at' => null, 'domain' => 'wiki2.com' ] ),
Wiki::factory()->create( [ 'deleted_at' => Carbon::now()->subDays(90)->timestamp, 'domain' => 'wiki3.com' ] ),
Wiki::factory()->create( [ 'deleted_at' => null, 'domain' => 'wiki4.com' ] )

];

foreach($testWikis as $wiki) {
Expand Down Expand Up @@ -145,13 +143,8 @@ public function testGroupings()
],
];

$users = [
User::factory()->create([ "created_at" => Carbon::now()->subDays(90)->timestamp ]),
User::factory()->create([ "created_at" => Carbon::now()->subHours(1)->timestamp ]),
User::factory()->create([ "created_at" => Carbon::now()->subHours(48)->timestamp ]),
];

$groups = $job->prepareStats($stats, $testWikis, $users);
$groups = $job->prepareStats($stats, $testWikis);

$this->assertEquals(
[
Expand All @@ -164,15 +157,34 @@ public function testGroupings()
"total_non_deleted_active_users" => 1,
"total_non_deleted_pages" => 2,
"total_non_deleted_edits" => 1,
"platform_summary_version" => "v1",
"wikis_created_PT24H" => 1,
"wikis_created_P30D" => 2,
"users_created_PT24H" => 1,
"users_created_P30D" => 2,
"platform_summary_version" => "v1"
],
$groups,
);
}

function testCreationStats() {
$mockJob = $this->createMock(Job::class);
$mockJob->expects($this->never())->method('fail');

$job = new PlatformStatsSummaryJob();
$job->setJob($mockJob);

$testWikis = [];
$testUsers = [];

$stats = $job->getCreationStats();

$this->assertEquals(
[
'wikis_created_PT24H' => 0,
'wikis_created_P30D' => 0,
'users_created_PT24H' => 0,
'users_created_P30D' => 0,
],
$stats,
);

}

}

0 comments on commit 2077e1b

Please sign in to comment.