Skip to content

Commit

Permalink
feat(platform-stats): provide number of recently created wikis
Browse files Browse the repository at this point in the history
  • Loading branch information
@m90
m90 committed Jun 28, 2023
1 parent ecc032f commit f2eef36
Show file tree
Hide file tree
Showing 3 changed files with 47 additions and 28 deletions.
51 changes: 34 additions & 17 deletions app/Jobs/PlatformStatsSummaryJob.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,7 @@
use PDO;
use Carbon\Carbon;
use Illuminate\Support\Facades\Log;
use Illuminate\Notifications\Notifiable;
use App\Notifications\PlatformStatsSummaryNotification;
use Illuminate\Support\Facades\Config;
use Illuminate\Support\Facades\Notification;
use Illuminate\Support\Facades\App;

/*
Expand All @@ -30,10 +27,12 @@
class PlatformStatsSummaryJob extends Job
{
private $inactiveThreshold;
private $newRanges;

private $platformSummaryStatsVersion = "v1";
public function __construct() {
$this->inactiveThreshold = Config::get('wbstack.platform_summary_inactive_threshold');
$this->newRanges = Config::get('wbstack.platform_summary_new_ranges');
}

private function isNullOrEmpty( $value ): bool {
Expand All @@ -43,12 +42,17 @@ private function isNullOrEmpty( $value ): bool {
public function prepareStats( array $allStats, $wikis ): array {

$deletedWikis = [];
$activeWikis= [];
$inactive = [];
$emptyWikis = [];

$activeWikis = [];
$inactive = [];
$emptyWikis = [];
$newWikis = [];
foreach ($this->newRanges as $range) {
$newWikis[$range] = [];
}
$nonDeletedStats = [];
$currentTime = Carbon::now()->timestamp;

$now = Carbon::now();
$currentTime = $now->timestamp;

foreach( $wikis as $wiki ) {

Expand All @@ -57,10 +61,17 @@ public function prepareStats( array $allStats, $wikis ): array {
continue;
}

$createdAt = new Carbon($wiki->created_at);
foreach ($newWikis as $range=>$matches) {
$lookback = new \DateInterval($range);
if ($createdAt >= $now->clone()->sub($lookback)) {
$newWikis[$range][] = $wiki;
}
}

$wikiDb = $wiki->wikiDb()->first();

if( !$wikiDb ) {

Log::error(__METHOD__ . ": Could not find WikiDB for {$wiki->domain}");
continue;
}
Expand All @@ -71,7 +82,7 @@ public function prepareStats( array $allStats, $wikis ): array {
Log::warning(__METHOD__ . ": Could not find stats for {$wiki->domain}");
continue;
}

$stats = $allStats[$found_key];

// is it empty?
Expand All @@ -86,7 +97,7 @@ public function prepareStats( array $allStats, $wikis ): array {
if(!is_null($stats['lastEdit'])){
$lastTimestamp = intVal($stats['lastEdit']);
$diff = $currentTime - $lastTimestamp;

if ($diff >= $this->inactiveThreshold) {
$inactive[] = $wiki;
continue;
Expand All @@ -95,13 +106,13 @@ public function prepareStats( array $allStats, $wikis ): array {

$activeWikis[] = $wiki;
}

$totalNonDeletedUsers = array_sum(array_column($nonDeletedStats, 'users'));
$totalNonDeletedActiveUsers = array_sum(array_column($nonDeletedStats, 'active_users'));
$totalNonDeletedPages = array_sum(array_column($nonDeletedStats, 'pages'));
$totalNonDeletedEdits = array_sum(array_column($nonDeletedStats, 'edits'));

return [
$result = [
'platform_summary_version' => $this->platformSummaryStatsVersion,
'total' => count($wikis),
'deleted' => count($deletedWikis),
Expand All @@ -113,6 +124,12 @@ public function prepareStats( array $allStats, $wikis ): array {
'total_non_deleted_pages' => $totalNonDeletedPages,
'total_non_deleted_edits' => $totalNonDeletedEdits,
];

foreach ($newWikis as $range=>$items) {
$result['new_wikis_'.$range] = count($items);
}

return $result;
}

public function handle( DatabaseManager $manager ): void
Expand All @@ -133,11 +150,11 @@ public function handle( DatabaseManager $manager ): void
$mediawikiPdo = $mwConn->getPdo();

$pdo->setAttribute(PDO::ATTR_EMULATE_PREPARES, 1);

// prepare the first query
$statement = $pdo->prepare($this->wikiStatsQuery);
$statement->execute();

// produces the stats query
$result = $statement->fetchAll(PDO::FETCH_ASSOC)[0];
$query = array_values($result)[0];
Expand All @@ -148,7 +165,7 @@ public function handle( DatabaseManager $manager ): void

$manager->purge('mw');
$manager->purge('mysql');

// Output to be scraped from logs
if( !App::runningUnitTests() ) {
print( json_encode($summary) . PHP_EOL );
Expand Down Expand Up @@ -183,7 +200,7 @@ public function handle( DatabaseManager $manager ): void
"
) SEPARATOR ' UNION ALL ')
FROM apidb.wiki_dbs
LEFT JOIN apidb.wikis ON wiki_dbs.wiki_id = wikis.id;
Expand Down
1 change: 1 addition & 0 deletions config/wbstack.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
'wiki_max_per_user' => env('WBSTACK_MAX_PER_USER', false),

'platform_summary_inactive_threshold' => env('WBSTACK_SUMMARY_INACTIVE_THRESHOLD', 60 * 60 * 24 * 90),
'platform_summary_new_ranges' => explode(',', env('WBSTACK_SUMMARY_NEW_RANGES', 'PT24H,P30D')),

'elasticsearch_host' => env('ELASTICSEARCH_HOST', false),
'elasticsearch_enabled_by_default' => env('WBSTACK_ELASTICSEARCH_ENABLED_BY_DEFAULT', false),
Expand Down
23 changes: 12 additions & 11 deletions tests/Jobs/PlatformStatsSummaryJobTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,13 +82,12 @@ public function testGroupings()

$job = new PlatformStatsSummaryJob();
$job->setJob($mockJob);

$testWikis = [
Wiki::factory()->create( [ 'deleted_at' => null, 'domain' => 'wiki1.com' ] ),
Wiki::factory()->create( [ 'deleted_at' => null, 'domain' => 'wiki2.com' ] ),
Wiki::factory()->create( [ 'deleted_at' => Carbon::now()->subDays(90)->timestamp, 'domain' => 'wiki3.com' ] ),
Wiki::factory()->create( [ 'deleted_at' => null, 'domain' => 'wiki4.com' ] )

$testWikis = [
Wiki::factory()->create( [ 'deleted_at' => null, 'domain' => 'wiki1.com', 'created_at' => Carbon::now()->subMinutes(4)->timestamp ] ),
Wiki::factory()->create( [ 'deleted_at' => null, 'domain' => 'wiki2.com', 'created_at' => Carbon::now()->subHours(72)->timestamp ] ),
Wiki::factory()->create( [ 'deleted_at' => Carbon::now()->subDays(90)->timestamp, 'domain' => 'wiki3.com', 'created_at' => Carbon::now()->subHours(72)->timestamp ] ),
Wiki::factory()->create( [ 'deleted_at' => null, 'domain' => 'wiki4.com', 'created_at' => Carbon::now()->subYears(6)->timestamp ] )
];

foreach($testWikis as $wiki) {
Expand All @@ -99,7 +98,7 @@ public function testGroupings()
'version' => 'asdasdasdas',
'prefix' => 'asdasd',
'wiki_id' => $wiki->id
]);
]);
}
$stats = [
[ // inactive
Expand Down Expand Up @@ -145,10 +144,10 @@ public function testGroupings()
"platform_summary_version" => "v1"
],
];


$groups = $job->prepareStats($stats, $testWikis);

$this->assertEquals(
[
"total" => 4,
Expand All @@ -160,9 +159,11 @@ public function testGroupings()
"total_non_deleted_active_users" => 1,
"total_non_deleted_pages" => 2,
"total_non_deleted_edits" => 1,
"platform_summary_version" => "v1"
"platform_summary_version" => "v1",
"new_wikis_PT24H" => 1,
"new_wikis_P30D" => 2,
],
$groups,
$groups,
);
}

Expand Down

0 comments on commit f2eef36

Please sign in to comment.