From 1bb060ad1dae5c1abcc8ff1af5fc5f48dc8e6b92 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Wed, 19 Jul 2023 09:44:23 +0200 Subject: [PATCH 01/18] Introduce `Job` & `Schedule` DB model --- library/X509/Job.php | 1 - library/X509/Model/Behavior/DERBase64.php | 2 + .../Model/Behavior/ExpressionInjector.php | 2 + library/X509/Model/Behavior/Ip.php | 2 + library/X509/Model/X509Certificate.php | 2 + library/X509/Model/X509CertificateChain.php | 2 + .../X509/Model/X509CertificateChainLink.php | 2 + .../Model/X509CertificateSubjectAltName.php | 2 + library/X509/Model/X509Dn.php | 2 + library/X509/Model/X509Job.php | 73 +++++++++++++++++++ library/X509/Model/X509JobRun.php | 52 ++++++++++--- library/X509/Model/X509Schedule.php | 70 ++++++++++++++++++ library/X509/Model/X509Target.php | 2 + schema/mysql-upgrades/1.3.0.sql | 39 ++++++++++ schema/mysql.schema.sql | 53 +++++++++++--- schema/pgsql-upgrades/1.3.0.sql | 37 ++++++++++ schema/pgsql.schema.sql | 49 ++++++++++--- 17 files changed, 359 insertions(+), 33 deletions(-) create mode 100644 library/X509/Model/X509Job.php create mode 100644 library/X509/Model/X509Schedule.php create mode 100644 schema/mysql-upgrades/1.3.0.sql create mode 100644 schema/pgsql-upgrades/1.3.0.sql diff --git a/library/X509/Job.php b/library/X509/Job.php index 1937812e..da396e2a 100644 --- a/library/X509/Job.php +++ b/library/X509/Job.php @@ -443,7 +443,6 @@ public function run(): Promise\ExtendedPromiseInterface $this->db->insert('x509_job_run', [ 'name' => $this->getName(), 'start_time' => $this->jobRunStart->getTimestamp() * 1000.0, - 'ctime' => new Expression('UNIX_TIMESTAMP() * 1000'), 'total_targets' => 0, 'finished_targets' => 0 ]); diff --git a/library/X509/Model/Behavior/DERBase64.php b/library/X509/Model/Behavior/DERBase64.php index 402a4194..f7b7215d 100644 --- a/library/X509/Model/Behavior/DERBase64.php +++ b/library/X509/Model/Behavior/DERBase64.php @@ -1,5 +1,7 @@ add(new MillisecondTimestamp([ + 'ctime', + 'mtime' + ])); + } + + public function createRelations(Relations $relations): void + { + $relations->hasMany('schedule', X509Schedule::class) + ->setForeignKey('job_id'); + $relations->hasMany('job_run', X509JobRun::class) + ->setForeignKey('job_id'); + } +} diff --git a/library/X509/Model/X509JobRun.php b/library/X509/Model/X509JobRun.php index 1fa4af58..d776622f 100644 --- a/library/X509/Model/X509JobRun.php +++ b/library/X509/Model/X509JobRun.php @@ -1,43 +1,77 @@ add(new MillisecondTimestamp([ 'start_time', 'end_time', - 'ctime', - 'mtime' ])); } + + public function createRelations(Relations $relations): void + { + $relations->belongsTo('job', X509Job::class) + ->setCandidateKey('job_id'); + $relations->belongsTo('schedule', X509Schedule::class) + ->setJoinType('LEFT') + ->setCandidateKey('schedule_id'); + } } diff --git a/library/X509/Model/X509Schedule.php b/library/X509/Model/X509Schedule.php new file mode 100644 index 00000000..476641aa --- /dev/null +++ b/library/X509/Model/X509Schedule.php @@ -0,0 +1,70 @@ +add(new MillisecondTimestamp([ + 'ctime', + 'mtime' + ])); + } + + public function createRelations(Relations $relations): void + { + $relations->belongsTo('job', X509Job::class) + ->setCandidateKey('job_id'); + $relations->hasMany('job_run', X509JobRun::class) + ->setForeignKey('schedule_id'); + } +} diff --git a/library/X509/Model/X509Target.php b/library/X509/Model/X509Target.php index 978e6f27..7705d573 100644 --- a/library/X509/Model/X509Target.php +++ b/library/X509/Model/X509Target.php @@ -1,5 +1,7 @@ Date: Wed, 19 Jul 2023 09:52:17 +0200 Subject: [PATCH 02/18] Render `Jobs` section as sub section of module main menu --- configuration.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/configuration.php b/configuration.php index 259e1fcc..9930753a 100644 --- a/configuration.php +++ b/configuration.php @@ -20,18 +20,18 @@ 'priority' => 20 )); +$section->add(N_('Jobs'), [ + 'url' => 'x509/jobs', + 'priority' => 100, + 'description' => $this->translate('Configure the scan jobs') +]); + $this->provideConfigTab('backend', array( 'title' => $this->translate('Configure the database backend'), 'label' => $this->translate('Backend'), 'url' => 'config/backend' )); -$this->provideConfigTab('jobs', array( - 'title' => $this->translate('Configure the scan jobs'), - 'label' => $this->translate('Jobs'), - 'url' => 'jobs' -)); - $this->provideConfigTab('sni', array( 'title' => $this->translate('Configure SNI'), 'label' => $this->translate('SNI'), From 329fdbd191ba6ec9e92b5cbdb70d1912ee3ee92e Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Wed, 19 Jul 2023 09:51:28 +0200 Subject: [PATCH 03/18] Separate `Job` & `Schedule` configs --- application/controllers/JobController.php | 228 +++++++++++++++++++++ application/controllers/JobsController.php | 123 +++-------- application/forms/Config/JobConfigForm.php | 174 ---------------- application/forms/Jobs/JobConfigForm.php | 155 ++++++++++++++ application/forms/Jobs/ScheduleForm.php | 201 ++++++++++++++++++ library/X509/Common/Links.php | 37 ++++ library/X509/Widget/JobDetails.php | 60 ++++++ library/X509/Widget/Jobs.php | 62 ++++++ library/X509/Widget/Schedules.php | 60 ++++++ public/css/module.less | 12 ++ schema/mysql.schema.sql | 2 +- 11 files changed, 847 insertions(+), 267 deletions(-) create mode 100644 application/controllers/JobController.php delete mode 100644 application/forms/Config/JobConfigForm.php create mode 100644 application/forms/Jobs/JobConfigForm.php create mode 100644 application/forms/Jobs/ScheduleForm.php create mode 100644 library/X509/Common/Links.php create mode 100644 library/X509/Widget/JobDetails.php create mode 100644 library/X509/Widget/Jobs.php create mode 100644 library/X509/Widget/Schedules.php diff --git a/application/controllers/JobController.php b/application/controllers/JobController.php new file mode 100644 index 00000000..ec39f330 --- /dev/null +++ b/application/controllers/JobController.php @@ -0,0 +1,228 @@ +getTabs()->disableLegacyExtensions(); + + /** @var int $jobId */ + $jobId = $this->params->getRequired('id'); + + /** @var X509Job $job */ + $job = X509Job::on($this->getDb()) + ->filter(Filter::equal('id', $jobId)) + ->first(); + + if ($job === null) { + $this->httpNotFound($this->translate('Job not found')); + } + + $this->job = $job; + } + + public function indexAction(): void + { + $this->assertPermission('config/x509'); + + $this->initTabs(); + $this->getTabs()->activate('job-activities'); + + $jobRuns = $this->job->job_run->with(['job', 'schedule']); + + $limitControl = $this->createLimitControl(); + $sortControl = $this->createSortControl($jobRuns, [ + 'schedule.name' => $this->translate('Schedule Name'), + 'schedule.author' => $this->translate('Author'), + 'total_targets' => $this->translate('Total Targets'), + 'finished_targets' => $this->translate('Finished Targets'), + 'start_time desc' => $this->translate('Started At'), + 'end_time' => $this->translate('Ended At') + ]); + + $this->controls->getAttributes()->add('class', 'default-layout'); + $this->addControl($sortControl); + $this->addControl($limitControl); + $this->addControl($this->createActionBar()); + + $this->addContent(new JobDetails($jobRuns)); + } + + public function updateAction(): void + { + $this->assertPermission('config/x509'); + + $this->addTitleTab($this->translate('Update Job')); + + $form = (new JobConfigForm($this->job)) + ->setAction((string) Url::fromRequest()) + ->populate([ + 'name' => $this->job->name, + 'cidrs' => $this->job->cidrs, + 'ports' => $this->job->ports, + 'exclude_targets' => $this->job->exclude_targets + ]) + ->on(JobConfigForm::ON_SUCCESS, function (JobConfigForm $form) { + /** @var FormSubmitElement $button */ + $button = $form->getPressedSubmitElement(); + if ($button->getName() === 'btn_remove') { + $this->switchToSingleColumnLayout(); + } else { + $this->closeModalAndRefreshRelatedView(Links::job($this->job)); + } + }) + ->handleRequest($this->getServerRequest()); + + $this->addContent($form); + } + + public function schedulesAction(): void + { + $this->assertPermission('config/x509'); + + $this->initTabs(); + $this->getTabs()->activate('schedules'); + + $schedules = $this->job->schedule->with(['job']); + + $sortControl = $this->createSortControl($schedules, [ + 'name' => $this->translate('Name'), + 'author' => $this->translate('Author'), + 'ctime' => $this->translate('Date Created'), + 'mtime' => $this->translate('Date Modified') + ]); + + $this->controls->getAttributes()->add('class', 'default-layout'); + $this->addControl( + (new ButtonLink($this->translate('New Schedule'), Links::scheduleJob($this->job), 'plus')) + ->openInModal() + ); + $this->addControl($sortControl); + + $this->addContent(new Schedules($schedules)); + } + + public function scheduleAction(): void + { + $this->assertPermission('config/x509'); + + $this->addTitleTab($this->translate('Schedule Job')); + + $form = (new ScheduleForm()) + ->setAction((string) Url::fromRequest()) + ->setJobId($this->job->id) + ->on(JobConfigForm::ON_SUCCESS, function () { + $this->redirectNow(Links::schedules($this->job)); + }) + ->handleRequest($this->getServerRequest()); + + $parts = $form->getPartUpdates(); + if (! empty($parts)) { + $this->sendMultipartUpdate(...$parts); + } + + $this->addContent($form); + } + + public function updateScheduleAction(): void + { + $this->assertPermission('config/x509'); + + $this->addTitleTab($this->translate('Update Schedule')); + + /** @var int $id */ + $id = $this->params->getRequired('scheduleId'); + /** @var X509Schedule $schedule */ + $schedule = X509Schedule::on($this->getDb()) + ->filter(Filter::equal('id', $id)) + ->first(); + if ($schedule === null) { + $this->httpNotFound($this->translate('Schedule not found')); + } + + /** @var stdClass $config */ + $config = Json::decode($schedule->config); + /** @var Frequency $type */ + $type = $config->type; + $frequency = $type::fromJson($config->frequency); + + $form = (new ScheduleForm($schedule)) + ->setAction((string) Url::fromRequest()) + ->populate([ + 'name' => $schedule->name, + 'full_scan' => $config->full_scan ?? 'n', + 'rescan' => $config->rescan ?? 'n', + 'since_last_scan' => $config->since_last_scan ?? null, + 'schedule_element' => $frequency + ]) + ->on(JobConfigForm::ON_SUCCESS, function () { + $this->redirectNow(Links::schedules($this->job)); + }) + ->handleRequest($this->getServerRequest()); + + $parts = $form->getPartUpdates(); + if (! empty($parts)) { + $this->sendMultipartUpdate(...$parts); + } + + $this->addContent($form); + } + + protected function createActionBar(): ValidHtml + { + $actions = new ActionBar(); + $actions->addHtml( + (new ActionLink($this->translate('Modify'), Links::updateJob($this->job), 'edit')) + ->openInModal(), + (new ActionLink($this->translate('Schedule'), Links::scheduleJob($this->job), 'calendar')) + ->openInModal() + ); + + return $actions; + } + + protected function initTabs(): void + { + $tabs = $this->getTabs(); + $tabs + ->add('job-activities', [ + 'label' => $this->translate('Job Activities'), + 'url' => Links::job($this->job) + ]) + ->add('schedules', [ + 'label' => $this->translate('Schedules'), + 'url' => Links::schedules($this->job) + ]); + } +} diff --git a/application/controllers/JobsController.php b/application/controllers/JobsController.php index 3beec87b..a51562f9 100644 --- a/application/controllers/JobsController.php +++ b/application/controllers/JobsController.php @@ -4,17 +4,18 @@ namespace Icinga\Module\X509\Controllers; -use Icinga\Module\X509\Forms\Config\JobConfigForm; -use Icinga\Module\X509\JobsIniRepository; -use Icinga\Web\Url; -use ipl\Html\HtmlElement; -use ipl\Scheduler\Contract\Frequency; -use ipl\Scheduler\Cron; +use Icinga\Module\X509\Common\Database; +use Icinga\Module\X509\Forms\Jobs\JobConfigForm; +use Icinga\Module\X509\Model\X509Job; +use Icinga\Module\X509\Widget\Jobs; use ipl\Web\Compat\CompatController; -use stdClass; +use ipl\Web\Url; +use ipl\Web\Widget\ButtonLink; class JobsController extends CompatController { + use Database; + protected function prepareInit() { parent::prepareInit(); @@ -27,104 +28,42 @@ protected function prepareInit() */ public function indexAction() { - $this->view->tabs = $this->Module()->getConfigTabs()->activate('jobs'); + $this->addTitleTab($this->translate('Jobs')); + + $jobs = X509Job::on($this->getDb()); + if ($this->hasPermission('config/x509')) { + $this->addControl( + (new ButtonLink($this->translate('New Job'), Url::fromPath('x509/jobs/new'), 'plus')) + ->openInModal() + ); + } + + $sortControl = $this->createSortControl($jobs, [ + 'name' => $this->translate('Name'), + 'author' => $this->translate('Author'), + 'ctime' => $this->translate('Date Created'), + 'mtime' => $this->translate('Date Modified') + ]); - $repo = new JobsIniRepository(); + $this->controls->getAttributes()->add('class', 'default-layout'); + $this->addControl($sortControl); - $this->view->jobs = $repo->select(array('name')); + $this->addContent(new Jobs($jobs)); } - /** - * Create a job - */ public function newAction() { - $form = $this->prepareForm(true); + $this->assertPermission('config/x509'); $this->addTitleTab($this->translate('New Job')); - $this->addContent($form); - } - /** - * Update a job - */ - public function updateAction() - { - $name = $this->params->getRequired('name'); - $form = $this->prepareForm(); - - $this->addTitleTab($this->translate('Update Job')); - - $this->addContent($form); - } - - /** - * Remove a job - */ - public function removeAction() - { - $name = $this->params->getRequired('name'); - $form = $this->prepareForm(); - - $this->addTitleTab($this->translate('Remove Job')); - - $form->prependHtml(HtmlElement::create('h1', null, sprintf($this->translate('Remove job %s'), $name))); - $this->addContent($form); - } - - /** - * Assert config permission and return a prepared RepositoryForm - * - * @return JobConfigForm - */ - protected function prepareForm(bool $isNew = false) - { - $this->assertPermission('config/x509'); - - $repo = new JobsIniRepository(); $form = (new JobConfigForm()) - ->setRedirectUrl(Url::fromPath('x509/jobs')) - ->setRepo($repo); - - $values = []; - if (! $isNew) { - $name = $this->params->getRequired('name'); - $query = $repo->select()->where('name', $name); - /** @var false|stdClass $data */ - $data = $query->fetchRow(); - if ($data === false) { - $this->httpNotFound($this->translate('Job not found')); - } - - if (! isset($data->frequencyType) && ! empty($data->schedule)) { - $frequency = new Cron($data->schedule); - } elseif (! empty($data->schedule)) { - /** @var Frequency $type */ - $type = $data->frequencyType; - $frequency = $type::fromJson($data->schedule); - } - - $values = [ - 'name' => $data->name, - 'cidrs' => $data->cidrs, - 'ports' => $data->ports, - 'exclude_targets' => $data->exclude_targets, - 'schedule-element' => $frequency ?? [] - ]; - } - - $form - ->populate($values) + ->setAction((string) Url::fromRequest()) ->on(JobConfigForm::ON_SUCCESS, function () { - $this->redirectNow(Url::fromPath('x509/jobs')); + $this->closeModalAndRefreshRelatedView(Url::fromPath('x509/jobs')); }) ->handleRequest($this->getServerRequest()); - $parts = $form->getPartUpdates(); - if (! empty($parts)) { - $this->sendMultipartUpdate(...$parts); - } - - return $form; + $this->addContent($form); } } diff --git a/application/forms/Config/JobConfigForm.php b/application/forms/Config/JobConfigForm.php deleted file mode 100644 index a7ecbbaa..00000000 --- a/application/forms/Config/JobConfigForm.php +++ /dev/null @@ -1,174 +0,0 @@ -identifier = Url::fromRequest()->getParam('name'); - $this->scheduleElement = new ScheduleElement('schedule-element'); - $this->scheduleElement->setIdProtector([Icinga::app()->getRequest(), 'protectId']); - } - - public function setRepo(JobsIniRepository $repo): self - { - $this->repo = $repo; - - return $this; - } - - protected function createFilter() - { - return Filter::where('name', $this->identifier); - } - - protected function isUpdating(): bool - { - return Url::fromRequest()->getPath() === 'x509/jobs/update'; - } - - protected function isRemoving(): bool - { - return Url::fromRequest()->getPath() === 'x509/jobs/remove'; - } - - /** - * Get multipart updates - * - * @return array - */ - public function getPartUpdates(): array - { - if ($this->scheduleElement->getFrequency() === 'none') { - // Workaround for https://github.com/Icinga/ipl-web/issues/130 - return []; - } - - return $this->scheduleElement->prepareMultipartUpdate($this->getRequest()); - } - - protected function assemble() - { - if (! $this->isRemoving()) { - $this->addElement('text', 'name', [ - 'required' => true, - 'description' => t('Job name'), - 'label' => t('Name'), - ]); - $this->addElement('textarea', 'cidrs', [ - 'description' => t('Comma-separated list of CIDR addresses to scan'), - 'label' => t('CIDRs'), - 'required' => true, - 'validators' => [ - new CallbackValidator(function ($value, CallbackValidator $validator): bool { - $cidrValidator = new CidrValidator(); - $cidrs = Str::trimSplit($value); - - foreach ($cidrs as $cidr) { - if (! $cidrValidator->isValid($cidr)) { - $validator->addMessage(...$cidrValidator->getMessages()); - - return false; - } - } - - return true; - }) - ] - ]); - $this->addElement('textarea', 'ports', [ - 'required' => true, - 'description' => t('Comma-separated list of ports to scan'), - 'label' => t('Ports'), - ]); - - $this->addElement('textarea', 'exclude_targets', [ - 'description' => $this->translate('Comma-separated list of addresses/hostnames to exclude'), - 'label' => $this->translate('Exclude Targets'), - 'required' => false - ]); - - $this->addHtml(HtmlElement::create('div', ['class' => 'schedule-element-separator'])); - $this->addElement($this->scheduleElement); - } - - $this->addElement('submit', 'submit', [ - 'label' => $this->isRemoving() ? t('Yes') : ($this->isUpdating() ? t('Update') : t('Create')) - ]); - } - - protected function onSuccess() - { - if ($this->isRemoving()) { - try { - $this->repo->delete($this->repo->getBaseTable(), $this->createFilter()); - - Notification::success(t('Job removed')); - } catch (StatementException $err) { - Notification::error(t('Failed to remove job')); - } - } else { - /** @var Frequency $frequency */ - $frequency = $this->scheduleElement->getValue(); - $data = [ - 'name' => $this->getValue('name'), - 'cidrs' => $this->getValue('cidrs'), - 'ports' => $this->getValue('ports'), - 'schedule' => json_encode($frequency), - 'frequencyType' => get_php_type($frequency), - ]; - - $excludes = $this->getValue('exclude_targets'); - if (! empty($excludes)) { - $data['exclude_targets'] = $excludes; - } - - try { - if ($this->isUpdating()) { - $message = t('Job updated'); - $this->repo->update($this->repo->getBaseTable(), $data, $this->createFilter()); - } else { - $message = t('Job created'); - $this->repo->insert($this->repo->getBaseTable(), $data); - } - - Notification::success($message); - } catch (StatementException $err) { - $message = $this->isUpdating() ? t('Failed to update job') : t('Failed to create job'); - - Notification::error($message . ': ' . $err->getMessage()); - } - } - } -} diff --git a/application/forms/Jobs/JobConfigForm.php b/application/forms/Jobs/JobConfigForm.php new file mode 100644 index 00000000..81355bb4 --- /dev/null +++ b/application/forms/Jobs/JobConfigForm.php @@ -0,0 +1,155 @@ +job = $job; + } + + protected function isUpdating(): bool + { + return $this->job !== null; + } + + public function hasBeenSubmitted(): bool + { + if (! $this->hasBeenSent()) { + return false; + } + + $button = $this->getPressedSubmitElement(); + return $button && ($button->getName() === 'btn_submit' || $button->getName() === 'btn_remove'); + } + + protected function assemble(): void + { + $this->addElement('text', 'name', [ + 'required' => true, + 'label' => $this->translate('Name'), + 'description' => $this->translate('Job name'), + ]); + + $this->addElement('textarea', 'cidrs', [ + 'required' => true, + 'label' => $this->translate('CIDRs'), + 'description' => $this->translate('Comma-separated list of CIDR addresses to scan'), + 'validators' => [ + new CallbackValidator(function ($value, CallbackValidator $validator): bool { + $cidrValidator = new CidrValidator(); + $cidrs = Str::trimSplit($value); + + foreach ($cidrs as $cidr) { + if (! $cidrValidator->isValid($cidr)) { + $validator->addMessage(...$cidrValidator->getMessages()); + + return false; + } + } + + return true; + }) + ] + ]); + + $this->addElement('textarea', 'ports', [ + 'required' => true, + 'label' => $this->translate('Ports'), + 'description' => $this->translate('Comma-separated list of ports to scan'), + ]); + + $this->addElement('textarea', 'exclude_targets', [ + 'required' => false, + 'label' => $this->translate('Exclude Targets'), + 'description' => $this->translate('Comma-separated list of addresses/hostnames to exclude'), + ]); + + $this->addElement('submit', 'btn_submit', [ + 'label' => $this->isUpdating() ? $this->translate('Update') : $this->translate('Create') + ]); + + if ($this->isUpdating()) { + $removeButton = $this->createElement('submit', 'btn_remove', [ + 'class' => 'btn-remove', + 'label' => $this->translate('Remove Job'), + ]); + $this->registerElement($removeButton); + + /** @var HtmlDocument $wrapper */ + $wrapper = $this->getElement('btn_submit')->getWrapper(); + $wrapper->prepend($removeButton); + } + } + + protected function onSuccess(): void + { + $conn = $this->getDb(); + /** @var FormSubmitElement $submitElement */ + $submitElement = $this->getPressedSubmitElement(); + if ($submitElement->getName() === 'btn_remove') { + try { + /** @var X509Job $job */ + $job = $this->job; + $conn->delete('x509_job', ['id = ?' => $job->id]); + + Notification::success($this->translate('Removed job successfully')); + } catch (Exception $err) { + Notification::error($this->translate('Failed to remove job') . ': ' . $err->getMessage()); + } + } else { + $values = $this->getValues(); + + try { + /** @var User $user */ + $user = Auth::getInstance()->getUser(); + if ($this->job === null) { + $values['author'] = $user->getUsername(); + $values['ctime'] = (new DateTime())->getTimestamp() * 1000.0; + $values['mtime'] = (new DateTime())->getTimestamp() * 1000.0; + + $conn->insert('x509_job', $values); + $message = $this->translate('Created job successfully'); + } else { + $values['mtime'] = (new DateTime())->getTimestamp() * 1000.0; + + $conn->update('x509_job', $values, ['id = ?' => $this->job->id]); + $message = $this->translate('Updated job successfully'); + } + + Notification::success($message); + } catch (Exception $err) { + $message = $this->isUpdating() + ? $this->translate('Failed to update job') + : $this->translate('Failed to create job'); + + Notification::error($message . ': ' . $err->getMessage()); + } + } + } +} diff --git a/application/forms/Jobs/ScheduleForm.php b/application/forms/Jobs/ScheduleForm.php new file mode 100644 index 00000000..ea74dcd1 --- /dev/null +++ b/application/forms/Jobs/ScheduleForm.php @@ -0,0 +1,201 @@ +schedule = $schedule; + $this->scheduleElement = new ScheduleElement('schedule_element'); + + /** @var Web $app */ + $app = Icinga::app(); + $this->scheduleElement->setIdProtector([$app->getRequest(), 'protectId']); + } + + protected function isUpdating(): bool + { + return $this->schedule !== null; + } + + public function setJobId(int $jobId): self + { + $this->jobId = $jobId; + + return $this; + } + + /** + * Get multipart updates + * + * @return array + */ + public function getPartUpdates(): array + { + /** @var RequestInterface $request */ + $request = $this->getRequest(); + return $this->scheduleElement->prepareMultipartUpdate($request); + } + + public function hasBeenSubmitted(): bool + { + if (! $this->hasBeenSent()) { + return false; + } + + $button = $this->getPressedSubmitElement(); + return $button && ($button->getName() === 'submit' || $button->getName() === 'btn_remove'); + } + + protected function assemble(): void + { + $this->addElement('text', 'name', [ + 'required' => true, + 'label' => $this->translate('Name'), + 'description' => $this->translate('Schedule name'), + ]); + + $this->addElement('checkbox', 'full_scan', [ + 'required' => false, + 'class' => 'autosubmit', + 'label' => $this->translate('Full Scan'), + 'description' => $this->translate( + 'Scan all known and unknown targets of this job. (Defaults to only scan unknown targets)' + ) + ]); + + if ($this->getPopulatedValue('full_scan', 'n') === 'n') { + $this->addElement('checkbox', 'rescan', [ + 'required' => false, + 'class' => 'autosubmit', + 'label' => $this->translate('Rescan'), + 'description' => $this->translate('Rescan only targets that have been scanned before') + ]); + + $this->addElement('text', 'since_last_scan', [ + 'required' => false, + 'label' => $this->translate('Since Last Scan'), + 'placeholder' => '-24 hours', + 'description' => $this->translate( + 'Scan targets whose last scan is older than the specified date/time, which can also be an' + . ' English textual datetime description like "2 days". If you want to scan only unknown targets' + . ' you can set this to "null".' + ), + 'validators' => [ + new CallbackValidator(function ($value, CallbackValidator $validator) { + if ($value !== null && $value !== 'null') { + try { + new DateTime($value); + } catch (Exception $_) { + $validator->addMessage($this->translate('Invalid textual date time')); + + return false; + } + } + + return true; + }) + ] + ]); + } + + $this->addHtml(HtmlElement::create('div', ['class' => 'schedule-element-separator'])); + $this->addElement($this->scheduleElement); + + $this->addElement('submit', 'submit', [ + 'label' => $this->isUpdating() ? $this->translate('Update') : $this->translate('Schedule') + ]); + + if ($this->isUpdating()) { + $removeButton = $this->createElement('submit', 'btn_remove', [ + 'class' => 'btn-remove', + 'label' => $this->translate('Remove Schedule'), + ]); + $this->registerElement($removeButton); + + /** @var HtmlDocument $wrapper */ + $wrapper = $this->getElement('submit')->getWrapper(); + $wrapper->prepend($removeButton); + } + } + + protected function onSuccess(): void + { + /** @var X509Schedule $schedule */ + $schedule = $this->schedule; + $conn = $this->getDb(); + /** @var FormSubmitElement $submitElement */ + $submitElement = $this->getPressedSubmitElement(); + if ($submitElement->getName() === 'btn_remove') { + $conn->delete('x509_schedule', ['id = ?' => $schedule->id]); + + Notification::success($this->translate('Deleted schedule successfully')); + } else { + $config = $this->getValues(); + unset($config['name']); + unset($config['schedule_element']); + + $frequency = $this->scheduleElement->getValue(); + $config['type'] = get_php_type($frequency); + $config['frequency'] = Json::encode($frequency); + + /** @var User $user */ + $user = Auth::getInstance()->getUser(); + if (! $this->isUpdating()) { + $conn->insert('x509_schedule', [ + 'job_id' => $this->schedule ? $this->schedule->job_id : $this->jobId, + 'name' => $this->getValue('name'), + 'author' => $user->getUsername(), + 'config' => Json::encode($config), + 'ctime' => (new DateTime())->getTimestamp() * 1000.0, + 'mtime' => (new DateTime())->getTimestamp() * 1000.0 + ]); + $message = $this->translate('Created schedule successfully'); + } else { + $conn->update('x509_schedule', [ + 'name' => $this->getValue('name'), + 'config' => Json::encode($config), + 'mtime' => (new DateTime())->getTimestamp() * 1000.0 + ], ['id = ?' => $schedule->id]); + $message = $this->translate('Updated schedule successfully'); + } + + Notification::success($message); + } + } +} diff --git a/library/X509/Common/Links.php b/library/X509/Common/Links.php new file mode 100644 index 00000000..c1570dcb --- /dev/null +++ b/library/X509/Common/Links.php @@ -0,0 +1,37 @@ + $job->id]); + } + + public static function updateJob(X509Job $job): Url + { + return Url::fromPath('x509/job/update', ['id' => $job->id]); + } + + public static function schedules(X509Job $job): Url + { + return Url::fromPath('x509/job/schedules', ['id' => $job->id]); + } + + public static function scheduleJob(X509Job $job): Url + { + return Url::fromPath('x509/job/schedule', ['id' => $job->id]); + } + + public static function updateSchedule(X509Schedule $schedule): Url + { + return Url::fromPath('x509/job/update-schedule', ['id' => $schedule->job->id, 'scheduleId' => $schedule->id]); + } +} diff --git a/library/X509/Widget/JobDetails.php b/library/X509/Widget/JobDetails.php new file mode 100644 index 00000000..c3f8522d --- /dev/null +++ b/library/X509/Widget/JobDetails.php @@ -0,0 +1,60 @@ + 'common-table']; + + /** @var Query */ + protected $runs; + + public function __construct(Query $runs) + { + $this->runs = $runs; + } + + protected function assemble(): void + { + /** @var X509JobRun $run */ + foreach ($this->runs as $run) { + $row = static::tr(); + $row->addHtml( + static::td($run->job->name), + static::td($run->schedule->name ?: $this->translate('N/A')), + static::td((string) $run->total_targets), + static::td((string) $run->finished_targets), + static::td($run->start_time->format('Y-m-d H:i')), + static::td($run->end_time ? $run->end_time->format('Y-m-d H:i') : 'N/A') + ); + + $this->addHtml($row); + } + + if ($this->isEmpty()) { + $this->addHtml(new EmptyState($this->translate('Job never run.'))); + } else { + $row = static::tr(); + $row->addHtml( + static::th($this->translate('Name')), + static::th($this->translate('Schedule Name')), + static::th($this->translate('Total')), + static::th($this->translate('Scanned')), + static::th($this->translate('Started')), + static::th($this->translate('Finished')) + ); + + $this->getHeader()->addHtml($row); + } + } +} diff --git a/library/X509/Widget/Jobs.php b/library/X509/Widget/Jobs.php new file mode 100644 index 00000000..171d0519 --- /dev/null +++ b/library/X509/Widget/Jobs.php @@ -0,0 +1,62 @@ + 'common-table table-row-selectable', + 'data-base-target' => '_next' + ]; + + public function __construct(Query $jobs) + { + $this->jobs = $jobs; + } + + protected function assemble(): void + { + $jobs = $this->jobs->execute(); + if (! $jobs->hasResult()) { + $this->addHtml(new EmptyState($this->translate('No jobs configured yet.'))); + return; + } + + $headers = static::tr(); + $headers->addHtml( + static::th($this->translate('Name')), + static::th($this->translate('Author')), + static::th($this->translate('Date Created')), + static::th($this->translate('Date Modified')) + ); + $this->getHeader()->addHtml($headers); + + /** @var X509Job $job */ + foreach ($jobs as $job) { + $row = static::tr(); + $row->addHtml( + static::td(new Link($job->name, Links::job($job))), + static::td($job->author), + static::td($job->ctime->format('Y-m-d H:i')), + static::td($job->mtime->format('Y-m-d H:i')) + ); + + $this->addHtml($row); + } + } +} diff --git a/library/X509/Widget/Schedules.php b/library/X509/Widget/Schedules.php new file mode 100644 index 00000000..6c3576a8 --- /dev/null +++ b/library/X509/Widget/Schedules.php @@ -0,0 +1,60 @@ + 'common-table table-row-selectable', + 'data-base-target' => '_next' + ]; + + /** @var Query */ + protected $schedules; + + public function __construct(Query $schedules) + { + $this->schedules = $schedules; + } + + protected function assemble(): void + { + /** @var X509Schedule $schedule */ + foreach ($this->schedules as $schedule) { + $row = ModalOpener::addTo(static::tr()); + $row->addHtml( + static::td((new Link(Links::updateSchedule($schedule), $schedule->name))->openInModal()), + static::td($schedule->author), + static::td($schedule->ctime->format('Y-m-d H:i')), + static::td($schedule->mtime->format('Y-m-d H:i')) + ); + + $this->addHtml($row); + } + + if ($this->isEmpty()) { + $this->addHtml(new EmptyState($this->translate('No job schedules.'))); + } else { + $row = static::tr(); + $row->addHtml( + static::th($this->translate('Name')), + static::th($this->translate('Author')), + static::th($this->translate('Date Created')), + static::th($this->translate('Date Modified')) + ); + $this->getHeader()->addHtml($row); + } + } +} diff --git a/public/css/module.less b/public/css/module.less index 3665d339..d4ce29bb 100644 --- a/public/css/module.less +++ b/public/css/module.less @@ -6,6 +6,18 @@ @cert-segment-color-3: #1982C4; @cert-segment-color-4: #6A4C93; +.empty-state { + .rounded-corners(); + background-color: @gray-lighter; + margin: 0 1em; + padding: 1em; + text-align: center; +} + +.action-bar { + line-height: 2.5em; +} + .cert-details { .iicon-certificate { font-size: 5em; diff --git a/schema/mysql.schema.sql b/schema/mysql.schema.sql index cf32c220..8892b357 100644 --- a/schema/mysql.schema.sql +++ b/schema/mysql.schema.sql @@ -96,7 +96,7 @@ CREATE TABLE x509_job ( CREATE TABLE x509_schedule ( id int(10) unsigned NOT NULL AUTO_INCREMENT, - job_id int(10) NOT NULL, + job_id int(10) unsigned NOT NULL, name varchar(255) NOT NULL COLLATE utf8mb4_unicode_ci, author varchar(255) NOT NULL COLLATE utf8mb4_unicode_ci, config text NOT NULL, -- json From 382dda4f10df9273ed40466f4d4eb98907e8ef85 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Wed, 19 Jul 2023 09:54:34 +0200 Subject: [PATCH 04/18] Load `Jobs` & `Schedules` from database for scanning --- application/clicommands/JobsCommand.php | 181 ++++++++++++++------- application/clicommands/ScanCommand.php | 54 +++---- library/X509/Common/JobOptions.php | 145 +++++++++++++++++ library/X509/Common/JobUtils.php | 108 +++++++------ library/X509/Job.php | 206 ++++++++++-------------- library/X509/Schedule.php | 125 ++++++++++++++ 6 files changed, 562 insertions(+), 257 deletions(-) create mode 100644 library/X509/Common/JobOptions.php create mode 100644 library/X509/Schedule.php diff --git a/application/clicommands/JobsCommand.php b/application/clicommands/JobsCommand.php index 378498c4..6ab5a577 100644 --- a/application/clicommands/JobsCommand.php +++ b/application/clicommands/JobsCommand.php @@ -6,18 +6,23 @@ use DateTime; use Exception; -use Icinga\Application\Config; use Icinga\Application\Logger; use Icinga\Module\X509\CertificateUtils; use Icinga\Module\X509\Command; use Icinga\Module\X509\Hook\SniHook; use Icinga\Module\X509\Job; +use Icinga\Module\X509\Model\X509Job; +use Icinga\Module\X509\Model\X509Schedule; +use Icinga\Module\X509\Schedule; +use InvalidArgumentException; +use ipl\Orm\Query; use ipl\Scheduler\Contract\Frequency; -use ipl\Scheduler\Contract\Task; -use ipl\Scheduler\Cron; use ipl\Scheduler\Scheduler; +use ipl\Stdlib\Filter; +use Ramsey\Uuid\Uuid; use React\EventLoop\Loop; use React\Promise\ExtendedPromiseInterface; +use stdClass; use Throwable; class JobsCommand extends Command @@ -27,15 +32,34 @@ class JobsCommand extends Command * * USAGE: * - * icingacli x509 jobs run + * icingacli x509 jobs run [OPTIONS] + * + * OPTIONS + * + * --job= + * Run all configured schedules only of the specified job. + * + * --schedule= + * Run only the given schedule of the specified job. Providing a schedule name + * without a job will fail immediately. + * + * --parallel= + * Allow parallel scanning of targets up to the specified number. Defaults to 256. + * May cause **too many open files** error if set to a number higher than the configured one (ulimit). */ - public function runAction() + public function runAction(): void { - $parallel = (int) $this->Config()->get('scan', 'parallel', 256); + $parallel = (int) $this->params->get('parallel', Job::DEFAULT_PARALLEL); if ($parallel <= 0) { $this->fail("The 'parallel' option must be set to at least 1"); } + $jobName = (string) $this->params->get('job'); + $scheduleName = (string) $this->params->get('schedule'); + if (! $jobName && $scheduleName) { + throw new InvalidArgumentException('You cannot provide a schedule without a job'); + } + $scheduler = new Scheduler(); $this->attachJobsLogging($scheduler); @@ -53,13 +77,14 @@ public function runAction() $scheduled = []; // Periodically check configuration changes to ensure that new jobs are scheduled, jobs are updated, // and deleted jobs are canceled. - $watchdog = function () use (&$watchdog, $scheduler, &$scheduled, $parallel) { - $jobs = $this->fetchJobs(); + $watchdog = function () use (&$watchdog, &$scheduled, $scheduler, $parallel, $jobName, $scheduleName) { + $jobs = $this->fetchSchedules($jobName, $scheduleName); $outdatedJobs = array_diff_key($scheduled, $jobs); foreach ($outdatedJobs as $job) { Logger::info( - 'Removing scheduled job %s, as it either no longer exists in the configuration or its config has' - . ' been changed', + 'Removing schedule %s of job %s, as it either no longer exists in the configuration or its' + . ' config has been changed', + $job->getSchedule()->getName(), $job->getName() ); @@ -70,30 +95,21 @@ public function runAction() foreach ($newJobs as $job) { $job->setParallel($parallel); - $config = $job->getConfig(); - if (! isset($config->frequencyType)) { - if (! Cron::isValid($config->schedule)) { - Logger::error('Job %s has invalid schedule expression %s', $job->getName(), $config->schedule); - - continue; - } - - $frequency = new Cron($config->schedule); - } else { - try { - /** @var Frequency $type */ - $type = $config->frequencyType; - $frequency = $type::fromJson($config->schedule); - } catch (Exception $err) { - Logger::error( - 'Job %s has invalid schedule expression %s: %s', - $job->getName(), - $config->schedule, - $err->getMessage() - ); - - continue; - } + /** @var stdClass $config */ + $config = $job->getSchedule()->getConfig(); + try { + /** @var Frequency $type */ + $type = $config->type; + $frequency = $type::fromJson($config->frequency); + } catch (Throwable $err) { + Logger::error( + 'Cannot create schedule %s of job %s: %s', + $job->getSchedule()->getName(), + $job->getName(), + $err->getMessage() + ); + + continue; } $scheduler->schedule($job, $frequency); @@ -108,28 +124,51 @@ public function runAction() } /** - * Fetch jobs from disk + * Fetch job schedules from database + * + * @param ?string $jobName + * @param ?string $scheduleName * * @return Job[] */ - protected function fetchJobs(): array + protected function fetchSchedules(?string $jobName, ?string $scheduleName): array { - $configs = Config::module($this->getModuleName(), 'jobs', true); - $defaultSchedule = $configs->get('jobs', 'default_schedule'); + $jobs = X509Job::on($this->getDb()); + if ($jobName) { + $jobs->filter(Filter::equal('name', $jobName)); + } + + $jobSchedules = []; + $snimap = SniHook::getAll(); + /** @var X509Job $jobConfig */ + foreach ($jobs as $jobConfig) { + $job = (new Job($jobConfig->name, $jobConfig->cidrs, $jobConfig->ports, $snimap)) + ->setId($jobConfig->id) + ->setExcludes($jobConfig->exclude_targets); + + /** @var Query $schedules */ + $schedules = $jobConfig->schedule; + if ($scheduleName) { + $schedules->filter(Filter::equal('name', $scheduleName)); + } - $jobs = []; - foreach ($configs as $name => $config) { - if (! $config->get('schedule', $defaultSchedule)) { - Logger::debug('Job %s cannot be scheduled', $name); + $jobSchedules = []; + /** @var X509Schedule $scheduleModel */ + foreach ($schedules as $scheduleModel) { + $schedule = Schedule::fromModel($scheduleModel); + $job = (clone $job) + ->setSchedule($schedule) + ->setUuid(Uuid::fromBytes($job->getChecksum())); - continue; + $jobSchedules[$job->getUuid()->toString()] = $job; } - $job = new Job($name, $config, SniHook::getAll()); - $jobs[$job->getUuid()->toString()] = $job; + if (! isset($jobSchedules[$job->getUuid()->toString()])) { + Logger::info('Skipping job %s because no schedules are configured', $job->getName()); + } } - return $jobs; + return $jobSchedules; } /** @@ -139,15 +178,24 @@ protected function fetchJobs(): array */ protected function attachJobsLogging(Scheduler $scheduler): void { - $scheduler->on(Scheduler::ON_TASK_CANCEL, function (Task $job, array $_) { - Logger::info('Job %s canceled', $job->getName()); + $scheduler->on(Scheduler::ON_TASK_CANCEL, function (Job $task, array $_) { + Logger::info('Schedule %s of job %s canceled', $task->getSchedule()->getName(), $task->getName()); }); - $scheduler->on(Scheduler::ON_TASK_DONE, function (Task $job, $targets = 0) { + $scheduler->on(Scheduler::ON_TASK_DONE, function (Job $task, $targets = 0) { if ($targets === 0) { - Logger::warning('The job %s does not have any targets', $job->getName()); + Logger::warning( + 'Schedule %s of job %s does not have any targets', + $task->getSchedule()->getName(), + $task->getName() + ); } else { - Logger::info('Scanned %d target(s) from job %s', $targets, $job->getName()); + Logger::info( + 'Scanned %d target(s) by schedule %s of job %s', + $targets, + $task->getSchedule()->getName(), + $task->getName() + ); try { $verified = CertificateUtils::verifyCertificates($this->getDb()); @@ -160,21 +208,36 @@ protected function attachJobsLogging(Scheduler $scheduler): void } }); - $scheduler->on(Scheduler::ON_TASK_FAILED, function (Task $job, Throwable $e) { - Logger::error('Failed to run job %s: %s', $job->getName(), $e->getMessage()); + $scheduler->on(Scheduler::ON_TASK_FAILED, function (Job $task, Throwable $e) { + Logger::error( + 'Failed to run schedule %s of job %s: %s', + $task->getSchedule()->getName(), + $task->getName(), + $e->getMessage() + ); Logger::debug($e->getTraceAsString()); }); - $scheduler->on(Scheduler::ON_TASK_RUN, function (Task $job, ExtendedPromiseInterface $_) { - Logger::info('Running job %s', $job->getName()); + $scheduler->on(Scheduler::ON_TASK_RUN, function (Job $task, ExtendedPromiseInterface $_) { + Logger::info('Running schedule %s of job %s', $task->getSchedule()->getName(), $task->getName()); }); - $scheduler->on(Scheduler::ON_TASK_SCHEDULED, function (Task $job, DateTime $dateTime) { - Logger::info('Scheduling job %s to run at %s', $job->getName(), $dateTime->format('Y-m-d H:i:s')); + $scheduler->on(Scheduler::ON_TASK_SCHEDULED, function (Job $task, DateTime $dateTime) { + Logger::info( + 'Scheduling %s of job %s to run at %s', + $task->getSchedule()->getName(), + $task->getName(), + $dateTime->format('Y-m-d H:i:s') + ); }); - $scheduler->on(Scheduler::ON_TASK_EXPIRED, function (Task $task, DateTime $dateTime) { - Logger::info(sprintf('Detaching expired job %s at %s', $task->getName(), $dateTime->format('Y-m-d H:i:s'))); + $scheduler->on(Scheduler::ON_TASK_EXPIRED, function (Job $task, DateTime $dateTime) { + Logger::info( + 'Detaching expired schedule %s of job %s at %s', + $task->getSchedule()->getName(), + $task->getName(), + $dateTime->format('Y-m-d H:i:s') + ); }); } } diff --git a/application/clicommands/ScanCommand.php b/application/clicommands/ScanCommand.php index c633ecec..eb3123ea 100644 --- a/application/clicommands/ScanCommand.php +++ b/application/clicommands/ScanCommand.php @@ -4,14 +4,14 @@ namespace Icinga\Module\X509\Clicommands; -use DateTime; use Exception; use Icinga\Application\Logger; use Icinga\Module\X509\CertificateUtils; use Icinga\Module\X509\Command; use Icinga\Module\X509\Hook\SniHook; use Icinga\Module\X509\Job; -use InvalidArgumentException; +use Icinga\Module\X509\Model\X509Job; +use ipl\Stdlib\Filter; use React\EventLoop\Loop; use Throwable; @@ -45,6 +45,10 @@ class ScanCommand extends Command * which can also be an English textual datetime description like "2 days". * Defaults to "-24 hours". * + * --parallel= + * Allow parallel scanning of targets up to the specified number. Defaults to 256. + * May cause **too many open files** error if set to a number higher than the configured one (ulimit). + * * --rescan * Rescan only targets that have been scanned before. * @@ -74,51 +78,43 @@ class ScanCommand extends Command * * icingacli x509 scan --job --full */ - public function indexAction() + public function indexAction(): void { + /** @var string $name */ $name = $this->params->shiftRequired('job'); $fullScan = (bool) $this->params->get('full', false); $rescan = (bool) $this->params->get('rescan', false); - $parallel = (int) $this->Config()->get('scan', 'parallel', 256); + /** @var string $sinceLastScan */ + $sinceLastScan = $this->params->get('since-last-scan', Job::DEFAULT_SINCE_LAST_SCAN); + if ($sinceLastScan === 'null') { + $sinceLastScan = null; + } + + /** @var int $parallel */ + $parallel = $this->params->get('parallel', Job::DEFAULT_PARALLEL); if ($parallel <= 0) { throw new Exception('The \'parallel\' option must be set to at least 1'); } - $jobs = $this->Config('jobs'); - if (! $jobs->hasSection($name)) { + /** @var X509Job $jobConfig */ + $jobConfig = X509Job::on($this->getDb()) + ->filter(Filter::equal('name', $name)) + ->first(); + if ($jobConfig === null) { throw new Exception(sprintf('Job %s not found', $name)); } - $jobDescription = $this->Config('jobs')->getSection($name); - if (! strlen($jobDescription->get('cidrs'))) { + if (! strlen($jobConfig->cidrs)) { throw new Exception(sprintf('The job %s does not specify any CIDRs', $name)); } - $sinceLastScan = $this->params->get('since-last-scan', '-24 hours'); - if ($sinceLastScan === 'null') { - $sinceLastScan = null; - } else { - if ($sinceLastScan[0] !== '-') { - // When the user specified "2 days" as a threshold strtotime() will compute the - // timestamp NOW() + 2 days, but it has to be NOW() + (-2 days) - $sinceLastScan = "-$sinceLastScan"; - } - - try { - $sinceLastScan = new DateTime($sinceLastScan); - } catch (Exception $_) { - throw new InvalidArgumentException(sprintf( - 'The specified last scan time is in an unknown format: %s', - $this->params->get('since-last-scan') - )); - } - } - - $job = (new Job($name, $jobDescription, SniHook::getAll())) + $job = (new Job($name, $jobConfig->cidrs, $jobConfig->ports, SniHook::getAll())) + ->setId($jobConfig->id) ->setFullScan($fullScan) ->setRescan($rescan) ->setParallel($parallel) + ->setExcludes($jobConfig->exclude_targets) ->setLastScan($sinceLastScan); $promise = $job->run(); diff --git a/library/X509/Common/JobOptions.php b/library/X509/Common/JobOptions.php new file mode 100644 index 00000000..b0fcb99b --- /dev/null +++ b/library/X509/Common/JobOptions.php @@ -0,0 +1,145 @@ +rescan; + } + + /** + * Set whether this job should do only a rescan or full scan + * + * @param bool $rescan + * + * @return $this + */ + public function setRescan(bool $rescan): self + { + $this->rescan = $rescan; + + return $this; + } + + public function getParallel(): int + { + return $this->parallel; + } + + public function setParallel(int $parallel): self + { + $this->parallel = $parallel; + + return $this; + } + + /** + * Set whether this job should scan all known and unknown targets + * + * @param bool $fullScan + * + * @return $this + */ + public function setFullScan(bool $fullScan): self + { + $this->fullScan = $fullScan; + + return $this; + } + + /** + * Set since last scan threshold for the targets to rescan + * + * @param ?string $time + * + * @return $this + */ + public function setLastScan(?string $time): self + { + if ($time && $time !== 'null') { + $sinceLastScan = $time; + if ($sinceLastScan[0] !== '-') { + // When the user specified "2 days" as a threshold strtotime() will compute the + // timestamp NOW() + 2 days, but it has to be NOW() + (-2 days) + $sinceLastScan = "-$sinceLastScan"; + } + + try { + $this->sinceLastScan = new DateTime($sinceLastScan); + } catch (Exception $_) { + throw new InvalidArgumentException(sprintf( + 'The specified last scan time is in an unknown format: %s', + $time + )); + } + } + + return $this; + } + + /** + * Get the schedule config of this job + * + * @return Schedule + */ + public function getSchedule(): Schedule + { + if (! $this->schedule) { + throw new LogicException('You are accessing an unset property. Please make sure to set it beforehand.'); + } + + return $this->schedule; + } + + /** + * Set the schedule config of this job + * + * @param Schedule $schedule + * + * @return $this + */ + public function setSchedule(Schedule $schedule): self + { + $this->schedule = $schedule; + + /** @var stdClass $config */ + $config = $schedule->getConfig(); + $this->setRescan($config->rescan); + $this->setFullScan($config->full_scan); + $this->setLastScan($config->since_last_scan ?? Job::DEFAULT_SINCE_LAST_SCAN); + + return $this; + } +} diff --git a/library/X509/Common/JobUtils.php b/library/X509/Common/JobUtils.php index d868a4ee..4a81c5bd 100644 --- a/library/X509/Common/JobUtils.php +++ b/library/X509/Common/JobUtils.php @@ -6,91 +6,109 @@ use GMP; use Icinga\Application\Logger; -use Icinga\Data\ConfigObject; -use ipl\Scheduler\Common\TaskProperties; use ipl\Stdlib\Str; trait JobUtils { - use TaskProperties; - - /** @var ConfigObject A config for this job loaded from the jobs.ini file */ - private $config; + /** @var array A list of excluded IP addresses and host names */ + private $excludedTargets = []; + /** @var array> */ private $cidrs = []; + /** @var array> */ private $ports = []; /** - * Get this job's config + * Get the configured job CIDRS * - * @return ConfigObject + * @return array> */ - public function getConfig(): ConfigObject + public function getCIDRs(): array { - return $this->config; + return $this->cidrs; } /** - * Set the config of this job + * Set the CIDRs of this job * - * @param ConfigObject $jobConfig + * @param string $cidrs * * @return $this */ - public function setConfig(ConfigObject $jobConfig): self + public function setCIDRs(string $cidrs): self { - $this->config = $jobConfig; + foreach (Str::trimSplit($cidrs) as $cidr) { + $pieces = Str::trimSplit($cidr, '/'); + if (count($pieces) !== 2) { + Logger::warning('CIDR %s is in the wrong format', $cidr); + continue; + } + + $this->cidrs[$cidr] = $pieces; + } return $this; } /** - * Get the configured job CIDRS as an array + * Get the configured ports of this job + * + * @return array> + */ + public function getPorts(): array + { + return $this->ports; + } + + /** + * Set the ports of this job to be scanned * - * @return array + * @param string $ports + * + * @return $this */ - public function getCidrs(): array + public function setPorts(string $ports): self { - if (empty($this->cidrs) && ! $this->config->isEmpty()) { - $cidrs = Str::trimSplit($this->config->get('cidrs')); - foreach ($cidrs as $cidr) { - $pieces = Str::trimSplit($cidr, '/'); - if (count($pieces) !== 2) { - Logger::warning('CIDR %s is in the wrong format', $cidr); - continue; - } - - $this->cidrs[$cidr] = $pieces; + foreach (Str::trimSplit($ports) as $portRange) { + $pieces = Str::trimSplit($portRange, '-'); + if (count($pieces) === 2) { + list($start, $end) = $pieces; + } else { + $start = $pieces[0]; + $end = $pieces[0]; } + + $this->ports[] = [$start, $end]; } - return $this->cidrs; + return $this; } /** - * Get the configured ports of this job + * Get excluded IPs and host names * - * @return array + * @return array */ - public function getPorts(): array + public function getExcludes(): array { - if (empty($this->ports) && ! $this->config->isEmpty()) { - $ports = Str::trimSplit($this->config->get('ports')); - foreach ($ports as $portRange) { - $pieces = Str::trimSplit($portRange, '-'); - if (count($pieces) === 2) { - list($start, $end) = $pieces; - } else { - $start = $pieces[0]; - $end = $pieces[0]; - } - - $this->ports[] = [$start, $end]; - } + return $this->excludedTargets; + } + + /** + * Set a set of IPs and host names to be excluded from scan + * + * @param ?string $targets + * + * @return $this + */ + public function setExcludes(?string $targets): self + { + if (! empty($targets)) { + $this->excludedTargets = array_flip(Str::trimSplit($targets)); } - return $this->ports; + return $this; } /** diff --git a/library/X509/Job.php b/library/X509/Job.php index da396e2a..20a89241 100644 --- a/library/X509/Job.php +++ b/library/X509/Job.php @@ -8,19 +8,20 @@ use Exception; use Generator; use Icinga\Application\Logger; -use Icinga\Data\ConfigObject; use Icinga\Module\X509\Common\Database; +use Icinga\Module\X509\Common\JobOptions; use Icinga\Module\X509\Common\JobUtils; use Icinga\Module\X509\Model\X509Certificate; use Icinga\Module\X509\Model\X509CertificateChain; +use Icinga\Module\X509\Model\X509JobRun; use Icinga\Module\X509\Model\X509Target; use Icinga\Module\X509\React\StreamOptsCaptureConnector; use Icinga\Util\Json; +use ipl\Scheduler\Common\TaskProperties; use ipl\Scheduler\Contract\Task; use ipl\Sql\Connection; use ipl\Sql\Expression; use ipl\Stdlib\Filter; -use ipl\Stdlib\Str; use LogicException; use Ramsey\Uuid\Uuid; use React\EventLoop\Loop; @@ -35,7 +36,17 @@ class Job implements Task { use Database; + use JobOptions; use JobUtils; + use TaskProperties; + + /** @var int Number of targets to be scanned in parallel by default */ + public const DEFAULT_PARALLEL = 256; + + public const DEFAULT_SINCE_LAST_SCAN = '-24 hours'; + + /** @var int The database id of this job */ + protected $id; /** @var Connection x509 database connection */ private $db; @@ -43,67 +54,71 @@ class Job implements Task /** @var DbTool Database utils for marshalling and unmarshalling binary data */ private $dbTool; + /** @var int Number of pending targets to be scanned */ private $pendingTargets = 0; + + /** @var int Total number of scan targets */ private $totalTargets = 0; + + /** @var int Number of scanned targets */ private $finishedTargets = 0; - /** @var Generator */ + /** @var Generator Scan targets generator */ private $targets; + + /** @var array> The configured SNI maps */ private $snimap; - protected $jobId; + /** @var int The id of the last inserted job run entry */ + private $jobRunId; /** @var Promise\Deferred React promise deferred instance used to resolve the running promise */ protected $deferred; - /** @var int Used to control how many targets can be scanned in parallel */ - protected $parallel; - - /** @var DateTime A formatted date time of this job start time */ + /** @var DateTime The start time of this job */ protected $jobRunStart; - /** @var ?array A list of excluded IP addresses and host names */ - protected $excludedTargets = null; - - /** @var ?DateTime Since last scan threshold used to filter out scan targets */ - protected $sinceLastScan; - - /** @var bool Whether job run should only perform a rescan */ - protected $rescan = false; - - /** @var bool Whether the job run should perform a full scan */ - protected $fullScan = false; - - public function __construct(string $name, ConfigObject $config, array $snimap) + public function __construct(string $name, string $cidrs, string $ports, array $snimap, Schedule $schedule = null) { $this->db = $this->getDb(); $this->dbTool = new DbTool($this->db); $this->snimap = $snimap; - $this->setConfig($config); + if ($schedule) { + $this->setSchedule($schedule); + } + $this->setName($name); + $this->setCIDRs($cidrs); + $this->setPorts($ports); $this->setUuid(Uuid::fromBytes($this->getChecksum())); } /** - * Get excluded IPs and host names + * Get the database id of this job * - * @return array + * @return int */ - protected function getExcludes(): array + public function getId(): int { - if ($this->excludedTargets === null) { - $config = $this->getConfig(); - $this->excludedTargets = []; - if (isset($config['exclude_targets']) && ! empty($config['exclude_targets'])) { - $this->excludedTargets = array_flip(Str::trimSplit($config['exclude_targets'])); - } - } + return $this->id; + } - return $this->excludedTargets; + /** + * Set the database id of this job + * + * @param int $id + * + * @return $this + */ + public function setId(int $id): self + { + $this->id = $id; + + return $this; } - private function getConnector($peerName) + private function getConnector($peerName): array { $simpleConnector = new Connector(); $streamCaptureConnector = new StreamOptsCaptureConnector($simpleConnector); @@ -118,7 +133,7 @@ private function getConnector($peerName) } /** - * Get whether this task has been completed + * Get whether this job has been completed scanning all targets * * @return bool */ @@ -127,71 +142,7 @@ public function isFinished(): bool return ! $this->targets->valid() && $this->pendingTargets === 0; } - public function getParallel(): int - { - return $this->parallel; - } - - public function setParallel(int $parallel): self - { - $this->parallel = $parallel; - - return $this; - } - - /** - * Get whether this job run should do only a rescan - * - * @return bool - */ - public function isRescan(): bool - { - return $this->rescan; - } - - /** - * Set whether this job run should do only a rescan or full scan - * - * @param bool $rescan - * - * @return $this - */ - public function setRescan(bool $rescan): self - { - $this->rescan = $rescan; - - return $this; - } - - /** - * Set whether this job run should scan all known and unknown targets - * - * @param bool $fullScan - * - * @return $this - */ - public function setFullScan(bool $fullScan): self - { - $this->fullScan = $fullScan; - - return $this; - } - - /** - * Set since last scan threshold for the targets to rescan - * - * @param ?DateTime $dateTime - * - * @return $this - */ - public function setLastScan(?DateTime $dateTime): self - { - $this->sinceLastScan = $dateTime; - - return $this; - } - - protected function updateLastScan($target) + public function updateLastScan($target) { if (! $this->isRescan()) { return; @@ -202,12 +153,21 @@ protected function updateLastScan($target) ], ['id = ?' => $target->id]); } - protected function getChecksum() + public function getChecksum(): string { - $config = $this->getConfig()->toArray(); - ksort($config); + $data = [ + 'name' => $this->getName(), + 'cidrs' => $this->getCIDRs(), + 'ports' => $this->getPorts(), + 'exclude_targets' => $this->getExcludes(), + ]; + + $schedule = null; + if ($this->schedule) { + $schedule = $this->getSchedule(); + } - return md5($this->getName() . Json::encode($config), true); + return md5(Json::encode($data) . ($schedule ? bin2hex($schedule->getChecksum()) : ''), true); } protected function getScanTargets(): Generator @@ -225,9 +185,9 @@ protected function getScanTargets(): Generator foreach ($targets as $target) { $addr = static::addrToNumber($target->ip); $addrFound = false; - foreach ($this->getCidrs() as $cidr) { + foreach ($this->getCIDRs() as $cidr) { list($subnet, $mask) = $cidr; - if (static::isAddrInside($addr, $subnet, $mask)) { + if (static::isAddrInside($addr, (string) $subnet, (int) $mask)) { $target->ip = static::numberToAddr($addr, static::isIPV6($subnet)); $addrFound = true; @@ -245,15 +205,15 @@ protected function getScanTargets(): Generator private function generateTargets(): Generator { $excludes = $this->getExcludes(); - foreach ($this->getCidrs() as $cidr) { + foreach ($this->getCIDRs() as $cidr) { list($startIp, $prefix) = $cidr; $ipv6 = static::isIPV6($startIp); $subnet = $ipv6 ? 128 : 32; - $numIps = pow(2, ($subnet - $prefix)); + $numIps = pow(2, ($subnet - (int) $prefix)); Logger::info('Scanning %d IPs in the CIDR %s', $numIps, implode('/', $cidr)); - $start = static::addrToNumber($startIp); + $start = static::addrToNumber((string) $startIp); for ($i = 0; $i < $numIps; $i++) { $ip = static::numberToAddr(gmp_add($start, $i), $ipv6); if (isset($excludes[$ip])) { @@ -265,7 +225,7 @@ private function generateTargets(): Generator list($startPort, $endPort) = $portRange; foreach (range($startPort, $endPort) as $port) { foreach ($this->snimap[$ip] ?? [null] as $hostname) { - if (array_key_exists($hostname, $excludes)) { + if (array_key_exists((string) $hostname, $excludes)) { Logger::debug('Excluding host %s from scan', $hostname); continue; } @@ -302,24 +262,16 @@ private function generateTargets(): Generator public function updateJobStats(bool $finished = false): void { - $fields = [ - 'finished_targets' => $this->finishedTargets, - 'mtime' => new Expression('UNIX_TIMESTAMP() * 1000') - ]; - + $fields = ['finished_targets' => $this->finishedTargets]; if ($finished) { $fields['end_time'] = new Expression('UNIX_TIMESTAMP() * 1000'); $fields['total_targets'] = $this->totalTargets; } - $this->db->update( - 'x509_job_run', - $fields, - ['id = ?' => $this->jobId] - ); + $this->db->update('x509_job_run', $fields, ['id = ?' => $this->jobRunId]); } - private static function formatTarget($target) + private static function formatTarget($target): string { $result = "tls://[{$target->ip}]:{$target->port}"; @@ -440,14 +392,20 @@ public function run(): Promise\ExtendedPromiseInterface $this->finishedTargets = 0; $this->pendingTargets = 0; + if ($this->schedule) { + $scheduleId = $this->getSchedule()->getId(); + } else { + $scheduleId = new Expression('NULL'); + } + $this->db->insert('x509_job_run', [ - 'name' => $this->getName(), + 'job_id' => $this->getId(), + 'schedule_id' => $scheduleId, 'start_time' => $this->jobRunStart->getTimestamp() * 1000.0, 'total_targets' => 0, 'finished_targets' => 0 ]); - - $this->jobId = $this->db->lastInsertId(); + $this->jobRunId = (int) $this->db->lastInsertId(); $this->targets = $this->getScanTargets(); diff --git a/library/X509/Schedule.php b/library/X509/Schedule.php new file mode 100644 index 00000000..3f809328 --- /dev/null +++ b/library/X509/Schedule.php @@ -0,0 +1,125 @@ +id = $id; + $this->name = $name; + $this->config = $config; + } + + public static function fromModel(X509Schedule $schedule): self + { + /** @var stdClass $config */ + $config = Json::decode($schedule->config); + if (isset($config->rescan)) { + $config->rescan = $config->rescan === 'y'; + } + + if (isset($config->full_scan)) { + $config->full_scan = $config->full_scan === 'y'; + } + + return new static($schedule->name, $schedule->id, $config); + } + + /** + * Get the name of this schedule + * + * @return string + */ + public function getName(): string + { + return $this->name; + } + + /** + * Set the name of this schedule + * + * @param string $name + * + * @return $this + */ + public function setName(string $name): self + { + $this->name = $name; + + return $this; + } + + /** + * Get the database id of this job + * + * @return int + */ + public function getId(): int + { + return $this->id; + } + + /** + * Set the database id of this job + * + * @param int $id + * + * @return $this + */ + public function setId(int $id): self + { + $this->id = $id; + + return $this; + } + + /** + * Get the config of this schedule + * + * @return object + */ + public function getConfig(): object + { + return $this->config; + } + + /** + * Set the config of this schedule + * + * @param object $config + * + * @return $this + */ + public function setConfig(object $config): self + { + $this->config = $config; + + return $this; + } + + /** + * Get the checksum of this schedule + * + * @return string + */ + public function getChecksum(): string + { + return md5($this->getName() . Json::encode($this->getConfig()), true); + } +} From 6ea1e560e7e7a89783c7e3a1ae2ff593420f2f94 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Wed, 19 Jul 2023 09:56:24 +0200 Subject: [PATCH 05/18] Document cli commands & enhance existing configuration docs --- doc/03-Configuration.md | 89 +++++++++++++++-------------------- doc/04-Scanning.md | 85 +++++++++++++++++++++++++++++++++ doc/res/weekly-schedules.png | Bin 0 -> 140091 bytes 3 files changed, 124 insertions(+), 50 deletions(-) create mode 100644 doc/04-Scanning.md create mode 100644 doc/res/weekly-schedules.png diff --git a/doc/03-Configuration.md b/doc/03-Configuration.md index a70bf0bf..62a1649f 100644 --- a/doc/03-Configuration.md +++ b/doc/03-Configuration.md @@ -2,7 +2,7 @@ ## Importing CA certificates -The module tries to verify certificates using its own trust store. By default this trust store is empty and it +The module tries to verify certificates using its own trust store. By default, this trust store is empty, and it is up to the Icinga Web 2 admin to import CA certificates into it. Using the `icingacli x509 import` command CA certificates can be imported. The certificate chain file that is specified @@ -13,19 +13,14 @@ store: icingacli x509 import --file /etc/ssl/certs/ca-certificates.crt ``` -## Scan Jobs - -The module needs to know which IP address ranges and ports to scan. These can be configured in -`Configuration -> Modules -> x509 -> Jobs`. +## Configure Jobs Scan jobs have a name which uniquely identifies them, e.g. `lan`. These names are used by the CLI command to start scanning for specific jobs. Each scan job can have one or more IP address ranges and one or more port ranges. The module scans each port in -a job's port ranges for all the individual IP addresses in the IP ranges. - -IP address ranges have to be specified using the CIDR format. Multiple IP address ranges can be separated with commas, -e.g.: +a job's port ranges for all the individual IP addresses in the IP ranges. IP address ranges have to be specified using +the CIDR format. Multiple IP address ranges can be separated with commas, e.g.: `192.0.2.0/24,10.0.10.0/24` @@ -34,17 +29,37 @@ port: `443,5665-5669` -Scan jobs can be executed using the `icingacli x509 scan` CLI command. The `--job` option is used to specify the scan -job which should be run: +Additionally, each job may also exclude specific **hosts** and **IP** addresses from scan. These hosts won't be scanned +when you run the [scan](04-Scanning.md#scan-command) or [jobs](04-Scanning.md#scheduling-jobs) command. Excluding an +entire network and specifying IP addresses in CIDR format will not work. You must specify a comma-separated concrete +**IP** and **host CN**, e.g: -``` -icingacli x509 scan --job lan -``` +`192.0.2.2,192.0.2.5,icinga.com` + +### Job Schedules + +Schedules are `cron` and [rrule](https://www.rfc-editor.org/rfc/rfc5545) based configs used to run periodically +at the given interval. Every job is allowed to have multiple schedules that can be run independently of each other. +Don't worry, you don't need to know anything about rrule to create **rrule** based schedules. All you need to do is +clicking some buttons over the UI. On the other hand, you should know what cron is and how to configure it to create +**cron**-based schedules. `Cron` based examples can be found [here](#cron-schedules). + +Each job schedule provides different options that you can use to control the scheduling behavior of the +[jobs command](04-Scanning.md#scheduling-jobs). -## Scheduling Jobs +#### Examples -Each job may specify a `cron` compatible `schedule` to run periodically at the given interval. The `cron` format is as -follows: +##### RRule Schedules + +A schedule that runs weekly on **Friday** and scans all targets that have not yet been scanned, or +whose last scan is older than `1 week`. + +![Weekly Schedules](res/weekly-schedules.png "Weekly Schedules") + + +##### Cron Schedules + +The `cron` format is as follows: ``` * * * * * @@ -60,39 +75,13 @@ follows: Example definitions: -Description | Definition -------------------------------------------------------------| ---------- -Run once a year at midnight of 1 January | 0 0 1 1 * -Run once a month at midnight of the first day of the month | 0 0 1 * * -Run once a week at midnight on Sunday morning | 0 0 * * 0 -Run once a day at midnight | 0 0 * * * -Run once an hour at the beginning of the hour | 0 * * * * - -Jobs are executed on CLI with the `jobs` command: - -``` -icingacli x509 jobs run -``` - -This command runs all jobs which are currently due and schedules the next execution of all jobs. - -You may configure this command as `systemd` service. Just copy the example service definition from -`config/systemd/icinga-x509.service` to `/etc/systemd/system/icinga-x509.service` and enable it afterwards: - -``` -systemctl enable icinga-x509.service -``` - -As an alternative if you want scan jobs to be run periodically, you can use the `cron(8)` daemon to run them on a -schedule: - -``` -vi /etc/crontab -[...] - -# Runs job 'lan' daily at 2:30 AM -30 2 * * * www-data icingacli x509 scan --job lan -``` +| Description | Definition | +|------------------------------------------------------------|------------| +| Run once a year at midnight of 1 January | 0 0 1 1 * | +| Run once a month at midnight of the first day of the month | 0 0 1 * * | +| Run once a week at midnight on Sunday morning | 0 0 * * 0 | +| Run once a day at midnight | 0 0 * * * | +| Run once an hour at the beginning of the hour | 0 * * * * | ## Server Name Indication diff --git a/doc/04-Scanning.md b/doc/04-Scanning.md new file mode 100644 index 00000000..3dc89206 --- /dev/null +++ b/doc/04-Scanning.md @@ -0,0 +1,85 @@ +# Scanning + +The Icinga Certificate Monitoring provides CLI commands to scan arbitrary **hosts** and **IPs** in various ways. +These commands are listed below and can be used individually. It is necessary for all commands to know which IP address +ranges and ports to scan. These can be configured as described [here](03-Configuration.md#configure-jobs). + +## Scan Command + +The scan command, as its name implies, scans targets to find their X.509 certificates and track changes to them. +A **target** is an **IP-port** combination that is generated from the job configuration, taking into account configured +[**SNI**](03-Configuration.md#server-name-indication) maps, so that targets with multiple certificates are also properly +scanned. + +By default, successive calls to the scan command perform partial scans, checking both targets not yet scanned and +targets whose scan is older than 24 hours, to ensure that all targets are rescanned over time and new certificates +are collected. This behavior can be customized through the command [options](#usage-1). + +> **Note** +> +> When rescanning due targets, they will be rescanned regardless of whether the target previously provided a certificate +> or not, to collect new certificates, track changed certificates, and remove decommissioned certificates. + +### Usage + +This scan command can be used like any other Icinga Web 2 cli operations like this: `icingacli x509 scan [OPTIONS]` + +**Options:** + +``` +--job= Scan targets that belong to the specified job. (Required) +--since-last-scan=