diff --git a/application/clicommands/JobsCommand.php b/application/clicommands/JobsCommand.php index 378498c4..7272e5a4 100644 --- a/application/clicommands/JobsCommand.php +++ b/application/clicommands/JobsCommand.php @@ -6,36 +6,63 @@ use DateTime; use Exception; -use Icinga\Application\Config; use Icinga\Application\Logger; use Icinga\Module\X509\CertificateUtils; use Icinga\Module\X509\Command; +use Icinga\Module\X509\Common\JobUtils; use Icinga\Module\X509\Hook\SniHook; use Icinga\Module\X509\Job; +use Icinga\Module\X509\Model\X509Job; +use Icinga\Module\X509\Model\X509Schedule; +use Icinga\Module\X509\Schedule; +use InvalidArgumentException; +use ipl\Orm\Query; use ipl\Scheduler\Contract\Frequency; -use ipl\Scheduler\Contract\Task; -use ipl\Scheduler\Cron; use ipl\Scheduler\Scheduler; +use ipl\Stdlib\Filter; +use Ramsey\Uuid\Uuid; use React\EventLoop\Loop; use React\Promise\ExtendedPromiseInterface; +use stdClass; use Throwable; class JobsCommand extends Command { + use JobUtils; + /** * Run all configured jobs based on their schedule * * USAGE: * - * icingacli x509 jobs run + * icingacli x509 jobs run [OPTIONS] + * + * OPTIONS + * + * --job= + * Run all configured schedules only of the specified job. + * + * --schedule= + * Run only the given schedule of the specified job. Providing a schedule name + * without a job will fail immediately. + * + * --parallel= + * Allow parallel scanning of targets up to the specified number. Defaults to 256. + * May cause **too many open files** error if set to a number higher than the configured one (ulimit). */ - public function runAction() + public function runAction(): void { - $parallel = (int) $this->Config()->get('scan', 'parallel', 256); + $parallel = (int) $this->params->get('parallel', Job::DEFAULT_PARALLEL); if ($parallel <= 0) { $this->fail("The 'parallel' option must be set to at least 1"); } + $jobName = (string) $this->params->get('job'); + $scheduleName = (string) $this->params->get('schedule'); + if (! $jobName && $scheduleName) { + throw new InvalidArgumentException('You cannot provide a schedule without a job'); + } + $scheduler = new Scheduler(); $this->attachJobsLogging($scheduler); @@ -53,13 +80,14 @@ public function runAction() $scheduled = []; // Periodically check configuration changes to ensure that new jobs are scheduled, jobs are updated, // and deleted jobs are canceled. - $watchdog = function () use (&$watchdog, $scheduler, &$scheduled, $parallel) { - $jobs = $this->fetchJobs(); + $watchdog = function () use (&$watchdog, &$scheduled, $scheduler, $parallel, $jobName, $scheduleName) { + $jobs = $this->fetchSchedules($jobName, $scheduleName); $outdatedJobs = array_diff_key($scheduled, $jobs); foreach ($outdatedJobs as $job) { Logger::info( - 'Removing scheduled job %s, as it either no longer exists in the configuration or its config has' - . ' been changed', + 'Removing schedule %s of job %s, as it either no longer exists in the configuration or its' + . ' config has been changed', + $job->getSchedule()->getName(), $job->getName() ); @@ -70,30 +98,21 @@ public function runAction() foreach ($newJobs as $job) { $job->setParallel($parallel); - $config = $job->getConfig(); - if (! isset($config->frequencyType)) { - if (! Cron::isValid($config->schedule)) { - Logger::error('Job %s has invalid schedule expression %s', $job->getName(), $config->schedule); - - continue; - } + /** @var stdClass $config */ + $config = $job->getSchedule()->getConfig(); + try { + /** @var Frequency $type */ + $type = $config->type; + $frequency = $type::fromJson($config->frequency); + } catch (Throwable $err) { + Logger::error( + 'Cannot create schedule %s of job %s: %s', + $job->getSchedule()->getName(), + $job->getName(), + $err->getMessage() + ); - $frequency = new Cron($config->schedule); - } else { - try { - /** @var Frequency $type */ - $type = $config->frequencyType; - $frequency = $type::fromJson($config->schedule); - } catch (Exception $err) { - Logger::error( - 'Job %s has invalid schedule expression %s: %s', - $job->getName(), - $config->schedule, - $err->getMessage() - ); - - continue; - } + continue; } $scheduler->schedule($job, $frequency); @@ -108,28 +127,53 @@ public function runAction() } /** - * Fetch jobs from disk + * Fetch job schedules from database + * + * @param ?string $jobName + * @param ?string $scheduleName * * @return Job[] */ - protected function fetchJobs(): array + protected function fetchSchedules(?string $jobName, ?string $scheduleName): array { - $configs = Config::module($this->getModuleName(), 'jobs', true); - $defaultSchedule = $configs->get('jobs', 'default_schedule'); + $jobs = X509Job::on($this->getDb()); + if ($jobName) { + $jobs->filter(Filter::equal('name', $jobName)); + } + + $jobSchedules = []; + $snimap = SniHook::getAll(); + /** @var X509Job $jobConfig */ + foreach ($jobs as $jobConfig) { + $cidrs = $this->parseCIDRs($jobConfig->cidrs); + $ports = $this->parsePorts($jobConfig->ports); + $job = (new Job($jobConfig->name, $cidrs, $ports, $snimap)) + ->setId($jobConfig->id) + ->setExcludes($this->parseExcludes($jobConfig->exclude_targets)); + + /** @var Query $schedules */ + $schedules = $jobConfig->schedule; + if ($scheduleName) { + $schedules->filter(Filter::equal('name', $scheduleName)); + } - $jobs = []; - foreach ($configs as $name => $config) { - if (! $config->get('schedule', $defaultSchedule)) { - Logger::debug('Job %s cannot be scheduled', $name); + $jobSchedules = []; + /** @var X509Schedule $scheduleModel */ + foreach ($schedules as $scheduleModel) { + $schedule = Schedule::fromModel($scheduleModel); + $job = (clone $job) + ->setSchedule($schedule) + ->setUuid(Uuid::fromBytes($job->getChecksum())); - continue; + $jobSchedules[$job->getUuid()->toString()] = $job; } - $job = new Job($name, $config, SniHook::getAll()); - $jobs[$job->getUuid()->toString()] = $job; + if (! isset($jobSchedules[$job->getUuid()->toString()])) { + Logger::info('Skipping job %s because no schedules are configured', $job->getName()); + } } - return $jobs; + return $jobSchedules; } /** @@ -139,15 +183,34 @@ protected function fetchJobs(): array */ protected function attachJobsLogging(Scheduler $scheduler): void { - $scheduler->on(Scheduler::ON_TASK_CANCEL, function (Task $job, array $_) { - Logger::info('Job %s canceled', $job->getName()); + $scheduler->on(Scheduler::ON_TASK_CANCEL, function (Job $task, array $_) { + Logger::info('Schedule %s of job %s canceled', $task->getSchedule()->getName(), $task->getName()); }); - $scheduler->on(Scheduler::ON_TASK_DONE, function (Task $job, $targets = 0) { + $scheduler->on(Scheduler::ON_TASK_DONE, function (Job $task, $targets = 0) { if ($targets === 0) { - Logger::warning('The job %s does not have any targets', $job->getName()); + $sinceLastScan = $task->getSinceLastScan(); + if ($sinceLastScan) { + Logger::info( + 'Schedule %s of job %s does not have any targets to be rescanned matching since last scan: %s', + $task->getSchedule()->getName(), + $task->getName(), + $sinceLastScan->format('Y-m-d H:i:s') + ); + } else { + Logger::warning( + 'Schedule %s of job %s does not have any targets', + $task->getSchedule()->getName(), + $task->getName() + ); + } } else { - Logger::info('Scanned %d target(s) from job %s', $targets, $job->getName()); + Logger::info( + 'Scanned %d target(s) by schedule %s of job %s', + $targets, + $task->getSchedule()->getName(), + $task->getName() + ); try { $verified = CertificateUtils::verifyCertificates($this->getDb()); @@ -160,21 +223,36 @@ protected function attachJobsLogging(Scheduler $scheduler): void } }); - $scheduler->on(Scheduler::ON_TASK_FAILED, function (Task $job, Throwable $e) { - Logger::error('Failed to run job %s: %s', $job->getName(), $e->getMessage()); + $scheduler->on(Scheduler::ON_TASK_FAILED, function (Job $task, Throwable $e) { + Logger::error( + 'Failed to run schedule %s of job %s: %s', + $task->getSchedule()->getName(), + $task->getName(), + $e->getMessage() + ); Logger::debug($e->getTraceAsString()); }); - $scheduler->on(Scheduler::ON_TASK_RUN, function (Task $job, ExtendedPromiseInterface $_) { - Logger::info('Running job %s', $job->getName()); + $scheduler->on(Scheduler::ON_TASK_RUN, function (Job $task, ExtendedPromiseInterface $_) { + Logger::info('Running schedule %s of job %s', $task->getSchedule()->getName(), $task->getName()); }); - $scheduler->on(Scheduler::ON_TASK_SCHEDULED, function (Task $job, DateTime $dateTime) { - Logger::info('Scheduling job %s to run at %s', $job->getName(), $dateTime->format('Y-m-d H:i:s')); + $scheduler->on(Scheduler::ON_TASK_SCHEDULED, function (Job $task, DateTime $dateTime) { + Logger::info( + 'Scheduling %s of job %s to run at %s', + $task->getSchedule()->getName(), + $task->getName(), + $dateTime->format('Y-m-d H:i:s') + ); }); - $scheduler->on(Scheduler::ON_TASK_EXPIRED, function (Task $task, DateTime $dateTime) { - Logger::info(sprintf('Detaching expired job %s at %s', $task->getName(), $dateTime->format('Y-m-d H:i:s'))); + $scheduler->on(Scheduler::ON_TASK_EXPIRED, function (Job $task, DateTime $dateTime) { + Logger::info( + 'Detaching expired schedule %s of job %s at %s', + $task->getSchedule()->getName(), + $task->getName(), + $dateTime->format('Y-m-d H:i:s') + ); }); } } diff --git a/application/clicommands/ScanCommand.php b/application/clicommands/ScanCommand.php index c633ecec..422a59ec 100644 --- a/application/clicommands/ScanCommand.php +++ b/application/clicommands/ScanCommand.php @@ -4,19 +4,22 @@ namespace Icinga\Module\X509\Clicommands; -use DateTime; use Exception; use Icinga\Application\Logger; use Icinga\Module\X509\CertificateUtils; use Icinga\Module\X509\Command; +use Icinga\Module\X509\Common\JobUtils; use Icinga\Module\X509\Hook\SniHook; use Icinga\Module\X509\Job; -use InvalidArgumentException; +use Icinga\Module\X509\Model\X509Job; +use ipl\Stdlib\Filter; use React\EventLoop\Loop; use Throwable; class ScanCommand extends Command { + use JobUtils; + /** * Scan targets to find their X.509 certificates and track changes to them. * @@ -45,6 +48,10 @@ class ScanCommand extends Command * which can also be an English textual datetime description like "2 days". * Defaults to "-24 hours". * + * --parallel= + * Allow parallel scanning of targets up to the specified number. Defaults to 256. + * May cause **too many open files** error if set to a number higher than the configured one (ulimit). + * * --rescan * Rescan only targets that have been scanned before. * @@ -74,51 +81,45 @@ class ScanCommand extends Command * * icingacli x509 scan --job --full */ - public function indexAction() + public function indexAction(): void { + /** @var string $name */ $name = $this->params->shiftRequired('job'); $fullScan = (bool) $this->params->get('full', false); $rescan = (bool) $this->params->get('rescan', false); - $parallel = (int) $this->Config()->get('scan', 'parallel', 256); + /** @var string $sinceLastScan */ + $sinceLastScan = $this->params->get('since-last-scan', Job::DEFAULT_SINCE_LAST_SCAN); + if ($sinceLastScan === 'null') { + $sinceLastScan = null; + } + + /** @var int $parallel */ + $parallel = $this->params->get('parallel', Job::DEFAULT_PARALLEL); if ($parallel <= 0) { throw new Exception('The \'parallel\' option must be set to at least 1'); } - $jobs = $this->Config('jobs'); - if (! $jobs->hasSection($name)) { + /** @var X509Job $jobConfig */ + $jobConfig = X509Job::on($this->getDb()) + ->filter(Filter::equal('name', $name)) + ->first(); + if ($jobConfig === null) { throw new Exception(sprintf('Job %s not found', $name)); } - $jobDescription = $this->Config('jobs')->getSection($name); - if (! strlen($jobDescription->get('cidrs'))) { + if (! strlen($jobConfig->cidrs)) { throw new Exception(sprintf('The job %s does not specify any CIDRs', $name)); } - $sinceLastScan = $this->params->get('since-last-scan', '-24 hours'); - if ($sinceLastScan === 'null') { - $sinceLastScan = null; - } else { - if ($sinceLastScan[0] !== '-') { - // When the user specified "2 days" as a threshold strtotime() will compute the - // timestamp NOW() + 2 days, but it has to be NOW() + (-2 days) - $sinceLastScan = "-$sinceLastScan"; - } - - try { - $sinceLastScan = new DateTime($sinceLastScan); - } catch (Exception $_) { - throw new InvalidArgumentException(sprintf( - 'The specified last scan time is in an unknown format: %s', - $this->params->get('since-last-scan') - )); - } - } - - $job = (new Job($name, $jobDescription, SniHook::getAll())) + $cidrs = $this->parseCIDRs($jobConfig->cidrs); + $ports = $this->parsePorts($jobConfig->ports); + $job = (new Job($name, $cidrs, $ports, SniHook::getAll())) + ->setId($jobConfig->id) ->setFullScan($fullScan) ->setRescan($rescan) ->setParallel($parallel) + ->setExcludes($this->parseExcludes($jobConfig->exclude_targets)) ->setLastScan($sinceLastScan); $promise = $job->run(); diff --git a/application/controllers/JobController.php b/application/controllers/JobController.php new file mode 100644 index 00000000..c97d6c88 --- /dev/null +++ b/application/controllers/JobController.php @@ -0,0 +1,228 @@ +getTabs()->disableLegacyExtensions(); + + /** @var int $jobId */ + $jobId = $this->params->getRequired('id'); + + /** @var X509Job $job */ + $job = X509Job::on($this->getDb()) + ->filter(Filter::equal('id', $jobId)) + ->first(); + + if ($job === null) { + $this->httpNotFound($this->translate('Job not found')); + } + + $this->job = $job; + } + + public function indexAction(): void + { + $this->assertPermission('config/x509'); + + $this->initTabs(); + $this->getTabs()->activate('job-activities'); + + $jobRuns = $this->job->job_run->with(['job', 'schedule']); + + $limitControl = $this->createLimitControl(); + $sortControl = $this->createSortControl($jobRuns, [ + 'schedule.name' => $this->translate('Schedule Name'), + 'schedule.author' => $this->translate('Author'), + 'total_targets' => $this->translate('Total Targets'), + 'finished_targets' => $this->translate('Finished Targets'), + 'start_time desc' => $this->translate('Started At'), + 'end_time' => $this->translate('Ended At') + ]); + + $this->controls->getAttributes()->add('class', 'default-layout'); + $this->addControl($sortControl); + $this->addControl($limitControl); + $this->addControl($this->createActionBar()); + + $this->addContent(new JobDetails($jobRuns)); + } + + public function updateAction(): void + { + $this->assertPermission('config/x509'); + + $this->addTitleTab($this->translate('Update Job')); + + $form = (new JobConfigForm($this->job)) + ->setAction((string) Url::fromRequest()) + ->populate([ + 'name' => $this->job->name, + 'cidrs' => $this->job->cidrs, + 'ports' => $this->job->ports, + 'exclude_targets' => $this->job->exclude_targets + ]) + ->on(JobConfigForm::ON_SUCCESS, function (JobConfigForm $form) { + /** @var FormSubmitElement $button */ + $button = $form->getPressedSubmitElement(); + if ($button->getName() === 'btn_remove') { + $this->switchToSingleColumnLayout(); + } else { + $this->closeModalAndRefreshRelatedView(Links::job($this->job)); + } + }) + ->handleRequest($this->getServerRequest()); + + $this->addContent($form); + } + + public function schedulesAction(): void + { + $this->assertPermission('config/x509'); + + $this->initTabs(); + $this->getTabs()->activate('schedules'); + + $schedules = $this->job->schedule->with(['job']); + + $sortControl = $this->createSortControl($schedules, [ + 'name' => $this->translate('Name'), + 'author' => $this->translate('Author'), + 'ctime' => $this->translate('Date Created'), + 'mtime' => $this->translate('Date Modified') + ]); + + $this->controls->getAttributes()->add('class', 'default-layout'); + $this->addControl( + (new ButtonLink($this->translate('New Schedule'), Links::scheduleJob($this->job), 'plus')) + ->openInModal() + ); + $this->addControl($sortControl); + + $this->addContent(new Schedules($schedules)); + } + + public function scheduleAction(): void + { + $this->assertPermission('config/x509'); + + $this->addTitleTab($this->translate('Schedule Job')); + + $form = (new ScheduleForm()) + ->setAction((string) Url::fromRequest()) + ->setJobId($this->job->id) + ->on(JobConfigForm::ON_SUCCESS, function () { + $this->redirectNow(Links::schedules($this->job)); + }) + ->handleRequest($this->getServerRequest()); + + $parts = $form->getPartUpdates(); + if (! empty($parts)) { + $this->sendMultipartUpdate(...$parts); + } + + $this->addContent($form); + } + + public function updateScheduleAction(): void + { + $this->assertPermission('config/x509'); + + $this->addTitleTab($this->translate('Update Schedule')); + + /** @var int $id */ + $id = $this->params->getRequired('scheduleId'); + /** @var X509Schedule $schedule */ + $schedule = X509Schedule::on($this->getDb()) + ->filter(Filter::equal('id', $id)) + ->first(); + if ($schedule === null) { + $this->httpNotFound($this->translate('Schedule not found')); + } + + /** @var stdClass $config */ + $config = Json::decode($schedule->config); + /** @var Frequency $type */ + $type = $config->type; + $frequency = $type::fromJson($config->frequency); + + $form = (new ScheduleForm($schedule)) + ->setAction((string) Url::fromRequest()) + ->populate([ + 'name' => $schedule->name, + 'full_scan' => $config->full_scan ?? 'n', + 'rescan' => $config->rescan ?? 'n', + 'since_last_scan' => $config->since_last_scan ?? null, + 'schedule_element' => $frequency + ]) + ->on(JobConfigForm::ON_SUCCESS, function () { + $this->redirectNow('__BACK__'); + }) + ->handleRequest($this->getServerRequest()); + + $parts = $form->getPartUpdates(); + if (! empty($parts)) { + $this->sendMultipartUpdate(...$parts); + } + + $this->addContent($form); + } + + protected function createActionBar(): ValidHtml + { + $actions = new ActionBar(); + $actions->addHtml( + (new ActionLink($this->translate('Modify'), Links::updateJob($this->job), 'edit')) + ->openInModal(), + (new ActionLink($this->translate('Schedule'), Links::scheduleJob($this->job), 'calendar')) + ->openInModal() + ); + + return $actions; + } + + protected function initTabs(): void + { + $tabs = $this->getTabs(); + $tabs + ->add('job-activities', [ + 'label' => $this->translate('Job Activities'), + 'url' => Links::job($this->job) + ]) + ->add('schedules', [ + 'label' => $this->translate('Schedules'), + 'url' => Links::schedules($this->job) + ]); + } +} diff --git a/application/controllers/JobsController.php b/application/controllers/JobsController.php index 3beec87b..132d6216 100644 --- a/application/controllers/JobsController.php +++ b/application/controllers/JobsController.php @@ -4,127 +4,59 @@ namespace Icinga\Module\X509\Controllers; -use Icinga\Module\X509\Forms\Config\JobConfigForm; -use Icinga\Module\X509\JobsIniRepository; -use Icinga\Web\Url; -use ipl\Html\HtmlElement; -use ipl\Scheduler\Contract\Frequency; -use ipl\Scheduler\Cron; +use Icinga\Module\X509\Common\Database; +use Icinga\Module\X509\Forms\Jobs\JobConfigForm; +use Icinga\Module\X509\Model\X509Job; +use Icinga\Module\X509\Widget\Jobs; use ipl\Web\Compat\CompatController; -use stdClass; +use ipl\Web\Url; +use ipl\Web\Widget\ButtonLink; class JobsController extends CompatController { - protected function prepareInit() - { - parent::prepareInit(); - - $this->getTabs()->disableLegacyExtensions(); - } + use Database; /** * List all jobs */ public function indexAction() { - $this->view->tabs = $this->Module()->getConfigTabs()->activate('jobs'); + $this->addTitleTab($this->translate('Jobs')); + + $jobs = X509Job::on($this->getDb()); + if ($this->hasPermission('config/x509')) { + $this->addControl( + (new ButtonLink($this->translate('New Job'), Url::fromPath('x509/jobs/new'), 'plus')) + ->openInModal() + ); + } + + $sortControl = $this->createSortControl($jobs, [ + 'name' => $this->translate('Name'), + 'author' => $this->translate('Author'), + 'ctime' => $this->translate('Date Created'), + 'mtime' => $this->translate('Date Modified') + ]); - $repo = new JobsIniRepository(); + $this->controls->getAttributes()->add('class', 'default-layout'); + $this->addControl($sortControl); - $this->view->jobs = $repo->select(array('name')); + $this->addContent(new Jobs($jobs)); } - /** - * Create a job - */ public function newAction() { - $form = $this->prepareForm(true); + $this->assertPermission('config/x509'); $this->addTitleTab($this->translate('New Job')); - $this->addContent($form); - } - /** - * Update a job - */ - public function updateAction() - { - $name = $this->params->getRequired('name'); - $form = $this->prepareForm(); - - $this->addTitleTab($this->translate('Update Job')); - - $this->addContent($form); - } - - /** - * Remove a job - */ - public function removeAction() - { - $name = $this->params->getRequired('name'); - $form = $this->prepareForm(); - - $this->addTitleTab($this->translate('Remove Job')); - - $form->prependHtml(HtmlElement::create('h1', null, sprintf($this->translate('Remove job %s'), $name))); - $this->addContent($form); - } - - /** - * Assert config permission and return a prepared RepositoryForm - * - * @return JobConfigForm - */ - protected function prepareForm(bool $isNew = false) - { - $this->assertPermission('config/x509'); - - $repo = new JobsIniRepository(); $form = (new JobConfigForm()) - ->setRedirectUrl(Url::fromPath('x509/jobs')) - ->setRepo($repo); - - $values = []; - if (! $isNew) { - $name = $this->params->getRequired('name'); - $query = $repo->select()->where('name', $name); - /** @var false|stdClass $data */ - $data = $query->fetchRow(); - if ($data === false) { - $this->httpNotFound($this->translate('Job not found')); - } - - if (! isset($data->frequencyType) && ! empty($data->schedule)) { - $frequency = new Cron($data->schedule); - } elseif (! empty($data->schedule)) { - /** @var Frequency $type */ - $type = $data->frequencyType; - $frequency = $type::fromJson($data->schedule); - } - - $values = [ - 'name' => $data->name, - 'cidrs' => $data->cidrs, - 'ports' => $data->ports, - 'exclude_targets' => $data->exclude_targets, - 'schedule-element' => $frequency ?? [] - ]; - } - - $form - ->populate($values) + ->setAction((string) Url::fromRequest()) ->on(JobConfigForm::ON_SUCCESS, function () { - $this->redirectNow(Url::fromPath('x509/jobs')); + $this->closeModalAndRefreshRelatedView(Url::fromPath('x509/jobs')); }) ->handleRequest($this->getServerRequest()); - $parts = $form->getPartUpdates(); - if (! empty($parts)) { - $this->sendMultipartUpdate(...$parts); - } - - return $form; + $this->addContent($form); } } diff --git a/application/forms/Config/JobConfigForm.php b/application/forms/Config/JobConfigForm.php deleted file mode 100644 index a7ecbbaa..00000000 --- a/application/forms/Config/JobConfigForm.php +++ /dev/null @@ -1,174 +0,0 @@ -identifier = Url::fromRequest()->getParam('name'); - $this->scheduleElement = new ScheduleElement('schedule-element'); - $this->scheduleElement->setIdProtector([Icinga::app()->getRequest(), 'protectId']); - } - - public function setRepo(JobsIniRepository $repo): self - { - $this->repo = $repo; - - return $this; - } - - protected function createFilter() - { - return Filter::where('name', $this->identifier); - } - - protected function isUpdating(): bool - { - return Url::fromRequest()->getPath() === 'x509/jobs/update'; - } - - protected function isRemoving(): bool - { - return Url::fromRequest()->getPath() === 'x509/jobs/remove'; - } - - /** - * Get multipart updates - * - * @return array - */ - public function getPartUpdates(): array - { - if ($this->scheduleElement->getFrequency() === 'none') { - // Workaround for https://github.com/Icinga/ipl-web/issues/130 - return []; - } - - return $this->scheduleElement->prepareMultipartUpdate($this->getRequest()); - } - - protected function assemble() - { - if (! $this->isRemoving()) { - $this->addElement('text', 'name', [ - 'required' => true, - 'description' => t('Job name'), - 'label' => t('Name'), - ]); - $this->addElement('textarea', 'cidrs', [ - 'description' => t('Comma-separated list of CIDR addresses to scan'), - 'label' => t('CIDRs'), - 'required' => true, - 'validators' => [ - new CallbackValidator(function ($value, CallbackValidator $validator): bool { - $cidrValidator = new CidrValidator(); - $cidrs = Str::trimSplit($value); - - foreach ($cidrs as $cidr) { - if (! $cidrValidator->isValid($cidr)) { - $validator->addMessage(...$cidrValidator->getMessages()); - - return false; - } - } - - return true; - }) - ] - ]); - $this->addElement('textarea', 'ports', [ - 'required' => true, - 'description' => t('Comma-separated list of ports to scan'), - 'label' => t('Ports'), - ]); - - $this->addElement('textarea', 'exclude_targets', [ - 'description' => $this->translate('Comma-separated list of addresses/hostnames to exclude'), - 'label' => $this->translate('Exclude Targets'), - 'required' => false - ]); - - $this->addHtml(HtmlElement::create('div', ['class' => 'schedule-element-separator'])); - $this->addElement($this->scheduleElement); - } - - $this->addElement('submit', 'submit', [ - 'label' => $this->isRemoving() ? t('Yes') : ($this->isUpdating() ? t('Update') : t('Create')) - ]); - } - - protected function onSuccess() - { - if ($this->isRemoving()) { - try { - $this->repo->delete($this->repo->getBaseTable(), $this->createFilter()); - - Notification::success(t('Job removed')); - } catch (StatementException $err) { - Notification::error(t('Failed to remove job')); - } - } else { - /** @var Frequency $frequency */ - $frequency = $this->scheduleElement->getValue(); - $data = [ - 'name' => $this->getValue('name'), - 'cidrs' => $this->getValue('cidrs'), - 'ports' => $this->getValue('ports'), - 'schedule' => json_encode($frequency), - 'frequencyType' => get_php_type($frequency), - ]; - - $excludes = $this->getValue('exclude_targets'); - if (! empty($excludes)) { - $data['exclude_targets'] = $excludes; - } - - try { - if ($this->isUpdating()) { - $message = t('Job updated'); - $this->repo->update($this->repo->getBaseTable(), $data, $this->createFilter()); - } else { - $message = t('Job created'); - $this->repo->insert($this->repo->getBaseTable(), $data); - } - - Notification::success($message); - } catch (StatementException $err) { - $message = $this->isUpdating() ? t('Failed to update job') : t('Failed to create job'); - - Notification::error($message . ': ' . $err->getMessage()); - } - } - } -} diff --git a/application/forms/Jobs/JobConfigForm.php b/application/forms/Jobs/JobConfigForm.php new file mode 100644 index 00000000..8b46773d --- /dev/null +++ b/application/forms/Jobs/JobConfigForm.php @@ -0,0 +1,156 @@ +job = $job; + } + + protected function isUpdating(): bool + { + return $this->job !== null; + } + + public function hasBeenSubmitted(): bool + { + if (! $this->hasBeenSent()) { + return false; + } + + $button = $this->getPressedSubmitElement(); + + return $button && ($button->getName() === 'btn_submit' || $button->getName() === 'btn_remove'); + } + + protected function assemble(): void + { + $this->addElement('text', 'name', [ + 'required' => true, + 'label' => $this->translate('Name'), + 'description' => $this->translate('Job name'), + ]); + + $this->addElement('textarea', 'cidrs', [ + 'required' => true, + 'label' => $this->translate('CIDRs'), + 'description' => $this->translate('Comma-separated list of CIDR addresses to scan'), + 'validators' => [ + new CallbackValidator(function ($value, CallbackValidator $validator): bool { + $cidrValidator = new CidrValidator(); + $cidrs = Str::trimSplit($value); + + foreach ($cidrs as $cidr) { + if (! $cidrValidator->isValid($cidr)) { + $validator->addMessage(...$cidrValidator->getMessages()); + + return false; + } + } + + return true; + }) + ] + ]); + + $this->addElement('textarea', 'ports', [ + 'required' => true, + 'label' => $this->translate('Ports'), + 'description' => $this->translate('Comma-separated list of ports to scan'), + ]); + + $this->addElement('textarea', 'exclude_targets', [ + 'required' => false, + 'label' => $this->translate('Exclude Targets'), + 'description' => $this->translate('Comma-separated list of addresses/hostnames to exclude'), + ]); + + $this->addElement('submit', 'btn_submit', [ + 'label' => $this->isUpdating() ? $this->translate('Update') : $this->translate('Create') + ]); + + if ($this->isUpdating()) { + $removeButton = $this->createElement('submit', 'btn_remove', [ + 'class' => 'btn-remove', + 'label' => $this->translate('Remove Job'), + ]); + $this->registerElement($removeButton); + + /** @var HtmlDocument $wrapper */ + $wrapper = $this->getElement('btn_submit')->getWrapper(); + $wrapper->prepend($removeButton); + } + } + + protected function onSuccess(): void + { + $conn = $this->getDb(); + /** @var FormSubmitElement $submitElement */ + $submitElement = $this->getPressedSubmitElement(); + if ($submitElement->getName() === 'btn_remove') { + try { + /** @var X509Job $job */ + $job = $this->job; + $conn->delete('x509_job', ['id = ?' => $job->id]); + + Notification::success($this->translate('Removed job successfully')); + } catch (Exception $err) { + Notification::error($this->translate('Failed to remove job') . ': ' . $err->getMessage()); + } + } else { + $values = $this->getValues(); + + try { + /** @var User $user */ + $user = Auth::getInstance()->getUser(); + if ($this->job === null) { + $values['author'] = $user->getUsername(); + $values['ctime'] = (new DateTime())->getTimestamp() * 1000.0; + $values['mtime'] = (new DateTime())->getTimestamp() * 1000.0; + + $conn->insert('x509_job', $values); + $message = $this->translate('Created job successfully'); + } else { + $values['mtime'] = (new DateTime())->getTimestamp() * 1000.0; + + $conn->update('x509_job', $values, ['id = ?' => $this->job->id]); + $message = $this->translate('Updated job successfully'); + } + + Notification::success($message); + } catch (Exception $err) { + $message = $this->isUpdating() + ? $this->translate('Failed to update job') + : $this->translate('Failed to create job'); + + Notification::error($message . ': ' . $err->getMessage()); + } + } + } +} diff --git a/application/forms/Jobs/ScheduleForm.php b/application/forms/Jobs/ScheduleForm.php new file mode 100644 index 00000000..3903e2a3 --- /dev/null +++ b/application/forms/Jobs/ScheduleForm.php @@ -0,0 +1,203 @@ +schedule = $schedule; + $this->scheduleElement = new ScheduleElement('schedule_element'); + + /** @var Web $app */ + $app = Icinga::app(); + $this->scheduleElement->setIdProtector([$app->getRequest(), 'protectId']); + } + + protected function isUpdating(): bool + { + return $this->schedule !== null; + } + + public function setJobId(int $jobId): self + { + $this->jobId = $jobId; + + return $this; + } + + /** + * Get multipart updates + * + * @return array + */ + public function getPartUpdates(): array + { + /** @var RequestInterface $request */ + $request = $this->getRequest(); + + return $this->scheduleElement->prepareMultipartUpdate($request); + } + + public function hasBeenSubmitted(): bool + { + if (! $this->hasBeenSent()) { + return false; + } + + $button = $this->getPressedSubmitElement(); + + return $button && ($button->getName() === 'submit' || $button->getName() === 'btn_remove'); + } + + protected function assemble(): void + { + $this->addElement('text', 'name', [ + 'required' => true, + 'label' => $this->translate('Name'), + 'description' => $this->translate('Schedule name'), + ]); + + $this->addElement('checkbox', 'full_scan', [ + 'required' => false, + 'class' => 'autosubmit', + 'label' => $this->translate('Full Scan'), + 'description' => $this->translate( + 'Scan all known and unknown targets of this job. (Defaults to only scan unknown targets)' + ) + ]); + + if ($this->getPopulatedValue('full_scan', 'n') === 'n') { + $this->addElement('checkbox', 'rescan', [ + 'required' => false, + 'class' => 'autosubmit', + 'label' => $this->translate('Rescan'), + 'description' => $this->translate('Rescan only targets that have been scanned before') + ]); + + $this->addElement('text', 'since_last_scan', [ + 'required' => false, + 'label' => $this->translate('Since Last Scan'), + 'placeholder' => '-24 hours', + 'description' => $this->translate( + 'Scan targets whose last scan is older than the specified date/time, which can also be an' + . ' English textual datetime description like "2 days". If you want to scan only unknown targets' + . ' you can set this to "null".' + ), + 'validators' => [ + new CallbackValidator(function ($value, CallbackValidator $validator) { + if ($value !== null && $value !== 'null') { + try { + new DateTime($value); + } catch (Exception $_) { + $validator->addMessage($this->translate('Invalid textual date time')); + + return false; + } + } + + return true; + }) + ] + ]); + } + + $this->addHtml(HtmlElement::create('div', ['class' => 'schedule-element-separator'])); + $this->addElement($this->scheduleElement); + + $this->addElement('submit', 'submit', [ + 'label' => $this->isUpdating() ? $this->translate('Update') : $this->translate('Schedule') + ]); + + if ($this->isUpdating()) { + $removeButton = $this->createElement('submit', 'btn_remove', [ + 'class' => 'btn-remove', + 'label' => $this->translate('Remove Schedule'), + ]); + $this->registerElement($removeButton); + + /** @var HtmlDocument $wrapper */ + $wrapper = $this->getElement('submit')->getWrapper(); + $wrapper->prepend($removeButton); + } + } + + protected function onSuccess(): void + { + /** @var X509Schedule $schedule */ + $schedule = $this->schedule; + $conn = $this->getDb(); + /** @var FormSubmitElement $submitElement */ + $submitElement = $this->getPressedSubmitElement(); + if ($submitElement->getName() === 'btn_remove') { + $conn->delete('x509_schedule', ['id = ?' => $schedule->id]); + + Notification::success($this->translate('Deleted schedule successfully')); + } else { + $config = $this->getValues(); + unset($config['name']); + unset($config['schedule_element']); + + $frequency = $this->scheduleElement->getValue(); + $config['type'] = get_php_type($frequency); + $config['frequency'] = Json::encode($frequency); + + /** @var User $user */ + $user = Auth::getInstance()->getUser(); + if (! $this->isUpdating()) { + $conn->insert('x509_schedule', [ + 'job_id' => $this->schedule ? $this->schedule->job_id : $this->jobId, + 'name' => $this->getValue('name'), + 'author' => $user->getUsername(), + 'config' => Json::encode($config), + 'ctime' => (new DateTime())->getTimestamp() * 1000.0, + 'mtime' => (new DateTime())->getTimestamp() * 1000.0 + ]); + $message = $this->translate('Created schedule successfully'); + } else { + $conn->update('x509_schedule', [ + 'name' => $this->getValue('name'), + 'config' => Json::encode($config), + 'mtime' => (new DateTime())->getTimestamp() * 1000.0 + ], ['id = ?' => $schedule->id]); + $message = $this->translate('Updated schedule successfully'); + } + + Notification::success($message); + } + } +} diff --git a/configuration.php b/configuration.php index 259e1fcc..9930753a 100644 --- a/configuration.php +++ b/configuration.php @@ -20,18 +20,18 @@ 'priority' => 20 )); +$section->add(N_('Jobs'), [ + 'url' => 'x509/jobs', + 'priority' => 100, + 'description' => $this->translate('Configure the scan jobs') +]); + $this->provideConfigTab('backend', array( 'title' => $this->translate('Configure the database backend'), 'label' => $this->translate('Backend'), 'url' => 'config/backend' )); -$this->provideConfigTab('jobs', array( - 'title' => $this->translate('Configure the scan jobs'), - 'label' => $this->translate('Jobs'), - 'url' => 'jobs' -)); - $this->provideConfigTab('sni', array( 'title' => $this->translate('Configure SNI'), 'label' => $this->translate('SNI'), diff --git a/doc/03-Configuration.md b/doc/03-Configuration.md index a70bf0bf..26a1736a 100644 --- a/doc/03-Configuration.md +++ b/doc/03-Configuration.md @@ -2,7 +2,7 @@ ## Importing CA certificates -The module tries to verify certificates using its own trust store. By default this trust store is empty and it +The module tries to verify certificates using its own trust store. By default, this trust store is empty, and it is up to the Icinga Web 2 admin to import CA certificates into it. Using the `icingacli x509 import` command CA certificates can be imported. The certificate chain file that is specified @@ -13,19 +13,14 @@ store: icingacli x509 import --file /etc/ssl/certs/ca-certificates.crt ``` -## Scan Jobs - -The module needs to know which IP address ranges and ports to scan. These can be configured in -`Configuration -> Modules -> x509 -> Jobs`. +## Configure Jobs Scan jobs have a name which uniquely identifies them, e.g. `lan`. These names are used by the CLI command to start scanning for specific jobs. Each scan job can have one or more IP address ranges and one or more port ranges. The module scans each port in -a job's port ranges for all the individual IP addresses in the IP ranges. - -IP address ranges have to be specified using the CIDR format. Multiple IP address ranges can be separated with commas, -e.g.: +a job's port ranges for all the individual IP addresses in the IP ranges. IP address ranges have to be specified using +the CIDR format. Multiple IP address ranges can be separated with commas, e.g.: `192.0.2.0/24,10.0.10.0/24` @@ -34,65 +29,24 @@ port: `443,5665-5669` -Scan jobs can be executed using the `icingacli x509 scan` CLI command. The `--job` option is used to specify the scan -job which should be run: - -``` -icingacli x509 scan --job lan -``` - -## Scheduling Jobs - -Each job may specify a `cron` compatible `schedule` to run periodically at the given interval. The `cron` format is as -follows: - -``` -* * * * * -- - - - - -| | | | | -| | | | | -| | | | +----- day of week (0 - 6) (Sunday to Saturday) -| | | +---------- month (1 - 12) -| | +--------------- day of month (1 - 31) -| +-------------------- hour (0 - 23) -+------------------------- minute (0 - 59) -``` - -Example definitions: - -Description | Definition -------------------------------------------------------------| ---------- -Run once a year at midnight of 1 January | 0 0 1 1 * -Run once a month at midnight of the first day of the month | 0 0 1 * * -Run once a week at midnight on Sunday morning | 0 0 * * 0 -Run once a day at midnight | 0 0 * * * -Run once an hour at the beginning of the hour | 0 * * * * - -Jobs are executed on CLI with the `jobs` command: - -``` -icingacli x509 jobs run -``` +Additionally, each job may also exclude specific **hosts** and **IP** addresses from scan. These hosts won't be scanned +when you run the [scan](04-Scanning.md#scan-command) or [jobs](04-Scanning.md#scheduling-jobs) command. Excluding an entire network and specifying IP addresses in CIDR +format will not work. You must specify concrete **IP**s and **host CN**s separated with commas, e.g: -This command runs all jobs which are currently due and schedules the next execution of all jobs. +`192.0.2.2,192.0.2.5,icinga.com` -You may configure this command as `systemd` service. Just copy the example service definition from -`config/systemd/icinga-x509.service` to `/etc/systemd/system/icinga-x509.service` and enable it afterwards: +### Job Schedules -``` -systemctl enable icinga-x509.service -``` +Schedules are [`cron`](https://crontab.guru) and rule based configs used to run jobs periodically at the given interval. +Every job is allowed to have multiple schedules that can be run independently of each other. Each job schedule provides +different options that you can use to control the scheduling behavior of the [jobs command](04-Scanning.md#scheduling-jobs). -As an alternative if you want scan jobs to be run periodically, you can use the `cron(8)` daemon to run them on a -schedule: +#### Examples -``` -vi /etc/crontab -[...] +A schedule that runs weekly on **Friday** and scans all targets that have not yet been scanned, or +whose last scan is older than `1 week`. -# Runs job 'lan' daily at 2:30 AM -30 2 * * * www-data icingacli x509 scan --job lan -``` +![Weekly Schedules](res/weekly-schedules.png "Weekly Schedules") ## Server Name Indication diff --git a/doc/04-Scanning.md b/doc/04-Scanning.md new file mode 100644 index 00000000..608d18ab --- /dev/null +++ b/doc/04-Scanning.md @@ -0,0 +1,85 @@ +# Scanning + +The Icinga Certificate Monitoring provides CLI commands to scan **hosts** and **IPs** in various ways. +These commands are listed below and can be used individually. It is necessary for all commands to know which IP address +ranges and ports to scan. These can be configured as described [here](03-Configuration.md#configure-jobs). + +## Scan Command + +The scan command, scans targets to find their X.509 certificates and track changes to them. +A **target** is an **IP-port** combination that is generated from the job configuration, taking into account configured +[**SNI**](03-Configuration.md#server-name-indication) maps, so that targets with multiple certificates are also properly +scanned. + +By default, successive calls to the scan command perform partial scans, checking both targets not yet scanned and +targets whose scan is older than 24 hours, to ensure that all targets are rescanned over time and new certificates +are collected. This behavior can be customized through the command [options](#usage-1). + +> **Note** +> +> When rescanning due targets, they will be rescanned regardless of whether the target previously provided a certificate +> or not, to collect new certificates, track changed certificates, and remove decommissioned certificates. + +### Usage + +This scan command can be used like any other Icinga Web cli operations like this: `icingacli x509 scan [OPTIONS]` + +**Options:** + +``` +--job= Scan targets that belong to the specified job. (Required) +--since-last-scan=