diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index b9228f996cf140..00000000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,209 +0,0 @@ -version: 2.1 - -orbs: - ruby: circleci/ruby@1.4.1 - node: circleci/node@5.0.1 - -executors: - default: - parameters: - ruby-version: - type: string - docker: - - image: cimg/ruby:<< parameters.ruby-version >> - environment: - BUNDLE_JOBS: 3 - BUNDLE_RETRY: 3 - CONTINUOUS_INTEGRATION: true - DB_HOST: localhost - DB_USER: root - DISABLE_SIMPLECOV: true - RAILS_ENV: test - - image: cimg/postgres:14.0 - environment: - POSTGRES_USER: root - POSTGRES_HOST_AUTH_METHOD: trust - - image: cimg/redis:6.2 - -commands: - install-system-dependencies: - steps: - - run: - name: Install system dependencies - command: | - sudo apt-get update - sudo apt-get install -y libicu-dev libidn11-dev - install-ruby-dependencies: - parameters: - ruby-version: - type: string - steps: - - run: - command: | - bundle config clean 'true' - bundle config frozen 'true' - bundle config without 'development production' - name: Set bundler settings - - ruby/install-deps: - bundler-version: '2.3.8' - key: ruby<< parameters.ruby-version >>-gems-v1 - wait-db: - steps: - - run: - command: dockerize -wait tcp://localhost:5432 -wait tcp://localhost:6379 -timeout 1m - name: Wait for PostgreSQL and Redis - -jobs: - build: - docker: - - image: cimg/ruby:3.0-node - environment: - RAILS_ENV: test - steps: - - checkout - - install-system-dependencies - - install-ruby-dependencies: - ruby-version: '3.0' - - node/install-packages: - cache-version: v1 - pkg-manager: yarn - - run: - command: ./bin/rails assets:precompile - name: Precompile assets - - persist_to_workspace: - paths: - - public/assets - - public/packs-test - root: . - - test: - parameters: - ruby-version: - type: string - executor: - name: default - ruby-version: << parameters.ruby-version >> - environment: - ALLOW_NOPAM: true - PAM_ENABLED: true - PAM_DEFAULT_SERVICE: pam_test - PAM_CONTROLLED_SERVICE: pam_test_controlled - parallelism: 4 - steps: - - checkout - - install-system-dependencies - - run: - command: sudo apt-get install -y ffmpeg imagemagick libpam-dev - name: Install additional system dependencies - - run: - command: bundle config with 'pam_authentication' - name: Enable PAM authentication - - install-ruby-dependencies: - ruby-version: << parameters.ruby-version >> - - attach_workspace: - at: . - - wait-db - - run: - command: ./bin/rails db:create db:schema:load db:seed - name: Load database schema - - ruby/rspec-test - - test-migrations: - executor: - name: default - ruby-version: '3.0' - steps: - - checkout - - install-system-dependencies - - install-ruby-dependencies: - ruby-version: '3.0' - - wait-db - - run: - command: ./bin/rails db:create - name: Create database - - run: - command: ./bin/rails db:migrate VERSION=20171010025614 - name: Run migrations up to v2.0.0 - - run: - command: ./bin/rails tests:migrations:populate_v2 - name: Populate database with test data - - run: - command: ./bin/rails db:migrate VERSION=20180514140000 - name: Run migrations up to v2.4.0 - - run: - command: ./bin/rails tests:migrations:populate_v2_4 - name: Populate database with test data - - run: - command: ./bin/rails db:migrate - name: Run all remaining migrations - - run: - command: ./bin/rails tests:migrations:check_database - name: Check migration result - - test-two-step-migrations: - executor: - name: default - ruby-version: '3.0' - steps: - - checkout - - install-system-dependencies - - install-ruby-dependencies: - ruby-version: '3.0' - - wait-db - - run: - command: ./bin/rails db:create - name: Create database - - run: - command: ./bin/rails db:migrate VERSION=20171010025614 - name: Run migrations up to v2.0.0 - - run: - command: ./bin/rails tests:migrations:populate_v2 - name: Populate database with test data - - run: - command: ./bin/rails db:migrate VERSION=20180514140000 - name: Run pre-deployment migrations up to v2.4.0 - environment: - SKIP_POST_DEPLOYMENT_MIGRATIONS: true - - run: - command: ./bin/rails tests:migrations:populate_v2_4 - name: Populate database with test data - - run: - command: ./bin/rails db:migrate - name: Run all pre-deployment migrations - environment: - SKIP_POST_DEPLOYMENT_MIGRATIONS: true - - run: - command: ./bin/rails db:migrate - name: Run all post-deployment remaining migrations - - run: - command: ./bin/rails tests:migrations:check_database - name: Check migration result - -workflows: - version: 2 - build-and-test: - jobs: - - build - - test: - matrix: - parameters: - ruby-version: - - '2.7' - - '3.0' - name: test-ruby<< matrix.ruby-version >> - requires: - - build - - test-migrations: - requires: - - build - - test-two-step-migrations: - requires: - - build - - node/run: - cache-version: v1 - name: test-webui - pkg-manager: yarn - requires: - - build - version: lts - yarn-run: test:jest diff --git a/.github/workflows/build-container-image.yml b/.github/workflows/build-container-image.yml new file mode 100644 index 00000000000000..b9aebcc46c60d3 --- /dev/null +++ b/.github/workflows/build-container-image.yml @@ -0,0 +1,92 @@ +on: + workflow_call: + inputs: + platforms: + required: true + type: string + cache: + type: boolean + default: true + use_native_arm64_builder: + type: boolean + push_to_images: + type: string + flavor: + type: string + tags: + type: string + labels: + type: string + +jobs: + build-image: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - uses: docker/setup-qemu-action@v2 + if: contains(inputs.platforms, 'linux/arm64') && !inputs.use_native_arm64_builder + + - uses: docker/setup-buildx-action@v2 + id: buildx + if: ${{ !(inputs.use_native_arm64_builder && contains(inputs.platforms, 'linux/arm64')) }} + + - name: Start a local Docker Builder + if: inputs.use_native_arm64_builder && contains(inputs.platforms, 'linux/arm64') + run: | + docker run --rm -d --name buildkitd -p 1234:1234 --privileged moby/buildkit:latest --addr tcp://0.0.0.0:1234 + + - uses: docker/setup-buildx-action@v2 + id: buildx-native + if: inputs.use_native_arm64_builder && contains(inputs.platforms, 'linux/arm64') + with: + driver: remote + endpoint: tcp://localhost:1234 + platforms: linux/amd64 + append: | + - endpoint: tcp://${{ vars.DOCKER_BUILDER_HETZNER_ARM64_01_HOST }}:13865 + platforms: linux/arm64 + name: mastodon-docker-builder-arm64-01 + driver-opts: + - servername=mastodon-docker-builder-arm64-01 + env: + BUILDER_NODE_1_AUTH_TLS_CACERT: ${{ secrets.DOCKER_BUILDER_HETZNER_ARM64_01_CACERT }} + BUILDER_NODE_1_AUTH_TLS_CERT: ${{ secrets.DOCKER_BUILDER_HETZNER_ARM64_01_CERT }} + BUILDER_NODE_1_AUTH_TLS_KEY: ${{ secrets.DOCKER_BUILDER_HETZNER_ARM64_01_KEY }} + + - name: Log in to Docker Hub + if: contains(inputs.push_to_images, 'tootsuite') + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Log in to the Github Container registry + if: contains(inputs.push_to_images, 'ghcr.io') + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: docker/metadata-action@v4 + id: meta + if: ${{ inputs.push_to_images != '' }} + with: + images: ${{ inputs.push_to_images }} + flavor: ${{ inputs.flavor }} + tags: ${{ inputs.tags }} + labels: ${{ inputs.labels }} + + - uses: docker/build-push-action@v4 + with: + context: . + platforms: ${{ inputs.platforms }} + provenance: false + builder: ${{ steps.buildx.outputs.name || steps.buildx-native.outputs.name }} + push: ${{ inputs.push_to_images != '' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: ${{ inputs.cache && 'type=gha' || '' }} + cache-to: ${{ inputs.cache && 'type=gha,mode=max' || '' }} diff --git a/.github/workflows/build-releases.yml b/.github/workflows/build-releases.yml new file mode 100644 index 00000000000000..c19766b1862ff6 --- /dev/null +++ b/.github/workflows/build-releases.yml @@ -0,0 +1,27 @@ +name: Build container release images +on: + push: + tags: + - '*' + +permissions: + contents: read + packages: write + +jobs: + build-image: + uses: ./.github/workflows/build-container-image.yml + with: + platforms: linux/amd64,linux/arm64 + use_native_arm64_builder: true + push_to_images: | + tootsuite/mastodon + ghcr.io/mastodon/mastodon + # Do not use cache when building releases, so apt update is always ran and the release always contain the latest packages + cache: false + flavor: | + latest=false + tags: | + type=pep440,pattern={{raw}} + type=pep440,pattern=v{{major}}.{{minor}} + secrets: inherit diff --git a/.github/workflows/test-image-build.yml b/.github/workflows/test-image-build.yml new file mode 100644 index 00000000000000..71344c0046aa01 --- /dev/null +++ b/.github/workflows/test-image-build.yml @@ -0,0 +1,15 @@ +name: Test container image build +on: + pull_request: +permissions: + contents: read + +jobs: + build-image: + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + + uses: ./.github/workflows/build-container-image.yml + with: + platforms: linux/amd64 # Testing only on native platform so it is performant diff --git a/.ruby-version b/.ruby-version index 75a22a26ac4a92..818bd47abfc912 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -3.0.3 +3.0.6 diff --git a/CHANGELOG.md b/CHANGELOG.md index 129519519bc297..b0e875a0eb4771 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,235 @@ Changelog All notable changes to this project will be documented in this file. -## [3.4.5] - 2022-11-14 +## End of life notice + +**The 3.5.x branch has reached its end of life and will not receive any further update.** +This means that no security fix will be made available for this branch after this date, and you will need to update to a more recent version (such as the 4.2.x branch) to receive security fixes. + +## [3.5.19] - 2024-02-16 + +### Fixed + +- Fix OmniAuth tests and edge cases in error handling ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29201), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/29207)) + +### Security + +- Fix insufficient checking of remote posts ([GHSA-jhrq-qvrm-qr36](https://github.com/mastodon/mastodon/security/advisories/GHSA-jhrq-qvrm-qr36)) + +## [3.5.18] - 2024-02-14 + +### Security + +- Update the `sidekiq-unique-jobs` dependency (see [GHSA-cmh9-rx85-xj38](https://github.com/mhenrixon/sidekiq-unique-jobs/security/advisories/GHSA-cmh9-rx85-xj38)) + In addition, we have disabled the web interface for `sidekiq-unique-jobs` out of caution. + If you need it, you can re-enable it by setting `ENABLE_SIDEKIQ_UNIQUE_JOBS_UI=true`. + If you only need to clear all locks, you can now use `bundle exec rake sidekiq_unique_jobs:delete_all_locks`. +- Update the `nokogiri` dependency (see [GHSA-xc9x-jj77-9p9j](https://github.com/sparklemotion/nokogiri/security/advisories/GHSA-xc9x-jj77-9p9j)) +- Disable administrative Doorkeeper routes ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/29187)) +- Fix ongoing streaming sessions not being invalidated when applications get deleted in some cases ([GHSA-7w3c-p9j8-mq3x](https://github.com/mastodon/mastodon/security/advisories/GHSA-7w3c-p9j8-mq3x)) + In some rare cases, the streaming server was not notified of access tokens revocation on application deletion. +- Change external authentication behavior to never reattach a new identity to an existing user by default ([GHSA-vm39-j3vx-pch3](https://github.com/mastodon/mastodon/security/advisories/GHSA-vm39-j3vx-pch3)) + Up until now, Mastodon has allowed new identities from external authentication providers to attach to an existing local user based on their verified e-mail address. + This allowed upgrading users from a database-stored password to an external authentication provider, or move from one authentication provider to another. + However, this behavior may be unexpected, and means that when multiple authentication providers are configured, the overall security would be that of the least secure authentication provider. + For these reasons, this behavior is now locked under the `ALLOW_UNSAFE_AUTH_PROVIDER_REATTACH` environment variable. + In addition, regardless of this environment variable, Mastodon will refuse to attach two identities from the same authentication provider to the same account. + +## [3.5.17] - 2024-02-01 + +### Security + +- Fix insufficient origin validation (CVE-2024-23832, [GHSA-3fjr-858r-92rw](https://github.com/mastodon/mastodon/security/advisories/GHSA-3fjr-858r-92rw)) + +## [3.5.16] - 2023-12-04 + +### Changed + +- Change GIF max matrix size error to explicitly mention GIF files ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27927)) +- Change `Follow` activities delivery to bypass availability check ([ShadowJonathan](https://github.com/mastodon/mastodon/pull/27586)) +- Change Content-Security-Policy to be tighter on media paths ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26889)) + +### Fixed + +- Fix incoming status creation date not being restricted to standard ISO8601 ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27655), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/28081)) +- Fix posts from force-sensitized accounts being able to trend ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27620)) +- Fix processing LDSigned activities from actors with unknown public keys ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27474)) + +## [3.5.15] - 2023-10-10 + +### Changed + +- Change user archive export allowed period from 7 days to 6 days ([suddjian](https://github.com/mastodon/mastodon/pull/27200)) + +### Fixed + +- Fix mentions being matched in some URL query strings ([mjankowski](https://github.com/mastodon/mastodon/pull/25656)) +- Fix importer returning negative row estimates ([jgillich](https://github.com/mastodon/mastodon/pull/27258)) +- Fix filtering audit log for entries about disabling 2FA ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27186)) +- Fix tIME chunk not being properly removed from PNG uploads ([TheEssem](https://github.com/mastodon/mastodon/pull/27111)) +- Fix inefficient queries in “Follows and followers” as well as several admin pages ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27116), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/27306)) + +## [3.5.14] - 2023-09-19 + +### Fixed + +- Fix moderator rights inconsistencies ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26729)) +- Fix crash when encountering invalid URL ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26814)) +- Fix cached posts including stale stats ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26409)) +- Fix uploading of video files for which `ffprobe` reports `0/0` average framerate ([NicolaiSoeborg](https://github.com/mastodon/mastodon/pull/26500)) +- Fix unexpected audio stream transcoding when uploaded video is eligible to passthrough ([yufushiro](https://github.com/mastodon/mastodon/pull/26608)) + +### Security + +- Fix incorrect domain name normalization (CVE-2023-42451) + +## [3.5.13] - 2023-09-05 + +### Changed + +- Change remote report processing to accept reports with long comments, but truncate them ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25028)) + +### Fixed + +- **Fix blocking subdomains of an already-blocked domain** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26392)) +- Fix `/api/v1/timelines/tag/:hashtag` allowing for unauthenticated access when public preview is disabled ([danielmbrasil](https://github.com/mastodon/mastodon/pull/26237)) +- Fix inefficiencies in `PlainTextFormatter` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26727)) + +## [3.5.12] - 2023-07-31 + +### Fixed + +- Fix memory leak in streaming server ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/26228)) +- Fix incorrect connect timeout in outgoing requests ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26116)) + +## [3.5.11] - 2023-07-21 + +### Added + +- Add check preventing Sidekiq workers from running with Makara configured ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25850)) + +### Changed + +- Change request timeout handling to use a longer deadline ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26055)) + +### Fixed + +- Fix moderation interface for remote instances with a .zip TLD ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25886)) +- Fix remote accounts being possibly persisted to database with incomplete protocol values ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25886)) +- Fix trending publishers table not rendering correctly on narrow screens ([vmstan](https://github.com/mastodon/mastodon/pull/25945)) + +### Security + +- Fix CSP headers being unintentionally wide ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26105)) + +## [3.5.10] - 2023-07-07 + +### Fixed + +- Fix crash in admin interface when viewing a remote user with verified links ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25796)) +- Fix processing of media files with unusual names ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25788)) + +## [3.5.9] - 2023-07-06 + +### Changed + +- Change OpenGraph-based embeds to allow fullscreen ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25058)) +- Change profile updates to be sent to recently-mentioned servers ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24852)) +- Change auto-linking to allow carets in URL query params ([renchap](https://github.com/mastodon/mastodon/pull/25216)) + +### Removed + +- Remove invalid `X-Frame-Options: ALLOWALL` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25070)) + +### Fixed + +- Fix soft-deleted post cleanup scheduler overwhelming the streaming server ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25519)) +- Fix incorrect pagination headers in `/api/v2/admin/accounts` ([danielmbrasil](https://github.com/mastodon/mastodon/pull/25477)) +- Fix performance of streaming by parsing message JSON once ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25278), [ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25361)) +- Fix CSP headers when `S3_ALIAS_HOST` includes a path component ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25273)) +- Fix `tootctl accounts approve --number N` not aproving N earliest registrations ([danielmbrasil](https://github.com/mastodon/mastodon/pull/24605)) +- Fix being able to vote on your own polls ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25015)) +- Fix race condition when reblogging a status ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25016)) +- Fix “Authorized applications” inefficiently and incorrectly getting last use date ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25060)) +- Fix multiple N+1s in ConversationsController ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25134), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25399), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25499)) +- Fix user archive takeouts when using OpenStack Swift ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24431)) +- Fix inefficiencies in indexing content for search ([VyrCossont](https://github.com/mastodon/mastodon/pull/24285), [VyrCossont](https://github.com/mastodon/mastodon/pull/24342)) + +### Security + +- Update dependencies +- Add hardening headers for user-uploaded files ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25756)) +- Fix verified links possibly hiding important parts of the URL (CVE-2023-36462) +- Fix timeout handling of outbound HTTP requests (CVE-2023-36461) +- Fix arbitrary file creation through media processing (CVE-2023-36460) +- Fix possible XSS in preview cards (CVE-2023-36459) + +## [3.5.8] - 2023-04-04 + +### Fixed + +- Fix crash in `tootctl` commands making use of parallelization when Elasticsearch is enabled ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24182), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/24377)) +- Fix crash in `db:setup` when Elasticsearch is enabled ([rrgeorge](https://github.com/mastodon/mastodon/pull/24302)) +- Fix user archive takeout when using OpenStack Swift or S3 providers with no ACL support ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24200)) +- Fix invalid/expired invites being processed on sign-up ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24337)) + +### Security + +- Update Ruby to 3.0.6 due to ReDoS vulnerabilities ([saizai](https://github.com/mastodon/mastodon/pull/24332)) +- Fix unescaped user input in LDAP query ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24379)) + +# [3.5.7] - 2023-03-16 + +### Added + +- Add `lang` attribute to native language names in language picker in Web UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23749)) +- Add headers to outgoing mails to avoid auto-replies ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23597)) + +### Fixed + +- Fix “Remove all followers from the selected domains” being more destructive than it claims ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23805)) +- Fix case-sensitive check for previously used hashtags in hashtag autocompletion ([deanveloper](https://github.com/mastodon/mastodon/pull/23526)) +- Fix inefficiency when searching accounts per username in admin interface ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23801)) +- Fix server error when failing to follow back followers from `/relationships` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23787)) +- Fix original account being unfollowed on migration before the follow request to the new account could be sent ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/21957)) +- Fix pgBouncer resetting application name on every transaction ([Gargron](https://github.com/mastodon/mastodon/pull/23958)) +- Fix unconfirmed accounts being counted as active users ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23803)) +- Fix `/api/v1/streaming` sub-paths not being redirected ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23988)) +- Fix drag'n'drop upload area text that spans multiple lines not being centered ([vintprox](https://github.com/mastodon/mastodon/pull/24029)) +- Fix sidekiq jobs not triggering Elasticsearch index updates ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24046)) +- Fix missing null check on applications on strike disputes ([kescherCode](https://github.com/mastodon/mastodon/pull/19851)) +- Fix dashboard crash on ElasticSearch server error ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23751)) +- Fix incorrect post links in strikes when the account is remote ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23611)) +- Fix misleading error code when receiving invalid WebAuthn credentials ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23568)) + +### Security + +- Change user backups to use expiring URLs for download when possible ([Gargron](https://github.com/mastodon/mastodon/pull/24136)) +- Add warning for object storage misconfiguration ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24137)) + +## [3.5.6] - 2023-02-09 +### Fixed + +- **Fix changing domain block severity not undoing individual account effects** ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23480)) +- Fix suspension worker crashing on S3-compatible setups without ACL support ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23481)) +- Fix possible race conditions when suspending/unsuspending accounts ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23482)) +- Fix some performance issues with `/admin/instances` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23483)) +- Fix voter count not being cleared when a poll is reset ([afontenot](https://github.com/mastodon/mastodon/pull/23484)) +- Fix attachments of edited statuses not being fetched ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23485)) +- Fix 500 error when marking posts as sensitive while some of them are deleted ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23486)) +- Fix user clean-up scheduler crash when an unconfirmed account has a moderation note ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23487)) +- Fix pending account approval and rejection not being recorded in the admin audit log ([FrancisMurillo](https://github.com/mastodon/mastodon/pull/23488)) +- Fix replies sometimes being delivered to user-blocked domains ([tribela](https://github.com/mastodon/mastodon/pull/23490)) +- Fix sanitizer parsing link text as HTML when stripping unsupported links ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23491)) +- Fix REST API serializer for `Account` not including `moved` when the moved account has itself moved ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23492)) + +### Security + +- Add `form-action` CSP directive ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/23478)) +- Fix unbounded recursion in account discovery ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/22026)) +- Fix unbounded recursion in post discovery ([ClearlyClaire,nametoolong](https://github.com/mastodon/mastodon/pull/23507)) + +## [3.5.5] - 2022-11-14 ## Fixed - Fix nodes order being sometimes mangled when rewriting emoji ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/20677)) diff --git a/Dockerfile b/Dockerfile index 2073cbebff1541..4458be1d986dc7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,7 @@ RUN ARCH= && \ esac && \ echo "Etc/UTC" > /etc/localtime && \ apt-get update && \ + apt-get -yq dist-upgrade && \ apt-get install -y --no-install-recommends ca-certificates wget python apt-utils && \ cd ~ && \ wget -q https://nodejs.org/download/release/v$NODE_VER/node-v$NODE_VER-linux-$ARCH.tar.gz && \ @@ -27,7 +28,7 @@ RUN ARCH= && \ mv node-v$NODE_VER-linux-$ARCH /opt/node # Install Ruby 3.0 -ENV RUBY_VER="3.0.3" +ENV RUBY_VER="3.0.6" RUN apt-get update && \ apt-get install -y --no-install-recommends build-essential \ bison libyaml-dev libgdbm-dev libreadline-dev libjemalloc-dev \ @@ -46,7 +47,7 @@ RUN apt-get update && \ ENV PATH="${PATH}:/opt/ruby/bin:/opt/node/bin" -RUN npm install -g npm@latest && \ +RUN npm install -g npm@9 && \ npm install -g yarn && \ gem install bundler && \ apt-get update && \ diff --git a/Gemfile b/Gemfile index 2e77fb42a69025..f76776c8cf4fce 100644 --- a/Gemfile +++ b/Gemfile @@ -66,6 +66,7 @@ gem 'oj', '~> 3.13' gem 'ox', '~> 2.14' gem 'parslet' gem 'posix-spawn' +gem 'public_suffix', '~> 4.0.7' gem 'pundit', '~> 2.2' gem 'premailer-rails' gem 'rack-attack', '~> 6.6' diff --git a/Gemfile.lock b/Gemfile.lock index e12fdc237dc4a4..fd708aca648b7d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,40 +1,40 @@ GEM remote: https://rubygems.org/ specs: - actioncable (6.1.6) - actionpack (= 6.1.6) - activesupport (= 6.1.6) + actioncable (6.1.7.4) + actionpack (= 6.1.7.4) + activesupport (= 6.1.7.4) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailbox (6.1.6) - actionpack (= 6.1.6) - activejob (= 6.1.6) - activerecord (= 6.1.6) - activestorage (= 6.1.6) - activesupport (= 6.1.6) + actionmailbox (6.1.7.4) + actionpack (= 6.1.7.4) + activejob (= 6.1.7.4) + activerecord (= 6.1.7.4) + activestorage (= 6.1.7.4) + activesupport (= 6.1.7.4) mail (>= 2.7.1) - actionmailer (6.1.6) - actionpack (= 6.1.6) - actionview (= 6.1.6) - activejob (= 6.1.6) - activesupport (= 6.1.6) + actionmailer (6.1.7.4) + actionpack (= 6.1.7.4) + actionview (= 6.1.7.4) + activejob (= 6.1.7.4) + activesupport (= 6.1.7.4) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (6.1.6) - actionview (= 6.1.6) - activesupport (= 6.1.6) + actionpack (6.1.7.4) + actionview (= 6.1.7.4) + activesupport (= 6.1.7.4) rack (~> 2.0, >= 2.0.9) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0) - actiontext (6.1.6) - actionpack (= 6.1.6) - activerecord (= 6.1.6) - activestorage (= 6.1.6) - activesupport (= 6.1.6) + actiontext (6.1.7.4) + actionpack (= 6.1.7.4) + activerecord (= 6.1.7.4) + activestorage (= 6.1.7.4) + activesupport (= 6.1.7.4) nokogiri (>= 1.8.5) - actionview (6.1.6) - activesupport (= 6.1.6) + actionview (6.1.7.4) + activesupport (= 6.1.7.4) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) @@ -45,22 +45,22 @@ GEM case_transform (>= 0.2) jsonapi-renderer (>= 0.1.1.beta1, < 0.3) active_record_query_trace (1.8) - activejob (6.1.6) - activesupport (= 6.1.6) + activejob (6.1.7.4) + activesupport (= 6.1.7.4) globalid (>= 0.3.6) - activemodel (6.1.6) - activesupport (= 6.1.6) - activerecord (6.1.6) - activemodel (= 6.1.6) - activesupport (= 6.1.6) - activestorage (6.1.6) - actionpack (= 6.1.6) - activejob (= 6.1.6) - activerecord (= 6.1.6) - activesupport (= 6.1.6) + activemodel (6.1.7.4) + activesupport (= 6.1.7.4) + activerecord (6.1.7.4) + activemodel (= 6.1.7.4) + activesupport (= 6.1.7.4) + activestorage (6.1.7.4) + actionpack (= 6.1.7.4) + activejob (= 6.1.7.4) + activerecord (= 6.1.7.4) + activesupport (= 6.1.7.4) marcel (~> 1.0) mini_mime (>= 1.1.0) - activesupport (6.1.6) + activesupport (6.1.7.4) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) @@ -165,7 +165,7 @@ GEM climate_control (0.2.0) coderay (1.1.3) color_diff (0.1) - concurrent-ruby (1.1.10) + concurrent-ruby (1.2.2) connection_pool (2.2.5) cose (1.0.0) cbor (~> 0.5.9) @@ -197,7 +197,7 @@ GEM docile (1.3.4) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - doorkeeper (5.5.4) + doorkeeper (5.6.6) railties (>= 5) dotenv (2.7.6) dotenv-rails (2.7.6) @@ -214,7 +214,7 @@ GEM faraday (~> 1) multi_json encryptor (3.0.0) - erubi (1.10.0) + erubi (1.12.0) et-orbi (1.2.7) tzinfo excon (0.76.0) @@ -273,7 +273,7 @@ GEM addressable (~> 2.7) omniauth (~> 1.9) openid_connect (~> 1.2) - globalid (1.0.0) + globalid (1.0.1) activesupport (>= 5.0) hamlit (2.13.0) temple (>= 0.8.2) @@ -304,7 +304,7 @@ GEM httplog (1.5.0) rack (>= 1.0) rainbow (>= 2.0.0) - i18n (1.10.0) + i18n (1.14.1) concurrent-ruby (~> 1.0) i18n-tasks (1.0.10) activesupport (>= 4.0.2) @@ -374,9 +374,9 @@ GEM activesupport (>= 4) railties (>= 4) request_store (~> 1.0) - loofah (2.18.0) + loofah (2.21.3) crass (~> 1.0.2) - nokogiri (>= 1.5.9) + nokogiri (>= 1.12.0) mail (2.7.1) mini_mime (>= 0.1.1) makara (0.5.1) @@ -394,8 +394,8 @@ GEM mime-types-data (~> 3.2015) mime-types-data (3.2022.0105) mini_mime (1.1.2) - mini_portile2 (2.8.0) - minitest (5.15.0) + mini_portile2 (2.8.5) + minitest (5.18.1) msgpack (1.5.1) multi_json (1.15.0) multipart-post (2.1.1) @@ -403,9 +403,9 @@ GEM net-scp (3.0.0) net-ssh (>= 2.6.5, < 7.0.0) net-ssh (6.1.0) - nio4r (2.5.8) - nokogiri (1.13.6) - mini_portile2 (~> 2.8.0) + nio4r (2.5.9) + nokogiri (1.16.2) + mini_portile2 (~> 2.8.2) racc (~> 1.4) nsa (0.2.8) activesupport (>= 4.2, < 7) @@ -413,7 +413,7 @@ GEM sidekiq (>= 3.5) statsd-ruby (~> 1.4, >= 1.4.0) oj (3.13.11) - omniauth (1.9.1) + omniauth (1.9.2) hashie (>= 3.4.6) rack (>= 1.6.2, < 3) omniauth-cas (2.0.0) @@ -473,8 +473,8 @@ GEM pundit (2.2.0) activesupport (>= 3.0.0) raabro (1.4.0) - racc (1.6.0) - rack (2.2.3) + racc (1.7.3) + rack (2.2.7) rack-attack (6.6.1) rack (>= 1.0, < 3) rack-cors (1.1.1) @@ -489,20 +489,20 @@ GEM rack rack-test (1.1.0) rack (>= 1.0, < 3) - rails (6.1.6) - actioncable (= 6.1.6) - actionmailbox (= 6.1.6) - actionmailer (= 6.1.6) - actionpack (= 6.1.6) - actiontext (= 6.1.6) - actionview (= 6.1.6) - activejob (= 6.1.6) - activemodel (= 6.1.6) - activerecord (= 6.1.6) - activestorage (= 6.1.6) - activesupport (= 6.1.6) + rails (6.1.7.4) + actioncable (= 6.1.7.4) + actionmailbox (= 6.1.7.4) + actionmailer (= 6.1.7.4) + actionpack (= 6.1.7.4) + actiontext (= 6.1.7.4) + actionview (= 6.1.7.4) + activejob (= 6.1.7.4) + activemodel (= 6.1.7.4) + activerecord (= 6.1.7.4) + activestorage (= 6.1.7.4) + activesupport (= 6.1.7.4) bundler (>= 1.15.0) - railties (= 6.1.6) + railties (= 6.1.7.4) sprockets-rails (>= 2.0.0) rails-controller-testing (1.0.5) actionpack (>= 5.0.1.rc1) @@ -511,16 +511,17 @@ GEM rails-dom-testing (2.0.3) activesupport (>= 4.2.0) nokogiri (>= 1.6) - rails-html-sanitizer (1.4.2) - loofah (~> 2.3) + rails-html-sanitizer (1.6.0) + loofah (~> 2.21) + nokogiri (~> 1.14) rails-i18n (6.0.0) i18n (>= 0.7, < 2) railties (>= 6.0.0, < 7) rails-settings-cached (0.6.6) rails (>= 4.2.0) - railties (6.1.6) - actionpack (= 6.1.6) - activesupport (= 6.1.6) + railties (6.1.7.4) + actionpack (= 6.1.7.4) + activesupport (= 6.1.7.4) method_source rake (>= 12.2) thor (~> 1.0) @@ -592,7 +593,7 @@ GEM fugit (~> 1.1, >= 1.1.6) safety_net_attestation (0.4.0) jwt (~> 2.0) - sanitize (6.0.0) + sanitize (6.0.1) crass (~> 1.0.2) nokogiri (>= 1.12.0) scenic (1.6.0) @@ -611,10 +612,11 @@ GEM rufus-scheduler (~> 3.2) sidekiq (>= 4) tilt (>= 1.4.0) - sidekiq-unique-jobs (7.1.22) + sidekiq-unique-jobs (7.1.33) brpoplpush-redis_script (> 0.1.1, <= 2.0.0) concurrent-ruby (~> 1.0, >= 1.0.5) - sidekiq (>= 5.0, < 8.0) + redis (< 5.0) + sidekiq (>= 5.0, < 7.0) thor (>= 0.20, < 3.0) simple-navigation (4.3.0) activesupport (>= 2.3.2) @@ -652,7 +654,7 @@ GEM unicode-display_width (>= 1.1.1, < 3) terrapin (0.6.0) climate_control (>= 0.0.3, < 1.0) - thor (1.2.1) + thor (1.2.2) tilt (2.0.10) tpm-key_attestation (0.9.0) bindata (~> 2.4) @@ -670,7 +672,7 @@ GEM twitter-text (3.1.0) idn-ruby unf (~> 0.1.0) - tzinfo (2.0.4) + tzinfo (2.0.6) concurrent-ruby (~> 1.0) tzinfo-data (1.2022.1) tzinfo (>= 1.0.0) @@ -719,7 +721,7 @@ GEM xorcist (1.1.2) xpath (3.2.0) nokogiri (~> 1.8) - zeitwerk (2.5.4) + zeitwerk (2.6.8) PLATFORMS ruby @@ -803,6 +805,7 @@ DEPENDENCIES private_address_check (~> 0.5) pry-byebug (~> 3.9) pry-rails (~> 0.3) + public_suffix (~> 4.0.7) puma (~> 5.6) pundit (~> 2.2) rack (~> 2.2.3) diff --git a/README.md b/README.md index 4b48e071d8f641..10b356ae76ff27 100644 --- a/README.md +++ b/README.md @@ -5,13 +5,11 @@ [![Build Status](https://img.shields.io/circleci/project/github/mastodon/mastodon.svg)][circleci] [![Code Climate](https://img.shields.io/codeclimate/maintainability/mastodon/mastodon.svg)][code_climate] [![Crowdin](https://d322cqt584bo4o.cloudfront.net/mastodon/localized.svg)][crowdin] -[![Docker Pulls](https://img.shields.io/docker/pulls/tootsuite/mastodon.svg)][docker] [releases]: https://github.com/mastodon/mastodon/releases [circleci]: https://circleci.com/gh/mastodon/mastodon [code_climate]: https://codeclimate.com/github/mastodon/mastodon [crowdin]: https://crowdin.com/project/mastodon -[docker]: https://hub.docker.com/r/tootsuite/mastodon/ Mastodon is a **free, open-source social network server** based on ActivityPub where users can follow friends and discover new ones. On Mastodon, users can publish anything they want: links, pictures, text, video. All Mastodon servers are interoperable as a federated network (users on one server can seamlessly communicate with users from another one, including non-Mastodon software that implements ActivityPub)! @@ -28,6 +26,7 @@ Click below to **learn more** in a video: - [View sponsors](https://joinmastodon.org/sponsors) - [Blog](https://blog.joinmastodon.org) - [Documentation](https://docs.joinmastodon.org) +- [Official Docker image](https://github.com/mastodon/mastodon/pkgs/container/mastodon) - [Browse Mastodon servers](https://joinmastodon.org/communities) - [Browse Mastodon apps](https://joinmastodon.org/apps) diff --git a/SECURITY.md b/SECURITY.md index 62e23f73609fd1..1b46ac8c550634 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -10,11 +10,10 @@ A "vulnerability in Mastodon" is a vulnerability in the code distributed through ## Supported Versions -| Version | Supported | -| ------- | ------------------ | -| 3.5.x | Yes | -| 3.4.x | Yes | -| 3.3.x | No | -| < 3.3 | No | - -[bug-bounty]: https://app.intigriti.com/programs/mastodon/mastodonio/detail +| Version | Supported | +| ------- | ---------------- | +| 4.2.x | Yes | +| 4.1.x | Yes | +| 4.0.x | No | +| 3.5.x | No | +| < 3.5 | No | diff --git a/app/chewy/accounts_index.rb b/app/chewy/accounts_index.rb index e38e14a10699ad..9b78af85d6f623 100644 --- a/app/chewy/accounts_index.rb +++ b/app/chewy/accounts_index.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class AccountsIndex < Chewy::Index + include DatetimeClampingConcern + settings index: { refresh_interval: '30s' }, analysis: { analyzer: { content: { @@ -38,6 +40,6 @@ class AccountsIndex < Chewy::Index field :following_count, type: 'long', value: ->(account) { account.following_count } field :followers_count, type: 'long', value: ->(account) { account.followers_count } - field :last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at } + field :last_status_at, type: 'date', value: ->(account) { clamp_date(account.last_status_at || account.created_at) } end end diff --git a/app/chewy/concerns/datetime_clamping_concern.rb b/app/chewy/concerns/datetime_clamping_concern.rb new file mode 100644 index 00000000000000..7f176b6e5489f4 --- /dev/null +++ b/app/chewy/concerns/datetime_clamping_concern.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module DatetimeClampingConcern + extend ActiveSupport::Concern + + MIN_ISO8601_DATETIME = '0000-01-01T00:00:00Z'.to_datetime.freeze + MAX_ISO8601_DATETIME = '9999-12-31T23:59:59Z'.to_datetime.freeze + + class_methods do + def clamp_date(datetime) + datetime.clamp(MIN_ISO8601_DATETIME, MAX_ISO8601_DATETIME) + end + end +end diff --git a/app/chewy/tags_index.rb b/app/chewy/tags_index.rb index df3d9e4cce2920..8c778dc65d09d6 100644 --- a/app/chewy/tags_index.rb +++ b/app/chewy/tags_index.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class TagsIndex < Chewy::Index + include DatetimeClampingConcern + settings index: { refresh_interval: '30s' }, analysis: { analyzer: { content: { @@ -36,6 +38,6 @@ class TagsIndex < Chewy::Index field :reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? } field :usage, type: 'long', value: ->(tag, crutches) { tag.history.aggregate(crutches.time_period).accounts } - field :last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at } + field :last_status_at, type: 'date', value: ->(tag) { clamp_date(tag.last_status_at || tag.created_at) } end end diff --git a/app/controllers/admin/accounts_controller.rb b/app/controllers/admin/accounts_controller.rb index e0ae71b9f2feda..6ad393676dc853 100644 --- a/app/controllers/admin/accounts_controller.rb +++ b/app/controllers/admin/accounts_controller.rb @@ -49,12 +49,14 @@ def enable def approve authorize @account.user, :approve? @account.user.approve! + log_action :approve, @account.user redirect_to admin_accounts_path(status: 'pending'), notice: I18n.t('admin.accounts.approved_msg', username: @account.acct) end def reject authorize @account.user, :reject? DeleteAccountService.new.call(@account, reserve_email: false, reserve_username: false) + log_action :reject, @account.user redirect_to admin_accounts_path(status: 'pending'), notice: I18n.t('admin.accounts.rejected_msg', username: @account.acct) end diff --git a/app/controllers/admin/domain_blocks_controller.rb b/app/controllers/admin/domain_blocks_controller.rb index 16defc1ea87b6c..4dbbe6dd6adb23 100644 --- a/app/controllers/admin/domain_blocks_controller.rb +++ b/app/controllers/admin/domain_blocks_controller.rb @@ -25,7 +25,7 @@ def create @domain_block.errors.delete(:domain) render :new else - if existing_domain_block.present? + if existing_domain_block.present? && existing_domain_block.domain == TagManager.instance.normalize_domain(@domain_block.domain.strip) @domain_block = existing_domain_block @domain_block.update(resource_params) end @@ -43,12 +43,8 @@ def create def update authorize :domain_block, :update? - @domain_block.update(update_params) - - severity_changed = @domain_block.severity_changed? - - if @domain_block.save - DomainBlockWorker.perform_async(@domain_block.id, severity_changed) + if @domain_block.update(update_params) + DomainBlockWorker.perform_async(@domain_block.id, @domain_block.severity_previously_changed?) log_action :update, @domain_block redirect_to admin_instances_path(limited: '1'), notice: I18n.t('admin.domain_blocks.created_msg') else diff --git a/app/controllers/admin/instances_controller.rb b/app/controllers/admin/instances_controller.rb index 5c82331dea2f78..7c44e88b7b227b 100644 --- a/app/controllers/admin/instances_controller.rb +++ b/app/controllers/admin/instances_controller.rb @@ -57,7 +57,7 @@ def set_instances end def preload_delivery_failures! - warning_domains_map = DeliveryFailureTracker.warning_domains_map + warning_domains_map = DeliveryFailureTracker.warning_domains_map(@instances.map(&:domain)) @instances.each do |instance| instance.failure_days = warning_domains_map[instance.domain] diff --git a/app/controllers/api/v1/admin/accounts_controller.rb b/app/controllers/api/v1/admin/accounts_controller.rb index 65ed69f7ba444e..423ef7cc45dcfa 100644 --- a/app/controllers/api/v1/admin/accounts_controller.rb +++ b/app/controllers/api/v1/admin/accounts_controller.rb @@ -54,12 +54,14 @@ def enable def approve authorize @account.user, :approve? @account.user.approve! + log_action :approve, @account.user render json: @account, serializer: REST::Admin::AccountSerializer end def reject authorize @account.user, :reject? DeleteAccountService.new.call(@account, reserve_email: false, reserve_username: false) + log_action :reject, @account.user render json: @account, serializer: REST::Admin::AccountSerializer end diff --git a/app/controllers/api/v1/conversations_controller.rb b/app/controllers/api/v1/conversations_controller.rb index 6c7583403758e2..818ba6ebba2ebb 100644 --- a/app/controllers/api/v1/conversations_controller.rb +++ b/app/controllers/api/v1/conversations_controller.rb @@ -11,7 +11,7 @@ class Api::V1::ConversationsController < Api::BaseController def index @conversations = paginated_conversations - render json: @conversations, each_serializer: REST::ConversationSerializer + render json: @conversations, each_serializer: REST::ConversationSerializer, relationships: StatusRelationshipsPresenter.new(@conversations.map(&:last_status), current_user&.account_id) end def read @@ -32,6 +32,19 @@ def set_conversation def paginated_conversations AccountConversation.where(account: current_account) + .includes( + account: :account_stat, + last_status: [ + :media_attachments, + :preview_cards, + :status_stat, + :tags, + { + active_mentions: [account: :account_stat], + account: :account_stat, + }, + ] + ) .to_a_paginated_by_id(limit_param(LIMIT), params_slice(:max_id, :since_id, :min_id)) end diff --git a/app/controllers/api/v1/statuses/reblogs_controller.rb b/app/controllers/api/v1/statuses/reblogs_controller.rb index 1be15a5a439604..a4079a16db09b4 100644 --- a/app/controllers/api/v1/statuses/reblogs_controller.rb +++ b/app/controllers/api/v1/statuses/reblogs_controller.rb @@ -2,6 +2,8 @@ class Api::V1::Statuses::ReblogsController < Api::BaseController include Authorization + include Redisable + include Lockable before_action -> { doorkeeper_authorize! :write, :'write:statuses' } before_action :require_user! @@ -10,7 +12,9 @@ class Api::V1::Statuses::ReblogsController < Api::BaseController override_rate_limit_headers :create, family: :statuses def create - @status = ReblogService.new.call(current_account, @reblog, reblog_params) + with_lock("reblog:#{current_account.id}:#{@reblog.id}") do + @status = ReblogService.new.call(current_account, @reblog, reblog_params) + end render json: @status, serializer: REST::StatusSerializer end diff --git a/app/controllers/api/v1/timelines/tag_controller.rb b/app/controllers/api/v1/timelines/tag_controller.rb index e6746ba6d1c45b..7da6c2043d7a99 100644 --- a/app/controllers/api/v1/timelines/tag_controller.rb +++ b/app/controllers/api/v1/timelines/tag_controller.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true class Api::V1::Timelines::TagController < Api::BaseController + before_action -> { doorkeeper_authorize! :read, :'read:statuses' }, only: :show, if: :require_auth? before_action :load_tag after_action :insert_pagination_headers, unless: -> { @statuses.empty? } @@ -15,6 +16,10 @@ def show private + def require_auth? + !Setting.timeline_preview + end + def load_tag @tag = Tag.find_normalized(params[:id]) end diff --git a/app/controllers/api/v2/admin/accounts_controller.rb b/app/controllers/api/v2/admin/accounts_controller.rb index a89e6835ee10ed..ed39deced22150 100644 --- a/app/controllers/api/v2/admin/accounts_controller.rb +++ b/app/controllers/api/v2/admin/accounts_controller.rb @@ -17,6 +17,14 @@ class Api::V2::Admin::AccountsController < Api::V1::Admin::AccountsController private + def next_path + api_v2_admin_accounts_url(pagination_params(max_id: pagination_max_id)) if records_continue? + end + + def prev_path + api_v2_admin_accounts_url(pagination_params(min_id: pagination_since_id)) unless @accounts.empty? + end + def filtered_accounts AccountFilter.new(filter_params).results end diff --git a/app/controllers/auth/omniauth_callbacks_controller.rb b/app/controllers/auth/omniauth_callbacks_controller.rb index f9cf6d6551f681..8beba06fa76c09 100644 --- a/app/controllers/auth/omniauth_callbacks_controller.rb +++ b/app/controllers/auth/omniauth_callbacks_controller.rb @@ -5,7 +5,7 @@ class Auth::OmniauthCallbacksController < Devise::OmniauthCallbacksController def self.provides_callback_for(provider) define_method provider do - @user = User.find_for_oauth(request.env['omniauth.auth'], current_user) + @user = User.find_for_omniauth(request.env['omniauth.auth'], current_user) if @user.persisted? LoginActivity.create( @@ -23,6 +23,9 @@ def self.provides_callback_for(provider) session["devise.#{provider}_data"] = request.env['omniauth.auth'] redirect_to new_user_registration_url end + rescue ActiveRecord::RecordInvalid + flash[:alert] = I18n.t('devise.failure.omniauth_user_creation_failure') if is_navigational_format? + redirect_to new_user_session_url end end diff --git a/app/controllers/auth/registrations_controller.rb b/app/controllers/auth/registrations_controller.rb index 1c3adbd786bd91..e04069d74ee31b 100644 --- a/app/controllers/auth/registrations_controller.rb +++ b/app/controllers/auth/registrations_controller.rb @@ -46,7 +46,7 @@ def build_resource(hash = nil) super(hash) resource.locale = I18n.locale - resource.invite_code = params[:invite_code] if resource.invite_code.blank? + resource.invite_code = @invite&.code if resource.invite_code.blank? resource.registration_form_time = session[:registration_form_time] resource.sign_up_ip = request.remote_ip diff --git a/app/controllers/auth/sessions_controller.rb b/app/controllers/auth/sessions_controller.rb index c4c8151e33314b..656e80e0230d2e 100644 --- a/app/controllers/auth/sessions_controller.rb +++ b/app/controllers/auth/sessions_controller.rb @@ -12,6 +12,10 @@ class Auth::SessionsController < Devise::SessionsController before_action :set_instance_presenter, only: [:new] before_action :set_body_classes + content_security_policy only: :new do |p| + p.form_action(false) + end + def create super do |resource| # We only need to call this if this hasn't already been diff --git a/app/controllers/backups_controller.rb b/app/controllers/backups_controller.rb new file mode 100644 index 00000000000000..5891da6f6d62a2 --- /dev/null +++ b/app/controllers/backups_controller.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class BackupsController < ApplicationController + include RoutingHelper + + skip_before_action :require_functional! + + before_action :authenticate_user! + before_action :set_backup + + def download + case Paperclip::Attachment.default_options[:storage] + when :s3 + redirect_to @backup.dump.expiring_url(10) + when :fog + if Paperclip::Attachment.default_options.dig(:fog_credentials, :openstack_temp_url_key).present? + redirect_to @backup.dump.expiring_url(Time.now.utc + 10) + else + redirect_to full_asset_url(@backup.dump.url) + end + when :filesystem + redirect_to full_asset_url(@backup.dump.url) + end + end + + private + + def set_backup + @backup = current_user.backups.find(params[:id]) + end +end diff --git a/app/controllers/concerns/signature_verification.rb b/app/controllers/concerns/signature_verification.rb index 4dd0cac55da060..65f45e004b9119 100644 --- a/app/controllers/concerns/signature_verification.rb +++ b/app/controllers/concerns/signature_verification.rb @@ -219,7 +219,7 @@ def account_from_key_id(key_id) stoplight_wrap_request { ResolveAccountService.new.call(key_id.gsub(/\Aacct:/, '')) } elsif !ActivityPub::TagManager.instance.local_uri?(key_id) account = ActivityPub::TagManager.instance.uri_to_resource(key_id, Account) - account ||= stoplight_wrap_request { ActivityPub::FetchRemoteKeyService.new.call(key_id, id: false) } + account ||= stoplight_wrap_request { ActivityPub::FetchRemoteKeyService.new.call(key_id) } account end rescue Mastodon::HostValidationError diff --git a/app/controllers/media_controller.rb b/app/controllers/media_controller.rb index ee82625a03e9f9..42e32ce2aee259 100644 --- a/app/controllers/media_controller.rb +++ b/app/controllers/media_controller.rb @@ -46,6 +46,6 @@ def check_playable end def allow_iframing - response.headers['X-Frame-Options'] = 'ALLOWALL' + response.headers.delete('X-Frame-Options') end end diff --git a/app/controllers/oauth/authorizations_controller.rb b/app/controllers/oauth/authorizations_controller.rb index bb5d639ced6a9d..bddf15eb537d53 100644 --- a/app/controllers/oauth/authorizations_controller.rb +++ b/app/controllers/oauth/authorizations_controller.rb @@ -7,6 +7,10 @@ class Oauth::AuthorizationsController < Doorkeeper::AuthorizationsController before_action :authenticate_resource_owner! before_action :set_cache_headers + content_security_policy do |p| + p.form_action(false) + end + include Localized private diff --git a/app/controllers/oauth/authorized_applications_controller.rb b/app/controllers/oauth/authorized_applications_controller.rb index 45151cdd7754d0..63afc4c068f134 100644 --- a/app/controllers/oauth/authorized_applications_controller.rb +++ b/app/controllers/oauth/authorized_applications_controller.rb @@ -8,6 +8,8 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio before_action :require_not_suspended!, only: :destroy before_action :set_body_classes + before_action :set_last_used_at_by_app, only: :index, unless: -> { request.format == :json } + skip_before_action :require_functional! include Localized @@ -30,4 +32,14 @@ def store_current_location def require_not_suspended! forbidden if current_account.suspended? end + + def set_last_used_at_by_app + @last_used_at_by_app = Doorkeeper::AccessToken + .select('DISTINCT ON (application_id) application_id, last_used_at') + .where(resource_owner_id: current_resource_owner.id) + .where.not(last_used_at: nil) + .order(application_id: :desc, last_used_at: :desc) + .pluck(:application_id, :last_used_at) + .to_h + end end diff --git a/app/controllers/relationships_controller.rb b/app/controllers/relationships_controller.rb index 96cce55e9e4f6e..de5dc5879280d6 100644 --- a/app/controllers/relationships_controller.rb +++ b/app/controllers/relationships_controller.rb @@ -19,6 +19,8 @@ def update @form.save rescue ActionController::ParameterMissing # Do nothing + rescue Mastodon::NotPermittedError, ActiveRecord::RecordNotFound + flash[:alert] = I18n.t('relationships.follow_failure') if action_from_button == 'follow' ensure redirect_to relationships_path(filter_params) end @@ -60,8 +62,8 @@ def action_from_button 'unfollow' elsif params[:remove_from_followers] 'remove_from_followers' - elsif params[:block_domains] - 'block_domains' + elsif params[:block_domains] || params[:remove_domains_from_followers] + 'remove_domains_from_followers' end end diff --git a/app/controllers/settings/two_factor_authentication/webauthn_credentials_controller.rb b/app/controllers/settings/two_factor_authentication/webauthn_credentials_controller.rb index a50d30f06f32c6..8435155ddcefe7 100644 --- a/app/controllers/settings/two_factor_authentication/webauthn_credentials_controller.rb +++ b/app/controllers/settings/two_factor_authentication/webauthn_credentials_controller.rb @@ -52,7 +52,7 @@ def create end else flash[:error] = I18n.t('webauthn_credentials.create.error') - status = :internal_server_error + status = :unprocessable_entity end else flash[:error] = t('webauthn_credentials.create.error') diff --git a/app/controllers/statuses_controller.rb b/app/controllers/statuses_controller.rb index c52170d08188ac..3d1b74489b549c 100644 --- a/app/controllers/statuses_controller.rb +++ b/app/controllers/statuses_controller.rb @@ -48,7 +48,7 @@ def embed return not_found if @status.hidden? || @status.reblog? expires_in 180, public: true - response.headers['X-Frame-Options'] = 'ALLOWALL' + response.headers.delete('X-Frame-Options') render layout: 'embedded' end diff --git a/app/helpers/formatting_helper.rb b/app/helpers/formatting_helper.rb index 6adb1393a94b12..8a0fa085778850 100644 --- a/app/helpers/formatting_helper.rb +++ b/app/helpers/formatting_helper.rb @@ -56,6 +56,10 @@ def account_bio_format(account) end def account_field_value_format(field, with_rel_me: true) - html_aware_format(field.value, field.account.local?, with_rel_me: with_rel_me, with_domains: true, multiline: false) + if field.verified? && !field.account.local? + TextFormatter.shortened_link(field.value_for_verification) + else + html_aware_format(field.value, field.account.local?, with_rel_me: with_rel_me, with_domains: true, multiline: false) + end end end diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb index 102e4b13281ad7..332e623f316c7b 100644 --- a/app/helpers/jsonld_helper.rb +++ b/app/helpers/jsonld_helper.rb @@ -157,8 +157,8 @@ def safe_for_forwarding?(original, compacted) end end - def fetch_resource(uri, id, on_behalf_of = nil) - unless id + def fetch_resource(uri, id_is_known, on_behalf_of = nil) + unless id_is_known json = fetch_resource_without_id_validation(uri, on_behalf_of) return if !json.is_a?(Hash) || unsupported_uri_scheme?(json['id']) @@ -176,7 +176,19 @@ def fetch_resource_without_id_validation(uri, on_behalf_of = nil, raise_on_tempo build_request(uri, on_behalf_of).perform do |response| raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error - body_to_json(response.body_with_limit) if response.code == 200 + body_to_json(response.body_with_limit) if response.code == 200 && valid_activitypub_content_type?(response) + end + end + + def valid_activitypub_content_type?(response) + return true if response.mime_type == 'application/activity+json' + + # When the mime type is `application/ld+json`, we need to check the profile, + # but `http.rb` does not parse it for us. + return false unless response.mime_type == 'application/ld+json' + + response.headers[HTTP::Headers::CONTENT_TYPE]&.split(';')&.map(&:strip)&.any? do |str| + str.start_with?('profile="') && str[9...-1].split.include?('https://www.w3.org/ns/activitystreams') end end diff --git a/app/javascript/mastodon/actions/importer/normalizer.js b/app/javascript/mastodon/actions/importer/normalizer.js index 48be2a600c7209..b4fba27ca50ce1 100644 --- a/app/javascript/mastodon/actions/importer/normalizer.js +++ b/app/javascript/mastodon/actions/importer/normalizer.js @@ -82,6 +82,7 @@ export function normalizeStatus(status, normalOldStatus) { } const spoilerText = normalStatus.spoiler_text || ''; + const searchContent = ([spoilerText, status.content].concat((status.poll && status.poll.options) ? status.poll.options.map(option => option.title) : [])).concat(status.media_attachments.map(att => att.description)).join('\n\n').replace(//g, '\n').replace(/<\/p>

/g, '\n\n'); const emojiMap = makeEmojiMap(normalStatus); normalStatus.search_index = searchTextFromRawStatus(status); diff --git a/app/javascript/mastodon/actions/notifications.js b/app/javascript/mastodon/actions/notifications.js index fa500d7e06557a..96cf628d693a47 100644 --- a/app/javascript/mastodon/actions/notifications.js +++ b/app/javascript/mastodon/actions/notifications.js @@ -45,7 +45,7 @@ defineMessages({ }); const fetchRelatedRelationships = (dispatch, notifications) => { - const accountIds = notifications.map(item => item.account.id); + const accountIds = notifications.filter(item => ['follow', 'follow_request', 'admin.sign_up'].indexOf(item.type) !== -1).map(item => item.account.id); if (accountIds.length > 0) { dispatch(fetchRelationships(accountIds)); diff --git a/app/javascript/mastodon/features/compose/components/language_dropdown.js b/app/javascript/mastodon/features/compose/components/language_dropdown.js index d76490c775f89b..94ac8cd4eeb72a 100644 --- a/app/javascript/mastodon/features/compose/components/language_dropdown.js +++ b/app/javascript/mastodon/features/compose/components/language_dropdown.js @@ -222,7 +222,7 @@ class LanguageDropdownMenu extends React.PureComponent { return (

- {lang[2]} ({lang[1]}) + {lang[2]} ({lang[1]})
); } diff --git a/app/javascript/mastodon/features/status/containers/detailed_status_container.js b/app/javascript/mastodon/features/status/containers/detailed_status_container.js index d307e05bccf5c9..bf53fca2041d0b 100644 --- a/app/javascript/mastodon/features/status/containers/detailed_status_container.js +++ b/app/javascript/mastodon/features/status/containers/detailed_status_container.js @@ -172,6 +172,7 @@ const mapDispatchToProps = (dispatch, { intl }) => ({ dispatch(hideStatus(status.get('id'))); } }, + }); export default injectIntl(connect(makeMapStateToProps, mapDispatchToProps)(DetailedStatus)); diff --git a/app/javascript/mastodon/locales/en.json b/app/javascript/mastodon/locales/en.json index 3017133f66969a..e0d54059ba7596 100644 --- a/app/javascript/mastodon/locales/en.json +++ b/app/javascript/mastodon/locales/en.json @@ -192,11 +192,6 @@ "error.unexpected_crash.next_steps_addons": "Try disabling them and refreshing the page. If that does not help, you may still be able to use Mastodon through a different browser or native app.", "errors.unexpected_crash.copy_stacktrace": "Copy stacktrace to clipboard", "errors.unexpected_crash.report_issue": "Report issue", - "federation.change": "Adjust status federation", - "federation.federated.long": "Allow toot to reach other instances", - "federation.federated.short": "Federated", - "federation.local_only.long": "Restrict this toot only to my instance", - "federation.local_only.short": "Local-only", "explore.search_results": "Search results", "explore.suggested_follows": "For you", "explore.title": "Explore", diff --git a/app/javascript/mastodon/locales/pt-BR.json b/app/javascript/mastodon/locales/pt-BR.json index 20a74ed0fe0949..d6f0ead947e536 100644 --- a/app/javascript/mastodon/locales/pt-BR.json +++ b/app/javascript/mastodon/locales/pt-BR.json @@ -190,11 +190,6 @@ "error.unexpected_crash.next_steps_addons": "Tente desativá-los e atualizar a página. Se isso não ajudar, você ainda poderá usar o Mastodon por meio de um navegador diferente ou de um aplicativo nativo.", "errors.unexpected_crash.copy_stacktrace": "Copiar dados do erro para área de transferência", "errors.unexpected_crash.report_issue": "Reportar problema", - "federation.change": "Ajustar federação do toot", - "federation.federated.long": "Permitir que o toot chegue a outras instâncias", - "federation.federated.short": "Federado", - "federation.local_only.long": "Restringir o toot somente à minha instância", - "federation.local_only.short": "Somente local", "explore.search_results": "Resultado da pesquisa", "explore.suggested_follows": "Para você", "explore.title": "Explorar", @@ -469,7 +464,6 @@ "status.history.created": "{name} criou {date}", "status.history.edited": "{name} editou {date}", "status.load_more": "Ver mais", - "status.local_only": "Esse post só é visível para outros usuários da sua instância", "status.media_hidden": "Mídia sensível", "status.mention": "Mencionar @{name}", "status.more": "Mais", diff --git a/app/javascript/mastodon/reducers/compose.js b/app/javascript/mastodon/reducers/compose.js index 6bf2b23d7643e4..8bb87c695a0fc4 100644 --- a/app/javascript/mastodon/reducers/compose.js +++ b/app/javascript/mastodon/reducers/compose.js @@ -192,11 +192,12 @@ const ignoreSuggestion = (state, position, token, completion, path) => { }; const sortHashtagsByUse = (state, tags) => { - const personalHistory = state.get('tagHistory'); + const personalHistory = state.get('tagHistory').map(tag => tag.toLowerCase()); - return tags.sort((a, b) => { - const usedA = personalHistory.includes(a.name); - const usedB = personalHistory.includes(b.name); + const tagsWithLowercase = tags.map(t => ({ ...t, lowerName: t.name.toLowerCase() })); + const sorted = tagsWithLowercase.sort((a, b) => { + const usedA = personalHistory.includes(a.lowerName); + const usedB = personalHistory.includes(b.lowerName); if (usedA === usedB) { return 0; @@ -206,6 +207,8 @@ const sortHashtagsByUse = (state, tags) => { return 1; } }); + sorted.forEach(tag => delete tag.lowerName); + return sorted; }; const insertEmoji = (state, position, emojiData, needsSpace) => { @@ -381,20 +384,6 @@ export default function compose(state = initialState, action) { map.set('spoiler_text', ''); } }); - case COMPOSE_REPLY_CANCEL: - case COMPOSE_QUOTE_CANCEL: - case COMPOSE_RESET: - return state.withMutations(map => { - map.set('id', null); - map.set('in_reply_to', null); - map.set('quote_from', null); - map.set('text', ''); - map.set('spoiler', false); - map.set('spoiler_text', ''); - map.set('privacy', state.get('default_privacy')); - map.set('poll', null); - map.set('idempotencyKey', uuid()); - }); case COMPOSE_SUBMIT_REQUEST: return state.set('is_submitting', true); case COMPOSE_UPLOAD_CHANGE_REQUEST: @@ -572,6 +561,35 @@ export default function compose(state = initialState, action) { map.set('spoiler_text', ''); } + if (action.status.get('poll')) { + map.set('poll', ImmutableMap({ + options: action.status.getIn(['poll', 'options']).map(x => x.get('title')), + multiple: action.status.getIn(['poll', 'multiple']), + expires_in: expiresInFromExpiresAt(action.status.getIn(['poll', 'expires_at'])), + })); + } + }); + case COMPOSE_SET_STATUS: + return state.withMutations(map => { + map.set('id', action.status.get('id')); + map.set('text', action.text); + map.set('in_reply_to', action.status.get('in_reply_to_id')); + map.set('privacy', action.status.get('visibility')); + map.set('media_attachments', action.status.get('media_attachments')); + map.set('focusDate', new Date()); + map.set('caretPosition', null); + map.set('idempotencyKey', uuid()); + map.set('sensitive', action.status.get('sensitive')); + map.set('language', action.status.get('language')); + + if (action.spoiler_text.length > 0) { + map.set('spoiler', true); + map.set('spoiler_text', action.spoiler_text); + } else { + map.set('spoiler', false); + map.set('spoiler_text', ''); + } + if (action.status.get('poll')) { map.set('poll', ImmutableMap({ options: action.status.getIn(['poll', 'options']).map(x => x.get('title')), diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index e291163307cc5d..6702f13430a7a3 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -4288,6 +4288,7 @@ a.status-card.compact:hover { display: flex; align-items: center; justify-content: center; + text-align: center; color: $secondary-text-color; font-size: 18px; font-weight: 500; diff --git a/app/lib/account_reach_finder.rb b/app/lib/account_reach_finder.rb index 706ce8c1fbbbf7..481e254396e453 100644 --- a/app/lib/account_reach_finder.rb +++ b/app/lib/account_reach_finder.rb @@ -6,7 +6,7 @@ def initialize(account) end def inboxes - (followers_inboxes + reporters_inboxes + relay_inboxes).uniq + (followers_inboxes + reporters_inboxes + recently_mentioned_inboxes + relay_inboxes).uniq end private @@ -19,6 +19,13 @@ def reporters_inboxes Account.where(id: @account.targeted_reports.select(:account_id)).inboxes end + def recently_mentioned_inboxes + cutoff_id = Mastodon::Snowflake.id_at(2.days.ago, with_random: false) + recent_statuses = @account.statuses.recent.where(id: cutoff_id...).limit(200) + + Account.joins(:mentions).where(mentions: { status: recent_statuses }).inboxes.take(2000) + end + def relay_inboxes Relay.enabled.pluck(:inbox_url) end diff --git a/app/models/account_statuses_filter.rb b/app/lib/account_statuses_filter.rb similarity index 100% rename from app/models/account_statuses_filter.rb rename to app/lib/account_statuses_filter.rb diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index 7ff06ea390cf14..0a909b18b32a33 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -106,7 +106,8 @@ def status_from_object actor_id = value_or_id(first_of_value(@object['attributedTo'])) if actor_id == @account.uri - return ActivityPub::Activity.factory({ 'type' => 'Create', 'actor' => actor_id, 'object' => @object }, @account).perform + virtual_object = { 'type' => 'Create', 'actor' => actor_id, 'object' => @object } + return ActivityPub::Activity.factory(virtual_object, @account, request_id: @options[:request_id]).perform end end @@ -152,9 +153,10 @@ def follow_from_object def fetch_remote_original_status if object_uri.start_with?('http') return if ActivityPub::TagManager.instance.local_uri?(object_uri) - ActivityPub::FetchRemoteStatusService.new.call(object_uri, id: true, on_behalf_of: @account.followers.local.first) + + ActivityPub::FetchRemoteStatusService.new.call(object_uri, on_behalf_of: @account.followers.local.first, request_id: @options[:request_id]) elsif @object['url'].present? - ::FetchRemoteStatusService.new.call(@object['url']) + ::FetchRemoteStatusService.new.call(@object['url'], request_id: @options[:request_id]) end end diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 63d5c0af9f81b9..fe1ae94fba621c 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -224,7 +224,7 @@ def process_mention(tag) return if tag['href'].blank? account = account_from_uri(tag['href']) - account = ActivityPub::FetchRemoteAccountService.new.call(tag['href']) if account.nil? + account = ActivityPub::FetchRemoteAccountService.new.call(tag['href'], request_id: @options[:request_id]) if account.nil? return if account.nil? @@ -329,18 +329,18 @@ def poll_vote! def resolve_thread(status) return unless status.reply? && status.thread.nil? && Request.valid_url?(in_reply_to_uri) - ThreadResolveWorker.perform_async(status.id, in_reply_to_uri) + ThreadResolveWorker.perform_async(status.id, in_reply_to_uri, { 'request_id' => @options[:request_id]}) end def fetch_replies(status) collection = @object['replies'] return if collection.nil? - replies = ActivityPub::FetchRepliesService.new.call(status, collection, false) + replies = ActivityPub::FetchRepliesService.new.call(status, collection, allow_synchronous_requests: false, request_id: @options[:request_id]) return unless replies.nil? uri = value_or_id(collection) - ActivityPub::FetchRepliesWorker.perform_async(status.id, uri) unless uri.nil? + ActivityPub::FetchRepliesWorker.perform_async(status.id, uri, { 'request_id' => @options[:request_id]}) unless uri.nil? end def conversation_from_uri(uri) diff --git a/app/lib/activitypub/activity/flag.rb b/app/lib/activitypub/activity/flag.rb index b0443849a6b8b5..7539bda422ff38 100644 --- a/app/lib/activitypub/activity/flag.rb +++ b/app/lib/activitypub/activity/flag.rb @@ -16,7 +16,7 @@ def perform @account, target_account, status_ids: target_statuses.nil? ? [] : target_statuses.map(&:id), - comment: @json['content'] || '', + comment: report_comment, uri: report_uri ) end @@ -35,4 +35,8 @@ def object_uris def report_uri @json['id'] unless @json['id'].nil? || invalid_origin?(@json['id']) end + + def report_comment + (@json['content'] || '')[0...5000] + end end diff --git a/app/lib/activitypub/activity/update.rb b/app/lib/activitypub/activity/update.rb index 5b3238ece5f1eb..e7c3bc9bf83dec 100644 --- a/app/lib/activitypub/activity/update.rb +++ b/app/lib/activitypub/activity/update.rb @@ -18,7 +18,7 @@ def perform def update_account return reject_payload! if @account.uri != object_uri - ActivityPub::ProcessAccountService.new.call(@account.username, @account.domain, @object, signed_with_known_key: true) + ActivityPub::ProcessAccountService.new.call(@account.username, @account.domain, @object, signed_with_known_key: true, request_id: @options[:request_id]) end def update_status @@ -28,6 +28,6 @@ def update_status return if @status.nil? - ActivityPub::ProcessStatusUpdateService.new.call(@status, @object) + ActivityPub::ProcessStatusUpdateService.new.call(@status, @object, request_id: @options[:request_id]) end end diff --git a/app/lib/activitypub/linked_data_signature.rb b/app/lib/activitypub/linked_data_signature.rb index e853a970e81d22..d49c7896cd7008 100644 --- a/app/lib/activitypub/linked_data_signature.rb +++ b/app/lib/activitypub/linked_data_signature.rb @@ -18,8 +18,8 @@ def verify_account! return unless type == 'RsaSignature2017' - creator = ActivityPub::TagManager.instance.uri_to_resource(creator_uri, Account) - creator ||= ActivityPub::FetchRemoteKeyService.new.call(creator_uri, id: false) + creator = ActivityPub::TagManager.instance.uri_to_resource(creator_uri, Account) + creator = ActivityPub::FetchRemoteKeyService.new.call(creator_uri) if creator&.public_key.blank? return if creator.nil? @@ -27,9 +27,9 @@ def verify_account! document_hash = hash(@json.without('signature')) to_be_verified = options_hash + document_hash - if creator.keypair.public_key.verify(OpenSSL::Digest.new('SHA256'), Base64.decode64(signature), to_be_verified) - creator - end + creator if creator.keypair.public_key.verify(OpenSSL::Digest.new('SHA256'), Base64.decode64(signature), to_be_verified) + rescue OpenSSL::PKey::RSAError + false end def sign!(creator, sign_with: nil) diff --git a/app/lib/activitypub/parser/status_parser.rb b/app/lib/activitypub/parser/status_parser.rb index 028d0950c288b3..d6d4d11563bafa 100644 --- a/app/lib/activitypub/parser/status_parser.rb +++ b/app/lib/activitypub/parser/status_parser.rb @@ -2,7 +2,6 @@ class ActivityPub::Parser::StatusParser include JsonLdHelper - include FormattingHelper # @param [Hash] json # @param [Hash] magic_values @@ -56,7 +55,8 @@ def title end def created_at - @object['published']&.to_datetime + datetime = @object['published']&.to_datetime + datetime if datetime.present? && (0..9999).cover?(datetime.year) rescue ArgumentError nil end diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb index f6b9741fa1820e..a809ce0902daf3 100644 --- a/app/lib/activitypub/tag_manager.rb +++ b/app/lib/activitypub/tag_manager.rb @@ -27,6 +27,8 @@ def url_for(target) when :note, :comment, :activity return activity_account_status_url(target.account, target) if target.reblog? short_account_status_url(target.account, target) + when :flag + target.uri end end @@ -41,6 +43,8 @@ def uri_for(target) account_status_url(target.account, target) when :emoji emoji_url(target) + when :flag + target.uri end end diff --git a/app/lib/admin/account_statuses_filter.rb b/app/lib/admin/account_statuses_filter.rb new file mode 100644 index 00000000000000..94927e4b6806c9 --- /dev/null +++ b/app/lib/admin/account_statuses_filter.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class Admin::AccountStatusesFilter < AccountStatusesFilter + private + + def blocked? + false + end +end diff --git a/app/lib/admin/system_check.rb b/app/lib/admin/system_check.rb index 877a42ef639100..c4cca0406416ab 100644 --- a/app/lib/admin/system_check.rb +++ b/app/lib/admin/system_check.rb @@ -2,6 +2,7 @@ class Admin::SystemCheck ACTIVE_CHECKS = [ + Admin::SystemCheck::MediaPrivacyCheck, Admin::SystemCheck::DatabaseSchemaCheck, Admin::SystemCheck::SidekiqProcessCheck, Admin::SystemCheck::RulesCheck, diff --git a/app/lib/admin/system_check/elasticsearch_check.rb b/app/lib/admin/system_check/elasticsearch_check.rb index 1b48a5415a3a72..fa603bf8967bd0 100644 --- a/app/lib/admin/system_check/elasticsearch_check.rb +++ b/app/lib/admin/system_check/elasticsearch_check.rb @@ -20,7 +20,7 @@ def message def running_version @running_version ||= begin Chewy.client.info['version']['number'] - rescue Faraday::ConnectionFailed + rescue Faraday::ConnectionFailed, Elasticsearch::Transport::Transport::Error nil end end diff --git a/app/lib/admin/system_check/media_privacy_check.rb b/app/lib/admin/system_check/media_privacy_check.rb new file mode 100644 index 00000000000000..1df05b120ea80a --- /dev/null +++ b/app/lib/admin/system_check/media_privacy_check.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +class Admin::SystemCheck::MediaPrivacyCheck < Admin::SystemCheck::BaseCheck + include RoutingHelper + + def skip? + !current_user.can?(:view_devops) + end + + def pass? + check_media_uploads! + @failure_message.nil? + end + + def message + Admin::SystemCheck::Message.new(@failure_message, @failure_value, @failure_action, true) + end + + private + + def check_media_uploads! + if Rails.configuration.x.use_s3 + check_media_listing_inaccessible_s3! + else + check_media_listing_inaccessible! + end + end + + def check_media_listing_inaccessible! + full_url = full_asset_url(media_attachment.file.url(:original, false)) + + # Check if we can list the uploaded file. If true, that's an error + directory_url = Addressable::URI.parse(full_url) + directory_url.query = nil + filename = directory_url.path.gsub(%r{.*/}, '') + directory_url.path = directory_url.path.gsub(%r{/[^/]+\Z}, '/') + Request.new(:get, directory_url, allow_local: true).perform do |res| + if res.truncated_body&.include?(filename) + @failure_message = use_storage? ? :upload_check_privacy_error_object_storage : :upload_check_privacy_error + @failure_action = 'https://docs.joinmastodon.org/admin/optional/object-storage/#FS' + end + end + rescue + nil + end + + def check_media_listing_inaccessible_s3! + urls_to_check = [] + paperclip_options = Paperclip::Attachment.default_options + s3_protocol = paperclip_options[:s3_protocol] + s3_host_alias = paperclip_options[:s3_host_alias] + s3_host_name = paperclip_options[:s3_host_name] + bucket_name = paperclip_options.dig(:s3_credentials, :bucket) + + urls_to_check << "#{s3_protocol}://#{s3_host_alias}/" if s3_host_alias.present? + urls_to_check << "#{s3_protocol}://#{s3_host_name}/#{bucket_name}/" + urls_to_check.uniq.each do |full_url| + check_s3_listing!(full_url) + break if @failure_message.present? + end + rescue + nil + end + + def check_s3_listing!(full_url) + bucket_url = Addressable::URI.parse(full_url) + bucket_url.path = bucket_url.path.delete_suffix(media_attachment.file.path(:original)) + bucket_url.query = "max-keys=1&x-random=#{SecureRandom.hex(10)}" + Request.new(:get, bucket_url, allow_local: true).perform do |res| + if res.truncated_body&.include?('ListBucketResult') + @failure_message = :upload_check_privacy_error_object_storage + @failure_action = 'https://docs.joinmastodon.org/admin/optional/object-storage/#S3' + end + end + end + + def media_attachment + @media_attachment ||= begin + attachment = Account.representative.media_attachments.first + if attachment.present? + attachment.touch # rubocop:disable Rails/SkipsModelValidations + attachment + else + create_test_attachment! + end + end + end + + def create_test_attachment! + Tempfile.create(%w(test-upload .jpg), binmode: true) do |tmp_file| + tmp_file.write( + Base64.decode64( + '/9j/4QAiRXhpZgAATU0AKgAAAAgAAQESAAMAAAABAAYAAAA' \ + 'AAAD/2wCEAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBA' \ + 'QEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE' \ + 'BAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAf/AABEIAAEAAgMBEQACEQEDEQH/x' \ + 'ABKAAEAAAAAAAAAAAAAAAAAAAALEAEAAAAAAAAAAAAAAAAAAAAAAQEAAAAAAAAAAAAAAAA' \ + 'AAAAAEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwA/8H//2Q==' + ) + ) + tmp_file.flush + Account.representative.media_attachments.create!(file: tmp_file) + end + end +end diff --git a/app/lib/admin/system_check/message.rb b/app/lib/admin/system_check/message.rb index bfcad3bf3d00a6..ad8d4b6073872a 100644 --- a/app/lib/admin/system_check/message.rb +++ b/app/lib/admin/system_check/message.rb @@ -1,11 +1,12 @@ # frozen_string_literal: true class Admin::SystemCheck::Message - attr_reader :key, :value, :action + attr_reader :key, :value, :action, :critical - def initialize(key, value = nil, action = nil) - @key = key - @value = value - @action = action + def initialize(key, value = nil, action = nil, critical = false) + @key = key + @value = value + @action = action + @critical = critical end end diff --git a/app/lib/application_extension.rb b/app/lib/application_extension.rb index d61ec0e6e7f121..d1222656b75e18 100644 --- a/app/lib/application_extension.rb +++ b/app/lib/application_extension.rb @@ -4,16 +4,32 @@ module ApplicationExtension extend ActiveSupport::Concern included do + include Redisable + validates :name, length: { maximum: 60 } validates :website, url: true, length: { maximum: 2_000 }, if: :website? validates :redirect_uri, length: { maximum: 2_000 } - end - def most_recently_used_access_token - @most_recently_used_access_token ||= access_tokens.where.not(last_used_at: nil).order(last_used_at: :desc).first + # The relationship used between Applications and AccessTokens is using + # dependent: delete_all, which means the ActiveRecord callback in + # AccessTokenExtension is not run, so instead we manually announce to + # streaming that these tokens are being deleted. + before_destroy :push_to_streaming_api, prepend: true end def confirmation_redirect_uri redirect_uri.lines.first.strip end + + def push_to_streaming_api + # TODO: #28793 Combine into a single topic + payload = Oj.dump(event: :kill) + access_tokens.in_batches do |tokens| + redis.pipelined do |pipeline| + tokens.ids.each do |id| + pipeline.publish("timeline:access_token:#{id}", payload) + end + end + end + end end diff --git a/app/lib/delivery_failure_tracker.rb b/app/lib/delivery_failure_tracker.rb index 7c4e28eb79f74f..66c1fd8c00da8d 100644 --- a/app/lib/delivery_failure_tracker.rb +++ b/app/lib/delivery_failure_tracker.rb @@ -65,8 +65,13 @@ def warning_domains domains - UnavailableDomain.all.pluck(:domain) end - def warning_domains_map - warning_domains.index_with { |domain| redis.scard(exhausted_deliveries_key_by(domain)) } + def warning_domains_map(domains = nil) + if domains.nil? + warning_domains.index_with { |domain| redis.scard(exhausted_deliveries_key_by(domain)) } + else + domains -= UnavailableDomain.where(domain: domains).pluck(:domain) + domains.index_with { |domain| redis.scard(exhausted_deliveries_key_by(domain)) }.filter { |_, days| days.positive? } + end end private diff --git a/app/lib/importer/base_importer.rb b/app/lib/importer/base_importer.rb index ea522c600cf2e4..7009db11f7bb26 100644 --- a/app/lib/importer/base_importer.rb +++ b/app/lib/importer/base_importer.rb @@ -34,7 +34,9 @@ def optimize_for_search! # Estimate the amount of documents that would be indexed. Not exact! # @returns [Integer] def estimate! - ActiveRecord::Base.connection_pool.with_connection { |connection| connection.select_one("SELECT reltuples AS estimate FROM pg_class WHERE relname = '#{index.adapter.target.table_name}'")['estimate'].to_i } + reltuples = ActiveRecord::Base.connection_pool.with_connection { |connection| connection.select_one("SELECT reltuples FROM pg_class WHERE relname = '#{index.adapter.target.table_name}'")['reltuples'].to_i } + # If the table has never yet been vacuumed or analyzed, reltuples contains -1 + [reltuples, 0].max end # Import data from the database into the index diff --git a/app/lib/link_details_extractor.rb b/app/lib/link_details_extractor.rb index b0c4e4f425e37b..78265fa1cbeb5f 100644 --- a/app/lib/link_details_extractor.rb +++ b/app/lib/link_details_extractor.rb @@ -140,7 +140,7 @@ def link_type end def html - player_url.present? ? content_tag(:iframe, nil, src: player_url, width: width, height: height, allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil + player_url.present? ? content_tag(:iframe, nil, src: player_url, width: width, height: height, allowfullscreen: 'true', allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil end def width diff --git a/app/lib/plain_text_formatter.rb b/app/lib/plain_text_formatter.rb index 08aa29696450a8..d1ff6808b2a995 100644 --- a/app/lib/plain_text_formatter.rb +++ b/app/lib/plain_text_formatter.rb @@ -1,9 +1,7 @@ # frozen_string_literal: true class PlainTextFormatter - include ActionView::Helpers::TextHelper - - NEWLINE_TAGS_RE = /(
|
|<\/p>)+/.freeze + NEWLINE_TAGS_RE = %r{(
|
|

)+} attr_reader :text, :local @@ -18,7 +16,10 @@ def to_s if local? text else - strip_tags(insert_newlines).chomp + node = Nokogiri::HTML.fragment(insert_newlines) + # Elements that are entirely removed with our Sanitize config + node.xpath('.//iframe|.//math|.//noembed|.//noframes|.//noscript|.//plaintext|.//script|.//style|.//svg|.//xmp').remove + node.text.chomp end end diff --git a/app/lib/request.rb b/app/lib/request.rb index 4289da9333de82..6be62c45762cf4 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -4,14 +4,60 @@ require 'socket' require 'resolv' -# Monkey-patch the HTTP.rb timeout class to avoid using a timeout block +# Use our own timeout class to avoid using HTTP.rb's timeout block # around the Socket#open method, since we use our own timeout blocks inside # that method -class HTTP::Timeout::PerOperation +# +# Also changes how the read timeout behaves so that it is cumulative (closer +# to HTTP::Timeout::Global, but still having distinct timeouts for other +# operation types) +class PerOperationWithDeadline < HTTP::Timeout::PerOperation + READ_DEADLINE = 30 + + def initialize(*args) + super + + @read_deadline = options.fetch(:read_deadline, READ_DEADLINE) + end + def connect(socket_class, host, port, nodelay = false) @socket = socket_class.open(host, port) @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay end + + # Reset deadline when the connection is re-used for different requests + def reset_counter + @deadline = nil + end + + # Read data from the socket + def readpartial(size, buffer = nil) + @deadline ||= Process.clock_gettime(Process::CLOCK_MONOTONIC) + @read_deadline + + timeout = false + loop do + result = @socket.read_nonblock(size, buffer, exception: false) + + return :eof if result.nil? + + remaining_time = @deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC) + raise HTTP::TimeoutError, "Read timed out after #{@read_timeout} seconds" if timeout + raise HTTP::TimeoutError, "Read timed out after a total of #{@read_deadline} seconds" if remaining_time <= 0 + return result if result != :wait_readable + + # marking the socket for timeout. Why is this not being raised immediately? + # it seems there is some race-condition on the network level between calling + # #read_nonblock and #wait_readable, in which #read_nonblock signalizes waiting + # for reads, and when waiting for x seconds, it returns nil suddenly without completing + # the x seconds. In a normal case this would be a timeout on wait/read, but it can + # also mean that the socket has been closed by the server. Therefore we "mark" the + # socket for timeout and try to read more bytes. If it returns :eof, it's all good, no + # timeout. Else, the first timeout was a proper timeout. + # This hack has to be done because io/wait#wait_readable doesn't provide a value for when + # the socket is closed by the server, and HTTP::Parser doesn't provide the limit for the chunks. + timeout = true unless @socket.to_io.wait_readable([remaining_time, @read_timeout].min) + end + end end class Request @@ -20,7 +66,7 @@ class Request # We enforce a 5s timeout on DNS resolving, 5s timeout on socket opening # and 5s timeout on the TLS handshake, meaning the worst case should take # about 15s in total - TIMEOUT = { connect: 5, read: 10, write: 10 }.freeze + TIMEOUT = { connect_timeout: 5, read_timeout: 10, write_timeout: 10, read_deadline: 30 }.freeze include RoutingHelper @@ -31,6 +77,7 @@ def initialize(verb, url, **options) @url = Addressable::URI.parse(url).normalize @http_client = options.delete(:http_client) @options = options.merge(socket_class: use_proxy? ? ProxySocket : Socket) + @options = @options.merge(timeout_class: PerOperationWithDeadline, timeout_options: TIMEOUT) @options = @options.merge(Rails.configuration.x.http_client_proxy) if use_proxy? @headers = {} @@ -94,7 +141,7 @@ def valid_url?(url) end def http_client - HTTP.use(:auto_inflate).timeout(TIMEOUT.dup).follow(max_hops: 3) + HTTP.use(:auto_inflate).follow(max_hops: 3) end end @@ -218,11 +265,11 @@ def open(host, *args) end until socks.empty? - _, available_socks, = IO.select(nil, socks, nil, Request::TIMEOUT[:connect]) + _, available_socks, = IO.select(nil, socks, nil, Request::TIMEOUT[:connect_timeout]) if available_socks.nil? socks.each(&:close) - raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect]} seconds" + raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect_timeout]} seconds" end available_socks.each do |sock| diff --git a/app/lib/status_reach_finder.rb b/app/lib/status_reach_finder.rb index 98e502bb68c7ec..546fef9e23cc9f 100644 --- a/app/lib/status_reach_finder.rb +++ b/app/lib/status_reach_finder.rb @@ -70,7 +70,7 @@ def replies_account_ids def followers_inboxes if @status.in_reply_to_local_account? && distributable? - @status.account.followers.or(@status.thread.account.followers).inboxes + @status.account.followers.or(@status.thread.account.followers.not_domain_blocked_by_account(@status.account)).inboxes elsif @status.direct_visibility? || @status.limited_visibility? [] else diff --git a/app/lib/tag_manager.rb b/app/lib/tag_manager.rb index 39a98c3eb92db7..b4671a0fb0e75a 100644 --- a/app/lib/tag_manager.rb +++ b/app/lib/tag_manager.rb @@ -7,18 +7,18 @@ class TagManager include RoutingHelper def web_domain?(domain) - domain.nil? || domain.gsub(/[\/]/, '').casecmp(Rails.configuration.x.web_domain).zero? + domain.nil? || domain.delete_suffix('/').casecmp(Rails.configuration.x.web_domain).zero? end def local_domain?(domain) - domain.nil? || domain.gsub(/[\/]/, '').casecmp(Rails.configuration.x.local_domain).zero? + domain.nil? || domain.delete_suffix('/').casecmp(Rails.configuration.x.local_domain).zero? end def normalize_domain(domain) return if domain.nil? uri = Addressable::URI.new - uri.host = domain.gsub(/[\/]/, '') + uri.host = domain.delete_suffix('/') uri.normalized_host end @@ -27,5 +27,7 @@ def local_url?(url) domain = uri.host + (uri.port ? ":#{uri.port}" : '') TagManager.instance.web_domain?(domain) + rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError + false end end diff --git a/app/lib/text_formatter.rb b/app/lib/text_formatter.rb index c596700747384c..3a729dd5d4486f 100644 --- a/app/lib/text_formatter.rb +++ b/app/lib/text_formatter.rb @@ -50,6 +50,26 @@ def to_s html.html_safe # rubocop:disable Rails/OutputSafety end + class << self + include ERB::Util + + def shortened_link(url, rel_me: false) + url = Addressable::URI.parse(url).to_s + rel = rel_me ? (DEFAULT_REL + %w(me)) : DEFAULT_REL + + prefix = url.match(URL_PREFIX_REGEX).to_s + display_url = url[prefix.length, 30] + suffix = url[prefix.length + 30..-1] + cutoff = url[prefix.length..-1].length > 30 + + <<~HTML.squish.html_safe # rubocop:disable Rails/OutputSafety + #{h(display_url)} + HTML + rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError + h(url) + end + end + private def quotify(html, quote_uri) @@ -77,19 +97,7 @@ def rewrite end def link_to_url(entity) - url = Addressable::URI.parse(entity[:url]).to_s - rel = with_rel_me? ? (DEFAULT_REL + %w(me)) : DEFAULT_REL - - prefix = url.match(URL_PREFIX_REGEX).to_s - display_url = url[prefix.length, 30] - suffix = url[prefix.length + 30..-1] - cutoff = url[prefix.length..-1].length > 30 - - <<~HTML.squish - #{h(display_url)} - HTML - rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError - h(entity[:url]) + TextFormatter.shortened_link(entity[:url], rel_me: with_rel_me?) end def link_to_hashtag(entity) diff --git a/app/lib/video_metadata_extractor.rb b/app/lib/video_metadata_extractor.rb index 2896620cb21b09..f27d34868a2798 100644 --- a/app/lib/video_metadata_extractor.rb +++ b/app/lib/video_metadata_extractor.rb @@ -43,6 +43,9 @@ def parse_metadata @height = video_stream[:height] @frame_rate = video_stream[:avg_frame_rate] == '0/0' ? nil : Rational(video_stream[:avg_frame_rate]) @r_frame_rate = video_stream[:r_frame_rate] == '0/0' ? nil : Rational(video_stream[:r_frame_rate]) + # For some video streams the frame_rate reported by `ffprobe` will be 0/0, but for these streams we + # should use `r_frame_rate` instead. Video screencast generated by Gnome Screencast have this issue. + @frame_rate ||= @r_frame_rate end if (audio_stream = audio_streams.first) diff --git a/app/mailers/application_mailer.rb b/app/mailers/application_mailer.rb index a37682eca63ab6..4edcb75f31bef4 100644 --- a/app/mailers/application_mailer.rb +++ b/app/mailers/application_mailer.rb @@ -7,6 +7,8 @@ class ApplicationMailer < ActionMailer::Base helper :instance helper :formatting + after_action :set_autoreply_headers! + protected def locale_for_account(account) @@ -14,4 +16,10 @@ def locale_for_account(account) yield end end + + def set_autoreply_headers! + headers['Precedence'] = 'list' + headers['X-Auto-Response-Suppress'] = 'All' + headers['Auto-Submitted'] = 'auto-generated' + end end diff --git a/app/models/account.rb b/app/models/account.rb index bd94142c430f68..3c49dd0ee7df8e 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -61,9 +61,9 @@ class Account < ApplicationRecord trust_level ) - USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i - MENTION_RE = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[[:word:]\.\-]+[[:word:]]+)?)/i - URL_PREFIX_RE = /\Ahttp(s?):\/\/[^\/]+/ + USERNAME_RE = /[a-z0-9_]+([a-z0-9_.-]+[a-z0-9_]+)?/i + MENTION_RE = %r{(? { where(actor_type: %w(Application Service)) } scope :groups, -> { where(actor_type: 'Group') } scope :alphabetic, -> { order(domain: :asc, username: :asc) } - scope :matches_username, ->(value) { where(arel_table[:username].matches("#{value}%")) } + scope :matches_username, ->(value) { where('lower((username)::text) LIKE lower(?)', "#{value}%") } scope :matches_display_name, ->(value) { where(arel_table[:display_name].matches("#{value}%")) } scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) } scope :without_unapproved, -> { left_outer_joins(:user).remote.or(left_outer_joins(:user).merge(User.approved.confirmed)) } scope :searchable, -> { without_unapproved.without_suspended.where(moved_to_account_id: nil) } scope :discoverable, -> { searchable.without_silenced.where(discoverable: true).left_outer_joins(:account_stat) } scope :followable_by, ->(account) { joins(arel_table.join(Follow.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(Follow.arel_table[:target_account_id]).and(Follow.arel_table[:account_id].eq(account.id))).join_sources).where(Follow.arel_table[:id].eq(nil)).joins(arel_table.join(FollowRequest.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(FollowRequest.arel_table[:target_account_id]).and(FollowRequest.arel_table[:account_id].eq(account.id))).join_sources).where(FollowRequest.arel_table[:id].eq(nil)) } - scope :by_recent_status, -> { order(Arel.sql('(case when account_stats.last_status_at is null then 1 else 0 end) asc, account_stats.last_status_at desc, accounts.id desc')) } - scope :by_recent_sign_in, -> { order(Arel.sql('(case when users.current_sign_in_at is null then 1 else 0 end) asc, users.current_sign_in_at desc, accounts.id desc')) } + scope :by_recent_status, -> { includes(:account_stat).merge(AccountStat.order('last_status_at DESC NULLS LAST')).references(:account_stat) } + scope :by_recent_sign_in, -> { order(Arel.sql('users.current_sign_in_at DESC NULLS LAST')) } scope :popular, -> { order('account_stats.followers_count desc') } scope :by_domain_and_subdomains, ->(domain) { where(domain: domain).or(where(arel_table[:domain].matches('%.' + domain))) } scope :not_excluded_by_account, ->(account) { where.not(id: account.excluded_from_timeline_account_ids) } diff --git a/app/models/account_conversation.rb b/app/models/account_conversation.rb index 45e74bbeb31920..38ee247cfdc206 100644 --- a/app/models/account_conversation.rb +++ b/app/models/account_conversation.rb @@ -16,34 +16,44 @@ class AccountConversation < ApplicationRecord include Redisable + attr_writer :participant_accounts + + before_validation :set_last_status after_commit :push_to_streaming_api belongs_to :account belongs_to :conversation belongs_to :last_status, class_name: 'Status' - before_validation :set_last_status - def participant_account_ids=(arr) self[:participant_account_ids] = arr.sort + @participant_accounts = nil end def participant_accounts - if participant_account_ids.empty? - [account] - else - participants = Account.where(id: participant_account_ids) - participants.empty? ? [account] : participants - end + @participant_accounts ||= Account.where(id: participant_account_ids).to_a + @participant_accounts.presence || [account] end class << self def to_a_paginated_by_id(limit, options = {}) - if options[:min_id] - paginate_by_min_id(limit, options[:min_id], options[:max_id]).reverse - else - paginate_by_max_id(limit, options[:max_id], options[:since_id]).to_a + array = begin + if options[:min_id] + paginate_by_min_id(limit, options[:min_id], options[:max_id]).reverse + else + paginate_by_max_id(limit, options[:max_id], options[:since_id]).to_a + end end + + # Preload participants + participant_ids = array.flat_map(&:participant_account_ids) + accounts_by_id = Account.where(id: participant_ids).index_by(&:id) + + array.each do |conversation| + conversation.participant_accounts = conversation.participant_account_ids.filter_map { |id| accounts_by_id[id] } + end + + array end def paginate_by_min_id(limit, min_id = nil, max_id = nil) diff --git a/app/models/admin/action_log_filter.rb b/app/models/admin/action_log_filter.rb index 0f2f712a255e39..5723ae554a898e 100644 --- a/app/models/admin/action_log_filter.rb +++ b/app/models/admin/action_log_filter.rb @@ -31,7 +31,7 @@ class Admin::ActionLogFilter destroy_instance: { target_type: 'Instance', action: 'destroy' }.freeze, destroy_unavailable_domain: { target_type: 'UnavailableDomain', action: 'destroy' }.freeze, destroy_status: { target_type: 'Status', action: 'destroy' }.freeze, - disable_2fa_user: { target_type: 'User', action: 'disable' }.freeze, + disable_2fa_user: { target_type: 'User', action: 'disable_2fa' }.freeze, disable_custom_emoji: { target_type: 'CustomEmoji', action: 'disable' }.freeze, disable_user: { target_type: 'User', action: 'disable' }.freeze, enable_custom_emoji: { target_type: 'CustomEmoji', action: 'enable' }.freeze, diff --git a/app/models/admin/status_batch_action.rb b/app/models/admin/status_batch_action.rb index 7bf6fa6dafe79f..f2925db278c5eb 100644 --- a/app/models/admin/status_batch_action.rb +++ b/app/models/admin/status_batch_action.rb @@ -73,7 +73,7 @@ def handle_mark_as_sensitive! # Can't use a transaction here because UpdateStatusService queues # Sidekiq jobs statuses.includes(:media_attachments, :preview_cards).find_each do |status| - next unless status.with_media? || status.with_preview_card? + next if status.discarded? || !(status.with_media? || status.with_preview_card?) authorize(status, :update?) @@ -89,15 +89,15 @@ def handle_mark_as_sensitive! report.resolve!(current_account) log_action(:resolve, report) end - - @warning = target_account.strikes.create!( - action: :mark_statuses_as_sensitive, - account: current_account, - report: report, - status_ids: status_ids - ) end + @warning = target_account.strikes.create!( + action: :mark_statuses_as_sensitive, + account: current_account, + report: report, + status_ids: status_ids + ) + UserMailer.warning(target_account.user, @warning).deliver_later! if warnable? end @@ -137,6 +137,6 @@ def report_params end def allowed_status_ids - AccountStatusesFilter.new(@report.target_account, current_account).results.with_discarded.where(id: status_ids).pluck(:id) + Admin::AccountStatusesFilter.new(@report.target_account, current_account).results.with_discarded.where(id: status_ids).pluck(:id) end end diff --git a/app/models/backup.rb b/app/models/backup.rb index d242fd62c19d1d..8823e7cae56488 100644 --- a/app/models/backup.rb +++ b/app/models/backup.rb @@ -17,6 +17,6 @@ class Backup < ApplicationRecord belongs_to :user, inverse_of: :backups - has_attached_file :dump + has_attached_file :dump, s3_permissions: ->(*) { ENV['S3_PERMISSION'] == '' ? nil : 'private' } do_not_validate_attachment_file_type :dump end diff --git a/app/models/concerns/attachmentable.rb b/app/models/concerns/attachmentable.rb index 01fae4236fea12..a61c78dda170ee 100644 --- a/app/models/concerns/attachmentable.rb +++ b/app/models/concerns/attachmentable.rb @@ -22,15 +22,14 @@ module Attachmentable included do def self.has_attached_file(name, options = {}) # rubocop:disable Naming/PredicateName - options = { validate_media_type: false }.merge(options) super(name, options) - send(:"before_#{name}_post_process") do + + send(:"before_#{name}_validate", prepend: true) do attachment = send(name) check_image_dimension(attachment) set_file_content_type(attachment) obfuscate_file_name(attachment) set_file_extension(attachment) - Paperclip::Validators::MediaTypeSpoofDetectionValidator.new(attributes: [name]).validate(self) end end end @@ -53,9 +52,13 @@ def check_image_dimension(attachment) return if attachment.blank? || !/image.*/.match?(attachment.content_type) || attachment.queued_for_write[:original].blank? width, height = FastImage.size(attachment.queued_for_write[:original].path) - matrix_limit = attachment.content_type == 'image/gif' ? GIF_MATRIX_LIMIT : MAX_MATRIX_LIMIT + return unless width.present? && height.present? - raise Mastodon::DimensionsValidationError, "#{width}x#{height} images are not supported" if width.present? && height.present? && (width * height > matrix_limit) + if attachment.content_type == 'image/gif' && width * height > GIF_MATRIX_LIMIT + raise Mastodon::DimensionsValidationError, "#{width}x#{height} GIF files are not supported" + elsif width * height > MAX_MATRIX_LIMIT + raise Mastodon::DimensionsValidationError, "#{width}x#{height} images are not supported" + end end def appropriate_extension(attachment) diff --git a/app/models/concerns/domain_materializable.rb b/app/models/concerns/domain_materializable.rb index 88337f8c00094c..0eac6878ed4cec 100644 --- a/app/models/concerns/domain_materializable.rb +++ b/app/models/concerns/domain_materializable.rb @@ -3,11 +3,24 @@ module DomainMaterializable extend ActiveSupport::Concern + include Redisable + included do after_create_commit :refresh_instances_view end def refresh_instances_view - Instance.refresh unless domain.nil? || Instance.where(domain: domain).exists? + return if domain.nil? || Instance.exists?(domain: domain) + + Instance.refresh + count_unique_subdomains! + end + + def count_unique_subdomains! + second_and_top_level_domain = PublicSuffix.domain(domain, ignore_private: true) + with_redis do |redis| + redis.pfadd("unique_subdomains_for:#{second_and_top_level_domain}", domain) + redis.expire("unique_subdomains_for:#{second_and_top_level_domain}", 1.minute.seconds) + end end end diff --git a/app/models/concerns/ldap_authenticable.rb b/app/models/concerns/ldap_authenticable.rb index dc5abcd5accefe..775df081764d96 100644 --- a/app/models/concerns/ldap_authenticable.rb +++ b/app/models/concerns/ldap_authenticable.rb @@ -6,7 +6,7 @@ module LdapAuthenticable class_methods do def authenticate_with_ldap(params = {}) ldap = Net::LDAP.new(ldap_options) - filter = format(Devise.ldap_search_filter, uid: Devise.ldap_uid, mail: Devise.ldap_mail, email: params[:email]) + filter = format(Devise.ldap_search_filter, uid: Devise.ldap_uid, mail: Devise.ldap_mail, email: Net::LDAP::Filter.escape(params[:email])) if (user_info = ldap.bind_as(base: Devise.ldap_base, filter: filter, password: params[:password])) ldap_get_user(user_info.first) diff --git a/app/models/concerns/omniauthable.rb b/app/models/concerns/omniauthable.rb index a90d5d888a5aa6..37d8efd4843ad2 100644 --- a/app/models/concerns/omniauthable.rb +++ b/app/models/concerns/omniauthable.rb @@ -19,17 +19,18 @@ def email_present? end class_methods do - def find_for_oauth(auth, signed_in_resource = nil) + def find_for_omniauth(auth, signed_in_resource = nil) # EOLE-SSO Patch auth.uid = (auth.uid[0][:uid] || auth.uid[0][:user]) if auth.uid.is_a? Hashie::Array - identity = Identity.find_for_oauth(auth) + identity = Identity.find_for_omniauth(auth) # If a signed_in_resource is provided it always overrides the existing user # to prevent the identity being locked with accidentally created accounts. # Note that this may leave zombie accounts (with no associated identity) which # can be cleaned up at a later date. user = signed_in_resource || identity.user - user ||= create_for_oauth(auth) + user ||= reattach_for_auth(auth) + user ||= create_for_auth(auth) if identity.user.nil? identity.user = user @@ -39,19 +40,35 @@ def find_for_oauth(auth, signed_in_resource = nil) user end - def create_for_oauth(auth) - # Check if the user exists with provided email. If no email was provided, - # we assign a temporary email and ask the user to verify it on - # the next step via Auth::SetupController.show + private - strategy = Devise.omniauth_configs[auth.provider.to_sym].strategy - assume_verified = strategy&.security&.assume_email_is_verified - email_is_verified = auth.info.verified || auth.info.verified_email || auth.info.email_verified || assume_verified - email = auth.info.verified_email || auth.info.email + def reattach_for_auth(auth) + # If allowed, check if a user exists with the provided email address, + # and return it if they does not have an associated identity with the + # current authentication provider. + + # This can be used to provide a choice of alternative auth providers + # or provide smooth gradual transition between multiple auth providers, + # but this is discouraged because any insecure provider will put *all* + # local users at risk, regardless of which provider they registered with. + + return unless ENV['ALLOW_UNSAFE_AUTH_PROVIDER_REATTACH'] == 'true' - user = User.find_by(email: email) if email_is_verified + email, email_is_verified = email_from_auth(auth) + return unless email_is_verified - return user unless user.nil? + user = User.find_by(email: email) + return if user.nil? || Identity.exists?(provider: auth.provider, user_id: user.id) + + user + end + + def create_for_auth(auth) + # Create a user for the given auth params. If no email was provided, + # we assign a temporary email and ask the user to verify it on + # the next step via Auth::SetupController.show + + email, email_is_verified = email_from_auth(auth) user = User.new(user_params_from_auth(email, auth)) @@ -61,7 +78,14 @@ def create_for_oauth(auth) user end - private + def email_from_auth(auth) + strategy = Devise.omniauth_configs[auth.provider.to_sym].strategy + assume_verified = strategy&.security&.assume_email_is_verified + email_is_verified = auth.info.verified || auth.info.verified_email || auth.info.email_verified || assume_verified + email = auth.info.verified_email || auth.info.email + + [email, email_is_verified] + end def user_params_from_auth(email, auth) { diff --git a/app/models/form/account_batch.rb b/app/models/form/account_batch.rb index dcf15584038964..306d979c683241 100644 --- a/app/models/form/account_batch.rb +++ b/app/models/form/account_batch.rb @@ -16,8 +16,8 @@ def save unfollow! when 'remove_from_followers' remove_from_followers! - when 'block_domains' - block_domains! + when 'remove_domains_from_followers' + remove_domains_from_followers! when 'approve' approve! when 'reject' @@ -34,9 +34,15 @@ def save private def follow! + error = nil + accounts.each do |target_account| FollowService.new.call(current_account, target_account) + rescue Mastodon::NotPermittedError, ActiveRecord::RecordNotFound => e + error ||= e end + + raise error if error.present? end def unfollow! @@ -49,10 +55,8 @@ def remove_from_followers! RemoveFromFollowersService.new.call(current_account, account_ids) end - def block_domains! - AfterAccountDomainBlockWorker.push_bulk(account_domains) do |domain| - [current_account.id, domain] - end + def remove_domains_from_followers! + RemoveDomainsFromFollowersService.new.call(current_account, account_domains) end def account_domains diff --git a/app/models/identity.rb b/app/models/identity.rb index 8cc65aef413d13..a396d76234369a 100644 --- a/app/models/identity.rb +++ b/app/models/identity.rb @@ -12,11 +12,11 @@ # class Identity < ApplicationRecord - belongs_to :user, dependent: :destroy + belongs_to :user validates :uid, presence: true, uniqueness: { scope: :provider } validates :provider, presence: true - def self.find_for_oauth(auth) + def self.find_for_omniauth(auth) find_or_create_by(uid: auth.uid, provider: auth.provider) end end diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index 21c663e47a86ae..bc7849abc7126a 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -156,7 +156,7 @@ class MediaAttachment < ApplicationRecord }.freeze GLOBAL_CONVERT_OPTIONS = { - all: '-quality 90 -strip +set modify-date +set create-date', + all: '-quality 90 -strip +set date:modify +set date:create +set date:timestamp', }.freeze belongs_to :account, inverse_of: :media_attachments, optional: true diff --git a/app/models/poll.rb b/app/models/poll.rb index 1a326e452c6fae..af3b09315c87ab 100644 --- a/app/models/poll.rb +++ b/app/models/poll.rb @@ -85,6 +85,7 @@ def initialize(poll, id, title, votes_count) def reset_votes! self.cached_tallies = options.map { 0 } self.votes_count = 0 + self.voters_count = 0 votes.delete_all unless new_record? end diff --git a/app/models/relationship_filter.rb b/app/models/relationship_filter.rb index 9135ff144c9a66..33f5a4a4c06a21 100644 --- a/app/models/relationship_filter.rb +++ b/app/models/relationship_filter.rb @@ -60,13 +60,13 @@ def scope_for(key, value) def relationship_scope(value) case value when 'following' - account.following.eager_load(:account_stat).reorder(nil) + account.following.includes(:account_stat).reorder(nil) when 'followed_by' - account.followers.eager_load(:account_stat).reorder(nil) + account.followers.includes(:account_stat).reorder(nil) when 'mutual' - account.followers.eager_load(:account_stat).reorder(nil).merge(Account.where(id: account.following)) + account.followers.includes(:account_stat).reorder(nil).merge(Account.where(id: account.following)) when 'invited' - Account.joins(user: :invite).merge(Invite.where(user: account.user)).eager_load(:account_stat).reorder(nil) + Account.joins(user: :invite).merge(Invite.where(user: account.user)).includes(:account_stat).reorder(nil) else raise "Unknown relationship: #{value}" end @@ -112,7 +112,7 @@ def order_scope(value) def activity_scope(value) case value when 'dormant' - AccountStat.where(last_status_at: nil).or(AccountStat.where(AccountStat.arel_table[:last_status_at].lt(1.month.ago))) + Account.joins(:account_stat).where(account_stat: { last_status_at: [nil, ...1.month.ago] }) else raise "Unknown activity: #{value}" end diff --git a/app/models/report.rb b/app/models/report.rb index 6d41665401624a..dbf01abdc95cf6 100644 --- a/app/models/report.rb +++ b/app/models/report.rb @@ -38,7 +38,10 @@ class Report < ApplicationRecord scope :resolved, -> { where.not(action_taken_at: nil) } scope :with_accounts, -> { includes([:account, :target_account, :action_taken_by_account, :assigned_account].index_with({ user: [:invite_request, :invite] })) } - validates :comment, length: { maximum: 1_000 } + # A report is considered local if the reporter is local + delegate :local?, to: :account + + validates :comment, length: { maximum: 1_000 }, if: :local? validates :rule_ids, absence: true, unless: :violation? validate :validate_rule_ids @@ -49,10 +52,6 @@ class Report < ApplicationRecord violation: 2_000, } - def local? - false # Force uri_for to use uri attribute - end - before_validation :set_uri, only: :create def object_type diff --git a/app/models/status.rb b/app/models/status.rb index 7e7309befe5ac5..0f20f4b84ac1ce 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -285,6 +285,15 @@ def ordered_media_attachments end end + def ordered_media_attachments + if ordered_media_attachment_ids.nil? + media_attachments + else + map = media_attachments.index_by(&:id) + ordered_media_attachment_ids.filter_map { |media_attachment_id| map[media_attachment_id] } + end + end + def replies_count status_stat&.replies_count || 0 end @@ -374,13 +383,25 @@ def reload_stale_associations!(cached_items) account_ids.uniq! + status_ids = cached_items.map { |item| item.reblog? ? item.reblog_of_id : item.id }.uniq + return if account_ids.empty? accounts = Account.where(id: account_ids).includes(:account_stat, :user).index_by(&:id) + status_stats = StatusStat.where(status_id: status_ids).index_by(&:status_id) + cached_items.each do |item| item.account = accounts[item.account_id] item.reblog.account = accounts[item.reblog.account_id] if item.reblog? + + if item.reblog? + status_stat = status_stats[item.reblog.id] + item.reblog.status_stat = status_stat if status_stat.present? + else + status_stat = status_stats[item.id] + item.status_stat = status_stat if status_stat.present? + end end end diff --git a/app/models/trends/statuses.rb b/app/models/trends/statuses.rb index 777065d3e35d93..cae3a86f719ca4 100644 --- a/app/models/trends/statuses.rb +++ b/app/models/trends/statuses.rb @@ -75,7 +75,7 @@ def klass private def eligible?(status) - status.public_visibility? && status.account.discoverable? && !status.account.silenced? && status.spoiler_text.blank? && !status.sensitive? && !status.reply? + status.public_visibility? && status.account.discoverable? && !status.account.silenced? && !status.account.sensitized? && status.spoiler_text.blank? && !status.sensitive? && !status.reply? end def calculate_scores(statuses, at_time) diff --git a/app/models/user.rb b/app/models/user.rb index 4b92d57e94fe54..337daa6167b784 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -340,6 +340,25 @@ def reset_password(new_password, new_password_confirmation) super end + def revoke_access! + Doorkeeper::AccessGrant.by_resource_owner(self).update_all(revoked_at: Time.now.utc) + + Doorkeeper::AccessToken.by_resource_owner(self).in_batches do |batch| + batch.update_all(revoked_at: Time.now.utc) + Web::PushSubscription.where(access_token_id: batch).delete_all + + # Revoke each access token for the Streaming API, since `update_all`` + # doesn't trigger ActiveRecord Callbacks: + # TODO: #28793 Combine into a single topic + payload = Oj.dump(event: :kill) + redis.pipelined do |pipeline| + batch.ids.each do |id| + pipeline.publish("timeline:access_token:#{id}", payload) + end + end + end + end + def reset_password! # First, change password to something random and deactivate all sessions transaction do @@ -348,12 +367,7 @@ def reset_password! end # Then, remove all authorized applications and connected push subscriptions - Doorkeeper::AccessGrant.by_resource_owner(self).in_batches.update_all(revoked_at: Time.now.utc) - - Doorkeeper::AccessToken.by_resource_owner(self).in_batches do |batch| - batch.update_all(revoked_at: Time.now.utc) - Web::PushSubscription.where(access_token_id: batch).delete_all - end + revoke_access! # Finally, send a reset password prompt to the user send_reset_password_instructions @@ -442,10 +456,13 @@ def sanitize_languages def prepare_new_user! BootstrapTimelineWorker.perform_async(account_id) ActivityTracker.increment('activity:accounts:local') + ActivityTracker.record('activity:logins', id) UserMailer.welcome(self).deliver_later end def prepare_returning_user! + return unless confirmed? + ActivityTracker.record('activity:logins', id) regenerate_feed! if needs_feed_update? end diff --git a/app/policies/backup_policy.rb b/app/policies/backup_policy.rb index 0ef89a8d0c8fb9..86b8efbe96fa7d 100644 --- a/app/policies/backup_policy.rb +++ b/app/policies/backup_policy.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class BackupPolicy < ApplicationPolicy - MIN_AGE = 1.week + MIN_AGE = 6.days def create? user_signed_in? && current_user.backups.where('created_at >= ?', MIN_AGE.ago).count.zero? diff --git a/app/serializers/rest/account_serializer.rb b/app/serializers/rest/account_serializer.rb index c52a89d8723ad1..8a088f4463514d 100644 --- a/app/serializers/rest/account_serializer.rb +++ b/app/serializers/rest/account_serializer.rb @@ -15,6 +15,16 @@ class REST::AccountSerializer < ActiveModel::Serializer attribute :suspended, if: :suspended? attribute :silenced, key: :limited, if: :silenced? + class AccountDecorator < SimpleDelegator + def self.model_name + Account.model_name + end + + def moved? + false + end + end + class FieldSerializer < ActiveModel::Serializer include FormattingHelper @@ -84,7 +94,7 @@ def discoverable end def moved_to_account - object.suspended? ? nil : object.moved_to_account + object.suspended? ? nil : AccountDecorator.new(object.moved_to_account) end def emojis @@ -106,6 +116,6 @@ def silenced delegate :suspended?, :silenced?, to: :object def moved_and_not_nested? - object.moved? && object.moved_to_account.moved_to_account_id.nil? + object.moved? end end diff --git a/app/serializers/rest/preview_card_serializer.rb b/app/serializers/rest/preview_card_serializer.rb index 66ff47d22ea50e..e6d204fec3c076 100644 --- a/app/serializers/rest/preview_card_serializer.rb +++ b/app/serializers/rest/preview_card_serializer.rb @@ -11,4 +11,8 @@ class REST::PreviewCardSerializer < ActiveModel::Serializer def image object.image? ? full_asset_url(object.image.url(:original)) : nil end + + def html + Sanitize.fragment(object.html, Sanitize::Config::MASTODON_OEMBED) + end end diff --git a/app/services/activitypub/fetch_featured_collection_service.rb b/app/services/activitypub/fetch_featured_collection_service.rb index 37d05e05564194..b2b132fd813ca5 100644 --- a/app/services/activitypub/fetch_featured_collection_service.rb +++ b/app/services/activitypub/fetch_featured_collection_service.rb @@ -3,10 +3,11 @@ class ActivityPub::FetchFeaturedCollectionService < BaseService include JsonLdHelper - def call(account) + def call(account, **options) return if account.featured_collection_url.blank? || account.suspended? || account.local? @account = account + @options = options @json = fetch_resource(@account.featured_collection_url, true, local_follower) return unless supported_context?(@json) @@ -38,9 +39,9 @@ def fetch_collection(collection_or_uri) def process_items(items) status_ids = items.filter_map do |item| uri = value_or_id(item) - next if ActivityPub::TagManager.instance.local_uri?(uri) + next if ActivityPub::TagManager.instance.local_uri?(uri) || invalid_origin?(uri) - status = ActivityPub::FetchRemoteStatusService.new.call(uri, on_behalf_of: local_follower) + status = ActivityPub::FetchRemoteStatusService.new.call(uri, on_behalf_of: local_follower, expected_actor_uri: @account.uri, request_id: @options[:request_id]) next unless status&.account_id == @account.id status.id diff --git a/app/services/activitypub/fetch_remote_account_service.rb b/app/services/activitypub/fetch_remote_account_service.rb index 9d01f538687ed2..582506a7e31e93 100644 --- a/app/services/activitypub/fetch_remote_account_service.rb +++ b/app/services/activitypub/fetch_remote_account_service.rb @@ -8,15 +8,15 @@ class ActivityPub::FetchRemoteAccountService < BaseService SUPPORTED_TYPES = %w(Application Group Organization Person Service).freeze # Does a WebFinger roundtrip on each call, unless `only_key` is true - def call(uri, id: true, prefetched_body: nil, break_on_redirect: false, only_key: false) + def call(uri, prefetched_body: nil, break_on_redirect: false, only_key: false, request_id: nil) return if domain_not_allowed?(uri) return ActivityPub::TagManager.instance.uri_to_resource(uri, Account) if ActivityPub::TagManager.instance.local_uri?(uri) @json = begin if prefetched_body.nil? - fetch_resource(uri, id) + fetch_resource(uri, true) else - body_to_json(prefetched_body, compare_id: id ? uri : nil) + body_to_json(prefetched_body, compare_id: uri) end end @@ -28,7 +28,7 @@ def call(uri, id: true, prefetched_body: nil, break_on_redirect: false, only_key return unless only_key || verified_webfinger? - ActivityPub::ProcessAccountService.new.call(@username, @domain, @json, only_key: only_key, verified_webfinger: !only_key) + ActivityPub::ProcessAccountService.new.call(@username, @domain, @json, only_key: only_key, verified_webfinger: !only_key, request_id: request_id) rescue Oj::ParseError nil end diff --git a/app/services/activitypub/fetch_remote_key_service.rb b/app/services/activitypub/fetch_remote_key_service.rb index c48288b3ba715c..7142cae1e371a5 100644 --- a/app/services/activitypub/fetch_remote_key_service.rb +++ b/app/services/activitypub/fetch_remote_key_service.rb @@ -4,23 +4,10 @@ class ActivityPub::FetchRemoteKeyService < BaseService include JsonLdHelper # Returns account that owns the key - def call(uri, id: true, prefetched_body: nil) + def call(uri) return if uri.blank? - if prefetched_body.nil? - if id - @json = fetch_resource_without_id_validation(uri) - if person? - @json = fetch_resource(@json['id'], true) - elsif uri != @json['id'] - return - end - else - @json = fetch_resource(uri, id) - end - else - @json = body_to_json(prefetched_body, compare_id: id ? uri : nil) - end + @json = fetch_resource(uri, false) return unless supported_context?(@json) && expected_type? return find_account(@json['id'], @json) if person? diff --git a/app/services/activitypub/fetch_remote_status_service.rb b/app/services/activitypub/fetch_remote_status_service.rb index 80309824509a47..fa0884fdbae68d 100644 --- a/app/services/activitypub/fetch_remote_status_service.rb +++ b/app/services/activitypub/fetch_remote_status_service.rb @@ -2,14 +2,18 @@ class ActivityPub::FetchRemoteStatusService < BaseService include JsonLdHelper + include Redisable + + DISCOVERIES_PER_REQUEST = 1000 # Should be called when uri has already been checked for locality - def call(uri, id: true, prefetched_body: nil, on_behalf_of: nil) + def call(uri, prefetched_body: nil, on_behalf_of: nil, expected_actor_uri: nil, request_id: nil) + @request_id = request_id || "#{Time.now.utc.to_i}-status-#{uri}" @json = begin if prefetched_body.nil? - fetch_resource(uri, id, on_behalf_of) + fetch_resource(uri, true, on_behalf_of) else - body_to_json(prefetched_body, compare_id: id ? uri : nil) + body_to_json(prefetched_body, compare_id: uri) end end @@ -30,6 +34,7 @@ def call(uri, id: true, prefetched_body: nil, on_behalf_of: nil) end return if activity_json.nil? || object_uri.nil? || !trustworthy_attribution?(@json['id'], actor_uri) + return if expected_actor_uri.present? && actor_uri != expected_actor_uri return ActivityPub::TagManager.instance.uri_to_resource(object_uri, Status) if ActivityPub::TagManager.instance.local_uri?(object_uri) actor = account_from_uri(actor_uri) @@ -40,7 +45,13 @@ def call(uri, id: true, prefetched_body: nil, on_behalf_of: nil) # activity as an update rather than create activity_json['type'] = 'Update' if equals_or_includes_any?(activity_json['type'], %w(Create)) && Status.where(uri: object_uri, account_id: actor.id).exists? - ActivityPub::Activity.factory(activity_json, actor).perform + with_redis do |redis| + discoveries = redis.incr("status_discovery_per_request:#{@request_id}") + redis.expire("status_discovery_per_request:#{@request_id}", 5.minutes.seconds) + return nil if discoveries > DISCOVERIES_PER_REQUEST + end + + ActivityPub::Activity.factory(activity_json, actor, request_id: @request_id).perform end private @@ -52,7 +63,7 @@ def trustworthy_attribution?(uri, attributed_to) def account_from_uri(uri) actor = ActivityPub::TagManager.instance.uri_to_resource(uri, Account) - actor = ActivityPub::FetchRemoteAccountService.new.call(uri, id: true) if actor.nil? || actor.possibly_stale? + actor = ActivityPub::FetchRemoteAccountService.new.call(uri, request_id: @request_id) if actor.nil? || actor.possibly_stale? actor end diff --git a/app/services/activitypub/fetch_replies_service.rb b/app/services/activitypub/fetch_replies_service.rb index 8cb309e52a8633..18a27e851d7bb2 100644 --- a/app/services/activitypub/fetch_replies_service.rb +++ b/app/services/activitypub/fetch_replies_service.rb @@ -3,14 +3,14 @@ class ActivityPub::FetchRepliesService < BaseService include JsonLdHelper - def call(parent_status, collection_or_uri, allow_synchronous_requests = true) + def call(parent_status, collection_or_uri, allow_synchronous_requests: true, request_id: nil) @account = parent_status.account @allow_synchronous_requests = allow_synchronous_requests @items = collection_items(collection_or_uri) return if @items.nil? - FetchReplyWorker.push_bulk(filtered_replies) + FetchReplyWorker.push_bulk(filtered_replies) { |reply_uri| [reply_uri, { 'request_id' => request_id}] } @items end diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index 4449a54270143a..da9d77d1336a54 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -6,6 +6,9 @@ class ActivityPub::ProcessAccountService < BaseService include Redisable include Lockable + SUBDOMAINS_RATELIMIT = 10 + DISCOVERIES_PER_REQUEST = 400 + # Should be called with confirmed valid JSON # and WebFinger-resolved username and domain def call(username, domain, json, options = {}) @@ -15,9 +18,12 @@ def call(username, domain, json, options = {}) @json = json @uri = @json['id'] @username = username - @domain = domain + @domain = TagManager.instance.normalize_domain(domain) @collections = {} + # The key does not need to be unguessable, it just needs to be somewhat unique + @options[:request_id] ||= "#{Time.now.utc.to_i}-#{username}@#{domain}" + with_lock("process_account:#{@uri}") do @account = Account.remote.find_by(uri: @uri) if @options[:only_key] @account ||= Account.find_remote(@username, @domain) @@ -25,7 +31,18 @@ def call(username, domain, json, options = {}) @old_protocol = @account&.protocol @suspension_changed = false - create_account if @account.nil? + if @account.nil? + with_redis do |redis| + return nil if redis.pfcount("unique_subdomains_for:#{PublicSuffix.domain(@domain, ignore_private: true)}") >= SUBDOMAINS_RATELIMIT + + discoveries = redis.incr("discovery_per_request:#{@options[:request_id]}") + redis.expire("discovery_per_request:#{@options[:request_id]}", 5.minutes.seconds) + return nil if discoveries > DISCOVERIES_PER_REQUEST + end + + create_account + end + update_account process_tags @@ -60,6 +77,9 @@ def create_account @account.suspended_at = domain_block.created_at if auto_suspend? @account.suspension_origin = :local if auto_suspend? @account.silenced_at = domain_block.created_at if auto_silence? + + set_immediate_protocol_attributes! + @account.save end @@ -149,7 +169,7 @@ def after_suspension_change! end def check_featured_collection! - ActivityPub::SynchronizeFeaturedCollectionWorker.perform_async(@account.id) + ActivityPub::SynchronizeFeaturedCollectionWorker.perform_async(@account.id, { 'request_id' => @options[:request_id] }) end def check_links! @@ -249,7 +269,7 @@ def collection_info(type) def moved_account account = ActivityPub::TagManager.instance.uri_to_resource(@json['movedTo'], Account) - account ||= ActivityPub::FetchRemoteAccountService.new.call(@json['movedTo'], id: true, break_on_redirect: true) + account ||= ActivityPub::FetchRemoteAccountService.new.call(@json['movedTo'], break_on_redirect: true, request_id: @options[:request_id]) account end diff --git a/app/services/activitypub/process_status_update_service.rb b/app/services/activitypub/process_status_update_service.rb index addd5fc274cbf2..fad19f87fd339c 100644 --- a/app/services/activitypub/process_status_update_service.rb +++ b/app/services/activitypub/process_status_update_service.rb @@ -5,7 +5,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService include Redisable include Lockable - def call(status, json) + def call(status, json, request_id: nil) raise ArgumentError, 'Status has unsaved changes' if status.changed? @json = json @@ -15,6 +15,7 @@ def call(status, json) @account = status.account @media_attachments_changed = false @poll_changed = false + @request_id = request_id # Only native types can be updated at the moment return @status if !expected_type? || already_updated_more_recently? @@ -92,7 +93,13 @@ def update_media_attachments! next if unsupported_media_type?(media_attachment_parser.file_content_type) || skip_download? - RedownloadMediaWorker.perform_async(media_attachment.id) if media_attachment.remote_url_previously_changed? || media_attachment.thumbnail_remote_url_previously_changed? + begin + media_attachment.download_file! if media_attachment.remote_url_previously_changed? + media_attachment.download_thumbnail! if media_attachment.thumbnail_remote_url_previously_changed? + media_attachment.save + rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError + RedownloadMediaWorker.perform_in(rand(30..600).seconds, media_attachment.id) + end rescue Addressable::URI::InvalidURIError => e Rails.logger.debug "Invalid URL in attachment: #{e}" end @@ -185,7 +192,7 @@ def update_mentions! next if href.blank? account = ActivityPub::TagManager.instance.uri_to_resource(href, Account) - account ||= ActivityPub::FetchRemoteAccountService.new.call(href) + account ||= ActivityPub::FetchRemoteAccountService.new.call(href, request_id: @request_id) next if account.nil? diff --git a/app/services/fetch_remote_status_service.rb b/app/services/fetch_remote_status_service.rb index eafde4d4a5d6de..08c2d24ba17fdf 100644 --- a/app/services/fetch_remote_status_service.rb +++ b/app/services/fetch_remote_status_service.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class FetchRemoteStatusService < BaseService - def call(url, prefetched_body = nil) + def call(url, prefetched_body: nil, request_id: nil) if prefetched_body.nil? resource_url, resource_options = FetchResourceService.new.call(url) else @@ -9,6 +9,6 @@ def call(url, prefetched_body = nil) resource_options = { prefetched_body: prefetched_body } end - ActivityPub::FetchRemoteStatusService.new.call(resource_url, **resource_options) unless resource_url.nil? + ActivityPub::FetchRemoteStatusService.new.call(resource_url, **resource_options.merge(request_id: request_id)) unless resource_url.nil? end end diff --git a/app/services/fetch_resource_service.rb b/app/services/fetch_resource_service.rb index 6c0093cd45e988..d6fd63eca0d489 100644 --- a/app/services/fetch_resource_service.rb +++ b/app/services/fetch_resource_service.rb @@ -43,11 +43,19 @@ def process_response(response, terminal = false) @response_code = response.code return nil if response.code != 200 - if ['application/activity+json', 'application/ld+json'].include?(response.mime_type) + if valid_activitypub_content_type?(response) body = response.body_with_limit json = body_to_json(body) - [json['id'], { prefetched_body: body, id: true }] if supported_context?(json) && (equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) || expected_type?(json)) + return unless supported_context?(json) && (equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) || expected_type?(json)) + + if json['id'] != @url + return if terminal + + return process(json['id'], terminal: true) + end + + [@url, { prefetched_body: body }] elsif !terminal link_header = response['Link'] && parse_link_header(response) diff --git a/app/services/follow_migration_service.rb b/app/services/follow_migration_service.rb new file mode 100644 index 00000000000000..f642b6b2d8c8be --- /dev/null +++ b/app/services/follow_migration_service.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +class FollowMigrationService < FollowService + # Follow an account with the same settings as another account, and unfollow the old account once the request is sent + # @param [Account] source_account From which to follow + # @param [Account] target_account Account to follow + # @param [Account] old_target_account Account to unfollow once the follow request has been sent to the new one + # @option [Boolean] bypass_locked Whether to immediately follow the new account even if it is locked + def call(source_account, target_account, old_target_account, bypass_locked: false) + @old_target_account = old_target_account + + follow = source_account.active_relationships.find_by(target_account: old_target_account) + reblogs = follow&.show_reblogs? + notify = follow&.notify? + + super(source_account, target_account, reblogs: reblogs, notify: notify, bypass_locked: bypass_locked, bypass_limit: true) + end + + private + + def request_follow! + follow_request = @source_account.request_follow!(@target_account, **follow_options.merge(rate_limit: @options[:with_rate_limit], bypass_limit: @options[:bypass_limit])) + + if @target_account.local? + LocalNotificationWorker.perform_async(@target_account.id, follow_request.id, follow_request.class.name, 'follow_request') + UnfollowService.new.call(@source_account, @old_target_account, skip_unmerge: true) + elsif @target_account.activitypub? + ActivityPub::MigratedFollowDeliveryWorker.perform_async(build_json(follow_request), @source_account.id, @target_account.inbox_url, @old_target_account.id) + end + + follow_request + end + + def direct_follow! + follow = super + UnfollowService.new.call(@source_account, @old_target_account, skip_unmerge: true) + follow + end + + def follow_options + @options.slice(:reblogs, :notify) + end +end diff --git a/app/services/follow_service.rb b/app/services/follow_service.rb index ed28e13718e11b..9f958c81f4bd2a 100644 --- a/app/services/follow_service.rb +++ b/app/services/follow_service.rb @@ -70,7 +70,7 @@ def request_follow! if @target_account.local? LocalNotificationWorker.perform_async(@target_account.id, follow_request.id, follow_request.class.name, 'follow_request') elsif @target_account.activitypub? - ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), @source_account.id, @target_account.inbox_url) + ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), @source_account.id, @target_account.inbox_url, { 'bypass_availability' => true }) end follow_request diff --git a/app/services/remove_domains_from_followers_service.rb b/app/services/remove_domains_from_followers_service.rb new file mode 100644 index 00000000000000..d76763409d3b0d --- /dev/null +++ b/app/services/remove_domains_from_followers_service.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class RemoveDomainsFromFollowersService < BaseService + include Payloadable + + def call(source_account, target_domains) + source_account.passive_relationships.where(account_id: Account.where(domain: target_domains)).find_each do |follow| + follow.destroy + + create_notification(follow) if source_account.local? && !follow.account.local? && follow.account.activitypub? + end + end + + private + + def create_notification(follow) + ActivityPub::DeliveryWorker.perform_async(build_json(follow), follow.target_account_id, follow.account.inbox_url) + end + + def build_json(follow) + Oj.dump(serialize_payload(follow, ActivityPub::RejectFollowSerializer)) + end +end diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb index 8dc521eedfb0b5..f039ab47603567 100644 --- a/app/services/remove_status_service.rb +++ b/app/services/remove_status_service.rb @@ -12,6 +12,7 @@ class RemoveStatusService < BaseService # @option [Boolean] :immediate # @option [Boolean] :preserve # @option [Boolean] :original_removed + # @option [Boolean] :skip_streaming def call(status, **options) @payload = Oj.dump(event: :delete, payload: status.id.to_s) @status = status @@ -50,6 +51,9 @@ def call(status, **options) private + # The following FeedManager calls all do not result in redis publishes for + # streaming, as the `:update` option is false + def remove_from_self FeedManager.instance.unpush_from_home(@account, @status) end @@ -73,6 +77,8 @@ def remove_from_mentions # followers. Here we send a delete to actively mentioned accounts # that may not follow the account + return if skip_streaming? + @status.active_mentions.find_each do |mention| redis.publish("timeline:#{mention.account_id}", @payload) end @@ -101,7 +107,7 @@ def remove_reblogs # without us being able to do all the fancy stuff @status.reblogs.includes(:account).reorder(nil).find_each do |reblog| - RemoveStatusService.new.call(reblog, original_removed: true) + RemoveStatusService.new.call(reblog, original_removed: true, skip_streaming: skip_streaming?) end end @@ -112,6 +118,8 @@ def remove_from_hashtags return unless @status.public_visibility? + return if skip_streaming? + @status.tags.map(&:name).each do |hashtag| redis.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}", @payload) redis.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}:local", @payload) if @status.local? @@ -121,6 +129,8 @@ def remove_from_hashtags def remove_from_public return unless @status.public_visibility? + return if skip_streaming? + redis.publish('timeline:public', @payload) redis.publish(@status.local? ? 'timeline:public:local' : 'timeline:public:remote', @payload) end @@ -128,6 +138,8 @@ def remove_from_public def remove_from_media return unless @status.public_visibility? + return if skip_streaming? + redis.publish('timeline:public:media', @payload) redis.publish(@status.local? ? 'timeline:public:local:media' : 'timeline:public:remote:media', @payload) end @@ -141,4 +153,8 @@ def remove_media def permanently? @options[:immediate] || !(@options[:preserve] || @status.reported?) end + + def skip_streaming? + !!@options[:skip_streaming] + end end diff --git a/app/services/resolve_url_service.rb b/app/services/resolve_url_service.rb index e2c745673e572d..e40233048e96f7 100644 --- a/app/services/resolve_url_service.rb +++ b/app/services/resolve_url_service.rb @@ -23,7 +23,7 @@ def process_url if equals_or_includes_any?(type, ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) ActivityPub::FetchRemoteAccountService.new.call(resource_url, prefetched_body: body) elsif equals_or_includes_any?(type, ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES) - status = FetchRemoteStatusService.new.call(resource_url, body) + status = FetchRemoteStatusService.new.call(resource_url, prefetched_body: body) authorize_with @on_behalf_of, status, :show? unless status.nil? status end diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb index b8dc8d5e0948b3..211544fea6c891 100644 --- a/app/services/suspend_account_service.rb +++ b/app/services/suspend_account_service.rb @@ -3,10 +3,13 @@ class SuspendAccountService < BaseService include Payloadable + # Carry out the suspension of a recently-suspended account + # @param [Account] account Account to suspend def call(account) + return unless account.suspended? + @account = account - suspend! reject_remote_follows! distribute_update_actor! unmerge_from_home_timelines! @@ -16,10 +19,6 @@ def call(account) private - def suspend! - @account.suspend! unless @account.suspended? - end - def reject_remote_follows! return if @account.local? || !@account.activitypub? @@ -76,10 +75,15 @@ def privatize_media_attachments! styles.each do |style| case Paperclip::Attachment.default_options[:storage] when :s3 + # Prevent useless S3 calls if ACLs are disabled + next if ENV['S3_PERMISSION'] == '' + begin attachment.s3_object(style).acl.put(acl: 'private') rescue Aws::S3::Errors::NoSuchKey Rails.logger.warn "Tried to change acl on non-existent key #{attachment.s3_object(style).key}" + rescue Aws::S3::Errors::NotImplemented => e + Rails.logger.error "Error trying to change ACL on #{attachment.s3_object(style).key}: #{e.message}" end when :fog # Not supported diff --git a/app/services/unsuspend_account_service.rb b/app/services/unsuspend_account_service.rb index 39d8a6ba7f37bd..70667308ecccdf 100644 --- a/app/services/unsuspend_account_service.rb +++ b/app/services/unsuspend_account_service.rb @@ -2,10 +2,12 @@ class UnsuspendAccountService < BaseService include Payloadable + + # Restores a recently-unsuspended account + # @param [Account] account Account to restore def call(account) @account = account - unsuspend! refresh_remote_account! return if @account.nil? || @account.suspended? @@ -18,10 +20,6 @@ def call(account) private - def unsuspend! - @account.unsuspend! if @account.suspended? - end - def refresh_remote_account! return if @account.local? @@ -73,10 +71,15 @@ def publish_media_attachments! styles.each do |style| case Paperclip::Attachment.default_options[:storage] when :s3 + # Prevent useless S3 calls if ACLs are disabled + next if ENV['S3_PERMISSION'] == '' + begin attachment.s3_object(style).acl.put(acl: Paperclip::Attachment.default_options[:s3_permissions]) rescue Aws::S3::Errors::NoSuchKey Rails.logger.warn "Tried to change acl on non-existent key #{attachment.s3_object(style).key}" + rescue Aws::S3::Errors::NotImplemented => e + Rails.logger.error "Error trying to change ACL on #{attachment.s3_object(style).key}: #{e.message}" end when :fog # Not supported diff --git a/app/validators/vote_validator.rb b/app/validators/vote_validator.rb index b1692562d41598..4316c59ef09bb2 100644 --- a/app/validators/vote_validator.rb +++ b/app/validators/vote_validator.rb @@ -3,8 +3,8 @@ class VoteValidator < ActiveModel::Validator def validate(vote) vote.errors.add(:base, I18n.t('polls.errors.expired')) if vote.poll.expired? - vote.errors.add(:base, I18n.t('polls.errors.invalid_choice')) if invalid_choice?(vote) + vote.errors.add(:base, I18n.t('polls.errors.self_vote')) if self_vote?(vote) if vote.poll.multiple? && vote.poll.votes.where(account: vote.account, choice: vote.choice).exists? vote.errors.add(:base, I18n.t('polls.errors.already_voted')) @@ -18,4 +18,8 @@ def validate(vote) def invalid_choice?(vote) vote.choice.negative? || vote.choice >= vote.poll.options.size end + + def self_vote?(vote) + vote.account_id == vote.poll.account_id + end end diff --git a/app/views/admin/dashboard/index.html.haml b/app/views/admin/dashboard/index.html.haml index 8354f0b9f5a18e..425472abdecaf0 100644 --- a/app/views/admin/dashboard/index.html.haml +++ b/app/views/admin/dashboard/index.html.haml @@ -12,7 +12,7 @@ - unless @system_checks.empty? .flash-message-stack - @system_checks.each do |message| - .flash-message.warning + .flash-message{ class: message.critical ? 'alert' : 'warning' } = t("admin.system_checks.#{message.key}.message_html", value: message.value ? content_tag(:strong, message.value) : nil) - if message.action = link_to t("admin.system_checks.#{message.key}.action"), message.action diff --git a/app/views/admin/instances/_instance.html.haml b/app/views/admin/instances/_instance.html.haml index 0b382bc22759dd..93f9bd4181610c 100644 --- a/app/views/admin/instances/_instance.html.haml +++ b/app/views/admin/instances/_instance.html.haml @@ -11,4 +11,5 @@ = t('admin.accounts.whitelisted') - else = t('admin.accounts.no_limits_imposed') + .trends__item__current{ title: t('admin.instances.known_accounts', count: instance.accounts_count) }= friendly_number_to_human instance.accounts_count diff --git a/app/views/admin/reports/_actions.html.haml b/app/views/admin/reports/_actions.html.haml index 404d53a7736a5e..486eb486c724ad 100644 --- a/app/views/admin/reports/_actions.html.haml +++ b/app/views/admin/reports/_actions.html.haml @@ -5,7 +5,7 @@ = link_to t('admin.reports.mark_as_resolved'), resolve_admin_report_path(@report), method: :post, class: 'button' .report-actions__item__description = t('admin.reports.actions.resolve_description_html') - - if @statuses.any? { |status| status.with_media? || status.with_preview_card? } + - if @statuses.any? { |status| (status.with_media? || status.with_preview_card?) && !status.discarded? } .report-actions__item .report-actions__item__button = button_tag t('admin.reports.mark_as_sensitive'), name: :mark_as_sensitive, class: 'button' diff --git a/app/views/admin/trends/links/preview_card_providers/index.html.haml b/app/views/admin/trends/links/preview_card_providers/index.html.haml index c3648c35e97bd3..025270c128fbc9 100644 --- a/app/views/admin/trends/links/preview_card_providers/index.html.haml +++ b/app/views/admin/trends/links/preview_card_providers/index.html.haml @@ -29,7 +29,7 @@ - Trends::PreviewCardProviderFilter::KEYS.each do |key| = hidden_field_tag key, params[key] if params[key].present? - .batch-table.optional + .batch-table .batch-table__toolbar %label.batch-table__toolbar__select.batch-checkbox-all = check_box_tag :batch_checkbox_all, nil, false diff --git a/app/views/disputes/strikes/show.html.haml b/app/views/disputes/strikes/show.html.haml index 1be50331a4eab8..edf7463e55f76b 100644 --- a/app/views/disputes/strikes/show.html.haml +++ b/app/views/disputes/strikes/show.html.haml @@ -50,17 +50,18 @@ .strike-card__statuses-list__item - if (status = status_map[status_id.to_i]) .one-liner - = link_to short_account_status_url(@strike.target_account, status_id), class: 'emojify' do - = one_line_preview(status) + .emojify= one_line_preview(status) - - status.ordered_media_attachments.each do |media_attachment| - %abbr{ title: media_attachment.description } - = fa_icon 'link' - = media_attachment.file_file_name + - status.ordered_media_attachments.each do |media_attachment| + %abbr{ title: media_attachment.description } + = fa_icon 'link' + = media_attachment.file_file_name .strike-card__statuses-list__item__meta - %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at) - · - = status.application.name + = link_to ActivityPub::TagManager.instance.url_for(status), target: '_blank' do + %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at) + - unless status.application.nil? + · + = status.application.name - else .one-liner= t('disputes.strikes.status', id: status_id) .strike-card__statuses-list__item__meta diff --git a/app/views/oauth/authorized_applications/index.html.haml b/app/views/oauth/authorized_applications/index.html.haml index 0280d8aef84a88..55d8524dbe91e8 100644 --- a/app/views/oauth/authorized_applications/index.html.haml +++ b/app/views/oauth/authorized_applications/index.html.haml @@ -18,8 +18,8 @@ .announcements-list__item__action-bar .announcements-list__item__meta - - if application.most_recently_used_access_token - = t('doorkeeper.authorized_applications.index.last_used_at', date: l(application.most_recently_used_access_token.last_used_at.to_date)) + - if @last_used_at_by_app[application.id] + = t('doorkeeper.authorized_applications.index.last_used_at', date: l(@last_used_at_by_app[application.id].to_date)) - else = t('doorkeeper.authorized_applications.index.never_used') diff --git a/app/views/relationships/show.html.haml b/app/views/relationships/show.html.haml index c82e639e0ed605..cd5152b4075a9c 100644 --- a/app/views/relationships/show.html.haml +++ b/app/views/relationships/show.html.haml @@ -48,7 +48,7 @@ = f.button safe_join([fa_icon('trash'), t('relationships.remove_selected_followers')]), name: :remove_from_followers, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } unless following_relationship? - = f.button safe_join([fa_icon('trash'), t('relationships.remove_selected_domains')]), name: :block_domains, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } if followed_by_relationship? + = f.button safe_join([fa_icon('trash'), t('relationships.remove_selected_domains')]), name: :remove_domains_from_followers, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } if followed_by_relationship? .batch-table__body - if @accounts.empty? = nothing_here 'nothing-here--under-tabs' diff --git a/app/views/settings/exports/show.html.haml b/app/views/settings/exports/show.html.haml index c49613fdc0c61f..d7b59af270a43b 100644 --- a/app/views/settings/exports/show.html.haml +++ b/app/views/settings/exports/show.html.haml @@ -64,6 +64,6 @@ %td= l backup.created_at - if backup.processed? %td= number_to_human_size backup.dump_file_size - %td= table_link_to 'download', t('exports.archive_takeout.download'), backup.dump.url + %td= table_link_to 'download', t('exports.archive_takeout.download'), download_backup_url(backup) - else %td{ colspan: 2 }= t('exports.archive_takeout.in_progress') diff --git a/app/views/user_mailer/backup_ready.html.haml b/app/views/user_mailer/backup_ready.html.haml index 85140b08be3b4e..465ead2c8bac23 100644 --- a/app/views/user_mailer/backup_ready.html.haml +++ b/app/views/user_mailer/backup_ready.html.haml @@ -55,5 +55,5 @@ %tbody %tr %td.button-primary - = link_to full_asset_url(@backup.dump.url) do + = link_to download_backup_url(@backup) do %span= t 'exports.archive_takeout.download' diff --git a/app/views/user_mailer/backup_ready.text.erb b/app/views/user_mailer/backup_ready.text.erb index eb89e7d743fba4..8ebbaae85a5b82 100644 --- a/app/views/user_mailer/backup_ready.text.erb +++ b/app/views/user_mailer/backup_ready.text.erb @@ -4,4 +4,4 @@ <%= t 'user_mailer.backup_ready.explanation' %> -=> <%= full_asset_url(@backup.dump.url) %> +=> <%= download_backup_url(@backup) %> diff --git a/app/workers/activitypub/delivery_worker.rb b/app/workers/activitypub/delivery_worker.rb index 788f2cf8094264..055ba791b8123e 100644 --- a/app/workers/activitypub/delivery_worker.rb +++ b/app/workers/activitypub/delivery_worker.rb @@ -13,9 +13,10 @@ class ActivityPub::DeliveryWorker HEADERS = { 'Content-Type' => 'application/activity+json' }.freeze def perform(json, source_account_id, inbox_url, options = {}) - return unless DeliveryFailureTracker.available?(inbox_url) - @options = options.with_indifferent_access + + return unless @options[:bypass_availability] || DeliveryFailureTracker.available?(inbox_url) + @json = json @source_account = Account.find(source_account_id) @inbox_url = inbox_url diff --git a/app/workers/activitypub/fetch_replies_worker.rb b/app/workers/activitypub/fetch_replies_worker.rb index 54d98f228ba73f..d72bad745261d3 100644 --- a/app/workers/activitypub/fetch_replies_worker.rb +++ b/app/workers/activitypub/fetch_replies_worker.rb @@ -6,8 +6,8 @@ class ActivityPub::FetchRepliesWorker sidekiq_options queue: 'pull', retry: 3 - def perform(parent_status_id, replies_uri) - ActivityPub::FetchRepliesService.new.call(Status.find(parent_status_id), replies_uri) + def perform(parent_status_id, replies_uri, options = {}) + ActivityPub::FetchRepliesService.new.call(Status.find(parent_status_id), replies_uri, **options.deep_symbolize_keys) rescue ActiveRecord::RecordNotFound true end diff --git a/app/workers/activitypub/migrated_follow_delivery_worker.rb b/app/workers/activitypub/migrated_follow_delivery_worker.rb new file mode 100644 index 00000000000000..17a9e515efad08 --- /dev/null +++ b/app/workers/activitypub/migrated_follow_delivery_worker.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class ActivityPub::MigratedFollowDeliveryWorker < ActivityPub::DeliveryWorker + def perform(json, source_account_id, inbox_url, old_target_account_id, options = {}) + super(json, source_account_id, inbox_url, options) + unfollow_old_account!(old_target_account_id) + end + + private + + def unfollow_old_account!(old_target_account_id) + old_target_account = Account.find(old_target_account_id) + UnfollowService.new.call(@source_account, old_target_account, skip_unmerge: true) + rescue StandardError + true + end +end diff --git a/app/workers/activitypub/synchronize_featured_collection_worker.rb b/app/workers/activitypub/synchronize_featured_collection_worker.rb index 7a0898e89de431..f67d693cb3ab3c 100644 --- a/app/workers/activitypub/synchronize_featured_collection_worker.rb +++ b/app/workers/activitypub/synchronize_featured_collection_worker.rb @@ -5,8 +5,10 @@ class ActivityPub::SynchronizeFeaturedCollectionWorker sidekiq_options queue: 'pull', lock: :until_executed - def perform(account_id) - ActivityPub::FetchFeaturedCollectionService.new.call(Account.find(account_id)) + def perform(account_id, options = {}) + options = { note: true, hashtag: false }.deep_merge(options.deep_symbolize_keys) + + ActivityPub::FetchFeaturedCollectionService.new.call(Account.find(account_id), **options) rescue ActiveRecord::RecordNotFound true end diff --git a/app/workers/fetch_reply_worker.rb b/app/workers/fetch_reply_worker.rb index f7aa25e815b236..68a7414bebeaa0 100644 --- a/app/workers/fetch_reply_worker.rb +++ b/app/workers/fetch_reply_worker.rb @@ -6,7 +6,7 @@ class FetchReplyWorker sidekiq_options queue: 'pull', retry: 3 - def perform(child_url) - FetchRemoteStatusService.new.call(child_url) + def perform(child_url, options = {}) + FetchRemoteStatusService.new.call(child_url, **options.deep_symbolize_keys) end end diff --git a/app/workers/scheduler/indexing_scheduler.rb b/app/workers/scheduler/indexing_scheduler.rb index 3a6f47a29a4e13..2f05b5f81b11c5 100644 --- a/app/workers/scheduler/indexing_scheduler.rb +++ b/app/workers/scheduler/indexing_scheduler.rb @@ -6,15 +6,17 @@ class Scheduler::IndexingScheduler sidekiq_options retry: 0 + IMPORT_BATCH_SIZE = 1000 + SCAN_BATCH_SIZE = 10 * IMPORT_BATCH_SIZE + def perform indexes.each do |type| with_redis do |redis| - ids = redis.smembers("chewy:queue:#{type.name}") - - type.import!(ids) - - redis.pipelined do |pipeline| - ids.each { |id| pipeline.srem("chewy:queue:#{type.name}", id) } + redis.sscan_each("chewy:queue:#{type.name}", count: SCAN_BATCH_SIZE).each_slice(IMPORT_BATCH_SIZE) do |ids| + type.import!(ids) + redis.pipelined do |pipeline| + pipeline.srem("chewy:queue:#{type.name}", ids) + end end end end diff --git a/app/workers/scheduler/user_cleanup_scheduler.rb b/app/workers/scheduler/user_cleanup_scheduler.rb index d1f00c47fbbae2..77715ac1d714fc 100644 --- a/app/workers/scheduler/user_cleanup_scheduler.rb +++ b/app/workers/scheduler/user_cleanup_scheduler.rb @@ -15,6 +15,8 @@ def perform def clean_unconfirmed_accounts! User.where('confirmed_at is NULL AND confirmation_sent_at <= ?', 2.days.ago).reorder(nil).find_in_batches do |batch| + # We have to do it separately because of missing database constraints + AccountModerationNote.where(target_account_id: batch.map(&:account_id)).delete_all Account.where(id: batch.map(&:account_id)).delete_all User.where(id: batch.map(&:id)).delete_all end @@ -29,7 +31,7 @@ def clean_suspended_accounts! def clean_discarded_statuses! Status.unscoped.discarded.where('deleted_at <= ?', 30.days.ago).find_in_batches do |statuses| RemovalWorker.push_bulk(statuses) do |status| - [status.id, { 'immediate' => true }] + [status.id, { 'immediate' => true, 'skip_streaming' => true }] end end end diff --git a/app/workers/thread_resolve_worker.rb b/app/workers/thread_resolve_worker.rb index 1b77dfdd932cba..3206c45f6399e5 100644 --- a/app/workers/thread_resolve_worker.rb +++ b/app/workers/thread_resolve_worker.rb @@ -6,9 +6,9 @@ class ThreadResolveWorker sidekiq_options queue: 'pull', retry: 3 - def perform(child_status_id, parent_url) + def perform(child_status_id, parent_url, options = {}) child_status = Status.find(child_status_id) - parent_status = FetchRemoteStatusService.new.call(parent_url) + parent_status = FetchRemoteStatusService.new.call(parent_url, **options.deep_symbolize_keys) return if parent_status.nil? diff --git a/app/workers/unfollow_follow_worker.rb b/app/workers/unfollow_follow_worker.rb index 0bd5ff472e845d..a4d57839de96c8 100644 --- a/app/workers/unfollow_follow_worker.rb +++ b/app/workers/unfollow_follow_worker.rb @@ -10,12 +10,7 @@ def perform(follower_account_id, old_target_account_id, new_target_account_id, b old_target_account = Account.find(old_target_account_id) new_target_account = Account.find(new_target_account_id) - follow = follower_account.active_relationships.find_by(target_account: old_target_account) - reblogs = follow&.show_reblogs? - notify = follow&.notify? - - FollowService.new.call(follower_account, new_target_account, reblogs: reblogs, notify: notify, bypass_locked: bypass_locked, bypass_limit: true) - UnfollowService.new.call(follower_account, old_target_account, skip_unmerge: true) + FollowMigrationService.new.call(follower_account, new_target_account, old_target_account, bypass_locked: bypass_locked) rescue ActiveRecord::RecordNotFound, Mastodon::NotPermittedError true end diff --git a/bin/tootctl b/bin/tootctl index a9ebb22c6dc5ac..9c7ae8b8712104 100755 --- a/bin/tootctl +++ b/bin/tootctl @@ -5,7 +5,9 @@ require_relative '../config/boot' require_relative '../lib/cli' begin - Mastodon::CLI.start(ARGV) + Chewy.strategy(:mastodon) do + Mastodon::CLI.start(ARGV) + end rescue Interrupt exit(130) end diff --git a/config/application.rb b/config/application.rb index 24fa2a978197ef..0c644ba0d89ad2 100644 --- a/config/application.rb +++ b/config/application.rb @@ -29,6 +29,7 @@ require_relative '../lib/paperclip/attachment_extensions' require_relative '../lib/paperclip/lazy_thumbnail' require_relative '../lib/paperclip/gif_transcoder' +require_relative '../lib/paperclip/media_type_spoof_detector_extensions' require_relative '../lib/paperclip/transcoder' require_relative '../lib/paperclip/type_corrector' require_relative '../lib/paperclip/response_with_limit_adapter' @@ -39,6 +40,7 @@ require_relative '../lib/devise/two_factor_ldap_authenticatable' require_relative '../lib/devise/two_factor_pam_authenticatable' require_relative '../lib/chewy/strategy/mastodon' +require_relative '../lib/chewy/strategy/bypass_with_warning' require_relative '../lib/webpacker/manifest_extensions' require_relative '../lib/webpacker/helper_extensions' require_relative '../lib/rails/engine_extensions' @@ -158,6 +160,12 @@ class Application < Rails::Application end end + config.active_record.yaml_column_permitted_classes = [Symbol, Date, Time, ActiveSupport::HashWithIndifferentAccess, ActiveSupport::TimeWithZone, ActiveSupport::TimeZone] + + config.public_file_server.headers = { + 'X-Content-Type-Options' => 'nosniff', + } + # config.paths.add File.join('app', 'api'), glob: File.join('**', '*.rb') # config.autoload_paths += Dir[Rails.root.join('app', 'api', '*')] diff --git a/config/database.yml b/config/database.yml index 127a78abfab39c..6bfde618bd76d7 100644 --- a/config/database.yml +++ b/config/database.yml @@ -4,6 +4,7 @@ default: &default timeout: 5000 encoding: unicode sslmode: <%= ENV['DB_SSLMODE'] || "prefer" %> + application_name: '' development: <<: *default diff --git a/config/imagemagick/policy.xml b/config/imagemagick/policy.xml new file mode 100644 index 00000000000000..1052476b319eb3 --- /dev/null +++ b/config/imagemagick/policy.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + diff --git a/config/initializers/chewy.rb b/config/initializers/chewy.rb index 752fc3c6dfe551..daf4a5f3260cd6 100644 --- a/config/initializers/chewy.rb +++ b/config/initializers/chewy.rb @@ -19,7 +19,7 @@ # cycle, which takes care of checking if Elasticsearch is enabled # or not. However, mind that for the Rails console, the :urgent # strategy is set automatically with no way to override it. -Chewy.root_strategy = :mastodon +Chewy.root_strategy = :bypass_with_warning if Rails.env.production? Chewy.request_strategy = :mastodon Chewy.use_after_commit_callbacks = false diff --git a/config/initializers/content_security_policy.rb b/config/initializers/content_security_policy.rb index c113b0f8b98060..c980b948ab7d90 100644 --- a/config/initializers/content_security_policy.rb +++ b/config/initializers/content_security_policy.rb @@ -3,7 +3,11 @@ # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy def host_to_url(str) - "http#{Rails.configuration.x.use_https ? 's' : ''}://#{str}" unless str.blank? + return if str.blank? + + uri = Addressable::URI.parse("http#{Rails.configuration.x.use_https ? 's' : ''}://#{str}") + uri.path += '/' unless uri.path.blank? || uri.path.end_with?('/') + uri.to_s end base_host = Rails.configuration.x.web_domain @@ -26,6 +30,7 @@ def host_to_url(str) p.media_src :self, :https, :data, assets_host p.frame_src :self, :https p.manifest_src :self, assets_host + p.form_action :self if Rails.env.development? webpacker_urls = %w(ws http).map { |protocol| "#{protocol}#{Webpacker.dev_server.https? ? 's' : ''}://#{Webpacker.dev_server.host_with_port}" } diff --git a/config/initializers/doorkeeper.rb b/config/initializers/doorkeeper.rb index 84b649f5c1017c..8097e7c882efd6 100644 --- a/config/initializers/doorkeeper.rb +++ b/config/initializers/doorkeeper.rb @@ -19,9 +19,14 @@ user unless user&.otp_required_for_login? end - # If you want to restrict access to the web interface for adding oauth authorized applications, you need to declare the block below. + # Doorkeeper provides some administrative interfaces for managing OAuth + # Applications, allowing creation, edit, and deletion of applications from the + # server. At present, these administrative routes are not integrated into + # Mastodon, and as such, we've disabled them by always return a 403 forbidden + # response for them. This does not affect the ability for users to manage + # their own OAuth Applications. admin_authenticator do - current_user&.admin? || redirect_to(new_user_session_url) + head 403 end # Authorization Code expiration time (default 10 minutes). diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb index 26b0a2f7cd9b6e..53830c276cca00 100644 --- a/config/initializers/paperclip.rb +++ b/config/initializers/paperclip.rb @@ -118,6 +118,7 @@ def copy_to_local_file(style, local_dest_path) openstack_domain_name: ENV.fetch('SWIFT_DOMAIN_NAME') { 'default' }, openstack_region: ENV['SWIFT_REGION'], openstack_cache_ttl: ENV.fetch('SWIFT_CACHE_TTL') { 60 }, + openstack_temp_url_key: ENV['SWIFT_TEMP_URL_KEY'], }, fog_directory: ENV['SWIFT_CONTAINER'], @@ -146,3 +147,10 @@ class NetworkingError < StandardError; end end end end + +# Set our ImageMagick security policy, but allow admins to override it +ENV['MAGICK_CONFIGURE_PATH'] = begin + imagemagick_config_paths = ENV.fetch('MAGICK_CONFIGURE_PATH', '').split(File::PATH_SEPARATOR) + imagemagick_config_paths << Rails.root.join('config', 'imagemagick').expand_path.to_s + imagemagick_config_paths.join(File::PATH_SEPARATOR) +end diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index c1327053df7961..bf35152a96551b 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -3,6 +3,11 @@ require_relative '../../lib/mastodon/sidekiq_middleware' Sidekiq.configure_server do |config| + if Rails.configuration.database_configuration.dig('production', 'adapter') == 'postgresql_makara' + STDERR.puts 'ERROR: Database replication is not currently supported in Sidekiq workers. Check your configuration.' + exit 1 + end + config.redis = REDIS_SIDEKIQ_PARAMS config.server_middleware do |chain| diff --git a/config/initializers/twitter_regex.rb b/config/initializers/twitter_regex.rb index 6a7723fd213c77..e65b05dfdea083 100644 --- a/config/initializers/twitter_regex.rb +++ b/config/initializers/twitter_regex.rb @@ -25,7 +25,7 @@ class Regex \) /iox UCHARS = '\u{A0}-\u{D7FF}\u{F900}-\u{FDCF}\u{FDF0}-\u{FFEF}\u{10000}-\u{1FFFD}\u{20000}-\u{2FFFD}\u{30000}-\u{3FFFD}\u{40000}-\u{4FFFD}\u{50000}-\u{5FFFD}\u{60000}-\u{6FFFD}\u{70000}-\u{7FFFD}\u{80000}-\u{8FFFD}\u{90000}-\u{9FFFD}\u{A0000}-\u{AFFFD}\u{B0000}-\u{BFFFD}\u{C0000}-\u{CFFFD}\u{D0000}-\u{DFFFD}\u{E1000}-\u{EFFFD}\u{E000}-\u{F8FF}\u{F0000}-\u{FFFFD}\u{100000}-\u{10FFFD}' - REGEXEN[:valid_url_query_chars] = /[a-z0-9!?\*'\(\);:&=\+\$\/%#\[\]\-_\.,~|@#{UCHARS}]/iou + REGEXEN[:valid_url_query_chars] = /[a-z0-9!?\*'\(\);:&=\+\$\/%#\[\]\-_\.,~|@\^#{UCHARS}]/iou REGEXEN[:valid_url_query_ending_chars] = /[a-z0-9_&=#\/\-#{UCHARS}]/iou REGEXEN[:valid_url_path] = /(?: (?: diff --git a/config/locales/devise.en.yml b/config/locales/devise.en.yml index 458fa6d7596e54..d47b38321b6d6a 100644 --- a/config/locales/devise.en.yml +++ b/config/locales/devise.en.yml @@ -12,6 +12,7 @@ en: last_attempt: You have one more attempt before your account is locked. locked: Your account is locked. not_found_in_database: Invalid %{authentication_keys} or password. + omniauth_user_creation_failure: Error creating an account for this identity. pending: Your account is still under review. timeout: Your session expired. Please sign in again to continue. unauthenticated: You need to sign in or sign up before continuing. diff --git a/config/locales/en.yml b/config/locales/en.yml index db76860e9e9c66..b21c8eb60139e4 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -783,6 +783,12 @@ en: message_html: You haven't defined any server rules. sidekiq_process_check: message_html: No Sidekiq process running for the %{value} queue(s). Please review your Sidekiq configuration + upload_check_privacy_error: + action: Check here for more information + message_html: "Your web server is misconfigured. The privacy of your users is at risk." + upload_check_privacy_error_object_storage: + action: Check here for more information + message_html: "Your object storage is misconfigured. The privacy of your users is at risk." tags: review: Review status updated_msg: Hashtag settings updated successfully @@ -1310,6 +1316,7 @@ en: expired: The poll has already ended invalid_choice: The chosen vote option does not exist over_character_limit: cannot be longer than %{max} characters each + self_vote: You cannot vote in your own polls too_few_options: must have more than one item too_many_options: can't contain more than %{max} items preferences: @@ -1323,6 +1330,7 @@ en: relationships: activity: Account activity dormant: Dormant + follow_failure: Could not follow some of the selected accounts. follow_selected_followers: Follow selected followers followers: Followers following: Following @@ -1452,7 +1460,6 @@ en: edited_at_html: Edited %{date} errors: in_reply_not_found: The post you are trying to reply to does not appear to exist. - language_detection: Automatically detect language local_only: Local-only open_in_web: Open in web over_character_limit: character limit of %{max} exceeded diff --git a/config/locales/zh-CN.yml b/config/locales/zh-CN.yml index 56ada1f05dbc8e..deef5cc72f3182 100644 --- a/config/locales/zh-CN.yml +++ b/config/locales/zh-CN.yml @@ -1426,7 +1426,6 @@ zh-CN: edited_at_html: 编辑于 %{date} errors: in_reply_not_found: 你回复的嘟文似乎不存在 - language_detection: 自动检测语言 local_only: 仅本实例可见 open_in_web: 在站内打开 over_character_limit: 超过了 %{max} 字的限制 diff --git a/config/locales/zh-HK.yml b/config/locales/zh-HK.yml index 37c44618664722..ced23ef60800c3 100644 --- a/config/locales/zh-HK.yml +++ b/config/locales/zh-HK.yml @@ -1154,7 +1154,6 @@ zh-HK: other: 包含不允許的標籤： %{tags} errors: in_reply_not_found: 你所回覆的嘟文並不存在。 - language_detection: 自動偵測語言 local_only: 僅本實例可見 open_in_web: 開啟網頁 over_character_limit: 超過了 %{max} 字的限制 diff --git a/config/locales/zh-TW.yml b/config/locales/zh-TW.yml index 5ed690a3ae17a6..a6c5a2d08980df 100644 --- a/config/locales/zh-TW.yml +++ b/config/locales/zh-TW.yml @@ -1426,7 +1426,6 @@ zh-TW: edited_at_html: 編輯於 %{date} errors: in_reply_not_found: 您嘗試回覆的嘟文看起來不存在。 - language_detection: 自動偵測語言 local_only: 僅本實例可見 open_in_web: 以網頁開啟 over_character_limit: 超過了 %{max} 字的限制 diff --git a/config/routes.rb b/config/routes.rb index ce467f447c7d29..f10715a936bcf2 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require 'sidekiq_unique_jobs/web' +require 'sidekiq_unique_jobs/web' if ENV['ENABLE_SIDEKIQ_UNIQUE_JOBS_UI'] == true require 'sidekiq-scheduler/web' Rails.application.routes.draw do @@ -183,6 +183,7 @@ get '/public', to: 'public_timelines#show', as: :public_timeline get '/media_proxy/:id/(*any)', to: 'media_proxy#show', as: :media_proxy, format: false + get '/backups/:id/download', to: 'backups#download', as: :download_backup, format: false resource :authorize_interaction, only: [:show, :create] resource :share, only: [:show, :create] @@ -225,7 +226,7 @@ end end - resources :instances, only: [:index, :show, :destroy], constraints: { id: /[^\/]+/ } do + resources :instances, only: [:index, :show, :destroy], constraints: { id: /[^\/]+/ }, format: 'html' do member do post :clear_delivery_errors post :restart_delivery @@ -394,7 +395,9 @@ resources :list, only: :show end - resources :streaming, only: [:index] + get '/streaming', to: 'streaming#index' + get '/streaming/(*any)', to: 'streaming#index' + resources :custom_emojis, only: [:index] resources :suggestions, only: [:index, :destroy] resources :scheduled_statuses, only: [:index, :show, :update, :destroy] diff --git a/db/seeds.rb b/db/seeds.rb index 0bfb5d0db5a709..695b5f1e78781f 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -1,11 +1,13 @@ -Doorkeeper::Application.create!(name: 'Web', superapp: true, redirect_uri: Doorkeeper.configuration.native_redirect_uri, scopes: 'read write follow push') +Chewy.strategy(:mastodon) do + Doorkeeper::Application.create!(name: 'Web', superapp: true, redirect_uri: Doorkeeper.configuration.native_redirect_uri, scopes: 'read write follow push') -domain = ENV['LOCAL_DOMAIN'] || Rails.configuration.x.local_domain -account = Account.find_or_initialize_by(id: -99, actor_type: 'Application', locked: true, username: domain) -account.save! + domain = ENV['LOCAL_DOMAIN'] || Rails.configuration.x.local_domain + account = Account.find_or_initialize_by(id: -99, actor_type: 'Application', locked: true, username: domain) + account.save! -if Rails.env.development? - admin = Account.where(username: 'admin').first_or_initialize(username: 'admin') - admin.save(validate: false) - User.where(email: "admin@#{domain}").first_or_initialize(email: "admin@#{domain}", password: 'mastodonadmin', password_confirmation: 'mastodonadmin', confirmed_at: Time.now.utc, admin: true, account: admin, agreement: true, approved: true).save! + if Rails.env.development? + admin = Account.where(username: 'admin').first_or_initialize(username: 'admin') + admin.save(validate: false) + User.where(email: "admin@#{domain}").first_or_initialize(email: "admin@#{domain}", password: 'mastodonadmin', password_confirmation: 'mastodonadmin', confirmed_at: Time.now.utc, admin: true, account: admin, agreement: true, approved: true).save! + end end diff --git a/dist/nginx.conf b/dist/nginx.conf index 7e03343680c6b6..cbcf328a6eaee4 100644 --- a/dist/nginx.conf +++ b/dist/nginx.conf @@ -61,12 +61,15 @@ server { location ~ ^/(emoji|packs|system/accounts/avatars|system/media_attachments/files) { add_header Cache-Control "public, max-age=31536000, immutable"; add_header Strict-Transport-Security "max-age=31536000" always; + add_header X-Content-Type-Options nosniff; + add_header Content-Security-Policy "default-src 'none'; form-action 'none'"; try_files $uri @proxy; } location /sw.js { add_header Cache-Control "public, max-age=0"; add_header Strict-Transport-Security "max-age=31536000" always; + add_header X-Content-Type-Options nosniff; try_files $uri @proxy; } diff --git a/docker-compose.yml b/docker-compose.yml index bb74d0cb26cf40..dd1488e7444ab2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,7 +44,7 @@ services: web: build: . - image: tootsuite/mastodon:v3.5.5 + image: ghcr.io/mastodon/mastodon:v3.5.19 restart: always env_file: .env.production command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000" @@ -65,7 +65,7 @@ services: streaming: build: . - image: tootsuite/mastodon:v3.5.5 + image: ghcr.io/mastodon/mastodon:v3.5.19 restart: always env_file: .env.production command: node ./streaming @@ -83,7 +83,7 @@ services: sidekiq: build: . - image: tootsuite/mastodon:v3.5.5 + image: ghcr.io/mastodon/mastodon:v3.5.19 restart: always env_file: .env.production command: bundle exec sidekiq diff --git a/lib/chewy/strategy/bypass_with_warning.rb b/lib/chewy/strategy/bypass_with_warning.rb new file mode 100644 index 00000000000000..eb6fbaab167603 --- /dev/null +++ b/lib/chewy/strategy/bypass_with_warning.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Chewy + class Strategy + class BypassWithWarning < Base + def update(...) + Rails.logger.warn 'Chewy update without a root strategy' unless @warning_issued + @warning_issued = true + end + end + end +end diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb index 7256d1da98f440..8bb258d9151984 100644 --- a/lib/mastodon/accounts_cli.rb +++ b/lib/mastodon/accounts_cli.rb @@ -492,7 +492,7 @@ def approve(username = nil) User.pending.find_each(&:approve!) say('OK', :green) elsif options[:number] - User.pending.limit(options[:number]).each(&:approve!) + User.pending.order(created_at: :asc).limit(options[:number]).each(&:approve!) say('OK', :green) elsif username.present? account = Account.find_local(username) diff --git a/lib/mastodon/cli_helper.rb b/lib/mastodon/cli_helper.rb index a78a28e2734b2d..4e304c903539b5 100644 --- a/lib/mastodon/cli_helper.rb +++ b/lib/mastodon/cli_helper.rb @@ -53,14 +53,16 @@ def parallelize_with_progress(scope) progress.log("Processing #{item.id}") if options[:verbose] - result = ActiveRecord::Base.connection_pool.with_connection do - yield(item) - ensure - RedisConfiguration.pool.checkin if Thread.current[:redis] - Thread.current[:redis] = nil + Chewy.strategy(:mastodon) do + result = ActiveRecord::Base.connection_pool.with_connection do + yield(item) + ensure + RedisConfiguration.pool.checkin if Thread.current[:redis] + Thread.current[:redis] = nil + end + + aggregate.increment(result) if result.is_a?(Integer) end - - aggregate.increment(result) if result.is_a?(Integer) rescue => e progress.log pastel.red("Error processing #{item.id}: #{e}") ensure diff --git a/lib/mastodon/sidekiq_middleware.rb b/lib/mastodon/sidekiq_middleware.rb index c75e8401f5fbc2..9832e1a27c96ca 100644 --- a/lib/mastodon/sidekiq_middleware.rb +++ b/lib/mastodon/sidekiq_middleware.rb @@ -3,8 +3,8 @@ class Mastodon::SidekiqMiddleware BACKTRACE_LIMIT = 3 - def call(*) - yield + def call(*, &block) + Chewy.strategy(:mastodon, &block) rescue Mastodon::HostValidationError # Do not retry rescue => e diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 24303323bee89d..f032d133d552f9 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ def minor end def patch - 5 + 19 end def flags diff --git a/lib/paperclip/media_type_spoof_detector_extensions.rb b/lib/paperclip/media_type_spoof_detector_extensions.rb new file mode 100644 index 00000000000000..a406ef312fd3c7 --- /dev/null +++ b/lib/paperclip/media_type_spoof_detector_extensions.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Paperclip + module MediaTypeSpoofDetectorExtensions + def calculated_content_type + return @calculated_content_type if defined?(@calculated_content_type) + + @calculated_content_type = type_from_file_command.chomp + + # The `file` command fails to recognize some MP3 files as such + @calculated_content_type = type_from_marcel if @calculated_content_type == 'application/octet-stream' && type_from_marcel == 'audio/mpeg' + @calculated_content_type + end + + def type_from_marcel + @type_from_marcel ||= Marcel::MimeType.for Pathname.new(@file.path), + name: @file.path + end + end +end + +Paperclip::MediaTypeSpoofDetector.prepend(Paperclip::MediaTypeSpoofDetectorExtensions) diff --git a/lib/paperclip/transcoder.rb b/lib/paperclip/transcoder.rb index afd9f58ff695c0..0f2e30f7d5e45e 100644 --- a/lib/paperclip/transcoder.rb +++ b/lib/paperclip/transcoder.rb @@ -19,10 +19,7 @@ def initialize(file, options = {}, attachment = nil) def make metadata = VideoMetadataExtractor.new(@file.path) - unless metadata.valid? - Paperclip.log("Unsupported file #{@file.path}") - return File.open(@file.path) - end + raise Paperclip::Error, "Error while transcoding #{@file.path}: unsupported file" unless metadata.valid? update_attachment_type(metadata) update_options_from_metadata(metadata) @@ -40,12 +37,14 @@ def make @output_options['f'] = 'image2' @output_options['vframes'] = 1 when 'mp4' - @output_options['acodec'] = 'aac' - @output_options['strict'] = 'experimental' - - if high_vfr?(metadata) && !eligible_to_passthrough?(metadata) - @output_options['vsync'] = 'vfr' - @output_options['r'] = @vfr_threshold + unless eligible_to_passthrough?(metadata) + @output_options['acodec'] = 'aac' + @output_options['strict'] = 'experimental' + + if high_vfr?(metadata) + @output_options['vsync'] = 'vfr' + @output_options['r'] = @vfr_threshold + end end end diff --git a/lib/sanitize_ext/sanitize_config.rb b/lib/sanitize_ext/sanitize_config.rb index de6b132ee960ab..f754308704a67e 100644 --- a/lib/sanitize_ext/sanitize_config.rb +++ b/lib/sanitize_ext/sanitize_config.rb @@ -50,7 +50,7 @@ module Config end end - current_node.replace(current_node.text) unless LINK_PROTOCOLS.include?(scheme) + current_node.replace(Nokogiri::XML::Text.new(current_node.text, current_node.document)) unless LINK_PROTOCOLS.include?(scheme) end UNSUPPORTED_ELEMENTS_TRANSFORMER = lambda do |env| @@ -95,26 +95,26 @@ module Config ] ) - MASTODON_OEMBED ||= freeze_config merge( - RELAXED, - elements: RELAXED[:elements] + %w(audio embed iframe source video), + MASTODON_OEMBED ||= freeze_config( + elements: %w(audio embed iframe source video), - attributes: merge( - RELAXED[:attributes], + attributes: { 'audio' => %w(controls), 'embed' => %w(height src type width), 'iframe' => %w(allowfullscreen frameborder height scrolling src width), 'source' => %w(src type), 'video' => %w(controls height loop width), - 'div' => [:data] - ), + }, - protocols: merge( - RELAXED[:protocols], + protocols: { 'embed' => { 'src' => HTTP_PROTOCOLS }, 'iframe' => { 'src' => HTTP_PROTOCOLS }, - 'source' => { 'src' => HTTP_PROTOCOLS } - ) + 'source' => { 'src' => HTTP_PROTOCOLS }, + }, + + add_attributes: { + 'iframe' => { 'sandbox' => 'allow-scripts allow-same-origin allow-popups allow-popups-to-escape-sandbox allow-forms' }, + } ) end end diff --git a/lib/tasks/sidekiq_unique_jobs.rake b/lib/tasks/sidekiq_unique_jobs.rake new file mode 100644 index 00000000000000..bedc8fe4c650c4 --- /dev/null +++ b/lib/tasks/sidekiq_unique_jobs.rake @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +namespace :sidekiq_unique_jobs do + task delete_all_locks: :environment do + digests = SidekiqUniqueJobs::Digests.new + digests.delete_by_pattern('*', count: digests.count) + + expiring_digests = SidekiqUniqueJobs::ExpiringDigests.new + expiring_digests.delete_by_pattern('*', count: expiring_digests.count) + end +end diff --git a/spec/controllers/admin/accounts_controller_spec.rb b/spec/controllers/admin/accounts_controller_spec.rb index 1779fb7c0be227..1d820913a0e0a1 100644 --- a/spec/controllers/admin/accounts_controller_spec.rb +++ b/spec/controllers/admin/accounts_controller_spec.rb @@ -147,6 +147,87 @@ end end + describe 'POST #approve' do + subject { post :approve, params: { id: account.id } } + + let(:current_user) { Fabricate(:user, role: role) } + let(:account) { user.account } + let(:user) { Fabricate(:user) } + + before do + account.user.update(approved: false) + end + + context 'when user is admin' do + let(:role) { 'admin' } + + it 'succeeds in approving account' do + is_expected.to redirect_to admin_accounts_path(status: 'pending') + expect(user.reload).to be_approved + end + + it 'logs action' do + is_expected.to have_http_status :found + + log_item = Admin::ActionLog.last + + expect(log_item).to_not be_nil + expect(log_item.action).to eq :approve + expect(log_item.account_id).to eq current_user.account_id + expect(log_item.target_id).to eq account.user.id + end + end + + context 'when user is not admin' do + let(:role) { 'user' } + + it 'fails to approve account' do + is_expected.to have_http_status :forbidden + expect(user.reload).not_to be_approved + end + end + end + + describe 'POST #reject' do + subject { post :reject, params: { id: account.id } } + + let(:current_user) { Fabricate(:user, role: role) } + let(:account) { user.account } + let(:user) { Fabricate(:user) } + + before do + account.user.update(approved: false) + end + + context 'when user is admin' do + let(:role) { 'admin' } + + it 'succeeds in rejecting account' do + is_expected.to redirect_to admin_accounts_path(status: 'pending') + end + + it 'logs action' do + is_expected.to have_http_status :found + + log_item = Admin::ActionLog.last + + expect(log_item).to_not be_nil + expect(log_item.action).to eq :reject + expect(log_item.account_id).to eq current_user.account_id + expect(log_item.target_id).to eq account.user.id + end + end + + context 'when user is not admin' do + let(:role) { 'user' } + + it 'fails to reject account' do + is_expected.to have_http_status :forbidden + expect(user.reload).not_to be_approved + end + end + end + describe 'POST #redownload' do subject { post :redownload, params: { id: account.id } } diff --git a/spec/controllers/admin/domain_blocks_controller_spec.rb b/spec/controllers/admin/domain_blocks_controller_spec.rb index ecc79292b41a38..03e804cf6a21b2 100644 --- a/spec/controllers/admin/domain_blocks_controller_spec.rb +++ b/spec/controllers/admin/domain_blocks_controller_spec.rb @@ -49,6 +49,53 @@ end end + describe 'PUT #update' do + let!(:remote_account) { Fabricate(:account, domain: 'example.com') } + let(:domain_block) { Fabricate(:domain_block, domain: 'example.com', severity: original_severity) } + + before do + BlockDomainService.new.call(domain_block) + end + + let(:subject) do + post :update, params: { id: domain_block.id, domain_block: { domain: 'example.com', severity: new_severity } } + end + + context 'downgrading a domain suspension to silence' do + let(:original_severity) { 'suspend' } + let(:new_severity) { 'silence' } + + it 'changes the block severity' do + expect { subject }.to change { domain_block.reload.severity }.from('suspend').to('silence') + end + + it 'undoes individual suspensions' do + expect { subject }.to change { remote_account.reload.suspended? }.from(true).to(false) + end + + it 'performs individual silences' do + expect { subject }.to change { remote_account.reload.silenced? }.from(false).to(true) + end + end + + context 'upgrading a domain silence to suspend' do + let(:original_severity) { 'silence' } + let(:new_severity) { 'suspend' } + + it 'changes the block severity' do + expect { subject }.to change { domain_block.reload.severity }.from('silence').to('suspend') + end + + it 'undoes individual silences' do + expect { subject }.to change { remote_account.reload.silenced? }.from(true).to(false) + end + + it 'performs individual suspends' do + expect { subject }.to change { remote_account.reload.suspended? }.from(false).to(true) + end + end + end + describe 'DELETE #destroy' do it 'unblocks the domain' do service = double(call: true) diff --git a/spec/controllers/admin/reports/actions_controller_spec.rb b/spec/controllers/admin/reports/actions_controller_spec.rb new file mode 100644 index 00000000000000..e252f7634adf2a --- /dev/null +++ b/spec/controllers/admin/reports/actions_controller_spec.rb @@ -0,0 +1,42 @@ +require 'rails_helper' + +describe Admin::Reports::ActionsController do + render_views + + let(:user) { Fabricate(:user, role: 'moderator') } + let(:account) { Fabricate(:account) } + let!(:status) { Fabricate(:status, account: account) } + let(:media_attached_status) { Fabricate(:status, account: account) } + let!(:media_attachment) { Fabricate(:media_attachment, account: account, status: media_attached_status) } + let(:media_attached_deleted_status) { Fabricate(:status, account: account, deleted_at: 1.day.ago) } + let!(:media_attachment2) { Fabricate(:media_attachment, account: account, status: media_attached_deleted_status) } + let(:last_media_attached_status) { Fabricate(:status, account: account) } + let!(:last_media_attachment) { Fabricate(:media_attachment, account: account, status: last_media_attached_status) } + let!(:last_status) { Fabricate(:status, account: account) } + + before do + sign_in user, scope: :user + end + + describe 'POST #create' do + let(:report) { Fabricate(:report, status_ids: status_ids, account: user.account, target_account: account) } + let(:status_ids) { [media_attached_status.id, media_attached_deleted_status.id] } + + before do + post :create, params: { report_id: report.id, action => '' } + end + + context 'when action is mark_as_sensitive' do + + let(:action) { 'mark_as_sensitive' } + + it 'resolves the report' do + expect(report.reload.action_taken_at).to_not be_nil + end + + it 'marks the non-deleted as sensitive' do + expect(media_attached_status.reload.sensitive).to eq true + end + end + end +end diff --git a/spec/controllers/admin/statuses_controller_spec.rb b/spec/controllers/admin/statuses_controller_spec.rb index de32fd18e1dc68..e23258e167f123 100644 --- a/spec/controllers/admin/statuses_controller_spec.rb +++ b/spec/controllers/admin/statuses_controller_spec.rb @@ -40,24 +40,36 @@ end describe 'POST #batch' do - before do - post :batch, params: { :account_id => account.id, action => '', :admin_status_batch_action => { status_ids: status_ids } } - end + subject { post :batch, params: { :account_id => account.id, action => '', :admin_status_batch_action => { status_ids: status_ids } } } let(:status_ids) { [media_attached_status.id] } - context 'when action is report' do + shared_examples 'when action is report' do let(:action) { 'report' } it 'creates a report' do + subject + report = Report.last expect(report.target_account_id).to eq account.id expect(report.status_ids).to eq status_ids end it 'redirects to report page' do + subject + expect(response).to redirect_to(admin_report_path(Report.last.id)) end end + + it_behaves_like 'when action is report' + + context 'when the moderator is blocked by the author' do + before do + account.block!(user.account) + end + + it_behaves_like 'when action is report' + end end end diff --git a/spec/controllers/api/v1/admin/accounts_controller_spec.rb b/spec/controllers/api/v1/admin/accounts_controller_spec.rb index b69595f7e422d6..ae2d09e07120f1 100644 --- a/spec/controllers/api/v1/admin/accounts_controller_spec.rb +++ b/spec/controllers/api/v1/admin/accounts_controller_spec.rb @@ -100,6 +100,15 @@ it 'approves user' do expect(account.reload.user_approved?).to be true end + + it 'logs action' do + log_item = Admin::ActionLog.last + + expect(log_item).to_not be_nil + expect(log_item.action).to eq :approve + expect(log_item.account_id).to eq user.account_id + expect(log_item.target_id).to eq account.user.id + end end describe 'POST #reject' do @@ -118,6 +127,15 @@ it 'removes user' do expect(User.where(id: account.user.id).count).to eq 0 end + + it 'logs action' do + log_item = Admin::ActionLog.last + + expect(log_item).to_not be_nil + expect(log_item.action).to eq :reject + expect(log_item.account_id).to eq user.account_id + expect(log_item.target_id).to eq account.user.id + end end describe 'POST #enable' do diff --git a/spec/controllers/api/v1/conversations_controller_spec.rb b/spec/controllers/api/v1/conversations_controller_spec.rb index 5add7cf1d4214f..1ec26d52036fa6 100644 --- a/spec/controllers/api/v1/conversations_controller_spec.rb +++ b/spec/controllers/api/v1/conversations_controller_spec.rb @@ -16,6 +16,7 @@ before do PostStatusService.new.call(other.account, text: 'Hey @alice', visibility: 'direct') + PostStatusService.new.call(user.account, text: 'Hey, nobody here', visibility: 'direct') end it 'returns http success' do @@ -31,7 +32,26 @@ it 'returns conversations' do get :index json = body_as_json - expect(json.size).to eq 1 + expect(json.size).to eq 2 + expect(json[0][:accounts].size).to eq 1 + end + + context 'with since_id' do + context 'when requesting old posts' do + it 'returns conversations' do + get :index, params: { since_id: Mastodon::Snowflake.id_at(1.hour.ago, with_random: false) } + json = body_as_json + expect(json.size).to eq 2 + end + end + + context 'when requesting posts in the future' do + it 'returns no conversation' do + get :index, params: { since_id: Mastodon::Snowflake.id_at(1.hour.from_now, with_random: false) } + json = body_as_json + expect(json.size).to eq 0 + end + end end end end diff --git a/spec/controllers/api/v1/timelines/tag_controller_spec.rb b/spec/controllers/api/v1/timelines/tag_controller_spec.rb index 718911083362de..d7f41d2f809816 100644 --- a/spec/controllers/api/v1/timelines/tag_controller_spec.rb +++ b/spec/controllers/api/v1/timelines/tag_controller_spec.rb @@ -5,36 +5,71 @@ describe Api::V1::Timelines::TagController do render_views - let(:user) { Fabricate(:user) } + let(:user) { Fabricate(:user) } + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:statuses') } before do allow(controller).to receive(:doorkeeper_token) { token } end - context 'with a user context' do - let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id) } + describe 'GET #show' do + subject do + get :show, params: { id: 'test' } + end + + before do + PostStatusService.new.call(user.account, text: 'It is a #test') + end + + context 'when the instance allows public preview' do + context 'when the user is not authenticated' do + let(:token) { nil } - describe 'GET #show' do - before do - PostStatusService.new.call(user.account, text: 'It is a #test') + it 'returns http success', :aggregate_failures do + subject + + expect(response).to have_http_status(200) + expect(response.headers['Link'].links.size).to eq(2) + end end - it 'returns http success' do - get :show, params: { id: 'test' } - expect(response).to have_http_status(200) - expect(response.headers['Link'].links.size).to eq(2) + context 'when the user is authenticated' do + it 'returns http success', :aggregate_failures do + subject + + expect(response).to have_http_status(200) + expect(response.headers['Link'].links.size).to eq(2) + end end end - end - context 'without a user context' do - let(:token) { Fabricate(:accessible_access_token, resource_owner_id: nil) } + context 'when the instance does not allow public preview' do + around do |example| + timeline_preview = Setting.timeline_preview + Setting.timeline_preview = false + + example.run + + Setting.timeline_preview = timeline_preview + end + + context 'when the user is not authenticated' do + let(:token) { nil } + + it 'returns http unauthorized' do + subject + + expect(response).to have_http_status(401) + end + end + + context 'when the user is authenticated' do + it 'returns http success', :aggregate_failures do + subject - describe 'GET #show' do - it 'returns http success' do - get :show, params: { id: 'test' } - expect(response).to have_http_status(200) - expect(response.headers['Link']).to be_nil + expect(response).to have_http_status(200) + expect(response.headers['Link'].links.size).to eq(2) + end end end end diff --git a/spec/controllers/api/v2/admin/accounts_controller_spec.rb b/spec/controllers/api/v2/admin/accounts_controller_spec.rb index 3212ddb844e67e..01a50607dc3193 100644 --- a/spec/controllers/api/v2/admin/accounts_controller_spec.rb +++ b/spec/controllers/api/v2/admin/accounts_controller_spec.rb @@ -69,5 +69,13 @@ end end end + + context 'with limit param' do + let(:params) { { limit: 1 } } + + it 'sets the correct pagination headers' do + expect(response.headers['Link'].find_link(%w(rel next)).href).to eq api_v2_admin_accounts_url(limit: 1, max_id: admin_account.id) + end + end end end diff --git a/spec/controllers/concerns/cache_concern_spec.rb b/spec/controllers/concerns/cache_concern_spec.rb index a34d7d72676964..21daa19921007e 100644 --- a/spec/controllers/concerns/cache_concern_spec.rb +++ b/spec/controllers/concerns/cache_concern_spec.rb @@ -13,12 +13,17 @@ def empty_array def empty_relation render plain: cache_collection(Status.none, Status).size end + + def account_statuses_favourites + render plain: cache_collection(Status.where(account_id: params[:id]), Status).map(&:favourites_count) + end end before do routes.draw do - get 'empty_array' => 'anonymous#empty_array' - post 'empty_relation' => 'anonymous#empty_relation' + get 'empty_array' => 'anonymous#empty_array' + get 'empty_relation' => 'anonymous#empty_relation' + get 'account_statuses_favourites' => 'anonymous#account_statuses_favourites' end end @@ -36,5 +41,20 @@ def empty_relation expect(response.body).to eq '0' end end + + context 'when given a collection of statuses' do + let!(:account) { Fabricate(:account) } + let!(:status) { Fabricate(:status, account: account) } + + it 'correctly updates with new interactions' do + get :account_statuses_favourites, params: { id: account.id } + expect(response.body).to eq '[0]' + + FavouriteService.new.call(account, status) + + get :account_statuses_favourites, params: { id: account.id } + expect(response.body).to eq '[1]' + end + end end end diff --git a/spec/controllers/relationships_controller_spec.rb b/spec/controllers/relationships_controller_spec.rb index 2056a2ac294176..bcdcfa9051f3e9 100644 --- a/spec/controllers/relationships_controller_spec.rb +++ b/spec/controllers/relationships_controller_spec.rb @@ -55,7 +55,7 @@ end context 'when select parameter is provided' do - subject { patch :update, params: { form_account_batch: { account_ids: [poopfeast.id] }, block_domains: '' } } + subject { patch :update, params: { form_account_batch: { account_ids: [poopfeast.id] }, remove_domains_from_followers: '' } } it 'soft-blocks followers from selected domains' do poopfeast.follow!(user.account) @@ -66,6 +66,15 @@ expect(poopfeast.following?(user.account)).to be false end + it 'does not unfollow users from selected domains' do + user.account.follow!(poopfeast) + + sign_in user, scope: :user + subject + + expect(user.account.following?(poopfeast)).to be true + end + include_examples 'authenticate user' include_examples 'redirects back to followers page' end diff --git a/spec/controllers/settings/two_factor_authentication/webauthn_credentials_controller_spec.rb b/spec/controllers/settings/two_factor_authentication/webauthn_credentials_controller_spec.rb index fe53b4dfc26c17..269c4d685a3f6a 100644 --- a/spec/controllers/settings/two_factor_authentication/webauthn_credentials_controller_spec.rb +++ b/spec/controllers/settings/two_factor_authentication/webauthn_credentials_controller_spec.rb @@ -248,7 +248,7 @@ def add_webauthn_credential(user) post :create, params: { credential: new_webauthn_credential, nickname: 'USB Key' } - expect(response).to have_http_status(500) + expect(response).to have_http_status(422) expect(flash[:error]).to be_present end end @@ -268,7 +268,7 @@ def add_webauthn_credential(user) post :create, params: { credential: new_webauthn_credential, nickname: nickname } - expect(response).to have_http_status(500) + expect(response).to have_http_status(422) expect(flash[:error]).to be_present end end diff --git a/spec/fabricators/account_stat_fabricator.rb b/spec/fabricators/account_stat_fabricator.rb index 2b06b4790920de..20272fb22f202d 100644 --- a/spec/fabricators/account_stat_fabricator.rb +++ b/spec/fabricators/account_stat_fabricator.rb @@ -1,6 +1,8 @@ +# frozen_string_literal: true + Fabricator(:account_stat) do - account nil - statuses_count "" - following_count "" - followers_count "" + account { Fabricate.build(:account) } + statuses_count '123' + following_count '456' + followers_count '789' end diff --git a/spec/fixtures/files/attachment-jpg.123456_abcd b/spec/fixtures/files/attachment-jpg.123456_abcd new file mode 100644 index 00000000000000..f1d40539ac0484 Binary files /dev/null and b/spec/fixtures/files/attachment-jpg.123456_abcd differ diff --git a/spec/fixtures/files/boop.mp3 b/spec/fixtures/files/boop.mp3 new file mode 100644 index 00000000000000..ba106a3a32d414 Binary files /dev/null and b/spec/fixtures/files/boop.mp3 differ diff --git a/spec/helpers/jsonld_helper_spec.rb b/spec/helpers/jsonld_helper_spec.rb index 744a14f26096f7..54355b8482647a 100644 --- a/spec/helpers/jsonld_helper_spec.rb +++ b/spec/helpers/jsonld_helper_spec.rb @@ -56,15 +56,15 @@ describe '#fetch_resource' do context 'when the second argument is false' do it 'returns resource even if the retrieved ID and the given URI does not match' do - stub_request(:get, 'https://bob.test/').to_return body: '{"id": "https://alice.test/"}' - stub_request(:get, 'https://alice.test/').to_return body: '{"id": "https://alice.test/"}' + stub_request(:get, 'https://bob.test/').to_return(body: '{"id": "https://alice.test/"}', headers: { 'Content-Type': 'application/activity+json' }) + stub_request(:get, 'https://alice.test/').to_return(body: '{"id": "https://alice.test/"}', headers: { 'Content-Type': 'application/activity+json' }) expect(fetch_resource('https://bob.test/', false)).to eq({ 'id' => 'https://alice.test/' }) end it 'returns nil if the object identified by the given URI and the object identified by the retrieved ID does not match' do - stub_request(:get, 'https://mallory.test/').to_return body: '{"id": "https://marvin.test/"}' - stub_request(:get, 'https://marvin.test/').to_return body: '{"id": "https://alice.test/"}' + stub_request(:get, 'https://mallory.test/').to_return(body: '{"id": "https://marvin.test/"}', headers: { 'Content-Type': 'application/activity+json' }) + stub_request(:get, 'https://marvin.test/').to_return(body: '{"id": "https://alice.test/"}', headers: { 'Content-Type': 'application/activity+json' }) expect(fetch_resource('https://mallory.test/', false)).to eq nil end @@ -72,7 +72,7 @@ context 'when the second argument is true' do it 'returns nil if the retrieved ID and the given URI does not match' do - stub_request(:get, 'https://mallory.test/').to_return body: '{"id": "https://alice.test/"}' + stub_request(:get, 'https://mallory.test/').to_return(body: '{"id": "https://alice.test/"}', headers: { 'Content-Type': 'application/activity+json' }) expect(fetch_resource('https://mallory.test/', true)).to eq nil end end @@ -80,12 +80,12 @@ describe '#fetch_resource_without_id_validation' do it 'returns nil if the status code is not 200' do - stub_request(:get, 'https://host.test/').to_return status: 400, body: '{}' + stub_request(:get, 'https://host.test/').to_return(status: 400, body: '{}', headers: { 'Content-Type': 'application/activity+json' }) expect(fetch_resource_without_id_validation('https://host.test/')).to eq nil end it 'returns hash' do - stub_request(:get, 'https://host.test/').to_return status: 200, body: '{}' + stub_request(:get, 'https://host.test/').to_return(status: 200, body: '{}', headers: { 'Content-Type': 'application/activity+json' }) expect(fetch_resource_without_id_validation('https://host.test/')).to eq({}) end end diff --git a/spec/lib/account_reach_finder_spec.rb b/spec/lib/account_reach_finder_spec.rb new file mode 100644 index 00000000000000..1da95ba6b3a66e --- /dev/null +++ b/spec/lib/account_reach_finder_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe AccountReachFinder do + let(:account) { Fabricate(:account) } + + let(:follower1) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://example.com/inbox-1') } + let(:follower2) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://example.com/inbox-2') } + let(:follower3) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://foo.bar/users/a/inbox', shared_inbox_url: 'https://foo.bar/inbox') } + + let(:mentioned1) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://foo.bar/users/b/inbox', shared_inbox_url: 'https://foo.bar/inbox') } + let(:mentioned2) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://example.com/inbox-3') } + let(:mentioned3) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://example.com/inbox-4') } + + let(:unrelated_account) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://example.com/unrelated-inbox') } + + before do + follower1.follow!(account) + follower2.follow!(account) + follower3.follow!(account) + + Fabricate(:status, account: account).tap do |status| + status.mentions << Mention.new(account: follower1) + status.mentions << Mention.new(account: mentioned1) + end + + Fabricate(:status, account: account) + + Fabricate(:status, account: account).tap do |status| + status.mentions << Mention.new(account: mentioned2) + status.mentions << Mention.new(account: mentioned3) + end + + Fabricate(:status).tap do |status| + status.mentions << Mention.new(account: unrelated_account) + end + end + + describe '#inboxes' do + it 'includes the preferred inbox URL of followers' do + expect(described_class.new(account).inboxes).to include(*[follower1, follower2, follower3].map(&:preferred_inbox_url)) + end + + it 'includes the preferred inbox URL of recently-mentioned accounts' do + expect(described_class.new(account).inboxes).to include(*[mentioned1, mentioned2, mentioned3].map(&:preferred_inbox_url)) + end + + it 'does not include the inbox of unrelated users' do + expect(described_class.new(account).inboxes).to_not include(unrelated_account.preferred_inbox_url) + end + end +end diff --git a/spec/lib/activitypub/activity/add_spec.rb b/spec/lib/activitypub/activity/add_spec.rb index e6408b610f99cf..0b08e2924a3de1 100644 --- a/spec/lib/activitypub/activity/add_spec.rb +++ b/spec/lib/activitypub/activity/add_spec.rb @@ -48,7 +48,7 @@ end it 'fetches the status and pins it' do - allow(service_stub).to receive(:call) do |uri, id: true, on_behalf_of: nil| + allow(service_stub).to receive(:call) do |uri, id: true, on_behalf_of: nil, request_id: nil| expect(uri).to eq 'https://example.com/unknown' expect(id).to eq true expect(on_behalf_of&.following?(sender)).to eq true @@ -62,7 +62,7 @@ context 'when there is no local follower' do it 'tries to fetch the status' do - allow(service_stub).to receive(:call) do |uri, id: true, on_behalf_of: nil| + allow(service_stub).to receive(:call) do |uri, id: true, on_behalf_of: nil, request_id: nil| expect(uri).to eq 'https://example.com/unknown' expect(id).to eq true expect(on_behalf_of).to eq nil diff --git a/spec/lib/activitypub/activity/announce_spec.rb b/spec/lib/activitypub/activity/announce_spec.rb index 41806b25829a26..67a99434688252 100644 --- a/spec/lib/activitypub/activity/announce_spec.rb +++ b/spec/lib/activitypub/activity/announce_spec.rb @@ -33,7 +33,7 @@ context 'when sender is followed by a local account' do before do Fabricate(:account).follow!(sender) - stub_request(:get, 'https://example.com/actor/hello-world').to_return(body: Oj.dump(unknown_object_json)) + stub_request(:get, 'https://example.com/actor/hello-world').to_return(body: Oj.dump(unknown_object_json), headers: { 'Content-Type': 'application/activity+json' }) subject.perform end @@ -118,7 +118,7 @@ subject { described_class.new(json, sender, relayed_through_account: relay_account) } before do - stub_request(:get, 'https://example.com/actor/hello-world').to_return(body: Oj.dump(unknown_object_json)) + stub_request(:get, 'https://example.com/actor/hello-world').to_return(body: Oj.dump(unknown_object_json), headers: { 'Content-Type': 'application/activity+json' }) end context 'and the relay is enabled' do diff --git a/spec/lib/activitypub/activity/create_spec.rb b/spec/lib/activitypub/activity/create_spec.rb index 1a25395fad390f..378ba0cd1b5c87 100644 --- a/spec/lib/activitypub/activity/create_spec.rb +++ b/spec/lib/activitypub/activity/create_spec.rb @@ -29,29 +29,67 @@ subject.perform end - context 'object has been edited' do + context 'when object publication date is below ISO8601 range' do let(:object_json) do { id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, type: 'Note', content: 'Lorem ipsum', - published: '2022-01-22T15:00:00Z', - updated: '2022-01-22T16:00:00Z', + published: '-0977-11-03T08:31:22Z', } end - it 'creates status' do + it 'creates status with a valid creation date', :aggregate_failures do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.text).to eq 'Lorem ipsum' + + expect(status.created_at).to be_within(30).of(Time.now.utc) + end + end + + context 'when object publication date is above ISO8601 range' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + published: '10000-11-03T08:31:22Z', + } + end + + it 'creates status with a valid creation date', :aggregate_failures do status = sender.statuses.first expect(status).to_not be_nil expect(status.text).to eq 'Lorem ipsum' + + expect(status.created_at).to be_within(30).of(Time.now.utc) end + end - it 'marks status as edited' do + context 'when object has been edited' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + published: '2022-01-22T15:00:00Z', + updated: '2022-01-22T16:00:00Z', + } + end + + it 'creates status with appropriate creation and edition dates', :aggregate_failures do status = sender.statuses.first expect(status).to_not be_nil - expect(status.edited?).to eq true + expect(status.text).to eq 'Lorem ipsum' + + expect(status.created_at).to eq '2022-01-22T15:00:00Z'.to_datetime + + expect(status.edited?).to be true + expect(status.edited_at).to eq '2022-01-22T16:00:00Z'.to_datetime end end diff --git a/spec/lib/activitypub/activity/flag_spec.rb b/spec/lib/activitypub/activity/flag_spec.rb index 2f2d13876760df..6d7a8a7ec2e8dd 100644 --- a/spec/lib/activitypub/activity/flag_spec.rb +++ b/spec/lib/activitypub/activity/flag_spec.rb @@ -37,6 +37,37 @@ end end + context 'when the report comment is excessively long' do + subject do + described_class.new({ + '@context': 'https://www.w3.org/ns/activitystreams', + id: flag_id, + type: 'Flag', + content: long_comment, + actor: ActivityPub::TagManager.instance.uri_for(sender), + object: [ + ActivityPub::TagManager.instance.uri_for(flagged), + ActivityPub::TagManager.instance.uri_for(status), + ], + }.with_indifferent_access, sender) + end + + let(:long_comment) { Faker::Lorem.characters(number: 6000) } + + before do + subject.perform + end + + it 'creates a report but with a truncated comment' do + report = Report.find_by(account: sender, target_account: flagged) + + expect(report).to_not be_nil + expect(report.comment.length).to eq 5000 + expect(report.comment).to eq long_comment[0...5000] + expect(report.status_ids).to eq [status.id] + end + end + context 'when the reported status is private and should not be visible to the remote server' do let(:status) { Fabricate(:status, account: flagged, uri: 'foobar', visibility: :private) } diff --git a/spec/lib/activitypub/linked_data_signature_spec.rb b/spec/lib/activitypub/linked_data_signature_spec.rb index 2222c46fb5575c..b2c34039089d1d 100644 --- a/spec/lib/activitypub/linked_data_signature_spec.rb +++ b/spec/lib/activitypub/linked_data_signature_spec.rb @@ -36,6 +36,40 @@ end end + context 'when local account record is missing a public key' do + let(:raw_signature) do + { + 'creator' => 'http://example.com/alice', + 'created' => '2017-09-23T20:21:34Z', + } + end + + let(:signature) { raw_signature.merge('type' => 'RsaSignature2017', 'signatureValue' => sign(sender, raw_signature, raw_json)) } + + let(:service_stub) { instance_double(ActivityPub::FetchRemoteKeyService) } + + before do + # Ensure signature is computed with the old key + signature + + # Unset key + old_key = sender.public_key + sender.update!(private_key: '', public_key: '') + + allow(ActivityPub::FetchRemoteKeyService).to receive(:new).and_return(service_stub) + + allow(service_stub).to receive(:call).with('http://example.com/alice') do + sender.update!(public_key: old_key) + sender + end + end + + it 'fetches key and returns creator' do + expect(subject.verify_account!).to eq sender + expect(service_stub).to have_received(:call).with('http://example.com/alice').once + end + end + context 'when signature is missing' do let(:signature) { nil } diff --git a/spec/lib/sanitize_config_spec.rb b/spec/lib/sanitize_config_spec.rb index 747d81158da454..c9543ceb0c1983 100644 --- a/spec/lib/sanitize_config_spec.rb +++ b/spec/lib/sanitize_config_spec.rb @@ -38,6 +38,10 @@ expect(Sanitize.fragment('Test', subject)).to eq 'Test' end + it 'does not re-interpret HTML when removing unsupported links' do + expect(Sanitize.fragment('Test<a href="https://example.com">test</a>', subject)).to eq 'Test<a href="https://example.com">test</a>' + end + it 'keeps a with href' do expect(Sanitize.fragment('Test', subject)).to eq 'Test' end diff --git a/spec/models/account_spec.rb b/spec/models/account_spec.rb index dc0ca3da3742db..b4e8a7b3672eda 100644 --- a/spec/models/account_spec.rb +++ b/spec/models/account_spec.rb @@ -689,7 +689,7 @@ expect(subject.match('Check this out https://medium.com/@alice/some-article#.abcdef123')).to be_nil end - xit 'does not match URL querystring' do + it 'does not match URL query string' do expect(subject.match('https://example.com/?x=@alice')).to be_nil end end diff --git a/spec/models/identity_spec.rb b/spec/models/identity_spec.rb index 689c9b797f4f63..081c254d8200f3 100644 --- a/spec/models/identity_spec.rb +++ b/spec/models/identity_spec.rb @@ -1,16 +1,16 @@ require 'rails_helper' RSpec.describe Identity, type: :model do - describe '.find_for_oauth' do + describe '.find_for_omniauth' do let(:auth) { Fabricate(:identity, user: Fabricate(:user)) } it 'calls .find_or_create_by' do expect(described_class).to receive(:find_or_create_by).with(uid: auth.uid, provider: auth.provider) - described_class.find_for_oauth(auth) + described_class.find_for_omniauth(auth) end it 'returns an instance of Identity' do - expect(described_class.find_for_oauth(auth)).to be_instance_of Identity + expect(described_class.find_for_omniauth(auth)).to be_instance_of Identity end end end diff --git a/spec/models/media_attachment_spec.rb b/spec/models/media_attachment_spec.rb index cbd9a09c552b25..140d7af9b04d4f 100644 --- a/spec/models/media_attachment_spec.rb +++ b/spec/models/media_attachment_spec.rb @@ -150,6 +150,26 @@ end end + describe 'mp3 with large cover art' do + let(:media) { described_class.create(account: Fabricate(:account), file: attachment_fixture('boop.mp3')) } + + it 'detects it as an audio file' do + expect(media.type).to eq 'audio' + end + + it 'sets meta for the duration' do + expect(media.file.meta['original']['duration']).to be_within(0.05).of(0.235102) + end + + it 'extracts thumbnail' do + expect(media.thumbnail.present?).to be true + end + + it 'gives the file a random name' do + expect(media.file_file_name).to_not eq 'boop.mp3' + end + end + describe 'jpeg' do let(:media) { MediaAttachment.create(account: Fabricate(:account), file: attachment_fixture('attachment.jpg')) } diff --git a/spec/models/relationship_filter_spec.rb b/spec/models/relationship_filter_spec.rb index 7c0f37a06f299e..fccd42aaad0622 100644 --- a/spec/models/relationship_filter_spec.rb +++ b/spec/models/relationship_filter_spec.rb @@ -6,32 +6,60 @@ let(:account) { Fabricate(:account) } describe '#results' do - context 'when default params are used' do - let(:subject) do - RelationshipFilter.new(account, 'order' => 'active').results - end + let(:account_of_7_months) { Fabricate(:account_stat, statuses_count: 1, last_status_at: 7.months.ago).account } + let(:account_of_1_day) { Fabricate(:account_stat, statuses_count: 1, last_status_at: 1.day.ago).account } + let(:account_of_3_days) { Fabricate(:account_stat, statuses_count: 1, last_status_at: 3.days.ago).account } + let(:silent_account) { Fabricate(:account_stat, statuses_count: 0, last_status_at: nil).account } + + before do + account.follow!(account_of_7_months) + account.follow!(account_of_1_day) + account.follow!(account_of_3_days) + account.follow!(silent_account) + end - before do - add_following_account_with(last_status_at: 7.days.ago) - add_following_account_with(last_status_at: 1.day.ago) - add_following_account_with(last_status_at: 3.days.ago) + context 'when ordering by last activity' do + context 'when not filtering' do + subject do + described_class.new(account, 'order' => 'active').results + end + + it 'returns followings ordered by last activity' do + expect(subject).to eq [account_of_1_day, account_of_3_days, account_of_7_months, silent_account] + end end - it 'returns followings ordered by last activity' do - expected_result = account.following.eager_load(:account_stat).reorder(nil).by_recent_status + context 'when filtering for dormant accounts' do + subject do + described_class.new(account, 'order' => 'active', 'activity' => 'dormant').results + end - expect(subject).to eq expected_result + it 'returns dormant followings ordered by last activity' do + expect(subject).to eq [account_of_7_months, silent_account] + end end end - end - def add_following_account_with(last_status_at:) - following_account = Fabricate(:account) - Fabricate(:account_stat, account: following_account, - last_status_at: last_status_at, - statuses_count: 1, - following_count: 0, - followers_count: 0) - Fabricate(:follow, account: account, target_account: following_account).account + context 'when ordering by account creation' do + context 'when not filtering' do + subject do + described_class.new(account, 'order' => 'recent').results + end + + it 'returns followings ordered by last account creation' do + expect(subject).to eq [silent_account, account_of_3_days, account_of_1_day, account_of_7_months] + end + end + + context 'when filtering for dormant accounts' do + subject do + described_class.new(account, 'order' => 'recent', 'activity' => 'dormant').results + end + + it 'returns dormant followings ordered by last activity' do + expect(subject).to eq [silent_account, account_of_7_months] + end + end + end end end diff --git a/spec/models/report_spec.rb b/spec/models/report_spec.rb index 874be41328cb50..c485a4a3c9ad12 100644 --- a/spec/models/report_spec.rb +++ b/spec/models/report_spec.rb @@ -125,10 +125,17 @@ expect(report).to be_valid end - it 'is invalid if comment is longer than 1000 characters' do + let(:remote_account) { Fabricate(:account, domain: 'example.com', protocol: :activitypub, inbox_url: 'http://example.com/inbox') } + + it 'is invalid if comment is longer than 1000 characters only if reporter is local' do report = Fabricate.build(:report, comment: Faker::Lorem.characters(number: 1001)) - report.valid? + expect(report.valid?).to be false expect(report).to model_have_error_on_field(:comment) end + + it 'is valid if comment is longer than 1000 characters and reporter is not local' do + report = Fabricate.build(:report, account: remote_account, comment: Faker::Lorem.characters(number: 1001)) + expect(report.valid?).to be true + end end end diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index 1645ab59e04ebc..f3f0b8ad95e5f3 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -364,7 +364,10 @@ def fabricate let!(:access_token) { Fabricate(:access_token, resource_owner_id: user.id) } let!(:web_push_subscription) { Fabricate(:web_push_subscription, access_token: access_token) } + let(:redis_pipeline_stub) { instance_double(Redis::Namespace, publish: nil) } + before do + allow(redis).to receive(:pipelined).and_yield(redis_pipeline_stub) user.reset_password! end @@ -380,6 +383,10 @@ def fabricate expect(Doorkeeper::AccessToken.active_for(user).count).to eq 0 end + it 'revokes streaming access for all access tokens' do + expect(redis_pipeline_stub).to have_received(:publish).with("timeline:access_token:#{access_token.id}", Oj.dump(event: :kill)).once + end + it 'removes push subscriptions' do expect(Web::PushSubscription.where(user: user).or(Web::PushSubscription.where(access_token: access_token)).count).to eq 0 end diff --git a/spec/requests/api/v2/media_spec.rb b/spec/requests/api/v2/media_spec.rb new file mode 100644 index 00000000000000..67308cc42909c7 --- /dev/null +++ b/spec/requests/api/v2/media_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe 'Media API', paperclip_processing: true do + let(:user) { Fabricate(:user) } + let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: scopes) } + let(:scopes) { 'write' } + let(:headers) { { 'Authorization' => "Bearer #{token.token}" } } + + describe 'POST /api/v2/media' do + it 'returns http success' do + post '/api/v2/media', headers: headers, params: { file: fixture_file_upload('attachment-jpg.123456_abcd', 'image/jpeg') } + expect(File.exist?(user.account.media_attachments.first.file.path(:small))).to be true + expect(response).to have_http_status(202) + end + end +end diff --git a/spec/requests/content_security_policy_spec.rb b/spec/requests/content_security_policy_spec.rb new file mode 100644 index 00000000000000..8e1ba0b00a906c --- /dev/null +++ b/spec/requests/content_security_policy_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe 'Content-Security-Policy' do + it 'sets the expected CSP headers' do + allow(SecureRandom).to receive(:base64).with(16).and_return('ZbA+JmE7+bK8F5qvADZHuQ==') + + get '/' + expect(response.headers['Content-Security-Policy'].split(';').map(&:strip)).to contain_exactly( + "base-uri 'none'", + "default-src 'none'", + "frame-ancestors 'none'", + "font-src 'self' https://cb6e6126.ngrok.io", + "img-src 'self' https: data: blob: https://cb6e6126.ngrok.io", + "style-src 'self' https://cb6e6126.ngrok.io 'nonce-ZbA+JmE7+bK8F5qvADZHuQ=='", + "media-src 'self' https: data: https://cb6e6126.ngrok.io", + "frame-src 'self' https:", + "manifest-src 'self' https://cb6e6126.ngrok.io", + "form-action 'self'", + "child-src 'self' blob: https://cb6e6126.ngrok.io", + "worker-src 'self' blob: https://cb6e6126.ngrok.io", + "connect-src 'self' data: blob: https://cb6e6126.ngrok.io https://cb6e6126.ngrok.io ws://localhost:4000", + "script-src 'self' https://cb6e6126.ngrok.io" + ) + end +end diff --git a/spec/requests/disabled_oauth_endpoints_spec.rb b/spec/requests/disabled_oauth_endpoints_spec.rb new file mode 100644 index 00000000000000..7c2c09f3804bf3 --- /dev/null +++ b/spec/requests/disabled_oauth_endpoints_spec.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe 'Disabled OAuth routes' do + # These routes are disabled via the doorkeeper configuration for + # `admin_authenticator`, as these routes should only be accessible by server + # administrators. For now, these routes are not properly designed and + # integrated into Mastodon, so we're disabling them completely + describe 'GET /oauth/applications' do + it 'returns 403 forbidden' do + get oauth_applications_path + + expect(response).to have_http_status(403) + end + end + + describe 'POST /oauth/applications' do + it 'returns 403 forbidden' do + post oauth_applications_path + + expect(response).to have_http_status(403) + end + end + + describe 'GET /oauth/applications/new' do + it 'returns 403 forbidden' do + get new_oauth_application_path + + expect(response).to have_http_status(403) + end + end + + describe 'GET /oauth/applications/:id' do + let(:application) { Fabricate(:application, scopes: 'read') } + + it 'returns 403 forbidden' do + get oauth_application_path(application) + + expect(response).to have_http_status(403) + end + end + + describe 'PATCH /oauth/applications/:id' do + let(:application) { Fabricate(:application, scopes: 'read') } + + it 'returns 403 forbidden' do + patch oauth_application_path(application) + + expect(response).to have_http_status(403) + end + end + + describe 'PUT /oauth/applications/:id' do + let(:application) { Fabricate(:application, scopes: 'read') } + + it 'returns 403 forbidden' do + put oauth_application_path(application) + + expect(response).to have_http_status(403) + end + end + + describe 'DELETE /oauth/applications/:id' do + let(:application) { Fabricate(:application, scopes: 'read') } + + it 'returns 403 forbidden' do + delete oauth_application_path(application) + + expect(response).to have_http_status(403) + end + end + + describe 'GET /oauth/applications/:id/edit' do + let(:application) { Fabricate(:application, scopes: 'read') } + + it 'returns 403 forbidden' do + get edit_oauth_application_path(application) + + expect(response).to have_http_status(403) + end + end +end diff --git a/spec/requests/omniauth_callbacks_spec.rb b/spec/requests/omniauth_callbacks_spec.rb new file mode 100644 index 00000000000000..095535e48598e0 --- /dev/null +++ b/spec/requests/omniauth_callbacks_spec.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe 'OmniAuth callbacks' do + shared_examples 'omniauth provider callbacks' do |provider| + subject { post send :"user_#{provider}_omniauth_callback_path" } + + context 'with full information in response' do + before do + mock_omniauth(provider, { + provider: provider.to_s, + uid: '123', + info: { + verified: 'true', + email: 'user@host.example', + }, + }) + end + + context 'without a matching user' do + it 'creates a user and an identity and redirects to root path' do + expect { subject } + .to change(User, :count) + .by(1) + .and change(Identity, :count) + .by(1) + .and change(LoginActivity, :count) + .by(1) + + expect(User.last.email).to eq('user@host.example') + expect(Identity.find_by(user: User.last).uid).to eq('123') + expect(response).to redirect_to(root_path) + end + end + + context 'with a matching user and no matching identity' do + before do + Fabricate(:user, email: 'user@host.example') + end + + context 'when ALLOW_UNSAFE_AUTH_PROVIDER_REATTACH is set to true' do + around do |example| + ClimateControl.modify ALLOW_UNSAFE_AUTH_PROVIDER_REATTACH: 'true' do + example.run + end + end + + it 'matches the existing user, creates an identity, and redirects to root path' do + expect { subject } + .to not_change(User, :count) + .and change(Identity, :count) + .by(1) + .and change(LoginActivity, :count) + .by(1) + + expect(Identity.find_by(user: User.last).uid).to eq('123') + expect(response).to redirect_to(root_path) + end + end + + context 'when ALLOW_UNSAFE_AUTH_PROVIDER_REATTACH is not set to true' do + it 'does not match the existing user or create an identity, and redirects to login page' do + expect { subject } + .to not_change(User, :count) + .and not_change(Identity, :count) + .and not_change(LoginActivity, :count) + + expect(response).to redirect_to(new_user_session_url) + end + end + end + + context 'with a matching user and a matching identity' do + before do + user = Fabricate(:user, email: 'user@host.example') + Fabricate(:identity, user: user, uid: '123', provider: provider) + end + + it 'matches the existing records and redirects to root path' do + expect { subject } + .to not_change(User, :count) + .and not_change(Identity, :count) + .and change(LoginActivity, :count) + .by(1) + + expect(response).to redirect_to(root_path) + end + end + end + + context 'with a response missing email address' do + before do + mock_omniauth(provider, { + provider: provider.to_s, + uid: '123', + info: { + verified: 'true', + }, + }) + end + + it 'redirects to the auth setup page' do + expect { subject } + .to change(User, :count) + .by(1) + .and change(Identity, :count) + .by(1) + .and change(LoginActivity, :count) + .by(1) + + expect(response).to redirect_to(auth_setup_path(missing_email: '1')) + end + end + + context 'when a user cannot be built' do + before do + allow(User).to receive(:find_for_omniauth).and_return(User.new) + end + + it 'redirects to the new user signup page' do + expect { subject } + .to not_change(User, :count) + .and not_change(Identity, :count) + .and not_change(LoginActivity, :count) + + expect(response).to redirect_to(new_user_registration_url) + end + end + end + + describe '#openid_connect', if: ENV['OIDC_ENABLED'] == 'true' && ENV['OIDC_SCOPE'].present? do + include_examples 'omniauth provider callbacks', :openid_connect + end + + describe '#cas', if: ENV['CAS_ENABLED'] == 'true' do + include_examples 'omniauth provider callbacks', :cas + end + + describe '#saml', if: ENV['SAML_ENABLED'] == 'true' do + include_examples 'omniauth provider callbacks', :saml + end +end diff --git a/spec/services/activitypub/fetch_featured_collection_service_spec.rb b/spec/services/activitypub/fetch_featured_collection_service_spec.rb index f552b9dc07d669..e7351088ba5ae5 100644 --- a/spec/services/activitypub/fetch_featured_collection_service_spec.rb +++ b/spec/services/activitypub/fetch_featured_collection_service_spec.rb @@ -60,10 +60,10 @@ shared_examples 'sets pinned posts' do before do - stub_request(:get, 'https://example.com/account/pinned/1').to_return(status: 200, body: Oj.dump(status_json_1)) - stub_request(:get, 'https://example.com/account/pinned/2').to_return(status: 200, body: Oj.dump(status_json_2)) + stub_request(:get, 'https://example.com/account/pinned/1').to_return(status: 200, body: Oj.dump(status_json_1), headers: { 'Content-Type': 'application/activity+json' }) + stub_request(:get, 'https://example.com/account/pinned/2').to_return(status: 200, body: Oj.dump(status_json_2), headers: { 'Content-Type': 'application/activity+json' }) stub_request(:get, 'https://example.com/account/pinned/3').to_return(status: 404) - stub_request(:get, 'https://example.com/account/pinned/4').to_return(status: 200, body: Oj.dump(status_json_4)) + stub_request(:get, 'https://example.com/account/pinned/4').to_return(status: 200, body: Oj.dump(status_json_4), headers: { 'Content-Type': 'application/activity+json' }) subject.call(actor) end @@ -76,7 +76,7 @@ describe '#call' do context 'when the endpoint is a Collection' do before do - stub_request(:get, actor.featured_collection_url).to_return(status: 200, body: Oj.dump(payload)) + stub_request(:get, actor.featured_collection_url).to_return(status: 200, body: Oj.dump(payload), headers: { 'Content-Type': 'application/activity+json' }) end it_behaves_like 'sets pinned posts' @@ -93,7 +93,7 @@ end before do - stub_request(:get, actor.featured_collection_url).to_return(status: 200, body: Oj.dump(payload)) + stub_request(:get, actor.featured_collection_url).to_return(status: 200, body: Oj.dump(payload), headers: { 'Content-Type': 'application/activity+json' }) end it_behaves_like 'sets pinned posts' @@ -114,7 +114,7 @@ end before do - stub_request(:get, actor.featured_collection_url).to_return(status: 200, body: Oj.dump(payload)) + stub_request(:get, actor.featured_collection_url).to_return(status: 200, body: Oj.dump(payload), headers: { 'Content-Type': 'application/activity+json' }) end it_behaves_like 'sets pinned posts' diff --git a/spec/services/activitypub/fetch_remote_account_service_spec.rb b/spec/services/activitypub/fetch_remote_account_service_spec.rb index aa13f0a9b79367..1a649c9540c831 100644 --- a/spec/services/activitypub/fetch_remote_account_service_spec.rb +++ b/spec/services/activitypub/fetch_remote_account_service_spec.rb @@ -16,7 +16,7 @@ end describe '#call' do - let(:account) { subject.call('https://example.com/alice', id: true) } + let(:account) { subject.call('https://example.com/alice') } shared_examples 'sets profile data' do it 'returns an account' do @@ -42,7 +42,7 @@ before do actor[:inbox] = nil - stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor)) + stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor), headers: { 'Content-Type': 'application/activity+json' }) stub_request(:get, 'https://example.com/.well-known/webfinger?resource=acct:alice@example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' }) end @@ -65,7 +65,7 @@ let!(:webfinger) { { subject: 'acct:alice@example.com', links: [{ rel: 'self', href: 'https://example.com/alice' }] } } before do - stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor)) + stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor), headers: { 'Content-Type': 'application/activity+json' }) stub_request(:get, 'https://example.com/.well-known/webfinger?resource=acct:alice@example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' }) end @@ -91,7 +91,7 @@ let!(:webfinger) { { subject: 'acct:alice@iscool.af', links: [{ rel: 'self', href: 'https://example.com/alice' }] } } before do - stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor)) + stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor), headers: { 'Content-Type': 'application/activity+json' }) stub_request(:get, 'https://example.com/.well-known/webfinger?resource=acct:alice@example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' }) stub_request(:get, 'https://iscool.af/.well-known/webfinger?resource=acct:alice@iscool.af').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' }) end diff --git a/spec/services/activitypub/fetch_remote_status_service_spec.rb b/spec/services/activitypub/fetch_remote_status_service_spec.rb index 7359ca0b43b6a0..a81dcad81852bf 100644 --- a/spec/services/activitypub/fetch_remote_status_service_spec.rb +++ b/spec/services/activitypub/fetch_remote_status_service_spec.rb @@ -223,4 +223,98 @@ end end end + + context 'statuses referencing other statuses' do + before do + stub_const 'ActivityPub::FetchRemoteStatusService::DISCOVERIES_PER_REQUEST', 5 + end + + context 'using inReplyTo' do + let(:object) do + { + '@context': 'https://www.w3.org/ns/activitystreams', + id: "https://foo.bar/@foo/1", + type: 'Note', + content: 'Lorem ipsum', + inReplyTo: 'https://foo.bar/@foo/2', + attributedTo: ActivityPub::TagManager.instance.uri_for(sender), + } + end + + before do + 8.times do |i| + status_json = { + '@context': 'https://www.w3.org/ns/activitystreams', + id: "https://foo.bar/@foo/#{i}", + type: 'Note', + content: 'Lorem ipsum', + inReplyTo: "https://foo.bar/@foo/#{i + 1}", + attributedTo: ActivityPub::TagManager.instance.uri_for(sender), + to: 'as:Public', + }.with_indifferent_access + stub_request(:get, "https://foo.bar/@foo/#{i}").to_return(status: 200, body: status_json.to_json, headers: { 'Content-Type': 'application/activity+json' }) + end + end + + it 'creates at least some statuses' do + expect { subject.call(object[:id], prefetched_body: Oj.dump(object)) }.to change { sender.statuses.count }.by_at_least(2) + end + + it 'creates no more account than the limit allows' do + expect { subject.call(object[:id], prefetched_body: Oj.dump(object)) }.to change { sender.statuses.count }.by_at_most(5) + end + end + + context 'using replies' do + let(:object) do + { + '@context': 'https://www.w3.org/ns/activitystreams', + id: "https://foo.bar/@foo/1", + type: 'Note', + content: 'Lorem ipsum', + replies: { + type: 'Collection', + id: 'https://foo.bar/@foo/1/replies', + first: { + type: 'CollectionPage', + partOf: 'https://foo.bar/@foo/1/replies', + items: ['https://foo.bar/@foo/2'], + }, + }, + attributedTo: ActivityPub::TagManager.instance.uri_for(sender), + } + end + + before do + 8.times do |i| + status_json = { + '@context': 'https://www.w3.org/ns/activitystreams', + id: "https://foo.bar/@foo/#{i}", + type: 'Note', + content: 'Lorem ipsum', + replies: { + type: 'Collection', + id: "https://foo.bar/@foo/#{i}/replies", + first: { + type: 'CollectionPage', + partOf: "https://foo.bar/@foo/#{i}/replies", + items: ["https://foo.bar/@foo/#{i+1}"], + }, + }, + attributedTo: ActivityPub::TagManager.instance.uri_for(sender), + to: 'as:Public', + }.with_indifferent_access + stub_request(:get, "https://foo.bar/@foo/#{i}").to_return(status: 200, body: status_json.to_json, headers: { 'Content-Type': 'application/activity+json' }) + end + end + + it 'creates at least some statuses' do + expect { subject.call(object[:id], prefetched_body: Oj.dump(object)) }.to change { sender.statuses.count }.by_at_least(2) + end + + it 'creates no more account than the limit allows' do + expect { subject.call(object[:id], prefetched_body: Oj.dump(object)) }.to change { sender.statuses.count }.by_at_most(5) + end + end + end end diff --git a/spec/services/activitypub/fetch_replies_service_spec.rb b/spec/services/activitypub/fetch_replies_service_spec.rb index fe49b18c195db8..ec40abd5b25ea9 100644 --- a/spec/services/activitypub/fetch_replies_service_spec.rb +++ b/spec/services/activitypub/fetch_replies_service_spec.rb @@ -41,7 +41,7 @@ context 'when passing the URL to the collection' do before do - stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload)) + stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload), headers: { 'Content-Type': 'application/activity+json' }) end it 'spawns workers for up to 5 replies on the same server' do @@ -70,7 +70,7 @@ context 'when passing the URL to the collection' do before do - stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload)) + stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload), headers: { 'Content-Type': 'application/activity+json' }) end it 'spawns workers for up to 5 replies on the same server' do @@ -103,7 +103,7 @@ context 'when passing the URL to the collection' do before do - stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload)) + stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload), headers: { 'Content-Type': 'application/activity+json' }) end it 'spawns workers for up to 5 replies on the same server' do diff --git a/spec/services/activitypub/process_account_service_spec.rb b/spec/services/activitypub/process_account_service_spec.rb index 7728b9ba829e22..2b20d17b1bc984 100644 --- a/spec/services/activitypub/process_account_service_spec.rb +++ b/spec/services/activitypub/process_account_service_spec.rb @@ -109,4 +109,98 @@ end end end + + context 'discovering many subdomains in a short timeframe' do + before do + stub_const 'ActivityPub::ProcessAccountService::SUBDOMAINS_RATELIMIT', 5 + end + + let(:subject) do + 8.times do |i| + domain = "test#{i}.testdomain.com" + json = { + id: "https://#{domain}/users/1", + type: 'Actor', + inbox: "https://#{domain}/inbox", + }.with_indifferent_access + described_class.new.call('alice', domain, json) + end + end + + it 'creates at least some accounts' do + expect { subject }.to change { Account.remote.count }.by_at_least(2) + end + + it 'creates no more account than the limit allows' do + expect { subject }.to change { Account.remote.count }.by_at_most(5) + end + end + + context 'accounts referencing other accounts' do + before do + stub_const 'ActivityPub::ProcessAccountService::DISCOVERIES_PER_REQUEST', 5 + end + + let(:payload) do + { + '@context': ['https://www.w3.org/ns/activitystreams'], + id: 'https://foo.test/users/1', + type: 'Person', + inbox: 'https://foo.test/inbox', + featured: 'https://foo.test/users/1/featured', + preferredUsername: 'user1', + }.with_indifferent_access + end + + before do + 8.times do |i| + actor_json = { + '@context': ['https://www.w3.org/ns/activitystreams'], + id: "https://foo.test/users/#{i}", + type: 'Person', + inbox: 'https://foo.test/inbox', + featured: "https://foo.test/users/#{i}/featured", + preferredUsername: "user#{i}", + }.with_indifferent_access + status_json = { + '@context': ['https://www.w3.org/ns/activitystreams'], + id: "https://foo.test/users/#{i}/status", + attributedTo: "https://foo.test/users/#{i}", + type: 'Note', + content: "@user#{i + 1} test", + tag: [ + { + type: 'Mention', + href: "https://foo.test/users/#{i + 1}", + name: "@user#{i + 1 }", + } + ], + to: [ 'as:Public', "https://foo.test/users/#{i + 1}" ] + }.with_indifferent_access + featured_json = { + '@context': ['https://www.w3.org/ns/activitystreams'], + id: "https://foo.test/users/#{i}/featured", + type: 'OrderedCollection', + totelItems: 1, + orderedItems: [status_json], + }.with_indifferent_access + webfinger = { + subject: "acct:user#{i}@foo.test", + links: [{ rel: 'self', href: "https://foo.test/users/#{i}" }], + }.with_indifferent_access + stub_request(:get, "https://foo.test/users/#{i}").to_return(status: 200, body: actor_json.to_json, headers: { 'Content-Type': 'application/activity+json' }) + stub_request(:get, "https://foo.test/users/#{i}/featured").to_return(status: 200, body: featured_json.to_json, headers: { 'Content-Type': 'application/activity+json' }) + stub_request(:get, "https://foo.test/users/#{i}/status").to_return(status: 200, body: status_json.to_json, headers: { 'Content-Type': 'application/activity+json' }) + stub_request(:get, "https://foo.test/.well-known/webfinger?resource=acct:user#{i}@foo.test").to_return(body: webfinger.to_json, headers: { 'Content-Type': 'application/jrd+json' }) + end + end + + it 'creates at least some accounts' do + expect { subject.call('user1', 'foo.test', payload) }.to change { Account.remote.count }.by_at_least(2) + end + + it 'creates no more account than the limit allows' do + expect { subject.call('user1', 'foo.test', payload) }.to change { Account.remote.count }.by_at_most(5) + end + end end diff --git a/spec/services/activitypub/process_status_update_service_spec.rb b/spec/services/activitypub/process_status_update_service_spec.rb index 481572742b3bd5..750369d57fbfcf 100644 --- a/spec/services/activitypub/process_status_update_service_spec.rb +++ b/spec/services/activitypub/process_status_update_service_spec.rb @@ -331,7 +331,7 @@ def poll_option_json(name, votes) context 'originally without media attachments' do before do - allow(RedownloadMediaWorker).to receive(:perform_async) + stub_request(:get, 'https://example.com/foo.png').to_return(body: attachment_fixture('emojo.png')) subject.call(status, json) end @@ -355,8 +355,8 @@ def poll_option_json(name, votes) expect(media_attachment.remote_url).to eq 'https://example.com/foo.png' end - it 'queues download of media attachments' do - expect(RedownloadMediaWorker).to have_received(:perform_async) + it 'fetches the attachment' do + expect(a_request(:get, 'https://example.com/foo.png')).to have_been_made end it 'records media change in edit' do diff --git a/spec/services/activitypub/synchronize_followers_service_spec.rb b/spec/services/activitypub/synchronize_followers_service_spec.rb index 75dcf204b79517..7b4a5f8ffe2393 100644 --- a/spec/services/activitypub/synchronize_followers_service_spec.rb +++ b/spec/services/activitypub/synchronize_followers_service_spec.rb @@ -58,7 +58,7 @@ describe '#call' do context 'when the endpoint is a Collection of actor URIs' do before do - stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload)) + stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload), headers: { 'Content-Type': 'application/activity+json' }) end it_behaves_like 'synchronizes followers' @@ -75,7 +75,7 @@ end before do - stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload)) + stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload), headers: { 'Content-Type': 'application/activity+json' }) end it_behaves_like 'synchronizes followers' @@ -96,7 +96,7 @@ end before do - stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload)) + stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload), headers: { 'Content-Type': 'application/activity+json' }) end it_behaves_like 'synchronizes followers' diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb index 4914c275326eb9..7a758f910fbf93 100644 --- a/spec/services/fetch_link_card_service_spec.rb +++ b/spec/services/fetch_link_card_service_spec.rb @@ -10,6 +10,7 @@ stub_request(:get, 'http://example.com/koi8-r').to_return(request_fixture('koi8-r.txt')) stub_request(:get, 'http://example.com/日本語').to_return(request_fixture('sjis.txt')) stub_request(:get, 'https://github.com/qbi/WannaCry').to_return(status: 404) + stub_request(:get, 'http://example.com/test?data=file.gpx%5E1').to_return(status: 200) stub_request(:get, 'http://example.com/test-').to_return(request_fixture('idn.txt')) stub_request(:get, 'http://example.com/windows-1251').to_return(request_fixture('windows-1251.txt')) @@ -85,6 +86,15 @@ expect(a_request(:get, 'http://example.com/sjis')).to_not have_been_made end end + + context do + let(:status) { Fabricate(:status, text: 'test http://example.com/test?data=file.gpx^1') } + + it 'does fetch URLs with a caret in search params' do + expect(a_request(:get, 'http://example.com/test?data=file.gpx')).to_not have_been_made + expect(a_request(:get, 'http://example.com/test?data=file.gpx%5E1')).to have_been_made.once + end + end end context 'in a remote status' do diff --git a/spec/services/fetch_remote_status_service_spec.rb b/spec/services/fetch_remote_status_service_spec.rb index fe5f1aed19c7de..4f6ad6496720e0 100644 --- a/spec/services/fetch_remote_status_service_spec.rb +++ b/spec/services/fetch_remote_status_service_spec.rb @@ -15,7 +15,7 @@ end context 'protocol is :activitypub' do - subject { described_class.new.call(note[:id], prefetched_body) } + subject { described_class.new.call(note[:id], prefetched_body: prefetched_body) } let(:prefetched_body) { Oj.dump(note) } before do diff --git a/spec/services/fetch_resource_service_spec.rb b/spec/services/fetch_resource_service_spec.rb index ded05ffbc70254..1697ad76898743 100644 --- a/spec/services/fetch_resource_service_spec.rb +++ b/spec/services/fetch_resource_service_spec.rb @@ -54,7 +54,7 @@ let(:json) do { - id: 1, + id: 'http://example.com/foo', '@context': ActivityPub::TagManager::CONTEXT, type: 'Note', }.to_json @@ -79,14 +79,14 @@ let(:content_type) { 'application/activity+json; charset=utf-8' } let(:body) { json } - it { is_expected.to eq [1, { prefetched_body: body, id: true }] } + it { is_expected.to eq ['http://example.com/foo', { prefetched_body: body }] } end context 'when content type is ld+json with profile' do let(:content_type) { 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"' } let(:body) { json } - it { is_expected.to eq [1, { prefetched_body: body, id: true }] } + it { is_expected.to eq ['http://example.com/foo', { prefetched_body: body }] } end before do @@ -97,14 +97,14 @@ context 'when link header is present' do let(:headers) { { 'Link' => '; rel="alternate"; type="application/activity+json"', } } - it { is_expected.to eq [1, { prefetched_body: json, id: true }] } + it { is_expected.to eq ['http://example.com/foo', { prefetched_body: json }] } end context 'when content type is text/html' do let(:content_type) { 'text/html' } let(:body) { '' } - it { is_expected.to eq [1, { prefetched_body: json, id: true }] } + it { is_expected.to eq ['http://example.com/foo', { prefetched_body: json }] } end end end diff --git a/spec/services/report_service_spec.rb b/spec/services/report_service_spec.rb index ea68b3344dc6d3..9f69bdcf89b9d4 100644 --- a/spec/services/report_service_spec.rb +++ b/spec/services/report_service_spec.rb @@ -4,6 +4,14 @@ subject { described_class.new } let(:source_account) { Fabricate(:account) } + let(:target_account) { Fabricate(:account) } + + context 'with a local account' do + it 'has a uri' do + report = subject.call(source_account, target_account) + expect(report.uri).to_not be_nil + end + end context 'for a remote account' do let(:remote_account) { Fabricate(:account, domain: 'example.com', protocol: :activitypub, inbox_url: 'http://example.com/inbox') } diff --git a/spec/services/resolve_url_service_spec.rb b/spec/services/resolve_url_service_spec.rb index b3e3defbff2ccd..5b6b7b9a66a0f2 100644 --- a/spec/services/resolve_url_service_spec.rb +++ b/spec/services/resolve_url_service_spec.rb @@ -139,6 +139,7 @@ stub_request(:get, url).to_return(status: 302, headers: { 'Location' => status_url }) body = ActiveModelSerializers::SerializableResource.new(status, serializer: ActivityPub::NoteSerializer, adapter: ActivityPub::Adapter).to_json stub_request(:get, status_url).to_return(body: body, headers: { 'Content-Type' => 'application/activity+json' }) + stub_request(:get, uri).to_return(body: body, headers: { 'Content-Type' => 'application/activity+json' }) end it 'returns status by url' do diff --git a/spec/services/suspend_account_service_spec.rb b/spec/services/suspend_account_service_spec.rb index 5d45e4ffdba303..126b13986b6ca0 100644 --- a/spec/services/suspend_account_service_spec.rb +++ b/spec/services/suspend_account_service_spec.rb @@ -13,6 +13,8 @@ local_follower.follow!(account) list.accounts << account + + account.suspend! end it "unmerges from local followers' feeds" do @@ -21,8 +23,8 @@ expect(FeedManager.instance).to have_received(:unmerge_from_list).with(account, list) end - it 'marks account as suspended' do - expect { subject }.to change { account.suspended? }.from(false).to(true) + it 'does not change the “suspended” flag' do + expect { subject }.to_not change { account.suspended? } end end diff --git a/spec/services/unsuspend_account_service_spec.rb b/spec/services/unsuspend_account_service_spec.rb index 3ac4cc085e32c4..987eb09e234d8c 100644 --- a/spec/services/unsuspend_account_service_spec.rb +++ b/spec/services/unsuspend_account_service_spec.rb @@ -14,7 +14,7 @@ local_follower.follow!(account) list.accounts << account - account.suspend!(origin: :local) + account.unsuspend! end end @@ -30,8 +30,8 @@ def match_update_actor_request(req, account) stub_request(:post, 'https://bob.com/inbox').to_return(status: 201) end - it 'marks account as unsuspended' do - expect { subject }.to change { account.suspended? }.from(true).to(false) + it 'does not change the “suspended” flag' do + expect { subject }.to_not change { account.suspended? } end include_examples 'common behavior' do @@ -83,8 +83,8 @@ def match_update_actor_request(req, account) expect(FeedManager.instance).to have_received(:merge_into_list).with(account, list) end - it 'marks account as unsuspended' do - expect { subject }.to change { account.suspended? }.from(true).to(false) + it 'does not change the “suspended” flag' do + expect { subject }.to_not change { account.suspended? } end end @@ -107,8 +107,8 @@ def match_update_actor_request(req, account) expect(FeedManager.instance).to_not have_received(:merge_into_list).with(account, list) end - it 'does not mark the account as unsuspended' do - expect { subject }.not_to change { account.suspended? } + it 'marks account as suspended' do + expect { subject }.to change { account.suspended? }.from(false).to(true) end end diff --git a/spec/support/omniauth_mocks.rb b/spec/support/omniauth_mocks.rb new file mode 100644 index 00000000000000..9883adec7a6cad --- /dev/null +++ b/spec/support/omniauth_mocks.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +OmniAuth.config.test_mode = true + +def mock_omniauth(provider, data) + OmniAuth.config.mock_auth[provider] = OmniAuth::AuthHash.new(data) +end diff --git a/spec/workers/activitypub/fetch_replies_worker_spec.rb b/spec/workers/activitypub/fetch_replies_worker_spec.rb index 91ef3c4b928fd7..64cfcd8cbf74d4 100644 --- a/spec/workers/activitypub/fetch_replies_worker_spec.rb +++ b/spec/workers/activitypub/fetch_replies_worker_spec.rb @@ -21,7 +21,7 @@ describe 'perform' do it 'performs a request if the collection URI is from the same host' do - stub_request(:get, 'https://example.com/statuses_replies/1').to_return(status: 200, body: json) + stub_request(:get, 'https://example.com/statuses_replies/1').to_return(status: 200, body: json, headers: { 'Content-Type': 'application/activity+json' }) subject.perform(status.id, 'https://example.com/statuses_replies/1') expect(a_request(:get, 'https://example.com/statuses_replies/1')).to have_been_made.once end diff --git a/spec/workers/scheduler/user_cleanup_scheduler_spec.rb b/spec/workers/scheduler/user_cleanup_scheduler_spec.rb new file mode 100644 index 00000000000000..da99f10f97f326 --- /dev/null +++ b/spec/workers/scheduler/user_cleanup_scheduler_spec.rb @@ -0,0 +1,39 @@ +require 'rails_helper' + +describe Scheduler::UserCleanupScheduler do + subject { described_class.new } + + let!(:new_unconfirmed_user) { Fabricate(:user) } + let!(:old_unconfirmed_user) { Fabricate(:user) } + let!(:confirmed_user) { Fabricate(:user) } + let!(:moderation_note) { Fabricate(:account_moderation_note, account: Fabricate(:account), target_account: old_unconfirmed_user.account) } + + describe '#perform' do + before do + # Need to update the already-existing users because their initialization overrides confirmation_sent_at + new_unconfirmed_user.update!(confirmed_at: nil, confirmation_sent_at: Time.now.utc) + old_unconfirmed_user.update!(confirmed_at: nil, confirmation_sent_at: 1.week.ago) + confirmed_user.update!(confirmed_at: 1.day.ago) + end + + it 'deletes the old unconfirmed user' do + expect { subject.perform }.to change { User.exists?(old_unconfirmed_user.id) }.from(true).to(false) + end + + it "deletes the old unconfirmed user's account" do + expect { subject.perform }.to change { Account.exists?(old_unconfirmed_user.account_id) }.from(true).to(false) + end + + it 'does not delete the new unconfirmed user or their account' do + subject.perform + expect(User.exists?(new_unconfirmed_user.id)).to be true + expect(Account.exists?(new_unconfirmed_user.account_id)).to be true + end + + it 'does not delete the confirmed user or their account' do + subject.perform + expect(User.exists?(confirmed_user.id)).to be true + expect(Account.exists?(confirmed_user.account_id)).to be true + end + end +end diff --git a/streaming/index.js b/streaming/index.js index 6935c47645cc24..ab3a16030f037a 100644 --- a/streaming/index.js +++ b/streaming/index.js @@ -91,18 +91,31 @@ const redisUrlToClient = async (defaultConfig, redisUrl) => { const numWorkers = +process.env.STREAMING_CLUSTER_NUM || (env === 'development' ? 1 : Math.max(os.cpus().length - 1, 1)); /** + * Attempts to safely parse a string as JSON, used when both receiving a message + * from redis and when receiving a message from a client over a websocket + * connection, this is why it accepts a `req` argument. * @param {string} json - * @param {any} req - * @return {Object.|null} + * @param {any?} req + * @returns {Object.|null} */ const parseJSON = (json, req) => { try { return JSON.parse(json); } catch (err) { - if (req.accountId) { - log.warn(req.requestId, `Error parsing message from user ${req.accountId}: ${err}`); + /* FIXME: This logging isn't great, and should probably be done at the + * call-site of parseJSON, not in the method, but this would require changing + * the signature of parseJSON to return something akin to a Result type: + * [Error|null, null|Object { const redisPrefix = redisNamespace ? `${redisNamespace}:` : ''; /** - * @type {Object.>} + * @type {Object.): void>>} */ const subs = {}; @@ -208,12 +221,21 @@ const startWorker = async (workerId) => { return; } - callbacks.forEach(callback => callback(message)); + const json = parseJSON(message, null); + if (!json) return; + + callbacks.forEach(callback => callback(json)); }; + /** + * @callback SubscriptionListener + * @param {ReturnType} json of the message + * @returns void + */ + /** * @param {string} channel - * @param {function(string): void} callback + * @param {SubscriptionListener} callback */ const subscribe = (channel, callback) => { log.silly(`Adding listener for ${channel}`); @@ -230,6 +252,7 @@ const startWorker = async (workerId) => { /** * @param {string} channel + * @param {SubscriptionListener} callback */ const unsubscribe = (channel, callback) => { log.silly(`Removing listener for ${channel}`); @@ -379,7 +402,7 @@ const startWorker = async (workerId) => { /** * @param {any} req - * @return {string} + * @returns {string|undefined} */ const channelNameFromPath = req => { const { path, query } = req; @@ -488,15 +511,11 @@ const startWorker = async (workerId) => { /** * @param {any} req * @param {SystemMessageHandlers} eventHandlers - * @return {function(string): void} + * @returns {function(object): void} */ const createSystemMessageListener = (req, eventHandlers) => { return message => { - const json = parseJSON(message, req); - - if (!json) return; - - const { event } = json; + const { event } = message; log.silly(req.requestId, `System message for ${req.accountId}: ${event}`); @@ -605,21 +624,18 @@ const startWorker = async (workerId) => { * @param {string[]} ids * @param {any} req * @param {function(string, string): void} output - * @param {function(string[], function(string): void): void} attachCloseHandler + * @param {undefined | function(string[], SubscriptionListener): void} attachCloseHandler * @param {boolean=} needsFiltering - * @return {function(string): void} + * @returns {SubscriptionListener} */ const streamFrom = (ids, req, output, attachCloseHandler, needsFiltering = false) => { const accountId = req.accountId || req.remoteAddress; log.verbose(req.requestId, `Starting stream from ${ids.join(', ')} for ${accountId}`); + // Currently message is of type string, soon it'll be Record const listener = message => { - const json = parseJSON(message, req); - - if (!json) return; - - const { event, payload, queued_at } = json; + const { event, payload, queued_at } = message; const transmit = () => { const now = new Date().getTime(); @@ -693,7 +709,7 @@ const startWorker = async (workerId) => { subscribe(`${redisPrefix}${id}`, listener); }); - if (attachCloseHandler) { + if (typeof attachCloseHandler === 'function') { attachCloseHandler(ids.map(id => `${redisPrefix}${id}`), listener); } @@ -730,12 +746,13 @@ const startWorker = async (workerId) => { /** * @param {any} req * @param {function(): void} [closeHandler] - * @return {function(string[]): void} + * @returns {function(string[], SubscriptionListener): void} */ - const streamHttpEnd = (req, closeHandler = undefined) => (ids) => { + + const streamHttpEnd = (req, closeHandler = undefined) => (ids, listener) => { req.on('close', () => { ids.forEach(id => { - unsubscribe(id); + unsubscribe(id, listener); }); if (closeHandler) { @@ -946,7 +963,7 @@ const startWorker = async (workerId) => { * @typedef WebSocketSession * @property {any} socket * @property {any} request - * @property {Object.} subscriptions + * @property {Object.} subscriptions */ /** @@ -1078,8 +1095,15 @@ const startWorker = async (workerId) => { ws.on('close', onEnd); ws.on('error', onEnd); - ws.on('message', data => { - const json = parseJSON(data, session.request); + ws.on('message', (data, isBinary) => { + if (isBinary) { + log.warn('socket', 'Received binary data, closing connection'); + ws.close(1003, 'The mastodon streaming server does not support binary messages'); + return; + } + const message = data.toString('utf8'); + + const json = parseJSON(message, session.request); if (!json) return;