From 23d279e03ee1f7a1285614754738711359bc4b81 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 1 Aug 2019 17:28:00 +0300 Subject: [PATCH 001/138] [#1149] Replaced RetryQueue with oban-based retries. --- config/config.exs | 17 +- config/test.exs | 4 + docs/config.md | 7 - lib/pleroma/application.ex | 4 +- lib/pleroma/web/activity_pub/publisher.ex | 16 +- lib/pleroma/web/federator/federator.ex | 14 - lib/pleroma/web/federator/publisher.ex | 22 +- lib/pleroma/web/federator/retry_queue.ex | 239 ------------------ lib/pleroma/web/salmon/salmon.ex | 11 +- lib/pleroma/workers/publisher.ex | 14 + mix.exs | 1 + mix.lock | 1 + .../20190730055101_add_oban_jobs_table.exs | 6 + test/user_test.exs | 15 +- test/web/activity_pub/publisher_test.exs | 2 +- test/web/federator_test.exs | 78 +++--- test/web/retry_queue_test.exs | 48 ---- test/web/salmon/salmon_test.exs | 2 +- 18 files changed, 106 insertions(+), 395 deletions(-) delete mode 100644 lib/pleroma/web/federator/retry_queue.ex create mode 100644 lib/pleroma/workers/publisher.ex create mode 100644 priv/repo/migrations/20190730055101_add_oban_jobs_table.exs delete mode 100644 test/web/retry_queue_test.exs diff --git a/config/config.exs b/config/config.exs index 17770640..1bb325bf 100644 --- a/config/config.exs +++ b/config/config.exs @@ -440,13 +440,7 @@ config :pleroma, Pleroma.User, "web" ] -config :pleroma, Pleroma.Web.Federator.RetryQueue, - enabled: false, - max_jobs: 20, - initial_timeout: 30, - max_retries: 5 - -config :pleroma_job_queue, :queues, +job_queues = [ federator_incoming: 50, federator_outgoing: 50, web_push: 50, @@ -454,6 +448,15 @@ config :pleroma_job_queue, :queues, transmogrifier: 20, scheduled_activities: 10, background: 5 +] + +config :pleroma_job_queue, :queues, job_queues + +config :pleroma, Oban, + repo: Pleroma.Repo, + verbose: false, + prune: {:maxage, 60 * 60 * 24 * 7}, + queues: job_queues config :pleroma, :fetch_initial_posts, enabled: false, diff --git a/config/test.exs b/config/test.exs index 92dca18b..23d9bf77 100644 --- a/config/test.exs +++ b/config/test.exs @@ -62,6 +62,10 @@ config :web_push_encryption, :http_client, Pleroma.Web.WebPushHttpClientMock config :pleroma_job_queue, disabled: true +config :pleroma, Oban, + queues: false, + prune: :disabled + config :pleroma, Pleroma.ScheduledActivity, daily_user_limit: 2, total_user_limit: 3, diff --git a/docs/config.md b/docs/config.md index 02f86dc1..5c18ffdb 100644 --- a/docs/config.md +++ b/docs/config.md @@ -412,13 +412,6 @@ config :pleroma_job_queue, :queues, This config contains two queues: `federator_incoming` and `federator_outgoing`. Both have the `max_jobs` set to `50`. -## Pleroma.Web.Federator.RetryQueue - -* `enabled`: If set to `true`, failed federation jobs will be retried -* `max_jobs`: The maximum amount of parallel federation jobs running at the same time. -* `initial_timeout`: The initial timeout in seconds -* `max_retries`: The maximum number of times a federation job is retried - ## Pleroma.Web.Metadata * `providers`: a list of metadata providers to enable. Providers available: * Pleroma.Web.Metadata.Providers.OpenGraph diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 03533149..ce7d8c4b 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -120,8 +120,8 @@ defmodule Pleroma.Application do hackney_pool_children() ++ [ %{ - id: Pleroma.Web.Federator.RetryQueue, - start: {Pleroma.Web.Federator.RetryQueue, :start_link, []} + id: Oban, + start: {Oban, :start_link, [Application.get_env(:pleroma, Oban)]} }, %{ id: Pleroma.Web.OAuth.Token.CleanWorker, diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 46edab0b..29f3221d 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -85,6 +85,15 @@ defmodule Pleroma.Web.ActivityPub.Publisher do end end + def publish_one(%{actor_id: actor_id} = params) do + actor = User.get_by_id(actor_id) + + params + |> Map.delete(:actor_id) + |> Map.put(:actor, actor) + |> publish_one() + end + defp should_federate?(inbox, public) do if public do true @@ -160,7 +169,8 @@ defmodule Pleroma.Web.ActivityPub.Publisher do Publishes an activity with BCC to all relevant peers. """ - def publish(actor, %{data: %{"bcc" => bcc}} = activity) when is_list(bcc) and bcc != [] do + def publish(%User{} = actor, %{data: %{"bcc" => bcc}} = activity) + when is_list(bcc) and bcc != [] do public = is_public?(activity) {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) @@ -187,7 +197,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do Pleroma.Web.Federator.Publisher.enqueue_one(__MODULE__, %{ inbox: inbox, json: json, - actor: actor, + actor_id: actor.id, id: activity.data["id"], unreachable_since: unreachable_since }) @@ -222,7 +232,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do %{ inbox: inbox, json: json, - actor: actor, + actor_id: actor.id, id: activity.data["id"], unreachable_since: unreachable_since } diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index f4f9e83e..97ec9d54 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -10,7 +10,6 @@ defmodule Pleroma.Web.Federator do alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.Federator.Publisher - alias Pleroma.Web.Federator.RetryQueue alias Pleroma.Web.OStatus alias Pleroma.Web.Websub @@ -130,19 +129,6 @@ defmodule Pleroma.Web.Federator do end end - def perform( - :publish_single_websub, - %{xml: _xml, topic: _topic, callback: _callback, secret: _secret} = params - ) do - case Websub.publish_one(params) do - {:ok, _} -> - :ok - - {:error, _} -> - RetryQueue.enqueue(params, Websub) - end - end - def perform(type, _) do Logger.debug(fn -> "Unknown task: #{type}" end) {:error, "Don't know what to do with this"} diff --git a/lib/pleroma/web/federator/publisher.ex b/lib/pleroma/web/federator/publisher.ex index 70f87024..e8c1bf17 100644 --- a/lib/pleroma/web/federator/publisher.ex +++ b/lib/pleroma/web/federator/publisher.ex @@ -6,7 +6,6 @@ defmodule Pleroma.Web.Federator.Publisher do alias Pleroma.Activity alias Pleroma.Config alias Pleroma.User - alias Pleroma.Web.Federator.RetryQueue require Logger @@ -30,23 +29,10 @@ defmodule Pleroma.Web.Federator.Publisher do Enqueue publishing a single activity. """ @spec enqueue_one(module(), Map.t()) :: :ok - def enqueue_one(module, %{} = params), - do: PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:publish_one, module, params]) - - @spec perform(atom(), module(), any()) :: {:ok, any()} | {:error, any()} - def perform(:publish_one, module, params) do - case apply(module, :publish_one, [params]) do - {:ok, _} -> - :ok - - {:error, _e} -> - RetryQueue.enqueue(params, module) - end - end - - def perform(type, _, _) do - Logger.debug("Unknown task: #{type}") - {:error, "Don't know what to do with this"} + def enqueue_one(module, %{} = params) do + %{module: to_string(module), params: params} + |> Pleroma.Workers.Publisher.new() + |> Pleroma.Repo.insert() end @doc """ diff --git a/lib/pleroma/web/federator/retry_queue.ex b/lib/pleroma/web/federator/retry_queue.ex deleted file mode 100644 index 3db948c2..00000000 --- a/lib/pleroma/web/federator/retry_queue.ex +++ /dev/null @@ -1,239 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.Federator.RetryQueue do - use GenServer - - require Logger - - def init(args) do - queue_table = :ets.new(:pleroma_retry_queue, [:bag, :protected]) - - {:ok, %{args | queue_table: queue_table, running_jobs: :sets.new()}} - end - - def start_link do - enabled = - if Pleroma.Config.get(:env) == :test, - do: true, - else: Pleroma.Config.get([__MODULE__, :enabled], false) - - if enabled do - Logger.info("Starting retry queue") - - linkres = - GenServer.start_link( - __MODULE__, - %{delivered: 0, dropped: 0, queue_table: nil, running_jobs: nil}, - name: __MODULE__ - ) - - maybe_kickoff_timer() - linkres - else - Logger.info("Retry queue disabled") - :ignore - end - end - - def enqueue(data, transport, retries \\ 0) do - GenServer.cast(__MODULE__, {:maybe_enqueue, data, transport, retries + 1}) - end - - def get_stats do - GenServer.call(__MODULE__, :get_stats) - end - - def reset_stats do - GenServer.call(__MODULE__, :reset_stats) - end - - def get_retry_params(retries) do - if retries > Pleroma.Config.get([__MODULE__, :max_retries]) do - {:drop, "Max retries reached"} - else - {:retry, growth_function(retries)} - end - end - - def get_retry_timer_interval do - Pleroma.Config.get([:retry_queue, :interval], 1000) - end - - defp ets_count_expires(table, current_time) do - :ets.select_count( - table, - [ - { - {:"$1", :"$2"}, - [{:"=<", :"$1", {:const, current_time}}], - [true] - } - ] - ) - end - - defp ets_pop_n_expired(table, current_time, desired) do - {popped, _continuation} = - :ets.select( - table, - [ - { - {:"$1", :"$2"}, - [{:"=<", :"$1", {:const, current_time}}], - [:"$_"] - } - ], - desired - ) - - popped - |> Enum.each(fn e -> - :ets.delete_object(table, e) - end) - - popped - end - - def maybe_start_job(running_jobs, queue_table) do - # we don't want to hit the ets or the DateTime more times than we have to - # could optimize slightly further by not using the count, and instead grabbing - # up to N objects early... - current_time = DateTime.to_unix(DateTime.utc_now()) - n_running_jobs = :sets.size(running_jobs) - - if n_running_jobs < Pleroma.Config.get([__MODULE__, :max_jobs]) do - n_ready_jobs = ets_count_expires(queue_table, current_time) - - if n_ready_jobs > 0 do - # figure out how many we could start - available_job_slots = Pleroma.Config.get([__MODULE__, :max_jobs]) - n_running_jobs - start_n_jobs(running_jobs, queue_table, current_time, available_job_slots) - else - running_jobs - end - else - running_jobs - end - end - - defp start_n_jobs(running_jobs, _queue_table, _current_time, 0) do - running_jobs - end - - defp start_n_jobs(running_jobs, queue_table, current_time, available_job_slots) - when available_job_slots > 0 do - candidates = ets_pop_n_expired(queue_table, current_time, available_job_slots) - - candidates - |> List.foldl(running_jobs, fn {_, e}, rj -> - {:ok, pid} = Task.start(fn -> worker(e) end) - mref = Process.monitor(pid) - :sets.add_element(mref, rj) - end) - end - - def worker({:send, data, transport, retries}) do - case transport.publish_one(data) do - {:ok, _} -> - GenServer.cast(__MODULE__, :inc_delivered) - :delivered - - {:error, _reason} -> - enqueue(data, transport, retries) - :retry - end - end - - def handle_call(:get_stats, _from, %{delivered: delivery_count, dropped: drop_count} = state) do - {:reply, %{delivered: delivery_count, dropped: drop_count}, state} - end - - def handle_call(:reset_stats, _from, %{delivered: delivery_count, dropped: drop_count} = state) do - {:reply, %{delivered: delivery_count, dropped: drop_count}, - %{state | delivered: 0, dropped: 0}} - end - - def handle_cast(:reset_stats, state) do - {:noreply, %{state | delivered: 0, dropped: 0}} - end - - def handle_cast( - {:maybe_enqueue, data, transport, retries}, - %{dropped: drop_count, queue_table: queue_table, running_jobs: running_jobs} = state - ) do - case get_retry_params(retries) do - {:retry, timeout} -> - :ets.insert(queue_table, {timeout, {:send, data, transport, retries}}) - running_jobs = maybe_start_job(running_jobs, queue_table) - {:noreply, %{state | running_jobs: running_jobs}} - - {:drop, message} -> - Logger.debug(message) - {:noreply, %{state | dropped: drop_count + 1}} - end - end - - def handle_cast(:kickoff_timer, state) do - retry_interval = get_retry_timer_interval() - Process.send_after(__MODULE__, :retry_timer_run, retry_interval) - {:noreply, state} - end - - def handle_cast(:inc_delivered, %{delivered: delivery_count} = state) do - {:noreply, %{state | delivered: delivery_count + 1}} - end - - def handle_cast(:inc_dropped, %{dropped: drop_count} = state) do - {:noreply, %{state | dropped: drop_count + 1}} - end - - def handle_info({:send, data, transport, retries}, %{delivered: delivery_count} = state) do - case transport.publish_one(data) do - {:ok, _} -> - {:noreply, %{state | delivered: delivery_count + 1}} - - {:error, _reason} -> - enqueue(data, transport, retries) - {:noreply, state} - end - end - - def handle_info( - :retry_timer_run, - %{queue_table: queue_table, running_jobs: running_jobs} = state - ) do - maybe_kickoff_timer() - running_jobs = maybe_start_job(running_jobs, queue_table) - {:noreply, %{state | running_jobs: running_jobs}} - end - - def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do - %{running_jobs: running_jobs, queue_table: queue_table} = state - running_jobs = :sets.del_element(ref, running_jobs) - running_jobs = maybe_start_job(running_jobs, queue_table) - {:noreply, %{state | running_jobs: running_jobs}} - end - - def handle_info(unknown, state) do - Logger.debug("RetryQueue: don't know what to do with #{inspect(unknown)}, ignoring") - {:noreply, state} - end - - if Pleroma.Config.get(:env) == :test do - defp growth_function(_retries) do - _shutit = Pleroma.Config.get([__MODULE__, :initial_timeout]) - DateTime.to_unix(DateTime.utc_now()) - 1 - end - else - defp growth_function(retries) do - round(Pleroma.Config.get([__MODULE__, :initial_timeout]) * :math.pow(retries, 3)) + - DateTime.to_unix(DateTime.utc_now()) - end - end - - defp maybe_kickoff_timer do - GenServer.cast(__MODULE__, :kickoff_timer) - end -end diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index 9b01ebcc..bbaa293f 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -170,6 +170,15 @@ defmodule Pleroma.Web.Salmon do end end + def publish_one(%{recipient_id: recipient_id} = params) do + recipient = User.get_by_id(recipient_id) + + params + |> Map.delete(:recipient_id) + |> Map.put(:recipient, recipient) + |> publish_one() + end + def publish_one(_), do: :noop @supported_activities [ @@ -218,7 +227,7 @@ defmodule Pleroma.Web.Salmon do Logger.debug(fn -> "Sending Salmon to #{remote_user.ap_id}" end) Publisher.enqueue_one(__MODULE__, %{ - recipient: remote_user, + recipient_id: remote_user.id, feed: feed, unreachable_since: reachable_urls_metadata[remote_user.info.salmon] }) diff --git a/lib/pleroma/workers/publisher.ex b/lib/pleroma/workers/publisher.ex new file mode 100644 index 00000000..63979483 --- /dev/null +++ b/lib/pleroma/workers/publisher.ex @@ -0,0 +1,14 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Publisher do + use Oban.Worker, queue: "federator_outgoing", max_attempts: 5 + + @impl Oban.Worker + def perform(%Oban.Job{args: %{module: module_name, params: params}}) do + module_name + |> String.to_atom() + |> apply(:publish_one, [params]) + end +end diff --git a/mix.exs b/mix.exs index 2a8fe2e9..1ca7a4a7 100644 --- a/mix.exs +++ b/mix.exs @@ -101,6 +101,7 @@ defmodule Pleroma.Mixfile do {:phoenix_ecto, "~> 4.0"}, {:ecto_sql, "~> 3.1"}, {:postgrex, ">= 0.13.5"}, + {:oban, "~> 0.6"}, {:gettext, "~> 0.15"}, {:comeonin, "~> 4.1.1"}, {:pbkdf2_elixir, "~> 0.12.3"}, diff --git a/mix.lock b/mix.lock index 65da7be8..8c0b9734 100644 --- a/mix.lock +++ b/mix.lock @@ -55,6 +55,7 @@ "mogrify": {:hex, :mogrify, "0.6.1", "de1b527514f2d95a7bbe9642eb556061afb337e220cf97adbf3a4e6438ed70af", [:mix], [], "hexpm"}, "mox": {:hex, :mox, "0.5.1", "f86bb36026aac1e6f924a4b6d024b05e9adbed5c63e8daa069bd66fb3292165b", [:mix], [], "hexpm"}, "nimble_parsec": {:hex, :nimble_parsec, "0.5.0", "90e2eca3d0266e5c53f8fbe0079694740b9c91b6747f2b7e3c5d21966bba8300", [:mix], [], "hexpm"}, + "oban": {:hex, :oban, "0.6.0", "8b9b861355610e703e58a878bc29959f3f0e1b4cd1e90d785cf2bb2498d3b893", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.3", "6706a148809a29c306062862c803406e88f048277f6e85b68faf73291e820b84", [:mix], [], "hexpm"}, "phoenix": {:hex, :phoenix, "1.4.9", "746d098e10741c334d88143d3c94cab1756435f94387a63441792e66ec0ee974", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/priv/repo/migrations/20190730055101_add_oban_jobs_table.exs b/priv/repo/migrations/20190730055101_add_oban_jobs_table.exs new file mode 100644 index 00000000..2f201bd0 --- /dev/null +++ b/priv/repo/migrations/20190730055101_add_oban_jobs_table.exs @@ -0,0 +1,6 @@ +defmodule Pleroma.Repo.Migrations.AddObanJobsTable do + use Ecto.Migration + + defdelegate up, to: Oban.Migrations + defdelegate down, to: Oban.Migrations +end diff --git a/test/user_test.exs b/test/user_test.exs index 556df45f..70c37638 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -12,9 +12,9 @@ defmodule Pleroma.UserTest do alias Pleroma.Web.CommonAPI use Pleroma.DataCase + use Oban.Testing, repo: Pleroma.Repo import Pleroma.Factory - import Mock setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -1034,11 +1034,7 @@ defmodule Pleroma.UserTest do refute Activity.get_by_id(repeat.id) end - test_with_mock "it sends out User Delete activity", - %{user: user}, - Pleroma.Web.ActivityPub.Publisher, - [:passthrough], - [] do + test "it sends out User Delete activity", %{user: user} do config_path = [:instance, :federating] initial_setting = Pleroma.Config.get(config_path) Pleroma.Config.put(config_path, true) @@ -1048,11 +1044,8 @@ defmodule Pleroma.UserTest do {:ok, _user} = User.delete(user) - assert called( - Pleroma.Web.ActivityPub.Publisher.publish_one(%{ - inbox: "http://mastodon.example.org/inbox" - }) - ) + assert [%{args: %{"params" => %{"inbox" => "http://mastodon.example.org/inbox"}}}] = + all_enqueued(worker: Pleroma.Workers.Publisher) Pleroma.Config.put(config_path, initial_setting) end diff --git a/test/web/activity_pub/publisher_test.exs b/test/web/activity_pub/publisher_test.exs index 36a39c84..26d01987 100644 --- a/test/web/activity_pub/publisher_test.exs +++ b/test/web/activity_pub/publisher_test.exs @@ -257,7 +257,7 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do assert called( Pleroma.Web.Federator.Publisher.enqueue_one(Publisher, %{ inbox: "https://domain.com/users/nick1/inbox", - actor: actor, + actor_id: actor.id, id: note_activity.data["id"] }) ) diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index 6e143eee..5c170454 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -6,7 +6,10 @@ defmodule Pleroma.Web.FederatorTest do alias Pleroma.Instances alias Pleroma.Web.CommonAPI alias Pleroma.Web.Federator + use Pleroma.DataCase + use Oban.Testing, repo: Pleroma.Repo + import Pleroma.Factory import Mock @@ -22,15 +25,6 @@ defmodule Pleroma.Web.FederatorTest do :ok end - describe "Publisher.perform" do - test "call `perform` with unknown task" do - assert { - :error, - "Don't know what to do with this" - } = Pleroma.Web.Federator.Publisher.perform("test", :ok, :ok) - end - end - describe "Publish an activity" do setup do user = insert(:user) @@ -73,10 +67,7 @@ defmodule Pleroma.Web.FederatorTest do end describe "Targets reachability filtering in `publish`" do - test_with_mock "it federates only to reachable instances via AP", - Pleroma.Web.ActivityPub.Publisher, - [:passthrough], - [] do + test "it federates only to reachable instances via AP" do user = insert(:user) {inbox1, inbox2} = @@ -104,20 +95,13 @@ defmodule Pleroma.Web.FederatorTest do {:ok, _activity} = CommonAPI.post(user, %{"status" => "HI @nick1@domain.com, @nick2@domain2.com!"}) - assert called( - Pleroma.Web.ActivityPub.Publisher.publish_one(%{ - inbox: inbox1, - unreachable_since: dt - }) - ) + expected_dt = NaiveDateTime.to_iso8601(dt) - refute called(Pleroma.Web.ActivityPub.Publisher.publish_one(%{inbox: inbox2})) + assert [%{args: %{"params" => %{"inbox" => ^inbox1, "unreachable_since" => ^expected_dt}}}] = + all_enqueued(worker: Pleroma.Workers.Publisher) end - test_with_mock "it federates only to reachable instances via Websub", - Pleroma.Web.Websub, - [:passthrough], - [] do + test "it federates only to reachable instances via Websub" do user = insert(:user) websub_topic = Pleroma.Web.OStatus.feed_path(user) @@ -142,23 +126,25 @@ defmodule Pleroma.Web.FederatorTest do {:ok, _activity} = CommonAPI.post(user, %{"status" => "HI"}) - assert called( - Pleroma.Web.Websub.publish_one(%{ - callback: sub2.callback, - unreachable_since: dt - }) - ) + expected_callback = sub2.callback + expected_dt = NaiveDateTime.to_iso8601(dt) - refute called(Pleroma.Web.Websub.publish_one(%{callback: sub1.callback})) + assert [ + %{ + args: %{ + "params" => %{ + "callback" => ^expected_callback, + "unreachable_since" => ^expected_dt + } + } + } + ] = all_enqueued(worker: Pleroma.Workers.Publisher) end - test_with_mock "it federates only to reachable instances via Salmon", - Pleroma.Web.Salmon, - [:passthrough], - [] do + test "it federates only to reachable instances via Salmon" do user = insert(:user) - remote_user1 = + _remote_user1 = insert(:user, %{ local: false, nickname: "nick1@domain.com", @@ -174,6 +160,8 @@ defmodule Pleroma.Web.FederatorTest do info: %{salmon: "https://domain2.com/salmon"} }) + remote_user2_id = remote_user2.id + dt = NaiveDateTime.utc_now() Instances.set_unreachable(remote_user2.ap_id, dt) @@ -182,14 +170,18 @@ defmodule Pleroma.Web.FederatorTest do {:ok, _activity} = CommonAPI.post(user, %{"status" => "HI @nick1@domain.com, @nick2@domain2.com!"}) - assert called( - Pleroma.Web.Salmon.publish_one(%{ - recipient: remote_user2, - unreachable_since: dt - }) - ) + expected_dt = NaiveDateTime.to_iso8601(dt) - refute called(Pleroma.Web.Salmon.publish_one(%{recipient: remote_user1})) + assert [ + %{ + args: %{ + "params" => %{ + "recipient_id" => ^remote_user2_id, + "unreachable_since" => ^expected_dt + } + } + } + ] = all_enqueued(worker: Pleroma.Workers.Publisher) end end diff --git a/test/web/retry_queue_test.exs b/test/web/retry_queue_test.exs deleted file mode 100644 index ecb3ce5d..00000000 --- a/test/web/retry_queue_test.exs +++ /dev/null @@ -1,48 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule MockActivityPub do - def publish_one({ret, waiter}) do - send(waiter, :complete) - {ret, "success"} - end -end - -defmodule Pleroma.Web.Federator.RetryQueueTest do - use Pleroma.DataCase - alias Pleroma.Web.Federator.RetryQueue - - @small_retry_count 0 - @hopeless_retry_count 10 - - setup do - RetryQueue.reset_stats() - end - - test "RetryQueue responds to stats request" do - assert %{delivered: 0, dropped: 0} == RetryQueue.get_stats() - end - - test "failed posts are retried" do - {:retry, _timeout} = RetryQueue.get_retry_params(@small_retry_count) - - wait_task = - Task.async(fn -> - receive do - :complete -> :ok - end - end) - - RetryQueue.enqueue({:ok, wait_task.pid}, MockActivityPub, @small_retry_count) - Task.await(wait_task) - assert %{delivered: 1, dropped: 0} == RetryQueue.get_stats() - end - - test "posts that have been tried too many times are dropped" do - {:drop, _timeout} = RetryQueue.get_retry_params(@hopeless_retry_count) - - RetryQueue.enqueue({:ok, nil}, MockActivityPub, @hopeless_retry_count) - assert %{delivered: 0, dropped: 1} == RetryQueue.get_stats() - end -end diff --git a/test/web/salmon/salmon_test.exs b/test/web/salmon/salmon_test.exs index e86e76fe..0186f3fe 100644 --- a/test/web/salmon/salmon_test.exs +++ b/test/web/salmon/salmon_test.exs @@ -96,6 +96,6 @@ defmodule Pleroma.Web.Salmon.SalmonTest do Salmon.publish(user, activity) - assert called(Publisher.enqueue_one(Salmon, %{recipient: mentioned_user})) + assert called(Publisher.enqueue_one(Salmon, %{recipient_id: mentioned_user.id})) end end From b7fad8d395c2bd1afe445a370e539571f5ec0c18 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 9 Aug 2019 20:08:01 +0300 Subject: [PATCH 002/138] [#1149] Oban jobs implementation for :federator_incoming and :federator_outgoing queues. --- config/config.exs | 7 + lib/pleroma/web/activity_pub/utils.ex | 9 +- lib/pleroma/web/federator/federator.ex | 134 +++++------------- lib/pleroma/web/federator/publisher.ex | 12 +- lib/pleroma/workers/publisher.ex | 25 +++- lib/pleroma/workers/receiver.ex | 61 ++++++++ lib/pleroma/workers/subscriber.ex | 44 ++++++ test/activity_test.exs | 4 +- test/support/oban_helpers.ex | 36 +++++ test/user_test.exs | 11 +- .../activity_pub_controller_test.exs | 14 +- test/web/federator_test.exs | 57 +++++--- test/web/websub/websub_test.exs | 4 + 13 files changed, 280 insertions(+), 138 deletions(-) create mode 100644 lib/pleroma/workers/receiver.ex create mode 100644 lib/pleroma/workers/subscriber.ex create mode 100644 test/support/oban_helpers.ex diff --git a/config/config.exs b/config/config.exs index 1bb325bf..5fd64365 100644 --- a/config/config.exs +++ b/config/config.exs @@ -458,6 +458,13 @@ config :pleroma, Oban, prune: {:maxage, 60 * 60 * 24 * 7}, queues: job_queues +config :pleroma, :workers, + retries: [ + compile_time_default: 1, + federator_incoming: 5, + federator_outgoing: 5 + ] + config :pleroma, :fetch_initial_posts, enabled: false, pages: 5 diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 39074888..f0917f9d 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -168,14 +168,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do """ def maybe_federate(%Activity{local: true} = activity) do if Pleroma.Config.get!([:instance, :federating]) do - priority = - case activity.data["type"] do - "Delete" -> 10 - "Create" -> 1 - _ -> 5 - end - - Pleroma.Web.Federator.publish(activity, priority) + Pleroma.Web.Federator.publish(activity) end :ok diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 97ec9d54..bb9eadfe 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -3,22 +3,15 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Federator do - alias Pleroma.Activity - alias Pleroma.Object.Containment - alias Pleroma.User - alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.ActivityPub.Transmogrifier - alias Pleroma.Web.ActivityPub.Utils - alias Pleroma.Web.Federator.Publisher - alias Pleroma.Web.OStatus - alias Pleroma.Web.Websub + alias Pleroma.Workers.Publisher, as: PublisherWorker + alias Pleroma.Workers.Receiver, as: ReceiverWorker + alias Pleroma.Workers.Subscriber, as: SubscriberWorker require Logger def init do # 1 minute - Process.sleep(1000 * 60) - refresh_subscriptions() + refresh_subscriptions(schedule_in: 60) end @doc "Addresses [memory leaks on recursive replies fetching](https://git.pleroma.social/pleroma/pleroma/issues/161)" @@ -36,111 +29,50 @@ defmodule Pleroma.Web.Federator do # Client API def incoming_doc(doc) do - PleromaJobQueue.enqueue(:federator_incoming, __MODULE__, [:incoming_doc, doc]) + %{"op" => "incoming_doc", "body" => doc} + |> ReceiverWorker.new(worker_args(:federator_incoming)) + |> Pleroma.Repo.insert() end def incoming_ap_doc(params) do - PleromaJobQueue.enqueue(:federator_incoming, __MODULE__, [:incoming_ap_doc, params]) + %{"op" => "incoming_ap_doc", "params" => params} + |> ReceiverWorker.new(worker_args(:federator_incoming)) + |> Pleroma.Repo.insert() end - def publish(activity, priority \\ 1) do - PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:publish, activity], priority) + def publish(%{id: "pleroma:fakeid"} = activity) do + PublisherWorker.perform_publish(activity) + end + + def publish(activity) do + %{"op" => "publish", "activity_id" => activity.id} + |> PublisherWorker.new(worker_args(:federator_outgoing)) + |> Pleroma.Repo.insert() end def verify_websub(websub) do - PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:verify_websub, websub]) + %{"op" => "verify_websub", "websub_id" => websub.id} + |> SubscriberWorker.new(worker_args(:federator_outgoing)) + |> Pleroma.Repo.insert() end - def request_subscription(sub) do - PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:request_subscription, sub]) + def request_subscription(websub) do + %{"op" => "request_subscription", "websub_id" => websub.id} + |> SubscriberWorker.new(worker_args(:federator_outgoing)) + |> Pleroma.Repo.insert() end - def refresh_subscriptions do - PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:refresh_subscriptions]) + def refresh_subscriptions(worker_args \\ []) do + %{"op" => "refresh_subscriptions"} + |> SubscriberWorker.new(worker_args ++ [max_attempts: 1] ++ worker_args(:federator_outgoing)) + |> Pleroma.Repo.insert() end - # Job Worker Callbacks - - def perform(:refresh_subscriptions) do - Logger.debug("Federator running refresh subscriptions") - Websub.refresh_subscriptions() - - spawn(fn -> - # 6 hours - Process.sleep(1000 * 60 * 60 * 6) - refresh_subscriptions() - end) - end - - def perform(:request_subscription, websub) do - Logger.debug("Refreshing #{websub.topic}") - - with {:ok, websub} <- Websub.request_subscription(websub) do - Logger.debug("Successfully refreshed #{websub.topic}") + defp worker_args(queue) do + if max_attempts = Pleroma.Config.get([:workers, :retries, queue]) do + [max_attempts: max_attempts] else - _e -> Logger.debug("Couldn't refresh #{websub.topic}") - end - end - - def perform(:publish, activity) do - Logger.debug(fn -> "Running publish for #{activity.data["id"]}" end) - - with %User{} = actor <- User.get_cached_by_ap_id(activity.data["actor"]), - {:ok, actor} <- User.ensure_keys_present(actor) do - Publisher.publish(actor, activity) - end - end - - def perform(:verify_websub, websub) do - Logger.debug(fn -> - "Running WebSub verification for #{websub.id} (#{websub.topic}, #{websub.callback})" - end) - - Websub.verify(websub) - end - - def perform(:incoming_doc, doc) do - Logger.info("Got document, trying to parse") - OStatus.handle_incoming(doc) - end - - def perform(:incoming_ap_doc, params) do - Logger.info("Handling incoming AP activity") - - params = Utils.normalize_params(params) - - # NOTE: we use the actor ID to do the containment, this is fine because an - # actor shouldn't be acting on objects outside their own AP server. - with {:ok, _user} <- ap_enabled_actor(params["actor"]), - nil <- Activity.normalize(params["id"]), - :ok <- Containment.contain_origin_from_id(params["actor"], params), - {:ok, activity} <- Transmogrifier.handle_incoming(params) do - {:ok, activity} - else - %Activity{} -> - Logger.info("Already had #{params["id"]}") - :error - - _e -> - # Just drop those for now - Logger.info("Unhandled activity") - Logger.info(Jason.encode!(params, pretty: true)) - :error - end - end - - def perform(type, _) do - Logger.debug(fn -> "Unknown task: #{type}" end) - {:error, "Don't know what to do with this"} - end - - def ap_enabled_actor(id) do - user = User.get_cached_by_ap_id(id) - - if User.ap_enabled?(user) do - {:ok, user} - else - ActivityPub.make_user_from_ap_id(id) + [] end end end diff --git a/lib/pleroma/web/federator/publisher.ex b/lib/pleroma/web/federator/publisher.ex index e8c1bf17..05d2be61 100644 --- a/lib/pleroma/web/federator/publisher.ex +++ b/lib/pleroma/web/federator/publisher.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.Federator.Publisher do alias Pleroma.Activity alias Pleroma.Config alias Pleroma.User + alias Pleroma.Workers.Publisher, as: PublisherWorker require Logger @@ -30,8 +31,15 @@ defmodule Pleroma.Web.Federator.Publisher do """ @spec enqueue_one(module(), Map.t()) :: :ok def enqueue_one(module, %{} = params) do - %{module: to_string(module), params: params} - |> Pleroma.Workers.Publisher.new() + worker_args = + if max_attempts = Pleroma.Config.get([:workers, :retries, :federator_outgoing]) do + [max_attempts: max_attempts] + else + [] + end + + %{"op" => "publish_one", "module" => to_string(module), "params" => params} + |> PublisherWorker.new(worker_args) |> Pleroma.Repo.insert() end diff --git a/lib/pleroma/workers/publisher.ex b/lib/pleroma/workers/publisher.ex index 63979483..67871977 100644 --- a/lib/pleroma/workers/publisher.ex +++ b/lib/pleroma/workers/publisher.ex @@ -3,12 +3,33 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.Publisher do - use Oban.Worker, queue: "federator_outgoing", max_attempts: 5 + alias Pleroma.Activity + alias Pleroma.User + + # Note: `max_attempts` is intended to be overridden in `new/1` call + use Oban.Worker, + queue: "federator_outgoing", + max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) @impl Oban.Worker - def perform(%Oban.Job{args: %{module: module_name, params: params}}) do + def perform(%{"op" => "publish", "activity_id" => activity_id}) do + with %Activity{} = activity <- Activity.get_by_id(activity_id) do + perform_publish(activity) + else + _ -> raise "Non-existing activity: #{activity_id}" + end + end + + def perform(%{"op" => "publish_one", "module" => module_name, "params" => params}) do module_name |> String.to_atom() |> apply(:publish_one, [params]) end + + def perform_publish(%Activity{} = activity) do + with %User{} = actor <- User.get_cached_by_ap_id(activity.data["actor"]), + {:ok, actor} <- User.ensure_keys_present(actor) do + Pleroma.Web.Federator.Publisher.publish(actor, activity) + end + end end diff --git a/lib/pleroma/workers/receiver.ex b/lib/pleroma/workers/receiver.ex new file mode 100644 index 00000000..43558b4e --- /dev/null +++ b/lib/pleroma/workers/receiver.ex @@ -0,0 +1,61 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Receiver do + alias Pleroma.Activity + alias Pleroma.Object.Containment + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.Web.OStatus + + require Logger + + # Note: `max_attempts` is intended to be overridden in `new/1` call + use Oban.Worker, + queue: "federator_incoming", + max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + + @impl Oban.Worker + def perform(%{"op" => "incoming_doc", "body" => doc}) do + Logger.info("Got incoming document, trying to parse") + OStatus.handle_incoming(doc) + end + + def perform(%{"op" => "incoming_ap_doc", "params" => params}) do + Logger.info("Handling incoming AP activity") + + params = Utils.normalize_params(params) + + # NOTE: we use the actor ID to do the containment, this is fine because an + # actor shouldn't be acting on objects outside their own AP server. + with {:ok, _user} <- ap_enabled_actor(params["actor"]), + nil <- Activity.normalize(params["id"]), + :ok <- Containment.contain_origin_from_id(params["actor"], params), + {:ok, activity} <- Transmogrifier.handle_incoming(params) do + {:ok, activity} + else + %Activity{} -> + Logger.info("Already had #{params["id"]}") + :error + + _e -> + # Just drop those for now + Logger.info("Unhandled activity") + Logger.info(Jason.encode!(params, pretty: true)) + :error + end + end + + defp ap_enabled_actor(id) do + user = User.get_cached_by_ap_id(id) + + if User.ap_enabled?(user) do + {:ok, user} + else + ActivityPub.make_user_from_ap_id(id) + end + end +end diff --git a/lib/pleroma/workers/subscriber.ex b/lib/pleroma/workers/subscriber.ex new file mode 100644 index 00000000..a8c01bb1 --- /dev/null +++ b/lib/pleroma/workers/subscriber.ex @@ -0,0 +1,44 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Subscriber do + alias Pleroma.Repo + alias Pleroma.Web.Websub + alias Pleroma.Web.Websub.WebsubClientSubscription + + require Logger + + # Note: `max_attempts` is intended to be overridden in `new/1` call + use Oban.Worker, + queue: "federator_outgoing", + max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + + @impl Oban.Worker + def perform(%{"op" => "refresh_subscriptions"}) do + Websub.refresh_subscriptions() + # Schedule the next run in 6 hours + Pleroma.Web.Federator.refresh_subscriptions(schedule_in: 3600 * 6) + end + + def perform(%{"op" => "request_subscription", "websub_id" => websub_id}) do + websub = Repo.get(WebsubClientSubscription, websub_id) + Logger.debug("Refreshing #{websub.topic}") + + with {:ok, websub} <- Websub.request_subscription(websub) do + Logger.debug("Successfully refreshed #{websub.topic}") + else + _e -> Logger.debug("Couldn't refresh #{websub.topic}") + end + end + + def perform(%{"op" => "verify_websub", "websub_id" => websub_id}) do + websub = Repo.get(WebsubClientSubscription, websub_id) + + Logger.debug(fn -> + "Running WebSub verification for #{websub.id} (#{websub.topic}, #{websub.callback})" + end) + + Websub.verify(websub) + end +end diff --git a/test/activity_test.exs b/test/activity_test.exs index b27f6fd3..b9c12adb 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.ActivityTest do use Pleroma.DataCase alias Pleroma.Activity alias Pleroma.Bookmark + alias Pleroma.ObanHelpers alias Pleroma.Object alias Pleroma.ThreadMute import Pleroma.Factory @@ -125,7 +126,8 @@ defmodule Pleroma.ActivityTest do } {:ok, local_activity} = Pleroma.Web.CommonAPI.post(user, %{"status" => "find me!"}) - {:ok, remote_activity} = Pleroma.Web.Federator.incoming_ap_doc(params) + {:ok, job} = Pleroma.Web.Federator.incoming_ap_doc(params) + {:ok, remote_activity} = ObanHelpers.perform(job) %{local_activity: local_activity, remote_activity: remote_activity, user: user} end diff --git a/test/support/oban_helpers.ex b/test/support/oban_helpers.ex new file mode 100644 index 00000000..54b5a956 --- /dev/null +++ b/test/support/oban_helpers.ex @@ -0,0 +1,36 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ObanHelpers do + @moduledoc """ + Oban test helpers. + """ + + alias Pleroma.Repo + + def perform(%Oban.Job{} = job) do + res = apply(String.to_existing_atom("Elixir." <> job.worker), :perform, [job]) + Repo.delete(job) + res + end + + def perform(jobs) when is_list(jobs) do + for job <- jobs, do: perform(job) + end + + def member?(%{} = job_args, jobs) when is_list(jobs) do + Enum.any?(jobs, fn job -> + member?(job_args, job.args) + end) + end + + def member?(%{} = test_attrs, %{} = attrs) do + Enum.all?( + test_attrs, + fn {k, _v} -> member?(test_attrs[k], attrs[k]) end + ) + end + + def member?(x, y), do: x == y +end diff --git a/test/user_test.exs b/test/user_test.exs index 70c37638..ee6d8e8f 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -5,6 +5,7 @@ defmodule Pleroma.UserTest do alias Pleroma.Activity alias Pleroma.Builders.UserBuilder + alias Pleroma.ObanHelpers alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User @@ -1044,8 +1045,16 @@ defmodule Pleroma.UserTest do {:ok, _user} = User.delete(user) - assert [%{args: %{"params" => %{"inbox" => "http://mastodon.example.org/inbox"}}}] = + assert ObanHelpers.member?( + %{ + "op" => "publish_one", + "params" => %{ + "inbox" => "http://mastodon.example.org/inbox", + "id" => "pleroma:fakeid" + } + }, all_enqueued(worker: Pleroma.Workers.Publisher) + ) Pleroma.Config.put(config_path, initial_setting) end diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index 40344f17..1d809164 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -4,15 +4,19 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do use Pleroma.Web.ConnCase + use Oban.Testing, repo: Pleroma.Repo + import Pleroma.Factory alias Pleroma.Activity alias Pleroma.Instances + alias Pleroma.ObanHelpers alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.ActivityPub.ObjectView alias Pleroma.Web.ActivityPub.UserView alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.CommonAPI + alias Pleroma.Workers.Receiver, as: ReceiverWorker setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -232,7 +236,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do |> post("/inbox", data) assert "ok" == json_response(conn, 200) - :timer.sleep(500) + + ObanHelpers.perform(all_enqueued(worker: ReceiverWorker)) assert Activity.get_by_ap_id(data["id"]) end @@ -274,7 +279,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do |> post("/users/#{user.nickname}/inbox", data) assert "ok" == json_response(conn, 200) - :timer.sleep(500) + ObanHelpers.perform(all_enqueued(worker: ReceiverWorker)) assert Activity.get_by_ap_id(data["id"]) end @@ -303,7 +308,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do |> post("/users/#{recipient.nickname}/inbox", data) assert "ok" == json_response(conn, 200) - :timer.sleep(500) + ObanHelpers.perform(all_enqueued(worker: ReceiverWorker)) assert Activity.get_by_ap_id(data["id"]) end @@ -382,6 +387,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do |> post("/users/#{recipient.nickname}/inbox", data) |> json_response(200) + ObanHelpers.perform(all_enqueued(worker: ReceiverWorker)) + activity = Activity.get_by_ap_id(data["id"]) assert activity.id @@ -457,6 +464,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do |> post("/users/#{user.nickname}/outbox", data) result = json_response(conn, 201) + assert Activity.get_by_ap_id(result["id"]) end diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index 5c170454..ebe962da 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -4,8 +4,10 @@ defmodule Pleroma.Web.FederatorTest do alias Pleroma.Instances + alias Pleroma.ObanHelpers alias Pleroma.Web.CommonAPI alias Pleroma.Web.Federator + alias Pleroma.Workers.Publisher, as: PublisherWorker use Pleroma.DataCase use Oban.Testing, repo: Pleroma.Repo @@ -45,6 +47,7 @@ defmodule Pleroma.Web.FederatorTest do } do with_mocks([relay_mock]) do Federator.publish(activity) + ObanHelpers.perform(all_enqueued(worker: PublisherWorker)) end assert_received :relay_publish @@ -58,6 +61,7 @@ defmodule Pleroma.Web.FederatorTest do with_mocks([relay_mock]) do Federator.publish(activity) + ObanHelpers.perform(all_enqueued(worker: PublisherWorker)) end refute_received :relay_publish @@ -97,8 +101,15 @@ defmodule Pleroma.Web.FederatorTest do expected_dt = NaiveDateTime.to_iso8601(dt) - assert [%{args: %{"params" => %{"inbox" => ^inbox1, "unreachable_since" => ^expected_dt}}}] = - all_enqueued(worker: Pleroma.Workers.Publisher) + ObanHelpers.perform(all_enqueued(worker: PublisherWorker)) + + assert ObanHelpers.member?( + %{ + "op" => "publish_one", + "params" => %{"inbox" => inbox1, "unreachable_since" => expected_dt} + }, + all_enqueued(worker: PublisherWorker) + ) end test "it federates only to reachable instances via Websub" do @@ -129,16 +140,18 @@ defmodule Pleroma.Web.FederatorTest do expected_callback = sub2.callback expected_dt = NaiveDateTime.to_iso8601(dt) - assert [ + ObanHelpers.perform(all_enqueued(worker: PublisherWorker)) + + assert ObanHelpers.member?( %{ - args: %{ - "params" => %{ - "callback" => ^expected_callback, - "unreachable_since" => ^expected_dt - } + "op" => "publish_one", + "params" => %{ + "callback" => expected_callback, + "unreachable_since" => expected_dt } - } - ] = all_enqueued(worker: Pleroma.Workers.Publisher) + }, + all_enqueued(worker: PublisherWorker) + ) end test "it federates only to reachable instances via Salmon" do @@ -172,16 +185,18 @@ defmodule Pleroma.Web.FederatorTest do expected_dt = NaiveDateTime.to_iso8601(dt) - assert [ + ObanHelpers.perform(all_enqueued(worker: PublisherWorker)) + + assert ObanHelpers.member?( %{ - args: %{ - "params" => %{ - "recipient_id" => ^remote_user2_id, - "unreachable_since" => ^expected_dt - } + "op" => "publish_one", + "params" => %{ + "recipient_id" => remote_user2_id, + "unreachable_since" => expected_dt } - } - ] = all_enqueued(worker: Pleroma.Workers.Publisher) + }, + all_enqueued(worker: PublisherWorker) + ) end end @@ -201,7 +216,8 @@ defmodule Pleroma.Web.FederatorTest do "to" => ["https://www.w3.org/ns/activitystreams#Public"] } - {:ok, _activity} = Federator.incoming_ap_doc(params) + assert {:ok, job} = Federator.incoming_ap_doc(params) + assert {:ok, _activity} = ObanHelpers.perform(job) end test "rejects incoming AP docs with incorrect origin" do @@ -219,7 +235,8 @@ defmodule Pleroma.Web.FederatorTest do "to" => ["https://www.w3.org/ns/activitystreams#Public"] } - :error = Federator.incoming_ap_doc(params) + assert {:ok, job} = Federator.incoming_ap_doc(params) + assert :error = ObanHelpers.perform(job) end end end diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index 74386d7d..b704a558 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -4,11 +4,14 @@ defmodule Pleroma.Web.WebsubTest do use Pleroma.DataCase + use Oban.Testing, repo: Pleroma.Repo + alias Pleroma.ObanHelpers alias Pleroma.Web.Router.Helpers alias Pleroma.Web.Websub alias Pleroma.Web.Websub.WebsubClientSubscription alias Pleroma.Web.Websub.WebsubServerSubscription + alias Pleroma.Workers.Subscriber, as: SubscriberWorker import Pleroma.Factory import Tesla.Mock @@ -224,6 +227,7 @@ defmodule Pleroma.Web.WebsubTest do }) _refresh = Websub.refresh_subscriptions() + ObanHelpers.perform(all_enqueued(worker: SubscriberWorker)) assert still_good == Repo.get(WebsubClientSubscription, still_good.id) refute needs_refresh == Repo.get(WebsubClientSubscription, needs_refresh.id) From 33a5fc4a70b6f9b8c2d8c03a412d7eec8d5b3db1 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 10 Aug 2019 20:38:31 +0300 Subject: [PATCH 003/138] [#1149] Fixed failing tests. Ensured Instance.set_unreachable/2 supports ISO 8601 datetime. --- lib/pleroma/digest_email_worker.ex | 4 +--- lib/pleroma/instances/instance.ex | 8 +++++++- test/conversation_test.exs | 2 ++ test/support/oban_helpers.ex | 6 ++++++ test/web/federator_test.exs | 3 ++- test/web/instances/instance_test.exs | 3 ++- 6 files changed, 20 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/digest_email_worker.ex b/lib/pleroma/digest_email_worker.ex index 18e67d39..3b0e2bca 100644 --- a/lib/pleroma/digest_email_worker.ex +++ b/lib/pleroma/digest_email_worker.ex @@ -1,8 +1,6 @@ defmodule Pleroma.DigestEmailWorker do import Ecto.Query - @queue_name :digest_emails - def perform do config = Pleroma.Config.get([:email_notifications, :digest]) negative_interval = -Map.fetch!(config, :interval) @@ -17,7 +15,7 @@ defmodule Pleroma.DigestEmailWorker do select: u ) |> Pleroma.Repo.all() - |> Enum.each(&PleromaJobQueue.enqueue(@queue_name, __MODULE__, [&1])) + |> Enum.each(&PleromaJobQueue.enqueue(:digest_emails, __MODULE__, [&1])) end @doc """ diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 4d7ed4ca..544c4b68 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -90,7 +90,7 @@ defmodule Pleroma.Instances.Instance do def set_unreachable(url_or_host, unreachable_since \\ nil) def set_unreachable(url_or_host, unreachable_since) when is_binary(url_or_host) do - unreachable_since = unreachable_since || DateTime.utc_now() + unreachable_since = parse_datetime(unreachable_since) || NaiveDateTime.utc_now() host = host(url_or_host) existing_record = Repo.get_by(Instance, %{host: host}) @@ -114,4 +114,10 @@ defmodule Pleroma.Instances.Instance do end def set_unreachable(_, _), do: {:error, nil} + + defp parse_datetime(datetime) when is_binary(datetime) do + NaiveDateTime.from_iso8601(datetime) + end + + defp parse_datetime(datetime), do: datetime end diff --git a/test/conversation_test.exs b/test/conversation_test.exs index aa193e0d..2ebbcab7 100644 --- a/test/conversation_test.exs +++ b/test/conversation_test.exs @@ -28,6 +28,8 @@ defmodule Pleroma.ConversationTest do {:ok, _activity} = CommonAPI.post(user, %{"visibility" => "direct", "status" => "hey @#{other_user.nickname}"}) + Pleroma.ObanHelpers.perform_all() + Repo.delete_all(Conversation) Repo.delete_all(Conversation.Participation) diff --git a/test/support/oban_helpers.ex b/test/support/oban_helpers.ex index 54b5a956..ecc03ba1 100644 --- a/test/support/oban_helpers.ex +++ b/test/support/oban_helpers.ex @@ -9,6 +9,12 @@ defmodule Pleroma.ObanHelpers do alias Pleroma.Repo + def perform_all do + Oban.Job + |> Repo.all() + |> perform() + end + def perform(%Oban.Job{} = job) do res = apply(String.to_existing_atom("Elixir." <> job.worker), :perform, [job]) Repo.delete(job) diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index d3a28d50..e0be4342 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -249,7 +249,8 @@ defmodule Pleroma.Web.FederatorTest do File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() - assert Federator.incoming_ap_doc(params) == :error + assert {:ok, job} = Federator.incoming_ap_doc(params) + assert :error = ObanHelpers.perform(job) Pleroma.Config.put([:instance, :rewrite_policy], policies) Pleroma.Config.put(:mrf_keyword, mrf_keyword_policy) diff --git a/test/web/instances/instance_test.exs b/test/web/instances/instance_test.exs index d2873099..a1bdd45d 100644 --- a/test/web/instances/instance_test.exs +++ b/test/web/instances/instance_test.exs @@ -22,7 +22,8 @@ defmodule Pleroma.Instances.InstanceTest do describe "set_reachable/1" do test "clears `unreachable_since` of existing matching Instance record having non-nil `unreachable_since`" do - instance = insert(:instance, unreachable_since: NaiveDateTime.utc_now()) + unreachable_since = NaiveDateTime.to_iso8601(NaiveDateTime.utc_now()) + instance = insert(:instance, unreachable_since: unreachable_since) assert {:ok, instance} = Instance.set_reachable(instance.host) refute instance.unreachable_since From 0e1c481a94392b69833fbe6afc184ebbd90e1330 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 13 Aug 2019 20:20:26 +0300 Subject: [PATCH 004/138] [#1149] Added more oban workers. Refactoring. --- lib/pleroma/digest_email_worker.ex | 11 ++- lib/pleroma/scheduled_activity_worker.ex | 8 +- lib/pleroma/user.ex | 55 +++++++---- lib/pleroma/web/activity_pub/activity_pub.ex | 7 +- .../mrf/mediaproxy_warming_policy.ex | 12 ++- .../web/activity_pub/transmogrifier.ex | 7 +- lib/pleroma/web/federator/federator.ex | 98 ++++++++++++++++++- lib/pleroma/web/oauth/token/clean_worker.ex | 10 +- lib/pleroma/web/push/push.ex | 12 ++- .../controllers/util_controller.ex | 14 +-- lib/pleroma/workers/background_worker.ex | 66 +++++++++++++ lib/pleroma/workers/helper.ex | 13 +++ lib/pleroma/workers/mailer.ex | 18 ++++ lib/pleroma/workers/publisher.ex | 20 +--- lib/pleroma/workers/receiver.ex | 46 +-------- .../workers/scheduled_activity_worker.ex | 15 +++ lib/pleroma/workers/subscriber.ex | 23 +---- lib/pleroma/workers/transmogrifier.ex | 18 ++++ lib/pleroma/workers/web_pusher.ex | 19 ++++ test/activity_test.exs | 2 +- test/conversation_test.exs | 2 +- test/notification_test.exs | 5 +- test/support/oban_helpers.ex | 2 +- test/user_test.exs | 19 ++-- .../activity_pub_controller_test.exs | 2 +- .../mrf/mediaproxy_warming_policy_test.exs | 6 ++ test/web/activity_pub/transmogrifier_test.exs | 4 + test/web/federator_test.exs | 2 +- test/web/twitter_api/util_controller_test.exs | 43 ++++---- test/web/websub/websub_test.exs | 2 +- 30 files changed, 402 insertions(+), 159 deletions(-) create mode 100644 lib/pleroma/workers/background_worker.ex create mode 100644 lib/pleroma/workers/helper.ex create mode 100644 lib/pleroma/workers/mailer.ex create mode 100644 lib/pleroma/workers/scheduled_activity_worker.ex create mode 100644 lib/pleroma/workers/transmogrifier.ex create mode 100644 lib/pleroma/workers/web_pusher.ex diff --git a/lib/pleroma/digest_email_worker.ex b/lib/pleroma/digest_email_worker.ex index 3b0e2bca..6e44cc95 100644 --- a/lib/pleroma/digest_email_worker.ex +++ b/lib/pleroma/digest_email_worker.ex @@ -1,6 +1,11 @@ defmodule Pleroma.DigestEmailWorker do + alias Pleroma.Repo + alias Pleroma.Workers.Mailer, as: MailerWorker + import Ecto.Query + defdelegate worker_args(queue), to: Pleroma.Workers.Helper + def perform do config = Pleroma.Config.get([:email_notifications, :digest]) negative_interval = -Map.fetch!(config, :interval) @@ -15,7 +20,11 @@ defmodule Pleroma.DigestEmailWorker do select: u ) |> Pleroma.Repo.all() - |> Enum.each(&PleromaJobQueue.enqueue(:digest_emails, __MODULE__, [&1])) + |> Enum.each(fn user -> + %{"op" => "digest_email", "user_id" => user.id} + |> MailerWorker.new([queue: "digest_emails"] ++ worker_args(:digest_emails)) + |> Repo.insert() + end) end @doc """ diff --git a/lib/pleroma/scheduled_activity_worker.ex b/lib/pleroma/scheduled_activity_worker.ex index 65b38622..cabea51c 100644 --- a/lib/pleroma/scheduled_activity_worker.ex +++ b/lib/pleroma/scheduled_activity_worker.ex @@ -8,14 +8,18 @@ defmodule Pleroma.ScheduledActivityWorker do """ alias Pleroma.Config + alias Pleroma.Repo alias Pleroma.ScheduledActivity alias Pleroma.User alias Pleroma.Web.CommonAPI + use GenServer require Logger @schedule_interval :timer.minutes(1) + defdelegate worker_args(queue), to: Pleroma.Workers.Helper + def start_link do GenServer.start_link(__MODULE__, nil) end @@ -45,7 +49,9 @@ defmodule Pleroma.ScheduledActivityWorker do def handle_info(:perform, state) do ScheduledActivity.due_activities(@schedule_interval) |> Enum.each(fn scheduled_activity -> - PleromaJobQueue.enqueue(:scheduled_activities, __MODULE__, [:execute, scheduled_activity.id]) + %{"op" => "execute", "activity_id" => scheduled_activity.id} + |> Pleroma.Workers.ScheduledActivityWorker.new(worker_args(:scheduled_activities)) + |> Repo.insert() end) schedule_next() diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 7d18f099..bc2102ca 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -26,6 +26,7 @@ defmodule Pleroma.User do alias Pleroma.Web.OStatus alias Pleroma.Web.RelMe alias Pleroma.Web.Websub + alias Pleroma.Workers.BackgroundWorker require Logger @@ -39,6 +40,8 @@ defmodule Pleroma.User do @strict_local_nickname_regex ~r/^[a-zA-Z\d]+$/ @extended_local_nickname_regex ~r/^[a-zA-Z\d_-]+$/ + defdelegate worker_args(queue), to: Pleroma.Workers.Helper + schema "users" do field(:bio, :string) field(:email, :string) @@ -579,8 +582,11 @@ defmodule Pleroma.User do end @doc "Fetch some posts when the user has just been federated with" - def fetch_initial_posts(user), - do: PleromaJobQueue.enqueue(:background, __MODULE__, [:fetch_initial_posts, user]) + def fetch_initial_posts(user) do + %{"op" => "fetch_initial_posts", "user_id" => user.id} + |> BackgroundWorker.new(worker_args(:background)) + |> Repo.insert() + end @spec get_followers_query(User.t(), pos_integer() | nil) :: Ecto.Query.t() def get_followers_query(%User{} = user, nil) do @@ -1001,7 +1007,9 @@ defmodule Pleroma.User do end def deactivate_async(user, status \\ true) do - PleromaJobQueue.enqueue(:background, __MODULE__, [:deactivate_async, user, status]) + %{"op" => "deactivate_user", "user_id" => user.id, "status" => status} + |> BackgroundWorker.new(worker_args(:background)) + |> Repo.insert() end def deactivate(%User{} = user, status \\ true) do @@ -1029,9 +1037,11 @@ defmodule Pleroma.User do |> update_and_set_cache() end - @spec delete(User.t()) :: :ok - def delete(%User{} = user), - do: PleromaJobQueue.enqueue(:background, __MODULE__, [:delete, user]) + def delete(%User{} = user) do + %{"op" => "delete_user", "user_id" => user.id} + |> BackgroundWorker.new(worker_args(:background)) + |> Repo.insert() + end @spec perform(atom(), User.t()) :: {:ok, User.t()} def perform(:delete, %User{} = user) do @@ -1138,21 +1148,26 @@ defmodule Pleroma.User do Repo.all(query) end - def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers), - do: - PleromaJobQueue.enqueue(:background, __MODULE__, [ - :blocks_import, - blocker, - blocked_identifiers - ]) + def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers) do + %{ + "op" => "blocks_import", + "blocker_id" => blocker.id, + "blocked_identifiers" => blocked_identifiers + } + |> BackgroundWorker.new(worker_args(:background)) + |> Repo.insert() + end - def follow_import(%User{} = follower, followed_identifiers) when is_list(followed_identifiers), - do: - PleromaJobQueue.enqueue(:background, __MODULE__, [ - :follow_import, - follower, - followed_identifiers - ]) + def follow_import(%User{} = follower, followed_identifiers) + when is_list(followed_identifiers) do + %{ + "op" => "follow_import", + "follower_id" => follower.id, + "followed_identifiers" => followed_identifiers + } + |> BackgroundWorker.new(worker_args(:background)) + |> Repo.insert() + end def delete_user_activities(%User{ap_id: ap_id} = user) do ap_id diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 1a279a7d..8be8ac86 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -17,6 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.WebFinger + alias Pleroma.Workers.BackgroundWorker import Ecto.Query import Pleroma.Web.ActivityPub.Utils @@ -25,6 +26,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do require Logger require Pleroma.Constants + defdelegate worker_args(queue), to: Pleroma.Workers.Helper + # For Announce activities, we filter the recipients based on following status for any actors # that match actual users. See issue #164 for more information about why this is necessary. defp get_recipients(%{"type" => "Announce"} = data) do @@ -145,7 +148,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do activity end - PleromaJobQueue.enqueue(:background, Pleroma.Web.RichMedia.Helpers, [:fetch, activity]) + %{"op" => "fetch_data_for_activity", "activity_id" => activity.id} + |> BackgroundWorker.new(worker_args(:background)) + |> Repo.insert() Notification.create_notifications(activity) diff --git a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex index 01d21a29..1df3bb5b 100644 --- a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex @@ -7,7 +7,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do @behaviour Pleroma.Web.ActivityPub.MRF alias Pleroma.HTTP + alias Pleroma.Repo alias Pleroma.Web.MediaProxy + alias Pleroma.Workers.BackgroundWorker require Logger @@ -16,6 +18,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do recv_timeout: 10_000 ] + defdelegate worker_args(queue), to: Pleroma.Workers.Helper + def perform(:prefetch, url) do Logger.info("Prefetching #{inspect(url)}") @@ -30,7 +34,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do url |> Enum.each(fn %{"href" => href} -> - PleromaJobQueue.enqueue(:background, __MODULE__, [:prefetch, href]) + %{"op" => "media_proxy_prefetch", "url" => href} + |> BackgroundWorker.new(worker_args(:background)) + |> Repo.insert() x -> Logger.debug("Unhandled attachment URL object #{inspect(x)}") @@ -46,7 +52,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do %{"type" => "Create", "object" => %{"attachment" => attachments} = _object} = message ) when is_list(attachments) and length(attachments) > 0 do - PleromaJobQueue.enqueue(:background, __MODULE__, [:preload, message]) + %{"op" => "media_proxy_preload", "message" => message} + |> BackgroundWorker.new(worker_args(:background)) + |> Repo.insert() {:ok, message} end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 5403b71d..0f117cd0 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -15,12 +15,15 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.Federator + alias Pleroma.Workers.Transmogrifier, as: TransmogrifierWorker import Ecto.Query require Logger require Pleroma.Constants + defdelegate worker_args(queue), to: Pleroma.Workers.Helper + @doc """ Modifies an incoming AP object (mastodon format) to our internal format. """ @@ -1073,7 +1076,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do already_ap <- User.ap_enabled?(user), {:ok, user} <- user |> User.upgrade_changeset(data) |> User.update_and_set_cache() do unless already_ap do - PleromaJobQueue.enqueue(:transmogrifier, __MODULE__, [:user_upgrade, user]) + %{"op" => "user_upgrade", "user_id" => user.id} + |> TransmogrifierWorker.new(worker_args(:transmogrifier)) + |> Repo.insert() end {:ok, user} diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index bb9eadfe..d85fe824 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -3,12 +3,23 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Federator do + alias Pleroma.Activity + alias Pleroma.Object.Containment + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.Web.Federator.Publisher + alias Pleroma.Web.OStatus + alias Pleroma.Web.Websub alias Pleroma.Workers.Publisher, as: PublisherWorker alias Pleroma.Workers.Receiver, as: ReceiverWorker alias Pleroma.Workers.Subscriber, as: SubscriberWorker require Logger + defdelegate worker_args(queue), to: Pleroma.Workers.Helper + def init do # 1 minute refresh_subscriptions(schedule_in: 60) @@ -41,7 +52,7 @@ defmodule Pleroma.Web.Federator do end def publish(%{id: "pleroma:fakeid"} = activity) do - PublisherWorker.perform_publish(activity) + perform(:publish, activity) end def publish(activity) do @@ -68,11 +79,88 @@ defmodule Pleroma.Web.Federator do |> Pleroma.Repo.insert() end - defp worker_args(queue) do - if max_attempts = Pleroma.Config.get([:workers, :retries, queue]) do - [max_attempts: max_attempts] + # Job Worker Callbacks + + @spec perform(atom(), module(), any()) :: {:ok, any()} | {:error, any()} + def perform(:publish_one, module, params) do + apply(module, :publish_one, [params]) + end + + def perform(:publish, activity) do + Logger.debug(fn -> "Running publish for #{activity.data["id"]}" end) + + with %User{} = actor <- User.get_cached_by_ap_id(activity.data["actor"]), + {:ok, actor} <- User.ensure_keys_present(actor) do + Publisher.publish(actor, activity) + end + end + + def perform(:incoming_doc, doc) do + Logger.info("Got document, trying to parse") + OStatus.handle_incoming(doc) + end + + def perform(:incoming_ap_doc, params) do + Logger.info("Handling incoming AP activity") + + params = Utils.normalize_params(params) + + # NOTE: we use the actor ID to do the containment, this is fine because an + # actor shouldn't be acting on objects outside their own AP server. + with {:ok, _user} <- ap_enabled_actor(params["actor"]), + nil <- Activity.normalize(params["id"]), + :ok <- Containment.contain_origin_from_id(params["actor"], params), + {:ok, activity} <- Transmogrifier.handle_incoming(params) do + {:ok, activity} else - [] + %Activity{} -> + Logger.info("Already had #{params["id"]}") + :error + + _e -> + # Just drop those for now + Logger.info("Unhandled activity") + Logger.info(Jason.encode!(params, pretty: true)) + :error + end + end + + def perform(:request_subscription, websub) do + Logger.debug("Refreshing #{websub.topic}") + + with {:ok, websub} <- Websub.request_subscription(websub) do + Logger.debug("Successfully refreshed #{websub.topic}") + else + _e -> Logger.debug("Couldn't refresh #{websub.topic}") + end + end + + def perform(:verify_websub, websub) do + Logger.debug(fn -> + "Running WebSub verification for #{websub.id} (#{websub.topic}, #{websub.callback})" + end) + + Websub.verify(websub) + end + + def perform(:refresh_subscriptions) do + Logger.debug("Federator running refresh subscriptions") + Websub.refresh_subscriptions() + + spawn(fn -> + # 6 hours + Process.sleep(1000 * 60 * 60 * 6) + refresh_subscriptions() + end) + end + + def ap_enabled_actor(id) do + user = User.get_cached_by_ap_id(id) + + if User.ap_enabled?(user) do + {:ok, user} + else + ActivityPub.make_user_from_ap_id(id) end end end diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex index dca85244..c0c9c365 100644 --- a/lib/pleroma/web/oauth/token/clean_worker.ex +++ b/lib/pleroma/web/oauth/token/clean_worker.ex @@ -14,9 +14,12 @@ defmodule Pleroma.Web.OAuth.Token.CleanWorker do [:oauth2, :clean_expired_tokens_interval], 86_400_000 ) - @queue :background + alias Pleroma.Repo alias Pleroma.Web.OAuth.Token + alias Pleroma.Workers.BackgroundWorker + + defdelegate worker_args(queue), to: Pleroma.Workers.Helper def start_link, do: GenServer.start_link(__MODULE__, nil) @@ -31,8 +34,11 @@ defmodule Pleroma.Web.OAuth.Token.CleanWorker do @doc false def handle_info(:perform, state) do + %{"op" => "clean_expired_tokens"} + |> BackgroundWorker.new(worker_args(:background)) + |> Repo.insert() + Process.send_after(self(), :perform, @interval) - PleromaJobQueue.enqueue(@queue, __MODULE__, [:clean]) {:noreply, state} end diff --git a/lib/pleroma/web/push/push.ex b/lib/pleroma/web/push/push.ex index 729dad02..b4f0e512 100644 --- a/lib/pleroma/web/push/push.ex +++ b/lib/pleroma/web/push/push.ex @@ -3,10 +3,13 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Push do - alias Pleroma.Web.Push.Impl + alias Pleroma.Repo + alias Pleroma.Workers.WebPusher require Logger + defdelegate worker_args(queue), to: Pleroma.Workers.Helper + def init do unless enabled() do Logger.warn(""" @@ -31,6 +34,9 @@ defmodule Pleroma.Web.Push do end end - def send(notification), - do: PleromaJobQueue.enqueue(:web_push, Impl, [notification]) + def send(notification) do + %{"op" => "web_push", "notification_id" => notification.id} + |> WebPusher.new(worker_args(:web_push)) + |> Repo.insert() + end end diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 3405bd3b..7ba4ad30 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -265,12 +265,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do String.split(line, ",") |> List.first() end) |> List.delete("Account address") do - PleromaJobQueue.enqueue(:background, User, [ - :follow_import, - follower, - followed_identifiers - ]) - + User.follow_import(follower, followed_identifiers) json(conn, "job started") end end @@ -281,12 +276,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do def blocks_import(%{assigns: %{user: blocker}} = conn, %{"list" => list}) do with blocked_identifiers <- String.split(list) do - PleromaJobQueue.enqueue(:background, User, [ - :blocks_import, - blocker, - blocked_identifiers - ]) - + User.blocks_import(blocker, blocked_identifiers) json(conn, "job started") end end diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex new file mode 100644 index 00000000..3ab2b6bc --- /dev/null +++ b/lib/pleroma/workers/background_worker.ex @@ -0,0 +1,66 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.BackgroundWorker do + alias Pleroma.Activity + alias Pleroma.User + alias Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy + alias Pleroma.Web.OAuth.Token.CleanWorker + + # Note: `max_attempts` is intended to be overridden in `new/1` call + use Oban.Worker, + queue: "background", + max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + + @impl Oban.Worker + def perform(%{"op" => "fetch_initial_posts", "user_id" => user_id}) do + user = User.get_by_id(user_id) + User.perform(:fetch_initial_posts, user) + end + + def perform(%{"op" => "deactivate_user", "user_id" => user_id, "status" => status}) do + user = User.get_by_id(user_id) + User.perform(:deactivate_async, user, status) + end + + def perform(%{"op" => "delete_user", "user_id" => user_id}) do + user = User.get_by_id(user_id) + User.perform(:delete, user) + end + + def perform(%{ + "op" => "blocks_import", + "blocker_id" => blocker_id, + "blocked_identifiers" => blocked_identifiers + }) do + blocker = User.get_by_id(blocker_id) + User.perform(:blocks_import, blocker, blocked_identifiers) + end + + def perform(%{ + "op" => "follow_import", + "follower_id" => follower_id, + "followed_identifiers" => followed_identifiers + }) do + follower = User.get_by_id(follower_id) + User.perform(:follow_import, follower, followed_identifiers) + end + + def perform(%{"op" => "clean_expired_tokens"}) do + CleanWorker.perform(:clean) + end + + def perform(%{"op" => "media_proxy_preload", "message" => message}) do + MediaProxyWarmingPolicy.perform(:preload, message) + end + + def perform(%{"op" => "media_proxy_prefetch", "url" => url}) do + MediaProxyWarmingPolicy.perform(:prefetch, url) + end + + def perform(%{"op" => "fetch_data_for_activity", "activity_id" => activity_id}) do + activity = Activity.get_by_id(activity_id) + Pleroma.Web.RichMedia.Helpers.perform(:fetch, activity) + end +end diff --git a/lib/pleroma/workers/helper.ex b/lib/pleroma/workers/helper.ex new file mode 100644 index 00000000..3286ce0e --- /dev/null +++ b/lib/pleroma/workers/helper.ex @@ -0,0 +1,13 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Helper do + def worker_args(queue) do + if max_attempts = Pleroma.Config.get([:workers, :retries, queue]) do + [max_attempts: max_attempts] + else + [] + end + end +end diff --git a/lib/pleroma/workers/mailer.ex b/lib/pleroma/workers/mailer.ex new file mode 100644 index 00000000..da7fa6fd --- /dev/null +++ b/lib/pleroma/workers/mailer.ex @@ -0,0 +1,18 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Mailer do + alias Pleroma.User + + # Note: `max_attempts` is intended to be overridden in `new/1` call + use Oban.Worker, + queue: "mailer", + max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + + @impl Oban.Worker + def perform(%{"op" => "digest_email", "user_id" => user_id}) do + user = User.get_by_id(user_id) + Pleroma.DigestEmailWorker.perform(user) + end +end diff --git a/lib/pleroma/workers/publisher.ex b/lib/pleroma/workers/publisher.ex index 67871977..c890ffb7 100644 --- a/lib/pleroma/workers/publisher.ex +++ b/lib/pleroma/workers/publisher.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Workers.Publisher do alias Pleroma.Activity - alias Pleroma.User + alias Pleroma.Web.Federator # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, @@ -13,23 +13,11 @@ defmodule Pleroma.Workers.Publisher do @impl Oban.Worker def perform(%{"op" => "publish", "activity_id" => activity_id}) do - with %Activity{} = activity <- Activity.get_by_id(activity_id) do - perform_publish(activity) - else - _ -> raise "Non-existing activity: #{activity_id}" - end + activity = Activity.get_by_id(activity_id) + Federator.perform(:publish, activity) end def perform(%{"op" => "publish_one", "module" => module_name, "params" => params}) do - module_name - |> String.to_atom() - |> apply(:publish_one, [params]) - end - - def perform_publish(%Activity{} = activity) do - with %User{} = actor <- User.get_cached_by_ap_id(activity.data["actor"]), - {:ok, actor} <- User.ensure_keys_present(actor) do - Pleroma.Web.Federator.Publisher.publish(actor, activity) - end + Federator.perform(:publish_one, String.to_atom(module_name), params) end end diff --git a/lib/pleroma/workers/receiver.ex b/lib/pleroma/workers/receiver.ex index 43558b4e..d3de9571 100644 --- a/lib/pleroma/workers/receiver.ex +++ b/lib/pleroma/workers/receiver.ex @@ -3,15 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.Receiver do - alias Pleroma.Activity - alias Pleroma.Object.Containment - alias Pleroma.User - alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.ActivityPub.Transmogrifier - alias Pleroma.Web.ActivityPub.Utils - alias Pleroma.Web.OStatus - - require Logger + alias Pleroma.Web.Federator # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, @@ -20,42 +12,10 @@ defmodule Pleroma.Workers.Receiver do @impl Oban.Worker def perform(%{"op" => "incoming_doc", "body" => doc}) do - Logger.info("Got incoming document, trying to parse") - OStatus.handle_incoming(doc) + Federator.perform(:incoming_doc, doc) end def perform(%{"op" => "incoming_ap_doc", "params" => params}) do - Logger.info("Handling incoming AP activity") - - params = Utils.normalize_params(params) - - # NOTE: we use the actor ID to do the containment, this is fine because an - # actor shouldn't be acting on objects outside their own AP server. - with {:ok, _user} <- ap_enabled_actor(params["actor"]), - nil <- Activity.normalize(params["id"]), - :ok <- Containment.contain_origin_from_id(params["actor"], params), - {:ok, activity} <- Transmogrifier.handle_incoming(params) do - {:ok, activity} - else - %Activity{} -> - Logger.info("Already had #{params["id"]}") - :error - - _e -> - # Just drop those for now - Logger.info("Unhandled activity") - Logger.info(Jason.encode!(params, pretty: true)) - :error - end - end - - defp ap_enabled_actor(id) do - user = User.get_cached_by_ap_id(id) - - if User.ap_enabled?(user) do - {:ok, user} - else - ActivityPub.make_user_from_ap_id(id) - end + Federator.perform(:incoming_ap_doc, params) end end diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex new file mode 100644 index 00000000..a49834fd --- /dev/null +++ b/lib/pleroma/workers/scheduled_activity_worker.ex @@ -0,0 +1,15 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.ScheduledActivityWorker do + # Note: `max_attempts` is intended to be overridden in `new/1` call + use Oban.Worker, + queue: "scheduled_activities", + max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + + @impl Oban.Worker + def perform(%{"op" => "execute", "activity_id" => activity_id}) do + Pleroma.ScheduledActivityWorker.perform(:execute, activity_id) + end +end diff --git a/lib/pleroma/workers/subscriber.ex b/lib/pleroma/workers/subscriber.ex index a8c01bb1..6af3ad0a 100644 --- a/lib/pleroma/workers/subscriber.ex +++ b/lib/pleroma/workers/subscriber.ex @@ -4,11 +4,9 @@ defmodule Pleroma.Workers.Subscriber do alias Pleroma.Repo - alias Pleroma.Web.Websub + alias Pleroma.Web.Federator alias Pleroma.Web.Websub.WebsubClientSubscription - require Logger - # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, queue: "federator_outgoing", @@ -16,29 +14,16 @@ defmodule Pleroma.Workers.Subscriber do @impl Oban.Worker def perform(%{"op" => "refresh_subscriptions"}) do - Websub.refresh_subscriptions() - # Schedule the next run in 6 hours - Pleroma.Web.Federator.refresh_subscriptions(schedule_in: 3600 * 6) + Federator.perform(:refresh_subscriptions) end def perform(%{"op" => "request_subscription", "websub_id" => websub_id}) do websub = Repo.get(WebsubClientSubscription, websub_id) - Logger.debug("Refreshing #{websub.topic}") - - with {:ok, websub} <- Websub.request_subscription(websub) do - Logger.debug("Successfully refreshed #{websub.topic}") - else - _e -> Logger.debug("Couldn't refresh #{websub.topic}") - end + Federator.perform(:request_subscription, websub) end def perform(%{"op" => "verify_websub", "websub_id" => websub_id}) do websub = Repo.get(WebsubClientSubscription, websub_id) - - Logger.debug(fn -> - "Running WebSub verification for #{websub.id} (#{websub.topic}, #{websub.callback})" - end) - - Websub.verify(websub) + Federator.perform(:verify_websub, websub) end end diff --git a/lib/pleroma/workers/transmogrifier.ex b/lib/pleroma/workers/transmogrifier.ex new file mode 100644 index 00000000..c6b4fab4 --- /dev/null +++ b/lib/pleroma/workers/transmogrifier.ex @@ -0,0 +1,18 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Transmogrifier do + alias Pleroma.User + + # Note: `max_attempts` is intended to be overridden in `new/1` call + use Oban.Worker, + queue: "transmogrifier", + max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + + @impl Oban.Worker + def perform(%{"op" => "user_upgrade", "user_id" => user_id}) do + user = User.get_by_id(user_id) + Pleroma.Web.ActivityPub.Transmogrifier.perform(:user_upgrade, user) + end +end diff --git a/lib/pleroma/workers/web_pusher.ex b/lib/pleroma/workers/web_pusher.ex new file mode 100644 index 00000000..b99581eb --- /dev/null +++ b/lib/pleroma/workers/web_pusher.ex @@ -0,0 +1,19 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.WebPusher do + alias Pleroma.Notification + alias Pleroma.Repo + + # Note: `max_attempts` is intended to be overridden in `new/1` call + use Oban.Worker, + queue: "web_push", + max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + + @impl Oban.Worker + def perform(%{"op" => "web_push", "notification_id" => notification_id}) do + notification = Repo.get(Notification, notification_id) + Pleroma.Web.Push.Impl.perform(notification) + end +end diff --git a/test/activity_test.exs b/test/activity_test.exs index b9c12adb..658c4783 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -6,8 +6,8 @@ defmodule Pleroma.ActivityTest do use Pleroma.DataCase alias Pleroma.Activity alias Pleroma.Bookmark - alias Pleroma.ObanHelpers alias Pleroma.Object + alias Pleroma.Tests.ObanHelpers alias Pleroma.ThreadMute import Pleroma.Factory diff --git a/test/conversation_test.exs b/test/conversation_test.exs index 2ebbcab7..f917aa69 100644 --- a/test/conversation_test.exs +++ b/test/conversation_test.exs @@ -28,7 +28,7 @@ defmodule Pleroma.ConversationTest do {:ok, _activity} = CommonAPI.post(user, %{"visibility" => "direct", "status" => "hey @#{other_user.nickname}"}) - Pleroma.ObanHelpers.perform_all() + Pleroma.Tests.ObanHelpers.perform_all() Repo.delete_all(Conversation) Repo.delete_all(Conversation.Participation) diff --git a/test/notification_test.exs b/test/notification_test.exs index 80ea2a08..e1c9f4f9 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -8,6 +8,7 @@ defmodule Pleroma.NotificationTest do import Pleroma.Factory alias Pleroma.Notification + alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.CommonAPI @@ -621,7 +622,8 @@ defmodule Pleroma.NotificationTest do refute Enum.empty?(Notification.for_user(other_user)) - User.delete(user) + {:ok, job} = User.delete(user) + ObanHelpers.perform(job) assert Enum.empty?(Notification.for_user(other_user)) end @@ -666,6 +668,7 @@ defmodule Pleroma.NotificationTest do } {:ok, _delete_activity} = Transmogrifier.handle_incoming(delete_user_message) + ObanHelpers.perform_all() assert Enum.empty?(Notification.for_user(local_user)) end diff --git a/test/support/oban_helpers.ex b/test/support/oban_helpers.ex index ecc03ba1..d379c9ec 100644 --- a/test/support/oban_helpers.ex +++ b/test/support/oban_helpers.ex @@ -2,7 +2,7 @@ # Copyright © 2017-2018 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.ObanHelpers do +defmodule Pleroma.Tests.ObanHelpers do @moduledoc """ Oban test helpers. """ diff --git a/test/user_test.exs b/test/user_test.exs index 8617752d..9c2117a0 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -5,9 +5,9 @@ defmodule Pleroma.UserTest do alias Pleroma.Activity alias Pleroma.Builders.UserBuilder - alias Pleroma.ObanHelpers alias Pleroma.Object alias Pleroma.Repo + alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.CommonAPI @@ -676,7 +676,9 @@ defmodule Pleroma.UserTest do user3.nickname ] - result = User.follow_import(user1, identifiers) + {:ok, job} = User.follow_import(user1, identifiers) + result = ObanHelpers.perform(job) + assert is_list(result) assert result == [user2, user3] end @@ -887,7 +889,9 @@ defmodule Pleroma.UserTest do user3.nickname ] - result = User.blocks_import(user1, identifiers) + {:ok, job} = User.blocks_import(user1, identifiers) + result = ObanHelpers.perform(job) + assert is_list(result) assert result == [user2, user3] end @@ -1013,7 +1017,8 @@ defmodule Pleroma.UserTest do {:ok, like_two, _} = CommonAPI.favorite(activity.id, follower) {:ok, repeat, _} = CommonAPI.repeat(activity_two.id, user) - {:ok, _} = User.delete(user) + {:ok, job} = User.delete(user) + {:ok, _user} = ObanHelpers.perform(job) follower = User.get_cached_by_id(follower.id) @@ -1043,7 +1048,8 @@ defmodule Pleroma.UserTest do {:ok, follower} = User.get_or_fetch_by_ap_id("http://mastodon.example.org/users/admin") {:ok, _} = User.follow(follower, user) - {:ok, _user} = User.delete(user) + {:ok, job} = User.delete(user) + {:ok, _user} = ObanHelpers.perform(job) assert ObanHelpers.member?( %{ @@ -1100,7 +1106,8 @@ defmodule Pleroma.UserTest do test "User.delete() plugs any possible zombie objects" do user = insert(:user) - {:ok, _} = User.delete(user) + {:ok, job} = User.delete(user) + {:ok, _} = ObanHelpers.perform(job) {:ok, cached_user} = Cachex.get(:user_cache, "ap_id:#{user.ap_id}") diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index d7f0a826..f46353fd 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -9,8 +9,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do import Pleroma.Factory alias Pleroma.Activity alias Pleroma.Instances - alias Pleroma.ObanHelpers alias Pleroma.Object + alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.ActivityPub.ObjectView alias Pleroma.Web.ActivityPub.UserView diff --git a/test/web/activity_pub/mrf/mediaproxy_warming_policy_test.exs b/test/web/activity_pub/mrf/mediaproxy_warming_policy_test.exs index 372e789b..95a809d2 100644 --- a/test/web/activity_pub/mrf/mediaproxy_warming_policy_test.exs +++ b/test/web/activity_pub/mrf/mediaproxy_warming_policy_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicyTest do use Pleroma.DataCase alias Pleroma.HTTP + alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy import Mock @@ -24,6 +25,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicyTest do test "it prefetches media proxy URIs" do with_mock HTTP, get: fn _, _, _ -> {:ok, []} end do MediaProxyWarmingPolicy.filter(@message) + + ObanHelpers.perform_all() + # Performing jobs which has been just enqueued + ObanHelpers.perform_all() + assert called(HTTP.get(:_, :_, :_)) end end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index e7498e00..52f46c14 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -8,6 +8,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do alias Pleroma.Object alias Pleroma.Object.Fetcher alias Pleroma.Repo + alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier @@ -563,6 +564,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do |> Poison.decode!() {:ok, _} = Transmogrifier.handle_incoming(data) + ObanHelpers.perform_all() refute User.get_cached_by_ap_id(ap_id) end @@ -1132,6 +1134,8 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert user.info.note_count == 1 {:ok, user} = Transmogrifier.upgrade_user_from_ap_id("https://niu.moe/users/rye") + ObanHelpers.perform_all() + assert user.info.ap_enabled assert user.info.note_count == 1 assert user.follower_address == "https://niu.moe/users/rye/followers" diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index e0be4342..9ca341b6 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -4,7 +4,7 @@ defmodule Pleroma.Web.FederatorTest do alias Pleroma.Instances - alias Pleroma.ObanHelpers + alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.CommonAPI alias Pleroma.Web.Federator alias Pleroma.Workers.Publisher, as: PublisherWorker diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index 640579c0..e3f129f7 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -4,9 +4,11 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do use Pleroma.Web.ConnCase + use Oban.Testing, repo: Pleroma.Repo alias Pleroma.Notification alias Pleroma.Repo + alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.CommonAPI import Pleroma.Factory @@ -50,8 +52,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do {File, [], read!: fn "follow_list.txt" -> "Account address,Show boosts\n#{user2.ap_id},true" - end}, - {PleromaJobQueue, [:passthrough], []} + end} ]) do response = conn @@ -59,15 +60,16 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do |> post("/api/pleroma/follow_import", %{"list" => %Plug.Upload{path: "follow_list.txt"}}) |> json_response(:ok) - assert called( - PleromaJobQueue.enqueue( - :background, - User, - [:follow_import, user1, [user2.ap_id]] - ) - ) - assert response == "job started" + + assert ObanHelpers.member?( + %{ + "op" => "follow_import", + "follower_id" => user1.id, + "followed_identifiers" => [user2.ap_id] + }, + all_enqueued(worker: Pleroma.Workers.BackgroundWorker) + ) end end @@ -126,8 +128,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do user3 = insert(:user) with_mocks([ - {File, [], read!: fn "blocks_list.txt" -> "#{user2.ap_id} #{user3.ap_id}" end}, - {PleromaJobQueue, [:passthrough], []} + {File, [], read!: fn "blocks_list.txt" -> "#{user2.ap_id} #{user3.ap_id}" end} ]) do response = conn @@ -135,15 +136,16 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do |> post("/api/pleroma/blocks_import", %{"list" => %Plug.Upload{path: "blocks_list.txt"}}) |> json_response(:ok) - assert called( - PleromaJobQueue.enqueue( - :background, - User, - [:blocks_import, user1, [user2.ap_id, user3.ap_id]] - ) - ) - assert response == "job started" + + assert ObanHelpers.member?( + %{ + "op" => "blocks_import", + "blocker_id" => user1.id, + "blocked_identifiers" => [user2.ap_id, user3.ap_id] + }, + all_enqueued(worker: Pleroma.Workers.BackgroundWorker) + ) end end end @@ -607,6 +609,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do |> json_response(:ok) assert response == %{"status" => "success"} + ObanHelpers.perform_all() user = User.get_cached_by_id(user.id) diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index b704a558..41461087 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -6,7 +6,7 @@ defmodule Pleroma.Web.WebsubTest do use Pleroma.DataCase use Oban.Testing, repo: Pleroma.Repo - alias Pleroma.ObanHelpers + alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.Router.Helpers alias Pleroma.Web.Websub alias Pleroma.Web.Websub.WebsubClientSubscription From a180c1360ecdbed76eccf3435bb2c831356746bc Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 14 Aug 2019 21:42:21 +0300 Subject: [PATCH 005/138] [#1149] Oban mailer job. Adjusted tests. --- lib/pleroma/application.ex | 1 + lib/pleroma/emails/mailer.ex | 13 ++++++++++++- lib/pleroma/workers/mailer.ex | 9 +++++++++ test/mix/tasks/pleroma.digest_test.exs | 3 +++ .../mastodon_api/mastodon_api_controller_test.exs | 4 ++++ .../web/twitter_api/twitter_api_controller_test.exs | 4 ++++ test/web/twitter_api/twitter_api_test.exs | 2 ++ 7 files changed, 35 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 5550a490..7cf60f44 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -233,6 +233,7 @@ defmodule Pleroma.Application do defp after_supervisor_start do with digest_config <- Application.get_env(:pleroma, :email_notifications)[:digest], true <- digest_config[:active] do + # TODO: consider replacing with `quantum` scheduler PleromaJobQueue.schedule( digest_config[:schedule], :digest_emails, diff --git a/lib/pleroma/emails/mailer.ex b/lib/pleroma/emails/mailer.ex index 2e4657b7..bb534f60 100644 --- a/lib/pleroma/emails/mailer.ex +++ b/lib/pleroma/emails/mailer.ex @@ -9,6 +9,8 @@ defmodule Pleroma.Emails.Mailer do The module contains functions to delivery email using Swoosh.Mailer. """ + alias Pleroma.Repo + alias Pleroma.Workers.Mailer, as: MailerWorker alias Swoosh.DeliveryError @otp_app :pleroma @@ -17,9 +19,18 @@ defmodule Pleroma.Emails.Mailer do @spec enabled?() :: boolean() def enabled?, do: Pleroma.Config.get([__MODULE__, :enabled]) + defdelegate worker_args(queue), to: Pleroma.Workers.Helper + @doc "add email to queue" def deliver_async(email, config \\ []) do - PleromaJobQueue.enqueue(:mailer, __MODULE__, [:deliver_async, email, config]) + encoded_email = + email + |> :erlang.term_to_binary() + |> Base.encode64() + + %{"op" => "email", "encoded_email" => encoded_email, "config" => config} + |> MailerWorker.new(worker_args(:mailer)) + |> Repo.insert() end @doc "callback to perform send email from queue" diff --git a/lib/pleroma/workers/mailer.ex b/lib/pleroma/workers/mailer.ex index da7fa6fd..8bf9952b 100644 --- a/lib/pleroma/workers/mailer.ex +++ b/lib/pleroma/workers/mailer.ex @@ -11,6 +11,15 @@ defmodule Pleroma.Workers.Mailer do max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) @impl Oban.Worker + def perform(%{"op" => "email", "encoded_email" => encoded_email, "config" => config}) do + email = + encoded_email + |> Base.decode64!() + |> :erlang.binary_to_term() + + Pleroma.Emails.Mailer.deliver(email, config) + end + def perform(%{"op" => "digest_email", "user_id" => user_id}) do user = User.get_by_id(user_id) Pleroma.DigestEmailWorker.perform(user) diff --git a/test/mix/tasks/pleroma.digest_test.exs b/test/mix/tasks/pleroma.digest_test.exs index 595f64ed..5fbeac0d 100644 --- a/test/mix/tasks/pleroma.digest_test.exs +++ b/test/mix/tasks/pleroma.digest_test.exs @@ -4,6 +4,7 @@ defmodule Mix.Tasks.Pleroma.DigestTest do import Pleroma.Factory import Swoosh.TestAssertions + alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.CommonAPI setup_all do @@ -39,6 +40,8 @@ defmodule Mix.Tasks.Pleroma.DigestTest do :ok = Mix.Tasks.Pleroma.Digest.run(["test", user2.nickname, yesterday_date]) + ObanHelpers.perform_all() + assert_receive {:mix_shell, :info, [message]} assert message =~ "Digest email have been sent" diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index e49c4cc2..be9ff256 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -11,6 +11,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do alias Pleroma.Object alias Pleroma.Repo alias Pleroma.ScheduledActivity + alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.CommonAPI @@ -3871,6 +3872,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do end test "it sends an email to user", %{user: user} do + ObanHelpers.perform_all() token_record = Repo.get_by(Pleroma.PasswordResetToken, user_id: user.id) email = Pleroma.Emails.UserEmail.password_reset_email(user, token_record.token) @@ -3934,6 +3936,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do |> post("/api/v1/pleroma/accounts/confirmation_resend?email=#{user.email}") |> json_response(:no_content) + ObanHelpers.perform_all() + email = Pleroma.Emails.UserEmail.account_confirmation_email(user) notify_email = Pleroma.Config.get([:instance, :notify_email]) instance_name = Pleroma.Config.get([:instance, :name]) diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index 8bb8aa36..9ac4ff92 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -12,6 +12,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo + alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.CommonAPI @@ -1099,6 +1100,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do end test "it sends an email to user", %{user: user} do + ObanHelpers.perform_all() token_record = Repo.get_by(Pleroma.PasswordResetToken, user_id: user.id) email = Pleroma.Emails.UserEmail.password_reset_email(user, token_record.token) @@ -1209,6 +1211,8 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do |> assign(:user, user) |> post("/api/account/resend_confirmation_email?email=#{user.email}") + ObanHelpers.perform_all() + email = Pleroma.Emails.UserEmail.account_confirmation_email(user) notify_email = Pleroma.Config.get([:instance, :notify_email]) instance_name = Pleroma.Config.get([:instance, :name]) diff --git a/test/web/twitter_api/twitter_api_test.exs b/test/web/twitter_api/twitter_api_test.exs index cbe83852..bf063a0d 100644 --- a/test/web/twitter_api/twitter_api_test.exs +++ b/test/web/twitter_api/twitter_api_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPITest do alias Pleroma.Activity alias Pleroma.Object alias Pleroma.Repo + alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.UserInviteToken alias Pleroma.Web.ActivityPub.ActivityPub @@ -321,6 +322,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPITest do } {:ok, user} = TwitterAPI.register_user(data) + ObanHelpers.perform_all() assert user.info.confirmation_pending From c29686309eaf2cdae039ce813755c0e23cdc4a03 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 23 Aug 2019 09:23:10 +0300 Subject: [PATCH 006/138] [#1149] Upgraded `oban` from 0.6.0 to 0.7.1. --- config/config.exs | 1 - lib/pleroma/application.ex | 5 +-- lib/pleroma/workers/background_worker.ex | 42 +++++++++++-------- lib/pleroma/workers/mailer.ex | 6 +-- lib/pleroma/workers/publisher.ex | 6 +-- lib/pleroma/workers/receiver.ex | 6 +-- .../workers/scheduled_activity_worker.ex | 4 +- lib/pleroma/workers/subscriber.ex | 8 ++-- lib/pleroma/workers/transmogrifier.ex | 4 +- lib/pleroma/workers/web_pusher.ex | 4 +- mix.exs | 2 +- mix.lock | 10 ++--- test/support/oban_helpers.ex | 2 +- 13 files changed, 51 insertions(+), 49 deletions(-) diff --git a/config/config.exs b/config/config.exs index 9794997d..1a6348bc 100644 --- a/config/config.exs +++ b/config/config.exs @@ -469,7 +469,6 @@ config :pleroma, Oban, config :pleroma, :workers, retries: [ - compile_time_default: 1, federator_incoming: 5, federator_outgoing: 5 ] diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 2e2922d2..384b03aa 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -41,10 +41,7 @@ defmodule Pleroma.Application do hackney_pool_children() ++ [ Pleroma.Stats, - %{ - id: Oban, - start: {Oban, :start_link, [Application.get_env(:pleroma, Oban)]} - }, + {Oban, Application.get_env(:pleroma, Oban)}, %{ id: :web_push_init, start: {Task, :start_link, [&Pleroma.Web.Push.init/0]}, diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index 3ab2b6bc..3c021b9b 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -11,55 +11,61 @@ defmodule Pleroma.Workers.BackgroundWorker do # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, queue: "background", - max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + max_attempts: 1 @impl Oban.Worker - def perform(%{"op" => "fetch_initial_posts", "user_id" => user_id}) do + def perform(%{"op" => "fetch_initial_posts", "user_id" => user_id}, _job) do user = User.get_by_id(user_id) User.perform(:fetch_initial_posts, user) end - def perform(%{"op" => "deactivate_user", "user_id" => user_id, "status" => status}) do + def perform(%{"op" => "deactivate_user", "user_id" => user_id, "status" => status}, _job) do user = User.get_by_id(user_id) User.perform(:deactivate_async, user, status) end - def perform(%{"op" => "delete_user", "user_id" => user_id}) do + def perform(%{"op" => "delete_user", "user_id" => user_id}, _job) do user = User.get_by_id(user_id) User.perform(:delete, user) end - def perform(%{ - "op" => "blocks_import", - "blocker_id" => blocker_id, - "blocked_identifiers" => blocked_identifiers - }) do + def perform( + %{ + "op" => "blocks_import", + "blocker_id" => blocker_id, + "blocked_identifiers" => blocked_identifiers + }, + _job + ) do blocker = User.get_by_id(blocker_id) User.perform(:blocks_import, blocker, blocked_identifiers) end - def perform(%{ - "op" => "follow_import", - "follower_id" => follower_id, - "followed_identifiers" => followed_identifiers - }) do + def perform( + %{ + "op" => "follow_import", + "follower_id" => follower_id, + "followed_identifiers" => followed_identifiers + }, + _job + ) do follower = User.get_by_id(follower_id) User.perform(:follow_import, follower, followed_identifiers) end - def perform(%{"op" => "clean_expired_tokens"}) do + def perform(%{"op" => "clean_expired_tokens"}, _job) do CleanWorker.perform(:clean) end - def perform(%{"op" => "media_proxy_preload", "message" => message}) do + def perform(%{"op" => "media_proxy_preload", "message" => message}, _job) do MediaProxyWarmingPolicy.perform(:preload, message) end - def perform(%{"op" => "media_proxy_prefetch", "url" => url}) do + def perform(%{"op" => "media_proxy_prefetch", "url" => url}, _job) do MediaProxyWarmingPolicy.perform(:prefetch, url) end - def perform(%{"op" => "fetch_data_for_activity", "activity_id" => activity_id}) do + def perform(%{"op" => "fetch_data_for_activity", "activity_id" => activity_id}, _job) do activity = Activity.get_by_id(activity_id) Pleroma.Web.RichMedia.Helpers.perform(:fetch, activity) end diff --git a/lib/pleroma/workers/mailer.ex b/lib/pleroma/workers/mailer.ex index 8bf9952b..1cce2ea0 100644 --- a/lib/pleroma/workers/mailer.ex +++ b/lib/pleroma/workers/mailer.ex @@ -8,10 +8,10 @@ defmodule Pleroma.Workers.Mailer do # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, queue: "mailer", - max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + max_attempts: 1 @impl Oban.Worker - def perform(%{"op" => "email", "encoded_email" => encoded_email, "config" => config}) do + def perform(%{"op" => "email", "encoded_email" => encoded_email, "config" => config}, _job) do email = encoded_email |> Base.decode64!() @@ -20,7 +20,7 @@ defmodule Pleroma.Workers.Mailer do Pleroma.Emails.Mailer.deliver(email, config) end - def perform(%{"op" => "digest_email", "user_id" => user_id}) do + def perform(%{"op" => "digest_email", "user_id" => user_id}, _job) do user = User.get_by_id(user_id) Pleroma.DigestEmailWorker.perform(user) end diff --git a/lib/pleroma/workers/publisher.ex b/lib/pleroma/workers/publisher.ex index c890ffb7..0a908458 100644 --- a/lib/pleroma/workers/publisher.ex +++ b/lib/pleroma/workers/publisher.ex @@ -9,15 +9,15 @@ defmodule Pleroma.Workers.Publisher do # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, queue: "federator_outgoing", - max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + max_attempts: 1 @impl Oban.Worker - def perform(%{"op" => "publish", "activity_id" => activity_id}) do + def perform(%{"op" => "publish", "activity_id" => activity_id}, _job) do activity = Activity.get_by_id(activity_id) Federator.perform(:publish, activity) end - def perform(%{"op" => "publish_one", "module" => module_name, "params" => params}) do + def perform(%{"op" => "publish_one", "module" => module_name, "params" => params}, _job) do Federator.perform(:publish_one, String.to_atom(module_name), params) end end diff --git a/lib/pleroma/workers/receiver.ex b/lib/pleroma/workers/receiver.ex index d3de9571..4ee270d7 100644 --- a/lib/pleroma/workers/receiver.ex +++ b/lib/pleroma/workers/receiver.ex @@ -8,14 +8,14 @@ defmodule Pleroma.Workers.Receiver do # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, queue: "federator_incoming", - max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + max_attempts: 1 @impl Oban.Worker - def perform(%{"op" => "incoming_doc", "body" => doc}) do + def perform(%{"op" => "incoming_doc", "body" => doc}, _job) do Federator.perform(:incoming_doc, doc) end - def perform(%{"op" => "incoming_ap_doc", "params" => params}) do + def perform(%{"op" => "incoming_ap_doc", "params" => params}, _job) do Federator.perform(:incoming_ap_doc, params) end end diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex index a49834fd..d9724c78 100644 --- a/lib/pleroma/workers/scheduled_activity_worker.ex +++ b/lib/pleroma/workers/scheduled_activity_worker.ex @@ -6,10 +6,10 @@ defmodule Pleroma.Workers.ScheduledActivityWorker do # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, queue: "scheduled_activities", - max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + max_attempts: 1 @impl Oban.Worker - def perform(%{"op" => "execute", "activity_id" => activity_id}) do + def perform(%{"op" => "execute", "activity_id" => activity_id}, _job) do Pleroma.ScheduledActivityWorker.perform(:execute, activity_id) end end diff --git a/lib/pleroma/workers/subscriber.ex b/lib/pleroma/workers/subscriber.ex index 6af3ad0a..783c4417 100644 --- a/lib/pleroma/workers/subscriber.ex +++ b/lib/pleroma/workers/subscriber.ex @@ -10,19 +10,19 @@ defmodule Pleroma.Workers.Subscriber do # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, queue: "federator_outgoing", - max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + max_attempts: 1 @impl Oban.Worker - def perform(%{"op" => "refresh_subscriptions"}) do + def perform(%{"op" => "refresh_subscriptions"}, _job) do Federator.perform(:refresh_subscriptions) end - def perform(%{"op" => "request_subscription", "websub_id" => websub_id}) do + def perform(%{"op" => "request_subscription", "websub_id" => websub_id}, _job) do websub = Repo.get(WebsubClientSubscription, websub_id) Federator.perform(:request_subscription, websub) end - def perform(%{"op" => "verify_websub", "websub_id" => websub_id}) do + def perform(%{"op" => "verify_websub", "websub_id" => websub_id}, _job) do websub = Repo.get(WebsubClientSubscription, websub_id) Federator.perform(:verify_websub, websub) end diff --git a/lib/pleroma/workers/transmogrifier.ex b/lib/pleroma/workers/transmogrifier.ex index c6b4fab4..e13202c0 100644 --- a/lib/pleroma/workers/transmogrifier.ex +++ b/lib/pleroma/workers/transmogrifier.ex @@ -8,10 +8,10 @@ defmodule Pleroma.Workers.Transmogrifier do # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, queue: "transmogrifier", - max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + max_attempts: 1 @impl Oban.Worker - def perform(%{"op" => "user_upgrade", "user_id" => user_id}) do + def perform(%{"op" => "user_upgrade", "user_id" => user_id}, _job) do user = User.get_by_id(user_id) Pleroma.Web.ActivityPub.Transmogrifier.perform(:user_upgrade, user) end diff --git a/lib/pleroma/workers/web_pusher.ex b/lib/pleroma/workers/web_pusher.ex index b99581eb..7b78bb3e 100644 --- a/lib/pleroma/workers/web_pusher.ex +++ b/lib/pleroma/workers/web_pusher.ex @@ -9,10 +9,10 @@ defmodule Pleroma.Workers.WebPusher do # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, queue: "web_push", - max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + max_attempts: 1 @impl Oban.Worker - def perform(%{"op" => "web_push", "notification_id" => notification_id}) do + def perform(%{"op" => "web_push", "notification_id" => notification_id}, _job) do notification = Repo.get(Notification, notification_id) Pleroma.Web.Push.Impl.perform(notification) end diff --git a/mix.exs b/mix.exs index b651520e..eb023313 100644 --- a/mix.exs +++ b/mix.exs @@ -101,7 +101,7 @@ defmodule Pleroma.Mixfile do {:phoenix_ecto, "~> 4.0"}, {:ecto_sql, "~> 3.1"}, {:postgrex, ">= 0.13.5"}, - {:oban, "~> 0.6"}, + {:oban, "~> 0.7"}, {:gettext, "~> 0.15"}, {:comeonin, "~> 4.1.1"}, {:pbkdf2_elixir, "~> 0.12.3"}, diff --git a/mix.lock b/mix.lock index 52932c9e..8b859637 100644 --- a/mix.lock +++ b/mix.lock @@ -17,12 +17,12 @@ "credo": {:hex, :credo, "0.9.3", "76fa3e9e497ab282e0cf64b98a624aa11da702854c52c82db1bf24e54ab7c97a", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, "crontab": {:hex, :crontab, "1.1.7", "b9219f0bdc8678b94143655a8f229716c5810c0636a4489f98c0956137e53985", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]}, - "db_connection": {:hex, :db_connection, "2.0.6", "bde2f85d047969c5b5800cb8f4b3ed6316c8cb11487afedac4aa5f93fd39abfa", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, + "db_connection": {:hex, :db_connection, "2.1.1", "a51e8a2ee54ef2ae6ec41a668c85787ed40cb8944928c191280fe34c15b76ae5", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, "decimal": {:hex, :decimal, "1.8.0", "ca462e0d885f09a1c5a342dbd7c1dcf27ea63548c65a65e67334f4b61803822e", [:mix], [], "hexpm"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"}, "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"}, - "ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, - "ecto_sql": {:hex, :ecto_sql, "3.1.3", "2c536139190492d9de33c5fefac7323c5eaaa82e1b9bf93482a14649042f7cd9", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "ecto": {:hex, :ecto, "3.1.7", "fa21d06ef56cdc2fdaa62574e8c3ba34a2751d44ea34c30bc65f0728421043e5", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "ecto_sql": {:hex, :ecto_sql, "3.1.6", "1e80e30d16138a729c717f73dcb938590bcdb3a4502f3012414d0cbb261045d8", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0 or ~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"}, "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"}, "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"}, @@ -57,7 +57,7 @@ "mogrify": {:hex, :mogrify, "0.6.1", "de1b527514f2d95a7bbe9642eb556061afb337e220cf97adbf3a4e6438ed70af", [:mix], [], "hexpm"}, "mox": {:hex, :mox, "0.5.1", "f86bb36026aac1e6f924a4b6d024b05e9adbed5c63e8daa069bd66fb3292165b", [:mix], [], "hexpm"}, "nimble_parsec": {:hex, :nimble_parsec, "0.5.0", "90e2eca3d0266e5c53f8fbe0079694740b9c91b6747f2b7e3c5d21966bba8300", [:mix], [], "hexpm"}, - "oban": {:hex, :oban, "0.6.0", "8b9b861355610e703e58a878bc29959f3f0e1b4cd1e90d785cf2bb2498d3b893", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "oban": {:hex, :oban, "0.7.1", "171bdd1b69c1a4a839f8c768f5e962fc22d1de1513d459fb6b8e0cbd34817a9a", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.3", "6706a148809a29c306062862c803406e88f048277f6e85b68faf73291e820b84", [:mix], [], "hexpm"}, "phoenix": {:hex, :phoenix, "1.4.9", "746d098e10741c334d88143d3c94cab1756435f94387a63441792e66ec0ee974", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, @@ -71,7 +71,7 @@ "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"}, "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, - "postgrex": {:hex, :postgrex, "0.14.3", "5754dee2fdf6e9e508cbf49ab138df964278700b764177e8f3871e658b345a1e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "postgrex": {:hex, :postgrex, "0.15.0", "dd5349161019caeea93efa42f9b22f9d79995c3a86bdffb796427b4c9863b0f0", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, "prometheus": {:hex, :prometheus, "4.4.1", "1e96073b3ed7788053768fea779cbc896ddc3bdd9ba60687f2ad50b252ac87d6", [:mix, :rebar3], [], "hexpm"}, "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/support/oban_helpers.ex b/test/support/oban_helpers.ex index d379c9ec..98977092 100644 --- a/test/support/oban_helpers.ex +++ b/test/support/oban_helpers.ex @@ -16,7 +16,7 @@ defmodule Pleroma.Tests.ObanHelpers do end def perform(%Oban.Job{} = job) do - res = apply(String.to_existing_atom("Elixir." <> job.worker), :perform, [job]) + res = apply(String.to_existing_atom("Elixir." <> job.worker), :perform, [job.args, job]) Repo.delete(job) res end From c056736daaedb2a08557ee6c6a9bcb6bf44110ca Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 23 Aug 2019 16:11:39 +0300 Subject: [PATCH 007/138] [#1149] Publisher worker fix (atomized `params` keys). --- lib/pleroma/workers/publisher.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/workers/publisher.ex b/lib/pleroma/workers/publisher.ex index 0a908458..00fae99c 100644 --- a/lib/pleroma/workers/publisher.ex +++ b/lib/pleroma/workers/publisher.ex @@ -18,6 +18,7 @@ defmodule Pleroma.Workers.Publisher do end def perform(%{"op" => "publish_one", "module" => module_name, "params" => params}, _job) do + params = Map.new(params, fn {k, v} -> {String.to_atom(k), v} end) Federator.perform(:publish_one, String.to_atom(module_name), params) end end From 581123f8bb703023cb652267a1fc34292f862852 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 23 Aug 2019 18:28:23 +0300 Subject: [PATCH 008/138] [#1149] Introduced `quantum` job scheduler. Documentation & config changes. --- CHANGELOG.md | 2 ++ config/config.exs | 40 +++++++++++++++++--------- config/test.exs | 2 -- docs/config.md | 15 ++++++---- lib/pleroma/application.ex | 19 ++---------- lib/pleroma/scheduler.ex | 7 +++++ lib/pleroma/web/federator/federator.ex | 8 +----- mix.exs | 2 +- mix.lock | 6 +++- 9 files changed, 54 insertions(+), 47 deletions(-) create mode 100644 lib/pleroma/scheduler.ex diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b0f4f40..6dc19e79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Unsubscribe followers when they unfollow a user - AdminAPI: Add "godmode" while fetching user statuses (i.e. admin can see private statuses) - Improve digest email template +- Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) with [Oban](https://github.com/sorentwo/oban) +- Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler ### Fixed - Not being able to pin unlisted posts diff --git a/config/config.exs b/config/config.exs index 1a6348bc..43d114d7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -51,6 +51,24 @@ config :pleroma, Pleroma.Repo, telemetry_event: [Pleroma.Repo.Instrumenter], migration_lock: nil +scheduled_jobs = + with digest_config <- Application.get_env(:pleroma, :email_notifications)[:digest], + true <- digest_config[:active] do + [{digest_config[:schedule], {Pleroma.DigestEmailWorker, :perform, []}}] + else + _ -> [] + end + +scheduled_jobs = + scheduled_jobs ++ + [{"0 */6 * * * *", {Pleroma.Web.Websub, :refresh_subscriptions, []}}] + +config :pleroma, Pleroma.Scheduler, + global: true, + overlap: true, + timezone: :utc, + jobs: scheduled_jobs + config :pleroma, Pleroma.Captcha, enabled: false, seconds_valid: 60, @@ -449,23 +467,19 @@ config :pleroma, Pleroma.User, "web" ] -job_queues = [ - federator_incoming: 50, - federator_outgoing: 50, - web_push: 50, - mailer: 10, - transmogrifier: 20, - scheduled_activities: 10, - background: 5 -] - -config :pleroma_job_queue, :queues, job_queues - config :pleroma, Oban, repo: Pleroma.Repo, verbose: false, prune: {:maxage, 60 * 60 * 24 * 7}, - queues: job_queues + queues: [ + federator_incoming: 50, + federator_outgoing: 50, + web_push: 50, + mailer: 10, + transmogrifier: 20, + scheduled_activities: 10, + background: 5 + ] config :pleroma, :workers, retries: [ diff --git a/config/test.exs b/config/test.exs index a0fa6751..62f2a04d 100644 --- a/config/test.exs +++ b/config/test.exs @@ -61,8 +61,6 @@ config :web_push_encryption, :vapid_details, config :web_push_encryption, :http_client, Pleroma.Web.WebPushHttpClientMock -config :pleroma_job_queue, disabled: true - config :pleroma, Oban, queues: false, prune: :disabled diff --git a/docs/config.md b/docs/config.md index ae8afad8..81923c64 100644 --- a/docs/config.md +++ b/docs/config.md @@ -400,9 +400,9 @@ You can then do curl "http://localhost:4000/api/pleroma/admin/invite_token?admin_token=somerandomtoken" ``` -## :pleroma_job_queue +## Oban -[Pleroma Job Queue](https://git.pleroma.social/pleroma/pleroma_job_queue) configuration: a list of queues with maximum concurrent jobs. +[Oban](https://github.com/sorentwo/oban) asynchronous job processor configuration. Pleroma has the following queues: @@ -416,12 +416,15 @@ Pleroma has the following queues: Example: ```elixir -config :pleroma_job_queue, :queues, - federator_incoming: 50, - federator_outgoing: 50 +config :pleroma, Oban, + repo: Pleroma.Repo, + queues: [ + federator_incoming: 50, + federator_outgoing: 50 + ] ``` -This config contains two queues: `federator_incoming` and `federator_outgoing`. Both have the `max_jobs` set to `50`. +This config contains two queues: `federator_incoming` and `federator_outgoing`. Both have the number of max concurrent jobs set to `50`. ## Pleroma.Web.Metadata * `providers`: a list of metadata providers to enable. Providers available: diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 384b03aa..ce2d3ab5 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -31,6 +31,7 @@ defmodule Pleroma.Application do children = [ Pleroma.Repo, + Pleroma.Scheduler, Pleroma.Config.TransferTask, Pleroma.Emoji, Pleroma.Captcha, @@ -69,9 +70,7 @@ defmodule Pleroma.Application do # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html # for other strategies and supported options opts = [strategy: :one_for_one, name: Pleroma.Supervisor] - result = Supervisor.start_link(children, opts) - :ok = after_supervisor_start() - result + Supervisor.start_link(children, opts) end defp setup_instrumenters do @@ -162,18 +161,4 @@ defmodule Pleroma.Application do :hackney_pool.child_spec(pool, options) end end - - defp after_supervisor_start do - with digest_config <- Application.get_env(:pleroma, :email_notifications)[:digest], - true <- digest_config[:active] do - # TODO: consider replacing with `quantum` scheduler - PleromaJobQueue.schedule( - digest_config[:schedule], - :digest_emails, - Pleroma.DigestEmailWorker - ) - end - - :ok - end end diff --git a/lib/pleroma/scheduler.ex b/lib/pleroma/scheduler.ex new file mode 100644 index 00000000..d84cd99a --- /dev/null +++ b/lib/pleroma/scheduler.ex @@ -0,0 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Scheduler do + use Quantum.Scheduler, otp_app: :pleroma +end diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index d85fe824..cf7e50fe 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -21,7 +21,7 @@ defmodule Pleroma.Web.Federator do defdelegate worker_args(queue), to: Pleroma.Workers.Helper def init do - # 1 minute + # To do: consider removing this call in favor of scheduled execution (`quantum`-based) refresh_subscriptions(schedule_in: 60) end @@ -146,12 +146,6 @@ defmodule Pleroma.Web.Federator do def perform(:refresh_subscriptions) do Logger.debug("Federator running refresh subscriptions") Websub.refresh_subscriptions() - - spawn(fn -> - # 6 hours - Process.sleep(1000 * 60 * 60 * 6) - refresh_subscriptions() - end) end def ap_enabled_actor(id) do diff --git a/mix.exs b/mix.exs index eb023313..9d8ded1f 100644 --- a/mix.exs +++ b/mix.exs @@ -102,6 +102,7 @@ defmodule Pleroma.Mixfile do {:ecto_sql, "~> 3.1"}, {:postgrex, ">= 0.13.5"}, {:oban, "~> 0.7"}, + {:quantum, "~> 2.3"}, {:gettext, "~> 0.15"}, {:comeonin, "~> 4.1.1"}, {:pbkdf2_elixir, "~> 0.12.3"}, @@ -142,7 +143,6 @@ defmodule Pleroma.Mixfile do {:http_signatures, git: "https://git.pleroma.social/pleroma/http_signatures.git", ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"}, - {:pleroma_job_queue, "~> 0.3"}, {:telemetry, "~> 0.3"}, {:prometheus_ex, "~> 3.0"}, {:prometheus_plugs, "~> 1.1"}, diff --git a/mix.lock b/mix.lock index 8b859637..6ebc6627 100644 --- a/mix.lock +++ b/mix.lock @@ -36,6 +36,8 @@ "excoveralls": {:hex, :excoveralls, "0.11.1", "dd677fbdd49114fdbdbf445540ec735808250d56b011077798316505064edb2c", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, "floki": {:hex, :floki, "0.20.4", "be42ac911fece24b4c72f3b5846774b6e61b83fe685c2fc9d62093277fb3bc86", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}, {:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, "gen_smtp": {:hex, :gen_smtp, "0.14.0", "39846a03522456077c6429b4badfd1d55e5e7d0fdfb65e935b7c5e38549d9202", [:rebar3], [], "hexpm"}, + "gen_stage": {:hex, :gen_stage, "0.14.2", "6a2a578a510c5bfca8a45e6b27552f613b41cf584b58210f017088d3d17d0b14", [:mix], [], "hexpm"}, + "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"}, "gettext": {:hex, :gettext, "0.17.0", "abe21542c831887a2b16f4c94556db9c421ab301aee417b7c4fbde7fbdbe01ec", [:mix], [], "hexpm"}, "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, "html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm"}, @@ -46,6 +48,7 @@ "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"}, "joken": {:hex, :joken, "2.0.1", "ec9ab31bf660f343380da033b3316855197c8d4c6ef597fa3fcb451b326beb14", [:mix], [{:jose, "~> 1.9", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm"}, "jose": {:hex, :jose, "1.9.0", "4167c5f6d06ffaebffd15cdb8da61a108445ef5e85ab8f5a7ad926fdf3ada154", [:mix, :rebar3], [{:base64url, "~> 0.0.1", [hex: :base64url, repo: "hexpm", optional: false]}], "hexpm"}, + "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm"}, "makeup": {:hex, :makeup, "0.8.0", "9cf32aea71c7fe0a4b2e9246c2c4978f9070257e5c9ce6d4a28ec450a839b55f", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"}, "makeup_elixir": {:hex, :makeup_elixir, "0.13.0", "be7a477997dcac2e48a9d695ec730b2d22418292675c75aa2d34ba0909dcdeda", [:mix], [{:makeup, "~> 0.8", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"}, "meck": {:hex, :meck, "0.8.13", "ffedb39f99b0b99703b8601c6f17c7f76313ee12de6b646e671e3188401f7866", [:rebar3], [], "hexpm"}, @@ -65,7 +68,6 @@ "phoenix_html": {:hex, :phoenix_html, "2.13.1", "fa8f034b5328e2dfa0e4131b5569379003f34bc1fafdaa84985b0b9d2f12e68b", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "1.1.2", "496c303bdf1b2e98a9d26e89af5bba3ab487ba3a3735f74bf1f4064d2a845a3e", [:mix], [], "hexpm"}, "phoenix_swoosh": {:hex, :phoenix_swoosh, "0.2.0", "a7e0b32077cd6d2323ae15198839b05d9caddfa20663fd85787479e81f89520e", [:mix], [{:phoenix, "~> 1.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.2", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:swoosh, "~> 0.1", [hex: :swoosh, repo: "hexpm", optional: false]}], "hexpm"}, - "pleroma_job_queue": {:hex, :pleroma_job_queue, "0.3.0", "b84538d621f0c3d6fcc1cff9d5648d3faaf873b8b21b94e6503428a07a48ec47", [:mix], [{:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}], "hexpm"}, "plug": {:hex, :plug, "1.8.2", "0bcce1daa420f189a6491f3940cc77ea7fb1919761175c9c3b59800d897440fc", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm"}, "plug_cowboy": {:hex, :plug_cowboy, "2.1.0", "b75768153c3a8a9e8039d4b25bb9b14efbc58e9c4a6e6a270abff1cd30cbe320", [:mix], [{:cowboy, "~> 2.5", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"}, @@ -78,9 +80,11 @@ "prometheus_phoenix": {:hex, :prometheus_phoenix, "1.3.0", "c4b527e0b3a9ef1af26bdcfbfad3998f37795b9185d475ca610fe4388fdd3bb5", [:mix], [{:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.3 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, "prometheus_plugs": {:hex, :prometheus_plugs, "1.1.5", "25933d48f8af3a5941dd7b621c889749894d8a1082a6ff7c67cc99dec26377c5", [:mix], [{:accept, "~> 0.1", [hex: :accept, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}, {:prometheus_process_collector, "~> 1.1", [hex: :prometheus_process_collector, repo: "hexpm", optional: true]}], "hexpm"}, "quack": {:hex, :quack, "0.1.1", "cca7b4da1a233757fdb44b3334fce80c94785b3ad5a602053b7a002b5a8967bf", [:mix], [{:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: false]}, {:tesla, "~> 1.2.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm"}, + "quantum": {:hex, :quantum, "2.3.4", "72a0e8855e2adc101459eac8454787cb74ab4169de6ca50f670e72142d4960e9", [:mix], [{:calendar, "~> 0.17", [hex: :calendar, repo: "hexpm", optional: true]}, {:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.12", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:swarm, "~> 3.3", [hex: :swarm, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: true]}], "hexpm"}, "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, + "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm"}, "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"}, "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, From 71700ea6d4104ecd2cc0afb0ac103e722b30fbb5 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 24 Aug 2019 09:27:32 +0300 Subject: [PATCH 009/138] [#1149] Updated docs & tests. --- docs/config.md | 6 ++++++ test/web/admin_api/admin_api_controller_test.exs | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/config.md b/docs/config.md index 81923c64..5b2c3a02 100644 --- a/docs/config.md +++ b/docs/config.md @@ -426,6 +426,12 @@ config :pleroma, Oban, This config contains two queues: `federator_incoming` and `federator_outgoing`. Both have the number of max concurrent jobs set to `50`. +## :workers + +Includes custom worker options not interpretable directly by `Oban`. + +* `retries` — keyword lists where keys are `Oban` queues (see above) and values are numbers of max attempts for failed jobs. + ## Pleroma.Web.Metadata * `providers`: a list of metadata providers to enable. Providers available: * Pleroma.Web.Metadata.Providers.OpenGraph diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 844cd073..a867ac99 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -1861,7 +1861,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do post(conn, "/api/pleroma/admin/config", %{ configs: [ %{ - "group" => "pleroma_job_queue", + "group" => "oban", "key" => ":queues", "value" => [ %{"tuple" => [":federator_incoming", 50]}, @@ -1879,7 +1879,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do assert json_response(conn, 200) == %{ "configs" => [ %{ - "group" => "pleroma_job_queue", + "group" => "oban", "key" => ":queues", "value" => [ %{"tuple" => [":federator_incoming", 50]}, From cd78e63a2528ab813088d5e44a026f6bb05b344b Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 27 Aug 2019 14:34:37 +0300 Subject: [PATCH 010/138] [#1149] Bugfix: Pleroma.Workers.Subscriber / "verify_websub" works with WebsubServerSubscription. --- lib/pleroma/workers/subscriber.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/workers/subscriber.ex b/lib/pleroma/workers/subscriber.ex index 783c4417..e960b35b 100644 --- a/lib/pleroma/workers/subscriber.ex +++ b/lib/pleroma/workers/subscriber.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Workers.Subscriber do alias Pleroma.Repo alias Pleroma.Web.Federator - alias Pleroma.Web.Websub.WebsubClientSubscription + alias Pleroma.Web.Websub # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, @@ -18,12 +18,12 @@ defmodule Pleroma.Workers.Subscriber do end def perform(%{"op" => "request_subscription", "websub_id" => websub_id}, _job) do - websub = Repo.get(WebsubClientSubscription, websub_id) + websub = Repo.get(Websub.WebsubClientSubscription, websub_id) Federator.perform(:request_subscription, websub) end def perform(%{"op" => "verify_websub", "websub_id" => websub_id}, _job) do - websub = Repo.get(WebsubClientSubscription, websub_id) + websub = Repo.get(Websub.WebsubServerSubscription, websub_id) Federator.perform(:verify_websub, websub) end end From a90ea8ba1562818b025f677ffeea35f7ca08ddf2 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 31 Aug 2019 19:08:56 +0300 Subject: [PATCH 011/138] [#1149] Addressed code review comments (code style, jobs pruning etc.). --- CHANGELOG.md | 2 +- config/config.exs | 2 +- config/test.exs | 2 + docs/config.md | 56 ++++++++++++++++++- lib/pleroma/activity_expiration_worker.ex | 6 +- lib/pleroma/application.ex | 2 +- lib/pleroma/digest_email_worker.ex | 4 +- lib/pleroma/emails/mailer.ex | 4 +- lib/pleroma/scheduled_activity_worker.ex | 2 +- lib/pleroma/user.ex | 2 +- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- .../mrf/mediaproxy_warming_policy.ex | 2 +- lib/pleroma/web/activity_pub/publisher.ex | 2 +- .../web/activity_pub/transmogrifier.ex | 4 +- lib/pleroma/web/federator/federator.ex | 8 +-- lib/pleroma/web/federator/publisher.ex | 9 +-- lib/pleroma/web/oauth/token/clean_worker.ex | 2 +- lib/pleroma/web/push/push.ex | 6 +- lib/pleroma/web/salmon/salmon.ex | 2 +- .../workers/activity_expiration_worker.ex | 21 +++++++ lib/pleroma/workers/background_worker.ex | 19 ++----- lib/pleroma/workers/helper.ex | 13 ----- .../workers/{mailer.ex => mailer_worker.ex} | 19 +++---- .../{publisher.ex => publisher_worker.ex} | 8 ++- .../{receiver.ex => receiver_worker.ex} | 4 +- .../workers/scheduled_activity_worker.ex | 2 +- .../{subscriber.ex => subscriber_worker.ex} | 4 +- ...smogrifier.ex => transmogrifier_worker.ex} | 6 +- .../{web_pusher.ex => web_pusher_worker.ex} | 4 +- lib/pleroma/workers/worker_helper.ex | 23 ++++++++ test/user_test.exs | 2 +- .../activity_pub_controller_test.exs | 2 +- test/web/federator_test.exs | 2 +- test/web/websub/websub_test.exs | 2 +- 34 files changed, 163 insertions(+), 87 deletions(-) create mode 100644 lib/pleroma/workers/activity_expiration_worker.ex delete mode 100644 lib/pleroma/workers/helper.ex rename lib/pleroma/workers/{mailer.ex => mailer_worker.ex} (58%) rename lib/pleroma/workers/{publisher.ex => publisher_worker.ex} (76%) rename lib/pleroma/workers/{receiver.ex => receiver_worker.ex} (83%) rename lib/pleroma/workers/{subscriber.ex => subscriber_worker.ex} (88%) rename lib/pleroma/workers/{transmogrifier.ex => transmogrifier_worker.ex} (73%) rename lib/pleroma/workers/{web_pusher.ex => web_pusher_worker.ex} (82%) create mode 100644 lib/pleroma/workers/worker_helper.ex diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b73c783..c9d6fef1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Unsubscribe followers when they unfollow a user - AdminAPI: Add "godmode" while fetching user statuses (i.e. admin can see private statuses) - Improve digest email template -- Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) with [Oban](https://github.com/sorentwo/oban) +- Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings). - Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler ### Fixed diff --git a/config/config.exs b/config/config.exs index da89aa3e..6fb4a096 100644 --- a/config/config.exs +++ b/config/config.exs @@ -470,7 +470,7 @@ config :pleroma, Pleroma.User, config :pleroma, Oban, repo: Pleroma.Repo, verbose: false, - prune: {:maxage, 60 * 60 * 24 * 7}, + prune: {:maxlen, 1500}, queues: [ activity_expiration: 10, federator_incoming: 50, diff --git a/config/test.exs b/config/test.exs index 0ef809ac..df512b5d 100644 --- a/config/test.exs +++ b/config/test.exs @@ -65,6 +65,8 @@ config :pleroma, Oban, queues: false, prune: :disabled +config :pleroma, Pleroma.Scheduler, jobs: [] + config :pleroma, Pleroma.ScheduledActivity, daily_user_limit: 2, total_user_limit: 3, diff --git a/docs/config.md b/docs/config.md index 2e351e27..29a4d4c9 100644 --- a/docs/config.md +++ b/docs/config.md @@ -404,20 +404,29 @@ curl "http://localhost:4000/api/pleroma/admin/invite_token?admin_token=somerando [Oban](https://github.com/sorentwo/oban) asynchronous job processor configuration. +Configuration options described in [Oban readme](https://github.com/sorentwo/oban#usage): +* `repo` - app's Ecto repo (`Pleroma.Repo`) +* `verbose` - logs verbosity +* `prune` - non-retryable jobs [pruning settings](https://github.com/sorentwo/oban#pruning) (`:disabled` / `{:maxlen, value}` / `{:maxage, value}`) +* `queues` - job queues (see below) + Pleroma has the following queues: +* `activity_expiration` - Activity expiration * `federator_outgoing` - Outgoing federation * `federator_incoming` - Incoming federation -* `mailer` - Email sender, see [`Pleroma.Emails.Mailer`](#pleroma-emails-mailer) +* `mailer` - Email sender, see [`Pleroma.Emails.Mailer`](#pleromaemailsmailer) * `transmogrifier` - Transmogrifier * `web_push` - Web push notifications -* `scheduled_activities` - Scheduled activities, see [`Pleroma.ScheduledActivities`](#pleromascheduledactivity) +* `scheduled_activities` - Scheduled activities, see [`Pleroma.ScheduledActivity`](#pleromascheduledactivity) Example: ```elixir config :pleroma, Oban, repo: Pleroma.Repo, + verbose: false, + prune: {:maxlen, 1500}, queues: [ federator_incoming: 50, federator_outgoing: 50 @@ -426,12 +435,37 @@ config :pleroma, Oban, This config contains two queues: `federator_incoming` and `federator_outgoing`. Both have the number of max concurrent jobs set to `50`. +### Migrating `pleroma_job_queue` settings + +`config :pleroma_job_queue, :queues` is replaced by `config :pleroma, Oban, :queues` and uses the same format (keys are queues' names, values are max concurrent jobs numbers). + +### Note on running with PostgreSQL in silent mode + +If you are running PostgreSQL in [`silent_mode`](https://postgresqlco.nf/en/doc/param/silent_mode?version=9.1), it's advised to set [`log_destination`](https://postgresqlco.nf/en/doc/param/log_destination?version=9.1) to `syslog`, +otherwise `postmaster.log` file may grow because of "you don't own a lock of type ShareLock" warnings (see https://github.com/sorentwo/oban/issues/52). + ## :workers Includes custom worker options not interpretable directly by `Oban`. * `retries` — keyword lists where keys are `Oban` queues (see above) and values are numbers of max attempts for failed jobs. +Example: + +```elixir +config :pleroma, :workers, + retries: [ + federator_incoming: 5, + federator_outgoing: 5 + ] +``` + +### Migrating `Pleroma.Web.Federator.RetryQueue` settings + +* `max_retries` is replaced with `config :pleroma, :workers, retries: [federator_outgoing: 5]` +* `enabled: false` corresponds to `config :pleroma, :workers, retries: [federator_outgoing: 1]` +* deprecated options: `max_jobs`, `initial_timeout` + ## Pleroma.Web.Metadata * `providers`: a list of metadata providers to enable. Providers available: * Pleroma.Web.Metadata.Providers.OpenGraph @@ -491,6 +525,24 @@ config :auto_linker, ] ``` +## Pleroma.Scheduler + +Configuration for [Quantum](https://github.com/quantum-elixir/quantum-core) jobs scheduler. + +See [Quantum readme](https://github.com/quantum-elixir/quantum-core#usage) for the list of supported options. + +Example: + +```elixir +config :pleroma, Pleroma.Scheduler, + global: true, + overlap: true, + timezone: :utc, + jobs: [{"0 */6 * * * *", {Pleroma.Web.Websub, :refresh_subscriptions, []}}] +``` + +The above example defines a single job which invokes `Pleroma.Web.Websub.refresh_subscriptions()` every 6 hours ("0 */6 * * * *", [crontab format](https://en.wikipedia.org/wiki/Cron)). + ## Pleroma.ScheduledActivity * `daily_user_limit`: the number of scheduled activities a user is allowed to create in a single day (Default: `25`) diff --git a/lib/pleroma/activity_expiration_worker.ex b/lib/pleroma/activity_expiration_worker.ex index 5c0c5323..7aba7eec 100644 --- a/lib/pleroma/activity_expiration_worker.ex +++ b/lib/pleroma/activity_expiration_worker.ex @@ -9,13 +9,13 @@ defmodule Pleroma.ActivityExpirationWorker do alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.CommonAPI - alias Pleroma.Workers.BackgroundWorker + alias Pleroma.Workers.ActivityExpirationWorker require Logger use GenServer import Ecto.Query - defdelegate worker_args(queue), to: Pleroma.Workers.Helper + import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] @schedule_interval :timer.minutes(1) @@ -57,7 +57,7 @@ defmodule Pleroma.ActivityExpirationWorker do "op" => "activity_expiration", "activity_expiration_id" => expiration.id } - |> BackgroundWorker.new(worker_args(:activity_expiration)) + |> ActivityExpirationWorker.new(worker_args(:activity_expiration)) |> Repo.insert() end) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 7d38ed5c..f8f866db 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -43,7 +43,7 @@ defmodule Pleroma.Application do hackney_pool_children() ++ [ Pleroma.Stats, - {Oban, Application.get_env(:pleroma, Oban)}, + {Oban, Pleroma.Config.get(Oban)}, %{ id: :web_push_init, start: {Task, :start_link, [&Pleroma.Web.Push.init/0]}, diff --git a/lib/pleroma/digest_email_worker.ex b/lib/pleroma/digest_email_worker.ex index ffc48bfa..4ab2a4ef 100644 --- a/lib/pleroma/digest_email_worker.ex +++ b/lib/pleroma/digest_email_worker.ex @@ -4,11 +4,11 @@ defmodule Pleroma.DigestEmailWorker do alias Pleroma.Repo - alias Pleroma.Workers.Mailer, as: MailerWorker + alias Pleroma.Workers.MailerWorker import Ecto.Query - defdelegate worker_args(queue), to: Pleroma.Workers.Helper + import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] def perform do config = Pleroma.Config.get([:email_notifications, :digest]) diff --git a/lib/pleroma/emails/mailer.ex b/lib/pleroma/emails/mailer.ex index bb534f60..9cbe7313 100644 --- a/lib/pleroma/emails/mailer.ex +++ b/lib/pleroma/emails/mailer.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Emails.Mailer do """ alias Pleroma.Repo - alias Pleroma.Workers.Mailer, as: MailerWorker + alias Pleroma.Workers.MailerWorker alias Swoosh.DeliveryError @otp_app :pleroma @@ -19,7 +19,7 @@ defmodule Pleroma.Emails.Mailer do @spec enabled?() :: boolean() def enabled?, do: Pleroma.Config.get([__MODULE__, :enabled]) - defdelegate worker_args(queue), to: Pleroma.Workers.Helper + import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] @doc "add email to queue" def deliver_async(email, config \\ []) do diff --git a/lib/pleroma/scheduled_activity_worker.ex b/lib/pleroma/scheduled_activity_worker.ex index a01fb4fc..8bf534f4 100644 --- a/lib/pleroma/scheduled_activity_worker.ex +++ b/lib/pleroma/scheduled_activity_worker.ex @@ -18,7 +18,7 @@ defmodule Pleroma.ScheduledActivityWorker do @schedule_interval :timer.minutes(1) - defdelegate worker_args(queue), to: Pleroma.Workers.Helper + import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] def start_link(_) do GenServer.start_link(__MODULE__, nil) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 18bba0fb..abfa063f 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -41,7 +41,7 @@ defmodule Pleroma.User do @strict_local_nickname_regex ~r/^[a-zA-Z\d]+$/ @extended_local_nickname_regex ~r/^[a-zA-Z\d_-]+$/ - defdelegate worker_args(queue), to: Pleroma.Workers.Helper + import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] schema "users" do field(:bio, :string) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 50279cca..74c5eb91 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -26,7 +26,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do require Logger require Pleroma.Constants - defdelegate worker_args(queue), to: Pleroma.Workers.Helper + import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] # For Announce activities, we filter the recipients based on following status for any actors # that match actual users. See issue #164 for more information about why this is necessary. diff --git a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex index b188164e..17832155 100644 --- a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex @@ -18,7 +18,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do recv_timeout: 10_000 ] - defdelegate worker_args(queue), to: Pleroma.Workers.Helper + import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] def perform(:prefetch, url) do Logger.info("Prefetching #{inspect(url)}") diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 24d101dc..a6322e25 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -85,7 +85,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do end def publish_one(%{actor_id: actor_id} = params) do - actor = User.get_by_id(actor_id) + actor = User.get_cached_by_id(actor_id) params |> Map.delete(:actor_id) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index b068d28a..9437f9a1 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -15,14 +15,14 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.Federator - alias Pleroma.Workers.Transmogrifier, as: TransmogrifierWorker + alias Pleroma.Workers.TransmogrifierWorker import Ecto.Query require Logger require Pleroma.Constants - defdelegate worker_args(queue), to: Pleroma.Workers.Helper + import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] @doc """ Modifies an incoming AP object (mastodon format) to our internal format. diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index cf7e50fe..8f43066e 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -12,13 +12,13 @@ defmodule Pleroma.Web.Federator do alias Pleroma.Web.Federator.Publisher alias Pleroma.Web.OStatus alias Pleroma.Web.Websub - alias Pleroma.Workers.Publisher, as: PublisherWorker - alias Pleroma.Workers.Receiver, as: ReceiverWorker - alias Pleroma.Workers.Subscriber, as: SubscriberWorker + alias Pleroma.Workers.PublisherWorker + alias Pleroma.Workers.ReceiverWorker + alias Pleroma.Workers.SubscriberWorker require Logger - defdelegate worker_args(queue), to: Pleroma.Workers.Helper + import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] def init do # To do: consider removing this call in favor of scheduled execution (`quantum`-based) diff --git a/lib/pleroma/web/federator/publisher.ex b/lib/pleroma/web/federator/publisher.ex index 05d2be61..42be109a 100644 --- a/lib/pleroma/web/federator/publisher.ex +++ b/lib/pleroma/web/federator/publisher.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Web.Federator.Publisher do alias Pleroma.Activity alias Pleroma.Config alias Pleroma.User - alias Pleroma.Workers.Publisher, as: PublisherWorker + alias Pleroma.Workers.PublisherWorker require Logger @@ -31,12 +31,7 @@ defmodule Pleroma.Web.Federator.Publisher do """ @spec enqueue_one(module(), Map.t()) :: :ok def enqueue_one(module, %{} = params) do - worker_args = - if max_attempts = Pleroma.Config.get([:workers, :retries, :federator_outgoing]) do - [max_attempts: max_attempts] - else - [] - end + worker_args = Pleroma.Workers.WorkerHelper.worker_args(:federator_outgoing) %{"op" => "publish_one", "module" => to_string(module), "params" => params} |> PublisherWorker.new(worker_args) diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex index 943e7328..b150a68a 100644 --- a/lib/pleroma/web/oauth/token/clean_worker.ex +++ b/lib/pleroma/web/oauth/token/clean_worker.ex @@ -20,7 +20,7 @@ defmodule Pleroma.Web.OAuth.Token.CleanWorker do alias Pleroma.Web.OAuth.Token alias Pleroma.Workers.BackgroundWorker - defdelegate worker_args(queue), to: Pleroma.Workers.Helper + import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] def start_link(_), do: GenServer.start_link(__MODULE__, %{}) diff --git a/lib/pleroma/web/push/push.ex b/lib/pleroma/web/push/push.ex index b4f0e512..4973b529 100644 --- a/lib/pleroma/web/push/push.ex +++ b/lib/pleroma/web/push/push.ex @@ -4,11 +4,11 @@ defmodule Pleroma.Web.Push do alias Pleroma.Repo - alias Pleroma.Workers.WebPusher + alias Pleroma.Workers.WebPusherWorker require Logger - defdelegate worker_args(queue), to: Pleroma.Workers.Helper + import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] def init do unless enabled() do @@ -36,7 +36,7 @@ defmodule Pleroma.Web.Push do def send(notification) do %{"op" => "web_push", "notification_id" => notification.id} - |> WebPusher.new(worker_args(:web_push)) + |> WebPusherWorker.new(worker_args(:web_push)) |> Repo.insert() end end diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index bbaa293f..8ba7380c 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -171,7 +171,7 @@ defmodule Pleroma.Web.Salmon do end def publish_one(%{recipient_id: recipient_id} = params) do - recipient = User.get_by_id(recipient_id) + recipient = User.get_cached_by_id(recipient_id) params |> Map.delete(:recipient_id) diff --git a/lib/pleroma/workers/activity_expiration_worker.ex b/lib/pleroma/workers/activity_expiration_worker.ex new file mode 100644 index 00000000..0b491eab --- /dev/null +++ b/lib/pleroma/workers/activity_expiration_worker.ex @@ -0,0 +1,21 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.ActivityExpirationWorker do + # Note: `max_attempts` is intended to be overridden in `new/2` call + use Oban.Worker, + queue: "activity_expiration", + max_attempts: 1 + + @impl Oban.Worker + def perform( + %{ + "op" => "activity_expiration", + "activity_expiration_id" => activity_expiration_id + }, + _job + ) do + Pleroma.ActivityExpirationWorker.perform(:execute, activity_expiration_id) + end +end diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index fbce7d78..7b5575a5 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -8,24 +8,24 @@ defmodule Pleroma.Workers.BackgroundWorker do alias Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy alias Pleroma.Web.OAuth.Token.CleanWorker - # Note: `max_attempts` is intended to be overridden in `new/1` call + # Note: `max_attempts` is intended to be overridden in `new/2` call use Oban.Worker, queue: "background", max_attempts: 1 @impl Oban.Worker def perform(%{"op" => "fetch_initial_posts", "user_id" => user_id}, _job) do - user = User.get_by_id(user_id) + user = User.get_cached_by_id(user_id) User.perform(:fetch_initial_posts, user) end def perform(%{"op" => "deactivate_user", "user_id" => user_id, "status" => status}, _job) do - user = User.get_by_id(user_id) + user = User.get_cached_by_id(user_id) User.perform(:deactivate_async, user, status) end def perform(%{"op" => "delete_user", "user_id" => user_id}, _job) do - user = User.get_by_id(user_id) + user = User.get_cached_by_id(user_id) User.perform(:delete, user) end @@ -37,7 +37,7 @@ defmodule Pleroma.Workers.BackgroundWorker do }, _job ) do - blocker = User.get_by_id(blocker_id) + blocker = User.get_cached_by_id(blocker_id) User.perform(:blocks_import, blocker, blocked_identifiers) end @@ -49,7 +49,7 @@ defmodule Pleroma.Workers.BackgroundWorker do }, _job ) do - follower = User.get_by_id(follower_id) + follower = User.get_cached_by_id(follower_id) User.perform(:follow_import, follower, followed_identifiers) end @@ -69,11 +69,4 @@ defmodule Pleroma.Workers.BackgroundWorker do activity = Activity.get_by_id(activity_id) Pleroma.Web.RichMedia.Helpers.perform(:fetch, activity) end - - def perform( - %{"op" => "activity_expiration", "activity_expiration_id" => activity_expiration_id}, - _job - ) do - Pleroma.ActivityExpirationWorker.perform(:execute, activity_expiration_id) - end end diff --git a/lib/pleroma/workers/helper.ex b/lib/pleroma/workers/helper.ex deleted file mode 100644 index 3286ce0e..00000000 --- a/lib/pleroma/workers/helper.ex +++ /dev/null @@ -1,13 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.Helper do - def worker_args(queue) do - if max_attempts = Pleroma.Config.get([:workers, :retries, queue]) do - [max_attempts: max_attempts] - else - [] - end - end -end diff --git a/lib/pleroma/workers/mailer.ex b/lib/pleroma/workers/mailer_worker.ex similarity index 58% rename from lib/pleroma/workers/mailer.ex rename to lib/pleroma/workers/mailer_worker.ex index 1cce2ea0..4f73d61b 100644 --- a/lib/pleroma/workers/mailer.ex +++ b/lib/pleroma/workers/mailer_worker.ex @@ -2,26 +2,25 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Workers.Mailer do +defmodule Pleroma.Workers.MailerWorker do alias Pleroma.User - # Note: `max_attempts` is intended to be overridden in `new/1` call + # Note: `max_attempts` is intended to be overridden in `new/2` call use Oban.Worker, queue: "mailer", max_attempts: 1 @impl Oban.Worker def perform(%{"op" => "email", "encoded_email" => encoded_email, "config" => config}, _job) do - email = - encoded_email - |> Base.decode64!() - |> :erlang.binary_to_term() - - Pleroma.Emails.Mailer.deliver(email, config) + encoded_email + |> Base.decode64!() + |> :erlang.binary_to_term() + |> Pleroma.Emails.Mailer.deliver(config) end def perform(%{"op" => "digest_email", "user_id" => user_id}, _job) do - user = User.get_by_id(user_id) - Pleroma.DigestEmailWorker.perform(user) + user_id + |> User.get_cached_by_id() + |> Pleroma.DigestEmailWorker.perform() end end diff --git a/lib/pleroma/workers/publisher.ex b/lib/pleroma/workers/publisher_worker.ex similarity index 76% rename from lib/pleroma/workers/publisher.ex rename to lib/pleroma/workers/publisher_worker.ex index 00fae99c..5671d2a2 100644 --- a/lib/pleroma/workers/publisher.ex +++ b/lib/pleroma/workers/publisher_worker.ex @@ -2,15 +2,19 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Workers.Publisher do +defmodule Pleroma.Workers.PublisherWorker do alias Pleroma.Activity alias Pleroma.Web.Federator - # Note: `max_attempts` is intended to be overridden in `new/1` call + # Note: `max_attempts` is intended to be overridden in `new/2` call use Oban.Worker, queue: "federator_outgoing", max_attempts: 1 + def backoff(attempt) when is_integer(attempt) do + Pleroma.Workers.WorkerHelper.sidekiq_backoff(attempt, 5) + end + @impl Oban.Worker def perform(%{"op" => "publish", "activity_id" => activity_id}, _job) do activity = Activity.get_by_id(activity_id) diff --git a/lib/pleroma/workers/receiver.ex b/lib/pleroma/workers/receiver_worker.ex similarity index 83% rename from lib/pleroma/workers/receiver.ex rename to lib/pleroma/workers/receiver_worker.ex index 4ee270d7..cdce630f 100644 --- a/lib/pleroma/workers/receiver.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -2,10 +2,10 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Workers.Receiver do +defmodule Pleroma.Workers.ReceiverWorker do alias Pleroma.Web.Federator - # Note: `max_attempts` is intended to be overridden in `new/1` call + # Note: `max_attempts` is intended to be overridden in `new/2` call use Oban.Worker, queue: "federator_incoming", max_attempts: 1 diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex index d9724c78..4094411a 100644 --- a/lib/pleroma/workers/scheduled_activity_worker.ex +++ b/lib/pleroma/workers/scheduled_activity_worker.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.ScheduledActivityWorker do - # Note: `max_attempts` is intended to be overridden in `new/1` call + # Note: `max_attempts` is intended to be overridden in `new/2` call use Oban.Worker, queue: "scheduled_activities", max_attempts: 1 diff --git a/lib/pleroma/workers/subscriber.ex b/lib/pleroma/workers/subscriber_worker.ex similarity index 88% rename from lib/pleroma/workers/subscriber.ex rename to lib/pleroma/workers/subscriber_worker.ex index e960b35b..22d1dc95 100644 --- a/lib/pleroma/workers/subscriber.ex +++ b/lib/pleroma/workers/subscriber_worker.ex @@ -2,12 +2,12 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Workers.Subscriber do +defmodule Pleroma.Workers.SubscriberWorker do alias Pleroma.Repo alias Pleroma.Web.Federator alias Pleroma.Web.Websub - # Note: `max_attempts` is intended to be overridden in `new/1` call + # Note: `max_attempts` is intended to be overridden in `new/2` call use Oban.Worker, queue: "federator_outgoing", max_attempts: 1 diff --git a/lib/pleroma/workers/transmogrifier.ex b/lib/pleroma/workers/transmogrifier_worker.ex similarity index 73% rename from lib/pleroma/workers/transmogrifier.ex rename to lib/pleroma/workers/transmogrifier_worker.ex index e13202c0..6f5c1a2f 100644 --- a/lib/pleroma/workers/transmogrifier.ex +++ b/lib/pleroma/workers/transmogrifier_worker.ex @@ -2,17 +2,17 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Workers.Transmogrifier do +defmodule Pleroma.Workers.TransmogrifierWorker do alias Pleroma.User - # Note: `max_attempts` is intended to be overridden in `new/1` call + # Note: `max_attempts` is intended to be overridden in `new/2` call use Oban.Worker, queue: "transmogrifier", max_attempts: 1 @impl Oban.Worker def perform(%{"op" => "user_upgrade", "user_id" => user_id}, _job) do - user = User.get_by_id(user_id) + user = User.get_cached_by_id(user_id) Pleroma.Web.ActivityPub.Transmogrifier.perform(:user_upgrade, user) end end diff --git a/lib/pleroma/workers/web_pusher.ex b/lib/pleroma/workers/web_pusher_worker.ex similarity index 82% rename from lib/pleroma/workers/web_pusher.ex rename to lib/pleroma/workers/web_pusher_worker.ex index 7b78bb3e..2b1d3b99 100644 --- a/lib/pleroma/workers/web_pusher.ex +++ b/lib/pleroma/workers/web_pusher_worker.ex @@ -2,11 +2,11 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Workers.WebPusher do +defmodule Pleroma.Workers.WebPusherWorker do alias Pleroma.Notification alias Pleroma.Repo - # Note: `max_attempts` is intended to be overridden in `new/1` call + # Note: `max_attempts` is intended to be overridden in `new/2` call use Oban.Worker, queue: "web_push", max_attempts: 1 diff --git a/lib/pleroma/workers/worker_helper.ex b/lib/pleroma/workers/worker_helper.ex new file mode 100644 index 00000000..f9ed2e64 --- /dev/null +++ b/lib/pleroma/workers/worker_helper.ex @@ -0,0 +1,23 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.WorkerHelper do + alias Pleroma.Config + + def worker_args(queue) do + case Config.get([:workers, :retries, queue]) do + nil -> [] + max_attempts -> [max_attempts: max_attempts] + end + end + + def sidekiq_backoff(attempt, pow \\ 4, base_backoff \\ 15) do + backoff = + :math.pow(attempt, pow) + + base_backoff + + :rand.uniform(2 * base_backoff) * attempt + + trunc(backoff) + end +end diff --git a/test/user_test.exs b/test/user_test.exs index 86232de9..0acd0db4 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1123,7 +1123,7 @@ defmodule Pleroma.UserTest do "id" => "pleroma:fakeid" } }, - all_enqueued(worker: Pleroma.Workers.Publisher) + all_enqueued(worker: Pleroma.Workers.PublisherWorker) ) end end diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index a1b567a4..f1c1bb50 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -17,7 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do alias Pleroma.Web.ActivityPub.UserView alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.CommonAPI - alias Pleroma.Workers.Receiver, as: ReceiverWorker + alias Pleroma.Workers.ReceiverWorker setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index 5724672f..4096d469 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -7,7 +7,7 @@ defmodule Pleroma.Web.FederatorTest do alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.CommonAPI alias Pleroma.Web.Federator - alias Pleroma.Workers.Publisher, as: PublisherWorker + alias Pleroma.Workers.PublisherWorker use Pleroma.DataCase use Oban.Testing, repo: Pleroma.Repo diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index 41461087..929acf5a 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -11,7 +11,7 @@ defmodule Pleroma.Web.WebsubTest do alias Pleroma.Web.Websub alias Pleroma.Web.Websub.WebsubClientSubscription alias Pleroma.Web.Websub.WebsubServerSubscription - alias Pleroma.Workers.Subscriber, as: SubscriberWorker + alias Pleroma.Workers.SubscriberWorker import Pleroma.Factory import Tesla.Mock From dd017c65a4b86501c435f5cb01804300e6b7c6dd Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 31 Aug 2019 21:58:42 +0300 Subject: [PATCH 012/138] [#1149] Refactored Oban workers API (introduced `enqueue/3`). --- lib/pleroma/activity_expiration_worker.ex | 13 +++------ lib/pleroma/digest_email_worker.ex | 10 ++----- lib/pleroma/emails/mailer.ex | 7 +---- lib/pleroma/scheduled_activity_worker.ex | 10 +++---- lib/pleroma/user.ex | 28 +++++-------------- lib/pleroma/web/activity_pub/activity_pub.ex | 6 +--- .../mrf/mediaproxy_warming_policy.ex | 11 ++------ .../web/activity_pub/transmogrifier.ex | 6 +--- lib/pleroma/web/federator/federator.ex | 26 ++++------------- lib/pleroma/web/federator/publisher.ex | 9 +++--- lib/pleroma/web/oauth/token/clean_worker.ex | 7 +---- lib/pleroma/web/push/push.ex | 7 +---- .../workers/activity_expiration_worker.ex | 2 ++ lib/pleroma/workers/background_worker.ex | 2 ++ lib/pleroma/workers/digest_emails_worker.ex | 21 ++++++++++++++ lib/pleroma/workers/mailer_worker.ex | 10 ++----- lib/pleroma/workers/publisher_worker.ex | 2 ++ lib/pleroma/workers/receiver_worker.ex | 2 ++ .../workers/scheduled_activity_worker.ex | 2 ++ lib/pleroma/workers/subscriber_worker.ex | 2 ++ lib/pleroma/workers/transmogrifier_worker.ex | 2 ++ lib/pleroma/workers/web_pusher_worker.ex | 2 ++ lib/pleroma/workers/worker_helper.ex | 18 ++++++++++++ 23 files changed, 92 insertions(+), 113 deletions(-) create mode 100644 lib/pleroma/workers/digest_emails_worker.ex diff --git a/lib/pleroma/activity_expiration_worker.ex b/lib/pleroma/activity_expiration_worker.ex index 7aba7eec..c0820c20 100644 --- a/lib/pleroma/activity_expiration_worker.ex +++ b/lib/pleroma/activity_expiration_worker.ex @@ -9,14 +9,11 @@ defmodule Pleroma.ActivityExpirationWorker do alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.CommonAPI - alias Pleroma.Workers.ActivityExpirationWorker require Logger use GenServer import Ecto.Query - import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] - @schedule_interval :timer.minutes(1) def start_link(_) do @@ -53,12 +50,10 @@ defmodule Pleroma.ActivityExpirationWorker do def handle_info(:perform, state) do ActivityExpiration.due_expirations(@schedule_interval) |> Enum.each(fn expiration -> - %{ - "op" => "activity_expiration", - "activity_expiration_id" => expiration.id - } - |> ActivityExpirationWorker.new(worker_args(:activity_expiration)) - |> Repo.insert() + Pleroma.Workers.ActivityExpirationWorker.enqueue( + "activity_expiration", + %{"activity_expiration_id" => expiration.id} + ) end) schedule_next() diff --git a/lib/pleroma/digest_email_worker.ex b/lib/pleroma/digest_email_worker.ex index 4ab2a4ef..5be7cf26 100644 --- a/lib/pleroma/digest_email_worker.ex +++ b/lib/pleroma/digest_email_worker.ex @@ -4,12 +4,10 @@ defmodule Pleroma.DigestEmailWorker do alias Pleroma.Repo - alias Pleroma.Workers.MailerWorker + alias Pleroma.Workers.DigestEmailsWorker import Ecto.Query - import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] - def perform do config = Pleroma.Config.get([:email_notifications, :digest]) negative_interval = -Map.fetch!(config, :interval) @@ -23,11 +21,9 @@ defmodule Pleroma.DigestEmailWorker do where: u.last_digest_emailed_at < datetime_add(^now, ^negative_interval, "day"), select: u ) - |> Pleroma.Repo.all() + |> Repo.all() |> Enum.each(fn user -> - %{"op" => "digest_email", "user_id" => user.id} - |> MailerWorker.new([queue: "digest_emails"] ++ worker_args(:digest_emails)) - |> Repo.insert() + DigestEmailsWorker.enqueue("digest_email", %{"user_id" => user.id}) end) end diff --git a/lib/pleroma/emails/mailer.ex b/lib/pleroma/emails/mailer.ex index 9cbe7313..eb96f2e8 100644 --- a/lib/pleroma/emails/mailer.ex +++ b/lib/pleroma/emails/mailer.ex @@ -9,7 +9,6 @@ defmodule Pleroma.Emails.Mailer do The module contains functions to delivery email using Swoosh.Mailer. """ - alias Pleroma.Repo alias Pleroma.Workers.MailerWorker alias Swoosh.DeliveryError @@ -19,8 +18,6 @@ defmodule Pleroma.Emails.Mailer do @spec enabled?() :: boolean() def enabled?, do: Pleroma.Config.get([__MODULE__, :enabled]) - import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] - @doc "add email to queue" def deliver_async(email, config \\ []) do encoded_email = @@ -28,9 +25,7 @@ defmodule Pleroma.Emails.Mailer do |> :erlang.term_to_binary() |> Base.encode64() - %{"op" => "email", "encoded_email" => encoded_email, "config" => config} - |> MailerWorker.new(worker_args(:mailer)) - |> Repo.insert() + MailerWorker.enqueue("email", %{"encoded_email" => encoded_email, "config" => config}) end @doc "callback to perform send email from queue" diff --git a/lib/pleroma/scheduled_activity_worker.ex b/lib/pleroma/scheduled_activity_worker.ex index 8bf534f4..c41a542d 100644 --- a/lib/pleroma/scheduled_activity_worker.ex +++ b/lib/pleroma/scheduled_activity_worker.ex @@ -8,7 +8,6 @@ defmodule Pleroma.ScheduledActivityWorker do """ alias Pleroma.Config - alias Pleroma.Repo alias Pleroma.ScheduledActivity alias Pleroma.User alias Pleroma.Web.CommonAPI @@ -18,8 +17,6 @@ defmodule Pleroma.ScheduledActivityWorker do @schedule_interval :timer.minutes(1) - import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] - def start_link(_) do GenServer.start_link(__MODULE__, nil) end @@ -49,9 +46,10 @@ defmodule Pleroma.ScheduledActivityWorker do def handle_info(:perform, state) do ScheduledActivity.due_activities(@schedule_interval) |> Enum.each(fn scheduled_activity -> - %{"op" => "execute", "activity_id" => scheduled_activity.id} - |> Pleroma.Workers.ScheduledActivityWorker.new(worker_args(:scheduled_activities)) - |> Repo.insert() + Pleroma.Workers.ScheduledActivityWorker.enqueue( + "execute", + %{"activity_id" => scheduled_activity.id} + ) end) schedule_next() diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index abfa063f..2fe7e174 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -41,8 +41,6 @@ defmodule Pleroma.User do @strict_local_nickname_regex ~r/^[a-zA-Z\d]+$/ @extended_local_nickname_regex ~r/^[a-zA-Z\d_-]+$/ - import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] - schema "users" do field(:bio, :string) field(:email, :string) @@ -623,9 +621,7 @@ defmodule Pleroma.User do @doc "Fetch some posts when the user has just been federated with" def fetch_initial_posts(user) do - %{"op" => "fetch_initial_posts", "user_id" => user.id} - |> BackgroundWorker.new(worker_args(:background)) - |> Repo.insert() + BackgroundWorker.enqueue("fetch_initial_posts", %{"user_id" => user.id}) end @spec get_followers_query(User.t(), pos_integer() | nil) :: Ecto.Query.t() @@ -1056,9 +1052,7 @@ defmodule Pleroma.User do end def deactivate_async(user, status \\ true) do - %{"op" => "deactivate_user", "user_id" => user.id, "status" => status} - |> BackgroundWorker.new(worker_args(:background)) - |> Repo.insert() + BackgroundWorker.enqueue("deactivate_user", %{"user_id" => user.id, "status" => status}) end def deactivate(%User{} = user, status \\ true) do @@ -1087,9 +1081,7 @@ defmodule Pleroma.User do end def delete(%User{} = user) do - %{"op" => "delete_user", "user_id" => user.id} - |> BackgroundWorker.new(worker_args(:background)) - |> Repo.insert() + BackgroundWorker.enqueue("delete_user", %{"user_id" => user.id}) end @spec perform(atom(), User.t()) :: {:ok, User.t()} @@ -1198,24 +1190,18 @@ defmodule Pleroma.User do end def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers) do - %{ - "op" => "blocks_import", + BackgroundWorker.enqueue("blocks_import", %{ "blocker_id" => blocker.id, "blocked_identifiers" => blocked_identifiers - } - |> BackgroundWorker.new(worker_args(:background)) - |> Repo.insert() + }) end def follow_import(%User{} = follower, followed_identifiers) when is_list(followed_identifiers) do - %{ - "op" => "follow_import", + BackgroundWorker.enqueue("follow_import", %{ "follower_id" => follower.id, "followed_identifiers" => followed_identifiers - } - |> BackgroundWorker.new(worker_args(:background)) - |> Repo.insert() + }) end def delete_user_activities(%User{ap_id: ap_id} = user) do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 74c5eb91..90b40960 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -26,8 +26,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do require Logger require Pleroma.Constants - import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] - # For Announce activities, we filter the recipients based on following status for any actors # that match actual users. See issue #164 for more information about why this is necessary. defp get_recipients(%{"type" => "Announce"} = data) do @@ -148,9 +146,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do activity end - %{"op" => "fetch_data_for_activity", "activity_id" => activity.id} - |> BackgroundWorker.new(worker_args(:background)) - |> Repo.insert() + BackgroundWorker.enqueue("fetch_data_for_activity", %{"activity_id" => activity.id}) Notification.create_notifications(activity) diff --git a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex index 17832155..26b8539f 100644 --- a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex @@ -7,7 +7,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do @behaviour Pleroma.Web.ActivityPub.MRF alias Pleroma.HTTP - alias Pleroma.Repo alias Pleroma.Web.MediaProxy alias Pleroma.Workers.BackgroundWorker @@ -18,8 +17,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do recv_timeout: 10_000 ] - import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] - def perform(:prefetch, url) do Logger.info("Prefetching #{inspect(url)}") @@ -34,9 +31,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do url |> Enum.each(fn %{"href" => href} -> - %{"op" => "media_proxy_prefetch", "url" => href} - |> BackgroundWorker.new(worker_args(:background)) - |> Repo.insert() + BackgroundWorker.enqueue("media_proxy_prefetch", %{"url" => href}) x -> Logger.debug("Unhandled attachment URL object #{inspect(x)}") @@ -52,9 +47,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do %{"type" => "Create", "object" => %{"attachment" => attachments} = _object} = message ) when is_list(attachments) and length(attachments) > 0 do - %{"op" => "media_proxy_preload", "message" => message} - |> BackgroundWorker.new(worker_args(:background)) - |> Repo.insert() + BackgroundWorker.enqueue("media_proxy_preload", %{"message" => message}) {:ok, message} end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 9437f9a1..f27455e8 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -22,8 +22,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do require Logger require Pleroma.Constants - import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] - @doc """ Modifies an incoming AP object (mastodon format) to our internal format. """ @@ -1054,9 +1052,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do already_ap <- User.ap_enabled?(user), {:ok, user} <- user |> User.upgrade_changeset(data) |> User.update_and_set_cache() do unless already_ap do - %{"op" => "user_upgrade", "user_id" => user.id} - |> TransmogrifierWorker.new(worker_args(:transmogrifier)) - |> Repo.insert() + TransmogrifierWorker.enqueue("user_upgrade", %{"user_id" => user.id}) end {:ok, user} diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 8f43066e..1a2da014 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -18,8 +18,6 @@ defmodule Pleroma.Web.Federator do require Logger - import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] - def init do # To do: consider removing this call in favor of scheduled execution (`quantum`-based) refresh_subscriptions(schedule_in: 60) @@ -40,15 +38,11 @@ defmodule Pleroma.Web.Federator do # Client API def incoming_doc(doc) do - %{"op" => "incoming_doc", "body" => doc} - |> ReceiverWorker.new(worker_args(:federator_incoming)) - |> Pleroma.Repo.insert() + ReceiverWorker.enqueue("incoming_doc", %{"body" => doc}) end def incoming_ap_doc(params) do - %{"op" => "incoming_ap_doc", "params" => params} - |> ReceiverWorker.new(worker_args(:federator_incoming)) - |> Pleroma.Repo.insert() + ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}) end def publish(%{id: "pleroma:fakeid"} = activity) do @@ -56,27 +50,19 @@ defmodule Pleroma.Web.Federator do end def publish(activity) do - %{"op" => "publish", "activity_id" => activity.id} - |> PublisherWorker.new(worker_args(:federator_outgoing)) - |> Pleroma.Repo.insert() + PublisherWorker.enqueue("publish", %{"activity_id" => activity.id}) end def verify_websub(websub) do - %{"op" => "verify_websub", "websub_id" => websub.id} - |> SubscriberWorker.new(worker_args(:federator_outgoing)) - |> Pleroma.Repo.insert() + SubscriberWorker.enqueue("verify_websub", %{"websub_id" => websub.id}) end def request_subscription(websub) do - %{"op" => "request_subscription", "websub_id" => websub.id} - |> SubscriberWorker.new(worker_args(:federator_outgoing)) - |> Pleroma.Repo.insert() + SubscriberWorker.enqueue("request_subscription", %{"websub_id" => websub.id}) end def refresh_subscriptions(worker_args \\ []) do - %{"op" => "refresh_subscriptions"} - |> SubscriberWorker.new(worker_args ++ [max_attempts: 1] ++ worker_args(:federator_outgoing)) - |> Pleroma.Repo.insert() + SubscriberWorker.enqueue("refresh_subscriptions", %{}, worker_args ++ [max_attempts: 1]) end # Job Worker Callbacks diff --git a/lib/pleroma/web/federator/publisher.ex b/lib/pleroma/web/federator/publisher.ex index 42be109a..93706463 100644 --- a/lib/pleroma/web/federator/publisher.ex +++ b/lib/pleroma/web/federator/publisher.ex @@ -31,11 +31,10 @@ defmodule Pleroma.Web.Federator.Publisher do """ @spec enqueue_one(module(), Map.t()) :: :ok def enqueue_one(module, %{} = params) do - worker_args = Pleroma.Workers.WorkerHelper.worker_args(:federator_outgoing) - - %{"op" => "publish_one", "module" => to_string(module), "params" => params} - |> PublisherWorker.new(worker_args) - |> Pleroma.Repo.insert() + PublisherWorker.enqueue( + "publish_one", + %{"module" => to_string(module), "params" => params} + ) end @doc """ diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex index b150a68a..eb94bf86 100644 --- a/lib/pleroma/web/oauth/token/clean_worker.ex +++ b/lib/pleroma/web/oauth/token/clean_worker.ex @@ -16,12 +16,9 @@ defmodule Pleroma.Web.OAuth.Token.CleanWorker do @one_day ) - alias Pleroma.Repo alias Pleroma.Web.OAuth.Token alias Pleroma.Workers.BackgroundWorker - import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] - def start_link(_), do: GenServer.start_link(__MODULE__, %{}) def init(_) do @@ -31,9 +28,7 @@ defmodule Pleroma.Web.OAuth.Token.CleanWorker do @doc false def handle_info(:perform, state) do - %{"op" => "clean_expired_tokens"} - |> BackgroundWorker.new(worker_args(:background)) - |> Repo.insert() + BackgroundWorker.enqueue("clean_expired_tokens", %{}) Process.send_after(self(), :perform, @interval) {:noreply, state} diff --git a/lib/pleroma/web/push/push.ex b/lib/pleroma/web/push/push.ex index 4973b529..7ef1532a 100644 --- a/lib/pleroma/web/push/push.ex +++ b/lib/pleroma/web/push/push.ex @@ -3,13 +3,10 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Push do - alias Pleroma.Repo alias Pleroma.Workers.WebPusherWorker require Logger - import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] - def init do unless enabled() do Logger.warn(""" @@ -35,8 +32,6 @@ defmodule Pleroma.Web.Push do end def send(notification) do - %{"op" => "web_push", "notification_id" => notification.id} - |> WebPusherWorker.new(worker_args(:web_push)) - |> Repo.insert() + WebPusherWorker.enqueue("web_push", %{"notification_id" => notification.id}) end end diff --git a/lib/pleroma/workers/activity_expiration_worker.ex b/lib/pleroma/workers/activity_expiration_worker.ex index 0b491eab..60dd3feb 100644 --- a/lib/pleroma/workers/activity_expiration_worker.ex +++ b/lib/pleroma/workers/activity_expiration_worker.ex @@ -8,6 +8,8 @@ defmodule Pleroma.Workers.ActivityExpirationWorker do queue: "activity_expiration", max_attempts: 1 + use Pleroma.Workers.WorkerHelper, queue: "activity_expiration" + @impl Oban.Worker def perform( %{ diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index 7b5575a5..b9aef3a9 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -13,6 +13,8 @@ defmodule Pleroma.Workers.BackgroundWorker do queue: "background", max_attempts: 1 + use Pleroma.Workers.WorkerHelper, queue: "background" + @impl Oban.Worker def perform(%{"op" => "fetch_initial_posts", "user_id" => user_id}, _job) do user = User.get_cached_by_id(user_id) diff --git a/lib/pleroma/workers/digest_emails_worker.ex b/lib/pleroma/workers/digest_emails_worker.ex new file mode 100644 index 00000000..ca073ce6 --- /dev/null +++ b/lib/pleroma/workers/digest_emails_worker.ex @@ -0,0 +1,21 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.DigestEmailsWorker do + alias Pleroma.User + + # Note: `max_attempts` is intended to be overridden in `new/2` call + use Oban.Worker, + queue: "digest_emails", + max_attempts: 1 + + use Pleroma.Workers.WorkerHelper, queue: "digest_emails" + + @impl Oban.Worker + def perform(%{"op" => "digest_email", "user_id" => user_id}, _job) do + user_id + |> User.get_cached_by_id() + |> Pleroma.DigestEmailWorker.perform() + end +end diff --git a/lib/pleroma/workers/mailer_worker.ex b/lib/pleroma/workers/mailer_worker.ex index 4f73d61b..a4bd54a6 100644 --- a/lib/pleroma/workers/mailer_worker.ex +++ b/lib/pleroma/workers/mailer_worker.ex @@ -3,13 +3,13 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.MailerWorker do - alias Pleroma.User - # Note: `max_attempts` is intended to be overridden in `new/2` call use Oban.Worker, queue: "mailer", max_attempts: 1 + use Pleroma.Workers.WorkerHelper, queue: "mailer" + @impl Oban.Worker def perform(%{"op" => "email", "encoded_email" => encoded_email, "config" => config}, _job) do encoded_email @@ -17,10 +17,4 @@ defmodule Pleroma.Workers.MailerWorker do |> :erlang.binary_to_term() |> Pleroma.Emails.Mailer.deliver(config) end - - def perform(%{"op" => "digest_email", "user_id" => user_id}, _job) do - user_id - |> User.get_cached_by_id() - |> Pleroma.DigestEmailWorker.perform() - end end diff --git a/lib/pleroma/workers/publisher_worker.ex b/lib/pleroma/workers/publisher_worker.ex index 5671d2a2..a3ac2263 100644 --- a/lib/pleroma/workers/publisher_worker.ex +++ b/lib/pleroma/workers/publisher_worker.ex @@ -11,6 +11,8 @@ defmodule Pleroma.Workers.PublisherWorker do queue: "federator_outgoing", max_attempts: 1 + use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing" + def backoff(attempt) when is_integer(attempt) do Pleroma.Workers.WorkerHelper.sidekiq_backoff(attempt, 5) end diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index cdce630f..3cc415ce 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -10,6 +10,8 @@ defmodule Pleroma.Workers.ReceiverWorker do queue: "federator_incoming", max_attempts: 1 + use Pleroma.Workers.WorkerHelper, queue: "federator_incoming" + @impl Oban.Worker def perform(%{"op" => "incoming_doc", "body" => doc}, _job) do Federator.perform(:incoming_doc, doc) diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex index 4094411a..936bb64d 100644 --- a/lib/pleroma/workers/scheduled_activity_worker.ex +++ b/lib/pleroma/workers/scheduled_activity_worker.ex @@ -8,6 +8,8 @@ defmodule Pleroma.Workers.ScheduledActivityWorker do queue: "scheduled_activities", max_attempts: 1 + use Pleroma.Workers.WorkerHelper, queue: "scheduled_activities" + @impl Oban.Worker def perform(%{"op" => "execute", "activity_id" => activity_id}, _job) do Pleroma.ScheduledActivityWorker.perform(:execute, activity_id) diff --git a/lib/pleroma/workers/subscriber_worker.ex b/lib/pleroma/workers/subscriber_worker.ex index 22d1dc95..4fb99455 100644 --- a/lib/pleroma/workers/subscriber_worker.ex +++ b/lib/pleroma/workers/subscriber_worker.ex @@ -12,6 +12,8 @@ defmodule Pleroma.Workers.SubscriberWorker do queue: "federator_outgoing", max_attempts: 1 + use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing" + @impl Oban.Worker def perform(%{"op" => "refresh_subscriptions"}, _job) do Federator.perform(:refresh_subscriptions) diff --git a/lib/pleroma/workers/transmogrifier_worker.ex b/lib/pleroma/workers/transmogrifier_worker.ex index 6f5c1a2f..6fecc2bf 100644 --- a/lib/pleroma/workers/transmogrifier_worker.ex +++ b/lib/pleroma/workers/transmogrifier_worker.ex @@ -10,6 +10,8 @@ defmodule Pleroma.Workers.TransmogrifierWorker do queue: "transmogrifier", max_attempts: 1 + use Pleroma.Workers.WorkerHelper, queue: "transmogrifier" + @impl Oban.Worker def perform(%{"op" => "user_upgrade", "user_id" => user_id}, _job) do user = User.get_cached_by_id(user_id) diff --git a/lib/pleroma/workers/web_pusher_worker.ex b/lib/pleroma/workers/web_pusher_worker.ex index 2b1d3b99..4c2591a5 100644 --- a/lib/pleroma/workers/web_pusher_worker.ex +++ b/lib/pleroma/workers/web_pusher_worker.ex @@ -11,6 +11,8 @@ defmodule Pleroma.Workers.WebPusherWorker do queue: "web_push", max_attempts: 1 + use Pleroma.Workers.WorkerHelper, queue: "web_push" + @impl Oban.Worker def perform(%{"op" => "web_push", "notification_id" => notification_id}, _job) do notification = Repo.get(Notification, notification_id) diff --git a/lib/pleroma/workers/worker_helper.ex b/lib/pleroma/workers/worker_helper.ex index f9ed2e64..b12f198d 100644 --- a/lib/pleroma/workers/worker_helper.ex +++ b/lib/pleroma/workers/worker_helper.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Workers.WorkerHelper do alias Pleroma.Config + alias Pleroma.Workers.WorkerHelper def worker_args(queue) do case Config.get([:workers, :retries, queue]) do @@ -20,4 +21,21 @@ defmodule Pleroma.Workers.WorkerHelper do trunc(backoff) end + + defmacro __using__(opts) do + caller_module = __CALLER__.module + queue = Keyword.fetch!(opts, :queue) + + quote do + def enqueue(op, params, worker_args \\ []) do + params = Map.merge(%{"op" => op}, params) + queue_atom = String.to_atom(unquote(queue)) + worker_args = worker_args ++ WorkerHelper.worker_args(queue_atom) + + unquote(caller_module) + |> apply(:new, [params, worker_args]) + |> Pleroma.Repo.insert() + end + end + end end From 35ef470d000c53e21c6f867d53ca3a83260d93b8 Mon Sep 17 00:00:00 2001 From: Sadposter Date: Mon, 2 Sep 2019 12:15:21 +0100 Subject: [PATCH 013/138] truncate fields for remote users instead --- lib/pleroma/user/info.ex | 7 +++++++ test/user_test.exs | 17 +++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 779bfbc1..0beb2f72 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -242,6 +242,7 @@ defmodule Pleroma.User.Info do end def remote_user_creation(info, params) do + params = Map.put(params, "fields", Enum.map(params["fields"], &truncate_field/1)) info |> cast(params, [ :ap_enabled, @@ -326,6 +327,12 @@ defmodule Pleroma.User.Info do defp valid_field?(_), do: false + defp truncate_field(%{"name" => name, "value" => value}) do + {name, _chopped} = String.split_at(name, Pleroma.Config.get([:instance, :account_field_name_length], 255)) + {value, _chopped} = String.split_at(value, Pleroma.Config.get([:instance, :account_field_value_length], 255)) + %{"name" => name, "value" => value} + end + @spec confirmation_changeset(Info.t(), keyword()) :: Changeset.t() def confirmation_changeset(info, opts) do need_confirmation? = Keyword.get(opts, :need_confirmation) diff --git a/test/user_test.exs b/test/user_test.exs index 2cbc1f52..68a469fe 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1117,11 +1117,20 @@ defmodule Pleroma.UserTest do assert {:ok, _key} = User.get_public_key_for_ap_id("http://mastodon.example.org/users/admin") end - test "insert or update a user from given data" do - user = insert(:user, %{nickname: "nick@name.de"}) - data = %{ap_id: user.ap_id <> "xxx", name: user.name, nickname: user.nickname} + describe "insert or update a user from given data" do + test "with normal data" do + user = insert(:user, %{nickname: "nick@name.de"}) + data = %{ap_id: user.ap_id <> "xxx", name: user.name, nickname: user.nickname} - assert {:ok, %User{}} = User.insert_or_update_user(data) + assert {:ok, %User{}} = User.insert_or_update_user(data) + end + + test "with overly long fields" do + current_max_length = Pleroma.Config.get([:instance, :account_field_value_length], 255) + user = insert(:user, nickname: "nickname@supergood.domain") + data = %{ap_id: user.ap_id, info: %{ fields: [%{"name" => "myfield", "value" => String.duplicate("h", current_max_length + 1)}] }} + assert {:ok, %User{}} = User.insert_or_update_user(data) + end end describe "per-user rich-text filtering" do From 05c935c3961e4c1a20c7713611920318d45d4b57 Mon Sep 17 00:00:00 2001 From: Sadposter Date: Mon, 2 Sep 2019 12:15:40 +0100 Subject: [PATCH 014/138] mix format --- lib/pleroma/user/info.ex | 9 +++++++-- test/user_test.exs | 23 ++++++++++++++++------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 0beb2f72..ca1282d0 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -243,6 +243,7 @@ defmodule Pleroma.User.Info do def remote_user_creation(info, params) do params = Map.put(params, "fields", Enum.map(params["fields"], &truncate_field/1)) + info |> cast(params, [ :ap_enabled, @@ -328,8 +329,12 @@ defmodule Pleroma.User.Info do defp valid_field?(_), do: false defp truncate_field(%{"name" => name, "value" => value}) do - {name, _chopped} = String.split_at(name, Pleroma.Config.get([:instance, :account_field_name_length], 255)) - {value, _chopped} = String.split_at(value, Pleroma.Config.get([:instance, :account_field_value_length], 255)) + {name, _chopped} = + String.split_at(name, Pleroma.Config.get([:instance, :account_field_name_length], 255)) + + {value, _chopped} = + String.split_at(value, Pleroma.Config.get([:instance, :account_field_value_length], 255)) + %{"name" => name, "value" => value} end diff --git a/test/user_test.exs b/test/user_test.exs index 68a469fe..0ca31033 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1119,17 +1119,26 @@ defmodule Pleroma.UserTest do describe "insert or update a user from given data" do test "with normal data" do - user = insert(:user, %{nickname: "nick@name.de"}) - data = %{ap_id: user.ap_id <> "xxx", name: user.name, nickname: user.nickname} + user = insert(:user, %{nickname: "nick@name.de"}) + data = %{ap_id: user.ap_id <> "xxx", name: user.name, nickname: user.nickname} - assert {:ok, %User{}} = User.insert_or_update_user(data) + assert {:ok, %User{}} = User.insert_or_update_user(data) end test "with overly long fields" do - current_max_length = Pleroma.Config.get([:instance, :account_field_value_length], 255) - user = insert(:user, nickname: "nickname@supergood.domain") - data = %{ap_id: user.ap_id, info: %{ fields: [%{"name" => "myfield", "value" => String.duplicate("h", current_max_length + 1)}] }} - assert {:ok, %User{}} = User.insert_or_update_user(data) + current_max_length = Pleroma.Config.get([:instance, :account_field_value_length], 255) + user = insert(:user, nickname: "nickname@supergood.domain") + + data = %{ + ap_id: user.ap_id, + info: %{ + fields: [ + %{"name" => "myfield", "value" => String.duplicate("h", current_max_length + 1)} + ] + } + } + + assert {:ok, %User{}} = User.insert_or_update_user(data) end end From d0f07e55d28d25684130cb1090d0bdbb48807548 Mon Sep 17 00:00:00 2001 From: Sadposter Date: Mon, 2 Sep 2019 12:31:23 +0100 Subject: [PATCH 015/138] use atom key for fields --- lib/pleroma/user/info.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index ca1282d0..151e025d 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -242,7 +242,12 @@ defmodule Pleroma.User.Info do end def remote_user_creation(info, params) do - params = Map.put(params, "fields", Enum.map(params["fields"], &truncate_field/1)) + params = + if Map.has_key?(params, :fields) do + Map.put(params, :fields, Enum.map(params[:fields], &truncate_field/1)) + else + params + end info |> cast(params, [ From e73685834c1797404c943f66417ffa30add87e04 Mon Sep 17 00:00:00 2001 From: Sadposter Date: Mon, 2 Sep 2019 12:35:55 +0100 Subject: [PATCH 016/138] add mandatory fields for user update --- test/user_test.exs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/user_test.exs b/test/user_test.exs index 0ca31033..92a48f63 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1131,6 +1131,8 @@ defmodule Pleroma.UserTest do data = %{ ap_id: user.ap_id, + name: user.name, + nickname: user.nickname, info: %{ fields: [ %{"name" => "myfield", "value" => String.duplicate("h", current_max_length + 1)} From b49085c156a6a4449c95c2c315f6250317122735 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 2 Sep 2019 14:57:40 +0300 Subject: [PATCH 017/138] [#1149] Refactoring: GenServer workers renamed to daemons, `use Oban.Worker` moved to helper. --- config/config.exs | 2 +- lib/pleroma/application.ex | 4 ++-- .../activity_expiration_daemon.ex} | 2 +- .../digest_email_daemon.ex} | 2 +- .../scheduled_activity_daemon.ex} | 2 +- lib/pleroma/workers/activity_expiration_worker.ex | 7 +------ lib/pleroma/workers/background_worker.ex | 5 ----- lib/pleroma/workers/digest_emails_worker.ex | 7 +------ lib/pleroma/workers/mailer_worker.ex | 5 ----- lib/pleroma/workers/publisher_worker.ex | 5 ----- lib/pleroma/workers/receiver_worker.ex | 5 ----- lib/pleroma/workers/scheduled_activity_worker.ex | 7 +------ lib/pleroma/workers/subscriber_worker.ex | 5 ----- lib/pleroma/workers/transmogrifier_worker.ex | 5 ----- lib/pleroma/workers/web_pusher_worker.ex | 5 ----- lib/pleroma/workers/worker_helper.ex | 5 +++++ .../activity_expiration_daemon_test.exs} | 2 +- .../digest_email_daemon_test.exs} | 6 +++--- .../scheduled_activity_daemon_test.exs} | 4 ++-- 19 files changed, 20 insertions(+), 65 deletions(-) rename lib/pleroma/{activity_expiration_worker.ex => daemons/activity_expiration_daemon.ex} (96%) rename lib/pleroma/{digest_email_worker.ex => daemons/digest_email_daemon.ex} (96%) rename lib/pleroma/{scheduled_activity_worker.ex => daemons/scheduled_activity_daemon.ex} (96%) rename test/{activity_expiration_worker_test.exs => daemons/activity_expiration_daemon_test.exs} (86%) rename test/{web/digest_email_worker_test.exs => daemons/digest_email_daemon_test.exs} (88%) rename test/{scheduled_activity_worker_test.exs => daemons/scheduled_activity_daemon_test.exs} (82%) diff --git a/config/config.exs b/config/config.exs index 6fb4a096..b742a650 100644 --- a/config/config.exs +++ b/config/config.exs @@ -54,7 +54,7 @@ config :pleroma, Pleroma.Repo, scheduled_jobs = with digest_config <- Application.get_env(:pleroma, :email_notifications)[:digest], true <- digest_config[:active] do - [{digest_config[:schedule], {Pleroma.DigestEmailWorker, :perform, []}}] + [{digest_config[:schedule], {Pleroma.Daemons.DigestEmailDaemon, :perform, []}}] else _ -> [] end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index f8f866db..0c27027a 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -36,8 +36,8 @@ defmodule Pleroma.Application do Pleroma.Emoji, Pleroma.Captcha, Pleroma.FlakeId, - Pleroma.ScheduledActivityWorker, - Pleroma.ActivityExpirationWorker + Pleroma.Daemons.ScheduledActivityDaemon, + Pleroma.Daemons.ActivityExpirationDaemon ] ++ cachex_children() ++ hackney_pool_children() ++ diff --git a/lib/pleroma/activity_expiration_worker.ex b/lib/pleroma/daemons/activity_expiration_daemon.ex similarity index 96% rename from lib/pleroma/activity_expiration_worker.ex rename to lib/pleroma/daemons/activity_expiration_daemon.ex index c0820c20..cab7628c 100644 --- a/lib/pleroma/activity_expiration_worker.ex +++ b/lib/pleroma/daemons/activity_expiration_daemon.ex @@ -2,7 +2,7 @@ # Copyright © 2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.ActivityExpirationWorker do +defmodule Pleroma.Daemons.ActivityExpirationDaemon do alias Pleroma.Activity alias Pleroma.ActivityExpiration alias Pleroma.Config diff --git a/lib/pleroma/digest_email_worker.ex b/lib/pleroma/daemons/digest_email_daemon.ex similarity index 96% rename from lib/pleroma/digest_email_worker.ex rename to lib/pleroma/daemons/digest_email_daemon.ex index 5be7cf26..462ad2c5 100644 --- a/lib/pleroma/digest_email_worker.ex +++ b/lib/pleroma/daemons/digest_email_daemon.ex @@ -2,7 +2,7 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.DigestEmailWorker do +defmodule Pleroma.Daemons.DigestEmailDaemon do alias Pleroma.Repo alias Pleroma.Workers.DigestEmailsWorker diff --git a/lib/pleroma/scheduled_activity_worker.ex b/lib/pleroma/daemons/scheduled_activity_daemon.ex similarity index 96% rename from lib/pleroma/scheduled_activity_worker.ex rename to lib/pleroma/daemons/scheduled_activity_daemon.ex index c41a542d..aee5f723 100644 --- a/lib/pleroma/scheduled_activity_worker.ex +++ b/lib/pleroma/daemons/scheduled_activity_daemon.ex @@ -2,7 +2,7 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.ScheduledActivityWorker do +defmodule Pleroma.Daemons.ScheduledActivityDaemon do @moduledoc """ Sends scheduled activities to the job queue. """ diff --git a/lib/pleroma/workers/activity_expiration_worker.ex b/lib/pleroma/workers/activity_expiration_worker.ex index 60dd3feb..4e3e4195 100644 --- a/lib/pleroma/workers/activity_expiration_worker.ex +++ b/lib/pleroma/workers/activity_expiration_worker.ex @@ -3,11 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.ActivityExpirationWorker do - # Note: `max_attempts` is intended to be overridden in `new/2` call - use Oban.Worker, - queue: "activity_expiration", - max_attempts: 1 - use Pleroma.Workers.WorkerHelper, queue: "activity_expiration" @impl Oban.Worker @@ -18,6 +13,6 @@ defmodule Pleroma.Workers.ActivityExpirationWorker do }, _job ) do - Pleroma.ActivityExpirationWorker.perform(:execute, activity_expiration_id) + Pleroma.Daemons.ActivityExpirationDaemon.perform(:execute, activity_expiration_id) end end diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index b9aef3a9..082f20ab 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -8,11 +8,6 @@ defmodule Pleroma.Workers.BackgroundWorker do alias Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy alias Pleroma.Web.OAuth.Token.CleanWorker - # Note: `max_attempts` is intended to be overridden in `new/2` call - use Oban.Worker, - queue: "background", - max_attempts: 1 - use Pleroma.Workers.WorkerHelper, queue: "background" @impl Oban.Worker diff --git a/lib/pleroma/workers/digest_emails_worker.ex b/lib/pleroma/workers/digest_emails_worker.ex index ca073ce6..3e5a836d 100644 --- a/lib/pleroma/workers/digest_emails_worker.ex +++ b/lib/pleroma/workers/digest_emails_worker.ex @@ -5,17 +5,12 @@ defmodule Pleroma.Workers.DigestEmailsWorker do alias Pleroma.User - # Note: `max_attempts` is intended to be overridden in `new/2` call - use Oban.Worker, - queue: "digest_emails", - max_attempts: 1 - use Pleroma.Workers.WorkerHelper, queue: "digest_emails" @impl Oban.Worker def perform(%{"op" => "digest_email", "user_id" => user_id}, _job) do user_id |> User.get_cached_by_id() - |> Pleroma.DigestEmailWorker.perform() + |> Pleroma.Daemons.DigestEmailDaemon.perform() end end diff --git a/lib/pleroma/workers/mailer_worker.ex b/lib/pleroma/workers/mailer_worker.ex index a4bd54a6..1b7a0eb3 100644 --- a/lib/pleroma/workers/mailer_worker.ex +++ b/lib/pleroma/workers/mailer_worker.ex @@ -3,11 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.MailerWorker do - # Note: `max_attempts` is intended to be overridden in `new/2` call - use Oban.Worker, - queue: "mailer", - max_attempts: 1 - use Pleroma.Workers.WorkerHelper, queue: "mailer" @impl Oban.Worker diff --git a/lib/pleroma/workers/publisher_worker.ex b/lib/pleroma/workers/publisher_worker.ex index a3ac2263..455f7fc7 100644 --- a/lib/pleroma/workers/publisher_worker.ex +++ b/lib/pleroma/workers/publisher_worker.ex @@ -6,11 +6,6 @@ defmodule Pleroma.Workers.PublisherWorker do alias Pleroma.Activity alias Pleroma.Web.Federator - # Note: `max_attempts` is intended to be overridden in `new/2` call - use Oban.Worker, - queue: "federator_outgoing", - max_attempts: 1 - use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing" def backoff(attempt) when is_integer(attempt) do diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index 3cc415ce..83d528a6 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -5,11 +5,6 @@ defmodule Pleroma.Workers.ReceiverWorker do alias Pleroma.Web.Federator - # Note: `max_attempts` is intended to be overridden in `new/2` call - use Oban.Worker, - queue: "federator_incoming", - max_attempts: 1 - use Pleroma.Workers.WorkerHelper, queue: "federator_incoming" @impl Oban.Worker diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex index 936bb64d..ca7d53af 100644 --- a/lib/pleroma/workers/scheduled_activity_worker.ex +++ b/lib/pleroma/workers/scheduled_activity_worker.ex @@ -3,15 +3,10 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.ScheduledActivityWorker do - # Note: `max_attempts` is intended to be overridden in `new/2` call - use Oban.Worker, - queue: "scheduled_activities", - max_attempts: 1 - use Pleroma.Workers.WorkerHelper, queue: "scheduled_activities" @impl Oban.Worker def perform(%{"op" => "execute", "activity_id" => activity_id}, _job) do - Pleroma.ScheduledActivityWorker.perform(:execute, activity_id) + Pleroma.Daemons.ScheduledActivityDaemon.perform(:execute, activity_id) end end diff --git a/lib/pleroma/workers/subscriber_worker.ex b/lib/pleroma/workers/subscriber_worker.ex index 4fb99455..fc490e30 100644 --- a/lib/pleroma/workers/subscriber_worker.ex +++ b/lib/pleroma/workers/subscriber_worker.ex @@ -7,11 +7,6 @@ defmodule Pleroma.Workers.SubscriberWorker do alias Pleroma.Web.Federator alias Pleroma.Web.Websub - # Note: `max_attempts` is intended to be overridden in `new/2` call - use Oban.Worker, - queue: "federator_outgoing", - max_attempts: 1 - use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing" @impl Oban.Worker diff --git a/lib/pleroma/workers/transmogrifier_worker.ex b/lib/pleroma/workers/transmogrifier_worker.ex index 6fecc2bf..b581a2f8 100644 --- a/lib/pleroma/workers/transmogrifier_worker.ex +++ b/lib/pleroma/workers/transmogrifier_worker.ex @@ -5,11 +5,6 @@ defmodule Pleroma.Workers.TransmogrifierWorker do alias Pleroma.User - # Note: `max_attempts` is intended to be overridden in `new/2` call - use Oban.Worker, - queue: "transmogrifier", - max_attempts: 1 - use Pleroma.Workers.WorkerHelper, queue: "transmogrifier" @impl Oban.Worker diff --git a/lib/pleroma/workers/web_pusher_worker.ex b/lib/pleroma/workers/web_pusher_worker.ex index 4c2591a5..bea2baff 100644 --- a/lib/pleroma/workers/web_pusher_worker.ex +++ b/lib/pleroma/workers/web_pusher_worker.ex @@ -6,11 +6,6 @@ defmodule Pleroma.Workers.WebPusherWorker do alias Pleroma.Notification alias Pleroma.Repo - # Note: `max_attempts` is intended to be overridden in `new/2` call - use Oban.Worker, - queue: "web_push", - max_attempts: 1 - use Pleroma.Workers.WorkerHelper, queue: "web_push" @impl Oban.Worker diff --git a/lib/pleroma/workers/worker_helper.ex b/lib/pleroma/workers/worker_helper.ex index b12f198d..358efa14 100644 --- a/lib/pleroma/workers/worker_helper.ex +++ b/lib/pleroma/workers/worker_helper.ex @@ -27,6 +27,11 @@ defmodule Pleroma.Workers.WorkerHelper do queue = Keyword.fetch!(opts, :queue) quote do + # Note: `max_attempts` is intended to be overridden in `new/2` call + use Oban.Worker, + queue: unquote(queue), + max_attempts: 1 + def enqueue(op, params, worker_args \\ []) do params = Map.merge(%{"op" => op}, params) queue_atom = String.to_atom(unquote(queue)) diff --git a/test/activity_expiration_worker_test.exs b/test/daemons/activity_expiration_daemon_test.exs similarity index 86% rename from test/activity_expiration_worker_test.exs rename to test/daemons/activity_expiration_daemon_test.exs index 939d912f..31f4a70a 100644 --- a/test/activity_expiration_worker_test.exs +++ b/test/daemons/activity_expiration_daemon_test.exs @@ -10,7 +10,7 @@ defmodule Pleroma.ActivityExpirationWorkerTest do test "deletes an activity" do activity = insert(:note_activity) expiration = insert(:expiration_in_the_past, %{activity_id: activity.id}) - Pleroma.ActivityExpirationWorker.perform(:execute, expiration.id) + Pleroma.Daemons.ActivityExpirationDaemon.perform(:execute, expiration.id) refute Repo.get(Activity, activity.id) end diff --git a/test/web/digest_email_worker_test.exs b/test/daemons/digest_email_daemon_test.exs similarity index 88% rename from test/web/digest_email_worker_test.exs rename to test/daemons/digest_email_daemon_test.exs index 5dfd920f..3168f3b9 100644 --- a/test/web/digest_email_worker_test.exs +++ b/test/daemons/digest_email_daemon_test.exs @@ -2,11 +2,11 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.DigestEmailWorkerTest do +defmodule Pleroma.DigestEmailDaemonTest do use Pleroma.DataCase import Pleroma.Factory - alias Pleroma.DigestEmailWorker + alias Pleroma.Daemons.DigestEmailDaemon alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.CommonAPI @@ -23,7 +23,7 @@ defmodule Pleroma.DigestEmailWorkerTest do User.switch_email_notifications(user2, "digest", true) CommonAPI.post(user, %{"status" => "hey @#{user2.nickname}!"}) - DigestEmailWorker.perform() + DigestEmailDaemon.perform() ObanHelpers.perform_all() # Performing job(s) enqueued at previous step ObanHelpers.perform_all() diff --git a/test/scheduled_activity_worker_test.exs b/test/daemons/scheduled_activity_daemon_test.exs similarity index 82% rename from test/scheduled_activity_worker_test.exs rename to test/daemons/scheduled_activity_daemon_test.exs index e3ad1244..32820b2b 100644 --- a/test/scheduled_activity_worker_test.exs +++ b/test/daemons/scheduled_activity_daemon_test.exs @@ -2,7 +2,7 @@ # Copyright © 2017-2018 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.ScheduledActivityWorkerTest do +defmodule Pleroma.ScheduledActivityDaemonTest do use Pleroma.DataCase alias Pleroma.ScheduledActivity import Pleroma.Factory @@ -10,7 +10,7 @@ defmodule Pleroma.ScheduledActivityWorkerTest do test "creates a status from the scheduled activity" do user = insert(:user) scheduled_activity = insert(:scheduled_activity, user: user, params: %{status: "hi"}) - Pleroma.ScheduledActivityWorker.perform(:execute, scheduled_activity.id) + Pleroma.Daemons.ScheduledActivityDaemon.perform(:execute, scheduled_activity.id) refute Repo.get(ScheduledActivity, scheduled_activity.id) activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) From 8cbad5500cefbba1e0bb67604960fc331b75b498 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 4 Sep 2019 15:25:12 +0300 Subject: [PATCH 018/138] add tests for activity_pub/utils.ex --- lib/pleroma/user.ex | 1 + lib/pleroma/web/activity_pub/activity_pub.ex | 12 +- lib/pleroma/web/activity_pub/utils.ex | 298 +++++++++---------- test/web/activity_pub/utils_test.exs | 232 ++++++++++++++- 4 files changed, 371 insertions(+), 172 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 29fd6d2e..424ed772 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -147,6 +147,7 @@ defmodule Pleroma.User do Cachex.fetch!(:user_cache, key, fn _ -> {:commit, follow_state(user, target)} end) end + @spec set_follow_state_cache(String.t(), String.t(), String.t()) :: {:ok | :error, boolean()} def set_follow_state_cache(user_ap_id, target_ap_id, state) do Cachex.put( :user_cache, diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index eeb82681..39b46a59 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -435,6 +435,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end + @spec block(User.t(), User.t(), String.t() | nil, boolean) :: {:ok, Activity.t() | nil} def block(blocker, blocked, activity_id \\ nil, local \\ true) do outgoing_blocks = Config.get([:activitypub, :outgoing_blocks]) unfollow_blocked = Config.get([:activitypub, :unfollow_blocked]) @@ -463,10 +464,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end + @spec flag(map()) :: {:ok, Activity.t()} | any def flag( %{ actor: actor, - context: context, + context: _context, account: account, statuses: statuses, content: content @@ -478,14 +480,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do additional = params[:additional] || %{} - params = %{ - actor: actor, - context: context, - account: account, - statuses: statuses, - content: content - } - additional = if forward do Map.merge(additional, %{"to" => [], "cc" => [account.ap_id]}) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index c9c0c376..cf82d1a9 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -33,50 +33,40 @@ defmodule Pleroma.Web.ActivityPub.Utils do Map.put(params, "actor", get_ap_id(params["actor"])) end - def determine_explicit_mentions(%{"tag" => tag} = _object) when is_list(tag) do - tag - |> Enum.filter(fn x -> is_map(x) end) - |> Enum.filter(fn x -> x["type"] == "Mention" end) - |> Enum.map(fn x -> x["href"] end) + @spec determine_explicit_mentions(map()) :: map() + def determine_explicit_mentions(%{"tag" => tag} = _) when is_list(tag) do + Enum.flat_map(tag, fn + %{"type" => "Mention", "href" => href} -> [href] + _ -> [] + end) end def determine_explicit_mentions(%{"tag" => tag} = object) when is_map(tag) do - Map.put(object, "tag", [tag]) + object + |> Map.put("tag", [tag]) |> determine_explicit_mentions() end def determine_explicit_mentions(_), do: [] + @spec recipient_in_collection(any(), any()) :: boolean() defp recipient_in_collection(ap_id, coll) when is_binary(coll), do: ap_id == coll defp recipient_in_collection(ap_id, coll) when is_list(coll), do: ap_id in coll defp recipient_in_collection(_, _), do: false + @spec recipient_in_message(User.t(), User.t(), map()) :: boolean() def recipient_in_message(%User{ap_id: ap_id} = recipient, %User{} = actor, params) do + addresses = [params["to"], params["cc"], params["bto"], params["bcc"]] + cond do - recipient_in_collection(ap_id, params["to"]) -> - true - - recipient_in_collection(ap_id, params["cc"]) -> - true - - recipient_in_collection(ap_id, params["bto"]) -> - true - - recipient_in_collection(ap_id, params["bcc"]) -> - true - + Enum.any?(addresses, &recipient_in_collection(ap_id, &1)) -> true # if the message is unaddressed at all, then assume it is directly addressed # to the recipient - !params["to"] && !params["cc"] && !params["bto"] && !params["bcc"] -> - true - + Enum.all?(addresses, &is_nil(&1)) -> true # if the message is sent from somebody the user is following, then assume it # is addressed to the recipient - User.following?(recipient, actor) -> - true - - true -> - false + User.following?(recipient, actor) -> true + true -> false end end @@ -188,53 +178,58 @@ defmodule Pleroma.Web.ActivityPub.Utils do Adds an id and a published data if they aren't there, also adds it to an included object """ - def lazy_put_activity_defaults(map, fake \\ false) do - map = - unless fake do - %{data: %{"id" => context}, id: context_id} = create_context(map["context"]) + @spec lazy_put_activity_defaults(map(), boolean) :: map() + def lazy_put_activity_defaults(map, fake \\ false) - map - |> Map.put_new_lazy("id", &generate_activity_id/0) - |> Map.put_new_lazy("published", &make_date/0) - |> Map.put_new("context", context) - |> Map.put_new("context_id", context_id) - else - map - |> Map.put_new("id", "pleroma:fakeid") - |> Map.put_new_lazy("published", &make_date/0) - |> Map.put_new("context", "pleroma:fakecontext") - |> Map.put_new("context_id", -1) - end + def lazy_put_activity_defaults(map, true) do + map + |> Map.put_new("id", "pleroma:fakeid") + |> Map.put_new_lazy("published", &make_date/0) + |> Map.put_new("context", "pleroma:fakecontext") + |> Map.put_new("context_id", -1) + |> lazy_put_object_defaults(true) + end - if is_map(map["object"]) do - object = lazy_put_object_defaults(map["object"], map, fake) - %{map | "object" => object} - else + def lazy_put_activity_defaults(map, _fake) do + %{data: %{"id" => context}, id: context_id} = create_context(map["context"]) + + map + |> Map.put_new_lazy("id", &generate_activity_id/0) + |> Map.put_new_lazy("published", &make_date/0) + |> Map.put_new("context", context) + |> Map.put_new("context_id", context_id) + |> lazy_put_object_defaults(false) + end + + # Adds an id and published date if they aren't there. + # + @spec lazy_put_object_defaults(map(), boolean()) :: map() + defp lazy_put_object_defaults(%{"object" => map} = activity, true) + when is_map(map) do + object = map - end + |> Map.put_new("id", "pleroma:fake_object_id") + |> Map.put_new_lazy("published", &make_date/0) + |> Map.put_new("context", activity["context"]) + |> Map.put_new("context_id", activity["context_id"]) + |> Map.put_new("fake", true) + + %{activity | "object" => object} end - @doc """ - Adds an id and published date if they aren't there. - """ - def lazy_put_object_defaults(map, activity \\ %{}, fake) + defp lazy_put_object_defaults(%{"object" => map} = activity, _) + when is_map(map) do + object = + map + |> Map.put_new_lazy("id", &generate_object_id/0) + |> Map.put_new_lazy("published", &make_date/0) + |> Map.put_new("context", activity["context"]) + |> Map.put_new("context_id", activity["context_id"]) - def lazy_put_object_defaults(map, activity, true = _fake) do - map - |> Map.put_new_lazy("published", &make_date/0) - |> Map.put_new("id", "pleroma:fake_object_id") - |> Map.put_new("context", activity["context"]) - |> Map.put_new("fake", true) - |> Map.put_new("context_id", activity["context_id"]) + %{activity | "object" => object} end - def lazy_put_object_defaults(map, activity, _fake) do - map - |> Map.put_new_lazy("id", &generate_object_id/0) - |> Map.put_new_lazy("published", &make_date/0) - |> Map.put_new("context", activity["context"]) - |> Map.put_new("context_id", activity["context_id"]) - end + defp lazy_put_object_defaults(activity, _), do: activity @doc """ Inserts a full object if it is contained in an activity. @@ -356,23 +351,30 @@ defmodule Pleroma.Web.ActivityPub.Utils do @doc """ Updates a follow activity's state (for locked accounts). """ + @spec update_follow_state_for_all(Activity.t(), String.t()) :: {:ok, Activity} | {:error, any()} def update_follow_state_for_all( %Activity{data: %{"actor" => actor, "object" => object}} = activity, state ) do - try do - Ecto.Adapters.SQL.query!( - Repo, - "UPDATE activities SET data = jsonb_set(data, '{state}', $1) WHERE data->>'type' = 'Follow' AND data->>'actor' = $2 AND data->>'object' = $3 AND data->>'state' = 'pending'", - [state, actor, object] + query = + from(activity in Activity, + where: fragment("data->>'type' = 'Follow'"), + where: fragment("data->>'state' = 'pending'"), + where: fragment("data->>'actor' = ?", ^actor), + where: fragment("data->>'object' = ?", ^object), + update: [ + set: [ + data: fragment("jsonb_set(data, '{state}', ?)", ^state) + ] + ] ) - User.set_follow_state_cache(actor, object, state) - activity = Activity.get_by_id(activity.id) + with {_, _} <- Repo.update_all(query, []), + {_, _} <- User.set_follow_state_cache(actor, object, state), + %Activity{} = activity <- Activity.get_by_id(activity.id) do {:ok, activity} - rescue - e -> - {:error, e} + else + e -> {:error, e} end end @@ -380,9 +382,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do %Activity{data: %{"actor" => actor, "object" => object}} = activity, state ) do - with new_data <- - activity.data - |> Map.put("state", state), + with new_data <- Map.put(activity.data, "state", state), changeset <- Changeset.change(activity, data: new_data), {:ok, activity} <- Repo.update(changeset), _ <- User.set_follow_state_cache(actor, object, state) do @@ -411,27 +411,17 @@ defmodule Pleroma.Web.ActivityPub.Utils do def fetch_latest_follow(%User{ap_id: follower_id}, %User{ap_id: followed_id}) do query = - from( - activity in Activity, - where: - fragment( - "? ->> 'type' = 'Follow'", - activity.data - ), - where: activity.actor == ^follower_id, - # this is to use the index - where: - fragment( - "coalesce((?)->'object'->>'id', (?)->>'object') = ?", - activity.data, - activity.data, - ^followed_id - ), - order_by: [fragment("? desc nulls last", activity.id)], - limit: 1 - ) + follower_id + |> Activity.Queries.by_actor() + |> Activity.Queries.by_type("Follow") + |> Activity.Queries.by_object_id(followed_id) + |> Activity.Queries.limit(1) - Repo.one(query) + from( + activity in query, + order_by: [fragment("? desc nulls last", activity.id)] + ) + |> Repo.one() end #### Announce-related helpers @@ -439,23 +429,14 @@ defmodule Pleroma.Web.ActivityPub.Utils do @doc """ Retruns an existing announce activity if the notice has already been announced """ + @spec get_existing_announce(String.t(), map()) :: Activity.t() | nil def get_existing_announce(actor, %{data: %{"id" => id}}) do - query = - from( - activity in Activity, - where: activity.actor == ^actor, - # this is to use the index - where: - fragment( - "coalesce((?)->'object'->>'id', (?)->>'object') = ?", - activity.data, - activity.data, - ^id - ), - where: fragment("(?)->>'type' = 'Announce'", activity.data) - ) - - Repo.one(query) + actor + |> Activity.Queries.by_actor() + |> Activity.Queries.by_type("Announce") + |> Activity.Queries.by_object_id(id) + |> Activity.Queries.limit(1) + |> Repo.one() end @doc """ @@ -531,31 +512,35 @@ defmodule Pleroma.Web.ActivityPub.Utils do |> maybe_put("id", activity_id) end + @spec add_announce_to_object(Activity.t(), Object.t()) :: + {:ok, Object.t()} | {:error, Ecto.Changeset.t()} def add_announce_to_object( - %Activity{ - data: %{"actor" => actor, "cc" => [Pleroma.Constants.as_public()]} - }, + %Activity{data: %{"actor" => actor, "cc" => [Pleroma.Constants.as_public()]}}, object ) do - announcements = - if is_list(object.data["announcements"]), do: object.data["announcements"], else: [] + announcements = fetch_announcements(object) - with announcements <- [actor | announcements] |> Enum.uniq() do + with announcements <- Enum.uniq([actor | announcements]) do update_element_in_object("announcement", announcements, object) end end def add_announce_to_object(_, object), do: {:ok, object} + @spec remove_announce_from_object(Activity.t(), Object.t()) :: + {:ok, Object.t()} | {:error, Ecto.Changeset.t()} def remove_announce_from_object(%Activity{data: %{"actor" => actor}}, object) do - announcements = - if is_list(object.data["announcements"]), do: object.data["announcements"], else: [] - - with announcements <- announcements |> List.delete(actor) do + with announcements <- List.delete(fetch_announcements(object), actor) do update_element_in_object("announcement", announcements, object) end end + defp fetch_announcements(%{data: %{"announcements" => announcements}} = _) + when is_list(announcements), + do: announcements + + defp fetch_announcements(_), do: [] + #### Unfollow-related helpers def make_unfollow_data(follower, followed, follow_activity, activity_id) do @@ -569,29 +554,20 @@ defmodule Pleroma.Web.ActivityPub.Utils do end #### Block-related helpers + @spec fetch_latest_block(User.t(), User.t()) :: Activity.t() | nil def fetch_latest_block(%User{ap_id: blocker_id}, %User{ap_id: blocked_id}) do query = - from( - activity in Activity, - where: - fragment( - "? ->> 'type' = 'Block'", - activity.data - ), - where: activity.actor == ^blocker_id, - # this is to use the index - where: - fragment( - "coalesce((?)->'object'->>'id', (?)->>'object') = ?", - activity.data, - activity.data, - ^blocked_id - ), - order_by: [fragment("? desc nulls last", activity.id)], - limit: 1 - ) + blocker_id + |> Activity.Queries.by_actor() + |> Activity.Queries.by_type("Block") + |> Activity.Queries.by_object_id(blocked_id) + |> Activity.Queries.limit(1) - Repo.one(query) + from( + activity in query, + order_by: [fragment("? desc nulls last", activity.id)] + ) + |> Repo.one() end def make_block_data(blocker, blocked, activity_id) do @@ -631,28 +607,32 @@ defmodule Pleroma.Web.ActivityPub.Utils do end #### Flag-related helpers - - def make_flag_data(params, additional) do - status_ap_ids = - Enum.map(params.statuses || [], fn - %Activity{} = act -> act.data["id"] - act when is_map(act) -> act["id"] - act when is_binary(act) -> act - end) - - object = [params.account.ap_id] ++ status_ap_ids - + @spec make_flag_data(map(), map()) :: map() + def make_flag_data(%{actor: actor, context: context, content: content} = params, additional) do %{ "type" => "Flag", - "actor" => params.actor.ap_id, - "content" => params.content, - "object" => object, - "context" => params.context, + "actor" => actor.ap_id, + "content" => content, + "object" => build_flag_object(params), + "context" => context, "state" => "open" } |> Map.merge(additional) end + def make_flag_data(_, _), do: %{} + + defp build_flag_object(%{account: account, statuses: statuses} = _) do + [account.ap_id] ++ + Enum.map(statuses || [], fn + %Activity{} = act -> act.data["id"] + act when is_map(act) -> act["id"] + act when is_binary(act) -> act + end) + end + + defp build_flag_object(_), do: [] + @doc """ Fetches the OrderedCollection/OrderedCollectionPage from `from`, limiting the amount of pages fetched after the first one to `pages_left` pages. diff --git a/test/web/activity_pub/utils_test.exs b/test/web/activity_pub/utils_test.exs index eb429b2c..b1c1d6f7 100644 --- a/test/web/activity_pub/utils_test.exs +++ b/test/web/activity_pub/utils_test.exs @@ -87,6 +87,18 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do assert Utils.determine_explicit_mentions(object) == [] end + + test "works with an object has tags as map" do + object = %{ + "tag" => %{ + "type" => "Mention", + "href" => "https://example.com/~alyssa", + "name" => "Alyssa P. Hacker" + } + } + + assert Utils.determine_explicit_mentions(object) == ["https://example.com/~alyssa"] + end end describe "make_unlike_data/3" do @@ -300,8 +312,8 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do {:ok, follow_activity_two} = Utils.update_follow_state_for_all(follow_activity_two, "accept") - assert Repo.get(Activity, follow_activity.id).data["state"] == "accept" - assert Repo.get(Activity, follow_activity_two.id).data["state"] == "accept" + assert refresh_record(follow_activity).data["state"] == "accept" + assert refresh_record(follow_activity_two).data["state"] == "accept" end end @@ -323,8 +335,8 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do {:ok, follow_activity_two} = Utils.update_follow_state(follow_activity_two, "reject") - assert Repo.get(Activity, follow_activity.id).data["state"] == "pending" - assert Repo.get(Activity, follow_activity_two.id).data["state"] == "reject" + assert refresh_record(follow_activity).data["state"] == "pending" + assert refresh_record(follow_activity_two).data["state"] == "reject" end end @@ -401,4 +413,216 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do assert ^like_activity = Utils.get_existing_like(user.ap_id, object) end end + + describe "get_get_existing_announce/2" do + test "returns nil if announce not found" do + actor = insert(:user) + refute Utils.get_existing_announce(actor.ap_id, %{data: %{"id" => "test"}}) + end + + test "fetches existing announce" do + note_activity = insert(:note_activity) + assert object = Object.normalize(note_activity) + actor = insert(:user) + + {:ok, announce, _object} = ActivityPub.announce(actor, object) + assert Utils.get_existing_announce(actor.ap_id, object) == announce + end + end + + describe "fetch_latest_block/2" do + test "fetches last block activities" do + user1 = insert(:user) + user2 = insert(:user) + + assert {:ok, %Activity{} = _} = ActivityPub.block(user1, user2) + assert {:ok, %Activity{} = _} = ActivityPub.block(user1, user2) + assert {:ok, %Activity{} = activity} = ActivityPub.block(user1, user2) + + assert Utils.fetch_latest_block(user1, user2) == activity + end + end + + describe "recipient_in_message/3" do + test "returns true when recipient in `to`" do + recipient = insert(:user) + actor = insert(:user) + assert Utils.recipient_in_message(recipient, actor, %{"to" => recipient.ap_id}) + + assert Utils.recipient_in_message( + recipient, + actor, + %{"to" => [recipient.ap_id], "cc" => ""} + ) + end + + test "returns true when recipient in `cc`" do + recipient = insert(:user) + actor = insert(:user) + assert Utils.recipient_in_message(recipient, actor, %{"cc" => recipient.ap_id}) + + assert Utils.recipient_in_message( + recipient, + actor, + %{"cc" => [recipient.ap_id], "to" => ""} + ) + end + + test "returns true when recipient in `bto`" do + recipient = insert(:user) + actor = insert(:user) + assert Utils.recipient_in_message(recipient, actor, %{"bto" => recipient.ap_id}) + + assert Utils.recipient_in_message( + recipient, + actor, + %{"bcc" => "", "bto" => [recipient.ap_id]} + ) + end + + test "returns true when recipient in `bcc`" do + recipient = insert(:user) + actor = insert(:user) + assert Utils.recipient_in_message(recipient, actor, %{"bcc" => recipient.ap_id}) + + assert Utils.recipient_in_message( + recipient, + actor, + %{"bto" => "", "bcc" => [recipient.ap_id]} + ) + end + + test "returns true when message without addresses fields" do + recipient = insert(:user) + actor = insert(:user) + assert Utils.recipient_in_message(recipient, actor, %{"bccc" => recipient.ap_id}) + + assert Utils.recipient_in_message( + recipient, + actor, + %{"btod" => "", "bccc" => [recipient.ap_id]} + ) + end + + test "returns false" do + recipient = insert(:user) + actor = insert(:user) + refute Utils.recipient_in_message(recipient, actor, %{"to" => "ap_id"}) + end + end + + describe "lazy_put_activity_defaults/2" do + test "returns map with id and published data" do + note_activity = insert(:note_activity) + object = Object.normalize(note_activity) + res = Utils.lazy_put_activity_defaults(%{"context" => object.data["id"]}) + assert res["context"] == object.data["id"] + assert res["context_id"] == object.id + assert res["id"] + assert res["published"] + end + + test "returns map with fake id and published data" do + assert %{ + "context" => "pleroma:fakecontext", + "context_id" => -1, + "id" => "pleroma:fakeid", + "published" => _ + } = Utils.lazy_put_activity_defaults(%{}, true) + end + + test "returns activity data with object" do + note_activity = insert(:note_activity) + object = Object.normalize(note_activity) + + res = + Utils.lazy_put_activity_defaults(%{ + "context" => object.data["id"], + "object" => %{} + }) + + assert res["context"] == object.data["id"] + assert res["context_id"] == object.id + assert res["id"] + assert res["published"] + assert res["object"]["id"] + assert res["object"]["published"] + assert res["object"]["context"] == object.data["id"] + assert res["object"]["context_id"] == object.id + end + end + + describe "make_flag_data" do + test "returns empty map when params is invalid" do + assert Utils.make_flag_data(%{}, %{}) == %{} + end + + test "returns map with Flag object" do + reporter = insert(:user) + target_account = insert(:user) + {:ok, activity} = CommonAPI.post(target_account, %{"status" => "foobar"}) + context = Utils.generate_context_id() + content = "foobar" + + target_ap_id = target_account.ap_id + activity_ap_id = activity.data["id"] + + res = + Utils.make_flag_data( + %{ + actor: reporter, + context: context, + account: target_account, + statuses: [%{"id" => activity.data["id"]}], + content: content + }, + %{} + ) + + assert %{ + "type" => "Flag", + "content" => ^content, + "context" => ^context, + "object" => [^target_ap_id, ^activity_ap_id], + "state" => "open" + } = res + end + end + + describe "add_announce_to_object/2" do + test "adds actor to announcement" do + user = insert(:user) + object = insert(:note) + + activity = + insert(:note_activity, + data: %{ + "actor" => user.ap_id, + "cc" => [Pleroma.Constants.as_public()] + } + ) + + assert {:ok, updated_object} = Utils.add_announce_to_object(activity, object) + assert updated_object.data["announcements"] == [user.ap_id] + assert updated_object.data["announcement_count"] == 1 + end + end + + describe "remove_announce_from_object/2" do + test "removes actor from announcements" do + user = insert(:user) + user2 = insert(:user) + + object = + insert(:note, + data: %{"announcements" => [user.ap_id, user2.ap_id], "announcement_count" => 2} + ) + + activity = insert(:note_activity, data: %{"actor" => user.ap_id}) + + assert {:ok, updated_object} = Utils.remove_announce_from_object(activity, object) + assert updated_object.data["announcements"] == [user2.ap_id] + assert updated_object.data["announcement_count"] == 1 + end + end end From a890451187f0b1507be96ccf144b18fdb8294dd8 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 4 Sep 2019 17:42:27 +0300 Subject: [PATCH 019/138] fetch_announcements -> take_announcements --- lib/pleroma/web/activity_pub/utils.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index cf82d1a9..0d87b922 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -518,7 +518,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do %Activity{data: %{"actor" => actor, "cc" => [Pleroma.Constants.as_public()]}}, object ) do - announcements = fetch_announcements(object) + announcements = take_announcements(object) with announcements <- Enum.uniq([actor | announcements]) do update_element_in_object("announcement", announcements, object) @@ -530,16 +530,16 @@ defmodule Pleroma.Web.ActivityPub.Utils do @spec remove_announce_from_object(Activity.t(), Object.t()) :: {:ok, Object.t()} | {:error, Ecto.Changeset.t()} def remove_announce_from_object(%Activity{data: %{"actor" => actor}}, object) do - with announcements <- List.delete(fetch_announcements(object), actor) do + with announcements <- List.delete(take_announcements(object), actor) do update_element_in_object("announcement", announcements, object) end end - defp fetch_announcements(%{data: %{"announcements" => announcements}} = _) + defp take_announcements(%{data: %{"announcements" => announcements}} = _) when is_list(announcements), do: announcements - defp fetch_announcements(_), do: [] + defp take_announcements(_), do: [] #### Unfollow-related helpers From 2975da284b75c846a99a56ce70a91ebc3cc43f33 Mon Sep 17 00:00:00 2001 From: Sadposter Date: Wed, 4 Sep 2019 15:45:40 +0100 Subject: [PATCH 020/138] truncate remote user bio/display name --- lib/pleroma/user.ex | 16 +++++++++++++++- test/user_test.exs | 45 +++++++++++++++++++++++++++++---------------- 2 files changed, 44 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 29fd6d2e..87e56b5b 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -174,11 +174,25 @@ defmodule Pleroma.User do |> Repo.aggregate(:count, :id) end + defp truncate_if_exists(params, key, max_length) do + if Map.has_key?(params, key) do + {value, _chopped} = String.split_at(params[key], max_length) + Map.put(params, key, value) + else + params + end + end + def remote_user_creation(params) do bio_limit = Pleroma.Config.get([:instance, :user_bio_length], 5000) name_limit = Pleroma.Config.get([:instance, :user_name_length], 100) - params = Map.put(params, :info, params[:info] || %{}) + params = + params + |> Map.put(:info, params[:info] || %{}) + |> truncate_if_exists(:name, name_limit) + |> truncate_if_exists(:bio, bio_limit) + info_cng = User.Info.remote_user_creation(%User.Info{}, params[:info]) changes = diff --git a/test/user_test.exs b/test/user_test.exs index 92a48f63..45f998ff 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -570,22 +570,6 @@ defmodule Pleroma.UserTest do refute cs.valid? end) end - - test "it restricts some sizes" do - bio_limit = Pleroma.Config.get([:instance, :user_bio_length], 5000) - name_limit = Pleroma.Config.get([:instance, :user_name_length], 100) - - [bio: bio_limit, name: name_limit] - |> Enum.each(fn {field, size} -> - string = String.pad_leading(".", size) - cs = User.remote_user_creation(Map.put(@valid_remote, field, string)) - assert cs.valid? - - string = String.pad_leading(".", size + 1) - cs = User.remote_user_creation(Map.put(@valid_remote, field, string)) - refute cs.valid? - end) - end end describe "followers and friends" do @@ -1142,6 +1126,35 @@ defmodule Pleroma.UserTest do assert {:ok, %User{}} = User.insert_or_update_user(data) end + + test "with an overly long bio" do + current_max_length = Pleroma.Config.get([:instance, :user_bio_length], 5000) + user = insert(:user, nickname: "nickname@supergood.domain") + + data = %{ + ap_id: user.ap_id, + name: user.name, + nickname: user.nickname, + bio: String.duplicate("h", current_max_length + 1), + info: %{} + } + + assert {:ok, %User{}} = User.insert_or_update_user(data) + end + + test "with an overly long display name" do + current_max_length = Pleroma.Config.get([:instance, :user_name_length], 100) + user = insert(:user, nickname: "nickname@supergood.domain") + + data = %{ + ap_id: user.ap_id, + name: String.duplicate("h", current_max_length + 1), + nickname: user.nickname, + info: %{} + } + + assert {:ok, %User{}} = User.insert_or_update_user(data) + end end describe "per-user rich-text filtering" do From cb99cfcc65f57f0044117ebd12d040488343d9ef Mon Sep 17 00:00:00 2001 From: Sadposter Date: Wed, 4 Sep 2019 15:57:42 +0100 Subject: [PATCH 021/138] don't try to truncate non-strings --- lib/pleroma/user.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 87e56b5b..e2ebce6f 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -175,7 +175,7 @@ defmodule Pleroma.User do end defp truncate_if_exists(params, key, max_length) do - if Map.has_key?(params, key) do + if Map.has_key?(params, key) and is_binary(params[key]) do {value, _chopped} = String.split_at(params[key], max_length) Map.put(params, key, value) else From af746fa4a814dbacd4fe4a3e58b1ee1732363d22 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Wed, 4 Sep 2019 20:08:13 +0300 Subject: [PATCH 022/138] Return total for reports --- CHANGELOG.md | 3 ++- docs/api/admin_api.md | 1 + lib/pleroma/web/admin_api/admin_api_controller.ex | 6 ++---- lib/pleroma/web/admin_api/views/report_view.ex | 3 ++- test/web/admin_api/admin_api_controller_test.exs | 8 ++++++++ 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a414ba5e..942605f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Unsubscribe followers when they unfollow a user - AdminAPI: Add "godmode" while fetching user statuses (i.e. admin can see private statuses) - Improve digest email template -– Pagination: (optional) return `total` alongside with `items` when paginating +- Pagination: (optional) return `total` alongside with `items` when paginating +- Admin API: Return `total` when querying for reports ### Fixed - Following from Osada diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index d79c342b..5a090c72 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -313,6 +313,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret ```json { + "total" : 1, "reports": [ { "account": { diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 544b9d7d..2a1cc59e 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -442,11 +442,9 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do params |> Map.put("type", "Flag") |> Map.put("skip_preload", true) + |> Map.put("total", true) - reports = - [] - |> ActivityPub.fetch_activities(params) - |> Enum.reverse() + reports = ActivityPub.fetch_activities([], params) conn |> put_view(ReportView) diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex index a25f3f1f..0b8745b2 100644 --- a/lib/pleroma/web/admin_api/views/report_view.ex +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -12,7 +12,8 @@ defmodule Pleroma.Web.AdminAPI.ReportView do def render("index.json", %{reports: reports}) do %{ - reports: render_many(reports, __MODULE__, "show.json", as: :report) + reports: render_many(reports[:items], __MODULE__, "show.json", as: :report), + total: reports[:total] } end diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 4e2c2743..b1ddd898 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -1309,6 +1309,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do |> json_response(:ok) assert Enum.empty?(response["reports"]) + assert response["total"] == 0 end test "returns reports", %{conn: conn} do @@ -1331,6 +1332,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do assert length(response["reports"]) == 1 assert report["id"] == report_id + + assert response["total"] == 1 end test "returns reports with specified state", %{conn: conn} do @@ -1364,6 +1367,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do assert length(response["reports"]) == 1 assert open_report["id"] == first_report_id + assert response["total"] == 1 + response = conn |> get("/api/pleroma/admin/reports", %{ @@ -1376,6 +1381,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do assert length(response["reports"]) == 1 assert closed_report["id"] == second_report_id + assert response["total"] == 1 + response = conn |> get("/api/pleroma/admin/reports", %{ @@ -1384,6 +1391,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do |> json_response(:ok) assert Enum.empty?(response["reports"]) + assert response["total"] == 0 end test "returns 403 when requested by a non-admin" do From 8306078de1abade082f932cda5b8d9297bdcdc80 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 4 Sep 2019 17:31:14 +0000 Subject: [PATCH 023/138] Apply suggestion to lib/pleroma/web/activity_pub/utils.ex --- lib/pleroma/web/activity_pub/utils.ex | 33 +++++++++++---------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 0d87b922..2de02f60 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -356,26 +356,19 @@ defmodule Pleroma.Web.ActivityPub.Utils do %Activity{data: %{"actor" => actor, "object" => object}} = activity, state ) do - query = - from(activity in Activity, - where: fragment("data->>'type' = 'Follow'"), - where: fragment("data->>'state' = 'pending'"), - where: fragment("data->>'actor' = ?", ^actor), - where: fragment("data->>'object' = ?", ^object), - update: [ - set: [ - data: fragment("jsonb_set(data, '{state}', ?)", ^state) - ] - ] - ) - - with {_, _} <- Repo.update_all(query, []), - {_, _} <- User.set_follow_state_cache(actor, object, state), - %Activity{} = activity <- Activity.get_by_id(activity.id) do - {:ok, activity} - else - e -> {:error, e} - end + "Follow" + |> Activity.Queries.by_type() + |> Activity.Queries.by_actor(actor) + |> Activity.Queries.by_object_id(object["id"]) + |> where(fragment("data->>'state' = 'pending'")) + |> update(set: [data: fragment("jsonb_set(data, '{state}', ?)", ^state)]) + |> Repo.update_all([]) + + User.set_follow_state_cache(actor, object, state) + + activity = Activity.get_by_id(activity.id) + + {:ok, activity} end def update_follow_state( From e2011a667cdf5e67f257c9c30a02c206fb4df913 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 4 Sep 2019 18:35:01 +0000 Subject: [PATCH 024/138] Apply suggestion to lib/pleroma/web/activity_pub/utils.ex --- lib/pleroma/web/activity_pub/utils.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 2de02f60..011acd48 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -359,7 +359,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do "Follow" |> Activity.Queries.by_type() |> Activity.Queries.by_actor(actor) - |> Activity.Queries.by_object_id(object["id"]) + |> Activity.Queries.by_object_id(object) |> where(fragment("data->>'state' = 'pending'")) |> update(set: [data: fragment("jsonb_set(data, '{state}', ?)", ^state)]) |> Repo.update_all([]) From ae506ca997619f118d18703a9b0802246eb427d5 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 4 Sep 2019 21:40:53 +0300 Subject: [PATCH 025/138] fix formatting --- lib/pleroma/web/activity_pub/utils.ex | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 011acd48..72e07b59 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -356,19 +356,19 @@ defmodule Pleroma.Web.ActivityPub.Utils do %Activity{data: %{"actor" => actor, "object" => object}} = activity, state ) do - "Follow" - |> Activity.Queries.by_type() - |> Activity.Queries.by_actor(actor) - |> Activity.Queries.by_object_id(object) - |> where(fragment("data->>'state' = 'pending'")) - |> update(set: [data: fragment("jsonb_set(data, '{state}', ?)", ^state)]) - |> Repo.update_all([]) - - User.set_follow_state_cache(actor, object, state) - - activity = Activity.get_by_id(activity.id) - - {:ok, activity} + "Follow" + |> Activity.Queries.by_type() + |> Activity.Queries.by_actor(actor) + |> Activity.Queries.by_object_id(object) + |> where(fragment("data->>'state' = 'pending'")) + |> update(set: [data: fragment("jsonb_set(data, '{state}', ?)", ^state)]) + |> Repo.update_all([]) + + User.set_follow_state_cache(actor, object, state) + + activity = Activity.get_by_id(activity.id) + + {:ok, activity} end def update_follow_state( From 736165c082d34ef4d52367ea8315c228a1df3944 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Thu, 5 Sep 2019 16:54:34 +0300 Subject: [PATCH 026/138] Reverse reports list --- lib/pleroma/web/admin_api/views/report_view.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex index 0b8745b2..51b95ad5 100644 --- a/lib/pleroma/web/admin_api/views/report_view.ex +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -12,7 +12,8 @@ defmodule Pleroma.Web.AdminAPI.ReportView do def render("index.json", %{reports: reports}) do %{ - reports: render_many(reports[:items], __MODULE__, "show.json", as: :report), + reports: + render_many(reports[:items], __MODULE__, "show.json", as: :report) |> Enum.reverse(), total: reports[:total] } end From 40b3289c26137ee4d07c7fb79faf232714cc7592 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 6 Sep 2019 17:08:47 +0700 Subject: [PATCH 027/138] Refactor `add_link_headers/7` -> `add_link_headers/3` --- lib/pleroma/web/controller_helper.ex | 95 ++++++------------- .../controllers/mastodon_api_controller.ex | 28 +++--- .../web/pleroma_api/pleroma_api_controller.ex | 27 ++---- 3 files changed, 50 insertions(+), 100 deletions(-) diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index eeac9f50..b53a0195 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -34,79 +34,38 @@ defmodule Pleroma.Web.ControllerHelper do defp param_to_integer(_, default), do: default - def add_link_headers( - conn, - method, - activities, - param \\ nil, - params \\ %{}, - func3 \\ nil, - func4 \\ nil - ) do - params = - conn.params - |> Map.drop(["since_id", "max_id", "min_id"]) - |> Map.merge(params) + def add_link_headers(conn, activities, extra_params \\ %{}) do + case List.last(activities) do + %{id: max_id} -> + params = + conn.params + |> Map.drop(Map.keys(conn.path_params)) + |> Map.drop(["since_id", "max_id", "min_id"]) + |> Map.merge(extra_params) - last = List.last(activities) + limit = + params + |> Map.get("limit", "20") + |> String.to_integer() - func3 = func3 || (&mastodon_api_url/3) - func4 = func4 || (&mastodon_api_url/4) + min_id = + if length(activities) <= limit do + activities + |> List.first() + |> Map.get(:id) + else + activities + |> Enum.at(limit * -1) + |> Map.get(:id) + end - if last do - max_id = last.id + next_url = current_url(conn, Map.merge(params, %{max_id: max_id})) + prev_url = current_url(conn, Map.merge(params, %{min_id: min_id})) - limit = - params - |> Map.get("limit", "20") - |> String.to_integer() + put_resp_header(conn, "link", "<#{next_url}>; rel=\"next\", <#{prev_url}>; rel=\"prev\"") - min_id = - if length(activities) <= limit do - activities - |> List.first() - |> Map.get(:id) - else - activities - |> Enum.at(limit * -1) - |> Map.get(:id) - end - - {next_url, prev_url} = - if param do - { - func4.( - Pleroma.Web.Endpoint, - method, - param, - Map.merge(params, %{max_id: max_id}) - ), - func4.( - Pleroma.Web.Endpoint, - method, - param, - Map.merge(params, %{min_id: min_id}) - ) - } - else - { - func3.( - Pleroma.Web.Endpoint, - method, - Map.merge(params, %{max_id: max_id}) - ), - func3.( - Pleroma.Web.Endpoint, - method, - Map.merge(params, %{min_id: min_id}) - ) - } - end - - conn - |> put_resp_header("link", "<#{next_url}>; rel=\"next\", <#{prev_url}>; rel=\"prev\"") - else - conn + _ -> + conn end end end diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex index 8dfad7a5..f30a21bc 100644 --- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do use Pleroma.Web, :controller import Pleroma.Web.ControllerHelper, - only: [json_response: 3, add_link_headers: 5, add_link_headers: 4, add_link_headers: 3] + only: [json_response: 3, add_link_headers: 2, add_link_headers: 3] alias Ecto.Changeset alias Pleroma.Activity @@ -365,7 +365,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do |> Enum.reverse() conn - |> add_link_headers(:home_timeline, activities) + |> add_link_headers(activities) |> put_view(StatusView) |> render("index.json", %{activities: activities, for: user, as: :activity}) end @@ -384,7 +384,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do |> Enum.reverse() conn - |> add_link_headers(:public_timeline, activities, false, %{"local" => local_only}) + |> add_link_headers(activities, %{"local" => local_only}) |> put_view(StatusView) |> render("index.json", %{activities: activities, for: user, as: :activity}) end @@ -398,7 +398,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do activities = ActivityPub.fetch_user_activities(user, reading_user, params) conn - |> add_link_headers(:user_statuses, activities, params["id"]) + |> add_link_headers(activities) |> put_view(StatusView) |> render("index.json", %{ activities: activities, @@ -422,7 +422,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do |> Pagination.fetch_paginated(params) conn - |> add_link_headers(:dm_timeline, activities) + |> add_link_headers(activities) |> put_view(StatusView) |> render("index.json", %{activities: activities, for: user, as: :activity}) end @@ -523,7 +523,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do def scheduled_statuses(%{assigns: %{user: user}} = conn, params) do with scheduled_activities <- MastodonAPI.get_scheduled_activities(user, params) do conn - |> add_link_headers(:scheduled_statuses, scheduled_activities) + |> add_link_headers(scheduled_activities) |> put_view(ScheduledActivityView) |> render("index.json", %{scheduled_activities: scheduled_activities}) end @@ -706,7 +706,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do notifications = MastodonAPI.get_notifications(user, params) conn - |> add_link_headers(:notifications, notifications) + |> add_link_headers(notifications) |> put_view(NotificationView) |> render("index.json", %{notifications: notifications, for: user}) end @@ -894,7 +894,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do |> Enum.reverse() conn - |> add_link_headers(:hashtag_timeline, activities, params["tag"], %{"local" => local_only}) + |> add_link_headers(activities, %{"local" => local_only}) |> put_view(StatusView) |> render("index.json", %{activities: activities, for: user, as: :activity}) end @@ -910,7 +910,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end conn - |> add_link_headers(:followers, followers, user) + |> add_link_headers(followers) |> put_view(AccountView) |> render("accounts.json", %{for: for_user, users: followers, as: :user}) end @@ -927,7 +927,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end conn - |> add_link_headers(:following, followers, user) + |> add_link_headers(followers) |> put_view(AccountView) |> render("accounts.json", %{for: for_user, users: followers, as: :user}) end @@ -1152,7 +1152,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do |> Enum.reverse() conn - |> add_link_headers(:favourites, activities) + |> add_link_headers(activities) |> put_view(StatusView) |> render("index.json", %{activities: activities, for: user, as: :activity}) end @@ -1179,7 +1179,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do |> Enum.reverse() conn - |> add_link_headers(:favourites, activities) + |> add_link_headers(activities) |> put_view(StatusView) |> render("index.json", %{activities: activities, for: for_user, as: :activity}) else @@ -1200,7 +1200,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do |> Enum.map(fn b -> Map.put(b.activity, :bookmark, Map.delete(b, :activity)) end) conn - |> add_link_headers(:bookmarks, bookmarks) + |> add_link_headers(bookmarks) |> put_view(StatusView) |> render("index.json", %{activities: activities, for: user, as: :activity}) end @@ -1640,7 +1640,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end) conn - |> add_link_headers(:conversations, participations) + |> add_link_headers(participations) |> json(conversations) end diff --git a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex index f4df3b02..d17ccf84 100644 --- a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do use Pleroma.Web, :controller - import Pleroma.Web.ControllerHelper, only: [add_link_headers: 7] + import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2] alias Pleroma.Conversation.Participation alias Pleroma.Notification @@ -27,31 +27,22 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do %{assigns: %{user: user}} = conn, %{"id" => participation_id} = params ) do - params = - params - |> Map.put("blocking_user", user) - |> Map.put("muting_user", user) - |> Map.put("user", user) - - participation = - participation_id - |> Participation.get(preload: [:conversation]) + participation = Participation.get(participation_id, preload: [:conversation]) if user.id == participation.user_id do + params = + params + |> Map.put("blocking_user", user) + |> Map.put("muting_user", user) + |> Map.put("user", user) + activities = participation.conversation.ap_id |> ActivityPub.fetch_activities_for_context(params) |> Enum.reverse() conn - |> add_link_headers( - :conversation_statuses, - activities, - participation_id, - params, - nil, - &pleroma_api_url/4 - ) + |> add_link_headers(activities) |> put_view(StatusView) |> render("index.json", %{activities: activities, for: user, as: :activity}) end From e5c6bf3673a8361d1417eba1ccc44edec7658ac4 Mon Sep 17 00:00:00 2001 From: shadowfacts Date: Sat, 7 Sep 2019 19:50:45 +0000 Subject: [PATCH 028/138] Mastodon API: URI encode hashtag name in generated URLs Otherwise hashtags with word characters other than those allowed in URLs (e.g. Japanese characters) produce hashtag URLs that are invalid. --- lib/pleroma/web/mastodon_api/views/status_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index e71083b9..708b8c2f 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -499,7 +499,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do object_tags = for tag when is_binary(tag) <- object_tags, do: tag Enum.reduce(object_tags, [], fn tag, tags -> - tags ++ [%{name: tag, url: "/tag/#{tag}"}] + tags ++ [%{name: tag, url: "/tag/#{URI.encode(tag)}"}] end) end From e0f84d0043d922f7cc88875a0bc52f2db8972b76 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 10 Sep 2019 01:11:57 +0700 Subject: [PATCH 029/138] Fix `ActivityPubController.read_inbox/2` --- .../activity_pub/activity_pub_controller.ex | 44 ++++++++++++------- .../activity_pub_controller_test.exs | 11 +++++ 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 08bf1c75..7b007547 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -251,22 +251,36 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do def whoami(_conn, _params), do: {:error, :not_found} - def read_inbox(%{assigns: %{user: user}} = conn, %{"nickname" => nickname} = params) do - if nickname == user.nickname do - conn - |> put_resp_content_type("application/activity+json") - |> json(UserView.render("inbox.json", %{user: user, max_id: params["max_id"]})) - else - err = - dgettext("errors", "can't read inbox of %{nickname} as %{as_nickname}", - nickname: nickname, - as_nickname: user.nickname - ) + def read_inbox( + %{assigns: %{user: %{nickname: nickname} = user}} = conn, + %{"nickname" => nickname} = params + ) do + conn + |> put_resp_content_type("application/activity+json") + |> put_view(UserView) + |> render("inbox.json", user: user, max_id: params["max_id"]) + end - conn - |> put_status(:forbidden) - |> json(err) - end + def read_inbox(%{assigns: %{user: nil}} = conn, %{"nickname" => nickname}) do + err = dgettext("errors", "can't read inbox of %{nickname}", nickname: nickname) + + conn + |> put_status(:forbidden) + |> json(err) + end + + def read_inbox(%{assigns: %{user: %{nickname: as_nickname}}} = conn, %{ + "nickname" => nickname + }) do + err = + dgettext("errors", "can't read inbox of %{nickname} as %{as_nickname}", + nickname: nickname, + as_nickname: as_nickname + ) + + conn + |> put_status(:forbidden) + |> json(err) end def handle_user_activity(user, %{"type" => "Create"} = params) do diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index 5192e734..4388538c 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -365,6 +365,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert json_response(conn, 403) end + test "it doesn't crash without an authenticated user", %{conn: conn} do + user = insert(:user) + + conn = + conn + |> put_req_header("accept", "application/activity+json") + |> get("/users/#{user.nickname}/inbox") + + assert json_response(conn, 403) + end + test "it returns a note activity in a collection", %{conn: conn} do note_activity = insert(:direct_note_activity) note_object = Object.normalize(note_activity) From e34de00052e628579a2915c95383d3bfd48f5ca5 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 10 Sep 2019 01:30:02 +0700 Subject: [PATCH 030/138] Update CHANGELOG (add a note about !1649) --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50a4467e..17d178b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Reverse Proxy limiting `max_body_length` was incorrectly defined and only checked `Content-Length` headers which may not be sufficient in some circumstances - MRF: fix use of unserializable keyword lists in describe() implementations - ActivityPub: Deactivated user deletion +- ActivityPub: Fix `/users/:nickname/inbox` crashing without an authenticated user - MRF: fix ability to follow a relay when AntiFollowbotPolicy was enabled ### Added From 11e12b5761bcd67aa609d91f6f8d1f6755b2312b Mon Sep 17 00:00:00 2001 From: minibikini Date: Mon, 9 Sep 2019 18:53:08 +0000 Subject: [PATCH 031/138] Add Pleroma.Plugs.Cache --- CHANGELOG.md | 1 + config/config.exs | 4 + docs/config.md | 9 + lib/pleroma/activity.ex | 9 + lib/pleroma/application.ex | 3 +- lib/pleroma/object.ex | 6 +- lib/pleroma/plugs/cache.ex | 122 ++++++++++++ .../activity_pub/activity_pub_controller.ex | 33 +++- test/object_test.exs | 3 + test/plugs/cache_test.exs | 186 ++++++++++++++++++ .../activity_pub_controller_test.exs | 88 +++++++++ 11 files changed, 457 insertions(+), 7 deletions(-) create mode 100644 lib/pleroma/plugs/cache.ex create mode 100644 test/plugs/cache_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index bcf31287..a3fa9dfb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -109,6 +109,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mix Tasks: `mix pleroma.database fix_likes_collections` - Federation: Remove `likes` from objects. - Admin API: Added moderation log +- Web response cache (currently, enabled for ActivityPub) ### Changed - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text diff --git a/config/config.exs b/config/config.exs index f630771a..5206fe37 100644 --- a/config/config.exs +++ b/config/config.exs @@ -560,6 +560,10 @@ config :pleroma, :rate_limit, nil config :pleroma, Pleroma.ActivityExpiration, enabled: true +config :pleroma, :web_cache_ttl, + activity_pub: nil, + activity_pub_question: 30_000 + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/docs/config.md b/docs/config.md index 7a8819c9..9136532e 100644 --- a/docs/config.md +++ b/docs/config.md @@ -690,3 +690,12 @@ Supported rate limiters: * `:relation_id_action` for actions on relation with a specific user (follow, unfollow) * `:statuses_actions` for create / delete / fav / unfav / reblog / unreblog actions on any statuses * `:status_id_action` for fav / unfav or reblog / unreblog actions on the same status by the same user + +## :web_cache_ttl + +The expiration time for the web responses cache. Values should be in milliseconds or `nil` to disable expiration. + +Available caches: + +* `:activity_pub` - activity pub routes (except question activities). Defaults to `nil` (no expiration). +* `:activity_pub_question` - activity pub routes (question activities). Defaults to `30_000` (30 seconds). diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index a7844c36..6a51d4cf 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -308,10 +308,19 @@ defmodule Pleroma.Activity do %{data: %{"type" => "Create", "object" => %{"id" => ap_id}}} -> ap_id == id _ -> nil end) + |> purge_web_resp_cache() end def delete_by_ap_id(_), do: nil + defp purge_web_resp_cache(%Activity{} = activity) do + %{path: path} = URI.parse(activity.data["id"]) + Cachex.del(:web_resp_cache, path) + activity + end + + defp purge_web_resp_cache(nil), do: nil + for {ap_type, type} <- @mastodon_notification_types do def mastodon_notification_type(%Activity{data: %{"type" => unquote(ap_type)}}), do: unquote(type) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 483ac1f3..1d46925f 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -116,7 +116,8 @@ defmodule Pleroma.Application do build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500), build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000), build_cachex("scrubber", limit: 2500), - build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500) + build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500), + build_cachex("web_resp", limit: 2500) ] end diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index d58eb7f7..5033798a 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -130,14 +130,16 @@ defmodule Pleroma.Object do def delete(%Object{data: %{"id" => id}} = object) do with {:ok, _obj} = swap_object_with_tombstone(object), deleted_activity = Activity.delete_by_ap_id(id), - {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do + {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"), + {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do {:ok, object, deleted_activity} end end def prune(%Object{data: %{"id" => id}} = object) do with {:ok, object} <- Repo.delete(object), - {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do + {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"), + {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do {:ok, object} end end diff --git a/lib/pleroma/plugs/cache.ex b/lib/pleroma/plugs/cache.ex new file mode 100644 index 00000000..a81a861d --- /dev/null +++ b/lib/pleroma/plugs/cache.ex @@ -0,0 +1,122 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Plugs.Cache do + @moduledoc """ + Caches successful GET responses. + + To enable the cache add the plug to a router pipeline or controller: + + plug(Pleroma.Plugs.Cache) + + ## Configuration + + To configure the plug you need to pass settings as the second argument to the `plug/2` macro: + + plug(Pleroma.Plugs.Cache, [ttl: nil, query_params: true]) + + Available options: + + - `ttl`: An expiration time (time-to-live). This value should be in milliseconds or `nil` to disable expiration. Defaults to `nil`. + - `query_params`: Take URL query string into account (`true`), ignore it (`false`) or limit to specific params only (list). Defaults to `true`. + + Additionally, you can overwrite the TTL inside a controller action by assigning `cache_ttl` to the connection struct: + + def index(conn, _params) do + ttl = 60_000 # one minute + + conn + |> assign(:cache_ttl, ttl) + |> render("index.html") + end + + """ + + import Phoenix.Controller, only: [current_path: 1, json: 2] + import Plug.Conn + + @behaviour Plug + + @defaults %{ttl: nil, query_params: true} + + @impl true + def init([]), do: @defaults + + def init(opts) do + opts = Map.new(opts) + Map.merge(@defaults, opts) + end + + @impl true + def call(%{method: "GET"} = conn, opts) do + key = cache_key(conn, opts) + + case Cachex.get(:web_resp_cache, key) do + {:ok, nil} -> + cache_resp(conn, opts) + + {:ok, record} -> + send_cached(conn, record) + + {atom, message} when atom in [:ignore, :error] -> + render_error(conn, message) + end + end + + def call(conn, _), do: conn + + # full path including query params + defp cache_key(conn, %{query_params: true}), do: current_path(conn) + + # request path without query params + defp cache_key(conn, %{query_params: false}), do: conn.request_path + + # request path with specific query params + defp cache_key(conn, %{query_params: query_params}) when is_list(query_params) do + query_string = + conn.params + |> Map.take(query_params) + |> URI.encode_query() + + conn.request_path <> "?" <> query_string + end + + defp cache_resp(conn, opts) do + register_before_send(conn, fn + %{status: 200, resp_body: body} = conn -> + ttl = Map.get(conn.assigns, :cache_ttl, opts.ttl) + key = cache_key(conn, opts) + content_type = content_type(conn) + record = {content_type, body} + + Cachex.put(:web_resp_cache, key, record, ttl: ttl) + + put_resp_header(conn, "x-cache", "MISS from Pleroma") + + conn -> + conn + end) + end + + defp content_type(conn) do + conn + |> Plug.Conn.get_resp_header("content-type") + |> hd() + end + + defp send_cached(conn, {content_type, body}) do + conn + |> put_resp_content_type(content_type, nil) + |> put_resp_header("x-cache", "HIT from Pleroma") + |> send_resp(:ok, body) + |> halt() + end + + defp render_error(conn, message) do + conn + |> put_status(:internal_server_error) + |> json(%{error: message}) + |> halt() + end +end diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 7b007547..705dbc1c 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -23,6 +23,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do action_fallback(:errors) + plug(Pleroma.Plugs.Cache, [query_params: false] when action in [:activity, :object]) plug(Pleroma.Web.FederatingPlug when action in [:inbox, :relay]) plug(:set_requester_reachable when action in [:inbox]) plug(:relay_active? when action in [:relay]) @@ -53,8 +54,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do %Object{} = object <- Object.get_cached_by_ap_id(ap_id), {_, true} <- {:public?, Visibility.is_public?(object)} do conn + |> set_cache_ttl_for(object) |> put_resp_content_type("application/activity+json") - |> json(ObjectView.render("object.json", %{object: object})) + |> put_view(ObjectView) + |> render("object.json", object: object) else {:public?, false} -> {:error, :not_found} @@ -96,14 +99,36 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do %Activity{} = activity <- Activity.normalize(ap_id), {_, true} <- {:public?, Visibility.is_public?(activity)} do conn + |> set_cache_ttl_for(activity) |> put_resp_content_type("application/activity+json") - |> json(ObjectView.render("object.json", %{object: activity})) + |> put_view(ObjectView) + |> render("object.json", object: activity) else - {:public?, false} -> - {:error, :not_found} + {:public?, false} -> {:error, :not_found} + nil -> {:error, :not_found} end end + defp set_cache_ttl_for(conn, %Activity{object: object}) do + set_cache_ttl_for(conn, object) + end + + defp set_cache_ttl_for(conn, entity) do + ttl = + case entity do + %Object{data: %{"type" => "Question"}} -> + Pleroma.Config.get([:web_cache_ttl, :activity_pub_question]) + + %Object{} -> + Pleroma.Config.get([:web_cache_ttl, :activity_pub]) + + _ -> + nil + end + + assign(conn, :cache_ttl, ttl) + end + # GET /relay/following def following(%{assigns: %{relay: true}} = conn, _params) do conn diff --git a/test/object_test.exs b/test/object_test.exs index d138ee09..ba96aeea 100644 --- a/test/object_test.exs +++ b/test/object_test.exs @@ -53,9 +53,12 @@ defmodule Pleroma.ObjectTest do assert object == cached_object + Cachex.put(:web_resp_cache, URI.parse(object.data["id"]).path, "cofe") + Object.delete(cached_object) {:ok, nil} = Cachex.get(:object_cache, "object:#{object.data["id"]}") + {:ok, nil} = Cachex.get(:web_resp_cache, URI.parse(object.data["id"]).path) cached_object = Object.get_cached_by_ap_id(object.data["id"]) diff --git a/test/plugs/cache_test.exs b/test/plugs/cache_test.exs new file mode 100644 index 00000000..e6e7f409 --- /dev/null +++ b/test/plugs/cache_test.exs @@ -0,0 +1,186 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Plugs.CacheTest do + use ExUnit.Case, async: true + use Plug.Test + + alias Pleroma.Plugs.Cache + + @miss_resp {200, + [ + {"cache-control", "max-age=0, private, must-revalidate"}, + {"content-type", "cofe/hot; charset=utf-8"}, + {"x-cache", "MISS from Pleroma"} + ], "cofe"} + + @hit_resp {200, + [ + {"cache-control", "max-age=0, private, must-revalidate"}, + {"content-type", "cofe/hot; charset=utf-8"}, + {"x-cache", "HIT from Pleroma"} + ], "cofe"} + + @ttl 5 + + setup do + Cachex.clear(:web_resp_cache) + :ok + end + + test "caches a response" do + assert @miss_resp == + conn(:get, "/") + |> Cache.call(%{query_params: false, ttl: nil}) + |> put_resp_content_type("cofe/hot") + |> send_resp(:ok, "cofe") + |> sent_resp() + + assert_raise(Plug.Conn.AlreadySentError, fn -> + conn(:get, "/") + |> Cache.call(%{query_params: false, ttl: nil}) + |> put_resp_content_type("cofe/hot") + |> send_resp(:ok, "cofe") + |> sent_resp() + end) + + assert @hit_resp == + conn(:get, "/") + |> Cache.call(%{query_params: false, ttl: nil}) + |> sent_resp() + end + + test "ttl is set" do + assert @miss_resp == + conn(:get, "/") + |> Cache.call(%{query_params: false, ttl: @ttl}) + |> put_resp_content_type("cofe/hot") + |> send_resp(:ok, "cofe") + |> sent_resp() + + assert @hit_resp == + conn(:get, "/") + |> Cache.call(%{query_params: false, ttl: @ttl}) + |> sent_resp() + + :timer.sleep(@ttl + 1) + + assert @miss_resp == + conn(:get, "/") + |> Cache.call(%{query_params: false, ttl: @ttl}) + |> put_resp_content_type("cofe/hot") + |> send_resp(:ok, "cofe") + |> sent_resp() + end + + test "set ttl via conn.assigns" do + assert @miss_resp == + conn(:get, "/") + |> Cache.call(%{query_params: false, ttl: nil}) + |> put_resp_content_type("cofe/hot") + |> assign(:cache_ttl, @ttl) + |> send_resp(:ok, "cofe") + |> sent_resp() + + assert @hit_resp == + conn(:get, "/") + |> Cache.call(%{query_params: false, ttl: nil}) + |> sent_resp() + + :timer.sleep(@ttl + 1) + + assert @miss_resp == + conn(:get, "/") + |> Cache.call(%{query_params: false, ttl: nil}) + |> put_resp_content_type("cofe/hot") + |> send_resp(:ok, "cofe") + |> sent_resp() + end + + test "ignore query string when `query_params` is false" do + assert @miss_resp == + conn(:get, "/?cofe") + |> Cache.call(%{query_params: false, ttl: nil}) + |> put_resp_content_type("cofe/hot") + |> send_resp(:ok, "cofe") + |> sent_resp() + + assert @hit_resp == + conn(:get, "/?cofefe") + |> Cache.call(%{query_params: false, ttl: nil}) + |> sent_resp() + end + + test "take query string into account when `query_params` is true" do + assert @miss_resp == + conn(:get, "/?cofe") + |> Cache.call(%{query_params: true, ttl: nil}) + |> put_resp_content_type("cofe/hot") + |> send_resp(:ok, "cofe") + |> sent_resp() + + assert @miss_resp == + conn(:get, "/?cofefe") + |> Cache.call(%{query_params: true, ttl: nil}) + |> put_resp_content_type("cofe/hot") + |> send_resp(:ok, "cofe") + |> sent_resp() + end + + test "take specific query params into account when `query_params` is list" do + assert @miss_resp == + conn(:get, "/?a=1&b=2&c=3&foo=bar") + |> fetch_query_params() + |> Cache.call(%{query_params: ["a", "b", "c"], ttl: nil}) + |> put_resp_content_type("cofe/hot") + |> send_resp(:ok, "cofe") + |> sent_resp() + + assert @hit_resp == + conn(:get, "/?bar=foo&c=3&b=2&a=1") + |> fetch_query_params() + |> Cache.call(%{query_params: ["a", "b", "c"], ttl: nil}) + |> sent_resp() + + assert @miss_resp == + conn(:get, "/?bar=foo&c=3&b=2&a=2") + |> fetch_query_params() + |> Cache.call(%{query_params: ["a", "b", "c"], ttl: nil}) + |> put_resp_content_type("cofe/hot") + |> send_resp(:ok, "cofe") + |> sent_resp() + end + + test "ignore not GET requests" do + expected = + {200, + [ + {"cache-control", "max-age=0, private, must-revalidate"}, + {"content-type", "cofe/hot; charset=utf-8"} + ], "cofe"} + + assert expected == + conn(:post, "/") + |> Cache.call(%{query_params: true, ttl: nil}) + |> put_resp_content_type("cofe/hot") + |> send_resp(:ok, "cofe") + |> sent_resp() + end + + test "ignore non-successful responses" do + expected = + {418, + [ + {"cache-control", "max-age=0, private, must-revalidate"}, + {"content-type", "tea/iced; charset=utf-8"} + ], "🥤"} + + assert expected == + conn(:get, "/cofe") + |> Cache.call(%{query_params: true, ttl: nil}) + |> put_resp_content_type("tea/iced") + |> send_resp(:im_a_teapot, "🥤") + |> sent_resp() + end +end diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index 4388538c..9698c709 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -175,6 +175,49 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert json_response(conn, 404) end + + test "it caches a response", %{conn: conn} do + note = insert(:note) + uuid = String.split(note.data["id"], "/") |> List.last() + + conn1 = + conn + |> put_req_header("accept", "application/activity+json") + |> get("/objects/#{uuid}") + + assert json_response(conn1, :ok) + assert Enum.any?(conn1.resp_headers, &(&1 == {"x-cache", "MISS from Pleroma"})) + + conn2 = + conn + |> put_req_header("accept", "application/activity+json") + |> get("/objects/#{uuid}") + + assert json_response(conn1, :ok) == json_response(conn2, :ok) + assert Enum.any?(conn2.resp_headers, &(&1 == {"x-cache", "HIT from Pleroma"})) + end + + test "cached purged after object deletion", %{conn: conn} do + note = insert(:note) + uuid = String.split(note.data["id"], "/") |> List.last() + + conn1 = + conn + |> put_req_header("accept", "application/activity+json") + |> get("/objects/#{uuid}") + + assert json_response(conn1, :ok) + assert Enum.any?(conn1.resp_headers, &(&1 == {"x-cache", "MISS from Pleroma"})) + + Object.delete(note) + + conn2 = + conn + |> put_req_header("accept", "application/activity+json") + |> get("/objects/#{uuid}") + + assert "Not found" == json_response(conn2, :not_found) + end end describe "/object/:uuid/likes" do @@ -264,6 +307,51 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert json_response(conn, 404) end + + test "it caches a response", %{conn: conn} do + activity = insert(:note_activity) + uuid = String.split(activity.data["id"], "/") |> List.last() + + conn1 = + conn + |> put_req_header("accept", "application/activity+json") + |> get("/activities/#{uuid}") + + assert json_response(conn1, :ok) + assert Enum.any?(conn1.resp_headers, &(&1 == {"x-cache", "MISS from Pleroma"})) + + conn2 = + conn + |> put_req_header("accept", "application/activity+json") + |> get("/activities/#{uuid}") + + assert json_response(conn1, :ok) == json_response(conn2, :ok) + assert Enum.any?(conn2.resp_headers, &(&1 == {"x-cache", "HIT from Pleroma"})) + end + + test "cached purged after activity deletion", %{conn: conn} do + user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{"status" => "cofe"}) + + uuid = String.split(activity.data["id"], "/") |> List.last() + + conn1 = + conn + |> put_req_header("accept", "application/activity+json") + |> get("/activities/#{uuid}") + + assert json_response(conn1, :ok) + assert Enum.any?(conn1.resp_headers, &(&1 == {"x-cache", "MISS from Pleroma"})) + + Activity.delete_by_ap_id(activity.object.data["id"]) + + conn2 = + conn + |> put_req_header("accept", "application/activity+json") + |> get("/activities/#{uuid}") + + assert "Not found" == json_response(conn2, :not_found) + end end describe "/inbox" do From b40b10b53d00d13f24b5667acc02b1642abc6ec4 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 3 Sep 2019 16:23:03 +0700 Subject: [PATCH 032/138] Add an endpoint to get multiple statuses by IDs --- CHANGELOG.md | 1 + docs/api/differences_in_mastoapi_responses.md | 12 ++++++++++++ lib/pleroma/activity.ex | 7 +++++++ .../controllers/mastodon_api_controller.ex | 14 ++++++++++++++ lib/pleroma/web/router.ex | 1 + test/activity_test.exs | 8 ++++++++ .../mastodon_api/mastodon_api_controller_test.exs | 10 ++++++++++ 7 files changed, 53 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3fa9dfb..f7f1aee0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,6 +110,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Federation: Remove `likes` from objects. - Admin API: Added moderation log - Web response cache (currently, enabled for ActivityPub) +- Mastodon API: Added an endpoint to get multiple statuses by IDs (`GET /api/v1/statuses/?ids[]=1&ids[]=2`) ### Changed - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index 02f90f3e..a21eebc9 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -91,6 +91,18 @@ Additional parameters can be added to the JSON body/Form data: - `expires_in`: The number of seconds the posted activity should expire in. When a posted activity expires it will be deleted from the server, and a delete request for it will be federated. This needs to be longer than an hour. - `in_reply_to_conversation_id`: Will reply to a given conversation, addressing only the people who are part of the recipient set of that conversation. Sets the visibility to `direct`. +## GET `/api/v1/statuses` + +An endpoint to get multiple statuses by IDs. + +Required parameters: + +- `ids`: array of activity ids + +Usage example: `GET /api/v1/statuses/?ids[]=1&ids[]=2`. + +Returns: array of Status. + ## PATCH `/api/v1/update_credentials` Additional parameters can be added to the JSON body/Form data: diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 6a51d4cf..44f1e301 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -173,6 +173,13 @@ defmodule Pleroma.Activity do |> Repo.one() end + def all_by_ids_with_object(ids) do + Activity + |> where([a], a.id in ^ids) + |> with_preloaded_object() + |> Repo.all() + end + def by_object_ap_id(ap_id) do from( activity in Activity, diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex index 8dfad7a5..c54462bb 100644 --- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex @@ -427,6 +427,20 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do |> render("index.json", %{activities: activities, for: user, as: :activity}) end + def get_statuses(%{assigns: %{user: user}} = conn, %{"ids" => ids}) do + limit = 100 + + activities = + ids + |> Enum.take(limit) + |> Activity.all_by_ids_with_object() + |> Enum.filter(&Visibility.visible_for_user?(&1, user)) + + conn + |> put_view(StatusView) + |> render("index.json", activities: activities, for: user, as: :activity) + end + def get_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do with %Activity{} = activity <- Activity.get_by_id_with_object(id), true <- Visibility.visible_for_user?(activity, user) do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index cfb973f5..7cd59acb 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -443,6 +443,7 @@ defmodule Pleroma.Web.Router do get("/timelines/tag/:tag", MastodonAPIController, :hashtag_timeline) get("/timelines/list/:list_id", MastodonAPIController, :list_timeline) + get("/statuses", MastodonAPIController, :get_statuses) get("/statuses/:id", MastodonAPIController, :get_status) get("/statuses/:id/context", MastodonAPIController, :get_context) diff --git a/test/activity_test.exs b/test/activity_test.exs index 785c4b3c..49654bd6 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -173,4 +173,12 @@ defmodule Pleroma.ActivityTest do |> where([a], a.activity_id == ^activity.id) |> Repo.one!() end + + test "all_by_ids_with_object/1" do + %{id: id1} = insert(:note_activity) + %{id: id2} = insert(:note_activity) + + assert [%{id: ^id1, object: %Object{}}, %{id: ^id2, object: %Object{}}] = + Activity.all_by_ids_with_object([id1, id2]) + end end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index e18f8f0d..f4902d04 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -744,6 +744,16 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do assert id == to_string(activity.id) end + test "get statuses by IDs", %{conn: conn} do + %{id: id1} = insert(:note_activity) + %{id: id2} = insert(:note_activity) + + query_string = "ids[]=#{id1}&ids[]=#{id2}" + conn = get(conn, "/api/v1/statuses/?#{query_string}") + + assert [%{"id" => ^id1}, %{"id" => ^id2}] = json_response(conn, :ok) + end + describe "deleting a status" do test "when you created it", %{conn: conn} do activity = insert(:note_activity) From 30f0cec49a39a2f17cd7fae89830bf0d7922a738 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 3 Sep 2019 16:58:33 +0700 Subject: [PATCH 033/138] Add note about limit to the doc --- docs/api/differences_in_mastoapi_responses.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index a21eebc9..9b32baf3 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -103,6 +103,8 @@ Usage example: `GET /api/v1/statuses/?ids[]=1&ids[]=2`. Returns: array of Status. +The maximum number of statuses is limited to 100 per request. + ## PATCH `/api/v1/update_credentials` Additional parameters can be added to the JSON body/Form data: From 3d12e05f43763d3ebc693e69f3f945044939bcc2 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 11 Sep 2019 01:08:37 +0700 Subject: [PATCH 034/138] Fix `Activity.all_by_ids_with_object/1` test --- test/activity_test.exs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/activity_test.exs b/test/activity_test.exs index 49654bd6..4152aaa7 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -178,7 +178,11 @@ defmodule Pleroma.ActivityTest do %{id: id1} = insert(:note_activity) %{id: id2} = insert(:note_activity) - assert [%{id: ^id1, object: %Object{}}, %{id: ^id2, object: %Object{}}] = - Activity.all_by_ids_with_object([id1, id2]) + activities = + [id1, id2] + |> Activity.all_by_ids_with_object() + |> Enum.sort(&(&1.id < &2.id)) + + assert [%{id: ^id1, object: %Object{}}, %{id: ^id2, object: %Object{}}] = activities end end From 43f02dfe38547e07fb189aab78539af9e02302b3 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 10 Sep 2019 22:01:45 +0300 Subject: [PATCH 035/138] Revert "Parallelize template rendering" This reverts commit 1ad71592adb47762287aec8c36d0fca565c38362. Since it had no limit on the number on concurrent processes it OOM killed instances while rendering hellthreads. When I tried introducing a concurrency limit with Task.async_stream/manual folds it lead to about 3 times worse performance on threads larger than 1000 activities (we are talking 30s vs 1.2 minutes), I think this is not worth the about 1.5 times performance increase on smaller threads when using it. --- lib/mix/tasks/pleroma/benchmark.ex | 38 +++++-------------- .../web/mastodon_api/views/status_view.ex | 4 +- lib/pleroma/web/web.ex | 18 +-------- 3 files changed, 12 insertions(+), 48 deletions(-) diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex index 4cc63472..a45940bf 100644 --- a/lib/mix/tasks/pleroma/benchmark.ex +++ b/lib/mix/tasks/pleroma/benchmark.ex @@ -37,37 +37,17 @@ defmodule Mix.Tasks.Pleroma.Benchmark do |> Map.put("blocking_user", user) |> Map.put("muting_user", user) |> Map.put("user", user) - |> Map.put("limit", 80) |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities() |> Enum.reverse() - inputs = %{ - "One activity" => Enum.take_random(activities, 1), - "Ten activities" => Enum.take_random(activities, 10), - "Twenty activities" => Enum.take_random(activities, 20), - "Forty activities" => Enum.take_random(activities, 40), - "Eighty activities" => Enum.take_random(activities, 80) - } - - Benchee.run( - %{ - "Parallel rendering" => fn activities -> - Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ - activities: activities, - for: user, - as: :activity - }) - end, - "Standart rendering" => fn activities -> - Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ - activities: activities, - for: user, - as: :activity, - parallel: false - }) - end - }, - inputs: inputs - ) + Benchee.run(%{ + "render_timeline" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: activities, + for: user, + as: :activity + }) + end + }) end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index e71083b9..b6a3431f 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -73,14 +73,12 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do def render("index.json", opts) do replied_to_activities = get_replied_to_activities(opts.activities) - parallel = unless is_nil(opts[:parallel]), do: opts[:parallel], else: true opts.activities |> safe_render_many( StatusView, "status.json", - Map.put(opts, :replied_to_activities, replied_to_activities), - parallel + Map.put(opts, :replied_to_activities, replied_to_activities) ) end diff --git a/lib/pleroma/web/web.ex b/lib/pleroma/web/web.ex index bfb6c728..68734655 100644 --- a/lib/pleroma/web/web.ex +++ b/lib/pleroma/web/web.ex @@ -66,23 +66,9 @@ defmodule Pleroma.Web do end @doc """ - Same as `render_many/4` but wrapped in rescue block and parallelized (unless disabled by passing false as a fifth argument). + Same as `render_many/4` but wrapped in rescue block. """ - def safe_render_many(collection, view, template, assigns \\ %{}, parallel \\ true) - - def safe_render_many(collection, view, template, assigns, true) do - Enum.map(collection, fn resource -> - Task.async(fn -> - as = Map.get(assigns, :as) || view.__resource__ - assigns = Map.put(assigns, as, resource) - safe_render(view, template, assigns) - end) - end) - |> Enum.map(&Task.await(&1, :infinity)) - |> Enum.filter(& &1) - end - - def safe_render_many(collection, view, template, assigns, false) do + def safe_render_many(collection, view, template, assigns \\ %{}) do Enum.map(collection, fn resource -> as = Map.get(assigns, :as) || view.__resource__ assigns = Map.put(assigns, as, resource) From ab4960cc7f6703d93e3bf2d316978150a6dba59f Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 30 Aug 2019 12:00:36 +0300 Subject: [PATCH 036/138] deleting old config.md file --- docs/config.md | 701 ------------------------------------------------- 1 file changed, 701 deletions(-) delete mode 100644 docs/config.md diff --git a/docs/config.md b/docs/config.md deleted file mode 100644 index 9136532e..00000000 --- a/docs/config.md +++ /dev/null @@ -1,701 +0,0 @@ -# Configuration - -This file describe the configuration, it is recommended to edit the relevant *.secret.exs file instead of the others founds in the ``config`` directory. -If you run Pleroma with ``MIX_ENV=prod`` the file is ``prod.secret.exs``, otherwise it is ``dev.secret.exs``. - -## Pleroma.Upload -* `uploader`: Select which `Pleroma.Uploaders` to use -* `filters`: List of `Pleroma.Upload.Filter` to use. -* `link_name`: When enabled Pleroma will add a `name` parameter to the url of the upload, for example `https://instance.tld/media/corndog.png?name=corndog.png`. This is needed to provide the correct filename in Content-Disposition headers when using filters like `Pleroma.Upload.Filter.Dedupe` -* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host. -* `proxy_remote`: If you're using a remote uploader, Pleroma will proxy media requests instead of redirecting to it. -* `proxy_opts`: Proxy options, see `Pleroma.ReverseProxy` documentation. - -Note: `strip_exif` has been replaced by `Pleroma.Upload.Filter.Mogrify`. - -## Pleroma.Uploaders.Local -* `uploads`: Which directory to store the user-uploads in, relative to pleroma’s working directory - -## Pleroma.Uploaders.S3 -* `bucket`: S3 bucket name -* `bucket_namespace`: S3 bucket namespace -* `public_endpoint`: S3 endpoint that the user finally accesses(ex. "https://s3.dualstack.ap-northeast-1.amazonaws.com") -* `truncated_namespace`: If you use S3 compatible service such as Digital Ocean Spaces or CDN, set folder name or "" etc. -For example, when using CDN to S3 virtual host format, set "". -At this time, write CNAME to CDN in public_endpoint. - -## Pleroma.Upload.Filter.Mogrify - -* `args`: List of actions for the `mogrify` command like `"strip"` or `["strip", "auto-orient", {"implode", "1"}]`. - -## Pleroma.Upload.Filter.Dedupe - -No specific configuration. - -## Pleroma.Upload.Filter.AnonymizeFilename - -This filter replaces the filename (not the path) of an upload. For complete obfuscation, add -`Pleroma.Upload.Filter.Dedupe` before AnonymizeFilename. - -* `text`: Text to replace filenames in links. If empty, `{random}.extension` will be used. You can get the original filename extension by using `{extension}`, for example `custom-file-name.{extension}`. - -## Pleroma.Emails.Mailer -* `adapter`: one of the mail adapters listed in [Swoosh readme](https://github.com/swoosh/swoosh#adapters), or `Swoosh.Adapters.Local` for in-memory mailbox. -* `api_key` / `password` and / or other adapter-specific settings, per the above documentation. -* `enabled`: Allows enable/disable send emails. Default: `false`. - -An example for Sendgrid adapter: - -```elixir -config :pleroma, Pleroma.Emails.Mailer, - adapter: Swoosh.Adapters.Sendgrid, - api_key: "YOUR_API_KEY" -``` - -An example for SMTP adapter: - -```elixir -config :pleroma, Pleroma.Emails.Mailer, - adapter: Swoosh.Adapters.SMTP, - relay: "smtp.gmail.com", - username: "YOUR_USERNAME@gmail.com", - password: "YOUR_SMTP_PASSWORD", - port: 465, - ssl: true, - tls: :always, - auth: :always -``` - -## :uri_schemes -* `valid_schemes`: List of the scheme part that is considered valid to be an URL - -## :instance -* `name`: The instance’s name -* `email`: Email used to reach an Administrator/Moderator of the instance -* `notify_email`: Email used for notifications. -* `description`: The instance’s description, can be seen in nodeinfo and ``/api/v1/instance`` -* `limit`: Posts character limit (CW/Subject included in the counter) -* `remote_limit`: Hard character limit beyond which remote posts will be dropped. -* `upload_limit`: File size limit of uploads (except for avatar, background, banner) -* `avatar_upload_limit`: File size limit of user’s profile avatars -* `background_upload_limit`: File size limit of user’s profile backgrounds -* `banner_upload_limit`: File size limit of user’s profile banners -* `poll_limits`: A map with poll limits for **local** polls - * `max_options`: Maximum number of options - * `max_option_chars`: Maximum number of characters per option - * `min_expiration`: Minimum expiration time (in seconds) - * `max_expiration`: Maximum expiration time (in seconds) -* `registrations_open`: Enable registrations for anyone, invitations can be enabled when false. -* `invites_enabled`: Enable user invitations for admins (depends on `registrations_open: false`). -* `account_activation_required`: Require users to confirm their emails before signing in. -* `federating`: Enable federation with other instances -* `federation_incoming_replies_max_depth`: Max. depth of reply-to activities fetching on incoming federation, to prevent out-of-memory situations while fetching very long threads. If set to `nil`, threads of any depth will be fetched. Lower this value if you experience out-of-memory crashes. -* `federation_reachability_timeout_days`: Timeout (in days) of each external federation target being unreachable prior to pausing federating to it. -* `allow_relay`: Enable Pleroma’s Relay, which makes it possible to follow a whole instance -* `rewrite_policy`: Message Rewrite Policy, either one or a list. Here are the ones available by default: - * `Pleroma.Web.ActivityPub.MRF.NoOpPolicy`: Doesn’t modify activities (default) - * `Pleroma.Web.ActivityPub.MRF.DropPolicy`: Drops all activities. It generally doesn’t makes sense to use in production - * `Pleroma.Web.ActivityPub.MRF.SimplePolicy`: Restrict the visibility of activities from certains instances (See ``:mrf_simple`` section) - * `Pleroma.Web.ActivityPub.MRF.TagPolicy`: Applies policies to individual users based on tags, which can be set using pleroma-fe/admin-fe/any other app that supports Pleroma Admin API. For example it allows marking posts from individual users nsfw (sensitive) - * `Pleroma.Web.ActivityPub.MRF.SubchainPolicy`: Selectively runs other MRF policies when messages match (see ``:mrf_subchain`` section) - * `Pleroma.Web.ActivityPub.MRF.RejectNonPublic`: Drops posts with non-public visibility settings (See ``:mrf_rejectnonpublic`` section) - * `Pleroma.Web.ActivityPub.MRF.EnsureRePrepended`: Rewrites posts to ensure that replies to posts with subjects do not have an identical subject and instead begin with re:. - * `Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy`: Rejects posts from likely spambots by rejecting posts from new users that contain links. - * `Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`: Crawls attachments using their MediaProxy URLs so that the MediaProxy cache is primed. - * `Pleroma.Web.ActivityPub.MRF.MentionPolicy`: Drops posts mentioning configurable users. (see `:mrf_mention` section) - * `Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`: Restricts activities to a configured set of vocabulary. (see `:mrf_vocabulary` section) -* `public`: Makes the client API in authentificated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. -* `quarantined_instances`: List of ActivityPub instances where private(DMs, followers-only) activities will not be send. -* `managed_config`: Whenether the config for pleroma-fe is configured in this config or in ``static/config.json`` -* `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML) -* `mrf_transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo). -* `mrf_transparency_exclusions`: Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value. -* `scope_copy`: Copy the scope (private/unlisted/public) in replies to posts by default. -* `subject_line_behavior`: Allows changing the default behaviour of subject lines in replies. Valid values: - * "email": Copy and preprend re:, as in email. - * "masto": Copy verbatim, as in Mastodon. - * "noop": Don't copy the subject. -* `always_show_subject_input`: When set to false, auto-hide the subject field when it's empty. -* `extended_nickname_format`: Set to `true` to use extended local nicknames format (allows underscores/dashes). This will break federation with - older software for theses nicknames. -* `max_pinned_statuses`: The maximum number of pinned statuses. `0` will disable the feature. -* `autofollowed_nicknames`: Set to nicknames of (local) users that every new user should automatically follow. -* `no_attachment_links`: Set to true to disable automatically adding attachment link text to statuses -* `welcome_message`: A message that will be send to a newly registered users as a direct message. -* `welcome_user_nickname`: The nickname of the local user that sends the welcome message. -* `max_report_comment_size`: The maximum size of the report comment (Default: `1000`) -* `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). Default: `false`. -* `healthcheck`: If set to true, system data will be shown on ``/api/pleroma/healthcheck``. -* `remote_post_retention_days`: The default amount of days to retain remote posts when pruning the database. -* `user_bio_length`: A user bio maximum length (default: `5000`) -* `user_name_length`: A user name maximum length (default: `100`) -* `skip_thread_containment`: Skip filter out broken threads. The default is `false`. -* `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. -* `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. -* `max_account_fields`: The maximum number of custom fields in the user profile (default: `10`) -* `max_remote_account_fields`: The maximum number of custom fields in the remote user profile (default: `20`) -* `account_field_name_length`: An account field name maximum length (default: `512`) -* `account_field_value_length`: An account field value maximum length (default: `512`) -* `external_user_synchronization`: Enabling following/followers counters synchronization for external users. - - - -## :logger -* `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog, and `Quack.Logger` to log to Slack - -An example to enable ONLY ExSyslogger (f/ex in ``prod.secret.exs``) with info and debug suppressed: -```elixir -config :logger, - backends: [{ExSyslogger, :ex_syslogger}] - -config :logger, :ex_syslogger, - level: :warn -``` - -Another example, keeping console output and adding the pid to syslog output: -```elixir -config :logger, - backends: [:console, {ExSyslogger, :ex_syslogger}] - -config :logger, :ex_syslogger, - level: :warn, - option: [:pid, :ndelay] -``` - -See: [logger’s documentation](https://hexdocs.pm/logger/Logger.html) and [ex_syslogger’s documentation](https://hexdocs.pm/ex_syslogger/) - -An example of logging info to local syslog, but warn to a Slack channel: -```elixir -config :logger, - backends: [ {ExSyslogger, :ex_syslogger}, Quack.Logger ], - level: :info - -config :logger, :ex_syslogger, - level: :info, - ident: "pleroma", - format: "$metadata[$level] $message" - -config :quack, - level: :warn, - meta: [:all], - webhook_url: "https://hooks.slack.com/services/YOUR-API-KEY-HERE" -``` - -See the [Quack Github](https://github.com/azohra/quack) for more details - -## :frontend_configurations - -This can be used to configure a keyword list that keeps the configuration data for any kind of frontend. By default, settings for `pleroma_fe` and `masto_fe` are configured. - -Frontends can access these settings at `/api/pleroma/frontend_configurations` - -To add your own configuration for PleromaFE, use it like this: - -```elixir -config :pleroma, :frontend_configurations, - pleroma_fe: %{ - theme: "pleroma-dark", - # ... see /priv/static/static/config.json for the available keys. -}, - masto_fe: %{ - showInstanceSpecificPanel: true - } -``` - -These settings **need to be complete**, they will override the defaults. - -NOTE: for versions < 1.0, you need to set [`:fe`](#fe) to false, as shown a few lines below. - -## :fe -__THIS IS DEPRECATED__ - -If you are using this method, please change it to the [`frontend_configurations`](#frontend_configurations) method. -Please **set this option to false** in your config like this: - -```elixir -config :pleroma, :fe, false -``` - -This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:instance`` is set to false. - -* `theme`: Which theme to use, they are defined in ``styles.json`` -* `logo`: URL of the logo, defaults to Pleroma’s logo -* `logo_mask`: Whether to use only the logo's shape as a mask (true) or as a regular image (false) -* `logo_margin`: What margin to use around the logo -* `background`: URL of the background, unless viewing a user profile with a background that is set -* `redirect_root_no_login`: relative URL which indicates where to redirect when a user isn’t logged in. -* `redirect_root_login`: relative URL which indicates where to redirect when a user is logged in. -* `show_instance_panel`: Whenether to show the instance’s specific panel. -* `scope_options_enabled`: Enable setting an notice visibility and subject/CW when posting -* `formatting_options_enabled`: Enable setting a formatting different than plain-text (ie. HTML, Markdown) when posting, relates to ``:instance, allowed_post_formats`` -* `collapse_message_with_subjects`: When a message has a subject(aka Content Warning), collapse it by default -* `hide_post_stats`: Hide notices statistics(repeats, favorites, …) -* `hide_user_stats`: Hide profile statistics(posts, posts per day, followers, followings, …) - -## :assets - -This section configures assets to be used with various frontends. Currently the only option -relates to mascots on the mastodon frontend - -* `mascots`: KeywordList of mascots, each element __MUST__ contain both a `url` and a - `mime_type` key. -* `default_mascot`: An element from `mascots` - This will be used as the default mascot - on MastoFE (default: `:pleroma_fox_tan`) - -## :mrf_simple -* `media_removal`: List of instances to remove medias from -* `media_nsfw`: List of instances to put medias as NSFW(sensitive) from -* `federated_timeline_removal`: List of instances to remove from Federated (aka The Whole Known Network) Timeline -* `reject`: List of instances to reject any activities from -* `accept`: List of instances to accept any activities from -* `report_removal`: List of instances to reject reports from -* `avatar_removal`: List of instances to strip avatars from -* `banner_removal`: List of instances to strip banners from - -## :mrf_subchain -This policy processes messages through an alternate pipeline when a given message matches certain criteria. -All criteria are configured as a map of regular expressions to lists of policy modules. - -* `match_actor`: Matches a series of regular expressions against the actor field. - -Example: - -``` -config :pleroma, :mrf_subchain, - match_actor: %{ - ~r/https:\/\/example.com/s => [Pleroma.Web.ActivityPub.MRF.DropPolicy] - } -``` - -## :mrf_rejectnonpublic -* `allow_followersonly`: whether to allow followers-only posts -* `allow_direct`: whether to allow direct messages - -## :mrf_hellthread -* `delist_threshold`: Number of mentioned users after which the message gets delisted (the message can still be seen, but it will not show up in public timelines and mentioned users won't get notifications about it). Set to 0 to disable. -* `reject_threshold`: Number of mentioned users after which the messaged gets rejected. Set to 0 to disable. - -## :mrf_keyword -* `reject`: A list of patterns which result in message being rejected, each pattern can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) -* `federated_timeline_removal`: A list of patterns which result in message being removed from federated timelines (a.k.a unlisted), each pattern can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) -* `replace`: A list of tuples containing `{pattern, replacement}`, `pattern` can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) - -## :mrf_mention -* `actors`: A list of actors, for which to drop any posts mentioning. - -## :mrf_vocabulary -* `accept`: A list of ActivityStreams terms to accept. If empty, all supported messages are accepted. -* `reject`: A list of ActivityStreams terms to reject. If empty, no messages are rejected. - -## :media_proxy -* `enabled`: Enables proxying of remote media to the instance’s proxy -* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts. -* `proxy_opts`: All options defined in `Pleroma.ReverseProxy` documentation, defaults to `[max_body_length: (25*1_048_576)]`. -* `whitelist`: List of domains to bypass the mediaproxy - -## :gopher -* `enabled`: Enables the gopher interface -* `ip`: IP address to bind to -* `port`: Port to bind to -* `dstport`: Port advertised in urls (optional, defaults to `port`) - -## Pleroma.Web.Endpoint -`Phoenix` endpoint configuration, all configuration options can be viewed [here](https://hexdocs.pm/phoenix/Phoenix.Endpoint.html#module-dynamic-configuration), only common options are listed here -* `http` - a list containing http protocol configuration, all configuration options can be viewed [here](https://hexdocs.pm/plug_cowboy/Plug.Cowboy.html#module-options), only common options are listed here. For deployment using docker, you need to set this to `[ip: {0,0,0,0}, port: 4000]` to make pleroma accessible from other containers (such as your nginx server). - - `ip` - a tuple consisting of 4 integers - - `port` -* `url` - a list containing the configuration for generating urls, accepts - - `host` - the host without the scheme and a post (e.g `example.com`, not `https://example.com:2020`) - - `scheme` - e.g `http`, `https` - - `port` - - `path` -* `extra_cookie_attrs` - a list of `Key=Value` strings to be added as non-standard cookie attributes. Defaults to `["SameSite=Lax"]`. See the [SameSite article](https://www.owasp.org/index.php/SameSite) on OWASP for more info. - - - -**Important note**: if you modify anything inside these lists, default `config.exs` values will be overwritten, which may result in breakage, to make sure this does not happen please copy the default value for the list from `config.exs` and modify/add only what you need - -Example: -```elixir -config :pleroma, Pleroma.Web.Endpoint, - url: [host: "example.com", port: 2020, scheme: "https"], - http: [ - # start copied from config.exs - dispatch: [ - {:_, - [ - {"/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, []}, - {"/websocket", Phoenix.Endpoint.CowboyWebSocket, - {Phoenix.Transports.WebSocket, - {Pleroma.Web.Endpoint, Pleroma.Web.UserSocket, websocket_config}}}, - {:_, Phoenix.Endpoint.Cowboy2Handler, {Pleroma.Web.Endpoint, []}} - ]} - # end copied from config.exs - ], - port: 8080, - ip: {127, 0, 0, 1} - ] -``` - -This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls starting with `https://example.com:2020` - -## :activitypub -* ``unfollow_blocked``: Whether blocks result in people getting unfollowed -* ``outgoing_blocks``: Whether to federate blocks to other instances -* ``deny_follow_blocked``: Whether to disallow following an account that has blocked the user in question -* ``sign_object_fetches``: Sign object fetches with HTTP signatures - -## :http_security -* ``enabled``: Whether the managed content security policy is enabled -* ``sts``: Whether to additionally send a `Strict-Transport-Security` header -* ``sts_max_age``: The maximum age for the `Strict-Transport-Security` header if sent -* ``ct_max_age``: The maximum age for the `Expect-CT` header if sent -* ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"` -* ``report_uri``: Adds the specified url to `report-uri` and `report-to` group in CSP header. - -## :mrf_user_allowlist - -The keys in this section are the domain names that the policy should apply to. -Each key should be assigned a list of users that should be allowed through by -their ActivityPub ID. - -An example: - -```elixir -config :pleroma, :mrf_user_allowlist, - "example.org": ["https://example.org/users/admin"] -``` - -## :web_push_encryption, :vapid_details - -Web Push Notifications configuration. You can use the mix task `mix web_push.gen.keypair` to generate it. - -* ``subject``: a mailto link for the administrative contact. It’s best if this email is not a personal email address, but rather a group email so that if a person leaves an organization, is unavailable for an extended period, or otherwise can’t respond, someone else on the list can. -* ``public_key``: VAPID public key -* ``private_key``: VAPID private key - -## Pleroma.Captcha -* `enabled`: Whether the captcha should be shown on registration -* `method`: The method/service to use for captcha -* `seconds_valid`: The time in seconds for which the captcha is valid - -### Pleroma.Captcha.Kocaptcha -Kocaptcha is a very simple captcha service with a single API endpoint, -the source code is here: https://github.com/koto-bank/kocaptcha. The default endpoint -`https://captcha.kotobank.ch` is hosted by the developer. - -* `endpoint`: the kocaptcha endpoint to use - -## :admin_token - -Allows to set a token that can be used to authenticate with the admin api without using an actual user by giving it as the 'admin_token' parameter. Example: - -```elixir -config :pleroma, :admin_token, "somerandomtoken" -``` - -You can then do - -```sh -curl "http://localhost:4000/api/pleroma/admin/invite_token?admin_token=somerandomtoken" -``` - -## :pleroma_job_queue - -[Pleroma Job Queue](https://git.pleroma.social/pleroma/pleroma_job_queue) configuration: a list of queues with maximum concurrent jobs. - -Pleroma has the following queues: - -* `federator_outgoing` - Outgoing federation -* `federator_incoming` - Incoming federation -* `mailer` - Email sender, see [`Pleroma.Emails.Mailer`](#pleroma-emails-mailer) -* `transmogrifier` - Transmogrifier -* `web_push` - Web push notifications -* `scheduled_activities` - Scheduled activities, see [`Pleroma.ScheduledActivities`](#pleromascheduledactivity) - -Example: - -```elixir -config :pleroma_job_queue, :queues, - federator_incoming: 50, - federator_outgoing: 50 -``` - -This config contains two queues: `federator_incoming` and `federator_outgoing`. Both have the `max_jobs` set to `50`. - -## Pleroma.Web.Federator.RetryQueue - -* `enabled`: If set to `true`, failed federation jobs will be retried -* `max_jobs`: The maximum amount of parallel federation jobs running at the same time. -* `initial_timeout`: The initial timeout in seconds -* `max_retries`: The maximum number of times a federation job is retried - -## Pleroma.Web.Metadata -* `providers`: a list of metadata providers to enable. Providers available: - * Pleroma.Web.Metadata.Providers.OpenGraph - * Pleroma.Web.Metadata.Providers.TwitterCard - * Pleroma.Web.Metadata.Providers.RelMe - add links from user bio with rel=me into the `
` as `` -* `unfurl_nsfw`: If set to `true` nsfw attachments will be shown in previews - -## :rich_media -* `enabled`: if enabled the instance will parse metadata from attached links to generate link previews -* `ignore_hosts`: list of hosts which will be ignored by the metadata parser. For example `["accounts.google.com", "xss.website"]`, defaults to `[]`. -* `ignore_tld`: list TLDs (top-level domains) which will ignore for parse metadata. default is ["local", "localdomain", "lan"] -* `parsers`: list of Rich Media parsers - -## :fetch_initial_posts -* `enabled`: if enabled, when a new user is federated with, fetch some of their latest posts -* `pages`: the amount of pages to fetch - -## :hackney_pools - -Advanced. Tweaks Hackney (http client) connections pools. - -There's three pools used: - -* `:federation` for the federation jobs. - You may want this pool max_connections to be at least equal to the number of federator jobs + retry queue jobs. -* `:media` for rich media, media proxy -* `:upload` for uploaded media (if using a remote uploader and `proxy_remote: true`) - -For each pool, the options are: - -* `max_connections` - how much connections a pool can hold -* `timeout` - retention duration for connections - -## :auto_linker - -Configuration for the `auto_linker` library: - -* `class: "auto-linker"` - specify the class to be added to the generated link. false to clear -* `rel: "noopener noreferrer"` - override the rel attribute. false to clear -* `new_window: true` - set to false to remove `target='_blank'` attribute -* `scheme: false` - Set to true to link urls with schema `http://google.com` -* `truncate: false` - Set to a number to truncate urls longer then the number. Truncated urls will end in `..` -* `strip_prefix: true` - Strip the scheme prefix -* `extra: false` - link urls with rarely used schemes (magnet, ipfs, irc, etc.) - -Example: - -```elixir -config :auto_linker, - opts: [ - scheme: true, - extra: true, - class: false, - strip_prefix: false, - new_window: false, - rel: false - ] -``` - -## Pleroma.ScheduledActivity - -* `daily_user_limit`: the number of scheduled activities a user is allowed to create in a single day (Default: `25`) -* `total_user_limit`: the number of scheduled activities a user is allowed to create in total (Default: `300`) -* `enabled`: whether scheduled activities are sent to the job queue to be executed - -## Pleroma.ActivityExpiration - -# `enabled`: whether expired activities will be sent to the job queue to be deleted - -## Pleroma.Web.Auth.Authenticator - -* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator -* `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication - -## :ldap - -Use LDAP for user authentication. When a user logs in to the Pleroma -instance, the name and password will be verified by trying to authenticate -(bind) to an LDAP server. If a user exists in the LDAP directory but there -is no account with the same name yet on the Pleroma instance then a new -Pleroma account will be created with the same name as the LDAP user name. - -* `enabled`: enables LDAP authentication -* `host`: LDAP server hostname -* `port`: LDAP port, e.g. 389 or 636 -* `ssl`: true to use SSL, usually implies the port 636 -* `sslopts`: additional SSL options -* `tls`: true to start TLS, usually implies the port 389 -* `tlsopts`: additional TLS options -* `base`: LDAP base, e.g. "dc=example,dc=com" -* `uid`: LDAP attribute name to authenticate the user, e.g. when "cn", the filter will be "cn=username,base" - -## BBS / SSH access - -To enable simple command line interface accessible over ssh, add a setting like this to your configuration file: - -```exs -app_dir = File.cwd! -priv_dir = Path.join([app_dir, "priv/ssh_keys"]) - -config :esshd, - enabled: true, - priv_dir: priv_dir, - handler: "Pleroma.BBS.Handler", - port: 10_022, - password_authenticator: "Pleroma.BBS.Authenticator" -``` - -Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -m PEM -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT` - -## :auth - -* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator -* `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication - -Authentication / authorization settings. - -* `auth_template`: authentication form template. By default it's `show.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/show.html.eex`. -* `oauth_consumer_template`: OAuth consumer mode authentication form template. By default it's `consumer.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex`. -* `oauth_consumer_strategies`: the list of enabled OAuth consumer strategies; by default it's set by `OAUTH_CONSUMER_STRATEGIES` environment variable. Each entry in this space-delimited string should be of format `` or `:` (e.g. `twitter` or `keycloak:ueberauth_keycloak_strategy` in case dependency is named differently than `ueberauth_`). - -## :email_notifications - -Email notifications settings. - - - digest - emails of "what you've missed" for users who have been - inactive for a while. - - active: globally enable or disable digest emails - - schedule: When to send digest email, in [crontab format](https://en.wikipedia.org/wiki/Cron). - "0 0 * * 0" is the default, meaning "once a week at midnight on Sunday morning" - - interval: Minimum interval between digest emails to one user - - inactivity_threshold: Minimum user inactivity threshold - -## Pleroma.Emails.UserEmail - -- `:logo` - a path to a custom logo. Set it to `nil` to use the default Pleroma logo. -- `:styling` - a map with color settings for email templates. - -## OAuth consumer mode - -OAuth consumer mode allows sign in / sign up via external OAuth providers (e.g. Twitter, Facebook, Google, Microsoft, etc.). -Implementation is based on Ueberauth; see the list of [available strategies](https://github.com/ueberauth/ueberauth/wiki/List-of-Strategies). - -Note: each strategy is shipped as a separate dependency; in order to get the strategies, run `OAUTH_CONSUMER_STRATEGIES="..." mix deps.get`, -e.g. `OAUTH_CONSUMER_STRATEGIES="twitter facebook google microsoft" mix deps.get`. -The server should also be started with `OAUTH_CONSUMER_STRATEGIES="..." mix phx.server` in case you enable any strategies. - -Note: each strategy requires separate setup (on external provider side and Pleroma side). Below are the guidelines on setting up most popular strategies. - -Note: make sure that `"SameSite=Lax"` is set in `extra_cookie_attrs` when you have this feature enabled. OAuth consumer mode will not work with `"SameSite=Strict"` - -* For Twitter, [register an app](https://developer.twitter.com/en/apps), configure callback URL to https:///oauth/twitter/callback - -* For Facebook, [register an app](https://developers.facebook.com/apps), configure callback URL to https:///oauth/facebook/callback, enable Facebook Login service at https://developers.facebook.com/apps//fb-login/settings/ - -* For Google, [register an app](https://console.developers.google.com), configure callback URL to https:///oauth/google/callback - -* For Microsoft, [register an app](https://portal.azure.com), configure callback URL to https:///oauth/microsoft/callback - -Once the app is configured on external OAuth provider side, add app's credentials and strategy-specific settings (if any — e.g. see Microsoft below) to `config/prod.secret.exs`, -per strategy's documentation (e.g. [ueberauth_twitter](https://github.com/ueberauth/ueberauth_twitter)). Example config basing on environment variables: - -```elixir -# Twitter -config :ueberauth, Ueberauth.Strategy.Twitter.OAuth, - consumer_key: System.get_env("TWITTER_CONSUMER_KEY"), - consumer_secret: System.get_env("TWITTER_CONSUMER_SECRET") - -# Facebook -config :ueberauth, Ueberauth.Strategy.Facebook.OAuth, - client_id: System.get_env("FACEBOOK_APP_ID"), - client_secret: System.get_env("FACEBOOK_APP_SECRET"), - redirect_uri: System.get_env("FACEBOOK_REDIRECT_URI") - -# Google -config :ueberauth, Ueberauth.Strategy.Google.OAuth, - client_id: System.get_env("GOOGLE_CLIENT_ID"), - client_secret: System.get_env("GOOGLE_CLIENT_SECRET"), - redirect_uri: System.get_env("GOOGLE_REDIRECT_URI") - -# Microsoft -config :ueberauth, Ueberauth.Strategy.Microsoft.OAuth, - client_id: System.get_env("MICROSOFT_CLIENT_ID"), - client_secret: System.get_env("MICROSOFT_CLIENT_SECRET") - -config :ueberauth, Ueberauth, - providers: [ - microsoft: {Ueberauth.Strategy.Microsoft, [callback_params: []]} - ] - -# Keycloak -# Note: make sure to add `keycloak:ueberauth_keycloak_strategy` entry to `OAUTH_CONSUMER_STRATEGIES` environment variable -keycloak_url = "https://publicly-reachable-keycloak-instance.org:8080" - -config :ueberauth, Ueberauth.Strategy.Keycloak.OAuth, - client_id: System.get_env("KEYCLOAK_CLIENT_ID"), - client_secret: System.get_env("KEYCLOAK_CLIENT_SECRET"), - site: keycloak_url, - authorize_url: "#{keycloak_url}/auth/realms/master/protocol/openid-connect/auth", - token_url: "#{keycloak_url}/auth/realms/master/protocol/openid-connect/token", - userinfo_url: "#{keycloak_url}/auth/realms/master/protocol/openid-connect/userinfo", - token_method: :post - -config :ueberauth, Ueberauth, - providers: [ - keycloak: {Ueberauth.Strategy.Keycloak, [uid_field: :email]} - ] -``` - -## OAuth 2.0 provider - :oauth2 - -Configure OAuth 2 provider capabilities: - -* `token_expires_in` - The lifetime in seconds of the access token. -* `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token. -* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`. -* `clean_expired_tokens_interval` - Interval to run the job to clean expired tokens. Defaults to `86_400_000` (24 hours). - -## :emoji -* `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]` -* `pack_extensions`: A list of file extensions for emojis, when no emoji.txt for a pack is present. Example `[".png", ".gif"]` -* `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]` -* `default_manifest`: Location of the JSON-manifest. This manifest contains information about the emoji-packs you can download. Currently only one manifest can be added (no arrays). - -## Database options - -### RUM indexing for full text search -* `rum_enabled`: If RUM indexes should be used. Defaults to `false`. - -RUM indexes are an alternative indexing scheme that is not included in PostgreSQL by default. While they may eventually be mainlined, for now they have to be installed as a PostgreSQL extension from https://github.com/postgrespro/rum. - -Their advantage over the standard GIN indexes is that they allow efficient ordering of search results by timestamp, which makes search queries a lot faster on larger servers, by one or two orders of magnitude. They take up around 3 times as much space as GIN indexes. - -To enable them, both the `rum_enabled` flag has to be set and the following special migration has to be run: - -`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/` - -This will probably take a long time. - -## :rate_limit - -This is an advanced feature and disabled by default. - -A keyword list of rate limiters where a key is a limiter name and value is the limiter configuration. The basic configuration is a tuple where: - -* The first element: `scale` (Integer). The time scale in milliseconds. -* The second element: `limit` (Integer). How many requests to limit in the time scale provided. - -It is also possible to have different limits for unauthenticated and authenticated users: the keyword value must be a list of two tuples where the first one is a config for unauthenticated users and the second one is for authenticated. - -See [`Pleroma.Plugs.RateLimiter`](Pleroma.Plugs.RateLimiter.html) documentation for examples. - -Supported rate limiters: - -* `:search` for the search requests (account & status search etc.) -* `:app_account_creation` for registering user accounts from the same IP address -* `:relations_actions` for actions on relations with all users (follow, unfollow) -* `:relation_id_action` for actions on relation with a specific user (follow, unfollow) -* `:statuses_actions` for create / delete / fav / unfav / reblog / unreblog actions on any statuses -* `:status_id_action` for fav / unfav or reblog / unreblog actions on the same status by the same user - -## :web_cache_ttl - -The expiration time for the web responses cache. Values should be in milliseconds or `nil` to disable expiration. - -Available caches: - -* `:activity_pub` - activity pub routes (except question activities). Defaults to `nil` (no expiration). -* `:activity_pub_question` - activity pub routes (question activities). Defaults to `30_000` (30 seconds). From 171cefd88972c6ec1f37a2e014a9a484ae91ab9a Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 30 Aug 2019 13:20:09 +0300 Subject: [PATCH 037/138] description.exs --- config/description.exs | 2813 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 2813 insertions(+) create mode 100644 config/description.exs diff --git a/config/description.exs b/config/description.exs new file mode 100644 index 00000000..537b9d99 --- /dev/null +++ b/config/description.exs @@ -0,0 +1,2813 @@ +use Mix.Config +alias Pleroma.Docs.Formatter + +websocket_config = [ + path: "/websocket", + serializer: [ + {Phoenix.Socket.V1.JSONSerializer, "~> 1.0.0"}, + {Phoenix.Socket.V2.JSONSerializer, "~> 2.0.0"} + ], + timeout: 60_000, + transport_log: false, + compress: false +] + +config :pleroma, :config_description, [ + %{ + group: :pleroma, + key: Pleroma.Upload, + type: :group, + description: "Upload general settings", + children: [ + %{ + key: :uploader, + type: :module, + description: "Module which will be used for uploads", + suggestions: [ + Formatter.uploaders_list() + ] + }, + %{ + key: :filters, + type: {:list, :module}, + description: "List of filter modules for uploads", + suggestions: [ + Formatter.filters_list() + ] + }, + %{ + key: :link_name, + type: :boolean, + description: "If enabled Pleroma will add name parameter to the url off the upload", + suggestions: [ + true, + false + ] + }, + %{ + key: :base_url, + type: :string, + description: "Base url for the uploads, needed if you use CDN", + suggestions: [ + "https://cdn-host.com" + ] + }, + %{ + key: :proxy_remote, + type: :boolean, + description: "If enabled, Pleroma will proxy media requests instead of redirecting to it", + suggestions: [ + true, + false + ] + }, + %{ + key: :proxy_opts, + type: :keyword, + description: "Proxy options, see `Pleroma.ReverseProxy` documentation", + suggestions: ["somehow created link to Pleroma.ReverseProxy options"] + } + ] + }, + %{ + group: :pleroma, + key: Pleroma.Uploaders.Local, + type: :group, + description: "Local uploader-related settings", + children: [ + %{ + key: :uploads, + type: :string, + description: "Path where user uploads will be saved", + suggestions: [ + "uploads" + ] + } + ] + }, + %{ + group: :pleroma, + key: Pleroma.Uploaders.S3, + type: :group, + description: "S3 uploader-related settings", + children: [ + %{ + key: :bucket, + type: :strings, + description: "S3 bucket", + suggestions: [ + "bucket" + ] + }, + %{ + key: :bucket_namespace, + type: :string, + description: "S3 bucket namespace", + suggestions: ["pleroma"] + }, + %{ + key: :public_endpoint, + type: :string, + description: "S3 endpoint", + suggestions: ["https://s3.amazonaws.com"] + }, + %{ + key: :truncated_namespace, + type: :string, + description: + "If you use S3 compatible service such as Digital Ocean Spaces or CDN, set folder name or \"\" etc." <> + " For example, when using CDN to S3 virtual host format, set \"\". At this time, write CNAME to CDN in public_endpoint.", + suggestions: [""] + } + ] + }, + %{ + group: :pleroma, + key: Pleroma.Upload.Filter.Mogrify, + type: :group, + description: "Uploads mogrify filter settings", + children: [ + %{ + key: :args, + type: [:string, {:list, :string}, {:list, :tuple}], + description: "List of actions for the mogrify command", + suggestions: [ + "strip", + ["strip", "auto-orient"], + [{"implode", "1"}], + ["strip", "auto-orient", {"implode", "1"}] + ] + } + ] + }, + %{ + group: :pleroma, + key: Pleroma.Upload.Filter.AnonymizeFilename, + type: :group, + description: "Filter replaces the filename of the upload", + children: [ + %{ + key: :text, + type: :string, + description: + "Text to replace filenames in links. If no setting, {random}.extension will be used. You can get the original" <> + " filename extension by using {extension}, for example custom-file-name.{extension}", + suggestions: [ + "custom-file-name.{extension}", + nil + ] + } + ] + }, + %{ + group: :pleroma, + key: Pleroma.Emails.Mailer, + type: :group, + description: "Mailer-related settings", + children: [ + %{ + key: :adapter, + type: :module, + description: + "One of the mail adapters listed in [Swoosh readme](https://github.com/swoosh/swoosh#adapters)," <> + " or Swoosh.Adapters.Local for in-memory mailbox", + suggestions: [ + Swoosh.Adapters.SMTP, + Swoosh.Adapters.Sendgrid, + Swoosh.Adapters.Sendmail, + Swoosh.Adapters.Mandrill, + Swoosh.Adapters.Mailgun, + Swoosh.Adapters.Mailjet, + Swoosh.Adapters.Postmark, + Swoosh.Adapters.SparkPost, + Swoosh.Adapters.AmazonSES, + Swoosh.Adapters.Dyn, + Swoosh.Adapters.SocketLabs, + Swoosh.Adapters.Gmail + ] + }, + %{ + key: :enabled, + type: :boolean, + description: "Allow/disallow send emails", + suggestions: [ + true, + false + ] + }, + %{ + group: {:subgroup, Swoosh.Adapters.SMTP}, + key: :relay, + type: :string, + description: "`Swoosh.Adapters.SMTP` adapter specific setting", + suggestions: ["smtp.gmail.com"] + }, + %{ + group: {:subgroup, Swoosh.Adapters.SMTP}, + key: :username, + type: :string, + description: "`Swoosh.Adapters.SMTP` adapter specific setting", + suggestions: ["pleroma"] + }, + %{ + group: {:subgroup, Swoosh.Adapters.SMTP}, + key: :password, + type: :string, + description: "`Swoosh.Adapters.SMTP` adapter specific setting", + suggestions: ["password"] + }, + %{ + group: {:subgroup, Swoosh.Adapters.SMTP}, + key: :ssl, + type: :boolean, + description: "`Swoosh.Adapters.SMTP` adapter specific setting", + suggestions: [true, false] + }, + %{ + group: {:subgroup, Swoosh.Adapters.SMTP}, + key: :tls, + type: :atom, + description: "`Swoosh.Adapters.SMTP` adapter specific setting", + suggestions: [:always, :never, :if_available] + }, + %{ + group: {:subgroup, Swoosh.Adapters.SMTP}, + key: :auth, + type: :atom, + description: "`Swoosh.Adapters.SMTP` adapter specific setting", + suggestions: [:always, :never, :if_available] + }, + %{ + group: {:subgroup, Swoosh.Adapters.SMTP}, + key: :port, + type: :integer, + description: "`Swoosh.Adapters.SMTP` adapter specific setting", + suggestions: [1025] + }, + %{ + group: {:subgroup, Swoosh.Adapters.SMTP}, + key: :retries, + type: :integer, + description: "`Swoosh.Adapters.SMTP` adapter specific setting", + suggestions: [5] + }, + %{ + group: {:subgroup, Swoosh.Adapters.SMTP}, + key: :no_mx_lookups, + type: :boolean, + description: "`Swoosh.Adapters.SMTP` adapter specific setting", + suggestions: [true, false] + }, + %{ + group: {:subgroup, Swoosh.Adapters.Sendgrid}, + key: :api_key, + type: :string, + description: "`Swoosh.Adapters.Sendgrid` adapter specific setting", + suggestions: ["my-api-key"] + }, + %{ + group: {:subgroup, Swoosh.Adapters.Sendmail}, + key: :cmd_path, + type: :string, + description: "`Swoosh.Adapters.Sendmail` adapter specific setting", + suggestions: ["/usr/bin/sendmail"] + }, + %{ + group: {:subgroup, Swoosh.Adapters.Sendmail}, + key: :cmd_args, + type: :string, + description: "`Swoosh.Adapters.Sendmail` adapter specific setting", + suggestions: ["-N delay,failure,success"] + }, + %{ + group: {:subgroup, Swoosh.Adapters.Sendmail}, + key: :qmail, + type: :boolean, + description: "`Swoosh.Adapters.Sendmail` adapter specific setting", + suggestions: [true, false] + }, + %{ + group: {:subgroup, Swoosh.Adapters.Mandrill}, + key: :api_key, + type: :string, + description: "`Swoosh.Adapters.Mandrill` adapter specific setting", + suggestions: ["my-api-key"] + }, + %{ + group: {:subgroup, Swoosh.Adapters.Mailgun}, + key: :api_key, + type: :string, + description: "`Swoosh.Adapters.Mailgun` adapter specific setting", + suggestions: ["my-api-key"] + }, + %{ + group: {:subgroup, Swoosh.Adapters.Mailgun}, + key: :domain, + type: :string, + description: "`Swoosh.Adapters.Mailgun` adapter specific setting", + suggestions: ["pleroma.com"] + }, + %{ + group: {:subgroup, Swoosh.Adapters.Mailjet}, + key: :api_key, + type: :string, + description: "`Swoosh.Adapters.Mailjet` adapter specific setting", + suggestions: ["my-api-key"] + }, + %{ + group: {:subgroup, Swoosh.Adapters.Mailjet}, + key: :secret, + type: :string, + description: "`Swoosh.Adapters.Mailjet` adapter specific setting", + suggestions: ["my-secret-key"] + }, + %{ + group: {:subgroup, Swoosh.Adapters.Postmark}, + key: :api_key, + type: :string, + description: "`Swoosh.Adapters.Postmark` adapter specific setting", + suggestions: ["my-api-key"] + }, + %{ + group: {:subgroup, Swoosh.Adapters.SparkPost}, + key: :api_key, + type: :string, + description: "`Swoosh.Adapters.SparkPost` adapter specific setting", + suggestions: ["my-api-key"] + }, + %{ + group: {:subgroup, Swoosh.Adapters.SparkPost}, + key: :endpoint, + type: :string, + description: "`Swoosh.Adapters.SparkPost` adapter specific setting", + suggestions: ["https://api.sparkpost.com/api/v1"] + }, + %{ + group: {:subgroup, Swoosh.Adapters.AmazonSES}, + key: :region, + type: {:string}, + description: "`Swoosh.Adapters.AmazonSES` adapter specific setting", + suggestions: ["us-east-1", "us-east-2"] + }, + %{ + group: {:subgroup, Swoosh.Adapters.AmazonSES}, + key: :access_key, + type: :string, + description: "`Swoosh.Adapters.AmazonSES` adapter specific setting", + suggestions: ["aws-access-key"] + }, + %{ + group: {:subgroup, Swoosh.Adapters.AmazonSES}, + key: :secret, + type: :string, + description: "`Swoosh.Adapters.AmazonSES` adapter specific setting", + suggestions: ["aws-secret-key"] + }, + %{ + group: {:subgroup, Swoosh.Adapters.Dyn}, + key: :api_key, + type: :string, + description: "`Swoosh.Adapters.Dyn` adapter specific setting", + suggestions: ["my-api-key"] + }, + %{ + group: {:subgroup, Swoosh.Adapters.SocketLabs}, + key: :server_id, + type: :string, + description: "`Swoosh.Adapters.SocketLabs` adapter specific setting", + suggestions: [""] + }, + %{ + group: {:subgroup, Swoosh.Adapters.SocketLabs}, + key: :api_key, + type: :string, + description: "`Swoosh.Adapters.SocketLabs` adapter specific setting", + suggestions: [""] + }, + %{ + group: {:subgroup, Swoosh.Adapters.Gmail}, + key: :access_token, + type: :string, + description: "`Swoosh.Adapters.Gmail` adapter specific setting", + suggestions: [""] + } + ] + }, + %{ + group: :pleroma, + key: :uri_schemes, + type: :group, + description: "URI schemes related settings", + children: [ + %{ + key: :valid_schemes, + type: {:list, :string}, + description: "List of the scheme part that is considered valid to be an URL", + suggestions: [ + [ + "https", + "http", + "dat", + "dweb", + "gopher", + "ipfs", + "ipns", + "irc", + "ircs", + "magnet", + "mailto", + "mumble", + "ssb", + "xmpp" + ] + ] + } + ] + }, + %{ + group: :pleroma, + key: :instance, + type: :group, + description: "Instance-related settings", + children: [ + %{ + key: :name, + type: :string, + description: "Name of the instance", + suggestions: [ + "Pleroma" + ] + }, + %{ + key: :email, + type: :string, + description: "Email used to reach an Administrator/Moderator of the instance", + suggestions: [ + "email@example.com" + ] + }, + %{ + key: :notify_email, + type: :string, + description: "Email used for notifications", + suggestions: [ + "notify@example.com" + ] + }, + %{ + key: :description, + type: :string, + description: "The instance's description, can be seen in nodeinfo and /api/v1/instance", + suggestions: [ + "Very cool instance" + ] + }, + %{ + key: :limit, + type: :integer, + description: "Posts character limit (CW/Subject included in the counter)", + suggestions: [ + 5_000 + ] + }, + %{ + key: :remote_limit, + type: :integer, + description: "Hard character limit beyond which remote posts will be dropped", + suggestions: [ + 100_000 + ] + }, + %{ + key: :upload_limit, + type: :integer, + description: "File size limit of uploads (except for avatar, background, banner)", + suggestions: [ + 16_000_000 + ] + }, + %{ + key: :avatar_upload_limit, + type: :integer, + description: "File size limit of user's profile avatars", + suggestions: [ + 2_000_000 + ] + }, + %{ + key: :background_upload_limit, + type: :integer, + description: "File size limit of user's profile backgrounds", + suggestions: [ + 4_000_000 + ] + }, + %{ + key: :banner_upload_limit, + type: :integer, + description: "File size limit of user's profile banners", + suggestions: [ + 4_000_000 + ] + }, + %{ + key: :poll_limits, + type: :map, + description: "A map with poll limits for local polls", + suggestions: [ + %{ + max_options: 20, + max_option_chars: 200, + min_expiration: 0, + max_expiration: 31_536_000 + } + ], + children: [ + %{ + key: :max_options, + type: :integer, + description: "Maximum number of options", + suggestions: [20] + }, + %{ + key: :max_option_chars, + type: :integer, + description: "Maximum number of characters per option", + suggestions: [200] + }, + %{ + key: :min_expiration, + type: :integer, + description: "Minimum expiration time (in seconds)", + suggestions: [0] + }, + %{ + key: :max_expiration, + type: :integer, + description: "Maximum expiration time (in seconds)", + suggestions: [3600] + } + ] + }, + %{ + key: :registrations_open, + type: :boolean, + description: "Enable registrations for anyone, invitations can be enabled when false", + suggestions: [ + true, + false + ] + }, + %{ + key: :invites_enabled, + type: :boolean, + description: "Enable user invitations for admins (depends on registrations_open: false)", + suggestions: [ + true, + false + ] + }, + %{ + key: :account_activation_required, + type: :boolean, + description: "Require users to confirm their emails before signing in", + suggestions: [ + true, + false + ] + }, + %{ + key: :federating, + type: :boolean, + description: "Enable federation with other instances", + suggestions: [ + true, + false + ] + }, + %{ + key: :federation_incoming_replies_max_depth, + type: :integer, + description: + "Max. depth of reply-to activities fetching on incoming federation, to prevent out-of-memory situations while" <> + " fetching very long threads. If set to nil, threads of any depth will be fetched. Lower this value if you experience out-of-memory crashes", + suggestions: [ + 100 + ] + }, + %{ + key: :federation_reachability_timeout_days, + type: :integer, + description: + "Timeout (in days) of each external federation target being unreachable prior to pausing federating to it", + suggestions: [ + 7 + ] + }, + %{ + key: :federation_publisher_modules, + type: [:list, :module], + description: "List of modules for federation publishing", + suggestions: [ + Pleroma.Web.ActivityPub.Publisher, + Pleroma.Web.Websub, + Pleroma.Web.Salmo + ] + }, + %{ + key: :allow_relay, + type: :boolean, + description: "Enable Pleroma's Relay, which makes it possible to follow a whole instance", + suggestions: [ + true, + false + ] + }, + %{ + key: :rewrite_policy, + type: {:list, :module}, + description: "A list of MRF policies enabled", + suggestions: [ + Pleroma.Web.ActivityPub.MRF.NoOpPolicy, + Formatter.mrf_list() + ] + }, + %{ + key: :public, + type: :boolean, + description: + "Makes the client API in authentificated mode-only except for user-profiles." <> + " Useful for disabling the Local Timeline and The Whole Known Network", + suggestions: [ + true, + false + ] + }, + %{ + key: :quarantined_instances, + type: {:list, :string}, + description: + "List of ActivityPub instances where private(DMs, followers-only) activities will not be send", + suggestions: [ + "quarantined.com", + "*.quarantined.com" + ] + }, + %{ + key: :managed_config, + type: :boolean, + description: + "Whenether the config for pleroma-fe is configured in this config or in static/config.json", + suggestions: [ + true, + false + ] + }, + %{ + key: :static_dir, + type: :string, + description: "Instance static directory", + suggestions: [ + "instance/static/" + ] + }, + %{ + key: :allowed_post_formats, + type: {:list, :string}, + description: "MIME-type list of formats allowed to be posted (transformed into HTML)", + suggestions: [ + [ + "text/plain", + "text/html", + "text/markdown", + "text/bbcode" + ] + ] + }, + %{ + key: :mrf_transparency, + type: :boolean, + description: + "Make the content of your Message Rewrite Facility settings public (via nodeinfo)", + suggestions: [ + true, + false + ] + }, + %{ + key: :mrf_transparency_exclusions, + type: {:list, :string}, + description: + "Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value", + suggestions: [ + ["exclusion.com"] + ] + }, + %{ + key: :extended_nickname_format, + type: :boolean, + description: + "Set to true to use extended local nicknames format (allows underscores/dashes)." <> + " This will break federation with older software for theses nicknames", + suggestions: [ + true, + false + ] + }, + %{ + key: :max_pinned_statuses, + type: :integer, + description: "The maximum number of pinned statuses. 0 will disable the feature", + suggestions: [ + 0, + 1, + 3 + ] + }, + %{ + key: :autofollowed_nicknames, + type: {:list, :string}, + description: + "Set to nicknames of (local) users that every new user should automatically follow", + suggestions: [ + "lain", + "kaniini", + "lanodan", + "rinpatch" + ] + }, + %{ + key: :no_attachment_links, + type: :boolean, + description: + "Set to true to disable automatically adding attachment link text to statuses", + suggestions: [ + true, + false + ] + }, + %{ + key: :welcome_message, + type: :string, + description: + "A message that will be send to a newly registered users as a direct message", + suggestions: [ + "Hi, @username! Welcome to the board!", + nil + ] + }, + %{ + key: :welcome_user_nickname, + type: :string, + description: "The nickname of the local user that sends the welcome message", + suggestions: [ + "lain", + nil + ] + }, + %{ + key: :max_report_comment_size, + type: :integer, + description: "The maximum size of the report comment (Default: 1000)", + suggestions: [ + 1_000 + ] + }, + %{ + key: :safe_dm_mentions, + type: :boolean, + description: + "If set to true, only mentions at the beginning of a post will be used to address people in direct messages." <> + " This is to prevent accidental mentioning of people when talking about them (e.g. \"@friend hey i really don't like @enemy\")." <> + " Default: false", + suggestions: [ + true, + false + ] + }, + %{ + key: :healthcheck, + type: :boolean, + description: "If set to true, system data will be shown on /api/pleroma/healthcheck", + suggestions: [ + true, + false + ] + }, + %{ + key: :remote_post_retention_days, + type: :integer, + description: + "The default amount of days to retain remote posts when pruning the database", + suggestions: [ + 90 + ] + }, + %{ + key: :user_bio_length, + type: :integer, + description: "A user bio maximum length (default: 5000)", + suggestions: [ + 5_000 + ] + }, + %{ + key: :user_name_length, + type: :integer, + description: "A user name maximum length (default: 100)", + suggestions: [ + 100 + ] + }, + %{ + key: :skip_thread_containment, + type: :boolean, + description: "Skip filter out broken threads. The default is true", + suggestions: [ + true, + false + ] + }, + %{ + key: :limit_to_local_content, + type: [:atom, false], + description: + "Limit unauthenticated users to search for local statutes and users only. The default is :unauthenticated ", + suggestions: [ + :unauthenticated, + :all, + false + ] + }, + %{ + key: :dynamic_configuration, + type: :boolean, + description: + "Allow transferring configuration to DB with the subsequent customization from Admin api. Defaults to `false`", + suggestions: [ + true, + false + ] + }, + %{ + key: :max_account_fields, + type: :integer, + description: "The maximum number of custom fields in the user profile (default: 10)", + suggestions: [ + 10 + ] + }, + %{ + key: :max_remote_account_fields, + type: :integer, + description: + "The maximum number of custom fields in the remote user profile (default: 20)", + suggestions: [ + 20 + ] + }, + %{ + key: :account_field_name_length, + type: :integer, + description: "An account field name maximum length (default: 512)", + suggestions: [ + 512 + ] + }, + %{ + key: :account_field_value_length, + type: :integer, + description: "An account field value maximum length (default: 512)", + suggestions: [ + 512 + ] + }, + %{ + key: :external_user_synchronization, + type: :boolean, + description: "Enabling following/followers counters synchronization for external users", + suggestions: [ + true, + false + ] + } + ] + }, + %{ + group: :logger, + type: :group, + description: "Logger-related settings", + children: [ + %{ + key: :backends, + type: [:atom, :tuple, :module], + description: + "Where logs will be send, :console - send logs to stdout, {ExSyslogger, :ex_syslogger} - to syslog, Quack.Logger - to Slack.", + suggestions: [[:console, {ExSyslogger, :ex_syslogger}, Quack.Logger]] + } + ] + }, + %{ + group: :logger, + type: :group, + key: :ex_syslogger, + description: "ExSyslogger-related settings", + children: [ + %{ + key: :level, + type: :atom, + description: "Log level", + suggestions: [:debug, :info, :warn, :error] + }, + %{ + key: :ident, + type: :string, + description: + "A string that's prepended to every message, and is typically set to the app name", + suggestions: ["pleroma"] + }, + %{ + key: :format, + type: :string, + description: "It defaults to \"$date $time [$level] $levelpad$node $metadata $message\"", + suggestions: ["$metadata[$level] $message"] + }, + %{ + key: :metadata, + type: {:list, :atom}, + description: "", + suggestions: [[:request_id]] + } + ] + }, + %{ + group: :logger, + type: :group, + key: :console, + description: "Console logger settings", + children: [ + %{ + key: :level, + type: :atom, + description: "Log level", + suggestions: [:debug, :info, :warn, :error] + }, + %{ + key: :format, + type: :string, + description: "It defaults to \"$date $time [$level] $levelpad$node $metadata $message\"", + suggestions: ["$metadata[$level] $message"] + }, + %{ + key: :metadata, + type: {:list, :atom}, + description: "", + suggestions: [[:request_id]] + } + ] + }, + %{ + group: :quack, + type: :group, + description: "Quack-related settings", + children: [ + %{ + key: :level, + type: :atom, + description: "Log level", + suggestions: [:debug, :info, :warn, :error] + }, + %{ + key: :meta, + type: {:list, :atom}, + description: "Configure which metadata you want to report on", + suggestions: [ + :application, + :module, + :file, + :function, + :line, + :pid, + :crash_reason, + :initial_call, + :registered_name, + :all, + :none + ] + }, + %{ + key: :webhook_url, + type: :string, + description: "Configure the Slack incoming webhook", + suggestions: ["https://hooks.slack.com/services/YOUR-KEY-HERE"] + } + ] + }, + %{ + group: :pleroma, + key: :frontend_configurations, + type: :group, + description: "A keyword list that keeps the configuration data for any kind of frontend", + children: [ + %{ + key: :pleroma_fe, + type: :map, + description: "Settings for Pleroma FE", + suggestions: [ + %{ + theme: "pleroma-dark", + logo: "/static/logo.png", + background: "/images/city.jpg", + redirectRootNoLogin: "/main/all", + redirectRootLogin: "/main/friends", + showInstanceSpecificPanel: true, + scopeOptionsEnabled: false, + formattingOptionsEnabled: false, + collapseMessageWithSubject: false, + hidePostStats: false, + hideUserStats: false, + scopeCopy: true, + subjectLineBehavior: "email", + alwaysShowSubjectInput: true + } + ], + children: [ + %{ + key: :theme, + type: :string, + description: "Which theme to use, they are defined in styles.json", + suggestions: ["pleroma-dark"] + }, + %{ + key: :logo, + type: :string, + description: "URL of the logo, defaults to Pleroma's logo", + suggestions: ["/static/logo.png"] + }, + %{ + key: :background, + type: :string, + description: + "URL of the background, unless viewing a user profile with a background that is set", + suggestions: ["/images/city.jpg"] + }, + %{ + key: :redirectRootNoLogin, + type: :string, + description: + "relative URL which indicates where to redirect when a user isn't logged in", + suggestions: ["/main/all"] + }, + %{ + key: :redirectRootLogin, + type: :string, + description: + "relative URL which indicates where to redirect when a user is logged in", + suggestions: ["/main/friends"] + }, + %{ + key: :showInstanceSpecificPanel, + type: :boolean, + description: "Whenether to show the instance's specific panel", + suggestions: [true, false] + }, + %{ + key: :scopeOptionsEnabled, + type: :boolean, + description: "Enable setting an notice visibility and subject/CW when posting", + suggestions: [true, false] + }, + %{ + key: :formattingOptionsEnabled, + type: :boolean, + description: + "Enable setting a formatting different than plain-text (ie. HTML, Markdown) when posting, relates to :instance, allowed_post_formats", + suggestions: [true, false] + }, + %{ + key: :collapseMessageWithSubject, + type: :boolean, + description: + "When a message has a subject(aka Content Warning), collapse it by default", + suggestions: [true, false] + }, + %{ + key: :hidePostStats, + type: :boolean, + description: "Hide notices statistics(repeats, favorites, ...)", + suggestions: [true, false] + }, + %{ + key: :hideUserStats, + type: :boolean, + description: + "Hide profile statistics(posts, posts per day, followers, followings, ...)", + suggestions: [true, false] + }, + %{ + key: :scopeCopy, + type: :boolean, + description: + "Copy the scope (private/unlisted/public) in replies to posts by default", + suggestions: [true, false] + }, + %{ + key: :subjectLineBehavior, + type: :string, + description: "Allows changing the default behaviour of subject lines in replies. + `email`: Copy and preprend re:, as in email, + `masto`: Copy verbatim, as in Mastodon, + `noop`: Don't copy the subjec", + suggestions: ["email", "masto", "noop"] + }, + %{ + key: :alwaysShowSubjectInput, + type: :boolean, + description: "When set to false, auto-hide the subject field when it's empty", + suggestions: [true, false] + } + ] + }, + %{ + key: :masto_fe, + type: :map, + description: "Settings for Masto FE", + suggestions: [ + %{ + showInstanceSpecificPanel: true + } + ], + children: [ + %{ + key: :showInstanceSpecificPanel, + type: :boolean, + description: "Whenether to show the instance's specific panel", + suggestions: [true, false] + } + ] + } + ] + }, + %{ + group: :pleroma, + key: :assets, + type: :group, + description: + "This section configures assets to be used with various frontends. Currently the only option relates to mascots on the mastodon frontend", + children: [ + %{ + key: :mascots, + type: :keyword, + description: + "Keyword of mascots, each element MUST contain both a url and a mime_type key", + suggestions: [ + [ + pleroma_fox_tan: %{ + url: "/images/pleroma-fox-tan-smol.png", + mime_type: "image/png" + }, + pleroma_fox_tan_shy: %{ + url: "/images/pleroma-fox-tan-shy.png", + mime_type: "image/png" + } + ] + ] + }, + %{ + key: :default_mascot, + type: :atom, + description: + "This will be used as the default mascot on MastoFE (default: :pleroma_fox_tan)", + suggestions: [ + :pleroma_fox_tan + ] + } + ] + }, + %{ + group: :pleroma, + key: :mrf_simple, + type: :group, + description: "Message Rewrite Facility", + children: [ + %{ + key: :media_removal, + type: {:list, :string}, + description: "List of instances to remove medias from", + suggestions: ["example.com", "*.example.com"] + }, + %{ + key: :media_nsfw, + type: {:list, :string}, + description: "List of instances to put medias as NSFW(sensitive) from", + suggestions: ["example.com", "*.example.com"] + }, + %{ + key: :federated_timeline_removal, + type: {:list, :string}, + description: + "List of instances to remove from Federated (aka The Whole Known Network) Timeline", + suggestions: ["example.com", "*.example.com"] + }, + %{ + key: :reject, + type: {:list, :string}, + description: "List of instances to reject any activities from", + suggestions: ["example.com", "*.example.com"] + }, + %{ + key: :accept, + type: {:list, :string}, + description: "List of instances to accept any activities from", + suggestions: ["example.com", "*.example.com"] + }, + %{ + key: :report_removal, + type: {:list, :string}, + description: "List of instances to reject reports from", + suggestions: ["example.com", "*.example.com"] + }, + %{ + key: :avatar_removal, + type: {:list, :string}, + description: "List of instances to strip avatars from", + suggestions: ["example.com", "*.example.com"] + }, + %{ + key: :banner_removal, + type: {:list, :string}, + description: "List of instances to strip banners from", + suggestions: ["example.com", "*.example.com"] + } + ] + }, + %{ + group: :pleroma, + key: :mrf_subchain, + type: :group, + description: + "This policy processes messages through an alternate pipeline when a given message matches certain criteria." <> + " All criteria are configured as a map of regular expressions to lists of policy modules.", + children: [ + %{ + key: :match_actor, + type: :map, + description: "Matches a series of regular expressions against the actor field", + suggestions: [ + %{ + ~r/https:\/\/example.com/s => [Pleroma.Web.ActivityPub.MRF.DropPolicy] + } + ] + } + ] + }, + %{ + group: :pleroma, + key: :mrf_rejectnonpublic, + type: :group, + description: "", + children: [ + %{ + key: :allow_followersonly, + type: :boolean, + description: "whether to allow followers-only posts", + suggestions: [true, false] + }, + %{ + key: :allow_direct, + type: :boolean, + description: "whether to allow direct messages", + suggestions: [true, false] + } + ] + }, + %{ + group: :pleroma, + key: :mrf_hellthread, + type: :group, + description: "Block messages with too much mentions", + children: [ + %{ + key: :delist_threshold, + type: :integer, + description: + "Number of mentioned users after which the message gets delisted (the message can still be seen, " <> + " but it will not show up in public timelines and mentioned users won't get notifications about it). Set to 0 to disable", + suggestions: [10] + }, + %{ + key: :reject_threshold, + type: :integer, + description: + "Number of mentioned users after which the messaged gets rejected. Set to 0 to disable", + suggestions: [20] + } + ] + }, + %{ + group: :pleroma, + key: :mrf_keyword, + type: :group, + description: "Reject or Word-Replace messages with a keyword or regex", + children: [ + %{ + key: :reject, + type: [:string, :regex], + description: + "A list of patterns which result in message being rejected, each pattern can be a string or a regular expression", + suggestions: ["foo", ~r/foo/iu] + }, + %{ + key: :federated_timeline_removal, + type: [:string, :regex], + description: + "A list of patterns which result in message being removed from federated timelines (a.k.a unlisted), each pattern can be a string or a regular expression", + suggestions: ["foo", ~r/foo/iu] + }, + %{ + key: :replace, + type: [{:string, :string}, {:regex, :string}], + description: + "A list of patterns which result in message being removed from federated timelines (a.k.a unlisted), each pattern can be a string or a regular expression", + suggestions: [{"foo", "bar"}, {~r/foo/iu, "bar"}] + } + ] + }, + %{ + group: :pleroma, + key: :mrf_mention, + type: :group, + description: "Block messages which mention a user", + children: [ + %{ + key: :actors, + type: {:list, :string}, + description: "A list of actors, for which to drop any posts mentioning", + suggestions: [["actor1", "actor2"]] + } + ] + }, + %{ + group: :pleroma, + key: :mrf_vocabulary, + type: :group, + description: "Filter messages which belong to certain activity vocabularies", + children: [ + %{ + key: :accept, + type: {:list, :string}, + description: + "A list of ActivityStreams terms to accept. If empty, all supported messages are accepted", + suggestions: [["Create", "Follow", "Mention", "Announce", "Like"]] + }, + %{ + key: :reject, + type: {:list, :string}, + description: + "A list of ActivityStreams terms to reject. If empty, no messages are rejected", + suggestions: [["Create", "Follow", "Mention", "Announce", "Like"]] + } + ] + }, + # %{ + # group: :pleroma, + # key: :mrf_user_allowlist, + # type: :group, + # description: + # "The keys in this section are the domain names that the policy should apply to." <> + # " Each key should be assigned a list of users that should be allowed through by their ActivityPub ID", + # children: [ + # ["example.org": ["https://example.org/users/admin"]], + # suggestions: [ + # ["example.org": ["https://example.org/users/admin"]] + # ] + # ] + # }, + %{ + group: :pleroma, + key: :media_proxy, + type: :group, + description: "Media proxy", + children: [ + %{ + key: :enabled, + type: :boolean, + description: "Enables proxying of remote media to the instance's proxy", + suggestions: [true, false] + }, + %{ + key: :base_url, + type: :string, + description: + "The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts", + suggestions: ["https://example.com"] + }, + %{ + key: :proxy_opts, + type: :keyword, + description: "Options for Pleroma.ReverseProxy", + suggestions: [[max_body_length: 25 * 1_048_576, redirect_on_failure: false]] + }, + %{ + key: :whitelist, + type: {:list, :string}, + description: "List of domains to bypass the mediaproxy", + suggestions: ["example.com"] + } + ] + }, + %{ + group: :pleroma, + key: :gopher, + type: :group, + description: "Gopher settings", + children: [ + %{ + key: :enabled, + type: :boolean, + description: "Enables the gopher interface", + suggestions: [true, false] + }, + %{ + key: :ip, + type: :tuple, + description: "IP address to bind to", + suggestions: [{0, 0, 0, 0}] + }, + %{ + key: :port, + type: :integer, + description: "Port to bind to", + suggestions: [9999] + }, + %{ + key: :dstport, + type: :integer, + description: "Port advertised in urls (optional, defaults to port)", + suggestions: [9999] + } + ] + }, + %{ + group: :pleroma, + key: Pleroma.Web.Endpoint, + type: :group, + description: "Phoenix endpoint configuration", + children: [ + %{ + key: :http, + type: :keyword, + description: "http protocol configuration", + suggestions: [ + [port: 8080, ip: {127, 0, 0, 1}] + ], + children: [ + %{ + key: :dispatch, + type: {:list, :tuple}, + description: "dispatch settings", + suggestions: [ + [ + {:_, + [ + {"/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, []}, + {"/websocket", Phoenix.Endpoint.CowboyWebSocket, + {Phoenix.Transports.WebSocket, + {Pleroma.Web.Endpoint, Pleroma.Web.UserSocket, websocket_config}}}, + {:_, Phoenix.Endpoint.Cowboy2Handler, {Pleroma.Web.Endpoint, []}} + ]} + # end copied from config.exs + ] + ] + }, + %{ + key: :ip, + type: :tuple, + description: "ip", + suggestions: [ + {0, 0, 0, 0} + ] + }, + %{ + key: :port, + type: :integer, + description: "port", + suggestions: [ + 2020 + ] + } + ] + }, + %{ + key: :url, + type: :keyword, + description: "configuration for generating urls", + suggestions: [ + [host: "example.com", port: 2020, scheme: "https"] + ], + children: [ + %{ + key: :host, + type: :string, + description: "Host", + suggestions: [ + "example.com" + ] + }, + %{ + key: :port, + type: :integer, + description: "port", + suggestions: [ + 2020 + ] + }, + %{ + key: :scheme, + type: :string, + description: "Scheme", + suggestions: [ + "https", + "https" + ] + } + ] + }, + %{ + key: :instrumenters, + type: {:list, :module}, + description: "", + suggestions: [Pleroma.Web.Endpoint.Instrumenter] + }, + %{ + key: :protocol, + type: :string, + description: "", + suggestions: ["https"] + }, + %{ + key: :secret_key_base, + type: :string, + description: "", + suggestions: ["aK4Abxf29xU9TTDKre9coZPUgevcVCFQJe/5xP/7Lt4BEif6idBIbjupVbOrbKxl"] + }, + %{ + key: :signing_salt, + type: :string, + description: "", + suggestions: ["CqaoopA2"] + }, + %{ + key: :render_errors, + type: :keyword, + description: "", + suggestions: [[view: Pleroma.Web.ErrorView, accepts: ~w(json)]], + children: [ + %{ + key: :view, + type: :module, + description: "", + suggestions: [Pleroma.Web.ErrorView] + }, + %{ + key: :accepts, + type: {:list, :string}, + description: "", + suggestions: ["json"] + } + ] + }, + %{ + key: :pubsub, + type: :keyword, + description: "", + suggestions: [[name: Pleroma.PubSub, adapter: Phoenix.PubSub.PG2]], + children: [ + %{ + key: :name, + type: :module, + description: "", + suggestions: [Pleroma.PubSub] + }, + %{ + key: :adapter, + type: :module, + description: "", + suggestions: [Phoenix.PubSub.PG2] + } + ] + }, + %{ + key: :secure_cookie_flag, + type: :boolean, + description: "", + suggestions: [true, false] + }, + %{ + key: :extra_cookie_attrs, + type: {:list, :string}, + description: "", + suggestions: ["SameSite=Lax"] + } + ] + }, + %{ + group: :pleroma, + key: :activitypub, + type: :group, + description: "ActivityPub-related settings", + children: [ + %{ + key: :unfollow_blocked, + type: :boolean, + description: "Whether blocks result in people getting unfollowed", + suggestions: [true, false] + }, + %{ + key: :outgoing_blocks, + type: :boolean, + description: "Whether to federate blocks to other instances", + suggestions: [true, false] + }, + %{ + key: :sign_object_fetches, + type: :boolean, + description: "Sign object fetches with HTTP signatures", + suggestions: [true, false] + }, + %{ + key: :follow_handshake_timeout, + type: :integer, + description: "Following handshake timeout", + suggestions: [500] + } + ] + }, + %{ + group: :pleroma, + key: :http_security, + type: :group, + description: "HTTP security settings", + children: [ + %{ + key: :enabled, + type: :boolean, + description: "Whether the managed content security policy is enabled", + suggestions: [true, false] + }, + %{ + key: :sts, + type: :boolean, + description: "Whether to additionally send a Strict-Transport-Security header", + suggestions: [true, false] + }, + %{ + key: :sts_max_age, + type: :integer, + description: "The maximum age for the Strict-Transport-Security header if sent", + suggestions: [31_536_000] + }, + %{ + key: :ct_max_age, + type: :integer, + description: "The maximum age for the Expect-CT header if sent", + suggestions: [2_592_000] + }, + %{ + key: :referrer_policy, + type: :string, + description: "The referrer policy to use, either \"same-origin\" or \"no-referrer\"", + suggestions: ["same-origin", "no-referrer"] + }, + %{ + key: :report_uri, + type: :string, + description: "Adds the specified url to report-uri and report-to group in CSP header", + suggestions: ["https://example.com/report-uri"] + } + ] + }, + %{ + group: :web_push_encryption, + key: :vapid_details, + type: :group, + description: + "Web Push Notifications configuration. You can use the mix task mix web_push.gen.keypair to generate it", + children: [ + %{ + key: :subject, + type: :string, + description: + "a mailto link for the administrative contact." <> + " It's best if this email is not a personal email address, but rather a group email so that if a person leaves an organization," <> + " is unavailable for an extended period, or otherwise can't respond, someone else on the list can", + suggestions: ["Subject"] + }, + %{ + key: :public_key, + type: :string, + description: "VAPID public key", + suggestions: ["Public key"] + }, + %{ + key: :private_key, + type: :string, + description: "VAPID private keyn", + suggestions: ["Private key"] + } + ] + }, + %{ + group: :pleroma, + key: Pleroma.Captcha, + type: :group, + description: "Captcha-related settings", + children: [ + %{ + key: :enabled, + type: :boolean, + description: "Whether the captcha should be shown on registration", + suggestions: [true, false] + }, + %{ + key: :method, + type: :module, + description: "The method/service to use for captcha", + suggestions: [Pleroma.Captcha.Kocaptcha] + }, + %{ + key: :seconds_valid, + type: :integer, + description: "The time in seconds for which the captcha is valid", + suggestions: [60] + } + ] + }, + %{ + group: :pleroma, + key: Pleroma.Captcha.Kocaptcha, + type: :group, + description: + "Kocaptcha is a very simple captcha service with a single API endpoint, the source code is" <> + " here: https://github.com/koto-bank/kocaptcha. The default endpoint https://captcha.kotobank.ch is hosted by the developer", + children: [ + %{ + key: :endpoint, + type: :string, + description: "the kocaptcha endpoint to use", + suggestions: ["https://captcha.kotobank.ch"] + } + ] + }, + %{ + group: :pleroma, + type: :group, + description: + "Allows to set a token that can be used to authenticate with the admin api without using an actual user by giving it as the 'admin_token' parameter", + children: [ + %{ + key: :admin_token, + type: :string, + description: "Token", + suggestions: ["some_random_token"] + } + ] + }, + %{ + group: :pleroma_job_queue, + key: :queues, + type: :group, + description: "Pleroma Job Queue configuration: a list of queues with maximum concurrent jobs", + children: [ + %{ + key: :federator_outgoing, + type: :integer, + description: "Outgoing federation queue", + suggestions: [50] + }, + %{ + key: :federator_incoming, + type: :integer, + description: "Incoming federation queue", + suggestions: [50] + }, + %{ + key: :mailer, + type: :integer, + description: "Email sender queue, see Pleroma.Emails.Mailer", + suggestions: [10] + }, + %{ + key: :web_push, + type: :integer, + description: "Web push notifications queue", + suggestions: [50] + }, + %{ + key: :transmogrifier, + type: :integer, + description: "Transmogrifier queue", + suggestions: [20] + }, + %{ + key: :scheduled_activities, + type: :integer, + description: "Scheduled activities queue, see Pleroma.ScheduledActivities", + suggestions: [10] + }, + %{ + key: :activity_expiration, + type: :integer, + description: "Activity expiration queue", + suggestions: [10] + }, + %{ + key: :background, + type: :integer, + description: "Background queue", + suggestions: [5] + } + ] + }, + %{ + group: :pleroma, + key: Pleroma.Web.Federator.RetryQueue, + type: :group, + description: "", + children: [ + %{ + key: :enabled, + type: :boolean, + description: "If set to true, failed federation jobs will be retried", + suggestions: [true, false] + }, + %{ + key: :max_jobs, + type: :integer, + description: "The maximum amount of parallel federation jobs running at the same time", + suggestions: [20] + }, + %{ + key: :initial_timeout, + type: :integer, + description: "The initial timeout in seconds", + suggestions: [30] + }, + %{ + key: :max_retries, + type: :integer, + description: "The maximum number of times a federation job is retried", + suggestions: [5] + } + ] + }, + %{ + group: :pleroma, + key: Pleroma.Web.Metadata, + type: :group, + decsription: "Metadata-related settings", + children: [ + %{ + key: :providers, + type: {:list, :module}, + description: "List of metadata providers to enable", + suggestions: [ + [ + Pleroma.Web.Metadata.Providers.OpenGraph, + Pleroma.Web.Metadata.Providers.TwitterCard, + Pleroma.Web.Metadata.Providers.RelMe + ] + ] + }, + %{ + key: :unfurl_nsfw, + type: :boolean, + description: "If set to true nsfw attachments will be shown in previews", + suggestions: [ + true, + false + ] + } + ] + }, + %{ + group: :pleroma, + key: :rich_media, + type: :group, + description: "", + children: [ + %{ + key: :enabled, + type: :boolean, + description: + "if enabled the instance will parse metadata from attached links to generate link previews", + suggestions: [true, false] + }, + %{ + key: :ignore_hosts, + type: {:list, :string}, + description: "list of hosts which will be ignored by the metadata parser", + suggestions: [["accounts.google.com", "xss.website"]] + }, + %{ + key: :ignore_tld, + type: {:list, :string}, + description: "list TLDs (top-level domains) which will ignore for parse metadata", + suggestions: [["local", "localdomain", "lan"]] + }, + %{ + key: :parsers, + type: {:list, :module}, + description: "list of Rich Media parsers", + suggestions: [ + Formatter.richmedia_parsers() + ] + }, + %{ + key: :ttl_setters, + type: {:list, :module}, + description: "list of rich media ttl setters", + suggestions: [ + [Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl] + ] + } + ] + }, + %{ + group: :pleroma, + key: :fetch_initial_posts, + type: :group, + description: "Fetching initial posts settings", + children: [ + %{ + key: :enabled, + type: :boolean, + description: + "if enabled, when a new user is federated with, fetch some of their latest posts", + suggestions: [true, false] + }, + %{ + key: :pages, + type: :integer, + description: "the amount of pages to fetch", + suggestions: [5] + } + ] + }, + %{ + group: :auto_linker, + key: :opts, + type: :group, + description: "Configuration for the auto_linker library", + children: [ + %{ + key: :class, + type: [:string, false], + description: "specify the class to be added to the generated link. false to clear", + suggestions: ["auto-linker", false] + }, + %{ + key: :rel, + type: [:string, false], + description: "override the rel attribute. false to clear", + suggestions: ["noopener noreferrer", false] + }, + %{ + key: :new_window, + type: :boolean, + description: "set to false to remove target='_blank' attribute", + suggestions: [true, false] + }, + %{ + key: :scheme, + type: :boolean, + description: "Set to true to link urls with schema http://google.com", + suggestions: [true, false] + }, + %{ + key: :truncate, + type: [:integer, false], + description: + "Set to a number to truncate urls longer then the number. Truncated urls will end in `..`", + suggestions: [15, false] + }, + %{ + key: :strip_prefix, + type: :boolean, + description: "Strip the scheme prefix", + suggestions: [true, false] + }, + %{ + key: :extra, + type: :boolean, + description: "link urls with rarely used schemes (magnet, ipfs, irc, etc.)", + suggestions: [true, false] + } + ] + }, + %{ + group: :pleroma, + key: Pleroma.ScheduledActivity, + type: :group, + description: "Scheduled activities settings", + children: [ + %{ + key: :daily_user_limit, + type: :integer, + description: + "the number of scheduled activities a user is allowed to create in a single day (Default: 25)", + suggestions: [25] + }, + %{ + key: :total_user_limit, + type: :integer, + description: + "the number of scheduled activities a user is allowed to create in total (Default: 300)", + suggestions: [300] + }, + %{ + key: :enabled, + type: :boolean, + description: "whether scheduled activities are sent to the job queue to be executed", + suggestions: [true, false] + } + ] + }, + %{ + group: :pleroma, + key: Pleroma.ActivityExpiration, + type: :group, + description: "Expired activity settings", + children: [ + %{ + key: :enabled, + type: :boolean, + description: "whether expired activities will be sent to the job queue to be deleted", + suggestions: [true, false] + } + ] + }, + %{ + group: :pleroma, + type: :group, + description: "Authenticator", + children: [ + %{ + key: Pleroma.Web.Auth.Authenticator, + type: :module, + description: "", + suggestions: [Pleroma.Web.Auth.PleromaAuthenticator, Pleroma.Web.Auth.LDAPAuthenticator] + } + ] + }, + %{ + group: :pleroma, + key: :ldap, + type: :group, + description: + "Use LDAP for user authentication. When a user logs in to the Pleroma instance, the name and password" <> + " will be verified by trying to authenticate (bind) to an LDAP server." <> + " If a user exists in the LDAP directory but there is no account with the same name yet on the" <> + " Pleroma instance then a new Pleroma account will be created with the same name as the LDAP user name.", + children: [ + %{ + key: :enabled, + type: :boolean, + description: "enables LDAP authentication", + suggestions: [true, false] + }, + %{ + key: :host, + type: :string, + description: "LDAP server hostname", + suggestions: ["localhosts"] + }, + %{ + key: :port, + type: :integer, + description: "LDAP port, e.g. 389 or 636", + suggestions: [389, 636] + }, + %{ + key: :ssl, + type: :boolean, + description: "true to use SSL, usually implies the port 636", + suggestions: [true, false] + }, + %{ + key: :sslopts, + type: :keyword, + description: "additional SSL options", + suggestions: [] + }, + %{ + key: :tls, + type: :boolean, + description: "true to start TLS, usually implies the port 389", + suggestions: [true, false] + }, + %{ + key: :tlsopts, + type: :keyword, + description: "additional TLS options", + suggestions: [] + }, + %{ + key: :base, + type: :string, + description: "LDAP base, e.g. \"dc=example,dc=com\"", + suggestions: ["dc=example,dc=com"] + }, + %{ + key: :uid, + type: :string, + description: + "LDAP attribute name to authenticate the user, e.g. when \"cn\", the filter will be \"cn=username,base\"", + suggestions: ["cn"] + } + ] + }, + %{ + group: :pleroma, + key: :auth, + type: :group, + description: "Authentication / authorization settings", + children: [ + %{ + key: :auth_template, + type: :string, + description: + "authentication form template. By default it's show.html which corresponds to lib/pleroma/web/templates/o_auth/o_auth/show.html.ee", + suggestions: ["show.html"] + }, + %{ + key: :oauth_consumer_template, + type: :string, + description: + "OAuth consumer mode authentication form template. By default it's consumer.html which corresponds to" <> + " lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex", + suggestions: ["consumer.html"] + }, + %{ + key: :oauth_consumer_strategies, + type: :string, + description: + "the list of enabled OAuth consumer strategies; by default it's set by OAUTH_CONSUMER_STRATEGIES environment variable." <> + " Each entry in this space-delimited string should be of format or :" <> + " (e.g. twitter or keycloak:ueberauth_keycloak_strategy in case dependency is named differently than ueberauth_).", + suggestions: ["twitter", "keycloak:ueberauth_keycloak_strategy"] + } + ] + }, + %{ + group: :pleroma, + key: :email_notifications, + type: :group, + description: "Email notifications settings", + children: [ + %{ + key: :digest, + type: :map, + description: + "emails of \"what you've missed\" for users who have been inactive for a while", + suggestions: [ + %{ + active: false, + schedule: "0 0 * * 0", + interval: 7, + inactivity_threshold: 7 + } + ], + children: [ + %{ + key: :active, + type: :boolean, + description: "globally enable or disable digest emails", + suggestions: [true, false] + }, + %{ + key: :schedule, + type: :string, + description: + "When to send digest email, in crontab format. \"0 0 0\" is the default, meaning \"once a week at midnight on Sunday morning\"", + suggestions: ["0 0 * * 0"] + }, + %{ + key: :interval, + type: :ininteger, + description: "Minimum interval between digest emails to one user", + suggestions: [7] + }, + %{ + key: :inactivity_threshold, + type: :integer, + description: "Minimum user inactivity threshold", + suggestions: [7] + } + ] + } + ] + }, + %{ + group: :pleroma, + key: Pleroma.Emails.UserEmail, + type: :group, + description: "Email template settings", + children: [ + %{ + key: :logo, + # type: [:string, nil], + description: "a path to a custom logo. Set it to nil to use the default Pleroma logo", + suggestions: ["some/path/logo.png", nil] + }, + %{ + key: :styling, + type: :map, + description: "a map with color settings for email templates.", + suggestions: [ + %{ + link_color: "#d8a070", + background_color: "#2C3645", + content_background_color: "#1B2635", + header_color: "#d8a070", + text_color: "#b9b9ba", + text_muted_color: "#b9b9ba" + } + ], + children: [ + %{ + key: :link_color, + type: :string, + description: "", + suggestions: ["#d8a070"] + }, + %{ + key: :background_color, + type: :string, + description: "", + suggestions: ["#2C3645"] + }, + %{ + key: :content_background_color, + type: :string, + description: "", + suggestions: ["#1B2635"] + }, + %{ + key: :header_color, + type: :string, + description: "", + suggestions: ["#d8a070"] + }, + %{ + key: :text_color, + type: :string, + description: "", + suggestions: ["#b9b9ba"] + }, + %{ + key: :text_muted_color, + type: :string, + description: "", + suggestions: ["#b9b9ba"] + } + ] + } + ] + }, + %{ + group: :pleroma, + key: :oauth2, + type: :group, + description: "Configure OAuth 2 provider capabilities", + children: [ + %{ + key: :token_expires_in, + type: :integer, + description: "The lifetime in seconds of the access token", + suggestions: [600] + }, + %{ + key: :issue_new_refresh_token, + type: :boolean, + description: + "Keeps old refresh token or generate new refresh token when to obtain an access token", + suggestions: [true, false] + }, + %{ + key: :clean_expired_tokens, + type: :boolean, + description: "Enable a background job to clean expired oauth tokens. Defaults to false", + suggestions: [true, false] + }, + %{ + key: :clean_expired_tokens_interval, + type: :integer, + description: + "Interval to run the job to clean expired tokens. Defaults to 86_400_000 (24 hours).", + suggestions: [86_400_000] + } + ] + }, + %{ + group: :pleroma, + key: :emoji, + type: :group, + description: "", + children: [ + %{ + key: :shortcode_globs, + type: {:list, :string}, + description: "Location of custom emoji files. * can be used as a wildcard", + suggestions: [["/emoji/custom/**/*.png"]] + }, + %{ + key: :pack_extensions, + type: {:list, :string}, + description: + "A list of file extensions for emojis, when no emoji.txt for a pack is present", + suggestions: [[".png", ".gif"]] + }, + %{ + key: :groups, + type: :keyword, + description: + "Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname" <> + " and the value the location or array of locations. * can be used as a wildcard", + suggestions: [ + [ + # Put groups that have higher priority than defaults here. Example in `docs/config/custom_emoji.md` + Custom: ["/emoji/*.png", "/emoji/**/*.png"] + ] + ] + }, + %{ + key: :default_manifest, + type: :string, + description: + "Location of the JSON-manifest. This manifest contains information about the emoji-packs you can download." <> + " Currently only one manifest can be added (no arrays)", + suggestions: ["https://git.pleroma.social/pleroma/emoji-index/raw/master/index.json"] + } + ] + }, + %{ + group: :pleroma, + key: :database, + type: :group, + description: "Database related settings", + children: [ + %{ + key: :rum_enabled, + type: :boolean, + description: "If RUM indexes should be used. Defaults to false", + suggestions: [true, false] + } + ] + }, + %{ + group: :pleroma, + key: :rate_limit, + type: :group, + description: "Rate limit settings. This is an advanced feature and disabled by default.", + children: [ + %{ + key: :search, + type: [:tuple, {:list, :tuple}], + description: "for the search requests (account & status search etc.)", + suggestions: [{1000, 10}, [{10_000, 10}, {10_000, 50}]] + }, + %{ + key: :app_account_creation, + type: [:tuple, {:list, :tuple}], + description: "for registering user accounts from the same IP address", + suggestions: [{1000, 10}, [{10_000, 10}, {10_000, 50}]] + }, + %{ + key: :relations_actions, + type: [:tuple, {:list, :tuple}], + description: "for actions on relations with all users (follow, unfollow)", + suggestions: [{1000, 10}, [{10_000, 10}, {10_000, 50}]] + }, + %{ + key: :relation_id_action, + type: [:tuple, {:list, :tuple}], + description: "for actions on relation with a specific user (follow, unfollow)", + suggestions: [{1000, 10}, [{10_000, 10}, {10_000, 50}]] + }, + %{ + key: :statuses_actions, + type: [:tuple, {:list, :tuple}], + description: + "for create / delete / fav / unfav / reblog / unreblog actions on any statuses", + suggestions: [{1000, 10}, [{10_000, 10}, {10_000, 50}]] + }, + %{ + key: :status_id_action, + type: [:tuple, {:list, :tuple}], + description: + "for fav / unfav or reblog / unreblog actions on the same status by the same user", + suggestions: [{1000, 10}, [{10_000, 10}, {10_000, 50}]] + } + ] + }, + %{ + group: :esshd, + type: :group, + description: + "To enable simple command line interface accessible over ssh, add a setting like this to your configuration file", + children: [ + %{ + key: :enabled, + type: :boolean, + description: "Enables ssh", + suggestions: [true, false] + }, + %{ + key: :priv_dir, + type: :string, + description: "Dir with ssh keys", + suggestions: ["/some/path/ssh_keys"] + }, + %{ + key: :handler, + type: :string, + description: "Handler module", + suggestions: ["Pleroma.BBS.Handler"] + }, + %{ + key: :port, + type: :integer, + description: "Port to connect", + suggestions: [10_022] + }, + %{ + key: :password_authenticator, + type: :string, + description: "Authenticator module", + suggestions: ["Pleroma.BBS.Authenticator"] + } + ] + }, + %{ + group: :mime, + type: :group, + description: "Mime types", + children: [ + %{ + key: :types, + type: :map, + description: "", + suggestions: [ + %{ + "application/xml" => ["xml"], + "application/xrd+xml" => ["xrd+xml"], + "application/jrd+json" => ["jrd+json"], + "application/activity+json" => ["activity+json"], + "application/ld+json" => ["activity+json"] + } + ], + children: [ + %{ + key: "application/xml", + type: {:list, :string}, + description: "", + suggestions: [["xml"]] + }, + %{ + key: "application/xrd+xml", + type: {:list, :string}, + description: "", + suggestions: [["xrd+xml"]] + }, + %{ + key: "application/jrd+json", + type: {:list, :string}, + description: "", + suggestions: [["jrd+json"]] + }, + %{ + key: "application/activity+json", + type: {:list, :string}, + description: "", + suggestions: [["activity+json"]] + }, + %{ + key: "application/ld+json", + type: {:list, :string}, + description: "", + suggestions: [["activity+json"]] + } + ] + } + ] + }, + %{ + group: :tesla, + type: :group, + description: "Tesla settings", + children: [ + %{ + key: :adapter, + type: :module, + description: "Tesla adapter", + suggestions: [Tesla.Adapter.Hackney] + } + ] + }, + %{ + group: :pleroma, + key: :chat, + type: :group, + description: "Pleroma chat settings", + children: [ + %{ + key: :enabled, + type: :boolean, + description: "", + suggestions: [true, false] + } + ] + }, + %{ + group: :pleroma, + key: :suggestions, + type: :group, + description: "", + children: [ + %{ + key: :enabled, + type: :boolean, + description: "Enables suggestions", + suggestions: [] + }, + %{ + key: :third_party_engine, + type: :string, + description: "URL for third party engine", + suggestions: [ + "http://vinayaka.distsn.org/cgi-bin/vinayaka-user-match-suggestions-api.cgi?{{host}}+{{user}}" + ] + }, + %{ + key: :timeout, + type: :integer, + description: "Request timeout to third party engine", + suggestions: [300_000] + }, + %{ + key: :limit, + type: :integer, + description: "Limit for suggestions", + suggestions: [40] + }, + %{ + key: :web, + type: :string, + description: "", + suggestions: ["https://vinayaka.distsn.org"] + } + ] + }, + %{ + group: :prometheus, + key: Pleroma.Web.Endpoint.MetricsExporter, + type: :group, + description: "Prometheus settings", + children: [ + %{ + key: :path, + type: :string, + description: "API endpoint with metrics", + suggestions: ["/api/pleroma/app_metrics"] + } + ] + }, + %{ + group: :http_signatures, + type: :group, + description: "HTTP Signatures settings", + children: [ + %{ + key: :adapter, + type: :module, + description: "", + suggestions: [Pleroma.Signature] + } + ] + }, + %{ + group: :pleroma, + key: Pleroma.Uploaders.MDII, + type: :group, + description: "", + children: [ + %{ + key: :cgi, + type: :string, + description: "", + suggestions: ["https://mdii.sakura.ne.jp/mdii-post.cgi"] + }, + %{ + key: :files, + type: :string, + description: "", + suggestions: ["https://mdii.sakura.ne.jp"] + } + ] + }, + %{ + group: :pleroma, + key: :http, + type: :group, + description: "HTTP settings", + children: [ + %{ + key: :proxy_url, + type: [:string, :atom, nil], + description: "", + suggestions: ["localhost:9020", {:socks5, :localhost, 3090}, nil] + }, + %{ + key: :send_user_agent, + type: :boolean, + description: "", + suggestions: [true, false] + }, + %{ + key: :adapter, + type: :keyword, + description: "", + suggestions: [ + [ + ssl_options: [ + # Workaround for remote server certificate chain issues + partial_chain: &:hackney_connect.partial_chain/1, + # We don't support TLS v1.3 yet + versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"] + ] + ] + ] + } + ] + }, + %{ + group: :pleroma, + key: :markup, + type: :group, + description: "", + children: [ + %{ + key: :allow_inline_images, + type: :boolean, + description: "", + suggestions: [true, false] + }, + %{ + key: :allow_headings, + type: :boolean, + description: "", + suggestions: [true, false] + }, + %{ + key: :allow_tables, + type: :boolean, + description: "", + suggestions: [true, false] + }, + %{ + key: :allow_fonts, + type: :boolean, + description: "", + suggestions: [true, false] + }, + %{ + key: :scrub_policy, + type: {:list, :module}, + description: "", + suggestions: [[Pleroma.HTML.Transform.MediaProxy, Pleroma.HTML.Scrubber.Default]] + } + ] + }, + %{ + group: :pleroma, + key: :user, + type: :group, + description: "", + children: [ + %{ + key: :deny_follow_blocked, + type: :boolean, + description: "", + suggestions: [true, false] + } + ] + }, + %{ + group: :pleroma, + key: :mrf_normalize_markup, + type: :group, + description: "", + children: [ + %{ + key: :scrub_policy, + type: :module, + description: "", + suggestions: [Pleroma.HTML.Scrubber.Default] + } + ] + }, + %{ + group: :pleroma, + key: Pleroma.User, + type: :group, + description: "", + children: [ + %{ + key: :restricted_nicknames, + type: {:list, :string}, + description: "", + suggestions: [ + [ + ".well-known", + "~", + "about", + "activities", + "api", + "auth", + "check_password", + "dev", + "friend-requests", + "inbox", + "internal", + "main", + "media", + "nodeinfo", + "notice", + "oauth", + "objects", + "ostatus_subscribe", + "pleroma", + "proxy", + "push", + "registration", + "relay", + "settings", + "status", + "tag", + "user-search", + "user_exists", + "users", + "web" + ] + ] + } + ] + }, + %{ + group: :cors_plug, + type: :group, + description: "", + children: [ + %{ + key: :max_age, + type: :integer, + description: "", + suggestions: [86_400] + }, + %{ + key: :methods, + type: {:list, :string}, + description: "", + suggestions: [["POST", "PUT", "DELETE", "GET", "PATCH", "OPTIONS"]] + }, + %{ + key: :expose, + type: :string, + description: "", + suggestions: [ + [ + "Link", + "X-RateLimit-Reset", + "X-RateLimit-Limit", + "X-RateLimit-Remaining", + "X-Request-Id", + "Idempotency-Key" + ] + ] + }, + %{ + key: :credentials, + type: :boolean, + description: "", + suggestions: [true, false] + }, + %{ + key: :headers, + type: {:list, :string}, + description: "", + suggestions: [["Authorization", "Content-Type", "Idempotency-Key"]] + } + ] + } +] From 67e430093187c50f307810e88ed0e73afe825b75 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 30 Aug 2019 13:21:48 +0300 Subject: [PATCH 038/138] description formatters --- lib/pleroma/docs/formatter.ex | 69 +++++++++++++++++++++++++++++++++++ lib/pleroma/docs/json.ex | 15 ++++++++ lib/pleroma/docs/markdown.ex | 67 ++++++++++++++++++++++++++++++++++ 3 files changed, 151 insertions(+) create mode 100644 lib/pleroma/docs/formatter.ex create mode 100644 lib/pleroma/docs/json.ex create mode 100644 lib/pleroma/docs/markdown.ex diff --git a/lib/pleroma/docs/formatter.ex b/lib/pleroma/docs/formatter.ex new file mode 100644 index 00000000..a1c75793 --- /dev/null +++ b/lib/pleroma/docs/formatter.ex @@ -0,0 +1,69 @@ +defmodule Pleroma.Docs.Formatter do + @callback process(keyword()) :: {:ok, String.t()} + + @spec process(module(), keyword()) :: {:ok, String.t()} + def process(implementation, descriptions) do + implementation.process(descriptions) + end + + def uploaders_list do + {:ok, modules} = :application.get_key(:pleroma, :modules) + + Enum.filter(modules, fn module -> + name_as_list = Module.split(module) + + List.starts_with?(name_as_list, ["Pleroma", "Uploaders"]) and + List.last(name_as_list) in ["S3", "Local", "MDII"] + end) + end + + def filters_list do + {:ok, modules} = :application.get_key(:pleroma, :modules) + + Enum.filter(modules, fn module -> + name_as_list = Module.split(module) + + List.starts_with?(name_as_list, ["Pleroma", "Upload", "Filter"]) + end) + end + + def mrf_list do + {:ok, modules} = :application.get_key(:pleroma, :modules) + + Enum.filter(modules, fn module -> + name_as_list = Module.split(module) + + List.starts_with?(name_as_list, ["Pleroma", "Web", "ActivityPub", "MRF"]) and + length(name_as_list) > 4 + end) + end + + def richmedia_parsers do + {:ok, modules} = :application.get_key(:pleroma, :modules) + + Enum.filter(modules, fn module -> + name_as_list = Module.split(module) + + List.starts_with?(name_as_list, ["Pleroma", "Web", "RichMedia", "Parsers"]) and + length(name_as_list) == 5 + end) + end +end + +defimpl Jason.Encoder, for: Tuple do + def encode(tuple, opts) do + Jason.Encode.list(Tuple.to_list(tuple), opts) + end +end + +defimpl Jason.Encoder, for: [Regex, Function] do + def encode(term, opts) do + Jason.Encode.string(inspect(term), opts) + end +end + +defimpl String.Chars, for: Regex do + def to_string(term) do + inspect(term) + end +end diff --git a/lib/pleroma/docs/json.ex b/lib/pleroma/docs/json.ex new file mode 100644 index 00000000..38f01501 --- /dev/null +++ b/lib/pleroma/docs/json.ex @@ -0,0 +1,15 @@ +defmodule Pleroma.Docs.JSON do + @behaviour Pleroma.Docs.Formatter + def process(descriptions) do + config_path = "docs/generate_config.json" + {:ok, file} = File.open(config_path, [:write]) + json = generate_json(descriptions) + IO.write(file, json) + :ok = File.close(file) + {:ok, config_path} + end + + def generate_json(descriptions) do + Jason.encode!(descriptions) + end +end diff --git a/lib/pleroma/docs/markdown.ex b/lib/pleroma/docs/markdown.ex new file mode 100644 index 00000000..27a09663 --- /dev/null +++ b/lib/pleroma/docs/markdown.ex @@ -0,0 +1,67 @@ +defmodule Pleroma.Docs.Markdown do + @behaviour Pleroma.Docs.Formatter + + def process(descriptions) do + config_path = "docs/config.md" + {:ok, file} = File.open(config_path, [:write]) + IO.write(file, "# Generated configuration\r\n\r\n") + IO.write(file, "Date of generation: #{Date.utc_today()}\r\n\r\n") + + IO.write( + file, + "This file describe the configuration, it is recommended to edit the relevant *.secret.exs file instead of the others founds in the ``config`` directory. +If you run Pleroma with ``MIX_ENV=prod`` the file is ``prod.secret.exs``, otherwise it is ``dev.secret.exs``.\r\n\r\n" + ) + + for group <- descriptions do + if is_nil(group[:key]) do + IO.write(file, "## #{inspect(group[:group])}\r\n\r\n") + else + IO.write(file, "## #{inspect(group[:key])}\r\n\r\n") + end + + IO.write(file, "Type: `#{group[:type]}` \r\n") + IO.write(file, "#{group[:description]} \r\n\r\n") + + for child <- group[:children] do + print_child_header(file, child) + + print_suggestions(file, child[:suggestions]) + + if child[:children] do + for subchild <- child[:children] do + print_child_header(file, subchild) + + print_suggestions(file, subchild[:suggestions]) + end + end + end + + IO.write(file, "\r\n") + end + + :ok = File.close(file) + {:ok, config_path} + end + + defp print_suggestion(file, suggestion) when is_function(suggestion) do + IO.write(file, " `#{inspect(suggestion.())}`\r\n") + end + + defp print_suggestion(file, suggestion) do + IO.write(file, " `#{inspect(suggestion)}`\r\n") + end + + defp print_suggestions(file, suggestions) do + IO.write(file, "Suggestions: \r\n") + + for suggestion <- suggestions do + print_suggestion(file, suggestion) + end + end + + defp print_child_header(file, child) do + IO.write(file, "* `#{inspect(child[:key])}`: #{child[:description]} \r\n") + IO.write(file, "Type: `#{inspect(child[:type])}` \r\n") + end +end From 511d93fa5486a58b63576ce1b8af551bbafe703f Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 30 Aug 2019 13:22:21 +0300 Subject: [PATCH 039/138] mix docs generates config.md --- lib/mix/tasks/pleroma/docs.ex | 40 +++++++++++++++++++++++++++++++++++ mix.exs | 3 ++- 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 lib/mix/tasks/pleroma/docs.ex diff --git a/lib/mix/tasks/pleroma/docs.ex b/lib/mix/tasks/pleroma/docs.ex new file mode 100644 index 00000000..5ae3b871 --- /dev/null +++ b/lib/mix/tasks/pleroma/docs.ex @@ -0,0 +1,40 @@ +defmodule Mix.Tasks.Pleroma.Docs do + use Mix.Task + import Mix.Pleroma + + @shortdoc "Generates docs from descriptions.exs" + @moduledoc """ + Generates docs from `descriptions.exs`. + + Supports two formats: `markdown` and `json`. + + ## Generate markdown docs + + `mix pleroma.docs` + + ## Generate json docs + + `mix pleroma.docs json`s + """ + + def run(["json"]) do + do_run(Pleroma.Docs.JSON) + end + + def run(_) do + do_run(Pleroma.Docs.Markdown) + end + + defp do_run(implementation) do + start_pleroma() + descriptions = Config.Reader.read!("config/description.exs") + + {:ok, file_path} = + Pleroma.Docs.Formatter.process( + implementation, + descriptions[:pleroma][:config_description] + ) + + Mix.shell().info([:green, "Markdown docs successfully generated to #{file_path}."]) + end +end diff --git a/mix.exs b/mix.exs index 3170d6f2..c52ab08b 100644 --- a/mix.exs +++ b/mix.exs @@ -172,7 +172,8 @@ defmodule Pleroma.Mixfile do "ecto.rollback": ["pleroma.ecto.rollback"], "ecto.setup": ["ecto.create", "ecto.migrate", "run priv/repo/seeds.exs"], "ecto.reset": ["ecto.drop", "ecto.setup"], - test: ["ecto.create --quiet", "ecto.migrate", "test"] + test: ["ecto.create --quiet", "ecto.migrate", "test"], + docs: ["pleroma.docs", "docs"] ] end From a1f2dfb10a777592ea85d4bf8c5f91c859ec225b Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 30 Aug 2019 14:04:21 +0300 Subject: [PATCH 040/138] expanding regex sigils to use modifiers --- lib/pleroma/web/admin_api/config.ex | 15 +++++++++--- .../admin_api/admin_api_controller_test.exs | 12 ++++++++-- test/web/admin_api/config_test.exs | 24 +++++++++++++++++++ 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/admin_api/config.ex b/lib/pleroma/web/admin_api/config.ex index a10cc779..1917a558 100644 --- a/lib/pleroma/web/admin_api/config.ex +++ b/lib/pleroma/web/admin_api/config.ex @@ -90,6 +90,8 @@ defmodule Pleroma.Web.AdminAPI.Config do for v <- entity, into: [], do: do_convert(v) end + defp do_convert(%Regex{} = entity), do: inspect(entity) + defp do_convert(entity) when is_map(entity) do for {k, v} <- entity, into: %{}, do: {do_convert(k), do_convert(v)} end @@ -122,7 +124,7 @@ defmodule Pleroma.Web.AdminAPI.Config do def transform(entity), do: :erlang.term_to_binary(entity) - defp do_transform(%Regex{} = entity) when is_map(entity), do: entity + defp do_transform(%Regex{} = entity), do: entity defp do_transform(%{"tuple" => [":dispatch", [entity]]}) do {dispatch_settings, []} = do_eval(entity) @@ -154,8 +156,15 @@ defmodule Pleroma.Web.AdminAPI.Config do defp do_transform(entity), do: entity defp do_transform_string("~r/" <> pattern) do - pattern = String.trim_trailing(pattern, "/") - ~r/#{pattern}/ + modificator = String.split(pattern, "/") |> List.last() + pattern = String.trim_trailing(pattern, "/" <> modificator) + + case modificator do + "" -> ~r/#{pattern}/ + "i" -> ~r/#{pattern}/i + "u" -> ~r/#{pattern}/u + "s" -> ~r/#{pattern}/s + end end defp do_transform_string(":" <> atom), do: String.to_atom(atom) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 4e2c2743..3b6d75a4 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -1779,7 +1779,11 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do %{"tuple" => [":seconds_valid", 60]}, %{"tuple" => [":path", ""]}, %{"tuple" => [":key1", nil]}, - %{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]} + %{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]}, + %{"tuple" => [":regex1", "~r/https:\/\/example.com/"]}, + %{"tuple" => [":regex2", "~r/https:\/\/example.com/u"]}, + %{"tuple" => [":regex3", "~r/https:\/\/example.com/i"]}, + %{"tuple" => [":regex4", "~r/https:\/\/example.com/s"]} ] } ] @@ -1796,7 +1800,11 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do %{"tuple" => [":seconds_valid", 60]}, %{"tuple" => [":path", ""]}, %{"tuple" => [":key1", nil]}, - %{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]} + %{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]}, + %{"tuple" => [":regex1", "~r/https:\\/\\/example.com/"]}, + %{"tuple" => [":regex2", "~r/https:\\/\\/example.com/u"]}, + %{"tuple" => [":regex3", "~r/https:\\/\\/example.com/i"]}, + %{"tuple" => [":regex4", "~r/https:\\/\\/example.com/s"]} ] } ] diff --git a/test/web/admin_api/config_test.exs b/test/web/admin_api/config_test.exs index 3190dc1c..204446b7 100644 --- a/test/web/admin_api/config_test.exs +++ b/test/web/admin_api/config_test.exs @@ -103,6 +103,30 @@ defmodule Pleroma.Web.AdminAPI.ConfigTest do assert Config.from_binary(binary) == ~r/comp[lL][aA][iI][nN]er/ end + test "link sigil" do + binary = Config.transform("~r/https:\/\/example.com/") + assert binary == :erlang.term_to_binary(~r/https:\/\/example.com/) + assert Config.from_binary(binary) == ~r/https:\/\/example.com/ + end + + test "link sigil with u modifier" do + binary = Config.transform("~r/https:\/\/example.com/u") + assert binary == :erlang.term_to_binary(~r/https:\/\/example.com/u) + assert Config.from_binary(binary) == ~r/https:\/\/example.com/u + end + + test "link sigil with i modifier" do + binary = Config.transform("~r/https:\/\/example.com/i") + assert binary == :erlang.term_to_binary(~r/https:\/\/example.com/i) + assert Config.from_binary(binary) == ~r/https:\/\/example.com/i + end + + test "link sigil with s modifier" do + binary = Config.transform("~r/https:\/\/example.com/s") + assert binary == :erlang.term_to_binary(~r/https:\/\/example.com/s) + assert Config.from_binary(binary) == ~r/https:\/\/example.com/s + end + test "2 child tuple" do binary = Config.transform(%{"tuple" => ["v1", ":v2"]}) assert binary == :erlang.term_to_binary({"v1", :v2}) From 68e45a327f7dbe0fdc98878202d6a72d8856b96b Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 30 Aug 2019 14:09:07 +0300 Subject: [PATCH 041/138] changelog --- CHANGELOG.md | 3 ++- mix.lock | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7f1aee0..3d9c6a4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,11 +20,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **Breaking:** `/api/pleroma/notifications/read` is moved to `/api/v1/pleroma/notifications/read` and now supports `max_id` and responds with Mastodon API entities. - Configuration: OpenGraph and TwitterCard providers enabled by default - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text -- Mastodon API: `pleroma.thread_muted` key in the Status entity +- Configuration: added `config/description.exs`, from which `docs/config.md` is generated - Federation: Return 403 errors when trying to request pages from a user's follower/following collections if they have `hide_followers`/`hide_follows` set - NodeInfo: Return `skipThreadContainment` in `metadata` for the `skip_thread_containment` option - NodeInfo: Return `mailerEnabled` in `metadata` - Mastodon API: Unsubscribe followers when they unfollow a user +- Mastodon API: `pleroma.thread_muted` key in the Status entity - AdminAPI: Add "godmode" while fetching user statuses (i.e. admin can see private statuses) - Improve digest email template – Pagination: (optional) return `total` alongside with `items` when paginating diff --git a/mix.lock b/mix.lock index 2639e96e..07bc8d48 100644 --- a/mix.lock +++ b/mix.lock @@ -20,7 +20,7 @@ "db_connection": {:hex, :db_connection, "2.0.6", "bde2f85d047969c5b5800cb8f4b3ed6316c8cb11487afedac4aa5f93fd39abfa", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, "decimal": {:hex, :decimal, "1.8.0", "ca462e0d885f09a1c5a342dbd7c1dcf27ea63548c65a65e67334f4b61803822e", [:mix], [], "hexpm"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"}, - "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"}, + "earmark": {:hex, :earmark, "1.3.6", "ce1d0675e10a5bb46b007549362bd3f5f08908843957687d8484fe7f37466b19", [:mix], [], "hexpm"}, "ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, "ecto_sql": {:hex, :ecto_sql, "3.1.3", "2c536139190492d9de33c5fefac7323c5eaaa82e1b9bf93482a14649042f7cd9", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"}, @@ -46,8 +46,8 @@ "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"}, "joken": {:hex, :joken, "2.0.1", "ec9ab31bf660f343380da033b3316855197c8d4c6ef597fa3fcb451b326beb14", [:mix], [{:jose, "~> 1.9", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm"}, "jose": {:hex, :jose, "1.9.0", "4167c5f6d06ffaebffd15cdb8da61a108445ef5e85ab8f5a7ad926fdf3ada154", [:mix, :rebar3], [{:base64url, "~> 0.0.1", [hex: :base64url, repo: "hexpm", optional: false]}], "hexpm"}, - "makeup": {:hex, :makeup, "0.8.0", "9cf32aea71c7fe0a4b2e9246c2c4978f9070257e5c9ce6d4a28ec450a839b55f", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"}, - "makeup_elixir": {:hex, :makeup_elixir, "0.13.0", "be7a477997dcac2e48a9d695ec730b2d22418292675c75aa2d34ba0909dcdeda", [:mix], [{:makeup, "~> 0.8", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"}, + "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"}, + "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"}, "meck": {:hex, :meck, "0.8.13", "ffedb39f99b0b99703b8601c6f17c7f76313ee12de6b646e671e3188401f7866", [:rebar3], [], "hexpm"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"}, "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm"}, @@ -56,7 +56,7 @@ "mock": {:hex, :mock, "0.3.3", "42a433794b1291a9cf1525c6d26b38e039e0d3a360732b5e467bfc77ef26c914", [:mix], [{:meck, "~> 0.8.13", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"}, "mogrify": {:hex, :mogrify, "0.6.1", "de1b527514f2d95a7bbe9642eb556061afb337e220cf97adbf3a4e6438ed70af", [:mix], [], "hexpm"}, "mox": {:hex, :mox, "0.5.1", "f86bb36026aac1e6f924a4b6d024b05e9adbed5c63e8daa069bd66fb3292165b", [:mix], [], "hexpm"}, - "nimble_parsec": {:hex, :nimble_parsec, "0.5.0", "90e2eca3d0266e5c53f8fbe0079694740b9c91b6747f2b7e3c5d21966bba8300", [:mix], [], "hexpm"}, + "nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.3", "6706a148809a29c306062862c803406e88f048277f6e85b68faf73291e820b84", [:mix], [], "hexpm"}, "phoenix": {:hex, :phoenix, "1.4.9", "746d098e10741c334d88143d3c94cab1756435f94387a63441792e66ec0ee974", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, From 0559c82bdb39cba019099d404723f38fed6e2aba Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 30 Aug 2019 14:27:55 +0300 Subject: [PATCH 042/138] fix --- lib/mix/tasks/pleroma/docs.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/docs.ex b/lib/mix/tasks/pleroma/docs.ex index 5ae3b871..d68e0238 100644 --- a/lib/mix/tasks/pleroma/docs.ex +++ b/lib/mix/tasks/pleroma/docs.ex @@ -27,7 +27,7 @@ defmodule Mix.Tasks.Pleroma.Docs do defp do_run(implementation) do start_pleroma() - descriptions = Config.Reader.read!("config/description.exs") + {descriptions, _paths} = Mix.Config.eval!("config/description.exs") {:ok, file_path} = Pleroma.Docs.Formatter.process( From 6721301086674afe721f9eea478a2037756be93c Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 30 Aug 2019 19:14:01 +0300 Subject: [PATCH 043/138] some changes --- config/description.exs | 15 ++++---- lib/mix/tasks/pleroma/docs.ex | 6 +-- .../docs/{formatter.ex => generator.ex} | 6 ++- lib/pleroma/docs/json.ex | 5 ++- lib/pleroma/docs/markdown.ex | 38 ++++++++++++------- 5 files changed, 44 insertions(+), 26 deletions(-) rename lib/pleroma/docs/{formatter.ex => generator.ex} (90%) diff --git a/config/description.exs b/config/description.exs index 537b9d99..684d3fc0 100644 --- a/config/description.exs +++ b/config/description.exs @@ -1,5 +1,5 @@ use Mix.Config -alias Pleroma.Docs.Formatter +alias Pleroma.Docs.Generator websocket_config = [ path: "/websocket", @@ -24,7 +24,7 @@ config :pleroma, :config_description, [ type: :module, description: "Module which will be used for uploads", suggestions: [ - Formatter.uploaders_list() + Generator.uploaders_list() ] }, %{ @@ -32,7 +32,7 @@ config :pleroma, :config_description, [ type: {:list, :module}, description: "List of filter modules for uploads", suggestions: [ - Formatter.filters_list() + Generator.filters_list() ] }, %{ @@ -64,8 +64,7 @@ config :pleroma, :config_description, [ %{ key: :proxy_opts, type: :keyword, - description: "Proxy options, see `Pleroma.ReverseProxy` documentation", - suggestions: ["somehow created link to Pleroma.ReverseProxy options"] + description: "Proxy options, see `Pleroma.ReverseProxy` documentation" } ] }, @@ -93,7 +92,7 @@ config :pleroma, :config_description, [ children: [ %{ key: :bucket, - type: :strings, + type: :string, description: "S3 bucket", suggestions: [ "bucket" @@ -629,7 +628,7 @@ config :pleroma, :config_description, [ description: "A list of MRF policies enabled", suggestions: [ Pleroma.Web.ActivityPub.MRF.NoOpPolicy, - Formatter.mrf_list() + Generator.mrf_list() ] }, %{ @@ -1920,7 +1919,7 @@ config :pleroma, :config_description, [ type: {:list, :module}, description: "list of Rich Media parsers", suggestions: [ - Formatter.richmedia_parsers() + Generator.richmedia_parsers() ] }, %{ diff --git a/lib/mix/tasks/pleroma/docs.ex b/lib/mix/tasks/pleroma/docs.ex index d68e0238..821ee74f 100644 --- a/lib/mix/tasks/pleroma/docs.ex +++ b/lib/mix/tasks/pleroma/docs.ex @@ -8,11 +8,11 @@ defmodule Mix.Tasks.Pleroma.Docs do Supports two formats: `markdown` and `json`. - ## Generate markdown docs + ## Generate Markdown docs `mix pleroma.docs` - ## Generate json docs + ## Generate JSON docs `mix pleroma.docs json`s """ @@ -30,7 +30,7 @@ defmodule Mix.Tasks.Pleroma.Docs do {descriptions, _paths} = Mix.Config.eval!("config/description.exs") {:ok, file_path} = - Pleroma.Docs.Formatter.process( + Pleroma.Docs.Generator.process( implementation, descriptions[:pleroma][:config_description] ) diff --git a/lib/pleroma/docs/formatter.ex b/lib/pleroma/docs/generator.ex similarity index 90% rename from lib/pleroma/docs/formatter.ex rename to lib/pleroma/docs/generator.ex index a1c75793..e788712c 100644 --- a/lib/pleroma/docs/formatter.ex +++ b/lib/pleroma/docs/generator.ex @@ -1,4 +1,4 @@ -defmodule Pleroma.Docs.Formatter do +defmodule Pleroma.Docs.Generator do @callback process(keyword()) :: {:ok, String.t()} @spec process(module(), keyword()) :: {:ok, String.t()} @@ -6,6 +6,7 @@ defmodule Pleroma.Docs.Formatter do implementation.process(descriptions) end + @spec uploaders_list() :: [module()] def uploaders_list do {:ok, modules} = :application.get_key(:pleroma, :modules) @@ -17,6 +18,7 @@ defmodule Pleroma.Docs.Formatter do end) end + @spec filters_list() :: [module()] def filters_list do {:ok, modules} = :application.get_key(:pleroma, :modules) @@ -27,6 +29,7 @@ defmodule Pleroma.Docs.Formatter do end) end + @spec mrf_list() :: [module()] def mrf_list do {:ok, modules} = :application.get_key(:pleroma, :modules) @@ -38,6 +41,7 @@ defmodule Pleroma.Docs.Formatter do end) end + @spec richmedia_parsers() :: [module()] def richmedia_parsers do {:ok, modules} = :application.get_key(:pleroma, :modules) diff --git a/lib/pleroma/docs/json.ex b/lib/pleroma/docs/json.ex index 38f01501..aed730e7 100644 --- a/lib/pleroma/docs/json.ex +++ b/lib/pleroma/docs/json.ex @@ -1,5 +1,7 @@ defmodule Pleroma.Docs.JSON do - @behaviour Pleroma.Docs.Formatter + @behaviour Pleroma.Docs.Generator + + @spec process(keyword()) :: {:ok, String.t()} def process(descriptions) do config_path = "docs/generate_config.json" {:ok, file} = File.open(config_path, [:write]) @@ -9,6 +11,7 @@ defmodule Pleroma.Docs.JSON do {:ok, config_path} end + @spec generate_json([keyword()]) :: String.t() def generate_json(descriptions) do Jason.encode!(descriptions) end diff --git a/lib/pleroma/docs/markdown.ex b/lib/pleroma/docs/markdown.ex index 27a09663..c66640bf 100644 --- a/lib/pleroma/docs/markdown.ex +++ b/lib/pleroma/docs/markdown.ex @@ -1,16 +1,17 @@ defmodule Pleroma.Docs.Markdown do - @behaviour Pleroma.Docs.Formatter + @behaviour Pleroma.Docs.Generator + @spec process(keyword()) :: {:ok, String.t()} def process(descriptions) do config_path = "docs/config.md" {:ok, file} = File.open(config_path, [:write]) - IO.write(file, "# Generated configuration\r\n\r\n") + IO.write(file, "# Configuration\r\n\r\n") IO.write(file, "Date of generation: #{Date.utc_today()}\r\n\r\n") IO.write( file, - "This file describe the configuration, it is recommended to edit the relevant *.secret.exs file instead of the others founds in the ``config`` directory. -If you run Pleroma with ``MIX_ENV=prod`` the file is ``prod.secret.exs``, otherwise it is ``dev.secret.exs``.\r\n\r\n" + "This file describe the configuration, it is recommended to edit the relevant `*.secret.exs` file instead of the others founds in the ``config`` directory. \r\n + If you run Pleroma with ``MIX_ENV=prod`` the file is ``prod.secret.exs``, otherwise it is ``dev.secret.exs``.\r\n\r\n" ) for group <- descriptions do @@ -20,7 +21,6 @@ If you run Pleroma with ``MIX_ENV=prod`` the file is ``prod.secret.exs``, otherw IO.write(file, "## #{inspect(group[:key])}\r\n\r\n") end - IO.write(file, "Type: `#{group[:type]}` \r\n") IO.write(file, "#{group[:description]} \r\n\r\n") for child <- group[:children] do @@ -44,24 +44,36 @@ If you run Pleroma with ``MIX_ENV=prod`` the file is ``prod.secret.exs``, otherw {:ok, config_path} end + defp print_suggestion(file, suggestion) when is_list(suggestion) do + IO.write(file, " `#{inspect(suggestion)}`\r\n") + end + defp print_suggestion(file, suggestion) when is_function(suggestion) do IO.write(file, " `#{inspect(suggestion.())}`\r\n") end - defp print_suggestion(file, suggestion) do - IO.write(file, " `#{inspect(suggestion)}`\r\n") + defp print_suggestion(file, suggestion, as_list \\ false) do + list_mark = if as_list, do: "*", else: "" + IO.write(file, " #{list_mark} `#{inspect(suggestion)}`\r\n") end - defp print_suggestions(file, suggestions) do - IO.write(file, "Suggestions: \r\n") + defp print_suggestions(_file, nil), do: nil - for suggestion <- suggestions do - print_suggestion(file, suggestion) + defp print_suggestions(file, suggestions) do + IO.write(file, " Suggestions: \r\n") + + if length(suggestions) > 1 do + for suggestion <- suggestions do + print_suggestion(file, suggestion, true) + end + else + print_suggestion(file, List.first(suggestions)) end end defp print_child_header(file, child) do - IO.write(file, "* `#{inspect(child[:key])}`: #{child[:description]} \r\n") - IO.write(file, "Type: `#{inspect(child[:type])}` \r\n") + IO.write(file, "* `#{inspect(child[:key])}` \r\n") + IO.write(file, " #{child[:description]} \r\n") + IO.write(file, " Type: `#{inspect(child[:type])}` \r\n") end end From 8f5ee7db06e379e44505744fd21e59cc46432ac3 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 30 Aug 2019 19:59:13 +0300 Subject: [PATCH 044/138] typo fix --- lib/mix/tasks/pleroma/docs.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/docs.ex b/lib/mix/tasks/pleroma/docs.ex index 821ee74f..fb8a2a01 100644 --- a/lib/mix/tasks/pleroma/docs.ex +++ b/lib/mix/tasks/pleroma/docs.ex @@ -14,7 +14,7 @@ defmodule Mix.Tasks.Pleroma.Docs do ## Generate JSON docs - `mix pleroma.docs json`s + `mix pleroma.docs json` """ def run(["json"]) do From 0624e06a9c981264f238041a2ad22392d4a8f792 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 30 Aug 2019 20:14:18 +0300 Subject: [PATCH 045/138] little fix --- lib/pleroma/docs/markdown.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/docs/markdown.ex b/lib/pleroma/docs/markdown.ex index c66640bf..797ce73b 100644 --- a/lib/pleroma/docs/markdown.ex +++ b/lib/pleroma/docs/markdown.ex @@ -10,8 +10,8 @@ defmodule Pleroma.Docs.Markdown do IO.write( file, - "This file describe the configuration, it is recommended to edit the relevant `*.secret.exs` file instead of the others founds in the ``config`` directory. \r\n - If you run Pleroma with ``MIX_ENV=prod`` the file is ``prod.secret.exs``, otherwise it is ``dev.secret.exs``.\r\n\r\n" + "This file describe the configuration, it is recommended to edit the relevant `*.secret.exs` file instead of the others founds in the ``config`` directory. \r\n\r\n" <> + " If you run Pleroma with ``MIX_ENV=prod`` the file is ``prod.secret.exs``, otherwise it is ``dev.secret.exs``.\r\n\r\n" ) for group <- descriptions do From 65bc9e66ad3848594df67c4b0d5539b1c8540161 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Sep 2019 06:45:02 +0000 Subject: [PATCH 046/138] Apply suggestion to config/description.exs --- config/description.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/description.exs b/config/description.exs index 684d3fc0..48cc70e6 100644 --- a/config/description.exs +++ b/config/description.exs @@ -38,7 +38,7 @@ config :pleroma, :config_description, [ %{ key: :link_name, type: :boolean, - description: "If enabled Pleroma will add name parameter to the url off the upload", + description: "If enabled, a name parameter will be added to the url of the upload. For example `https://instance.tld/media/imagehash.png?name=realname.png`", suggestions: [ true, false From 5db2920644d8a82e53eaef228e39edef1e6af5aa Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 3 Sep 2019 06:45:16 +0000 Subject: [PATCH 047/138] Apply suggestion to config/description.exs --- config/description.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/description.exs b/config/description.exs index 48cc70e6..37a8a566 100644 --- a/config/description.exs +++ b/config/description.exs @@ -55,7 +55,7 @@ config :pleroma, :config_description, [ %{ key: :proxy_remote, type: :boolean, - description: "If enabled, Pleroma will proxy media requests instead of redirecting to it", + description: "If enabled, requests to media stored using a remote uploader will be proxied instead of being redirected.", suggestions: [ true, false From 35757b6d0eb7b59d511bfea6a166683e18d6aa97 Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 3 Sep 2019 09:45:54 +0300 Subject: [PATCH 048/138] don't add behaviour to suggestions --- lib/pleroma/docs/generator.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/docs/generator.ex b/lib/pleroma/docs/generator.ex index e788712c..aa578eee 100644 --- a/lib/pleroma/docs/generator.ex +++ b/lib/pleroma/docs/generator.ex @@ -14,7 +14,7 @@ defmodule Pleroma.Docs.Generator do name_as_list = Module.split(module) List.starts_with?(name_as_list, ["Pleroma", "Uploaders"]) and - List.last(name_as_list) in ["S3", "Local", "MDII"] + List.last(name_as_list) != "Uploader" end) end From 10827eecad5ef521dff3673c0891cbec2c8bb44b Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 3 Sep 2019 11:56:21 +0300 Subject: [PATCH 049/138] formatting --- config/description.exs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/config/description.exs b/config/description.exs index 37a8a566..a56a9508 100644 --- a/config/description.exs +++ b/config/description.exs @@ -38,7 +38,8 @@ config :pleroma, :config_description, [ %{ key: :link_name, type: :boolean, - description: "If enabled, a name parameter will be added to the url of the upload. For example `https://instance.tld/media/imagehash.png?name=realname.png`", + description: + "If enabled, a name parameter will be added to the url of the upload. For example `https://instance.tld/media/imagehash.png?name=realname.png`", suggestions: [ true, false @@ -55,7 +56,8 @@ config :pleroma, :config_description, [ %{ key: :proxy_remote, type: :boolean, - description: "If enabled, requests to media stored using a remote uploader will be proxied instead of being redirected.", + description: + "If enabled, requests to media stored using a remote uploader will be proxied instead of being redirected.", suggestions: [ true, false From 6b0e8b73df32ce926551713972d9d6af484ccc2e Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 3 Sep 2019 12:30:44 +0300 Subject: [PATCH 050/138] bump ex_doc version --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index c52ab08b..96ef723b 100644 --- a/mix.exs +++ b/mix.exs @@ -125,7 +125,7 @@ defmodule Pleroma.Mixfile do {:crypt, git: "https://github.com/msantos/crypt", ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"}, {:cors_plug, "~> 1.5"}, - {:ex_doc, "~> 0.20.2", only: :dev, runtime: false}, + {:ex_doc, "~> 0.21", only: :dev, runtime: false}, {:web_push_encryption, "~> 0.2.1"}, {:swoosh, "~> 0.23.2"}, {:phoenix_swoosh, "~> 0.2"}, diff --git a/mix.lock b/mix.lock index 07bc8d48..5762dae4 100644 --- a/mix.lock +++ b/mix.lock @@ -29,7 +29,7 @@ "ex_aws": {:hex, :ex_aws, "2.1.0", "b92651527d6c09c479f9013caa9c7331f19cba38a650590d82ebf2c6c16a1d8a", [:mix], [{:configparser_ex, "~> 2.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:xml_builder, "~> 0.1.0", [hex: :xml_builder, repo: "hexpm", optional: true]}], "hexpm"}, "ex_aws_s3": {:hex, :ex_aws_s3, "2.0.1", "9e09366e77f25d3d88c5393824e613344631be8db0d1839faca49686e99b6704", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm"}, "ex_const": {:hex, :ex_const, "0.2.4", "d06e540c9d834865b012a17407761455efa71d0ce91e5831e86881b9c9d82448", [:mix], [], "hexpm"}, - "ex_doc": {:hex, :ex_doc, "0.20.2", "1bd0dfb0304bade58beb77f20f21ee3558cc3c753743ae0ddbb0fd7ba2912331", [:mix], [{:earmark, "~> 1.3", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, + "ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, "ex_machina": {:hex, :ex_machina, "2.3.0", "92a5ad0a8b10ea6314b876a99c8c9e3f25f4dde71a2a835845b136b9adaf199a", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm"}, "ex_rated": {:hex, :ex_rated, "1.3.3", "30ecbdabe91f7eaa9d37fa4e81c85ba420f371babeb9d1910adbcd79ec798d27", [:mix], [{:ex2ms, "~> 1.5", [hex: :ex2ms, repo: "hexpm", optional: false]}], "hexpm"}, "ex_syslogger": {:git, "https://github.com/slashmili/ex_syslogger.git", "f3963399047af17e038897c69e20d552e6899e1d", [tag: "1.4.0"]}, From 57dc59d98d0ae5b4315bbc5c8a9c7ca59f6341f9 Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 3 Sep 2019 12:31:43 +0300 Subject: [PATCH 051/138] little fix --- lib/mix/tasks/pleroma/docs.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/docs.ex b/lib/mix/tasks/pleroma/docs.ex index fb8a2a01..4be53ce7 100644 --- a/lib/mix/tasks/pleroma/docs.ex +++ b/lib/mix/tasks/pleroma/docs.ex @@ -35,6 +35,8 @@ defmodule Mix.Tasks.Pleroma.Docs do descriptions[:pleroma][:config_description] ) - Mix.shell().info([:green, "Markdown docs successfully generated to #{file_path}."]) + type = if implementation == Pleroma.Docs.Markdown, do: "Markdown", else: "JSON" + + Mix.shell().info([:green, "#{type} docs successfully generated to #{file_path}."]) end end From aa7fb22008abab1a48a5dac56a32fc9e37d04f88 Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 3 Sep 2019 12:32:00 +0300 Subject: [PATCH 052/138] placeholder for config.md --- docs/config.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/config.md diff --git a/docs/config.md b/docs/config.md new file mode 100644 index 00000000..7f54a34b --- /dev/null +++ b/docs/config.md @@ -0,0 +1 @@ +This file is a placeholder, please run mix pleroma.docs to generate it. From 5ff12e7df10dd70f70e5929ede5dc7570e066c57 Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 3 Sep 2019 19:22:25 +0300 Subject: [PATCH 053/138] some changes --- lib/pleroma/docs/markdown.ex | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/docs/markdown.ex b/lib/pleroma/docs/markdown.ex index 797ce73b..24930cc9 100644 --- a/lib/pleroma/docs/markdown.ex +++ b/lib/pleroma/docs/markdown.ex @@ -4,24 +4,24 @@ defmodule Pleroma.Docs.Markdown do @spec process(keyword()) :: {:ok, String.t()} def process(descriptions) do config_path = "docs/config.md" - {:ok, file} = File.open(config_path, [:write]) - IO.write(file, "# Configuration\r\n\r\n") - IO.write(file, "Date of generation: #{Date.utc_today()}\r\n\r\n") + {:ok, file} = File.open(config_path, [:utf8, :write]) + IO.write(file, "# Configuration\n") + IO.write(file, "Date of generation: #{Date.utc_today()}\n\n") IO.write( file, - "This file describe the configuration, it is recommended to edit the relevant `*.secret.exs` file instead of the others founds in the ``config`` directory. \r\n\r\n" <> - " If you run Pleroma with ``MIX_ENV=prod`` the file is ``prod.secret.exs``, otherwise it is ``dev.secret.exs``.\r\n\r\n" + "This file describe the configuration, it is recommended to edit the relevant `*.secret.exs` file instead of the others founds in the ``config`` directory.\n\n" <> + "If you run Pleroma with ``MIX_ENV=prod`` the file is ``prod.secret.exs``, otherwise it is ``dev.secret.exs``.\n\n" ) for group <- descriptions do if is_nil(group[:key]) do - IO.write(file, "## #{inspect(group[:group])}\r\n\r\n") + IO.write(file, "## #{inspect(group[:group])}\n") else - IO.write(file, "## #{inspect(group[:key])}\r\n\r\n") + IO.write(file, "## #{inspect(group[:key])}\n") end - IO.write(file, "#{group[:description]} \r\n\r\n") + IO.write(file, "#{group[:description]}\n") for child <- group[:children] do print_child_header(file, child) @@ -37,7 +37,7 @@ defmodule Pleroma.Docs.Markdown do end end - IO.write(file, "\r\n") + IO.write(file, "\n") end :ok = File.close(file) @@ -45,22 +45,22 @@ defmodule Pleroma.Docs.Markdown do end defp print_suggestion(file, suggestion) when is_list(suggestion) do - IO.write(file, " `#{inspect(suggestion)}`\r\n") + IO.write(file, " `#{inspect(suggestion)}`\n") end defp print_suggestion(file, suggestion) when is_function(suggestion) do - IO.write(file, " `#{inspect(suggestion.())}`\r\n") + IO.write(file, " `#{inspect(suggestion.())}`\n") end defp print_suggestion(file, suggestion, as_list \\ false) do - list_mark = if as_list, do: "*", else: "" - IO.write(file, " #{list_mark} `#{inspect(suggestion)}`\r\n") + list_mark = if as_list, do: "- ", else: "" + IO.write(file, " #{list_mark}`#{inspect(suggestion)}`\n") end defp print_suggestions(_file, nil), do: nil defp print_suggestions(file, suggestions) do - IO.write(file, " Suggestions: \r\n") + IO.write(file, "Suggestions:\n") if length(suggestions) > 1 do for suggestion <- suggestions do @@ -72,8 +72,7 @@ defmodule Pleroma.Docs.Markdown do end defp print_child_header(file, child) do - IO.write(file, "* `#{inspect(child[:key])}` \r\n") - IO.write(file, " #{child[:description]} \r\n") - IO.write(file, " Type: `#{inspect(child[:type])}` \r\n") + IO.write(file, "- `#{inspect(child[:key])}` -`#{inspect(child[:type])}` \n") + IO.write(file, "#{child[:description]} \n") end end From be32d90a0cdee77f4cd6ecbbbade1748c88637b8 Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 3 Sep 2019 20:06:22 +0300 Subject: [PATCH 054/138] little refactor --- lib/pleroma/docs/json.ex | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/docs/json.ex b/lib/pleroma/docs/json.ex index aed730e7..18ba01d5 100644 --- a/lib/pleroma/docs/json.ex +++ b/lib/pleroma/docs/json.ex @@ -4,11 +4,13 @@ defmodule Pleroma.Docs.JSON do @spec process(keyword()) :: {:ok, String.t()} def process(descriptions) do config_path = "docs/generate_config.json" - {:ok, file} = File.open(config_path, [:write]) - json = generate_json(descriptions) - IO.write(file, json) - :ok = File.close(file) - {:ok, config_path} + + with {:ok, file} <- File.open(config_path, [:write]), + json <- generate_json(descriptions), + :ok <- IO.write(file, json), + :ok <- File.close(file) do + {:ok, config_path} + end end @spec generate_json([keyword()]) :: String.t() From 38b29779c3372741ce1d7b652922058ad2fda79c Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 3 Sep 2019 20:11:32 +0300 Subject: [PATCH 055/138] refactoring --- lib/mix/tasks/pleroma/docs.ex | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/mix/tasks/pleroma/docs.ex b/lib/mix/tasks/pleroma/docs.ex index 4be53ce7..0d266364 100644 --- a/lib/mix/tasks/pleroma/docs.ex +++ b/lib/mix/tasks/pleroma/docs.ex @@ -27,16 +27,16 @@ defmodule Mix.Tasks.Pleroma.Docs do defp do_run(implementation) do start_pleroma() - {descriptions, _paths} = Mix.Config.eval!("config/description.exs") - {:ok, file_path} = - Pleroma.Docs.Generator.process( - implementation, - descriptions[:pleroma][:config_description] - ) + with {descriptions, _paths} <- Mix.Config.eval!("config/description.exs"), + {:ok, file_path} <- + Pleroma.Docs.Generator.process( + implementation, + descriptions[:pleroma][:config_description] + ) do + type = if implementation == Pleroma.Docs.Markdown, do: "Markdown", else: "JSON" - type = if implementation == Pleroma.Docs.Markdown, do: "Markdown", else: "JSON" - - Mix.shell().info([:green, "#{type} docs successfully generated to #{file_path}."]) + Mix.shell().info([:green, "#{type} docs successfully generated to #{file_path}."]) + end end end From e47089cf5506952befd6995f04e2683a168ff9e2 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 11 Sep 2019 09:32:58 +0300 Subject: [PATCH 056/138] web_cache_ttl description --- config/description.exs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/config/description.exs b/config/description.exs index a56a9508..c5ae6391 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2810,5 +2810,28 @@ config :pleroma, :config_description, [ suggestions: [["Authorization", "Content-Type", "Idempotency-Key"]] } ] + }, + %{ + group: :pleroma, + key: :web_cache_ttl, + type: :group, + description: + "The expiration time for the web responses cache. Values should be in milliseconds or `nil` to disable expiration.", + children: [ + %{ + key: :activity_pub, + type: :integer, + description: + "activity pub routes (except question activities). Defaults to `nil` (no expiration).", + suggestions: [30_000, nil] + }, + %{ + key: :activity_pub_question, + type: :integer, + description: + "activity pub routes (question activities). Defaults to `30_000` (30 seconds).", + suggestions: [30_000] + } + ] } ] From 5a76d5d2391f0fa6b4c58bf190b0cdbfff96014f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 10 Sep 2019 23:08:15 +0300 Subject: [PATCH 057/138] Add extended benchmark --- lib/mix/tasks/pleroma/benchmark.ex | 40 ++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex index a45940bf..84dccf7f 100644 --- a/lib/mix/tasks/pleroma/benchmark.ex +++ b/lib/mix/tasks/pleroma/benchmark.ex @@ -27,7 +27,7 @@ defmodule Mix.Tasks.Pleroma.Benchmark do }) end - def run(["render_timeline", nickname]) do + def run(["render_timeline", nickname | _] = args) do start_pleroma() user = Pleroma.User.get_by_nickname(nickname) @@ -37,17 +37,41 @@ defmodule Mix.Tasks.Pleroma.Benchmark do |> Map.put("blocking_user", user) |> Map.put("muting_user", user) |> Map.put("user", user) + |> Map.put("limit", 4096) |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities() |> Enum.reverse() - Benchee.run(%{ - "render_timeline" => fn -> - Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ - activities: activities, - for: user, - as: :activity + inputs = %{ + "1 activity" => Enum.take_random(activities, 1), + "10 activities" => Enum.take_random(activities, 10), + "20 activities" => Enum.take_random(activities, 20), + "40 activities" => Enum.take_random(activities, 40), + "80 activities" => Enum.take_random(activities, 80) + } + + inputs = + if Enum.at(args, 2) == "extended" do + Map.merge(inputs, %{ + "200 activities" => Enum.take_random(activities, 200), + "500 activities" => Enum.take_random(activities, 500), + "2000 activities" => Enum.take_random(activities, 2000), + "4096 activities" => Enum.take_random(activities, 4096) }) + else + inputs end - }) + + Benchee.run( + %{ + "Standart rendering" => fn activities -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: activities, + for: user, + as: :activity + }) + end + }, + inputs: inputs + ) end end From 56828abf6de81a52079b26cf899b91fdd822f2ce Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 11 Sep 2019 23:04:01 +0300 Subject: [PATCH 058/138] Use Jason for rendering responses Although Jason readme says Phoenix 1.4+ already does it by default, [it actually does it only for new projects](https://github.com/phoenixframework/phoenix/blob/3bfb9f6e900c9a2e31cb95736e2cb5bdad329b61/lib/phoenix.ex#L58-L59) --- config/config.exs | 2 ++ lib/{ => pleroma}/healthcheck.ex | 1 + 2 files changed, 3 insertions(+) rename lib/{ => pleroma}/healthcheck.ex (98%) diff --git a/config/config.exs b/config/config.exs index 5206fe37..0e91df88 100644 --- a/config/config.exs +++ b/config/config.exs @@ -373,6 +373,8 @@ config :pleroma, :chat, enabled: true config :phoenix, :format_encoders, json: Jason +config :phoenix, :json_library, Jason + config :pleroma, :gopher, enabled: false, ip: {0, 0, 0, 0}, diff --git a/lib/healthcheck.ex b/lib/pleroma/healthcheck.ex similarity index 98% rename from lib/healthcheck.ex rename to lib/pleroma/healthcheck.ex index f97d1443..977b78c2 100644 --- a/lib/healthcheck.ex +++ b/lib/pleroma/healthcheck.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Healthcheck do alias Pleroma.Healthcheck alias Pleroma.Repo + @derive Jason.Encoder defstruct pool_size: 0, active: 0, idle: 0, From 74e4c72c4a0a079e8e2a245ec9b1d22684baf2e1 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 11 Sep 2019 16:16:09 -0500 Subject: [PATCH 059/138] Fix double quotes in error logs Example: pleroma: [error] Couldn't fetch ""https://pleroma.soykaf.com/objects/6288a14b-0623-40fc-a26a-0d358f8a11ca"", error: nil --- lib/pleroma/web/activity_pub/transmogrifier.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 468961bd..350b83ab 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -185,12 +185,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Map.put("context", replied_object.data["context"] || object["conversation"]) else e -> - Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}") + Logger.error("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}") object end e -> - Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}") + Logger.error("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}") object end else From 4f548cb2b7b4a16a956a4f4a0ff64d279777925e Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Thu, 12 Sep 2019 09:59:34 +0300 Subject: [PATCH 060/138] added test for Ostatus --- .../web/activity_pub/transmogrifier.ex | 10 +- lib/pleroma/web/ostatus/ostatus.ex | 99 ++++++++----------- lib/pleroma/web/ostatus/ostatus_controller.ex | 12 +-- test/web/ostatus/ostatus_test.exs | 14 +++ 4 files changed, 68 insertions(+), 67 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 468961bd..acd61bda 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -1049,8 +1049,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do with %User{local: false} = user <- User.get_cached_by_ap_id(ap_id), {:ok, data} <- ActivityPub.fetch_and_prepare_user_from_ap_id(ap_id), already_ap <- User.ap_enabled?(user), - {:ok, user} <- user |> User.upgrade_changeset(data) |> User.update_and_set_cache() do - unless already_ap do + {:ok, user} <- upgrade_user(user, data) do + if not already_ap do PleromaJobQueue.enqueue(:transmogrifier, __MODULE__, [:user_upgrade, user]) end @@ -1061,6 +1061,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end + defp upgrade_user(user, data) do + user + |> User.upgrade_changeset(data) + |> User.update_and_set_cache() + end + def maybe_retire_websub(ap_id) do # some sanity checks if is_binary(ap_id) && String.length(ap_id) > 8 do diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 331cbc0b..5de1ceef 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -3,14 +3,12 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.OStatus do - import Ecto.Query import Pleroma.Web.XML require Logger alias Pleroma.Activity alias Pleroma.HTTP alias Pleroma.Object - alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web alias Pleroma.Web.ActivityPub.ActivityPub @@ -38,21 +36,13 @@ defmodule Pleroma.Web.OStatus do end end - def feed_path(user) do - "#{user.ap_id}/feed.atom" - end + def feed_path(user), do: "#{user.ap_id}/feed.atom" - def pubsub_path(user) do - "#{Web.base_url()}/push/hub/#{user.nickname}" - end + def pubsub_path(user), do: "#{Web.base_url()}/push/hub/#{user.nickname}" - def salmon_path(user) do - "#{user.ap_id}/salmon" - end + def salmon_path(user), do: "#{user.ap_id}/salmon" - def remote_follow_path do - "#{Web.base_url()}/ostatus_subscribe?acct={uri}" - end + def remote_follow_path, do: "#{Web.base_url()}/ostatus_subscribe?acct={uri}" def handle_incoming(xml_string, options \\ []) do with doc when doc != :error <- parse_document(xml_string) do @@ -217,10 +207,9 @@ defmodule Pleroma.Web.OStatus do Get the cw that mastodon uses. """ def get_cw(entry) do - with cw when not is_nil(cw) <- string_from_xpath("/*/summary", entry) do - cw - else - _e -> nil + case string_from_xpath("/*/summary", entry) do + cw when not is_nil(cw) -> cw + _ -> nil end end @@ -232,19 +221,17 @@ defmodule Pleroma.Web.OStatus do end def maybe_update(doc, user) do - if "true" == string_from_xpath("//author[1]/ap_enabled", doc) do - Transmogrifier.upgrade_user_from_ap_id(user.ap_id) - else - maybe_update_ostatus(doc, user) + case string_from_xpath("//author[1]/ap_enabled", doc) do + "true" -> + Transmogrifier.upgrade_user_from_ap_id(user.ap_id) + + _ -> + maybe_update_ostatus(doc, user) end end def maybe_update_ostatus(doc, user) do - old_data = %{ - avatar: user.avatar, - bio: user.bio, - name: user.name - } + old_data = Map.take(user, [:bio, :avatar, :name]) with false <- user.local, avatar <- make_avatar_object(doc), @@ -279,38 +266,37 @@ defmodule Pleroma.Web.OStatus do end end + @spec find_or_make_user(String.t()) :: {:ok, User.t()} def find_or_make_user(uri) do - query = from(user in User, where: user.ap_id == ^uri) - - user = Repo.one(query) - - if is_nil(user) do - make_user(uri) - else - {:ok, user} + case User.get_by_ap_id(uri) do + %User{} = user -> {:ok, user} + _ -> make_user(uri) end end + @spec make_user(String.t(), boolean()) :: {:ok, User.t()} | {:error, any()} def make_user(uri, update \\ false) do with {:ok, info} <- gather_user_info(uri) do - data = %{ - name: info["name"], - nickname: info["nickname"] <> "@" <> info["host"], - ap_id: info["uri"], - info: info, - avatar: info["avatar"], - bio: info["bio"] - } - with false <- update, - %User{} = user <- User.get_cached_by_ap_id(data.ap_id) do + %User{} = user <- User.get_cached_by_ap_id(info["uri"]) do {:ok, user} else - _e -> User.insert_or_update_user(data) + _e -> User.insert_or_update_user(build_user_data(info)) end end end + defp build_user_data(info) do + %{ + name: info["name"], + nickname: info["nickname"] <> "@" <> info["host"], + ap_id: info["uri"], + info: info, + avatar: info["avatar"], + bio: info["bio"] + } + end + # TODO: Just takes the first one for now. def make_avatar_object(author_doc, rel \\ "avatar") do href = string_from_xpath("//author[1]/link[@rel=\"#{rel}\"]/@href", author_doc) @@ -319,23 +305,23 @@ defmodule Pleroma.Web.OStatus do if href do %{ "type" => "Image", - "url" => [ - %{ - "type" => "Link", - "mediaType" => type, - "href" => href - } - ] + "url" => [%{"type" => "Link", "mediaType" => type, "href" => href}] } else nil end end + @spec gather_user_info(String.t()) :: {:ok, map()} | {:error, any()} def gather_user_info(username) do with {:ok, webfinger_data} <- WebFinger.finger(username), {:ok, feed_data} <- Websub.gather_feed_data(webfinger_data["topic"]) do - {:ok, Map.merge(webfinger_data, feed_data) |> Map.put("fqn", username)} + data = + webfinger_data + |> Map.merge(feed_data) + |> Map.put("fqn", username) + + {:ok, data} else e -> Logger.debug(fn -> "Couldn't gather info for #{username}" end) @@ -371,10 +357,7 @@ defmodule Pleroma.Web.OStatus do def fetch_activity_from_atom_url(url, options \\ []) do with true <- String.starts_with?(url, "http"), {:ok, %{body: body, status: code}} when code in 200..299 <- - HTTP.get( - url, - [{:Accept, "application/atom+xml"}] - ) do + HTTP.get(url, [{:Accept, "application/atom+xml"}]) do Logger.debug("Got document from #{url}, handling...") handle_incoming(body, options) else diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 07e2a4c2..64b2c64b 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -55,12 +55,11 @@ defmodule Pleroma.Web.OStatus.OStatusController do def feed(conn, %{"nickname" => nickname} = params) do with {_, %User{} = user} <- {:fetch_user, User.get_cached_by_nickname(nickname)} do - query_params = - Map.take(params, ["max_id"]) - |> Map.merge(%{"whole_db" => true, "actor_id" => user.ap_id}) - activities = - ActivityPub.fetch_public_activities(query_params) + params + |> Map.take(["max_id"]) + |> Map.merge(%{"whole_db" => true, "actor_id" => user.ap_id}) + |> ActivityPub.fetch_public_activities() |> Enum.reverse() response = @@ -98,8 +97,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do Federator.incoming_doc(doc) - conn - |> send_resp(200, "") + send_resp(conn, 200, "") end def object(%{assigns: %{format: format}} = conn, %{"uuid" => _uuid}) diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 803a9769..ff00c53e 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -628,4 +628,18 @@ defmodule Pleroma.Web.OStatusTest do refute OStatus.is_representable?(note_activity) end end + + describe "make_user/2" do + test "creates new user" do + {:ok, user} = OStatus.make_user("https://social.heldscal.la/user/23211") + + created_user = + User + |> Repo.get_by(ap_id: "https://social.heldscal.la/user/23211") + |> Map.put(:last_digest_emailed_at, nil) + + assert user.info + assert user == created_user + end + end end From 102eb4455c6285378bf7c25822d82378c7024aa4 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 12 Sep 2019 12:29:08 -0500 Subject: [PATCH 061/138] Fix associated test as well --- test/web/activity_pub/transmogrifier_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 0661d5d7..bebecce8 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -102,7 +102,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert capture_log(fn -> {:ok, _returned_activity} = Transmogrifier.handle_incoming(data) - end) =~ "[error] Couldn't fetch \"\"https://404.site/whatever\"\", error: nil" + end) =~ "[error] Couldn't fetch \"https://404.site/whatever\", error: nil" end test "it works for incoming notices" do From 769fb778d41df77c2514b5e3c663f3f624c0a266 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 12 Sep 2019 21:37:36 +0300 Subject: [PATCH 062/138] Track object/create activity fetches --- lib/pleroma/delivery.ex | 58 +++++++++++++ lib/pleroma/plugs/cache.ex | 16 +++- lib/pleroma/user.ex | 10 +++ .../activity_pub/activity_pub_controller.ex | 29 ++++++- .../20190912065617_create_deliveries.exs | 12 +++ .../activity_pub_controller_test.exs | 83 +++++++++++++++++++ 6 files changed, 206 insertions(+), 2 deletions(-) create mode 100644 lib/pleroma/delivery.ex create mode 100644 priv/repo/migrations/20190912065617_create_deliveries.exs diff --git a/lib/pleroma/delivery.ex b/lib/pleroma/delivery.ex new file mode 100644 index 00000000..f9a9e35c --- /dev/null +++ b/lib/pleroma/delivery.ex @@ -0,0 +1,58 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Delivery do + use Ecto.Schema + + alias Pleroma.Delivery + alias Pleroma.FlakeId + alias Pleroma.User + alias Pleroma.Repo + alias Pleroma.Object + alias Pleroma.User + + import Ecto.Changeset + import Ecto.Query + + schema "deliveries" do + belongs_to(:user, User, type: FlakeId) + belongs_to(:object, Object) + end + + def changeset(delivery, params \\ %{}) do + delivery + |> cast(params, [:user_id, :object_id]) + |> foreign_key_constraint(:object_id) + |> foreign_key_constraint(:user_id) + |> unique_constraint(:user_id, name: :deliveries_user_id_object_id_index) + end + + def create(object_id, user_id) do + %Delivery{} + |> changeset(%{user_id: user_id, object_id: object_id}) + |> Repo.insert() + end + + def get(object_id, user_id) do + from(d in Delivery, where: d.user_id == ^user_id and d.object_id == ^object_id) + |> Repo.one() + end + + def get_or_create(object_id, user_id) do + case get(object_id, user_id) do + %Delivery{} = delivery -> {:ok, delivery} + nil -> create(object_id, user_id) + end + end + + def delete_all_by_object_id(object_id) do + from(d in Delivery, where: d.object_id == ^object_id) + |> Repo.delete_all() + end + + def get_all_by_object_id(object_id) do + from(d in Delivery, where: d.object_id == ^object_id) + |> Repo.all() + end +end diff --git a/lib/pleroma/plugs/cache.ex b/lib/pleroma/plugs/cache.ex index a81a861d..42d77fc1 100644 --- a/lib/pleroma/plugs/cache.ex +++ b/lib/pleroma/plugs/cache.ex @@ -20,6 +20,7 @@ defmodule Pleroma.Plugs.Cache do - `ttl`: An expiration time (time-to-live). This value should be in milliseconds or `nil` to disable expiration. Defaults to `nil`. - `query_params`: Take URL query string into account (`true`), ignore it (`false`) or limit to specific params only (list). Defaults to `true`. + - `tracking_fun`: A function that is called on successfull responses, no matter if the request is cached or not. It should accept a conn as the first argument and the value assigned to `tracking_fun_data` as the second. Additionally, you can overwrite the TTL inside a controller action by assigning `cache_ttl` to the connection struct: @@ -56,6 +57,10 @@ defmodule Pleroma.Plugs.Cache do {:ok, nil} -> cache_resp(conn, opts) + {:ok, {content_type, body, tracking_fun_data}} -> + conn = opts.tracking_fun(conn, tracking_fun_data) + send_cached(conn, {content_type, body}) + {:ok, record} -> send_cached(conn, record) @@ -90,7 +95,16 @@ defmodule Pleroma.Plugs.Cache do content_type = content_type(conn) record = {content_type, body} - Cachex.put(:web_resp_cache, key, record, ttl: ttl) + conn = + unless opts[:tracking_fun] do + Cachex.put(:web_resp_cache, key, {content_type, body}, ttl: ttl) + conn + else + tracking_fun_data = Map.get(conn.assigns, :tracking_fun_data, nil) + Cachex.put(:web_resp_cache, {content_type, body, tracking_fun_data}, record, ttl: ttl) + + opts.tracking_fun.(conn, tracking_fun_data) + end put_resp_header(conn, "x-cache", "MISS from Pleroma") diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 3aa245f2..9614acda 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -11,6 +11,7 @@ defmodule Pleroma.User do alias Comeonin.Pbkdf2 alias Ecto.Multi alias Pleroma.Activity + alias Pleroma.Delivery alias Pleroma.Keys alias Pleroma.Notification alias Pleroma.Object @@ -61,6 +62,7 @@ defmodule Pleroma.User do field(:last_digest_emailed_at, :naive_datetime) has_many(:notifications, Notification) has_many(:registrations, Registration) + has_many(:deliveries, Delivery) embeds_one(:info, User.Info) timestamps() @@ -1624,4 +1626,12 @@ defmodule Pleroma.User do def is_internal_user?(%User{nickname: nil}), do: true def is_internal_user?(%User{local: true, nickname: "internal." <> _}), do: true def is_internal_user?(_), do: false + + def get_delivered_users_by_object_id(object_id) do + from(u in User, + inner_join: delivery in assoc(u, :deliveries), + where: delivery.object_id == ^object_id + ) + |> Repo.all() + end end diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 705dbc1c..009260d3 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do use Pleroma.Web, :controller alias Pleroma.Activity + alias Pleroma.Delivery alias Pleroma.Object alias Pleroma.Object.Fetcher alias Pleroma.User @@ -23,7 +24,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do action_fallback(:errors) - plug(Pleroma.Plugs.Cache, [query_params: false] when action in [:activity, :object]) + plug( + Pleroma.Plugs.Cache, + [ + query_params: false, + tracking_fun: &Pleroma.Web.ActivityPub.ActivityPubController.track_object_fetch/2 + ] + when action in [:activity, :object] + ) + plug(Pleroma.Web.FederatingPlug when action in [:inbox, :relay]) plug(:set_requester_reachable when action in [:inbox]) plug(:relay_active? when action in [:relay]) @@ -54,6 +63,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do %Object{} = object <- Object.get_cached_by_ap_id(ap_id), {_, true} <- {:public?, Visibility.is_public?(object)} do conn + |> assign(:tracking_fun_data, object.id) |> set_cache_ttl_for(object) |> put_resp_content_type("application/activity+json") |> put_view(ObjectView) @@ -64,6 +74,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do end end + def track_object_fetch(conn, object_id) do + case conn.assigns[:user] do + %User{id: user_id} -> Delivery.create(object_id, user_id) + _ -> nil + end + + conn + end + def object_likes(conn, %{"uuid" => uuid, "page" => page}) do with ap_id <- o_status_url(conn, :object, uuid), %Object{} = object <- Object.get_cached_by_ap_id(ap_id), @@ -99,6 +118,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do %Activity{} = activity <- Activity.normalize(ap_id), {_, true} <- {:public?, Visibility.is_public?(activity)} do conn + |> maybe_set_tracking_data(activity) |> set_cache_ttl_for(activity) |> put_resp_content_type("application/activity+json") |> put_view(ObjectView) @@ -109,6 +129,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do end end + defp maybe_set_tracking_data(conn, %Activity{data: %{"type" => "Create"}} = activity) do + object_id = Object.normalize(activity).id + assign(conn, :tracking_fun_data, object_id) + end + + defp maybe_set_tracking_data(conn, _activity), do: assign(conn, :tracking_fun_data, nil) + defp set_cache_ttl_for(conn, %Activity{object: object}) do set_cache_ttl_for(conn, object) end diff --git a/priv/repo/migrations/20190912065617_create_deliveries.exs b/priv/repo/migrations/20190912065617_create_deliveries.exs new file mode 100644 index 00000000..92ca5650 --- /dev/null +++ b/priv/repo/migrations/20190912065617_create_deliveries.exs @@ -0,0 +1,12 @@ +defmodule Pleroma.Repo.Migrations.CreateDeliveries do + use Ecto.Migration + + def change do + create_if_not_exists table(:deliveries) do + add(:object_id, references(:objects, type: :id)) + add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) + end + create_if_not_exists index(:deliveries, :object_id, name: :deliveries_object_id) + create_if_not_exists(unique_index(:deliveries, [:user_id, :object_id])) + end +end diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index 9698c709..0bab555b 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do use Pleroma.Web.ConnCase import Pleroma.Factory alias Pleroma.Activity + alias Pleroma.Delivery alias Pleroma.Instances alias Pleroma.Object alias Pleroma.User @@ -885,4 +886,86 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert result["totalItems"] == 15 end end + + describe "delivery tracking" do + test "it tracks a signed object fetch", %{conn: conn} do + user = insert(:user, local: false) + activity = insert(:note_activity) + object = Object.normalize(activity) + + object_path = String.trim_leading(object.data["id"], Pleroma.Web.Endpoint.url()) + + conn + |> put_req_header("accept", "application/activity+json") + |> assign(:user, user) + |> get(object_path) + |> json_response(200) + + assert Delivery.get(object.id, user.id) + end + + test "it tracks a signed activity fetch", %{conn: conn} do + user = insert(:user, local: false) + activity = insert(:note_activity) + object = Object.normalize(activity) + + activity_path = String.trim_leading(activity.data["id"], Pleroma.Web.Endpoint.url()) + + conn + |> put_req_header("accept", "application/activity+json") + |> assign(:user, user) + |> get(activity_path) + |> json_response(200) + + assert Delivery.get(object.id, user.id) + end + + test "it tracks a signed object fetch when the json is cached", %{conn: conn} do + user = insert(:user, local: false) + other_user = insert(:user, local: false) + activity = insert(:note_activity) + object = Object.normalize(activity) + + object_path = String.trim_leading(object.data["id"], Pleroma.Web.Endpoint.url()) + + conn + |> put_req_header("accept", "application/activity+json") + |> assign(:user, user) + |> get(object_path) + |> json_response(200) + + build_conn() + |> put_req_header("accept", "application/activity+json") + |> assign(:user, other_user) + |> get(object_path) + |> json_response(200) + + assert Delivery.get(object.id, user.id) + assert Delivery.get(object.id, other_user.id) + end + + test "it tracks a signed activity fetch when the json is cached", %{conn: conn} do + user = insert(:user, local: false) + other_user = insert(:user, local: false) + activity = insert(:note_activity) + object = Object.normalize(activity) + + activity_path = String.trim_leading(activity.data["id"], Pleroma.Web.Endpoint.url()) + + conn + |> put_req_header("accept", "application/activity+json") + |> assign(:user, user) + |> get(activity_path) + |> json_response(200) + + build_conn() + |> put_req_header("accept", "application/activity+json") + |> assign(:user, other_user) + |> get(activity_path) + |> json_response(200) + + assert Delivery.get(object.id, user.id) + assert Delivery.get(object.id, other_user.id) + end + end end From dabc4a00f5cf08dac75f701457a24fce8735175f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 12 Sep 2019 22:10:15 +0300 Subject: [PATCH 063/138] Put the cache with the right key when using a tracking function --- lib/pleroma/plugs/cache.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/plugs/cache.ex b/lib/pleroma/plugs/cache.ex index 42d77fc1..50b534e7 100644 --- a/lib/pleroma/plugs/cache.ex +++ b/lib/pleroma/plugs/cache.ex @@ -58,7 +58,8 @@ defmodule Pleroma.Plugs.Cache do cache_resp(conn, opts) {:ok, {content_type, body, tracking_fun_data}} -> - conn = opts.tracking_fun(conn, tracking_fun_data) + conn = opts.tracking_fun.(conn, tracking_fun_data) + send_cached(conn, {content_type, body}) {:ok, record} -> @@ -93,7 +94,6 @@ defmodule Pleroma.Plugs.Cache do ttl = Map.get(conn.assigns, :cache_ttl, opts.ttl) key = cache_key(conn, opts) content_type = content_type(conn) - record = {content_type, body} conn = unless opts[:tracking_fun] do @@ -101,7 +101,7 @@ defmodule Pleroma.Plugs.Cache do conn else tracking_fun_data = Map.get(conn.assigns, :tracking_fun_data, nil) - Cachex.put(:web_resp_cache, {content_type, body, tracking_fun_data}, record, ttl: ttl) + Cachex.put(:web_resp_cache, key, {content_type, body, tracking_fun_data}, ttl: ttl) opts.tracking_fun.(conn, tracking_fun_data) end From b0e60580215e26caae6452099fa1fbace525937c Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 12 Sep 2019 22:40:53 +0300 Subject: [PATCH 064/138] Parse http signature for request to objects/activities --- lib/pleroma/plugs/http_signature.ex | 3 ++- lib/pleroma/web/router.ex | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/plugs/http_signature.ex b/lib/pleroma/plugs/http_signature.ex index d87fa52f..23d22a71 100644 --- a/lib/pleroma/plugs/http_signature.ex +++ b/lib/pleroma/plugs/http_signature.ex @@ -15,7 +15,8 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do end def call(conn, _opts) do - [signature | _] = get_req_header(conn, "signature") + headers = get_req_header(conn, "signature") + signature = Enum.at(headers, 0) if signature do # set (request-target) header to the appropriate value diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 7cd59acb..badc7e04 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -135,6 +135,7 @@ defmodule Pleroma.Web.Router do pipeline :http_signature do plug(Pleroma.Web.Plugs.HTTPSignaturePlug) + plug(Pleroma.Web.Plugs.MappedSignatureToIdentityPlug) end scope "/api/pleroma", Pleroma.Web.TwitterAPI do @@ -513,6 +514,7 @@ defmodule Pleroma.Web.Router do scope "/", Pleroma.Web do pipe_through(:ostatus) + pipe_through(:http_signature) get("/objects/:uuid", OStatus.OStatusController, :object) get("/activities/:uuid", OStatus.OStatusController, :activity) From 528a88a68665a5bc9ed2d19ccfe99119839fe6e5 Mon Sep 17 00:00:00 2001 From: Angelina Filippova Date: Fri, 13 Sep 2019 03:31:16 +0000 Subject: [PATCH 065/138] Fix admin api docs for creating users --- docs/api/admin_api.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index d79c342b..fd608c45 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -60,9 +60,13 @@ Authentication is required and the user must be an admin. - Method: `POST` - Params: - - `nickname` - - `email` - - `password` + `users`: [ + { + `nickname`, + `email`, + `password` + } + ] - Response: User’s nickname ## `/api/pleroma/admin/users/follow` From 39dc9b470c7ad8348a13f181039f11d14a42fa2b Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 3 Sep 2019 21:58:30 +0700 Subject: [PATCH 066/138] Cleanup Pleroma.Activity and Pleroma.Web.ActivityPub.Utils --- lib/pleroma/activity.ex | 193 ++++++-------------------- lib/pleroma/activity/queries.ex | 32 ++++- lib/pleroma/user.ex | 2 +- lib/pleroma/web/activity_pub/utils.ex | 167 ++++++++-------------- test/user_test.exs | 2 +- 5 files changed, 128 insertions(+), 268 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 44f1e301..ec558168 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Activity do use Ecto.Schema alias Pleroma.Activity + alias Pleroma.Activity.Queries alias Pleroma.ActivityExpiration alias Pleroma.Bookmark alias Pleroma.Notification @@ -65,8 +66,8 @@ defmodule Pleroma.Activity do timestamps() end - def with_joined_object(query) do - join(query, :inner, [activity], o in Object, + def with_joined_object(query, join_type \\ :inner) do + join(query, join_type, [activity], o in Object, on: fragment( "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')", @@ -78,10 +79,10 @@ defmodule Pleroma.Activity do ) end - def with_preloaded_object(query) do + def with_preloaded_object(query, join_type \\ :inner) do query |> has_named_binding?(:object) - |> if(do: query, else: with_joined_object(query)) + |> if(do: query, else: with_joined_object(query, join_type)) |> preload([activity, object: object], object: object) end @@ -107,12 +108,9 @@ defmodule Pleroma.Activity do def with_set_thread_muted_field(query, _), do: query def get_by_ap_id(ap_id) do - Repo.one( - from( - activity in Activity, - where: fragment("(?)->>'id' = ?", activity.data, ^to_string(ap_id)) - ) - ) + ap_id + |> Queries.by_ap_id() + |> Repo.one() end def get_bookmark(%Activity{} = activity, %User{} = user) do @@ -133,21 +131,10 @@ defmodule Pleroma.Activity do end def get_by_ap_id_with_object(ap_id) do - Repo.one( - from( - activity in Activity, - where: fragment("(?)->>'id' = ?", activity.data, ^to_string(ap_id)), - left_join: o in Object, - on: - fragment( - "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')", - o.data, - activity.data, - activity.data - ), - preload: [object: o] - ) - ) + ap_id + |> Queries.by_ap_id() + |> with_preloaded_object(:left) + |> Repo.one() end def get_by_id(id) do @@ -158,18 +145,9 @@ defmodule Pleroma.Activity do end def get_by_id_with_object(id) do - from(activity in Activity, - where: activity.id == ^id, - inner_join: o in Object, - on: - fragment( - "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')", - o.data, - activity.data, - activity.data - ), - preload: [object: o] - ) + Activity + |> where(id: ^id) + |> with_preloaded_object() |> Repo.one() end @@ -180,51 +158,21 @@ defmodule Pleroma.Activity do |> Repo.all() end - def by_object_ap_id(ap_id) do - from( - activity in Activity, - where: - fragment( - "coalesce((?)->'object'->>'id', (?)->>'object') = ?", - activity.data, - activity.data, - ^to_string(ap_id) - ) - ) + @doc """ + Accepts `ap_id` or list of `ap_id`. + Returns a query. + """ + @spec create_by_object_ap_id(String.t() | [String.t()]) :: Ecto.Queryable.t() + def create_by_object_ap_id(ap_id) do + ap_id + |> Queries.by_object_id() + |> Queries.by_type("Create") end - def create_by_object_ap_id(ap_ids) when is_list(ap_ids) do - from( - activity in Activity, - where: - fragment( - "coalesce((?)->'object'->>'id', (?)->>'object') = ANY(?)", - activity.data, - activity.data, - ^ap_ids - ), - where: fragment("(?)->>'type' = 'Create'", activity.data) - ) - end - - def create_by_object_ap_id(ap_id) when is_binary(ap_id) do - from( - activity in Activity, - where: - fragment( - "coalesce((?)->'object'->>'id', (?)->>'object') = ?", - activity.data, - activity.data, - ^to_string(ap_id) - ), - where: fragment("(?)->>'type' = 'Create'", activity.data) - ) - end - - def create_by_object_ap_id(_), do: nil - def get_all_create_by_object_ap_id(ap_id) do - Repo.all(create_by_object_ap_id(ap_id)) + ap_id + |> create_by_object_ap_id() + |> Repo.all() end def get_create_by_object_ap_id(ap_id) when is_binary(ap_id) do @@ -235,54 +183,17 @@ defmodule Pleroma.Activity do def get_create_by_object_ap_id(_), do: nil - def create_by_object_ap_id_with_object(ap_ids) when is_list(ap_ids) do - from( - activity in Activity, - where: - fragment( - "coalesce((?)->'object'->>'id', (?)->>'object') = ANY(?)", - activity.data, - activity.data, - ^ap_ids - ), - where: fragment("(?)->>'type' = 'Create'", activity.data), - inner_join: o in Object, - on: - fragment( - "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')", - o.data, - activity.data, - activity.data - ), - preload: [object: o] - ) + @doc """ + Accepts `ap_id` or list of `ap_id`. + Returns a query. + """ + @spec create_by_object_ap_id_with_object(String.t() | [String.t()]) :: Ecto.Queryable.t() + def create_by_object_ap_id_with_object(ap_id) do + ap_id + |> create_by_object_ap_id() + |> with_preloaded_object() end - def create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do - from( - activity in Activity, - where: - fragment( - "coalesce((?)->'object'->>'id', (?)->>'object') = ?", - activity.data, - activity.data, - ^to_string(ap_id) - ), - where: fragment("(?)->>'type' = 'Create'", activity.data), - inner_join: o in Object, - on: - fragment( - "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')", - o.data, - activity.data, - activity.data - ), - preload: [object: o] - ) - end - - def create_by_object_ap_id_with_object(_), do: nil - def get_create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do ap_id |> create_by_object_ap_id_with_object() @@ -306,7 +217,8 @@ defmodule Pleroma.Activity do def normalize(_), do: nil def delete_by_ap_id(id) when is_binary(id) do - by_object_ap_id(id) + id + |> Queries.by_object_id() |> select([u], u) |> Repo.delete_all() |> elem(1) @@ -350,31 +262,10 @@ defmodule Pleroma.Activity do end def follow_requests_for_actor(%Pleroma.User{ap_id: ap_id}) do - from( - a in Activity, - where: - fragment( - "? ->> 'type' = 'Follow'", - a.data - ), - where: - fragment( - "? ->> 'state' = 'pending'", - a.data - ), - where: - fragment( - "coalesce((?)->'object'->>'id', (?)->>'object') = ?", - a.data, - a.data, - ^ap_id - ) - ) - end - - @spec query_by_actor(actor()) :: Ecto.Query.t() - def query_by_actor(actor) do - from(a in Activity, where: a.actor == ^actor) + ap_id + |> Queries.by_object_id() + |> Queries.by_type("Follow") + |> where([a], fragment("? ->> 'state' = 'pending'", a.data)) end def restrict_deactivated_users(query) do diff --git a/lib/pleroma/activity/queries.ex b/lib/pleroma/activity/queries.ex index aa5b2956..13fa3383 100644 --- a/lib/pleroma/activity/queries.ex +++ b/lib/pleroma/activity/queries.ex @@ -13,6 +13,14 @@ defmodule Pleroma.Activity.Queries do alias Pleroma.Activity + @spec by_ap_id(query, String.t()) :: query + def by_ap_id(query \\ Activity, ap_id) do + from( + activity in query, + where: fragment("(?)->>'id' = ?", activity.data, ^to_string(ap_id)) + ) + end + @spec by_actor(query, String.t()) :: query def by_actor(query \\ Activity, actor) do from( @@ -21,8 +29,23 @@ defmodule Pleroma.Activity.Queries do ) end - @spec by_object_id(query, String.t()) :: query - def by_object_id(query \\ Activity, object_id) do + @spec by_object_id(query, String.t() | [String.t()]) :: query + def by_object_id(query \\ Activity, object_id) + + def by_object_id(query, object_ids) when is_list(object_ids) do + from( + activity in query, + where: + fragment( + "coalesce((?)->'object'->>'id', (?)->>'object') = ANY(?)", + activity.data, + activity.data, + ^object_ids + ) + ) + end + + def by_object_id(query, object_id) when is_binary(object_id) do from(activity in query, where: fragment( @@ -41,9 +64,4 @@ defmodule Pleroma.Activity.Queries do where: fragment("(?)->>'type' = ?", activity.data, ^activity_type) ) end - - @spec limit(query, pos_integer()) :: query - def limit(query \\ Activity, limit) do - from(activity in query, limit: ^limit) - end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 3aa245f2..ceca11de 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1219,7 +1219,7 @@ defmodule Pleroma.User do def delete_user_activities(%User{ap_id: ap_id} = user) do ap_id - |> Activity.query_by_actor() + |> Activity.Queries.by_actor() |> RepoStreamer.chunk_stream(50) |> Stream.each(fn activities -> Enum.each(activities, &delete_activity(&1)) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index c9c0c376..47917f5d 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -85,15 +85,13 @@ defmodule Pleroma.Web.ActivityPub.Utils do defp extract_list(_), do: [] def maybe_splice_recipient(ap_id, params) do - need_splice = + need_splice? = !recipient_in_collection(ap_id, params["to"]) && !recipient_in_collection(ap_id, params["cc"]) - cc_list = extract_list(params["cc"]) - - if need_splice do - params - |> Map.put("cc", [ap_id | cc_list]) + if need_splice? do + cc_list = extract_list(params["cc"]) + Map.put(params, "cc", [ap_id | cc_list]) else params end @@ -139,7 +137,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do "object" => object } - Notification.get_notified_from_activity(%Activity{data: fake_create_activity}, false) + get_notified_from_object(fake_create_activity) end def get_notified_from_object(object) do @@ -188,9 +186,9 @@ defmodule Pleroma.Web.ActivityPub.Utils do Adds an id and a published data if they aren't there, also adds it to an included object """ - def lazy_put_activity_defaults(map, fake \\ false) do + def lazy_put_activity_defaults(map, fake? \\ false) do map = - unless fake do + if not fake? do %{data: %{"id" => context}, id: context_id} = create_context(map["context"]) map @@ -207,7 +205,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do end if is_map(map["object"]) do - object = lazy_put_object_defaults(map["object"], map, fake) + object = lazy_put_object_defaults(map["object"], map, fake?) %{map | "object" => object} else map @@ -217,9 +215,9 @@ defmodule Pleroma.Web.ActivityPub.Utils do @doc """ Adds an id and published date if they aren't there. """ - def lazy_put_object_defaults(map, activity \\ %{}, fake) + def lazy_put_object_defaults(map, activity \\ %{}, fake?) - def lazy_put_object_defaults(map, activity, true = _fake) do + def lazy_put_object_defaults(map, activity, true = _fake?) do map |> Map.put_new_lazy("published", &make_date/0) |> Map.put_new("id", "pleroma:fake_object_id") @@ -228,7 +226,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do |> Map.put_new("context_id", activity["context_id"]) end - def lazy_put_object_defaults(map, activity, _fake) do + def lazy_put_object_defaults(map, activity, _fake?) do map |> Map.put_new_lazy("id", &generate_object_id/0) |> Map.put_new_lazy("published", &make_date/0) @@ -242,9 +240,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do def insert_full_object(%{"object" => %{"type" => type} = object_data} = map) when is_map(object_data) and type in @supported_object_types do with {:ok, object} <- Object.create(object_data) do - map = - map - |> Map.put("object", object.data["id"]) + map = Map.put(map, "object", object.data["id"]) {:ok, map, object} end @@ -263,7 +259,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do |> Activity.Queries.by_actor() |> Activity.Queries.by_object_id(id) |> Activity.Queries.by_type("Like") - |> Activity.Queries.limit(1) + |> limit(1) |> Repo.one() end @@ -380,12 +376,11 @@ defmodule Pleroma.Web.ActivityPub.Utils do %Activity{data: %{"actor" => actor, "object" => object}} = activity, state ) do - with new_data <- - activity.data - |> Map.put("state", state), - changeset <- Changeset.change(activity, data: new_data), - {:ok, activity} <- Repo.update(changeset), - _ <- User.set_follow_state_cache(actor, object, state) do + new_data = Map.put(activity.data, "state", state) + changeset = Changeset.change(activity, data: new_data) + + with {:ok, activity} <- Repo.update(changeset) do + User.set_follow_state_cache(actor, object, state) {:ok, activity} end end @@ -410,28 +405,14 @@ defmodule Pleroma.Web.ActivityPub.Utils do end def fetch_latest_follow(%User{ap_id: follower_id}, %User{ap_id: followed_id}) do - query = - from( - activity in Activity, - where: - fragment( - "? ->> 'type' = 'Follow'", - activity.data - ), - where: activity.actor == ^follower_id, - # this is to use the index - where: - fragment( - "coalesce((?)->'object'->>'id', (?)->>'object') = ?", - activity.data, - activity.data, - ^followed_id - ), - order_by: [fragment("? desc nulls last", activity.id)], - limit: 1 - ) - - Repo.one(query) + "Follow" + |> Activity.Queries.by_type() + |> where(actor: ^follower_id) + # this is to use the index + |> Activity.Queries.by_object_id(followed_id) + |> order_by([activity], fragment("? desc nulls last", activity.id)) + |> limit(1) + |> Repo.one() end #### Announce-related helpers @@ -439,23 +420,13 @@ defmodule Pleroma.Web.ActivityPub.Utils do @doc """ Retruns an existing announce activity if the notice has already been announced """ - def get_existing_announce(actor, %{data: %{"id" => id}}) do - query = - from( - activity in Activity, - where: activity.actor == ^actor, - # this is to use the index - where: - fragment( - "coalesce((?)->'object'->>'id', (?)->>'object') = ?", - activity.data, - activity.data, - ^id - ), - where: fragment("(?)->>'type' = 'Announce'", activity.data) - ) - - Repo.one(query) + def get_existing_announce(actor, %{data: %{"id" => ap_id}}) do + "Announce" + |> Activity.Queries.by_type() + |> where(actor: ^actor) + # this is to use the index + |> Activity.Queries.by_object_id(ap_id) + |> Repo.one() end @doc """ @@ -538,11 +509,13 @@ defmodule Pleroma.Web.ActivityPub.Utils do object ) do announcements = - if is_list(object.data["announcements"]), do: object.data["announcements"], else: [] + if is_list(object.data["announcements"]) do + Enum.uniq([actor | object.data["announcements"]]) + else + [actor] + end - with announcements <- [actor | announcements] |> Enum.uniq() do - update_element_in_object("announcement", announcements, object) - end + update_element_in_object("announcement", announcements, object) end def add_announce_to_object(_, object), do: {:ok, object} @@ -570,28 +543,14 @@ defmodule Pleroma.Web.ActivityPub.Utils do #### Block-related helpers def fetch_latest_block(%User{ap_id: blocker_id}, %User{ap_id: blocked_id}) do - query = - from( - activity in Activity, - where: - fragment( - "? ->> 'type' = 'Block'", - activity.data - ), - where: activity.actor == ^blocker_id, - # this is to use the index - where: - fragment( - "coalesce((?)->'object'->>'id', (?)->>'object') = ?", - activity.data, - activity.data, - ^blocked_id - ), - order_by: [fragment("? desc nulls last", activity.id)], - limit: 1 - ) - - Repo.one(query) + "Block" + |> Activity.Queries.by_type() + |> where(actor: ^blocker_id) + # this is to use the index + |> Activity.Queries.by_object_id(blocked_id) + |> order_by([activity], fragment("? desc nulls last", activity.id)) + |> limit(1) + |> Repo.one() end def make_block_data(blocker, blocked, activity_id) do @@ -695,11 +654,11 @@ defmodule Pleroma.Web.ActivityPub.Utils do #### Report-related helpers def update_report_state(%Activity{} = activity, state) when state in @supported_report_states do - with new_data <- Map.put(activity.data, "state", state), - changeset <- Changeset.change(activity, data: new_data), - {:ok, activity} <- Repo.update(changeset) do - {:ok, activity} - end + new_data = Map.put(activity.data, "state", state) + + activity + |> Changeset.change(data: new_data) + |> Repo.update() end def update_report_state(_, _), do: {:error, "Unsupported state"} @@ -766,21 +725,13 @@ defmodule Pleroma.Web.ActivityPub.Utils do end def get_existing_votes(actor, %{data: %{"id" => id}}) do - query = - from( - [activity, object: object] in Activity.with_preloaded_object(Activity), - where: fragment("(?)->>'type' = 'Create'", activity.data), - where: fragment("(?)->>'actor' = ?", activity.data, ^actor), - where: - fragment( - "(?)->>'inReplyTo' = ?", - object.data, - ^to_string(id) - ), - where: fragment("(?)->>'type' = 'Answer'", object.data) - ) - - Repo.all(query) + actor + |> Activity.Queries.by_actor() + |> Activity.Queries.by_type("Create") + |> Activity.with_preloaded_object() + |> where([a, object: o], fragment("(?)->>'inReplyTo' = ?", o.data, ^to_string(id))) + |> where([a, object: o], fragment("(?)->>'type' = 'Answer'", o.data)) + |> Repo.all() end defp maybe_put(map, _key, nil), do: map diff --git a/test/user_test.exs b/test/user_test.exs index a25b72f4..206258fe 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1081,7 +1081,7 @@ defmodule Pleroma.UserTest do user_activities = user.ap_id - |> Activity.query_by_actor() + |> Activity.Queries.by_actor() |> Repo.all() |> Enum.map(fn act -> act.data["type"] end) From 5bfbad13ad4dd009b172748d81f56ead21c700de Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 3 Sep 2019 21:33:02 +0700 Subject: [PATCH 067/138] Add more tests for Pleroma.Activity --- test/activity_test.exs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/test/activity_test.exs b/test/activity_test.exs index 4152aaa7..f9f789a7 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -185,4 +185,35 @@ defmodule Pleroma.ActivityTest do assert [%{id: ^id1, object: %Object{}}, %{id: ^id2, object: %Object{}}] = activities end + + test "get_by_id_with_object/1" do + %{id: id} = insert(:note_activity) + + assert %Activity{id: ^id, object: %Object{}} = Activity.get_by_id_with_object(id) + end + + test "get_by_ap_id_with_object/1" do + %{data: %{"id" => ap_id}} = insert(:note_activity) + + assert %Activity{data: %{"id" => ^ap_id}, object: %Object{}} = + Activity.get_by_ap_id_with_object(ap_id) + end + + test "get_by_id/1" do + %{id: id} = insert(:note_activity) + + assert %Activity{id: ^id} = Activity.get_by_id(id) + end + + test "all_by_actor_and_id/2" do + user = insert(:user) + + {:ok, %{id: id1}} = Pleroma.Web.CommonAPI.post(user, %{"status" => "cofe"}) + {:ok, %{id: id2}} = Pleroma.Web.CommonAPI.post(user, %{"status" => "cofefe"}) + + assert [] == Activity.all_by_actor_and_id(user, []) + + assert [%Activity{id: ^id2}, %Activity{id: ^id1}] = + Activity.all_by_actor_and_id(user.ap_id, [id1, id2]) + end end From 25d8216804c7742cd8549799a7785723f2a70afa Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 13 Sep 2019 13:09:35 +0700 Subject: [PATCH 068/138] Add email change endpoint --- CHANGELOG.md | 1 + docs/api/pleroma_api.md | 11 +- lib/pleroma/user.ex | 9 ++ lib/pleroma/web/router.ex | 1 + .../controllers/util_controller.ex | 19 ++++ test/user_test.exs | 27 +++++ test/web/twitter_api/util_controller_test.exs | 107 ++++++++++++++++++ 7 files changed, 174 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7f1aee0..d5bb2e07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,6 +105,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - ActivityPub: Optional signing of ActivityPub object fetches. - Admin API: Endpoint for fetching latest user's statuses - Pleroma API: Add `/api/v1/pleroma/accounts/confirmation_resend?email=` for resending account confirmation. +- Pleroma API: Email change endpoint. - Relays: Added a task to list relay subscriptions. - Mix Tasks: `mix pleroma.database fix_likes_collections` - Federation: Remove `likes` from objects. diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md index 7d343e97..8a726a7c 100644 --- a/docs/api/pleroma_api.md +++ b/docs/api/pleroma_api.md @@ -252,7 +252,7 @@ See [Admin-API](Admin-API.md) * Params: * `email`: email of that needs to be verified * Authentication: not required -* Response: 204 No Content +* Response: 204 No Content ## `/api/v1/pleroma/mascot` ### Gets user mascot image @@ -321,6 +321,15 @@ See [Admin-API](Admin-API.md) } ``` +## `/api/pleroma/change_email` +### Change account email +* Method `POST` +* Authentication: required +* Params: + * `password`: user's password + * `email`: new email +* Response: JSON. Returns `{"status": "success"}` if the change was successful, `{"error": "[error message]"}` otherwise + # Pleroma Conversations Pleroma Conversations have the same general structure that Mastodon Conversations have. The behavior differs in the following ways when using these endpoints: diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 3aa245f2..1f6a75d0 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1624,4 +1624,13 @@ defmodule Pleroma.User do def is_internal_user?(%User{nickname: nil}), do: true def is_internal_user?(%User{local: true, nickname: "internal." <> _}), do: true def is_internal_user?(_), do: false + + def change_email(user, email) do + user + |> cast(%{email: email}, [:email]) + |> validate_required([:email]) + |> unique_constraint(:email) + |> validate_format(:email, @email_regex) + |> update_and_set_cache() + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 7cd59acb..b0464037 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -224,6 +224,7 @@ defmodule Pleroma.Web.Router do scope [] do pipe_through(:oauth_write) + post("/change_email", UtilController, :change_email) post("/change_password", UtilController, :change_password) post("/delete_account", UtilController, :delete_account) put("/notification_settings", UtilController, :update_notificaton_settings) diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 3405bd3b..867787c5 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -314,6 +314,25 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end end + def change_email(%{assigns: %{user: user}} = conn, params) do + case CommonAPI.Utils.confirm_current_password(user, params["password"]) do + {:ok, user} -> + with {:ok, _user} <- User.change_email(user, params["email"]) do + json(conn, %{status: "success"}) + else + {:error, changeset} -> + {_, {error, _}} = Enum.at(changeset.errors, 0) + json(conn, %{error: "Email #{error}."}) + + _ -> + json(conn, %{error: "Unable to change email."}) + end + + {:error, msg} -> + json(conn, %{error: msg}) + end + end + def delete_account(%{assigns: %{user: user}} = conn, params) do case CommonAPI.Utils.confirm_current_password(user, params["password"]) do {:ok, user} -> diff --git a/test/user_test.exs b/test/user_test.exs index a25b72f4..ed8cdbe3 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1614,4 +1614,31 @@ defmodule Pleroma.UserTest do assert User.user_info(other_user).following_count == 152 end end + + describe "change_email/2" do + setup do + [user: insert(:user)] + end + + test "blank email returns error", %{user: user} do + assert {:error, %{errors: [email: {"can't be blank", _}]}} = User.change_email(user, "") + assert {:error, %{errors: [email: {"can't be blank", _}]}} = User.change_email(user, nil) + end + + test "non unique email returns error", %{user: user} do + %{email: email} = insert(:user) + + assert {:error, %{errors: [email: {"has already been taken", _}]}} = + User.change_email(user, email) + end + + test "invalid email returns error", %{user: user} do + assert {:error, %{errors: [email: {"has invalid format", _}]}} = + User.change_email(user, "cofe") + end + + test "changes email", %{user: user} do + assert {:ok, %User{email: "cofe@cofe.party"}} = User.change_email(user, "cofe@cofe.party") + end + end end diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index cf8e69d2..a3c6145c 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -662,4 +662,111 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do assert called(Pleroma.Captcha.new()) end end + + defp with_credentials(conn, username, password) do + header_content = "Basic " <> Base.encode64("#{username}:#{password}") + put_req_header(conn, "authorization", header_content) + end + + defp valid_user(_context) do + user = insert(:user) + [user: user] + end + + describe "POST /api/pleroma/change_email" do + setup [:valid_user] + + test "without credentials", %{conn: conn} do + conn = post(conn, "/api/pleroma/change_email") + assert json_response(conn, 403) == %{"error" => "Invalid credentials."} + end + + test "with credentials and invalid password", %{conn: conn, user: current_user} do + conn = + conn + |> with_credentials(current_user.nickname, "test") + |> post("/api/pleroma/change_email", %{ + "password" => "hi", + "email" => "test@test.com" + }) + + assert json_response(conn, 200) == %{"error" => "Invalid password."} + end + + test "with credentials, valid password and invalid email", %{ + conn: conn, + user: current_user + } do + conn = + conn + |> with_credentials(current_user.nickname, "test") + |> post("/api/pleroma/change_email", %{ + "password" => "test", + "email" => "foobar" + }) + + assert json_response(conn, 200) == %{"error" => "Email has invalid format."} + end + + test "with credentials, valid password and no email", %{ + conn: conn, + user: current_user + } do + conn = + conn + |> with_credentials(current_user.nickname, "test") + |> post("/api/pleroma/change_email", %{ + "password" => "test" + }) + + assert json_response(conn, 200) == %{"error" => "Email can't be blank."} + end + + test "with credentials, valid password and blank email", %{ + conn: conn, + user: current_user + } do + conn = + conn + |> with_credentials(current_user.nickname, "test") + |> post("/api/pleroma/change_email", %{ + "password" => "test", + "email" => "" + }) + + assert json_response(conn, 200) == %{"error" => "Email can't be blank."} + end + + test "with credentials, valid password and non unique email", %{ + conn: conn, + user: current_user + } do + user = insert(:user) + + conn = + conn + |> with_credentials(current_user.nickname, "test") + |> post("/api/pleroma/change_email", %{ + "password" => "test", + "email" => user.email + }) + + assert json_response(conn, 200) == %{"error" => "Email has already been taken."} + end + + test "with credentials, valid password and valid email", %{ + conn: conn, + user: current_user + } do + conn = + conn + |> with_credentials(current_user.nickname, "test") + |> post("/api/pleroma/change_email", %{ + "password" => "test", + "email" => "cofe@foobar.com" + }) + + assert json_response(conn, 200) == %{"status" => "success"} + end + end end From 7b5c81b3918b7ff99f00e72194075f43c3f81165 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 28 Aug 2019 14:50:58 +0700 Subject: [PATCH 069/138] Add a note about compatibility with Mastodon --- docs/api/pleroma_api.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md index 8a726a7c..30fac77d 100644 --- a/docs/api/pleroma_api.md +++ b/docs/api/pleroma_api.md @@ -329,12 +329,13 @@ See [Admin-API](Admin-API.md) * `password`: user's password * `email`: new email * Response: JSON. Returns `{"status": "success"}` if the change was successful, `{"error": "[error message]"}` otherwise +* Note: Currently, Mastodon has no API for changing email. If they add it in future it might be incompatible with Pleroma. # Pleroma Conversations Pleroma Conversations have the same general structure that Mastodon Conversations have. The behavior differs in the following ways when using these endpoints: -1. Pleroma Conversations never add or remove recipients, unless explicitly changed by the user. +1. Pleroma Conversations never add or remove recipients, unless explicitly changed by the user. 2. Pleroma Conversations statuses can be requested by Conversation id. 3. Pleroma Conversations can be replied to. From ce23529d917c1830b270a29e774e4ed7768bfeff Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 13 Sep 2019 11:36:49 +0300 Subject: [PATCH 070/138] Use delivery info when federating deletes --- lib/pleroma/delivery.ex | 4 ++ lib/pleroma/user.ex | 4 ++ lib/pleroma/web/activity_pub/publisher.ex | 15 ++++- test/web/activity_pub/publisher_test.exs | 68 ++++++++++++++++++++++- 4 files changed, 89 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/delivery.ex b/lib/pleroma/delivery.ex index f9a9e35c..2e7c019f 100644 --- a/lib/pleroma/delivery.ex +++ b/lib/pleroma/delivery.ex @@ -46,6 +46,10 @@ defmodule Pleroma.Delivery do end end + # A hack because user delete activities have a fake id for whatever reason + # TODO: Get rid of this + def delete_all_by_object_id("pleroma:fake_object_id"), do: {0, []} + def delete_all_by_object_id(object_id) do from(d in Delivery, where: d.object_id == ^object_id) |> Repo.delete_all() diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 9614acda..785b2264 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1627,6 +1627,10 @@ defmodule Pleroma.User do def is_internal_user?(%User{local: true, nickname: "internal." <> _}), do: true def is_internal_user?(_), do: false + # A hack because user delete activities have a fake id for whatever reason + # TODO: Get rid of this + def get_delivered_users_by_object_id("pleroma:fake_object_id"), do: [] + def get_delivered_users_by_object_id(object_id) do from(u in User, inner_join: delivery in assoc(u, :deliveries), diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index c9740569..db64fd2f 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -5,9 +5,11 @@ defmodule Pleroma.Web.ActivityPub.Publisher do alias Pleroma.Activity alias Pleroma.Config + alias Pleroma.Delivery alias Pleroma.HTTP alias Pleroma.Instances alias Pleroma.User + alias Pleroma.Object alias Pleroma.Web.ActivityPub.Relay alias Pleroma.Web.ActivityPub.Transmogrifier @@ -107,7 +109,18 @@ defmodule Pleroma.Web.ActivityPub.Publisher do {:ok, []} end - Pleroma.Web.Salmon.remote_users(actor, activity) ++ followers + fetchers = + with %Activity{data: %{"type" => "Delete"}} <- activity, + %Object{id: object_id} <- Object.normalize(activity), + fetchers <- User.get_delivered_users_by_object_id(object_id), + _ <- Delivery.delete_all_by_object_id(object_id) do + fetchers + else + _ -> + [] + end + + Pleroma.Web.Salmon.remote_users(actor, activity) ++ followers ++ fetchers end defp get_cc_ap_ids(ap_id, recipients) do diff --git a/test/web/activity_pub/publisher_test.exs b/test/web/activity_pub/publisher_test.exs index 36a39c84..32b7a242 100644 --- a/test/web/activity_pub/publisher_test.exs +++ b/test/web/activity_pub/publisher_test.exs @@ -3,15 +3,17 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.PublisherTest do - use Pleroma.DataCase + use Pleroma.Web.ConnCase import Pleroma.Factory import Tesla.Mock import Mock alias Pleroma.Activity + alias Pleroma.Object alias Pleroma.Instances alias Pleroma.Web.ActivityPub.Publisher + alias Pleroma.Web.CommonAPI @as_public "https://www.w3.org/ns/activitystreams#Public" @@ -262,5 +264,69 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do }) ) end + + test_with_mock "publishes a delete activity to peers who signed fetch requests to the create acitvity/object.", + Pleroma.Web.Federator.Publisher, + [:passthrough], + [] do + fetcher = + insert(:user, + local: false, + info: %{ + ap_enabled: true, + source_data: %{"inbox" => "https://domain.com/users/nick1/inbox"} + } + ) + + another_fetcher = + insert(:user, + local: false, + info: %{ + ap_enabled: true, + source_data: %{"inbox" => "https://domain2.com/users/nick1/inbox"} + } + ) + + actor = insert(:user) + + note_activity = insert(:note_activity, user: actor) + object = Object.normalize(note_activity) + + activity_path = String.trim_leading(note_activity.data["id"], Pleroma.Web.Endpoint.url()) + object_path = String.trim_leading(object.data["id"], Pleroma.Web.Endpoint.url()) + + build_conn() + |> put_req_header("accept", "application/activity+json") + |> assign(:user, fetcher) + |> get(object_path) + |> json_response(200) + + build_conn() + |> put_req_header("accept", "application/activity+json") + |> assign(:user, another_fetcher) + |> get(activity_path) + |> json_response(200) + + {:ok, delete} = CommonAPI.delete(note_activity.id, actor) + + res = Publisher.publish(actor, delete) + assert res == :ok + + assert called( + Pleroma.Web.Federator.Publisher.enqueue_one(Publisher, %{ + inbox: "https://domain.com/users/nick1/inbox", + actor: actor, + id: delete.data["id"] + }) + ) + + assert called( + Pleroma.Web.Federator.Publisher.enqueue_one(Publisher, %{ + inbox: "https://domain2.com/users/nick1/inbox", + actor: actor, + id: delete.data["id"] + }) + ) + end end end From fb96facc32fb275efffeefa2892a1098ecd68b77 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 13 Sep 2019 12:06:31 +0300 Subject: [PATCH 071/138] Remove unused functions and fix credo issues --- lib/pleroma/delivery.ex | 16 ++-------------- lib/pleroma/web/activity_pub/publisher.ex | 2 +- test/web/activity_pub/publisher_test.exs | 2 +- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/delivery.ex b/lib/pleroma/delivery.ex index 2e7c019f..ce8fb96f 100644 --- a/lib/pleroma/delivery.ex +++ b/lib/pleroma/delivery.ex @@ -7,9 +7,9 @@ defmodule Pleroma.Delivery do alias Pleroma.Delivery alias Pleroma.FlakeId - alias Pleroma.User - alias Pleroma.Repo alias Pleroma.Object + alias Pleroma.Repo + alias Pleroma.User alias Pleroma.User import Ecto.Changeset @@ -39,13 +39,6 @@ defmodule Pleroma.Delivery do |> Repo.one() end - def get_or_create(object_id, user_id) do - case get(object_id, user_id) do - %Delivery{} = delivery -> {:ok, delivery} - nil -> create(object_id, user_id) - end - end - # A hack because user delete activities have a fake id for whatever reason # TODO: Get rid of this def delete_all_by_object_id("pleroma:fake_object_id"), do: {0, []} @@ -54,9 +47,4 @@ defmodule Pleroma.Delivery do from(d in Delivery, where: d.object_id == ^object_id) |> Repo.delete_all() end - - def get_all_by_object_id(object_id) do - from(d in Delivery, where: d.object_id == ^object_id) - |> Repo.all() - end end diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index db64fd2f..c39e89a6 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -8,8 +8,8 @@ defmodule Pleroma.Web.ActivityPub.Publisher do alias Pleroma.Delivery alias Pleroma.HTTP alias Pleroma.Instances - alias Pleroma.User alias Pleroma.Object + alias Pleroma.User alias Pleroma.Web.ActivityPub.Relay alias Pleroma.Web.ActivityPub.Transmogrifier diff --git a/test/web/activity_pub/publisher_test.exs b/test/web/activity_pub/publisher_test.exs index 32b7a242..0ef97464 100644 --- a/test/web/activity_pub/publisher_test.exs +++ b/test/web/activity_pub/publisher_test.exs @@ -10,8 +10,8 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do import Mock alias Pleroma.Activity - alias Pleroma.Object alias Pleroma.Instances + alias Pleroma.Object alias Pleroma.Web.ActivityPub.Publisher alias Pleroma.Web.CommonAPI From 517017048316a52172d60d26b03beddb85af7b39 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 13 Sep 2019 10:09:46 +0000 Subject: [PATCH 072/138] Apply suggestion to lib/pleroma/web/activity_pub/activity_pub_controller.ex --- lib/pleroma/web/activity_pub/activity_pub_controller.ex | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 009260d3..02564172 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -26,10 +26,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do plug( Pleroma.Plugs.Cache, - [ - query_params: false, - tracking_fun: &Pleroma.Web.ActivityPub.ActivityPubController.track_object_fetch/2 - ] + [query_params: false, tracking_fun: &__MODULE__.track_object_fetch/2] when action in [:activity, :object] ) From 3896a51b8aefe6fe54251ffd559c636980faa87e Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 13 Sep 2019 10:09:56 +0000 Subject: [PATCH 073/138] Apply suggestion to lib/pleroma/web/activity_pub/activity_pub_controller.ex --- lib/pleroma/web/activity_pub/activity_pub_controller.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 02564172..4bd13def 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -72,9 +72,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do end def track_object_fetch(conn, object_id) do - case conn.assigns[:user] do - %User{id: user_id} -> Delivery.create(object_id, user_id) - _ -> nil + with %{assigns: %{user: %User{id: user_id}}} <- conn do + Delivery.create(object_id, user_id) end conn From 2784962dba295ee35677e93996df53d1711e5768 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 13 Sep 2019 15:23:03 +0000 Subject: [PATCH 074/138] Apply suggestion to lib/pleroma/web/activity_pub/activity_pub_controller.ex --- lib/pleroma/web/activity_pub/activity_pub_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 4bd13def..70d4a5ba 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -130,7 +130,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do assign(conn, :tracking_fun_data, object_id) end - defp maybe_set_tracking_data(conn, _activity), do: assign(conn, :tracking_fun_data, nil) + defp maybe_set_tracking_data(conn, _activity), do: conn defp set_cache_ttl_for(conn, %Activity{object: object}) do set_cache_ttl_for(conn, object) From 05f8a066a107af2f7151aee8d85af97cf6a4835c Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 13 Sep 2019 15:23:26 +0000 Subject: [PATCH 075/138] Apply suggestion to lib/pleroma/delivery.ex --- lib/pleroma/delivery.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/delivery.ex b/lib/pleroma/delivery.ex index ce8fb96f..38c148c3 100644 --- a/lib/pleroma/delivery.ex +++ b/lib/pleroma/delivery.ex @@ -31,7 +31,7 @@ defmodule Pleroma.Delivery do def create(object_id, user_id) do %Delivery{} |> changeset(%{user_id: user_id, object_id: object_id}) - |> Repo.insert() + |> Repo.insert(on_conflict: :nothing) end def get(object_id, user_id) do From 8900cb68aef535dbf60de87fce47d85b91909077 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 13 Sep 2019 15:25:15 +0000 Subject: [PATCH 076/138] Apply suggestion to lib/pleroma/web/activity_pub/activity_pub_controller.ex --- lib/pleroma/web/activity_pub/activity_pub_controller.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 70d4a5ba..01b34fb1 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -71,6 +71,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do end end + def track_object_fetch(conn, nil), do: conn + def track_object_fetch(conn, object_id) do with %{assigns: %{user: %User{id: user_id}}} <- conn do Delivery.create(object_id, user_id) From 25a64a4aa0a10bf06c2ccdf9a6c493f184170a89 Mon Sep 17 00:00:00 2001 From: stwf Date: Fri, 13 Sep 2019 11:46:41 -0400 Subject: [PATCH 077/138] Capture test error messages where appropriate --- test/integration/mastodon_websocket_test.exs | 29 ++++++++++++++----- test/web/activity_pub/publisher_test.exs | 24 +++++++++------ test/web/activity_pub/relay_test.exs | 13 +++++++-- .../mastodon_api_controller_test.exs | 14 +++++---- test/web/twitter_api/util_controller_test.exs | 13 +++++---- .../web_finger/web_finger_controller_test.exs | 13 +++++---- 6 files changed, 70 insertions(+), 36 deletions(-) diff --git a/test/integration/mastodon_websocket_test.exs b/test/integration/mastodon_websocket_test.exs index 3975cdcd..63bf7341 100644 --- a/test/integration/mastodon_websocket_test.exs +++ b/test/integration/mastodon_websocket_test.exs @@ -5,6 +5,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do use Pleroma.DataCase + import ExUnit.CaptureLog import Pleroma.Factory alias Pleroma.Integration.WebsocketClient @@ -39,13 +40,17 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do end test "refuses invalid requests" do - assert {:error, {400, _}} = start_socket() - assert {:error, {404, _}} = start_socket("?stream=ncjdk") + capture_log(fn -> + assert {:error, {400, _}} = start_socket() + assert {:error, {404, _}} = start_socket("?stream=ncjdk") + end) end test "requires authentication and a valid token for protected streams" do - assert {:error, {403, _}} = start_socket("?stream=user&access_token=aaaaaaaaaaaa") - assert {:error, {403, _}} = start_socket("?stream=user") + capture_log(fn -> + assert {:error, {403, _}} = start_socket("?stream=user&access_token=aaaaaaaaaaaa") + assert {:error, {403, _}} = start_socket("?stream=user") + end) end test "allows public streams without authentication" do @@ -100,19 +105,27 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do test "accepts the 'user' stream", %{token: token} = _state do assert {:ok, _} = start_socket("?stream=user&access_token=#{token.token}") - assert {:error, {403, "Forbidden"}} = start_socket("?stream=user") + + assert capture_log(fn -> + assert {:error, {403, "Forbidden"}} = start_socket("?stream=user") + end) =~ ":badarg" end test "accepts the 'user:notification' stream", %{token: token} = _state do assert {:ok, _} = start_socket("?stream=user:notification&access_token=#{token.token}") - assert {:error, {403, "Forbidden"}} = start_socket("?stream=user:notification") + + assert capture_log(fn -> + assert {:error, {403, "Forbidden"}} = start_socket("?stream=user:notification") + end) =~ ":badarg" end test "accepts valid token on Sec-WebSocket-Protocol header", %{token: token} do assert {:ok, _} = start_socket("?stream=user", [{"Sec-WebSocket-Protocol", token.token}]) - assert {:error, {403, "Forbidden"}} = - start_socket("?stream=user", [{"Sec-WebSocket-Protocol", "I am a friend"}]) + assert capture_log(fn -> + assert {:error, {403, "Forbidden"}} = + start_socket("?stream=user", [{"Sec-WebSocket-Protocol", "I am a friend"}]) + end) =~ ":badarg" end end end diff --git a/test/web/activity_pub/publisher_test.exs b/test/web/activity_pub/publisher_test.exs index 36a39c84..381757e1 100644 --- a/test/web/activity_pub/publisher_test.exs +++ b/test/web/activity_pub/publisher_test.exs @@ -5,6 +5,7 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do use Pleroma.DataCase + import ExUnit.CaptureLog import Pleroma.Factory import Tesla.Mock import Mock @@ -188,7 +189,10 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do actor = insert(:user) inbox = "http://connrefused.site/users/nick1/inbox" - assert {:error, _} = Publisher.publish_one(%{inbox: inbox, json: "{}", actor: actor, id: 1}) + assert capture_log(fn -> + assert {:error, _} = + Publisher.publish_one(%{inbox: inbox, json: "{}", actor: actor, id: 1}) + end) =~ "connrefused" assert called(Instances.set_unreachable(inbox)) end @@ -212,14 +216,16 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do actor = insert(:user) inbox = "http://connrefused.site/users/nick1/inbox" - assert {:error, _} = - Publisher.publish_one(%{ - inbox: inbox, - json: "{}", - actor: actor, - id: 1, - unreachable_since: NaiveDateTime.utc_now() - }) + assert capture_log(fn -> + assert {:error, _} = + Publisher.publish_one(%{ + inbox: inbox, + json: "{}", + actor: actor, + id: 1, + unreachable_since: NaiveDateTime.utc_now() + }) + end) =~ "connrefused" refute called(Instances.set_unreachable(inbox)) end diff --git a/test/web/activity_pub/relay_test.exs b/test/web/activity_pub/relay_test.exs index 4f7d592a..9db4255d 100644 --- a/test/web/activity_pub/relay_test.exs +++ b/test/web/activity_pub/relay_test.exs @@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.RelayTest do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Relay + import ExUnit.CaptureLog import Pleroma.Factory import Mock @@ -20,7 +21,9 @@ defmodule Pleroma.Web.ActivityPub.RelayTest do describe "follow/1" do test "returns errors when user not found" do - assert Relay.follow("test-ap-id") == {:error, "Could not fetch by AP id"} + assert capture_log(fn -> + assert Relay.follow("test-ap-id") == {:error, "Could not fetch by AP id"} + end) =~ "Could not fetch by AP id" end test "returns activity" do @@ -37,7 +40,9 @@ defmodule Pleroma.Web.ActivityPub.RelayTest do describe "unfollow/1" do test "returns errors when user not found" do - assert Relay.unfollow("test-ap-id") == {:error, "Could not fetch by AP id"} + assert capture_log(fn -> + assert Relay.unfollow("test-ap-id") == {:error, "Could not fetch by AP id"} + end) =~ "Could not fetch by AP id" end test "returns activity" do @@ -78,7 +83,9 @@ defmodule Pleroma.Web.ActivityPub.RelayTest do } ) - assert Relay.publish(activity) == {:error, nil} + assert capture_log(fn -> + assert Relay.publish(activity) == {:error, nil} + end) =~ "[error] error: nil" end test_with_mock "returns announce activity and publish to federate", diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index f4902d04..806ae7e6 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -3963,13 +3963,15 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do Config.put([:suggestions, :enabled], true) Config.put([:suggestions, :third_party_engine], "http://test500?{{host}}&{{user}}") - res = - conn - |> assign(:user, user) - |> get("/api/v1/suggestions") - |> json_response(500) + assert capture_log(fn -> + res = + conn + |> assign(:user, user) + |> get("/api/v1/suggestions") + |> json_response(500) - assert res == "Something went wrong" + assert res == "Something went wrong" + end) =~ "Could not retrieve suggestions" end test "returns suggestions", %{conn: conn, user: user, other_user: other_user} do diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index cf8e69d2..e36d3130 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -8,6 +8,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.CommonAPI + import ExUnit.CaptureLog import Pleroma.Factory import Mock @@ -338,12 +339,14 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do test "show follow page with error when user cannot fecth by `acct` link", %{conn: conn} do user = insert(:user) - response = - conn - |> assign(:user, user) - |> get("/ostatus_subscribe?acct=https://mastodon.social/users/not_found") + assert capture_log(fn -> + response = + conn + |> assign(:user, user) + |> get("/ostatus_subscribe?acct=https://mastodon.social/users/not_found") - assert html_response(response, 200) =~ "Error fetching user" + assert html_response(response, 200) =~ "Error fetching user" + end) =~ "Object has been deleted" end end diff --git a/test/web/web_finger/web_finger_controller_test.exs b/test/web/web_finger/web_finger_controller_test.exs index e23086b2..bd3ccaaf 100644 --- a/test/web/web_finger/web_finger_controller_test.exs +++ b/test/web/web_finger/web_finger_controller_test.exs @@ -5,6 +5,7 @@ defmodule Pleroma.Web.WebFinger.WebFingerControllerTest do use Pleroma.Web.ConnCase + import ExUnit.CaptureLog import Pleroma.Factory import Tesla.Mock @@ -75,11 +76,13 @@ defmodule Pleroma.Web.WebFinger.WebFingerControllerTest do test "Sends a 404 when invalid format" do user = insert(:user) - assert_raise Phoenix.NotAcceptableError, fn -> - build_conn() - |> put_req_header("accept", "text/html") - |> get("/.well-known/webfinger?resource=acct:#{user.nickname}@localhost") - end + assert capture_log(fn -> + assert_raise Phoenix.NotAcceptableError, fn -> + build_conn() + |> put_req_header("accept", "text/html") + |> get("/.well-known/webfinger?resource=acct:#{user.nickname}@localhost") + end + end) =~ "no supported media type in accept header" end test "Sends a 400 when resource param is missing" do From 69faec031d62f4e87a1791ae0c71ca4b0f02f12b Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 13 Sep 2019 19:02:42 +0300 Subject: [PATCH 078/138] markdown generation to the new file --- lib/pleroma/docs/markdown.ex | 4 ++-- mix.exs | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/docs/markdown.ex b/lib/pleroma/docs/markdown.ex index 24930cc9..8386dc2f 100644 --- a/lib/pleroma/docs/markdown.ex +++ b/lib/pleroma/docs/markdown.ex @@ -3,9 +3,9 @@ defmodule Pleroma.Docs.Markdown do @spec process(keyword()) :: {:ok, String.t()} def process(descriptions) do - config_path = "docs/config.md" + config_path = "docs/generated_config.md" {:ok, file} = File.open(config_path, [:utf8, :write]) - IO.write(file, "# Configuration\n") + IO.write(file, "# Generated configuration\n") IO.write(file, "Date of generation: #{Date.utc_today()}\n\n") IO.write( diff --git a/mix.exs b/mix.exs index 96ef723b..dfa53035 100644 --- a/mix.exs +++ b/mix.exs @@ -172,8 +172,7 @@ defmodule Pleroma.Mixfile do "ecto.rollback": ["pleroma.ecto.rollback"], "ecto.setup": ["ecto.create", "ecto.migrate", "run priv/repo/seeds.exs"], "ecto.reset": ["ecto.drop", "ecto.setup"], - test: ["ecto.create --quiet", "ecto.migrate", "test"], - docs: ["pleroma.docs", "docs"] + test: ["ecto.create --quiet", "ecto.migrate", "test"] ] end From c625fe6f09a308f10e98c9e5ea4bf14500a0b58a Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 13 Sep 2019 19:03:39 +0300 Subject: [PATCH 079/138] config.md back --- docs/config.md | 702 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 701 insertions(+), 1 deletion(-) diff --git a/docs/config.md b/docs/config.md index 7f54a34b..066547bb 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1 +1,701 @@ -This file is a placeholder, please run mix pleroma.docs to generate it. +# Configuration + +This file describe the configuration, it is recommended to edit the relevant *.secret.exs file instead of the others founds in the ``config`` directory. +If you run Pleroma with ``MIX_ENV=prod`` the file is ``prod.secret.exs``, otherwise it is ``dev.secret.exs``. + +## Pleroma.Upload +* `uploader`: Select which `Pleroma.Uploaders` to use +* `filters`: List of `Pleroma.Upload.Filter` to use. +* `link_name`: When enabled Pleroma will add a `name` parameter to the url of the upload, for example `https://instance.tld/media/corndog.png?name=corndog.png`. This is needed to provide the correct filename in Content-Disposition headers when using filters like `Pleroma.Upload.Filter.Dedupe` +* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host. +* `proxy_remote`: If you're using a remote uploader, Pleroma will proxy media requests instead of redirecting to it. +* `proxy_opts`: Proxy options, see `Pleroma.ReverseProxy` documentation. + +Note: `strip_exif` has been replaced by `Pleroma.Upload.Filter.Mogrify`. + +## Pleroma.Uploaders.Local +* `uploads`: Which directory to store the user-uploads in, relative to pleroma’s working directory + +## Pleroma.Uploaders.S3 +* `bucket`: S3 bucket name +* `bucket_namespace`: S3 bucket namespace +* `public_endpoint`: S3 endpoint that the user finally accesses(ex. "https://s3.dualstack.ap-northeast-1.amazonaws.com") +* `truncated_namespace`: If you use S3 compatible service such as Digital Ocean Spaces or CDN, set folder name or "" etc. +For example, when using CDN to S3 virtual host format, set "". +At this time, write CNAME to CDN in public_endpoint. + +## Pleroma.Upload.Filter.Mogrify + +* `args`: List of actions for the `mogrify` command like `"strip"` or `["strip", "auto-orient", {"implode", "1"}]`. + +## Pleroma.Upload.Filter.Dedupe + +No specific configuration. + +## Pleroma.Upload.Filter.AnonymizeFilename + +This filter replaces the filename (not the path) of an upload. For complete obfuscation, add +`Pleroma.Upload.Filter.Dedupe` before AnonymizeFilename. + +* `text`: Text to replace filenames in links. If empty, `{random}.extension` will be used. You can get the original filename extension by using `{extension}`, for example `custom-file-name.{extension}`. + +## Pleroma.Emails.Mailer +* `adapter`: one of the mail adapters listed in [Swoosh readme](https://github.com/swoosh/swoosh#adapters), or `Swoosh.Adapters.Local` for in-memory mailbox. +* `api_key` / `password` and / or other adapter-specific settings, per the above documentation. +* `enabled`: Allows enable/disable send emails. Default: `false`. + +An example for Sendgrid adapter: + +```elixir +config :pleroma, Pleroma.Emails.Mailer, + adapter: Swoosh.Adapters.Sendgrid, + api_key: "YOUR_API_KEY" +``` + +An example for SMTP adapter: + +```elixir +config :pleroma, Pleroma.Emails.Mailer, + adapter: Swoosh.Adapters.SMTP, + relay: "smtp.gmail.com", + username: "YOUR_USERNAME@gmail.com", + password: "YOUR_SMTP_PASSWORD", + port: 465, + ssl: true, + tls: :always, + auth: :always +``` + +## :uri_schemes +* `valid_schemes`: List of the scheme part that is considered valid to be an URL + +## :instance +* `name`: The instance’s name +* `email`: Email used to reach an Administrator/Moderator of the instance +* `notify_email`: Email used for notifications. +* `description`: The instance’s description, can be seen in nodeinfo and ``/api/v1/instance`` +* `limit`: Posts character limit (CW/Subject included in the counter) +* `remote_limit`: Hard character limit beyond which remote posts will be dropped. +* `upload_limit`: File size limit of uploads (except for avatar, background, banner) +* `avatar_upload_limit`: File size limit of user’s profile avatars +* `background_upload_limit`: File size limit of user’s profile backgrounds +* `banner_upload_limit`: File size limit of user’s profile banners +* `poll_limits`: A map with poll limits for **local** polls + * `max_options`: Maximum number of options + * `max_option_chars`: Maximum number of characters per option + * `min_expiration`: Minimum expiration time (in seconds) + * `max_expiration`: Maximum expiration time (in seconds) +* `registrations_open`: Enable registrations for anyone, invitations can be enabled when false. +* `invites_enabled`: Enable user invitations for admins (depends on `registrations_open: false`). +* `account_activation_required`: Require users to confirm their emails before signing in. +* `federating`: Enable federation with other instances +* `federation_incoming_replies_max_depth`: Max. depth of reply-to activities fetching on incoming federation, to prevent out-of-memory situations while fetching very long threads. If set to `nil`, threads of any depth will be fetched. Lower this value if you experience out-of-memory crashes. +* `federation_reachability_timeout_days`: Timeout (in days) of each external federation target being unreachable prior to pausing federating to it. +* `allow_relay`: Enable Pleroma’s Relay, which makes it possible to follow a whole instance +* `rewrite_policy`: Message Rewrite Policy, either one or a list. Here are the ones available by default: + * `Pleroma.Web.ActivityPub.MRF.NoOpPolicy`: Doesn’t modify activities (default) + * `Pleroma.Web.ActivityPub.MRF.DropPolicy`: Drops all activities. It generally doesn’t makes sense to use in production + * `Pleroma.Web.ActivityPub.MRF.SimplePolicy`: Restrict the visibility of activities from certains instances (See ``:mrf_simple`` section) + * `Pleroma.Web.ActivityPub.MRF.TagPolicy`: Applies policies to individual users based on tags, which can be set using pleroma-fe/admin-fe/any other app that supports Pleroma Admin API. For example it allows marking posts from individual users nsfw (sensitive) + * `Pleroma.Web.ActivityPub.MRF.SubchainPolicy`: Selectively runs other MRF policies when messages match (see ``:mrf_subchain`` section) + * `Pleroma.Web.ActivityPub.MRF.RejectNonPublic`: Drops posts with non-public visibility settings (See ``:mrf_rejectnonpublic`` section) + * `Pleroma.Web.ActivityPub.MRF.EnsureRePrepended`: Rewrites posts to ensure that replies to posts with subjects do not have an identical subject and instead begin with re:. + * `Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy`: Rejects posts from likely spambots by rejecting posts from new users that contain links. + * `Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`: Crawls attachments using their MediaProxy URLs so that the MediaProxy cache is primed. + * `Pleroma.Web.ActivityPub.MRF.MentionPolicy`: Drops posts mentioning configurable users. (see `:mrf_mention` section) + * `Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`: Restricts activities to a configured set of vocabulary. (see `:mrf_vocabulary` section) +* `public`: Makes the client API in authentificated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. +* `quarantined_instances`: List of ActivityPub instances where private(DMs, followers-only) activities will not be send. +* `managed_config`: Whenether the config for pleroma-fe is configured in this config or in ``static/config.json`` +* `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML) +* `mrf_transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo). +* `mrf_transparency_exclusions`: Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value. +* `scope_copy`: Copy the scope (private/unlisted/public) in replies to posts by default. +* `subject_line_behavior`: Allows changing the default behaviour of subject lines in replies. Valid values: + * "email": Copy and preprend re:, as in email. + * "masto": Copy verbatim, as in Mastodon. + * "noop": Don't copy the subject. +* `always_show_subject_input`: When set to false, auto-hide the subject field when it's empty. +* `extended_nickname_format`: Set to `true` to use extended local nicknames format (allows underscores/dashes). This will break federation with + older software for theses nicknames. +* `max_pinned_statuses`: The maximum number of pinned statuses. `0` will disable the feature. +* `autofollowed_nicknames`: Set to nicknames of (local) users that every new user should automatically follow. +* `no_attachment_links`: Set to true to disable automatically adding attachment link text to statuses +* `welcome_message`: A message that will be send to a newly registered users as a direct message. +* `welcome_user_nickname`: The nickname of the local user that sends the welcome message. +* `max_report_comment_size`: The maximum size of the report comment (Default: `1000`) +* `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). Default: `false`. +* `healthcheck`: If set to true, system data will be shown on ``/api/pleroma/healthcheck``. +* `remote_post_retention_days`: The default amount of days to retain remote posts when pruning the database. +* `user_bio_length`: A user bio maximum length (default: `5000`) +* `user_name_length`: A user name maximum length (default: `100`) +* `skip_thread_containment`: Skip filter out broken threads. The default is `false`. +* `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. +* `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. +* `max_account_fields`: The maximum number of custom fields in the user profile (default: `10`) +* `max_remote_account_fields`: The maximum number of custom fields in the remote user profile (default: `20`) +* `account_field_name_length`: An account field name maximum length (default: `512`) +* `account_field_value_length`: An account field value maximum length (default: `512`) +* `external_user_synchronization`: Enabling following/followers counters synchronization for external users. + + + +## :logger +* `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog, and `Quack.Logger` to log to Slack + +An example to enable ONLY ExSyslogger (f/ex in ``prod.secret.exs``) with info and debug suppressed: +```elixir +config :logger, + backends: [{ExSyslogger, :ex_syslogger}] + +config :logger, :ex_syslogger, + level: :warn +``` + +Another example, keeping console output and adding the pid to syslog output: +```elixir +config :logger, + backends: [:console, {ExSyslogger, :ex_syslogger}] + +config :logger, :ex_syslogger, + level: :warn, + option: [:pid, :ndelay] +``` + +See: [logger’s documentation](https://hexdocs.pm/logger/Logger.html) and [ex_syslogger’s documentation](https://hexdocs.pm/ex_syslogger/) + +An example of logging info to local syslog, but warn to a Slack channel: +```elixir +config :logger, + backends: [ {ExSyslogger, :ex_syslogger}, Quack.Logger ], + level: :info + +config :logger, :ex_syslogger, + level: :info, + ident: "pleroma", + format: "$metadata[$level] $message" + +config :quack, + level: :warn, + meta: [:all], + webhook_url: "https://hooks.slack.com/services/YOUR-API-KEY-HERE" +``` + +See the [Quack Github](https://github.com/azohra/quack) for more details + +## :frontend_configurations + +This can be used to configure a keyword list that keeps the configuration data for any kind of frontend. By default, settings for `pleroma_fe` and `masto_fe` are configured. + +Frontends can access these settings at `/api/pleroma/frontend_configurations` + +To add your own configuration for PleromaFE, use it like this: + +```elixir +config :pleroma, :frontend_configurations, + pleroma_fe: %{ + theme: "pleroma-dark", + # ... see /priv/static/static/config.json for the available keys. +}, + masto_fe: %{ + showInstanceSpecificPanel: true + } +``` + +These settings **need to be complete**, they will override the defaults. + +NOTE: for versions < 1.0, you need to set [`:fe`](#fe) to false, as shown a few lines below. + +## :fe +__THIS IS DEPRECATED__ + +If you are using this method, please change it to the [`frontend_configurations`](#frontend_configurations) method. +Please **set this option to false** in your config like this: + +```elixir +config :pleroma, :fe, false +``` + +This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:instance`` is set to false. + +* `theme`: Which theme to use, they are defined in ``styles.json`` +* `logo`: URL of the logo, defaults to Pleroma’s logo +* `logo_mask`: Whether to use only the logo's shape as a mask (true) or as a regular image (false) +* `logo_margin`: What margin to use around the logo +* `background`: URL of the background, unless viewing a user profile with a background that is set +* `redirect_root_no_login`: relative URL which indicates where to redirect when a user isn’t logged in. +* `redirect_root_login`: relative URL which indicates where to redirect when a user is logged in. +* `show_instance_panel`: Whenether to show the instance’s specific panel. +* `scope_options_enabled`: Enable setting an notice visibility and subject/CW when posting +* `formatting_options_enabled`: Enable setting a formatting different than plain-text (ie. HTML, Markdown) when posting, relates to ``:instance, allowed_post_formats`` +* `collapse_message_with_subjects`: When a message has a subject(aka Content Warning), collapse it by default +* `hide_post_stats`: Hide notices statistics(repeats, favorites, …) +* `hide_user_stats`: Hide profile statistics(posts, posts per day, followers, followings, …) + +## :assets + +This section configures assets to be used with various frontends. Currently the only option +relates to mascots on the mastodon frontend + +* `mascots`: KeywordList of mascots, each element __MUST__ contain both a `url` and a + `mime_type` key. +* `default_mascot`: An element from `mascots` - This will be used as the default mascot + on MastoFE (default: `:pleroma_fox_tan`) + +## :mrf_simple +* `media_removal`: List of instances to remove medias from +* `media_nsfw`: List of instances to put medias as NSFW(sensitive) from +* `federated_timeline_removal`: List of instances to remove from Federated (aka The Whole Known Network) Timeline +* `reject`: List of instances to reject any activities from +* `accept`: List of instances to accept any activities from +* `report_removal`: List of instances to reject reports from +* `avatar_removal`: List of instances to strip avatars from +* `banner_removal`: List of instances to strip banners from + +## :mrf_subchain +This policy processes messages through an alternate pipeline when a given message matches certain criteria. +All criteria are configured as a map of regular expressions to lists of policy modules. + +* `match_actor`: Matches a series of regular expressions against the actor field. + +Example: + +``` +config :pleroma, :mrf_subchain, + match_actor: %{ + ~r/https:\/\/example.com/s => [Pleroma.Web.ActivityPub.MRF.DropPolicy] + } +``` + +## :mrf_rejectnonpublic +* `allow_followersonly`: whether to allow followers-only posts +* `allow_direct`: whether to allow direct messages + +## :mrf_hellthread +* `delist_threshold`: Number of mentioned users after which the message gets delisted (the message can still be seen, but it will not show up in public timelines and mentioned users won't get notifications about it). Set to 0 to disable. +* `reject_threshold`: Number of mentioned users after which the messaged gets rejected. Set to 0 to disable. + +## :mrf_keyword +* `reject`: A list of patterns which result in message being rejected, each pattern can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) +* `federated_timeline_removal`: A list of patterns which result in message being removed from federated timelines (a.k.a unlisted), each pattern can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) +* `replace`: A list of tuples containing `{pattern, replacement}`, `pattern` can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) + +## :mrf_mention +* `actors`: A list of actors, for which to drop any posts mentioning. + +## :mrf_vocabulary +* `accept`: A list of ActivityStreams terms to accept. If empty, all supported messages are accepted. +* `reject`: A list of ActivityStreams terms to reject. If empty, no messages are rejected. + +## :media_proxy +* `enabled`: Enables proxying of remote media to the instance’s proxy +* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts. +* `proxy_opts`: All options defined in `Pleroma.ReverseProxy` documentation, defaults to `[max_body_length: (25*1_048_576)]`. +* `whitelist`: List of domains to bypass the mediaproxy + +## :gopher +* `enabled`: Enables the gopher interface +* `ip`: IP address to bind to +* `port`: Port to bind to +* `dstport`: Port advertised in urls (optional, defaults to `port`) + +## Pleroma.Web.Endpoint +`Phoenix` endpoint configuration, all configuration options can be viewed [here](https://hexdocs.pm/phoenix/Phoenix.Endpoint.html#module-dynamic-configuration), only common options are listed here +* `http` - a list containing http protocol configuration, all configuration options can be viewed [here](https://hexdocs.pm/plug_cowboy/Plug.Cowboy.html#module-options), only common options are listed here. For deployment using docker, you need to set this to `[ip: {0,0,0,0}, port: 4000]` to make pleroma accessible from other containers (such as your nginx server). + - `ip` - a tuple consisting of 4 integers + - `port` +* `url` - a list containing the configuration for generating urls, accepts + - `host` - the host without the scheme and a post (e.g `example.com`, not `https://example.com:2020`) + - `scheme` - e.g `http`, `https` + - `port` + - `path` +* `extra_cookie_attrs` - a list of `Key=Value` strings to be added as non-standard cookie attributes. Defaults to `["SameSite=Lax"]`. See the [SameSite article](https://www.owasp.org/index.php/SameSite) on OWASP for more info. + + + +**Important note**: if you modify anything inside these lists, default `config.exs` values will be overwritten, which may result in breakage, to make sure this does not happen please copy the default value for the list from `config.exs` and modify/add only what you need + +Example: +```elixir +config :pleroma, Pleroma.Web.Endpoint, + url: [host: "example.com", port: 2020, scheme: "https"], + http: [ + # start copied from config.exs + dispatch: [ + {:_, + [ + {"/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, []}, + {"/websocket", Phoenix.Endpoint.CowboyWebSocket, + {Phoenix.Transports.WebSocket, + {Pleroma.Web.Endpoint, Pleroma.Web.UserSocket, websocket_config}}}, + {:_, Phoenix.Endpoint.Cowboy2Handler, {Pleroma.Web.Endpoint, []}} + ]} + # end copied from config.exs + ], + port: 8080, + ip: {127, 0, 0, 1} + ] +``` + +This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls starting with `https://example.com:2020` + +## :activitypub +* ``unfollow_blocked``: Whether blocks result in people getting unfollowed +* ``outgoing_blocks``: Whether to federate blocks to other instances +* ``deny_follow_blocked``: Whether to disallow following an account that has blocked the user in question +* ``sign_object_fetches``: Sign object fetches with HTTP signatures + +## :http_security +* ``enabled``: Whether the managed content security policy is enabled +* ``sts``: Whether to additionally send a `Strict-Transport-Security` header +* ``sts_max_age``: The maximum age for the `Strict-Transport-Security` header if sent +* ``ct_max_age``: The maximum age for the `Expect-CT` header if sent +* ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"` +* ``report_uri``: Adds the specified url to `report-uri` and `report-to` group in CSP header. + +## :mrf_user_allowlist + +The keys in this section are the domain names that the policy should apply to. +Each key should be assigned a list of users that should be allowed through by +their ActivityPub ID. + +An example: + +```elixir +config :pleroma, :mrf_user_allowlist, + "example.org": ["https://example.org/users/admin"] +``` + +## :web_push_encryption, :vapid_details + +Web Push Notifications configuration. You can use the mix task `mix web_push.gen.keypair` to generate it. + +* ``subject``: a mailto link for the administrative contact. It’s best if this email is not a personal email address, but rather a group email so that if a person leaves an organization, is unavailable for an extended period, or otherwise can’t respond, someone else on the list can. +* ``public_key``: VAPID public key +* ``private_key``: VAPID private key + +## Pleroma.Captcha +* `enabled`: Whether the captcha should be shown on registration +* `method`: The method/service to use for captcha +* `seconds_valid`: The time in seconds for which the captcha is valid + +### Pleroma.Captcha.Kocaptcha +Kocaptcha is a very simple captcha service with a single API endpoint, +the source code is here: https://github.com/koto-bank/kocaptcha. The default endpoint +`https://captcha.kotobank.ch` is hosted by the developer. + +* `endpoint`: the kocaptcha endpoint to use + +## :admin_token + +Allows to set a token that can be used to authenticate with the admin api without using an actual user by giving it as the 'admin_token' parameter. Example: + +```elixir +config :pleroma, :admin_token, "somerandomtoken" +``` + +You can then do + +```sh +curl "http://localhost:4000/api/pleroma/admin/invite_token?admin_token=somerandomtoken" +``` + +## :pleroma_job_queue + +[Pleroma Job Queue](https://git.pleroma.social/pleroma/pleroma_job_queue) configuration: a list of queues with maximum concurrent jobs. + +Pleroma has the following queues: + +* `federator_outgoing` - Outgoing federation +* `federator_incoming` - Incoming federation +* `mailer` - Email sender, see [`Pleroma.Emails.Mailer`](#pleroma-emails-mailer) +* `transmogrifier` - Transmogrifier +* `web_push` - Web push notifications +* `scheduled_activities` - Scheduled activities, see [`Pleroma.ScheduledActivities`](#pleromascheduledactivity) + +Example: + +```elixir +config :pleroma_job_queue, :queues, + federator_incoming: 50, + federator_outgoing: 50 +``` + +This config contains two queues: `federator_incoming` and `federator_outgoing`. Both have the `max_jobs` set to `50`. + +## Pleroma.Web.Federator.RetryQueue + +* `enabled`: If set to `true`, failed federation jobs will be retried +* `max_jobs`: The maximum amount of parallel federation jobs running at the same time. +* `initial_timeout`: The initial timeout in seconds +* `max_retries`: The maximum number of times a federation job is retried + +## Pleroma.Web.Metadata +* `providers`: a list of metadata providers to enable. Providers available: + * Pleroma.Web.Metadata.Providers.OpenGraph + * Pleroma.Web.Metadata.Providers.TwitterCard + * Pleroma.Web.Metadata.Providers.RelMe - add links from user bio with rel=me into the `
` as `` +* `unfurl_nsfw`: If set to `true` nsfw attachments will be shown in previews + +## :rich_media +* `enabled`: if enabled the instance will parse metadata from attached links to generate link previews +* `ignore_hosts`: list of hosts which will be ignored by the metadata parser. For example `["accounts.google.com", "xss.website"]`, defaults to `[]`. +* `ignore_tld`: list TLDs (top-level domains) which will ignore for parse metadata. default is ["local", "localdomain", "lan"] +* `parsers`: list of Rich Media parsers + +## :fetch_initial_posts +* `enabled`: if enabled, when a new user is federated with, fetch some of their latest posts +* `pages`: the amount of pages to fetch + +## :hackney_pools + +Advanced. Tweaks Hackney (http client) connections pools. + +There's three pools used: + +* `:federation` for the federation jobs. + You may want this pool max_connections to be at least equal to the number of federator jobs + retry queue jobs. +* `:media` for rich media, media proxy +* `:upload` for uploaded media (if using a remote uploader and `proxy_remote: true`) + +For each pool, the options are: + +* `max_connections` - how much connections a pool can hold +* `timeout` - retention duration for connections + +## :auto_linker + +Configuration for the `auto_linker` library: + +* `class: "auto-linker"` - specify the class to be added to the generated link. false to clear +* `rel: "noopener noreferrer"` - override the rel attribute. false to clear +* `new_window: true` - set to false to remove `target='_blank'` attribute +* `scheme: false` - Set to true to link urls with schema `http://google.com` +* `truncate: false` - Set to a number to truncate urls longer then the number. Truncated urls will end in `..` +* `strip_prefix: true` - Strip the scheme prefix +* `extra: false` - link urls with rarely used schemes (magnet, ipfs, irc, etc.) + +Example: + +```elixir +config :auto_linker, + opts: [ + scheme: true, + extra: true, + class: false, + strip_prefix: false, + new_window: false, + rel: false + ] +``` + +## Pleroma.ScheduledActivity + +* `daily_user_limit`: the number of scheduled activities a user is allowed to create in a single day (Default: `25`) +* `total_user_limit`: the number of scheduled activities a user is allowed to create in total (Default: `300`) +* `enabled`: whether scheduled activities are sent to the job queue to be executed + +## Pleroma.ActivityExpiration + +# `enabled`: whether expired activities will be sent to the job queue to be deleted + +## Pleroma.Web.Auth.Authenticator + +* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator +* `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication + +## :ldap + +Use LDAP for user authentication. When a user logs in to the Pleroma +instance, the name and password will be verified by trying to authenticate +(bind) to an LDAP server. If a user exists in the LDAP directory but there +is no account with the same name yet on the Pleroma instance then a new +Pleroma account will be created with the same name as the LDAP user name. + +* `enabled`: enables LDAP authentication +* `host`: LDAP server hostname +* `port`: LDAP port, e.g. 389 or 636 +* `ssl`: true to use SSL, usually implies the port 636 +* `sslopts`: additional SSL options +* `tls`: true to start TLS, usually implies the port 389 +* `tlsopts`: additional TLS options +* `base`: LDAP base, e.g. "dc=example,dc=com" +* `uid`: LDAP attribute name to authenticate the user, e.g. when "cn", the filter will be "cn=username,base" + +## BBS / SSH access + +To enable simple command line interface accessible over ssh, add a setting like this to your configuration file: + +```exs +app_dir = File.cwd! +priv_dir = Path.join([app_dir, "priv/ssh_keys"]) + +config :esshd, + enabled: true, + priv_dir: priv_dir, + handler: "Pleroma.BBS.Handler", + port: 10_022, + password_authenticator: "Pleroma.BBS.Authenticator" +``` + +Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -m PEM -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT` + +## :auth + +* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator +* `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication + +Authentication / authorization settings. + +* `auth_template`: authentication form template. By default it's `show.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/show.html.eex`. +* `oauth_consumer_template`: OAuth consumer mode authentication form template. By default it's `consumer.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex`. +* `oauth_consumer_strategies`: the list of enabled OAuth consumer strategies; by default it's set by `OAUTH_CONSUMER_STRATEGIES` environment variable. Each entry in this space-delimited string should be of format `` or `:` (e.g. `twitter` or `keycloak:ueberauth_keycloak_strategy` in case dependency is named differently than `ueberauth_`). + +## :email_notifications + +Email notifications settings. + + - digest - emails of "what you've missed" for users who have been + inactive for a while. + - active: globally enable or disable digest emails + - schedule: When to send digest email, in [crontab format](https://en.wikipedia.org/wiki/Cron). + "0 0 * * 0" is the default, meaning "once a week at midnight on Sunday morning" + - interval: Minimum interval between digest emails to one user + - inactivity_threshold: Minimum user inactivity threshold + +## Pleroma.Emails.UserEmail + +- `:logo` - a path to a custom logo. Set it to `nil` to use the default Pleroma logo. +- `:styling` - a map with color settings for email templates. + +## OAuth consumer mode + +OAuth consumer mode allows sign in / sign up via external OAuth providers (e.g. Twitter, Facebook, Google, Microsoft, etc.). +Implementation is based on Ueberauth; see the list of [available strategies](https://github.com/ueberauth/ueberauth/wiki/List-of-Strategies). + +Note: each strategy is shipped as a separate dependency; in order to get the strategies, run `OAUTH_CONSUMER_STRATEGIES="..." mix deps.get`, +e.g. `OAUTH_CONSUMER_STRATEGIES="twitter facebook google microsoft" mix deps.get`. +The server should also be started with `OAUTH_CONSUMER_STRATEGIES="..." mix phx.server` in case you enable any strategies. + +Note: each strategy requires separate setup (on external provider side and Pleroma side). Below are the guidelines on setting up most popular strategies. + +Note: make sure that `"SameSite=Lax"` is set in `extra_cookie_attrs` when you have this feature enabled. OAuth consumer mode will not work with `"SameSite=Strict"` + +* For Twitter, [register an app](https://developer.twitter.com/en/apps), configure callback URL to https:///oauth/twitter/callback + +* For Facebook, [register an app](https://developers.facebook.com/apps), configure callback URL to https:///oauth/facebook/callback, enable Facebook Login service at https://developers.facebook.com/apps//fb-login/settings/ + +* For Google, [register an app](https://console.developers.google.com), configure callback URL to https:///oauth/google/callback + +* For Microsoft, [register an app](https://portal.azure.com), configure callback URL to https:///oauth/microsoft/callback + +Once the app is configured on external OAuth provider side, add app's credentials and strategy-specific settings (if any — e.g. see Microsoft below) to `config/prod.secret.exs`, +per strategy's documentation (e.g. [ueberauth_twitter](https://github.com/ueberauth/ueberauth_twitter)). Example config basing on environment variables: + +```elixir +# Twitter +config :ueberauth, Ueberauth.Strategy.Twitter.OAuth, + consumer_key: System.get_env("TWITTER_CONSUMER_KEY"), + consumer_secret: System.get_env("TWITTER_CONSUMER_SECRET") + +# Facebook +config :ueberauth, Ueberauth.Strategy.Facebook.OAuth, + client_id: System.get_env("FACEBOOK_APP_ID"), + client_secret: System.get_env("FACEBOOK_APP_SECRET"), + redirect_uri: System.get_env("FACEBOOK_REDIRECT_URI") + +# Google +config :ueberauth, Ueberauth.Strategy.Google.OAuth, + client_id: System.get_env("GOOGLE_CLIENT_ID"), + client_secret: System.get_env("GOOGLE_CLIENT_SECRET"), + redirect_uri: System.get_env("GOOGLE_REDIRECT_URI") + +# Microsoft +config :ueberauth, Ueberauth.Strategy.Microsoft.OAuth, + client_id: System.get_env("MICROSOFT_CLIENT_ID"), + client_secret: System.get_env("MICROSOFT_CLIENT_SECRET") + +config :ueberauth, Ueberauth, + providers: [ + microsoft: {Ueberauth.Strategy.Microsoft, [callback_params: []]} + ] + +# Keycloak +# Note: make sure to add `keycloak:ueberauth_keycloak_strategy` entry to `OAUTH_CONSUMER_STRATEGIES` environment variable +keycloak_url = "https://publicly-reachable-keycloak-instance.org:8080" + +config :ueberauth, Ueberauth.Strategy.Keycloak.OAuth, + client_id: System.get_env("KEYCLOAK_CLIENT_ID"), + client_secret: System.get_env("KEYCLOAK_CLIENT_SECRET"), + site: keycloak_url, + authorize_url: "#{keycloak_url}/auth/realms/master/protocol/openid-connect/auth", + token_url: "#{keycloak_url}/auth/realms/master/protocol/openid-connect/token", + userinfo_url: "#{keycloak_url}/auth/realms/master/protocol/openid-connect/userinfo", + token_method: :post + +config :ueberauth, Ueberauth, + providers: [ + keycloak: {Ueberauth.Strategy.Keycloak, [uid_field: :email]} + ] +``` + +## OAuth 2.0 provider - :oauth2 + +Configure OAuth 2 provider capabilities: + +* `token_expires_in` - The lifetime in seconds of the access token. +* `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token. +* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`. +* `clean_expired_tokens_interval` - Interval to run the job to clean expired tokens. Defaults to `86_400_000` (24 hours). + +## :emoji +* `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]` +* `pack_extensions`: A list of file extensions for emojis, when no emoji.txt for a pack is present. Example `[".png", ".gif"]` +* `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]` +* `default_manifest`: Location of the JSON-manifest. This manifest contains information about the emoji-packs you can download. Currently only one manifest can be added (no arrays). + +## Database options + +### RUM indexing for full text search +* `rum_enabled`: If RUM indexes should be used. Defaults to `false`. + +RUM indexes are an alternative indexing scheme that is not included in PostgreSQL by default. While they may eventually be mainlined, for now they have to be installed as a PostgreSQL extension from https://github.com/postgrespro/rum. + +Their advantage over the standard GIN indexes is that they allow efficient ordering of search results by timestamp, which makes search queries a lot faster on larger servers, by one or two orders of magnitude. They take up around 3 times as much space as GIN indexes. + +To enable them, both the `rum_enabled` flag has to be set and the following special migration has to be run: + +`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/` + +This will probably take a long time. + +## :rate_limit + +This is an advanced feature and disabled by default. + +A keyword list of rate limiters where a key is a limiter name and value is the limiter configuration. The basic configuration is a tuple where: + +* The first element: `scale` (Integer). The time scale in milliseconds. +* The second element: `limit` (Integer). How many requests to limit in the time scale provided. + +It is also possible to have different limits for unauthenticated and authenticated users: the keyword value must be a list of two tuples where the first one is a config for unauthenticated users and the second one is for authenticated. + +See [`Pleroma.Plugs.RateLimiter`](Pleroma.Plugs.RateLimiter.html) documentation for examples. + +Supported rate limiters: + +* `:search` for the search requests (account & status search etc.) +* `:app_account_creation` for registering user accounts from the same IP address +* `:relations_actions` for actions on relations with all users (follow, unfollow) +* `:relation_id_action` for actions on relation with a specific user (follow, unfollow) +* `:statuses_actions` for create / delete / fav / unfav / reblog / unreblog actions on any statuses +* `:status_id_action` for fav / unfav or reblog / unreblog actions on the same status by the same user + +## :web_cache_ttl + +The expiration time for the web responses cache. Values should be in milliseconds or `nil` to disable expiration. + +Available caches: + +* `:activity_pub` - activity pub routes (except question activities). Defaults to `nil` (no expiration). +* `:activity_pub_question` - activity pub routes (question activities). Defaults to `30_000` (30 seconds). From 4da0da9aa46f0970735f398a1786902f4e3a86eb Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 13 Sep 2019 19:13:04 +0300 Subject: [PATCH 080/138] don't track generated_config.md --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9591f997..4e71a7df 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ erl_crash.dump # Prevent committing docs files /priv/static/doc/* +docs/generated_config.md # Code test coverage /cover From ac4a748fad34c02647bf72e802cd9d74205681fe Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 13 Sep 2019 19:28:35 +0300 Subject: [PATCH 081/138] Disallow NULLs in deliveries --- lib/pleroma/delivery.ex | 1 + priv/repo/migrations/20190912065617_create_deliveries.exs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/delivery.ex b/lib/pleroma/delivery.ex index 38c148c3..29a1e5a7 100644 --- a/lib/pleroma/delivery.ex +++ b/lib/pleroma/delivery.ex @@ -23,6 +23,7 @@ defmodule Pleroma.Delivery do def changeset(delivery, params \\ %{}) do delivery |> cast(params, [:user_id, :object_id]) + |> validate_required([:user_id, :object_id]) |> foreign_key_constraint(:object_id) |> foreign_key_constraint(:user_id) |> unique_constraint(:user_id, name: :deliveries_user_id_object_id_index) diff --git a/priv/repo/migrations/20190912065617_create_deliveries.exs b/priv/repo/migrations/20190912065617_create_deliveries.exs index 92ca5650..79071a79 100644 --- a/priv/repo/migrations/20190912065617_create_deliveries.exs +++ b/priv/repo/migrations/20190912065617_create_deliveries.exs @@ -3,8 +3,8 @@ defmodule Pleroma.Repo.Migrations.CreateDeliveries do def change do create_if_not_exists table(:deliveries) do - add(:object_id, references(:objects, type: :id)) - add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) + add(:object_id, references(:objects, type: :id), null: false) + add(:user_id, references(:users, type: :uuid, on_delete: :delete_all), null: false) end create_if_not_exists index(:deliveries, :object_id, name: :deliveries_object_id) create_if_not_exists(unique_index(:deliveries, [:user_id, :object_id])) From 5c5ebd38619bb853a58374918fd8983569ba7c0b Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 14 Sep 2019 01:50:15 +0300 Subject: [PATCH 082/138] Mastodon API: Respect post privacy in favourited/reblogged endpoints --- CHANGELOG.md | 1 + .../controllers/mastodon_api_controller.ex | 4 ++ .../mastodon_api_controller_test.exs | 53 ++++++++++++++++++- 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fe3bf68..0c5e4312 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Security - OStatus: eliminate the possibility of a protocol downgrade attack. - OStatus: prevent following locked accounts, bypassing the approval process. +- Mastodon API: respect post privacy in `/api/v1/statuses/:id/{favourited,reblogged}_by` ### Removed - **Breaking:** GNU Social API with Qvitter extensions support diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex index 0940e07a..060137b8 100644 --- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex @@ -842,6 +842,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do with %Activity{} = activity <- Activity.get_by_id_with_object(id), + {:visible, true} <- {:visible, Visibility.visible_for_user?(activity, user)}, %Object{data: %{"likes" => likes}} <- Object.normalize(activity) do q = from(u in User, where: u.ap_id in ^likes) @@ -853,12 +854,14 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do |> put_view(AccountView) |> render("accounts.json", %{for: user, users: users, as: :user}) else + {:visible, false} -> {:error, :not_found} _ -> json(conn, []) end end def reblogged_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do with %Activity{} = activity <- Activity.get_by_id_with_object(id), + {:visible, true} <- {:visible, Visibility.visible_for_user?(activity, user)}, %Object{data: %{"announcements" => announces}} <- Object.normalize(activity) do q = from(u in User, where: u.ap_id in ^announces) @@ -870,6 +873,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do |> put_view(AccountView) |> render("accounts.json", %{for: user, users: users, as: :user}) else + {:visible, false} -> {:error, :not_found} _ -> json(conn, []) end end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 806ae7e6..c9bce143 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -3698,7 +3698,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do build_conn() |> assign(:user, user) - [conn: conn, activity: activity] + [conn: conn, activity: activity, user: user] end test "returns users who have favorited the status", %{conn: conn, activity: activity} do @@ -3758,6 +3758,32 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do [%{"id" => id}] = response assert id == other_user.id end + + test "requires authentifucation for private posts", %{conn: conn, user: user} do + other_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "@#{other_user.nickname} wanna get some #cofe together?", + "visibility" => "direct" + }) + + {:ok, _, _} = CommonAPI.favorite(activity.id, other_user) + + conn + |> assign(:user, nil) + |> get("/api/v1/statuses/#{activity.id}/favourited_by") + |> json_response(404) + + response = + build_conn() + |> assign(:user, other_user) + |> get("/api/v1/statuses/#{activity.id}/favourited_by") + |> json_response(200) + + [%{"id" => id}] = response + assert id == other_user.id + end end describe "GET /api/v1/statuses/:id/reblogged_by" do @@ -3769,7 +3795,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do build_conn() |> assign(:user, user) - [conn: conn, activity: activity] + [conn: conn, activity: activity, user: user] end test "returns users who have reblogged the status", %{conn: conn, activity: activity} do @@ -3829,6 +3855,29 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do [%{"id" => id}] = response assert id == other_user.id end + + test "requires authentifucation for private posts", %{conn: conn, user: user} do + other_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "@#{other_user.nickname} wanna get some #cofe together?", + "visibility" => "direct" + }) + + conn + |> assign(:user, nil) + |> get("/api/v1/statuses/#{activity.id}/reblogged_by") + |> json_response(404) + + response = + build_conn() + |> assign(:user, other_user) + |> get("/api/v1/statuses/#{activity.id}/reblogged_by") + |> json_response(200) + + assert [] == response + end end describe "POST /auth/password, with valid parameters" do From 85b6144ffd9e96fc79608847fe739a40ca094207 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 14 Sep 2019 10:46:35 +0000 Subject: [PATCH 083/138] Apply suggestion to test/web/mastodon_api/mastodon_api_controller_test.exs --- test/web/mastodon_api/mastodon_api_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index c9bce143..013a838b 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -3759,7 +3759,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do assert id == other_user.id end - test "requires authentifucation for private posts", %{conn: conn, user: user} do + test "requires authentification for private posts", %{conn: conn, user: user} do other_user = insert(:user) {:ok, activity} = From a78a7ee455c4e8f4c2aab15a15626237b2b90399 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 14 Sep 2019 10:50:08 +0000 Subject: [PATCH 084/138] Apply suggestion to test/web/mastodon_api/mastodon_api_controller_test.exs --- test/web/mastodon_api/mastodon_api_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 013a838b..061c3a8a 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -3856,7 +3856,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do assert id == other_user.id end - test "requires authentifucation for private posts", %{conn: conn, user: user} do + test "requires authentification for private posts", %{conn: conn, user: user} do other_user = insert(:user) {:ok, activity} = From b870ae08fd19acd7c40e4353e657e6a1ed1b34c5 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sat, 14 Sep 2019 19:31:20 +0700 Subject: [PATCH 085/138] Fix `Activity.all_by_actor_and_id/2` test --- test/activity_test.exs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/activity_test.exs b/test/activity_test.exs index f9f789a7..275cab81 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -213,7 +213,11 @@ defmodule Pleroma.ActivityTest do assert [] == Activity.all_by_actor_and_id(user, []) - assert [%Activity{id: ^id2}, %Activity{id: ^id1}] = - Activity.all_by_actor_and_id(user.ap_id, [id1, id2]) + activities = + user.ap_id + |> Activity.all_by_actor_and_id([id1, id2]) + |> Enum.sort(&(&1.id < &2.id)) + + assert [%Activity{id: ^id1}, %Activity{id: ^id2}] = activities end end From e127b9ab6d01da48ebad188d2b9fcf7cb8a41578 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 14 Sep 2019 16:28:59 +0300 Subject: [PATCH 086/138] [#1149] Rewritten readme as config/description.exs. --- config/description.exs | 198 ++++++++++++++++++++++++++--------------- 1 file changed, 126 insertions(+), 72 deletions(-) diff --git a/config/description.exs b/config/description.exs index c5ae6391..be5eb0cc 100644 --- a/config/description.exs +++ b/config/description.exs @@ -1778,87 +1778,141 @@ config :pleroma, :config_description, [ group: :pleroma_job_queue, key: :queues, type: :group, - description: "Pleroma Job Queue configuration: a list of queues with maximum concurrent jobs", - children: [ - %{ - key: :federator_outgoing, - type: :integer, - description: "Outgoing federation queue", - suggestions: [50] - }, - %{ - key: :federator_incoming, - type: :integer, - description: "Incoming federation queue", - suggestions: [50] - }, - %{ - key: :mailer, - type: :integer, - description: "Email sender queue, see Pleroma.Emails.Mailer", - suggestions: [10] - }, - %{ - key: :web_push, - type: :integer, - description: "Web push notifications queue", - suggestions: [50] - }, - %{ - key: :transmogrifier, - type: :integer, - description: "Transmogrifier queue", - suggestions: [20] - }, - %{ - key: :scheduled_activities, - type: :integer, - description: "Scheduled activities queue, see Pleroma.ScheduledActivities", - suggestions: [10] - }, - %{ - key: :activity_expiration, - type: :integer, - description: "Activity expiration queue", - suggestions: [10] - }, - %{ - key: :background, - type: :integer, - description: "Background queue", - suggestions: [5] - } - ] + description: "[Deprecated] Replaced with `Oban`/`:queues` (keeping the same format)", + children: [] }, %{ group: :pleroma, key: Pleroma.Web.Federator.RetryQueue, type: :group, - description: "", + description: "[Deprecated] See `Oban` and `:workers` sections for configuration notes", children: [ - %{ - key: :enabled, - type: :boolean, - description: "If set to true, failed federation jobs will be retried", - suggestions: [true, false] - }, - %{ - key: :max_jobs, - type: :integer, - description: "The maximum amount of parallel federation jobs running at the same time", - suggestions: [20] - }, - %{ - key: :initial_timeout, - type: :integer, - description: "The initial timeout in seconds", - suggestions: [30] - }, %{ key: :max_retries, type: :integer, - description: "The maximum number of times a federation job is retried", - suggestions: [5] + description: "[Deprecated] Replaced as `Oban`/`:queues`/`:outgoing_federation` value", + suggestions: [] + } + ] + }, + %{ + group: :pleroma, + key: Oban, + type: :group, + description: """ + [Oban](https://github.com/sorentwo/oban) asynchronous job processor configuration. + + Note: if you are running PostgreSQL in [`silent_mode`](https://postgresqlco.nf/en/doc/param/silent_mode?version=9.1), + it's advised to set [`log_destination`](https://postgresqlco.nf/en/doc/param/log_destination?version=9.1) to `syslog`, + otherwise `postmaster.log` file may grow because of "you don't own a lock of type ShareLock" warnings + (see https://github.com/sorentwo/oban/issues/52). + """, + children: [ + %{ + key: :repo, + type: :module, + description: "Application's Ecto repo", + suggestions: [Pleroma.Repo] + }, + %{ + key: :verbose, + type: :boolean, + description: "Logs verbose mode", + suggestions: [false, true] + }, + %{ + key: :prune, + type: [:atom, :tuple], + description: + "Non-retryable jobs [pruning settings](https://github.com/sorentwo/oban#pruning)", + suggestions: [:disabled, {:maxlen, 1500}, {:maxage, 60 * 60}] + }, + %{ + key: :queues, + type: :keyword, + description: + "Background jobs queues (keys: queues, values: max numbers of concurrent jobs)", + suggestions: [ + [ + activity_expiration: 10, + background: 5, + federator_incoming: 50, + federator_outgoing: 50, + mailer: 10, + scheduled_activities: 10, + transmogrifier: 20, + web_push: 50 + ] + ], + children: [ + %{ + key: :activity_expiration, + type: :integer, + description: "Activity expiration queue", + suggestions: [10] + }, + %{ + key: :background, + type: :integer, + description: "Background queue", + suggestions: [5] + }, + %{ + key: :federator_incoming, + type: :integer, + description: "Incoming federation queue", + suggestions: [50] + }, + %{ + key: :federator_outgoing, + type: :integer, + description: "Outgoing federation queue", + suggestions: [50] + }, + %{ + key: :mailer, + type: :integer, + description: "Email sender queue, see Pleroma.Emails.Mailer", + suggestions: [10] + }, + %{ + key: :scheduled_activities, + type: :integer, + description: "Scheduled activities queue, see Pleroma.ScheduledActivities", + suggestions: [10] + }, + %{ + key: :transmogrifier, + type: :integer, + description: "Transmogrifier queue", + suggestions: [20] + }, + %{ + key: :web_push, + type: :integer, + description: "Web push notifications queue", + suggestions: [50] + } + ] + } + ] + }, + %{ + group: :pleroma, + key: :workers, + type: :group, + description: "Includes custom worker options not interpretable directly by `Oban`", + children: [ + %{ + key: :retries, + type: :keyword, + description: "Max retry attempts for failed jobs, per `Oban` queue", + suggestions: [ + [ + federator_incoming: 5, + federator_outgoing: 5 + ] + ] } ] }, From 3b8ec98b0e3b3fb2bd333f3be724676c4821366f Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 15 Sep 2019 10:15:57 +0300 Subject: [PATCH 087/138] [#1149] Reinstated docs/config.md changes. --- docs/config.md | 1456 +++++++++++++++++++++++++----------------------- 1 file changed, 755 insertions(+), 701 deletions(-) diff --git a/docs/config.md b/docs/config.md index 066547bb..270d7fce 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1,701 +1,755 @@ -# Configuration - -This file describe the configuration, it is recommended to edit the relevant *.secret.exs file instead of the others founds in the ``config`` directory. -If you run Pleroma with ``MIX_ENV=prod`` the file is ``prod.secret.exs``, otherwise it is ``dev.secret.exs``. - -## Pleroma.Upload -* `uploader`: Select which `Pleroma.Uploaders` to use -* `filters`: List of `Pleroma.Upload.Filter` to use. -* `link_name`: When enabled Pleroma will add a `name` parameter to the url of the upload, for example `https://instance.tld/media/corndog.png?name=corndog.png`. This is needed to provide the correct filename in Content-Disposition headers when using filters like `Pleroma.Upload.Filter.Dedupe` -* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host. -* `proxy_remote`: If you're using a remote uploader, Pleroma will proxy media requests instead of redirecting to it. -* `proxy_opts`: Proxy options, see `Pleroma.ReverseProxy` documentation. - -Note: `strip_exif` has been replaced by `Pleroma.Upload.Filter.Mogrify`. - -## Pleroma.Uploaders.Local -* `uploads`: Which directory to store the user-uploads in, relative to pleroma’s working directory - -## Pleroma.Uploaders.S3 -* `bucket`: S3 bucket name -* `bucket_namespace`: S3 bucket namespace -* `public_endpoint`: S3 endpoint that the user finally accesses(ex. "https://s3.dualstack.ap-northeast-1.amazonaws.com") -* `truncated_namespace`: If you use S3 compatible service such as Digital Ocean Spaces or CDN, set folder name or "" etc. -For example, when using CDN to S3 virtual host format, set "". -At this time, write CNAME to CDN in public_endpoint. - -## Pleroma.Upload.Filter.Mogrify - -* `args`: List of actions for the `mogrify` command like `"strip"` or `["strip", "auto-orient", {"implode", "1"}]`. - -## Pleroma.Upload.Filter.Dedupe - -No specific configuration. - -## Pleroma.Upload.Filter.AnonymizeFilename - -This filter replaces the filename (not the path) of an upload. For complete obfuscation, add -`Pleroma.Upload.Filter.Dedupe` before AnonymizeFilename. - -* `text`: Text to replace filenames in links. If empty, `{random}.extension` will be used. You can get the original filename extension by using `{extension}`, for example `custom-file-name.{extension}`. - -## Pleroma.Emails.Mailer -* `adapter`: one of the mail adapters listed in [Swoosh readme](https://github.com/swoosh/swoosh#adapters), or `Swoosh.Adapters.Local` for in-memory mailbox. -* `api_key` / `password` and / or other adapter-specific settings, per the above documentation. -* `enabled`: Allows enable/disable send emails. Default: `false`. - -An example for Sendgrid adapter: - -```elixir -config :pleroma, Pleroma.Emails.Mailer, - adapter: Swoosh.Adapters.Sendgrid, - api_key: "YOUR_API_KEY" -``` - -An example for SMTP adapter: - -```elixir -config :pleroma, Pleroma.Emails.Mailer, - adapter: Swoosh.Adapters.SMTP, - relay: "smtp.gmail.com", - username: "YOUR_USERNAME@gmail.com", - password: "YOUR_SMTP_PASSWORD", - port: 465, - ssl: true, - tls: :always, - auth: :always -``` - -## :uri_schemes -* `valid_schemes`: List of the scheme part that is considered valid to be an URL - -## :instance -* `name`: The instance’s name -* `email`: Email used to reach an Administrator/Moderator of the instance -* `notify_email`: Email used for notifications. -* `description`: The instance’s description, can be seen in nodeinfo and ``/api/v1/instance`` -* `limit`: Posts character limit (CW/Subject included in the counter) -* `remote_limit`: Hard character limit beyond which remote posts will be dropped. -* `upload_limit`: File size limit of uploads (except for avatar, background, banner) -* `avatar_upload_limit`: File size limit of user’s profile avatars -* `background_upload_limit`: File size limit of user’s profile backgrounds -* `banner_upload_limit`: File size limit of user’s profile banners -* `poll_limits`: A map with poll limits for **local** polls - * `max_options`: Maximum number of options - * `max_option_chars`: Maximum number of characters per option - * `min_expiration`: Minimum expiration time (in seconds) - * `max_expiration`: Maximum expiration time (in seconds) -* `registrations_open`: Enable registrations for anyone, invitations can be enabled when false. -* `invites_enabled`: Enable user invitations for admins (depends on `registrations_open: false`). -* `account_activation_required`: Require users to confirm their emails before signing in. -* `federating`: Enable federation with other instances -* `federation_incoming_replies_max_depth`: Max. depth of reply-to activities fetching on incoming federation, to prevent out-of-memory situations while fetching very long threads. If set to `nil`, threads of any depth will be fetched. Lower this value if you experience out-of-memory crashes. -* `federation_reachability_timeout_days`: Timeout (in days) of each external federation target being unreachable prior to pausing federating to it. -* `allow_relay`: Enable Pleroma’s Relay, which makes it possible to follow a whole instance -* `rewrite_policy`: Message Rewrite Policy, either one or a list. Here are the ones available by default: - * `Pleroma.Web.ActivityPub.MRF.NoOpPolicy`: Doesn’t modify activities (default) - * `Pleroma.Web.ActivityPub.MRF.DropPolicy`: Drops all activities. It generally doesn’t makes sense to use in production - * `Pleroma.Web.ActivityPub.MRF.SimplePolicy`: Restrict the visibility of activities from certains instances (See ``:mrf_simple`` section) - * `Pleroma.Web.ActivityPub.MRF.TagPolicy`: Applies policies to individual users based on tags, which can be set using pleroma-fe/admin-fe/any other app that supports Pleroma Admin API. For example it allows marking posts from individual users nsfw (sensitive) - * `Pleroma.Web.ActivityPub.MRF.SubchainPolicy`: Selectively runs other MRF policies when messages match (see ``:mrf_subchain`` section) - * `Pleroma.Web.ActivityPub.MRF.RejectNonPublic`: Drops posts with non-public visibility settings (See ``:mrf_rejectnonpublic`` section) - * `Pleroma.Web.ActivityPub.MRF.EnsureRePrepended`: Rewrites posts to ensure that replies to posts with subjects do not have an identical subject and instead begin with re:. - * `Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy`: Rejects posts from likely spambots by rejecting posts from new users that contain links. - * `Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`: Crawls attachments using their MediaProxy URLs so that the MediaProxy cache is primed. - * `Pleroma.Web.ActivityPub.MRF.MentionPolicy`: Drops posts mentioning configurable users. (see `:mrf_mention` section) - * `Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`: Restricts activities to a configured set of vocabulary. (see `:mrf_vocabulary` section) -* `public`: Makes the client API in authentificated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. -* `quarantined_instances`: List of ActivityPub instances where private(DMs, followers-only) activities will not be send. -* `managed_config`: Whenether the config for pleroma-fe is configured in this config or in ``static/config.json`` -* `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML) -* `mrf_transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo). -* `mrf_transparency_exclusions`: Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value. -* `scope_copy`: Copy the scope (private/unlisted/public) in replies to posts by default. -* `subject_line_behavior`: Allows changing the default behaviour of subject lines in replies. Valid values: - * "email": Copy and preprend re:, as in email. - * "masto": Copy verbatim, as in Mastodon. - * "noop": Don't copy the subject. -* `always_show_subject_input`: When set to false, auto-hide the subject field when it's empty. -* `extended_nickname_format`: Set to `true` to use extended local nicknames format (allows underscores/dashes). This will break federation with - older software for theses nicknames. -* `max_pinned_statuses`: The maximum number of pinned statuses. `0` will disable the feature. -* `autofollowed_nicknames`: Set to nicknames of (local) users that every new user should automatically follow. -* `no_attachment_links`: Set to true to disable automatically adding attachment link text to statuses -* `welcome_message`: A message that will be send to a newly registered users as a direct message. -* `welcome_user_nickname`: The nickname of the local user that sends the welcome message. -* `max_report_comment_size`: The maximum size of the report comment (Default: `1000`) -* `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). Default: `false`. -* `healthcheck`: If set to true, system data will be shown on ``/api/pleroma/healthcheck``. -* `remote_post_retention_days`: The default amount of days to retain remote posts when pruning the database. -* `user_bio_length`: A user bio maximum length (default: `5000`) -* `user_name_length`: A user name maximum length (default: `100`) -* `skip_thread_containment`: Skip filter out broken threads. The default is `false`. -* `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. -* `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. -* `max_account_fields`: The maximum number of custom fields in the user profile (default: `10`) -* `max_remote_account_fields`: The maximum number of custom fields in the remote user profile (default: `20`) -* `account_field_name_length`: An account field name maximum length (default: `512`) -* `account_field_value_length`: An account field value maximum length (default: `512`) -* `external_user_synchronization`: Enabling following/followers counters synchronization for external users. - - - -## :logger -* `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog, and `Quack.Logger` to log to Slack - -An example to enable ONLY ExSyslogger (f/ex in ``prod.secret.exs``) with info and debug suppressed: -```elixir -config :logger, - backends: [{ExSyslogger, :ex_syslogger}] - -config :logger, :ex_syslogger, - level: :warn -``` - -Another example, keeping console output and adding the pid to syslog output: -```elixir -config :logger, - backends: [:console, {ExSyslogger, :ex_syslogger}] - -config :logger, :ex_syslogger, - level: :warn, - option: [:pid, :ndelay] -``` - -See: [logger’s documentation](https://hexdocs.pm/logger/Logger.html) and [ex_syslogger’s documentation](https://hexdocs.pm/ex_syslogger/) - -An example of logging info to local syslog, but warn to a Slack channel: -```elixir -config :logger, - backends: [ {ExSyslogger, :ex_syslogger}, Quack.Logger ], - level: :info - -config :logger, :ex_syslogger, - level: :info, - ident: "pleroma", - format: "$metadata[$level] $message" - -config :quack, - level: :warn, - meta: [:all], - webhook_url: "https://hooks.slack.com/services/YOUR-API-KEY-HERE" -``` - -See the [Quack Github](https://github.com/azohra/quack) for more details - -## :frontend_configurations - -This can be used to configure a keyword list that keeps the configuration data for any kind of frontend. By default, settings for `pleroma_fe` and `masto_fe` are configured. - -Frontends can access these settings at `/api/pleroma/frontend_configurations` - -To add your own configuration for PleromaFE, use it like this: - -```elixir -config :pleroma, :frontend_configurations, - pleroma_fe: %{ - theme: "pleroma-dark", - # ... see /priv/static/static/config.json for the available keys. -}, - masto_fe: %{ - showInstanceSpecificPanel: true - } -``` - -These settings **need to be complete**, they will override the defaults. - -NOTE: for versions < 1.0, you need to set [`:fe`](#fe) to false, as shown a few lines below. - -## :fe -__THIS IS DEPRECATED__ - -If you are using this method, please change it to the [`frontend_configurations`](#frontend_configurations) method. -Please **set this option to false** in your config like this: - -```elixir -config :pleroma, :fe, false -``` - -This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:instance`` is set to false. - -* `theme`: Which theme to use, they are defined in ``styles.json`` -* `logo`: URL of the logo, defaults to Pleroma’s logo -* `logo_mask`: Whether to use only the logo's shape as a mask (true) or as a regular image (false) -* `logo_margin`: What margin to use around the logo -* `background`: URL of the background, unless viewing a user profile with a background that is set -* `redirect_root_no_login`: relative URL which indicates where to redirect when a user isn’t logged in. -* `redirect_root_login`: relative URL which indicates where to redirect when a user is logged in. -* `show_instance_panel`: Whenether to show the instance’s specific panel. -* `scope_options_enabled`: Enable setting an notice visibility and subject/CW when posting -* `formatting_options_enabled`: Enable setting a formatting different than plain-text (ie. HTML, Markdown) when posting, relates to ``:instance, allowed_post_formats`` -* `collapse_message_with_subjects`: When a message has a subject(aka Content Warning), collapse it by default -* `hide_post_stats`: Hide notices statistics(repeats, favorites, …) -* `hide_user_stats`: Hide profile statistics(posts, posts per day, followers, followings, …) - -## :assets - -This section configures assets to be used with various frontends. Currently the only option -relates to mascots on the mastodon frontend - -* `mascots`: KeywordList of mascots, each element __MUST__ contain both a `url` and a - `mime_type` key. -* `default_mascot`: An element from `mascots` - This will be used as the default mascot - on MastoFE (default: `:pleroma_fox_tan`) - -## :mrf_simple -* `media_removal`: List of instances to remove medias from -* `media_nsfw`: List of instances to put medias as NSFW(sensitive) from -* `federated_timeline_removal`: List of instances to remove from Federated (aka The Whole Known Network) Timeline -* `reject`: List of instances to reject any activities from -* `accept`: List of instances to accept any activities from -* `report_removal`: List of instances to reject reports from -* `avatar_removal`: List of instances to strip avatars from -* `banner_removal`: List of instances to strip banners from - -## :mrf_subchain -This policy processes messages through an alternate pipeline when a given message matches certain criteria. -All criteria are configured as a map of regular expressions to lists of policy modules. - -* `match_actor`: Matches a series of regular expressions against the actor field. - -Example: - -``` -config :pleroma, :mrf_subchain, - match_actor: %{ - ~r/https:\/\/example.com/s => [Pleroma.Web.ActivityPub.MRF.DropPolicy] - } -``` - -## :mrf_rejectnonpublic -* `allow_followersonly`: whether to allow followers-only posts -* `allow_direct`: whether to allow direct messages - -## :mrf_hellthread -* `delist_threshold`: Number of mentioned users after which the message gets delisted (the message can still be seen, but it will not show up in public timelines and mentioned users won't get notifications about it). Set to 0 to disable. -* `reject_threshold`: Number of mentioned users after which the messaged gets rejected. Set to 0 to disable. - -## :mrf_keyword -* `reject`: A list of patterns which result in message being rejected, each pattern can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) -* `federated_timeline_removal`: A list of patterns which result in message being removed from federated timelines (a.k.a unlisted), each pattern can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) -* `replace`: A list of tuples containing `{pattern, replacement}`, `pattern` can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) - -## :mrf_mention -* `actors`: A list of actors, for which to drop any posts mentioning. - -## :mrf_vocabulary -* `accept`: A list of ActivityStreams terms to accept. If empty, all supported messages are accepted. -* `reject`: A list of ActivityStreams terms to reject. If empty, no messages are rejected. - -## :media_proxy -* `enabled`: Enables proxying of remote media to the instance’s proxy -* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts. -* `proxy_opts`: All options defined in `Pleroma.ReverseProxy` documentation, defaults to `[max_body_length: (25*1_048_576)]`. -* `whitelist`: List of domains to bypass the mediaproxy - -## :gopher -* `enabled`: Enables the gopher interface -* `ip`: IP address to bind to -* `port`: Port to bind to -* `dstport`: Port advertised in urls (optional, defaults to `port`) - -## Pleroma.Web.Endpoint -`Phoenix` endpoint configuration, all configuration options can be viewed [here](https://hexdocs.pm/phoenix/Phoenix.Endpoint.html#module-dynamic-configuration), only common options are listed here -* `http` - a list containing http protocol configuration, all configuration options can be viewed [here](https://hexdocs.pm/plug_cowboy/Plug.Cowboy.html#module-options), only common options are listed here. For deployment using docker, you need to set this to `[ip: {0,0,0,0}, port: 4000]` to make pleroma accessible from other containers (such as your nginx server). - - `ip` - a tuple consisting of 4 integers - - `port` -* `url` - a list containing the configuration for generating urls, accepts - - `host` - the host without the scheme and a post (e.g `example.com`, not `https://example.com:2020`) - - `scheme` - e.g `http`, `https` - - `port` - - `path` -* `extra_cookie_attrs` - a list of `Key=Value` strings to be added as non-standard cookie attributes. Defaults to `["SameSite=Lax"]`. See the [SameSite article](https://www.owasp.org/index.php/SameSite) on OWASP for more info. - - - -**Important note**: if you modify anything inside these lists, default `config.exs` values will be overwritten, which may result in breakage, to make sure this does not happen please copy the default value for the list from `config.exs` and modify/add only what you need - -Example: -```elixir -config :pleroma, Pleroma.Web.Endpoint, - url: [host: "example.com", port: 2020, scheme: "https"], - http: [ - # start copied from config.exs - dispatch: [ - {:_, - [ - {"/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, []}, - {"/websocket", Phoenix.Endpoint.CowboyWebSocket, - {Phoenix.Transports.WebSocket, - {Pleroma.Web.Endpoint, Pleroma.Web.UserSocket, websocket_config}}}, - {:_, Phoenix.Endpoint.Cowboy2Handler, {Pleroma.Web.Endpoint, []}} - ]} - # end copied from config.exs - ], - port: 8080, - ip: {127, 0, 0, 1} - ] -``` - -This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls starting with `https://example.com:2020` - -## :activitypub -* ``unfollow_blocked``: Whether blocks result in people getting unfollowed -* ``outgoing_blocks``: Whether to federate blocks to other instances -* ``deny_follow_blocked``: Whether to disallow following an account that has blocked the user in question -* ``sign_object_fetches``: Sign object fetches with HTTP signatures - -## :http_security -* ``enabled``: Whether the managed content security policy is enabled -* ``sts``: Whether to additionally send a `Strict-Transport-Security` header -* ``sts_max_age``: The maximum age for the `Strict-Transport-Security` header if sent -* ``ct_max_age``: The maximum age for the `Expect-CT` header if sent -* ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"` -* ``report_uri``: Adds the specified url to `report-uri` and `report-to` group in CSP header. - -## :mrf_user_allowlist - -The keys in this section are the domain names that the policy should apply to. -Each key should be assigned a list of users that should be allowed through by -their ActivityPub ID. - -An example: - -```elixir -config :pleroma, :mrf_user_allowlist, - "example.org": ["https://example.org/users/admin"] -``` - -## :web_push_encryption, :vapid_details - -Web Push Notifications configuration. You can use the mix task `mix web_push.gen.keypair` to generate it. - -* ``subject``: a mailto link for the administrative contact. It’s best if this email is not a personal email address, but rather a group email so that if a person leaves an organization, is unavailable for an extended period, or otherwise can’t respond, someone else on the list can. -* ``public_key``: VAPID public key -* ``private_key``: VAPID private key - -## Pleroma.Captcha -* `enabled`: Whether the captcha should be shown on registration -* `method`: The method/service to use for captcha -* `seconds_valid`: The time in seconds for which the captcha is valid - -### Pleroma.Captcha.Kocaptcha -Kocaptcha is a very simple captcha service with a single API endpoint, -the source code is here: https://github.com/koto-bank/kocaptcha. The default endpoint -`https://captcha.kotobank.ch` is hosted by the developer. - -* `endpoint`: the kocaptcha endpoint to use - -## :admin_token - -Allows to set a token that can be used to authenticate with the admin api without using an actual user by giving it as the 'admin_token' parameter. Example: - -```elixir -config :pleroma, :admin_token, "somerandomtoken" -``` - -You can then do - -```sh -curl "http://localhost:4000/api/pleroma/admin/invite_token?admin_token=somerandomtoken" -``` - -## :pleroma_job_queue - -[Pleroma Job Queue](https://git.pleroma.social/pleroma/pleroma_job_queue) configuration: a list of queues with maximum concurrent jobs. - -Pleroma has the following queues: - -* `federator_outgoing` - Outgoing federation -* `federator_incoming` - Incoming federation -* `mailer` - Email sender, see [`Pleroma.Emails.Mailer`](#pleroma-emails-mailer) -* `transmogrifier` - Transmogrifier -* `web_push` - Web push notifications -* `scheduled_activities` - Scheduled activities, see [`Pleroma.ScheduledActivities`](#pleromascheduledactivity) - -Example: - -```elixir -config :pleroma_job_queue, :queues, - federator_incoming: 50, - federator_outgoing: 50 -``` - -This config contains two queues: `federator_incoming` and `federator_outgoing`. Both have the `max_jobs` set to `50`. - -## Pleroma.Web.Federator.RetryQueue - -* `enabled`: If set to `true`, failed federation jobs will be retried -* `max_jobs`: The maximum amount of parallel federation jobs running at the same time. -* `initial_timeout`: The initial timeout in seconds -* `max_retries`: The maximum number of times a federation job is retried - -## Pleroma.Web.Metadata -* `providers`: a list of metadata providers to enable. Providers available: - * Pleroma.Web.Metadata.Providers.OpenGraph - * Pleroma.Web.Metadata.Providers.TwitterCard - * Pleroma.Web.Metadata.Providers.RelMe - add links from user bio with rel=me into the `
` as `` -* `unfurl_nsfw`: If set to `true` nsfw attachments will be shown in previews - -## :rich_media -* `enabled`: if enabled the instance will parse metadata from attached links to generate link previews -* `ignore_hosts`: list of hosts which will be ignored by the metadata parser. For example `["accounts.google.com", "xss.website"]`, defaults to `[]`. -* `ignore_tld`: list TLDs (top-level domains) which will ignore for parse metadata. default is ["local", "localdomain", "lan"] -* `parsers`: list of Rich Media parsers - -## :fetch_initial_posts -* `enabled`: if enabled, when a new user is federated with, fetch some of their latest posts -* `pages`: the amount of pages to fetch - -## :hackney_pools - -Advanced. Tweaks Hackney (http client) connections pools. - -There's three pools used: - -* `:federation` for the federation jobs. - You may want this pool max_connections to be at least equal to the number of federator jobs + retry queue jobs. -* `:media` for rich media, media proxy -* `:upload` for uploaded media (if using a remote uploader and `proxy_remote: true`) - -For each pool, the options are: - -* `max_connections` - how much connections a pool can hold -* `timeout` - retention duration for connections - -## :auto_linker - -Configuration for the `auto_linker` library: - -* `class: "auto-linker"` - specify the class to be added to the generated link. false to clear -* `rel: "noopener noreferrer"` - override the rel attribute. false to clear -* `new_window: true` - set to false to remove `target='_blank'` attribute -* `scheme: false` - Set to true to link urls with schema `http://google.com` -* `truncate: false` - Set to a number to truncate urls longer then the number. Truncated urls will end in `..` -* `strip_prefix: true` - Strip the scheme prefix -* `extra: false` - link urls with rarely used schemes (magnet, ipfs, irc, etc.) - -Example: - -```elixir -config :auto_linker, - opts: [ - scheme: true, - extra: true, - class: false, - strip_prefix: false, - new_window: false, - rel: false - ] -``` - -## Pleroma.ScheduledActivity - -* `daily_user_limit`: the number of scheduled activities a user is allowed to create in a single day (Default: `25`) -* `total_user_limit`: the number of scheduled activities a user is allowed to create in total (Default: `300`) -* `enabled`: whether scheduled activities are sent to the job queue to be executed - -## Pleroma.ActivityExpiration - -# `enabled`: whether expired activities will be sent to the job queue to be deleted - -## Pleroma.Web.Auth.Authenticator - -* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator -* `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication - -## :ldap - -Use LDAP for user authentication. When a user logs in to the Pleroma -instance, the name and password will be verified by trying to authenticate -(bind) to an LDAP server. If a user exists in the LDAP directory but there -is no account with the same name yet on the Pleroma instance then a new -Pleroma account will be created with the same name as the LDAP user name. - -* `enabled`: enables LDAP authentication -* `host`: LDAP server hostname -* `port`: LDAP port, e.g. 389 or 636 -* `ssl`: true to use SSL, usually implies the port 636 -* `sslopts`: additional SSL options -* `tls`: true to start TLS, usually implies the port 389 -* `tlsopts`: additional TLS options -* `base`: LDAP base, e.g. "dc=example,dc=com" -* `uid`: LDAP attribute name to authenticate the user, e.g. when "cn", the filter will be "cn=username,base" - -## BBS / SSH access - -To enable simple command line interface accessible over ssh, add a setting like this to your configuration file: - -```exs -app_dir = File.cwd! -priv_dir = Path.join([app_dir, "priv/ssh_keys"]) - -config :esshd, - enabled: true, - priv_dir: priv_dir, - handler: "Pleroma.BBS.Handler", - port: 10_022, - password_authenticator: "Pleroma.BBS.Authenticator" -``` - -Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -m PEM -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT` - -## :auth - -* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator -* `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication - -Authentication / authorization settings. - -* `auth_template`: authentication form template. By default it's `show.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/show.html.eex`. -* `oauth_consumer_template`: OAuth consumer mode authentication form template. By default it's `consumer.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex`. -* `oauth_consumer_strategies`: the list of enabled OAuth consumer strategies; by default it's set by `OAUTH_CONSUMER_STRATEGIES` environment variable. Each entry in this space-delimited string should be of format `` or `:` (e.g. `twitter` or `keycloak:ueberauth_keycloak_strategy` in case dependency is named differently than `ueberauth_`). - -## :email_notifications - -Email notifications settings. - - - digest - emails of "what you've missed" for users who have been - inactive for a while. - - active: globally enable or disable digest emails - - schedule: When to send digest email, in [crontab format](https://en.wikipedia.org/wiki/Cron). - "0 0 * * 0" is the default, meaning "once a week at midnight on Sunday morning" - - interval: Minimum interval between digest emails to one user - - inactivity_threshold: Minimum user inactivity threshold - -## Pleroma.Emails.UserEmail - -- `:logo` - a path to a custom logo. Set it to `nil` to use the default Pleroma logo. -- `:styling` - a map with color settings for email templates. - -## OAuth consumer mode - -OAuth consumer mode allows sign in / sign up via external OAuth providers (e.g. Twitter, Facebook, Google, Microsoft, etc.). -Implementation is based on Ueberauth; see the list of [available strategies](https://github.com/ueberauth/ueberauth/wiki/List-of-Strategies). - -Note: each strategy is shipped as a separate dependency; in order to get the strategies, run `OAUTH_CONSUMER_STRATEGIES="..." mix deps.get`, -e.g. `OAUTH_CONSUMER_STRATEGIES="twitter facebook google microsoft" mix deps.get`. -The server should also be started with `OAUTH_CONSUMER_STRATEGIES="..." mix phx.server` in case you enable any strategies. - -Note: each strategy requires separate setup (on external provider side and Pleroma side). Below are the guidelines on setting up most popular strategies. - -Note: make sure that `"SameSite=Lax"` is set in `extra_cookie_attrs` when you have this feature enabled. OAuth consumer mode will not work with `"SameSite=Strict"` - -* For Twitter, [register an app](https://developer.twitter.com/en/apps), configure callback URL to https:///oauth/twitter/callback - -* For Facebook, [register an app](https://developers.facebook.com/apps), configure callback URL to https:///oauth/facebook/callback, enable Facebook Login service at https://developers.facebook.com/apps//fb-login/settings/ - -* For Google, [register an app](https://console.developers.google.com), configure callback URL to https:///oauth/google/callback - -* For Microsoft, [register an app](https://portal.azure.com), configure callback URL to https:///oauth/microsoft/callback - -Once the app is configured on external OAuth provider side, add app's credentials and strategy-specific settings (if any — e.g. see Microsoft below) to `config/prod.secret.exs`, -per strategy's documentation (e.g. [ueberauth_twitter](https://github.com/ueberauth/ueberauth_twitter)). Example config basing on environment variables: - -```elixir -# Twitter -config :ueberauth, Ueberauth.Strategy.Twitter.OAuth, - consumer_key: System.get_env("TWITTER_CONSUMER_KEY"), - consumer_secret: System.get_env("TWITTER_CONSUMER_SECRET") - -# Facebook -config :ueberauth, Ueberauth.Strategy.Facebook.OAuth, - client_id: System.get_env("FACEBOOK_APP_ID"), - client_secret: System.get_env("FACEBOOK_APP_SECRET"), - redirect_uri: System.get_env("FACEBOOK_REDIRECT_URI") - -# Google -config :ueberauth, Ueberauth.Strategy.Google.OAuth, - client_id: System.get_env("GOOGLE_CLIENT_ID"), - client_secret: System.get_env("GOOGLE_CLIENT_SECRET"), - redirect_uri: System.get_env("GOOGLE_REDIRECT_URI") - -# Microsoft -config :ueberauth, Ueberauth.Strategy.Microsoft.OAuth, - client_id: System.get_env("MICROSOFT_CLIENT_ID"), - client_secret: System.get_env("MICROSOFT_CLIENT_SECRET") - -config :ueberauth, Ueberauth, - providers: [ - microsoft: {Ueberauth.Strategy.Microsoft, [callback_params: []]} - ] - -# Keycloak -# Note: make sure to add `keycloak:ueberauth_keycloak_strategy` entry to `OAUTH_CONSUMER_STRATEGIES` environment variable -keycloak_url = "https://publicly-reachable-keycloak-instance.org:8080" - -config :ueberauth, Ueberauth.Strategy.Keycloak.OAuth, - client_id: System.get_env("KEYCLOAK_CLIENT_ID"), - client_secret: System.get_env("KEYCLOAK_CLIENT_SECRET"), - site: keycloak_url, - authorize_url: "#{keycloak_url}/auth/realms/master/protocol/openid-connect/auth", - token_url: "#{keycloak_url}/auth/realms/master/protocol/openid-connect/token", - userinfo_url: "#{keycloak_url}/auth/realms/master/protocol/openid-connect/userinfo", - token_method: :post - -config :ueberauth, Ueberauth, - providers: [ - keycloak: {Ueberauth.Strategy.Keycloak, [uid_field: :email]} - ] -``` - -## OAuth 2.0 provider - :oauth2 - -Configure OAuth 2 provider capabilities: - -* `token_expires_in` - The lifetime in seconds of the access token. -* `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token. -* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`. -* `clean_expired_tokens_interval` - Interval to run the job to clean expired tokens. Defaults to `86_400_000` (24 hours). - -## :emoji -* `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]` -* `pack_extensions`: A list of file extensions for emojis, when no emoji.txt for a pack is present. Example `[".png", ".gif"]` -* `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]` -* `default_manifest`: Location of the JSON-manifest. This manifest contains information about the emoji-packs you can download. Currently only one manifest can be added (no arrays). - -## Database options - -### RUM indexing for full text search -* `rum_enabled`: If RUM indexes should be used. Defaults to `false`. - -RUM indexes are an alternative indexing scheme that is not included in PostgreSQL by default. While they may eventually be mainlined, for now they have to be installed as a PostgreSQL extension from https://github.com/postgrespro/rum. - -Their advantage over the standard GIN indexes is that they allow efficient ordering of search results by timestamp, which makes search queries a lot faster on larger servers, by one or two orders of magnitude. They take up around 3 times as much space as GIN indexes. - -To enable them, both the `rum_enabled` flag has to be set and the following special migration has to be run: - -`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/` - -This will probably take a long time. - -## :rate_limit - -This is an advanced feature and disabled by default. - -A keyword list of rate limiters where a key is a limiter name and value is the limiter configuration. The basic configuration is a tuple where: - -* The first element: `scale` (Integer). The time scale in milliseconds. -* The second element: `limit` (Integer). How many requests to limit in the time scale provided. - -It is also possible to have different limits for unauthenticated and authenticated users: the keyword value must be a list of two tuples where the first one is a config for unauthenticated users and the second one is for authenticated. - -See [`Pleroma.Plugs.RateLimiter`](Pleroma.Plugs.RateLimiter.html) documentation for examples. - -Supported rate limiters: - -* `:search` for the search requests (account & status search etc.) -* `:app_account_creation` for registering user accounts from the same IP address -* `:relations_actions` for actions on relations with all users (follow, unfollow) -* `:relation_id_action` for actions on relation with a specific user (follow, unfollow) -* `:statuses_actions` for create / delete / fav / unfav / reblog / unreblog actions on any statuses -* `:status_id_action` for fav / unfav or reblog / unreblog actions on the same status by the same user - -## :web_cache_ttl - -The expiration time for the web responses cache. Values should be in milliseconds or `nil` to disable expiration. - -Available caches: - -* `:activity_pub` - activity pub routes (except question activities). Defaults to `nil` (no expiration). -* `:activity_pub_question` - activity pub routes (question activities). Defaults to `30_000` (30 seconds). +# Configuration + +This file describe the configuration, it is recommended to edit the relevant *.secret.exs file instead of the others founds in the ``config`` directory. +If you run Pleroma with ``MIX_ENV=prod`` the file is ``prod.secret.exs``, otherwise it is ``dev.secret.exs``. + +## Pleroma.Upload +* `uploader`: Select which `Pleroma.Uploaders` to use +* `filters`: List of `Pleroma.Upload.Filter` to use. +* `link_name`: When enabled Pleroma will add a `name` parameter to the url of the upload, for example `https://instance.tld/media/corndog.png?name=corndog.png`. This is needed to provide the correct filename in Content-Disposition headers when using filters like `Pleroma.Upload.Filter.Dedupe` +* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host. +* `proxy_remote`: If you're using a remote uploader, Pleroma will proxy media requests instead of redirecting to it. +* `proxy_opts`: Proxy options, see `Pleroma.ReverseProxy` documentation. + +Note: `strip_exif` has been replaced by `Pleroma.Upload.Filter.Mogrify`. + +## Pleroma.Uploaders.Local +* `uploads`: Which directory to store the user-uploads in, relative to pleroma’s working directory + +## Pleroma.Uploaders.S3 +* `bucket`: S3 bucket name +* `bucket_namespace`: S3 bucket namespace +* `public_endpoint`: S3 endpoint that the user finally accesses(ex. "https://s3.dualstack.ap-northeast-1.amazonaws.com") +* `truncated_namespace`: If you use S3 compatible service such as Digital Ocean Spaces or CDN, set folder name or "" etc. +For example, when using CDN to S3 virtual host format, set "". +At this time, write CNAME to CDN in public_endpoint. + +## Pleroma.Upload.Filter.Mogrify + +* `args`: List of actions for the `mogrify` command like `"strip"` or `["strip", "auto-orient", {"implode", "1"}]`. + +## Pleroma.Upload.Filter.Dedupe + +No specific configuration. + +## Pleroma.Upload.Filter.AnonymizeFilename + +This filter replaces the filename (not the path) of an upload. For complete obfuscation, add +`Pleroma.Upload.Filter.Dedupe` before AnonymizeFilename. + +* `text`: Text to replace filenames in links. If empty, `{random}.extension` will be used. You can get the original filename extension by using `{extension}`, for example `custom-file-name.{extension}`. + +## Pleroma.Emails.Mailer +* `adapter`: one of the mail adapters listed in [Swoosh readme](https://github.com/swoosh/swoosh#adapters), or `Swoosh.Adapters.Local` for in-memory mailbox. +* `api_key` / `password` and / or other adapter-specific settings, per the above documentation. +* `enabled`: Allows enable/disable send emails. Default: `false`. + +An example for Sendgrid adapter: + +```elixir +config :pleroma, Pleroma.Emails.Mailer, + adapter: Swoosh.Adapters.Sendgrid, + api_key: "YOUR_API_KEY" +``` + +An example for SMTP adapter: + +```elixir +config :pleroma, Pleroma.Emails.Mailer, + adapter: Swoosh.Adapters.SMTP, + relay: "smtp.gmail.com", + username: "YOUR_USERNAME@gmail.com", + password: "YOUR_SMTP_PASSWORD", + port: 465, + ssl: true, + tls: :always, + auth: :always +``` + +## :uri_schemes +* `valid_schemes`: List of the scheme part that is considered valid to be an URL + +## :instance +* `name`: The instance’s name +* `email`: Email used to reach an Administrator/Moderator of the instance +* `notify_email`: Email used for notifications. +* `description`: The instance’s description, can be seen in nodeinfo and ``/api/v1/instance`` +* `limit`: Posts character limit (CW/Subject included in the counter) +* `remote_limit`: Hard character limit beyond which remote posts will be dropped. +* `upload_limit`: File size limit of uploads (except for avatar, background, banner) +* `avatar_upload_limit`: File size limit of user’s profile avatars +* `background_upload_limit`: File size limit of user’s profile backgrounds +* `banner_upload_limit`: File size limit of user’s profile banners +* `poll_limits`: A map with poll limits for **local** polls + * `max_options`: Maximum number of options + * `max_option_chars`: Maximum number of characters per option + * `min_expiration`: Minimum expiration time (in seconds) + * `max_expiration`: Maximum expiration time (in seconds) +* `registrations_open`: Enable registrations for anyone, invitations can be enabled when false. +* `invites_enabled`: Enable user invitations for admins (depends on `registrations_open: false`). +* `account_activation_required`: Require users to confirm their emails before signing in. +* `federating`: Enable federation with other instances +* `federation_incoming_replies_max_depth`: Max. depth of reply-to activities fetching on incoming federation, to prevent out-of-memory situations while fetching very long threads. If set to `nil`, threads of any depth will be fetched. Lower this value if you experience out-of-memory crashes. +* `federation_reachability_timeout_days`: Timeout (in days) of each external federation target being unreachable prior to pausing federating to it. +* `allow_relay`: Enable Pleroma’s Relay, which makes it possible to follow a whole instance +* `rewrite_policy`: Message Rewrite Policy, either one or a list. Here are the ones available by default: + * `Pleroma.Web.ActivityPub.MRF.NoOpPolicy`: Doesn’t modify activities (default) + * `Pleroma.Web.ActivityPub.MRF.DropPolicy`: Drops all activities. It generally doesn’t makes sense to use in production + * `Pleroma.Web.ActivityPub.MRF.SimplePolicy`: Restrict the visibility of activities from certains instances (See ``:mrf_simple`` section) + * `Pleroma.Web.ActivityPub.MRF.TagPolicy`: Applies policies to individual users based on tags, which can be set using pleroma-fe/admin-fe/any other app that supports Pleroma Admin API. For example it allows marking posts from individual users nsfw (sensitive) + * `Pleroma.Web.ActivityPub.MRF.SubchainPolicy`: Selectively runs other MRF policies when messages match (see ``:mrf_subchain`` section) + * `Pleroma.Web.ActivityPub.MRF.RejectNonPublic`: Drops posts with non-public visibility settings (See ``:mrf_rejectnonpublic`` section) + * `Pleroma.Web.ActivityPub.MRF.EnsureRePrepended`: Rewrites posts to ensure that replies to posts with subjects do not have an identical subject and instead begin with re:. + * `Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy`: Rejects posts from likely spambots by rejecting posts from new users that contain links. + * `Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`: Crawls attachments using their MediaProxy URLs so that the MediaProxy cache is primed. + * `Pleroma.Web.ActivityPub.MRF.MentionPolicy`: Drops posts mentioning configurable users. (see `:mrf_mention` section) + * `Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`: Restricts activities to a configured set of vocabulary. (see `:mrf_vocabulary` section) +* `public`: Makes the client API in authentificated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. +* `quarantined_instances`: List of ActivityPub instances where private(DMs, followers-only) activities will not be send. +* `managed_config`: Whenether the config for pleroma-fe is configured in this config or in ``static/config.json`` +* `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML) +* `mrf_transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo). +* `mrf_transparency_exclusions`: Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value. +* `scope_copy`: Copy the scope (private/unlisted/public) in replies to posts by default. +* `subject_line_behavior`: Allows changing the default behaviour of subject lines in replies. Valid values: + * "email": Copy and preprend re:, as in email. + * "masto": Copy verbatim, as in Mastodon. + * "noop": Don't copy the subject. +* `always_show_subject_input`: When set to false, auto-hide the subject field when it's empty. +* `extended_nickname_format`: Set to `true` to use extended local nicknames format (allows underscores/dashes). This will break federation with + older software for theses nicknames. +* `max_pinned_statuses`: The maximum number of pinned statuses. `0` will disable the feature. +* `autofollowed_nicknames`: Set to nicknames of (local) users that every new user should automatically follow. +* `no_attachment_links`: Set to true to disable automatically adding attachment link text to statuses +* `welcome_message`: A message that will be send to a newly registered users as a direct message. +* `welcome_user_nickname`: The nickname of the local user that sends the welcome message. +* `max_report_comment_size`: The maximum size of the report comment (Default: `1000`) +* `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). Default: `false`. +* `healthcheck`: If set to true, system data will be shown on ``/api/pleroma/healthcheck``. +* `remote_post_retention_days`: The default amount of days to retain remote posts when pruning the database. +* `user_bio_length`: A user bio maximum length (default: `5000`) +* `user_name_length`: A user name maximum length (default: `100`) +* `skip_thread_containment`: Skip filter out broken threads. The default is `false`. +* `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. +* `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. +* `max_account_fields`: The maximum number of custom fields in the user profile (default: `10`) +* `max_remote_account_fields`: The maximum number of custom fields in the remote user profile (default: `20`) +* `account_field_name_length`: An account field name maximum length (default: `512`) +* `account_field_value_length`: An account field value maximum length (default: `512`) +* `external_user_synchronization`: Enabling following/followers counters synchronization for external users. + + + +## :logger +* `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog, and `Quack.Logger` to log to Slack + +An example to enable ONLY ExSyslogger (f/ex in ``prod.secret.exs``) with info and debug suppressed: +```elixir +config :logger, + backends: [{ExSyslogger, :ex_syslogger}] + +config :logger, :ex_syslogger, + level: :warn +``` + +Another example, keeping console output and adding the pid to syslog output: +```elixir +config :logger, + backends: [:console, {ExSyslogger, :ex_syslogger}] + +config :logger, :ex_syslogger, + level: :warn, + option: [:pid, :ndelay] +``` + +See: [logger’s documentation](https://hexdocs.pm/logger/Logger.html) and [ex_syslogger’s documentation](https://hexdocs.pm/ex_syslogger/) + +An example of logging info to local syslog, but warn to a Slack channel: +```elixir +config :logger, + backends: [ {ExSyslogger, :ex_syslogger}, Quack.Logger ], + level: :info + +config :logger, :ex_syslogger, + level: :info, + ident: "pleroma", + format: "$metadata[$level] $message" + +config :quack, + level: :warn, + meta: [:all], + webhook_url: "https://hooks.slack.com/services/YOUR-API-KEY-HERE" +``` + +See the [Quack Github](https://github.com/azohra/quack) for more details + +## :frontend_configurations + +This can be used to configure a keyword list that keeps the configuration data for any kind of frontend. By default, settings for `pleroma_fe` and `masto_fe` are configured. + +Frontends can access these settings at `/api/pleroma/frontend_configurations` + +To add your own configuration for PleromaFE, use it like this: + +```elixir +config :pleroma, :frontend_configurations, + pleroma_fe: %{ + theme: "pleroma-dark", + # ... see /priv/static/static/config.json for the available keys. +}, + masto_fe: %{ + showInstanceSpecificPanel: true + } +``` + +These settings **need to be complete**, they will override the defaults. + +NOTE: for versions < 1.0, you need to set [`:fe`](#fe) to false, as shown a few lines below. + +## :fe +__THIS IS DEPRECATED__ + +If you are using this method, please change it to the [`frontend_configurations`](#frontend_configurations) method. +Please **set this option to false** in your config like this: + +```elixir +config :pleroma, :fe, false +``` + +This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:instance`` is set to false. + +* `theme`: Which theme to use, they are defined in ``styles.json`` +* `logo`: URL of the logo, defaults to Pleroma’s logo +* `logo_mask`: Whether to use only the logo's shape as a mask (true) or as a regular image (false) +* `logo_margin`: What margin to use around the logo +* `background`: URL of the background, unless viewing a user profile with a background that is set +* `redirect_root_no_login`: relative URL which indicates where to redirect when a user isn’t logged in. +* `redirect_root_login`: relative URL which indicates where to redirect when a user is logged in. +* `show_instance_panel`: Whenether to show the instance’s specific panel. +* `scope_options_enabled`: Enable setting an notice visibility and subject/CW when posting +* `formatting_options_enabled`: Enable setting a formatting different than plain-text (ie. HTML, Markdown) when posting, relates to ``:instance, allowed_post_formats`` +* `collapse_message_with_subjects`: When a message has a subject(aka Content Warning), collapse it by default +* `hide_post_stats`: Hide notices statistics(repeats, favorites, …) +* `hide_user_stats`: Hide profile statistics(posts, posts per day, followers, followings, …) + +## :assets + +This section configures assets to be used with various frontends. Currently the only option +relates to mascots on the mastodon frontend + +* `mascots`: KeywordList of mascots, each element __MUST__ contain both a `url` and a + `mime_type` key. +* `default_mascot`: An element from `mascots` - This will be used as the default mascot + on MastoFE (default: `:pleroma_fox_tan`) + +## :mrf_simple +* `media_removal`: List of instances to remove medias from +* `media_nsfw`: List of instances to put medias as NSFW(sensitive) from +* `federated_timeline_removal`: List of instances to remove from Federated (aka The Whole Known Network) Timeline +* `reject`: List of instances to reject any activities from +* `accept`: List of instances to accept any activities from +* `report_removal`: List of instances to reject reports from +* `avatar_removal`: List of instances to strip avatars from +* `banner_removal`: List of instances to strip banners from + +## :mrf_subchain +This policy processes messages through an alternate pipeline when a given message matches certain criteria. +All criteria are configured as a map of regular expressions to lists of policy modules. + +* `match_actor`: Matches a series of regular expressions against the actor field. + +Example: + +``` +config :pleroma, :mrf_subchain, + match_actor: %{ + ~r/https:\/\/example.com/s => [Pleroma.Web.ActivityPub.MRF.DropPolicy] + } +``` + +## :mrf_rejectnonpublic +* `allow_followersonly`: whether to allow followers-only posts +* `allow_direct`: whether to allow direct messages + +## :mrf_hellthread +* `delist_threshold`: Number of mentioned users after which the message gets delisted (the message can still be seen, but it will not show up in public timelines and mentioned users won't get notifications about it). Set to 0 to disable. +* `reject_threshold`: Number of mentioned users after which the messaged gets rejected. Set to 0 to disable. + +## :mrf_keyword +* `reject`: A list of patterns which result in message being rejected, each pattern can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) +* `federated_timeline_removal`: A list of patterns which result in message being removed from federated timelines (a.k.a unlisted), each pattern can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) +* `replace`: A list of tuples containing `{pattern, replacement}`, `pattern` can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) + +## :mrf_mention +* `actors`: A list of actors, for which to drop any posts mentioning. + +## :mrf_vocabulary +* `accept`: A list of ActivityStreams terms to accept. If empty, all supported messages are accepted. +* `reject`: A list of ActivityStreams terms to reject. If empty, no messages are rejected. + +## :media_proxy +* `enabled`: Enables proxying of remote media to the instance’s proxy +* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts. +* `proxy_opts`: All options defined in `Pleroma.ReverseProxy` documentation, defaults to `[max_body_length: (25*1_048_576)]`. +* `whitelist`: List of domains to bypass the mediaproxy + +## :gopher +* `enabled`: Enables the gopher interface +* `ip`: IP address to bind to +* `port`: Port to bind to +* `dstport`: Port advertised in urls (optional, defaults to `port`) + +## Pleroma.Web.Endpoint +`Phoenix` endpoint configuration, all configuration options can be viewed [here](https://hexdocs.pm/phoenix/Phoenix.Endpoint.html#module-dynamic-configuration), only common options are listed here +* `http` - a list containing http protocol configuration, all configuration options can be viewed [here](https://hexdocs.pm/plug_cowboy/Plug.Cowboy.html#module-options), only common options are listed here. For deployment using docker, you need to set this to `[ip: {0,0,0,0}, port: 4000]` to make pleroma accessible from other containers (such as your nginx server). + - `ip` - a tuple consisting of 4 integers + - `port` +* `url` - a list containing the configuration for generating urls, accepts + - `host` - the host without the scheme and a post (e.g `example.com`, not `https://example.com:2020`) + - `scheme` - e.g `http`, `https` + - `port` + - `path` +* `extra_cookie_attrs` - a list of `Key=Value` strings to be added as non-standard cookie attributes. Defaults to `["SameSite=Lax"]`. See the [SameSite article](https://www.owasp.org/index.php/SameSite) on OWASP for more info. + + + +**Important note**: if you modify anything inside these lists, default `config.exs` values will be overwritten, which may result in breakage, to make sure this does not happen please copy the default value for the list from `config.exs` and modify/add only what you need + +Example: +```elixir +config :pleroma, Pleroma.Web.Endpoint, + url: [host: "example.com", port: 2020, scheme: "https"], + http: [ + # start copied from config.exs + dispatch: [ + {:_, + [ + {"/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, []}, + {"/websocket", Phoenix.Endpoint.CowboyWebSocket, + {Phoenix.Transports.WebSocket, + {Pleroma.Web.Endpoint, Pleroma.Web.UserSocket, websocket_config}}}, + {:_, Phoenix.Endpoint.Cowboy2Handler, {Pleroma.Web.Endpoint, []}} + ]} + # end copied from config.exs + ], + port: 8080, + ip: {127, 0, 0, 1} + ] +``` + +This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls starting with `https://example.com:2020` + +## :activitypub +* ``unfollow_blocked``: Whether blocks result in people getting unfollowed +* ``outgoing_blocks``: Whether to federate blocks to other instances +* ``deny_follow_blocked``: Whether to disallow following an account that has blocked the user in question +* ``sign_object_fetches``: Sign object fetches with HTTP signatures + +## :http_security +* ``enabled``: Whether the managed content security policy is enabled +* ``sts``: Whether to additionally send a `Strict-Transport-Security` header +* ``sts_max_age``: The maximum age for the `Strict-Transport-Security` header if sent +* ``ct_max_age``: The maximum age for the `Expect-CT` header if sent +* ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"` +* ``report_uri``: Adds the specified url to `report-uri` and `report-to` group in CSP header. + +## :mrf_user_allowlist + +The keys in this section are the domain names that the policy should apply to. +Each key should be assigned a list of users that should be allowed through by +their ActivityPub ID. + +An example: + +```elixir +config :pleroma, :mrf_user_allowlist, + "example.org": ["https://example.org/users/admin"] +``` + +## :web_push_encryption, :vapid_details + +Web Push Notifications configuration. You can use the mix task `mix web_push.gen.keypair` to generate it. + +* ``subject``: a mailto link for the administrative contact. It’s best if this email is not a personal email address, but rather a group email so that if a person leaves an organization, is unavailable for an extended period, or otherwise can’t respond, someone else on the list can. +* ``public_key``: VAPID public key +* ``private_key``: VAPID private key + +## Pleroma.Captcha +* `enabled`: Whether the captcha should be shown on registration +* `method`: The method/service to use for captcha +* `seconds_valid`: The time in seconds for which the captcha is valid + +### Pleroma.Captcha.Kocaptcha +Kocaptcha is a very simple captcha service with a single API endpoint, +the source code is here: https://github.com/koto-bank/kocaptcha. The default endpoint +`https://captcha.kotobank.ch` is hosted by the developer. + +* `endpoint`: the kocaptcha endpoint to use + +## :admin_token + +Allows to set a token that can be used to authenticate with the admin api without using an actual user by giving it as the 'admin_token' parameter. Example: + +```elixir +config :pleroma, :admin_token, "somerandomtoken" +``` + +You can then do + +```sh +curl "http://localhost:4000/api/pleroma/admin/invite_token?admin_token=somerandomtoken" +``` + +## Oban + +[Oban](https://github.com/sorentwo/oban) asynchronous job processor configuration. + +Configuration options described in [Oban readme](https://github.com/sorentwo/oban#usage): +* `repo` - app's Ecto repo (`Pleroma.Repo`) +* `verbose` - logs verbosity +* `prune` - non-retryable jobs [pruning settings](https://github.com/sorentwo/oban#pruning) (`:disabled` / `{:maxlen, value}` / `{:maxage, value}`) +* `queues` - job queues (see below) + +Pleroma has the following queues: + +* `activity_expiration` - Activity expiration +* `federator_outgoing` - Outgoing federation +* `federator_incoming` - Incoming federation +* `mailer` - Email sender, see [`Pleroma.Emails.Mailer`](#pleromaemailsmailer) +* `transmogrifier` - Transmogrifier +* `web_push` - Web push notifications +* `scheduled_activities` - Scheduled activities, see [`Pleroma.ScheduledActivity`](#pleromascheduledactivity) + +Example: + +```elixir +config :pleroma, Oban, + repo: Pleroma.Repo, + verbose: false, + prune: {:maxlen, 1500}, + queues: [ + federator_incoming: 50, + federator_outgoing: 50 + ] +``` + +This config contains two queues: `federator_incoming` and `federator_outgoing`. Both have the number of max concurrent jobs set to `50`. + +### Migrating `pleroma_job_queue` settings + +`config :pleroma_job_queue, :queues` is replaced by `config :pleroma, Oban, :queues` and uses the same format (keys are queues' names, values are max concurrent jobs numbers). + +### Note on running with PostgreSQL in silent mode + +If you are running PostgreSQL in [`silent_mode`](https://postgresqlco.nf/en/doc/param/silent_mode?version=9.1), it's advised to set [`log_destination`](https://postgresqlco.nf/en/doc/param/log_destination?version=9.1) to `syslog`, +otherwise `postmaster.log` file may grow because of "you don't own a lock of type ShareLock" warnings (see https://github.com/sorentwo/oban/issues/52). + +## :workers + +Includes custom worker options not interpretable directly by `Oban`. + +* `retries` — keyword lists where keys are `Oban` queues (see above) and values are numbers of max attempts for failed jobs. + +Example: + +```elixir +config :pleroma, :workers, + retries: [ + federator_incoming: 5, + federator_outgoing: 5 + ] +``` + +### Migrating `Pleroma.Web.Federator.RetryQueue` settings + +* `max_retries` is replaced with `config :pleroma, :workers, retries: [federator_outgoing: 5]` +* `enabled: false` corresponds to `config :pleroma, :workers, retries: [federator_outgoing: 1]` +* deprecated options: `max_jobs`, `initial_timeout` + +## Pleroma.Web.Metadata +* `providers`: a list of metadata providers to enable. Providers available: + * Pleroma.Web.Metadata.Providers.OpenGraph + * Pleroma.Web.Metadata.Providers.TwitterCard + * Pleroma.Web.Metadata.Providers.RelMe - add links from user bio with rel=me into the `
` as `` +* `unfurl_nsfw`: If set to `true` nsfw attachments will be shown in previews + +## :rich_media +* `enabled`: if enabled the instance will parse metadata from attached links to generate link previews +* `ignore_hosts`: list of hosts which will be ignored by the metadata parser. For example `["accounts.google.com", "xss.website"]`, defaults to `[]`. +* `ignore_tld`: list TLDs (top-level domains) which will ignore for parse metadata. default is ["local", "localdomain", "lan"] +* `parsers`: list of Rich Media parsers + +## :fetch_initial_posts +* `enabled`: if enabled, when a new user is federated with, fetch some of their latest posts +* `pages`: the amount of pages to fetch + +## :hackney_pools + +Advanced. Tweaks Hackney (http client) connections pools. + +There's three pools used: + +* `:federation` for the federation jobs. + You may want this pool max_connections to be at least equal to the number of federator jobs + retry queue jobs. +* `:media` for rich media, media proxy +* `:upload` for uploaded media (if using a remote uploader and `proxy_remote: true`) + +For each pool, the options are: + +* `max_connections` - how much connections a pool can hold +* `timeout` - retention duration for connections + +## :auto_linker + +Configuration for the `auto_linker` library: + +* `class: "auto-linker"` - specify the class to be added to the generated link. false to clear +* `rel: "noopener noreferrer"` - override the rel attribute. false to clear +* `new_window: true` - set to false to remove `target='_blank'` attribute +* `scheme: false` - Set to true to link urls with schema `http://google.com` +* `truncate: false` - Set to a number to truncate urls longer then the number. Truncated urls will end in `..` +* `strip_prefix: true` - Strip the scheme prefix +* `extra: false` - link urls with rarely used schemes (magnet, ipfs, irc, etc.) + +Example: + +```elixir +config :auto_linker, + opts: [ + scheme: true, + extra: true, + class: false, + strip_prefix: false, + new_window: false, + rel: false + ] +``` + +## Pleroma.Scheduler + +Configuration for [Quantum](https://github.com/quantum-elixir/quantum-core) jobs scheduler. + +See [Quantum readme](https://github.com/quantum-elixir/quantum-core#usage) for the list of supported options. + +Example: + +```elixir +config :pleroma, Pleroma.Scheduler, + global: true, + overlap: true, + timezone: :utc, + jobs: [{"0 */6 * * * *", {Pleroma.Web.Websub, :refresh_subscriptions, []}}] +``` + +The above example defines a single job which invokes `Pleroma.Web.Websub.refresh_subscriptions()` every 6 hours ("0 */6 * * * *", [crontab format](https://en.wikipedia.org/wiki/Cron)). + +## Pleroma.ScheduledActivity + +* `daily_user_limit`: the number of scheduled activities a user is allowed to create in a single day (Default: `25`) +* `total_user_limit`: the number of scheduled activities a user is allowed to create in total (Default: `300`) +* `enabled`: whether scheduled activities are sent to the job queue to be executed + +## Pleroma.ActivityExpiration + +# `enabled`: whether expired activities will be sent to the job queue to be deleted + +## Pleroma.Web.Auth.Authenticator + +* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator +* `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication + +## :ldap + +Use LDAP for user authentication. When a user logs in to the Pleroma +instance, the name and password will be verified by trying to authenticate +(bind) to an LDAP server. If a user exists in the LDAP directory but there +is no account with the same name yet on the Pleroma instance then a new +Pleroma account will be created with the same name as the LDAP user name. + +* `enabled`: enables LDAP authentication +* `host`: LDAP server hostname +* `port`: LDAP port, e.g. 389 or 636 +* `ssl`: true to use SSL, usually implies the port 636 +* `sslopts`: additional SSL options +* `tls`: true to start TLS, usually implies the port 389 +* `tlsopts`: additional TLS options +* `base`: LDAP base, e.g. "dc=example,dc=com" +* `uid`: LDAP attribute name to authenticate the user, e.g. when "cn", the filter will be "cn=username,base" + +## BBS / SSH access + +To enable simple command line interface accessible over ssh, add a setting like this to your configuration file: + +```exs +app_dir = File.cwd! +priv_dir = Path.join([app_dir, "priv/ssh_keys"]) + +config :esshd, + enabled: true, + priv_dir: priv_dir, + handler: "Pleroma.BBS.Handler", + port: 10_022, + password_authenticator: "Pleroma.BBS.Authenticator" +``` + +Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -m PEM -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT` + +## :auth + +* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator +* `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication + +Authentication / authorization settings. + +* `auth_template`: authentication form template. By default it's `show.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/show.html.eex`. +* `oauth_consumer_template`: OAuth consumer mode authentication form template. By default it's `consumer.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex`. +* `oauth_consumer_strategies`: the list of enabled OAuth consumer strategies; by default it's set by `OAUTH_CONSUMER_STRATEGIES` environment variable. Each entry in this space-delimited string should be of format `` or `:` (e.g. `twitter` or `keycloak:ueberauth_keycloak_strategy` in case dependency is named differently than `ueberauth_`). + +## :email_notifications + +Email notifications settings. + + - digest - emails of "what you've missed" for users who have been + inactive for a while. + - active: globally enable or disable digest emails + - schedule: When to send digest email, in [crontab format](https://en.wikipedia.org/wiki/Cron). + "0 0 * * 0" is the default, meaning "once a week at midnight on Sunday morning" + - interval: Minimum interval between digest emails to one user + - inactivity_threshold: Minimum user inactivity threshold + +## Pleroma.Emails.UserEmail + +- `:logo` - a path to a custom logo. Set it to `nil` to use the default Pleroma logo. +- `:styling` - a map with color settings for email templates. + +## OAuth consumer mode + +OAuth consumer mode allows sign in / sign up via external OAuth providers (e.g. Twitter, Facebook, Google, Microsoft, etc.). +Implementation is based on Ueberauth; see the list of [available strategies](https://github.com/ueberauth/ueberauth/wiki/List-of-Strategies). + +Note: each strategy is shipped as a separate dependency; in order to get the strategies, run `OAUTH_CONSUMER_STRATEGIES="..." mix deps.get`, +e.g. `OAUTH_CONSUMER_STRATEGIES="twitter facebook google microsoft" mix deps.get`. +The server should also be started with `OAUTH_CONSUMER_STRATEGIES="..." mix phx.server` in case you enable any strategies. + +Note: each strategy requires separate setup (on external provider side and Pleroma side). Below are the guidelines on setting up most popular strategies. + +Note: make sure that `"SameSite=Lax"` is set in `extra_cookie_attrs` when you have this feature enabled. OAuth consumer mode will not work with `"SameSite=Strict"` + +* For Twitter, [register an app](https://developer.twitter.com/en/apps), configure callback URL to https:///oauth/twitter/callback + +* For Facebook, [register an app](https://developers.facebook.com/apps), configure callback URL to https:///oauth/facebook/callback, enable Facebook Login service at https://developers.facebook.com/apps//fb-login/settings/ + +* For Google, [register an app](https://console.developers.google.com), configure callback URL to https:///oauth/google/callback + +* For Microsoft, [register an app](https://portal.azure.com), configure callback URL to https:///oauth/microsoft/callback + +Once the app is configured on external OAuth provider side, add app's credentials and strategy-specific settings (if any — e.g. see Microsoft below) to `config/prod.secret.exs`, +per strategy's documentation (e.g. [ueberauth_twitter](https://github.com/ueberauth/ueberauth_twitter)). Example config basing on environment variables: + +```elixir +# Twitter +config :ueberauth, Ueberauth.Strategy.Twitter.OAuth, + consumer_key: System.get_env("TWITTER_CONSUMER_KEY"), + consumer_secret: System.get_env("TWITTER_CONSUMER_SECRET") + +# Facebook +config :ueberauth, Ueberauth.Strategy.Facebook.OAuth, + client_id: System.get_env("FACEBOOK_APP_ID"), + client_secret: System.get_env("FACEBOOK_APP_SECRET"), + redirect_uri: System.get_env("FACEBOOK_REDIRECT_URI") + +# Google +config :ueberauth, Ueberauth.Strategy.Google.OAuth, + client_id: System.get_env("GOOGLE_CLIENT_ID"), + client_secret: System.get_env("GOOGLE_CLIENT_SECRET"), + redirect_uri: System.get_env("GOOGLE_REDIRECT_URI") + +# Microsoft +config :ueberauth, Ueberauth.Strategy.Microsoft.OAuth, + client_id: System.get_env("MICROSOFT_CLIENT_ID"), + client_secret: System.get_env("MICROSOFT_CLIENT_SECRET") + +config :ueberauth, Ueberauth, + providers: [ + microsoft: {Ueberauth.Strategy.Microsoft, [callback_params: []]} + ] + +# Keycloak +# Note: make sure to add `keycloak:ueberauth_keycloak_strategy` entry to `OAUTH_CONSUMER_STRATEGIES` environment variable +keycloak_url = "https://publicly-reachable-keycloak-instance.org:8080" + +config :ueberauth, Ueberauth.Strategy.Keycloak.OAuth, + client_id: System.get_env("KEYCLOAK_CLIENT_ID"), + client_secret: System.get_env("KEYCLOAK_CLIENT_SECRET"), + site: keycloak_url, + authorize_url: "#{keycloak_url}/auth/realms/master/protocol/openid-connect/auth", + token_url: "#{keycloak_url}/auth/realms/master/protocol/openid-connect/token", + userinfo_url: "#{keycloak_url}/auth/realms/master/protocol/openid-connect/userinfo", + token_method: :post + +config :ueberauth, Ueberauth, + providers: [ + keycloak: {Ueberauth.Strategy.Keycloak, [uid_field: :email]} + ] +``` + +## OAuth 2.0 provider - :oauth2 + +Configure OAuth 2 provider capabilities: + +* `token_expires_in` - The lifetime in seconds of the access token. +* `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token. +* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`. +* `clean_expired_tokens_interval` - Interval to run the job to clean expired tokens. Defaults to `86_400_000` (24 hours). + +## :emoji +* `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]` +* `pack_extensions`: A list of file extensions for emojis, when no emoji.txt for a pack is present. Example `[".png", ".gif"]` +* `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]` +* `default_manifest`: Location of the JSON-manifest. This manifest contains information about the emoji-packs you can download. Currently only one manifest can be added (no arrays). + +## Database options + +### RUM indexing for full text search +* `rum_enabled`: If RUM indexes should be used. Defaults to `false`. + +RUM indexes are an alternative indexing scheme that is not included in PostgreSQL by default. While they may eventually be mainlined, for now they have to be installed as a PostgreSQL extension from https://github.com/postgrespro/rum. + +Their advantage over the standard GIN indexes is that they allow efficient ordering of search results by timestamp, which makes search queries a lot faster on larger servers, by one or two orders of magnitude. They take up around 3 times as much space as GIN indexes. + +To enable them, both the `rum_enabled` flag has to be set and the following special migration has to be run: + +`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/` + +This will probably take a long time. + +## :rate_limit + +This is an advanced feature and disabled by default. + +A keyword list of rate limiters where a key is a limiter name and value is the limiter configuration. The basic configuration is a tuple where: + +* The first element: `scale` (Integer). The time scale in milliseconds. +* The second element: `limit` (Integer). How many requests to limit in the time scale provided. + +It is also possible to have different limits for unauthenticated and authenticated users: the keyword value must be a list of two tuples where the first one is a config for unauthenticated users and the second one is for authenticated. + +See [`Pleroma.Plugs.RateLimiter`](Pleroma.Plugs.RateLimiter.html) documentation for examples. + +Supported rate limiters: + +* `:search` for the search requests (account & status search etc.) +* `:app_account_creation` for registering user accounts from the same IP address +* `:relations_actions` for actions on relations with all users (follow, unfollow) +* `:relation_id_action` for actions on relation with a specific user (follow, unfollow) +* `:statuses_actions` for create / delete / fav / unfav / reblog / unreblog actions on any statuses +* `:status_id_action` for fav / unfav or reblog / unreblog actions on the same status by the same user + +## :web_cache_ttl + +The expiration time for the web responses cache. Values should be in milliseconds or `nil` to disable expiration. + +Available caches: + +* `:activity_pub` - activity pub routes (except question activities). Defaults to `nil` (no expiration). +* `:activity_pub_question` - activity pub routes (question activities). Defaults to `30_000` (30 seconds). From b4cf74c1067b866574a63fbd25ccb12cc1fed619 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Sun, 15 Sep 2019 14:53:58 +0300 Subject: [PATCH 088/138] added prepare html for RichMedia.Parser --- lib/pleroma/web/rich_media/parser.ex | 6 +++++- mix.exs | 2 +- mix.lock | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index f5f9e358..c06b0a0f 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -81,6 +81,7 @@ defmodule Pleroma.Web.RichMedia.Parser do {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options) html + |> parse_html |> maybe_parse() |> Map.put(:url, url) |> clean_parsed_data() @@ -91,6 +92,8 @@ defmodule Pleroma.Web.RichMedia.Parser do end end + defp parse_html(html), do: Floki.parse(html) + defp maybe_parse(html) do Enum.reduce_while(parsers(), %{}, fn parser, acc -> case parser.parse(html, acc) do @@ -100,7 +103,8 @@ defmodule Pleroma.Web.RichMedia.Parser do end) end - defp check_parsed_data(%{title: title} = data) when is_binary(title) and byte_size(title) > 0 do + defp check_parsed_data(%{title: title} = data) + when is_binary(title) and byte_size(title) > 0 do {:ok, data} end diff --git a/mix.exs b/mix.exs index dfa53035..6f952fa1 100644 --- a/mix.exs +++ b/mix.exs @@ -131,7 +131,7 @@ defmodule Pleroma.Mixfile do {:phoenix_swoosh, "~> 0.2"}, {:gen_smtp, "~> 0.13"}, {:websocket_client, git: "https://github.com/jeremyong/websocket_client.git", only: :test}, - {:floki, "~> 0.20.0"}, + {:floki, "~> 0.23.0"}, {:ex_syslogger, github: "slashmili/ex_syslogger", tag: "1.4.0"}, {:timex, "~> 3.5"}, {:ueberauth, "~> 0.4"}, diff --git a/mix.lock b/mix.lock index 5762dae4..2bce00de 100644 --- a/mix.lock +++ b/mix.lock @@ -34,7 +34,7 @@ "ex_rated": {:hex, :ex_rated, "1.3.3", "30ecbdabe91f7eaa9d37fa4e81c85ba420f371babeb9d1910adbcd79ec798d27", [:mix], [{:ex2ms, "~> 1.5", [hex: :ex2ms, repo: "hexpm", optional: false]}], "hexpm"}, "ex_syslogger": {:git, "https://github.com/slashmili/ex_syslogger.git", "f3963399047af17e038897c69e20d552e6899e1d", [tag: "1.4.0"]}, "excoveralls": {:hex, :excoveralls, "0.11.1", "dd677fbdd49114fdbdbf445540ec735808250d56b011077798316505064edb2c", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, - "floki": {:hex, :floki, "0.20.4", "be42ac911fece24b4c72f3b5846774b6e61b83fe685c2fc9d62093277fb3bc86", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}, {:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, + "floki": {:hex, :floki, "0.23.0", "956ab6dba828c96e732454809fb0bd8d43ce0979b75f34de6322e73d4c917829", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm"}, "gen_smtp": {:hex, :gen_smtp, "0.14.0", "39846a03522456077c6429b4badfd1d55e5e7d0fdfb65e935b7c5e38549d9202", [:rebar3], [], "hexpm"}, "gettext": {:hex, :gettext, "0.17.0", "abe21542c831887a2b16f4c94556db9c421ab301aee417b7c4fbde7fbdbe01ec", [:mix], [], "hexpm"}, "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, From 43f17c2e67cfb85ae469eee39b526a5baf7c7408 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 12 Sep 2019 19:04:55 +0700 Subject: [PATCH 089/138] Restore tests for `change_password` and `delete_account` --- test/web/twitter_api/util_controller_test.exs | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index 0a2a48fb..56e31818 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -775,4 +775,109 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do assert json_response(conn, 200) == %{"status" => "success"} end end + + describe "POST /api/pleroma/change_password" do + setup [:valid_user] + + test "without credentials", %{conn: conn} do + conn = post(conn, "/api/pleroma/change_password") + assert json_response(conn, 403) == %{"error" => "Invalid credentials."} + end + + test "with credentials and invalid password", %{conn: conn, user: current_user} do + conn = + conn + |> with_credentials(current_user.nickname, "test") + |> post("/api/pleroma/change_password", %{ + "password" => "hi", + "new_password" => "newpass", + "new_password_confirmation" => "newpass" + }) + + assert json_response(conn, 200) == %{"error" => "Invalid password."} + end + + test "with credentials, valid password and new password and confirmation not matching", %{ + conn: conn, + user: current_user + } do + conn = + conn + |> with_credentials(current_user.nickname, "test") + |> post("/api/pleroma/change_password", %{ + "password" => "test", + "new_password" => "newpass", + "new_password_confirmation" => "notnewpass" + }) + + assert json_response(conn, 200) == %{ + "error" => "New password does not match confirmation." + } + end + + test "with credentials, valid password and invalid new password", %{ + conn: conn, + user: current_user + } do + conn = + conn + |> with_credentials(current_user.nickname, "test") + |> post("/api/pleroma/change_password", %{ + "password" => "test", + "new_password" => "", + "new_password_confirmation" => "" + }) + + assert json_response(conn, 200) == %{ + "error" => "New password can't be blank." + } + end + + test "with credentials, valid password and matching new password and confirmation", %{ + conn: conn, + user: current_user + } do + conn = + conn + |> with_credentials(current_user.nickname, "test") + |> post("/api/pleroma/change_password", %{ + "password" => "test", + "new_password" => "newpass", + "new_password_confirmation" => "newpass" + }) + + assert json_response(conn, 200) == %{"status" => "success"} + fetched_user = User.get_cached_by_id(current_user.id) + assert Comeonin.Pbkdf2.checkpw("newpass", fetched_user.password_hash) == true + end + end + + describe "POST /api/pleroma/delete_account" do + setup [:valid_user] + + test "without credentials", %{conn: conn} do + conn = post(conn, "/api/pleroma/delete_account") + assert json_response(conn, 403) == %{"error" => "Invalid credentials."} + end + + test "with credentials and invalid password", %{conn: conn, user: current_user} do + conn = + conn + |> with_credentials(current_user.nickname, "test") + |> post("/api/pleroma/delete_account", %{"password" => "hi"}) + + assert json_response(conn, 200) == %{"error" => "Invalid password."} + end + + test "with credentials and valid password", %{conn: conn, user: current_user} do + conn = + conn + |> with_credentials(current_user.nickname, "test") + |> post("/api/pleroma/delete_account", %{"password" => "test"}) + + assert json_response(conn, 200) == %{"status" => "success"} + # Wait a second for the started task to end + :timer.sleep(1000) + end + end end From ca88e37a8f3b0b52771f94df676e26471fb44019 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 16 Sep 2019 12:55:05 +0700 Subject: [PATCH 090/138] Fix a race condition in tests --- test/web/mastodon_api/mastodon_api_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 9c5322cc..fb04748b 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -752,7 +752,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do query_string = "ids[]=#{id1}&ids[]=#{id2}" conn = get(conn, "/api/v1/statuses/?#{query_string}") - assert [%{"id" => ^id1}, %{"id" => ^id2}] = json_response(conn, :ok) + assert [%{"id" => ^id1}, %{"id" => ^id2}] = Enum.sort_by(json_response(conn, :ok), & &1["id"]) end describe "deleting a status" do From aab264db82054df470075c65ca25c42bbcc5d7a8 Mon Sep 17 00:00:00 2001 From: Steven Fuchs Date: Mon, 16 Sep 2019 07:44:03 +0000 Subject: [PATCH 091/138] Streamer refactoring --- .gitignore | 4 + config/config.exs | 4 + lib/pleroma/activity/ir/topics.ex | 63 ++++ lib/pleroma/application.ex | 2 +- lib/pleroma/notification.ex | 6 +- lib/pleroma/web/activity_pub/activity_pub.ex | 48 +-- .../web/mastodon_api/websocket_handler.ex | 7 +- lib/pleroma/web/streamer.ex | 318 ------------------ lib/pleroma/web/streamer/ping.ex | 33 ++ lib/pleroma/web/streamer/state.ex | 68 ++++ lib/pleroma/web/streamer/streamer.ex | 55 +++ lib/pleroma/web/streamer/streamer_socket.ex | 31 ++ lib/pleroma/web/streamer/supervisor.ex | 33 ++ lib/pleroma/web/streamer/worker.ex | 220 ++++++++++++ lib/pleroma/web/views/streamer_view.ex | 66 ++++ mix.exs | 1 + mix.lock | 1 + test/activity/ir/topics_test.exs | 141 ++++++++ test/integration/mastodon_websocket_test.exs | 16 +- test/notification_test.exs | 11 +- test/support/conn_case.ex | 4 + test/support/data_case.ex | 4 + test/web/activity_pub/activity_pub_test.exs | 4 +- test/web/streamer/ping_test.exs | 36 ++ test/web/streamer/state_test.exs | 54 +++ test/web/{ => streamer}/streamer_test.exs | 105 +++--- 26 files changed, 888 insertions(+), 447 deletions(-) create mode 100644 lib/pleroma/activity/ir/topics.ex delete mode 100644 lib/pleroma/web/streamer.ex create mode 100644 lib/pleroma/web/streamer/ping.ex create mode 100644 lib/pleroma/web/streamer/state.ex create mode 100644 lib/pleroma/web/streamer/streamer.ex create mode 100644 lib/pleroma/web/streamer/streamer_socket.ex create mode 100644 lib/pleroma/web/streamer/supervisor.ex create mode 100644 lib/pleroma/web/streamer/worker.ex create mode 100644 lib/pleroma/web/views/streamer_view.ex create mode 100644 test/activity/ir/topics_test.exs create mode 100644 test/web/streamer/ping_test.exs create mode 100644 test/web/streamer/state_test.exs rename test/web/{ => streamer}/streamer_test.exs (86%) diff --git a/.gitignore b/.gitignore index 4e71a7df..3b0c7d36 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,7 @@ docs/generated_config.md # Code test coverage /cover /Elixir.*.coverdata + +.idea +pleroma.iml + diff --git a/config/config.exs b/config/config.exs index ab6e00c9..b1b98af9 100644 --- a/config/config.exs +++ b/config/config.exs @@ -331,6 +331,10 @@ config :pleroma, :activitypub, follow_handshake_timeout: 500, sign_object_fetches: true +config :pleroma, :streamer, + workers: 3, + overflow_workers: 2 + config :pleroma, :user, deny_follow_blocked: true config :pleroma, :mrf_normalize_markup, scrub_policy: Pleroma.HTML.Scrubber.Default diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex new file mode 100644 index 00000000..010897ab --- /dev/null +++ b/lib/pleroma/activity/ir/topics.ex @@ -0,0 +1,63 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Activity.Ir.Topics do + alias Pleroma.Object + alias Pleroma.Web.ActivityPub.Visibility + + def get_activity_topics(activity) do + activity + |> Object.normalize() + |> generate_topics(activity) + |> List.flatten() + end + + defp generate_topics(%{data: %{"type" => "Answer"}}, _) do + [] + end + + defp generate_topics(object, activity) do + ["user", "list"] ++ visibility_tags(object, activity) + end + + defp visibility_tags(object, activity) do + case Visibility.get_visibility(activity) do + "public" -> + if activity.local do + ["public", "public:local"] + else + ["public"] + end + |> item_creation_tags(object, activity) + + "direct" -> + ["direct"] + + _ -> + [] + end + end + + defp item_creation_tags(tags, %{data: %{"type" => "Create"}} = object, activity) do + tags ++ hashtags_to_topics(object) ++ attachment_topics(object, activity) + end + + defp item_creation_tags(tags, _, _) do + tags + end + + defp hashtags_to_topics(%{data: %{"tag" => tags}}) do + tags + |> Enum.filter(&is_bitstring(&1)) + |> Enum.map(fn tag -> "hashtag:" <> tag end) + end + + defp hashtags_to_topics(_), do: [] + + defp attachment_topics(%{data: %{"attachment" => []}}, _act), do: [] + + defp attachment_topics(_object, %{local: true}), do: ["public:media", "public:local:media"] + + defp attachment_topics(_object, _act), do: ["public:media"] +end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 49094704..3b37ce63 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -141,7 +141,7 @@ defmodule Pleroma.Application do defp streamer_child(:test), do: [] defp streamer_child(_) do - [Pleroma.Web.Streamer] + [Pleroma.Web.Streamer.supervisor()] end defp oauth_cleanup_child(true), diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index b7c880c5..8012389a 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -210,8 +210,10 @@ defmodule Pleroma.Notification do unless skip?(activity, user) do notification = %Notification{user_id: user.id, activity: activity} {:ok, notification} = Repo.insert(notification) - Streamer.stream("user", notification) - Streamer.stream("user:notification", notification) + + ["user", "user:notification"] + |> Streamer.stream(notification) + Push.send(notification) notification end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 41f6a0f1..bc5ae7fb 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Activity + alias Pleroma.Activity.Ir.Topics alias Pleroma.Config alias Pleroma.Conversation alias Pleroma.Notification @@ -16,6 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.User alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.Streamer alias Pleroma.Web.WebFinger alias Pleroma.Workers.BackgroundWorker @@ -187,9 +189,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do participations |> Repo.preload(:user) - Enum.each(participations, fn participation -> - Pleroma.Web.Streamer.stream("participation", participation) - end) + Streamer.stream("participation", participations) end def stream_out_participations(%Object{data: %{"context" => context}}, user) do @@ -208,41 +208,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def stream_out_participations(_, _), do: :noop - def stream_out(activity) do - if activity.data["type"] in ["Create", "Announce", "Delete"] do - object = Object.normalize(activity) - # Do not stream out poll replies - unless object.data["type"] == "Answer" do - Pleroma.Web.Streamer.stream("user", activity) - Pleroma.Web.Streamer.stream("list", activity) + def stream_out(%Activity{data: %{"type" => data_type}} = activity) + when data_type in ["Create", "Announce", "Delete"] do + activity + |> Topics.get_activity_topics() + |> Streamer.stream(activity) + end - if get_visibility(activity) == "public" do - Pleroma.Web.Streamer.stream("public", activity) - - if activity.local do - Pleroma.Web.Streamer.stream("public:local", activity) - end - - if activity.data["type"] in ["Create"] do - object.data - |> Map.get("tag", []) - |> Enum.filter(fn tag -> is_bitstring(tag) end) - |> Enum.each(fn tag -> Pleroma.Web.Streamer.stream("hashtag:" <> tag, activity) end) - - if object.data["attachment"] != [] do - Pleroma.Web.Streamer.stream("public:media", activity) - - if activity.local do - Pleroma.Web.Streamer.stream("public:local:media", activity) - end - end - end - else - if get_visibility(activity) == "direct", - do: Pleroma.Web.Streamer.stream("direct", activity) - end - end - end + def stream_out(_activity) do + :noop end def create(%{to: to, actor: actor, context: context, object: object} = params, fake \\ false) do diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index dbd3542e..3c26eb40 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.OAuth.Token + alias Pleroma.Web.Streamer @behaviour :cowboy_websocket @@ -24,7 +25,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do ] @anonymous_streams ["public", "public:local", "hashtag"] - # Handled by periodic keepalive in Pleroma.Web.Streamer. + # Handled by periodic keepalive in Pleroma.Web.Streamer.Ping. @timeout :infinity def init(%{qs: qs} = req, state) do @@ -65,7 +66,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do }, topic #{state.topic}" ) - Pleroma.Web.Streamer.add_socket(state.topic, streamer_socket(state)) + Streamer.add_socket(state.topic, streamer_socket(state)) {:ok, state} end @@ -80,7 +81,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do }, topic #{state.topic || "?"}: #{inspect(reason)}" ) - Pleroma.Web.Streamer.remove_socket(state.topic, streamer_socket(state)) + Streamer.remove_socket(state.topic, streamer_socket(state)) :ok end diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex deleted file mode 100644 index 587c43f4..00000000 --- a/lib/pleroma/web/streamer.ex +++ /dev/null @@ -1,318 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.Streamer do - use GenServer - require Logger - alias Pleroma.Activity - alias Pleroma.Config - alias Pleroma.Conversation.Participation - alias Pleroma.Notification - alias Pleroma.Object - alias Pleroma.User - alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.ActivityPub.Visibility - alias Pleroma.Web.CommonAPI - alias Pleroma.Web.MastodonAPI.NotificationView - - @keepalive_interval :timer.seconds(30) - - def start_link(_) do - GenServer.start_link(__MODULE__, %{}, name: __MODULE__) - end - - def add_socket(topic, socket) do - GenServer.cast(__MODULE__, %{action: :add, socket: socket, topic: topic}) - end - - def remove_socket(topic, socket) do - GenServer.cast(__MODULE__, %{action: :remove, socket: socket, topic: topic}) - end - - def stream(topic, item) do - GenServer.cast(__MODULE__, %{action: :stream, topic: topic, item: item}) - end - - def init(args) do - Process.send_after(self(), %{action: :ping}, @keepalive_interval) - - {:ok, args} - end - - def handle_info(%{action: :ping}, topics) do - topics - |> Map.values() - |> List.flatten() - |> Enum.each(fn socket -> - Logger.debug("Sending keepalive ping") - send(socket.transport_pid, {:text, ""}) - end) - - Process.send_after(self(), %{action: :ping}, @keepalive_interval) - - {:noreply, topics} - end - - def handle_cast(%{action: :stream, topic: "direct", item: item}, topics) do - recipient_topics = - User.get_recipients_from_activity(item) - |> Enum.map(fn %{id: id} -> "direct:#{id}" end) - - Enum.each(recipient_topics || [], fn user_topic -> - Logger.debug("Trying to push direct message to #{user_topic}\n\n") - push_to_socket(topics, user_topic, item) - end) - - {:noreply, topics} - end - - def handle_cast(%{action: :stream, topic: "participation", item: participation}, topics) do - user_topic = "direct:#{participation.user_id}" - Logger.debug("Trying to push a conversation participation to #{user_topic}\n\n") - - push_to_socket(topics, user_topic, participation) - - {:noreply, topics} - end - - def handle_cast(%{action: :stream, topic: "list", item: item}, topics) do - # filter the recipient list if the activity is not public, see #270. - recipient_lists = - case Visibility.is_public?(item) do - true -> - Pleroma.List.get_lists_from_activity(item) - - _ -> - Pleroma.List.get_lists_from_activity(item) - |> Enum.filter(fn list -> - owner = User.get_cached_by_id(list.user_id) - - Visibility.visible_for_user?(item, owner) - end) - end - - recipient_topics = - recipient_lists - |> Enum.map(fn %{id: id} -> "list:#{id}" end) - - Enum.each(recipient_topics || [], fn list_topic -> - Logger.debug("Trying to push message to #{list_topic}\n\n") - push_to_socket(topics, list_topic, item) - end) - - {:noreply, topics} - end - - def handle_cast( - %{action: :stream, topic: topic, item: %Notification{} = item}, - topics - ) - when topic in ["user", "user:notification"] do - topics - |> Map.get("#{topic}:#{item.user_id}", []) - |> Enum.each(fn socket -> - with %User{} = user <- User.get_cached_by_ap_id(socket.assigns[:user].ap_id), - true <- should_send?(user, item) do - send( - socket.transport_pid, - {:text, represent_notification(socket.assigns[:user], item)} - ) - end - end) - - {:noreply, topics} - end - - def handle_cast(%{action: :stream, topic: "user", item: item}, topics) do - Logger.debug("Trying to push to users") - - recipient_topics = - User.get_recipients_from_activity(item) - |> Enum.map(fn %{id: id} -> "user:#{id}" end) - - Enum.each(recipient_topics, fn topic -> - push_to_socket(topics, topic, item) - end) - - {:noreply, topics} - end - - def handle_cast(%{action: :stream, topic: topic, item: item}, topics) do - Logger.debug("Trying to push to #{topic}") - Logger.debug("Pushing item to #{topic}") - push_to_socket(topics, topic, item) - {:noreply, topics} - end - - def handle_cast(%{action: :add, topic: topic, socket: socket}, sockets) do - topic = internal_topic(topic, socket) - sockets_for_topic = sockets[topic] || [] - sockets_for_topic = Enum.uniq([socket | sockets_for_topic]) - sockets = Map.put(sockets, topic, sockets_for_topic) - Logger.debug("Got new conn for #{topic}") - {:noreply, sockets} - end - - def handle_cast(%{action: :remove, topic: topic, socket: socket}, sockets) do - topic = internal_topic(topic, socket) - sockets_for_topic = sockets[topic] || [] - sockets_for_topic = List.delete(sockets_for_topic, socket) - sockets = Map.put(sockets, topic, sockets_for_topic) - Logger.debug("Removed conn for #{topic}") - {:noreply, sockets} - end - - def handle_cast(m, state) do - Logger.info("Unknown: #{inspect(m)}, #{inspect(state)}") - {:noreply, state} - end - - defp represent_update(%Activity{} = activity, %User{} = user) do - %{ - event: "update", - payload: - Pleroma.Web.MastodonAPI.StatusView.render( - "status.json", - activity: activity, - for: user - ) - |> Jason.encode!() - } - |> Jason.encode!() - end - - defp represent_update(%Activity{} = activity) do - %{ - event: "update", - payload: - Pleroma.Web.MastodonAPI.StatusView.render( - "status.json", - activity: activity - ) - |> Jason.encode!() - } - |> Jason.encode!() - end - - def represent_conversation(%Participation{} = participation) do - %{ - event: "conversation", - payload: - Pleroma.Web.MastodonAPI.ConversationView.render("participation.json", %{ - participation: participation, - for: participation.user - }) - |> Jason.encode!() - } - |> Jason.encode!() - end - - @spec represent_notification(User.t(), Notification.t()) :: binary() - defp represent_notification(%User{} = user, %Notification{} = notify) do - %{ - event: "notification", - payload: - NotificationView.render( - "show.json", - %{notification: notify, for: user} - ) - |> Jason.encode!() - } - |> Jason.encode!() - end - - defp should_send?(%User{} = user, %Activity{} = item) do - blocks = user.info.blocks || [] - mutes = user.info.mutes || [] - reblog_mutes = user.info.muted_reblogs || [] - domain_blocks = Pleroma.Web.ActivityPub.MRF.subdomains_regex(user.info.domain_blocks) - - with parent when not is_nil(parent) <- Object.normalize(item), - true <- Enum.all?([blocks, mutes, reblog_mutes], &(item.actor not in &1)), - true <- Enum.all?([blocks, mutes], &(parent.data["actor"] not in &1)), - %{host: item_host} <- URI.parse(item.actor), - %{host: parent_host} <- URI.parse(parent.data["actor"]), - false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, item_host), - false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, parent_host), - true <- thread_containment(item, user), - false <- CommonAPI.thread_muted?(user, item) do - true - else - _ -> false - end - end - - defp should_send?(%User{} = user, %Notification{activity: activity}) do - should_send?(user, activity) - end - - def push_to_socket(topics, topic, %Activity{data: %{"type" => "Announce"}} = item) do - Enum.each(topics[topic] || [], fn socket -> - # Get the current user so we have up-to-date blocks etc. - if socket.assigns[:user] do - user = User.get_cached_by_ap_id(socket.assigns[:user].ap_id) - - if should_send?(user, item) do - send(socket.transport_pid, {:text, represent_update(item, user)}) - end - else - send(socket.transport_pid, {:text, represent_update(item)}) - end - end) - end - - def push_to_socket(topics, topic, %Participation{} = participation) do - Enum.each(topics[topic] || [], fn socket -> - send(socket.transport_pid, {:text, represent_conversation(participation)}) - end) - end - - def push_to_socket(topics, topic, %Activity{ - data: %{"type" => "Delete", "deleted_activity_id" => deleted_activity_id} - }) do - Enum.each(topics[topic] || [], fn socket -> - send( - socket.transport_pid, - {:text, %{event: "delete", payload: to_string(deleted_activity_id)} |> Jason.encode!()} - ) - end) - end - - def push_to_socket(_topics, _topic, %Activity{data: %{"type" => "Delete"}}), do: :noop - - def push_to_socket(topics, topic, item) do - Enum.each(topics[topic] || [], fn socket -> - # Get the current user so we have up-to-date blocks etc. - if socket.assigns[:user] do - user = User.get_cached_by_ap_id(socket.assigns[:user].ap_id) - blocks = user.info.blocks || [] - mutes = user.info.mutes || [] - - with true <- Enum.all?([blocks, mutes], &(item.actor not in &1)), - true <- thread_containment(item, user) do - send(socket.transport_pid, {:text, represent_update(item, user)}) - end - else - send(socket.transport_pid, {:text, represent_update(item)}) - end - end) - end - - defp internal_topic(topic, socket) when topic in ~w[user user:notification direct] do - "#{topic}:#{socket.assigns[:user].id}" - end - - defp internal_topic(topic, _), do: topic - - @spec thread_containment(Activity.t(), User.t()) :: boolean() - defp thread_containment(_activity, %User{info: %{skip_thread_containment: true}}), do: true - - defp thread_containment(activity, user) do - if Config.get([:instance, :skip_thread_containment]) do - true - else - ActivityPub.contain_activity(activity, user) - end - end -end diff --git a/lib/pleroma/web/streamer/ping.ex b/lib/pleroma/web/streamer/ping.ex new file mode 100644 index 00000000..f77cbb95 --- /dev/null +++ b/lib/pleroma/web/streamer/ping.ex @@ -0,0 +1,33 @@ +defmodule Pleroma.Web.Streamer.Ping do + use GenServer + require Logger + + alias Pleroma.Web.Streamer.State + alias Pleroma.Web.Streamer.StreamerSocket + + @keepalive_interval :timer.seconds(30) + + def start_link(opts) do + ping_interval = Keyword.get(opts, :ping_interval, @keepalive_interval) + GenServer.start_link(__MODULE__, %{ping_interval: ping_interval}, name: __MODULE__) + end + + def init(%{ping_interval: ping_interval} = args) do + Process.send_after(self(), :ping, ping_interval) + {:ok, args} + end + + def handle_info(:ping, %{ping_interval: ping_interval} = state) do + State.get_sockets() + |> Map.values() + |> List.flatten() + |> Enum.each(fn %StreamerSocket{transport_pid: transport_pid} -> + Logger.debug("Sending keepalive ping") + send(transport_pid, {:text, ""}) + end) + + Process.send_after(self(), :ping, ping_interval) + + {:noreply, state} + end +end diff --git a/lib/pleroma/web/streamer/state.ex b/lib/pleroma/web/streamer/state.ex new file mode 100644 index 00000000..7b519906 --- /dev/null +++ b/lib/pleroma/web/streamer/state.ex @@ -0,0 +1,68 @@ +defmodule Pleroma.Web.Streamer.State do + use GenServer + require Logger + + alias Pleroma.Web.Streamer.StreamerSocket + + def start_link(_) do + GenServer.start_link(__MODULE__, %{sockets: %{}}, name: __MODULE__) + end + + def add_socket(topic, socket) do + GenServer.call(__MODULE__, {:add, socket, topic}) + end + + def remove_socket(topic, socket) do + GenServer.call(__MODULE__, {:remove, socket, topic}) + end + + def get_sockets do + %{sockets: stream_sockets} = GenServer.call(__MODULE__, :get_state) + stream_sockets + end + + def init(init_arg) do + {:ok, init_arg} + end + + def handle_call(:get_state, _from, state) do + {:reply, state, state} + end + + def handle_call({:add, socket, topic}, _from, %{sockets: sockets} = state) do + internal_topic = internal_topic(topic, socket) + stream_socket = StreamerSocket.from_socket(socket) + + sockets_for_topic = + sockets + |> Map.get(internal_topic, []) + |> List.insert_at(0, stream_socket) + |> Enum.uniq() + + state = put_in(state, [:sockets, internal_topic], sockets_for_topic) + Logger.debug("Got new conn for #{topic}") + {:reply, state, state} + end + + def handle_call({:remove, socket, topic}, _from, %{sockets: sockets} = state) do + internal_topic = internal_topic(topic, socket) + stream_socket = StreamerSocket.from_socket(socket) + + sockets_for_topic = + sockets + |> Map.get(internal_topic, []) + |> List.delete(stream_socket) + + state = Kernel.put_in(state, [:sockets, internal_topic], sockets_for_topic) + {:reply, state, state} + end + + defp internal_topic(topic, socket) + when topic in ~w[user user:notification direct] do + "#{topic}:#{socket.assigns[:user].id}" + end + + defp internal_topic(topic, _) do + topic + end +end diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex new file mode 100644 index 00000000..8cf71927 --- /dev/null +++ b/lib/pleroma/web/streamer/streamer.ex @@ -0,0 +1,55 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Streamer do + alias Pleroma.Web.Streamer.State + alias Pleroma.Web.Streamer.Worker + + @timeout 60_000 + @mix_env Mix.env() + + def add_socket(topic, socket) do + State.add_socket(topic, socket) + end + + def remove_socket(topic, socket) do + State.remove_socket(topic, socket) + end + + def get_sockets do + State.get_sockets() + end + + def stream(topics, items) do + if should_send?() do + Task.async(fn -> + :poolboy.transaction( + :streamer_worker, + &Worker.stream(&1, topics, items), + @timeout + ) + end) + end + end + + def supervisor, do: Pleroma.Web.Streamer.Supervisor + + defp should_send? do + handle_should_send(@mix_env) + end + + defp handle_should_send(:test) do + case Process.whereis(:streamer_worker) do + nil -> + false + + pid -> + Process.alive?(pid) + end + end + + defp handle_should_send(_) do + true + end +end diff --git a/lib/pleroma/web/streamer/streamer_socket.ex b/lib/pleroma/web/streamer/streamer_socket.ex new file mode 100644 index 00000000..f006c030 --- /dev/null +++ b/lib/pleroma/web/streamer/streamer_socket.ex @@ -0,0 +1,31 @@ +defmodule Pleroma.Web.Streamer.StreamerSocket do + defstruct transport_pid: nil, user: nil + + alias Pleroma.User + alias Pleroma.Web.Streamer.StreamerSocket + + def from_socket(%{ + transport_pid: transport_pid, + assigns: %{user: nil} + }) do + %StreamerSocket{ + transport_pid: transport_pid + } + end + + def from_socket(%{ + transport_pid: transport_pid, + assigns: %{user: %User{} = user} + }) do + %StreamerSocket{ + transport_pid: transport_pid, + user: user + } + end + + def from_socket(%{transport_pid: transport_pid}) do + %StreamerSocket{ + transport_pid: transport_pid + } + end +end diff --git a/lib/pleroma/web/streamer/supervisor.ex b/lib/pleroma/web/streamer/supervisor.ex new file mode 100644 index 00000000..6afe1932 --- /dev/null +++ b/lib/pleroma/web/streamer/supervisor.ex @@ -0,0 +1,33 @@ +defmodule Pleroma.Web.Streamer.Supervisor do + use Supervisor + + def start_link(opts) do + Supervisor.start_link(__MODULE__, opts, name: __MODULE__) + end + + def init(args) do + children = [ + {Pleroma.Web.Streamer.State, args}, + {Pleroma.Web.Streamer.Ping, args}, + :poolboy.child_spec(:streamer_worker, poolboy_config()) + ] + + opts = [strategy: :one_for_one, name: Pleroma.Web.Streamer.Supervisor] + Supervisor.init(children, opts) + end + + defp poolboy_config do + opts = + Pleroma.Config.get(:streamer, + workers: 3, + overflow_workers: 2 + ) + + [ + {:name, {:local, :streamer_worker}}, + {:worker_module, Pleroma.Web.Streamer.Worker}, + {:size, opts[:workers]}, + {:max_overflow, opts[:overflow_workers]} + ] + end +end diff --git a/lib/pleroma/web/streamer/worker.ex b/lib/pleroma/web/streamer/worker.ex new file mode 100644 index 00000000..5804508e --- /dev/null +++ b/lib/pleroma/web/streamer/worker.ex @@ -0,0 +1,220 @@ +defmodule Pleroma.Web.Streamer.Worker do + use GenServer + + require Logger + + alias Pleroma.Activity + alias Pleroma.Config + alias Pleroma.Conversation.Participation + alias Pleroma.Notification + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.Streamer.State + alias Pleroma.Web.Streamer.StreamerSocket + alias Pleroma.Web.StreamerView + + def start_link(_) do + GenServer.start_link(__MODULE__, %{}, []) + end + + def init(init_arg) do + {:ok, init_arg} + end + + def stream(pid, topics, items) do + GenServer.call(pid, {:stream, topics, items}) + end + + def handle_call({:stream, topics, item}, _from, state) when is_list(topics) do + Enum.each(topics, fn t -> + do_stream(%{topic: t, item: item}) + end) + + {:reply, state, state} + end + + def handle_call({:stream, topic, items}, _from, state) when is_list(items) do + Enum.each(items, fn i -> + do_stream(%{topic: topic, item: i}) + end) + + {:reply, state, state} + end + + def handle_call({:stream, topic, item}, _from, state) do + do_stream(%{topic: topic, item: item}) + + {:reply, state, state} + end + + defp do_stream(%{topic: "direct", item: item}) do + recipient_topics = + User.get_recipients_from_activity(item) + |> Enum.map(fn %{id: id} -> "direct:#{id}" end) + + Enum.each(recipient_topics, fn user_topic -> + Logger.debug("Trying to push direct message to #{user_topic}\n\n") + push_to_socket(State.get_sockets(), user_topic, item) + end) + end + + defp do_stream(%{topic: "participation", item: participation}) do + user_topic = "direct:#{participation.user_id}" + Logger.debug("Trying to push a conversation participation to #{user_topic}\n\n") + + push_to_socket(State.get_sockets(), user_topic, participation) + end + + defp do_stream(%{topic: "list", item: item}) do + # filter the recipient list if the activity is not public, see #270. + recipient_lists = + case Visibility.is_public?(item) do + true -> + Pleroma.List.get_lists_from_activity(item) + + _ -> + Pleroma.List.get_lists_from_activity(item) + |> Enum.filter(fn list -> + owner = User.get_cached_by_id(list.user_id) + + Visibility.visible_for_user?(item, owner) + end) + end + + recipient_topics = + recipient_lists + |> Enum.map(fn %{id: id} -> "list:#{id}" end) + + Enum.each(recipient_topics, fn list_topic -> + Logger.debug("Trying to push message to #{list_topic}\n\n") + push_to_socket(State.get_sockets(), list_topic, item) + end) + end + + defp do_stream(%{topic: topic, item: %Notification{} = item}) + when topic in ["user", "user:notification"] do + State.get_sockets() + |> Map.get("#{topic}:#{item.user_id}", []) + |> Enum.each(fn %StreamerSocket{transport_pid: transport_pid, user: socket_user} -> + with %User{} = user <- User.get_cached_by_ap_id(socket_user.ap_id), + true <- should_send?(user, item) do + send(transport_pid, {:text, StreamerView.render("notification.json", socket_user, item)}) + end + end) + end + + defp do_stream(%{topic: "user", item: item}) do + Logger.debug("Trying to push to users") + + recipient_topics = + User.get_recipients_from_activity(item) + |> Enum.map(fn %{id: id} -> "user:#{id}" end) + + Enum.each(recipient_topics, fn topic -> + push_to_socket(State.get_sockets(), topic, item) + end) + end + + defp do_stream(%{topic: topic, item: item}) do + Logger.debug("Trying to push to #{topic}") + Logger.debug("Pushing item to #{topic}") + push_to_socket(State.get_sockets(), topic, item) + end + + defp should_send?(%User{} = user, %Activity{} = item) do + blocks = user.info.blocks || [] + mutes = user.info.mutes || [] + reblog_mutes = user.info.muted_reblogs || [] + domain_blocks = Pleroma.Web.ActivityPub.MRF.subdomains_regex(user.info.domain_blocks) + + with parent when not is_nil(parent) <- Object.normalize(item), + true <- Enum.all?([blocks, mutes, reblog_mutes], &(item.actor not in &1)), + true <- Enum.all?([blocks, mutes], &(parent.data["actor"] not in &1)), + %{host: item_host} <- URI.parse(item.actor), + %{host: parent_host} <- URI.parse(parent.data["actor"]), + false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, item_host), + false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, parent_host), + true <- thread_containment(item, user), + false <- CommonAPI.thread_muted?(user, item) do + true + else + _ -> false + end + end + + defp should_send?(%User{} = user, %Notification{activity: activity}) do + should_send?(user, activity) + end + + def push_to_socket(topics, topic, %Activity{data: %{"type" => "Announce"}} = item) do + Enum.each(topics[topic] || [], fn %StreamerSocket{ + transport_pid: transport_pid, + user: socket_user + } -> + # Get the current user so we have up-to-date blocks etc. + if socket_user do + user = User.get_cached_by_ap_id(socket_user.ap_id) + + if should_send?(user, item) do + send(transport_pid, {:text, StreamerView.render("update.json", item, user)}) + end + else + send(transport_pid, {:text, StreamerView.render("update.json", item)}) + end + end) + end + + def push_to_socket(topics, topic, %Participation{} = participation) do + Enum.each(topics[topic] || [], fn %StreamerSocket{transport_pid: transport_pid} -> + send(transport_pid, {:text, StreamerView.render("conversation.json", participation)}) + end) + end + + def push_to_socket(topics, topic, %Activity{ + data: %{"type" => "Delete", "deleted_activity_id" => deleted_activity_id} + }) do + Enum.each(topics[topic] || [], fn %StreamerSocket{transport_pid: transport_pid} -> + send( + transport_pid, + {:text, %{event: "delete", payload: to_string(deleted_activity_id)} |> Jason.encode!()} + ) + end) + end + + def push_to_socket(_topics, _topic, %Activity{data: %{"type" => "Delete"}}), do: :noop + + def push_to_socket(topics, topic, item) do + Enum.each(topics[topic] || [], fn %StreamerSocket{ + transport_pid: transport_pid, + user: socket_user + } -> + # Get the current user so we have up-to-date blocks etc. + if socket_user do + user = User.get_cached_by_ap_id(socket_user.ap_id) + blocks = user.info.blocks || [] + mutes = user.info.mutes || [] + + with true <- Enum.all?([blocks, mutes], &(item.actor not in &1)), + true <- thread_containment(item, user) do + send(transport_pid, {:text, StreamerView.render("update.json", item, user)}) + end + else + send(transport_pid, {:text, StreamerView.render("update.json", item)}) + end + end) + end + + @spec thread_containment(Activity.t(), User.t()) :: boolean() + defp thread_containment(_activity, %User{info: %{skip_thread_containment: true}}), do: true + + defp thread_containment(activity, user) do + if Config.get([:instance, :skip_thread_containment]) do + true + else + ActivityPub.contain_activity(activity, user) + end + end +end diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex new file mode 100644 index 00000000..b13030fa --- /dev/null +++ b/lib/pleroma/web/views/streamer_view.ex @@ -0,0 +1,66 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.StreamerView do + use Pleroma.Web, :view + + alias Pleroma.Activity + alias Pleroma.Conversation.Participation + alias Pleroma.Notification + alias Pleroma.User + alias Pleroma.Web.MastodonAPI.NotificationView + + def render("update.json", %Activity{} = activity, %User{} = user) do + %{ + event: "update", + payload: + Pleroma.Web.MastodonAPI.StatusView.render( + "status.json", + activity: activity, + for: user + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + + def render("notification.json", %User{} = user, %Notification{} = notify) do + %{ + event: "notification", + payload: + NotificationView.render( + "show.json", + %{notification: notify, for: user} + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + + def render("update.json", %Activity{} = activity) do + %{ + event: "update", + payload: + Pleroma.Web.MastodonAPI.StatusView.render( + "status.json", + activity: activity + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + + def render("conversation.json", %Participation{} = participation) do + %{ + event: "conversation", + payload: + Pleroma.Web.MastodonAPI.ConversationView.render("participation.json", %{ + participation: participation, + for: participation.user + }) + |> Jason.encode!() + } + |> Jason.encode!() + end +end diff --git a/mix.exs b/mix.exs index f1e98585..911ebad1 100644 --- a/mix.exs +++ b/mix.exs @@ -144,6 +144,7 @@ defmodule Pleroma.Mixfile do git: "https://git.pleroma.social/pleroma/http_signatures.git", ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"}, {:telemetry, "~> 0.3"}, + {:poolboy, "~> 1.5"}, {:prometheus_ex, "~> 3.0"}, {:prometheus_plugs, "~> 1.1"}, {:prometheus_phoenix, "~> 1.3"}, diff --git a/mix.lock b/mix.lock index 41697dd5..0bf6a811 100644 --- a/mix.lock +++ b/mix.lock @@ -73,6 +73,7 @@ "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"}, "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, + "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, "postgrex": {:hex, :postgrex, "0.14.3", "5754dee2fdf6e9e508cbf49ab138df964278700b764177e8f3871e658b345a1e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, "prometheus": {:hex, :prometheus, "4.4.1", "1e96073b3ed7788053768fea779cbc896ddc3bdd9ba60687f2ad50b252ac87d6", [:mix, :rebar3], [], "hexpm"}, "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/activity/ir/topics_test.exs b/test/activity/ir/topics_test.exs new file mode 100644 index 00000000..e75f8358 --- /dev/null +++ b/test/activity/ir/topics_test.exs @@ -0,0 +1,141 @@ +defmodule Pleroma.Activity.Ir.TopicsTest do + use Pleroma.DataCase + + alias Pleroma.Activity + alias Pleroma.Activity.Ir.Topics + alias Pleroma.Object + + require Pleroma.Constants + + describe "poll answer" do + test "produce no topics" do + activity = %Activity{object: %Object{data: %{"type" => "Answer"}}} + + assert [] == Topics.get_activity_topics(activity) + end + end + + describe "non poll answer" do + test "always add user and list topics" do + activity = %Activity{object: %Object{data: %{"type" => "FooBar"}}} + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "user") + assert Enum.member?(topics, "list") + end + end + + describe "public visibility" do + setup do + activity = %Activity{ + object: %Object{data: %{"type" => "Note"}}, + data: %{"to" => [Pleroma.Constants.as_public()]} + } + + {:ok, activity: activity} + end + + test "produces public topic", %{activity: activity} do + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "public") + end + + test "local action produces public:local topic", %{activity: activity} do + activity = %{activity | local: true} + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "public:local") + end + + test "non-local action does not produce public:local topic", %{activity: activity} do + activity = %{activity | local: false} + topics = Topics.get_activity_topics(activity) + + refute Enum.member?(topics, "public:local") + end + end + + describe "public visibility create events" do + setup do + activity = %Activity{ + object: %Object{data: %{"type" => "Create", "attachment" => []}}, + data: %{"to" => [Pleroma.Constants.as_public()]} + } + + {:ok, activity: activity} + end + + test "with no attachments doesn't produce public:media topics", %{activity: activity} do + topics = Topics.get_activity_topics(activity) + + refute Enum.member?(topics, "public:media") + refute Enum.member?(topics, "public:local:media") + end + + test "converts tags to hash tags", %{activity: %{object: %{data: data} = object} = activity} do + tagged_data = Map.put(data, "tag", ["foo", "bar"]) + activity = %{activity | object: %{object | data: tagged_data}} + + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "hashtag:foo") + assert Enum.member?(topics, "hashtag:bar") + end + + test "only converts strinngs to hash tags", %{ + activity: %{object: %{data: data} = object} = activity + } do + tagged_data = Map.put(data, "tag", [2]) + activity = %{activity | object: %{object | data: tagged_data}} + + topics = Topics.get_activity_topics(activity) + + refute Enum.member?(topics, "hashtag:2") + end + end + + describe "public visibility create events with attachments" do + setup do + activity = %Activity{ + object: %Object{data: %{"type" => "Create", "attachment" => ["foo"]}}, + data: %{"to" => [Pleroma.Constants.as_public()]} + } + + {:ok, activity: activity} + end + + test "produce public:media topics", %{activity: activity} do + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "public:media") + end + + test "local produces public:local:media topics", %{activity: activity} do + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "public:local:media") + end + + test "non-local doesn't produce public:local:media topics", %{activity: activity} do + activity = %{activity | local: false} + + topics = Topics.get_activity_topics(activity) + + refute Enum.member?(topics, "public:local:media") + end + end + + describe "non-public visibility" do + test "produces direct topic" do + activity = %Activity{object: %Object{data: %{"type" => "Note"}}, data: %{"to" => []}} + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "direct") + refute Enum.member?(topics, "public") + refute Enum.member?(topics, "public:local") + refute Enum.member?(topics, "public:media") + refute Enum.member?(topics, "public:local:media") + end + end +end diff --git a/test/integration/mastodon_websocket_test.exs b/test/integration/mastodon_websocket_test.exs index 63bf7341..c0426280 100644 --- a/test/integration/mastodon_websocket_test.exs +++ b/test/integration/mastodon_websocket_test.exs @@ -11,7 +11,6 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do alias Pleroma.Integration.WebsocketClient alias Pleroma.Web.CommonAPI alias Pleroma.Web.OAuth - alias Pleroma.Web.Streamer @path Pleroma.Web.Endpoint.url() |> URI.parse() @@ -19,16 +18,6 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do |> Map.put(:path, "/api/v1/streaming") |> URI.to_string() - setup do - GenServer.start(Streamer, %{}, name: Streamer) - - on_exit(fn -> - if pid = Process.whereis(Streamer) do - Process.exit(pid, :kill) - end - end) - end - def start_socket(qs \\ nil, headers \\ []) do path = case qs do @@ -53,12 +42,14 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do end) end + @tag needs_streamer: true test "allows public streams without authentication" do assert {:ok, _} = start_socket("?stream=public") assert {:ok, _} = start_socket("?stream=public:local") assert {:ok, _} = start_socket("?stream=hashtag&tag=lain") end + @tag needs_streamer: true test "receives well formatted events" do user = insert(:user) {:ok, _} = start_socket("?stream=public") @@ -103,6 +94,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert {:ok, _} = start_socket("?stream=user&access_token=#{state.token.token}") end + @tag needs_streamer: true test "accepts the 'user' stream", %{token: token} = _state do assert {:ok, _} = start_socket("?stream=user&access_token=#{token.token}") @@ -111,6 +103,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do end) =~ ":badarg" end + @tag needs_streamer: true test "accepts the 'user:notification' stream", %{token: token} = _state do assert {:ok, _} = start_socket("?stream=user:notification&access_token=#{token.token}") @@ -119,6 +112,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do end) =~ ":badarg" end + @tag needs_streamer: true test "accepts valid token on Sec-WebSocket-Protocol header", %{token: token} do assert {:ok, _} = start_socket("?stream=user", [{"Sec-WebSocket-Protocol", token.token}]) diff --git a/test/notification_test.exs b/test/notification_test.exs index 3be9db09..3d2f9a8f 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -69,16 +69,7 @@ defmodule Pleroma.NotificationTest do end describe "create_notification" do - setup do - GenServer.start(Streamer, %{}, name: Streamer) - - on_exit(fn -> - if pid = Process.whereis(Streamer) do - Process.exit(pid, :kill) - end - end) - end - + @tag needs_streamer: true test "it creates a notification for user and send to the 'user' and the 'user:notification' stream" do user = insert(:user) task = Task.async(fn -> assert_receive {:text, _}, 4_000 end) diff --git a/test/support/conn_case.ex b/test/support/conn_case.ex index ec5892ff..b39c7067 100644 --- a/test/support/conn_case.ex +++ b/test/support/conn_case.ex @@ -40,6 +40,10 @@ defmodule Pleroma.Web.ConnCase do Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, {:shared, self()}) end + if tags[:needs_streamer] do + start_supervised(Pleroma.Web.Streamer.supervisor()) + end + {:ok, conn: Phoenix.ConnTest.build_conn()} end end diff --git a/test/support/data_case.ex b/test/support/data_case.ex index f3d98e7e..17fa1521 100644 --- a/test/support/data_case.ex +++ b/test/support/data_case.ex @@ -39,6 +39,10 @@ defmodule Pleroma.DataCase do Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, {:shared, self()}) end + if tags[:needs_streamer] do + start_supervised(Pleroma.Web.Streamer.supervisor()) + end + :ok end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index d0118fef..4100108a 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -38,9 +38,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do stream: fn _, _ -> nil end do ActivityPub.stream_out_participations(conversation.participations) - Enum.each(participations, fn participation -> - assert called(Pleroma.Web.Streamer.stream("participation", participation)) - end) + assert called(Pleroma.Web.Streamer.stream("participation", participations)) end end end diff --git a/test/web/streamer/ping_test.exs b/test/web/streamer/ping_test.exs new file mode 100644 index 00000000..3d52c00e --- /dev/null +++ b/test/web/streamer/ping_test.exs @@ -0,0 +1,36 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PingTest do + use Pleroma.DataCase + + import Pleroma.Factory + alias Pleroma.Web.Streamer + + setup do + start_supervised({Streamer.supervisor(), [ping_interval: 30]}) + + :ok + end + + describe "sockets" do + setup do + user = insert(:user) + {:ok, %{user: user}} + end + + test "it sends pings", %{user: user} do + task = + Task.async(fn -> + assert_receive {:text, received_event}, 40 + assert_receive {:text, received_event}, 40 + assert_receive {:text, received_event}, 40 + end) + + Streamer.add_socket("public", %{transport_pid: task.pid, assigns: %{user: user}}) + + Task.await(task) + end + end +end diff --git a/test/web/streamer/state_test.exs b/test/web/streamer/state_test.exs new file mode 100644 index 00000000..d1aeac54 --- /dev/null +++ b/test/web/streamer/state_test.exs @@ -0,0 +1,54 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.StateTest do + use Pleroma.DataCase + + import Pleroma.Factory + alias Pleroma.Web.Streamer + alias Pleroma.Web.Streamer.StreamerSocket + + @moduletag needs_streamer: true + + describe "sockets" do + setup do + user = insert(:user) + user2 = insert(:user) + {:ok, %{user: user, user2: user2}} + end + + test "it can add a socket", %{user: user} do + Streamer.add_socket("public", %{transport_pid: 1, assigns: %{user: user}}) + + assert(%{"public" => [%StreamerSocket{transport_pid: 1}]} = Streamer.get_sockets()) + end + + test "it can add multiple sockets per user", %{user: user} do + Streamer.add_socket("public", %{transport_pid: 1, assigns: %{user: user}}) + Streamer.add_socket("public", %{transport_pid: 2, assigns: %{user: user}}) + + assert( + %{ + "public" => [ + %StreamerSocket{transport_pid: 2}, + %StreamerSocket{transport_pid: 1} + ] + } = Streamer.get_sockets() + ) + end + + test "it will not add a duplicate socket", %{user: user} do + Streamer.add_socket("activity", %{transport_pid: 1, assigns: %{user: user}}) + Streamer.add_socket("activity", %{transport_pid: 1, assigns: %{user: user}}) + + assert( + %{ + "activity" => [ + %StreamerSocket{transport_pid: 1} + ] + } = Streamer.get_sockets() + ) + end + end +end diff --git a/test/web/streamer_test.exs b/test/web/streamer/streamer_test.exs similarity index 86% rename from test/web/streamer_test.exs rename to test/web/streamer/streamer_test.exs index 96fa7645..88847e20 100644 --- a/test/web/streamer_test.exs +++ b/test/web/streamer/streamer_test.exs @@ -5,24 +5,20 @@ defmodule Pleroma.Web.StreamerTest do use Pleroma.DataCase + import Pleroma.Factory + alias Pleroma.List alias Pleroma.User alias Pleroma.Web.CommonAPI alias Pleroma.Web.Streamer - import Pleroma.Factory + alias Pleroma.Web.Streamer.StreamerSocket + alias Pleroma.Web.Streamer.Worker + @moduletag needs_streamer: true clear_config_all([:instance, :skip_thread_containment]) describe "user streams" do setup do - GenServer.start(Streamer, %{}, name: Streamer) - - on_exit(fn -> - if pid = Process.whereis(Streamer) do - Process.exit(pid, :kill) - end - end) - user = insert(:user) notify = insert(:notification, user: user, activity: build(:note_activity)) {:ok, %{user: user, notify: notify}} @@ -125,11 +121,9 @@ defmodule Pleroma.Web.StreamerTest do assert_receive {:text, _}, 4_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user - } + user: user } {:ok, activity} = CommonAPI.post(other_user, %{"status" => "Test"}) @@ -138,7 +132,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) @@ -155,11 +149,9 @@ defmodule Pleroma.Web.StreamerTest do assert received_event == expected_event end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user - } + user: user } {:ok, activity} = CommonAPI.delete(activity.id, other_user) @@ -168,7 +160,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) end @@ -189,9 +181,9 @@ defmodule Pleroma.Web.StreamerTest do ) task = Task.async(fn -> refute_receive {:text, _}, 1_000 end) - fake_socket = %{transport_pid: task.pid, assigns: %{user: user}} + fake_socket = %StreamerSocket{transport_pid: task.pid, user: user} topics = %{"public" => [fake_socket]} - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) end @@ -211,9 +203,9 @@ defmodule Pleroma.Web.StreamerTest do ) task = Task.async(fn -> assert_receive {:text, _}, 1_000 end) - fake_socket = %{transport_pid: task.pid, assigns: %{user: user}} + fake_socket = %StreamerSocket{transport_pid: task.pid, user: user} topics = %{"public" => [fake_socket]} - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) end @@ -233,9 +225,9 @@ defmodule Pleroma.Web.StreamerTest do ) task = Task.async(fn -> assert_receive {:text, _}, 1_000 end) - fake_socket = %{transport_pid: task.pid, assigns: %{user: user}} + fake_socket = %StreamerSocket{transport_pid: task.pid, user: user} topics = %{"public" => [fake_socket]} - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) end @@ -251,11 +243,9 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user - } + user: user } {:ok, activity} = CommonAPI.post(blocked_user, %{"status" => "Test"}) @@ -264,7 +254,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) end @@ -284,11 +274,9 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user_a - } + user: user_a } {:ok, activity} = @@ -301,7 +289,7 @@ defmodule Pleroma.Web.StreamerTest do "list:#{list.id}" => [fake_socket] } - Streamer.handle_cast(%{action: :stream, topic: "list", item: activity}, topics) + Worker.handle_call({:stream, "list", activity}, self(), topics) Task.await(task) end @@ -318,11 +306,9 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user_a - } + user: user_a } {:ok, activity} = @@ -335,12 +321,12 @@ defmodule Pleroma.Web.StreamerTest do "list:#{list.id}" => [fake_socket] } - Streamer.handle_cast(%{action: :stream, topic: "list", item: activity}, topics) + Worker.handle_call({:stream, "list", activity}, self(), topics) Task.await(task) end - test "it send wanted private posts to list" do + test "it sends wanted private posts to list" do user_a = insert(:user) user_b = insert(:user) @@ -354,11 +340,9 @@ defmodule Pleroma.Web.StreamerTest do assert_receive {:text, _}, 1_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user_a - } + user: user_a } {:ok, activity} = @@ -367,11 +351,12 @@ defmodule Pleroma.Web.StreamerTest do "visibility" => "private" }) - topics = %{ - "list:#{list.id}" => [fake_socket] - } + Streamer.add_socket( + "list:#{list.id}", + fake_socket + ) - Streamer.handle_cast(%{action: :stream, topic: "list", item: activity}, topics) + Worker.handle_call({:stream, "list", activity}, self(), %{}) Task.await(task) end @@ -387,11 +372,9 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user1 - } + user: user1 } {:ok, create_activity} = CommonAPI.post(user3, %{"status" => "I'm kawen"}) @@ -401,7 +384,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Streamer.push_to_socket(topics, "public", announce_activity) + Worker.push_to_socket(topics, "public", announce_activity) Task.await(task) end @@ -417,6 +400,8 @@ defmodule Pleroma.Web.StreamerTest do task = Task.async(fn -> refute_receive {:text, _}, 4_000 end) + Process.sleep(4000) + Streamer.add_socket( "user", %{transport_pid: task.pid, assigns: %{user: user2}} @@ -428,14 +413,6 @@ defmodule Pleroma.Web.StreamerTest do describe "direct streams" do setup do - GenServer.start(Streamer, %{}, name: Streamer) - - on_exit(fn -> - if pid = Process.whereis(Streamer) do - Process.exit(pid, :kill) - end - end) - :ok end @@ -480,6 +457,8 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 4_000 end) + Process.sleep(1000) + Streamer.add_socket( "direct", %{transport_pid: task.pid, assigns: %{user: user}} @@ -521,6 +500,8 @@ defmodule Pleroma.Web.StreamerTest do assert last_status["id"] == to_string(create_activity.id) end) + Process.sleep(1000) + Streamer.add_socket( "direct", %{transport_pid: task.pid, assigns: %{user: user}} From c623b4324deaf236334a0f77a81435b5bffadf3c Mon Sep 17 00:00:00 2001 From: kaniini Date: Mon, 16 Sep 2019 09:09:21 +0000 Subject: [PATCH 092/138] Revert "Merge branch 'streamer-refactoring' into 'develop'" This reverts merge request !1653 --- .gitignore | 4 - config/config.exs | 4 - lib/pleroma/activity/ir/topics.ex | 63 ---- lib/pleroma/application.ex | 2 +- lib/pleroma/notification.ex | 6 +- lib/pleroma/web/activity_pub/activity_pub.ex | 48 ++- .../web/mastodon_api/websocket_handler.ex | 7 +- lib/pleroma/web/streamer.ex | 318 ++++++++++++++++++ lib/pleroma/web/streamer/ping.ex | 33 -- lib/pleroma/web/streamer/state.ex | 68 ---- lib/pleroma/web/streamer/streamer.ex | 55 --- lib/pleroma/web/streamer/streamer_socket.ex | 31 -- lib/pleroma/web/streamer/supervisor.ex | 33 -- lib/pleroma/web/streamer/worker.ex | 220 ------------ lib/pleroma/web/views/streamer_view.ex | 66 ---- mix.exs | 1 - mix.lock | 1 - test/activity/ir/topics_test.exs | 141 -------- test/integration/mastodon_websocket_test.exs | 16 +- test/notification_test.exs | 11 +- test/support/conn_case.ex | 4 - test/support/data_case.ex | 4 - test/web/activity_pub/activity_pub_test.exs | 4 +- test/web/streamer/ping_test.exs | 36 -- test/web/streamer/state_test.exs | 54 --- test/web/{streamer => }/streamer_test.exs | 105 +++--- 26 files changed, 447 insertions(+), 888 deletions(-) delete mode 100644 lib/pleroma/activity/ir/topics.ex create mode 100644 lib/pleroma/web/streamer.ex delete mode 100644 lib/pleroma/web/streamer/ping.ex delete mode 100644 lib/pleroma/web/streamer/state.ex delete mode 100644 lib/pleroma/web/streamer/streamer.ex delete mode 100644 lib/pleroma/web/streamer/streamer_socket.ex delete mode 100644 lib/pleroma/web/streamer/supervisor.ex delete mode 100644 lib/pleroma/web/streamer/worker.ex delete mode 100644 lib/pleroma/web/views/streamer_view.ex delete mode 100644 test/activity/ir/topics_test.exs delete mode 100644 test/web/streamer/ping_test.exs delete mode 100644 test/web/streamer/state_test.exs rename test/web/{streamer => }/streamer_test.exs (86%) diff --git a/.gitignore b/.gitignore index 3b0c7d36..4e71a7df 100644 --- a/.gitignore +++ b/.gitignore @@ -43,7 +43,3 @@ docs/generated_config.md # Code test coverage /cover /Elixir.*.coverdata - -.idea -pleroma.iml - diff --git a/config/config.exs b/config/config.exs index b1b98af9..ab6e00c9 100644 --- a/config/config.exs +++ b/config/config.exs @@ -331,10 +331,6 @@ config :pleroma, :activitypub, follow_handshake_timeout: 500, sign_object_fetches: true -config :pleroma, :streamer, - workers: 3, - overflow_workers: 2 - config :pleroma, :user, deny_follow_blocked: true config :pleroma, :mrf_normalize_markup, scrub_policy: Pleroma.HTML.Scrubber.Default diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex deleted file mode 100644 index 010897ab..00000000 --- a/lib/pleroma/activity/ir/topics.ex +++ /dev/null @@ -1,63 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Activity.Ir.Topics do - alias Pleroma.Object - alias Pleroma.Web.ActivityPub.Visibility - - def get_activity_topics(activity) do - activity - |> Object.normalize() - |> generate_topics(activity) - |> List.flatten() - end - - defp generate_topics(%{data: %{"type" => "Answer"}}, _) do - [] - end - - defp generate_topics(object, activity) do - ["user", "list"] ++ visibility_tags(object, activity) - end - - defp visibility_tags(object, activity) do - case Visibility.get_visibility(activity) do - "public" -> - if activity.local do - ["public", "public:local"] - else - ["public"] - end - |> item_creation_tags(object, activity) - - "direct" -> - ["direct"] - - _ -> - [] - end - end - - defp item_creation_tags(tags, %{data: %{"type" => "Create"}} = object, activity) do - tags ++ hashtags_to_topics(object) ++ attachment_topics(object, activity) - end - - defp item_creation_tags(tags, _, _) do - tags - end - - defp hashtags_to_topics(%{data: %{"tag" => tags}}) do - tags - |> Enum.filter(&is_bitstring(&1)) - |> Enum.map(fn tag -> "hashtag:" <> tag end) - end - - defp hashtags_to_topics(_), do: [] - - defp attachment_topics(%{data: %{"attachment" => []}}, _act), do: [] - - defp attachment_topics(_object, %{local: true}), do: ["public:media", "public:local:media"] - - defp attachment_topics(_object, _act), do: ["public:media"] -end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 3b37ce63..49094704 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -141,7 +141,7 @@ defmodule Pleroma.Application do defp streamer_child(:test), do: [] defp streamer_child(_) do - [Pleroma.Web.Streamer.supervisor()] + [Pleroma.Web.Streamer] end defp oauth_cleanup_child(true), diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 8012389a..b7c880c5 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -210,10 +210,8 @@ defmodule Pleroma.Notification do unless skip?(activity, user) do notification = %Notification{user_id: user.id, activity: activity} {:ok, notification} = Repo.insert(notification) - - ["user", "user:notification"] - |> Streamer.stream(notification) - + Streamer.stream("user", notification) + Streamer.stream("user:notification", notification) Push.send(notification) notification end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index bc5ae7fb..41f6a0f1 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -4,7 +4,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Activity - alias Pleroma.Activity.Ir.Topics alias Pleroma.Config alias Pleroma.Conversation alias Pleroma.Notification @@ -17,7 +16,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.User alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.Transmogrifier - alias Pleroma.Web.Streamer alias Pleroma.Web.WebFinger alias Pleroma.Workers.BackgroundWorker @@ -189,7 +187,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do participations |> Repo.preload(:user) - Streamer.stream("participation", participations) + Enum.each(participations, fn participation -> + Pleroma.Web.Streamer.stream("participation", participation) + end) end def stream_out_participations(%Object{data: %{"context" => context}}, user) do @@ -208,15 +208,41 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def stream_out_participations(_, _), do: :noop - def stream_out(%Activity{data: %{"type" => data_type}} = activity) - when data_type in ["Create", "Announce", "Delete"] do - activity - |> Topics.get_activity_topics() - |> Streamer.stream(activity) - end + def stream_out(activity) do + if activity.data["type"] in ["Create", "Announce", "Delete"] do + object = Object.normalize(activity) + # Do not stream out poll replies + unless object.data["type"] == "Answer" do + Pleroma.Web.Streamer.stream("user", activity) + Pleroma.Web.Streamer.stream("list", activity) - def stream_out(_activity) do - :noop + if get_visibility(activity) == "public" do + Pleroma.Web.Streamer.stream("public", activity) + + if activity.local do + Pleroma.Web.Streamer.stream("public:local", activity) + end + + if activity.data["type"] in ["Create"] do + object.data + |> Map.get("tag", []) + |> Enum.filter(fn tag -> is_bitstring(tag) end) + |> Enum.each(fn tag -> Pleroma.Web.Streamer.stream("hashtag:" <> tag, activity) end) + + if object.data["attachment"] != [] do + Pleroma.Web.Streamer.stream("public:media", activity) + + if activity.local do + Pleroma.Web.Streamer.stream("public:local:media", activity) + end + end + end + else + if get_visibility(activity) == "direct", + do: Pleroma.Web.Streamer.stream("direct", activity) + end + end + end end def create(%{to: to, actor: actor, context: context, object: object} = params, fake \\ false) do diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index 3c26eb40..dbd3542e 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.OAuth.Token - alias Pleroma.Web.Streamer @behaviour :cowboy_websocket @@ -25,7 +24,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do ] @anonymous_streams ["public", "public:local", "hashtag"] - # Handled by periodic keepalive in Pleroma.Web.Streamer.Ping. + # Handled by periodic keepalive in Pleroma.Web.Streamer. @timeout :infinity def init(%{qs: qs} = req, state) do @@ -66,7 +65,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do }, topic #{state.topic}" ) - Streamer.add_socket(state.topic, streamer_socket(state)) + Pleroma.Web.Streamer.add_socket(state.topic, streamer_socket(state)) {:ok, state} end @@ -81,7 +80,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do }, topic #{state.topic || "?"}: #{inspect(reason)}" ) - Streamer.remove_socket(state.topic, streamer_socket(state)) + Pleroma.Web.Streamer.remove_socket(state.topic, streamer_socket(state)) :ok end diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex new file mode 100644 index 00000000..587c43f4 --- /dev/null +++ b/lib/pleroma/web/streamer.ex @@ -0,0 +1,318 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Streamer do + use GenServer + require Logger + alias Pleroma.Activity + alias Pleroma.Config + alias Pleroma.Conversation.Participation + alias Pleroma.Notification + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.MastodonAPI.NotificationView + + @keepalive_interval :timer.seconds(30) + + def start_link(_) do + GenServer.start_link(__MODULE__, %{}, name: __MODULE__) + end + + def add_socket(topic, socket) do + GenServer.cast(__MODULE__, %{action: :add, socket: socket, topic: topic}) + end + + def remove_socket(topic, socket) do + GenServer.cast(__MODULE__, %{action: :remove, socket: socket, topic: topic}) + end + + def stream(topic, item) do + GenServer.cast(__MODULE__, %{action: :stream, topic: topic, item: item}) + end + + def init(args) do + Process.send_after(self(), %{action: :ping}, @keepalive_interval) + + {:ok, args} + end + + def handle_info(%{action: :ping}, topics) do + topics + |> Map.values() + |> List.flatten() + |> Enum.each(fn socket -> + Logger.debug("Sending keepalive ping") + send(socket.transport_pid, {:text, ""}) + end) + + Process.send_after(self(), %{action: :ping}, @keepalive_interval) + + {:noreply, topics} + end + + def handle_cast(%{action: :stream, topic: "direct", item: item}, topics) do + recipient_topics = + User.get_recipients_from_activity(item) + |> Enum.map(fn %{id: id} -> "direct:#{id}" end) + + Enum.each(recipient_topics || [], fn user_topic -> + Logger.debug("Trying to push direct message to #{user_topic}\n\n") + push_to_socket(topics, user_topic, item) + end) + + {:noreply, topics} + end + + def handle_cast(%{action: :stream, topic: "participation", item: participation}, topics) do + user_topic = "direct:#{participation.user_id}" + Logger.debug("Trying to push a conversation participation to #{user_topic}\n\n") + + push_to_socket(topics, user_topic, participation) + + {:noreply, topics} + end + + def handle_cast(%{action: :stream, topic: "list", item: item}, topics) do + # filter the recipient list if the activity is not public, see #270. + recipient_lists = + case Visibility.is_public?(item) do + true -> + Pleroma.List.get_lists_from_activity(item) + + _ -> + Pleroma.List.get_lists_from_activity(item) + |> Enum.filter(fn list -> + owner = User.get_cached_by_id(list.user_id) + + Visibility.visible_for_user?(item, owner) + end) + end + + recipient_topics = + recipient_lists + |> Enum.map(fn %{id: id} -> "list:#{id}" end) + + Enum.each(recipient_topics || [], fn list_topic -> + Logger.debug("Trying to push message to #{list_topic}\n\n") + push_to_socket(topics, list_topic, item) + end) + + {:noreply, topics} + end + + def handle_cast( + %{action: :stream, topic: topic, item: %Notification{} = item}, + topics + ) + when topic in ["user", "user:notification"] do + topics + |> Map.get("#{topic}:#{item.user_id}", []) + |> Enum.each(fn socket -> + with %User{} = user <- User.get_cached_by_ap_id(socket.assigns[:user].ap_id), + true <- should_send?(user, item) do + send( + socket.transport_pid, + {:text, represent_notification(socket.assigns[:user], item)} + ) + end + end) + + {:noreply, topics} + end + + def handle_cast(%{action: :stream, topic: "user", item: item}, topics) do + Logger.debug("Trying to push to users") + + recipient_topics = + User.get_recipients_from_activity(item) + |> Enum.map(fn %{id: id} -> "user:#{id}" end) + + Enum.each(recipient_topics, fn topic -> + push_to_socket(topics, topic, item) + end) + + {:noreply, topics} + end + + def handle_cast(%{action: :stream, topic: topic, item: item}, topics) do + Logger.debug("Trying to push to #{topic}") + Logger.debug("Pushing item to #{topic}") + push_to_socket(topics, topic, item) + {:noreply, topics} + end + + def handle_cast(%{action: :add, topic: topic, socket: socket}, sockets) do + topic = internal_topic(topic, socket) + sockets_for_topic = sockets[topic] || [] + sockets_for_topic = Enum.uniq([socket | sockets_for_topic]) + sockets = Map.put(sockets, topic, sockets_for_topic) + Logger.debug("Got new conn for #{topic}") + {:noreply, sockets} + end + + def handle_cast(%{action: :remove, topic: topic, socket: socket}, sockets) do + topic = internal_topic(topic, socket) + sockets_for_topic = sockets[topic] || [] + sockets_for_topic = List.delete(sockets_for_topic, socket) + sockets = Map.put(sockets, topic, sockets_for_topic) + Logger.debug("Removed conn for #{topic}") + {:noreply, sockets} + end + + def handle_cast(m, state) do + Logger.info("Unknown: #{inspect(m)}, #{inspect(state)}") + {:noreply, state} + end + + defp represent_update(%Activity{} = activity, %User{} = user) do + %{ + event: "update", + payload: + Pleroma.Web.MastodonAPI.StatusView.render( + "status.json", + activity: activity, + for: user + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + + defp represent_update(%Activity{} = activity) do + %{ + event: "update", + payload: + Pleroma.Web.MastodonAPI.StatusView.render( + "status.json", + activity: activity + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + + def represent_conversation(%Participation{} = participation) do + %{ + event: "conversation", + payload: + Pleroma.Web.MastodonAPI.ConversationView.render("participation.json", %{ + participation: participation, + for: participation.user + }) + |> Jason.encode!() + } + |> Jason.encode!() + end + + @spec represent_notification(User.t(), Notification.t()) :: binary() + defp represent_notification(%User{} = user, %Notification{} = notify) do + %{ + event: "notification", + payload: + NotificationView.render( + "show.json", + %{notification: notify, for: user} + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + + defp should_send?(%User{} = user, %Activity{} = item) do + blocks = user.info.blocks || [] + mutes = user.info.mutes || [] + reblog_mutes = user.info.muted_reblogs || [] + domain_blocks = Pleroma.Web.ActivityPub.MRF.subdomains_regex(user.info.domain_blocks) + + with parent when not is_nil(parent) <- Object.normalize(item), + true <- Enum.all?([blocks, mutes, reblog_mutes], &(item.actor not in &1)), + true <- Enum.all?([blocks, mutes], &(parent.data["actor"] not in &1)), + %{host: item_host} <- URI.parse(item.actor), + %{host: parent_host} <- URI.parse(parent.data["actor"]), + false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, item_host), + false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, parent_host), + true <- thread_containment(item, user), + false <- CommonAPI.thread_muted?(user, item) do + true + else + _ -> false + end + end + + defp should_send?(%User{} = user, %Notification{activity: activity}) do + should_send?(user, activity) + end + + def push_to_socket(topics, topic, %Activity{data: %{"type" => "Announce"}} = item) do + Enum.each(topics[topic] || [], fn socket -> + # Get the current user so we have up-to-date blocks etc. + if socket.assigns[:user] do + user = User.get_cached_by_ap_id(socket.assigns[:user].ap_id) + + if should_send?(user, item) do + send(socket.transport_pid, {:text, represent_update(item, user)}) + end + else + send(socket.transport_pid, {:text, represent_update(item)}) + end + end) + end + + def push_to_socket(topics, topic, %Participation{} = participation) do + Enum.each(topics[topic] || [], fn socket -> + send(socket.transport_pid, {:text, represent_conversation(participation)}) + end) + end + + def push_to_socket(topics, topic, %Activity{ + data: %{"type" => "Delete", "deleted_activity_id" => deleted_activity_id} + }) do + Enum.each(topics[topic] || [], fn socket -> + send( + socket.transport_pid, + {:text, %{event: "delete", payload: to_string(deleted_activity_id)} |> Jason.encode!()} + ) + end) + end + + def push_to_socket(_topics, _topic, %Activity{data: %{"type" => "Delete"}}), do: :noop + + def push_to_socket(topics, topic, item) do + Enum.each(topics[topic] || [], fn socket -> + # Get the current user so we have up-to-date blocks etc. + if socket.assigns[:user] do + user = User.get_cached_by_ap_id(socket.assigns[:user].ap_id) + blocks = user.info.blocks || [] + mutes = user.info.mutes || [] + + with true <- Enum.all?([blocks, mutes], &(item.actor not in &1)), + true <- thread_containment(item, user) do + send(socket.transport_pid, {:text, represent_update(item, user)}) + end + else + send(socket.transport_pid, {:text, represent_update(item)}) + end + end) + end + + defp internal_topic(topic, socket) when topic in ~w[user user:notification direct] do + "#{topic}:#{socket.assigns[:user].id}" + end + + defp internal_topic(topic, _), do: topic + + @spec thread_containment(Activity.t(), User.t()) :: boolean() + defp thread_containment(_activity, %User{info: %{skip_thread_containment: true}}), do: true + + defp thread_containment(activity, user) do + if Config.get([:instance, :skip_thread_containment]) do + true + else + ActivityPub.contain_activity(activity, user) + end + end +end diff --git a/lib/pleroma/web/streamer/ping.ex b/lib/pleroma/web/streamer/ping.ex deleted file mode 100644 index f77cbb95..00000000 --- a/lib/pleroma/web/streamer/ping.ex +++ /dev/null @@ -1,33 +0,0 @@ -defmodule Pleroma.Web.Streamer.Ping do - use GenServer - require Logger - - alias Pleroma.Web.Streamer.State - alias Pleroma.Web.Streamer.StreamerSocket - - @keepalive_interval :timer.seconds(30) - - def start_link(opts) do - ping_interval = Keyword.get(opts, :ping_interval, @keepalive_interval) - GenServer.start_link(__MODULE__, %{ping_interval: ping_interval}, name: __MODULE__) - end - - def init(%{ping_interval: ping_interval} = args) do - Process.send_after(self(), :ping, ping_interval) - {:ok, args} - end - - def handle_info(:ping, %{ping_interval: ping_interval} = state) do - State.get_sockets() - |> Map.values() - |> List.flatten() - |> Enum.each(fn %StreamerSocket{transport_pid: transport_pid} -> - Logger.debug("Sending keepalive ping") - send(transport_pid, {:text, ""}) - end) - - Process.send_after(self(), :ping, ping_interval) - - {:noreply, state} - end -end diff --git a/lib/pleroma/web/streamer/state.ex b/lib/pleroma/web/streamer/state.ex deleted file mode 100644 index 7b519906..00000000 --- a/lib/pleroma/web/streamer/state.ex +++ /dev/null @@ -1,68 +0,0 @@ -defmodule Pleroma.Web.Streamer.State do - use GenServer - require Logger - - alias Pleroma.Web.Streamer.StreamerSocket - - def start_link(_) do - GenServer.start_link(__MODULE__, %{sockets: %{}}, name: __MODULE__) - end - - def add_socket(topic, socket) do - GenServer.call(__MODULE__, {:add, socket, topic}) - end - - def remove_socket(topic, socket) do - GenServer.call(__MODULE__, {:remove, socket, topic}) - end - - def get_sockets do - %{sockets: stream_sockets} = GenServer.call(__MODULE__, :get_state) - stream_sockets - end - - def init(init_arg) do - {:ok, init_arg} - end - - def handle_call(:get_state, _from, state) do - {:reply, state, state} - end - - def handle_call({:add, socket, topic}, _from, %{sockets: sockets} = state) do - internal_topic = internal_topic(topic, socket) - stream_socket = StreamerSocket.from_socket(socket) - - sockets_for_topic = - sockets - |> Map.get(internal_topic, []) - |> List.insert_at(0, stream_socket) - |> Enum.uniq() - - state = put_in(state, [:sockets, internal_topic], sockets_for_topic) - Logger.debug("Got new conn for #{topic}") - {:reply, state, state} - end - - def handle_call({:remove, socket, topic}, _from, %{sockets: sockets} = state) do - internal_topic = internal_topic(topic, socket) - stream_socket = StreamerSocket.from_socket(socket) - - sockets_for_topic = - sockets - |> Map.get(internal_topic, []) - |> List.delete(stream_socket) - - state = Kernel.put_in(state, [:sockets, internal_topic], sockets_for_topic) - {:reply, state, state} - end - - defp internal_topic(topic, socket) - when topic in ~w[user user:notification direct] do - "#{topic}:#{socket.assigns[:user].id}" - end - - defp internal_topic(topic, _) do - topic - end -end diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex deleted file mode 100644 index 8cf71927..00000000 --- a/lib/pleroma/web/streamer/streamer.ex +++ /dev/null @@ -1,55 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.Streamer do - alias Pleroma.Web.Streamer.State - alias Pleroma.Web.Streamer.Worker - - @timeout 60_000 - @mix_env Mix.env() - - def add_socket(topic, socket) do - State.add_socket(topic, socket) - end - - def remove_socket(topic, socket) do - State.remove_socket(topic, socket) - end - - def get_sockets do - State.get_sockets() - end - - def stream(topics, items) do - if should_send?() do - Task.async(fn -> - :poolboy.transaction( - :streamer_worker, - &Worker.stream(&1, topics, items), - @timeout - ) - end) - end - end - - def supervisor, do: Pleroma.Web.Streamer.Supervisor - - defp should_send? do - handle_should_send(@mix_env) - end - - defp handle_should_send(:test) do - case Process.whereis(:streamer_worker) do - nil -> - false - - pid -> - Process.alive?(pid) - end - end - - defp handle_should_send(_) do - true - end -end diff --git a/lib/pleroma/web/streamer/streamer_socket.ex b/lib/pleroma/web/streamer/streamer_socket.ex deleted file mode 100644 index f006c030..00000000 --- a/lib/pleroma/web/streamer/streamer_socket.ex +++ /dev/null @@ -1,31 +0,0 @@ -defmodule Pleroma.Web.Streamer.StreamerSocket do - defstruct transport_pid: nil, user: nil - - alias Pleroma.User - alias Pleroma.Web.Streamer.StreamerSocket - - def from_socket(%{ - transport_pid: transport_pid, - assigns: %{user: nil} - }) do - %StreamerSocket{ - transport_pid: transport_pid - } - end - - def from_socket(%{ - transport_pid: transport_pid, - assigns: %{user: %User{} = user} - }) do - %StreamerSocket{ - transport_pid: transport_pid, - user: user - } - end - - def from_socket(%{transport_pid: transport_pid}) do - %StreamerSocket{ - transport_pid: transport_pid - } - end -end diff --git a/lib/pleroma/web/streamer/supervisor.ex b/lib/pleroma/web/streamer/supervisor.ex deleted file mode 100644 index 6afe1932..00000000 --- a/lib/pleroma/web/streamer/supervisor.ex +++ /dev/null @@ -1,33 +0,0 @@ -defmodule Pleroma.Web.Streamer.Supervisor do - use Supervisor - - def start_link(opts) do - Supervisor.start_link(__MODULE__, opts, name: __MODULE__) - end - - def init(args) do - children = [ - {Pleroma.Web.Streamer.State, args}, - {Pleroma.Web.Streamer.Ping, args}, - :poolboy.child_spec(:streamer_worker, poolboy_config()) - ] - - opts = [strategy: :one_for_one, name: Pleroma.Web.Streamer.Supervisor] - Supervisor.init(children, opts) - end - - defp poolboy_config do - opts = - Pleroma.Config.get(:streamer, - workers: 3, - overflow_workers: 2 - ) - - [ - {:name, {:local, :streamer_worker}}, - {:worker_module, Pleroma.Web.Streamer.Worker}, - {:size, opts[:workers]}, - {:max_overflow, opts[:overflow_workers]} - ] - end -end diff --git a/lib/pleroma/web/streamer/worker.ex b/lib/pleroma/web/streamer/worker.ex deleted file mode 100644 index 5804508e..00000000 --- a/lib/pleroma/web/streamer/worker.ex +++ /dev/null @@ -1,220 +0,0 @@ -defmodule Pleroma.Web.Streamer.Worker do - use GenServer - - require Logger - - alias Pleroma.Activity - alias Pleroma.Config - alias Pleroma.Conversation.Participation - alias Pleroma.Notification - alias Pleroma.Object - alias Pleroma.User - alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.ActivityPub.Visibility - alias Pleroma.Web.CommonAPI - alias Pleroma.Web.Streamer.State - alias Pleroma.Web.Streamer.StreamerSocket - alias Pleroma.Web.StreamerView - - def start_link(_) do - GenServer.start_link(__MODULE__, %{}, []) - end - - def init(init_arg) do - {:ok, init_arg} - end - - def stream(pid, topics, items) do - GenServer.call(pid, {:stream, topics, items}) - end - - def handle_call({:stream, topics, item}, _from, state) when is_list(topics) do - Enum.each(topics, fn t -> - do_stream(%{topic: t, item: item}) - end) - - {:reply, state, state} - end - - def handle_call({:stream, topic, items}, _from, state) when is_list(items) do - Enum.each(items, fn i -> - do_stream(%{topic: topic, item: i}) - end) - - {:reply, state, state} - end - - def handle_call({:stream, topic, item}, _from, state) do - do_stream(%{topic: topic, item: item}) - - {:reply, state, state} - end - - defp do_stream(%{topic: "direct", item: item}) do - recipient_topics = - User.get_recipients_from_activity(item) - |> Enum.map(fn %{id: id} -> "direct:#{id}" end) - - Enum.each(recipient_topics, fn user_topic -> - Logger.debug("Trying to push direct message to #{user_topic}\n\n") - push_to_socket(State.get_sockets(), user_topic, item) - end) - end - - defp do_stream(%{topic: "participation", item: participation}) do - user_topic = "direct:#{participation.user_id}" - Logger.debug("Trying to push a conversation participation to #{user_topic}\n\n") - - push_to_socket(State.get_sockets(), user_topic, participation) - end - - defp do_stream(%{topic: "list", item: item}) do - # filter the recipient list if the activity is not public, see #270. - recipient_lists = - case Visibility.is_public?(item) do - true -> - Pleroma.List.get_lists_from_activity(item) - - _ -> - Pleroma.List.get_lists_from_activity(item) - |> Enum.filter(fn list -> - owner = User.get_cached_by_id(list.user_id) - - Visibility.visible_for_user?(item, owner) - end) - end - - recipient_topics = - recipient_lists - |> Enum.map(fn %{id: id} -> "list:#{id}" end) - - Enum.each(recipient_topics, fn list_topic -> - Logger.debug("Trying to push message to #{list_topic}\n\n") - push_to_socket(State.get_sockets(), list_topic, item) - end) - end - - defp do_stream(%{topic: topic, item: %Notification{} = item}) - when topic in ["user", "user:notification"] do - State.get_sockets() - |> Map.get("#{topic}:#{item.user_id}", []) - |> Enum.each(fn %StreamerSocket{transport_pid: transport_pid, user: socket_user} -> - with %User{} = user <- User.get_cached_by_ap_id(socket_user.ap_id), - true <- should_send?(user, item) do - send(transport_pid, {:text, StreamerView.render("notification.json", socket_user, item)}) - end - end) - end - - defp do_stream(%{topic: "user", item: item}) do - Logger.debug("Trying to push to users") - - recipient_topics = - User.get_recipients_from_activity(item) - |> Enum.map(fn %{id: id} -> "user:#{id}" end) - - Enum.each(recipient_topics, fn topic -> - push_to_socket(State.get_sockets(), topic, item) - end) - end - - defp do_stream(%{topic: topic, item: item}) do - Logger.debug("Trying to push to #{topic}") - Logger.debug("Pushing item to #{topic}") - push_to_socket(State.get_sockets(), topic, item) - end - - defp should_send?(%User{} = user, %Activity{} = item) do - blocks = user.info.blocks || [] - mutes = user.info.mutes || [] - reblog_mutes = user.info.muted_reblogs || [] - domain_blocks = Pleroma.Web.ActivityPub.MRF.subdomains_regex(user.info.domain_blocks) - - with parent when not is_nil(parent) <- Object.normalize(item), - true <- Enum.all?([blocks, mutes, reblog_mutes], &(item.actor not in &1)), - true <- Enum.all?([blocks, mutes], &(parent.data["actor"] not in &1)), - %{host: item_host} <- URI.parse(item.actor), - %{host: parent_host} <- URI.parse(parent.data["actor"]), - false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, item_host), - false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, parent_host), - true <- thread_containment(item, user), - false <- CommonAPI.thread_muted?(user, item) do - true - else - _ -> false - end - end - - defp should_send?(%User{} = user, %Notification{activity: activity}) do - should_send?(user, activity) - end - - def push_to_socket(topics, topic, %Activity{data: %{"type" => "Announce"}} = item) do - Enum.each(topics[topic] || [], fn %StreamerSocket{ - transport_pid: transport_pid, - user: socket_user - } -> - # Get the current user so we have up-to-date blocks etc. - if socket_user do - user = User.get_cached_by_ap_id(socket_user.ap_id) - - if should_send?(user, item) do - send(transport_pid, {:text, StreamerView.render("update.json", item, user)}) - end - else - send(transport_pid, {:text, StreamerView.render("update.json", item)}) - end - end) - end - - def push_to_socket(topics, topic, %Participation{} = participation) do - Enum.each(topics[topic] || [], fn %StreamerSocket{transport_pid: transport_pid} -> - send(transport_pid, {:text, StreamerView.render("conversation.json", participation)}) - end) - end - - def push_to_socket(topics, topic, %Activity{ - data: %{"type" => "Delete", "deleted_activity_id" => deleted_activity_id} - }) do - Enum.each(topics[topic] || [], fn %StreamerSocket{transport_pid: transport_pid} -> - send( - transport_pid, - {:text, %{event: "delete", payload: to_string(deleted_activity_id)} |> Jason.encode!()} - ) - end) - end - - def push_to_socket(_topics, _topic, %Activity{data: %{"type" => "Delete"}}), do: :noop - - def push_to_socket(topics, topic, item) do - Enum.each(topics[topic] || [], fn %StreamerSocket{ - transport_pid: transport_pid, - user: socket_user - } -> - # Get the current user so we have up-to-date blocks etc. - if socket_user do - user = User.get_cached_by_ap_id(socket_user.ap_id) - blocks = user.info.blocks || [] - mutes = user.info.mutes || [] - - with true <- Enum.all?([blocks, mutes], &(item.actor not in &1)), - true <- thread_containment(item, user) do - send(transport_pid, {:text, StreamerView.render("update.json", item, user)}) - end - else - send(transport_pid, {:text, StreamerView.render("update.json", item)}) - end - end) - end - - @spec thread_containment(Activity.t(), User.t()) :: boolean() - defp thread_containment(_activity, %User{info: %{skip_thread_containment: true}}), do: true - - defp thread_containment(activity, user) do - if Config.get([:instance, :skip_thread_containment]) do - true - else - ActivityPub.contain_activity(activity, user) - end - end -end diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex deleted file mode 100644 index b13030fa..00000000 --- a/lib/pleroma/web/views/streamer_view.ex +++ /dev/null @@ -1,66 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.StreamerView do - use Pleroma.Web, :view - - alias Pleroma.Activity - alias Pleroma.Conversation.Participation - alias Pleroma.Notification - alias Pleroma.User - alias Pleroma.Web.MastodonAPI.NotificationView - - def render("update.json", %Activity{} = activity, %User{} = user) do - %{ - event: "update", - payload: - Pleroma.Web.MastodonAPI.StatusView.render( - "status.json", - activity: activity, - for: user - ) - |> Jason.encode!() - } - |> Jason.encode!() - end - - def render("notification.json", %User{} = user, %Notification{} = notify) do - %{ - event: "notification", - payload: - NotificationView.render( - "show.json", - %{notification: notify, for: user} - ) - |> Jason.encode!() - } - |> Jason.encode!() - end - - def render("update.json", %Activity{} = activity) do - %{ - event: "update", - payload: - Pleroma.Web.MastodonAPI.StatusView.render( - "status.json", - activity: activity - ) - |> Jason.encode!() - } - |> Jason.encode!() - end - - def render("conversation.json", %Participation{} = participation) do - %{ - event: "conversation", - payload: - Pleroma.Web.MastodonAPI.ConversationView.render("participation.json", %{ - participation: participation, - for: participation.user - }) - |> Jason.encode!() - } - |> Jason.encode!() - end -end diff --git a/mix.exs b/mix.exs index 911ebad1..f1e98585 100644 --- a/mix.exs +++ b/mix.exs @@ -144,7 +144,6 @@ defmodule Pleroma.Mixfile do git: "https://git.pleroma.social/pleroma/http_signatures.git", ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"}, {:telemetry, "~> 0.3"}, - {:poolboy, "~> 1.5"}, {:prometheus_ex, "~> 3.0"}, {:prometheus_plugs, "~> 1.1"}, {:prometheus_phoenix, "~> 1.3"}, diff --git a/mix.lock b/mix.lock index 0bf6a811..41697dd5 100644 --- a/mix.lock +++ b/mix.lock @@ -73,7 +73,6 @@ "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"}, "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, - "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, "postgrex": {:hex, :postgrex, "0.14.3", "5754dee2fdf6e9e508cbf49ab138df964278700b764177e8f3871e658b345a1e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, "prometheus": {:hex, :prometheus, "4.4.1", "1e96073b3ed7788053768fea779cbc896ddc3bdd9ba60687f2ad50b252ac87d6", [:mix, :rebar3], [], "hexpm"}, "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/activity/ir/topics_test.exs b/test/activity/ir/topics_test.exs deleted file mode 100644 index e75f8358..00000000 --- a/test/activity/ir/topics_test.exs +++ /dev/null @@ -1,141 +0,0 @@ -defmodule Pleroma.Activity.Ir.TopicsTest do - use Pleroma.DataCase - - alias Pleroma.Activity - alias Pleroma.Activity.Ir.Topics - alias Pleroma.Object - - require Pleroma.Constants - - describe "poll answer" do - test "produce no topics" do - activity = %Activity{object: %Object{data: %{"type" => "Answer"}}} - - assert [] == Topics.get_activity_topics(activity) - end - end - - describe "non poll answer" do - test "always add user and list topics" do - activity = %Activity{object: %Object{data: %{"type" => "FooBar"}}} - topics = Topics.get_activity_topics(activity) - - assert Enum.member?(topics, "user") - assert Enum.member?(topics, "list") - end - end - - describe "public visibility" do - setup do - activity = %Activity{ - object: %Object{data: %{"type" => "Note"}}, - data: %{"to" => [Pleroma.Constants.as_public()]} - } - - {:ok, activity: activity} - end - - test "produces public topic", %{activity: activity} do - topics = Topics.get_activity_topics(activity) - - assert Enum.member?(topics, "public") - end - - test "local action produces public:local topic", %{activity: activity} do - activity = %{activity | local: true} - topics = Topics.get_activity_topics(activity) - - assert Enum.member?(topics, "public:local") - end - - test "non-local action does not produce public:local topic", %{activity: activity} do - activity = %{activity | local: false} - topics = Topics.get_activity_topics(activity) - - refute Enum.member?(topics, "public:local") - end - end - - describe "public visibility create events" do - setup do - activity = %Activity{ - object: %Object{data: %{"type" => "Create", "attachment" => []}}, - data: %{"to" => [Pleroma.Constants.as_public()]} - } - - {:ok, activity: activity} - end - - test "with no attachments doesn't produce public:media topics", %{activity: activity} do - topics = Topics.get_activity_topics(activity) - - refute Enum.member?(topics, "public:media") - refute Enum.member?(topics, "public:local:media") - end - - test "converts tags to hash tags", %{activity: %{object: %{data: data} = object} = activity} do - tagged_data = Map.put(data, "tag", ["foo", "bar"]) - activity = %{activity | object: %{object | data: tagged_data}} - - topics = Topics.get_activity_topics(activity) - - assert Enum.member?(topics, "hashtag:foo") - assert Enum.member?(topics, "hashtag:bar") - end - - test "only converts strinngs to hash tags", %{ - activity: %{object: %{data: data} = object} = activity - } do - tagged_data = Map.put(data, "tag", [2]) - activity = %{activity | object: %{object | data: tagged_data}} - - topics = Topics.get_activity_topics(activity) - - refute Enum.member?(topics, "hashtag:2") - end - end - - describe "public visibility create events with attachments" do - setup do - activity = %Activity{ - object: %Object{data: %{"type" => "Create", "attachment" => ["foo"]}}, - data: %{"to" => [Pleroma.Constants.as_public()]} - } - - {:ok, activity: activity} - end - - test "produce public:media topics", %{activity: activity} do - topics = Topics.get_activity_topics(activity) - - assert Enum.member?(topics, "public:media") - end - - test "local produces public:local:media topics", %{activity: activity} do - topics = Topics.get_activity_topics(activity) - - assert Enum.member?(topics, "public:local:media") - end - - test "non-local doesn't produce public:local:media topics", %{activity: activity} do - activity = %{activity | local: false} - - topics = Topics.get_activity_topics(activity) - - refute Enum.member?(topics, "public:local:media") - end - end - - describe "non-public visibility" do - test "produces direct topic" do - activity = %Activity{object: %Object{data: %{"type" => "Note"}}, data: %{"to" => []}} - topics = Topics.get_activity_topics(activity) - - assert Enum.member?(topics, "direct") - refute Enum.member?(topics, "public") - refute Enum.member?(topics, "public:local") - refute Enum.member?(topics, "public:media") - refute Enum.member?(topics, "public:local:media") - end - end -end diff --git a/test/integration/mastodon_websocket_test.exs b/test/integration/mastodon_websocket_test.exs index c0426280..63bf7341 100644 --- a/test/integration/mastodon_websocket_test.exs +++ b/test/integration/mastodon_websocket_test.exs @@ -11,6 +11,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do alias Pleroma.Integration.WebsocketClient alias Pleroma.Web.CommonAPI alias Pleroma.Web.OAuth + alias Pleroma.Web.Streamer @path Pleroma.Web.Endpoint.url() |> URI.parse() @@ -18,6 +19,16 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do |> Map.put(:path, "/api/v1/streaming") |> URI.to_string() + setup do + GenServer.start(Streamer, %{}, name: Streamer) + + on_exit(fn -> + if pid = Process.whereis(Streamer) do + Process.exit(pid, :kill) + end + end) + end + def start_socket(qs \\ nil, headers \\ []) do path = case qs do @@ -42,14 +53,12 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do end) end - @tag needs_streamer: true test "allows public streams without authentication" do assert {:ok, _} = start_socket("?stream=public") assert {:ok, _} = start_socket("?stream=public:local") assert {:ok, _} = start_socket("?stream=hashtag&tag=lain") end - @tag needs_streamer: true test "receives well formatted events" do user = insert(:user) {:ok, _} = start_socket("?stream=public") @@ -94,7 +103,6 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert {:ok, _} = start_socket("?stream=user&access_token=#{state.token.token}") end - @tag needs_streamer: true test "accepts the 'user' stream", %{token: token} = _state do assert {:ok, _} = start_socket("?stream=user&access_token=#{token.token}") @@ -103,7 +111,6 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do end) =~ ":badarg" end - @tag needs_streamer: true test "accepts the 'user:notification' stream", %{token: token} = _state do assert {:ok, _} = start_socket("?stream=user:notification&access_token=#{token.token}") @@ -112,7 +119,6 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do end) =~ ":badarg" end - @tag needs_streamer: true test "accepts valid token on Sec-WebSocket-Protocol header", %{token: token} do assert {:ok, _} = start_socket("?stream=user", [{"Sec-WebSocket-Protocol", token.token}]) diff --git a/test/notification_test.exs b/test/notification_test.exs index 3d2f9a8f..3be9db09 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -69,7 +69,16 @@ defmodule Pleroma.NotificationTest do end describe "create_notification" do - @tag needs_streamer: true + setup do + GenServer.start(Streamer, %{}, name: Streamer) + + on_exit(fn -> + if pid = Process.whereis(Streamer) do + Process.exit(pid, :kill) + end + end) + end + test "it creates a notification for user and send to the 'user' and the 'user:notification' stream" do user = insert(:user) task = Task.async(fn -> assert_receive {:text, _}, 4_000 end) diff --git a/test/support/conn_case.ex b/test/support/conn_case.ex index b39c7067..ec5892ff 100644 --- a/test/support/conn_case.ex +++ b/test/support/conn_case.ex @@ -40,10 +40,6 @@ defmodule Pleroma.Web.ConnCase do Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, {:shared, self()}) end - if tags[:needs_streamer] do - start_supervised(Pleroma.Web.Streamer.supervisor()) - end - {:ok, conn: Phoenix.ConnTest.build_conn()} end end diff --git a/test/support/data_case.ex b/test/support/data_case.ex index 17fa1521..f3d98e7e 100644 --- a/test/support/data_case.ex +++ b/test/support/data_case.ex @@ -39,10 +39,6 @@ defmodule Pleroma.DataCase do Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, {:shared, self()}) end - if tags[:needs_streamer] do - start_supervised(Pleroma.Web.Streamer.supervisor()) - end - :ok end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 4100108a..d0118fef 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -38,7 +38,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do stream: fn _, _ -> nil end do ActivityPub.stream_out_participations(conversation.participations) - assert called(Pleroma.Web.Streamer.stream("participation", participations)) + Enum.each(participations, fn participation -> + assert called(Pleroma.Web.Streamer.stream("participation", participation)) + end) end end end diff --git a/test/web/streamer/ping_test.exs b/test/web/streamer/ping_test.exs deleted file mode 100644 index 3d52c00e..00000000 --- a/test/web/streamer/ping_test.exs +++ /dev/null @@ -1,36 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.PingTest do - use Pleroma.DataCase - - import Pleroma.Factory - alias Pleroma.Web.Streamer - - setup do - start_supervised({Streamer.supervisor(), [ping_interval: 30]}) - - :ok - end - - describe "sockets" do - setup do - user = insert(:user) - {:ok, %{user: user}} - end - - test "it sends pings", %{user: user} do - task = - Task.async(fn -> - assert_receive {:text, received_event}, 40 - assert_receive {:text, received_event}, 40 - assert_receive {:text, received_event}, 40 - end) - - Streamer.add_socket("public", %{transport_pid: task.pid, assigns: %{user: user}}) - - Task.await(task) - end - end -end diff --git a/test/web/streamer/state_test.exs b/test/web/streamer/state_test.exs deleted file mode 100644 index d1aeac54..00000000 --- a/test/web/streamer/state_test.exs +++ /dev/null @@ -1,54 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.StateTest do - use Pleroma.DataCase - - import Pleroma.Factory - alias Pleroma.Web.Streamer - alias Pleroma.Web.Streamer.StreamerSocket - - @moduletag needs_streamer: true - - describe "sockets" do - setup do - user = insert(:user) - user2 = insert(:user) - {:ok, %{user: user, user2: user2}} - end - - test "it can add a socket", %{user: user} do - Streamer.add_socket("public", %{transport_pid: 1, assigns: %{user: user}}) - - assert(%{"public" => [%StreamerSocket{transport_pid: 1}]} = Streamer.get_sockets()) - end - - test "it can add multiple sockets per user", %{user: user} do - Streamer.add_socket("public", %{transport_pid: 1, assigns: %{user: user}}) - Streamer.add_socket("public", %{transport_pid: 2, assigns: %{user: user}}) - - assert( - %{ - "public" => [ - %StreamerSocket{transport_pid: 2}, - %StreamerSocket{transport_pid: 1} - ] - } = Streamer.get_sockets() - ) - end - - test "it will not add a duplicate socket", %{user: user} do - Streamer.add_socket("activity", %{transport_pid: 1, assigns: %{user: user}}) - Streamer.add_socket("activity", %{transport_pid: 1, assigns: %{user: user}}) - - assert( - %{ - "activity" => [ - %StreamerSocket{transport_pid: 1} - ] - } = Streamer.get_sockets() - ) - end - end -end diff --git a/test/web/streamer/streamer_test.exs b/test/web/streamer_test.exs similarity index 86% rename from test/web/streamer/streamer_test.exs rename to test/web/streamer_test.exs index 88847e20..96fa7645 100644 --- a/test/web/streamer/streamer_test.exs +++ b/test/web/streamer_test.exs @@ -5,20 +5,24 @@ defmodule Pleroma.Web.StreamerTest do use Pleroma.DataCase - import Pleroma.Factory - alias Pleroma.List alias Pleroma.User alias Pleroma.Web.CommonAPI alias Pleroma.Web.Streamer - alias Pleroma.Web.Streamer.StreamerSocket - alias Pleroma.Web.Streamer.Worker + import Pleroma.Factory - @moduletag needs_streamer: true clear_config_all([:instance, :skip_thread_containment]) describe "user streams" do setup do + GenServer.start(Streamer, %{}, name: Streamer) + + on_exit(fn -> + if pid = Process.whereis(Streamer) do + Process.exit(pid, :kill) + end + end) + user = insert(:user) notify = insert(:notification, user: user, activity: build(:note_activity)) {:ok, %{user: user, notify: notify}} @@ -121,9 +125,11 @@ defmodule Pleroma.Web.StreamerTest do assert_receive {:text, _}, 4_000 end) - fake_socket = %StreamerSocket{ + fake_socket = %{ transport_pid: task.pid, - user: user + assigns: %{ + user: user + } } {:ok, activity} = CommonAPI.post(other_user, %{"status" => "Test"}) @@ -132,7 +138,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Worker.push_to_socket(topics, "public", activity) + Streamer.push_to_socket(topics, "public", activity) Task.await(task) @@ -149,9 +155,11 @@ defmodule Pleroma.Web.StreamerTest do assert received_event == expected_event end) - fake_socket = %StreamerSocket{ + fake_socket = %{ transport_pid: task.pid, - user: user + assigns: %{ + user: user + } } {:ok, activity} = CommonAPI.delete(activity.id, other_user) @@ -160,7 +168,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Worker.push_to_socket(topics, "public", activity) + Streamer.push_to_socket(topics, "public", activity) Task.await(task) end @@ -181,9 +189,9 @@ defmodule Pleroma.Web.StreamerTest do ) task = Task.async(fn -> refute_receive {:text, _}, 1_000 end) - fake_socket = %StreamerSocket{transport_pid: task.pid, user: user} + fake_socket = %{transport_pid: task.pid, assigns: %{user: user}} topics = %{"public" => [fake_socket]} - Worker.push_to_socket(topics, "public", activity) + Streamer.push_to_socket(topics, "public", activity) Task.await(task) end @@ -203,9 +211,9 @@ defmodule Pleroma.Web.StreamerTest do ) task = Task.async(fn -> assert_receive {:text, _}, 1_000 end) - fake_socket = %StreamerSocket{transport_pid: task.pid, user: user} + fake_socket = %{transport_pid: task.pid, assigns: %{user: user}} topics = %{"public" => [fake_socket]} - Worker.push_to_socket(topics, "public", activity) + Streamer.push_to_socket(topics, "public", activity) Task.await(task) end @@ -225,9 +233,9 @@ defmodule Pleroma.Web.StreamerTest do ) task = Task.async(fn -> assert_receive {:text, _}, 1_000 end) - fake_socket = %StreamerSocket{transport_pid: task.pid, user: user} + fake_socket = %{transport_pid: task.pid, assigns: %{user: user}} topics = %{"public" => [fake_socket]} - Worker.push_to_socket(topics, "public", activity) + Streamer.push_to_socket(topics, "public", activity) Task.await(task) end @@ -243,9 +251,11 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %StreamerSocket{ + fake_socket = %{ transport_pid: task.pid, - user: user + assigns: %{ + user: user + } } {:ok, activity} = CommonAPI.post(blocked_user, %{"status" => "Test"}) @@ -254,7 +264,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Worker.push_to_socket(topics, "public", activity) + Streamer.push_to_socket(topics, "public", activity) Task.await(task) end @@ -274,9 +284,11 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %StreamerSocket{ + fake_socket = %{ transport_pid: task.pid, - user: user_a + assigns: %{ + user: user_a + } } {:ok, activity} = @@ -289,7 +301,7 @@ defmodule Pleroma.Web.StreamerTest do "list:#{list.id}" => [fake_socket] } - Worker.handle_call({:stream, "list", activity}, self(), topics) + Streamer.handle_cast(%{action: :stream, topic: "list", item: activity}, topics) Task.await(task) end @@ -306,9 +318,11 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %StreamerSocket{ + fake_socket = %{ transport_pid: task.pid, - user: user_a + assigns: %{ + user: user_a + } } {:ok, activity} = @@ -321,12 +335,12 @@ defmodule Pleroma.Web.StreamerTest do "list:#{list.id}" => [fake_socket] } - Worker.handle_call({:stream, "list", activity}, self(), topics) + Streamer.handle_cast(%{action: :stream, topic: "list", item: activity}, topics) Task.await(task) end - test "it sends wanted private posts to list" do + test "it send wanted private posts to list" do user_a = insert(:user) user_b = insert(:user) @@ -340,9 +354,11 @@ defmodule Pleroma.Web.StreamerTest do assert_receive {:text, _}, 1_000 end) - fake_socket = %StreamerSocket{ + fake_socket = %{ transport_pid: task.pid, - user: user_a + assigns: %{ + user: user_a + } } {:ok, activity} = @@ -351,12 +367,11 @@ defmodule Pleroma.Web.StreamerTest do "visibility" => "private" }) - Streamer.add_socket( - "list:#{list.id}", - fake_socket - ) + topics = %{ + "list:#{list.id}" => [fake_socket] + } - Worker.handle_call({:stream, "list", activity}, self(), %{}) + Streamer.handle_cast(%{action: :stream, topic: "list", item: activity}, topics) Task.await(task) end @@ -372,9 +387,11 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %StreamerSocket{ + fake_socket = %{ transport_pid: task.pid, - user: user1 + assigns: %{ + user: user1 + } } {:ok, create_activity} = CommonAPI.post(user3, %{"status" => "I'm kawen"}) @@ -384,7 +401,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Worker.push_to_socket(topics, "public", announce_activity) + Streamer.push_to_socket(topics, "public", announce_activity) Task.await(task) end @@ -400,8 +417,6 @@ defmodule Pleroma.Web.StreamerTest do task = Task.async(fn -> refute_receive {:text, _}, 4_000 end) - Process.sleep(4000) - Streamer.add_socket( "user", %{transport_pid: task.pid, assigns: %{user: user2}} @@ -413,6 +428,14 @@ defmodule Pleroma.Web.StreamerTest do describe "direct streams" do setup do + GenServer.start(Streamer, %{}, name: Streamer) + + on_exit(fn -> + if pid = Process.whereis(Streamer) do + Process.exit(pid, :kill) + end + end) + :ok end @@ -457,8 +480,6 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 4_000 end) - Process.sleep(1000) - Streamer.add_socket( "direct", %{transport_pid: task.pid, assigns: %{user: user}} @@ -500,8 +521,6 @@ defmodule Pleroma.Web.StreamerTest do assert last_status["id"] == to_string(create_activity.id) end) - Process.sleep(1000) - Streamer.add_socket( "direct", %{transport_pid: task.pid, assigns: %{user: user}} From e8120944d8c016a1aa8fcefe34b1f0cc9089ea4f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 16 Sep 2019 13:23:06 +0300 Subject: [PATCH 093/138] Fix signed fetch inclusion publisher test Oban branch changed `actor` to `actor_id` and this test was not adjusted for that --- test/web/activity_pub/publisher_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/web/activity_pub/publisher_test.exs b/test/web/activity_pub/publisher_test.exs index c7d1d05a..df03b400 100644 --- a/test/web/activity_pub/publisher_test.exs +++ b/test/web/activity_pub/publisher_test.exs @@ -321,7 +321,7 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do assert called( Pleroma.Web.Federator.Publisher.enqueue_one(Publisher, %{ inbox: "https://domain.com/users/nick1/inbox", - actor: actor, + actor_id: actor.id, id: delete.data["id"] }) ) @@ -329,7 +329,7 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do assert called( Pleroma.Web.Federator.Publisher.enqueue_one(Publisher, %{ inbox: "https://domain2.com/users/nick1/inbox", - actor: actor, + actor_id: actor.id, id: delete.data["id"] }) ) From 96816ceaa25c21cec7677e75dcddd7ffb42d83c3 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 16 Sep 2019 17:03:37 +0700 Subject: [PATCH 094/138] Revert "Merge branch 'revert-4fabf83a' into 'develop'" This reverts commit fe7fd331263007e0fb2877ef7370a09a9704da36, reversing changes made to 4fabf83ad01352442906d79187aeab4c777f4df8. --- .gitignore | 4 + config/config.exs | 4 + lib/pleroma/activity/ir/topics.ex | 63 ++++ lib/pleroma/application.ex | 2 +- lib/pleroma/notification.ex | 6 +- lib/pleroma/web/activity_pub/activity_pub.ex | 48 +-- .../web/mastodon_api/websocket_handler.ex | 7 +- lib/pleroma/web/streamer.ex | 318 ------------------ lib/pleroma/web/streamer/ping.ex | 33 ++ lib/pleroma/web/streamer/state.ex | 68 ++++ lib/pleroma/web/streamer/streamer.ex | 55 +++ lib/pleroma/web/streamer/streamer_socket.ex | 31 ++ lib/pleroma/web/streamer/supervisor.ex | 33 ++ lib/pleroma/web/streamer/worker.ex | 220 ++++++++++++ lib/pleroma/web/views/streamer_view.ex | 66 ++++ mix.exs | 1 + mix.lock | 1 + test/activity/ir/topics_test.exs | 141 ++++++++ test/integration/mastodon_websocket_test.exs | 16 +- test/notification_test.exs | 11 +- test/support/conn_case.ex | 4 + test/support/data_case.ex | 4 + test/web/activity_pub/activity_pub_test.exs | 4 +- test/web/streamer/ping_test.exs | 36 ++ test/web/streamer/state_test.exs | 54 +++ test/web/{ => streamer}/streamer_test.exs | 105 +++--- 26 files changed, 888 insertions(+), 447 deletions(-) create mode 100644 lib/pleroma/activity/ir/topics.ex delete mode 100644 lib/pleroma/web/streamer.ex create mode 100644 lib/pleroma/web/streamer/ping.ex create mode 100644 lib/pleroma/web/streamer/state.ex create mode 100644 lib/pleroma/web/streamer/streamer.ex create mode 100644 lib/pleroma/web/streamer/streamer_socket.ex create mode 100644 lib/pleroma/web/streamer/supervisor.ex create mode 100644 lib/pleroma/web/streamer/worker.ex create mode 100644 lib/pleroma/web/views/streamer_view.ex create mode 100644 test/activity/ir/topics_test.exs create mode 100644 test/web/streamer/ping_test.exs create mode 100644 test/web/streamer/state_test.exs rename test/web/{ => streamer}/streamer_test.exs (86%) diff --git a/.gitignore b/.gitignore index 4e71a7df..3b0c7d36 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,7 @@ docs/generated_config.md # Code test coverage /cover /Elixir.*.coverdata + +.idea +pleroma.iml + diff --git a/config/config.exs b/config/config.exs index ab6e00c9..b1b98af9 100644 --- a/config/config.exs +++ b/config/config.exs @@ -331,6 +331,10 @@ config :pleroma, :activitypub, follow_handshake_timeout: 500, sign_object_fetches: true +config :pleroma, :streamer, + workers: 3, + overflow_workers: 2 + config :pleroma, :user, deny_follow_blocked: true config :pleroma, :mrf_normalize_markup, scrub_policy: Pleroma.HTML.Scrubber.Default diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex new file mode 100644 index 00000000..010897ab --- /dev/null +++ b/lib/pleroma/activity/ir/topics.ex @@ -0,0 +1,63 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Activity.Ir.Topics do + alias Pleroma.Object + alias Pleroma.Web.ActivityPub.Visibility + + def get_activity_topics(activity) do + activity + |> Object.normalize() + |> generate_topics(activity) + |> List.flatten() + end + + defp generate_topics(%{data: %{"type" => "Answer"}}, _) do + [] + end + + defp generate_topics(object, activity) do + ["user", "list"] ++ visibility_tags(object, activity) + end + + defp visibility_tags(object, activity) do + case Visibility.get_visibility(activity) do + "public" -> + if activity.local do + ["public", "public:local"] + else + ["public"] + end + |> item_creation_tags(object, activity) + + "direct" -> + ["direct"] + + _ -> + [] + end + end + + defp item_creation_tags(tags, %{data: %{"type" => "Create"}} = object, activity) do + tags ++ hashtags_to_topics(object) ++ attachment_topics(object, activity) + end + + defp item_creation_tags(tags, _, _) do + tags + end + + defp hashtags_to_topics(%{data: %{"tag" => tags}}) do + tags + |> Enum.filter(&is_bitstring(&1)) + |> Enum.map(fn tag -> "hashtag:" <> tag end) + end + + defp hashtags_to_topics(_), do: [] + + defp attachment_topics(%{data: %{"attachment" => []}}, _act), do: [] + + defp attachment_topics(_object, %{local: true}), do: ["public:media", "public:local:media"] + + defp attachment_topics(_object, _act), do: ["public:media"] +end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 49094704..3b37ce63 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -141,7 +141,7 @@ defmodule Pleroma.Application do defp streamer_child(:test), do: [] defp streamer_child(_) do - [Pleroma.Web.Streamer] + [Pleroma.Web.Streamer.supervisor()] end defp oauth_cleanup_child(true), diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index b7c880c5..8012389a 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -210,8 +210,10 @@ defmodule Pleroma.Notification do unless skip?(activity, user) do notification = %Notification{user_id: user.id, activity: activity} {:ok, notification} = Repo.insert(notification) - Streamer.stream("user", notification) - Streamer.stream("user:notification", notification) + + ["user", "user:notification"] + |> Streamer.stream(notification) + Push.send(notification) notification end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 41f6a0f1..bc5ae7fb 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Activity + alias Pleroma.Activity.Ir.Topics alias Pleroma.Config alias Pleroma.Conversation alias Pleroma.Notification @@ -16,6 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.User alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.Streamer alias Pleroma.Web.WebFinger alias Pleroma.Workers.BackgroundWorker @@ -187,9 +189,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do participations |> Repo.preload(:user) - Enum.each(participations, fn participation -> - Pleroma.Web.Streamer.stream("participation", participation) - end) + Streamer.stream("participation", participations) end def stream_out_participations(%Object{data: %{"context" => context}}, user) do @@ -208,41 +208,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def stream_out_participations(_, _), do: :noop - def stream_out(activity) do - if activity.data["type"] in ["Create", "Announce", "Delete"] do - object = Object.normalize(activity) - # Do not stream out poll replies - unless object.data["type"] == "Answer" do - Pleroma.Web.Streamer.stream("user", activity) - Pleroma.Web.Streamer.stream("list", activity) + def stream_out(%Activity{data: %{"type" => data_type}} = activity) + when data_type in ["Create", "Announce", "Delete"] do + activity + |> Topics.get_activity_topics() + |> Streamer.stream(activity) + end - if get_visibility(activity) == "public" do - Pleroma.Web.Streamer.stream("public", activity) - - if activity.local do - Pleroma.Web.Streamer.stream("public:local", activity) - end - - if activity.data["type"] in ["Create"] do - object.data - |> Map.get("tag", []) - |> Enum.filter(fn tag -> is_bitstring(tag) end) - |> Enum.each(fn tag -> Pleroma.Web.Streamer.stream("hashtag:" <> tag, activity) end) - - if object.data["attachment"] != [] do - Pleroma.Web.Streamer.stream("public:media", activity) - - if activity.local do - Pleroma.Web.Streamer.stream("public:local:media", activity) - end - end - end - else - if get_visibility(activity) == "direct", - do: Pleroma.Web.Streamer.stream("direct", activity) - end - end - end + def stream_out(_activity) do + :noop end def create(%{to: to, actor: actor, context: context, object: object} = params, fake \\ false) do diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index dbd3542e..3c26eb40 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.OAuth.Token + alias Pleroma.Web.Streamer @behaviour :cowboy_websocket @@ -24,7 +25,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do ] @anonymous_streams ["public", "public:local", "hashtag"] - # Handled by periodic keepalive in Pleroma.Web.Streamer. + # Handled by periodic keepalive in Pleroma.Web.Streamer.Ping. @timeout :infinity def init(%{qs: qs} = req, state) do @@ -65,7 +66,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do }, topic #{state.topic}" ) - Pleroma.Web.Streamer.add_socket(state.topic, streamer_socket(state)) + Streamer.add_socket(state.topic, streamer_socket(state)) {:ok, state} end @@ -80,7 +81,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do }, topic #{state.topic || "?"}: #{inspect(reason)}" ) - Pleroma.Web.Streamer.remove_socket(state.topic, streamer_socket(state)) + Streamer.remove_socket(state.topic, streamer_socket(state)) :ok end diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex deleted file mode 100644 index 587c43f4..00000000 --- a/lib/pleroma/web/streamer.ex +++ /dev/null @@ -1,318 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.Streamer do - use GenServer - require Logger - alias Pleroma.Activity - alias Pleroma.Config - alias Pleroma.Conversation.Participation - alias Pleroma.Notification - alias Pleroma.Object - alias Pleroma.User - alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.ActivityPub.Visibility - alias Pleroma.Web.CommonAPI - alias Pleroma.Web.MastodonAPI.NotificationView - - @keepalive_interval :timer.seconds(30) - - def start_link(_) do - GenServer.start_link(__MODULE__, %{}, name: __MODULE__) - end - - def add_socket(topic, socket) do - GenServer.cast(__MODULE__, %{action: :add, socket: socket, topic: topic}) - end - - def remove_socket(topic, socket) do - GenServer.cast(__MODULE__, %{action: :remove, socket: socket, topic: topic}) - end - - def stream(topic, item) do - GenServer.cast(__MODULE__, %{action: :stream, topic: topic, item: item}) - end - - def init(args) do - Process.send_after(self(), %{action: :ping}, @keepalive_interval) - - {:ok, args} - end - - def handle_info(%{action: :ping}, topics) do - topics - |> Map.values() - |> List.flatten() - |> Enum.each(fn socket -> - Logger.debug("Sending keepalive ping") - send(socket.transport_pid, {:text, ""}) - end) - - Process.send_after(self(), %{action: :ping}, @keepalive_interval) - - {:noreply, topics} - end - - def handle_cast(%{action: :stream, topic: "direct", item: item}, topics) do - recipient_topics = - User.get_recipients_from_activity(item) - |> Enum.map(fn %{id: id} -> "direct:#{id}" end) - - Enum.each(recipient_topics || [], fn user_topic -> - Logger.debug("Trying to push direct message to #{user_topic}\n\n") - push_to_socket(topics, user_topic, item) - end) - - {:noreply, topics} - end - - def handle_cast(%{action: :stream, topic: "participation", item: participation}, topics) do - user_topic = "direct:#{participation.user_id}" - Logger.debug("Trying to push a conversation participation to #{user_topic}\n\n") - - push_to_socket(topics, user_topic, participation) - - {:noreply, topics} - end - - def handle_cast(%{action: :stream, topic: "list", item: item}, topics) do - # filter the recipient list if the activity is not public, see #270. - recipient_lists = - case Visibility.is_public?(item) do - true -> - Pleroma.List.get_lists_from_activity(item) - - _ -> - Pleroma.List.get_lists_from_activity(item) - |> Enum.filter(fn list -> - owner = User.get_cached_by_id(list.user_id) - - Visibility.visible_for_user?(item, owner) - end) - end - - recipient_topics = - recipient_lists - |> Enum.map(fn %{id: id} -> "list:#{id}" end) - - Enum.each(recipient_topics || [], fn list_topic -> - Logger.debug("Trying to push message to #{list_topic}\n\n") - push_to_socket(topics, list_topic, item) - end) - - {:noreply, topics} - end - - def handle_cast( - %{action: :stream, topic: topic, item: %Notification{} = item}, - topics - ) - when topic in ["user", "user:notification"] do - topics - |> Map.get("#{topic}:#{item.user_id}", []) - |> Enum.each(fn socket -> - with %User{} = user <- User.get_cached_by_ap_id(socket.assigns[:user].ap_id), - true <- should_send?(user, item) do - send( - socket.transport_pid, - {:text, represent_notification(socket.assigns[:user], item)} - ) - end - end) - - {:noreply, topics} - end - - def handle_cast(%{action: :stream, topic: "user", item: item}, topics) do - Logger.debug("Trying to push to users") - - recipient_topics = - User.get_recipients_from_activity(item) - |> Enum.map(fn %{id: id} -> "user:#{id}" end) - - Enum.each(recipient_topics, fn topic -> - push_to_socket(topics, topic, item) - end) - - {:noreply, topics} - end - - def handle_cast(%{action: :stream, topic: topic, item: item}, topics) do - Logger.debug("Trying to push to #{topic}") - Logger.debug("Pushing item to #{topic}") - push_to_socket(topics, topic, item) - {:noreply, topics} - end - - def handle_cast(%{action: :add, topic: topic, socket: socket}, sockets) do - topic = internal_topic(topic, socket) - sockets_for_topic = sockets[topic] || [] - sockets_for_topic = Enum.uniq([socket | sockets_for_topic]) - sockets = Map.put(sockets, topic, sockets_for_topic) - Logger.debug("Got new conn for #{topic}") - {:noreply, sockets} - end - - def handle_cast(%{action: :remove, topic: topic, socket: socket}, sockets) do - topic = internal_topic(topic, socket) - sockets_for_topic = sockets[topic] || [] - sockets_for_topic = List.delete(sockets_for_topic, socket) - sockets = Map.put(sockets, topic, sockets_for_topic) - Logger.debug("Removed conn for #{topic}") - {:noreply, sockets} - end - - def handle_cast(m, state) do - Logger.info("Unknown: #{inspect(m)}, #{inspect(state)}") - {:noreply, state} - end - - defp represent_update(%Activity{} = activity, %User{} = user) do - %{ - event: "update", - payload: - Pleroma.Web.MastodonAPI.StatusView.render( - "status.json", - activity: activity, - for: user - ) - |> Jason.encode!() - } - |> Jason.encode!() - end - - defp represent_update(%Activity{} = activity) do - %{ - event: "update", - payload: - Pleroma.Web.MastodonAPI.StatusView.render( - "status.json", - activity: activity - ) - |> Jason.encode!() - } - |> Jason.encode!() - end - - def represent_conversation(%Participation{} = participation) do - %{ - event: "conversation", - payload: - Pleroma.Web.MastodonAPI.ConversationView.render("participation.json", %{ - participation: participation, - for: participation.user - }) - |> Jason.encode!() - } - |> Jason.encode!() - end - - @spec represent_notification(User.t(), Notification.t()) :: binary() - defp represent_notification(%User{} = user, %Notification{} = notify) do - %{ - event: "notification", - payload: - NotificationView.render( - "show.json", - %{notification: notify, for: user} - ) - |> Jason.encode!() - } - |> Jason.encode!() - end - - defp should_send?(%User{} = user, %Activity{} = item) do - blocks = user.info.blocks || [] - mutes = user.info.mutes || [] - reblog_mutes = user.info.muted_reblogs || [] - domain_blocks = Pleroma.Web.ActivityPub.MRF.subdomains_regex(user.info.domain_blocks) - - with parent when not is_nil(parent) <- Object.normalize(item), - true <- Enum.all?([blocks, mutes, reblog_mutes], &(item.actor not in &1)), - true <- Enum.all?([blocks, mutes], &(parent.data["actor"] not in &1)), - %{host: item_host} <- URI.parse(item.actor), - %{host: parent_host} <- URI.parse(parent.data["actor"]), - false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, item_host), - false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, parent_host), - true <- thread_containment(item, user), - false <- CommonAPI.thread_muted?(user, item) do - true - else - _ -> false - end - end - - defp should_send?(%User{} = user, %Notification{activity: activity}) do - should_send?(user, activity) - end - - def push_to_socket(topics, topic, %Activity{data: %{"type" => "Announce"}} = item) do - Enum.each(topics[topic] || [], fn socket -> - # Get the current user so we have up-to-date blocks etc. - if socket.assigns[:user] do - user = User.get_cached_by_ap_id(socket.assigns[:user].ap_id) - - if should_send?(user, item) do - send(socket.transport_pid, {:text, represent_update(item, user)}) - end - else - send(socket.transport_pid, {:text, represent_update(item)}) - end - end) - end - - def push_to_socket(topics, topic, %Participation{} = participation) do - Enum.each(topics[topic] || [], fn socket -> - send(socket.transport_pid, {:text, represent_conversation(participation)}) - end) - end - - def push_to_socket(topics, topic, %Activity{ - data: %{"type" => "Delete", "deleted_activity_id" => deleted_activity_id} - }) do - Enum.each(topics[topic] || [], fn socket -> - send( - socket.transport_pid, - {:text, %{event: "delete", payload: to_string(deleted_activity_id)} |> Jason.encode!()} - ) - end) - end - - def push_to_socket(_topics, _topic, %Activity{data: %{"type" => "Delete"}}), do: :noop - - def push_to_socket(topics, topic, item) do - Enum.each(topics[topic] || [], fn socket -> - # Get the current user so we have up-to-date blocks etc. - if socket.assigns[:user] do - user = User.get_cached_by_ap_id(socket.assigns[:user].ap_id) - blocks = user.info.blocks || [] - mutes = user.info.mutes || [] - - with true <- Enum.all?([blocks, mutes], &(item.actor not in &1)), - true <- thread_containment(item, user) do - send(socket.transport_pid, {:text, represent_update(item, user)}) - end - else - send(socket.transport_pid, {:text, represent_update(item)}) - end - end) - end - - defp internal_topic(topic, socket) when topic in ~w[user user:notification direct] do - "#{topic}:#{socket.assigns[:user].id}" - end - - defp internal_topic(topic, _), do: topic - - @spec thread_containment(Activity.t(), User.t()) :: boolean() - defp thread_containment(_activity, %User{info: %{skip_thread_containment: true}}), do: true - - defp thread_containment(activity, user) do - if Config.get([:instance, :skip_thread_containment]) do - true - else - ActivityPub.contain_activity(activity, user) - end - end -end diff --git a/lib/pleroma/web/streamer/ping.ex b/lib/pleroma/web/streamer/ping.ex new file mode 100644 index 00000000..f77cbb95 --- /dev/null +++ b/lib/pleroma/web/streamer/ping.ex @@ -0,0 +1,33 @@ +defmodule Pleroma.Web.Streamer.Ping do + use GenServer + require Logger + + alias Pleroma.Web.Streamer.State + alias Pleroma.Web.Streamer.StreamerSocket + + @keepalive_interval :timer.seconds(30) + + def start_link(opts) do + ping_interval = Keyword.get(opts, :ping_interval, @keepalive_interval) + GenServer.start_link(__MODULE__, %{ping_interval: ping_interval}, name: __MODULE__) + end + + def init(%{ping_interval: ping_interval} = args) do + Process.send_after(self(), :ping, ping_interval) + {:ok, args} + end + + def handle_info(:ping, %{ping_interval: ping_interval} = state) do + State.get_sockets() + |> Map.values() + |> List.flatten() + |> Enum.each(fn %StreamerSocket{transport_pid: transport_pid} -> + Logger.debug("Sending keepalive ping") + send(transport_pid, {:text, ""}) + end) + + Process.send_after(self(), :ping, ping_interval) + + {:noreply, state} + end +end diff --git a/lib/pleroma/web/streamer/state.ex b/lib/pleroma/web/streamer/state.ex new file mode 100644 index 00000000..7b519906 --- /dev/null +++ b/lib/pleroma/web/streamer/state.ex @@ -0,0 +1,68 @@ +defmodule Pleroma.Web.Streamer.State do + use GenServer + require Logger + + alias Pleroma.Web.Streamer.StreamerSocket + + def start_link(_) do + GenServer.start_link(__MODULE__, %{sockets: %{}}, name: __MODULE__) + end + + def add_socket(topic, socket) do + GenServer.call(__MODULE__, {:add, socket, topic}) + end + + def remove_socket(topic, socket) do + GenServer.call(__MODULE__, {:remove, socket, topic}) + end + + def get_sockets do + %{sockets: stream_sockets} = GenServer.call(__MODULE__, :get_state) + stream_sockets + end + + def init(init_arg) do + {:ok, init_arg} + end + + def handle_call(:get_state, _from, state) do + {:reply, state, state} + end + + def handle_call({:add, socket, topic}, _from, %{sockets: sockets} = state) do + internal_topic = internal_topic(topic, socket) + stream_socket = StreamerSocket.from_socket(socket) + + sockets_for_topic = + sockets + |> Map.get(internal_topic, []) + |> List.insert_at(0, stream_socket) + |> Enum.uniq() + + state = put_in(state, [:sockets, internal_topic], sockets_for_topic) + Logger.debug("Got new conn for #{topic}") + {:reply, state, state} + end + + def handle_call({:remove, socket, topic}, _from, %{sockets: sockets} = state) do + internal_topic = internal_topic(topic, socket) + stream_socket = StreamerSocket.from_socket(socket) + + sockets_for_topic = + sockets + |> Map.get(internal_topic, []) + |> List.delete(stream_socket) + + state = Kernel.put_in(state, [:sockets, internal_topic], sockets_for_topic) + {:reply, state, state} + end + + defp internal_topic(topic, socket) + when topic in ~w[user user:notification direct] do + "#{topic}:#{socket.assigns[:user].id}" + end + + defp internal_topic(topic, _) do + topic + end +end diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex new file mode 100644 index 00000000..8cf71927 --- /dev/null +++ b/lib/pleroma/web/streamer/streamer.ex @@ -0,0 +1,55 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Streamer do + alias Pleroma.Web.Streamer.State + alias Pleroma.Web.Streamer.Worker + + @timeout 60_000 + @mix_env Mix.env() + + def add_socket(topic, socket) do + State.add_socket(topic, socket) + end + + def remove_socket(topic, socket) do + State.remove_socket(topic, socket) + end + + def get_sockets do + State.get_sockets() + end + + def stream(topics, items) do + if should_send?() do + Task.async(fn -> + :poolboy.transaction( + :streamer_worker, + &Worker.stream(&1, topics, items), + @timeout + ) + end) + end + end + + def supervisor, do: Pleroma.Web.Streamer.Supervisor + + defp should_send? do + handle_should_send(@mix_env) + end + + defp handle_should_send(:test) do + case Process.whereis(:streamer_worker) do + nil -> + false + + pid -> + Process.alive?(pid) + end + end + + defp handle_should_send(_) do + true + end +end diff --git a/lib/pleroma/web/streamer/streamer_socket.ex b/lib/pleroma/web/streamer/streamer_socket.ex new file mode 100644 index 00000000..f006c030 --- /dev/null +++ b/lib/pleroma/web/streamer/streamer_socket.ex @@ -0,0 +1,31 @@ +defmodule Pleroma.Web.Streamer.StreamerSocket do + defstruct transport_pid: nil, user: nil + + alias Pleroma.User + alias Pleroma.Web.Streamer.StreamerSocket + + def from_socket(%{ + transport_pid: transport_pid, + assigns: %{user: nil} + }) do + %StreamerSocket{ + transport_pid: transport_pid + } + end + + def from_socket(%{ + transport_pid: transport_pid, + assigns: %{user: %User{} = user} + }) do + %StreamerSocket{ + transport_pid: transport_pid, + user: user + } + end + + def from_socket(%{transport_pid: transport_pid}) do + %StreamerSocket{ + transport_pid: transport_pid + } + end +end diff --git a/lib/pleroma/web/streamer/supervisor.ex b/lib/pleroma/web/streamer/supervisor.ex new file mode 100644 index 00000000..6afe1932 --- /dev/null +++ b/lib/pleroma/web/streamer/supervisor.ex @@ -0,0 +1,33 @@ +defmodule Pleroma.Web.Streamer.Supervisor do + use Supervisor + + def start_link(opts) do + Supervisor.start_link(__MODULE__, opts, name: __MODULE__) + end + + def init(args) do + children = [ + {Pleroma.Web.Streamer.State, args}, + {Pleroma.Web.Streamer.Ping, args}, + :poolboy.child_spec(:streamer_worker, poolboy_config()) + ] + + opts = [strategy: :one_for_one, name: Pleroma.Web.Streamer.Supervisor] + Supervisor.init(children, opts) + end + + defp poolboy_config do + opts = + Pleroma.Config.get(:streamer, + workers: 3, + overflow_workers: 2 + ) + + [ + {:name, {:local, :streamer_worker}}, + {:worker_module, Pleroma.Web.Streamer.Worker}, + {:size, opts[:workers]}, + {:max_overflow, opts[:overflow_workers]} + ] + end +end diff --git a/lib/pleroma/web/streamer/worker.ex b/lib/pleroma/web/streamer/worker.ex new file mode 100644 index 00000000..5804508e --- /dev/null +++ b/lib/pleroma/web/streamer/worker.ex @@ -0,0 +1,220 @@ +defmodule Pleroma.Web.Streamer.Worker do + use GenServer + + require Logger + + alias Pleroma.Activity + alias Pleroma.Config + alias Pleroma.Conversation.Participation + alias Pleroma.Notification + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.Streamer.State + alias Pleroma.Web.Streamer.StreamerSocket + alias Pleroma.Web.StreamerView + + def start_link(_) do + GenServer.start_link(__MODULE__, %{}, []) + end + + def init(init_arg) do + {:ok, init_arg} + end + + def stream(pid, topics, items) do + GenServer.call(pid, {:stream, topics, items}) + end + + def handle_call({:stream, topics, item}, _from, state) when is_list(topics) do + Enum.each(topics, fn t -> + do_stream(%{topic: t, item: item}) + end) + + {:reply, state, state} + end + + def handle_call({:stream, topic, items}, _from, state) when is_list(items) do + Enum.each(items, fn i -> + do_stream(%{topic: topic, item: i}) + end) + + {:reply, state, state} + end + + def handle_call({:stream, topic, item}, _from, state) do + do_stream(%{topic: topic, item: item}) + + {:reply, state, state} + end + + defp do_stream(%{topic: "direct", item: item}) do + recipient_topics = + User.get_recipients_from_activity(item) + |> Enum.map(fn %{id: id} -> "direct:#{id}" end) + + Enum.each(recipient_topics, fn user_topic -> + Logger.debug("Trying to push direct message to #{user_topic}\n\n") + push_to_socket(State.get_sockets(), user_topic, item) + end) + end + + defp do_stream(%{topic: "participation", item: participation}) do + user_topic = "direct:#{participation.user_id}" + Logger.debug("Trying to push a conversation participation to #{user_topic}\n\n") + + push_to_socket(State.get_sockets(), user_topic, participation) + end + + defp do_stream(%{topic: "list", item: item}) do + # filter the recipient list if the activity is not public, see #270. + recipient_lists = + case Visibility.is_public?(item) do + true -> + Pleroma.List.get_lists_from_activity(item) + + _ -> + Pleroma.List.get_lists_from_activity(item) + |> Enum.filter(fn list -> + owner = User.get_cached_by_id(list.user_id) + + Visibility.visible_for_user?(item, owner) + end) + end + + recipient_topics = + recipient_lists + |> Enum.map(fn %{id: id} -> "list:#{id}" end) + + Enum.each(recipient_topics, fn list_topic -> + Logger.debug("Trying to push message to #{list_topic}\n\n") + push_to_socket(State.get_sockets(), list_topic, item) + end) + end + + defp do_stream(%{topic: topic, item: %Notification{} = item}) + when topic in ["user", "user:notification"] do + State.get_sockets() + |> Map.get("#{topic}:#{item.user_id}", []) + |> Enum.each(fn %StreamerSocket{transport_pid: transport_pid, user: socket_user} -> + with %User{} = user <- User.get_cached_by_ap_id(socket_user.ap_id), + true <- should_send?(user, item) do + send(transport_pid, {:text, StreamerView.render("notification.json", socket_user, item)}) + end + end) + end + + defp do_stream(%{topic: "user", item: item}) do + Logger.debug("Trying to push to users") + + recipient_topics = + User.get_recipients_from_activity(item) + |> Enum.map(fn %{id: id} -> "user:#{id}" end) + + Enum.each(recipient_topics, fn topic -> + push_to_socket(State.get_sockets(), topic, item) + end) + end + + defp do_stream(%{topic: topic, item: item}) do + Logger.debug("Trying to push to #{topic}") + Logger.debug("Pushing item to #{topic}") + push_to_socket(State.get_sockets(), topic, item) + end + + defp should_send?(%User{} = user, %Activity{} = item) do + blocks = user.info.blocks || [] + mutes = user.info.mutes || [] + reblog_mutes = user.info.muted_reblogs || [] + domain_blocks = Pleroma.Web.ActivityPub.MRF.subdomains_regex(user.info.domain_blocks) + + with parent when not is_nil(parent) <- Object.normalize(item), + true <- Enum.all?([blocks, mutes, reblog_mutes], &(item.actor not in &1)), + true <- Enum.all?([blocks, mutes], &(parent.data["actor"] not in &1)), + %{host: item_host} <- URI.parse(item.actor), + %{host: parent_host} <- URI.parse(parent.data["actor"]), + false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, item_host), + false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, parent_host), + true <- thread_containment(item, user), + false <- CommonAPI.thread_muted?(user, item) do + true + else + _ -> false + end + end + + defp should_send?(%User{} = user, %Notification{activity: activity}) do + should_send?(user, activity) + end + + def push_to_socket(topics, topic, %Activity{data: %{"type" => "Announce"}} = item) do + Enum.each(topics[topic] || [], fn %StreamerSocket{ + transport_pid: transport_pid, + user: socket_user + } -> + # Get the current user so we have up-to-date blocks etc. + if socket_user do + user = User.get_cached_by_ap_id(socket_user.ap_id) + + if should_send?(user, item) do + send(transport_pid, {:text, StreamerView.render("update.json", item, user)}) + end + else + send(transport_pid, {:text, StreamerView.render("update.json", item)}) + end + end) + end + + def push_to_socket(topics, topic, %Participation{} = participation) do + Enum.each(topics[topic] || [], fn %StreamerSocket{transport_pid: transport_pid} -> + send(transport_pid, {:text, StreamerView.render("conversation.json", participation)}) + end) + end + + def push_to_socket(topics, topic, %Activity{ + data: %{"type" => "Delete", "deleted_activity_id" => deleted_activity_id} + }) do + Enum.each(topics[topic] || [], fn %StreamerSocket{transport_pid: transport_pid} -> + send( + transport_pid, + {:text, %{event: "delete", payload: to_string(deleted_activity_id)} |> Jason.encode!()} + ) + end) + end + + def push_to_socket(_topics, _topic, %Activity{data: %{"type" => "Delete"}}), do: :noop + + def push_to_socket(topics, topic, item) do + Enum.each(topics[topic] || [], fn %StreamerSocket{ + transport_pid: transport_pid, + user: socket_user + } -> + # Get the current user so we have up-to-date blocks etc. + if socket_user do + user = User.get_cached_by_ap_id(socket_user.ap_id) + blocks = user.info.blocks || [] + mutes = user.info.mutes || [] + + with true <- Enum.all?([blocks, mutes], &(item.actor not in &1)), + true <- thread_containment(item, user) do + send(transport_pid, {:text, StreamerView.render("update.json", item, user)}) + end + else + send(transport_pid, {:text, StreamerView.render("update.json", item)}) + end + end) + end + + @spec thread_containment(Activity.t(), User.t()) :: boolean() + defp thread_containment(_activity, %User{info: %{skip_thread_containment: true}}), do: true + + defp thread_containment(activity, user) do + if Config.get([:instance, :skip_thread_containment]) do + true + else + ActivityPub.contain_activity(activity, user) + end + end +end diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex new file mode 100644 index 00000000..b13030fa --- /dev/null +++ b/lib/pleroma/web/views/streamer_view.ex @@ -0,0 +1,66 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.StreamerView do + use Pleroma.Web, :view + + alias Pleroma.Activity + alias Pleroma.Conversation.Participation + alias Pleroma.Notification + alias Pleroma.User + alias Pleroma.Web.MastodonAPI.NotificationView + + def render("update.json", %Activity{} = activity, %User{} = user) do + %{ + event: "update", + payload: + Pleroma.Web.MastodonAPI.StatusView.render( + "status.json", + activity: activity, + for: user + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + + def render("notification.json", %User{} = user, %Notification{} = notify) do + %{ + event: "notification", + payload: + NotificationView.render( + "show.json", + %{notification: notify, for: user} + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + + def render("update.json", %Activity{} = activity) do + %{ + event: "update", + payload: + Pleroma.Web.MastodonAPI.StatusView.render( + "status.json", + activity: activity + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + + def render("conversation.json", %Participation{} = participation) do + %{ + event: "conversation", + payload: + Pleroma.Web.MastodonAPI.ConversationView.render("participation.json", %{ + participation: participation, + for: participation.user + }) + |> Jason.encode!() + } + |> Jason.encode!() + end +end diff --git a/mix.exs b/mix.exs index f1e98585..911ebad1 100644 --- a/mix.exs +++ b/mix.exs @@ -144,6 +144,7 @@ defmodule Pleroma.Mixfile do git: "https://git.pleroma.social/pleroma/http_signatures.git", ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"}, {:telemetry, "~> 0.3"}, + {:poolboy, "~> 1.5"}, {:prometheus_ex, "~> 3.0"}, {:prometheus_plugs, "~> 1.1"}, {:prometheus_phoenix, "~> 1.3"}, diff --git a/mix.lock b/mix.lock index 41697dd5..0bf6a811 100644 --- a/mix.lock +++ b/mix.lock @@ -73,6 +73,7 @@ "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"}, "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, + "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, "postgrex": {:hex, :postgrex, "0.14.3", "5754dee2fdf6e9e508cbf49ab138df964278700b764177e8f3871e658b345a1e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, "prometheus": {:hex, :prometheus, "4.4.1", "1e96073b3ed7788053768fea779cbc896ddc3bdd9ba60687f2ad50b252ac87d6", [:mix, :rebar3], [], "hexpm"}, "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/activity/ir/topics_test.exs b/test/activity/ir/topics_test.exs new file mode 100644 index 00000000..e75f8358 --- /dev/null +++ b/test/activity/ir/topics_test.exs @@ -0,0 +1,141 @@ +defmodule Pleroma.Activity.Ir.TopicsTest do + use Pleroma.DataCase + + alias Pleroma.Activity + alias Pleroma.Activity.Ir.Topics + alias Pleroma.Object + + require Pleroma.Constants + + describe "poll answer" do + test "produce no topics" do + activity = %Activity{object: %Object{data: %{"type" => "Answer"}}} + + assert [] == Topics.get_activity_topics(activity) + end + end + + describe "non poll answer" do + test "always add user and list topics" do + activity = %Activity{object: %Object{data: %{"type" => "FooBar"}}} + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "user") + assert Enum.member?(topics, "list") + end + end + + describe "public visibility" do + setup do + activity = %Activity{ + object: %Object{data: %{"type" => "Note"}}, + data: %{"to" => [Pleroma.Constants.as_public()]} + } + + {:ok, activity: activity} + end + + test "produces public topic", %{activity: activity} do + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "public") + end + + test "local action produces public:local topic", %{activity: activity} do + activity = %{activity | local: true} + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "public:local") + end + + test "non-local action does not produce public:local topic", %{activity: activity} do + activity = %{activity | local: false} + topics = Topics.get_activity_topics(activity) + + refute Enum.member?(topics, "public:local") + end + end + + describe "public visibility create events" do + setup do + activity = %Activity{ + object: %Object{data: %{"type" => "Create", "attachment" => []}}, + data: %{"to" => [Pleroma.Constants.as_public()]} + } + + {:ok, activity: activity} + end + + test "with no attachments doesn't produce public:media topics", %{activity: activity} do + topics = Topics.get_activity_topics(activity) + + refute Enum.member?(topics, "public:media") + refute Enum.member?(topics, "public:local:media") + end + + test "converts tags to hash tags", %{activity: %{object: %{data: data} = object} = activity} do + tagged_data = Map.put(data, "tag", ["foo", "bar"]) + activity = %{activity | object: %{object | data: tagged_data}} + + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "hashtag:foo") + assert Enum.member?(topics, "hashtag:bar") + end + + test "only converts strinngs to hash tags", %{ + activity: %{object: %{data: data} = object} = activity + } do + tagged_data = Map.put(data, "tag", [2]) + activity = %{activity | object: %{object | data: tagged_data}} + + topics = Topics.get_activity_topics(activity) + + refute Enum.member?(topics, "hashtag:2") + end + end + + describe "public visibility create events with attachments" do + setup do + activity = %Activity{ + object: %Object{data: %{"type" => "Create", "attachment" => ["foo"]}}, + data: %{"to" => [Pleroma.Constants.as_public()]} + } + + {:ok, activity: activity} + end + + test "produce public:media topics", %{activity: activity} do + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "public:media") + end + + test "local produces public:local:media topics", %{activity: activity} do + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "public:local:media") + end + + test "non-local doesn't produce public:local:media topics", %{activity: activity} do + activity = %{activity | local: false} + + topics = Topics.get_activity_topics(activity) + + refute Enum.member?(topics, "public:local:media") + end + end + + describe "non-public visibility" do + test "produces direct topic" do + activity = %Activity{object: %Object{data: %{"type" => "Note"}}, data: %{"to" => []}} + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "direct") + refute Enum.member?(topics, "public") + refute Enum.member?(topics, "public:local") + refute Enum.member?(topics, "public:media") + refute Enum.member?(topics, "public:local:media") + end + end +end diff --git a/test/integration/mastodon_websocket_test.exs b/test/integration/mastodon_websocket_test.exs index 63bf7341..c0426280 100644 --- a/test/integration/mastodon_websocket_test.exs +++ b/test/integration/mastodon_websocket_test.exs @@ -11,7 +11,6 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do alias Pleroma.Integration.WebsocketClient alias Pleroma.Web.CommonAPI alias Pleroma.Web.OAuth - alias Pleroma.Web.Streamer @path Pleroma.Web.Endpoint.url() |> URI.parse() @@ -19,16 +18,6 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do |> Map.put(:path, "/api/v1/streaming") |> URI.to_string() - setup do - GenServer.start(Streamer, %{}, name: Streamer) - - on_exit(fn -> - if pid = Process.whereis(Streamer) do - Process.exit(pid, :kill) - end - end) - end - def start_socket(qs \\ nil, headers \\ []) do path = case qs do @@ -53,12 +42,14 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do end) end + @tag needs_streamer: true test "allows public streams without authentication" do assert {:ok, _} = start_socket("?stream=public") assert {:ok, _} = start_socket("?stream=public:local") assert {:ok, _} = start_socket("?stream=hashtag&tag=lain") end + @tag needs_streamer: true test "receives well formatted events" do user = insert(:user) {:ok, _} = start_socket("?stream=public") @@ -103,6 +94,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert {:ok, _} = start_socket("?stream=user&access_token=#{state.token.token}") end + @tag needs_streamer: true test "accepts the 'user' stream", %{token: token} = _state do assert {:ok, _} = start_socket("?stream=user&access_token=#{token.token}") @@ -111,6 +103,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do end) =~ ":badarg" end + @tag needs_streamer: true test "accepts the 'user:notification' stream", %{token: token} = _state do assert {:ok, _} = start_socket("?stream=user:notification&access_token=#{token.token}") @@ -119,6 +112,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do end) =~ ":badarg" end + @tag needs_streamer: true test "accepts valid token on Sec-WebSocket-Protocol header", %{token: token} do assert {:ok, _} = start_socket("?stream=user", [{"Sec-WebSocket-Protocol", token.token}]) diff --git a/test/notification_test.exs b/test/notification_test.exs index 3be9db09..3d2f9a8f 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -69,16 +69,7 @@ defmodule Pleroma.NotificationTest do end describe "create_notification" do - setup do - GenServer.start(Streamer, %{}, name: Streamer) - - on_exit(fn -> - if pid = Process.whereis(Streamer) do - Process.exit(pid, :kill) - end - end) - end - + @tag needs_streamer: true test "it creates a notification for user and send to the 'user' and the 'user:notification' stream" do user = insert(:user) task = Task.async(fn -> assert_receive {:text, _}, 4_000 end) diff --git a/test/support/conn_case.ex b/test/support/conn_case.ex index ec5892ff..b39c7067 100644 --- a/test/support/conn_case.ex +++ b/test/support/conn_case.ex @@ -40,6 +40,10 @@ defmodule Pleroma.Web.ConnCase do Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, {:shared, self()}) end + if tags[:needs_streamer] do + start_supervised(Pleroma.Web.Streamer.supervisor()) + end + {:ok, conn: Phoenix.ConnTest.build_conn()} end end diff --git a/test/support/data_case.ex b/test/support/data_case.ex index f3d98e7e..17fa1521 100644 --- a/test/support/data_case.ex +++ b/test/support/data_case.ex @@ -39,6 +39,10 @@ defmodule Pleroma.DataCase do Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, {:shared, self()}) end + if tags[:needs_streamer] do + start_supervised(Pleroma.Web.Streamer.supervisor()) + end + :ok end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index d0118fef..4100108a 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -38,9 +38,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do stream: fn _, _ -> nil end do ActivityPub.stream_out_participations(conversation.participations) - Enum.each(participations, fn participation -> - assert called(Pleroma.Web.Streamer.stream("participation", participation)) - end) + assert called(Pleroma.Web.Streamer.stream("participation", participations)) end end end diff --git a/test/web/streamer/ping_test.exs b/test/web/streamer/ping_test.exs new file mode 100644 index 00000000..3d52c00e --- /dev/null +++ b/test/web/streamer/ping_test.exs @@ -0,0 +1,36 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PingTest do + use Pleroma.DataCase + + import Pleroma.Factory + alias Pleroma.Web.Streamer + + setup do + start_supervised({Streamer.supervisor(), [ping_interval: 30]}) + + :ok + end + + describe "sockets" do + setup do + user = insert(:user) + {:ok, %{user: user}} + end + + test "it sends pings", %{user: user} do + task = + Task.async(fn -> + assert_receive {:text, received_event}, 40 + assert_receive {:text, received_event}, 40 + assert_receive {:text, received_event}, 40 + end) + + Streamer.add_socket("public", %{transport_pid: task.pid, assigns: %{user: user}}) + + Task.await(task) + end + end +end diff --git a/test/web/streamer/state_test.exs b/test/web/streamer/state_test.exs new file mode 100644 index 00000000..d1aeac54 --- /dev/null +++ b/test/web/streamer/state_test.exs @@ -0,0 +1,54 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.StateTest do + use Pleroma.DataCase + + import Pleroma.Factory + alias Pleroma.Web.Streamer + alias Pleroma.Web.Streamer.StreamerSocket + + @moduletag needs_streamer: true + + describe "sockets" do + setup do + user = insert(:user) + user2 = insert(:user) + {:ok, %{user: user, user2: user2}} + end + + test "it can add a socket", %{user: user} do + Streamer.add_socket("public", %{transport_pid: 1, assigns: %{user: user}}) + + assert(%{"public" => [%StreamerSocket{transport_pid: 1}]} = Streamer.get_sockets()) + end + + test "it can add multiple sockets per user", %{user: user} do + Streamer.add_socket("public", %{transport_pid: 1, assigns: %{user: user}}) + Streamer.add_socket("public", %{transport_pid: 2, assigns: %{user: user}}) + + assert( + %{ + "public" => [ + %StreamerSocket{transport_pid: 2}, + %StreamerSocket{transport_pid: 1} + ] + } = Streamer.get_sockets() + ) + end + + test "it will not add a duplicate socket", %{user: user} do + Streamer.add_socket("activity", %{transport_pid: 1, assigns: %{user: user}}) + Streamer.add_socket("activity", %{transport_pid: 1, assigns: %{user: user}}) + + assert( + %{ + "activity" => [ + %StreamerSocket{transport_pid: 1} + ] + } = Streamer.get_sockets() + ) + end + end +end diff --git a/test/web/streamer_test.exs b/test/web/streamer/streamer_test.exs similarity index 86% rename from test/web/streamer_test.exs rename to test/web/streamer/streamer_test.exs index 96fa7645..88847e20 100644 --- a/test/web/streamer_test.exs +++ b/test/web/streamer/streamer_test.exs @@ -5,24 +5,20 @@ defmodule Pleroma.Web.StreamerTest do use Pleroma.DataCase + import Pleroma.Factory + alias Pleroma.List alias Pleroma.User alias Pleroma.Web.CommonAPI alias Pleroma.Web.Streamer - import Pleroma.Factory + alias Pleroma.Web.Streamer.StreamerSocket + alias Pleroma.Web.Streamer.Worker + @moduletag needs_streamer: true clear_config_all([:instance, :skip_thread_containment]) describe "user streams" do setup do - GenServer.start(Streamer, %{}, name: Streamer) - - on_exit(fn -> - if pid = Process.whereis(Streamer) do - Process.exit(pid, :kill) - end - end) - user = insert(:user) notify = insert(:notification, user: user, activity: build(:note_activity)) {:ok, %{user: user, notify: notify}} @@ -125,11 +121,9 @@ defmodule Pleroma.Web.StreamerTest do assert_receive {:text, _}, 4_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user - } + user: user } {:ok, activity} = CommonAPI.post(other_user, %{"status" => "Test"}) @@ -138,7 +132,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) @@ -155,11 +149,9 @@ defmodule Pleroma.Web.StreamerTest do assert received_event == expected_event end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user - } + user: user } {:ok, activity} = CommonAPI.delete(activity.id, other_user) @@ -168,7 +160,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) end @@ -189,9 +181,9 @@ defmodule Pleroma.Web.StreamerTest do ) task = Task.async(fn -> refute_receive {:text, _}, 1_000 end) - fake_socket = %{transport_pid: task.pid, assigns: %{user: user}} + fake_socket = %StreamerSocket{transport_pid: task.pid, user: user} topics = %{"public" => [fake_socket]} - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) end @@ -211,9 +203,9 @@ defmodule Pleroma.Web.StreamerTest do ) task = Task.async(fn -> assert_receive {:text, _}, 1_000 end) - fake_socket = %{transport_pid: task.pid, assigns: %{user: user}} + fake_socket = %StreamerSocket{transport_pid: task.pid, user: user} topics = %{"public" => [fake_socket]} - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) end @@ -233,9 +225,9 @@ defmodule Pleroma.Web.StreamerTest do ) task = Task.async(fn -> assert_receive {:text, _}, 1_000 end) - fake_socket = %{transport_pid: task.pid, assigns: %{user: user}} + fake_socket = %StreamerSocket{transport_pid: task.pid, user: user} topics = %{"public" => [fake_socket]} - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) end @@ -251,11 +243,9 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user - } + user: user } {:ok, activity} = CommonAPI.post(blocked_user, %{"status" => "Test"}) @@ -264,7 +254,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) end @@ -284,11 +274,9 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user_a - } + user: user_a } {:ok, activity} = @@ -301,7 +289,7 @@ defmodule Pleroma.Web.StreamerTest do "list:#{list.id}" => [fake_socket] } - Streamer.handle_cast(%{action: :stream, topic: "list", item: activity}, topics) + Worker.handle_call({:stream, "list", activity}, self(), topics) Task.await(task) end @@ -318,11 +306,9 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user_a - } + user: user_a } {:ok, activity} = @@ -335,12 +321,12 @@ defmodule Pleroma.Web.StreamerTest do "list:#{list.id}" => [fake_socket] } - Streamer.handle_cast(%{action: :stream, topic: "list", item: activity}, topics) + Worker.handle_call({:stream, "list", activity}, self(), topics) Task.await(task) end - test "it send wanted private posts to list" do + test "it sends wanted private posts to list" do user_a = insert(:user) user_b = insert(:user) @@ -354,11 +340,9 @@ defmodule Pleroma.Web.StreamerTest do assert_receive {:text, _}, 1_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user_a - } + user: user_a } {:ok, activity} = @@ -367,11 +351,12 @@ defmodule Pleroma.Web.StreamerTest do "visibility" => "private" }) - topics = %{ - "list:#{list.id}" => [fake_socket] - } + Streamer.add_socket( + "list:#{list.id}", + fake_socket + ) - Streamer.handle_cast(%{action: :stream, topic: "list", item: activity}, topics) + Worker.handle_call({:stream, "list", activity}, self(), %{}) Task.await(task) end @@ -387,11 +372,9 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user1 - } + user: user1 } {:ok, create_activity} = CommonAPI.post(user3, %{"status" => "I'm kawen"}) @@ -401,7 +384,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Streamer.push_to_socket(topics, "public", announce_activity) + Worker.push_to_socket(topics, "public", announce_activity) Task.await(task) end @@ -417,6 +400,8 @@ defmodule Pleroma.Web.StreamerTest do task = Task.async(fn -> refute_receive {:text, _}, 4_000 end) + Process.sleep(4000) + Streamer.add_socket( "user", %{transport_pid: task.pid, assigns: %{user: user2}} @@ -428,14 +413,6 @@ defmodule Pleroma.Web.StreamerTest do describe "direct streams" do setup do - GenServer.start(Streamer, %{}, name: Streamer) - - on_exit(fn -> - if pid = Process.whereis(Streamer) do - Process.exit(pid, :kill) - end - end) - :ok end @@ -480,6 +457,8 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 4_000 end) + Process.sleep(1000) + Streamer.add_socket( "direct", %{transport_pid: task.pid, assigns: %{user: user}} @@ -521,6 +500,8 @@ defmodule Pleroma.Web.StreamerTest do assert last_status["id"] == to_string(create_activity.id) end) + Process.sleep(1000) + Streamer.add_socket( "direct", %{transport_pid: task.pid, assigns: %{user: user}} From 085d014f0859b3b3e5023c423ae0361ec6ed6c67 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 16 Sep 2019 19:26:00 +0700 Subject: [PATCH 095/138] Fix `Transmogrifier.upgrade_user_from_ap_id/1` --- lib/pleroma/web/activity_pub/transmogrifier.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index acb3087d..8461b666 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -1050,7 +1050,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do with %User{local: false} = user <- User.get_cached_by_ap_id(ap_id), {:ok, data} <- ActivityPub.fetch_and_prepare_user_from_ap_id(ap_id), already_ap <- User.ap_enabled?(user), - {:ok, user} <- user |> User.upgrade_changeset(data) |> User.update_and_set_cache() do + {:ok, user} <- user |> User.upgrade_changeset(data, true) |> User.update_and_set_cache() do unless already_ap do TransmogrifierWorker.enqueue("user_upgrade", %{"user_id" => user.id}) end From d6ab78e610f16e97246ec9e83b3db72f04cf41e7 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 16 Sep 2019 21:48:01 +0700 Subject: [PATCH 096/138] Set `account_field_value_length` limit to 2048 by default --- config/config.exs | 2 +- config/description.exs | 4 ++-- docs/config.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/config.exs b/config/config.exs index b1b98af9..c7e0cf09 100644 --- a/config/config.exs +++ b/config/config.exs @@ -276,7 +276,7 @@ config :pleroma, :instance, max_account_fields: 10, max_remote_account_fields: 20, account_field_name_length: 512, - account_field_value_length: 512, + account_field_value_length: 2048, external_user_synchronization: true config :pleroma, :markup, diff --git a/config/description.exs b/config/description.exs index be5eb0cc..32d36d6d 100644 --- a/config/description.exs +++ b/config/description.exs @@ -878,9 +878,9 @@ config :pleroma, :config_description, [ %{ key: :account_field_value_length, type: :integer, - description: "An account field value maximum length (default: 512)", + description: "An account field value maximum length (default: 2048)", suggestions: [ - 512 + 2048 ] }, %{ diff --git a/docs/config.md b/docs/config.md index 270d7fce..3f37fa56 100644 --- a/docs/config.md +++ b/docs/config.md @@ -135,7 +135,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `max_account_fields`: The maximum number of custom fields in the user profile (default: `10`) * `max_remote_account_fields`: The maximum number of custom fields in the remote user profile (default: `20`) * `account_field_name_length`: An account field name maximum length (default: `512`) -* `account_field_value_length`: An account field value maximum length (default: `512`) +* `account_field_value_length`: An account field value maximum length (default: `2048`) * `external_user_synchronization`: Enabling following/followers counters synchronization for external users. From a21584556f2c3edb90db3c58ba2a4829a7e220c1 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 17 Sep 2019 13:04:43 +0300 Subject: [PATCH 097/138] Update oban to 0.8.1 This version uses a different locking mechanism, which gets rid of `WARNING: you don't own a lock of type ShareLock` log spam --- lib/pleroma/flake_id.ex | 2 +- mix.exs | 2 +- mix.lock | 8 ++++---- priv/repo/migrations/20190917100019_update_oban.exs | 11 +++++++++++ 4 files changed, 17 insertions(+), 6 deletions(-) create mode 100644 priv/repo/migrations/20190917100019_update_oban.exs diff --git a/lib/pleroma/flake_id.ex b/lib/pleroma/flake_id.ex index 47d61ca5..042cf865 100644 --- a/lib/pleroma/flake_id.ex +++ b/lib/pleroma/flake_id.ex @@ -14,7 +14,7 @@ defmodule Pleroma.FlakeId do @type t :: binary - @behaviour Ecto.Type + use Ecto.Type use GenServer require Logger alias __MODULE__ diff --git a/mix.exs b/mix.exs index 911ebad1..230f9024 100644 --- a/mix.exs +++ b/mix.exs @@ -101,7 +101,7 @@ defmodule Pleroma.Mixfile do {:phoenix_ecto, "~> 4.0"}, {:ecto_sql, "~> 3.1"}, {:postgrex, ">= 0.13.5"}, - {:oban, "~> 0.7"}, + {:oban, "~> 0.8.1"}, {:quantum, "~> 2.3"}, {:gettext, "~> 0.15"}, {:comeonin, "~> 4.1.1"}, diff --git a/mix.lock b/mix.lock index 0bf6a811..547ff6be 100644 --- a/mix.lock +++ b/mix.lock @@ -21,8 +21,8 @@ "decimal": {:hex, :decimal, "1.8.0", "ca462e0d885f09a1c5a342dbd7c1dcf27ea63548c65a65e67334f4b61803822e", [:mix], [], "hexpm"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"}, "earmark": {:hex, :earmark, "1.3.6", "ce1d0675e10a5bb46b007549362bd3f5f08908843957687d8484fe7f37466b19", [:mix], [], "hexpm"}, - "ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, - "ecto_sql": {:hex, :ecto_sql, "3.1.3", "2c536139190492d9de33c5fefac7323c5eaaa82e1b9bf93482a14649042f7cd9", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "ecto": {:hex, :ecto, "3.2.0", "940e2598813f205223d60c78d66e514afe1db5167ed8075510a59e496619cfb5", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "ecto_sql": {:hex, :ecto_sql, "3.2.0", "751cea597e8deb616084894dd75cbabfdbe7255ff01e8c058ca13f0353a3921b", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.2.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"}, "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"}, "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"}, @@ -60,7 +60,7 @@ "mogrify": {:hex, :mogrify, "0.6.1", "de1b527514f2d95a7bbe9642eb556061afb337e220cf97adbf3a4e6438ed70af", [:mix], [], "hexpm"}, "mox": {:hex, :mox, "0.5.1", "f86bb36026aac1e6f924a4b6d024b05e9adbed5c63e8daa069bd66fb3292165b", [:mix], [], "hexpm"}, "nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm"}, - "oban": {:hex, :oban, "0.7.1", "171bdd1b69c1a4a839f8c768f5e962fc22d1de1513d459fb6b8e0cbd34817a9a", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "oban": {:hex, :oban, "0.8.1", "4bbf62eb1829f856d69aeb5069ac7036afe07db8221a17de2a9169cc7a58a318", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.3", "6706a148809a29c306062862c803406e88f048277f6e85b68faf73291e820b84", [:mix], [], "hexpm"}, "phoenix": {:hex, :phoenix, "1.4.9", "746d098e10741c334d88143d3c94cab1756435f94387a63441792e66ec0ee974", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, @@ -74,7 +74,7 @@ "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, - "postgrex": {:hex, :postgrex, "0.14.3", "5754dee2fdf6e9e508cbf49ab138df964278700b764177e8f3871e658b345a1e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "postgrex": {:hex, :postgrex, "0.15.1", "23ce3417de70f4c0e9e7419ad85bdabcc6860a6925fe2c6f3b1b5b1e8e47bf2f", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, "prometheus": {:hex, :prometheus, "4.4.1", "1e96073b3ed7788053768fea779cbc896ddc3bdd9ba60687f2ad50b252ac87d6", [:mix, :rebar3], [], "hexpm"}, "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/priv/repo/migrations/20190917100019_update_oban.exs b/priv/repo/migrations/20190917100019_update_oban.exs new file mode 100644 index 00000000..157dc54f --- /dev/null +++ b/priv/repo/migrations/20190917100019_update_oban.exs @@ -0,0 +1,11 @@ +defmodule Pleroma.Repo.Migrations.UpdateOban do + use Ecto.Migration + + def up do + Oban.Migrations.up(version: 4) + end + + def down do + Oban.Migrations.down(version: 2) + end +end From 450bf7a63c39c2301d5985448a867e77f1dfe3b3 Mon Sep 17 00:00:00 2001 From: eugenijm Date: Fri, 13 Sep 2019 17:37:30 +0300 Subject: [PATCH 098/138] Mastodon API: Add a setting to hide follow/follower count from the user view (`hide_follows_count` and `hide_followers_count`) --- CHANGELOG.md | 1 + docs/api/differences_in_mastoapi_responses.md | 4 +++ lib/pleroma/user/info.ex | 14 ++++++-- .../web/activity_pub/views/user_view.ex | 32 +++++++++++-------- .../controllers/mastodon_api_controller.ex | 2 ++ .../web/mastodon_api/views/account_view.ex | 14 ++++++-- .../web/activity_pub/views/user_view_test.exs | 24 ++++++++++++-- .../update_credentials_test.exs | 16 ++++++++++ .../mastodon_api/views/account_view_test.exs | 31 ++++++++++++++++-- 9 files changed, 117 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4eb72c00..7dfa477b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -94,6 +94,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: added `/auth/password` endpoint for password reset with rate limit. - Mastodon API: /api/v1/accounts/:id/statuses now supports nicknames or user id - Mastodon API: Improve support for the user profile custom fields +- Mastodon API: follower/following counters are nullified when `hide_follows`/`hide_followers` and `hide_follows_count`/`hide_followers_count` are set - Admin API: Return users' tags when querying reports - Admin API: Return avatar and display name when querying users - Admin API: Allow querying user by ID diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index 9b32baf3..3c7f5dad 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -50,6 +50,8 @@ Has these additional fields under the `pleroma` object: - `confirmation_pending`: boolean, true if a new user account is waiting on email confirmation to be activated - `hide_followers`: boolean, true when the user has follower hiding enabled - `hide_follows`: boolean, true when the user has follow hiding enabled +- `hide_followers_count`: boolean, true when the user has follower stat hiding enabled +- `hide_follows_count`: boolean, true when the user has follow stat hiding enabled - `settings_store`: A generic map of settings for frontends. Opaque to the backend. Only returned in `verify_credentials` and `update_credentials` - `chat_token`: The token needed for Pleroma chat. Only returned in `verify_credentials` - `deactivated`: boolean, true when the user is deactivated @@ -112,6 +114,8 @@ Additional parameters can be added to the JSON body/Form data: - `no_rich_text` - if true, html tags are stripped from all statuses requested from the API - `hide_followers` - if true, user's followers will be hidden - `hide_follows` - if true, user's follows will be hidden +- `hide_followers_count` - if true, user's follower count will be hidden +- `hide_follows_count` - if true, user's follow count will be hidden - `hide_favorites` - if true, user's favorites timeline will be hidden - `show_role` - if true, user's role (e.g admin, moderator) will be exposed to anyone in the API - `default_scope` - the scope returned under `privacy` key in Source subentity diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 151e025d..b150a57c 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -41,6 +41,8 @@ defmodule Pleroma.User.Info do field(:topic, :string, default: nil) field(:hub, :string, default: nil) field(:salmon, :string, default: nil) + field(:hide_followers_count, :boolean, default: false) + field(:hide_follows_count, :boolean, default: false) field(:hide_followers, :boolean, default: false) field(:hide_follows, :boolean, default: false) field(:hide_favorites, :boolean, default: true) @@ -262,6 +264,8 @@ defmodule Pleroma.User.Info do :salmon, :hide_followers, :hide_follows, + :hide_followers_count, + :hide_follows_count, :follower_count, :fields, :following_count @@ -281,7 +285,9 @@ defmodule Pleroma.User.Info do :following_count, :hide_follows, :fields, - :hide_followers + :hide_followers, + :hide_followers_count, + :hide_follows_count ]) |> validate_fields(remote?) end @@ -295,6 +301,8 @@ defmodule Pleroma.User.Info do :banner, :hide_follows, :hide_followers, + :hide_followers_count, + :hide_follows_count, :hide_favorites, :background, :show_role, @@ -458,7 +466,9 @@ defmodule Pleroma.User.Info do :hide_followers, :hide_follows, :follower_count, - :following_count + :following_count, + :hide_followers_count, + :hide_follows_count ]) end end diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 7be734b2..164b973d 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -118,30 +118,34 @@ defmodule Pleroma.Web.ActivityPub.UserView do end def render("following.json", %{user: user, page: page} = opts) do - showing = (opts[:for] && opts[:for] == user) || !user.info.hide_follows + showing_items = (opts[:for] && opts[:for] == user) || !user.info.hide_follows + showing_count = showing_items || !user.info.hide_follows_count + query = User.get_friends_query(user) query = from(user in query, select: [:ap_id]) following = Repo.all(query) total = - if showing do + if showing_count do length(following) else 0 end - collection(following, "#{user.ap_id}/following", page, showing, total) + collection(following, "#{user.ap_id}/following", page, showing_items, total) |> Map.merge(Utils.make_json_ld_header()) end def render("following.json", %{user: user} = opts) do - showing = (opts[:for] && opts[:for] == user) || !user.info.hide_follows + showing_items = (opts[:for] && opts[:for] == user) || !user.info.hide_follows + showing_count = showing_items || !user.info.hide_follows_count + query = User.get_friends_query(user) query = from(user in query, select: [:ap_id]) following = Repo.all(query) total = - if showing do + if showing_count do length(following) else 0 @@ -152,7 +156,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do "type" => "OrderedCollection", "totalItems" => total, "first" => - if showing do + if showing_items do collection(following, "#{user.ap_id}/following", 1, !user.info.hide_follows) else "#{user.ap_id}/following?page=1" @@ -162,32 +166,34 @@ defmodule Pleroma.Web.ActivityPub.UserView do end def render("followers.json", %{user: user, page: page} = opts) do - showing = (opts[:for] && opts[:for] == user) || !user.info.hide_followers + showing_items = (opts[:for] && opts[:for] == user) || !user.info.hide_followers + showing_count = showing_items || !user.info.hide_followers_count query = User.get_followers_query(user) query = from(user in query, select: [:ap_id]) followers = Repo.all(query) total = - if showing do + if showing_count do length(followers) else 0 end - collection(followers, "#{user.ap_id}/followers", page, showing, total) + collection(followers, "#{user.ap_id}/followers", page, showing_items, total) |> Map.merge(Utils.make_json_ld_header()) end def render("followers.json", %{user: user} = opts) do - showing = (opts[:for] && opts[:for] == user) || !user.info.hide_followers + showing_items = (opts[:for] && opts[:for] == user) || !user.info.hide_followers + showing_count = showing_items || !user.info.hide_followers_count query = User.get_followers_query(user) query = from(user in query, select: [:ap_id]) followers = Repo.all(query) total = - if showing do + if showing_count do length(followers) else 0 @@ -198,8 +204,8 @@ defmodule Pleroma.Web.ActivityPub.UserView do "type" => "OrderedCollection", "totalItems" => total, "first" => - if showing do - collection(followers, "#{user.ap_id}/followers", 1, showing, total) + if showing_items do + collection(followers, "#{user.ap_id}/followers", 1, showing_items, total) else "#{user.ap_id}/followers?page=1" end diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex index 060137b8..1beb4bcf 100644 --- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex @@ -147,6 +147,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do [ :no_rich_text, :locked, + :hide_followers_count, + :hide_follows_count, :hide_followers, :hide_follows, :hide_favorites, diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 169116d0..195dd124 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -74,10 +74,18 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do user_info = User.get_cached_user_info(user) following_count = - ((!user.info.hide_follows or opts[:for] == user) && user_info.following_count) || 0 + if !user.info.hide_follows_count or !user.info.hide_follows or opts[:for] == user do + user_info.following_count + else + 0 + end followers_count = - ((!user.info.hide_followers or opts[:for] == user) && user_info.follower_count) || 0 + if !user.info.hide_followers_count or !user.info.hide_followers or opts[:for] == user do + user_info.follower_count + else + 0 + end bot = (user.info.source_data["type"] || "Person") in ["Application", "Service"] @@ -138,6 +146,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do pleroma: %{ confirmation_pending: user_info.confirmation_pending, tags: user.tags, + hide_followers_count: user.info.hide_followers_count, + hide_follows_count: user.info.hide_follows_count, hide_followers: user.info.hide_followers, hide_follows: user.info.hide_follows, hide_favorites: user.info.hide_favorites, diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs index fb7fd9e7..2b4a04af 100644 --- a/test/web/activity_pub/views/user_view_test.exs +++ b/test/web/activity_pub/views/user_view_test.exs @@ -105,10 +105,20 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do other_user = insert(:user) {:ok, _other_user, user, _activity} = CommonAPI.follow(other_user, user) assert %{"totalItems" => 1} = UserView.render("followers.json", %{user: user}) - info = Map.put(user.info, :hide_followers, true) + info = Map.merge(user.info, %{hide_followers_count: true, hide_followers: true}) user = Map.put(user, :info, info) assert %{"totalItems" => 0} = UserView.render("followers.json", %{user: user}) end + + test "sets correct totalItems when followers are hidden but the follower counter is not" do + user = insert(:user) + other_user = insert(:user) + {:ok, _other_user, user, _activity} = CommonAPI.follow(other_user, user) + assert %{"totalItems" => 1} = UserView.render("followers.json", %{user: user}) + info = Map.merge(user.info, %{hide_followers_count: false, hide_followers: true}) + user = Map.put(user, :info, info) + assert %{"totalItems" => 1} = UserView.render("followers.json", %{user: user}) + end end describe "following" do @@ -117,9 +127,19 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do other_user = insert(:user) {:ok, user, _other_user, _activity} = CommonAPI.follow(user, other_user) assert %{"totalItems" => 1} = UserView.render("following.json", %{user: user}) - info = Map.put(user.info, :hide_follows, true) + info = Map.merge(user.info, %{hide_follows_count: true, hide_follows: true}) user = Map.put(user, :info, info) assert %{"totalItems" => 0} = UserView.render("following.json", %{user: user}) end + + test "sets correct totalItems when follows are hidden but the follow counter is not" do + user = insert(:user) + other_user = insert(:user) + {:ok, user, _other_user, _activity} = CommonAPI.follow(user, other_user) + assert %{"totalItems" => 1} = UserView.render("following.json", %{user: user}) + info = Map.merge(user.info, %{hide_follows_count: false, hide_follows: true}) + user = Map.put(user, :info, info) + assert %{"totalItems" => 1} = UserView.render("following.json", %{user: user}) + end end end diff --git a/test/web/mastodon_api/controllers/mastodon_api_controller/update_credentials_test.exs b/test/web/mastodon_api/controllers/mastodon_api_controller/update_credentials_test.exs index 87ee8205..89d4ca37 100644 --- a/test/web/mastodon_api/controllers/mastodon_api_controller/update_credentials_test.exs +++ b/test/web/mastodon_api/controllers/mastodon_api_controller/update_credentials_test.exs @@ -128,6 +128,22 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController.UpdateCredentialsTest do assert user["pleroma"]["hide_followers"] == true end + test "updates the user's hide_followers_count and hide_follows_count", %{conn: conn} do + user = insert(:user) + + conn = + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_credentials", %{ + hide_followers_count: "true", + hide_follows_count: "true" + }) + + assert user = json_response(conn, 200) + assert user["pleroma"]["hide_followers_count"] == true + assert user["pleroma"]["hide_follows_count"] == true + end + test "updates the user's skip_thread_containment option", %{conn: conn} do user = insert(:user) diff --git a/test/web/mastodon_api/views/account_view_test.exs b/test/web/mastodon_api/views/account_view_test.exs index 1d8b2833..8ff6751d 100644 --- a/test/web/mastodon_api/views/account_view_test.exs +++ b/test/web/mastodon_api/views/account_view_test.exs @@ -79,6 +79,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do hide_favorites: true, hide_followers: false, hide_follows: false, + hide_followers_count: false, + hide_follows_count: false, relationship: %{}, skip_thread_containment: false } @@ -147,6 +149,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do hide_favorites: true, hide_followers: false, hide_follows: false, + hide_followers_count: false, + hide_follows_count: false, relationship: %{}, skip_thread_containment: false } @@ -318,6 +322,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do hide_favorites: true, hide_followers: false, hide_follows: false, + hide_followers_count: false, + hide_follows_count: false, relationship: %{ id: to_string(user.id), following: false, @@ -361,8 +367,16 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do end describe "hiding follows/following" do - test "shows when follows/following are hidden and sets follower/following count to 0" do - user = insert(:user, info: %{hide_followers: true, hide_follows: true}) + test "shows when follows/followers stats are hidden and sets follow/follower count to 0" do + info = %{ + hide_followers: true, + hide_followers_count: true, + hide_follows: true, + hide_follows_count: true + } + + user = insert(:user, info: info) + other_user = insert(:user) {:ok, user, other_user, _activity} = CommonAPI.follow(user, other_user) {:ok, _other_user, user, _activity} = CommonAPI.follow(other_user, user) @@ -370,6 +384,19 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do assert %{ followers_count: 0, following_count: 0, + pleroma: %{hide_follows_count: true, hide_followers_count: true} + } = AccountView.render("account.json", %{user: user}) + end + + test "shows when follows/followers are hidden" do + user = insert(:user, info: %{hide_followers: true, hide_follows: true}) + other_user = insert(:user) + {:ok, user, other_user, _activity} = CommonAPI.follow(user, other_user) + {:ok, _other_user, user, _activity} = CommonAPI.follow(other_user, user) + + assert %{ + followers_count: 1, + following_count: 1, pleroma: %{hide_follows: true, hide_followers: true} } = AccountView.render("account.json", %{user: user}) end From 80c5c3495bdd7939e576c8746a959f3f89f44042 Mon Sep 17 00:00:00 2001 From: Steven Fuchs Date: Tue, 17 Sep 2019 14:44:52 +0000 Subject: [PATCH 099/138] remove remaining errors from tests --- lib/pleroma/application.ex | 53 ++++++++++++++------ lib/pleroma/web/streamer/state.ex | 18 +++++-- test/integration/mastodon_websocket_test.exs | 16 ++++-- 3 files changed, 62 insertions(+), 25 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 3b37ce63..dabce771 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -43,23 +43,9 @@ defmodule Pleroma.Application do hackney_pool_children() ++ [ Pleroma.Stats, - {Oban, Pleroma.Config.get(Oban)}, - %{ - id: :web_push_init, - start: {Task, :start_link, [&Pleroma.Web.Push.init/0]}, - restart: :temporary - }, - %{ - id: :federator_init, - start: {Task, :start_link, [&Pleroma.Web.Federator.init/0]}, - restart: :temporary - }, - %{ - id: :internal_fetch_init, - start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]}, - restart: :temporary - } + {Oban, Pleroma.Config.get(Oban)} ] ++ + task_children(@env) ++ oauth_cleanup_child(oauth_cleanup_enabled?()) ++ streamer_child(@env) ++ chat_child(@env, chat_enabled?()) ++ @@ -163,4 +149,39 @@ defmodule Pleroma.Application do :hackney_pool.child_spec(pool, options) end end + + defp task_children(:test) do + [ + %{ + id: :web_push_init, + start: {Task, :start_link, [&Pleroma.Web.Push.init/0]}, + restart: :temporary + }, + %{ + id: :federator_init, + start: {Task, :start_link, [&Pleroma.Web.Federator.init/0]}, + restart: :temporary + } + ] + end + + defp task_children(_) do + [ + %{ + id: :web_push_init, + start: {Task, :start_link, [&Pleroma.Web.Push.init/0]}, + restart: :temporary + }, + %{ + id: :federator_init, + start: {Task, :start_link, [&Pleroma.Web.Federator.init/0]}, + restart: :temporary + }, + %{ + id: :internal_fetch_init, + start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]}, + restart: :temporary + } + ] + end end diff --git a/lib/pleroma/web/streamer/state.ex b/lib/pleroma/web/streamer/state.ex index 7b519906..c48752d9 100644 --- a/lib/pleroma/web/streamer/state.ex +++ b/lib/pleroma/web/streamer/state.ex @@ -4,16 +4,18 @@ defmodule Pleroma.Web.Streamer.State do alias Pleroma.Web.Streamer.StreamerSocket + @env Mix.env() + def start_link(_) do GenServer.start_link(__MODULE__, %{sockets: %{}}, name: __MODULE__) end def add_socket(topic, socket) do - GenServer.call(__MODULE__, {:add, socket, topic}) + GenServer.call(__MODULE__, {:add, topic, socket}) end def remove_socket(topic, socket) do - GenServer.call(__MODULE__, {:remove, socket, topic}) + do_remove_socket(@env, topic, socket) end def get_sockets do @@ -29,7 +31,7 @@ defmodule Pleroma.Web.Streamer.State do {:reply, state, state} end - def handle_call({:add, socket, topic}, _from, %{sockets: sockets} = state) do + def handle_call({:add, topic, socket}, _from, %{sockets: sockets} = state) do internal_topic = internal_topic(topic, socket) stream_socket = StreamerSocket.from_socket(socket) @@ -44,7 +46,7 @@ defmodule Pleroma.Web.Streamer.State do {:reply, state, state} end - def handle_call({:remove, socket, topic}, _from, %{sockets: sockets} = state) do + def handle_call({:remove, topic, socket}, _from, %{sockets: sockets} = state) do internal_topic = internal_topic(topic, socket) stream_socket = StreamerSocket.from_socket(socket) @@ -57,6 +59,14 @@ defmodule Pleroma.Web.Streamer.State do {:reply, state, state} end + defp do_remove_socket(:test, _, _) do + :ok + end + + defp do_remove_socket(_env, topic, socket) do + GenServer.call(__MODULE__, {:remove, topic, socket}) + end + defp internal_topic(topic, socket) when topic in ~w[user user:notification direct] do "#{topic}:#{socket.assigns[:user].id}" diff --git a/test/integration/mastodon_websocket_test.exs b/test/integration/mastodon_websocket_test.exs index c0426280..d02a3cc4 100644 --- a/test/integration/mastodon_websocket_test.exs +++ b/test/integration/mastodon_websocket_test.exs @@ -18,6 +18,11 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do |> Map.put(:path, "/api/v1/streaming") |> URI.to_string() + setup_all do + start_supervised(Pleroma.Web.Streamer.supervisor()) + :ok + end + def start_socket(qs \\ nil, headers \\ []) do path = case qs do @@ -32,6 +37,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do capture_log(fn -> assert {:error, {400, _}} = start_socket() assert {:error, {404, _}} = start_socket("?stream=ncjdk") + Process.sleep(30) end) end @@ -39,17 +45,16 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do capture_log(fn -> assert {:error, {403, _}} = start_socket("?stream=user&access_token=aaaaaaaaaaaa") assert {:error, {403, _}} = start_socket("?stream=user") + Process.sleep(30) end) end - @tag needs_streamer: true test "allows public streams without authentication" do assert {:ok, _} = start_socket("?stream=public") assert {:ok, _} = start_socket("?stream=public:local") assert {:ok, _} = start_socket("?stream=hashtag&tag=lain") end - @tag needs_streamer: true test "receives well formatted events" do user = insert(:user) {:ok, _} = start_socket("?stream=public") @@ -94,31 +99,32 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert {:ok, _} = start_socket("?stream=user&access_token=#{state.token.token}") end - @tag needs_streamer: true test "accepts the 'user' stream", %{token: token} = _state do assert {:ok, _} = start_socket("?stream=user&access_token=#{token.token}") assert capture_log(fn -> assert {:error, {403, "Forbidden"}} = start_socket("?stream=user") + Process.sleep(30) end) =~ ":badarg" end - @tag needs_streamer: true test "accepts the 'user:notification' stream", %{token: token} = _state do assert {:ok, _} = start_socket("?stream=user:notification&access_token=#{token.token}") assert capture_log(fn -> assert {:error, {403, "Forbidden"}} = start_socket("?stream=user:notification") + Process.sleep(30) end) =~ ":badarg" end - @tag needs_streamer: true test "accepts valid token on Sec-WebSocket-Protocol header", %{token: token} do assert {:ok, _} = start_socket("?stream=user", [{"Sec-WebSocket-Protocol", token.token}]) assert capture_log(fn -> assert {:error, {403, "Forbidden"}} = start_socket("?stream=user", [{"Sec-WebSocket-Protocol", "I am a friend"}]) + + Process.sleep(30) end) =~ ":badarg" end end From 6193157f1998b10ac6cb9f4d36dd863eced81b37 Mon Sep 17 00:00:00 2001 From: Steven Fuchs Date: Tue, 17 Sep 2019 18:12:27 +0000 Subject: [PATCH 100/138] Fix notification warnings --- lib/pleroma/workers/web_pusher_worker.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/workers/web_pusher_worker.ex b/lib/pleroma/workers/web_pusher_worker.ex index bea2baff..61b451e3 100644 --- a/lib/pleroma/workers/web_pusher_worker.ex +++ b/lib/pleroma/workers/web_pusher_worker.ex @@ -10,7 +10,11 @@ defmodule Pleroma.Workers.WebPusherWorker do @impl Oban.Worker def perform(%{"op" => "web_push", "notification_id" => notification_id}, _job) do - notification = Repo.get(Notification, notification_id) + notification = + Notification + |> Repo.get(notification_id) + |> Repo.preload([:activity]) + Pleroma.Web.Push.Impl.perform(notification) end end From 8d812c28a70ae174985000e98b9618dad746b22e Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 17 Sep 2019 21:51:50 +0300 Subject: [PATCH 101/138] Update Tesla to 1.3 This version includes a couple of fixes, adds Gun and Mint adapters and removes 0.x -> 1.0 config migrator, which for some reason fails under certain conditions. I had to set `override: true` because Quack pins Tesla to `1.2.0`, but I have looked through the source code and verified that updating Tesla doesn't break anything there. --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 230f9024..d8d3f802 100644 --- a/mix.exs +++ b/mix.exs @@ -113,7 +113,7 @@ defmodule Pleroma.Mixfile do {:calendar, "~> 0.17.4"}, {:cachex, "~> 3.0.2"}, {:poison, "~> 3.0", override: true}, - {:tesla, "~> 1.2"}, + {:tesla, "~> 1.3", override: true}, {:jason, "~> 1.0"}, {:mogrify, "~> 0.6.1"}, {:ex_aws, "~> 2.1"}, diff --git a/mix.lock b/mix.lock index 547ff6be..24b34c09 100644 --- a/mix.lock +++ b/mix.lock @@ -90,7 +90,7 @@ "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, - "tesla": {:hex, :tesla, "1.2.1", "864783cc27f71dd8c8969163704752476cec0f3a51eb3b06393b3971dc9733ff", [:mix], [{:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"}, + "tesla": {:hex, :tesla, "1.3.0", "f35d72f029e608f9cdc6f6d6fcc7c66cf6d6512a70cfef9206b21b8bd0203a30", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 0.4", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm"}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, From d201eec45cc5eb8c7b0c912c14be4704dbb4c1b1 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Tue, 17 Sep 2019 22:02:37 +0300 Subject: [PATCH 102/138] fixed ecto version --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 230f9024..58d1606d 100644 --- a/mix.exs +++ b/mix.exs @@ -99,7 +99,7 @@ defmodule Pleroma.Mixfile do {:plug_cowboy, "~> 2.0"}, {:phoenix_pubsub, "~> 1.1"}, {:phoenix_ecto, "~> 4.0"}, - {:ecto_sql, "~> 3.1"}, + {:ecto_sql, "~> 3.2"}, {:postgrex, ">= 0.13.5"}, {:oban, "~> 0.8.1"}, {:quantum, "~> 2.3"}, From 228bfd8a70cefadb8673ed6d11485944ef7c5666 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Tue, 17 Sep 2019 22:36:42 +0300 Subject: [PATCH 103/138] Bump elixir version to ~> 1.8 --- CHANGELOG.md | 1 + mix.exs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4eb72c00..f2d14930 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Remove `Reply-To` header from report emails for admins. ### Changed +- **Breaking:** Now pleroma requires Elixir ~> 1.8 (it was ~> 1.7) - **Breaking:** Configuration: A setting to explicitly disable the mailer was added, defaulting to true, if you are using a mailer add `config :pleroma, Pleroma.Emails.Mailer, enabled: true` to your config - **Breaking:** Configuration: `/media/` is now removed when `base_url` is configured, append `/media/` to your `base_url` config to keep the old behaviour if desired - **Breaking:** `/api/pleroma/notifications/read` is moved to `/api/v1/pleroma/notifications/read` and now supports `max_id` and responds with Mastodon API entities. diff --git a/mix.exs b/mix.exs index 230f9024..7d262a60 100644 --- a/mix.exs +++ b/mix.exs @@ -5,7 +5,7 @@ defmodule Pleroma.Mixfile do [ app: :pleroma, version: version("1.0.0"), - elixir: "~> 1.7", + elixir: "~> 1.8", elixirc_paths: elixirc_paths(Mix.env()), compilers: [:phoenix, :gettext] ++ Mix.compilers(), elixirc_options: [warnings_as_errors: true], From 35dcea3e13673b8a1a078a1f6fbc44f1e2098c22 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 13 Sep 2019 17:05:19 +0200 Subject: [PATCH 104/138] Remove [true,false] for booleans, it is implicit --- config/description.exs | 262 +++++++++++------------------------------ 1 file changed, 68 insertions(+), 194 deletions(-) diff --git a/config/description.exs b/config/description.exs index 32d36d6d..5ae32d0f 100644 --- a/config/description.exs +++ b/config/description.exs @@ -39,11 +39,7 @@ config :pleroma, :config_description, [ key: :link_name, type: :boolean, description: - "If enabled, a name parameter will be added to the url of the upload. For example `https://instance.tld/media/imagehash.png?name=realname.png`", - suggestions: [ - true, - false - ] + "If enabled, a name parameter will be added to the url of the upload. For example `https://instance.tld/media/imagehash.png?name=realname.png`" }, %{ key: :base_url, @@ -57,11 +53,7 @@ config :pleroma, :config_description, [ key: :proxy_remote, type: :boolean, description: - "If enabled, requests to media stored using a remote uploader will be proxied instead of being redirected.", - suggestions: [ - true, - false - ] + "If enabled, requests to media stored using a remote uploader will be proxied instead of being redirected." }, %{ key: :proxy_opts, @@ -190,11 +182,7 @@ config :pleroma, :config_description, [ %{ key: :enabled, type: :boolean, - description: "Allow/disallow send emails", - suggestions: [ - true, - false - ] + description: "Allow/disallow send emails" }, %{ group: {:subgroup, Swoosh.Adapters.SMTP}, @@ -221,8 +209,7 @@ config :pleroma, :config_description, [ group: {:subgroup, Swoosh.Adapters.SMTP}, key: :ssl, type: :boolean, - description: "`Swoosh.Adapters.SMTP` adapter specific setting", - suggestions: [true, false] + description: "`Swoosh.Adapters.SMTP` adapter specific setting" }, %{ group: {:subgroup, Swoosh.Adapters.SMTP}, @@ -256,8 +243,7 @@ config :pleroma, :config_description, [ group: {:subgroup, Swoosh.Adapters.SMTP}, key: :no_mx_lookups, type: :boolean, - description: "`Swoosh.Adapters.SMTP` adapter specific setting", - suggestions: [true, false] + description: "`Swoosh.Adapters.SMTP` adapter specific setting" }, %{ group: {:subgroup, Swoosh.Adapters.Sendgrid}, @@ -284,8 +270,7 @@ config :pleroma, :config_description, [ group: {:subgroup, Swoosh.Adapters.Sendmail}, key: :qmail, type: :boolean, - description: "`Swoosh.Adapters.Sendmail` adapter specific setting", - suggestions: [true, false] + description: "`Swoosh.Adapters.Sendmail` adapter specific setting" }, %{ group: {:subgroup, Swoosh.Adapters.Mandrill}, @@ -553,38 +538,22 @@ config :pleroma, :config_description, [ %{ key: :registrations_open, type: :boolean, - description: "Enable registrations for anyone, invitations can be enabled when false", - suggestions: [ - true, - false - ] + description: "Enable registrations for anyone, invitations can be enabled when false" }, %{ key: :invites_enabled, type: :boolean, - description: "Enable user invitations for admins (depends on registrations_open: false)", - suggestions: [ - true, - false - ] + description: "Enable user invitations for admins (depends on registrations_open: false)" }, %{ key: :account_activation_required, type: :boolean, - description: "Require users to confirm their emails before signing in", - suggestions: [ - true, - false - ] + description: "Require users to confirm their emails before signing in" }, %{ key: :federating, type: :boolean, - description: "Enable federation with other instances", - suggestions: [ - true, - false - ] + description: "Enable federation with other instances" }, %{ key: :federation_incoming_replies_max_depth, @@ -618,11 +587,7 @@ config :pleroma, :config_description, [ %{ key: :allow_relay, type: :boolean, - description: "Enable Pleroma's Relay, which makes it possible to follow a whole instance", - suggestions: [ - true, - false - ] + description: "Enable Pleroma's Relay, which makes it possible to follow a whole instance" }, %{ key: :rewrite_policy, @@ -638,11 +603,7 @@ config :pleroma, :config_description, [ type: :boolean, description: "Makes the client API in authentificated mode-only except for user-profiles." <> - " Useful for disabling the Local Timeline and The Whole Known Network", - suggestions: [ - true, - false - ] + " Useful for disabling the Local Timeline and The Whole Known Network" }, %{ key: :quarantined_instances, @@ -658,11 +619,7 @@ config :pleroma, :config_description, [ key: :managed_config, type: :boolean, description: - "Whenether the config for pleroma-fe is configured in this config or in static/config.json", - suggestions: [ - true, - false - ] + "Whenether the config for pleroma-fe is configured in this config or in static/config.json" }, %{ key: :static_dir, @@ -689,11 +646,7 @@ config :pleroma, :config_description, [ key: :mrf_transparency, type: :boolean, description: - "Make the content of your Message Rewrite Facility settings public (via nodeinfo)", - suggestions: [ - true, - false - ] + "Make the content of your Message Rewrite Facility settings public (via nodeinfo)" }, %{ key: :mrf_transparency_exclusions, @@ -709,11 +662,7 @@ config :pleroma, :config_description, [ type: :boolean, description: "Set to true to use extended local nicknames format (allows underscores/dashes)." <> - " This will break federation with older software for theses nicknames", - suggestions: [ - true, - false - ] + " This will break federation with older software for theses nicknames" }, %{ key: :max_pinned_statuses, @@ -741,11 +690,7 @@ config :pleroma, :config_description, [ key: :no_attachment_links, type: :boolean, description: - "Set to true to disable automatically adding attachment link text to statuses", - suggestions: [ - true, - false - ] + "Set to true to disable automatically adding attachment link text to statuses" }, %{ key: :welcome_message, @@ -780,20 +725,12 @@ config :pleroma, :config_description, [ description: "If set to true, only mentions at the beginning of a post will be used to address people in direct messages." <> " This is to prevent accidental mentioning of people when talking about them (e.g. \"@friend hey i really don't like @enemy\")." <> - " Default: false", - suggestions: [ - true, - false - ] + " Default: false" }, %{ key: :healthcheck, type: :boolean, - description: "If set to true, system data will be shown on /api/pleroma/healthcheck", - suggestions: [ - true, - false - ] + description: "If set to true, system data will be shown on /api/pleroma/healthcheck" }, %{ key: :remote_post_retention_days, @@ -823,11 +760,7 @@ config :pleroma, :config_description, [ %{ key: :skip_thread_containment, type: :boolean, - description: "Skip filter out broken threads. The default is true", - suggestions: [ - true, - false - ] + description: "Skip filter out broken threads. The default is true" }, %{ key: :limit_to_local_content, @@ -844,11 +777,7 @@ config :pleroma, :config_description, [ key: :dynamic_configuration, type: :boolean, description: - "Allow transferring configuration to DB with the subsequent customization from Admin api. Defaults to `false`", - suggestions: [ - true, - false - ] + "Allow transferring configuration to DB with the subsequent customization from Admin api. Defaults to `false`" }, %{ key: :max_account_fields, @@ -886,11 +815,7 @@ config :pleroma, :config_description, [ %{ key: :external_user_synchronization, type: :boolean, - description: "Enabling following/followers counters synchronization for external users", - suggestions: [ - true, - false - ] + description: "Enabling following/followers counters synchronization for external users" } ] }, @@ -1069,48 +994,40 @@ config :pleroma, :config_description, [ %{ key: :showInstanceSpecificPanel, type: :boolean, - description: "Whenether to show the instance's specific panel", - suggestions: [true, false] + description: "Whenether to show the instance's specific panel" }, %{ key: :scopeOptionsEnabled, type: :boolean, - description: "Enable setting an notice visibility and subject/CW when posting", - suggestions: [true, false] + description: "Enable setting an notice visibility and subject/CW when posting" }, %{ key: :formattingOptionsEnabled, type: :boolean, description: - "Enable setting a formatting different than plain-text (ie. HTML, Markdown) when posting, relates to :instance, allowed_post_formats", - suggestions: [true, false] + "Enable setting a formatting different than plain-text (ie. HTML, Markdown) when posting, relates to :instance, allowed_post_formats" }, %{ key: :collapseMessageWithSubject, type: :boolean, description: - "When a message has a subject(aka Content Warning), collapse it by default", - suggestions: [true, false] + "When a message has a subject(aka Content Warning), collapse it by default" }, %{ key: :hidePostStats, type: :boolean, - description: "Hide notices statistics(repeats, favorites, ...)", - suggestions: [true, false] + description: "Hide notices statistics(repeats, favorites, ...)" }, %{ key: :hideUserStats, type: :boolean, description: - "Hide profile statistics(posts, posts per day, followers, followings, ...)", - suggestions: [true, false] + "Hide profile statistics(posts, posts per day, followers, followings, ...)" }, %{ key: :scopeCopy, type: :boolean, - description: - "Copy the scope (private/unlisted/public) in replies to posts by default", - suggestions: [true, false] + description: "Copy the scope (private/unlisted/public) in replies to posts by default" }, %{ key: :subjectLineBehavior, @@ -1124,8 +1041,7 @@ config :pleroma, :config_description, [ %{ key: :alwaysShowSubjectInput, type: :boolean, - description: "When set to false, auto-hide the subject field when it's empty", - suggestions: [true, false] + description: "When set to false, auto-hide the subject field when it's empty" } ] }, @@ -1142,8 +1058,7 @@ config :pleroma, :config_description, [ %{ key: :showInstanceSpecificPanel, type: :boolean, - description: "Whenether to show the instance's specific panel", - suggestions: [true, false] + description: "Whenether to show the instance's specific panel" } ] } @@ -1271,14 +1186,12 @@ config :pleroma, :config_description, [ %{ key: :allow_followersonly, type: :boolean, - description: "whether to allow followers-only posts", - suggestions: [true, false] + description: "whether to allow followers-only posts" }, %{ key: :allow_direct, type: :boolean, - description: "whether to allow direct messages", - suggestions: [true, false] + description: "whether to allow direct messages" } ] }, @@ -1393,8 +1306,7 @@ config :pleroma, :config_description, [ %{ key: :enabled, type: :boolean, - description: "Enables proxying of remote media to the instance's proxy", - suggestions: [true, false] + description: "Enables proxying of remote media to the instance's proxy" }, %{ key: :base_url, @@ -1426,8 +1338,7 @@ config :pleroma, :config_description, [ %{ key: :enabled, type: :boolean, - description: "Enables the gopher interface", - suggestions: [true, false] + description: "Enables the gopher interface" }, %{ key: :ip, @@ -1601,8 +1512,7 @@ config :pleroma, :config_description, [ %{ key: :secure_cookie_flag, type: :boolean, - description: "", - suggestions: [true, false] + description: "" }, %{ key: :extra_cookie_attrs, @@ -1621,20 +1531,17 @@ config :pleroma, :config_description, [ %{ key: :unfollow_blocked, type: :boolean, - description: "Whether blocks result in people getting unfollowed", - suggestions: [true, false] + description: "Whether blocks result in people getting unfollowed" }, %{ key: :outgoing_blocks, type: :boolean, - description: "Whether to federate blocks to other instances", - suggestions: [true, false] + description: "Whether to federate blocks to other instances" }, %{ key: :sign_object_fetches, type: :boolean, - description: "Sign object fetches with HTTP signatures", - suggestions: [true, false] + description: "Sign object fetches with HTTP signatures" }, %{ key: :follow_handshake_timeout, @@ -1653,14 +1560,12 @@ config :pleroma, :config_description, [ %{ key: :enabled, type: :boolean, - description: "Whether the managed content security policy is enabled", - suggestions: [true, false] + description: "Whether the managed content security policy is enabled" }, %{ key: :sts, type: :boolean, - description: "Whether to additionally send a Strict-Transport-Security header", - suggestions: [true, false] + description: "Whether to additionally send a Strict-Transport-Security header" }, %{ key: :sts_max_age, @@ -1727,8 +1632,7 @@ config :pleroma, :config_description, [ %{ key: :enabled, type: :boolean, - description: "Whether the captcha should be shown on registration", - suggestions: [true, false] + description: "Whether the captcha should be shown on registration" }, %{ key: :method, @@ -1817,8 +1721,7 @@ config :pleroma, :config_description, [ %{ key: :verbose, type: :boolean, - description: "Logs verbose mode", - suggestions: [false, true] + description: "Logs verbose mode" }, %{ key: :prune, @@ -1937,11 +1840,7 @@ config :pleroma, :config_description, [ %{ key: :unfurl_nsfw, type: :boolean, - description: "If set to true nsfw attachments will be shown in previews", - suggestions: [ - true, - false - ] + description: "If set to true nsfw attachments will be shown in previews" } ] }, @@ -1955,8 +1854,7 @@ config :pleroma, :config_description, [ key: :enabled, type: :boolean, description: - "if enabled the instance will parse metadata from attached links to generate link previews", - suggestions: [true, false] + "if enabled the instance will parse metadata from attached links to generate link previews" }, %{ key: :ignore_hosts, @@ -1998,8 +1896,7 @@ config :pleroma, :config_description, [ key: :enabled, type: :boolean, description: - "if enabled, when a new user is federated with, fetch some of their latest posts", - suggestions: [true, false] + "if enabled, when a new user is federated with, fetch some of their latest posts" }, %{ key: :pages, @@ -2030,14 +1927,12 @@ config :pleroma, :config_description, [ %{ key: :new_window, type: :boolean, - description: "set to false to remove target='_blank' attribute", - suggestions: [true, false] + description: "set to false to remove target='_blank' attribute" }, %{ key: :scheme, type: :boolean, - description: "Set to true to link urls with schema http://google.com", - suggestions: [true, false] + description: "Set to true to link urls with schema http://google.com" }, %{ key: :truncate, @@ -2049,14 +1944,12 @@ config :pleroma, :config_description, [ %{ key: :strip_prefix, type: :boolean, - description: "Strip the scheme prefix", - suggestions: [true, false] + description: "Strip the scheme prefix" }, %{ key: :extra, type: :boolean, - description: "link urls with rarely used schemes (magnet, ipfs, irc, etc.)", - suggestions: [true, false] + description: "link urls with rarely used schemes (magnet, ipfs, irc, etc.)" } ] }, @@ -2083,8 +1976,7 @@ config :pleroma, :config_description, [ %{ key: :enabled, type: :boolean, - description: "whether scheduled activities are sent to the job queue to be executed", - suggestions: [true, false] + description: "whether scheduled activities are sent to the job queue to be executed" } ] }, @@ -2097,8 +1989,7 @@ config :pleroma, :config_description, [ %{ key: :enabled, type: :boolean, - description: "whether expired activities will be sent to the job queue to be deleted", - suggestions: [true, false] + description: "whether expired activities will be sent to the job queue to be deleted" } ] }, @@ -2128,8 +2019,7 @@ config :pleroma, :config_description, [ %{ key: :enabled, type: :boolean, - description: "enables LDAP authentication", - suggestions: [true, false] + description: "enables LDAP authentication" }, %{ key: :host, @@ -2146,8 +2036,7 @@ config :pleroma, :config_description, [ %{ key: :ssl, type: :boolean, - description: "true to use SSL, usually implies the port 636", - suggestions: [true, false] + description: "true to use SSL, usually implies the port 636" }, %{ key: :sslopts, @@ -2158,8 +2047,7 @@ config :pleroma, :config_description, [ %{ key: :tls, type: :boolean, - description: "true to start TLS, usually implies the port 389", - suggestions: [true, false] + description: "true to start TLS, usually implies the port 389" }, %{ key: :tlsopts, @@ -2237,8 +2125,7 @@ config :pleroma, :config_description, [ %{ key: :active, type: :boolean, - description: "globally enable or disable digest emails", - suggestions: [true, false] + description: "globally enable or disable digest emails" }, %{ key: :schedule, @@ -2346,14 +2233,12 @@ config :pleroma, :config_description, [ key: :issue_new_refresh_token, type: :boolean, description: - "Keeps old refresh token or generate new refresh token when to obtain an access token", - suggestions: [true, false] + "Keeps old refresh token or generate new refresh token when to obtain an access token" }, %{ key: :clean_expired_tokens, type: :boolean, - description: "Enable a background job to clean expired oauth tokens. Defaults to false", - suggestions: [true, false] + description: "Enable a background job to clean expired oauth tokens. Defaults to false" }, %{ key: :clean_expired_tokens_interval, @@ -2415,8 +2300,7 @@ config :pleroma, :config_description, [ %{ key: :rum_enabled, type: :boolean, - description: "If RUM indexes should be used. Defaults to false", - suggestions: [true, false] + description: "If RUM indexes should be used. Defaults to false" } ] }, @@ -2475,8 +2359,7 @@ config :pleroma, :config_description, [ %{ key: :enabled, type: :boolean, - description: "Enables ssh", - suggestions: [true, false] + description: "Enables ssh" }, %{ key: :priv_dir, @@ -2579,8 +2462,7 @@ config :pleroma, :config_description, [ %{ key: :enabled, type: :boolean, - description: "", - suggestions: [true, false] + description: "" } ] }, @@ -2593,8 +2475,7 @@ config :pleroma, :config_description, [ %{ key: :enabled, type: :boolean, - description: "Enables suggestions", - suggestions: [] + description: "Enables suggestions" }, %{ key: :third_party_engine, @@ -2686,8 +2567,7 @@ config :pleroma, :config_description, [ %{ key: :send_user_agent, type: :boolean, - description: "", - suggestions: [true, false] + description: "" }, %{ key: :adapter, @@ -2715,26 +2595,22 @@ config :pleroma, :config_description, [ %{ key: :allow_inline_images, type: :boolean, - description: "", - suggestions: [true, false] + description: "" }, %{ key: :allow_headings, type: :boolean, - description: "", - suggestions: [true, false] + description: "" }, %{ key: :allow_tables, type: :boolean, - description: "", - suggestions: [true, false] + description: "" }, %{ key: :allow_fonts, type: :boolean, - description: "", - suggestions: [true, false] + description: "" }, %{ key: :scrub_policy, @@ -2753,8 +2629,7 @@ config :pleroma, :config_description, [ %{ key: :deny_follow_blocked, type: :boolean, - description: "", - suggestions: [true, false] + description: "" } ] }, @@ -2854,8 +2729,7 @@ config :pleroma, :config_description, [ %{ key: :credentials, type: :boolean, - description: "", - suggestions: [true, false] + description: "" }, %{ key: :headers, From 7f211a48e0c443cbff90f028c5c92c496f66c62e Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 17 Sep 2019 21:43:27 +0200 Subject: [PATCH 105/138] docs/markdown.ex: child header as "- key (type): description" --- lib/pleroma/docs/markdown.ex | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/docs/markdown.ex b/lib/pleroma/docs/markdown.ex index 8386dc2f..58a42b32 100644 --- a/lib/pleroma/docs/markdown.ex +++ b/lib/pleroma/docs/markdown.ex @@ -44,6 +44,13 @@ defmodule Pleroma.Docs.Markdown do {:ok, config_path} end + defp print_child_header(file, child) do + IO.write( + file, + "- `#{inspect(child[:key])}` (`#{inspect(child[:type])}`): #{child[:description]}\n" + ) + end + defp print_suggestion(file, suggestion) when is_list(suggestion) do IO.write(file, " `#{inspect(suggestion)}`\n") end @@ -70,9 +77,4 @@ defmodule Pleroma.Docs.Markdown do print_suggestion(file, List.first(suggestions)) end end - - defp print_child_header(file, child) do - IO.write(file, "- `#{inspect(child[:key])}` -`#{inspect(child[:type])}` \n") - IO.write(file, "#{child[:description]} \n") - end end From e686f9be818ebddfba1aedcd9ae328d26147dca8 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Tue, 17 Sep 2019 20:04:57 +0000 Subject: [PATCH 106/138] Apply suggestion to CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2d14930..58618b7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Remove `Reply-To` header from report emails for admins. ### Changed -- **Breaking:** Now pleroma requires Elixir ~> 1.8 (it was ~> 1.7) +- **Breaking:** Elixir >=1.8 is now required (was >= 1.7) - **Breaking:** Configuration: A setting to explicitly disable the mailer was added, defaulting to true, if you are using a mailer add `config :pleroma, Pleroma.Emails.Mailer, enabled: true` to your config - **Breaking:** Configuration: `/media/` is now removed when `base_url` is configured, append `/media/` to your `base_url` config to keep the old behaviour if desired - **Breaking:** `/api/pleroma/notifications/read` is moved to `/api/v1/pleroma/notifications/read` and now supports `max_id` and responds with Mastodon API entities. From bf8567996c6149e5a7857ccf74697184adb42be9 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 17 Sep 2019 23:48:26 +0300 Subject: [PATCH 107/138] Sync develop changelog with master and create a new section for post-1.1 changes --- CHANGELOG.md | 92 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 62 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4426e3f7..1f0d55e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,24 +4,25 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] +### Changed +- **Breaking:** Elixir >=1.8 is now required (was >= 1.7) +- Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings) +- Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler +- Admin API: Return `total` when querying for reports + +## [1.1.0] - 2019-??-?? ### Security -- OStatus: eliminate the possibility of a protocol downgrade attack. -- OStatus: prevent following locked accounts, bypassing the approval process. - Mastodon API: respect post privacy in `/api/v1/statuses/:id/{favourited,reblogged}_by` ### Removed - **Breaking:** GNU Social API with Qvitter extensions support -- **Breaking:** ActivityPub: The `accept_blocks` configuration setting. - Emoji: Remove longfox emojis. - Remove `Reply-To` header from report emails for admins. ### Changed -- **Breaking:** Elixir >=1.8 is now required (was >= 1.7) - **Breaking:** Configuration: A setting to explicitly disable the mailer was added, defaulting to true, if you are using a mailer add `config :pleroma, Pleroma.Emails.Mailer, enabled: true` to your config - **Breaking:** Configuration: `/media/` is now removed when `base_url` is configured, append `/media/` to your `base_url` config to keep the old behaviour if desired - **Breaking:** `/api/pleroma/notifications/read` is moved to `/api/v1/pleroma/notifications/read` and now supports `max_id` and responds with Mastodon API entities. -- Configuration: OpenGraph and TwitterCard providers enabled by default -- Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text - Configuration: added `config/description.exs`, from which `docs/config.md` is generated - Federation: Return 403 errors when trying to request pages from a user's follower/following collections if they have `hide_followers`/`hide_follows` set - NodeInfo: Return `skipThreadContainment` in `metadata` for the `skip_thread_containment` option @@ -31,24 +32,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - AdminAPI: Add "godmode" while fetching user statuses (i.e. admin can see private statuses) - Improve digest email template – Pagination: (optional) return `total` alongside with `items` when paginating -- Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings) -- Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler -- Admin API: Return `total` when querying for reports ### Fixed - Following from Osada -- Not being able to pin unlisted posts -- Objects being re-embedded to activities after being updated (e.g faved/reposted). Running 'mix pleroma.database prune_objects' again is advised. - Favorites timeline doing database-intensive queries - Metadata rendering errors resulting in the entire page being inaccessible - `federation_incoming_replies_max_depth` option being ignored in certain cases -- Federation/MediaProxy not working with instances that have wrong certificate order - Mastodon API: Handling of search timeouts (`/api/v1/search` and `/api/v2/search`) - Mastodon API: Misskey's endless polls being unable to render - Mastodon API: Embedded relationships not being properly rendered in the Account entity of Status entity - Mastodon API: Notifications endpoint crashing if one notification failed to render -- Mastodon API: follower/following counters not being nullified, when `hide_follows`/`hide_followers` is set -- Mastodon API: `muted` in the Status entity, using author's account to determine if the tread was muted - Mastodon API: Add `account_id`, `type`, `offset`, and `limit` to search API (`/api/v1/search` and `/api/v2/search`) - Mastodon API, streaming: Fix filtering of notifications based on blocks/mutes/thread mutes - ActivityPub C2S: follower/following collection pages being inaccessible even when authentifucated if `hide_followers`/ `hide_follows` was set @@ -56,15 +49,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Rich Media: Parser failing when no TTL can be found by image TTL setters - Rich Media: The crawled URL is now spliced into the rich media data. - ActivityPub S2S: sharedInbox usage has been mostly aligned with the rules in the AP specification. -- ActivityPub S2S: remote user deletions now work the same as local user deletions. -- ActivityPub S2S: POST requests are now signed with `(request-target)` pseudo-header. -- Not being able to access the Mastodon FE login page on private instances -- Invalid SemVer version generation, when the current branch does not have commits ahead of tag/checked out on a tag - Pleroma.Upload base_url was not automatically whitelisted by MediaProxy. Now your custom CDN or file hosting will be accessed directly as expected. - Report email not being sent to admins when the reporter is a remote user -- MRF: ensure that subdomain_match calls are case-insensitive - Reverse Proxy limiting `max_body_length` was incorrectly defined and only checked `Content-Length` headers which may not be sufficient in some circumstances -- MRF: fix use of unserializable keyword lists in describe() implementations - ActivityPub: Deactivated user deletion - ActivityPub: Fix `/users/:nickname/inbox` crashing without an authenticated user - MRF: fix ability to follow a relay when AntiFollowbotPolicy was enabled @@ -75,16 +62,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: all status JSON responses contain a `pleroma.expires_at` item which states when an activity will expire. The value is only shown to the user who created the activity. To everyone else it's empty. - Configuration: `ActivityExpiration.enabled` controls whether expired activites will get deleted at the appropriate time. Enabled by default. - Conversations: Add Pleroma-specific conversation endpoints and status posting extensions. Run the `bump_all_conversations` task again to create the necessary data. -- **Breaking:** MRF describe API, which adds support for exposing configuration information about MRF policies to NodeInfo. - Custom modules will need to be updated by adding, at the very least, `def describe, do: {:ok, %{}}` to the MRF policy modules. - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) - MRF: Support for excluding specific domains from Transparency. - MRF: Support for filtering posts based on who they mention (`Pleroma.Web.ActivityPub.MRF.MentionPolicy`) -- MRF: Support for filtering posts based on ActivityStreams vocabulary (`Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`) -- MRF (Simple Policy): Support for wildcard domains. -- Support for wildcard domains in user domain blocks setting. -- Configuration: `quarantined_instances` support wildcard domains. -- Configuration: `federation_incoming_replies_max_depth` option - Mastodon API: Support for the [`tagged` filter](https://github.com/tootsuite/mastodon/pull/9755) in [`GET /api/v1/accounts/:id/statuses`](https://docs.joinmastodon.org/api/rest/accounts/#get-api-v1-accounts-id-statuses) - Mastodon API, streaming: Add support for passing the token in the `Sec-WebSocket-Protocol` header - Mastodon API, extension: Ability to reset avatar, profile banner, and background @@ -112,9 +92,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Admin API: Endpoint for fetching latest user's statuses - Pleroma API: Add `/api/v1/pleroma/accounts/confirmation_resend?email=` for resending account confirmation. - Pleroma API: Email change endpoint. -- Relays: Added a task to list relay subscriptions. -- Mix Tasks: `mix pleroma.database fix_likes_collections` -- Federation: Remove `likes` from objects. - Admin API: Added moderation log - Web response cache (currently, enabled for ActivityPub) - Mastodon API: Added an endpoint to get multiple statuses by IDs (`GET /api/v1/statuses/?ids[]=1&ids[]=2`) @@ -125,6 +102,61 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - RichMedia: parsers and their order are configured in `rich_media` config. - RichMedia: add the rich media ttl based on image expiration time. +## [1.0.6] - 2019-08-14 +### Fixed +- MRF: fix use of unserializable keyword lists in describe() implementations +- ActivityPub S2S: POST requests are now signed with `(request-target)` pseudo-header. + +## [1.0.5] - 2019-08-13 +### Fixed +- Mastodon API: follower/following counters not being nullified, when `hide_follows`/`hide_followers` is set +- Mastodon API: `muted` in the Status entity, using author's account to determine if the thread was muted +- Mastodon API: return the actual profile URL in the Account entity's `url` property when appropriate +- Templates: properly style anchor tags +- Objects being re-embedded to activities after being updated (e.g faved/reposted). Running 'mix pleroma.database prune_objects' again is advised. +- Not being able to access the Mastodon FE login page on private instances +- MRF: ensure that subdomain_match calls are case-insensitive +- Fix internal server error when using the healthcheck API. + +### Added +- **Breaking:** MRF describe API, which adds support for exposing configuration information about MRF policies to NodeInfo. + Custom modules will need to be updated by adding, at the very least, `def describe, do: {:ok, %{}}` to the MRF policy modules. +- Relays: Added a task to list relay subscriptions. +- MRF: Support for filtering posts based on ActivityStreams vocabulary (`Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`) +- MRF (Simple Policy): Support for wildcard domains. +- Support for wildcard domains in user domain blocks setting. +- Configuration: `quarantined_instances` support wildcard domains. +- Mix Tasks: `mix pleroma.database fix_likes_collections` +- Configuration: `federation_incoming_replies_max_depth` option + +### Removed +- Federation: Remove `likes` from objects. +- ActivityPub: The `accept_blocks` configuration setting. + +## [1.0.4] - 2019-08-01 +### Fixed +- Invalid SemVer version generation, when the current branch does not have commits ahead of tag/checked out on a tag + +## [1.0.3] - 2019-07-31 +### Security +- OStatus: eliminate the possibility of a protocol downgrade attack. +- OStatus: prevent following locked accounts, bypassing the approval process. +- TwitterAPI: use CommonAPI to handle remote follows instead of OStatus. + +## [1.0.2] - 2019-07-28 +### Fixed +- Not being able to pin unlisted posts +- Mastodon API: represent poll IDs as strings +- MediaProxy: fix matching filenames +- MediaProxy: fix filename encoding +- Migrations: fix a sporadic migration failure +- Metadata rendering errors resulting in the entire page being inaccessible +- Federation/MediaProxy not working with instances that have wrong certificate order +- ActivityPub S2S: remote user deletions now work the same as local user deletions. + +### Changed +- Configuration: OpenGraph and TwitterCard providers enabled by default +- Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text ## [1.0.1] - 2019-07-14 ### Security From 50ec445b2c1e45b0d3b3a2016650f3262ed00e75 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 17 Sep 2019 21:57:13 +0200 Subject: [PATCH 108/138] description.exs: remove empty strings and arrays --- config/description.exs | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/config/description.exs b/config/description.exs index 5ae32d0f..959d839b 100644 --- a/config/description.exs +++ b/config/description.exs @@ -109,8 +109,7 @@ config :pleroma, :config_description, [ type: :string, description: "If you use S3 compatible service such as Digital Ocean Spaces or CDN, set folder name or \"\" etc." <> - " For example, when using CDN to S3 virtual host format, set \"\". At this time, write CNAME to CDN in public_endpoint.", - suggestions: [""] + " For example, when using CDN to S3 virtual host format, set \"\". At this time, write CNAME to CDN in public_endpoint." } ] }, @@ -360,22 +359,19 @@ config :pleroma, :config_description, [ group: {:subgroup, Swoosh.Adapters.SocketLabs}, key: :server_id, type: :string, - description: "`Swoosh.Adapters.SocketLabs` adapter specific setting", - suggestions: [""] + description: "`Swoosh.Adapters.SocketLabs` adapter specific setting" }, %{ group: {:subgroup, Swoosh.Adapters.SocketLabs}, key: :api_key, type: :string, - description: "`Swoosh.Adapters.SocketLabs` adapter specific setting", - suggestions: [""] + description: "`Swoosh.Adapters.SocketLabs` adapter specific setting" }, %{ group: {:subgroup, Swoosh.Adapters.Gmail}, key: :access_token, type: :string, - description: "`Swoosh.Adapters.Gmail` adapter specific setting", - suggestions: [""] + description: "`Swoosh.Adapters.Gmail` adapter specific setting" } ] }, @@ -1682,8 +1678,7 @@ config :pleroma, :config_description, [ group: :pleroma_job_queue, key: :queues, type: :group, - description: "[Deprecated] Replaced with `Oban`/`:queues` (keeping the same format)", - children: [] + description: "[Deprecated] Replaced with `Oban`/`:queues` (keeping the same format)" }, %{ group: :pleroma, @@ -1694,8 +1689,7 @@ config :pleroma, :config_description, [ %{ key: :max_retries, type: :integer, - description: "[Deprecated] Replaced as `Oban`/`:queues`/`:outgoing_federation` value", - suggestions: [] + description: "[Deprecated] Replaced as `Oban`/`:queues`/`:outgoing_federation` value" } ] }, @@ -2041,8 +2035,7 @@ config :pleroma, :config_description, [ %{ key: :sslopts, type: :keyword, - description: "additional SSL options", - suggestions: [] + description: "additional SSL options" }, %{ key: :tls, @@ -2052,8 +2045,7 @@ config :pleroma, :config_description, [ %{ key: :tlsopts, type: :keyword, - description: "additional TLS options", - suggestions: [] + description: "additional TLS options" }, %{ key: :base, From e0d8c8897e46d20039b4c0a383bca0192c5eb2ec Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 17 Sep 2019 22:00:02 +0200 Subject: [PATCH 109/138] docs/markdown.ex: do no print empty suggestions --- lib/pleroma/docs/markdown.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pleroma/docs/markdown.ex b/lib/pleroma/docs/markdown.ex index 58a42b32..d7ca9795 100644 --- a/lib/pleroma/docs/markdown.ex +++ b/lib/pleroma/docs/markdown.ex @@ -66,6 +66,8 @@ defmodule Pleroma.Docs.Markdown do defp print_suggestions(_file, nil), do: nil + defp print_suggestions(_file, ""), do: nil + defp print_suggestions(file, suggestions) do IO.write(file, "Suggestions:\n") From 106afaed58da3a25d1c4593e13192ad2145643e4 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 17 Sep 2019 22:04:21 +0200 Subject: [PATCH 110/138] markdown.ex: do not fail if there is no children --- lib/pleroma/docs/markdown.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/docs/markdown.ex b/lib/pleroma/docs/markdown.ex index d7ca9795..20bd1c89 100644 --- a/lib/pleroma/docs/markdown.ex +++ b/lib/pleroma/docs/markdown.ex @@ -23,7 +23,7 @@ defmodule Pleroma.Docs.Markdown do IO.write(file, "#{group[:description]}\n") - for child <- group[:children] do + for child <- group[:children] || [] do print_child_header(file, child) print_suggestions(file, child[:suggestions]) From c0c56282007aff88a923bba4769af894cb6235af Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 17 Sep 2019 22:14:56 +0200 Subject: [PATCH 111/138] description.exs: remove empty strings on descriptions --- config/description.exs | 76 +++++------------------------------- lib/pleroma/docs/markdown.ex | 8 +++- 2 files changed, 15 insertions(+), 69 deletions(-) diff --git a/config/description.exs b/config/description.exs index 959d839b..821b7697 100644 --- a/config/description.exs +++ b/config/description.exs @@ -857,7 +857,6 @@ config :pleroma, :config_description, [ %{ key: :metadata, type: {:list, :atom}, - description: "", suggestions: [[:request_id]] } ] @@ -883,7 +882,6 @@ config :pleroma, :config_description, [ %{ key: :metadata, type: {:list, :atom}, - description: "", suggestions: [[:request_id]] } ] @@ -1177,7 +1175,6 @@ config :pleroma, :config_description, [ group: :pleroma, key: :mrf_rejectnonpublic, type: :group, - description: "", children: [ %{ key: :allow_followersonly, @@ -1444,43 +1441,36 @@ config :pleroma, :config_description, [ %{ key: :instrumenters, type: {:list, :module}, - description: "", suggestions: [Pleroma.Web.Endpoint.Instrumenter] }, %{ key: :protocol, type: :string, - description: "", suggestions: ["https"] }, %{ key: :secret_key_base, type: :string, - description: "", suggestions: ["aK4Abxf29xU9TTDKre9coZPUgevcVCFQJe/5xP/7Lt4BEif6idBIbjupVbOrbKxl"] }, %{ key: :signing_salt, type: :string, - description: "", suggestions: ["CqaoopA2"] }, %{ key: :render_errors, type: :keyword, - description: "", suggestions: [[view: Pleroma.Web.ErrorView, accepts: ~w(json)]], children: [ %{ key: :view, type: :module, - description: "", suggestions: [Pleroma.Web.ErrorView] }, %{ key: :accepts, type: {:list, :string}, - description: "", suggestions: ["json"] } ] @@ -1488,32 +1478,27 @@ config :pleroma, :config_description, [ %{ key: :pubsub, type: :keyword, - description: "", suggestions: [[name: Pleroma.PubSub, adapter: Phoenix.PubSub.PG2]], children: [ %{ key: :name, type: :module, - description: "", suggestions: [Pleroma.PubSub] }, %{ key: :adapter, type: :module, - description: "", suggestions: [Phoenix.PubSub.PG2] } ] }, %{ key: :secure_cookie_flag, - type: :boolean, - description: "" + type: :boolean }, %{ key: :extra_cookie_attrs, type: {:list, :string}, - description: "", suggestions: ["SameSite=Lax"] } ] @@ -1842,7 +1827,6 @@ config :pleroma, :config_description, [ group: :pleroma, key: :rich_media, type: :group, - description: "", children: [ %{ key: :enabled, @@ -1995,7 +1979,6 @@ config :pleroma, :config_description, [ %{ key: Pleroma.Web.Auth.Authenticator, type: :module, - description: "", suggestions: [Pleroma.Web.Auth.PleromaAuthenticator, Pleroma.Web.Auth.LDAPAuthenticator] } ] @@ -2172,37 +2155,31 @@ config :pleroma, :config_description, [ %{ key: :link_color, type: :string, - description: "", suggestions: ["#d8a070"] }, %{ key: :background_color, type: :string, - description: "", suggestions: ["#2C3645"] }, %{ key: :content_background_color, type: :string, - description: "", suggestions: ["#1B2635"] }, %{ key: :header_color, type: :string, - description: "", suggestions: ["#d8a070"] }, %{ key: :text_color, type: :string, - description: "", suggestions: ["#b9b9ba"] }, %{ key: :text_muted_color, type: :string, - description: "", suggestions: ["#b9b9ba"] } ] @@ -2245,7 +2222,6 @@ config :pleroma, :config_description, [ group: :pleroma, key: :emoji, type: :group, - description: "", children: [ %{ key: :shortcode_globs, @@ -2387,7 +2363,6 @@ config :pleroma, :config_description, [ %{ key: :types, type: :map, - description: "", suggestions: [ %{ "application/xml" => ["xml"], @@ -2401,31 +2376,26 @@ config :pleroma, :config_description, [ %{ key: "application/xml", type: {:list, :string}, - description: "", suggestions: [["xml"]] }, %{ key: "application/xrd+xml", type: {:list, :string}, - description: "", suggestions: [["xrd+xml"]] }, %{ key: "application/jrd+json", type: {:list, :string}, - description: "", suggestions: [["jrd+json"]] }, %{ key: "application/activity+json", type: {:list, :string}, - description: "", suggestions: [["activity+json"]] }, %{ key: "application/ld+json", type: {:list, :string}, - description: "", suggestions: [["activity+json"]] } ] @@ -2453,8 +2423,7 @@ config :pleroma, :config_description, [ children: [ %{ key: :enabled, - type: :boolean, - description: "" + type: :boolean } ] }, @@ -2462,7 +2431,6 @@ config :pleroma, :config_description, [ group: :pleroma, key: :suggestions, type: :group, - description: "", children: [ %{ key: :enabled, @@ -2492,7 +2460,6 @@ config :pleroma, :config_description, [ %{ key: :web, type: :string, - description: "", suggestions: ["https://vinayaka.distsn.org"] } ] @@ -2519,7 +2486,6 @@ config :pleroma, :config_description, [ %{ key: :adapter, type: :module, - description: "", suggestions: [Pleroma.Signature] } ] @@ -2528,18 +2494,15 @@ config :pleroma, :config_description, [ group: :pleroma, key: Pleroma.Uploaders.MDII, type: :group, - description: "", children: [ %{ key: :cgi, type: :string, - description: "", suggestions: ["https://mdii.sakura.ne.jp/mdii-post.cgi"] }, %{ key: :files, type: :string, - description: "", suggestions: ["https://mdii.sakura.ne.jp"] } ] @@ -2553,18 +2516,15 @@ config :pleroma, :config_description, [ %{ key: :proxy_url, type: [:string, :atom, nil], - description: "", suggestions: ["localhost:9020", {:socks5, :localhost, 3090}, nil] }, %{ key: :send_user_agent, - type: :boolean, - description: "" + type: :boolean }, %{ key: :adapter, type: :keyword, - description: "", suggestions: [ [ ssl_options: [ @@ -2582,32 +2542,26 @@ config :pleroma, :config_description, [ group: :pleroma, key: :markup, type: :group, - description: "", children: [ %{ key: :allow_inline_images, - type: :boolean, - description: "" + type: :boolean }, %{ key: :allow_headings, - type: :boolean, - description: "" + type: :boolean }, %{ key: :allow_tables, - type: :boolean, - description: "" + type: :boolean }, %{ key: :allow_fonts, - type: :boolean, - description: "" + type: :boolean }, %{ key: :scrub_policy, type: {:list, :module}, - description: "", suggestions: [[Pleroma.HTML.Transform.MediaProxy, Pleroma.HTML.Scrubber.Default]] } ] @@ -2616,12 +2570,10 @@ config :pleroma, :config_description, [ group: :pleroma, key: :user, type: :group, - description: "", children: [ %{ key: :deny_follow_blocked, - type: :boolean, - description: "" + type: :boolean } ] }, @@ -2629,12 +2581,10 @@ config :pleroma, :config_description, [ group: :pleroma, key: :mrf_normalize_markup, type: :group, - description: "", children: [ %{ key: :scrub_policy, type: :module, - description: "", suggestions: [Pleroma.HTML.Scrubber.Default] } ] @@ -2643,12 +2593,10 @@ config :pleroma, :config_description, [ group: :pleroma, key: Pleroma.User, type: :group, - description: "", children: [ %{ key: :restricted_nicknames, type: {:list, :string}, - description: "", suggestions: [ [ ".well-known", @@ -2689,24 +2637,20 @@ config :pleroma, :config_description, [ %{ group: :cors_plug, type: :group, - description: "", children: [ %{ key: :max_age, type: :integer, - description: "", suggestions: [86_400] }, %{ key: :methods, type: {:list, :string}, - description: "", suggestions: [["POST", "PUT", "DELETE", "GET", "PATCH", "OPTIONS"]] }, %{ key: :expose, type: :string, - description: "", suggestions: [ [ "Link", @@ -2720,13 +2664,11 @@ config :pleroma, :config_description, [ }, %{ key: :credentials, - type: :boolean, - description: "" + type: :boolean }, %{ key: :headers, type: {:list, :string}, - description: "", suggestions: [["Authorization", "Content-Type", "Idempotency-Key"]] } ] diff --git a/lib/pleroma/docs/markdown.ex b/lib/pleroma/docs/markdown.ex index 20bd1c89..739e4fce 100644 --- a/lib/pleroma/docs/markdown.ex +++ b/lib/pleroma/docs/markdown.ex @@ -44,13 +44,17 @@ defmodule Pleroma.Docs.Markdown do {:ok, config_path} end - defp print_child_header(file, child) do + defp print_child_header(file, %{key: key, type: type, description: description} = _child) do IO.write( file, - "- `#{inspect(child[:key])}` (`#{inspect(child[:type])}`): #{child[:description]}\n" + "- `#{inspect(key)}` (`#{inspect(type)}`): #{description}\n" ) end + defp print_child_header(file, %{key: key, type: type} = _child) do + IO.write(file, "- `#{inspect(key)}` (`#{inspect(type)}`)\n") + end + defp print_suggestion(file, suggestion) when is_list(suggestion) do IO.write(file, " `#{inspect(suggestion)}`\n") end From 32d64102cb2f8cf3b0f825f2ac0770563cfb457f Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 17 Sep 2019 22:19:30 +0200 Subject: [PATCH 112/138] description.exs: uncomment type for email logo --- config/description.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/description.exs b/config/description.exs index 821b7697..65ea6bf0 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2133,7 +2133,7 @@ config :pleroma, :config_description, [ children: [ %{ key: :logo, - # type: [:string, nil], + type: [:string, nil], description: "a path to a custom logo. Set it to nil to use the default Pleroma logo", suggestions: ["some/path/logo.png", nil] }, From d6182a3c8fef6377c20bb827a8e86bdac5bfb125 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 17 Sep 2019 22:22:54 +0200 Subject: [PATCH 113/138] markdown.ex: Make suggestion(s) plural only if on >1 --- lib/pleroma/docs/markdown.ex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/docs/markdown.ex b/lib/pleroma/docs/markdown.ex index 739e4fce..fc638906 100644 --- a/lib/pleroma/docs/markdown.ex +++ b/lib/pleroma/docs/markdown.ex @@ -73,13 +73,15 @@ defmodule Pleroma.Docs.Markdown do defp print_suggestions(_file, ""), do: nil defp print_suggestions(file, suggestions) do - IO.write(file, "Suggestions:\n") - if length(suggestions) > 1 do + IO.write(file, "Suggestions:\n") + for suggestion <- suggestions do print_suggestion(file, suggestion, true) end else + IO.write(file, "Suggestion:\n") + print_suggestion(file, List.first(suggestions)) end end From d2097fd0f5d5d6750de09243cb5720b161305790 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 17 Sep 2019 22:33:32 +0200 Subject: [PATCH 114/138] markdown.ex: \n\n on >1 suggestions, 2-spaces on one --- lib/pleroma/docs/markdown.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/docs/markdown.ex b/lib/pleroma/docs/markdown.ex index fc638906..280fe030 100644 --- a/lib/pleroma/docs/markdown.ex +++ b/lib/pleroma/docs/markdown.ex @@ -74,13 +74,13 @@ defmodule Pleroma.Docs.Markdown do defp print_suggestions(file, suggestions) do if length(suggestions) > 1 do - IO.write(file, "Suggestions:\n") + IO.write(file, "\n\nSuggestions:\n") for suggestion <- suggestions do print_suggestion(file, suggestion, true) end else - IO.write(file, "Suggestion:\n") + IO.write(file, " Suggestion: ") print_suggestion(file, List.first(suggestions)) end From 4785596a2cf638570b35afc91babbb0ac8309981 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 17 Sep 2019 22:55:29 +0200 Subject: [PATCH 115/138] markdown.ex: end suggestions list with a newline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Otherwise we end up with suggestion on the same level as the childs Markdown is a fuck… --- lib/pleroma/docs/markdown.ex | 2 ++ mix.exs | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/docs/markdown.ex b/lib/pleroma/docs/markdown.ex index 280fe030..27be1b09 100644 --- a/lib/pleroma/docs/markdown.ex +++ b/lib/pleroma/docs/markdown.ex @@ -79,6 +79,8 @@ defmodule Pleroma.Docs.Markdown do for suggestion <- suggestions do print_suggestion(file, suggestion, true) end + + IO.write(file, "\n") else IO.write(file, " Suggestion: ") diff --git a/mix.exs b/mix.exs index 58d1606d..e4fe5adf 100644 --- a/mix.exs +++ b/mix.exs @@ -174,7 +174,8 @@ defmodule Pleroma.Mixfile do "ecto.rollback": ["pleroma.ecto.rollback"], "ecto.setup": ["ecto.create", "ecto.migrate", "run priv/repo/seeds.exs"], "ecto.reset": ["ecto.drop", "ecto.setup"], - test: ["ecto.create --quiet", "ecto.migrate", "test"] + test: ["ecto.create --quiet", "ecto.migrate", "test"], + docs: ["pleroma.docs", "docs"] ] end From e501c822c98edb675b71b25d165fdf8df8447c27 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 17 Sep 2019 23:02:24 +0200 Subject: [PATCH 116/138] markdown.ex: put two-spaces before the description-newline --- lib/pleroma/docs/markdown.ex | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/docs/markdown.ex b/lib/pleroma/docs/markdown.ex index 27be1b09..68b10649 100644 --- a/lib/pleroma/docs/markdown.ex +++ b/lib/pleroma/docs/markdown.ex @@ -47,12 +47,12 @@ defmodule Pleroma.Docs.Markdown do defp print_child_header(file, %{key: key, type: type, description: description} = _child) do IO.write( file, - "- `#{inspect(key)}` (`#{inspect(type)}`): #{description}\n" + "- `#{inspect(key)}` (`#{inspect(type)}`): #{description} \n" ) end defp print_child_header(file, %{key: key, type: type} = _child) do - IO.write(file, "- `#{inspect(key)}` (`#{inspect(type)}`)\n") + IO.write(file, "- `#{inspect(key)}` (`#{inspect(type)}`) \n") end defp print_suggestion(file, suggestion) when is_list(suggestion) do @@ -74,13 +74,11 @@ defmodule Pleroma.Docs.Markdown do defp print_suggestions(file, suggestions) do if length(suggestions) > 1 do - IO.write(file, "\n\nSuggestions:\n") + IO.write(file, "Suggestions:\n") for suggestion <- suggestions do print_suggestion(file, suggestion, true) end - - IO.write(file, "\n") else IO.write(file, " Suggestion: ") From ea6d4137dac60ae23e15ab29901a84d8468baf3e Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 17 Sep 2019 21:24:21 +0000 Subject: [PATCH 117/138] Apply suggestion to CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f0d55e9..f3f38b81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -131,7 +131,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Removed - Federation: Remove `likes` from objects. -- ActivityPub: The `accept_blocks` configuration setting. +- **Breaking:** ActivityPub: The `accept_blocks` configuration setting. ## [1.0.4] - 2019-08-01 ### Fixed From 4faf2b1555f004664005e0efddb9815ebca4c5c7 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 6 Sep 2019 17:14:31 +0300 Subject: [PATCH 118/138] post for creating invite tokens in admin api --- CHANGELOG.md | 4 ++ docs/api/admin_api.md | 16 ++++++- .../web/admin_api/admin_api_controller.ex | 18 ++++++-- lib/pleroma/web/router.ex | 2 +- .../admin_api/admin_api_controller_test.exs | 46 +++++++------------ 5 files changed, 49 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3f38b81..a8342b16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **Breaking:** Configuration: A setting to explicitly disable the mailer was added, defaulting to true, if you are using a mailer add `config :pleroma, Pleroma.Emails.Mailer, enabled: true` to your config - **Breaking:** Configuration: `/media/` is now removed when `base_url` is configured, append `/media/` to your `base_url` config to keep the old behaviour if desired - **Breaking:** `/api/pleroma/notifications/read` is moved to `/api/v1/pleroma/notifications/read` and now supports `max_id` and responds with Mastodon API entities. +- **Breaking:** `/api/pleroma/admin/users/invite_token` now uses `POST`, changed accepted params and returns full invite in json instead of only token string. - Configuration: added `config/description.exs`, from which `docs/config.md` is generated +- Configuration: OpenGraph and TwitterCard providers enabled by default +- Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text +- Mastodon API: `pleroma.thread_muted` key in the Status entity - Federation: Return 403 errors when trying to request pages from a user's follower/following collections if they have `hide_followers`/`hide_follows` set - NodeInfo: Return `skipThreadContainment` in `metadata` for the `skip_thread_containment` option - NodeInfo: Return `mailerEnabled` in `metadata` diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index 9362e3d7..a8c75d93 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -226,13 +226,25 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret ### Get an account registration invite token -- Methods: `GET` +- Methods: `POST` - Params: - *optional* `invite` => [ - *optional* `max_use` (integer) - *optional* `expires_at` (date string e.g. "2019-04-07") ] -- Response: invite token (base64 string) +- Response: + +```json +{ + "id": integer, + "token": string, + "used": boolean, + "expires_at": date, + "uses": integer, + "max_use": integer, + "invite_type": string (possible values: `one_time`, `reusable`, `date_limited`, `reusable_date_limited`) +} +``` ## `/api/pleroma/admin/users/invites` diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 2a1cc59e..41ded734 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -402,11 +402,21 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do @doc "Get a account registeration invite token (base64 string)" def get_invite_token(conn, params) do - options = params["invite"] || %{} - {:ok, invite} = UserInviteToken.create_invite(options) + opts = %{} - conn - |> json(invite.token) + opts = + if params["max_use"], + do: Map.put(opts, :max_use, params["max_use"]), + else: opts + + opts = + if params["expires_at"], + do: Map.put(opts, :expires_at, params["expires_at"]), + else: opts + + {:ok, invite} = UserInviteToken.create_invite(opts) + + json(conn, AccountView.render("invite.json", %{invite: invite})) end @doc "Get list of created invites" diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 401133bf..5779d27d 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -180,7 +180,7 @@ defmodule Pleroma.Web.Router do post("/relay", AdminAPIController, :relay_follow) delete("/relay", AdminAPIController, :relay_unfollow) - get("/users/invite_token", AdminAPIController, :get_invite_token) + post("/users/invite_token", AdminAPIController, :get_invite_token) get("/users/invites", AdminAPIController, :invites) post("/users/revoke_invite", AdminAPIController, :revoke_invite) post("/users/email_invite", AdminAPIController, :email_invite) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index c497ea09..5f36d42e 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -574,18 +574,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do end end - test "/api/pleroma/admin/users/invite_token" do - admin = insert(:user, info: %{is_admin: true}) - - conn = - build_conn() - |> assign(:user, admin) - |> put_req_header("accept", "application/json") - |> get("/api/pleroma/admin/users/invite_token") - - assert conn.status == 200 - end - test "/api/pleroma/admin/users/:nickname/password_reset" do admin = insert(:user, info: %{is_admin: true}) user = insert(:user) @@ -1064,7 +1052,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do "@#{admin.nickname} deactivated user @#{user.nickname}" end - describe "GET /api/pleroma/admin/users/invite_token" do + describe "POST /api/pleroma/admin/users/invite_token" do setup do admin = insert(:user, info: %{is_admin: true}) @@ -1076,10 +1064,10 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do end test "without options", %{conn: conn} do - conn = get(conn, "/api/pleroma/admin/users/invite_token") + conn = post(conn, "/api/pleroma/admin/users/invite_token") - token = json_response(conn, 200) - invite = UserInviteToken.find_by_token!(token) + invite_json = json_response(conn, 200) + invite = UserInviteToken.find_by_token!(invite_json["token"]) refute invite.used refute invite.expires_at refute invite.max_use @@ -1088,12 +1076,12 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do test "with expires_at", %{conn: conn} do conn = - get(conn, "/api/pleroma/admin/users/invite_token", %{ - "invite" => %{"expires_at" => Date.to_string(Date.utc_today())} + post(conn, "/api/pleroma/admin/users/invite_token", %{ + "expires_at" => Date.to_string(Date.utc_today()) }) - token = json_response(conn, 200) - invite = UserInviteToken.find_by_token!(token) + invite_json = json_response(conn, 200) + invite = UserInviteToken.find_by_token!(invite_json["token"]) refute invite.used assert invite.expires_at == Date.utc_today() @@ -1102,13 +1090,10 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do end test "with max_use", %{conn: conn} do - conn = - get(conn, "/api/pleroma/admin/users/invite_token", %{ - "invite" => %{"max_use" => 150} - }) + conn = post(conn, "/api/pleroma/admin/users/invite_token", %{"max_use" => 150}) - token = json_response(conn, 200) - invite = UserInviteToken.find_by_token!(token) + invite_json = json_response(conn, 200) + invite = UserInviteToken.find_by_token!(invite_json["token"]) refute invite.used refute invite.expires_at assert invite.max_use == 150 @@ -1117,12 +1102,13 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do test "with max use and expires_at", %{conn: conn} do conn = - get(conn, "/api/pleroma/admin/users/invite_token", %{ - "invite" => %{"max_use" => 150, "expires_at" => Date.to_string(Date.utc_today())} + post(conn, "/api/pleroma/admin/users/invite_token", %{ + "max_use" => 150, + "expires_at" => Date.to_string(Date.utc_today()) }) - token = json_response(conn, 200) - invite = UserInviteToken.find_by_token!(token) + invite_json = json_response(conn, 200) + invite = UserInviteToken.find_by_token!(invite_json["token"]) refute invite.used assert invite.expires_at == Date.utc_today() assert invite.max_use == 150 From 2263c8b6b9260bee7dedeaff3d2ce955df12f08b Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 6 Sep 2019 17:20:44 +0300 Subject: [PATCH 119/138] little fixes --- lib/pleroma/web/admin_api/admin_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 41ded734..d25c21e3 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -400,7 +400,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do end end - @doc "Get a account registeration invite token (base64 string)" + @doc "Get an account registration invite token" def get_invite_token(conn, params) do opts = %{} From 17ff63b3c34e6e70580be98e71b353d1f0684222 Mon Sep 17 00:00:00 2001 From: Alex S Date: Sat, 7 Sep 2019 08:56:22 +0300 Subject: [PATCH 120/138] docs fix --- docs/api/admin_api.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index a8c75d93..577f802a 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -228,10 +228,8 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - Methods: `POST` - Params: - - *optional* `invite` => [ - - *optional* `max_use` (integer) - - *optional* `expires_at` (date string e.g. "2019-04-07") - ] + - *optional* `max_use` (integer) + - *optional* `expires_at` (date string e.g. "2019-04-07") - Response: ```json From a18f1e7cd7addf8aee9c56643f4f0531e1c5b5a0 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 13 Sep 2019 08:07:29 +0300 Subject: [PATCH 121/138] namings --- docs/api/admin_api.md | 2 +- lib/pleroma/web/admin_api/admin_api_controller.ex | 4 ++-- lib/pleroma/web/router.ex | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index 577f802a..7637fa0d 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -224,7 +224,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret ## `/api/pleroma/admin/users/invite_token` -### Get an account registration invite token +### Create an account registration invite token - Methods: `POST` - Params: diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index d25c21e3..8a8091da 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -400,8 +400,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do end end - @doc "Get an account registration invite token" - def get_invite_token(conn, params) do + @doc "Create an account registration invite token" + def create_invite_token(conn, params) do opts = %{} opts = diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 5779d27d..b9b85fd6 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -180,7 +180,7 @@ defmodule Pleroma.Web.Router do post("/relay", AdminAPIController, :relay_follow) delete("/relay", AdminAPIController, :relay_unfollow) - post("/users/invite_token", AdminAPIController, :get_invite_token) + post("/users/invite_token", AdminAPIController, :create_invite_token) get("/users/invites", AdminAPIController, :invites) post("/users/revoke_invite", AdminAPIController, :revoke_invite) post("/users/email_invite", AdminAPIController, :email_invite) From 7ef575d11e46247d1f64dd09d992e532cb8c5c37 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 18 Sep 2019 18:13:21 +0300 Subject: [PATCH 122/138] Initial poll refresh support Implement refreshing the object with an interval and call the function when getting the poll. --- lib/pleroma/object.ex | 18 ++++ lib/pleroma/object/fetcher.ex | 17 +++- .../controllers/mastodon_api_controller.ex | 2 +- test/fixtures/tesla_mock/poll_modified.json | 1 + test/fixtures/tesla_mock/poll_original.json | 1 + test/fixtures/tesla_mock/rin.json | 1 + test/object_test.exs | 86 +++++++++++++++++++ test/support/http_request_mock.ex | 4 + 8 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 test/fixtures/tesla_mock/poll_modified.json create mode 100644 test/fixtures/tesla_mock/poll_original.json create mode 100644 test/fixtures/tesla_mock/rin.json diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 5033798a..640e068e 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -38,6 +38,24 @@ defmodule Pleroma.Object do def get_by_id(nil), do: nil def get_by_id(id), do: Repo.get(Object, id) + def get_by_id_and_maybe_refetch(id, opts \\ []) do + %{updated_at: updated_at} = object = get_by_id(id) + + if opts[:interval] && + NaiveDateTime.diff(updated_at, NaiveDateTime.utc_now()) > opts[:interval] do + case Fetcher.refetch_object(object) do + {:ok, %Object{} = object} -> + object + + e -> + Logger.error("Couldn't refresh #{object.data["id"]}:\n#{inspect(e)}") + object + end + else + object + end + end + def get_by_ap_id(nil), do: nil def get_by_ap_id(ap_id) do diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index c1795ae0..da1ebd8b 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -7,17 +7,19 @@ defmodule Pleroma.Object.Fetcher do alias Pleroma.Object alias Pleroma.Object.Containment alias Pleroma.Signature + alias Pleroma.Repo alias Pleroma.Web.ActivityPub.InternalFetchActor alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.OStatus require Logger - defp reinject_object(data) do + defp reinject_object(struct, data) do Logger.debug("Reinjecting object #{data["id"]}") with data <- Transmogrifier.fix_object(data), - {:ok, object} <- Object.create(data) do + changeset <- Object.change(struct, %{data: data}), + {:ok, object} <- Repo.insert_or_update(changeset) do {:ok, object} else e -> @@ -26,6 +28,15 @@ defmodule Pleroma.Object.Fetcher do end end + def refetch_object(%Object{data: %{"id" => id}} = object) do + with {:ok, data} <- fetch_and_contain_remote_object_from_id(id), + {:ok, object} <- reinject_object(object, data) do + {:ok, object} + else + e -> {:error, e} + end + end + # TODO: # This will create a Create activity, which we need internally at the moment. def fetch_object_from_id(id, options \\ []) do @@ -57,7 +68,7 @@ defmodule Pleroma.Object.Fetcher do {:reject, nil} {:object, data, nil} -> - reinject_object(data) + reinject_object(%Object{}, data) {:normalize, object = %Object{}} -> {:ok, object} diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex index 060137b8..970cfd8d 100644 --- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex @@ -485,7 +485,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end def get_poll(%{assigns: %{user: user}} = conn, %{"id" => id}) do - with %Object{} = object <- Object.get_by_id(id), + with %Object{} = object <- Object.get_by_id_and_maybe_refetch(id, interval: 60), %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), true <- Visibility.visible_for_user?(activity, user) do conn diff --git a/test/fixtures/tesla_mock/poll_modified.json b/test/fixtures/tesla_mock/poll_modified.json new file mode 100644 index 00000000..1d026b59 --- /dev/null +++ b/test/fixtures/tesla_mock/poll_modified.json @@ -0,0 +1 @@ +{"@context":["https://www.w3.org/ns/activitystreams","https://patch.cx/schemas/litepub-0.1.jsonld",{"@language":"und"}],"actor":"https://patch.cx/users/rin","attachment":[],"attributedTo":"https://patch.cx/users/rin","cc":["https://patch.cx/users/rin/followers"],"closed":"2019-09-19T00:32:36.785333","content":"can you vote on this poll?","context":"https://patch.cx/contexts/626ecafd-3377-46c4-b908-3721a4d4373c","conversation":"https://patch.cx/contexts/626ecafd-3377-46c4-b908-3721a4d4373c","id":"https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d","oneOf":[{"name":"yes","replies":{"totalItems":8,"type":"Collection"},"type":"Note"},{"name":"no","replies":{"totalItems":3,"type":"Collection"},"type":"Note"}],"published":"2019-09-18T14:32:36.802152Z","sensitive":false,"summary":"","tag":[],"to":["https://www.w3.org/ns/activitystreams#Public"],"type":"Question"} \ No newline at end of file diff --git a/test/fixtures/tesla_mock/poll_original.json b/test/fixtures/tesla_mock/poll_original.json new file mode 100644 index 00000000..267876b3 --- /dev/null +++ b/test/fixtures/tesla_mock/poll_original.json @@ -0,0 +1 @@ +{"@context":["https://www.w3.org/ns/activitystreams","https://patch.cx/schemas/litepub-0.1.jsonld",{"@language":"und"}],"actor":"https://patch.cx/users/rin","attachment":[],"attributedTo":"https://patch.cx/users/rin","cc":["https://patch.cx/users/rin/followers"],"closed":"2019-09-19T00:32:36.785333","content":"can you vote on this poll?","context":"https://patch.cx/contexts/626ecafd-3377-46c4-b908-3721a4d4373c","conversation":"https://patch.cx/contexts/626ecafd-3377-46c4-b908-3721a4d4373c","id":"https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d","oneOf":[{"name":"yes","replies":{"totalItems":4,"type":"Collection"},"type":"Note"},{"name":"no","replies":{"totalItems":0,"type":"Collection"},"type":"Note"}],"published":"2019-09-18T14:32:36.802152Z","sensitive":false,"summary":"","tag":[],"to":["https://www.w3.org/ns/activitystreams#Public"],"type":"Question"} \ No newline at end of file diff --git a/test/fixtures/tesla_mock/rin.json b/test/fixtures/tesla_mock/rin.json new file mode 100644 index 00000000..2cf62376 --- /dev/null +++ b/test/fixtures/tesla_mock/rin.json @@ -0,0 +1 @@ +{"@context":["https://www.w3.org/ns/activitystreams","https://patch.cx/schemas/litepub-0.1.jsonld",{"@language":"und"}],"attachment":[],"endpoints":{"oauthAuthorizationEndpoint":"https://patch.cx/oauth/authorize","oauthRegistrationEndpoint":"https://patch.cx/api/v1/apps","oauthTokenEndpoint":"https://patch.cx/oauth/token","sharedInbox":"https://patch.cx/inbox"},"followers":"https://patch.cx/users/rin/followers","following":"https://patch.cx/users/rin/following","icon":{"type":"Image","url":"https://patch.cx/media/4e914f5b84e4a259a3f6c2d2edc9ab642f2ab05f3e3d9c52c81fc2d984b3d51e.jpg"},"id":"https://patch.cx/users/rin","image":{"type":"Image","url":"https://patch.cx/media/f739efddefeee49c6e67e947c4811fdc911785c16ae43da4c3684051fbf8da6a.jpg?name=f739efddefeee49c6e67e947c4811fdc911785c16ae43da4c3684051fbf8da6a.jpg"},"inbox":"https://patch.cx/users/rin/inbox","manuallyApprovesFollowers":false,"name":"rinpatch","outbox":"https://patch.cx/users/rin/outbox","preferredUsername":"rin","publicKey":{"id":"https://patch.cx/users/rin#main-key","owner":"https://patch.cx/users/rin","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5DLtwGXNZElJyxFGfcVc\nXANhaMadj/iYYQwZjOJTV9QsbtiNBeIK54PJrYuU0/0YIdrvS1iqheX5IwXRhcwa\nhm3ZyLz7XeN9st7FBni4BmZMBtMpxAuYuu5p/jbWy13qAiYOhPreCx0wrWgm/lBD\n9mkgaxIxPooBE0S4ZWEJIDIV1Vft3AWcRUyWW1vIBK0uZzs6GYshbQZB952S0yo4\nFzI1hABGHncH8UvuFauh4EZ8tY7/X5I0pGRnDOcRN1dAht5w5yTA+6r5kebiFQjP\nIzN/eCO/a9Flrj9YGW7HDNtjSOH0A31PLRGlJtJO3yK57dnf5ppyCZGfL4emShQo\ncQIDAQAB\n-----END PUBLIC KEY-----\n\n"},"summary":"your friendly neighborhood pleroma developer
I like cute things and distributed systems, and really hate delete and redrafts","tag":[],"type":"Person","url":"https://patch.cx/users/rin"} \ No newline at end of file diff --git a/test/object_test.exs b/test/object_test.exs index ba96aeea..72e36316 100644 --- a/test/object_test.exs +++ b/test/object_test.exs @@ -89,4 +89,90 @@ defmodule Pleroma.ObjectTest do ) end end + + describe "get_by_id_and_maybe_refetch" do + test "refetches if the time since the last refetch is greater than the interval" do + mock(fn + %{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} -> + %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/poll_original.json")} + + env -> + apply(HttpRequestMock, :request, [env]) + end) + + %Object{} = + object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d") + + assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4 + assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0 + + mock(fn + %{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} -> + %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/poll_modified.json")} + + env -> + apply(HttpRequestMock, :request, [env]) + end) + + updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1) + assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 8 + assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 3 + end + + test "returns the old object if refetch fails" do + mock(fn + %{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} -> + %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/poll_original.json")} + + env -> + apply(HttpRequestMock, :request, [env]) + end) + + %Object{} = + object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d") + + assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4 + assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0 + + mock(fn + %{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} -> + %Tesla.Env{status: 404, body: ""} + + env -> + apply(HttpRequestMock, :request, [env]) + end) + + updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1) + assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 4 + assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 0 + end + + test "does not refetch if the time since the last refetch is greater than the interval" do + mock(fn + %{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} -> + %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/poll_original.json")} + + env -> + apply(HttpRequestMock, :request, [env]) + end) + + %Object{} = + object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d") + + assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4 + assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0 + + mock(fn + %{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} -> + %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/poll_modified.json")} + + env -> + apply(HttpRequestMock, :request, [env]) + end) + + updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: 100) + assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 4 + assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 0 + end + end end diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index 231e7c49..833162a6 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -1004,6 +1004,10 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/sjw.json")}} end + def get("https://patch.cx/users/rin", _, _, _) do + {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/rin.json")}} + end + def get(url, query, body, headers) do {:error, "Mock response not implemented for GET #{inspect(url)}, #{query}, #{inspect(body)}, #{ From 84a40f6f266ea651578b3d641c000a3b762fc9f3 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 18 Sep 2019 18:31:24 +0300 Subject: [PATCH 123/138] Capture log in object refetch tests --- test/object_test.exs | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/test/object_test.exs b/test/object_test.exs index 72e36316..25e8d45d 100644 --- a/test/object_test.exs +++ b/test/object_test.exs @@ -4,6 +4,7 @@ defmodule Pleroma.ObjectTest do use Pleroma.DataCase + import ExUnit.CaptureLog import Pleroma.Factory import Tesla.Mock alias Pleroma.Object @@ -134,17 +135,23 @@ defmodule Pleroma.ObjectTest do assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4 assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0 - mock(fn - %{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} -> - %Tesla.Env{status: 404, body: ""} + assert capture_log(fn -> + mock(fn + %{ + method: :get, + url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d" + } -> + %Tesla.Env{status: 404, body: ""} - env -> - apply(HttpRequestMock, :request, [env]) - end) + env -> + apply(HttpRequestMock, :request, [env]) + end) - updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1) - assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 4 - assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 0 + updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1) + assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 4 + assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 0 + end) =~ + "[error] Couldn't refresh https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d" end test "does not refetch if the time since the last refetch is greater than the interval" do From a9c700ff1594bbd3c280dd6ac3a8dffa6ea7060b Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 18 Sep 2019 18:52:33 +0300 Subject: [PATCH 124/138] Fix wrong argument order when calling NaiveDateTime.diff --- lib/pleroma/object.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 640e068e..3fa40793 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -42,7 +42,7 @@ defmodule Pleroma.Object do %{updated_at: updated_at} = object = get_by_id(id) if opts[:interval] && - NaiveDateTime.diff(updated_at, NaiveDateTime.utc_now()) > opts[:interval] do + NaiveDateTime.diff(NaiveDateTime.utc_now(), updated_at) > opts[:interval] do case Fetcher.refetch_object(object) do {:ok, %Object{} = object} -> object From e3f902b3a1330f942ddaf6ff7b108bba8fc3120a Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 18 Sep 2019 19:07:25 +0300 Subject: [PATCH 125/138] Set updated_at even if the object stayed the same --- lib/pleroma/object/fetcher.ex | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index da1ebd8b..786e31cc 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -14,11 +14,20 @@ defmodule Pleroma.Object.Fetcher do require Logger + defp touch_changeset(changeset) do + updated_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.truncate(:second) + + Ecto.Changeset.put_change(changeset, :updated_at, updated_at) + end + defp reinject_object(struct, data) do Logger.debug("Reinjecting object #{data["id"]}") with data <- Transmogrifier.fix_object(data), changeset <- Object.change(struct, %{data: data}), + changeset <- touch_changeset(changeset), {:ok, object} <- Repo.insert_or_update(changeset) do {:ok, object} else From d32894ae512c1f4cff4d967b89a0772e105d456b Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 18 Sep 2019 19:24:20 +0300 Subject: [PATCH 126/138] Move object internal fields to a constant --- lib/pleroma/constants.ex | 12 ++++++++++++ lib/pleroma/web/activity_pub/transmogrifier.ex | 10 +--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index ef141854..0bf20cdd 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -6,4 +6,16 @@ defmodule Pleroma.Constants do use Const const(as_public, do: "https://www.w3.org/ns/activitystreams#Public") + + const(object_internal_fields, + do: [ + "likes", + "like_count", + "announcements", + "announcement_count", + "emoji", + "context_id", + "deleted_activity_id" + ] + ) end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 8461b666..9d2ddc1c 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -979,15 +979,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do defp strip_internal_fields(object) do object - |> Map.drop([ - "likes", - "like_count", - "announcements", - "announcement_count", - "emoji", - "context_id", - "deleted_activity_id" - ]) + |> Map.drop(Pleroma.Constants.object_internal_fields()) end defp strip_internal_tags(%{"tag" => tags} = object) do From eb87a86b5b3999f3e7ee119e839da3bd6d2ed4cf Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 18 Sep 2019 19:53:51 +0300 Subject: [PATCH 127/138] Preserve internal fields when reinjecting --- lib/pleroma/object/fetcher.ex | 10 ++++ test/object_test.exs | 102 ++++++++++++++++++++-------------- 2 files changed, 69 insertions(+), 43 deletions(-) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 786e31cc..fecc97c5 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -13,6 +13,7 @@ defmodule Pleroma.Object.Fetcher do alias Pleroma.Web.OStatus require Logger + require Pleroma.Constants defp touch_changeset(changeset) do updated_at = @@ -22,10 +23,19 @@ defmodule Pleroma.Object.Fetcher do Ecto.Changeset.put_change(changeset, :updated_at, updated_at) end + defp maybe_reinject_internal_fields(data, %{data: %{} = old_data}) do + internal_fields = Map.take(old_data, Pleroma.Constants.object_internal_fields()) + + Map.merge(data, internal_fields) + end + + defp maybe_reinject_internal_fields(data, _), do: data + defp reinject_object(struct, data) do Logger.debug("Reinjecting object #{data["id"]}") with data <- Transmogrifier.fix_object(data), + data <- maybe_reinject_internal_fields(data, struct), changeset <- Object.change(struct, %{data: data}), changeset <- touch_changeset(changeset), {:ok, object} <- Repo.insert_or_update(changeset) do diff --git a/test/object_test.exs b/test/object_test.exs index 25e8d45d..3d64fdb4 100644 --- a/test/object_test.exs +++ b/test/object_test.exs @@ -7,8 +7,10 @@ defmodule Pleroma.ObjectTest do import ExUnit.CaptureLog import Pleroma.Factory import Tesla.Mock + alias Pleroma.Activity alias Pleroma.Object alias Pleroma.Repo + alias Pleroma.Web.CommonAPI setup do mock(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -92,7 +94,7 @@ defmodule Pleroma.ObjectTest do end describe "get_by_id_and_maybe_refetch" do - test "refetches if the time since the last refetch is greater than the interval" do + setup do mock(fn %{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} -> %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/poll_original.json")} @@ -101,34 +103,41 @@ defmodule Pleroma.ObjectTest do apply(HttpRequestMock, :request, [env]) end) + mock_modified = fn resp -> + mock(fn + %{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} -> + resp + + env -> + apply(HttpRequestMock, :request, [env]) + end) + end + + on_exit(fn -> mock(fn env -> apply(HttpRequestMock, :request, [env]) end) end) + + [mock_modified: mock_modified] + end + + test "refetches if the time since the last refetch is greater than the interval", %{ + mock_modified: mock_modified + } do %Object{} = object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d") assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4 assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0 - mock(fn - %{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} -> - %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/poll_modified.json")} - - env -> - apply(HttpRequestMock, :request, [env]) - end) + mock_modified.(%Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/poll_modified.json") + }) updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1) assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 8 assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 3 end - test "returns the old object if refetch fails" do - mock(fn - %{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} -> - %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/poll_original.json")} - - env -> - apply(HttpRequestMock, :request, [env]) - end) - + test "returns the old object if refetch fails", %{mock_modified: mock_modified} do %Object{} = object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d") @@ -136,16 +145,7 @@ defmodule Pleroma.ObjectTest do assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0 assert capture_log(fn -> - mock(fn - %{ - method: :get, - url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d" - } -> - %Tesla.Env{status: 404, body: ""} - - env -> - apply(HttpRequestMock, :request, [env]) - end) + mock_modified.(%Tesla.Env{status: 404, body: ""}) updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1) assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 4 @@ -154,32 +154,48 @@ defmodule Pleroma.ObjectTest do "[error] Couldn't refresh https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d" end - test "does not refetch if the time since the last refetch is greater than the interval" do - mock(fn - %{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} -> - %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/poll_original.json")} - - env -> - apply(HttpRequestMock, :request, [env]) - end) - + test "does not refetch if the time since the last refetch is greater than the interval", %{ + mock_modified: mock_modified + } do %Object{} = object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d") assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4 assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0 - mock(fn - %{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} -> - %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/poll_modified.json")} - - env -> - apply(HttpRequestMock, :request, [env]) - end) + mock_modified.(%Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/poll_modified.json") + }) updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: 100) assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 4 assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 0 end + + test "preserves internal fields on refetch", %{mock_modified: mock_modified} do + %Object{} = + object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d") + + assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4 + assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0 + + user = insert(:user) + activity = Activity.get_create_by_object_ap_id(object.data["id"]) + {:ok, _activity, object} = CommonAPI.favorite(activity.id, user) + + assert object.data["like_count"] == 1 + + mock_modified.(%Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/poll_modified.json") + }) + + updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1) + assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 8 + assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 3 + + assert updated_object.data["like_count"] == 1 + end end end From c096dd86e5e4e3bdb9aa35c2c4f499efc17ddd16 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 18 Sep 2019 19:59:23 +0300 Subject: [PATCH 128/138] Do not refetch local objects --- lib/pleroma/object/fetcher.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index fecc97c5..91e6b6dc 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -48,10 +48,12 @@ defmodule Pleroma.Object.Fetcher do end def refetch_object(%Object{data: %{"id" => id}} = object) do - with {:ok, data} <- fetch_and_contain_remote_object_from_id(id), + with {:local, false} <- {:local, String.starts_with?(id, Pleroma.Web.base_url() <> "/")}, + {:ok, data} <- fetch_and_contain_remote_object_from_id(id), {:ok, object} <- reinject_object(object, data) do {:ok, object} else + {:local, true} -> object e -> {:error, e} end end From 5028b7b5780fbfd0904b2e48c05a05eeab0e623d Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 18 Sep 2019 22:09:03 +0300 Subject: [PATCH 129/138] Fix credo issues --- lib/pleroma/object/fetcher.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 91e6b6dc..cea33b5a 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -6,8 +6,8 @@ defmodule Pleroma.Object.Fetcher do alias Pleroma.HTTP alias Pleroma.Object alias Pleroma.Object.Containment - alias Pleroma.Signature alias Pleroma.Repo + alias Pleroma.Signature alias Pleroma.Web.ActivityPub.InternalFetchActor alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.OStatus From a12aeb09c42f1c120b78d517cfbad2fe29c88006 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 18 Sep 2019 23:34:13 +0300 Subject: [PATCH 130/138] Cleanup uploads after the tests are finished --- test/test_helper.exs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/test_helper.exs b/test/test_helper.exs index a927b2c3..6a389365 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -7,3 +7,8 @@ ExUnit.start(exclude: os_exclude) Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, :manual) Mox.defmock(Pleroma.ReverseProxy.ClientMock, for: Pleroma.ReverseProxy.Client) {:ok, _} = Application.ensure_all_started(:ex_machina) + +ExUnit.after_suite(fn _results -> + uploads = Pleroma.Config.get([Pleroma.Uploaders.Local, :uploads], "test/uploads") + File.rm_rf!(uploads) +end) From a22b87b30c8bf5a28465c732ac77c70631ddc4d9 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 19 Sep 2019 00:00:05 +0300 Subject: [PATCH 131/138] Add a changelog entry for poll refetching --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3f38b81..6a49bc4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] +### Added +- Refreshing poll results for remote polls ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) - Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings) From c3856bed0c4c177c3e6716d06d615a928d95f69c Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 18 Sep 2019 23:17:15 +0200 Subject: [PATCH 132/138] docs/clients.md: Update source code urls --- docs/clients.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/clients.md b/docs/clients.md index 9029361f..6c6180f7 100644 --- a/docs/clients.md +++ b/docs/clients.md @@ -39,7 +39,7 @@ Feel free to contact us to be added to this list! ### Nekonium - Homepage: [F-Droid Repository](https://repo.gdgd.jp.net/), [Google Play](https://play.google.com/store/apps/details?id=com.apps.nekonium), [Amazon](https://www.amazon.co.jp/dp/B076FXPRBC/) -- Source: +- Source: - Contact: [@lin@pleroma.gdgd.jp.net](https://pleroma.gdgd.jp.net/users/lin) - Platforms: Android - Features: Streaming Ready @@ -67,7 +67,7 @@ Feel free to contact us to be added to this list! ## Alternative Web Interfaces ### Brutaldon - Homepage: -- Source Code: +- Source Code: - Contact: [@gcupc@glitch.social](https://glitch.social/users/gcupc) - Features: No Streaming From 447514dfa2759e3415399412e82bf772ff119e04 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 18 Sep 2019 23:20:54 +0200 Subject: [PATCH 133/138] Bump copyright years of files changed in 2019 Done via the following command: git diff 1e6c102bfcfe0e4835a48f2483f2376f9bf86a20 --stat --name-only | cat - | xargs sed -i 's/2017-2018 Pleroma Authors/2017-2019 Pleroma Authors/' --- lib/mix/pleroma.ex | 2 +- lib/mix/tasks/pleroma/database.ex | 2 +- lib/mix/tasks/pleroma/ecto/ecto.ex | 2 +- lib/mix/tasks/pleroma/ecto/migrate.ex | 2 +- lib/mix/tasks/pleroma/ecto/rollback.ex | 2 +- lib/mix/tasks/pleroma/emoji.ex | 2 +- lib/mix/tasks/pleroma/instance.ex | 2 +- lib/mix/tasks/pleroma/relay.ex | 2 +- lib/mix/tasks/pleroma/uploads.ex | 2 +- lib/mix/tasks/pleroma/user.ex | 2 +- lib/pleroma/activity/queries.ex | 2 +- lib/pleroma/user/query.ex | 2 +- lib/pleroma/web/oauth/token/clean_worker.ex | 2 +- lib/pleroma/web/oauth/token/query.ex | 2 +- test/activity_test.exs | 2 +- test/captcha_test.exs | 2 +- test/config_test.exs | 2 +- test/daemons/activity_expiration_daemon_test.exs | 2 +- test/daemons/scheduled_activity_daemon_test.exs | 2 +- test/emails/admin_email_test.exs | 2 +- test/emails/mailer_test.exs | 2 +- test/emails/user_email_test.exs | 2 +- test/formatter_test.exs | 2 +- test/html_test.exs | 2 +- test/integration/mastodon_websocket_test.exs | 2 +- test/list_test.exs | 2 +- test/notification_test.exs | 2 +- test/object_test.exs | 2 +- test/plugs/authentication_plug_test.exs | 2 +- test/plugs/cache_control_test.exs | 2 +- test/plugs/ensure_public_or_authenticated_plug_test.exs | 2 +- test/plugs/http_security_plug_test.exs | 2 +- test/plugs/http_signature_plug_test.exs | 2 +- test/plugs/instance_static_test.exs | 2 +- test/plugs/legacy_authentication_plug_test.exs | 2 +- test/plugs/mapped_identity_to_signature_plug_test.exs | 2 +- test/plugs/oauth_plug_test.exs | 2 +- test/plugs/oauth_scopes_plug_test.exs | 2 +- test/plugs/set_format_plug_test.exs | 2 +- test/plugs/set_locale_plug_test.exs | 2 +- test/plugs/uploaded_media_plug_test.exs | 2 +- test/scheduled_activity_test.exs | 2 +- test/support/captcha_mock.ex | 2 +- test/support/conn_case.ex | 2 +- test/support/data_case.ex | 2 +- test/support/helpers.ex | 2 +- test/support/http_request_mock.ex | 2 +- test/support/mrf_module_mock.ex | 2 +- test/support/oban_helpers.ex | 2 +- test/support/web_push_http_client_mock.ex | 2 +- test/tasks/ecto/migrate_test.exs | 2 +- test/tasks/relay_test.exs | 2 +- test/tasks/user_test.exs | 2 +- test/test_helper.exs | 2 +- test/upload_test.exs | 2 +- test/user_search_test.exs | 2 +- test/user_test.exs | 2 +- test/web/activity_pub/activity_pub_controller_test.exs | 2 +- test/web/activity_pub/relay_test.exs | 2 +- test/web/activity_pub/transmogrifier/follow_handling_test.exs | 2 +- test/web/activity_pub/transmogrifier_test.exs | 2 +- test/web/admin_api/admin_api_controller_test.exs | 2 +- test/web/admin_api/search_test.exs | 2 +- test/web/common_api/common_api_utils_test.exs | 2 +- test/web/federator_test.exs | 2 +- test/web/instances/instance_test.exs | 2 +- test/web/instances/instances_test.exs | 2 +- test/web/mastodon_api/views/account_view_test.exs | 2 +- test/web/mastodon_api/views/list_view_test.exs | 2 +- test/web/mastodon_api/views/notification_view_test.exs | 2 +- test/web/mastodon_api/views/push_subscription_view_test.exs | 2 +- test/web/mastodon_api/views/scheduled_activity_view_test.exs | 2 +- test/web/mastodon_api/views/status_view_test.exs | 2 +- test/web/media_proxy/media_proxy_controller_test.exs | 2 +- test/web/media_proxy/media_proxy_test.exs | 2 +- test/web/node_info_test.exs | 2 +- test/web/oauth/authorization_test.exs | 2 +- test/web/oauth/oauth_controller_test.exs | 2 +- test/web/oauth/token/utils_test.exs | 2 +- test/web/oauth/token_test.exs | 2 +- test/web/ostatus/activity_representer_test.exs | 2 +- test/web/ostatus/feed_representer_test.exs | 2 +- test/web/ostatus/ostatus_controller_test.exs | 2 +- test/web/ostatus/ostatus_test.exs | 2 +- test/web/plugs/federating_plug_test.exs | 2 +- test/web/push/impl_test.exs | 2 +- test/web/salmon/salmon_test.exs | 2 +- test/web/streamer/streamer_test.exs | 2 +- test/web/twitter_api/twitter_api_test.exs | 2 +- test/web/uploader_controller_test.exs | 2 +- test/web/views/error_view_test.exs | 2 +- test/web/web_finger/web_finger_controller_test.exs | 2 +- test/web/web_finger/web_finger_test.exs | 2 +- test/web/websub/websub_controller_test.exs | 2 +- test/web/websub/websub_test.exs | 2 +- 95 files changed, 95 insertions(+), 95 deletions(-) diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index 1b758ea3..faeb30e1 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Pleroma do diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index bcc2052d..890a383d 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.Database do diff --git a/lib/mix/tasks/pleroma/ecto/ecto.ex b/lib/mix/tasks/pleroma/ecto/ecto.ex index b66f6337..36808b93 100644 --- a/lib/mix/tasks/pleroma/ecto/ecto.ex +++ b/lib/mix/tasks/pleroma/ecto/ecto.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-onl defmodule Mix.Tasks.Pleroma.Ecto do diff --git a/lib/mix/tasks/pleroma/ecto/migrate.ex b/lib/mix/tasks/pleroma/ecto/migrate.ex index 855c977f..d87b6957 100644 --- a/lib/mix/tasks/pleroma/ecto/migrate.ex +++ b/lib/mix/tasks/pleroma/ecto/migrate.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-onl defmodule Mix.Tasks.Pleroma.Ecto.Migrate do diff --git a/lib/mix/tasks/pleroma/ecto/rollback.ex b/lib/mix/tasks/pleroma/ecto/rollback.ex index 2ffb0901..a1af73fa 100644 --- a/lib/mix/tasks/pleroma/ecto/rollback.ex +++ b/lib/mix/tasks/pleroma/ecto/rollback.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-onl defmodule Mix.Tasks.Pleroma.Ecto.Rollback do diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index c2225af7..238d8dcd 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.Emoji do diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index b9b1991c..1a1634fe 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.Instance do diff --git a/lib/mix/tasks/pleroma/relay.ex b/lib/mix/tasks/pleroma/relay.ex index a738fae7..20072116 100644 --- a/lib/mix/tasks/pleroma/relay.ex +++ b/lib/mix/tasks/pleroma/relay.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.Relay do diff --git a/lib/mix/tasks/pleroma/uploads.ex b/lib/mix/tasks/pleroma/uploads.ex index be45383e..95392d81 100644 --- a/lib/mix/tasks/pleroma/uploads.ex +++ b/lib/mix/tasks/pleroma/uploads.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.Uploads do diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index a3f8bc94..eb005214 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.User do diff --git a/lib/pleroma/activity/queries.ex b/lib/pleroma/activity/queries.ex index 13fa3383..949f010a 100644 --- a/lib/pleroma/activity/queries.ex +++ b/lib/pleroma/activity/queries.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Activity.Queries do diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex index f9bcc9e1..2baf016c 100644 --- a/lib/pleroma/user/query.ex +++ b/lib/pleroma/user/query.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.User.Query do diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex index eb94bf86..f639f9c6 100644 --- a/lib/pleroma/web/oauth/token/clean_worker.ex +++ b/lib/pleroma/web/oauth/token/clean_worker.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.OAuth.Token.CleanWorker do diff --git a/lib/pleroma/web/oauth/token/query.ex b/lib/pleroma/web/oauth/token/query.ex index d92e1f07..9642103e 100644 --- a/lib/pleroma/web/oauth/token/query.ex +++ b/lib/pleroma/web/oauth/token/query.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.OAuth.Token.Query do diff --git a/test/activity_test.exs b/test/activity_test.exs index 6512d84a..95d9341c 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ActivityTest do diff --git a/test/captcha_test.exs b/test/captcha_test.exs index 7ca9a460..9f395d6b 100644 --- a/test/captcha_test.exs +++ b/test/captcha_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.CaptchaTest do diff --git a/test/config_test.exs b/test/config_test.exs index 73f3fcb0..438fe62e 100644 --- a/test/config_test.exs +++ b/test/config_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ConfigTest do diff --git a/test/daemons/activity_expiration_daemon_test.exs b/test/daemons/activity_expiration_daemon_test.exs index 31f4a70a..b51132fb 100644 --- a/test/daemons/activity_expiration_daemon_test.exs +++ b/test/daemons/activity_expiration_daemon_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ActivityExpirationWorkerTest do diff --git a/test/daemons/scheduled_activity_daemon_test.exs b/test/daemons/scheduled_activity_daemon_test.exs index 32820b2b..c8e46449 100644 --- a/test/daemons/scheduled_activity_daemon_test.exs +++ b/test/daemons/scheduled_activity_daemon_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ScheduledActivityDaemonTest do diff --git a/test/emails/admin_email_test.exs b/test/emails/admin_email_test.exs index 9e83c73c..31eac5f1 100644 --- a/test/emails/admin_email_test.exs +++ b/test/emails/admin_email_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Emails.AdminEmailTest do diff --git a/test/emails/mailer_test.exs b/test/emails/mailer_test.exs index ae5effb7..2425c85d 100644 --- a/test/emails/mailer_test.exs +++ b/test/emails/mailer_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Emails.MailerTest do diff --git a/test/emails/user_email_test.exs b/test/emails/user_email_test.exs index 7d8df6ab..963565f7 100644 --- a/test/emails/user_email_test.exs +++ b/test/emails/user_email_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Emails.UserEmailTest do diff --git a/test/formatter_test.exs b/test/formatter_test.exs index bfa67304..c443dfe7 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.FormatterTest do diff --git a/test/html_test.exs b/test/html_test.exs index b8906c46..306ad3b3 100644 --- a/test/html_test.exs +++ b/test/html_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTMLTest do diff --git a/test/integration/mastodon_websocket_test.exs b/test/integration/mastodon_websocket_test.exs index d02a3cc4..ed7ce8fe 100644 --- a/test/integration/mastodon_websocket_test.exs +++ b/test/integration/mastodon_websocket_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Integration.MastodonWebsocketTest do diff --git a/test/list_test.exs b/test/list_test.exs index 8efba75e..ba79251d 100644 --- a/test/list_test.exs +++ b/test/list_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ListTest do diff --git a/test/notification_test.exs b/test/notification_test.exs index 3d2f9a8f..54c0f987 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.NotificationTest do diff --git a/test/object_test.exs b/test/object_test.exs index ba96aeea..570213a6 100644 --- a/test/object_test.exs +++ b/test/object_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ObjectTest do diff --git a/test/plugs/authentication_plug_test.exs b/test/plugs/authentication_plug_test.exs index f7f8fd9f..9ae4c506 100644 --- a/test/plugs/authentication_plug_test.exs +++ b/test/plugs/authentication_plug_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Plugs.AuthenticationPlugTest do diff --git a/test/plugs/cache_control_test.exs b/test/plugs/cache_control_test.exs index 45151b28..69ce6cc7 100644 --- a/test/plugs/cache_control_test.exs +++ b/test/plugs/cache_control_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.CacheControlTest do diff --git a/test/plugs/ensure_public_or_authenticated_plug_test.exs b/test/plugs/ensure_public_or_authenticated_plug_test.exs index d45662a2..bae95e15 100644 --- a/test/plugs/ensure_public_or_authenticated_plug_test.exs +++ b/test/plugs/ensure_public_or_authenticated_plug_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Plugs.EnsurePublicOrAuthenticatedPlugTest do diff --git a/test/plugs/http_security_plug_test.exs b/test/plugs/http_security_plug_test.exs index 7a2835e3..9c1c2054 100644 --- a/test/plugs/http_security_plug_test.exs +++ b/test/plugs/http_security_plug_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do diff --git a/test/plugs/http_signature_plug_test.exs b/test/plugs/http_signature_plug_test.exs index d6fd9ea8..d8ace36d 100644 --- a/test/plugs/http_signature_plug_test.exs +++ b/test/plugs/http_signature_plug_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do diff --git a/test/plugs/instance_static_test.exs b/test/plugs/instance_static_test.exs index 6aabc45a..9b27246f 100644 --- a/test/plugs/instance_static_test.exs +++ b/test/plugs/instance_static_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RuntimeStaticPlugTest do diff --git a/test/plugs/legacy_authentication_plug_test.exs b/test/plugs/legacy_authentication_plug_test.exs index 9804e073..568ef5ab 100644 --- a/test/plugs/legacy_authentication_plug_test.exs +++ b/test/plugs/legacy_authentication_plug_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Plugs.LegacyAuthenticationPlugTest do diff --git a/test/plugs/mapped_identity_to_signature_plug_test.exs b/test/plugs/mapped_identity_to_signature_plug_test.exs index bb45d9ed..6b9d3649 100644 --- a/test/plugs/mapped_identity_to_signature_plug_test.exs +++ b/test/plugs/mapped_identity_to_signature_plug_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlugTest do diff --git a/test/plugs/oauth_plug_test.exs b/test/plugs/oauth_plug_test.exs index 5a2ed11c..dea11cdb 100644 --- a/test/plugs/oauth_plug_test.exs +++ b/test/plugs/oauth_plug_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Plugs.OAuthPlugTest do diff --git a/test/plugs/oauth_scopes_plug_test.exs b/test/plugs/oauth_scopes_plug_test.exs index f328026d..6a13ea81 100644 --- a/test/plugs/oauth_scopes_plug_test.exs +++ b/test/plugs/oauth_scopes_plug_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Plugs.OAuthScopesPlugTest do diff --git a/test/plugs/set_format_plug_test.exs b/test/plugs/set_format_plug_test.exs index bb21956b..27c026fd 100644 --- a/test/plugs/set_format_plug_test.exs +++ b/test/plugs/set_format_plug_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Plugs.SetFormatPlugTest do diff --git a/test/plugs/set_locale_plug_test.exs b/test/plugs/set_locale_plug_test.exs index b6c4c1ce..0aaeedc1 100644 --- a/test/plugs/set_locale_plug_test.exs +++ b/test/plugs/set_locale_plug_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Plugs.SetLocalePlugTest do diff --git a/test/plugs/uploaded_media_plug_test.exs b/test/plugs/uploaded_media_plug_test.exs index 49cf5396..5ba96313 100644 --- a/test/plugs/uploaded_media_plug_test.exs +++ b/test/plugs/uploaded_media_plug_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.UploadedMediaPlugTest do diff --git a/test/scheduled_activity_test.exs b/test/scheduled_activity_test.exs index edc7cc3f..dcf12fb4 100644 --- a/test/scheduled_activity_test.exs +++ b/test/scheduled_activity_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ScheduledActivityTest do diff --git a/test/support/captcha_mock.ex b/test/support/captcha_mock.ex index ef4e68bc..65ca6b3b 100644 --- a/test/support/captcha_mock.ex +++ b/test/support/captcha_mock.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Captcha.Mock do diff --git a/test/support/conn_case.ex b/test/support/conn_case.ex index b39c7067..9897f72c 100644 --- a/test/support/conn_case.ex +++ b/test/support/conn_case.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ConnCase do diff --git a/test/support/data_case.ex b/test/support/data_case.ex index 17fa1521..4ffcbac9 100644 --- a/test/support/data_case.ex +++ b/test/support/data_case.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.DataCase do diff --git a/test/support/helpers.ex b/test/support/helpers.ex index a601b3ec..ce39dd9d 100644 --- a/test/support/helpers.ex +++ b/test/support/helpers.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Tests.Helpers do diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index 231e7c49..6f988683 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule HttpRequestMock do diff --git a/test/support/mrf_module_mock.ex b/test/support/mrf_module_mock.ex index 12c7e22b..632c7ff1 100644 --- a/test/support/mrf_module_mock.ex +++ b/test/support/mrf_module_mock.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule MRFModuleMock do diff --git a/test/support/oban_helpers.ex b/test/support/oban_helpers.ex index 98977092..72792c06 100644 --- a/test/support/oban_helpers.ex +++ b/test/support/oban_helpers.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Tests.ObanHelpers do diff --git a/test/support/web_push_http_client_mock.ex b/test/support/web_push_http_client_mock.ex index d8accd21..1d6ccff7 100644 --- a/test/support/web_push_http_client_mock.ex +++ b/test/support/web_push_http_client_mock.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.WebPushHttpClientMock do diff --git a/test/tasks/ecto/migrate_test.exs b/test/tasks/ecto/migrate_test.exs index 0538a7b4..42f6cbf4 100644 --- a/test/tasks/ecto/migrate_test.exs +++ b/test/tasks/ecto/migrate_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-onl defmodule Mix.Tasks.Pleroma.Ecto.MigrateTest do diff --git a/test/tasks/relay_test.exs b/test/tasks/relay_test.exs index 7bde5660..c866608a 100644 --- a/test/tasks/relay_test.exs +++ b/test/tasks/relay_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.RelayTest do diff --git a/test/tasks/user_test.exs b/test/tasks/user_test.exs index 2b945304..cf12d9ed 100644 --- a/test/tasks/user_test.exs +++ b/test/tasks/user_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.UserTest do diff --git a/test/test_helper.exs b/test/test_helper.exs index a927b2c3..fb33e096 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only os_exclude = if :os.type() == {:unix, :darwin}, do: [skip_on_mac: true], else: [] diff --git a/test/upload_test.exs b/test/upload_test.exs index 6721fe82..0ca5ebce 100644 --- a/test/upload_test.exs +++ b/test/upload_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.UploadTest do diff --git a/test/user_search_test.exs b/test/user_search_test.exs index 48ce973a..f7ab3128 100644 --- a/test/user_search_test.exs +++ b/test/user_search_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.UserSearchTest do diff --git a/test/user_test.exs b/test/user_test.exs index b09e9311..39ba6966 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.UserTest do diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index f83b1445..9e8e420e 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do diff --git a/test/web/activity_pub/relay_test.exs b/test/web/activity_pub/relay_test.exs index 7315dce2..0f755653 100644 --- a/test/web/activity_pub/relay_test.exs +++ b/test/web/activity_pub/relay_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.RelayTest do diff --git a/test/web/activity_pub/transmogrifier/follow_handling_test.exs b/test/web/activity_pub/transmogrifier/follow_handling_test.exs index fe89f7cb..99ab573c 100644 --- a/test/web/activity_pub/transmogrifier/follow_handling_test.exs +++ b/test/web/activity_pub/transmogrifier/follow_handling_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.Transmogrifier.FollowHandlingTest do diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 6c296eb0..ebed65b7 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index c497ea09..41b4364f 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do diff --git a/test/web/admin_api/search_test.exs b/test/web/admin_api/search_test.exs index 501a8d00..9df4cd53 100644 --- a/test/web/admin_api/search_test.exs +++ b/test/web/admin_api/search_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.AdminAPI.SearchTest do diff --git a/test/web/common_api/common_api_utils_test.exs b/test/web/common_api/common_api_utils_test.exs index c281dd1f..23014645 100644 --- a/test/web/common_api/common_api_utils_test.exs +++ b/test/web/common_api/common_api_utils_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.CommonAPI.UtilsTest do diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index 4096d469..43a71570 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.FederatorTest do diff --git a/test/web/instances/instance_test.exs b/test/web/instances/instance_test.exs index 0b53bc6c..e54d708a 100644 --- a/test/web/instances/instance_test.exs +++ b/test/web/instances/instance_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Instances.InstanceTest do diff --git a/test/web/instances/instances_test.exs b/test/web/instances/instances_test.exs index dea8e2ae..65b03b15 100644 --- a/test/web/instances/instances_test.exs +++ b/test/web/instances/instances_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.InstancesTest do diff --git a/test/web/mastodon_api/views/account_view_test.exs b/test/web/mastodon_api/views/account_view_test.exs index 1d8b2833..2ea87c5f 100644 --- a/test/web/mastodon_api/views/account_view_test.exs +++ b/test/web/mastodon_api/views/account_view_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.MastodonAPI.AccountViewTest do diff --git a/test/web/mastodon_api/views/list_view_test.exs b/test/web/mastodon_api/views/list_view_test.exs index fb00310b..59e896a7 100644 --- a/test/web/mastodon_api/views/list_view_test.exs +++ b/test/web/mastodon_api/views/list_view_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.MastodonAPI.ListViewTest do diff --git a/test/web/mastodon_api/views/notification_view_test.exs b/test/web/mastodon_api/views/notification_view_test.exs index 977ea1e8..9231aaec 100644 --- a/test/web/mastodon_api/views/notification_view_test.exs +++ b/test/web/mastodon_api/views/notification_view_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do diff --git a/test/web/mastodon_api/views/push_subscription_view_test.exs b/test/web/mastodon_api/views/push_subscription_view_test.exs index dc935fc8..4e4f5b7e 100644 --- a/test/web/mastodon_api/views/push_subscription_view_test.exs +++ b/test/web/mastodon_api/views/push_subscription_view_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.MastodonAPI.PushSubscriptionViewTest do diff --git a/test/web/mastodon_api/views/scheduled_activity_view_test.exs b/test/web/mastodon_api/views/scheduled_activity_view_test.exs index ecbb855d..6387e455 100644 --- a/test/web/mastodon_api/views/scheduled_activity_view_test.exs +++ b/test/web/mastodon_api/views/scheduled_activity_view_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.MastodonAPI.ScheduledActivityViewTest do diff --git a/test/web/mastodon_api/views/status_view_test.exs b/test/web/mastodon_api/views/status_view_test.exs index fcdd7fbc..51f8434f 100644 --- a/test/web/mastodon_api/views/status_view_test.exs +++ b/test/web/mastodon_api/views/status_view_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.MastodonAPI.StatusViewTest do diff --git a/test/web/media_proxy/media_proxy_controller_test.exs b/test/web/media_proxy/media_proxy_controller_test.exs index 53b8f556..fdfdb5ec 100644 --- a/test/web/media_proxy/media_proxy_controller_test.exs +++ b/test/web/media_proxy/media_proxy_controller_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do diff --git a/test/web/media_proxy/media_proxy_test.exs b/test/web/media_proxy/media_proxy_test.exs index 79699cac..96bdde21 100644 --- a/test/web/media_proxy/media_proxy_test.exs +++ b/test/web/media_proxy/media_proxy_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.MediaProxyTest do diff --git a/test/web/node_info_test.exs b/test/web/node_info_test.exs index f6147c28..e15a0bff 100644 --- a/test/web/node_info_test.exs +++ b/test/web/node_info_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.NodeInfoTest do diff --git a/test/web/oauth/authorization_test.exs b/test/web/oauth/authorization_test.exs index d8b00843..2e82a7b7 100644 --- a/test/web/oauth/authorization_test.exs +++ b/test/web/oauth/authorization_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.OAuth.AuthorizationTest do diff --git a/test/web/oauth/oauth_controller_test.exs b/test/web/oauth/oauth_controller_test.exs index b492c779..2780e174 100644 --- a/test/web/oauth/oauth_controller_test.exs +++ b/test/web/oauth/oauth_controller_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.OAuth.OAuthControllerTest do diff --git a/test/web/oauth/token/utils_test.exs b/test/web/oauth/token/utils_test.exs index 20e338ca..dc1f9a98 100644 --- a/test/web/oauth/token/utils_test.exs +++ b/test/web/oauth/token/utils_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.OAuth.Token.UtilsTest do diff --git a/test/web/oauth/token_test.exs b/test/web/oauth/token_test.exs index 3c07309b..5359940f 100644 --- a/test/web/oauth/token_test.exs +++ b/test/web/oauth/token_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.OAuth.TokenTest do diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index a3a92ce5..a8d50089 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.OStatus.ActivityRepresenterTest do diff --git a/test/web/ostatus/feed_representer_test.exs b/test/web/ostatus/feed_representer_test.exs index 3c7b126e..d1cadf1e 100644 --- a/test/web/ostatus/feed_representer_test.exs +++ b/test/web/ostatus/feed_representer_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.OStatus.FeedRepresenterTest do diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs index 095ae704..ec96f001 100644 --- a/test/web/ostatus/ostatus_controller_test.exs +++ b/test/web/ostatus/ostatus_controller_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.OStatus.OStatusControllerTest do diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 803a9769..f04a5cfc 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.OStatusTest do diff --git a/test/web/plugs/federating_plug_test.exs b/test/web/plugs/federating_plug_test.exs index bb2e1687..9dcab93d 100644 --- a/test/web/plugs/federating_plug_test.exs +++ b/test/web/plugs/federating_plug_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.FederatingPlugTest do diff --git a/test/web/push/impl_test.exs b/test/web/push/impl_test.exs index e2f89f40..2f6ce4bd 100644 --- a/test/web/push/impl_test.exs +++ b/test/web/push/impl_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Push.ImplTest do diff --git a/test/web/salmon/salmon_test.exs b/test/web/salmon/salmon_test.exs index 0186f3fe..153ec41a 100644 --- a/test/web/salmon/salmon_test.exs +++ b/test/web/salmon/salmon_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Salmon.SalmonTest do diff --git a/test/web/streamer/streamer_test.exs b/test/web/streamer/streamer_test.exs index 88847e20..b8fcd41f 100644 --- a/test/web/streamer/streamer_test.exs +++ b/test/web/streamer/streamer_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.StreamerTest do diff --git a/test/web/twitter_api/twitter_api_test.exs b/test/web/twitter_api/twitter_api_test.exs index 3c052877..08f26443 100644 --- a/test/web/twitter_api/twitter_api_test.exs +++ b/test/web/twitter_api/twitter_api_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.TwitterAPI.TwitterAPITest do diff --git a/test/web/uploader_controller_test.exs b/test/web/uploader_controller_test.exs index 70028df1..7c7f9a6e 100644 --- a/test/web/uploader_controller_test.exs +++ b/test/web/uploader_controller_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.UploaderControllerTest do diff --git a/test/web/views/error_view_test.exs b/test/web/views/error_view_test.exs index 3857d585..4e5398c8 100644 --- a/test/web/views/error_view_test.exs +++ b/test/web/views/error_view_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ErrorViewTest do diff --git a/test/web/web_finger/web_finger_controller_test.exs b/test/web/web_finger/web_finger_controller_test.exs index bd3ccaaf..49cd1460 100644 --- a/test/web/web_finger/web_finger_controller_test.exs +++ b/test/web/web_finger/web_finger_controller_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.WebFinger.WebFingerControllerTest do diff --git a/test/web/web_finger/web_finger_test.exs b/test/web/web_finger/web_finger_test.exs index 8fdb9ade..696c1bd7 100644 --- a/test/web/web_finger/web_finger_test.exs +++ b/test/web/web_finger/web_finger_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.WebFingerTest do diff --git a/test/web/websub/websub_controller_test.exs b/test/web/websub/websub_controller_test.exs index 59cacbe6..f6d002b3 100644 --- a/test/web/websub/websub_controller_test.exs +++ b/test/web/websub/websub_controller_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Websub.WebsubControllerTest do diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index 929acf5a..46ca545d 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.WebsubTest do From fe5e0b784604b1352e98e7915c3c67d59ac4f709 Mon Sep 17 00:00:00 2001 From: eugenijm Date: Thu, 19 Sep 2019 08:27:55 +0300 Subject: [PATCH 134/138] Mastodon API: Return `pleroma.direct_conversation_id` when creating direct messages (`POST /api/v1/statuses`) --- CHANGELOG.md | 1 + .../mastodon_api/controllers/mastodon_api_controller.ex | 7 ++++++- test/web/mastodon_api/mastodon_api_controller_test.exs | 4 +++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 906aa985..84b64e2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings) - Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler - Admin API: Return `total` when querying for reports +- Mastodon API: Return `pleroma.direct_conversation_id` when creating a direct message (`POST /api/v1/statuses`) ## [1.1.0] - 2019-??-?? ### Security diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex index 37eeb2ac..6704ee7e 100644 --- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex @@ -611,7 +611,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do {:ok, activity} -> conn |> put_view(StatusView) - |> try_render("status.json", %{activity: activity, for: user, as: :activity}) + |> try_render("status.json", %{ + activity: activity, + for: user, + as: :activity, + with_direct_conversation_id: true + }) end end end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index fb04748b..35a0d3fe 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -296,7 +296,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do conn |> post("api/v1/statuses", %{"status" => content, "visibility" => "direct"}) - assert %{"id" => id, "visibility" => "direct"} = json_response(conn, 200) + assert %{"id" => id} = response = json_response(conn, 200) + assert response["visibility"] == "direct" + assert response["pleroma"]["direct_conversation_id"] assert activity = Activity.get_by_id(id) assert activity.recipients == [user2.ap_id, conn.assigns[:user].ap_id] assert activity.data["to"] == [user2.ap_id] From 0e6085da106cb966c340fac2d307d9e8e26e91ed Mon Sep 17 00:00:00 2001 From: D Anzorge Date: Thu, 19 Sep 2019 16:09:07 +0200 Subject: [PATCH 135/138] Fix pagination in AP outbox.json --- lib/pleroma/web/activity_pub/views/user_view.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 164b973d..a2f73e14 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -227,11 +227,12 @@ defmodule Pleroma.Web.ActivityPub.UserView do activities = ActivityPub.fetch_user_activities(user, nil, params) + # this is sorted chronologically, so first activity is the newest (max) {max_id, min_id, collection} = if length(activities) > 0 do { - Enum.at(Enum.reverse(activities), 0).id, Enum.at(activities, 0).id, + Enum.at(Enum.reverse(activities), 0).id, Enum.map(activities, fn act -> {:ok, data} = Transmogrifier.prepare_outgoing(act.data) data From 9aca2cc95d0d8886d35be17e5cdd683004b425d9 Mon Sep 17 00:00:00 2001 From: D Anzorge Date: Thu, 19 Sep 2019 16:09:24 +0200 Subject: [PATCH 136/138] Add test for correct AP outbox pagination --- .../web/activity_pub/views/user_view_test.exs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs index 2b4a04af..eda95e3e 100644 --- a/test/web/activity_pub/views/user_view_test.exs +++ b/test/web/activity_pub/views/user_view_test.exs @@ -142,4 +142,27 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do assert %{"totalItems" => 1} = UserView.render("following.json", %{user: user}) end end + + test "outbox paginates correctly" do + user = insert(:user) + + posts = + for i <- 0..25 do + {:ok, activity} = CommonAPI.post(user, %{"status" => "post #{i}"}) + activity + end + + # outbox sorts chronologically, newest first, with ten per page + posts = Enum.reverse(posts) + + %{"first" => %{"next" => next_url}} = + UserView.render("outbox.json", %{user: user, max_id: nil}) + + next_id = Enum.at(posts, 9).id + assert next_url =~ next_id + + %{"next" => next_url} = UserView.render("outbox.json", %{user: user, max_id: next_id}) + next_id = Enum.at(posts, 19).id + assert next_url =~ next_id + end end From fe4db3b94e71bafb913044de543472764671cd1a Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 19 Sep 2019 21:01:05 +0200 Subject: [PATCH 137/138] API Docs: Document conversation ids. --- docs/api/differences_in_mastoapi_responses.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index 3c7f5dad..d007a69c 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -21,7 +21,8 @@ Adding the parameter `with_muted=true` to the timeline queries will also return Has these additional fields under the `pleroma` object: - `local`: true if the post was made on the local instance -- `conversation_id`: the ID of the conversation the status is associated with (if any) +- `conversation_id`: the ID of the AP context the status is associated with (if any) +- `direct_conversation_id`: the ID of the Mastodon direct message conversation the status is associated with (if any) - `in_reply_to_account_acct`: the `acct` property of User entity for replied user (if any) - `content`: a map consisting of alternate representations of the `content` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain` - `spoiler_text`: a map consisting of alternate representations of the `spoiler_text` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain` From df3feb9412f1a6b9962aa5ad4a45e73aabc486d7 Mon Sep 17 00:00:00 2001 From: feld Date: Fri, 20 Sep 2019 13:21:07 +0000 Subject: [PATCH 138/138] Make it obvious how to support dual stack for MongooseIM --- installation/pleroma-mongooseim.cfg | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/installation/pleroma-mongooseim.cfg b/installation/pleroma-mongooseim.cfg index d7567321..576f8354 100755 --- a/installation/pleroma-mongooseim.cfg +++ b/installation/pleroma-mongooseim.cfg @@ -215,7 +215,9 @@ ]} ]}, - { 5222, ejabberd_c2s, [ + %% If you want dual stack, you have to clone this entire config stanza + %% and change the bind to "::" + { {5222, "0.0.0.0"}, ejabberd_c2s, [ %% %% If TLS is compiled in and you installed a SSL @@ -246,7 +248,9 @@ %% {max_stanza_size, 65536} %% ]}, - { 5269, ejabberd_s2s_in, [ + %% If you want dual stack, you have to clone this entire config stanza + %% and change the bind to "::" + { {5269, "0.0.0.0"}, ejabberd_s2s_in, [ {shaper, s2s_shaper}, {max_stanza_size, 131072}, {protocol_options, ["no_sslv3"]}