From 5c028b8f92aacb296afbd59130d848883f0c3a10 Mon Sep 17 00:00:00 2001 From: Sachin Joshi Date: Fri, 17 May 2019 12:20:31 +0545 Subject: [PATCH 01/83] user creation admin api will create multiple users --- CHANGELOG.md | 1 + .../web/admin_api/admin_api_controller.ex | 37 +++++---- .../web/admin_api/views/account_view.ex | 46 ++++++++++ lib/pleroma/web/router.ex | 2 +- .../admin_api/admin_api_controller_test.exs | 83 ++++++++++++++++++- 5 files changed, 149 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0e84928..5ee853c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Configuration: Added `extra_cookie_attrs` for setting non-standard cookie attributes. Defaults to ["SameSite=Lax"] so that remote follows work. - Timelines: Messages involving people you have blocked will be excluded from the timeline in all cases instead of just repeats. - Admin API: Move the user related API to `api/pleroma/admin/users` +- Admin API: `POST /api/pleroma/admin/users` will take list of users - Pleroma API: Support for emoji tags in `/api/pleroma/emoji` resulting in a breaking API change - Mastodon API: Support for `exclude_types`, `limit` and `min_id` in `/api/v1/notifications` - Mastodon API: Add `languages` and `registrations` to `/api/v1/instance` diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index e00b33ab..6048ed35 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -46,24 +46,31 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do |> json("ok") end - def user_create( - conn, - %{"nickname" => nickname, "email" => email, "password" => password} - ) do - user_data = %{ - nickname: nickname, - name: nickname, - email: email, - password: password, - password_confirmation: password, - bio: "." - } + def users_create(conn, %{"users" => users}) do + result = + Enum.map(users, fn %{"nickname" => nickname, "email" => email, "password" => password} -> + user_data = %{ + nickname: nickname, + name: nickname, + email: email, + password: password, + password_confirmation: password, + bio: "." + } - changeset = User.register_changeset(%User{}, user_data, need_confirmation: false) - {:ok, user} = User.register(changeset) + changeset = User.register_changeset(%User{}, user_data, need_confirmation: false) + + case User.register(changeset) do + {:ok, user} -> + AccountView.render("created.json", %{user: user}) + + {:error, changeset} -> + AccountView.render("create-error.json", %{changeset: changeset}) + end + end) conn - |> json(user.nickname) + |> json(result) end def user_show(conn, %{"nickname" => nickname}) do diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex index 28bb667d..e1825c5f 100644 --- a/lib/pleroma/web/admin_api/views/account_view.ex +++ b/lib/pleroma/web/admin_api/views/account_view.ex @@ -44,4 +44,50 @@ defmodule Pleroma.Web.AdminAPI.AccountView do invites: render_many(invites, AccountView, "invite.json", as: :invite) } end + + def render("created.json", %{user: user}) do + %{ + type: "success", + code: 201, + data: %{ + nickname: user.nickname, + email: user.email + } + } + end + + def render("create-error.json", %{changeset: %Ecto.Changeset{changes: changes, errors: errors}}) do + %{ + type: "error", + code: 409, + error: parse_error(errors), + data: %{ + nickname: Map.get(changes, :nickname), + email: Map.get(changes, :email) + } + } + end + + defp parse_error([]), do: "" + + defp parse_error(errors) do + ## when nickname is duplicate ap_id constraint error is raised + nickname_error = Keyword.get(errors, :nickname) || Keyword.get(errors, :ap_id) + email_error = Keyword.get(errors, :email) + password_error = Keyword.get(errors, :password) + + cond do + nickname_error -> + "nickname #{elem(nickname_error, 0)}" + + email_error -> + "email #{elem(email_error, 0)}" + + password_error -> + "password #{elem(password_error, 0)}" + + true -> + "" + end + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 7fef82f8..bbc2fda9 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -156,7 +156,7 @@ defmodule Pleroma.Web.Router do post("/user", AdminAPIController, :user_create) delete("/users", AdminAPIController, :user_delete) - post("/users", AdminAPIController, :user_create) + post("/users", AdminAPIController, :users_create) patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation) put("/users/tag", AdminAPIController, :tag_users) delete("/users/tag", AdminAPIController, :untag_users) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 6c1897b5..a0c9fd56 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -31,12 +31,87 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do |> assign(:user, admin) |> put_req_header("accept", "application/json") |> post("/api/pleroma/admin/users", %{ - "nickname" => "lain", - "email" => "lain@example.org", - "password" => "test" + "users" => [ + %{ + "nickname" => "lain", + "email" => "lain@example.org", + "password" => "test" + } + ] }) - assert json_response(conn, 200) == "lain" + assert json_response(conn, 200) == [ + %{ + "code" => 201, + "data" => %{ + "email" => "lain@example.org", + "nickname" => "lain" + }, + "type" => "success" + } + ] + end + + test "Cannot create user with exisiting email" do + admin = insert(:user, info: %{is_admin: true}) + user = insert(:user) + + conn = + build_conn() + |> assign(:user, admin) + |> put_req_header("accept", "application/json") + |> post("/api/pleroma/admin/users", %{ + "users" => [ + %{ + "nickname" => "lain", + "email" => user.email, + "password" => "test" + } + ] + }) + + assert json_response(conn, 200) == [ + %{ + "code" => 409, + "data" => %{ + "email" => user.email, + "nickname" => "lain" + }, + "error" => "email has already been taken", + "type" => "error" + } + ] + end + + test "Cannot create user with exisiting nickname" do + admin = insert(:user, info: %{is_admin: true}) + user = insert(:user) + + conn = + build_conn() + |> assign(:user, admin) + |> put_req_header("accept", "application/json") + |> post("/api/pleroma/admin/users", %{ + "users" => [ + %{ + "nickname" => user.nickname, + "email" => "someuser@plerama.social", + "password" => "test" + } + ] + }) + + assert json_response(conn, 200) == [ + %{ + "code" => 409, + "data" => %{ + "email" => "someuser@plerama.social", + "nickname" => user.nickname + }, + "error" => "nickname has already been taken", + "type" => "error" + } + ] end end From 5534d4c67675901ab272ee47355ad43dfae99033 Mon Sep 17 00:00:00 2001 From: Sachin Joshi Date: Sat, 1 Jun 2019 11:17:53 +0545 Subject: [PATCH 02/83] make bulk user creation from admin works as a transaction --- lib/pleroma/user.ex | 8 ++- .../web/admin_api/admin_api_controller.ex | 45 +++++++++---- .../web/admin_api/views/account_view.ex | 2 +- .../admin_api/admin_api_controller_test.exs | 66 ++++++++++++++++++- 4 files changed, 104 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index c6a562a6..722e8ff6 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -276,7 +276,13 @@ defmodule Pleroma.User do @doc "Inserts provided changeset, performs post-registration actions (confirmation email sending etc.)" def register(%Ecto.Changeset{} = changeset) do with {:ok, user} <- Repo.insert(changeset), - {:ok, user} <- autofollow_users(user), + {:ok, user} <- post_register_action(user) do + {:ok, user} + end + end + + def post_register_action(%User{} = user) do + with {:ok, user} <- autofollow_users(user), {:ok, user} <- set_cache(user), {:ok, _} <- Pleroma.User.WelcomeMessage.post_welcome_message_to_user(user), {:ok, _} <- try_send_confirmation_email(user) do diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 6048ed35..60fd4e57 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -47,7 +47,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do end def users_create(conn, %{"users" => users}) do - result = + changesets = Enum.map(users, fn %{"nickname" => nickname, "email" => email, "password" => password} -> user_data = %{ nickname: nickname, @@ -58,19 +58,40 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do bio: "." } - changeset = User.register_changeset(%User{}, user_data, need_confirmation: false) - - case User.register(changeset) do - {:ok, user} -> - AccountView.render("created.json", %{user: user}) - - {:error, changeset} -> - AccountView.render("create-error.json", %{changeset: changeset}) - end + User.register_changeset(%User{}, user_data, need_confirmation: false) + end) + |> Enum.reduce(Ecto.Multi.new(), fn changeset, multi -> + Ecto.Multi.insert(multi, Ecto.UUID.generate(), changeset) end) - conn - |> json(result) + case Pleroma.Repo.transaction(changesets) do + {:ok, users} -> + res = + users + |> Map.values() + |> Enum.map(fn user -> + {:ok, user} = User.post_register_action(user) + user + end) + |> Enum.map(&AccountView.render("created.json", %{user: &1})) + + conn + |> json(res) + + {:error, id, changeset, _} -> + res = + Enum.map(changesets.operations, fn + {current_id, {:changeset, _current_changeset, _}} when current_id == id -> + AccountView.render("create-error.json", %{changeset: changeset}) + + {_, {:changeset, current_changeset, _}} -> + AccountView.render("create-error.json", %{changeset: current_changeset}) + end) + + conn + |> put_status(:conflict) + |> json(res) + end end def user_show(conn, %{"nickname" => nickname}) do diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex index e1825c5f..cccdeff7 100644 --- a/lib/pleroma/web/admin_api/views/account_view.ex +++ b/lib/pleroma/web/admin_api/views/account_view.ex @@ -48,7 +48,7 @@ defmodule Pleroma.Web.AdminAPI.AccountView do def render("created.json", %{user: user}) do %{ type: "success", - code: 201, + code: 200, data: %{ nickname: user.nickname, email: user.email diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index a0c9fd56..01990513 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -36,18 +36,31 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do "nickname" => "lain", "email" => "lain@example.org", "password" => "test" + }, + %{ + "nickname" => "lain2", + "email" => "lain2@example.org", + "password" => "test" } ] }) assert json_response(conn, 200) == [ %{ - "code" => 201, + "code" => 200, "data" => %{ "email" => "lain@example.org", "nickname" => "lain" }, "type" => "success" + }, + %{ + "code" => 200, + "data" => %{ + "email" => "lain2@example.org", + "nickname" => "lain2" + }, + "type" => "success" } ] end @@ -70,7 +83,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do ] }) - assert json_response(conn, 200) == [ + assert json_response(conn, 409) == [ %{ "code" => 409, "data" => %{ @@ -101,7 +114,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do ] }) - assert json_response(conn, 200) == [ + assert json_response(conn, 409) == [ %{ "code" => 409, "data" => %{ @@ -113,6 +126,53 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do } ] end + + test "Multiple user creation works in transaction" do + admin = insert(:user, info: %{is_admin: true}) + user = insert(:user) + + conn = + build_conn() + |> assign(:user, admin) + |> put_req_header("accept", "application/json") + |> post("/api/pleroma/admin/users", %{ + "users" => [ + %{ + "nickname" => "newuser", + "email" => "newuser@pleroma.social", + "password" => "test" + }, + %{ + "nickname" => "lain", + "email" => user.email, + "password" => "test" + } + ] + }) + + assert json_response(conn, 409) == [ + %{ + "code" => 409, + "data" => %{ + "email" => user.email, + "nickname" => "lain" + }, + "error" => "email has already been taken", + "type" => "error" + }, + %{ + "code" => 409, + "data" => %{ + "email" => "newuser@pleroma.social", + "nickname" => "newuser" + }, + "error" => "", + "type" => "error" + } + ] + + assert User.get_by_nickname("newuser") === nil + end end describe "/api/pleroma/admin/users/:nickname" do From e394fc2eefdd7a4c7edd5fb3c04b445215d4a86c Mon Sep 17 00:00:00 2001 From: Sachin Joshi Date: Sun, 2 Jun 2019 09:48:45 +0545 Subject: [PATCH 03/83] fix the flaky test for users creation by admin --- .../admin_api/admin_api_controller_test.exs | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 9721a403..86b16024 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -47,24 +47,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do ] }) - assert json_response(conn, 200) == [ - %{ - "code" => 200, - "data" => %{ - "email" => "lain@example.org", - "nickname" => "lain" - }, - "type" => "success" - }, - %{ - "code" => 200, - "data" => %{ - "email" => "lain2@example.org", - "nickname" => "lain2" - }, - "type" => "success" - } - ] + response = json_response(conn, 200) |> Enum.map(&Map.get(&1, "type")) + assert response == ["success", "success"] end test "Cannot create user with exisiting email" do From 8ba7a151adf77c5cc47d6e1364a6078cc4bdef98 Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Mon, 22 Jul 2019 16:45:54 +0200 Subject: [PATCH 04/83] Cleanup: fix a comment --- test/web/mastodon_api/mastodon_api_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index ce2e4449..b5279412 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -405,7 +405,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do assert %{"visibility" => "direct"} = status assert status["url"] != direct.data["id"] - # User should be able to see his own direct message + # User should be able to see their own direct message res_conn = build_conn() |> assign(:user, user_one) From b72940277470c67802b979e4cab44f277e8fffb3 Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Mon, 22 Jul 2019 09:10:30 +0200 Subject: [PATCH 05/83] Make test.exs read config in the same way as dev.exs This way, if your test.secret.exs has an error, you'll actually see it. --- config/test.exs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/config/test.exs b/config/test.exs index 92dca18b..3f606aa8 100644 --- a/config/test.exs +++ b/config/test.exs @@ -82,11 +82,10 @@ IO.puts("RUM enabled: #{rum_enabled}") config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock -try do +if File.exists?("./config/test.secret.exs") do import_config "test.secret.exs" -rescue - _ -> - IO.puts( - "You may want to create test.secret.exs to declare custom database connection parameters." - ) +else + IO.puts( + "You may want to create test.secret.exs to declare custom database connection parameters." + ) end From 666514194a325e2463c05bae516b89d7c5f59316 Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Mon, 22 Jul 2019 14:16:20 +0200 Subject: [PATCH 06/83] Add activity expirations table Add a table to store activity expirations. An activity can have zero or one expirations. The expiration has a scheduled_at field which stores the time at which the activity should expire and be deleted. --- lib/pleroma/activity.ex | 3 ++ lib/pleroma/activity_expiration.ex | 31 +++++++++++++++++++ .../20190716100804_add_expirations_table.exs | 10 ++++++ test/activity_expiration_test.exs | 21 +++++++++++++ test/activity_test.exs | 9 ++++++ test/support/factory.ex | 19 +++++++++++- 6 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 lib/pleroma/activity_expiration.ex create mode 100644 priv/repo/migrations/20190716100804_add_expirations_table.exs create mode 100644 test/activity_expiration_test.exs diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 46552c7b..be485056 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Activity do use Ecto.Schema alias Pleroma.Activity + alias Pleroma.ActivityExpiration alias Pleroma.Bookmark alias Pleroma.Notification alias Pleroma.Object @@ -59,6 +60,8 @@ defmodule Pleroma.Activity do # typical case. has_one(:object, Object, on_delete: :nothing, foreign_key: :id) + has_one(:expiration, ActivityExpiration, on_delete: :delete_all) + timestamps() end diff --git a/lib/pleroma/activity_expiration.ex b/lib/pleroma/activity_expiration.ex new file mode 100644 index 00000000..d3d95f9e --- /dev/null +++ b/lib/pleroma/activity_expiration.ex @@ -0,0 +1,31 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ActivityExpiration do + use Ecto.Schema + + alias Pleroma.Activity + alias Pleroma.ActivityExpiration + alias Pleroma.FlakeId + alias Pleroma.Repo + + import Ecto.Query + + @type t :: %__MODULE__{} + + schema "activity_expirations" do + belongs_to(:activity, Activity, type: FlakeId) + field(:scheduled_at, :naive_datetime) + end + + def due_expirations(offset \\ 0) do + naive_datetime = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(offset, :millisecond) + + ActivityExpiration + |> where([exp], exp.scheduled_at < ^naive_datetime) + |> Repo.all() + end +end diff --git a/priv/repo/migrations/20190716100804_add_expirations_table.exs b/priv/repo/migrations/20190716100804_add_expirations_table.exs new file mode 100644 index 00000000..fbde8f9d --- /dev/null +++ b/priv/repo/migrations/20190716100804_add_expirations_table.exs @@ -0,0 +1,10 @@ +defmodule Pleroma.Repo.Migrations.AddExpirationsTable do + use Ecto.Migration + + def change do + create_if_not_exists table(:activity_expirations) do + add(:activity_id, references(:activities, type: :uuid, on_delete: :delete_all)) + add(:scheduled_at, :naive_datetime, null: false) + end + end +end diff --git a/test/activity_expiration_test.exs b/test/activity_expiration_test.exs new file mode 100644 index 00000000..20566a18 --- /dev/null +++ b/test/activity_expiration_test.exs @@ -0,0 +1,21 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ActivityExpirationTest do + use Pleroma.DataCase + alias Pleroma.ActivityExpiration + import Pleroma.Factory + + test "finds activities due to be deleted only" do + activity = insert(:note_activity) + expiration_due = insert(:expiration_in_the_past, %{activity_id: activity.id}) + activity2 = insert(:note_activity) + insert(:expiration_in_the_future, %{activity_id: activity2.id}) + + expirations = ActivityExpiration.due_expirations() + + assert length(expirations) == 1 + assert hd(expirations) == expiration_due + end +end diff --git a/test/activity_test.exs b/test/activity_test.exs index b27f6fd3..785c4b3c 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -164,4 +164,13 @@ defmodule Pleroma.ActivityTest do Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) end end + + test "add an activity with an expiration" do + activity = insert(:note_activity) + insert(:expiration_in_the_future, %{activity_id: activity.id}) + + Pleroma.ActivityExpiration + |> where([a], a.activity_id == ^activity.id) + |> Repo.one!() + end end diff --git a/test/support/factory.ex b/test/support/factory.ex index c751546c..7b52b132 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors +# Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Factory do @@ -142,6 +142,23 @@ defmodule Pleroma.Factory do |> Map.merge(attrs) end + defp expiration_offset_by_minutes(attrs, minutes) do + %Pleroma.ActivityExpiration{} + |> Map.merge(attrs) + |> Map.put( + :scheduled_at, + NaiveDateTime.add(NaiveDateTime.utc_now(), :timer.minutes(minutes), :millisecond) + ) + end + + def expiration_in_the_past_factory(attrs \\ %{}) do + expiration_offset_by_minutes(attrs, -60) + end + + def expiration_in_the_future_factory(attrs \\ %{}) do + expiration_offset_by_minutes(attrs, 60) + end + def article_activity_factory do article = insert(:article) From 378f5f0fbe21c2533719fed9afe8313586fda5d5 Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Mon, 22 Jul 2019 14:18:58 +0200 Subject: [PATCH 07/83] Add activity expiration worker This is a worker that runs every minute and deletes expired activities. It's based heavily on the scheduled activities worker. --- config/config.exs | 3 ++ docs/config.md | 4 ++ lib/pleroma/activity_expiration_worker.ex | 62 +++++++++++++++++++++++ lib/pleroma/application.ex | 4 ++ test/activity_expiration_worker_test.exs | 17 +++++++ 5 files changed, 90 insertions(+) create mode 100644 lib/pleroma/activity_expiration_worker.ex create mode 100644 test/activity_expiration_worker_test.exs diff --git a/config/config.exs b/config/config.exs index 56941186..2887353f 100644 --- a/config/config.exs +++ b/config/config.exs @@ -447,6 +447,7 @@ config :pleroma, Pleroma.Web.Federator.RetryQueue, max_retries: 5 config :pleroma_job_queue, :queues, + activity_expiration: 10, federator_incoming: 50, federator_outgoing: 50, web_push: 50, @@ -536,6 +537,8 @@ config :pleroma, :rate_limit, status_id_action: {60_000, 3}, password_reset: {1_800_000, 5} +config :pleroma, Pleroma.ActivityExpiration, enabled: true + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/docs/config.md b/docs/config.md index 02f86dc1..a20ed704 100644 --- a/docs/config.md +++ b/docs/config.md @@ -484,6 +484,10 @@ config :auto_linker, * `total_user_limit`: the number of scheduled activities a user is allowed to create in total (Default: `300`) * `enabled`: whether scheduled activities are sent to the job queue to be executed +## Pleroma.ActivityExpiration + +# `enabled`: whether expired activities will be sent to the job queue to be deleted + ## Pleroma.Web.Auth.Authenticator * `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator diff --git a/lib/pleroma/activity_expiration_worker.ex b/lib/pleroma/activity_expiration_worker.ex new file mode 100644 index 00000000..a341f58d --- /dev/null +++ b/lib/pleroma/activity_expiration_worker.ex @@ -0,0 +1,62 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ActivityExpirationWorker do + alias Pleroma.Activity + alias Pleroma.ActivityExpiration + alias Pleroma.Config + alias Pleroma.Repo + alias Pleroma.User + alias Pleroma.Web.CommonAPI + require Logger + use GenServer + import Ecto.Query + + @schedule_interval :timer.minutes(1) + + def start_link do + GenServer.start_link(__MODULE__, nil) + end + + @impl true + def init(_) do + if Config.get([ActivityExpiration, :enabled]) do + schedule_next() + {:ok, nil} + else + :ignore + end + end + + def perform(:execute, expiration_id) do + try do + expiration = + ActivityExpiration + |> where([e], e.id == ^expiration_id) + |> Repo.one!() + + activity = Activity.get_by_id_with_object(expiration.activity_id) + user = User.get_by_ap_id(activity.object.data["actor"]) + CommonAPI.delete(activity.id, user) + rescue + error -> + Logger.error("#{__MODULE__} Couldn't delete expired activity: #{inspect(error)}") + end + end + + @impl true + def handle_info(:perform, state) do + ActivityExpiration.due_expirations(@schedule_interval) + |> Enum.each(fn expiration -> + PleromaJobQueue.enqueue(:activity_expiration, __MODULE__, [:execute, expiration.id]) + end) + + schedule_next() + {:noreply, state} + end + + defp schedule_next do + Process.send_after(self(), :perform, @schedule_interval) + end +end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 03533149..42e4a1df 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -115,6 +115,10 @@ defmodule Pleroma.Application do %{ id: Pleroma.ScheduledActivityWorker, start: {Pleroma.ScheduledActivityWorker, :start_link, []} + }, + %{ + id: Pleroma.ActivityExpirationWorker, + start: {Pleroma.ActivityExpirationWorker, :start_link, []} } ] ++ hackney_pool_children() ++ diff --git a/test/activity_expiration_worker_test.exs b/test/activity_expiration_worker_test.exs new file mode 100644 index 00000000..939d912f --- /dev/null +++ b/test/activity_expiration_worker_test.exs @@ -0,0 +1,17 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ActivityExpirationWorkerTest do + use Pleroma.DataCase + alias Pleroma.Activity + import Pleroma.Factory + + test "deletes an activity" do + activity = insert(:note_activity) + expiration = insert(:expiration_in_the_past, %{activity_id: activity.id}) + Pleroma.ActivityExpirationWorker.perform(:execute, expiration.id) + + refute Repo.get(Activity, activity.id) + end +end From 704960b3c135d2e050308c68f5ccf5d7b7df40f8 Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Mon, 22 Jul 2019 16:46:20 +0200 Subject: [PATCH 08/83] Add support for activity expiration to common and Masto API The "expires_at" parameter accepts an ISO8601-formatted date which defines when the activity will expire. At this point the API will not give you any feedback about if your post will expire or not. --- docs/api/differences_in_mastoapi_responses.md | 1 + lib/pleroma/activity_expiration.ex | 19 ++++++++++++ lib/pleroma/web/common_api/common_api.ex | 29 +++++++++++++------ test/support/factory.ex | 10 ++++--- test/web/common_api/common_api_test.exs | 17 +++++++++++ .../mastodon_api_controller_test.exs | 19 ++++++++++++ 6 files changed, 82 insertions(+), 13 deletions(-) diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index 1907d70c..7d5be471 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -79,6 +79,7 @@ Additional parameters can be added to the JSON body/Form data: - `content_type`: string, contain the MIME type of the status, it is transformed into HTML by the backend. You can get the list of the supported MIME types with the nodeinfo endpoint. - `to`: A list of nicknames (like `lain@soykaf.club` or `lain` on the local server) that will be used to determine who is going to be addressed by this post. Using this will disable the implicit addressing by mentioned names in the `status` body, only the people in the `to` list will be addressed. The normal rules for for post visibility are not affected by this and will still apply. - `visibility`: string, besides standard MastoAPI values (`direct`, `private`, `unlisted` or `public`) it can be used to address a List by setting it to `list:LIST_ID`. +- `expires_on`: datetime (iso8601), sets when the posted activity should expire. When a posted activity expires it will be deleted from the server, and a delete request for it will be federated. ## PATCH `/api/v1/update_credentials` diff --git a/lib/pleroma/activity_expiration.ex b/lib/pleroma/activity_expiration.ex index d3d95f9e..a0af5255 100644 --- a/lib/pleroma/activity_expiration.ex +++ b/lib/pleroma/activity_expiration.ex @@ -10,6 +10,7 @@ defmodule Pleroma.ActivityExpiration do alias Pleroma.FlakeId alias Pleroma.Repo + import Ecto.Changeset import Ecto.Query @type t :: %__MODULE__{} @@ -19,6 +20,24 @@ defmodule Pleroma.ActivityExpiration do field(:scheduled_at, :naive_datetime) end + def changeset(%ActivityExpiration{} = expiration, attrs) do + expiration + |> cast(attrs, [:scheduled_at]) + |> validate_required([:scheduled_at]) + end + + def get_by_activity_id(activity_id) do + ActivityExpiration + |> where([exp], exp.activity_id == ^activity_id) + |> Repo.one() + end + + def create(%Activity{} = activity, scheduled_at) do + %ActivityExpiration{activity_id: activity.id} + |> changeset(%{scheduled_at: scheduled_at}) + |> Repo.insert() + end + def due_expirations(offset \\ 0) do naive_datetime = NaiveDateTime.utc_now() diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 44af6a77..0f287af4 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.Activity + alias Pleroma.ActivityExpiration alias Pleroma.Formatter alias Pleroma.Object alias Pleroma.ThreadMute @@ -218,6 +219,7 @@ defmodule Pleroma.Web.CommonAPI do context <- make_context(in_reply_to), cw <- data["spoiler_text"] || "", sensitive <- data["sensitive"] || Enum.member?(tags, {"#nsfw", "nsfw"}), + {:ok, expires_at} <- Ecto.Type.cast(:naive_datetime, data["expires_at"]), full_payload <- String.trim(status <> cw), :ok <- validate_character_limit(full_payload, attachments, limit), object <- @@ -243,15 +245,24 @@ defmodule Pleroma.Web.CommonAPI do preview? = Pleroma.Web.ControllerHelper.truthy_param?(data["preview"]) || false direct? = visibility == "direct" - %{ - to: to, - actor: user, - context: context, - object: object, - additional: %{"cc" => cc, "directMessage" => direct?} - } - |> maybe_add_list_data(user, visibility) - |> ActivityPub.create(preview?) + result = + %{ + to: to, + actor: user, + context: context, + object: object, + additional: %{"cc" => cc, "directMessage" => direct?} + } + |> maybe_add_list_data(user, visibility) + |> ActivityPub.create(preview?) + + if expires_at do + with {:ok, activity} <- result do + ActivityExpiration.create(activity, expires_at) + end + end + + result else {:private_to_public, true} -> {:error, dgettext("errors", "The message visibility must be direct")} diff --git a/test/support/factory.ex b/test/support/factory.ex index 7b52b132..63fe3a66 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -143,12 +143,14 @@ defmodule Pleroma.Factory do end defp expiration_offset_by_minutes(attrs, minutes) do + scheduled_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(minutes), :millisecond) + |> NaiveDateTime.truncate(:second) + %Pleroma.ActivityExpiration{} |> Map.merge(attrs) - |> Map.put( - :scheduled_at, - NaiveDateTime.add(NaiveDateTime.utc_now(), :timer.minutes(minutes), :millisecond) - ) + |> Map.put(:scheduled_at, scheduled_at) end def expiration_in_the_past_factory(attrs \\ %{}) do diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 16b3f121..210314a4 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -160,6 +160,23 @@ defmodule Pleroma.Web.CommonAPITest do Pleroma.Config.put([:instance, :limit], limit) end + + test "it can handle activities that expire" do + user = insert(:user) + + expires_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.truncate(:second) + |> NaiveDateTime.add(1_000_000, :second) + + expires_at_iso8601 = expires_at |> NaiveDateTime.to_iso8601() + + assert {:ok, activity} = + CommonAPI.post(user, %{"status" => "chai", "expires_at" => expires_at_iso8601}) + + assert expiration = Pleroma.ActivityExpiration.get_by_activity_id(activity.id) + assert expiration.scheduled_at == expires_at + end end describe "reactions" do diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index b5279412..24482a4a 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do alias Ecto.Changeset alias Pleroma.Activity + alias Pleroma.ActivityExpiration alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo @@ -151,6 +152,24 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do assert %{"id" => third_id} = json_response(conn_three, 200) refute id == third_id + + # An activity that will expire: + expires_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(120), :millisecond) + |> NaiveDateTime.truncate(:second) + + conn_four = + conn + |> post("api/v1/statuses", %{ + "status" => "oolong", + "expires_at" => expires_at + }) + + assert %{"id" => fourth_id} = json_response(conn_four, 200) + assert activity = Activity.get_by_id(fourth_id) + assert expiration = ActivityExpiration.get_by_activity_id(fourth_id) + assert expiration.scheduled_at == expires_at end test "replying to a status", %{conn: conn} do From 36012ef6c1dfea2489e61063e14783fa3fb52700 Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Tue, 23 Jul 2019 16:33:45 +0200 Subject: [PATCH 09/83] Require that ephemeral posts live for at least one hour If we didn't put some kind of lifetime requirement on these, I guess you could annoy people by sending large numbers of ephemeral posts that provoke notifications but then disappear before anyone can read them. --- lib/pleroma/activity_expiration.ex | 18 ++++++++++++++++++ lib/pleroma/web/common_api/common_api.ex | 14 ++++++++++++-- test/activity_expiration_test.exs | 6 ++++++ test/support/factory.ex | 2 +- 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/activity_expiration.ex b/lib/pleroma/activity_expiration.ex index a0af5255..bf57abca 100644 --- a/lib/pleroma/activity_expiration.ex +++ b/lib/pleroma/activity_expiration.ex @@ -14,6 +14,7 @@ defmodule Pleroma.ActivityExpiration do import Ecto.Query @type t :: %__MODULE__{} + @min_activity_lifetime :timer.hours(1) schema "activity_expirations" do belongs_to(:activity, Activity, type: FlakeId) @@ -24,6 +25,7 @@ defmodule Pleroma.ActivityExpiration do expiration |> cast(attrs, [:scheduled_at]) |> validate_required([:scheduled_at]) + |> validate_scheduled_at() end def get_by_activity_id(activity_id) do @@ -47,4 +49,20 @@ defmodule Pleroma.ActivityExpiration do |> where([exp], exp.scheduled_at < ^naive_datetime) |> Repo.all() end + + def validate_scheduled_at(changeset) do + validate_change(changeset, :scheduled_at, fn _, scheduled_at -> + if not expires_late_enough?(scheduled_at) do + [scheduled_at: "an ephemeral activity must live for at least one hour"] + else + [] + end + end) + end + + def expires_late_enough?(scheduled_at) do + now = NaiveDateTime.utc_now() + diff = NaiveDateTime.diff(scheduled_at, now, :millisecond) + diff >= @min_activity_lifetime + end end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 0f287af4..261d6039 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -196,6 +196,16 @@ defmodule Pleroma.Web.CommonAPI do end end + defp check_expiry_date(expiry_str) do + {:ok, expiry} = Ecto.Type.cast(:naive_datetime, expiry_str) + + if is_nil(expiry) || ActivityExpiration.expires_late_enough?(expiry) do + {:ok, expiry} + else + {:error, "Expiry date is too soon"} + end + end + def post(user, %{"status" => status} = data) do limit = Pleroma.Config.get([:instance, :limit]) @@ -219,7 +229,7 @@ defmodule Pleroma.Web.CommonAPI do context <- make_context(in_reply_to), cw <- data["spoiler_text"] || "", sensitive <- data["sensitive"] || Enum.member?(tags, {"#nsfw", "nsfw"}), - {:ok, expires_at} <- Ecto.Type.cast(:naive_datetime, data["expires_at"]), + {:ok, expires_at} <- check_expiry_date(data["expires_at"]), full_payload <- String.trim(status <> cw), :ok <- validate_character_limit(full_payload, attachments, limit), object <- @@ -258,7 +268,7 @@ defmodule Pleroma.Web.CommonAPI do if expires_at do with {:ok, activity} <- result do - ActivityExpiration.create(activity, expires_at) + {:ok, _} = ActivityExpiration.create(activity, expires_at) end end diff --git a/test/activity_expiration_test.exs b/test/activity_expiration_test.exs index 20566a18..4948fae1 100644 --- a/test/activity_expiration_test.exs +++ b/test/activity_expiration_test.exs @@ -18,4 +18,10 @@ defmodule Pleroma.ActivityExpirationTest do assert length(expirations) == 1 assert hd(expirations) == expiration_due end + + test "denies expirations that don't live long enough" do + activity = insert(:note_activity) + now = NaiveDateTime.utc_now() + assert {:error, _} = ActivityExpiration.create(activity, now) + end end diff --git a/test/support/factory.ex b/test/support/factory.ex index 63fe3a66..7a2ddcad 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -158,7 +158,7 @@ defmodule Pleroma.Factory do end def expiration_in_the_future_factory(attrs \\ %{}) do - expiration_offset_by_minutes(attrs, 60) + expiration_offset_by_minutes(attrs, 61) end def article_activity_factory do From 3cb471ec0688b81c8ef37dd27f2b82e6c858431f Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Wed, 24 Jul 2019 12:43:20 +0200 Subject: [PATCH 10/83] Expose expires_at datetime in mastoAPI only for the activity actor In the "pleroma" section of the MastoAPI for status activities you can see an expires_at item that states when the activity will expire, or nothing if the activity will not expire. The expires_at date is only visible to the person who posted the activity. This is the conservative approach in case some attacker decides to write a logger for expiring posts. However, in the future of OCAP, signed requests, and all that stuff, this attack might not be that likely. Some other pleroma dev should remove the restriction in the code at that time, if they're satisfied with the security implications of doing so. --- docs/api/differences_in_mastoapi_responses.md | 1 + lib/pleroma/web/mastodon_api/views/status_view.ex | 13 ++++++++++++- .../mastodon_api/mastodon_api_controller_test.exs | 3 ++- test/web/mastodon_api/status_view_test.exs | 3 ++- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index 7d5be471..168a13f4 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -25,6 +25,7 @@ Has these additional fields under the `pleroma` object: - `in_reply_to_account_acct`: the `acct` property of User entity for replied user (if any) - `content`: a map consisting of alternate representations of the `content` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain` - `spoiler_text`: a map consisting of alternate representations of the `spoiler_text` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain` +- `expires_on`: a datetime (iso8601) that states when the post will expire (be deleted automatically), or empty if the post won't expire ## Attachments diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index de942595..7264dcaf 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do use Pleroma.Web, :view alias Pleroma.Activity + alias Pleroma.ActivityExpiration alias Pleroma.HTML alias Pleroma.Object alias Pleroma.Repo @@ -165,6 +166,15 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do bookmarked = Activity.get_bookmark(activity, opts[:for]) != nil + client_posted_this_activity = opts[:for] && user.id == opts[:for].id + + expires_at = + with true <- client_posted_this_activity, + expiration when not is_nil(expiration) <- + ActivityExpiration.get_by_activity_id(activity.id) do + expiration.scheduled_at + end + thread_muted? = case activity.thread_muted? do thread_muted? when is_boolean(thread_muted?) -> thread_muted? @@ -262,7 +272,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do conversation_id: get_context_id(activity), in_reply_to_account_acct: reply_to_user && reply_to_user.nickname, content: %{"text/plain" => content_plaintext}, - spoiler_text: %{"text/plain" => summary_plaintext} + spoiler_text: %{"text/plain" => summary_plaintext}, + expires_at: expires_at } } end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 24482a4a..e5990897 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -166,10 +166,11 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do "expires_at" => expires_at }) - assert %{"id" => fourth_id} = json_response(conn_four, 200) + assert fourth_response = %{"id" => fourth_id} = json_response(conn_four, 200) assert activity = Activity.get_by_id(fourth_id) assert expiration = ActivityExpiration.get_by_activity_id(fourth_id) assert expiration.scheduled_at == expires_at + assert fourth_response["pleroma"]["expires_at"] == NaiveDateTime.to_iso8601(expires_at) end test "replying to a status", %{conn: conn} do diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index 3447c5b1..073c6965 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -133,7 +133,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do conversation_id: convo_id, in_reply_to_account_acct: nil, content: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["content"])}, - spoiler_text: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["summary"])} + spoiler_text: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["summary"])}, + expires_at: nil } } From 91d9fdc7decc664483625c11e44d4e053dd9c585 Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Wed, 24 Jul 2019 13:02:28 +0200 Subject: [PATCH 11/83] Update changelog to document expiring posts feature --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35a5a6c2..75d236af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] +### Added +- Expiring/ephemeral activites. All activities can have expires_on value set, which controls when they should be deleted automatically. +- Mastodon API: in post_status, expires_at datetime parameter lets you set when an activity should expire +- Mastodon API: all status JSON responses contain a `pleroma.expires_in` item which states the number of minutes until an activity expires. The value is only shown to the user who created the activity. To everyone else it's empty. +- Configuration: `ActivityExpiration.enabled` controls whether expired activites will get deleted at the appropriate time. Enabled by default. + ### Changed - **Breaking:** Configuration: A setting to explicitly disable the mailer was added, defaulting to true, if you are using a mailer add `config :pleroma, Pleroma.Emails.Mailer, enabled: true` to your config - Configuration: OpenGraph and TwitterCard providers enabled by default From 2981821db834448bf9b2ba26590314e36201664c Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Wed, 24 Jul 2019 16:51:09 +0200 Subject: [PATCH 12/83] squash! Expose expires_at datetime in mastoAPI only for the activity actor NOTE: rewrite the commit msg --- docs/api/differences_in_mastoapi_responses.md | 2 +- lib/pleroma/web/mastodon_api/views/status_view.ex | 10 +++++++--- test/web/mastodon_api/mastodon_api_controller_test.exs | 2 +- test/web/mastodon_api/status_view_test.exs | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index 168a13f4..829468b1 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -25,7 +25,7 @@ Has these additional fields under the `pleroma` object: - `in_reply_to_account_acct`: the `acct` property of User entity for replied user (if any) - `content`: a map consisting of alternate representations of the `content` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain` - `spoiler_text`: a map consisting of alternate representations of the `spoiler_text` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain` -- `expires_on`: a datetime (iso8601) that states when the post will expire (be deleted automatically), or empty if the post won't expire +- `expires_in`: the number of minutes until a post will expire (be deleted automatically), or empty if the post won't expire ## Attachments diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 7264dcaf..4a3686d7 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -168,11 +168,15 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do client_posted_this_activity = opts[:for] && user.id == opts[:for].id - expires_at = + expires_in = with true <- client_posted_this_activity, expiration when not is_nil(expiration) <- ActivityExpiration.get_by_activity_id(activity.id) do - expiration.scheduled_at + expires_in_seconds = + expiration.scheduled_at + |> NaiveDateTime.diff(NaiveDateTime.utc_now(), :second) + + round(expires_in_seconds / 60) end thread_muted? = @@ -273,7 +277,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do in_reply_to_account_acct: reply_to_user && reply_to_user.nickname, content: %{"text/plain" => content_plaintext}, spoiler_text: %{"text/plain" => summary_plaintext}, - expires_at: expires_at + expires_in: expires_in } } end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index e5990897..a9d38c06 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -170,7 +170,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do assert activity = Activity.get_by_id(fourth_id) assert expiration = ActivityExpiration.get_by_activity_id(fourth_id) assert expiration.scheduled_at == expires_at - assert fourth_response["pleroma"]["expires_at"] == NaiveDateTime.to_iso8601(expires_at) + assert fourth_response["pleroma"]["expires_in"] > 0 end test "replying to a status", %{conn: conn} do diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index 073c6965..eb0874ab 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -134,7 +134,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do in_reply_to_account_acct: nil, content: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["content"])}, spoiler_text: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["summary"])}, - expires_at: nil + expires_in: nil } } From 877575d0da830724e822eac2de243391aaea7ec8 Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Wed, 24 Jul 2019 17:07:51 +0200 Subject: [PATCH 13/83] fixup! Update changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75d236af..f6450663 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Added - Expiring/ephemeral activites. All activities can have expires_on value set, which controls when they should be deleted automatically. -- Mastodon API: in post_status, expires_at datetime parameter lets you set when an activity should expire -- Mastodon API: all status JSON responses contain a `pleroma.expires_in` item which states the number of minutes until an activity expires. The value is only shown to the user who created the activity. To everyone else it's empty. +- Mastodon API: in post_status, the expires_in parameter lets you set the number of minutes until an activity expires. It must be at least one hour. +- Mastodon API: all status JSON responses contain a `pleroma.expires_on` item which states when an activity will expire. The value is only shown to the user who created the activity. To everyone else it's empty. - Configuration: `ActivityExpiration.enabled` controls whether expired activites will get deleted at the appropriate time. Enabled by default. ### Changed From 2c83eb0b157b2f574f55341e9171f0b5ab7bd3b2 Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Wed, 24 Jul 2019 17:09:59 +0200 Subject: [PATCH 14/83] Revert "squash! Expose expires_at datetime in mastoAPI only for the activity actor" This reverts commit 2981821db834448bf9b2ba26590314e36201664c. --- docs/api/differences_in_mastoapi_responses.md | 2 +- lib/pleroma/web/mastodon_api/views/status_view.ex | 10 +++------- test/web/mastodon_api/mastodon_api_controller_test.exs | 2 +- test/web/mastodon_api/status_view_test.exs | 2 +- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index 829468b1..168a13f4 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -25,7 +25,7 @@ Has these additional fields under the `pleroma` object: - `in_reply_to_account_acct`: the `acct` property of User entity for replied user (if any) - `content`: a map consisting of alternate representations of the `content` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain` - `spoiler_text`: a map consisting of alternate representations of the `spoiler_text` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain` -- `expires_in`: the number of minutes until a post will expire (be deleted automatically), or empty if the post won't expire +- `expires_on`: a datetime (iso8601) that states when the post will expire (be deleted automatically), or empty if the post won't expire ## Attachments diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 4a3686d7..7264dcaf 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -168,15 +168,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do client_posted_this_activity = opts[:for] && user.id == opts[:for].id - expires_in = + expires_at = with true <- client_posted_this_activity, expiration when not is_nil(expiration) <- ActivityExpiration.get_by_activity_id(activity.id) do - expires_in_seconds = - expiration.scheduled_at - |> NaiveDateTime.diff(NaiveDateTime.utc_now(), :second) - - round(expires_in_seconds / 60) + expiration.scheduled_at end thread_muted? = @@ -277,7 +273,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do in_reply_to_account_acct: reply_to_user && reply_to_user.nickname, content: %{"text/plain" => content_plaintext}, spoiler_text: %{"text/plain" => summary_plaintext}, - expires_in: expires_in + expires_at: expires_at } } end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index a9d38c06..e5990897 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -170,7 +170,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do assert activity = Activity.get_by_id(fourth_id) assert expiration = ActivityExpiration.get_by_activity_id(fourth_id) assert expiration.scheduled_at == expires_at - assert fourth_response["pleroma"]["expires_in"] > 0 + assert fourth_response["pleroma"]["expires_at"] == NaiveDateTime.to_iso8601(expires_at) end test "replying to a status", %{conn: conn} do diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index eb0874ab..073c6965 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -134,7 +134,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do in_reply_to_account_acct: nil, content: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["content"])}, spoiler_text: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["summary"])}, - expires_in: nil + expires_at: nil } } From 0e2b5a3e6aed7947909c2a1ff1618403546f1572 Mon Sep 17 00:00:00 2001 From: Mike Verdone Date: Wed, 24 Jul 2019 17:25:11 +0200 Subject: [PATCH 15/83] WIP --- .../mastodon_api_controller_test.exs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index e5990897..fbe0ab37 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -154,23 +154,27 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do refute id == third_id # An activity that will expire: - expires_at = - NaiveDateTime.utc_now() - |> NaiveDateTime.add(:timer.minutes(120), :millisecond) - |> NaiveDateTime.truncate(:second) + expires_in = 120 conn_four = conn |> post("api/v1/statuses", %{ "status" => "oolong", - "expires_at" => expires_at + "expires_in" => expires_in }) assert fourth_response = %{"id" => fourth_id} = json_response(conn_four, 200) assert activity = Activity.get_by_id(fourth_id) assert expiration = ActivityExpiration.get_by_activity_id(fourth_id) - assert expiration.scheduled_at == expires_at - assert fourth_response["pleroma"]["expires_at"] == NaiveDateTime.to_iso8601(expires_at) + + estimated_expires_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(expires_in), :millisecond) + |> NaiveDateTime.truncate(:second) + + # This assert will fail if the test takes longer than a minute. I sure hope it never does: + assert abs(NaiveDateTime.diff(expiration.scheduled_at, estimated_expires_at, :second)) < 60 + assert fourth_response["pleroma"]["expires_at"] == NaiveDateTime.to_iso8601(expiration.scheduled_at) end test "replying to a status", %{conn: conn} do From dfae61c25c7ee2bb8add38b2cbaa8391f03c9550 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Fri, 9 Aug 2019 23:05:28 +0300 Subject: [PATCH 16/83] Fix deactivated user deletion --- lib/mix/tasks/pleroma/user.ex | 2 +- lib/pleroma/user.ex | 34 +++++++++---------- lib/pleroma/web/activity_pub/activity_pub.ex | 10 +++--- .../web/activity_pub/transmogrifier.ex | 2 +- .../web/admin_api/admin_api_controller.ex | 4 +-- .../web/ostatus/handlers/delete_handler.ex | 2 +- test/user_test.exs | 8 +++++ 7 files changed, 36 insertions(+), 26 deletions(-) diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index a3f8bc94..f33d0142 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -176,7 +176,7 @@ defmodule Mix.Tasks.Pleroma.User do start_pleroma() with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do - User.perform(:delete, user) + User.perform(:delete, user, nil) shell_info("User #{nickname} deleted.") else _ -> diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 7d18f099..14057a0e 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1029,13 +1029,26 @@ defmodule Pleroma.User do |> update_and_set_cache() end + @spec perform(atom(), User.t()) :: {:ok, User.t()} + def perform(:fetch_initial_posts, %User{} = user) do + pages = Pleroma.Config.get!([:fetch_initial_posts, :pages]) + + Enum.each( + # Insert all the posts in reverse order, so they're in the right order on the timeline + Enum.reverse(Utils.fetch_ordered_collection(user.info.source_data["outbox"], pages)), + &Pleroma.Web.Federator.incoming_ap_doc/1 + ) + + {:ok, user} + end + @spec delete(User.t()) :: :ok - def delete(%User{} = user), - do: PleromaJobQueue.enqueue(:background, __MODULE__, [:delete, user]) + def delete(%User{} = user, actor \\ nil), + do: PleromaJobQueue.enqueue(:background, __MODULE__, [:delete, user, actor]) @spec perform(atom(), User.t()) :: {:ok, User.t()} - def perform(:delete, %User{} = user) do - {:ok, _user} = ActivityPub.delete(user) + def perform(:delete, %User{} = user, actor) do + {:ok, _user} = ActivityPub.delete(user, actor: actor) # Remove all relationships {:ok, followers} = User.get_followers(user) @@ -1057,19 +1070,6 @@ defmodule Pleroma.User do Repo.delete(user) end - @spec perform(atom(), User.t()) :: {:ok, User.t()} - def perform(:fetch_initial_posts, %User{} = user) do - pages = Pleroma.Config.get!([:fetch_initial_posts, :pages]) - - Enum.each( - # Insert all the posts in reverse order, so they're in the right order on the timeline - Enum.reverse(Utils.fetch_ordered_collection(user.info.source_data["outbox"], pages)), - &Pleroma.Web.Federator.incoming_ap_doc/1 - ) - - {:ok, user} - end - def perform(:deactivate_async, user, status), do: deactivate(user, status) @spec perform(atom(), User.t(), list()) :: list() | {:error, any()} diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 1a279a7d..8f669acb 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -403,11 +403,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end - def delete(%User{ap_id: ap_id, follower_address: follower_address} = user) do + def delete(data, opts \\ %{actor: nil, local: true}) + + def delete(%User{ap_id: ap_id, follower_address: follower_address} = user, opts) do with data <- %{ "to" => [follower_address], "type" => "Delete", - "actor" => ap_id, + "actor" => opts[:actor] || ap_id, "object" => %{"type" => "Person", "id" => ap_id} }, {:ok, activity} <- insert(data, true, true), @@ -416,7 +418,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end - def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, local \\ true) do + def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, opts) do user = User.get_cached_by_ap_id(actor) to = (object.data["to"] || []) ++ (object.data["cc"] || []) @@ -428,7 +430,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do "to" => to, "deleted_activity_id" => activity && activity.id }, - {:ok, activity} <- insert(data, local, false), + {:ok, activity} <- insert(data, opts[:local], false), stream_out_participations(object, user), _ <- decrease_replies_count_if_reply(object), # Changing note count prior to enqueuing federation task in order to avoid diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 5403b71d..b34ef73c 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -649,7 +649,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id), :ok <- Containment.contain_origin(actor.ap_id, object.data), - {:ok, activity} <- ActivityPub.delete(object, false) do + {:ok, activity} <- ActivityPub.delete(object, local: false) do {:ok, activity} else nil -> diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 2d3d0adc..63c9a7d7 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -25,9 +25,9 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do action_fallback(:errors) - def user_delete(conn, %{"nickname" => nickname}) do + def user_delete(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do User.get_cached_by_nickname(nickname) - |> User.delete() + |> User.delete(admin.ap_id) conn |> json(nickname) diff --git a/lib/pleroma/web/ostatus/handlers/delete_handler.ex b/lib/pleroma/web/ostatus/handlers/delete_handler.ex index b2f9f394..ac2dc115 100644 --- a/lib/pleroma/web/ostatus/handlers/delete_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/delete_handler.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Web.OStatus.DeleteHandler do def handle_delete(entry, _doc \\ nil) do with id <- XML.string_from_xpath("//id", entry), %Object{} = object <- Object.normalize(id), - {:ok, delete} <- ActivityPub.delete(object, false) do + {:ok, delete} <- ActivityPub.delete(object, local: false) do delete end end diff --git a/test/user_test.exs b/test/user_test.exs index 8440d456..e2da8d84 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -998,6 +998,14 @@ defmodule Pleroma.UserTest do refute Activity.get_by_id(activity.id) end + test "it deletes deactivated user" do + admin = insert(:user, %{info: %{is_admin: true}}) + {:ok, user} = insert(:user, info: %{deactivated: true}) |> User.set_cache() + + assert {:ok, _} = User.delete(user, admin.ap_id) + refute User.get_by_id(user.id) + end + test "it deletes a user, all follow relationships and all activities", %{user: user} do follower = insert(:user) {:ok, follower} = User.follow(follower, user) From e0ac5c7a66664c897e1b3af9a55e0b73f32fa034 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 24 Jul 2019 19:26:35 +0700 Subject: [PATCH 17/83] Add custom profile fields --- config/config.exs | 1 + docs/config.md | 1 + lib/pleroma/user/info.ex | 31 +++++++++++++++ .../mastodon_api/mastodon_api_controller.ex | 11 ++++++ .../web/mastodon_api/views/account_view.ex | 9 ++--- .../update_credentials_test.exs | 39 +++++++++++++++++++ 6 files changed, 87 insertions(+), 5 deletions(-) diff --git a/config/config.exs b/config/config.exs index 75866112..109cf651 100644 --- a/config/config.exs +++ b/config/config.exs @@ -255,6 +255,7 @@ config :pleroma, :instance, dynamic_configuration: false, user_bio_length: 5000, user_name_length: 100, + max_account_fields: 4, external_user_synchronization: true config :pleroma, :markup, diff --git a/docs/config.md b/docs/config.md index 20311db5..ca5da7db 100644 --- a/docs/config.md +++ b/docs/config.md @@ -132,6 +132,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `skip_thread_containment`: Skip filter out broken threads. The default is `false`. * `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. * `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. +* `max_account_fields`: The maximum number of custom fields in the user profile (default: `4`) * `external_user_synchronization`: Enabling following/followers counters synchronization for external users. diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 22eb9a18..fa57052f 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -49,6 +49,7 @@ defmodule Pleroma.User.Info do field(:mascot, :map, default: nil) field(:emoji, {:array, :map}, default: []) field(:pleroma_settings_store, :map, default: %{}) + field(:fields, {:array, :map}, default: []) field(:notification_settings, :map, default: %{ @@ -286,10 +287,32 @@ defmodule Pleroma.User.Info do :background, :show_role, :skip_thread_containment, + :fields, :pleroma_settings_store ]) + |> validate_fields() end + def validate_fields(changeset) do + limit = Pleroma.Config.get([:instance, :max_account_fields], 0) + + changeset + |> validate_length(:fields, max: limit) + |> validate_change(:fields, fn :fields, fields -> + if Enum.all?(fields, &valid_field?/1) do + [] + else + [fields: "invalid"] + end + end) + end + + defp valid_field?(%{"name" => name, "value" => value}) do + is_binary(name) && is_binary(value) + end + + defp valid_field?(_), do: false + @spec confirmation_changeset(Info.t(), keyword()) :: Changeset.t() def confirmation_changeset(info, opts) do need_confirmation? = Keyword.get(opts, :need_confirmation) @@ -384,6 +407,14 @@ defmodule Pleroma.User.Info do cast(info, params, [:muted_reblogs]) end + def fields(%{source_data: %{"attachment" => attachment}}) do + attachment + |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) + |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) + end + + def fields(%{fields: fields}), do: fields + def follow_information_update(info, params) do info |> cast(params, [ diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 7ce2b5b0..e79a02ca 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -156,6 +156,17 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end) end) |> add_if_present(params, "default_scope", :default_scope) + |> add_if_present(params, "fields", :fields, fn fields -> + fields = + Enum.map(fields, fn field -> + %{ + "name" => Formatter.html_escape(field["name"], "text/plain"), + "value" => Formatter.html_escape(field["value"], "text/plain") + } + end) + + {:ok, fields} + end) |> add_if_present(params, "pleroma_settings_store", :pleroma_settings_store, fn value -> {:ok, Map.merge(user.info.pleroma_settings_store, value)} end) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 72c092f2..d2f3986f 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -93,10 +93,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do } end) - fields = - (user.info.source_data["attachment"] || []) - |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) - |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) + fields = User.Info.fields(user.info) + fields_html = Enum.map(fields, fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end) bio = HTML.filter_tags(user.bio, User.html_filter_policy(opts[:for])) @@ -119,11 +117,12 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do header: header, header_static: header, emojis: emojis, - fields: fields, + fields: fields_html, bot: bot, source: %{ note: HTML.strip_tags((user.bio || "") |> String.replace("
", "\n")), sensitive: false, + fields: fields, pleroma: %{} }, diff --git a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs index 71d0c8af..a3eadde1 100644 --- a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs @@ -300,5 +300,44 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController.UpdateCredentialsTest do assert user["display_name"] == name assert [%{"shortcode" => "blank"}, %{"shortcode" => "firefox"}] = user["emojis"] end + + test "update fields", %{conn: conn} do + user = insert(:user) + + fields = [ + %{"name" => "foo", "value" => "bar"}, + %{"name" => "link", "value" => "cofe.io"} + ] + + account = + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_credentials", %{"fields" => fields}) + |> json_response(200) + + assert account["fields"] == [ + %{"name" => "<b>foo<b>", "value" => "<i>bar</i>"}, + %{"name" => "link", "value" => "cofe.io"} + ] + + assert account["source"]["fields"] == [ + %{"name" => "<b>foo<b>", "value" => "<i>bar</i>"}, + %{"name" => "link", "value" => "cofe.io"} + ] + + Pleroma.Config.put([:instance, :max_account_fields], 1) + + fields = [ + %{"name" => "foo", "value" => "bar"}, + %{"name" => "link", "value" => "cofe.io"} + ] + + conn = + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_credentials", %{"fields" => fields}) + + assert %{"error" => "Invalid request"} == json_response(conn, 403) + end end end From d6094b405d1a744eaa1c17752b3088dfee16c8d2 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 24 Jul 2019 19:48:15 +0700 Subject: [PATCH 18/83] Fix tests --- test/web/mastodon_api/account_view_test.exs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs index a26f514a..1d8b2833 100644 --- a/test/web/mastodon_api/account_view_test.exs +++ b/test/web/mastodon_api/account_view_test.exs @@ -67,7 +67,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do source: %{ note: "valid html", sensitive: false, - pleroma: %{} + pleroma: %{}, + fields: [] }, pleroma: %{ background_image: "https://example.com/images/asuka_hospital.png", @@ -134,7 +135,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do source: %{ note: user.bio, sensitive: false, - pleroma: %{} + pleroma: %{}, + fields: [] }, pleroma: %{ background_image: nil, @@ -304,7 +306,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do source: %{ note: user.bio, sensitive: false, - pleroma: %{} + pleroma: %{}, + fields: [] }, pleroma: %{ background_image: nil, From 069951722f9b674a759e0b8683aff9c03019467b Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 25 Jul 2019 14:34:52 +0700 Subject: [PATCH 19/83] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35828709..a54b0465 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Add `pleroma.deactivated` to the Account entity - Mastodon API: added `/auth/password` endpoint for password reset with rate limit. - Mastodon API: /api/v1/accounts/:id/statuses now supports nicknames or user id +- Mastodon API: Add support for the user profile custom fields - Admin API: Return users' tags when querying reports - Admin API: Return avatar and display name when querying users - Admin API: Allow querying user by ID From a22f540fc42dd941631e94fe931d1f655b2904a1 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 25 Jul 2019 19:33:18 +0700 Subject: [PATCH 20/83] Add custom fields to TwitterAPI.UserView --- lib/pleroma/user/info.ex | 2 ++ lib/pleroma/web/twitter_api/views/user_view.ex | 7 +------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index fa57052f..98b89422 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -407,6 +407,8 @@ defmodule Pleroma.User.Info do cast(info, params, [:muted_reblogs]) end + # ``fields`` is an array of mastodon profile field, containing ``{"name": "…", "value": "…"}``. + # For example: [{"name": "Pronoun", "value": "she/her"}, …] def fields(%{source_data: %{"attachment" => attachment}}) do attachment |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex index 8d889206..3681773b 100644 --- a/lib/pleroma/web/twitter_api/views/user_view.ex +++ b/lib/pleroma/web/twitter_api/views/user_view.ex @@ -74,12 +74,7 @@ defmodule Pleroma.Web.TwitterAPI.UserView do |> HTML.filter_tags(User.html_filter_policy(for_user)) |> Formatter.emojify(emoji) - # ``fields`` is an array of mastodon profile field, containing ``{"name": "…", "value": "…"}``. - # For example: [{"name": "Pronoun", "value": "she/her"}, …] - fields = - (user.info.source_data["attachment"] || []) - |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) - |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) + fields = User.Info.fields(user.info) data = %{ From 88598c9bafcdcf89b0f1fb00d0785c77b583cd65 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 25 Jul 2019 19:35:34 +0700 Subject: [PATCH 21/83] Add profile custom fields to ActivityPub.UserView --- lib/pleroma/web/activity_pub/views/user_view.ex | 6 ++++++ test/web/activity_pub/views/user_view_test.exs | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 06c9e1c7..7b4bc998 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -80,6 +80,11 @@ defmodule Pleroma.Web.ActivityPub.UserView do |> Transmogrifier.add_emoji_tags() |> Map.get("tag", []) + fields = + user.info + |> User.Info.fields() + |> Enum.map(&Map.put(&1, "type", "PropertyValue")) + %{ "id" => user.ap_id, "type" => "Person", @@ -98,6 +103,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do "publicKeyPem" => public_key }, "endpoints" => endpoints, + "attachment" => fields, "tag" => (user.info.source_data["tag"] || []) ++ user_tags } |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user)) diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs index 86254117..48a522c6 100644 --- a/test/web/activity_pub/views/user_view_test.exs +++ b/test/web/activity_pub/views/user_view_test.exs @@ -22,6 +22,22 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do assert String.contains?(result["publicKey"]["publicKeyPem"], "BEGIN PUBLIC KEY") end + test "Renders profile fields" do + fields = [ + %{"name" => "foo", "value" => "bar"}, + %{"name" => "website", "value" => "cofe.my"} + ] + + user = insert(:user, info: %{fields: fields}) + + assert %{ + "attachment" => [ + %{"name" => "foo", "type" => "PropertyValue", "value" => "bar"}, + %{"name" => "website", "type" => "PropertyValue", "value" => "cofe.my"} + ] + } = UserView.render("user.json", %{user: user}) + end + test "Does not add an avatar image if the user hasn't set one" do user = insert(:user) {:ok, user} = User.ensure_keys_present(user) From 8ab87ce40b0b8a5be7cd3576eddbacc279d8d3a1 Mon Sep 17 00:00:00 2001 From: Egor Date: Fri, 26 Jul 2019 12:08:46 +0000 Subject: [PATCH 22/83] Apply suggestion to CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a54b0465..483c8807 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,7 +64,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Add `pleroma.deactivated` to the Account entity - Mastodon API: added `/auth/password` endpoint for password reset with rate limit. - Mastodon API: /api/v1/accounts/:id/statuses now supports nicknames or user id -- Mastodon API: Add support for the user profile custom fields +- Mastodon API: Improve support for the user profile custom fields - Admin API: Return users' tags when querying reports - Admin API: Return avatar and display name when querying users - Admin API: Allow querying user by ID From 5178f960c3f5a35e2071bd5463b537cadc9a53af Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 29 Jul 2019 19:01:15 +0700 Subject: [PATCH 23/83] Support user attachment update in Transmogrifier --- .../web/activity_pub/transmogrifier.ex | 2 + test/fixtures/mastodon-update.json | 36 +++++++++--- .../tesla_mock/admin@mastdon.example.org.json | 55 ++++++++++++++++++- test/web/activity_pub/transmogrifier_test.exs | 36 ++++++++++++ 4 files changed, 120 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 0fcc81bf..225c3487 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -598,11 +598,13 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do banner = new_user_data[:info][:banner] locked = new_user_data[:info][:locked] || false + attachment = get_in(new_user_data, [:info, "source_data", "attachment"]) update_data = new_user_data |> Map.take([:name, :bio, :avatar]) |> Map.put(:info, %{banner: banner, locked: locked}) + |> Map.put(:info, %{"banner" => banner, "locked" => locked, "source_data" => source_data}) actor |> User.upgrade_changeset(update_data) diff --git a/test/fixtures/mastodon-update.json b/test/fixtures/mastodon-update.json index f6713fea..dbf8b6df 100644 --- a/test/fixtures/mastodon-update.json +++ b/test/fixtures/mastodon-update.json @@ -1,10 +1,10 @@ -{ - "type": "Update", - "object": { - "url": "http://mastodon.example.org/@gargron", - "type": "Person", - "summary": "

Some bio

", - "publicKey": { +{ + "type": "Update", + "object": { + "url": "http://mastodon.example.org/@gargron", + "type": "Person", + "summary": "

Some bio

", + "publicKey": { "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0gs3VnQf6am3R+CeBV4H\nlfI1HZTNRIBHgvFszRZkCERbRgEWMu+P+I6/7GJC5H5jhVQ60z4MmXcyHOGmYMK/\n5XyuHQz7V2Ssu1AxLfRN5Biq1ayb0+DT/E7QxNXDJPqSTnstZ6C7zKH/uAETqg3l\nBonjCQWyds+IYbQYxf5Sp3yhvQ80lMwHML3DaNCMlXWLoOnrOX5/yK5+dedesg2\n/HIvGk+HEt36vm6hoH7bwPuEkgA++ACqwjXRe5Mta7i3eilHxFaF8XIrJFARV0t\nqOu4GID/jG6oA+swIWndGrtR2QRJIt9QIBFfK3HG5M0koZbY1eTqwNFRHFL3xaD\nUQIDAQAB\n-----END PUBLIC KEY-----\n", "owner": "http://mastodon.example.org/users/gargron", "id": "http://mastodon.example.org/users/gargron#main-key" @@ -20,7 +20,27 @@ "endpoints": { "sharedInbox": "http://mastodon.example.org/inbox" }, - "icon":{"type":"Image","mediaType":"image/jpeg","url":"https://cd.niu.moe/accounts/avatars/000/033/323/original/fd7f8ae0b3ffedc9.jpeg"},"image":{"type":"Image","mediaType":"image/png","url":"https://cd.niu.moe/accounts/headers/000/033/323/original/850b3448fa5fd477.png"} + "attachment": [{ + "type": "PropertyValue", + "name": "foo", + "value": "updated" + }, + { + "type": "PropertyValue", + "name": "foo1", + "value": "updated" + } + ], + "icon": { + "type": "Image", + "mediaType": "image/jpeg", + "url": "https://cd.niu.moe/accounts/avatars/000/033/323/original/fd7f8ae0b3ffedc9.jpeg" + }, + "image": { + "type": "Image", + "mediaType": "image/png", + "url": "https://cd.niu.moe/accounts/headers/000/033/323/original/850b3448fa5fd477.png" + } }, "id": "http://mastodon.example.org/users/gargron#updates/1519563538", "actor": "http://mastodon.example.org/users/gargron", diff --git a/test/fixtures/tesla_mock/admin@mastdon.example.org.json b/test/fixtures/tesla_mock/admin@mastdon.example.org.json index c297e434..8159dc20 100644 --- a/test/fixtures/tesla_mock/admin@mastdon.example.org.json +++ b/test/fixtures/tesla_mock/admin@mastdon.example.org.json @@ -1 +1,54 @@ -{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1",{"manuallyApprovesFollowers":"as:manuallyApprovesFollowers","sensitive":"as:sensitive","movedTo":"as:movedTo","Hashtag":"as:Hashtag","ostatus":"http://ostatus.org#","atomUri":"ostatus:atomUri","inReplyToAtomUri":"ostatus:inReplyToAtomUri","conversation":"ostatus:conversation","toot":"http://joinmastodon.org/ns#","Emoji":"toot:Emoji"}],"id":"http://mastodon.example.org/users/admin","type":"Person","following":"http://mastodon.example.org/users/admin/following","followers":"http://mastodon.example.org/users/admin/followers","inbox":"http://mastodon.example.org/users/admin/inbox","outbox":"http://mastodon.example.org/users/admin/outbox","preferredUsername":"admin","name":null,"summary":"\u003cp\u003e\u003c/p\u003e","url":"http://mastodon.example.org/@admin","manuallyApprovesFollowers":false,"publicKey":{"id":"http://mastodon.example.org/users/admin#main-key","owner":"http://mastodon.example.org/users/admin","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtc4Tir+3ADhSNF6VKrtW\nOU32T01w7V0yshmQei38YyiVwVvFu8XOP6ACchkdxbJ+C9mZud8qWaRJKVbFTMUG\nNX4+6Q+FobyuKrwN7CEwhDALZtaN2IPbaPd6uG1B7QhWorrY+yFa8f2TBM3BxnUy\nI4T+bMIZIEYG7KtljCBoQXuTQmGtuffO0UwJksidg2ffCF5Q+K//JfQagJ3UzrR+\nZXbKMJdAw4bCVJYs4Z5EhHYBwQWiXCyMGTd7BGlmMkY6Av7ZqHKC/owp3/0EWDNz\nNqF09Wcpr3y3e8nA10X40MJqp/wR+1xtxp+YGbq/Cj5hZGBG7etFOmIpVBrDOhry\nBwIDAQAB\n-----END PUBLIC KEY-----\n"},"endpoints":{"sharedInbox":"http://mastodon.example.org/inbox"},"icon":{"type":"Image","mediaType":"image/jpeg","url":"https://cdn.niu.moe/accounts/avatars/000/033/323/original/fd7f8ae0b3ffedc9.jpeg"},"image":{"type":"Image","mediaType":"image/png","url":"https://cdn.niu.moe/accounts/headers/000/033/323/original/850b3448fa5fd477.png"}} +{ + "@context": ["https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1", { + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + "sensitive": "as:sensitive", + "movedTo": "as:movedTo", + "Hashtag": "as:Hashtag", + "ostatus": "http://ostatus.org#", + "atomUri": "ostatus:atomUri", + "inReplyToAtomUri": "ostatus:inReplyToAtomUri", + "conversation": "ostatus:conversation", + "toot": "http://joinmastodon.org/ns#", + "Emoji": "toot:Emoji" + }], + "id": "http://mastodon.example.org/users/admin", + "type": "Person", + "following": "http://mastodon.example.org/users/admin/following", + "followers": "http://mastodon.example.org/users/admin/followers", + "inbox": "http://mastodon.example.org/users/admin/inbox", + "outbox": "http://mastodon.example.org/users/admin/outbox", + "preferredUsername": "admin", + "name": null, + "summary": "\u003cp\u003e\u003c/p\u003e", + "url": "http://mastodon.example.org/@admin", + "manuallyApprovesFollowers": false, + "publicKey": { + "id": "http://mastodon.example.org/users/admin#main-key", + "owner": "http://mastodon.example.org/users/admin", + "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtc4Tir+3ADhSNF6VKrtW\nOU32T01w7V0yshmQei38YyiVwVvFu8XOP6ACchkdxbJ+C9mZud8qWaRJKVbFTMUG\nNX4+6Q+FobyuKrwN7CEwhDALZtaN2IPbaPd6uG1B7QhWorrY+yFa8f2TBM3BxnUy\nI4T+bMIZIEYG7KtljCBoQXuTQmGtuffO0UwJksidg2ffCF5Q+K//JfQagJ3UzrR+\nZXbKMJdAw4bCVJYs4Z5EhHYBwQWiXCyMGTd7BGlmMkY6Av7ZqHKC/owp3/0EWDNz\nNqF09Wcpr3y3e8nA10X40MJqp/wR+1xtxp+YGbq/Cj5hZGBG7etFOmIpVBrDOhry\nBwIDAQAB\n-----END PUBLIC KEY-----\n" + }, + "attachment": [{ + "type": "PropertyValue", + "name": "foo", + "value": "bar" + }, + { + "type": "PropertyValue", + "name": "foo1", + "value": "bar1" + } + ], + "endpoints": { + "sharedInbox": "http://mastodon.example.org/inbox" + }, + "icon": { + "type": "Image", + "mediaType": "image/jpeg", + "url": "https://cdn.niu.moe/accounts/avatars/000/033/323/original/fd7f8ae0b3ffedc9.jpeg" + }, + "image": { + "type": "Image", + "mediaType": "image/png", + "url": "https://cdn.niu.moe/accounts/headers/000/033/323/original/850b3448fa5fd477.png" + } +} diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 060b91e2..05ec09ec 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -509,6 +509,42 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert user.bio == "

Some bio

" end + test "it works with custom profile fields" do + {:ok, activity} = + "test/fixtures/mastodon-post-activity.json" + |> File.read!() + |> Poison.decode!() + |> Transmogrifier.handle_incoming() + + user = User.get_cached_by_ap_id(activity.actor) + + assert user.info.source_data["attachment"] == [ + %{"name" => "foo", "type" => "PropertyValue", "value" => "bar"}, + %{"name" => "foo1", "type" => "PropertyValue", "value" => "bar1"} + ] + + update_data = File.read!("test/fixtures/mastodon-update.json") |> Poison.decode!() + + object = + update_data["object"] + |> Map.put("actor", user.ap_id) + |> Map.put("id", user.ap_id) + + update_data = + update_data + |> Map.put("actor", user.ap_id) + |> Map.put("object", object) + + {:ok, _update_activity} = Transmogrifier.handle_incoming(update_data) + + user = User.get_cached_by_ap_id(user.ap_id) + + assert user.info.source_data["attachment"] == [ + %{"name" => "foo", "type" => "PropertyValue", "value" => "updated"}, + %{"name" => "foo1", "type" => "PropertyValue", "value" => "updated"} + ] + end + test "it works for incoming update activities which lock the account" do data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() From 7d6f8a7fd75e5de4e0c9ce208ac9276dcbe044f5 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 29 Jul 2019 19:17:09 +0700 Subject: [PATCH 24/83] Linkify custom fields values in ActivityPub.UserViewx --- lib/pleroma/web/activity_pub/views/user_view.ex | 1 + test/web/activity_pub/views/user_view_test.exs | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 7b4bc998..b2a22478 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -84,6 +84,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do user.info |> User.Info.fields() |> Enum.map(&Map.put(&1, "type", "PropertyValue")) + |> Enum.map(fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end) %{ "id" => user.ap_id, diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs index 48a522c6..a2aa5238 100644 --- a/test/web/activity_pub/views/user_view_test.exs +++ b/test/web/activity_pub/views/user_view_test.exs @@ -33,7 +33,11 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do assert %{ "attachment" => [ %{"name" => "foo", "type" => "PropertyValue", "value" => "bar"}, - %{"name" => "website", "type" => "PropertyValue", "value" => "cofe.my"} + %{ + "name" => "website", + "type" => "PropertyValue", + "value" => "cofe.my" + } ] } = UserView.render("user.json", %{user: user}) end From db3c05f6b4c226733633a409cb1f1a290db4c48b Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 30 Jul 2019 17:22:52 +0700 Subject: [PATCH 25/83] Add configurable account field value length limit --- config/config.exs | 1 + docs/config.md | 1 + lib/pleroma/user/info.ex | 7 ++++- .../update_credentials_test.exs | 31 +++++++++++++++---- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/config/config.exs b/config/config.exs index 109cf651..21f4861f 100644 --- a/config/config.exs +++ b/config/config.exs @@ -256,6 +256,7 @@ config :pleroma, :instance, user_bio_length: 5000, user_name_length: 100, max_account_fields: 4, + account_field_value_length: 255, external_user_synchronization: true config :pleroma, :markup, diff --git a/docs/config.md b/docs/config.md index ca5da7db..fbb50609 100644 --- a/docs/config.md +++ b/docs/config.md @@ -133,6 +133,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. * `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. * `max_account_fields`: The maximum number of custom fields in the user profile (default: `4`) +* `account_field_value_length`: An account field value maximum length (default: `255`) * `external_user_synchronization`: Enabling following/followers counters synchronization for external users. diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 98b89422..9e4d381f 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -308,7 +308,12 @@ defmodule Pleroma.User.Info do end defp valid_field?(%{"name" => name, "value" => value}) do - is_binary(name) && is_binary(value) + value_limit = Pleroma.Config.get([:instance, :account_field_value_length], 255) + + is_binary(name) && + is_binary(value) && + String.length(name) <= 255 && + String.length(value) <= value_limit end defp valid_field?(_), do: false diff --git a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs index a3eadde1..992a692f 100644 --- a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs @@ -325,6 +325,26 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController.UpdateCredentialsTest do %{"name" => "link", "value" => "cofe.io"} ] + value_limit = Pleroma.Config.get([:instance, :account_field_value_length]) + + long_str = Enum.map(0..value_limit, fn _ -> "x" end) |> Enum.join() + + fields = [%{"name" => "foo", "value" => long_str}] + + assert %{"error" => "Invalid request"} == + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_credentials", %{"fields" => fields}) + |> json_response(403) + + fields = [%{"name" => long_str, "value" => "bar"}] + + assert %{"error" => "Invalid request"} == + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_credentials", %{"fields" => fields}) + |> json_response(403) + Pleroma.Config.put([:instance, :max_account_fields], 1) fields = [ @@ -332,12 +352,11 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController.UpdateCredentialsTest do %{"name" => "link", "value" => "cofe.io"} ] - conn = - conn - |> assign(:user, user) - |> patch("/api/v1/accounts/update_credentials", %{"fields" => fields}) - - assert %{"error" => "Invalid request"} == json_response(conn, 403) + assert %{"error" => "Invalid request"} == + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_credentials", %{"fields" => fields}) + |> json_response(403) end end end From 2c35d4b0b04e58368c51f2828536d295f72839a2 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 1 Aug 2019 15:09:15 +0700 Subject: [PATCH 26/83] Add configurable account field name length limit --- config/config.exs | 1 + docs/config.md | 1 + lib/pleroma/user/info.ex | 3 ++- .../mastodon_api_controller/update_credentials_test.exs | 9 ++++++--- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/config/config.exs b/config/config.exs index 21f4861f..4fd241e9 100644 --- a/config/config.exs +++ b/config/config.exs @@ -256,6 +256,7 @@ config :pleroma, :instance, user_bio_length: 5000, user_name_length: 100, max_account_fields: 4, + account_field_name_length: 255, account_field_value_length: 255, external_user_synchronization: true diff --git a/docs/config.md b/docs/config.md index fbb50609..6744f587 100644 --- a/docs/config.md +++ b/docs/config.md @@ -133,6 +133,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. * `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. * `max_account_fields`: The maximum number of custom fields in the user profile (default: `4`) +* `account_field_name_length`: An account field name maximum length (default: `255`) * `account_field_value_length`: An account field value maximum length (default: `255`) * `external_user_synchronization`: Enabling following/followers counters synchronization for external users. diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 9e4d381f..e54243f0 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -308,11 +308,12 @@ defmodule Pleroma.User.Info do end defp valid_field?(%{"name" => name, "value" => value}) do + name_limit = Pleroma.Config.get([:instance, :account_field_name_length], 255) value_limit = Pleroma.Config.get([:instance, :account_field_value_length], 255) is_binary(name) && is_binary(value) && - String.length(name) <= 255 && + String.length(name) <= name_limit && String.length(value) <= value_limit end diff --git a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs index 992a692f..e75f25d5 100644 --- a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs @@ -325,11 +325,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController.UpdateCredentialsTest do %{"name" => "link", "value" => "cofe.io"} ] + name_limit = Pleroma.Config.get([:instance, :account_field_name_length]) value_limit = Pleroma.Config.get([:instance, :account_field_value_length]) - long_str = Enum.map(0..value_limit, fn _ -> "x" end) |> Enum.join() + long_value = Enum.map(0..value_limit, fn _ -> "x" end) |> Enum.join() - fields = [%{"name" => "foo", "value" => long_str}] + fields = [%{"name" => "foo", "value" => long_value}] assert %{"error" => "Invalid request"} == conn @@ -337,7 +338,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController.UpdateCredentialsTest do |> patch("/api/v1/accounts/update_credentials", %{"fields" => fields}) |> json_response(403) - fields = [%{"name" => long_str, "value" => "bar"}] + long_name = Enum.map(0..name_limit, fn _ -> "x" end) |> Enum.join() + + fields = [%{"name" => long_name, "value" => "bar"}] assert %{"error" => "Invalid request"} == conn From f7bbf99caade7f06756e95e3a4e2f0e4d3e76579 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 6 Aug 2019 18:21:25 +0700 Subject: [PATCH 27/83] Use info.fields instead of source_data for remote users --- lib/pleroma/html.ex | 28 +++++++++++++++++++ lib/pleroma/user/info.ex | 6 +++- .../web/activity_pub/transmogrifier.ex | 10 +++++-- .../web/activity_pub/views/user_view.ex | 7 ++++- .../mastodon_api/mastodon_api_controller.ex | 13 ++++----- .../web/mastodon_api/views/account_view.ex | 18 ++++++++---- .../web/twitter_api/views/user_view.ex | 10 ++++++- test/web/activity_pub/transmogrifier_test.exs | 12 ++++---- .../web/activity_pub/views/user_view_test.exs | 17 ++++------- .../update_credentials_test.exs | 9 ++++-- 10 files changed, 91 insertions(+), 39 deletions(-) diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index 2fae7281..bf2000d9 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -280,3 +280,31 @@ defmodule Pleroma.HTML.Transform.MediaProxy do def scrub({_tag, children}), do: children def scrub(text), do: text end + +defmodule Pleroma.HTML.Scrubber.LinksOnly do + @moduledoc """ + An HTML scrubbing policy which limits to links only. + """ + + @valid_schemes Pleroma.Config.get([:uri_schemes, :valid_schemes], []) + + require HtmlSanitizeEx.Scrubber.Meta + alias HtmlSanitizeEx.Scrubber.Meta + + Meta.remove_cdata_sections_before_scrub() + Meta.strip_comments() + + # links + Meta.allow_tag_with_uri_attributes("a", ["href"], @valid_schemes) + + Meta.allow_tag_with_this_attribute_values("a", "rel", [ + "tag", + "nofollow", + "noopener", + "noreferrer", + "me" + ]) + + Meta.allow_tag_with_these_attributes("a", ["name", "title"]) + Meta.strip_everything_not_covered() +end diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index e54243f0..ada9fb68 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -50,6 +50,7 @@ defmodule Pleroma.User.Info do field(:emoji, {:array, :map}, default: []) field(:pleroma_settings_store, :map, default: %{}) field(:fields, {:array, :map}, default: []) + field(:raw_fields, {:array, :map}, default: []) field(:notification_settings, :map, default: %{ @@ -270,8 +271,10 @@ defmodule Pleroma.User.Info do :follower_count, :following_count, :hide_follows, + :fields, :hide_followers ]) + |> validate_fields() end def profile_update(info, params) do @@ -288,6 +291,7 @@ defmodule Pleroma.User.Info do :show_role, :skip_thread_containment, :fields, + :raw_fields, :pleroma_settings_store ]) |> validate_fields() @@ -415,7 +419,7 @@ defmodule Pleroma.User.Info do # ``fields`` is an array of mastodon profile field, containing ``{"name": "…", "value": "…"}``. # For example: [{"name": "Pronoun", "value": "she/her"}, …] - def fields(%{source_data: %{"attachment" => attachment}}) do + def fields(%{fields: [], source_data: %{"attachment" => attachment}}) do attachment |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 225c3487..2be2e329 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -598,13 +598,17 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do banner = new_user_data[:info][:banner] locked = new_user_data[:info][:locked] || false - attachment = get_in(new_user_data, [:info, "source_data", "attachment"]) + attachment = get_in(new_user_data, [:info, :source_data, "attachment"]) || [] + + fields = + attachment + |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) + |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) update_data = new_user_data |> Map.take([:name, :bio, :avatar]) - |> Map.put(:info, %{banner: banner, locked: locked}) - |> Map.put(:info, %{"banner" => banner, "locked" => locked, "source_data" => source_data}) + |> Map.put(:info, %{banner: banner, locked: locked, fields: fields}) actor |> User.upgrade_changeset(update_data) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index b2a22478..7be734b2 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -83,8 +83,13 @@ defmodule Pleroma.Web.ActivityPub.UserView do fields = user.info |> User.Info.fields() + |> Enum.map(fn %{"name" => name, "value" => value} -> + %{ + "name" => Pleroma.HTML.strip_tags(name), + "value" => Pleroma.HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly) + } + end) |> Enum.map(&Map.put(&1, "type", "PropertyValue")) - |> Enum.map(fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end) %{ "id" => user.ap_id, diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index e79a02ca..e8fac888 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -137,7 +137,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do emojis_text = (user_params["display_name"] || "") <> (user_params["note"] || "") user_info_emojis = - ((user.info.emoji || []) ++ Formatter.get_emoji_map(emojis_text)) + user.info + |> Map.get(:emoji, []) + |> Enum.concat(Formatter.get_emoji_map(emojis_text)) |> Enum.dedup() info_params = @@ -157,16 +159,11 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end) |> add_if_present(params, "default_scope", :default_scope) |> add_if_present(params, "fields", :fields, fn fields -> - fields = - Enum.map(fields, fn field -> - %{ - "name" => Formatter.html_escape(field["name"], "text/plain"), - "value" => Formatter.html_escape(field["value"], "text/plain") - } - end) + fields = Enum.map(fields, fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end) {:ok, fields} end) + |> add_if_present(params, "fields", :raw_fields) |> add_if_present(params, "pleroma_settings_store", :pleroma_settings_store, fn value -> {:ok, Map.merge(user.info.pleroma_settings_store, value)} end) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index d2f3986f..a2297a8e 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -93,11 +93,19 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do } end) - fields = User.Info.fields(user.info) - fields_html = Enum.map(fields, fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end) + fields = + user.info + |> User.Info.fields() + |> Enum.map(fn %{"name" => name, "value" => value} -> + %{ + "name" => Pleroma.HTML.strip_tags(name), + "value" => Pleroma.HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly) + } + end) + + raw_fields = Map.get(user.info, :raw_fields, []) bio = HTML.filter_tags(user.bio, User.html_filter_policy(opts[:for])) - relationship = render("relationship.json", %{user: opts[:for], target: user}) %{ @@ -117,12 +125,12 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do header: header, header_static: header, emojis: emojis, - fields: fields_html, + fields: fields, bot: bot, source: %{ note: HTML.strip_tags((user.bio || "") |> String.replace("
", "\n")), sensitive: false, - fields: fields, + fields: raw_fields, pleroma: %{} }, diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex index 3681773b..8a7d2fc7 100644 --- a/lib/pleroma/web/twitter_api/views/user_view.ex +++ b/lib/pleroma/web/twitter_api/views/user_view.ex @@ -74,7 +74,15 @@ defmodule Pleroma.Web.TwitterAPI.UserView do |> HTML.filter_tags(User.html_filter_policy(for_user)) |> Formatter.emojify(emoji) - fields = User.Info.fields(user.info) + fields = + user.info + |> User.Info.fields() + |> Enum.map(fn %{"name" => name, "value" => value} -> + %{ + "name" => Pleroma.HTML.strip_tags(name), + "value" => Pleroma.HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly) + } + end) data = %{ diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 05ec09ec..7e2c8769 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -518,9 +518,9 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do user = User.get_cached_by_ap_id(activity.actor) - assert user.info.source_data["attachment"] == [ - %{"name" => "foo", "type" => "PropertyValue", "value" => "bar"}, - %{"name" => "foo1", "type" => "PropertyValue", "value" => "bar1"} + assert User.Info.fields(user.info) == [ + %{"name" => "foo", "value" => "bar"}, + %{"name" => "foo1", "value" => "bar1"} ] update_data = File.read!("test/fixtures/mastodon-update.json") |> Poison.decode!() @@ -539,9 +539,9 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do user = User.get_cached_by_ap_id(user.ap_id) - assert user.info.source_data["attachment"] == [ - %{"name" => "foo", "type" => "PropertyValue", "value" => "updated"}, - %{"name" => "foo1", "type" => "PropertyValue", "value" => "updated"} + assert User.Info.fields(user.info) == [ + %{"name" => "foo", "value" => "updated"}, + %{"name" => "foo1", "value" => "updated"} ] end diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs index a2aa5238..fb7fd9e7 100644 --- a/test/web/activity_pub/views/user_view_test.exs +++ b/test/web/activity_pub/views/user_view_test.exs @@ -24,21 +24,16 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do test "Renders profile fields" do fields = [ - %{"name" => "foo", "value" => "bar"}, - %{"name" => "website", "value" => "cofe.my"} + %{"name" => "foo", "value" => "bar"} ] - user = insert(:user, info: %{fields: fields}) + {:ok, user} = + insert(:user) + |> User.upgrade_changeset(%{info: %{fields: fields}}) + |> User.update_and_set_cache() assert %{ - "attachment" => [ - %{"name" => "foo", "type" => "PropertyValue", "value" => "bar"}, - %{ - "name" => "website", - "type" => "PropertyValue", - "value" => "cofe.my" - } - ] + "attachment" => [%{"name" => "foo", "type" => "PropertyValue", "value" => "bar"}] } = UserView.render("user.json", %{user: user}) end diff --git a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs index e75f25d5..dd443495 100644 --- a/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs @@ -305,7 +305,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController.UpdateCredentialsTest do user = insert(:user) fields = [ - %{"name" => "foo", "value" => "bar"}, + %{"name" => "foo", "value" => ""}, %{"name" => "link", "value" => "cofe.io"} ] @@ -316,12 +316,15 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController.UpdateCredentialsTest do |> json_response(200) assert account["fields"] == [ - %{"name" => "<b>foo<b>", "value" => "<i>bar</i>"}, + %{"name" => "foo", "value" => "bar"}, %{"name" => "link", "value" => "cofe.io"} ] assert account["source"]["fields"] == [ - %{"name" => "<b>foo<b>", "value" => "<i>bar</i>"}, + %{ + "name" => "foo", + "value" => "" + }, %{"name" => "link", "value" => "cofe.io"} ] From e457fcc47971df6c76c3da096e6b45c2972e4029 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 7 Aug 2019 18:14:22 +0700 Subject: [PATCH 28/83] Add `:max_remote_account_fields` config option --- config/config.exs | 1 + docs/config.md | 1 + lib/pleroma/user.ex | 4 ++-- lib/pleroma/user/info.ex | 11 +++++++---- lib/pleroma/web/activity_pub/activity_pub.ex | 7 +++++++ lib/pleroma/web/activity_pub/transmogrifier.ex | 2 +- test/web/activity_pub/transmogrifier_test.exs | 18 ++++++++++++++++++ 7 files changed, 37 insertions(+), 7 deletions(-) diff --git a/config/config.exs b/config/config.exs index 4fd241e9..3937f7e7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -256,6 +256,7 @@ config :pleroma, :instance, user_bio_length: 5000, user_name_length: 100, max_account_fields: 4, + max_remote_account_fields: 10, account_field_name_length: 255, account_field_value_length: 255, external_user_synchronization: true diff --git a/docs/config.md b/docs/config.md index 6744f587..b4bdbd07 100644 --- a/docs/config.md +++ b/docs/config.md @@ -133,6 +133,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. * `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. * `max_account_fields`: The maximum number of custom fields in the user profile (default: `4`) +* `max_remote_account_fields`: The maximum number of custom fields in the remote user profile (default: `10`) * `account_field_name_length`: An account field name maximum length (default: `255`) * `account_field_value_length`: An account field value maximum length (default: `255`) * `external_user_synchronization`: Enabling following/followers counters synchronization for external users. diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index b6774384..faa1e3d5 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -199,12 +199,12 @@ defmodule Pleroma.User do |> validate_length(:name, min: 1, max: name_limit) end - def upgrade_changeset(struct, params \\ %{}) do + def upgrade_changeset(struct, params \\ %{}, remote? \\ false) do bio_limit = Pleroma.Config.get([:instance, :user_bio_length], 5000) name_limit = Pleroma.Config.get([:instance, :user_name_length], 100) params = Map.put(params, :last_refreshed_at, NaiveDateTime.utc_now()) - info_cng = User.Info.user_upgrade(struct.info, params[:info]) + info_cng = User.Info.user_upgrade(struct.info, params[:info], remote?) struct |> cast(params, [ diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index ada9fb68..47e7df91 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -256,11 +256,13 @@ defmodule Pleroma.User.Info do :hide_followers, :hide_follows, :follower_count, + :fields, :following_count ]) + |> validate_fields(true) end - def user_upgrade(info, params) do + def user_upgrade(info, params, remote? \\ false) do info |> cast(params, [ :ap_enabled, @@ -274,7 +276,7 @@ defmodule Pleroma.User.Info do :fields, :hide_followers ]) - |> validate_fields() + |> validate_fields(remote?) end def profile_update(info, params) do @@ -297,8 +299,9 @@ defmodule Pleroma.User.Info do |> validate_fields() end - def validate_fields(changeset) do - limit = Pleroma.Config.get([:instance, :max_account_fields], 0) + def validate_fields(changeset, remote? \\ false) do + limit_name = if remote?, do: :max_remote_account_fields, else: :max_account_fields + limit = Pleroma.Config.get([:instance, limit_name], 0) changeset |> validate_length(:fields, max: limit) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index cf55c952..7bb7740b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1016,6 +1016,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do "url" => [%{"href" => data["image"]["url"]}] } + fields = + data + |> Map.get("attachment", []) + |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) + |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) + locked = data["manuallyApprovesFollowers"] || false data = Transmogrifier.maybe_fix_user_object(data) @@ -1025,6 +1031,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do ap_enabled: true, source_data: data, banner: banner, + fields: fields, locked: locked }, avatar: avatar, diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 2be2e329..36340a3a 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -611,7 +611,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Map.put(:info, %{banner: banner, locked: locked, fields: fields}) actor - |> User.upgrade_changeset(update_data) + |> User.upgrade_changeset(update_data, true) |> User.update_and_set_cache() ActivityPub.update(%{ diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 7e2c8769..d8fbcd62 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -539,6 +539,24 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do user = User.get_cached_by_ap_id(user.ap_id) + assert User.Info.fields(user.info) == [ + %{"name" => "foo", "value" => "updated"}, + %{"name" => "foo1", "value" => "updated"} + ] + + Pleroma.Config.put([:instance, :max_remote_account_fields], 2) + + update_data = + put_in(update_data, ["object", "attachment"], [ + %{"name" => "foo", "type" => "PropertyValue", "value" => "bar"}, + %{"name" => "foo11", "type" => "PropertyValue", "value" => "bar11"}, + %{"name" => "foo22", "type" => "PropertyValue", "value" => "bar22"} + ]) + + {:ok, _} = Transmogrifier.handle_incoming(update_data) + + user = User.get_cached_by_ap_id(user.ap_id) + assert User.Info.fields(user.info) == [ %{"name" => "foo", "value" => "updated"}, %{"name" => "foo1", "value" => "updated"} From 672fcbc7b716f18346a17845d05c286b45dca5f3 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 7 Aug 2019 18:48:05 +0700 Subject: [PATCH 29/83] Limit custom fields for old remote users --- lib/pleroma/user/info.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 47e7df91..45a39924 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -423,9 +423,12 @@ defmodule Pleroma.User.Info do # ``fields`` is an array of mastodon profile field, containing ``{"name": "…", "value": "…"}``. # For example: [{"name": "Pronoun", "value": "she/her"}, …] def fields(%{fields: [], source_data: %{"attachment" => attachment}}) do + limit = Pleroma.Config.get([:instance, :max_remote_account_fields], 0) + attachment |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) + |> Enum.take(limit) end def fields(%{fields: fields}), do: fields From a2e1db56323d0f306ee42a1f58471eb55c8c1e68 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 8 Aug 2019 13:20:35 +0700 Subject: [PATCH 30/83] Increase max account fields limits --- config/config.exs | 4 ++-- docs/config.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/config.exs b/config/config.exs index 3937f7e7..7279b43b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -255,8 +255,8 @@ config :pleroma, :instance, dynamic_configuration: false, user_bio_length: 5000, user_name_length: 100, - max_account_fields: 4, - max_remote_account_fields: 10, + max_account_fields: 10, + max_remote_account_fields: 20, account_field_name_length: 255, account_field_value_length: 255, external_user_synchronization: true diff --git a/docs/config.md b/docs/config.md index b4bdbd07..7e4e96db 100644 --- a/docs/config.md +++ b/docs/config.md @@ -132,8 +132,8 @@ config :pleroma, Pleroma.Emails.Mailer, * `skip_thread_containment`: Skip filter out broken threads. The default is `false`. * `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. * `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. -* `max_account_fields`: The maximum number of custom fields in the user profile (default: `4`) -* `max_remote_account_fields`: The maximum number of custom fields in the remote user profile (default: `10`) +* `max_account_fields`: The maximum number of custom fields in the user profile (default: `10`) +* `max_remote_account_fields`: The maximum number of custom fields in the remote user profile (default: `20`) * `account_field_name_length`: An account field name maximum length (default: `255`) * `account_field_value_length`: An account field value maximum length (default: `255`) * `external_user_synchronization`: Enabling following/followers counters synchronization for external users. From 4b7f1c6995ca49c782e3e29d14245f18d4d11430 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 14 Aug 2019 20:46:05 +0700 Subject: [PATCH 31/83] Improve digest email template --- lib/mix/tasks/pleroma/digest.ex | 12 +- lib/pleroma/emails/user_email.ex | 17 + .../web/templates/email/digest.html.eex | 581 +++++++++++++++++- lib/pleroma/web/views/email_view.ex | 10 + test/mix/tasks/pleroma.digest_test.exs | 2 +- 5 files changed, 602 insertions(+), 20 deletions(-) diff --git a/lib/mix/tasks/pleroma/digest.ex b/lib/mix/tasks/pleroma/digest.ex index 81c207e1..430116a5 100644 --- a/lib/mix/tasks/pleroma/digest.ex +++ b/lib/mix/tasks/pleroma/digest.ex @@ -27,7 +27,15 @@ defmodule Mix.Tasks.Pleroma.Digest do patched_user = %{user | last_digest_emailed_at: last_digest_emailed_at} - _user = Pleroma.DigestEmailWorker.perform(patched_user) - Mix.shell().info("Digest email have been sent to #{nickname} (#{user.email})") + with %Swoosh.Email{} = email <- Pleroma.Emails.UserEmail.digest_email(patched_user) do + {:ok, _} = Pleroma.Emails.Mailer.deliver(email) + + Mix.shell().info("Digest email have been sent to #{nickname} (#{user.email})") + else + _ -> + Mix.shell().info( + "Cound't find any mentions for #{nickname} since #{last_digest_emailed_at}" + ) + end end end diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index 49046bb8..bf6b811b 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -123,6 +123,11 @@ defmodule Pleroma.Emails.UserEmail do end) with [_ | _] = mentions <- new_notifications.mentions do + mentions = + Enum.map(mentions, fn mention -> + update_in(mention.object.data["content"], &format_links/1) + end) + html_data = %{ instance: instance_name(), user: user, @@ -131,17 +136,29 @@ defmodule Pleroma.Emails.UserEmail do unsubscribe_link: unsubscribe_url(user, "digest") } + logo_path = Path.join(:code.priv_dir(:pleroma), "static/static/logo.png") + new() |> to(recipient(user)) |> from(sender()) |> subject("Your digest from #{instance_name()}") + |> put_layout(false) |> render_body("digest.html", html_data) + |> attachment(Swoosh.Attachment.new(logo_path, filename: "logo.png", type: :inline)) else _ -> nil end end + defp format_links(str) do + re = ~r//iU + + String.replace(str, re, fn link -> + String.replace(link, "Hey <%= @user.nickname %>, here is what you've missed! + -

New Mentions:

-
    -<%= for %{data: mention, object: object, from: from} <- @mentions do %> -
  • <%= link from.nickname, to: mention.activity.actor %>: <%= raw object.data["content"] %>
    • -<% end %> -
+ -<%= if @followers != [] do %> -

<%= length(@followers) %> New Followers:

-
    -<%= for %{data: follow, from: from} <- @followers do %> -
  • <%= link from.nickname, to: follow.activity.actor %>
    • -<% end %> -
-<% end %> + + + + + + + + <%= @email.subject %>< + + + + + + + + + + + + + + + + + + + diff --git a/lib/pleroma/web/views/email_view.ex b/lib/pleroma/web/views/email_view.ex index b63eb162..b506a234 100644 --- a/lib/pleroma/web/views/email_view.ex +++ b/lib/pleroma/web/views/email_view.ex @@ -2,4 +2,14 @@ defmodule Pleroma.Web.EmailView do use Pleroma.Web, :view import Phoenix.HTML import Phoenix.HTML.Link + + def avatar_url(user) do + Pleroma.User.avatar_url(user) + end + + def format_date(date) when is_binary(date) do + date + |> Timex.parse!("{ISO:Extended:Z}") + |> Timex.format!("{Mshort} {D}, {YYYY} {h24}:{m}") + end end diff --git a/test/mix/tasks/pleroma.digest_test.exs b/test/mix/tasks/pleroma.digest_test.exs index 595f64ed..4bfa1fb9 100644 --- a/test/mix/tasks/pleroma.digest_test.exs +++ b/test/mix/tasks/pleroma.digest_test.exs @@ -44,7 +44,7 @@ defmodule Mix.Tasks.Pleroma.DigestTest do assert_email_sent( to: {user2.name, user2.email}, - html_body: ~r/new mentions:/i + html_body: ~r/here is what you've missed!/i ) end end From c9970feee20f25375223e5f4a32bdbcff7b98607 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 14 Aug 2019 21:03:25 +0700 Subject: [PATCH 32/83] Fix compatibility with Elixir 1.8 --- lib/pleroma/emails/user_email.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index bf6b811b..3b5e6401 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -154,7 +154,7 @@ defmodule Pleroma.Emails.UserEmail do defp format_links(str) do re = ~r//iU - String.replace(str, re, fn link -> + Regex.replace(re, str, fn link -> String.replace(link, " Date: Thu, 15 Aug 2019 01:35:29 +0300 Subject: [PATCH 33/83] Do not check if actor is active when deleting a user --- CHANGELOG.md | 1 + lib/mix/tasks/pleroma/user.ex | 2 +- lib/pleroma/user.ex | 34 +++++++++---------- lib/pleroma/web/activity_pub/activity_pub.ex | 20 +++++------ .../web/activity_pub/transmogrifier.ex | 2 +- .../web/admin_api/admin_api_controller.ex | 4 +-- .../web/ostatus/handlers/delete_handler.ex | 2 +- test/user_test.exs | 3 +- 8 files changed, 34 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dccc3696..a7b776be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Security - OStatus: eliminate the possibility of a protocol downgrade attack. - OStatus: prevent following locked accounts, bypassing the approval process. +– ActivityPub: Do not check if actor is active when deleting a user ### Changed - **Breaking:** Configuration: A setting to explicitly disable the mailer was added, defaulting to true, if you are using a mailer add `config :pleroma, Pleroma.Emails.Mailer, enabled: true` to your config diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index f33d0142..a3f8bc94 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -176,7 +176,7 @@ defmodule Mix.Tasks.Pleroma.User do start_pleroma() with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do - User.perform(:delete, user, nil) + User.perform(:delete, user) shell_info("User #{nickname} deleted.") else _ -> diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 14057a0e..7d18f099 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1029,26 +1029,13 @@ defmodule Pleroma.User do |> update_and_set_cache() end - @spec perform(atom(), User.t()) :: {:ok, User.t()} - def perform(:fetch_initial_posts, %User{} = user) do - pages = Pleroma.Config.get!([:fetch_initial_posts, :pages]) - - Enum.each( - # Insert all the posts in reverse order, so they're in the right order on the timeline - Enum.reverse(Utils.fetch_ordered_collection(user.info.source_data["outbox"], pages)), - &Pleroma.Web.Federator.incoming_ap_doc/1 - ) - - {:ok, user} - end - @spec delete(User.t()) :: :ok - def delete(%User{} = user, actor \\ nil), - do: PleromaJobQueue.enqueue(:background, __MODULE__, [:delete, user, actor]) + def delete(%User{} = user), + do: PleromaJobQueue.enqueue(:background, __MODULE__, [:delete, user]) @spec perform(atom(), User.t()) :: {:ok, User.t()} - def perform(:delete, %User{} = user, actor) do - {:ok, _user} = ActivityPub.delete(user, actor: actor) + def perform(:delete, %User{} = user) do + {:ok, _user} = ActivityPub.delete(user) # Remove all relationships {:ok, followers} = User.get_followers(user) @@ -1070,6 +1057,19 @@ defmodule Pleroma.User do Repo.delete(user) end + @spec perform(atom(), User.t()) :: {:ok, User.t()} + def perform(:fetch_initial_posts, %User{} = user) do + pages = Pleroma.Config.get!([:fetch_initial_posts, :pages]) + + Enum.each( + # Insert all the posts in reverse order, so they're in the right order on the timeline + Enum.reverse(Utils.fetch_ordered_collection(user.info.source_data["outbox"], pages)), + &Pleroma.Web.Federator.incoming_ap_doc/1 + ) + + {:ok, user} + end + def perform(:deactivate_async, user, status), do: deactivate(user, status) @spec perform(atom(), User.t(), list()) :: list() | {:error, any()} diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 8f669acb..da873b7b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -61,7 +61,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do {recipients, to, cc} end - defp check_actor_is_active(actor) do + defp check_actor_is_active(true, _), do: :ok + + defp check_actor_is_active(false, actor) do if not is_nil(actor) do with user <- User.get_cached_by_ap_id(actor), false <- user.info.deactivated do @@ -119,10 +121,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def increase_poll_votes_if_vote(_create_data), do: :noop - def insert(map, local \\ true, fake \\ false) when is_map(map) do + def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when is_map(map) do with nil <- Activity.normalize(map), map <- lazy_put_activity_defaults(map, fake), - :ok <- check_actor_is_active(map["actor"]), + :ok <- check_actor_is_active(bypass_actor_check, map["actor"]), {_, true} <- {:remote_limit_error, check_remote_limit(map)}, {:ok, map} <- MRF.filter(map), {recipients, _, _} = get_recipients(map), @@ -403,22 +405,20 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end - def delete(data, opts \\ %{actor: nil, local: true}) - - def delete(%User{ap_id: ap_id, follower_address: follower_address} = user, opts) do + def delete(%User{ap_id: ap_id, follower_address: follower_address} = user) do with data <- %{ "to" => [follower_address], "type" => "Delete", - "actor" => opts[:actor] || ap_id, + "actor" => ap_id, "object" => %{"type" => "Person", "id" => ap_id} }, - {:ok, activity} <- insert(data, true, true), + {:ok, activity} <- insert(data, true, true, true), :ok <- maybe_federate(activity) do {:ok, user} end end - def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, opts) do + def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, local \\ true) do user = User.get_cached_by_ap_id(actor) to = (object.data["to"] || []) ++ (object.data["cc"] || []) @@ -430,7 +430,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do "to" => to, "deleted_activity_id" => activity && activity.id }, - {:ok, activity} <- insert(data, opts[:local], false), + {:ok, activity} <- insert(data, local, false), stream_out_participations(object, user), _ <- decrease_replies_count_if_reply(object), # Changing note count prior to enqueuing federation task in order to avoid diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index b34ef73c..5403b71d 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -649,7 +649,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id), :ok <- Containment.contain_origin(actor.ap_id, object.data), - {:ok, activity} <- ActivityPub.delete(object, local: false) do + {:ok, activity} <- ActivityPub.delete(object, false) do {:ok, activity} else nil -> diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 63c9a7d7..2d3d0adc 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -25,9 +25,9 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do action_fallback(:errors) - def user_delete(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do + def user_delete(conn, %{"nickname" => nickname}) do User.get_cached_by_nickname(nickname) - |> User.delete(admin.ap_id) + |> User.delete() conn |> json(nickname) diff --git a/lib/pleroma/web/ostatus/handlers/delete_handler.ex b/lib/pleroma/web/ostatus/handlers/delete_handler.ex index ac2dc115..b2f9f394 100644 --- a/lib/pleroma/web/ostatus/handlers/delete_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/delete_handler.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Web.OStatus.DeleteHandler do def handle_delete(entry, _doc \\ nil) do with id <- XML.string_from_xpath("//id", entry), %Object{} = object <- Object.normalize(id), - {:ok, delete} <- ActivityPub.delete(object, local: false) do + {:ok, delete} <- ActivityPub.delete(object, false) do delete end end diff --git a/test/user_test.exs b/test/user_test.exs index e2da8d84..755c6005 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -999,10 +999,9 @@ defmodule Pleroma.UserTest do end test "it deletes deactivated user" do - admin = insert(:user, %{info: %{is_admin: true}}) {:ok, user} = insert(:user, info: %{deactivated: true}) |> User.set_cache() - assert {:ok, _} = User.delete(user, admin.ap_id) + assert {:ok, _} = User.delete(user) refute User.get_by_id(user.id) end From b27fafe161241c954b713281bebd6ffe1e990884 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Thu, 15 Aug 2019 01:42:43 +0300 Subject: [PATCH 34/83] Fix CHANGELOG entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7b776be..161f8817 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Security - OStatus: eliminate the possibility of a protocol downgrade attack. - OStatus: prevent following locked accounts, bypassing the approval process. -– ActivityPub: Do not check if actor is active when deleting a user ### Changed - **Breaking:** Configuration: A setting to explicitly disable the mailer was added, defaulting to true, if you are using a mailer add `config :pleroma, Pleroma.Emails.Mailer, enabled: true` to your config @@ -39,6 +38,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Invalid SemVer version generation, when the current branch does not have commits ahead of tag/checked out on a tag - Pleroma.Upload base_url was not automatically whitelisted by MediaProxy. Now your custom CDN or file hosting will be accessed directly as expected. - Report email not being sent to admins when the reporter is a remote user +- ActivityPub: Deactivated user deletion ### Added - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) From 6a3b1a526eb1a3ea49aca7914ed7ba8736c52059 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 15 Aug 2019 15:34:41 -0500 Subject: [PATCH 35/83] max_body_size -> max_body_length, as it should be --- lib/pleroma/reverse_proxy/reverse_proxy.ex | 4 ++-- test/reverse_proxy_test.exs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 1f98f215..7518e4c3 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -109,7 +109,7 @@ defmodule Pleroma.ReverseProxy do end with {:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts), - :ok <- header_length_constraint(headers, Keyword.get(opts, :max_body_length)) do + :ok <- header_length_constraint(headers, Keyword.get(opts, :max_body_length, @max_body_length)) do response(conn, client, url, code, headers, opts) else {:ok, code, headers} -> @@ -200,7 +200,7 @@ defmodule Pleroma.ReverseProxy do {:ok, data} <- client().stream_body(client), {:ok, duration} <- increase_read_duration(duration), sent_so_far = sent_so_far + byte_size(data), - :ok <- body_size_constraint(sent_so_far, Keyword.get(opts, :max_body_size)), + :ok <- body_size_constraint(sent_so_far, Keyword.get(opts, :max_body_length, @max_body_length)), {:ok, conn} <- chunk(conn, data) do chunk_reply(conn, client, opts, sent_so_far, duration) else diff --git a/test/reverse_proxy_test.exs b/test/reverse_proxy_test.exs index f4b7d6ad..3a83c4c4 100644 --- a/test/reverse_proxy_test.exs +++ b/test/reverse_proxy_test.exs @@ -108,11 +108,11 @@ defmodule Pleroma.ReverseProxyTest do end end - test "max_body_size returns error if streaming body more than that option", %{conn: conn} do + test "max_body_length returns error if streaming body more than that option", %{conn: conn} do stream_mock(3, true) assert capture_log(fn -> - ReverseProxy.call(conn, "/stream-bytes/50", max_body_size: 30) + ReverseProxy.call(conn, "/stream-bytes/50", max_body_length: 30) end) =~ "[warn] Elixir.Pleroma.ReverseProxy request to /stream-bytes/50 failed while reading/chunking: :body_too_large" end From 158231cd20ceabf805ce49ae2b4d465e09e34d69 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 16 Aug 2019 18:32:25 +0700 Subject: [PATCH 36/83] Add configurable colors and logo for the digest template --- config/config.exs | 11 ++ docs/config.md | 5 + lib/pleroma/emails/user_email.ex | 88 +++++----- .../web/templates/email/digest.html.eex | 163 +++++++++--------- 4 files changed, 143 insertions(+), 124 deletions(-) diff --git a/config/config.exs b/config/config.exs index d2325edb..4941953e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -507,6 +507,17 @@ config :pleroma, :auth, oauth_consumer_strategies: oauth_consumer_strategies config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Sendmail, enabled: false +config :pleroma, Pleroma.Emails.UserEmail, + logo: nil, + styling: %{ + link_color: "#d8a070", + background_color: "#2C3645", + content_background_color: "#1B2635", + header_color: "#d8a070", + text_color: "#b9b9ba", + text_muted_color: "#b9b9ba" + } + config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, path: "/api/pleroma/app_metrics" config :pleroma, Pleroma.ScheduledActivity, diff --git a/docs/config.md b/docs/config.md index 703ef67d..8360e640 100644 --- a/docs/config.md +++ b/docs/config.md @@ -548,6 +548,11 @@ Email notifications settings. - interval: Minimum interval between digest emails to one user - inactivity_threshold: Minimum user inactivity threshold +## Pleroma.Emails.UserEmail + +- `:logo` - a path to a custom logo. Set it to `nil` to use the default Pleroma logo. +- `:styling` - a map with color settings for email templates. + ## OAuth consumer mode OAuth consumer mode allows sign in / sign up via external OAuth providers (e.g. Twitter, Facebook, Google, Microsoft, etc.). diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index 3b5e6401..40b67ff5 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -7,21 +7,21 @@ defmodule Pleroma.Emails.UserEmail do use Phoenix.Swoosh, view: Pleroma.Web.EmailView, layout: {Pleroma.Web.LayoutView, :email} + alias Pleroma.Config + alias Pleroma.User alias Pleroma.Web.Endpoint alias Pleroma.Web.Router - defp instance_config, do: Pleroma.Config.get(:instance) - - defp instance_name, do: instance_config()[:name] + defp instance_name, do: Config.get([:instance, :name]) defp sender do - email = Keyword.get(instance_config(), :notify_email, instance_config()[:email]) + email = Config.get([:instance, :notify_email]) || Config.get([:instance, :email]) {instance_name(), email} end defp recipient(email, nil), do: email defp recipient(email, name), do: {name, email} - defp recipient(%Pleroma.User{} = user), do: recipient(user.email, user.name) + defp recipient(%User{} = user), do: recipient(user.email, user.name) def password_reset_email(user, token) when is_binary(token) do password_reset_url = Router.Helpers.reset_password_url(Endpoint, :reset, token) @@ -93,50 +93,54 @@ defmodule Pleroma.Emails.UserEmail do Includes Mentions and New Followers data If there are no mentions (even when new followers exist), the function will return nil """ - @spec digest_email(Pleroma.User.t()) :: Swoosh.Email.t() | nil + @spec digest_email(User.t()) :: Swoosh.Email.t() | nil def digest_email(user) do - new_notifications = - Pleroma.Notification.for_user_since(user, user.last_digest_emailed_at) - |> Enum.reduce(%{followers: [], mentions: []}, fn - %{activity: %{data: %{"type" => "Create"}, actor: actor} = activity} = notification, - acc -> - new_mention = %{ - data: notification, - object: Pleroma.Object.normalize(activity), - from: Pleroma.User.get_by_ap_id(actor) - } + notifications = Pleroma.Notification.for_user_since(user, user.last_digest_emailed_at) - %{acc | mentions: [new_mention | acc.mentions]} + mentions = + notifications + |> Enum.filter(&(&1.activity.data["type"] == "Create")) + |> Enum.map(fn notification -> + object = Pleroma.Object.normalize(notification.activity) + object = update_in(object.data["content"], &format_links/1) - %{activity: %{data: %{"type" => "Follow"}, actor: actor} = activity} = notification, - acc -> - new_follower = %{ - data: notification, - object: Pleroma.Object.normalize(activity), - from: Pleroma.User.get_by_ap_id(actor) - } - - %{acc | followers: [new_follower | acc.followers]} - - _, acc -> - acc + %{ + data: notification, + object: object, + from: User.get_by_ap_id(notification.activity.actor) + } end) - with [_ | _] = mentions <- new_notifications.mentions do - mentions = - Enum.map(mentions, fn mention -> - update_in(mention.object.data["content"], &format_links/1) - end) + followers = + notifications + |> Enum.filter(&(&1.activity.data["type"] == "Follow")) + |> Enum.map(fn notification -> + %{ + data: notification, + object: Pleroma.Object.normalize(notification.activity), + from: User.get_by_ap_id(notification.activity.actor) + } + end) + + unless Enum.empty?(mentions) do + styling = Config.get([__MODULE__, :styling]) + logo = Config.get([__MODULE__, :logo]) html_data = %{ instance: instance_name(), user: user, mentions: mentions, - followers: new_notifications.followers, - unsubscribe_link: unsubscribe_url(user, "digest") + followers: followers, + unsubscribe_link: unsubscribe_url(user, "digest"), + styling: styling } - logo_path = Path.join(:code.priv_dir(:pleroma), "static/static/logo.png") + logo_path = + if is_nil(logo) do + Path.join(:code.priv_dir(:pleroma), "static/static/logo.png") + else + Path.join(Config.get([:instance, :static_dir]), logo) + end new() |> to(recipient(user)) @@ -145,17 +149,15 @@ defmodule Pleroma.Emails.UserEmail do |> put_layout(false) |> render_body("digest.html", html_data) |> attachment(Swoosh.Attachment.new(logo_path, filename: "logo.png", type: :inline)) - else - _ -> - nil end end defp format_links(str) do re = ~r//iU + %{link_color: color} = Config.get([__MODULE__, :styling]) Regex.replace(re, str, fn link -> - String.replace(link, " user.id, "act" => %{"unsubscribe" => notifications_type}, "exp" => false} |> Pleroma.JWT.generate_and_sign!() |> Base.encode64() - Router.Helpers.subscription_url(Pleroma.Web.Endpoint, :unsubscribe, token) + Router.Helpers.subscription_url(Endpoint, :unsubscribe, token) end end diff --git a/lib/pleroma/web/templates/email/digest.html.eex b/lib/pleroma/web/templates/email/digest.html.eex index 61d57093..860df5f9 100644 --- a/lib/pleroma/web/templates/email/digest.html.eex +++ b/lib/pleroma/web/templates/email/digest.html.eex @@ -21,7 +21,8 @@ } a { - color: #d8a070; + + color: <%= @styling.link_color %>; text-decoration: none; } @@ -100,21 +101,21 @@ - + -