Remove automatic language detection.
multiple creatures
parent
6c374b5153
commit
dca70079b1
1
Gemfile
1
Gemfile
|
|
@ -30,7 +30,6 @@ gem 'browser'
|
|||
gem 'charlock_holmes', '~> 0.7.6'
|
||||
gem 'iso-639'
|
||||
gem 'chewy', '~> 5.0'
|
||||
gem 'cld3', '~> 3.2.4'
|
||||
gem 'devise', '~> 4.6'
|
||||
gem 'devise-two-factor', '~> 3.0'
|
||||
|
||||
|
|
|
|||
|
|
@ -146,8 +146,6 @@ GEM
|
|||
elasticsearch (>= 2.0.0)
|
||||
elasticsearch-dsl
|
||||
chunky_png (1.3.10)
|
||||
cld3 (3.2.4)
|
||||
ffi (>= 1.1.0, < 1.11.0)
|
||||
climate_control (0.2.0)
|
||||
cocaine (0.5.8)
|
||||
climate_control (>= 0.0.3, < 1.0)
|
||||
|
|
@ -382,10 +380,6 @@ GEM
|
|||
omniauth (~> 1.3, >= 1.3.2)
|
||||
ruby-saml (~> 1.7)
|
||||
orm_adapter (0.5.0)
|
||||
ostatus2 (2.0.3)
|
||||
addressable (~> 2.5)
|
||||
http (~> 3.0)
|
||||
nokogiri (~> 1.8)
|
||||
ox (2.10.0)
|
||||
paperclip (6.0.0)
|
||||
activemodel (>= 4.2.0)
|
||||
|
|
@ -673,6 +667,7 @@ DEPENDENCIES
|
|||
brakeman (~> 4.5)
|
||||
browser
|
||||
bullet (~> 6.0)
|
||||
bundler (~> 1.17)
|
||||
bundler-audit (~> 0.6)
|
||||
capistrano (~> 3.11)
|
||||
capistrano-rails (~> 1.4)
|
||||
|
|
@ -681,7 +676,6 @@ DEPENDENCIES
|
|||
capybara (~> 3.20)
|
||||
charlock_holmes (~> 0.7.6)
|
||||
chewy (~> 5.0)
|
||||
cld3 (~> 3.2.4)
|
||||
climate_control (~> 0.2)
|
||||
concurrent-ruby
|
||||
derailed_benchmarks
|
||||
|
|
@ -728,7 +722,6 @@ DEPENDENCIES
|
|||
omniauth (~> 1.9)
|
||||
omniauth-cas (~> 1.1)
|
||||
omniauth-saml (~> 1.10)
|
||||
ostatus2 (~> 2.0)
|
||||
ox (~> 2.10)
|
||||
paperclip (~> 6.0)
|
||||
paperclip-av-transcoder (~> 0.6)
|
||||
|
|
|
|||
|
|
@ -67,10 +67,6 @@ module SettingsHelper
|
|||
HUMAN_LOCALES[locale]
|
||||
end
|
||||
|
||||
def filterable_languages
|
||||
LanguageDetector.instance.language_names.select(&HUMAN_LOCALES.method(:key?))
|
||||
end
|
||||
|
||||
def hash_to_object(hash)
|
||||
HashObject.new(hash)
|
||||
end
|
||||
|
|
|
|||
|
|
@ -338,7 +338,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
|
|||
elsif summary_language_map?
|
||||
@object['summaryMap'].keys.first
|
||||
elsif supported_object_type?
|
||||
LanguageDetector.instance.detect(text_from_content, @account)
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -1,99 +0,0 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class LanguageDetector
|
||||
include Singleton
|
||||
|
||||
WORDS_THRESHOLD = 4
|
||||
RELIABLE_CHARACTERS_RE = /[\p{Hebrew}\p{Arabic}\p{Syriac}\p{Thaana}\p{Nko}\p{Han}\p{Katakana}\p{Hiragana}\p{Hangul}]+/m
|
||||
|
||||
def initialize
|
||||
@identifier = CLD3::NNetLanguageIdentifier.new(1, 2048)
|
||||
end
|
||||
|
||||
def detect(text, account)
|
||||
input_text = prepare_text(text)
|
||||
|
||||
return if input_text.blank?
|
||||
|
||||
detect_language_code(input_text) || default_locale(account)
|
||||
end
|
||||
|
||||
def language_names
|
||||
@language_names = CLD3::TaskContextParams::LANGUAGE_NAMES.map { |name| iso6391(name.to_s).to_sym }.uniq
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def prepare_text(text)
|
||||
simplify_text(text).strip
|
||||
end
|
||||
|
||||
def unreliable_input?(text)
|
||||
!reliable_input?(text)
|
||||
end
|
||||
|
||||
def reliable_input?(text)
|
||||
sufficient_text_length?(text) || language_specific_character_set?(text)
|
||||
end
|
||||
|
||||
def sufficient_text_length?(text)
|
||||
text.split(/\s+/).size >= WORDS_THRESHOLD
|
||||
end
|
||||
|
||||
def language_specific_character_set?(text)
|
||||
words = text.scan(RELIABLE_CHARACTERS_RE)
|
||||
|
||||
if words.present?
|
||||
words.reduce(0) { |acc, elem| acc + elem.size }.to_f / text.size.to_f > 0.3
|
||||
else
|
||||
false
|
||||
end
|
||||
end
|
||||
|
||||
def detect_language_code(text)
|
||||
return if unreliable_input?(text)
|
||||
result = @identifier.find_language(text)
|
||||
iso6391(result.language.to_s).to_sym if result.reliable?
|
||||
end
|
||||
|
||||
def iso6391(bcp47)
|
||||
iso639 = bcp47.split('-').first
|
||||
|
||||
# CLD3 returns grandfathered language code for Hebrew
|
||||
return 'he' if iso639 == 'iw'
|
||||
|
||||
ISO_639.find(iso639).alpha2
|
||||
end
|
||||
|
||||
def simplify_text(text)
|
||||
new_text = remove_html(text)
|
||||
new_text.gsub!(FetchLinkCardService::URL_PATTERN, '')
|
||||
new_text.gsub!(Account::MENTION_RE, '')
|
||||
new_text.gsub!(Tag::HASHTAG_RE, '')
|
||||
new_text.gsub!(/:#{CustomEmoji::SHORTCODE_RE_FRAGMENT}:/, '')
|
||||
new_text.gsub!(/\s+/, ' ')
|
||||
new_text
|
||||
end
|
||||
|
||||
def new_scrubber
|
||||
scrubber = Rails::Html::PermitScrubber.new
|
||||
scrubber.tags = %w(br p)
|
||||
scrubber
|
||||
end
|
||||
|
||||
def scrubber
|
||||
@scrubber ||= new_scrubber
|
||||
end
|
||||
|
||||
def remove_html(text)
|
||||
text = Loofah.fragment(text).scrub!(scrubber).to_s
|
||||
text.gsub!('<br>', "\n")
|
||||
text.gsub!('</p><p>', "\n\n")
|
||||
text.gsub!(/(^<p>|<\/p>$)/, '')
|
||||
text
|
||||
end
|
||||
|
||||
def default_locale(account)
|
||||
account.user_locale&.to_sym || I18n.default_locale if account.local?
|
||||
end
|
||||
end
|
||||
|
|
@ -176,7 +176,7 @@ class PostStatusService < BaseService
|
|||
spoiler_text: @options[:spoiler_text] || '',
|
||||
visibility: @visibility,
|
||||
sharekey: @sharekey,
|
||||
language: language_from_option(@options[:language]) || @account.user&.setting_default_language&.presence || LanguageDetector.instance.detect(@text, @account),
|
||||
language: language_from_option(@options[:language]) || @account.user&.setting_default_language&.presence || nil,
|
||||
application: @options[:application],
|
||||
content_type: @options[:content_type] || @account.user&.setting_default_content_type,
|
||||
}.compact
|
||||
|
|
|
|||
|
|
@ -1,134 +0,0 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require 'rails_helper'
|
||||
|
||||
describe LanguageDetector do
|
||||
describe 'prepare_text' do
|
||||
it 'returns unmodified string without special cases' do
|
||||
string = 'just a regular string'
|
||||
result = described_class.instance.send(:prepare_text, string)
|
||||
|
||||
expect(result).to eq string
|
||||
end
|
||||
|
||||
it 'collapses spacing in strings' do
|
||||
string = 'The formatting in this is very odd'
|
||||
|
||||
result = described_class.instance.send(:prepare_text, string)
|
||||
expect(result).to eq 'The formatting in this is very odd'
|
||||
end
|
||||
|
||||
it 'strips usernames from strings before detection' do
|
||||
string = '@username Yeah, very surreal...! also @friend'
|
||||
|
||||
result = described_class.instance.send(:prepare_text, string)
|
||||
expect(result).to eq 'Yeah, very surreal...! also'
|
||||
end
|
||||
|
||||
it 'strips URLs from strings before detection' do
|
||||
string = 'Our website is https://example.com and also http://localhost.dev'
|
||||
|
||||
result = described_class.instance.send(:prepare_text, string)
|
||||
expect(result).to eq 'Our website is and also'
|
||||
end
|
||||
|
||||
it 'strips #hashtags from strings before detection' do
|
||||
string = 'Hey look at all the #animals and #fish'
|
||||
|
||||
result = described_class.instance.send(:prepare_text, string)
|
||||
expect(result).to eq 'Hey look at all the and'
|
||||
end
|
||||
end
|
||||
|
||||
describe 'detect' do
|
||||
let(:account_without_user_locale) { Fabricate(:user, locale: nil).account }
|
||||
let(:account_remote) { Fabricate(:account, domain: 'joinmastodon.org') }
|
||||
|
||||
it 'detects english language for basic strings' do
|
||||
strings = [
|
||||
"Hello and welcome to mastodon how are you today?",
|
||||
"I'd rather not!",
|
||||
"a lot of people just want to feel righteous all the time and that's all that matters",
|
||||
]
|
||||
strings.each do |string|
|
||||
result = described_class.instance.detect(string, account_without_user_locale)
|
||||
|
||||
expect(result).to eq(:en), string
|
||||
end
|
||||
end
|
||||
|
||||
it 'detects spanish language' do
|
||||
string = 'Obtener un Hola y bienvenidos a Mastodon. Obtener un Hola y bienvenidos a Mastodon. Obtener un Hola y bienvenidos a Mastodon. Obtener un Hola y bienvenidos a Mastodon'
|
||||
result = described_class.instance.detect(string, account_without_user_locale)
|
||||
|
||||
expect(result).to eq :es
|
||||
end
|
||||
|
||||
describe 'when language can\'t be detected' do
|
||||
it 'uses nil when sent an empty document' do
|
||||
result = described_class.instance.detect('', account_without_user_locale)
|
||||
expect(result).to eq nil
|
||||
end
|
||||
|
||||
describe 'because of a URL' do
|
||||
it 'uses nil when sent just a URL' do
|
||||
string = 'http://example.com/media/2kFTgOJLXhQf0g2nKB4'
|
||||
cld_result = CLD3::NNetLanguageIdentifier.new(0, 2048).find_language(string)
|
||||
expect(cld_result).not_to eq :en
|
||||
|
||||
result = described_class.instance.detect(string, account_without_user_locale)
|
||||
|
||||
expect(result).to eq nil
|
||||
end
|
||||
end
|
||||
|
||||
describe 'with an account' do
|
||||
it 'uses the account locale when present' do
|
||||
account = double(user_locale: 'fr')
|
||||
result = described_class.instance.detect('', account)
|
||||
|
||||
expect(result).to eq nil
|
||||
end
|
||||
|
||||
it 'uses nil when account is present but has no locale' do
|
||||
result = described_class.instance.detect('', account_without_user_locale)
|
||||
|
||||
expect(result).to eq nil
|
||||
end
|
||||
end
|
||||
|
||||
describe 'with an `en` default locale' do
|
||||
it 'uses nil for undetectable string' do
|
||||
result = described_class.instance.detect('', account_without_user_locale)
|
||||
|
||||
expect(result).to eq nil
|
||||
end
|
||||
end
|
||||
|
||||
describe 'remote user' do
|
||||
it 'detects Korean language' do
|
||||
string = '안녕하세요'
|
||||
result = described_class.instance.detect(string, account_remote)
|
||||
|
||||
expect(result).to eq :ko
|
||||
end
|
||||
end
|
||||
|
||||
describe 'with a non-`en` default locale' do
|
||||
around(:each) do |example|
|
||||
before = I18n.default_locale
|
||||
I18n.default_locale = :ja
|
||||
example.run
|
||||
I18n.default_locale = before
|
||||
end
|
||||
|
||||
it 'uses nil for undetectable string' do
|
||||
string = ''
|
||||
result = described_class.instance.detect(string, account_without_user_locale)
|
||||
|
||||
expect(result).to eq nil
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
Loading…
Reference in New Issue