From f57d43c1b3f317b66b42b9d748d88dd0227a2079 Mon Sep 17 00:00:00 2001 From: Francis Tyers Date: Sat, 14 Jul 2018 14:05:14 +0200 Subject: [PATCH 1/2] tatar.rb Add an example for Tatar with abbreviations. --- lib/pragmatic_segmenter/languages/tatar.rb | 28 ++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 lib/pragmatic_segmenter/languages/tatar.rb diff --git a/lib/pragmatic_segmenter/languages/tatar.rb b/lib/pragmatic_segmenter/languages/tatar.rb new file mode 100644 index 0000000..66c3c82 --- /dev/null +++ b/lib/pragmatic_segmenter/languages/tatar.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module PragmaticSegmenter + module Languages + module Tatar + include Languages::Common + + module Abbreviation + ABBREVIATIONS = Set.new(["биекл", "биол", "б.э", "б.э.к", "геогр", "геол", "е", "елл", "к", "кг", "км", "км²", "км³", "лат", "м", "м²", "м³", "мәс", "млн", "млрд", "мм", "обл", "окр", "пед", "см", "т", "тех", "тирәнл", "трлн", "һ.б", "хим", "экон", "эл"]).freeze + PREPOSITIVE_ABBREVIATIONS = [].freeze + NUMBER_ABBREVIATIONS = [].freeze + end + + class AbbreviationReplacer < AbbreviationReplacer + SENTENCE_STARTERS = [].freeze + + private + + def replace_period_of_abbr(txt, abbr) + txt.gsub!(/(?<=\s#{abbr.strip})\./, '∯') + txt.gsub!(/(?<=\A#{abbr.strip})\./, '∯') + txt.gsub!(/(?<=^#{abbr.strip})\./, '∯') + txt + end + end + end + end +end From 594726d4a0df9c7ece1def1304f5fc5e0422e4b4 Mon Sep 17 00:00:00 2001 From: Francis Tyers Date: Sat, 14 Jul 2018 14:14:16 +0200 Subject: [PATCH 2/2] Update tatar.rb Copy/paste the handling of 1. from Danish --- lib/pragmatic_segmenter/languages/tatar.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/pragmatic_segmenter/languages/tatar.rb b/lib/pragmatic_segmenter/languages/tatar.rb index 66c3c82..511ef61 100644 --- a/lib/pragmatic_segmenter/languages/tatar.rb +++ b/lib/pragmatic_segmenter/languages/tatar.rb @@ -10,6 +10,16 @@ module Abbreviation PREPOSITIVE_ABBREVIATIONS = [].freeze NUMBER_ABBREVIATIONS = [].freeze end + + # This handles the case where a dot is used to denote and ordinal (5. Juni) + module Numbers + NumberPeriodSpaceRule = Rule.new(/(?<=\s[0-9]|\s([1-9][0-9]))\.(?=\s)/, '∯') + + All = [ + Common::Numbers::All, + NumberPeriodSpaceRule + ] + end class AbbreviationReplacer < AbbreviationReplacer SENTENCE_STARTERS = [].freeze