From 57d19652a20300266b0a7973218655f3cf6117c2 Mon Sep 17 00:00:00 2001 From: Lucas Carlson Date: Mon, 14 Sep 2026 06:50:37 -0700 Subject: [PATCH 1/2] feat: add JSON Feed parsing Let importers consume JSON Feed 1.0 and 1.1 through the existing parser, fetcher, and normalized entry interface. Preserve source metadata and extensions while recovering optional date and attachment number errors. Keep XML parsing and serialization compatible. Document required-field errors, author inheritance, raw JSON snapshots, and object export rules. Verify format parity, local HTTP behavior, and the shared digest example. Closes #60 --- AGENTS.md | 5 +- CHANGELOG.md | 9 + README.md | 131 ++++++++++- Steepfile | 1 + lib/simple-rss.rb | 46 +++- lib/simple-rss/json_entry_normalizer.rb | 144 ++++++++++++ lib/simple-rss/json_feed.rb | 153 ++++++++++++ lib/simple-rss/normalized_entry.rb | 9 +- test/base/json_feed_test.rb | 295 ++++++++++++++++++++++++ test/base/normalized_fetch_test.rb | 40 ++++ test/data/json_feed_1.json | 75 ++++++ test/data/json_feed_1_1.json | 78 +++++++ 12 files changed, 975 insertions(+), 11 deletions(-) create mode 100644 lib/simple-rss/json_entry_normalizer.rb create mode 100644 lib/simple-rss/json_feed.rb create mode 100644 test/base/json_feed_test.rb create mode 100644 test/data/json_feed_1.json create mode 100644 test/data/json_feed_1_1.json diff --git a/AGENTS.md b/AGENTS.md index f1d729c..623aebc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,7 +23,10 @@ with malformed feeds. `lib/simple-rss/xml_element.rb` holds shared XML tokenizat and scoped element metadata. The optional normalized view lives in `entry_normalizer.rb` and the format-independent `normalized_entry.rb` value object under `lib/simple-rss/`. Preserve the existing raw parser and serialization -contracts when extending normalization. +contracts when extending normalization. `json_feed.rb` validates JSON Feed +structure and preserves the original document; `json_entry_normalizer.rb` maps +JSON fields into the same immutable entry type. Keep format-specific extraction +separate and use the existing JSON standard library dependency. **Tag Syntax** (extend via `SimpleRSS.item_tags <<`): - `tag` - simple element extraction diff --git a/CHANGELOG.md b/CHANGELOG.md index bb2a9ec..800f753 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ repository begins with a 1.1 import, so earlier releases are not reconstructed. ## Unreleased +- Parse JSON Feed 1.0 and 1.1 through `parse` and `fetch`, using the same + normalized entry interface as RSS/Atom. Support titleless and empty feeds, + inherited authors, opaque/numeric IDs, separate content and dates, tags, and + multiple attachments. Preserve original metadata and extensions in `raw_json` + and normalized entry `raw`; report recoverable date/number issues. Keep XML + serialization unchanged and reject JSON-to-XML conversion explicitly. + Fetch detects JSON regardless of Content-Type and retains conditional GET. + ([#60](https://github.com/cardmagic/simple-rss/issues/60)) + - Add `normalized_entries`, an optional immutable RSS/Atom view with consistent article URLs, separate publication/update dates, typed content and summaries, category terms, authors, and associated attachment metadata. Preserve raw diff --git a/README.md b/README.md index 1fc3bf7..1850acd 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,11 @@ [![CI](https://github.com/cardmagic/simple-rss/actions/workflows/ruby.yml/badge.svg)](https://github.com/cardmagic/simple-rss/actions/workflows/ruby.yml) [![License: LGPL](https://img.shields.io/badge/License-LGPL-blue.svg)](https://opensource.org/licenses/LGPL-3.0) -A simple, flexible, extensible, and liberal RSS and Atom reader for Ruby. Designed to be backwards compatible with Ruby's standard RSS parser while handling malformed feeds gracefully. +A simple, flexible, extensible, and liberal RSS, Atom, and JSON Feed reader for Ruby. Designed to be backwards compatible with Ruby's standard RSS parser while handling malformed feeds gracefully. ## Features -- Parses both RSS and Atom feeds +- Parses RSS, Atom, and JSON Feed 1.0/1.1 - Tolerant of malformed XML (regex-based parsing) - Built-in URL fetching with conditional GET support (ETags, Last-Modified) - JSON and XML serialization @@ -27,7 +27,7 @@ The 2.x releases add: - **JSON Serialization** - Export feeds with `to_json`, `to_hash`, and Rails-compatible `as_json`. Time objects serialize to ISO 8601. -- **XML Serialization** - Convert any parsed feed to clean RSS 2.0 or Atom XML with `to_xml(format: :rss2)` or `to_xml(format: :atom)`. +- **XML Serialization** - Convert parsed XML feeds to clean RSS 2.0 or Atom XML with `to_xml(format: :rss2)` or `to_xml(format: :atom)`. - **Array Tags** - Collect all occurrences of a tag (like multiple categories) with the `array_tags:` option. @@ -178,7 +178,7 @@ order, regardless of their dates. ### Normalized Entries -Use `normalized_entries` when an importer or digest should handle RSS and Atom +Use `normalized_entries` when an importer or digest should handle RSS, Atom, and JSON Feed through the same fields: ```ruby @@ -283,18 +283,133 @@ options raise `ArgumentError`. For migration, replace format-specific expressions such as `item[:link_alternate] || item[:link]` with `entry.url`, while retaining `entry.raw` for existing custom fields. The runnable [digest example](examples/digest.rb) -reads either format without testing which one it received: +reads all three formats without testing which one it received: ```bash ruby -Ilib examples/digest.rb test/data/normalized_rss.xml ruby -Ilib examples/digest.rb test/data/normalized_atom.xml +ruby -Ilib examples/digest.rb test/data/json_feed_1_1.json ``` The mapping follows the [Atom specification](https://www.rfc-editor.org/rfc/rfc4287.html), [RSS specification](https://www.rssboard.org/rss-specification), and [XML Base rules](https://www.w3.org/TR/xmlbase/). It is a tolerant extraction view, -not a standards validator. JSON Feed parsing is tracked separately in -[#60](https://github.com/cardmagic/simple-rss/issues/60). +not a standards validator. JSON Feed has the separate rules below. + +### JSON Feed Parsing + +JSON Feed 1.0 and 1.1 use the same `parse`, IO, and `fetch` entry points: + +```ruby +require "simple-rss" +require "json" + +source = JSON.generate( + version: "https://jsonfeed.org/version/1.1", + title: "Example", + authors: [{ name: "Example Editor" }], + items: [{ + id: "post:42", + url: "https://example.com/posts/42", + content_text: "Hello from JSON Feed", + date_published: "2026-09-12T10:00:00Z", + tags: ["ruby", "feeds"] + }] +) +feed = SimpleRSS.parse(source) +entry = feed.normalized_entries.first + +feed.feed_type # => :json_feed +entry.identifier # => "post:42" +entry.content_text # => "Hello from JSON Feed" +entry.authors.first[:name] # => "Example Editor" (inherited) +entry.categories # => ["ruby", "feeds"] +entry.published_at # => a Time +entry.raw["id"] # => "post:42" + +feed = SimpleRSS.fetch("https://example.com/feed.json", timeout: 10) +feed.next_url # Pagination metadata only; never fetched automatically +feed.raw_json # Frozen original JSON document, including extensions +``` + +`fetch` detects the response body independently of Content-Type and sends an +Accept header covering all three formats. Custom headers can override Accept. +ETag, Last-Modified, redirects, and `nil` for HTTP 304 work as for XML feeds. + +| Normalized field | JSON Feed mapping | +| --- | --- | +| `identifier` | `id`, preserved as an opaque string; numeric IDs become strings | +| `url`, `external_url` | Separate permalink and linkblog destination; no ID fallback | +| `published_at`, `updated_at` | `date_published`, `date_modified`, parsed separately as RFC 3339 | +| `content_html`, `content_text` | Corresponding fields, without decoding HTML entities or deriving one from the other | +| `summary`, `summary_type` | Plain text `summary`, with type `:text` | +| `image`, `banner_image` | Corresponding image URLs | +| `categories`, `category_details` | Nonblank `tags`, trimmed and deduplicated for categories; duplicate details retained | +| `authors` | Item authors, otherwise feed authors; includes `name`, `url`, `avatar`, and raw metadata | +| `language` | 1.1 item language, otherwise feed language | +| `attachments` | All attachments with URL, MIME type as `media_type`, title, size, duration, and raw metadata | + +In 1.0, authors come from singular `author`. In 1.1, `authors` takes precedence +over deprecated `author` within the same object; an item's authors take +precedence over the feed's. An explicit empty `authors` array prevents +inheritance. The later `authors` and `language` fields are retained as raw data +but not normalized in a 1.0 document. Matching attachment titles preserve the +publisher's grouping of alternate formats. `external_url`, `image`, +`banner_image`, and `language` are additive normalized fields; XML entries +currently return `nil` for these fields. + +Relative JSON URLs resolve against the supplied/fetched `source_url`, or the +JSON `feed_url` when no source URL is supplied. Per-call `source_url` overrides +still work. Missing bases and invalid URLs remain inspectable through `issues` +and raw data, using the same issue codes as XML. `content_base_url` is the item +URL when available, otherwise the source URL or feed URL. Content links are not +rewritten; no articles, attachments, hubs, or pagination URLs are fetched. + +Parsing requires a supported version, string feed title, an items array, and +objects with nonblank string/numeric IDs and at least one string content field. +Titleless items and empty feeds are supported. Malformed JSON, missing required +fields, malformed author/tag/attachment structures, and wrong known field types +raise `SimpleRSSError` with a field path. Invalid required item data rejects the +whole feed; items are never assigned invented IDs or returned partially parsed. +Invalid optional dates and attachment numbers are preserved in raw data and +reported in `issues`, with `nil` normalized values. Dates are never substituted +with the current time. `effective_at`, `latest`, and `items_since` safely use a +valid modification date when publication is invalid or absent. + +UTF-8 strings and readable IO accept ordinary leading JSON whitespace and one +UTF-8 BOM at the very start, before whitespace. Embedded or repeated BOMs are +rejected. `source` preserves the original input. This is a parser, not a complete +standards validator: it does not validate URL reachability, language tags, ID +uniqueness across updates, or publisher extension schemas. +`SimpleRSS.valid?(source)` reports parseability. A parsed JSON feed's instance +`valid?` is true, including an empty feed. XML retains its historical distinction: +the class method accepts parseable empty feeds, but instance `valid?` requires +items and a title or link. + +For JSON feeds, `raw_json` is an immutable snapshot of the entire decoded +document with string keys. Each normalized entry's `raw` is its original JSON +item, also with string keys; `raw_xml` is `nil`. Original numeric IDs, date +strings, nested data, and unknown fields remain intact. Normalized entries are +immutable snapshots based on the original JSON, so edits to `items` do not +rewrite their normalized fields. Reordering or deduplication retains the source +association; inserting an unrelated item raises an error during normalization. + +`items` remains an array of hashes with symbol keys and dot access. It retains +JSON fields and adds compatibility aliases: string `id`/`guid`, `link`, +`description`, `content`, category arrays, publication/update dates, and the first +attachment's enclosure metadata. Nested original JSON values are frozen. +XML-only tag configuration and `array_tags` do not affect JSON; passing XML +`mappings` to JSON normalization raises `ArgumentError`. + +`as_json`, `to_hash`, and `to_json` export that Ruby object view: original feed +metadata plus compatibility fields and the current items, with times converted +to ISO 8601. These operations preserve extensions but are **not JSON Feed +exporters**. Use `raw_json` to inspect the original document. XML serialization +behavior is unchanged; calling `to_xml` on a JSON feed raises `SimpleRSSError` +until a separate conversion contract exists. + +These mappings follow the [JSON Feed 1.0 specification](https://www.jsonfeed.org/version/1/) +and [JSON Feed 1.1 specification](https://www.jsonfeed.org/version/1.1/). ### JSON Serialization @@ -315,7 +430,7 @@ feed.as_json ### XML Serialization -Convert parsed feeds to standard RSS 2.0 or Atom format: +Convert parsed XML feeds to standard RSS 2.0 or Atom format: ```ruby feed = SimpleRSS.parse(xml) diff --git a/Steepfile b/Steepfile index 05878a0..aea3ace 100644 --- a/Steepfile +++ b/Steepfile @@ -5,6 +5,7 @@ target :lib do check "lib" library "time" + library "date" library "cgi" library "json" library "uri" diff --git a/lib/simple-rss.rb b/lib/simple-rss.rb index 21719f9..5d92de9 100644 --- a/lib/simple-rss.rb +++ b/lib/simple-rss.rb @@ -15,6 +15,7 @@ class SimpleRSS # rubocop:disable Metrics/ClassLength # @rbs @items: Array[Hash[Symbol, untyped]] # @rbs @source: String # @rbs @options: Hash[Symbol, untyped] + # @rbs @json_feed: JsonFeed? # @rbs @etag: String? # @rbs @last_modified: String? # @rbs @entry_contexts: Hash[Hash[Symbol, untyped], Hash[Symbol, untyped]] @@ -24,6 +25,9 @@ class SimpleRSS # rubocop:disable Metrics/ClassLength attr_reader :etag #: String? attr_reader :last_modified #: String? attr_reader :source_url #: String? + attr_reader :raw_json #: Hash[String, untyped]? + attr_reader :home_page_url, :feed_url, :favicon, :next_url, :user_comment #: String? + attr_reader :authors, :hubs, :expired #: untyped alias entries items #: Array[Hash[Symbol, untyped]] @@feed_tags = %i[ @@ -66,6 +70,8 @@ def initialize(source, options = {}) @options = {} #: Hash[Symbol, untyped] @options.update(options) @source_url = options[:source_url] + @json_feed = nil + @raw_json = nil @entry_contexts = {} #: Hash[Hash[Symbol, untyped], Hash[Symbol, untyped]] @entry_contexts.compare_by_identity @@ -80,6 +86,13 @@ def channel # @rbs (?source_url: String?, ?mappings: Hash[Symbol, untyped]) -> Array[NormalizedEntry] def normalized_entries(source_url: nil, mappings: {}) + json_feed = @json_feed + if json_feed + raise ArgumentError, "XML mappings are not supported for JSON Feed" unless mappings.empty? + + return items.map { |item| json_feed.normalized_entry(item, source_url: source_url || @source_url) } + end + EntryNormalizer.validate_mappings(mappings) feed_authors = normalized_feed_authors items.map do |item| @@ -117,6 +130,8 @@ def latest(count = 10) # @rbs () -> Symbol def feed_type + return :json_feed if @json_feed + atom_namespaced_feed = source.match?(/<(atom:)?feed\b[^>]*xmlns(:\w+)?=['"][^'"]*atom/i) return :atom if atom_namespaced_feed return :rss2 if source.match?(/]*version=['"]2/i) @@ -128,6 +143,8 @@ def feed_type # @rbs () -> bool def valid? + return true if @json_feed + return false if items.empty? title_value = instance_variable_get(:@title) @@ -212,7 +229,8 @@ def images # @rbs (?Hash[Symbol, untyped]) -> Hash[Symbol, untyped] def as_json(_options = {}) - hash = {} #: Hash[Symbol, untyped] + raw_json = @raw_json + hash = raw_json ? raw_json.transform_keys(&:to_sym) : {} #: Hash[Symbol, untyped] @@feed_tags.each do |tag| tag_cleaned = clean_tag(tag) @@ -237,6 +255,8 @@ def to_json(*) # @rbs (?format: Symbol) -> String def to_xml(format: :rss2) + raise SimpleRSSError, "JSON Feed to XML conversion is not supported" if @json_feed + case format when :rss2 then to_rss2_xml when :atom then to_atom_xml @@ -339,6 +359,7 @@ def build_http(uri, options) # @rbs (untyped, Hash[Symbol, untyped]) -> untyped def build_request(uri, options) request = Net::HTTP::Get.new(uri) + request["Accept"] = "application/feed+json, application/rss+xml, application/atom+xml, application/json, application/xml, text/xml, */*" request["User-Agent"] = "SimpleRSS/#{VERSION}" # Conditional GET headers @@ -386,6 +407,27 @@ def normalized_feed_authors # @rbs () -> void def parse + prefix = @source.b.sub(/\A\xEF\xBB\xBF/n, "").lstrip + return parse_xml unless prefix.match?(/\A(?:[\{\["0-9-]|true\b|false\b|null\b)/n) + + json_feed = JsonFeed.new(@source) + @json_feed = json_feed + @raw_json = json_feed.document + JsonFeed::FEED_FIELDS.each do |field| + instance_variable_set("@#{field}", json_feed.document[field]) + self.class.attr_reader(field) + end + @link = json_feed.document["home_page_url"] + self.class.attr_reader(:link) + @items = json_feed.items + @items.each do |item| + item.define_singleton_method(:method_missing) { |name, *_args| self[name] } + add_item_media_helpers(item) + end + end + + # @rbs () -> void + def parse_xml raise SimpleRSSError, "Poorly formatted feed" unless @source =~ %r{<(channel|feed).*?>.*?}mi # Feed's title and link @@ -916,6 +958,8 @@ def unescape(content) require_relative "simple-rss/xml_element" require_relative "simple-rss/normalized_entry" +require_relative "simple-rss/json_feed" +require_relative "simple-rss/json_entry_normalizer" require_relative "simple-rss/entry_normalizer" class SimpleRSSError < StandardError # rubocop:disable Style/OneClassPerFile diff --git a/lib/simple-rss/json_entry_normalizer.rb b/lib/simple-rss/json_entry_normalizer.rb new file mode 100644 index 0000000..2992a46 --- /dev/null +++ b/lib/simple-rss/json_entry_normalizer.rb @@ -0,0 +1,144 @@ +# rbs_inline: enabled + +require "uri" +require "date" + +class SimpleRSS::JsonEntryNormalizer + FIELDS = { + title: "title", content_html: "content_html", content_text: "content_text", summary: "summary" + }.freeze + URL_FIELDS = %i[url external_url image banner_image].freeze + + # @rbs @item: Hash[String, untyped] + # @rbs @feed: Hash[String, untyped] + # @rbs @source_url: String? + # @rbs @values: Hash[Symbol, untyped] + # @rbs @issues: Array[Hash[Symbol, untyped]] + # @rbs @sources: Hash[Symbol, untyped] + + # @rbs (Hash[String, untyped], Hash[String, untyped], ?source_url: String?) -> void + def initialize(item, feed, source_url: nil) + @item = item + @feed = feed + @source_url = source_url || feed["feed_url"] + @issues = [] #: Array[Hash[Symbol, untyped]] + @sources = {} #: Hash[Symbol, untyped] + @values = { raw: item, issues: @issues, field_sources: @sources } #: Hash[Symbol, untyped] + end + + # @rbs () -> SimpleRSS::NormalizedEntry + def entry + assign(:identifier, @item["id"].to_s, "id") + FIELDS.each { |field, source| assign(field, @item[source], source) } + URL_FIELDS.each { |field| assign(field, resolve_url(@item[field.to_s], field, field.to_s), field.to_s) } + assign(:published_at, read_date("date_published", :published_at), "date_published") + assign(:updated_at, read_date("date_modified", :updated_at), "date_modified") + @values[:summary_type] = :text if @item["summary"] + @values[:content_base_url] = @values[:url] || @source_url + read_language + read_categories + read_authors + @values[:attachments] = (@item["attachments"] || []).each_with_index.map { |attachment, index| read_attachment(attachment, index) } + @values[:links] = %i[url external_url].filter_map do |field| + next unless @values[field] + + { url: @values[field], rel: field == :url ? "alternate" : "related", media_type: nil, source: field.to_s, raw: @item[field.to_s] } + end + SimpleRSS::NormalizedEntry.new(@values) + end + + private + + # @rbs (Symbol, untyped, String) -> void + def assign(field, value, source) + return if value.nil? + + @values[field] = value + @sources[field] = source + end + + # @rbs (Symbol, Symbol, untyped, String) -> void + def issue(field, code, value, source) + @issues << { field: field, code: code, value: value, source: source } + end + + # @rbs (String, Symbol) -> Time? + def read_date(source, field) + value = @item[source] + return if value.nil? + return DateTime.rfc3339(value).to_time if value.is_a?(String) && value.match?(/\A\d{4}-\d\d-\d\d[tT]\d\d:\d\d:\d\d(?:\.\d+)?(?:[zZ]|[+-]\d\d:\d\d)\z/) + + issue(field, :invalid_date, value, source) + nil + rescue ArgumentError, RangeError + issue(field, :invalid_date, value, source) + nil + end + + # @rbs (String?, Symbol, String) -> String? + def resolve_url(value, field, source) + return if value.nil? || value.strip.empty? + + value = value.strip + return value if URI.parse(value).absolute? + + source_url = @source_url + return URI.join(source_url, value).to_s if source_url && URI.parse(source_url).absolute? + + issue(field, :relative_url_without_base, value, source) + value + rescue URI::Error + issue(field, :invalid_url, value, source) + value + end + + # @rbs () -> void + def read_language + return unless @feed["version"] == SimpleRSS::JsonFeed::VERSIONS.last + + object = @item.key?("language") ? @item : @feed + source = object.equal?(@item) ? "language" : "feed.language" + assign(:language, object["language"], source) + end + + # @rbs () -> void + def read_categories + tags = @item["tags"] || [] + categories = tags.map(&:strip).reject(&:empty?).uniq + assign(:categories, categories, "tags") if @item.key?("tags") + @values[:category_details] = tags.reject { |tag| tag.strip.empty? }.map { |tag| { term: tag.strip, label: nil, scheme: nil, source: "tags", raw: tag } } + end + + # @rbs () -> void + def read_authors + object = @item + plural = @feed["version"] == SimpleRSS::JsonFeed::VERSIONS.last + object = @feed unless (plural && object.key?("authors")) || object.key?("author") + source = plural && object.key?("authors") ? "authors" : "author" + authors = source == "authors" ? object[source] : [object[source]].compact + source = "feed.#{source}" if object.equal?(@feed) + records = (authors || []).map do |author| + { name: author["name"], email: nil, url: resolve_url(author["url"], :authors, source), + avatar: resolve_url(author["avatar"], :authors, source), raw: author } + end + assign(:authors, records, source) unless records.empty? + end + + # @rbs (Hash[String, untyped], Integer) -> Hash[Symbol, untyped] + def read_attachment(attachment, index) + source = "attachments[#{index}]" + { url: resolve_url(attachment["url"], :attachments, "#{source}.url"), media_type: attachment["mime_type"], + title: attachment["title"], size_in_bytes: read_number(attachment, "size_in_bytes", source), + duration_in_seconds: read_number(attachment, "duration_in_seconds", source), source: source, raw: attachment } + end + + # @rbs (Hash[String, untyped], String, String) -> Numeric? + def read_number(attachment, field, source) + value = attachment[field] + return if value.nil? + return value if value.is_a?(Numeric) && value >= 0 && value.finite? + + issue(:attachments, :invalid_number, value, "#{source}.#{field}") + nil + end +end diff --git a/lib/simple-rss/json_feed.rb b/lib/simple-rss/json_feed.rb new file mode 100644 index 0000000..3156807 --- /dev/null +++ b/lib/simple-rss/json_feed.rb @@ -0,0 +1,153 @@ +# rbs_inline: enabled + +require "json" + +class SimpleRSS::JsonFeed + VERSIONS = %w[https://jsonfeed.org/version/1 https://jsonfeed.org/version/1.1].freeze + FEED_FIELDS = %w[title description home_page_url feed_url icon favicon language author authors expired next_url hubs user_comment].freeze + FEED_STRINGS = %w[description home_page_url feed_url icon favicon next_url user_comment].freeze + ITEM_STRINGS = %w[url external_url title summary content_html content_text image banner_image].freeze + + attr_reader :document #: Hash[String, untyped] + attr_reader :items #: Array[Hash[Symbol, untyped]] + + # @rbs @originals: Hash[Hash[Symbol, untyped], Hash[String, untyped]] + + # @rbs (String) -> void + def initialize(source) + @document = JSON.parse(source.b.sub(/\A\xEF\xBB\xBF/n, "").force_encoding(Encoding::UTF_8)) + validate + freeze_data(@document) + @originals = {} #: Hash[Hash[Symbol, untyped], Hash[String, untyped]] + @originals.compare_by_identity + @items = @document.fetch("items").map do |original| + item = legacy_item(original) + @originals[item] = original + item + end + rescue JSON::ParserError, EncodingError => e + raise SimpleRSSError, "Malformed JSON Feed: #{e.message}" + end + + # @rbs (Hash[Symbol, untyped], ?source_url: String?) -> SimpleRSS::NormalizedEntry + def normalized_entry(item, source_url: nil) + original = @originals[item] || raise(SimpleRSSError, "Cannot normalize an item without its original JSON source") + SimpleRSS::JsonEntryNormalizer.new(original, document, source_url: source_url).entry + end + + private + + # @rbs () -> void + def validate + check_type(document, Hash, "feed") + required_string(document, "version", "feed") + raise SimpleRSSError, "Unsupported JSON Feed version: #{document["version"].inspect}" unless VERSIONS.include?(document["version"]) + + required_string(document, "title", "feed") + check_type(document["items"], Array, "items") + optional_strings(document, FEED_STRINGS, "feed") + optional_strings(document, ["language"], "feed") if version_1_1? + validate_authors(document, "feed") + optional_array(document, "hubs", "feed").each_with_index do |hub, index| + check_type(hub, Hash, "hubs[#{index}]") + %w[type url].each { |field| required_string(hub, field, "hubs[#{index}]") } + end + document["items"].each_with_index { |item, index| validate_item(item, "items[#{index}]") } + end + + # @rbs (untyped, String) -> void + def validate_item(item, path) + check_type(item, Hash, path) + identifier = item["id"] + unless (identifier.is_a?(String) || identifier.is_a?(Numeric)) && !identifier.to_s.strip.empty? + raise SimpleRSSError, "JSON Feed #{path}.id must be a nonblank string or number" + end + unless %w[content_html content_text].any? { |field| item[field].is_a?(String) } + raise SimpleRSSError, "JSON Feed #{path} requires content_html or content_text" + end + + optional_strings(item, ITEM_STRINGS, path) + optional_strings(item, ["language"], path) if version_1_1? + validate_authors(item, path) + optional_array(item, "tags", path).each { |tag| check_type(tag, String, "#{path}.tags[]") } + optional_array(item, "attachments", path).each_with_index do |attachment, index| + attachment_path = "#{path}.attachments[#{index}]" + check_type(attachment, Hash, attachment_path) + %w[url mime_type].each { |field| required_string(attachment, field, attachment_path) } + optional_strings(attachment, ["title"], attachment_path) + end + end + + # @rbs (Hash[String, untyped], String) -> void + def validate_authors(object, path) + if version_1_1? && object.key?("authors") + optional_array(object, "authors", path).each_with_index { |author, index| validate_author(author, "#{path}.authors[#{index}]") } + return + end + + validate_author(object["author"], "#{path}.author") if object.key?("author") + end + + # @rbs (untyped, String) -> void + def validate_author(author, path) + check_type(author, Hash, path) + fields = %w[name url avatar] + optional_strings(author, fields, path) + return if fields.any? { |field| author[field].is_a?(String) } + + raise SimpleRSSError, "JSON Feed #{path} requires name, url, or avatar" + end + + # @rbs (Hash[String, untyped], Array[String], String) -> void + def optional_strings(object, fields, path) + fields.each { |field| check_type(object[field], String, "#{path}.#{field}") if object.key?(field) } + end + + # @rbs (Hash[String, untyped], String, String) -> Array[untyped] + def optional_array(object, field, path) + return [] unless object.key?(field) + + check_type(object[field], Array, "#{path}.#{field}") + object[field] + end + + # @rbs (Hash[String, untyped], String, String) -> void + def required_string(object, field, path) + check_type(object[field], String, "#{path}.#{field}") + end + + # @rbs (untyped, untyped, String) -> void + def check_type(value, type, path) + return if value.is_a?(type) + + raise SimpleRSSError, "JSON Feed #{path} must be a #{type}" + end + + # @rbs () -> bool + def version_1_1? + document["version"] == VERSIONS.last + end + + # @rbs (Hash[String, untyped]) -> Hash[Symbol, untyped] + def legacy_item(original) + entry = SimpleRSS::JsonEntryNormalizer.new(original, document).entry + attachment = entry.attachments.first || {} + original.transform_keys(&:to_sym).merge( + id: entry.identifier, guid: entry.identifier, link: entry.url, + description: entry.summary || entry.content_text || entry.content_html, + content: entry.content_html || entry.content_text, category: entry.categories.dup, + pubDate: entry.published_at || original["date_published"], updated: entry.updated_at || original["date_modified"], + enclosure_url: attachment[:url], enclosure_type: attachment[:media_type], enclosure_length: attachment[:size_in_bytes], + media_thumbnail_url: entry.image, media_content_url: entry.banner_image + ) + end + + # @rbs (untyped) -> void + def freeze_data(value) + case value + when Hash then value.each_value { |child| freeze_data(child) } + when Array then value.each { |child| freeze_data(child) } + end + value.freeze + end +end diff --git a/lib/simple-rss/normalized_entry.rb b/lib/simple-rss/normalized_entry.rb index 8717332..f6639b7 100644 --- a/lib/simple-rss/normalized_entry.rb +++ b/lib/simple-rss/normalized_entry.rb @@ -1,13 +1,15 @@ # rbs_inline: enabled class SimpleRSS::NormalizedEntry + attr_reader :external_url, :image, :banner_image, :language #: String? attr_reader :identifier, :url, :title #: String? attr_reader :published_at, :updated_at #: Time? attr_reader :content_html, :content_text, :content_url, :content_base_url, :summary #: String? attr_reader :summary_type #: Symbol? attr_reader :categories #: Array[String] attr_reader :category_details, :attachments, :links, :authors, :issues #: Array[Hash[Symbol, untyped]] - attr_reader :raw, :field_sources #: Hash[Symbol, untyped] + attr_reader :raw #: Hash[Symbol | String, untyped] + attr_reader :field_sources #: Hash[Symbol, untyped] attr_reader :raw_xml #: String? # @rbs (Hash[Symbol, untyped]) -> void @@ -19,6 +21,10 @@ def initialize(attributes) empty_categories.freeze empty_records.freeze empty_data.freeze + @external_url = values[:external_url] + @image = values[:image] + @banner_image = values[:banner_image] + @language = values[:language] @identifier = values[:identifier] @url = values[:url] @title = values[:title] @@ -50,6 +56,7 @@ def effective_at # @rbs () -> Hash[Symbol, untyped] def to_h { + external_url: external_url, image: image, banner_image: banner_image, language: language, identifier: identifier, url: url, title: title, published_at: published_at, updated_at: updated_at, content_html: content_html, content_text: content_text, content_url: content_url, content_base_url: content_base_url, summary: summary, summary_type: summary_type, categories: categories, category_details: category_details, diff --git a/test/base/json_feed_test.rb b/test/base/json_feed_test.rb new file mode 100644 index 0000000..47a01b7 --- /dev/null +++ b/test/base/json_feed_test.rb @@ -0,0 +1,295 @@ +require "test_helper" +require "json" +require "stringio" + +class JsonFeedTest < Test::Unit::TestCase + def test_parses_json_feed_through_the_public_api + source = JSON.generate(version: "https://jsonfeed.org/version/1.1", title: "Example", + items: [{ id: "opaque:%2F:001", url: "https://example.com/1", content_text: "Hello" }]) + feed = SimpleRSS.parse(source) + entry = feed.normalized_entries.first + + assert_equal :json_feed, feed.feed_type + assert_equal "Example", feed.title + assert_equal "opaque:%2F:001", entry.identifier + assert_equal "https://example.com/1", entry.url + assert_equal "Hello", entry.content_text + end + + def test_accepts_an_empty_json_feed + source = JSON.generate(version: "https://jsonfeed.org/version/1", title: "Example", items: []) + feed = SimpleRSS.parse(StringIO.new(source)) + + assert_empty feed.items + assert_empty feed.normalized_entries + assert SimpleRSS.valid?(source) + assert feed.valid? + end + + def test_spec_fixtures_preserve_feed_metadata_and_extensions + %w[1 1_1].each do |version| + feed = fixture(version) + assert_equal "A feed of examples", feed.description + assert_equal "https://example.com/", feed.link + assert_equal "https://example.com/", feed.home_page_url + assert_equal "https://example.com/feed.json", feed.feed_url + assert_equal "https://example.com/older.json", feed.next_url + assert_equal "https://example.com/icon.png", feed.icon + assert_equal "https://example.com/favicon.png", feed.favicon + assert_equal true, feed.expired + assert_equal "WebSub", feed.hubs.first["type"] + assert_equal "Subscribe with a feed reader.", feed.user_comment + assert_equal [1, 2], feed.raw_json["_publisher"]["sequence"] + assert_equal "original", feed.normalized_entries.first.raw["_extension"]["value"] + end + end + + def test_rss_atom_and_json_share_normalized_core_fields + entries = %w[rss atom].map do |format| + SimpleRSS.parse(File.read(File.join(__dir__, "../data/normalized_#{format}.xml"))).normalized_entries.first + end + entries << fixture("1_1").normalized_entries.first + entries.each do |entry| + assert_equal "opaque:%2F:001", entry.identifier + assert_equal "Ruby & feeds", entry.title + assert_equal "https://example.com/blog/article?one=1&two=2", entry.url + assert_equal Time.utc(2026, 9, 12, 10), entry.published_at + assert_equal Time.utc(2026, 9, 13, 11), entry.updated_at + assert_equal "

Full & complete.

", entry.content_html + assert_equal %w[ruby feeds], entry.categories + assert_equal(["https://example.com/audio/one.mp3", "https://example.com/images/two.png"], entry.attachments.map { |attachment| attachment[:url] }) + assert_equal([1234, 4321], entry.attachments.map { |attachment| attachment[:size_in_bytes] }) + end + end + + def test_normalizes_json_specific_fields_without_losing_associations + entry = fixture("1_1").normalized_entries.first + assert_equal "Full & complete.", entry.content_text + assert_equal "A summary.", entry.summary + assert_equal :text, entry.summary_type + assert_equal "https://example.org/original", entry.external_url + assert_equal "https://example.com/image.png", entry.image + assert_equal "https://example.com/banner.png", entry.banner_image + assert_equal "en", entry.language + assert_equal(%w[alternate related], entry.links.map { |link| link[:rel] }) + assert_equal ["ruby", "feeds", "ruby", " "], entry.raw["tags"] + assert_equal 3, entry.category_details.size + assert_equal(%w[Episode Cover], entry.attachments.map { |attachment| attachment[:title] }) + assert_equal(%w[audio/mpeg image/png], entry.attachments.map { |attachment| attachment[:media_type] }) + assert_equal 62.5, entry.attachments.first[:duration_in_seconds] + assert_equal false, entry.attachments.first[:raw]["_codec"]["lossless"] + assert_equal "date_published", entry.field_sources[:published_at] + assert_empty entry.issues + assert_nil entry.raw_xml + end + + def test_authors_inherit_and_item_authors_take_precedence + %w[1 1_1].each do |version| + inherited = fixture(version).normalized_entries.first + assert_equal(["Example Editor"], inherited.authors.map { |author| author[:name] }) + assert_equal "https://example.com/avatar.png", inherited.authors.first[:avatar] + assert_equal "editor", inherited.authors.first[:raw]["_profile"]["role"] + end + feed = parse_items([{ id: "1", content_text: "Hello", authors: [{ name: "First" }, { url: "https://example.com/second" }], + author: { name: "Legacy" }, language: "fr" }], authors: [{ name: "Feed" }], language: "en") + entry = feed.normalized_entries.first + assert_equal(["First", nil], entry.authors.map { |author| author[:name] }) + assert_equal "https://example.com/second", entry.authors.last[:url] + assert_equal "authors", entry.field_sources[:authors] + assert_equal "fr", entry.language + assert_equal "Legacy", entry.raw["author"]["name"] + end + + def test_singular_author_and_empty_authors_have_deliberate_precedence + entry = parse_items([{ id: "1", content_text: "Hello", author: { name: "Item" } }], authors: [{ name: "Feed" }]).normalized_entries.first + assert_equal(["Item"], entry.authors.map { |author| author[:name] }) + entry = parse_items([{ id: "1", content_text: "Hello", authors: [], author: { name: "Legacy" } }], authors: [{ name: "Feed" }]).normalized_entries.first + assert_empty entry.authors + entry = parse_items([{ id: "1", content_text: "Hello" }], author: { name: "Legacy" }, authors: [{ name: "Current" }]).normalized_entries.first + assert_equal(["Current"], entry.authors.map { |author| author[:name] }) + end + + def test_version_one_ignores_later_version_fields_but_retains_them + feed = parse_items([{ id: "1", content_text: "Hello", authors: "unknown", language: 12 }], + version: "https://jsonfeed.org/version/1", author: { name: "Legacy" }, authors: "unknown", language: 12) + entry = feed.normalized_entries.first + assert_equal(["Legacy"], entry.authors.map { |author| author[:name] }) + assert_nil entry.language + assert_equal "unknown", entry.raw["authors"] + assert_equal 12, feed.raw_json["language"] + end + + def test_titleless_content_is_not_decoded_or_synthesized + entry = parse_items([{ id: " x:%2F&=1 ", content_text: "& 😀" }]).normalized_entries.first + assert_nil entry.title + assert_nil entry.url + assert_nil entry.content_html + assert_nil entry.summary + assert_equal " x:%2F&=1 ", entry.identifier + assert_equal "& 😀", entry.content_text + assert_equal "", parse_items([{ id: "empty", content_text: "" }]).normalized_entries.first.content_text + end + + def test_numeric_identifiers_are_coerced_only_in_the_item_view + feed = parse_items([{ id: 42, content_text: "Hello" }, { id: 1.5, content_html: "

Hi

" }]) + assert_equal ["42", "1.5"], feed.map(&:id) + assert_equal ["42", "1.5"], feed.normalized_entries.map(&:identifier) + assert_equal 42, feed.raw_json["items"].first["id"] + assert_equal 42, feed.normalized_entries.first.raw["id"] + end + + def test_invalid_dates_are_inspectable_and_do_not_break_effective_date_helpers + feed = parse_items([ + { id: "invalid", content_text: "One", date_published: "not a date", date_modified: "2026-09-13T11:00:00Z" }, + { id: "old", content_text: "Two", date_published: "1960-01-01T00:00:00Z" }, + { id: "undated", content_text: "Three", date_published: { bad: true }, date_modified: "2026-02-30T00:00:00Z" } + ]) + first, old, undated = feed.normalized_entries + assert_nil first.published_at + assert_equal Time.utc(2026, 9, 13, 11), first.effective_at + assert_equal "not a date", first.raw["date_published"] + assert_equal :invalid_date, first.issues.first[:code] + assert_equal :published_at, first.issues.first[:field] + assert_equal "date_published", first.issues.first[:source] + assert_nil old.updated_at + assert_nil undated.effective_at + assert_equal 2, undated.issues.size + assert_equal %w[invalid old undated], feed.latest.map(&:id) + assert_equal ["invalid"], feed.items_since(Time.utc(2020)).map(&:id) + %w[2026-09-12 tomorrow 2026-09-12T10:00:00 2026-09-12T25:00:00Z].each do |value| + entry = parse_items([{ id: "1", content_text: "Hi", date_published: value }]).normalized_entries.first + assert_nil entry.published_at, value + assert_equal :invalid_date, entry.issues.first[:code] + end + end + + def test_attachment_numeric_recovery_preserves_original_values + feed = parse_items([{ id: "1", content_text: "Hello", attachments: [ + { url: "https://example.com/one", mime_type: "audio/mpeg", title: "Episode", size_in_bytes: "big", duration_in_seconds: -1 }, + { url: "https://example.com/two", mime_type: "audio/ogg", title: "Episode", size_in_bytes: 0, duration_in_seconds: 0 } + ] }]) + entry = feed.normalized_entries.first + assert_nil entry.attachments.first[:size_in_bytes] + assert_nil entry.attachments.first[:duration_in_seconds] + assert_equal [0, 0], entry.attachments.last.values_at(:size_in_bytes, :duration_in_seconds) + assert_equal "big", entry.attachments.first[:raw]["size_in_bytes"] + assert_equal(%w[Episode Episode], entry.attachments.map { |attachment| attachment[:title] }) + assert_equal(["attachments[0].size_in_bytes", "attachments[0].duration_in_seconds"], entry.issues.map { |issue| issue[:source] }) + end + + def test_raw_and_serialized_representations_are_explicit_and_immutable + feed = fixture("1_1") + before = feed.to_json + entry = feed.normalized_entries.first + assert_equal before, feed.to_json + assert_equal JSON.parse(before), JSON.parse(JSON.generate(feed.as_json)) + assert_equal feed.as_json, feed.to_hash + assert_equal "2026-09-12T10:00:00+00:00", feed.to_hash[:items].first[:pubDate] + assert_equal "2026-09-12T10:00:00Z", feed.to_hash[:items].first[:date_published] + assert_equal [1, 2], feed.to_hash[:_publisher]["sequence"] + assert_equal "original", JSON.parse(before)["items"].first["_extension"]["value"] + assert_raise(FrozenError) { feed.raw_json["_publisher"]["sequence"] << 3 } + assert_raise(FrozenError) { entry.raw["_extension"]["value"].replace("changed") } + assert_raise(FrozenError) { entry.authors.first[:raw]["name"].replace("changed") } + assert_raise(SimpleRSSError) { feed.to_xml } + assert_raise(SimpleRSSError) { feed.to_xml(format: :atom) } + assert_raise(ArgumentError) { feed.normalized_entries(mappings: { content_text: "custom" }) } + end + + def test_legacy_helpers_and_dedupe_retain_the_original_entry_context + feed = fixture("1_1") + assert_equal feed.first, feed.search("Full").first + assert_equal feed.first, feed.items_by_category("RUBY").first + assert feed.first.has_media? + assert_equal "https://example.com/audio/one.mp3", feed.enclosures.first[:url] + assert_include feed.images, "https://example.com/image.png" + feed.items << feed.first + assert_equal 1, feed.dedupe.items.size + assert_equal "opaque:%2F:001", feed.normalized_entries.first.identifier + feed.items << { id: "unrelated" } + assert_raise(SimpleRSSError) { feed.normalized_entries } + end + + def test_utf8_bom_whitespace_and_readable_binary_io + source = JSON.generate(version: "https://jsonfeed.org/version/1.1", title: "😀", items: []) + [source, " \r\n\t#{source}", "\uFEFF \n#{source}"].each do |body| + feed = SimpleRSS.parse(StringIO.new(body.b)) + assert_equal "😀", feed.title + assert_equal body.b, feed.source + end + assert_raise(SimpleRSSError) { SimpleRSS.parse(" \uFEFF#{source}") } + assert_raise(SimpleRSSError) { SimpleRSS.parse("\uFEFF\uFEFF#{source}") } + end + + def test_invalid_json_and_structures_raise_library_errors + ['{"version":', "[]", "null", "42", "true", '"hello"', + '{"description":"not XML"}'].each do |source| + assert_raise(SimpleRSSError, source) { SimpleRSS.parse(source) } + assert_equal false, SimpleRSS.valid?(source) + end + [{ version: "https://jsonfeed.org/version/2" }, { version: nil }, { title: nil }, { items: nil }, + { items: {} }, { home_page_url: [] }, { authors: {} }, { authors: [{}] }, { author: false }, + { hubs: [nil] }, { hubs: [{ type: "WebSub" }] }].each do |override| + assert_raise(SimpleRSSError, override.inspect) { parse_items([], **override) } + end + [nil, [], {}, { id: "1" }, { id: nil, content_text: "Hi" }, { id: [], content_text: "Hi" }, + { id: true, content_text: "Hi" }, { id: " ", content_text: "Hi" }, + { id: "1", content_text: 1 }, { id: "1", content_text: "Hi", tags: {} }, + { id: "1", content_text: "Hi", tags: [false] }, { id: "1", content_text: "Hi", authors: [nil] }, + { id: "1", content_text: "Hi", attachments: [nil] }, + { id: "1", content_text: "Hi", attachments: [{ url: "https://example.com/file" }] }].each do |item| + error = assert_raise(SimpleRSSError, item.inspect) { parse_items([item]) } + assert_include error.message, "items[0]" + end + end + + def test_format_dispatch_preserves_tolerant_xml_prefixes + source = 'Unexpected prefix ExamplePost' + assert_equal "Post", SimpleRSS.parse(source).first.title + source = JSON.generate("Not an XML feedWrong") + assert_raise(SimpleRSSError) { SimpleRSS.parse(source) } + end + + def test_blank_urls_never_become_feed_urls + entry = parse_items([{ id: "1", url: " ", image: "", content_text: "Hi" }], feed_url: "https://example.com/feed.json").normalized_entries.first + assert_nil entry.url + assert_nil entry.image + end + + def test_xml_behavior_and_configuration_remain_unchanged + source = 'Empty' + tags = [SimpleRSS.feed_tags.dup, SimpleRSS.item_tags.dup] + feed = SimpleRSS.parse(source) + before = [feed.to_json, feed.to_hash, feed.to_xml] + fixture("1_1").normalized_entries + assert_equal before, [feed.to_json, feed.to_hash, feed.to_xml] + assert_equal tags, [SimpleRSS.feed_tags, SimpleRSS.item_tags] + assert_nil feed.raw_json + assert_equal false, feed.valid? + assert SimpleRSS.valid?(source) + end + + def test_relative_and_invalid_urls_are_preserved_with_normalization_issues + feed = parse_items([{ id: "relative", url: "article", external_url: "bad space", content_text: "Hi", image: "image.png" }]) + entry = feed.normalized_entries.first + assert_equal "article", entry.url + assert_equal(%i[relative_url_without_base invalid_url relative_url_without_base], entry.issues.map { |issue| issue[:code] }) + entry = feed.normalized_entries(source_url: "https://example.com/blog/feed.json").first + assert_equal "https://example.com/blog/article", entry.url + assert_equal "https://example.com/blog/image.png", entry.image + assert_equal "bad space", entry.external_url + assert_equal "article", entry.raw["url"] + assert_equal "https://example.com/blog/article", entry.content_base_url + end + + private + + def fixture(version) + SimpleRSS.parse(File.read(File.join(__dir__, "../data/json_feed_#{version}.json"))) + end + + def parse_items(items, **fields) + source = { version: "https://jsonfeed.org/version/1.1", title: "Example", items: items }.merge(fields) + SimpleRSS.parse(JSON.generate(source)) + end +end diff --git a/test/base/normalized_fetch_test.rb b/test/base/normalized_fetch_test.rb index 81389c8..be85d17 100644 --- a/test/base/normalized_fetch_test.rb +++ b/test/base/normalized_fetch_test.rb @@ -1,6 +1,7 @@ require "test_helper" require "socket" require "stringio" +require "json" class NormalizedFetchTest < Test::Unit::TestCase def test_fetch_uses_the_final_response_url_after_relative_redirects @@ -61,6 +62,45 @@ def test_fetched_source_url_wins_over_shared_parse_options_without_mutating_them end end + def test_json_fetch_detects_the_body_across_content_types_and_preserves_conditional_get + %w[application/feed+json application/json text/plain application/rss+xml].each do |content_type| + body = JSON.generate(version: "https://jsonfeed.org/version/1.1", title: "Example", + next_url: "/older.json", items: [{ id: "1", url: "article", content_text: "Hello" }]) + headers = { "Content-Type" => content_type, "ETag" => '"json-1"', "Last-Modified" => "Sat, 12 Sep 2026 10:00:00 GMT" } + with_server([[200, headers, body], [304, headers, ""]]) do |base_url, requests| + feed = SimpleRSS.fetch("#{base_url}/feed.json", timeout: 1) + assert_equal :json_feed, feed.feed_type + assert_equal "#{base_url}/article", feed.normalized_entries.first.url + assert_equal "/older.json", feed.next_url + assert_equal '"json-1"', feed.etag + assert_equal headers["Last-Modified"], feed.last_modified + assert_equal 1, requests.size + assert_include requests.first.find { |header| header.start_with?("Accept:") }, "application/feed+json" + assert_nil SimpleRSS.fetch("#{base_url}/feed.json", timeout: 1, etag: feed.etag, last_modified: feed.last_modified) + assert_include requests.last, 'If-None-Match: "json-1"' + assert_include requests.last, "If-Modified-Since: #{headers["Last-Modified"]}" + assert_equal 2, requests.size + end + end + end + + def test_json_fetch_handles_redirects_and_custom_accept_without_fetching_linked_resources + body = JSON.generate(version: "https://jsonfeed.org/version/1", title: "Example", next_url: "/older.json", + items: [{ id: "1", url: "article", content_text: "Hello", + attachments: [{ url: "episode.mp3", mime_type: "audio/mpeg" }] }]) + responses = [[302, { "Location" => "../feeds/final.json" }, ""], [200, {}, body]] + with_server(responses) do |base_url, requests| + feed = SimpleRSS.fetch("#{base_url}/start/feed", timeout: 1, headers: { "Accept" => "application/json" }) + entry = feed.normalized_entries.first + assert_equal "#{base_url}/feeds/final.json", feed.source_url + assert_equal "#{base_url}/feeds/article", entry.url + assert_equal "#{base_url}/feeds/episode.mp3", entry.attachments.first[:url] + assert_equal "/older.json", feed.next_url + assert_equal ["GET /start/feed HTTP/1.1", "GET /feeds/final.json HTTP/1.1"], requests.map(&:first) + requests.each { |request| assert_include request, "Accept: application/json" } + end + end + private def with_server(responses) diff --git a/test/data/json_feed_1.json b/test/data/json_feed_1.json new file mode 100644 index 0000000..231b38a --- /dev/null +++ b/test/data/json_feed_1.json @@ -0,0 +1,75 @@ +{ + "version": "https://jsonfeed.org/version/1", + "title": "Example", + "description": "A feed of examples", + "home_page_url": "https://example.com/", + "feed_url": "https://example.com/feed.json", + "icon": "https://example.com/icon.png", + "favicon": "https://example.com/favicon.png", + "expired": true, + "next_url": "https://example.com/older.json", + "user_comment": "Subscribe with a feed reader.", + "hubs": [ + { + "type": "WebSub", + "url": "https://example.com/hub" + } + ], + "_publisher": { + "enabled": true, + "sequence": [ + 1, + 2 + ] + }, + "items": [ + { + "id": "opaque:%2F:001", + "title": "Ruby & feeds", + "url": "https://example.com/blog/article?one=1&two=2", + "external_url": "https://example.org/original", + "content_html": "

Full & complete.

", + "content_text": "Full & complete.", + "summary": "A summary.", + "date_published": "2026-09-12T10:00:00Z", + "date_modified": "2026-09-13T11:00:00Z", + "image": "https://example.com/image.png", + "banner_image": "https://example.com/banner.png", + "tags": [ + "ruby", + "feeds", + "ruby", + " " + ], + "attachments": [ + { + "url": "https://example.com/audio/one.mp3", + "mime_type": "audio/mpeg", + "title": "Episode", + "size_in_bytes": 1234, + "duration_in_seconds": 62.5, + "_codec": { + "lossless": false + } + }, + { + "url": "https://example.com/images/two.png", + "mime_type": "image/png", + "title": "Cover", + "size_in_bytes": 4321 + } + ], + "_extension": { + "value": "original" + } + } + ], + "author": { + "name": "Example Editor", + "url": "https://example.com/editor", + "avatar": "https://example.com/avatar.png", + "_profile": { + "role": "editor" + } + } +} diff --git a/test/data/json_feed_1_1.json b/test/data/json_feed_1_1.json new file mode 100644 index 0000000..6b5c173 --- /dev/null +++ b/test/data/json_feed_1_1.json @@ -0,0 +1,78 @@ +{ + "version": "https://jsonfeed.org/version/1.1", + "title": "Example", + "description": "A feed of examples", + "home_page_url": "https://example.com/", + "feed_url": "https://example.com/feed.json", + "icon": "https://example.com/icon.png", + "favicon": "https://example.com/favicon.png", + "language": "en", + "expired": true, + "next_url": "https://example.com/older.json", + "user_comment": "Subscribe with a feed reader.", + "hubs": [ + { + "type": "WebSub", + "url": "https://example.com/hub" + } + ], + "authors": [ + { + "name": "Example Editor", + "url": "https://example.com/editor", + "avatar": "https://example.com/avatar.png", + "_profile": { + "role": "editor" + } + } + ], + "_publisher": { + "enabled": true, + "sequence": [ + 1, + 2 + ] + }, + "items": [ + { + "id": "opaque:%2F:001", + "title": "Ruby & feeds", + "url": "https://example.com/blog/article?one=1&two=2", + "external_url": "https://example.org/original", + "content_html": "

Full & complete.

", + "content_text": "Full & complete.", + "summary": "A summary.", + "date_published": "2026-09-12T10:00:00Z", + "date_modified": "2026-09-13T11:00:00Z", + "image": "https://example.com/image.png", + "banner_image": "https://example.com/banner.png", + "tags": [ + "ruby", + "feeds", + "ruby", + " " + ], + "attachments": [ + { + "url": "https://example.com/audio/one.mp3", + "mime_type": "audio/mpeg", + "title": "Episode", + "size_in_bytes": 1234, + "duration_in_seconds": 62.5, + "_codec": { + "lossless": false + } + }, + { + "url": "https://example.com/images/two.png", + "mime_type": "image/png", + "title": "Cover", + "size_in_bytes": 4321 + } + ], + "_extension": { + "value": "original" + } + } + ] +} From 45f74580579ba6e29d6d07680d8db5cb45651abd Mon Sep 17 00:00:00 2001 From: Lucas Carlson Date: Mon, 14 Sep 2026 06:57:18 -0700 Subject: [PATCH 2/2] fix: validate JSON Feed expiration flags Reject non-boolean expiration values so a string such as "false" cannot be mistaken for an expired feed. Preserve false in raw and serialized metadata and cover malformed values with a regression test. Flatten JSON normalization guards to follow the repository style. --- CHANGELOG.md | 4 +++- README.md | 3 ++- lib/simple-rss.rb | 10 ++++------ lib/simple-rss/json_feed.rb | 9 +++++++++ test/base/json_feed_test.rb | 14 ++++++++++++++ 5 files changed, 32 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 800f753..4512fb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,9 @@ repository begins with a 1.1 import, so earlier releases are not reconstructed. multiple attachments. Preserve original metadata and extensions in `raw_json` and normalized entry `raw`; report recoverable date/number issues. Keep XML serialization unchanged and reject JSON-to-XML conversion explicitly. - Fetch detects JSON regardless of Content-Type and retains conditional GET. + Validate supplied expiration flags as booleans so strings such as `"false"` + cannot be mistaken for expired feeds. Fetch detects JSON regardless of + Content-Type and retains conditional GET. ([#60](https://github.com/cardmagic/simple-rss/issues/60)) - Add `normalized_entries`, an optional immutable RSS/Atom view with consistent diff --git a/README.md b/README.md index 1850acd..cd658e3 100644 --- a/README.md +++ b/README.md @@ -369,7 +369,8 @@ Parsing requires a supported version, string feed title, an items array, and objects with nonblank string/numeric IDs and at least one string content field. Titleless items and empty feeds are supported. Malformed JSON, missing required fields, malformed author/tag/attachment structures, and wrong known field types -raise `SimpleRSSError` with a field path. Invalid required item data rejects the +raise `SimpleRSSError` with a field path. When supplied, `expired` must be a +JSON boolean; strings such as `"false"` are rejected. Invalid required item data rejects the whole feed; items are never assigned invented IDs or returned partially parsed. Invalid optional dates and attachment numbers are preserved in raw data and reported in `issues`, with `nil` normalized values. Dates are never substituted diff --git a/lib/simple-rss.rb b/lib/simple-rss.rb index 5d92de9..906a826 100644 --- a/lib/simple-rss.rb +++ b/lib/simple-rss.rb @@ -27,7 +27,8 @@ class SimpleRSS # rubocop:disable Metrics/ClassLength attr_reader :source_url #: String? attr_reader :raw_json #: Hash[String, untyped]? attr_reader :home_page_url, :feed_url, :favicon, :next_url, :user_comment #: String? - attr_reader :authors, :hubs, :expired #: untyped + attr_reader :authors, :hubs #: untyped + attr_reader :expired #: bool? alias entries items #: Array[Hash[Symbol, untyped]] @@feed_tags = %i[ @@ -87,11 +88,8 @@ def channel # @rbs (?source_url: String?, ?mappings: Hash[Symbol, untyped]) -> Array[NormalizedEntry] def normalized_entries(source_url: nil, mappings: {}) json_feed = @json_feed - if json_feed - raise ArgumentError, "XML mappings are not supported for JSON Feed" unless mappings.empty? - - return items.map { |item| json_feed.normalized_entry(item, source_url: source_url || @source_url) } - end + raise ArgumentError, "XML mappings are not supported for JSON Feed" if json_feed && !mappings.empty? + return items.map { |item| json_feed.normalized_entry(item, source_url: source_url || @source_url) } if json_feed EntryNormalizer.validate_mappings(mappings) feed_authors = normalized_feed_authors diff --git a/lib/simple-rss/json_feed.rb b/lib/simple-rss/json_feed.rb index 3156807..15eafbf 100644 --- a/lib/simple-rss/json_feed.rb +++ b/lib/simple-rss/json_feed.rb @@ -47,6 +47,7 @@ def validate check_type(document["items"], Array, "items") optional_strings(document, FEED_STRINGS, "feed") optional_strings(document, ["language"], "feed") if version_1_1? + validate_expiration validate_authors(document, "feed") optional_array(document, "hubs", "feed").each_with_index do |hub, index| check_type(hub, Hash, "hubs[#{index}]") @@ -55,6 +56,14 @@ def validate document["items"].each_with_index { |item, index| validate_item(item, "items[#{index}]") } end + # @rbs () -> void + def validate_expiration + return unless document.key?("expired") + return if [true, false].include?(document["expired"]) + + raise SimpleRSSError, "JSON Feed feed.expired must be a boolean" + end + # @rbs (untyped, String) -> void def validate_item(item, path) check_type(item, Hash, path) diff --git a/test/base/json_feed_test.rb b/test/base/json_feed_test.rb index 47a01b7..8b88bb6 100644 --- a/test/base/json_feed_test.rb +++ b/test/base/json_feed_test.rb @@ -26,6 +26,20 @@ def test_accepts_an_empty_json_feed assert feed.valid? end + def test_expired_accepts_only_booleans_when_present + assert_nil parse_items([]).expired + [true, false].each do |value| + feed = parse_items([], expired: value) + assert_equal value, feed.expired + assert_equal value, feed.raw_json["expired"] + assert_equal value, feed.to_hash[:expired] + end + ["false", "true", 0, 1, nil, [], {}].each do |value| + error = assert_raise(SimpleRSSError) { parse_items([], expired: value) } + assert_include error.message, "feed.expired must be a boolean" + end + end + def test_spec_fixtures_preserve_feed_metadata_and_extensions %w[1 1_1].each do |version| feed = fixture(version)